From 30904c1e5701b3fc4090d952dde19fbb4900d7c4 Mon Sep 17 00:00:00 2001 From: xq2010 Date: Mon, 9 Oct 2023 13:29:29 +0800 Subject: [PATCH 1/4] feat(load rdb file): add redis2kvrocks to load rdb file to kvrocks --- CMakeLists.txt | 6 + src/commands/cmd_server.cc | 6 +- src/common/rdb_stream.cc | 70 +++ src/common/rdb_stream.h | 80 +++ src/storage/rdb.cc | 409 ++++++++++--- src/storage/rdb.h | 43 +- src/storage/rdb_listpack.cc | 12 +- src/storage/rdb_zipmap.cc | 5 +- src/vendor/endianconv.h | 10 + tests/cppunit/rdb_stream_test.cc | 85 +++ tests/cppunit/rdb_test.cc | 337 ++++++++++ tests/testdata/corrupt_empty_keys.rdb | Bin 0 -> 280 bytes tests/testdata/corrupt_ziplist.rdb | Bin 0 -> 1415 bytes tests/testdata/encodings.rdb | Bin 0 -> 667 bytes tests/testdata/encodings_ver10.rdb | Bin 0 -> 707 bytes tests/testdata/hash-ziplist.rdb | Bin 0 -> 137 bytes tests/testdata/hash-zipmap.rdb | Bin 0 -> 35 bytes tests/testdata/list-quicklist.rdb | Bin 0 -> 123 bytes tests/testdata/scriptbackup.rdb | Bin 0 -> 225 bytes tests/testdata/test.conf | 847 ++++++++++++++++++++++++++ tests/testdata/zset-ziplist.rdb | Bin 0 -> 135 bytes utils/redis2kvrocks/README.md | 15 + utils/redis2kvrocks/main.cc | 252 ++++++++ x.py | 3 +- 24 files changed, 2074 insertions(+), 106 deletions(-) create mode 100644 src/common/rdb_stream.cc create mode 100644 src/common/rdb_stream.h create mode 100644 tests/cppunit/rdb_stream_test.cc create mode 100644 tests/cppunit/rdb_test.cc create mode 100644 tests/testdata/corrupt_empty_keys.rdb create mode 100644 tests/testdata/corrupt_ziplist.rdb create mode 100644 tests/testdata/encodings.rdb create mode 100644 tests/testdata/encodings_ver10.rdb create mode 100644 tests/testdata/hash-ziplist.rdb create mode 100644 tests/testdata/hash-zipmap.rdb create mode 100644 tests/testdata/list-quicklist.rdb create mode 100644 tests/testdata/scriptbackup.rdb create mode 100644 tests/testdata/test.conf create mode 100644 tests/testdata/zset-ziplist.rdb create mode 100644 utils/redis2kvrocks/README.md create mode 100644 utils/redis2kvrocks/main.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index 4aadc0d0acc..db2a37614ce 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -254,3 +254,9 @@ add_executable(unittest ${TESTS_SRCS}) target_include_directories(unittest PRIVATE tests/cppunit) target_link_libraries(unittest PRIVATE kvrocks_objs gtest_main ${EXTERNAL_LIBS}) + +# redis2kvrocks tool +file(GLOB_RECURSE REDIS2KVROCKS_SRCS utils/redis2kvrocks/*.cc) +add_executable(redis2kvrocks ${REDIS2KVROCKS_SRCS}) + +target_link_libraries(redis2kvrocks PRIVATE kvrocks_objs ${EXTERNAL_LIBS}) \ No newline at end of file diff --git a/src/commands/cmd_server.cc b/src/commands/cmd_server.cc index 232b17c8078..f1780b06cb1 100644 --- a/src/commands/cmd_server.cc +++ b/src/commands/cmd_server.cc @@ -22,6 +22,7 @@ #include "commander.h" #include "commands/scan_base.h" #include "common/io_util.h" +#include "common/rdb_stream.h" #include "config/config.h" #include "error_constants.h" #include "server/redis_connection.h" @@ -1058,8 +1059,9 @@ class CommandRestore : public Commander { ttl_ms_ -= now; } - RDB rdb(svr->storage, conn->GetNamespace(), args_[3]); - auto s = rdb.Restore(args_[1], ttl_ms_); + auto stream_ptr = std::make_shared(args_[3]); + RDB rdb(svr->storage, svr->GetConfig(), conn->GetNamespace(), stream_ptr); + auto s = rdb.Restore(args_[1], args_[3], ttl_ms_); if (!s.IsOK()) return {Status::RedisExecErr, s.Msg()}; *output = redis::SimpleString("OK"); return Status::OK(); diff --git a/src/common/rdb_stream.cc b/src/common/rdb_stream.cc new file mode 100644 index 00000000000..b029b728c99 --- /dev/null +++ b/src/common/rdb_stream.cc @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "rdb_stream.h" + +#include "fmt/format.h" +#include "vendor/crc64.h" +#include "vendor/endianconv.h" + +StatusOr RdbStringStream::Read(char *buf, size_t n) { + if (pos_ + n > input_.size()) { + return {Status::NotOK, "unexpected EOF"}; + } + memcpy(buf, input_.data() + pos_, n); + pos_ += n; + return n; +} + +StatusOr RdbStringStream::GetCheckSum() const { + if (input_.size() < 8) { + return {Status::NotOK, "invalid payload length"}; + } + uint64_t crc = crc64(0, reinterpret_cast(input_.data()), input_.size() - 8); + memrev64ifbe(&crc); + return crc; +} + +Status RdbFileStream::Open() { + ifs_.open(file_name_, std::ifstream::in | std::ifstream::binary); + if (!ifs_.is_open()) { + return {Status::NotOK, fmt::format("failed to open rdb file: '{}': {}", file_name_, strerror(errno))}; + } + + return Status::OK(); +} + +StatusOr RdbFileStream::Read(char *buf, size_t len) { + size_t n = 0; + while (len) { + size_t read_bytes = max_read_chunk_size_ < len ? max_read_chunk_size_ : len; + ifs_.read(buf, static_cast(read_bytes)); + if (!ifs_.good()) { + return Status(Status::NotOK, fmt::format("read failed: {}:", strerror(errno))); + } + check_sum_ = crc64(check_sum_, (const unsigned char *)buf, read_bytes); + buf = (char *)buf + read_bytes; + len -= read_bytes; + total_read_bytes_ += read_bytes; + n += read_bytes; + } + + return n; +} diff --git a/src/common/rdb_stream.h b/src/common/rdb_stream.h new file mode 100644 index 00000000000..7cea5ab5df2 --- /dev/null +++ b/src/common/rdb_stream.h @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#pragma once + +#include + +#include +#include + +#include "status.h" + +class RdbStream { + public: + RdbStream() = default; + virtual ~RdbStream() = default; + + virtual StatusOr Read(char *buf, size_t len) = 0; + virtual StatusOr GetCheckSum() const = 0; + StatusOr ReadByte() { + uint8_t value = 0; + auto s = Read(reinterpret_cast(&value), 1); + if (!s.IsOK()) { + return s; + } + return value; + } +}; + +class RdbStringStream : public RdbStream { + public: + explicit RdbStringStream(std::string_view input) : input_(input){}; + RdbStringStream(const RdbStringStream &) = delete; + RdbStringStream &operator=(const RdbStringStream &) = delete; + ~RdbStringStream() override = default; + + StatusOr Read(char *buf, size_t len) override; + StatusOr GetCheckSum() const override; + + private: + std::string input_; + size_t pos_ = 0; +}; + +class RdbFileStream : public RdbStream { + public: + explicit RdbFileStream(std::string file_name, size_t chunk_size = 1024 * 1024) + : file_name_(std::move(file_name)), check_sum_(0), total_read_bytes_(0), max_read_chunk_size_(chunk_size){}; + RdbFileStream(const RdbFileStream &) = delete; + RdbFileStream &operator=(const RdbFileStream &) = delete; + ~RdbFileStream() override = default; + + Status Open(); + StatusOr Read(char *buf, size_t len) override; + StatusOr GetCheckSum() const override { return check_sum_; } + + private: + std::ifstream ifs_; + std::string file_name_; + uint64_t check_sum_; + size_t total_read_bytes_; + size_t max_read_chunk_size_; // maximum single read chunk size +}; \ No newline at end of file diff --git a/src/storage/rdb.cc b/src/storage/rdb.cc index 23ddb0d0f50..2417091596a 100644 --- a/src/storage/rdb.cc +++ b/src/storage/rdb.cc @@ -20,7 +20,11 @@ #include "rdb.h" +#include + #include "common/encoding.h" +#include "common/rdb_stream.h" +#include "common/time_util.h" #include "rdb_intset.h" #include "rdb_listpack.h" #include "rdb_ziplist.h" @@ -39,32 +43,52 @@ constexpr const int RDB6BitLen = 0; constexpr const int RDB14BitLen = 1; constexpr const int RDBEncVal = 3; -constexpr const int RDB32BitLen = 0x08; +constexpr const int RDB32BitLen = 0x80; constexpr const int RDB64BitLen = 0x81; constexpr const int RDBEncInt8 = 0; constexpr const int RDBEncInt16 = 1; constexpr const int RDBEncInt32 = 2; constexpr const int RDBEncLzf = 3; -Status RDB::peekOk(size_t n) { - if (pos_ + n > input_.size()) { - return {Status::NotOK, "unexpected EOF"}; - } - return Status::OK(); -} - -Status RDB::VerifyPayloadChecksum() { - if (input_.size() < 10) { +/* Special RDB opcodes (saved/loaded with rdbSaveType/rdbLoadType). */ +constexpr const int RDBOpcodeFunction2 = 245; /* function library data */ +constexpr const int RDBOpcodeFunction = 246; /* old function library data for 7.0 rc1 and rc2 */ +constexpr const int RDBOpcodeModuleAux = 247; /* Module auxiliary data. */ +constexpr const int RDBOpcodeIdle = 248; /* LRU idle time. */ +constexpr const int RDBOpcodeFreq = 249; /* LFU frequency. */ +constexpr const int RDBOpcodeAux = 250; /* RDB aux field. */ +constexpr const int RDBOpcodeResizeDB = 251; /* Hash table resize hint. */ +constexpr const int RDBOpcodeExpireTimeMs = 252; /* Expire time in milliseconds. */ +constexpr const int RDBOpcodeExpireTime = 253; /* Old expire time in seconds. */ +constexpr const int RDBOpcodeSelectDB = 254; /* DB number of the following keys. */ +constexpr const int RDBOpcodeEof = 255; /* End of the RDB file. */ + +// The current support RDB version +constexpr const int RDBVersion = 10; + +// NOLINTNEXTLINE +#define GET_OR_RETWITHLOG(...) \ + ({ \ + auto &&status = (__VA_ARGS__); \ + if (!status) { \ + LOG(WARNING) << "Short read or unsupported type loading DB. Unrecoverable error, aborting now."; \ + LOG(ERROR) << "Unexpected EOF reading RDB file"; \ + return std::forward(status); \ + } \ + std::forward(status); \ + }).GetValue() + +Status RDB::VerifyPayloadChecksum(const std::string_view &payload) { + if (payload.size() < 10) { return {Status::NotOK, "invalid payload length"}; } - auto footer = input_.substr(input_.size() - 10); + auto footer = payload.substr(payload.size() - 10); auto rdb_version = (footer[1] << 8) | footer[0]; // For now, the max redis rdb version is 11 if (rdb_version > 11) { return {Status::NotOK, fmt::format("invalid or unsupported rdb version: {}", rdb_version)}; } - uint64_t crc = crc64(0, reinterpret_cast(input_.data()), input_.size() - 8); - memrev64ifbe(&crc); + auto crc = GET_OR_RET(stream_->GetCheckSum()); if (memcmp(&crc, footer.data() + 2, 8)) { return {Status::NotOK, "incorrect checksum"}; } @@ -72,20 +96,16 @@ Status RDB::VerifyPayloadChecksum() { } StatusOr RDB::LoadObjectType() { - GET_OR_RET(peekOk(1)); - auto type = input_[pos_++] & 0xFF; - // 0-5 is the basic type of Redis objects and 9-21 is the encoding type of Redis objects. - // Redis allow basic is 0-7 and 6/7 is for the module type which we don't support here. - if ((type >= 0 && type <= 5) || (type >= 9 && type <= 21)) { + auto type = GET_OR_RET(stream_->ReadByte()); + if (isObjectType(type)) { return type; } return {Status::NotOK, fmt::format("invalid or unsupported object type: {}", type)}; } StatusOr RDB::loadObjectLen(bool *is_encoded) { - GET_OR_RET(peekOk(1)); uint64_t len = 0; - auto c = input_[pos_++]; + auto c = GET_OR_RET(stream_->ReadByte()); auto type = (c & 0xC0) >> 6; switch (type) { case RDBEncVal: @@ -95,20 +115,17 @@ StatusOr RDB::loadObjectLen(bool *is_encoded) { return c & 0x3F; case RDB14BitLen: len = c & 0x3F; - GET_OR_RET(peekOk(1)); - return (len << 8) | input_[pos_++]; - case RDB32BitLen: - GET_OR_RET(peekOk(4)); - __builtin_memcpy(&len, input_.data() + pos_, 4); - pos_ += 4; - return len; - case RDB64BitLen: - GET_OR_RET(peekOk(8)); - __builtin_memcpy(&len, input_.data() + pos_, 8); - pos_ += 8; - return len; + return (len << 8) | GET_OR_RET(stream_->ReadByte()); default: - return {Status::NotOK, fmt::format("Unknown length encoding {} in loadObjectLen()", type)}; + if (c == RDB32BitLen) { + GET_OR_RET(stream_->Read(reinterpret_cast(&len), 4)); + return ntohl(len); + } else if (c == RDB64BitLen) { + GET_OR_RET(stream_->Read(reinterpret_cast(&len), 8)); + return ntohu64(len); + } else { + return {Status::NotOK, fmt::format("Unknown RDB string encoding type {} byte {}", type, c)}; + } } } @@ -118,13 +135,13 @@ StatusOr RDB::LoadStringObject() { return loadEncodedString(); } StatusOr RDB::loadLzfString() { auto compression_len = GET_OR_RET(loadObjectLen(nullptr)); auto len = GET_OR_RET(loadObjectLen(nullptr)); - GET_OR_RET(peekOk(static_cast(compression_len))); - std::string out_buf(len, 0); - if (lzf_decompress(input_.data() + pos_, compression_len, out_buf.data(), len) != len) { + std::vector vec(compression_len); + GET_OR_RET(stream_->Read(vec.data(), compression_len)); + + if (lzf_decompress(vec.data(), compression_len, out_buf.data(), len) != len) { return {Status::NotOK, "Invalid LZF compressed string"}; } - pos_ += compression_len; return out_buf; } @@ -133,20 +150,18 @@ StatusOr RDB::loadEncodedString() { auto len = GET_OR_RET(loadObjectLen(&is_encoded)); if (is_encoded) { // For integer type, needs to convert to uint8_t* to avoid signed extension - auto data = reinterpret_cast(input_.data()); + unsigned char buf[4] = {0}; if (len == RDBEncInt8) { - GET_OR_RET(peekOk(1)); - return std::to_string(data[pos_++]); + auto next = GET_OR_RET(stream_->ReadByte()); + return std::to_string(static_cast(next)); } else if (len == RDBEncInt16) { - GET_OR_RET(peekOk(2)); - auto value = static_cast(data[pos_]) | (static_cast(data[pos_ + 1]) << 8); - pos_ += 2; + GET_OR_RET(stream_->Read(reinterpret_cast(buf), 2)); + auto value = static_cast(buf[0]) | (static_cast(buf[1]) << 8); return std::to_string(static_cast(value)); } else if (len == RDBEncInt32) { - GET_OR_RET(peekOk(4)); - auto value = static_cast(data[pos_]) | (static_cast(data[pos_ + 1]) << 8) | - (static_cast(data[pos_ + 2]) << 16) | (static_cast(data[pos_ + 3]) << 24); - pos_ += 4; + GET_OR_RET(stream_->Read(reinterpret_cast(buf), 4)); + auto value = static_cast(buf[0]) | (static_cast(buf[1]) << 8) | + (static_cast(buf[2]) << 16) | (static_cast(buf[3]) << 24); return std::to_string(static_cast(value)); } else if (len == RDBEncLzf) { return loadLzfString(); @@ -156,10 +171,9 @@ StatusOr RDB::loadEncodedString() { } // Normal string - GET_OR_RET(peekOk(static_cast(len))); - auto value = std::string(input_.data() + pos_, len); - pos_ += len; - return value; + std::vector vec(len); + GET_OR_RET(stream_->Read(vec.data(), len)); + return std::string(vec.data(), len); } StatusOr> RDB::LoadListWithQuickList(int type) { @@ -177,10 +191,23 @@ StatusOr> RDB::LoadListWithQuickList(int type) { return {Status::NotOK, fmt::format("Unknown quicklist node container type {}", container)}; } } - auto list_pack_string = GET_OR_RET(loadEncodedString()); - ListPack lp(list_pack_string); - auto elements = GET_OR_RET(lp.Entries()); - list.insert(list.end(), elements.begin(), elements.end()); + + if (container == QuickListNodeContainerPlain) { + auto element = GET_OR_RET(loadEncodedString()); + list.push_back(element); + continue; + } + + auto encoded_string = GET_OR_RET(loadEncodedString()); + if (type == RDBTypeListQuickList2) { + ListPack lp(encoded_string); + auto elements = GET_OR_RET(lp.Entries()); + list.insert(list.end(), elements.begin(), elements.end()); + } else { + ZipList zip_list(encoded_string); + auto elements = GET_OR_RET(zip_list.Entries()); + list.insert(list.end(), elements.begin(), elements.end()); + } } return list; } @@ -279,18 +306,15 @@ StatusOr> RDB::LoadHashWithZipList() { } StatusOr RDB::loadBinaryDouble() { - GET_OR_RET(peekOk(8)); double value = 0; - memcpy(&value, input_.data() + pos_, 8); + GET_OR_RET(stream_->Read(reinterpret_cast(&value), 8)); memrev64ifbe(&value); - pos_ += 8; return value; } StatusOr RDB::loadDouble() { char buf[256]; - GET_OR_RET(peekOk(1)); - auto len = static_cast(input_[pos_++]); + auto len = GET_OR_RET(stream_->ReadByte()); switch (len) { case 255: return -INFINITY; /* Negative inf */ @@ -299,10 +323,8 @@ StatusOr RDB::loadDouble() { case 253: return NAN; /* NaN */ } - GET_OR_RET(peekOk(len)); - memcpy(buf, input_.data() + pos_, len); + GET_OR_RET(stream_->Read(buf, len)); buf[len] = '\0'; - pos_ += len; return ParseFloat(std::string(buf, len)); } @@ -356,17 +378,28 @@ StatusOr> RDB::LoadZSetWithZipList() { return zset; } -Status RDB::Restore(const std::string &key, uint64_t ttl_ms) { +Status RDB::Restore(const std::string &key, std::string_view payload, uint64_t ttl_ms) { rocksdb::Status db_status; // Check the checksum of the payload - GET_OR_RET(VerifyPayloadChecksum()); + GET_OR_RET(VerifyPayloadChecksum(payload)); auto type = GET_OR_RET(LoadObjectType()); + + auto value = GET_OR_RET(loadRdbObject(type, key)); + + return saveRdbObject(type, key, value, ttl_ms); // NOLINT +} + +StatusOr RDB::loadRdbType() { + auto type = GET_OR_RET(stream_->ReadByte()); + return type; +} + +StatusOr RDB::loadRdbObject(int type, const std::string &key) { if (type == RDBTypeString) { auto value = GET_OR_RET(LoadStringObject()); - redis::String string_db(storage_, ns_); - db_status = string_db.SetEX(key, value, ttl_ms); + return value; } else if (type == RDBTypeSet || type == RDBTypeSetIntSet || type == RDBTypeSetListPack) { std::vector members; if (type == RDBTypeSet) { @@ -376,14 +409,7 @@ Status RDB::Restore(const std::string &key, uint64_t ttl_ms) { } else { members = GET_OR_RET(LoadSetWithIntSet()); } - redis::Set set_db(storage_, ns_); - uint64_t count = 0; - std::vector insert_members; - insert_members.reserve(members.size()); - for (const auto &member : members) { - insert_members.emplace_back(member); - } - db_status = set_db.Add(key, insert_members, &count); + return members; } else if (type == RDBTypeZSet || type == RDBTypeZSet2 || type == RDBTypeZSetListPack || type == RDBTypeZSetZipList) { std::vector member_scores; if (type == RDBTypeZSet || type == RDBTypeZSet2) { @@ -393,9 +419,7 @@ Status RDB::Restore(const std::string &key, uint64_t ttl_ms) { } else { member_scores = GET_OR_RET(LoadZSetWithZipList()); } - redis::ZSet zset_db(storage_, ns_); - uint64_t count = 0; - db_status = zset_db.Add(key, ZAddFlags(0), (redis::ZSet::MemberScores *)&member_scores, &count); + return member_scores; } else if (type == RDBTypeHash || type == RDBTypeHashListPack || type == RDBTypeHashZipList || type == RDBTypeHashZipMap) { std::map entries; @@ -408,14 +432,7 @@ Status RDB::Restore(const std::string &key, uint64_t ttl_ms) { } else { entries = GET_OR_RET(LoadHashWithZipMap()); } - std::vector filed_values; - filed_values.reserve(entries.size()); - for (const auto &entry : entries) { - filed_values.emplace_back(entry.first, entry.second); - } - redis::Hash hash_db(storage_, ns_); - uint64_t count = 0; - db_status = hash_db.MSet(key, filed_values, false /*nx*/, &count); + return entries; } else if (type == RDBTypeList || type == RDBTypeListZipList || type == RDBTypeListQuickList || type == RDBTypeListQuickList2) { std::vector elements; @@ -426,6 +443,50 @@ Status RDB::Restore(const std::string &key, uint64_t ttl_ms) { } else { elements = GET_OR_RET(LoadListWithQuickList(type)); } + return elements; + } else { + return {Status::RedisParseErr, fmt::format("unsupported type: {}", type)}; + } + + // can't be here + return Status::OK(); +} + +Status RDB::saveRdbObject(int type, const std::string &key, const RedisObjValue &obj, uint64_t ttl_ms) { + rocksdb::Status db_status; + if (type == RDBTypeString) { + const auto &value = std::get(obj); + redis::String string_db(storage_, ns_); + db_status = string_db.SetEX(key, value, ttl_ms); + } else if (type == RDBTypeSet || type == RDBTypeSetIntSet || type == RDBTypeSetListPack) { + const auto &members = std::get>(obj); + redis::Set set_db(storage_, ns_); + uint64_t count = 0; + std::vector insert_members; + insert_members.reserve(members.size()); + for (const auto &member : members) { + insert_members.emplace_back(member); + } + db_status = set_db.Add(key, insert_members, &count); + } else if (type == RDBTypeZSet || type == RDBTypeZSet2 || type == RDBTypeZSetListPack || type == RDBTypeZSetZipList) { + const auto &member_scores = std::get>(obj); + redis::ZSet zset_db(storage_, ns_); + uint64_t count = 0; + db_status = zset_db.Add(key, ZAddFlags(0), (redis::ZSet::MemberScores *)&member_scores, &count); + } else if (type == RDBTypeHash || type == RDBTypeHashListPack || type == RDBTypeHashZipList || + type == RDBTypeHashZipMap) { + const auto &entries = std::get>(obj); + std::vector filed_values; + filed_values.reserve(entries.size()); + for (const auto &entry : entries) { + filed_values.emplace_back(entry.first, entry.second); + } + redis::Hash hash_db(storage_, ns_); + uint64_t count = 0; + db_status = hash_db.MSet(key, filed_values, false /*nx*/, &count); + } else if (type == RDBTypeList || type == RDBTypeListZipList || type == RDBTypeListQuickList || + type == RDBTypeListQuickList2) { + const auto &elements = std::get>(obj); if (!elements.empty()) { std::vector insert_elements; insert_elements.reserve(elements.size()); @@ -437,7 +498,7 @@ Status RDB::Restore(const std::string &key, uint64_t ttl_ms) { db_status = list_db.Push(key, insert_elements, false, &list_size); } } else { - return {Status::RedisExecErr, fmt::format("unsupported restore type: {}", type)}; + return {Status::RedisExecErr, fmt::format("unsupported save type: {}", type)}; } if (!db_status.ok()) { return {Status::RedisExecErr, db_status.ToString()}; @@ -449,3 +510,177 @@ Status RDB::Restore(const std::string &key, uint64_t ttl_ms) { } return db_status.ok() ? Status::OK() : Status{Status::RedisExecErr, db_status.ToString()}; } + +StatusOr RDB::loadTime() { + uint32_t t32 = 0; + GET_OR_RET(stream_->Read(reinterpret_cast(&t32), 4)); + return t32; +} + +StatusOr RDB::loadMillisecondTime(int rdb_version) { + uint64_t t64 = 0; + GET_OR_RET(stream_->Read(reinterpret_cast(&t64), 8)); + /* before Redis 5 (RDB version 9), the function + * failed to convert data to/from little endian, so RDB files with keys having + * expires could not be shared between big endian and little endian systems + * (because the expire time will be totally wrong). comment from src/rdb.c: rdbLoadMillisecondTime*/ + if (rdb_version >= 9) { + memrev64ifbe(&t64); + } + return t64; +} + +bool RDB::isEmptyRedisObject(const RedisObjValue &value) { + if (auto vec_str_ptr = std::get_if>(&value)) { + return vec_str_ptr->size() == 0; + } + if (auto vec_mem_ptr = std::get_if>(&value)) { + return vec_mem_ptr->size() == 0; + } + if (auto map_ptr = std::get_if>(&value)) { + return map_ptr->size() == 0; + } + + return false; +} + +// Load RDB file: copy from redis/src/rdb.c:branch 7.0, 76b9c13d. +Status RDB::LoadRdb() { + char buf[1024] = {0}; + GET_OR_RETWITHLOG(stream_->Read(buf, 9)); + buf[9] = '\0'; + + if (memcmp(buf, "REDIS", 5) != 0) { + LOG(WARNING) << "Wrong signature trying to load DB from file"; + return {Status::NotOK}; + } + + auto rdb_ver = std::atoi(buf + 5); + if (rdb_ver < 1 || rdb_ver > RDBVersion) { + LOG(WARNING) << "Can't handle RDB format version " << rdb_ver; + return {Status::NotOK}; + } + + uint64_t expire_time = 0; + int64_t expire_keys = 0; + int64_t load_keys = 0; + int64_t empty_keys_skipped = 0; + auto now = util::GetTimeStampMS(); + std::string origin_ns = ns_; // current namespace, when select db, will change ns to origin_ns + "_" + db_id + while (true) { + auto type = GET_OR_RETWITHLOG(loadRdbType()); + if (type == RDBOpcodeExpireTime) { + expire_time = static_cast(GET_OR_RETWITHLOG(loadTime())); + expire_time *= 1000; + continue; + } else if (type == RDBOpcodeExpireTimeMs) { + expire_time = GET_OR_RETWITHLOG(loadMillisecondTime(rdb_ver)); + continue; + } else if (type == RDBOpcodeFreq) { // LFU frequency: not use in kvrocks + GET_OR_RETWITHLOG(stream_->ReadByte()); // discard the value + continue; + } else if (type == RDBOpcodeIdle) { // LRU idle time: not use in kvrocks + uint64_t discard = 0; + GET_OR_RETWITHLOG(stream_->Read(reinterpret_cast(&discard), 8)); + continue; + } else if (type == RDBOpcodeEof) { + break; + } else if (type == RDBOpcodeSelectDB) { + auto db_id = GET_OR_RETWITHLOG(loadObjectLen(nullptr)); + if (db_id != 0) { // change namespace to ns + "_" + db_id + ns_ = origin_ns + "_" + std::to_string(db_id); + } else { // use origin namespace + ns_ = origin_ns; + } + LOG(INFO) << "select db: " << db_id << ", change to namespace: " << ns_; + // add namespace, password is the same as the namespace(expect the default namespace) + GET_OR_RET(addNamespace(ns_, ns_)); + continue; + } else if (type == RDBOpcodeResizeDB) { // not use in kvrocks, hint redis for hash table resize + GET_OR_RETWITHLOG(loadObjectLen(nullptr)); // db_size + GET_OR_RETWITHLOG(loadObjectLen(nullptr)); // expires_size + continue; + } else if (type == RDBOpcodeAux) { + /* AUX: generic string-string fields. Use to add state to RDB + * which is backward compatible. Implementations of RDB loading + * are required to skip AUX fields they don't understand. + * + * An AUX field is composed of two strings: key and value. */ + auto key = GET_OR_RETWITHLOG(LoadStringObject()); + auto value = GET_OR_RETWITHLOG(LoadStringObject()); + continue; + } else if (type == RDBOpcodeModuleAux) { + LOG(WARNING) << "RDB module not supported"; + return {Status::NotOK, "RDB module not supported"}; + } else if (type == RDBOpcodeFunction || type == RDBOpcodeFunction2) { + LOG(WARNING) << "RDB function not supported"; + return {Status::NotOK, "RDB function not supported"}; + } else { + if (!isObjectType(type)) { + LOG(WARNING) << "Invalid or Not supported object type: " << type; + return {Status::NotOK, "Invalid or Not supported object type"}; + } + } + + auto key = GET_OR_RETWITHLOG(LoadStringObject()); + auto value = GET_OR_RETWITHLOG(loadRdbObject(type, key)); + + if (isEmptyRedisObject(value)) { // compatible with empty value + /* Since we used to have bug that could lead to empty keys + * (See #8453), we rather not fail when empty key is encountered + * in an RDB file, instead we will silently discard it and + * continue loading. */ + if (empty_keys_skipped++ < 10) { + LOG(WARNING) << "skipping empty key: " << key; + } + continue; + } else if (expire_time != 0 && + expire_time < now) { // in redis this used to feed this deletion to any connected replicas + expire_keys++; + continue; + } + + auto ret = saveRdbObject(type, key, value, expire_time); + load_keys++; + if (!ret.IsOK()) { + LOG(WARNING) << "save rdb object key " << key << " failed: " << ret.Msg(); + } + } + + // Verify the checksum if RDB version is >= 5 + if (rdb_ver >= 5) { + uint64_t chk_sum = 0; + auto expected = GET_OR_RETWITHLOG(stream_->GetCheckSum()); + GET_OR_RETWITHLOG(stream_->Read(reinterpret_cast(&chk_sum), 8)); + if (chk_sum == 0) { + LOG(WARNING) << "RDB file was saved with checksum disabled: no check performed."; + } else if (chk_sum != expected) { + LOG(WARNING) << "Wrong RDB checksum expected: " << chk_sum << " got: " << expected; + return {Status::NotOK, "Wrong RDB checksum"}; + } + } + + if (empty_keys_skipped > 0) { + LOG(INFO) << "Done loading RDB, keys loaded: " << load_keys << ", keys expired:" << expire_keys + << ", empty keys skipped: " << empty_keys_skipped; + } else { + LOG(INFO) << "Done loading RDB, keys loaded: " << load_keys << ", keys expired:" << expire_keys; + } + + return Status::OK(); +} + +Status RDB::addNamespace(const std::string &ns, const std::string &token) { + if (ns == kDefaultNamespace) { + return Status::OK(); + } + + std::string old_token; + auto s = config_->GetNamespace(ns, &old_token); + if (s.IsOK()) { // namespace exist, use old token + return s; + } + + // add new namespace + return config_->AddNamespace(ns, token); +} \ No newline at end of file diff --git a/src/storage/rdb.h b/src/storage/rdb.h index 956df034065..2ee56ddb383 100644 --- a/src/storage/rdb.h +++ b/src/storage/rdb.h @@ -19,9 +19,15 @@ */ #pragma once +#include +#include +#include #include #include +#include +#include +#include "config/config.h" #include "status.h" #include "types/redis_zset.h" @@ -47,26 +53,31 @@ constexpr const int RDBTypeZSetListPack = 17; constexpr const int RDBTypeListQuickList2 = 18; constexpr const int RDBTypeStreamListPack2 = 19; constexpr const int RDBTypeSetListPack = 20; -// NOTE: when adding new Redis object encoding type, update LoadObjectType. +// NOTE: when adding new Redis object encoding type, update isObjectType. // Quick list node encoding constexpr const int QuickListNodeContainerPlain = 1; constexpr const int QuickListNodeContainerPacked = 2; +class RdbStream; + +using RedisObjValue = + std::variant, std::vector, std::map>; + class RDB { public: - explicit RDB(engine::Storage *storage, std::string ns, std::string_view input) - : storage_(storage), ns_(std::move(ns)), input_(input){}; + explicit RDB(engine::Storage *storage, Config *config, std::string ns, std::shared_ptr stream) + : storage_(storage), config_(config), ns_(std::move(ns)), stream_(std::move(stream)){}; ~RDB() = default; - Status VerifyPayloadChecksum(); + Status VerifyPayloadChecksum(const std::string_view &payload); StatusOr LoadObjectType(); - Status Restore(const std::string &key, uint64_t ttl_ms); + Status Restore(const std::string &key, std::string_view payload, uint64_t ttl_ms); // String StatusOr LoadStringObject(); - // List + // Hash StatusOr> LoadHashObject(); StatusOr> LoadHashWithZipMap(); StatusOr> LoadHashWithListPack(); @@ -87,16 +98,30 @@ class RDB { StatusOr> LoadListWithZipList(); StatusOr> LoadListWithQuickList(int type); + // Load rdb + Status LoadRdb(); + private: engine::Storage *storage_; + Config *config_; std::string ns_; - std::string_view input_; - size_t pos_ = 0; + std::shared_ptr stream_; StatusOr loadLzfString(); StatusOr loadEncodedString(); StatusOr loadObjectLen(bool *is_encoded); - Status peekOk(size_t n); StatusOr loadBinaryDouble(); StatusOr loadDouble(); + + StatusOr loadRdbType(); + StatusOr loadRdbObject(int rdbtype, const std::string &key); + Status saveRdbObject(int type, const std::string &key, const RedisObjValue &obj, uint64_t ttl_ms); + StatusOr loadTime(); + StatusOr loadMillisecondTime(int rdb_version); + Status addNamespace(const std::string &ns, const std::string &token); + + /*0-5 is the basic type of Redis objects and 9-21 is the encoding type of Redis objects. + Redis allow basic is 0-7 and 6/7 is for the module type which we don't support here.*/ + static bool isObjectType(int type) { return (type >= 0 && type <= 5) || (type >= 9 && type <= 21); }; + static bool isEmptyRedisObject(const RedisObjValue &value); }; diff --git a/src/storage/rdb_listpack.cc b/src/storage/rdb_listpack.cc index f2933bb968b..4eb3597d5b8 100644 --- a/src/storage/rdb_listpack.cc +++ b/src/storage/rdb_listpack.cc @@ -108,10 +108,12 @@ StatusOr ListPack::Length() { return {Status::NotOK, "invalid listpack length"}; } + auto data = reinterpret_cast( + input_.data()); // to avoid the sign extension in static_cast(input_[n])) // total bytes and number of elements are encoded in little endian - uint32_t total_bytes = (static_cast(input_[0])) | (static_cast(input_[1]) << 8) | - (static_cast(input_[2]) << 16) | (static_cast(input_[3]) << 24); - uint32_t len = (static_cast(input_[4])) | (static_cast(input_[5]) << 8); + uint32_t total_bytes = (static_cast(data[0])) | (static_cast(data[1]) << 8) | + (static_cast(data[2]) << 16) | (static_cast(data[3]) << 24); + uint32_t len = (static_cast(data[4])) | (static_cast(data[5]) << 8); pos_ += listPackHeaderSize; if (total_bytes != input_.size()) { @@ -166,7 +168,7 @@ StatusOr ListPack::Next() { pos_ += value_len + encodeBackLen(value_len + 1); } else if ((c & ListPack13BitIntMask) == ListPack13BitInt) { // 13bit int GET_OR_RET(peekOK(3)); - int_value = ((c & 0x1F) << 8) | data[pos_]; + int_value = ((c & 0x1F) << 8) | data[pos_ + 1]; value = std::to_string(int_value); pos_ += ListPack13BitIntEntrySize; } else if ((c & ListPack16BitIntMask) == ListPack16BitInt) { // 16bit int @@ -188,7 +190,7 @@ StatusOr ListPack::Next() { pos_ += ListPack32BitIntEntrySize; } else if ((c & ListPack64BitIntMask) == ListPack64BitInt) { // 64bit int GET_OR_RET(peekOK(10)); - int_value = (static_cast(data[pos_ + 1])) | (static_cast(input_[pos_ + 2]) << 8) | + int_value = (static_cast(data[pos_ + 1])) | (static_cast(data[pos_ + 2]) << 8) | (static_cast(data[pos_ + 3]) << 16) | (static_cast(data[pos_ + 4]) << 24) | (static_cast(data[pos_ + 5]) << 32) | (static_cast(data[pos_ + 6]) << 40) | (static_cast(data[pos_ + 7]) << 48) | (static_cast(data[pos_ + 8]) << 56); diff --git a/src/storage/rdb_zipmap.cc b/src/storage/rdb_zipmap.cc index 724ce852c01..6d82dad13fb 100644 --- a/src/storage/rdb_zipmap.cc +++ b/src/storage/rdb_zipmap.cc @@ -59,11 +59,12 @@ StatusOr> ZipMap::Next() { auto key_len = GET_OR_RET(decodeLength()); GET_OR_RET(peekOK(key_len)); auto key = input_.substr(pos_, key_len); - pos_ += key_len + getEncodedLengthSize(key_len); + pos_ += key_len; // decodeLength already process as getEncodedLengthSize(key_len); auto val_len = GET_OR_RET(decodeLength()); GET_OR_RET(peekOK(val_len + 1 /* free byte */)); + pos_ += 1; auto value = input_.substr(pos_, val_len); - pos_ += val_len + getEncodedLengthSize(val_len) + 1 /* free byte */; + pos_ += val_len; // + getEncodedLengthSize(val_len) + 1 /* free byte */; return std::make_pair(key, value); } diff --git a/src/vendor/endianconv.h b/src/vendor/endianconv.h index e27042754c0..3907578ad76 100644 --- a/src/vendor/endianconv.h +++ b/src/vendor/endianconv.h @@ -59,4 +59,14 @@ uint64_t intrev64(uint64_t v); #define intrev64ifbe(v) intrev64(v) #endif +/* The functions htonu64() and ntohu64() convert the specified value to + * network byte ordering and back. In big endian systems they are no-ops. */ +#if (BYTE_ORDER == BIG_ENDIAN) +#define htonu64(v) (v) +#define ntohu64(v) (v) +#else +#define htonu64(v) intrev64(v) +#define ntohu64(v) intrev64(v) +#endif + // NOLINTEND diff --git a/tests/cppunit/rdb_stream_test.cc b/tests/cppunit/rdb_stream_test.cc new file mode 100644 index 00000000000..2dc94abb207 --- /dev/null +++ b/tests/cppunit/rdb_stream_test.cc @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "common/rdb_stream.h" + +#include + +#include +#include + +TEST(RdbFileStreamOpenTest, FileNotExist) { + RdbFileStream reader("../tests/testdata/not_exist.rdb"); + ASSERT_FALSE(reader.Open().IsOK()); + ; +} + +TEST(RdbFileStreamOpenTest, FileExist) { + RdbFileStream reader("../tests/testdata/encodings.rdb"); + ASSERT_TRUE(reader.Open().IsOK()); +} + +TEST(RdbFileStreamReadTest, ReadRdb) { + const std::string file_path = "../tests/testdata/encodings.rdb"; + + std::ifstream file(file_path, std::ios::binary | std::ios::ate); + std::streamsize size = file.tellg(); + file.close(); + + RdbFileStream reader(file_path); + ASSERT_TRUE(reader.Open().IsOK()); + + char buf[16] = {0}; + ASSERT_EQ(reader.Read(buf, 5).GetValue(), 5); + ASSERT_EQ(strncmp(buf, "REDIS", 5), 0); + size -= 5; + + auto len = static_cast(sizeof(buf) / sizeof(buf[0])); + while (size >= len) { + ASSERT_EQ(reader.Read(buf, len).GetValue(), len); + size -= len; + } + + if (size > 0) { + ASSERT_EQ(reader.Read(buf, size).GetValue(), size); + } +} + +TEST(RdbFileStreamReadTest, ReadRdbLittleChunk) { + const std::string file_path = "../tests/testdata/encodings.rdb"; + + std::ifstream file(file_path, std::ios::binary | std::ios::ate); + std::streamsize size = file.tellg(); + file.close(); + + RdbFileStream reader(file_path, 16); + ASSERT_TRUE(reader.Open().IsOK()); + char buf[32] = {0}; + auto len = static_cast(sizeof(buf) / sizeof(buf[0])); + + while (size >= len) { + ASSERT_EQ(reader.Read(buf, len).GetValue(), len); + size -= len; + } + + if (size > 0) { + ASSERT_EQ(reader.Read(buf, size).GetValue(), size); + } +} \ No newline at end of file diff --git a/tests/cppunit/rdb_test.cc b/tests/cppunit/rdb_test.cc new file mode 100644 index 00000000000..6c3b69660dd --- /dev/null +++ b/tests/cppunit/rdb_test.cc @@ -0,0 +1,337 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include "storage/rdb.h" + +#include + +#include +#include +#include +#include +#include +#include + +#include "common/rdb_stream.h" +#include "config/config.h" +#include "storage/storage.h" +#include "types/redis_hash.h" +#include "types/redis_list.h" +#include "types/redis_set.h" +#include "types/redis_string.h" +#include "types/redis_zset.h" +#include "vendor/crc64.h" + +const std::string test_data_path = "../tests/testdata/"; +const std::string test_config = "test.conf"; + +class RDBTest : public testing::Test { + public: + RDBTest(const RDBTest &) = delete; + RDBTest &operator=(const RDBTest &) = delete; + + protected: + explicit RDBTest() : storage_(nullptr), ns_(kDefaultNamespace) {} + ~RDBTest() override { + if (nullptr != storage_) { + delete storage_; + } + } + void SetUp() override { + crc64_init(); + + std::string test_config_path = test_data_path + test_config; + auto s = config_.Load(CLIOptions(test_config_path)); + ASSERT_TRUE(s.IsOK()); + + ASSERT_TRUE(clearDBDir(config_.db_dir)); + + storage_ = new engine::Storage(&config_); + s = storage_->Open(); + ASSERT_TRUE(s.IsOK()); + } + + void TearDown() override { ASSERT_TRUE(clearDBDir(config_.db_dir)); } + + static std::string getRdbFullPath(const std::string &name) { return test_data_path + name; } + + void loadRdb(const std::string &path) { + auto stream_ptr = std::make_shared(path); + auto s = stream_ptr->Open(); + ASSERT_TRUE(s.IsOK()); + + RDB rdb(storage_, &config_, ns_, stream_ptr); + s = rdb.LoadRdb(); + ASSERT_TRUE(s.IsOK()); + } + + void stringCheck(const std::string &key, const std::string &expect) { + redis::String string_db(storage_, ns_); + std::string value; + auto s = string_db.Get(key, &value); + ASSERT_TRUE(s.ok()); + ASSERT_TRUE(expect == value); + } + + void setCheck(const std::string &key, const std::vector &expect) { + redis::Set set_db(storage_, ns_); + std::vector members; + auto s = set_db.Members(key, &members); + + ASSERT_TRUE(s.ok()); + ASSERT_TRUE(expect == members); + } + + void hashCheck(const std::string &key, const std::map &expect) { + redis::Hash hash_db(storage_, ns_); + std::vector field_values; + auto s = hash_db.GetAll(key, &field_values); + ASSERT_TRUE(s.ok()); + + // size check + ASSERT_TRUE(field_values.size() == expect.size()); + for (const auto &p : field_values) { + auto iter = expect.find(p.field); + if (iter == expect.end()) { + ASSERT_TRUE(false); + } + ASSERT_TRUE(iter->second == p.value); + } + } + + void listCheck(const std::string &key, const std::vector &expect) { + redis::List list_db(storage_, ns_); + std::vector values; + auto s = list_db.Range(key, 0, -1, &values); + ASSERT_TRUE(s.ok()); + ASSERT_TRUE(expect == values); + } + + void zsetCheck(const std::string &key, const std::vector &expect) { + redis::ZSet zset_db(storage_, ns_); + std::vector member_scores; + + RangeRankSpec spec; + auto s = zset_db.RangeByRank(key, spec, &member_scores, nullptr); + ASSERT_TRUE(s.ok()); + ASSERT_TRUE(expect.size() == member_scores.size()); + for (size_t i = 0; i < expect.size(); ++i) { + ASSERT_TRUE(expect[i].member == member_scores[i].member); + ASSERT_TRUE(std::fabs(expect[i].score - member_scores[i].score) < 0.000001); + } + } + + rocksdb::Status exists(const std::string &key) { + int cnt = 0; + std::vector keys; + keys.emplace_back(key); + redis::Database redis(storage_, ns_); + auto s = redis.Exists(keys, &cnt); + if (!s.ok()) { + return s; + } + if (cnt == 0) { + return rocksdb::Status::NotFound(); + } + return rocksdb::Status::OK(); + } + + void flushDB() { + redis::Database redis(storage_, ns_); + auto s = redis.FlushDB(); + ASSERT_TRUE(s.ok()); + } + + void encodingDataCheck(); + + engine::Storage *storage_; + std::string ns_; + Config config_; + + private: + static bool clearDBDir(const std::string &path) { + try { + std::filesystem::remove_all(path); + } catch (std::filesystem::filesystem_error &e) { + return false; + } + return true; + } +}; + +void RDBTest::encodingDataCheck() { + // string + stringCheck("compressible", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); + stringCheck("string", "Hello World"); + stringCheck("number", "10"); + + // list + std::vector list_expect = {"1", "2", "3", "a", "b", "c", "100000", "6000000000", + "1", "2", "3", "a", "b", "c", "100000", "6000000000", + "1", "2", "3", "a", "b", "c", "100000", "6000000000"}; + listCheck("list", list_expect); + + std::vector list_zipped_expect = {"1", "2", "3", "a", "b", "c", "100000", "6000000000"}; + listCheck("list_zipped", list_zipped_expect); + + // set + std::vector set_expect = {"1", "100000", "2", "3", "6000000000", "a", "b", "c"}; + setCheck("set", set_expect); + + std::vector set_zipped_1_expect = {"1", "2", "3", "4"}; + setCheck("set_zipped_1", set_zipped_1_expect); + + std::vector set_zipped_2_expect = {"100000", "200000", "300000", "400000"}; + setCheck("set_zipped_2", set_zipped_2_expect); + + std::vector set_zipped_3_expect = {"1000000000", "2000000000", "3000000000", + "4000000000", "5000000000", "6000000000"}; + setCheck("set_zipped_3", set_zipped_3_expect); + + // hash + std::map hash_expect = {{"a", "1"}, {"aa", "10"}, {"aaa", "100"}, {"b", "2"}, + {"bb", "20"}, {"bbb", "200"}, {"c", "3"}, {"cc", "30"}, + {"ccc", "300"}, {"ddd", "400"}, {"eee", "5000000000"}}; + hashCheck("hash", hash_expect); + + std::map hash_zipped_expect = { + {"a", "1"}, + {"b", "2"}, + {"c", "3"}, + }; + hashCheck("hash_zipped", hash_zipped_expect); + + // zset + std::vector zset_expect = { + {"a", 1}, {"b", 2}, {"c", 3}, {"aa", 10}, {"bb", 20}, {"cc", 30}, + {"aaa", 100}, {"bbb", 200}, {"ccc", 300}, {"aaaa", 1000}, {"cccc", 123456789}, {"bbbb", 5000000000}}; + zsetCheck("zset", zset_expect); + + std::vector zset_zipped_expect = { + {"a", 1}, + {"b", 2}, + {"c", 3}, + }; + zsetCheck("zset_zipped", zset_zipped_expect); +} + +TEST_F(RDBTest, LoadEncodings) { + for (const auto &entry : std::filesystem::directory_iterator(test_data_path)) { + std::string filename = entry.path().filename().string(); + if (filename.rfind("encodings", 0) != 0) { + continue; + } + auto full_path = getRdbFullPath(filename); + loadRdb(full_path); + encodingDataCheck(); + flushDB(); + } +} + +TEST_F(RDBTest, LoadHashZipMap) { + auto full_path = getRdbFullPath("hash-zipmap.rdb"); + loadRdb(full_path); + + // hash + std::map hash_expect = { + {"f1", "v1"}, + {"f2", "v2"}, + }; + hashCheck("hash", hash_expect); +} + +TEST_F(RDBTest, LoadHashZipList) { + auto full_path = getRdbFullPath("hash-ziplist.rdb"); + loadRdb(full_path); + + // hash + std::map hash_expect = { + {"f1", "v1"}, + {"f2", "v2"}, + }; + hashCheck("hash", hash_expect); +} + +TEST_F(RDBTest, LoadListQuickList) { + auto full_path = getRdbFullPath("list-quicklist.rdb"); + loadRdb(full_path); + + // list + std::vector list_expect = {"7"}; + listCheck("list", list_expect); +} + +TEST_F(RDBTest, LoadZSetZipList) { + auto full_path = getRdbFullPath("zset-ziplist.rdb"); + loadRdb(full_path); + + // zset + std::vector zset_expect = { + {"one", 1}, + {"two", 2}, + }; + zsetCheck("zset", zset_expect); +} + +TEST_F(RDBTest, LoadEmptyKeys) { + auto full_path = getRdbFullPath("corrupt_empty_keys.rdb"); + loadRdb(full_path); + + /* corrupt_empty_keys.rdb contains 9 keys with empty value: + "set" "hash" "list_ziplist" "zset" "zset_listpack" "hash_ziplist" "list_quicklist" "zset_ziplist" + "list_quicklist_empty_ziplist" + */ + + // string + rocksdb::Status s = exists("empty_string"); // empty_string not exist in rdb file + ASSERT_TRUE(s.IsNotFound()); + + // list + s = exists("list_ziplist"); + ASSERT_TRUE(s.IsNotFound()); + + s = exists("list_quicklist"); + ASSERT_TRUE(s.IsNotFound()); + + s = exists("list_quicklist_empty_ziplist"); + + // set + s = exists("set"); + ASSERT_TRUE(s.IsNotFound()); + + // hash + s = exists("hash"); + ASSERT_TRUE(s.IsNotFound()); + + s = exists("hash_ziplist"); + ASSERT_TRUE(s.IsNotFound()); + + // zset + s = exists("zset"); + ASSERT_TRUE(s.IsNotFound()); + + s = exists("zset_ziplist"); + ASSERT_TRUE(s.IsNotFound()); + + s = exists("zset_listpack"); + ASSERT_TRUE(s.IsNotFound()); +} \ No newline at end of file diff --git a/tests/testdata/corrupt_empty_keys.rdb b/tests/testdata/corrupt_empty_keys.rdb new file mode 100644 index 0000000000000000000000000000000000000000..98b6a143a14bef9d7692ed3e2ce23174a408869f GIT binary patch literal 280 zcmWG?b@2=~FfcUu#aWb^l3A=H&s-48J)Un3#)GOBh&KG7^h37`S+H zGK)*%t1=5fG&eT`0|OTjGcf#TWvK$G6XXR^@gUiP#N=#tc8IJ14@f&qA54^w4{Tat qX=ZXZh{eFe1J;ejcs?0arSYk`1tpa*0~wKB^8dqEw!6pj5}N^cYgv~7 literal 0 HcmV?d00001 diff --git a/tests/testdata/corrupt_ziplist.rdb b/tests/testdata/corrupt_ziplist.rdb new file mode 100644 index 0000000000000000000000000000000000000000..b40ada8d634d094ba2f20d728f20b19596e3e7ef GIT binary patch literal 1415 zcmaJ>yH3ME5WE!T6@e)D2D(_g=jTyTKtT%yJ;@ao7$n<4$tUnlG*No~f$uO9#!1cy z*|V&CI%%|?ot<6X?e*2o9T6q~& zFfZ+FlG}X!`94

MK_2^wBOaS~?d==q)zU!&yf={A4d*K4oP!@_nV$d3(I}+y1iu z&()~2>C61Nvc;cTI$n)>btn+(lX1ELPr^eUtzCYvzbM9d*I!=`yS@i_ig{uQPjG&y zy{3)DaZjgVn9o|!Pgo}tVIIahnXzxo0py_+*QFHeClYX>E=5=;8Qd3{ZcloCh5}D2 z%q77(nK4c&o(mJkN%B);{rtPnVE+R>+<$}TNQr=_xGrg--&n|J!OsZiGYkI2!oEp< bM#3J39Y65RY~Brk>u~62;CyS^7jGLs8LoRE literal 0 HcmV?d00001 diff --git a/tests/testdata/encodings.rdb b/tests/testdata/encodings.rdb new file mode 100644 index 0000000000000000000000000000000000000000..9fd9b705d16220065ee117a1c1c094f40fb122f2 GIT binary patch literal 667 zcmbVKu}UOC5UuKJ7oAzb-~v%NHW5S&dS+bpFfmX#Qw_|N?w&>$M|aur5EcU?;j)7> zGV&XY4SF>ZBR^rkz`zf1tzKQwOdP0r-Cfn)uU@~+^|g&HrPRU;knEK1xGIbhsS?(T zOp!5$Ql%uLiRxVU_K~%gGG1r2V@aAV)EAeQf1$=iXe|~B7q#x40{=g8Nhbr35aH0(af04| z31?FkfXdOIL*v>$`m`ZiW?H~$fQQSK0B})18Q{+2^#ErNo(A|lG8gy_c?x0;Mx(`{ zoa#1QiP|F?FVK2I8DyCB=mk$S8nlC&4|~3w8;|#Ox&QtGwHmXU=HND1)gUq}8+2xM zS?Yc@4yO2OwUpuPICQ~2@KI=m_~m`huJS+FRQ_l1RQDc;oztC1%JaPY56L?s)vH`;wZH<&56B04urfYGkq(PZZnX$ z(=em%Ml!Zt7XTX2jqs(E#Iaggwgb-@_i6xHXsQXt~ zpJc8Lf3Q9Yj`r*9mq(|xJ{=O;Z#d8{A-Nm*j9Fx@?|Pk%bnA7}D#sJ^n!3TG4ZGE% z-8z&ghZH5yIDUfY7)s*0uDbu4B~nWDD7<`GZcZHmtgkdw3MuB^O9XpbfzcZkUYx7I ztm=DFjBI$><`*~}XaFKZ1<8ew2<3pTko(6yiSu({j8@L0ku!zd^^~PJC>&8#uUY>` zsvt)PDw|t@kYQS#AX%=lF$n?#g_vq)(T0;-1yfEt=!7zgJmJfsVZTDoN6&#)s6=t- zZ9X;cNZ)VIKWK-(JCV40L#MMhn)t$0F{`#S;i_d2} z7e{GvYKm@dYVM&4Yk3%c@g(M_=@t~FCgvvPq#j`S$MBnxftMvCu{c9ont_2q42W46 c7?{!wS(wUzIL!!18~y+9Kj-3KvCJ9G0M4&2bN~PV literal 0 HcmV?d00001 diff --git a/tests/testdata/hash-zipmap.rdb b/tests/testdata/hash-zipmap.rdb new file mode 100644 index 0000000000000000000000000000000000000000..27a42ed4bb494cee9d67b7f0feccce7e4136d50a GIT binary patch literal 35 pcmWG?b@2=~FfcIw$H2*wkyxA|z{Heh$iz@)$dqOTq>TRm2LPS*34j0q literal 0 HcmV?d00001 diff --git a/tests/testdata/list-quicklist.rdb b/tests/testdata/list-quicklist.rdb new file mode 100644 index 0000000000000000000000000000000000000000..a9101a1ade0d167e5c3a741827944e4e0ee012bc GIT binary patch literal 123 zcmWG?b@2=~Ffg$A#aWb^l3A=OZT+eVEDxj7RgM}FgCGFO-)QoO*Txk zv`8~fGc_|ZH&3-RO-r^gOG-AeNHR7sGc`;!`o#^_k)M`UoLT}jfG06OO}7AKU{X#h zkk6b`ny6NkT2fk+rvP@AUUFhij)po=wYrXidRl%yh)PNP7 AH2?qr literal 0 HcmV?d00001 diff --git a/tests/testdata/test.conf b/tests/testdata/test.conf new file mode 100644 index 00000000000..00f6a1d39da --- /dev/null +++ b/tests/testdata/test.conf @@ -0,0 +1,847 @@ +################################ GENERAL ##################################### + +# By default kvrocks listens for connections from localhost interface. +# It is possible to listen to just one or multiple interfaces using +# the "bind" configuration directive, followed by one or more IP addresses. +# +# Examples: +# +# bind 192.168.1.100 10.0.0.1 +# bind 127.0.0.1 ::1 +# bind 0.0.0.0 +bind 127.0.0.1 + +# Unix socket. +# +# Specify the path for the unix socket that will be used to listen for +# incoming connections. There is no default, so kvrocks will not listen +# on a unix socket when not specified. +# +# unixsocket /tmp/kvrocks.sock +# unixsocketperm 777 + +# Accept connections on the specified port, default is 6666. +port 6666 + +# Close the connection after a client is idle for N seconds (0 to disable) +timeout 0 + +# The number of worker's threads, increase or decrease would affect the performance. +workers 8 + +# By default, kvrocks does not run as a daemon. Use 'yes' if you need it. +# Note that kvrocks will write a PID file in /var/run/kvrocks.pid when daemonized +daemonize no + +# Kvrocks implements the cluster solution that is similar to the Redis cluster solution. +# You can get cluster information by CLUSTER NODES|SLOTS|INFO command, it also is +# adapted to redis-cli, redis-benchmark, Redis cluster SDK, and Redis cluster proxy. +# But kvrocks doesn't support communicating with each other, so you must set +# cluster topology by CLUSTER SETNODES|SETNODEID commands, more details: #219. +# +# PLEASE NOTE: +# If you enable cluster, kvrocks will encode key with its slot id calculated by +# CRC16 and modulo 16384, encoding key with its slot id makes it efficient to +# migrate keys based on the slot. So if you enabled at first time, cluster mode must +# not be disabled after restarting, and vice versa. That is to say, data is not +# compatible between standalone mode with cluster mode, you must migrate data +# if you want to change mode, otherwise, kvrocks will make data corrupt. +# +# Default: no + +cluster-enabled no + + +# Persist the cluster nodes topology in local file($dir/nodes.conf). This configuration +# takes effect only if the cluster mode was enabled. +# +# If yes, it will try to load the cluster topology from the local file when starting, +# and dump the cluster nodes into the file if it was changed. +# +# Default: yes +# persist-cluster-nodes-enabled yes + +# Set the max number of connected clients at the same time. By default +# this limit is set to 10000 clients. However, if the server is not +# able to configure the process file limit to allow for the specified limit +# the max number of allowed clients is set to the current file limit +# +# Once the limit is reached the server will close all the new connections sending +# an error 'max number of clients reached'. +# +maxclients 10000 + +# Require clients to issue AUTH before processing any other +# commands. This might be useful in environments in which you do not trust +# others with access to the host running kvrocks. +# +# This should stay commented out for backward compatibility and because most +# people do not need auth (e.g. they run their own servers). +# +# Warning: since kvrocks is pretty fast an outside user can try up to +# 150k passwords per second against a good box. This means that you should +# use a very strong password otherwise it will be very easy to break. +# +# requirepass foobared + +# If the master is password protected (using the "masterauth" configuration +# directive below) it is possible to tell the slave to authenticate before +# starting the replication synchronization process. Otherwise, the master will +# refuse the slave request. +# +# masterauth foobared + +# Master-Salve replication would check db name is matched. if not, the slave should +# refuse to sync the db from master. Don't use the default value, set the db-name to identify +# the cluster. +db-name test.db + +# The working directory +# +# The DB will be written inside this directory +# Note that you must specify a directory here, not a file name. +dir /tmp/testdb/ + +# You can configure where to store your server logs by the log-dir. +# If you don't specify one, we will use the above `dir` as our default log directory. +# We also can send logs to stdout/stderr is as simple as: +# +log-dir stdout + +# Log level +# Possible values: info, warning, error, fatal +# Default: info +log-level info + +# You can configure log-retention-days to control whether to enable the log cleaner +# and the maximum retention days that the INFO level logs will be kept. +# +# if set to -1, that means to disable the log cleaner. +# if set to 0, all previous INFO level logs will be immediately removed. +# if set to between 0 to INT_MAX, that means it will retent latest N(log-retention-days) day logs. + +# By default the log-retention-days is -1. +log-retention-days -1 + +# When running in daemonize mode, kvrocks writes a PID file in ${CONFIG_DIR}/kvrocks.pid by +# default. You can specify a custom pid file location here. +# pidfile /var/run/kvrocks.pid +pidfile "" + +# You can configure a slave instance to accept writes or not. Writing against +# a slave instance may be useful to store some ephemeral data (because data +# written on a slave will be easily deleted after resync with the master) but +# may also cause problems if clients are writing to it because of a +# misconfiguration. +slave-read-only yes + +# The slave priority is an integer number published by Kvrocks in the INFO output. +# It is used by Redis Sentinel in order to select a slave to promote into a +# master if the master is no longer working correctly. +# +# A slave with a low priority number is considered better for promotion, so +# for instance if there are three slave with priority 10, 100, 25 Sentinel will +# pick the one with priority 10, that is the lowest. +# +# However a special priority of 0 marks the replica as not able to perform the +# role of master, so a slave with priority of 0 will never be selected by +# Redis Sentinel for promotion. +# +# By default the priority is 100. +slave-priority 100 + +# TCP listen() backlog. +# +# In high requests-per-second environments you need an high backlog in order +# to avoid slow clients connections issues. Note that the Linux kernel +# will silently truncate it to the value of /proc/sys/net/core/somaxconn so +# make sure to raise both the value of somaxconn and tcp_max_syn_backlog +# in order to Get the desired effect. +tcp-backlog 511 + +# If the master is an old version, it may have specified replication threads +# that use 'port + 1' as listening port, but in new versions, we don't use +# extra port to implement replication. In order to allow the new replicas to +# copy old masters, you should indicate that the master uses replication port +# or not. +# If yes, that indicates master uses replication port and replicas will connect +# to 'master's listening port + 1' when synchronization. +# If no, that indicates master doesn't use replication port and replicas will +# connect 'master's listening port' when synchronization. +master-use-repl-port no + +# Currently, master only checks sequence number when replica asks for PSYNC, +# that is not enough since they may have different replication histories even +# the replica asking sequence is in the range of the master current WAL. +# +# We design 'Replication Sequence ID' PSYNC, we add unique replication id for +# every write batch (the operation of each command on the storage engine), so +# the combination of replication id and sequence is unique for write batch. +# The master can identify whether the replica has the same replication history +# by checking replication id and sequence. +# +# By default, it is not enabled since this stricter check may easily lead to +# full synchronization. +use-rsid-psync no + +# Master-Slave replication. Use slaveof to make a kvrocks instance a copy of +# another kvrocks server. A few things to understand ASAP about kvrocks replication. +# +# 1) Kvrocks replication is asynchronous, but you can configure a master to +# stop accepting writes if it appears to be not connected with at least +# a given number of slaves. +# 2) Kvrocks slaves are able to perform a partial resynchronization with the +# master if the replication link is lost for a relatively small amount of +# time. You may want to configure the replication backlog size (see the next +# sections of this file) with a sensible value depending on your needs. +# 3) Replication is automatic and does not need user intervention. After a +# network partition slaves automatically try to reconnect to masters +# and resynchronize with them. +# +# slaveof +# slaveof 127.0.0.1 6379 + +# When a slave loses its connection with the master, or when the replication +# is still in progress, the slave can act in two different ways: +# +# 1) if slave-serve-stale-data is set to 'yes' (the default) the slave will +# still reply to client requests, possibly with out-of-date data, or the +# data set may just be empty if this is the first synchronization. +# +# 2) if slave-serve-stale-data is set to 'no' the slave will reply with +# an error "SYNC with master in progress" to all kinds of commands +# but to INFO and SLAVEOF. +# +slave-serve-stale-data yes + +# To guarantee slave's data safe and serve when it is in full synchronization +# state, slave still keep itself data. But this way needs to occupy much disk +# space, so we provide a way to reduce disk occupation, slave will delete itself +# entire database before fetching files from master during full synchronization. +# If you want to enable this way, you can set 'slave-delete-db-before-fullsync' +# to yes, but you must know that database will be lost if master is down during +# full synchronization, unless you have a backup of database. +# +# This option is similar redis replicas RDB diskless load option: +# repl-diskless-load on-empty-db +# +# Default: no +slave-empty-db-before-fullsync no + +# A Kvrocks master is able to list the address and port of the attached +# replicas in different ways. For example the "INFO replication" section +# offers this information, which is used, among other tools, by +# Redis Sentinel in order to discover replica instances. +# Another place where this info is available is in the output of the +# "ROLE" command of a master. +# +# The listed IP address and port normally reported by a replica is +# obtained in the following way: +# +# IP: The address is auto detected by checking the peer address +# of the socket used by the replica to connect with the master. +# +# Port: The port is communicated by the replica during the replication +# handshake, and is normally the port that the replica is using to +# listen for connections. +# +# However when port forwarding or Network Address Translation (NAT) is +# used, the replica may actually be reachable via different IP and port +# pairs. The following two options can be used by a replica in order to +# report to its master a specific set of IP and port, so that both INFO +# and ROLE will report those values. +# +# There is no need to use both the options if you need to override just +# the port or the IP address. +# +# replica-announce-ip 5.5.5.5 +# replica-announce-port 1234 + +# If replicas need full synchronization with master, master need to create +# checkpoint for feeding replicas, and replicas also stage a checkpoint of +# the master. If we also keep the backup, it maybe occupy extra disk space. +# You can enable 'purge-backup-on-fullsync' if disk is not sufficient, but +# that may cause remote backup copy failing. +# +# Default: no +purge-backup-on-fullsync no + +# The maximum allowed rate (in MB/s) that should be used by replication. +# If the rate exceeds max-replication-mb, replication will slow down. +# Default: 0 (i.e. no limit) +max-replication-mb 0 + +# The maximum allowed aggregated write rate of flush and compaction (in MB/s). +# If the rate exceeds max-io-mb, io will slow down. +# 0 is no limit +# Default: 0 +max-io-mb 0 + +# The maximum allowed space (in GB) that should be used by RocksDB. +# If the total size of the SST files exceeds max_allowed_space, writes to RocksDB will fail. +# Please see: https://github.com/facebook/rocksdb/wiki/Managing-Disk-Space-Utilization +# Default: 0 (i.e. no limit) +max-db-size 0 + +# The maximum backup to keep, server cron would run every minutes to check the num of current +# backup, and purge the old backup if exceed the max backup num to keep. If max-backup-to-keep +# is 0, no backup would be kept. But now, we only support 0 or 1. +max-backup-to-keep 1 + +# The maximum hours to keep the backup. If max-backup-keep-hours is 0, wouldn't purge any backup. +# default: 1 day +max-backup-keep-hours 24 + +# max-bitmap-to-string-mb use to limit the max size of bitmap to string transformation(MB). +# +# Default: 16 +max-bitmap-to-string-mb 16 + +# Whether to enable SCAN-like cursor compatible with Redis. +# If enabled, the cursor will be unsigned 64-bit integers. +# If disabled, the cursor will be a string. +# Default: no +redis-cursor-compatible no + +################################## TLS ################################### + +# By default, TLS/SSL is disabled, i.e. `tls-port` is set to 0. +# To enable it, `tls-port` can be used to define TLS-listening ports. +# tls-port 0 + +# Configure a X.509 certificate and private key to use for authenticating the +# server to connected clients, masters or cluster peers. +# These files should be PEM formatted. +# +# tls-cert-file kvrocks.crt +# tls-key-file kvrocks.key + +# If the key file is encrypted using a passphrase, it can be included here +# as well. +# +# tls-key-file-pass secret + +# Configure a CA certificate(s) bundle or directory to authenticate TLS/SSL +# clients and peers. Kvrocks requires an explicit configuration of at least one +# of these, and will not implicitly use the system wide configuration. +# +# tls-ca-cert-file ca.crt +# tls-ca-cert-dir /etc/ssl/certs + +# By default, clients on a TLS port are required +# to authenticate using valid client side certificates. +# +# If "no" is specified, client certificates are not required and not accepted. +# If "optional" is specified, client certificates are accepted and must be +# valid if provided, but are not required. +# +# tls-auth-clients no +# tls-auth-clients optional + +# By default, only TLSv1.2 and TLSv1.3 are enabled and it is highly recommended +# that older formally deprecated versions are kept disabled to reduce the attack surface. +# You can explicitly specify TLS versions to support. +# Allowed values are case insensitive and include "TLSv1", "TLSv1.1", "TLSv1.2", +# "TLSv1.3" (OpenSSL >= 1.1.1) or any combination. +# To enable only TLSv1.2 and TLSv1.3, use: +# +# tls-protocols "TLSv1.2 TLSv1.3" + +# Configure allowed ciphers. See the ciphers(1ssl) manpage for more information +# about the syntax of this string. +# +# Note: this configuration applies only to <= TLSv1.2. +# +# tls-ciphers DEFAULT:!MEDIUM + +# Configure allowed TLSv1.3 ciphersuites. See the ciphers(1ssl) manpage for more +# information about the syntax of this string, and specifically for TLSv1.3 +# ciphersuites. +# +# tls-ciphersuites TLS_CHACHA20_POLY1305_SHA256 + +# When choosing a cipher, use the server's preference instead of the client +# preference. By default, the server follows the client's preference. +# +# tls-prefer-server-ciphers yes + +# By default, TLS session caching is enabled to allow faster and less expensive +# reconnections by clients that support it. Use the following directive to disable +# caching. +# +# tls-session-caching no + +# Change the default number of TLS sessions cached. A zero value sets the cache +# to unlimited size. The default size is 20480. +# +# tls-session-cache-size 5000 + +# Change the default timeout of cached TLS sessions. The default timeout is 300 +# seconds. +# +# tls-session-cache-timeout 60 + +# By default, a replica does not attempt to establish a TLS connection +# with its master. +# +# Use the following directive to enable TLS on replication links. +# +# tls-replication yes + +################################## SLOW LOG ################################### + +# The Kvrocks Slow Log is a mechanism to log queries that exceeded a specified +# execution time. The execution time does not include the I/O operations +# like talking with the client, sending the reply and so forth, +# but just the time needed to actually execute the command (this is the only +# stage of command execution where the thread is blocked and can not serve +# other requests in the meantime). +# +# You can configure the slow log with two parameters: one tells Kvrocks +# what is the execution time, in microseconds, to exceed in order for the +# command to get logged, and the other parameter is the length of the +# slow log. When a new command is logged the oldest one is removed from the +# queue of logged commands. + +# The following time is expressed in microseconds, so 1000000 is equivalent +# to one second. Note that -1 value disables the slow log, while +# a value of zero forces the logging of every command. +slowlog-log-slower-than 100000 + +# There is no limit to this length. Just be aware that it will consume memory. +# You can reclaim memory used by the slow log with SLOWLOG RESET. +slowlog-max-len 128 + +# If you run kvrocks from upstart or systemd, kvrocks can interact with your +# supervision tree. Options: +# supervised no - no supervision interaction +# supervised upstart - signal upstart by putting kvrocks into SIGSTOP mode +# supervised systemd - signal systemd by writing READY=1 to $NOTIFY_SOCKET +# supervised auto - detect upstart or systemd method based on +# UPSTART_JOB or NOTIFY_SOCKET environment variables +# Note: these supervision methods only signal "process is ready." +# They do not enable continuous liveness pings back to your supervisor. +supervised no + +################################## PERF LOG ################################### + +# The Kvrocks Perf Log is a mechanism to log queries' performance context that +# exceeded a specified execution time. This mechanism uses rocksdb's +# Perf Context and IO Stats Context, Please see: +# https://github.com/facebook/rocksdb/wiki/Perf-Context-and-IO-Stats-Context +# +# This mechanism is enabled when profiling-sample-commands is not empty and +# profiling-sample-ratio greater than 0. +# It is important to note that this mechanism affects performance, but it is +# useful for troubleshooting performance bottlenecks, so it should only be +# enabled when performance problems occur. + +# The name of the commands you want to record. Must be original name of +# commands supported by Kvrocks. Use ',' to separate multiple commands and +# use '*' to record all commands supported by Kvrocks. +# Example: +# - Single command: profiling-sample-commands get +# - Multiple commands: profiling-sample-commands get,mget,hget +# +# Default: empty +# profiling-sample-commands "" + +# Ratio of the samples would be recorded. It is a number between 0 and 100. +# We simply use the rand to determine whether to record the sample or not. +# +# Default: 0 +profiling-sample-ratio 0 + +# There is no limit to this length. Just be aware that it will consume memory. +# You can reclaim memory used by the perf log with PERFLOG RESET. +# +# Default: 256 +profiling-sample-record-max-len 256 + +# profiling-sample-record-threshold-ms use to tell the kvrocks when to record. +# +# Default: 100 millisecond +profiling-sample-record-threshold-ms 100 + +################################## CRON ################################### + +# Compact Scheduler, auto compact at schedule time +# time expression format is the same as crontab(currently only support * and int) +# e.g. compact-cron 0 3 * * * 0 4 * * * +# would compact the db at 3am and 4am everyday +# compact-cron 0 3 * * * + +# The hour range that compaction checker would be active +# e.g. compaction-checker-range 0-7 means compaction checker would be worker between +# 0-7am every day. +compaction-checker-range 0-7 + +# When the compaction checker is triggered, the db will periodically pick the SST file +# with the highest "deleted percentage" (i.e. the percentage of deleted keys in the SST +# file) to compact, in order to free disk space. +# However, if a specific SST file was created more than "force-compact-file-age" seconds +# ago, and its percentage of deleted keys is higher than +# "force-compact-file-min-deleted-percentage", it will be forcely compacted as well. + +# Default: 172800 seconds; Range: [60, INT64_MAX]; +# force-compact-file-age 172800 +# Default: 10 %; Range: [1, 100]; +# force-compact-file-min-deleted-percentage 10 + +# Bgsave scheduler, auto bgsave at scheduled time +# time expression format is the same as crontab(currently only support * and int) +# e.g. bgsave-cron 0 3 * * * 0 4 * * * +# would bgsave the db at 3am and 4am every day + +# Command renaming. +# +# It is possible to change the name of dangerous commands in a shared +# environment. For instance, the KEYS command may be renamed into something +# hard to guess so that it will still be available for internal-use tools +# but not available for general clients. +# +# Example: +# +# rename-command KEYS b840fc02d524045429941cc15f59e41cb7be6c52 +# +# It is also possible to completely kill a command by renaming it into +# an empty string: +# +# rename-command KEYS "" + +################################ MIGRATE ##################################### +# If the network bandwidth is completely consumed by the migration task, +# it will affect the availability of kvrocks. To avoid this situation, +# migrate-speed is adopted to limit the migrating speed. +# Migrating speed is limited by controlling the duration between sending data, +# the duration is calculated by: 1000000 * migrate-pipeline-size / migrate-speed (us). +# Value: [0,INT_MAX], 0 means no limit +# +# Default: 4096 +migrate-speed 4096 + +# In order to reduce data transmission times and improve the efficiency of data migration, +# pipeline is adopted to send multiple data at once. Pipeline size can be set by this option. +# Value: [1, INT_MAX], it can't be 0 +# +# Default: 16 +migrate-pipeline-size 16 + +# In order to reduce the write forbidden time during migrating slot, we will migrate the incremental +# data several times to reduce the amount of incremental data. Until the quantity of incremental +# data is reduced to a certain threshold, slot will be forbidden write. The threshold is set by +# this option. +# Value: [1, INT_MAX], it can't be 0 +# +# Default: 10000 +migrate-sequence-gap 10000 + +################################ ROCKSDB ##################################### + +# Specify the capacity of column family block cache. A larger block cache +# may make requests faster while more keys would be cached. Max Size is 400*1024. +# Default: 4096MB +rocksdb.block_cache_size 4096 + +# A global cache for table-level rows in RocksDB. If almost always point +# lookups, enlarging row cache may improve read performance. Otherwise, +# if we enlarge this value, we can lessen metadata/subkey block cache size. +# +# Default: 0 (disabled) +rocksdb.row_cache_size 0 + +# Number of open files that can be used by the DB. You may need to +# increase this if your database has a large working set. Value -1 means +# files opened are always kept open. You can estimate number of files based +# on target_file_size_base and target_file_size_multiplier for level-based +# compaction. For universal-style compaction, you can usually set it to -1. +# Default: 8096 +rocksdb.max_open_files 8096 + +# Amount of data to build up in memory (backed by an unsorted log +# on disk) before converting to a sorted on-disk file. +# +# Larger values increase performance, especially during bulk loads. +# Up to max_write_buffer_number write buffers may be held in memory +# at the same time, +# so you may wish to adjust this parameter to control memory usage. +# Also, a larger write buffer will result in a longer recovery time +# the next time the database is opened. +# +# Note that write_buffer_size is enforced per column family. +# See db_write_buffer_size for sharing memory across column families. + +# default is 64MB +rocksdb.write_buffer_size 64 + +# Target file size for compaction, target file size for Level N can be calculated +# by target_file_size_base * (target_file_size_multiplier ^ (L-1)) +# +# Default: 128MB +rocksdb.target_file_size_base 128 + +# The maximum number of write buffers that are built up in memory. +# The default and the minimum number is 2, so that when 1 write buffer +# is being flushed to storage, new writes can continue to the other +# write buffer. +# If max_write_buffer_number > 3, writing will be slowed down to +# options.delayed_write_rate if we are writing to the last write buffer +# allowed. +rocksdb.max_write_buffer_number 4 + +# Maximum number of concurrent background jobs (compactions and flushes). +# For backwards compatibility we will set `max_background_jobs = +# max_background_compactions + max_background_flushes` in the case where user +# sets at least one of `max_background_compactions` or `max_background_flushes` +# (we replace -1 by 1 in case one option is unset). +rocksdb.max_background_jobs 4 + +# DEPRECATED: it is automatically decided based on the value of rocksdb.max_background_jobs +# Maximum number of concurrent background compaction jobs, submitted to +# the default LOW priority thread pool. +rocksdb.max_background_compactions -1 + +# DEPRECATED: it is automatically decided based on the value of rocksdb.max_background_jobs +# Maximum number of concurrent background memtable flush jobs, submitted by +# default to the HIGH priority thread pool. If the HIGH priority thread pool +# is configured to have zero threads, flush jobs will share the LOW priority +# thread pool with compaction jobs. +rocksdb.max_background_flushes -1 + +# This value represents the maximum number of threads that will +# concurrently perform a compaction job by breaking it into multiple, +# smaller ones that are run simultaneously. +# Default: 2 +rocksdb.max_sub_compactions 2 + +# In order to limit the size of WALs, RocksDB uses DBOptions::max_total_wal_size +# as the trigger of column family flush. Once WALs exceed this size, RocksDB +# will start forcing the flush of column families to allow deletion of some +# oldest WALs. This config can be useful when column families are updated at +# non-uniform frequencies. If there's no size limit, users may need to keep +# really old WALs when the infrequently-updated column families hasn't flushed +# for a while. +# +# In kvrocks, we use multiple column families to store metadata, subkeys, etc. +# If users always use string type, but use list, hash and other complex data types +# infrequently, there will be a lot of old WALs if we don't set size limit +# (0 by default in rocksdb), because rocksdb will dynamically choose the WAL size +# limit to be [sum of all write_buffer_size * max_write_buffer_number] * 4 if set to 0. +# +# Moreover, you should increase this value if you already set rocksdb.write_buffer_size +# to a big value, to avoid influencing the effect of rocksdb.write_buffer_size and +# rocksdb.max_write_buffer_number. +# +# default is 512MB +rocksdb.max_total_wal_size 512 + +# We implement the replication with rocksdb WAL, it would trigger full sync when the seq was out of range. +# wal_ttl_seconds and wal_size_limit_mb would affect how archived logs will be deleted. +# If WAL_ttl_seconds is not 0, then WAL files will be checked every WAL_ttl_seconds / 2 and those that +# are older than WAL_ttl_seconds will be deleted# +# +# Default: 3 Hours +rocksdb.wal_ttl_seconds 10800 + +# If WAL_ttl_seconds is 0 and WAL_size_limit_MB is not 0, +# WAL files will be checked every 10 min and if total size is greater +# then WAL_size_limit_MB, they will be deleted starting with the +# earliest until size_limit is met. All empty files will be deleted +# Default: 16GB +rocksdb.wal_size_limit_mb 16384 + +# Approximate size of user data packed per block. Note that the +# block size specified here corresponds to uncompressed data. The +# actual size of the unit read from disk may be smaller if +# compression is enabled. +# +# Default: 16KB +rocksdb.block_size 16384 + +# Indicating if we'd put index/filter blocks to the block cache +# +# Default: yes +rocksdb.cache_index_and_filter_blocks yes + +# Specify the compression to use. Only compress level greater +# than 2 to improve performance. +# Accept value: "no", "snappy", "lz4", "zstd", "zlib" +# default snappy +rocksdb.compression snappy + +# If non-zero, we perform bigger reads when doing compaction. If you're +# running RocksDB on spinning disks, you should set this to at least 2MB. +# That way RocksDB's compaction is doing sequential instead of random reads. +# When non-zero, we also force new_table_reader_for_compaction_inputs to +# true. +# +# Default: 2 MB +rocksdb.compaction_readahead_size 2097152 + +# he limited write rate to DB if soft_pending_compaction_bytes_limit or +# level0_slowdown_writes_trigger is triggered. + +# If the value is 0, we will infer a value from `rater_limiter` value +# if it is not empty, or 16MB if `rater_limiter` is empty. Note that +# if users change the rate in `rate_limiter` after DB is opened, +# `delayed_write_rate` won't be adjusted. +# +rocksdb.delayed_write_rate 0 +# If enable_pipelined_write is true, separate write thread queue is +# maintained for WAL write and memtable write. +# +# Default: no +rocksdb.enable_pipelined_write no + +# Soft limit on number of level-0 files. We start slowing down writes at this +# point. A value <0 means that no writing slow down will be triggered by +# number of files in level-0. +# +# Default: 20 +rocksdb.level0_slowdown_writes_trigger 20 + +# Maximum number of level-0 files. We stop writes at this point. +# +# Default: 40 +rocksdb.level0_stop_writes_trigger 40 + +# Number of files to trigger level-0 compaction. +# +# Default: 4 +rocksdb.level0_file_num_compaction_trigger 4 + +# if not zero, dump rocksdb.stats to LOG every stats_dump_period_sec +# +# Default: 0 +rocksdb.stats_dump_period_sec 0 + +# if yes, the auto compaction would be disabled, but the manual compaction remain works +# +# Default: no +rocksdb.disable_auto_compactions no + +# BlobDB(key-value separation) is essentially RocksDB for large-value use cases. +# Since 6.18.0, The new implementation is integrated into the RocksDB core. +# When set, large values (blobs) are written to separate blob files, and only +# pointers to them are stored in SST files. This can reduce write amplification +# for large-value use cases at the cost of introducing a level of indirection +# for reads. Please see: https://github.com/facebook/rocksdb/wiki/BlobDB. +# +# Note that when enable_blob_files is set to yes, BlobDB-related configuration +# items will take effect. +# +# Default: no +rocksdb.enable_blob_files no + +# The size of the smallest value to be stored separately in a blob file. Values +# which have an uncompressed size smaller than this threshold are stored alongside +# the keys in SST files in the usual fashion. +# +# Default: 4096 byte, 0 means that all values are stored in blob files +rocksdb.min_blob_size 4096 + +# The size limit for blob files. When writing blob files, a new file is +# opened once this limit is reached. +# +# Default: 268435456 bytes +rocksdb.blob_file_size 268435456 + +# Enables garbage collection of blobs. Valid blobs residing in blob files +# older than a cutoff get relocated to new files as they are encountered +# during compaction, which makes it possible to clean up blob files once +# they contain nothing but obsolete/garbage blobs. +# See also rocksdb.blob_garbage_collection_age_cutoff below. +# +# Default: yes +rocksdb.enable_blob_garbage_collection yes + +# The percentage cutoff in terms of blob file age for garbage collection. +# Blobs in the oldest N blob files will be relocated when encountered during +# compaction, where N = (garbage_collection_cutoff/100) * number_of_blob_files. +# Note that this value must belong to [0, 100]. +# +# Default: 25 +rocksdb.blob_garbage_collection_age_cutoff 25 + + +# The purpose of the following three options are to dynamically adjust the upper limit of +# the data that each layer can store according to the size of the different +# layers of the LSM. Enabling this option will bring some improvements in +# deletion efficiency and space amplification, but it will lose a certain +# amount of read performance. +# If you want to know more details about Levels' Target Size, you can read RocksDB wiki: +# https://github.com/facebook/rocksdb/wiki/Leveled-Compaction#levels-target-size +# +# Default: yes +rocksdb.level_compaction_dynamic_level_bytes yes + +# The total file size of level-1 sst. +# +# Default: 268435456 bytes +rocksdb.max_bytes_for_level_base 268435456 + +# Multiplication factor for the total file size of L(n+1) layers. +# This option is a double type number in RocksDB, but kvrocks is +# not support the double data type number yet, so we use integer +# number instead of double currently. +# +# Default: 10 +rocksdb.max_bytes_for_level_multiplier 10 + +# This feature only takes effect in Iterators and MultiGet. +# If yes, RocksDB will try to read asynchronously and in parallel as much as possible to hide IO latency. +# In iterators, it will prefetch data asynchronously in the background for each file being iterated on. +# In MultiGet, it will read the necessary data blocks from those files in parallel as much as possible. + +# Default no +rocksdb.read_options.async_io no + +# If yes, the write will be flushed from the operating system +# buffer cache before the write is considered complete. +# If this flag is enabled, writes will be slower. +# If this flag is disabled, and the machine crashes, some recent +# rites may be lost. Note that if it is just the process that +# crashes (i.e., the machine does not reboot), no writes will be +# lost even if sync==false. +# +# Default: no +rocksdb.write_options.sync no + +# If yes, writes will not first go to the write ahead log, +# and the write may get lost after a crash. +# +# Default: no +rocksdb.write_options.disable_wal no + +# If enabled and we need to wait or sleep for the write request, fails +# immediately. +# +# Default: no +rocksdb.write_options.no_slowdown no + +# If enabled, write requests are of lower priority if compaction is +# behind. In this case, no_slowdown = true, the request will be canceled +# immediately. Otherwise, it will be slowed down. +# The slowdown value is determined by RocksDB to guarantee +# it introduces minimum impacts to high priority writes. +# +# Default: no +rocksdb.write_options.low_pri no + +# If enabled, this writebatch will maintain the last insert positions of each +# memtable as hints in concurrent write. It can improve write performance +# in concurrent writes if keys in one writebatch are sequential. +# +# Default: no +rocksdb.write_options.memtable_insert_hint_per_batch no + + +# Support RocksDB auto-tune rate limiter for the background IO +# if enabled, Rate limiter will limit the compaction write if flush write is high +# Please see https://rocksdb.org/blog/2017/12/18/17-auto-tuned-rate-limiter.html +# +# Default: yes +rocksdb.rate_limiter_auto_tuned yes + +################################ NAMESPACE ##################################### +# namespace.test change.me diff --git a/tests/testdata/zset-ziplist.rdb b/tests/testdata/zset-ziplist.rdb new file mode 100644 index 0000000000000000000000000000000000000000..d5549475ad731d9ec22f1cf40f80e0e964e7d491 GIT binary patch literal 135 zcmWG?b@2=~FfcUu#aWb^l3A=+Af^< zi=(tSHAOc!HTO`!7aoRRJc;>fx&=k4iMdHRsRtPTG5ls^;9;pMPA!p?WME(r17a2i d2Il;{RMt;S%q8Xdte^k?m*4dF!_4}5k^p!pFg^eP literal 0 HcmV?d00001 diff --git a/utils/redis2kvrocks/README.md b/utils/redis2kvrocks/README.md new file mode 100644 index 00000000000..6c03150f803 --- /dev/null +++ b/utils/redis2kvrocks/README.md @@ -0,0 +1,15 @@ +# redis2kvrocks + +`redis2kvrocks` is used to load an RDB file into an empty kvrocks database. Therefore, you must ensure that the database directory does not contain any pre-existing databases. + +## Usage + +```bash + ./build/redis2kvrocks -c kvrocks.conf -r /path/dump.rdb -ns namespace +``` + +The default namespace to load is '__namespace'. If the RDB file contains multiple databases, the namespace will be set to `_`, where `n` is the database number ranging from 1 to `n`. The password will be the same as the namespace. This setup requires the `requirepass` setting in the kvrocks configuration file and will rewrite the config file with new namespaces. Database 0 will use the namespace ``. + +Current support rdb version is `10`, not support: +1. load stream type; +2. load module type; diff --git a/utils/redis2kvrocks/main.cc b/utils/redis2kvrocks/main.cc new file mode 100644 index 00000000000..c5873677283 --- /dev/null +++ b/utils/redis2kvrocks/main.cc @@ -0,0 +1,252 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +#include +#include +#include +#include +#include + +#include +#include + +#include "common/rdb_stream.h" +#include "config.h" +#include "scope_exit.h" +#include "storage/rdb.h" +#include "storage/storage.h" +#include "vendor/crc64.h" +#include "version.h" + +namespace google { +bool Symbolize(void *pc, char *out, size_t out_size); +} // namespace google + +Server *srv = nullptr; + +extern "C" void SignalHandler(int sig) { + LOG(INFO) << "Signal " << sig << " received, force exit now"; + exit(1); +} + +std::ostream &PrintVersion(std::ostream &os) { // TODO should define in common/utils + os << "redis2kvrocks "; + + if (VERSION != "unstable") { + os << "version "; + } + + os << VERSION; + + if (!GIT_COMMIT.empty()) { + os << " (commit " << GIT_COMMIT << ")"; + } + + return os; +} + +struct NewOpt { + friend auto &operator<<(std::ostream &os, NewOpt) { return os << std::string(4, ' ') << std::setw(32); } +} new_opt; + +static void PrintUsage(const char *program) { + std::cout << program << " load redis's rdb file to kvrocks, only used to create db from rdb file" << std::endl + << "Usage:" << std::endl + << std::left << new_opt << "-c, --config " + << "set config file to , or `-` for stdin" << std::endl + << new_opt << "-v, --version" + << "print version information" << std::endl + << new_opt << "-h, --help" + << "print this help message" << std::endl + << new_opt << "-- " + << "overwrite specific config option to " << std::endl + << new_opt << "-r --rdb " + << "load rdb file to rocksdb" << std::endl + << new_opt << "-ns --namespace [namespace]" + << "config the namespace to load, default ''" << std::endl; +} + +struct R2KCLIOptions { + CLIOptions options; + std::string rdb_file; + std::string namespace_; + R2KCLIOptions() : namespace_(kDefaultNamespace){}; +}; + +static R2KCLIOptions ParseCommandLineOptions(int argc, char **argv) { + using namespace std::string_view_literals; + R2KCLIOptions opts; + + for (int i = 1; i < argc; ++i) { + if ((argv[i] == "-c"sv || argv[i] == "--config"sv) && i + 1 < argc) { + opts.options.conf_file = argv[++i]; + } else if (argv[i] == "-v"sv || argv[i] == "--version"sv) { + std::cout << PrintVersion << std::endl; + std::exit(0); + } else if (argv[i] == "-h"sv || argv[i] == "--help"sv) { + PrintUsage(*argv); + std::exit(0); + } else if (std::string_view(argv[i], 2) == "--" && std::string_view(argv[i]).size() > 2 && i + 1 < argc) { + auto key = std::string_view(argv[i] + 2); + opts.options.cli_options.emplace_back(key, argv[++i]); + } else if ((argv[i] == "-r"sv || argv[i] == "--rdb"sv) && i + 1 < argc) { + opts.rdb_file = argv[++i]; + } else if ((argv[i] == "-ns"sv || argv[i] == "--rdb"sv) && i + 1 < argc) { + opts.namespace_ = argv[++i]; + } else { + PrintUsage(*argv); + std::exit(1); + } + } + + return opts; +} + +static void InitGoogleLog(const Config *config) { + FLAGS_minloglevel = config->log_level; + FLAGS_max_log_size = 100; + FLAGS_logbufsecs = 0; + + if (util::EqualICase(config->log_dir, "stdout")) { + for (int level = google::INFO; level <= google::FATAL; level++) { + google::SetLogDestination(level, ""); + } + FLAGS_stderrthreshold = google::ERROR; + FLAGS_logtostdout = true; + } else { + FLAGS_log_dir = config->log_dir + "/"; + if (config->log_retention_days != -1) { + google::EnableLogCleaner(config->log_retention_days); + } + } +} + +extern "C" void SegvHandler(int sig, siginfo_t *info, void *secret) { + void *trace[100]; + + LOG(ERROR) << "======= Ooops! " << PrintVersion << " got signal: " << strsignal(sig) << " (" << sig << ") ======="; + int trace_size = backtrace(trace, sizeof(trace) / sizeof(void *)); + char **messages = backtrace_symbols(trace, trace_size); + + size_t max_msg_len = 0; + for (int i = 1; i < trace_size; ++i) { + auto msg_len = strlen(messages[i]); + if (msg_len > max_msg_len) { + max_msg_len = msg_len; + } + } + + for (int i = 1; i < trace_size; ++i) { + char func_info[1024] = {}; + if (google::Symbolize(trace[i], func_info, sizeof(func_info) - 1)) { + LOG(ERROR) << std::left << std::setw(static_cast(max_msg_len)) << messages[i] << " " << func_info; + } else { + LOG(ERROR) << messages[i]; + } + } + + struct sigaction act; + /* Make sure we exit with the right signal at the end. So for instance + * the core will be dumped if enabled. + */ + sigemptyset(&act.sa_mask); + /* When the SA_SIGINFO flag is set in sa_flags then sa_sigaction + * is used. Otherwise, sa_handler is used + */ + act.sa_flags = SA_NODEFER | SA_ONSTACK | SA_RESETHAND; + act.sa_handler = SIG_DFL; + sigaction(sig, &act, nullptr); + kill(getpid(), sig); +} + +void SetupSigSegvAction() { + struct sigaction act; + + sigemptyset(&act.sa_mask); + /* When the SA_SIGINFO flag is set in sa_flags then sa_sigaction + * is used. Otherwise, sa_handler is used */ + act.sa_flags = SA_NODEFER | SA_ONSTACK | SA_RESETHAND | SA_SIGINFO; + act.sa_sigaction = SegvHandler; + sigaction(SIGSEGV, &act, nullptr); + sigaction(SIGBUS, &act, nullptr); + sigaction(SIGFPE, &act, nullptr); + sigaction(SIGILL, &act, nullptr); + sigaction(SIGABRT, &act, nullptr); + + act.sa_flags = SA_NODEFER | SA_ONSTACK | SA_RESETHAND; + act.sa_handler = SignalHandler; + sigaction(SIGTERM, &act, nullptr); + sigaction(SIGINT, &act, nullptr); +} + +bool checkDBExist(Config *config) { + engine::Storage storage(config); + auto s = storage.Open(true); + return s.IsOK(); +} + +int main(int argc, char *argv[]) { + google::InitGoogleLogging("redis2kvrocks"); + + SetupSigSegvAction(); + + auto opts = ParseCommandLineOptions(argc, argv); + + Config config; + Status s = config.Load(opts.options); + if (!s.IsOK()) { + std::cout << "Failed to load config. Error: " << s.Msg() << std::endl; + return 1; + } + + crc64_init(); + InitGoogleLog(&config); + LOG(INFO) << PrintVersion; + + // rdb file can only be loaded into empty database. + if (checkDBExist(&config)) { + LOG(ERROR) << " Database already exist, can't load from rdb file!"; + return 1; + } + + engine::Storage storage(&config); + s = storage.Open(); + if (!s.IsOK()) { + LOG(ERROR) << "Failed to open: " << s.Msg(); + return 1; + } + + auto stream_ptr = std::make_shared(opts.rdb_file); + s = stream_ptr->Open(); + if (!s.IsOK()) { + LOG(ERROR) << "Failed to open: " << s.Msg(); + return 1; + } + RDB rdb(&storage, &config, opts.namespace_, stream_ptr); + s = rdb.LoadRdb(); + if (!s.IsOK()) { + LOG(ERROR) << "Failed to load rdb: " << s.Msg(); + return 1; + } + LOG(INFO) << "Load rdb file success!"; + storage.CloseDB(); + LOG(INFO) << "Close DB success, redis2kvrocks exit!"; + return 0; +} diff --git a/x.py b/x.py index 08641028158..0ff59d0b097 100755 --- a/x.py +++ b/x.py @@ -125,7 +125,7 @@ def build(dir: str, jobs: Optional[int], ghproxy: bool, ninja: bool, unittest: b if skip_build: return - target = ["kvrocks", "kvrocks2redis"] + target = ["kvrocks", "kvrocks2redis", "redis2kvrocks"] if unittest: target.append("unittest") @@ -145,6 +145,7 @@ def get_source_files(dir: Path) -> List[str]: *glob(str(dir / "tests/cppunit/**/*.cc"), recursive=True), *glob(str(dir / "utils/kvrocks2redis/**/*.h"), recursive=True), *glob(str(dir / "utils/kvrocks2redis/**/*.cc"), recursive=True), + *glob(str(dir / "utils/redis2kvrocks/**/*.cc"), recursive=True), ] From 75355dedbc7605dba28ebf10d10b1bb5529c5a4b Mon Sep 17 00:00:00 2001 From: xqing2003 Date: Tue, 10 Oct 2023 12:01:03 +0800 Subject: [PATCH 2/4] refactor(load rdb file): use command: rdb load [NX] [DB index] to load rdb file, delete redis2kvrocks and rdb files --- CMakeLists.txt | 8 +- src/commands/cmd_server.cc | 45 ++++- src/storage/rdb.cc | 61 ++++--- src/storage/rdb.h | 10 +- tests/cppunit/rdb_test.cc | 222 ++++++++++++++++++++--- tests/testdata/corrupt_empty_keys.rdb | Bin 280 -> 0 bytes tests/testdata/corrupt_ziplist.rdb | Bin 1415 -> 0 bytes tests/testdata/encodings.rdb | Bin 667 -> 0 bytes tests/testdata/encodings_ver10.rdb | Bin 707 -> 0 bytes tests/testdata/hash-ziplist.rdb | Bin 137 -> 0 bytes tests/testdata/hash-zipmap.rdb | Bin 35 -> 0 bytes tests/testdata/list-quicklist.rdb | Bin 123 -> 0 bytes tests/testdata/scriptbackup.rdb | Bin 225 -> 0 bytes tests/testdata/zset-ziplist.rdb | Bin 135 -> 0 bytes utils/redis2kvrocks/README.md | 15 -- utils/redis2kvrocks/main.cc | 252 -------------------------- x.py | 3 +- 17 files changed, 285 insertions(+), 331 deletions(-) delete mode 100644 tests/testdata/corrupt_empty_keys.rdb delete mode 100644 tests/testdata/corrupt_ziplist.rdb delete mode 100644 tests/testdata/encodings.rdb delete mode 100644 tests/testdata/encodings_ver10.rdb delete mode 100644 tests/testdata/hash-ziplist.rdb delete mode 100644 tests/testdata/hash-zipmap.rdb delete mode 100644 tests/testdata/list-quicklist.rdb delete mode 100644 tests/testdata/scriptbackup.rdb delete mode 100644 tests/testdata/zset-ziplist.rdb delete mode 100644 utils/redis2kvrocks/README.md delete mode 100644 utils/redis2kvrocks/main.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index db2a37614ce..f5d5f6f6876 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -253,10 +253,4 @@ file(GLOB_RECURSE TESTS_SRCS tests/cppunit/*.cc) add_executable(unittest ${TESTS_SRCS}) target_include_directories(unittest PRIVATE tests/cppunit) -target_link_libraries(unittest PRIVATE kvrocks_objs gtest_main ${EXTERNAL_LIBS}) - -# redis2kvrocks tool -file(GLOB_RECURSE REDIS2KVROCKS_SRCS utils/redis2kvrocks/*.cc) -add_executable(redis2kvrocks ${REDIS2KVROCKS_SRCS}) - -target_link_libraries(redis2kvrocks PRIVATE kvrocks_objs ${EXTERNAL_LIBS}) \ No newline at end of file +target_link_libraries(unittest PRIVATE kvrocks_objs gtest_main ${EXTERNAL_LIBS}) \ No newline at end of file diff --git a/src/commands/cmd_server.cc b/src/commands/cmd_server.cc index 89237772d6b..c5dee98f3fa 100644 --- a/src/commands/cmd_server.cc +++ b/src/commands/cmd_server.cc @@ -1061,7 +1061,7 @@ class CommandRestore : public Commander { } auto stream_ptr = std::make_shared(args_[3]); - RDB rdb(svr->storage, svr->GetConfig(), conn->GetNamespace(), stream_ptr); + RDB rdb(svr->storage, conn->GetNamespace(), stream_ptr); auto s = rdb.Restore(args_[1], args_[3], ttl_ms_); if (!s.IsOK()) return {Status::RedisExecErr, s.Msg()}; *output = redis::SimpleString("OK"); @@ -1074,6 +1074,46 @@ class CommandRestore : public Commander { uint64_t ttl_ms_ = 0; }; +// command format: rdb load [NX] [DB index] +class CommandRdb : public Commander { + public: + Status Parse(const std::vector &args) override { + CommandParser parser(args, 3); + std::string_view ttl_flag, set_flag; + while (parser.Good()) { + if (parser.EatEqICase("NX")) { + is_nx_ = true; + } else if (parser.EatEqICase("DB")) { + db_index_ = GET_OR_RET(parser.TakeInt()); + } else { + return {Status::RedisParseErr, errInvalidSyntax}; + } + } + + return Status::OK(); + } + + Status Execute(Server *svr, Connection *conn, std::string *output) override { + rocksdb::Status db_status; + redis::Database redis(svr->storage, conn->GetNamespace()); + auto type = args_[1]; + auto path = args_[2]; + + auto stream_ptr = std::make_shared(path); + GET_OR_RET(stream_ptr->Open()); + + RDB rdb(svr->storage, conn->GetNamespace(), stream_ptr); + GET_OR_RET(rdb.LoadRdb(db_index_, is_nx_)); + + *output = redis::SimpleString("OK"); + return Status::OK(); + } + + private: + bool is_nx_ = false; + uint32_t db_index_ = 0; +}; + REDIS_REGISTER_COMMANDS(MakeCmdAttr("auth", 2, "read-only ok-loading", 0, 0, 0), MakeCmdAttr("ping", -1, "read-only", 0, 0, 0), MakeCmdAttr("select", 2, "read-only", 0, 0, 0), @@ -1107,6 +1147,7 @@ REDIS_REGISTER_COMMANDS(MakeCmdAttr("auth", 2, "read-only ok-loadin MakeCmdAttr("lastsave", 1, "read-only", 0, 0, 0), MakeCmdAttr("flushbackup", 1, "read-only no-script", 0, 0, 0), MakeCmdAttr("slaveof", 3, "read-only exclusive no-script", 0, 0, 0), - MakeCmdAttr("stats", 1, "read-only", 0, 0, 0), ) + MakeCmdAttr("stats", 1, "read-only", 0, 0, 0), + MakeCmdAttr("rdb", -3, "write exclusive", 0, 0, 0), ) } // namespace redis diff --git a/src/storage/rdb.cc b/src/storage/rdb.cc index 2417091596a..8433617f9bb 100644 --- a/src/storage/rdb.cc +++ b/src/storage/rdb.cc @@ -545,7 +545,7 @@ bool RDB::isEmptyRedisObject(const RedisObjValue &value) { } // Load RDB file: copy from redis/src/rdb.c:branch 7.0, 76b9c13d. -Status RDB::LoadRdb() { +Status RDB::LoadRdb(uint32_t db_index, bool is_nx) { char buf[1024] = {0}; GET_OR_RETWITHLOG(stream_->Read(buf, 9)); buf[9] = '\0'; @@ -566,7 +566,8 @@ Status RDB::LoadRdb() { int64_t load_keys = 0; int64_t empty_keys_skipped = 0; auto now = util::GetTimeStampMS(); - std::string origin_ns = ns_; // current namespace, when select db, will change ns to origin_ns + "_" + db_id + uint32_t db_id = 0; + uint64_t skip_exist_keys = 0; while (true) { auto type = GET_OR_RETWITHLOG(loadRdbType()); if (type == RDBOpcodeExpireTime) { @@ -586,15 +587,7 @@ Status RDB::LoadRdb() { } else if (type == RDBOpcodeEof) { break; } else if (type == RDBOpcodeSelectDB) { - auto db_id = GET_OR_RETWITHLOG(loadObjectLen(nullptr)); - if (db_id != 0) { // change namespace to ns + "_" + db_id - ns_ = origin_ns + "_" + std::to_string(db_id); - } else { // use origin namespace - ns_ = origin_ns; - } - LOG(INFO) << "select db: " << db_id << ", change to namespace: " << ns_; - // add namespace, password is the same as the namespace(expect the default namespace) - GET_OR_RET(addNamespace(ns_, ns_)); + db_id = GET_OR_RETWITHLOG(loadObjectLen(nullptr)); continue; } else if (type == RDBOpcodeResizeDB) { // not use in kvrocks, hint redis for hash table resize GET_OR_RETWITHLOG(loadObjectLen(nullptr)); // db_size @@ -625,6 +618,10 @@ Status RDB::LoadRdb() { auto key = GET_OR_RETWITHLOG(LoadStringObject()); auto value = GET_OR_RETWITHLOG(loadRdbObject(type, key)); + if (db_index != db_id) { // skip db not match + continue; + } + if (isEmptyRedisObject(value)) { // compatible with empty value /* Since we used to have bug that could lead to empty keys * (See #8453), we rather not fail when empty key is encountered @@ -640,10 +637,22 @@ Status RDB::LoadRdb() { continue; } + if (is_nx) { // only load not exist key + auto s = exist(key); + if (!s.IsNotFound()) { + skip_exist_keys++; // skip it even it's not okay + if (!s.ok()) { + LOG(ERROR) << "check key " << key << " exist failed: " << s.ToString(); + } + continue; + } + } + auto ret = saveRdbObject(type, key, value, expire_time); - load_keys++; if (!ret.IsOK()) { LOG(WARNING) << "save rdb object key " << key << " failed: " << ret.Msg(); + } else { + load_keys++; } } @@ -660,27 +669,29 @@ Status RDB::LoadRdb() { } } + std::string skip_info = (is_nx ? ", exist keys skipped: " + std::to_string(skip_exist_keys) : ""); + if (empty_keys_skipped > 0) { LOG(INFO) << "Done loading RDB, keys loaded: " << load_keys << ", keys expired:" << expire_keys - << ", empty keys skipped: " << empty_keys_skipped; + << ", empty keys skipped: " << empty_keys_skipped << skip_info; } else { - LOG(INFO) << "Done loading RDB, keys loaded: " << load_keys << ", keys expired:" << expire_keys; + LOG(INFO) << "Done loading RDB, keys loaded: " << load_keys << ", keys expired:" << expire_keys << skip_info; } return Status::OK(); } -Status RDB::addNamespace(const std::string &ns, const std::string &token) { - if (ns == kDefaultNamespace) { - return Status::OK(); - } - - std::string old_token; - auto s = config_->GetNamespace(ns, &old_token); - if (s.IsOK()) { // namespace exist, use old token +rocksdb::Status RDB::exist(const std::string &key) { + int cnt = 0; + std::vector keys; + keys.emplace_back(key); + redis::Database redis(storage_, ns_); + auto s = redis.Exists(keys, &cnt); + if (!s.ok()) { return s; } - - // add new namespace - return config_->AddNamespace(ns, token); + if (cnt == 0) { + return rocksdb::Status::NotFound(); + } + return rocksdb::Status::OK(); } \ No newline at end of file diff --git a/src/storage/rdb.h b/src/storage/rdb.h index 2ee56ddb383..8faabce5cd4 100644 --- a/src/storage/rdb.h +++ b/src/storage/rdb.h @@ -27,7 +27,6 @@ #include #include -#include "config/config.h" #include "status.h" #include "types/redis_zset.h" @@ -66,8 +65,8 @@ using RedisObjValue = class RDB { public: - explicit RDB(engine::Storage *storage, Config *config, std::string ns, std::shared_ptr stream) - : storage_(storage), config_(config), ns_(std::move(ns)), stream_(std::move(stream)){}; + explicit RDB(engine::Storage *storage, std::string ns, std::shared_ptr stream) + : storage_(storage), ns_(std::move(ns)), stream_(std::move(stream)){}; ~RDB() = default; Status VerifyPayloadChecksum(const std::string_view &payload); @@ -99,11 +98,10 @@ class RDB { StatusOr> LoadListWithQuickList(int type); // Load rdb - Status LoadRdb(); + Status LoadRdb(uint32_t db_index, bool is_nx = false); private: engine::Storage *storage_; - Config *config_; std::string ns_; std::shared_ptr stream_; @@ -118,7 +116,7 @@ class RDB { Status saveRdbObject(int type, const std::string &key, const RedisObjValue &obj, uint64_t ttl_ms); StatusOr loadTime(); StatusOr loadMillisecondTime(int rdb_version); - Status addNamespace(const std::string &ns, const std::string &token); + rocksdb::Status exist(const std::string &key); /*0-5 is the basic type of Redis objects and 9-21 is the encoding type of Redis objects. Redis allow basic is 0-7 and 6/7 is for the module type which we don't support here.*/ diff --git a/tests/cppunit/rdb_test.cc b/tests/cppunit/rdb_test.cc index 6c3b69660dd..c0fa8371614 100644 --- a/tests/cppunit/rdb_test.cc +++ b/tests/cppunit/rdb_test.cc @@ -42,6 +42,160 @@ const std::string test_data_path = "../tests/testdata/"; const std::string test_config = "test.conf"; +// RDB test data, copy from Redis's tests/asset/*rdb, not shellcode. + +// zset-ziplist.rdb +const char zset_ziplist_payload[] = + "\x52\x45\x44\x49\x53\x30\x30\x31\x30\xfa\x09\x72\x65\x64\x69\x73\x2d\x76\x65\x72\x0b\x32\x35\x35\x2e" + "\x32\x35\x35\x2e\x32\x35\x35\xfa\x0a\x72\x65\x64\x69\x73\x2d\x62\x69\x74\x73\xc0\x40\xfa\x05\x63\x74" + "\x69\x6d\x65\xc2\x62\xb7\x13\x61\xfa\x08\x75\x73\x65\x64\x2d\x6d\x65\x6d\xc2\x50\xf4\x0c\x00\xfa\x0c" + "\x61\x6f\x66\x2d\x70\x72\x65\x61\x6d\x62\x6c\x65\xc0\x00\xfe\x00\xfb\x01\x00\x0c\x04\x7a\x73\x65\x74" + "\x19\x19\x00\x00\x00\x16\x00\x00\x00\x04\x00\x00\x03\x6f\x6e\x65\x05\xf2\x02\x03\x74\x77\x6f\x05\xf3" + "\xff\xff\x1f\xb2\xfd\xf0\x99\x7f\x9e\x19"; + +// corrupt_empty_keys.rdb +const char corrupt_empty_keys_payload[] = + "\x52\x45\x44\x49\x53\x30\x30\x31\x30\xfa\x09\x72\x65\x64\x69\x73\x2d\x76\x65\x72\x0b\x32\x35\x35\x2e" + "\x32\x35\x35\x2e\x32\x35\x35\xfa\x0a\x72\x65\x64\x69\x73\x2d\x62\x69\x74\x73\xc0\x40\xfa\x05\x63\x74" + "\x69\x6d\x65\xc2\x7a\x18\x15\x61\xfa\x08\x75\x73\x65\x64\x2d\x6d\x65\x6d\xc2\x80\x31\x10\x00\xfa\x0c" + "\x61\x6f\x66\x2d\x70\x72\x65\x61\x6d\x62\x6c\x65\xc0\x00\xfe\x00\xfb\x09\x00\x02\x03\x73\x65\x74\x00" + "\x04\x04\x68\x61\x73\x68\x00\x0a\x0c\x6c\x69\x73\x74\x5f\x7a\x69\x70\x6c\x69\x73\x74\x0b\x0b\x00\x00" + "\x00\x0a\x00\x00\x00\x00\x00\xff\x05\x04\x7a\x73\x65\x74\x00\x11\x0d\x7a\x73\x65\x74\x5f\x6c\x69\x73" + "\x74\x70\x61\x63\x6b\x07\x07\x00\x00\x00\x00\x00\xff\x10\x0c\x68\x61\x73\x68\x5f\x7a\x69\x70\x6c\x69" + "\x73\x74\x07\x07\x00\x00\x00\x00\x00\xff\x0e\x0e\x6c\x69\x73\x74\x5f\x71\x75\x69\x63\x6b\x6c\x69\x73" + "\x74\x00\x0c\x0c\x7a\x73\x65\x74\x5f\x7a\x69\x70\x6c\x69\x73\x74\x0b\x0b\x00\x00\x00\x0a\x00\x00\x00" + "\x00\x00\xff\x0e\x1c\x6c\x69\x73\x74\x5f\x71\x75\x69\x63\x6b\x6c\x69\x73\x74\x5f\x65\x6d\x70\x74\x79" + "\x5f\x7a\x69\x70\x6c\x69\x73\x74\x01\x0b\x0b\x00\x00\x00\x0a\x00\x00\x00\x00\x00\xff\xff\xf0\xf5\x06" + "\xdd\xc6\x6e\x61\x83"; + +// encodings.rdb +const char encodings_rdb_payload[] = + "\x52\x45\x44\x49\x53\x30\x30\x30\x34\xfe\x00\x03\x04\x7a\x73\x65\x74\x0c\x02\x62\x62\x02\x32\x30\x02\x63\x63\x02" + "\x33\x30" + "\x03\x62\x62\x62\x03\x32\x30\x30\x04\x62\x62\x62\x62\x0a\x35\x30\x30\x30\x30\x30\x30\x30\x30\x30\x03\x63\x63\x63" + "\x03\x33" + "\x30\x30\x04\x63\x63\x63\x63\x09\x31\x32\x33\x34\x35\x36\x37\x38\x39\x01\x61\x01\x31\x02\x61\x61\x02\x31\x30\x01" + "\x62\x01" + "\x32\x03\x61\x61\x61\x03\x31\x30\x30\x01\x63\x01\x33\x04\x61\x61\x61\x61\x04\x31\x30\x30\x30\x0b\x0c\x73\x65\x74" + "\x5f\x7a" + "\x69\x70\x70\x65\x64\x5f\x31\x10\x02\x00\x00\x00\x04\x00\x00\x00\x01\x00\x02\x00\x03\x00\x04\x00\x0a\x0b\x6c\x69" + "\x73\x74" + "\x5f\x7a\x69\x70\x70\x65\x64\x30\x30\x00\x00\x00\x25\x00\x00\x00\x08\x00\x00\xc0\x01\x00\x04\xc0\x02\x00\x04\xc0" + "\x03\x00" + "\x04\x01\x61\x03\x01\x62\x03\x01\x63\x03\xd0\xa0\x86\x01\x00\x06\xe0\x00\xbc\xa0\x65\x01\x00\x00\x00\xff\x00\x06" + "\x73\x74" + "\x72\x69\x6e\x67\x0b\x48\x65\x6c\x6c\x6f\x20\x57\x6f\x72\x6c\x64\x00\x0c\x63\x6f\x6d\x70\x72\x65\x73\x73\x69\x62" + "\x6c\x65" + "\xc3\x09\x40\x89\x01\x61\x61\xe0\x7c\x00\x01\x61\x61\x0b\x0c\x73\x65\x74\x5f\x7a\x69\x70\x70\x65\x64\x5f\x32\x18" + "\x04\x00" + "\x00\x00\x04\x00\x00\x00\xa0\x86\x01\x00\x40\x0d\x03\x00\xe0\x93\x04\x00\x80\x1a\x06\x00\x00\x06\x6e\x75\x6d\x62" + "\x65\x72" + "\xc0\x0a\x0b\x0c\x73\x65\x74\x5f\x7a\x69\x70\x70\x65\x64\x5f\x33\x38\x08\x00\x00\x00\x06\x00\x00\x00\x00\xca\x9a" + "\x3b\x00" + "\x00\x00\x00\x00\x94\x35\x77\x00\x00\x00\x00\x00\x5e\xd0\xb2\x00\x00\x00\x00\x00\x28\x6b\xee\x00\x00\x00\x00\x00" + "\xf2\x05" + "\x2a\x01\x00\x00\x00\x00\xbc\xa0\x65\x01\x00\x00\x00\x02\x03\x73\x65\x74\x08\x0a\x36\x30\x30\x30\x30\x30\x30\x30" + "\x30\x30" + "\xc2\xa0\x86\x01\x00\x01\x61\xc0\x01\x01\x62\xc0\x02\x01\x63\xc0\x03\x01\x04\x6c\x69\x73\x74\x18\xc0\x01\xc0\x02" + "\xc0\x03" + "\x01\x61\x01\x62\x01\x63\xc2\xa0\x86\x01\x00\x0a\x36\x30\x30\x30\x30\x30\x30\x30\x30\x30\xc0\x01\xc0\x02\xc0\x03" + "\x01\x61" + "\x01\x62\x01\x63\xc2\xa0\x86\x01\x00\x0a\x36\x30\x30\x30\x30\x30\x30\x30\x30\x30\xc0\x01\xc0\x02\xc0\x03\x01\x61" + "\x01\x62" + "\x01\x63\xc2\xa0\x86\x01\x00\x0a\x36\x30\x30\x30\x30\x30\x30\x30\x30\x30\x0d\x0b\x68\x61\x73\x68\x5f\x7a\x69\x70" + "\x70\x65" + "\x64\x20\x20\x00\x00\x00\x1b\x00\x00\x00\x06\x00\x00\x01\x61\x03\xc0\x01\x00\x04\x01\x62\x03\xc0\x02\x00\x04\x01" + "\x63\x03" + "\xc0\x03\x00\xff\x0c\x0b\x7a\x73\x65\x74\x5f\x7a\x69\x70\x70\x65\x64\x20\x20\x00\x00\x00\x1b\x00\x00\x00\x06\x00" + "\x00\x01" + "\x61\x03\xc0\x01\x00\x04\x01\x62\x03\xc0\x02\x00\x04\x01\x63\x03\xc0\x03\x00\xff\x04\x04\x68\x61\x73\x68\x0b\x01" + "\x62\xc0" + "\x02\x02\x61\x61\xc0\x0a\x01\x63\xc0\x03\x03\x61\x61\x61\xc0\x64\x02\x62\x62\xc0\x14\x02\x63\x63\xc0\x1e\x03\x62" + "\x62\x62" + "\xc1\xc8\x00\x03\x63\x63\x63\xc1\x2c\x01\x03\x64\x64\x64\xc1\x90\x01\x03\x65\x65\x65\x0a\x35\x30\x30\x30\x30\x30" + "\x30\x30" + "\x30\x30\x01\x61\xc0\x01\xff"; + +// hash-ziplist.rdb +const char hash_ziplist_payload[] = + "\x52\x45\x44\x49\x53\x30\x30\x30\x39\xfa\x09\x72\x65\x64\x69\x73\x2d\x76\x65\x72\x0b\x32\x35\x35\x2e\x32\x35\x35" + "\x2e\x32" + "\x35\x35\xfa\x0a\x72\x65\x64\x69\x73\x2d\x62\x69\x74\x73\xc0\x40\xfa\x05\x63\x74\x69\x6d\x65\xc2\xc8\x5c\x96\x60" + "\xfa\x08" + "\x75\x73\x65\x64\x2d\x6d\x65\x6d\xc2\x90\xad\x0c\x00\xfa\x0c\x61\x6f\x66\x2d\x70\x72\x65\x61\x6d\x62\x6c\x65\xc0" + "\x00\xfe" + "\x00\xfb\x01\x00\x0d\x04\x68\x61\x73\x68\x1b\x1b\x00\x00\x00\x16\x00\x00\x00\x04\x00\x00\x02\x66\x31\x04\x02\x76" + "\x31\x04" + "\x02\x66\x32\x04\x02\x76\x32\xff\xff\x4f\x9c\xd1\xfd\x16\x69\x98\x83"; + +// hash-zipmap.rdb +const char hash_zipmap_payload[] = + "\x52\x45\x44\x49\x53\x30\x30\x30\x33\xfe\x00\x09\x04\x68\x61\x73\x68\x10\x02\x02\x66\x31\x02\x00\x76\x31\x02\x66" + "\x32\x02" + "\x00\x76\x32\xff\xff"; + +// list-quicklist.rdb +const char list_quicklist_payload[] = + "\x52\x45\x44\x49\x53\x30\x30\x30\x38\xfa\x09\x72\x65\x64\x69\x73\x2d\x76\x65\x72\x05\x34\x2e\x30\x2e\x39\xfa\x0a" + "\x72\x65" + "\x64\x69\x73\x2d\x62\x69\x74\x73\xc0\x40\xfa\x05\x63\x74\x69\x6d\x65\xc2\x9f\x06\x26\x61\xfa\x08\x75\x73\x65\x64" + "\x2d\x6d" + "\x65\x6d\xc2\x80\x92\x07\x00\xfa\x0c\x61\x6f\x66\x2d\x70\x72\x65\x61\x6d\x62\x6c\x65\xc0\x00\xfe\x00\xfb\x02\x00" + "\x0e\x04" + "\x6c\x69\x73\x74\x01\x0d\x0d\x00\x00\x00\x0a\x00\x00\x00\x01\x00\x00\xf8\xff\x00\x01\x78\xc0\x07\xff\x35\x72\xf8" + "\x54\x1a" + "\xc4\xd7\x40"; + +// dumped from redis-server 7.0, sourced from the 'encodings.rdb' file. +const char encodings_ver10_rdb_payload[] = + "\x52\x45\x44\x49\x53\x30\x30\x31\x30\xfa\x09\x72\x65\x64\x69\x73\x2d\x76\x65\x72\x05\x37\x2e\x30\x2e\x33\xfa\x0a" + "\x72\x65" + "\x64\x69\x73\x2d\x62\x69\x74\x73\xc0\x40\xfa\x05\x63\x74\x69\x6d\x65\xc2\x62\x65\x23\x65\xfa\x08\x75\x73\x65\x64" + "\x2d\x6d" + "\x65\x6d\xc2\x28\x4f\x0e\x00\xfa\x08\x61\x6f\x66\x2d\x62\x61\x73\x65\xc0\x00\xfe\x00\xfb\x0d\x00\x11\x04\x7a\x73" + "\x65\x74" + "\x40\x64\x64\x00\x00\x00\x18\x00\x81\x61\x02\x01\x01\x81\x62\x02\x02\x01\x81\x63\x02\x03\x01\x82\x61\x61\x03\x0a" + "\x01\x82" + "\x62\x62\x03\x14\x01\x82\x63\x63\x03\x1e\x01\x83\x61\x61\x61\x04\x64\x01\x83\x62\x62\x62\x04\xc0\xc8\x02\x83\x63" + "\x63\x63" + "\x04\xc1\x2c\x02\x84\x61\x61\x61\x61\x05\xc3\xe8\x02\x84\x63\x63\x63\x63\x05\xf3\x15\xcd\x5b\x07\x05\x84\x62\x62" + "\x62\x62" + "\x05\xf4\x00\xf2\x05\x2a\x01\x00\x00\x00\x09\xff\x11\x0b\x7a\x73\x65\x74\x5f\x7a\x69\x70\x70\x65\x64\x16\x16\x00" + "\x00\x00" + "\x06\x00\x81\x61\x02\x01\x01\x81\x62\x02\x02\x01\x81\x63\x02\x03\x01\xff\x10\x04\x68\x61\x73\x68\x40\x56\x56\x00" + "\x00\x00" + "\x16\x00\x81\x62\x02\x02\x01\x82\x61\x61\x03\x0a\x01\x81\x63\x02\x03\x01\x83\x61\x61\x61\x04\x64\x01\x82\x62\x62" + "\x03\x14" + "\x01\x82\x63\x63\x03\x1e\x01\x83\x62\x62\x62\x04\xc0\xc8\x02\x83\x63\x63\x63\x04\xc1\x2c\x02\x83\x64\x64\x64\x04" + "\xc1\x90" + "\x02\x83\x65\x65\x65\x04\xf4\x00\xf2\x05\x2a\x01\x00\x00\x00\x09\x81\x61\x02\x01\x01\xff\x0b\x0c\x73\x65\x74\x5f" + "\x7a\x69" + "\x70\x70\x65\x64\x5f\x32\x18\x04\x00\x00\x00\x04\x00\x00\x00\xa0\x86\x01\x00\x40\x0d\x03\x00\xe0\x93\x04\x00\x80" + "\x1a\x06" + "\x00\x12\x04\x6c\x69\x73\x74\x01\x02\xc3\x2b\x40\x61\x1f\x61\x00\x00\x00\x18\x00\x01\x01\x02\x01\x03\x01\x81\x61" + "\x02\x81" + "\x62\x02\x81\x63\x02\xf2\xa0\x86\x01\x04\xf4\x00\xbc\xa0\x65\x01\x20\x1e\x00\x09\xe0\x32\x1d\x01\x09\xff\x02\x03" + "\x73\x65" + "\x74\x08\xc0\x02\xc0\x01\xc2\xa0\x86\x01\x00\x01\x62\x0a\x36\x30\x30\x30\x30\x30\x30\x30\x30\x30\xc0\x03\x01\x61" + "\x01\x63" + "\x00\x06\x6e\x75\x6d\x62\x65\x72\xc0\x0a\x0b\x0c\x73\x65\x74\x5f\x7a\x69\x70\x70\x65\x64\x5f\x31\x10\x02\x00\x00" + "\x00\x04" + "\x00\x00\x00\x01\x00\x02\x00\x03\x00\x04\x00\x10\x0b\x68\x61\x73\x68\x5f\x7a\x69\x70\x70\x65\x64\x16\x16\x00\x00" + "\x00\x06" + "\x00\x81\x61\x02\x01\x01\x81\x62\x02\x02\x01\x81\x63\x02\x03\x01\xff\x00\x0c\x63\x6f\x6d\x70\x72\x65\x73\x73\x69" + "\x62\x6c" + "\x65\xc3\x09\x40\x89\x01\x61\x61\xe0\x7c\x00\x01\x61\x61\x00\x06\x73\x74\x72\x69\x6e\x67\x0b\x48\x65\x6c\x6c\x6f" + "\x20\x57" + "\x6f\x72\x6c\x64\x0b\x0c\x73\x65\x74\x5f\x7a\x69\x70\x70\x65\x64\x5f\x33\x38\x08\x00\x00\x00\x06\x00\x00\x00\x00" + "\xca\x9a" + "\x3b\x00\x00\x00\x00\x00\x94\x35\x77\x00\x00\x00\x00\x00\x5e\xd0\xb2\x00\x00\x00\x00\x00\x28\x6b\xee\x00\x00\x00" + "\x00\x00" + "\xf2\x05\x2a\x01\x00\x00\x00\x00\xbc\xa0\x65\x01\x00\x00\x00\x12\x0b\x6c\x69\x73\x74\x5f\x7a\x69\x70\x70\x65\x64" + "\x01\x02" + "\x25\x25\x00\x00\x00\x08\x00\x01\x01\x02\x01\x03\x01\x81\x61\x02\x81\x62\x02\x81\x63\x02\xf2\xa0\x86\x01\x04\xf4" + "\x00\xbc" + "\xa0\x65\x01\x00\x00\x00\x09\xff\xff\x58\xe7\x62\x56\x52\x9b\xdf\x6c"; + class RDBTest : public testing::Test { public: RDBTest(const RDBTest &) = delete; @@ -68,17 +222,18 @@ class RDBTest : public testing::Test { ASSERT_TRUE(s.IsOK()); } - void TearDown() override { ASSERT_TRUE(clearDBDir(config_.db_dir)); } - - static std::string getRdbFullPath(const std::string &name) { return test_data_path + name; } + void TearDown() override { + std::remove(tmp_rdb_.c_str()); + ASSERT_TRUE(clearDBDir(config_.db_dir)); + } void loadRdb(const std::string &path) { auto stream_ptr = std::make_shared(path); auto s = stream_ptr->Open(); ASSERT_TRUE(s.IsOK()); - RDB rdb(storage_, &config_, ns_, stream_ptr); - s = rdb.LoadRdb(); + RDB rdb(storage_, ns_, stream_ptr); + s = rdb.LoadRdb(0); ASSERT_TRUE(s.IsOK()); } @@ -159,11 +314,25 @@ class RDBTest : public testing::Test { ASSERT_TRUE(s.ok()); } + static void writeTestRdbFile(const std::string &name, const char *data, size_t len) { + std::ofstream out_file(name, std::ios::out | std::ios::binary); + if (!out_file) { + ASSERT_TRUE(false); + } + + out_file.write(data, static_cast(len)); + if (!out_file) { + ASSERT_TRUE(false); + } + out_file.close(); + } + void encodingDataCheck(); engine::Storage *storage_; std::string ns_; Config config_; + std::string tmp_rdb_; private: static bool clearDBDir(const std::string &path) { @@ -234,22 +403,27 @@ void RDBTest::encodingDataCheck() { zsetCheck("zset_zipped", zset_zipped_expect); } +std::string ConvertToString(const char *data, size_t len) { return {data, data + len}; } + TEST_F(RDBTest, LoadEncodings) { - for (const auto &entry : std::filesystem::directory_iterator(test_data_path)) { - std::string filename = entry.path().filename().string(); - if (filename.rfind("encodings", 0) != 0) { - continue; - } - auto full_path = getRdbFullPath(filename); - loadRdb(full_path); + std::map datas; + datas.insert({"encodings.rdb", ConvertToString(encodings_rdb_payload, sizeof(encodings_rdb_payload) - 1)}); + datas.insert( + {"encodings_ver10.rdb", ConvertToString(encodings_ver10_rdb_payload, sizeof(encodings_ver10_rdb_payload) - 1)}); + for (const auto &kv : datas) { + tmp_rdb_ = kv.first; + writeTestRdbFile(tmp_rdb_, kv.second.data(), kv.second.size()); + loadRdb(tmp_rdb_); encodingDataCheck(); flushDB(); + std::remove(tmp_rdb_.c_str()); } } TEST_F(RDBTest, LoadHashZipMap) { - auto full_path = getRdbFullPath("hash-zipmap.rdb"); - loadRdb(full_path); + tmp_rdb_ = "hash-zipmap.rdb"; + writeTestRdbFile(tmp_rdb_, hash_zipmap_payload, sizeof(hash_zipmap_payload) - 1); + loadRdb(tmp_rdb_); // hash std::map hash_expect = { @@ -260,8 +434,9 @@ TEST_F(RDBTest, LoadHashZipMap) { } TEST_F(RDBTest, LoadHashZipList) { - auto full_path = getRdbFullPath("hash-ziplist.rdb"); - loadRdb(full_path); + tmp_rdb_ = "hash-ziplist.rdb"; + writeTestRdbFile(tmp_rdb_, hash_ziplist_payload, sizeof(hash_ziplist_payload) - 1); + loadRdb(tmp_rdb_); // hash std::map hash_expect = { @@ -272,8 +447,9 @@ TEST_F(RDBTest, LoadHashZipList) { } TEST_F(RDBTest, LoadListQuickList) { - auto full_path = getRdbFullPath("list-quicklist.rdb"); - loadRdb(full_path); + tmp_rdb_ = "list-quicklist.rdb"; + writeTestRdbFile(tmp_rdb_, list_quicklist_payload, sizeof(list_quicklist_payload) - 1); + loadRdb(tmp_rdb_); // list std::vector list_expect = {"7"}; @@ -281,8 +457,9 @@ TEST_F(RDBTest, LoadListQuickList) { } TEST_F(RDBTest, LoadZSetZipList) { - auto full_path = getRdbFullPath("zset-ziplist.rdb"); - loadRdb(full_path); + tmp_rdb_ = "zset-ziplist.rdb"; + writeTestRdbFile(tmp_rdb_, zset_ziplist_payload, sizeof(zset_ziplist_payload) - 1); + loadRdb(tmp_rdb_); // zset std::vector zset_expect = { @@ -293,8 +470,9 @@ TEST_F(RDBTest, LoadZSetZipList) { } TEST_F(RDBTest, LoadEmptyKeys) { - auto full_path = getRdbFullPath("corrupt_empty_keys.rdb"); - loadRdb(full_path); + tmp_rdb_ = "corrupt_empty_keys.rdb"; + writeTestRdbFile(tmp_rdb_, corrupt_empty_keys_payload, sizeof(corrupt_empty_keys_payload) - 1); + loadRdb(tmp_rdb_); /* corrupt_empty_keys.rdb contains 9 keys with empty value: "set" "hash" "list_ziplist" "zset" "zset_listpack" "hash_ziplist" "list_quicklist" "zset_ziplist" diff --git a/tests/testdata/corrupt_empty_keys.rdb b/tests/testdata/corrupt_empty_keys.rdb deleted file mode 100644 index 98b6a143a14bef9d7692ed3e2ce23174a408869f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 280 zcmWG?b@2=~FfcUu#aWb^l3A=H&s-48J)Un3#)GOBh&KG7^h37`S+H zGK)*%t1=5fG&eT`0|OTjGcf#TWvK$G6XXR^@gUiP#N=#tc8IJ14@f&qA54^w4{Tat qX=ZXZh{eFe1J;ejcs?0arSYk`1tpa*0~wKB^8dqEw!6pj5}N^cYgv~7 diff --git a/tests/testdata/corrupt_ziplist.rdb b/tests/testdata/corrupt_ziplist.rdb deleted file mode 100644 index b40ada8d634d094ba2f20d728f20b19596e3e7ef..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1415 zcmaJ>yH3ME5WE!T6@e)D2D(_g=jTyTKtT%yJ;@ao7$n<4$tUnlG*No~f$uO9#!1cy z*|V&CI%%|?ot<6X?e*2o9T6q~& zFfZ+FlG}X!`94

MK_2^wBOaS~?d==q)zU!&yf={A4d*K4oP!@_nV$d3(I}+y1iu z&()~2>C61Nvc;cTI$n)>btn+(lX1ELPr^eUtzCYvzbM9d*I!=`yS@i_ig{uQPjG&y zy{3)DaZjgVn9o|!Pgo}tVIIahnXzxo0py_+*QFHeClYX>E=5=;8Qd3{ZcloCh5}D2 z%q77(nK4c&o(mJkN%B);{rtPnVE+R>+<$}TNQr=_xGrg--&n|J!OsZiGYkI2!oEp< bM#3J39Y65RY~Brk>u~62;CyS^7jGLs8LoRE diff --git a/tests/testdata/encodings.rdb b/tests/testdata/encodings.rdb deleted file mode 100644 index 9fd9b705d16220065ee117a1c1c094f40fb122f2..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 667 zcmbVKu}UOC5UuKJ7oAzb-~v%NHW5S&dS+bpFfmX#Qw_|N?w&>$M|aur5EcU?;j)7> zGV&XY4SF>ZBR^rkz`zf1tzKQwOdP0r-Cfn)uU@~+^|g&HrPRU;knEK1xGIbhsS?(T zOp!5$Ql%uLiRxVU_K~%gGG1r2V@aAV)EAeQf1$=iXe|~B7q#x40{=g8Nhbr35aH0(af04| z31?FkfXdOIL*v>$`m`ZiW?H~$fQQSK0B})18Q{+2^#ErNo(A|lG8gy_c?x0;Mx(`{ zoa#1QiP|F?FVK2I8DyCB=mk$S8nlC&4|~3w8;|#Ox&QtGwHmXU=HND1)gUq}8+2xM zS?Yc@4yO2OwUpuPICQ~2@KI=m_~m`huJS+FRQ_l1RQDc;oztC1%JaPY56L?s)vH`;wZH<&56B04urfYGkq(PZZnX$ z(=em%Ml!Zt7XTX2jqs(E#Iaggwgb-@_i6xHXsQXt~ zpJc8Lf3Q9Yj`r*9mq(|xJ{=O;Z#d8{A-Nm*j9Fx@?|Pk%bnA7}D#sJ^n!3TG4ZGE% z-8z&ghZH5yIDUfY7)s*0uDbu4B~nWDD7<`GZcZHmtgkdw3MuB^O9XpbfzcZkUYx7I ztm=DFjBI$><`*~}XaFKZ1<8ew2<3pTko(6yiSu({j8@L0ku!zd^^~PJC>&8#uUY>` zsvt)PDw|t@kYQS#AX%=lF$n?#g_vq)(T0;-1yfEt=!7zgJmJfsVZTDoN6&#)s6=t- zZ9X;cNZ)VIKWK-(JCV40L#MMhn)t$0F{`#S;i_d2} z7e{GvYKm@dYVM&4Yk3%c@g(M_=@t~FCgvvPq#j`S$MBnxftMvCu{c9ont_2q42W46 c7?{!wS(wUzIL!!18~y+9Kj-3KvCJ9G0M4&2bN~PV diff --git a/tests/testdata/hash-zipmap.rdb b/tests/testdata/hash-zipmap.rdb deleted file mode 100644 index 27a42ed4bb494cee9d67b7f0feccce7e4136d50a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 35 pcmWG?b@2=~FfcIw$H2*wkyxA|z{Heh$iz@)$dqOTq>TRm2LPS*34j0q diff --git a/tests/testdata/list-quicklist.rdb b/tests/testdata/list-quicklist.rdb deleted file mode 100644 index a9101a1ade0d167e5c3a741827944e4e0ee012bc..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 123 zcmWG?b@2=~Ffg$A#aWb^l3A=OZT+eVEDxj7RgM}FgCGFO-)QoO*Txk zv`8~fGc_|ZH&3-RO-r^gOG-AeNHR7sGc`;!`o#^_k)M`UoLT}jfG06OO}7AKU{X#h zkk6b`ny6NkT2fk+rvP@AUUFhij)po=wYrXidRl%yh)PNP7 AH2?qr diff --git a/tests/testdata/zset-ziplist.rdb b/tests/testdata/zset-ziplist.rdb deleted file mode 100644 index d5549475ad731d9ec22f1cf40f80e0e964e7d491..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 135 zcmWG?b@2=~FfcUu#aWb^l3A=+Af^< zi=(tSHAOc!HTO`!7aoRRJc;>fx&=k4iMdHRsRtPTG5ls^;9;pMPA!p?WME(r17a2i d2Il;{RMt;S%q8Xdte^k?m*4dF!_4}5k^p!pFg^eP diff --git a/utils/redis2kvrocks/README.md b/utils/redis2kvrocks/README.md deleted file mode 100644 index 6c03150f803..00000000000 --- a/utils/redis2kvrocks/README.md +++ /dev/null @@ -1,15 +0,0 @@ -# redis2kvrocks - -`redis2kvrocks` is used to load an RDB file into an empty kvrocks database. Therefore, you must ensure that the database directory does not contain any pre-existing databases. - -## Usage - -```bash - ./build/redis2kvrocks -c kvrocks.conf -r /path/dump.rdb -ns namespace -``` - -The default namespace to load is '__namespace'. If the RDB file contains multiple databases, the namespace will be set to `_`, where `n` is the database number ranging from 1 to `n`. The password will be the same as the namespace. This setup requires the `requirepass` setting in the kvrocks configuration file and will rewrite the config file with new namespaces. Database 0 will use the namespace ``. - -Current support rdb version is `10`, not support: -1. load stream type; -2. load module type; diff --git a/utils/redis2kvrocks/main.cc b/utils/redis2kvrocks/main.cc deleted file mode 100644 index c5873677283..00000000000 --- a/utils/redis2kvrocks/main.cc +++ /dev/null @@ -1,252 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ - -#include -#include -#include -#include -#include - -#include -#include - -#include "common/rdb_stream.h" -#include "config.h" -#include "scope_exit.h" -#include "storage/rdb.h" -#include "storage/storage.h" -#include "vendor/crc64.h" -#include "version.h" - -namespace google { -bool Symbolize(void *pc, char *out, size_t out_size); -} // namespace google - -Server *srv = nullptr; - -extern "C" void SignalHandler(int sig) { - LOG(INFO) << "Signal " << sig << " received, force exit now"; - exit(1); -} - -std::ostream &PrintVersion(std::ostream &os) { // TODO should define in common/utils - os << "redis2kvrocks "; - - if (VERSION != "unstable") { - os << "version "; - } - - os << VERSION; - - if (!GIT_COMMIT.empty()) { - os << " (commit " << GIT_COMMIT << ")"; - } - - return os; -} - -struct NewOpt { - friend auto &operator<<(std::ostream &os, NewOpt) { return os << std::string(4, ' ') << std::setw(32); } -} new_opt; - -static void PrintUsage(const char *program) { - std::cout << program << " load redis's rdb file to kvrocks, only used to create db from rdb file" << std::endl - << "Usage:" << std::endl - << std::left << new_opt << "-c, --config " - << "set config file to , or `-` for stdin" << std::endl - << new_opt << "-v, --version" - << "print version information" << std::endl - << new_opt << "-h, --help" - << "print this help message" << std::endl - << new_opt << "-- " - << "overwrite specific config option to " << std::endl - << new_opt << "-r --rdb " - << "load rdb file to rocksdb" << std::endl - << new_opt << "-ns --namespace [namespace]" - << "config the namespace to load, default ''" << std::endl; -} - -struct R2KCLIOptions { - CLIOptions options; - std::string rdb_file; - std::string namespace_; - R2KCLIOptions() : namespace_(kDefaultNamespace){}; -}; - -static R2KCLIOptions ParseCommandLineOptions(int argc, char **argv) { - using namespace std::string_view_literals; - R2KCLIOptions opts; - - for (int i = 1; i < argc; ++i) { - if ((argv[i] == "-c"sv || argv[i] == "--config"sv) && i + 1 < argc) { - opts.options.conf_file = argv[++i]; - } else if (argv[i] == "-v"sv || argv[i] == "--version"sv) { - std::cout << PrintVersion << std::endl; - std::exit(0); - } else if (argv[i] == "-h"sv || argv[i] == "--help"sv) { - PrintUsage(*argv); - std::exit(0); - } else if (std::string_view(argv[i], 2) == "--" && std::string_view(argv[i]).size() > 2 && i + 1 < argc) { - auto key = std::string_view(argv[i] + 2); - opts.options.cli_options.emplace_back(key, argv[++i]); - } else if ((argv[i] == "-r"sv || argv[i] == "--rdb"sv) && i + 1 < argc) { - opts.rdb_file = argv[++i]; - } else if ((argv[i] == "-ns"sv || argv[i] == "--rdb"sv) && i + 1 < argc) { - opts.namespace_ = argv[++i]; - } else { - PrintUsage(*argv); - std::exit(1); - } - } - - return opts; -} - -static void InitGoogleLog(const Config *config) { - FLAGS_minloglevel = config->log_level; - FLAGS_max_log_size = 100; - FLAGS_logbufsecs = 0; - - if (util::EqualICase(config->log_dir, "stdout")) { - for (int level = google::INFO; level <= google::FATAL; level++) { - google::SetLogDestination(level, ""); - } - FLAGS_stderrthreshold = google::ERROR; - FLAGS_logtostdout = true; - } else { - FLAGS_log_dir = config->log_dir + "/"; - if (config->log_retention_days != -1) { - google::EnableLogCleaner(config->log_retention_days); - } - } -} - -extern "C" void SegvHandler(int sig, siginfo_t *info, void *secret) { - void *trace[100]; - - LOG(ERROR) << "======= Ooops! " << PrintVersion << " got signal: " << strsignal(sig) << " (" << sig << ") ======="; - int trace_size = backtrace(trace, sizeof(trace) / sizeof(void *)); - char **messages = backtrace_symbols(trace, trace_size); - - size_t max_msg_len = 0; - for (int i = 1; i < trace_size; ++i) { - auto msg_len = strlen(messages[i]); - if (msg_len > max_msg_len) { - max_msg_len = msg_len; - } - } - - for (int i = 1; i < trace_size; ++i) { - char func_info[1024] = {}; - if (google::Symbolize(trace[i], func_info, sizeof(func_info) - 1)) { - LOG(ERROR) << std::left << std::setw(static_cast(max_msg_len)) << messages[i] << " " << func_info; - } else { - LOG(ERROR) << messages[i]; - } - } - - struct sigaction act; - /* Make sure we exit with the right signal at the end. So for instance - * the core will be dumped if enabled. - */ - sigemptyset(&act.sa_mask); - /* When the SA_SIGINFO flag is set in sa_flags then sa_sigaction - * is used. Otherwise, sa_handler is used - */ - act.sa_flags = SA_NODEFER | SA_ONSTACK | SA_RESETHAND; - act.sa_handler = SIG_DFL; - sigaction(sig, &act, nullptr); - kill(getpid(), sig); -} - -void SetupSigSegvAction() { - struct sigaction act; - - sigemptyset(&act.sa_mask); - /* When the SA_SIGINFO flag is set in sa_flags then sa_sigaction - * is used. Otherwise, sa_handler is used */ - act.sa_flags = SA_NODEFER | SA_ONSTACK | SA_RESETHAND | SA_SIGINFO; - act.sa_sigaction = SegvHandler; - sigaction(SIGSEGV, &act, nullptr); - sigaction(SIGBUS, &act, nullptr); - sigaction(SIGFPE, &act, nullptr); - sigaction(SIGILL, &act, nullptr); - sigaction(SIGABRT, &act, nullptr); - - act.sa_flags = SA_NODEFER | SA_ONSTACK | SA_RESETHAND; - act.sa_handler = SignalHandler; - sigaction(SIGTERM, &act, nullptr); - sigaction(SIGINT, &act, nullptr); -} - -bool checkDBExist(Config *config) { - engine::Storage storage(config); - auto s = storage.Open(true); - return s.IsOK(); -} - -int main(int argc, char *argv[]) { - google::InitGoogleLogging("redis2kvrocks"); - - SetupSigSegvAction(); - - auto opts = ParseCommandLineOptions(argc, argv); - - Config config; - Status s = config.Load(opts.options); - if (!s.IsOK()) { - std::cout << "Failed to load config. Error: " << s.Msg() << std::endl; - return 1; - } - - crc64_init(); - InitGoogleLog(&config); - LOG(INFO) << PrintVersion; - - // rdb file can only be loaded into empty database. - if (checkDBExist(&config)) { - LOG(ERROR) << " Database already exist, can't load from rdb file!"; - return 1; - } - - engine::Storage storage(&config); - s = storage.Open(); - if (!s.IsOK()) { - LOG(ERROR) << "Failed to open: " << s.Msg(); - return 1; - } - - auto stream_ptr = std::make_shared(opts.rdb_file); - s = stream_ptr->Open(); - if (!s.IsOK()) { - LOG(ERROR) << "Failed to open: " << s.Msg(); - return 1; - } - RDB rdb(&storage, &config, opts.namespace_, stream_ptr); - s = rdb.LoadRdb(); - if (!s.IsOK()) { - LOG(ERROR) << "Failed to load rdb: " << s.Msg(); - return 1; - } - LOG(INFO) << "Load rdb file success!"; - storage.CloseDB(); - LOG(INFO) << "Close DB success, redis2kvrocks exit!"; - return 0; -} diff --git a/x.py b/x.py index 0ff59d0b097..08641028158 100755 --- a/x.py +++ b/x.py @@ -125,7 +125,7 @@ def build(dir: str, jobs: Optional[int], ghproxy: bool, ninja: bool, unittest: b if skip_build: return - target = ["kvrocks", "kvrocks2redis", "redis2kvrocks"] + target = ["kvrocks", "kvrocks2redis"] if unittest: target.append("unittest") @@ -145,7 +145,6 @@ def get_source_files(dir: Path) -> List[str]: *glob(str(dir / "tests/cppunit/**/*.cc"), recursive=True), *glob(str(dir / "utils/kvrocks2redis/**/*.h"), recursive=True), *glob(str(dir / "utils/kvrocks2redis/**/*.cc"), recursive=True), - *glob(str(dir / "utils/redis2kvrocks/**/*.cc"), recursive=True), ] From b47820ee32a80868827c17c5b1193493fded6350 Mon Sep 17 00:00:00 2001 From: xq2010 Date: Wed, 11 Oct 2023 13:02:58 +0800 Subject: [PATCH 3/4] Revert changes in CMakeLists.txt, fix test errors of RdbFileStream, refactor rdb_test, and remove testdatas --- CMakeLists.txt | 2 +- tests/cppunit/rdb_stream_test.cc | 23 +- tests/cppunit/rdb_test.cc | 227 +-------- tests/cppunit/rdb_util.h | 203 ++++++++ tests/cppunit/test_base.h | 2 + tests/testdata/test.conf | 847 ------------------------------- 6 files changed, 237 insertions(+), 1067 deletions(-) create mode 100644 tests/cppunit/rdb_util.h delete mode 100644 tests/testdata/test.conf diff --git a/CMakeLists.txt b/CMakeLists.txt index f5d5f6f6876..4aadc0d0acc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -253,4 +253,4 @@ file(GLOB_RECURSE TESTS_SRCS tests/cppunit/*.cc) add_executable(unittest ${TESTS_SRCS}) target_include_directories(unittest PRIVATE tests/cppunit) -target_link_libraries(unittest PRIVATE kvrocks_objs gtest_main ${EXTERNAL_LIBS}) \ No newline at end of file +target_link_libraries(unittest PRIVATE kvrocks_objs gtest_main ${EXTERNAL_LIBS}) diff --git a/tests/cppunit/rdb_stream_test.cc b/tests/cppunit/rdb_stream_test.cc index 2dc94abb207..508f4f69b94 100644 --- a/tests/cppunit/rdb_stream_test.cc +++ b/tests/cppunit/rdb_stream_test.cc @@ -17,7 +17,6 @@ * under the License. * */ - #include "common/rdb_stream.h" #include @@ -25,25 +24,30 @@ #include #include +#include "rdb_util.h" + TEST(RdbFileStreamOpenTest, FileNotExist) { - RdbFileStream reader("../tests/testdata/not_exist.rdb"); + RdbFileStream reader("not_exist.rdb"); ASSERT_FALSE(reader.Open().IsOK()); ; } TEST(RdbFileStreamOpenTest, FileExist) { - RdbFileStream reader("../tests/testdata/encodings.rdb"); + std::string test_file = "hash-zipmap.rdb"; + ScopedTestRDBFile temp(test_file, hash_zipmap_payload, sizeof(hash_zipmap_payload) - 1); + RdbFileStream reader(test_file); ASSERT_TRUE(reader.Open().IsOK()); } TEST(RdbFileStreamReadTest, ReadRdb) { - const std::string file_path = "../tests/testdata/encodings.rdb"; + const std::string test_file = "encodings.rdb"; + ScopedTestRDBFile temp(test_file, encodings_rdb_payload, sizeof(encodings_rdb_payload) - 1); - std::ifstream file(file_path, std::ios::binary | std::ios::ate); + std::ifstream file(test_file, std::ios::binary | std::ios::ate); std::streamsize size = file.tellg(); file.close(); - RdbFileStream reader(file_path); + RdbFileStream reader(test_file); ASSERT_TRUE(reader.Open().IsOK()); char buf[16] = {0}; @@ -63,13 +67,14 @@ TEST(RdbFileStreamReadTest, ReadRdb) { } TEST(RdbFileStreamReadTest, ReadRdbLittleChunk) { - const std::string file_path = "../tests/testdata/encodings.rdb"; + const std::string test_file = "encodings.rdb"; + ScopedTestRDBFile temp(test_file, encodings_rdb_payload, sizeof(encodings_rdb_payload) - 1); - std::ifstream file(file_path, std::ios::binary | std::ios::ate); + std::ifstream file(test_file, std::ios::binary | std::ios::ate); std::streamsize size = file.tellg(); file.close(); - RdbFileStream reader(file_path, 16); + RdbFileStream reader(test_file, 16); ASSERT_TRUE(reader.Open().IsOK()); char buf[32] = {0}; auto len = static_cast(sizeof(buf) / sizeof(buf[0])); diff --git a/tests/cppunit/rdb_test.cc b/tests/cppunit/rdb_test.cc index c0fa8371614..d63060d3804 100644 --- a/tests/cppunit/rdb_test.cc +++ b/tests/cppunit/rdb_test.cc @@ -17,11 +17,8 @@ * under the License. * */ - #include "storage/rdb.h" -#include - #include #include #include @@ -31,7 +28,9 @@ #include "common/rdb_stream.h" #include "config/config.h" +#include "rdb_util.h" #include "storage/storage.h" +#include "test_base.h" #include "types/redis_hash.h" #include "types/redis_list.h" #include "types/redis_set.h" @@ -39,193 +38,17 @@ #include "types/redis_zset.h" #include "vendor/crc64.h" -const std::string test_data_path = "../tests/testdata/"; -const std::string test_config = "test.conf"; - -// RDB test data, copy from Redis's tests/asset/*rdb, not shellcode. - -// zset-ziplist.rdb -const char zset_ziplist_payload[] = - "\x52\x45\x44\x49\x53\x30\x30\x31\x30\xfa\x09\x72\x65\x64\x69\x73\x2d\x76\x65\x72\x0b\x32\x35\x35\x2e" - "\x32\x35\x35\x2e\x32\x35\x35\xfa\x0a\x72\x65\x64\x69\x73\x2d\x62\x69\x74\x73\xc0\x40\xfa\x05\x63\x74" - "\x69\x6d\x65\xc2\x62\xb7\x13\x61\xfa\x08\x75\x73\x65\x64\x2d\x6d\x65\x6d\xc2\x50\xf4\x0c\x00\xfa\x0c" - "\x61\x6f\x66\x2d\x70\x72\x65\x61\x6d\x62\x6c\x65\xc0\x00\xfe\x00\xfb\x01\x00\x0c\x04\x7a\x73\x65\x74" - "\x19\x19\x00\x00\x00\x16\x00\x00\x00\x04\x00\x00\x03\x6f\x6e\x65\x05\xf2\x02\x03\x74\x77\x6f\x05\xf3" - "\xff\xff\x1f\xb2\xfd\xf0\x99\x7f\x9e\x19"; - -// corrupt_empty_keys.rdb -const char corrupt_empty_keys_payload[] = - "\x52\x45\x44\x49\x53\x30\x30\x31\x30\xfa\x09\x72\x65\x64\x69\x73\x2d\x76\x65\x72\x0b\x32\x35\x35\x2e" - "\x32\x35\x35\x2e\x32\x35\x35\xfa\x0a\x72\x65\x64\x69\x73\x2d\x62\x69\x74\x73\xc0\x40\xfa\x05\x63\x74" - "\x69\x6d\x65\xc2\x7a\x18\x15\x61\xfa\x08\x75\x73\x65\x64\x2d\x6d\x65\x6d\xc2\x80\x31\x10\x00\xfa\x0c" - "\x61\x6f\x66\x2d\x70\x72\x65\x61\x6d\x62\x6c\x65\xc0\x00\xfe\x00\xfb\x09\x00\x02\x03\x73\x65\x74\x00" - "\x04\x04\x68\x61\x73\x68\x00\x0a\x0c\x6c\x69\x73\x74\x5f\x7a\x69\x70\x6c\x69\x73\x74\x0b\x0b\x00\x00" - "\x00\x0a\x00\x00\x00\x00\x00\xff\x05\x04\x7a\x73\x65\x74\x00\x11\x0d\x7a\x73\x65\x74\x5f\x6c\x69\x73" - "\x74\x70\x61\x63\x6b\x07\x07\x00\x00\x00\x00\x00\xff\x10\x0c\x68\x61\x73\x68\x5f\x7a\x69\x70\x6c\x69" - "\x73\x74\x07\x07\x00\x00\x00\x00\x00\xff\x0e\x0e\x6c\x69\x73\x74\x5f\x71\x75\x69\x63\x6b\x6c\x69\x73" - "\x74\x00\x0c\x0c\x7a\x73\x65\x74\x5f\x7a\x69\x70\x6c\x69\x73\x74\x0b\x0b\x00\x00\x00\x0a\x00\x00\x00" - "\x00\x00\xff\x0e\x1c\x6c\x69\x73\x74\x5f\x71\x75\x69\x63\x6b\x6c\x69\x73\x74\x5f\x65\x6d\x70\x74\x79" - "\x5f\x7a\x69\x70\x6c\x69\x73\x74\x01\x0b\x0b\x00\x00\x00\x0a\x00\x00\x00\x00\x00\xff\xff\xf0\xf5\x06" - "\xdd\xc6\x6e\x61\x83"; - -// encodings.rdb -const char encodings_rdb_payload[] = - "\x52\x45\x44\x49\x53\x30\x30\x30\x34\xfe\x00\x03\x04\x7a\x73\x65\x74\x0c\x02\x62\x62\x02\x32\x30\x02\x63\x63\x02" - "\x33\x30" - "\x03\x62\x62\x62\x03\x32\x30\x30\x04\x62\x62\x62\x62\x0a\x35\x30\x30\x30\x30\x30\x30\x30\x30\x30\x03\x63\x63\x63" - "\x03\x33" - "\x30\x30\x04\x63\x63\x63\x63\x09\x31\x32\x33\x34\x35\x36\x37\x38\x39\x01\x61\x01\x31\x02\x61\x61\x02\x31\x30\x01" - "\x62\x01" - "\x32\x03\x61\x61\x61\x03\x31\x30\x30\x01\x63\x01\x33\x04\x61\x61\x61\x61\x04\x31\x30\x30\x30\x0b\x0c\x73\x65\x74" - "\x5f\x7a" - "\x69\x70\x70\x65\x64\x5f\x31\x10\x02\x00\x00\x00\x04\x00\x00\x00\x01\x00\x02\x00\x03\x00\x04\x00\x0a\x0b\x6c\x69" - "\x73\x74" - "\x5f\x7a\x69\x70\x70\x65\x64\x30\x30\x00\x00\x00\x25\x00\x00\x00\x08\x00\x00\xc0\x01\x00\x04\xc0\x02\x00\x04\xc0" - "\x03\x00" - "\x04\x01\x61\x03\x01\x62\x03\x01\x63\x03\xd0\xa0\x86\x01\x00\x06\xe0\x00\xbc\xa0\x65\x01\x00\x00\x00\xff\x00\x06" - "\x73\x74" - "\x72\x69\x6e\x67\x0b\x48\x65\x6c\x6c\x6f\x20\x57\x6f\x72\x6c\x64\x00\x0c\x63\x6f\x6d\x70\x72\x65\x73\x73\x69\x62" - "\x6c\x65" - "\xc3\x09\x40\x89\x01\x61\x61\xe0\x7c\x00\x01\x61\x61\x0b\x0c\x73\x65\x74\x5f\x7a\x69\x70\x70\x65\x64\x5f\x32\x18" - "\x04\x00" - "\x00\x00\x04\x00\x00\x00\xa0\x86\x01\x00\x40\x0d\x03\x00\xe0\x93\x04\x00\x80\x1a\x06\x00\x00\x06\x6e\x75\x6d\x62" - "\x65\x72" - "\xc0\x0a\x0b\x0c\x73\x65\x74\x5f\x7a\x69\x70\x70\x65\x64\x5f\x33\x38\x08\x00\x00\x00\x06\x00\x00\x00\x00\xca\x9a" - "\x3b\x00" - "\x00\x00\x00\x00\x94\x35\x77\x00\x00\x00\x00\x00\x5e\xd0\xb2\x00\x00\x00\x00\x00\x28\x6b\xee\x00\x00\x00\x00\x00" - "\xf2\x05" - "\x2a\x01\x00\x00\x00\x00\xbc\xa0\x65\x01\x00\x00\x00\x02\x03\x73\x65\x74\x08\x0a\x36\x30\x30\x30\x30\x30\x30\x30" - "\x30\x30" - "\xc2\xa0\x86\x01\x00\x01\x61\xc0\x01\x01\x62\xc0\x02\x01\x63\xc0\x03\x01\x04\x6c\x69\x73\x74\x18\xc0\x01\xc0\x02" - "\xc0\x03" - "\x01\x61\x01\x62\x01\x63\xc2\xa0\x86\x01\x00\x0a\x36\x30\x30\x30\x30\x30\x30\x30\x30\x30\xc0\x01\xc0\x02\xc0\x03" - "\x01\x61" - "\x01\x62\x01\x63\xc2\xa0\x86\x01\x00\x0a\x36\x30\x30\x30\x30\x30\x30\x30\x30\x30\xc0\x01\xc0\x02\xc0\x03\x01\x61" - "\x01\x62" - "\x01\x63\xc2\xa0\x86\x01\x00\x0a\x36\x30\x30\x30\x30\x30\x30\x30\x30\x30\x0d\x0b\x68\x61\x73\x68\x5f\x7a\x69\x70" - "\x70\x65" - "\x64\x20\x20\x00\x00\x00\x1b\x00\x00\x00\x06\x00\x00\x01\x61\x03\xc0\x01\x00\x04\x01\x62\x03\xc0\x02\x00\x04\x01" - "\x63\x03" - "\xc0\x03\x00\xff\x0c\x0b\x7a\x73\x65\x74\x5f\x7a\x69\x70\x70\x65\x64\x20\x20\x00\x00\x00\x1b\x00\x00\x00\x06\x00" - "\x00\x01" - "\x61\x03\xc0\x01\x00\x04\x01\x62\x03\xc0\x02\x00\x04\x01\x63\x03\xc0\x03\x00\xff\x04\x04\x68\x61\x73\x68\x0b\x01" - "\x62\xc0" - "\x02\x02\x61\x61\xc0\x0a\x01\x63\xc0\x03\x03\x61\x61\x61\xc0\x64\x02\x62\x62\xc0\x14\x02\x63\x63\xc0\x1e\x03\x62" - "\x62\x62" - "\xc1\xc8\x00\x03\x63\x63\x63\xc1\x2c\x01\x03\x64\x64\x64\xc1\x90\x01\x03\x65\x65\x65\x0a\x35\x30\x30\x30\x30\x30" - "\x30\x30" - "\x30\x30\x01\x61\xc0\x01\xff"; - -// hash-ziplist.rdb -const char hash_ziplist_payload[] = - "\x52\x45\x44\x49\x53\x30\x30\x30\x39\xfa\x09\x72\x65\x64\x69\x73\x2d\x76\x65\x72\x0b\x32\x35\x35\x2e\x32\x35\x35" - "\x2e\x32" - "\x35\x35\xfa\x0a\x72\x65\x64\x69\x73\x2d\x62\x69\x74\x73\xc0\x40\xfa\x05\x63\x74\x69\x6d\x65\xc2\xc8\x5c\x96\x60" - "\xfa\x08" - "\x75\x73\x65\x64\x2d\x6d\x65\x6d\xc2\x90\xad\x0c\x00\xfa\x0c\x61\x6f\x66\x2d\x70\x72\x65\x61\x6d\x62\x6c\x65\xc0" - "\x00\xfe" - "\x00\xfb\x01\x00\x0d\x04\x68\x61\x73\x68\x1b\x1b\x00\x00\x00\x16\x00\x00\x00\x04\x00\x00\x02\x66\x31\x04\x02\x76" - "\x31\x04" - "\x02\x66\x32\x04\x02\x76\x32\xff\xff\x4f\x9c\xd1\xfd\x16\x69\x98\x83"; - -// hash-zipmap.rdb -const char hash_zipmap_payload[] = - "\x52\x45\x44\x49\x53\x30\x30\x30\x33\xfe\x00\x09\x04\x68\x61\x73\x68\x10\x02\x02\x66\x31\x02\x00\x76\x31\x02\x66" - "\x32\x02" - "\x00\x76\x32\xff\xff"; - -// list-quicklist.rdb -const char list_quicklist_payload[] = - "\x52\x45\x44\x49\x53\x30\x30\x30\x38\xfa\x09\x72\x65\x64\x69\x73\x2d\x76\x65\x72\x05\x34\x2e\x30\x2e\x39\xfa\x0a" - "\x72\x65" - "\x64\x69\x73\x2d\x62\x69\x74\x73\xc0\x40\xfa\x05\x63\x74\x69\x6d\x65\xc2\x9f\x06\x26\x61\xfa\x08\x75\x73\x65\x64" - "\x2d\x6d" - "\x65\x6d\xc2\x80\x92\x07\x00\xfa\x0c\x61\x6f\x66\x2d\x70\x72\x65\x61\x6d\x62\x6c\x65\xc0\x00\xfe\x00\xfb\x02\x00" - "\x0e\x04" - "\x6c\x69\x73\x74\x01\x0d\x0d\x00\x00\x00\x0a\x00\x00\x00\x01\x00\x00\xf8\xff\x00\x01\x78\xc0\x07\xff\x35\x72\xf8" - "\x54\x1a" - "\xc4\xd7\x40"; - -// dumped from redis-server 7.0, sourced from the 'encodings.rdb' file. -const char encodings_ver10_rdb_payload[] = - "\x52\x45\x44\x49\x53\x30\x30\x31\x30\xfa\x09\x72\x65\x64\x69\x73\x2d\x76\x65\x72\x05\x37\x2e\x30\x2e\x33\xfa\x0a" - "\x72\x65" - "\x64\x69\x73\x2d\x62\x69\x74\x73\xc0\x40\xfa\x05\x63\x74\x69\x6d\x65\xc2\x62\x65\x23\x65\xfa\x08\x75\x73\x65\x64" - "\x2d\x6d" - "\x65\x6d\xc2\x28\x4f\x0e\x00\xfa\x08\x61\x6f\x66\x2d\x62\x61\x73\x65\xc0\x00\xfe\x00\xfb\x0d\x00\x11\x04\x7a\x73" - "\x65\x74" - "\x40\x64\x64\x00\x00\x00\x18\x00\x81\x61\x02\x01\x01\x81\x62\x02\x02\x01\x81\x63\x02\x03\x01\x82\x61\x61\x03\x0a" - "\x01\x82" - "\x62\x62\x03\x14\x01\x82\x63\x63\x03\x1e\x01\x83\x61\x61\x61\x04\x64\x01\x83\x62\x62\x62\x04\xc0\xc8\x02\x83\x63" - "\x63\x63" - "\x04\xc1\x2c\x02\x84\x61\x61\x61\x61\x05\xc3\xe8\x02\x84\x63\x63\x63\x63\x05\xf3\x15\xcd\x5b\x07\x05\x84\x62\x62" - "\x62\x62" - "\x05\xf4\x00\xf2\x05\x2a\x01\x00\x00\x00\x09\xff\x11\x0b\x7a\x73\x65\x74\x5f\x7a\x69\x70\x70\x65\x64\x16\x16\x00" - "\x00\x00" - "\x06\x00\x81\x61\x02\x01\x01\x81\x62\x02\x02\x01\x81\x63\x02\x03\x01\xff\x10\x04\x68\x61\x73\x68\x40\x56\x56\x00" - "\x00\x00" - "\x16\x00\x81\x62\x02\x02\x01\x82\x61\x61\x03\x0a\x01\x81\x63\x02\x03\x01\x83\x61\x61\x61\x04\x64\x01\x82\x62\x62" - "\x03\x14" - "\x01\x82\x63\x63\x03\x1e\x01\x83\x62\x62\x62\x04\xc0\xc8\x02\x83\x63\x63\x63\x04\xc1\x2c\x02\x83\x64\x64\x64\x04" - "\xc1\x90" - "\x02\x83\x65\x65\x65\x04\xf4\x00\xf2\x05\x2a\x01\x00\x00\x00\x09\x81\x61\x02\x01\x01\xff\x0b\x0c\x73\x65\x74\x5f" - "\x7a\x69" - "\x70\x70\x65\x64\x5f\x32\x18\x04\x00\x00\x00\x04\x00\x00\x00\xa0\x86\x01\x00\x40\x0d\x03\x00\xe0\x93\x04\x00\x80" - "\x1a\x06" - "\x00\x12\x04\x6c\x69\x73\x74\x01\x02\xc3\x2b\x40\x61\x1f\x61\x00\x00\x00\x18\x00\x01\x01\x02\x01\x03\x01\x81\x61" - "\x02\x81" - "\x62\x02\x81\x63\x02\xf2\xa0\x86\x01\x04\xf4\x00\xbc\xa0\x65\x01\x20\x1e\x00\x09\xe0\x32\x1d\x01\x09\xff\x02\x03" - "\x73\x65" - "\x74\x08\xc0\x02\xc0\x01\xc2\xa0\x86\x01\x00\x01\x62\x0a\x36\x30\x30\x30\x30\x30\x30\x30\x30\x30\xc0\x03\x01\x61" - "\x01\x63" - "\x00\x06\x6e\x75\x6d\x62\x65\x72\xc0\x0a\x0b\x0c\x73\x65\x74\x5f\x7a\x69\x70\x70\x65\x64\x5f\x31\x10\x02\x00\x00" - "\x00\x04" - "\x00\x00\x00\x01\x00\x02\x00\x03\x00\x04\x00\x10\x0b\x68\x61\x73\x68\x5f\x7a\x69\x70\x70\x65\x64\x16\x16\x00\x00" - "\x00\x06" - "\x00\x81\x61\x02\x01\x01\x81\x62\x02\x02\x01\x81\x63\x02\x03\x01\xff\x00\x0c\x63\x6f\x6d\x70\x72\x65\x73\x73\x69" - "\x62\x6c" - "\x65\xc3\x09\x40\x89\x01\x61\x61\xe0\x7c\x00\x01\x61\x61\x00\x06\x73\x74\x72\x69\x6e\x67\x0b\x48\x65\x6c\x6c\x6f" - "\x20\x57" - "\x6f\x72\x6c\x64\x0b\x0c\x73\x65\x74\x5f\x7a\x69\x70\x70\x65\x64\x5f\x33\x38\x08\x00\x00\x00\x06\x00\x00\x00\x00" - "\xca\x9a" - "\x3b\x00\x00\x00\x00\x00\x94\x35\x77\x00\x00\x00\x00\x00\x5e\xd0\xb2\x00\x00\x00\x00\x00\x28\x6b\xee\x00\x00\x00" - "\x00\x00" - "\xf2\x05\x2a\x01\x00\x00\x00\x00\xbc\xa0\x65\x01\x00\x00\x00\x12\x0b\x6c\x69\x73\x74\x5f\x7a\x69\x70\x70\x65\x64" - "\x01\x02" - "\x25\x25\x00\x00\x00\x08\x00\x01\x01\x02\x01\x03\x01\x81\x61\x02\x81\x62\x02\x81\x63\x02\xf2\xa0\x86\x01\x04\xf4" - "\x00\xbc" - "\xa0\x65\x01\x00\x00\x00\x09\xff\xff\x58\xe7\x62\x56\x52\x9b\xdf\x6c"; - -class RDBTest : public testing::Test { +class RDBTest : public TestBase { public: RDBTest(const RDBTest &) = delete; RDBTest &operator=(const RDBTest &) = delete; protected: - explicit RDBTest() : storage_(nullptr), ns_(kDefaultNamespace) {} - ~RDBTest() override { - if (nullptr != storage_) { - delete storage_; - } - } - void SetUp() override { - crc64_init(); - - std::string test_config_path = test_data_path + test_config; - auto s = config_.Load(CLIOptions(test_config_path)); - ASSERT_TRUE(s.IsOK()); - - ASSERT_TRUE(clearDBDir(config_.db_dir)); - - storage_ = new engine::Storage(&config_); - s = storage_->Open(); - ASSERT_TRUE(s.IsOK()); - } + explicit RDBTest() : ns_(kDefaultNamespace) {} + ~RDBTest() override = default; + void SetUp() override { crc64_init(); } - void TearDown() override { - std::remove(tmp_rdb_.c_str()); - ASSERT_TRUE(clearDBDir(config_.db_dir)); - } + void TearDown() override { ASSERT_TRUE(clearDBDir(config_->db_dir)); } void loadRdb(const std::string &path) { auto stream_ptr = std::make_shared(path); @@ -314,24 +137,9 @@ class RDBTest : public testing::Test { ASSERT_TRUE(s.ok()); } - static void writeTestRdbFile(const std::string &name, const char *data, size_t len) { - std::ofstream out_file(name, std::ios::out | std::ios::binary); - if (!out_file) { - ASSERT_TRUE(false); - } - - out_file.write(data, static_cast(len)); - if (!out_file) { - ASSERT_TRUE(false); - } - out_file.close(); - } - void encodingDataCheck(); - engine::Storage *storage_; std::string ns_; - Config config_; std::string tmp_rdb_; private: @@ -406,23 +214,22 @@ void RDBTest::encodingDataCheck() { std::string ConvertToString(const char *data, size_t len) { return {data, data + len}; } TEST_F(RDBTest, LoadEncodings) { - std::map datas; - datas.insert({"encodings.rdb", ConvertToString(encodings_rdb_payload, sizeof(encodings_rdb_payload) - 1)}); - datas.insert( + std::map data; + data.insert({"encodings.rdb", ConvertToString(encodings_rdb_payload, sizeof(encodings_rdb_payload) - 1)}); + data.insert( {"encodings_ver10.rdb", ConvertToString(encodings_ver10_rdb_payload, sizeof(encodings_ver10_rdb_payload) - 1)}); - for (const auto &kv : datas) { + for (const auto &kv : data) { tmp_rdb_ = kv.first; - writeTestRdbFile(tmp_rdb_, kv.second.data(), kv.second.size()); + ScopedTestRDBFile temp(tmp_rdb_, kv.second.data(), kv.second.size()); loadRdb(tmp_rdb_); encodingDataCheck(); flushDB(); - std::remove(tmp_rdb_.c_str()); } } TEST_F(RDBTest, LoadHashZipMap) { tmp_rdb_ = "hash-zipmap.rdb"; - writeTestRdbFile(tmp_rdb_, hash_zipmap_payload, sizeof(hash_zipmap_payload) - 1); + ScopedTestRDBFile temp(tmp_rdb_, hash_zipmap_payload, sizeof(hash_zipmap_payload) - 1); loadRdb(tmp_rdb_); // hash @@ -435,7 +242,7 @@ TEST_F(RDBTest, LoadHashZipMap) { TEST_F(RDBTest, LoadHashZipList) { tmp_rdb_ = "hash-ziplist.rdb"; - writeTestRdbFile(tmp_rdb_, hash_ziplist_payload, sizeof(hash_ziplist_payload) - 1); + ScopedTestRDBFile temp(tmp_rdb_, hash_ziplist_payload, sizeof(hash_ziplist_payload) - 1); loadRdb(tmp_rdb_); // hash @@ -448,7 +255,7 @@ TEST_F(RDBTest, LoadHashZipList) { TEST_F(RDBTest, LoadListQuickList) { tmp_rdb_ = "list-quicklist.rdb"; - writeTestRdbFile(tmp_rdb_, list_quicklist_payload, sizeof(list_quicklist_payload) - 1); + ScopedTestRDBFile temp(tmp_rdb_, list_quicklist_payload, sizeof(list_quicklist_payload) - 1); loadRdb(tmp_rdb_); // list @@ -458,7 +265,7 @@ TEST_F(RDBTest, LoadListQuickList) { TEST_F(RDBTest, LoadZSetZipList) { tmp_rdb_ = "zset-ziplist.rdb"; - writeTestRdbFile(tmp_rdb_, zset_ziplist_payload, sizeof(zset_ziplist_payload) - 1); + ScopedTestRDBFile temp(tmp_rdb_, zset_ziplist_payload, sizeof(zset_ziplist_payload) - 1); loadRdb(tmp_rdb_); // zset @@ -471,7 +278,7 @@ TEST_F(RDBTest, LoadZSetZipList) { TEST_F(RDBTest, LoadEmptyKeys) { tmp_rdb_ = "corrupt_empty_keys.rdb"; - writeTestRdbFile(tmp_rdb_, corrupt_empty_keys_payload, sizeof(corrupt_empty_keys_payload) - 1); + ScopedTestRDBFile temp(tmp_rdb_, corrupt_empty_keys_payload, sizeof(corrupt_empty_keys_payload) - 1); loadRdb(tmp_rdb_); /* corrupt_empty_keys.rdb contains 9 keys with empty value: diff --git a/tests/cppunit/rdb_util.h b/tests/cppunit/rdb_util.h new file mode 100644 index 00000000000..8ce743d5a61 --- /dev/null +++ b/tests/cppunit/rdb_util.h @@ -0,0 +1,203 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +#pragma once + +#include + +#include +#include + +// RDB test data, copy from Redis's tests/asset/*rdb, not shellcode. + +// zset-ziplist.rdb +inline constexpr const char zset_ziplist_payload[] = + "\x52\x45\x44\x49\x53\x30\x30\x31\x30\xfa\x09\x72\x65\x64\x69\x73\x2d\x76\x65\x72\x0b\x32\x35\x35\x2e" + "\x32\x35\x35\x2e\x32\x35\x35\xfa\x0a\x72\x65\x64\x69\x73\x2d\x62\x69\x74\x73\xc0\x40\xfa\x05\x63\x74" + "\x69\x6d\x65\xc2\x62\xb7\x13\x61\xfa\x08\x75\x73\x65\x64\x2d\x6d\x65\x6d\xc2\x50\xf4\x0c\x00\xfa\x0c" + "\x61\x6f\x66\x2d\x70\x72\x65\x61\x6d\x62\x6c\x65\xc0\x00\xfe\x00\xfb\x01\x00\x0c\x04\x7a\x73\x65\x74" + "\x19\x19\x00\x00\x00\x16\x00\x00\x00\x04\x00\x00\x03\x6f\x6e\x65\x05\xf2\x02\x03\x74\x77\x6f\x05\xf3" + "\xff\xff\x1f\xb2\xfd\xf0\x99\x7f\x9e\x19"; + +// corrupt_empty_keys.rdb +inline constexpr const char corrupt_empty_keys_payload[] = + "\x52\x45\x44\x49\x53\x30\x30\x31\x30\xfa\x09\x72\x65\x64\x69\x73\x2d\x76\x65\x72\x0b\x32\x35\x35\x2e" + "\x32\x35\x35\x2e\x32\x35\x35\xfa\x0a\x72\x65\x64\x69\x73\x2d\x62\x69\x74\x73\xc0\x40\xfa\x05\x63\x74" + "\x69\x6d\x65\xc2\x7a\x18\x15\x61\xfa\x08\x75\x73\x65\x64\x2d\x6d\x65\x6d\xc2\x80\x31\x10\x00\xfa\x0c" + "\x61\x6f\x66\x2d\x70\x72\x65\x61\x6d\x62\x6c\x65\xc0\x00\xfe\x00\xfb\x09\x00\x02\x03\x73\x65\x74\x00" + "\x04\x04\x68\x61\x73\x68\x00\x0a\x0c\x6c\x69\x73\x74\x5f\x7a\x69\x70\x6c\x69\x73\x74\x0b\x0b\x00\x00" + "\x00\x0a\x00\x00\x00\x00\x00\xff\x05\x04\x7a\x73\x65\x74\x00\x11\x0d\x7a\x73\x65\x74\x5f\x6c\x69\x73" + "\x74\x70\x61\x63\x6b\x07\x07\x00\x00\x00\x00\x00\xff\x10\x0c\x68\x61\x73\x68\x5f\x7a\x69\x70\x6c\x69" + "\x73\x74\x07\x07\x00\x00\x00\x00\x00\xff\x0e\x0e\x6c\x69\x73\x74\x5f\x71\x75\x69\x63\x6b\x6c\x69\x73" + "\x74\x00\x0c\x0c\x7a\x73\x65\x74\x5f\x7a\x69\x70\x6c\x69\x73\x74\x0b\x0b\x00\x00\x00\x0a\x00\x00\x00" + "\x00\x00\xff\x0e\x1c\x6c\x69\x73\x74\x5f\x71\x75\x69\x63\x6b\x6c\x69\x73\x74\x5f\x65\x6d\x70\x74\x79" + "\x5f\x7a\x69\x70\x6c\x69\x73\x74\x01\x0b\x0b\x00\x00\x00\x0a\x00\x00\x00\x00\x00\xff\xff\xf0\xf5\x06" + "\xdd\xc6\x6e\x61\x83"; + +// encodings.rdb +inline constexpr const char encodings_rdb_payload[] = + "\x52\x45\x44\x49\x53\x30\x30\x30\x34\xfe\x00\x03\x04\x7a\x73\x65\x74\x0c\x02\x62\x62\x02\x32\x30\x02\x63\x63\x02" + "\x33\x30" + "\x03\x62\x62\x62\x03\x32\x30\x30\x04\x62\x62\x62\x62\x0a\x35\x30\x30\x30\x30\x30\x30\x30\x30\x30\x03\x63\x63\x63" + "\x03\x33" + "\x30\x30\x04\x63\x63\x63\x63\x09\x31\x32\x33\x34\x35\x36\x37\x38\x39\x01\x61\x01\x31\x02\x61\x61\x02\x31\x30\x01" + "\x62\x01" + "\x32\x03\x61\x61\x61\x03\x31\x30\x30\x01\x63\x01\x33\x04\x61\x61\x61\x61\x04\x31\x30\x30\x30\x0b\x0c\x73\x65\x74" + "\x5f\x7a" + "\x69\x70\x70\x65\x64\x5f\x31\x10\x02\x00\x00\x00\x04\x00\x00\x00\x01\x00\x02\x00\x03\x00\x04\x00\x0a\x0b\x6c\x69" + "\x73\x74" + "\x5f\x7a\x69\x70\x70\x65\x64\x30\x30\x00\x00\x00\x25\x00\x00\x00\x08\x00\x00\xc0\x01\x00\x04\xc0\x02\x00\x04\xc0" + "\x03\x00" + "\x04\x01\x61\x03\x01\x62\x03\x01\x63\x03\xd0\xa0\x86\x01\x00\x06\xe0\x00\xbc\xa0\x65\x01\x00\x00\x00\xff\x00\x06" + "\x73\x74" + "\x72\x69\x6e\x67\x0b\x48\x65\x6c\x6c\x6f\x20\x57\x6f\x72\x6c\x64\x00\x0c\x63\x6f\x6d\x70\x72\x65\x73\x73\x69\x62" + "\x6c\x65" + "\xc3\x09\x40\x89\x01\x61\x61\xe0\x7c\x00\x01\x61\x61\x0b\x0c\x73\x65\x74\x5f\x7a\x69\x70\x70\x65\x64\x5f\x32\x18" + "\x04\x00" + "\x00\x00\x04\x00\x00\x00\xa0\x86\x01\x00\x40\x0d\x03\x00\xe0\x93\x04\x00\x80\x1a\x06\x00\x00\x06\x6e\x75\x6d\x62" + "\x65\x72" + "\xc0\x0a\x0b\x0c\x73\x65\x74\x5f\x7a\x69\x70\x70\x65\x64\x5f\x33\x38\x08\x00\x00\x00\x06\x00\x00\x00\x00\xca\x9a" + "\x3b\x00" + "\x00\x00\x00\x00\x94\x35\x77\x00\x00\x00\x00\x00\x5e\xd0\xb2\x00\x00\x00\x00\x00\x28\x6b\xee\x00\x00\x00\x00\x00" + "\xf2\x05" + "\x2a\x01\x00\x00\x00\x00\xbc\xa0\x65\x01\x00\x00\x00\x02\x03\x73\x65\x74\x08\x0a\x36\x30\x30\x30\x30\x30\x30\x30" + "\x30\x30" + "\xc2\xa0\x86\x01\x00\x01\x61\xc0\x01\x01\x62\xc0\x02\x01\x63\xc0\x03\x01\x04\x6c\x69\x73\x74\x18\xc0\x01\xc0\x02" + "\xc0\x03" + "\x01\x61\x01\x62\x01\x63\xc2\xa0\x86\x01\x00\x0a\x36\x30\x30\x30\x30\x30\x30\x30\x30\x30\xc0\x01\xc0\x02\xc0\x03" + "\x01\x61" + "\x01\x62\x01\x63\xc2\xa0\x86\x01\x00\x0a\x36\x30\x30\x30\x30\x30\x30\x30\x30\x30\xc0\x01\xc0\x02\xc0\x03\x01\x61" + "\x01\x62" + "\x01\x63\xc2\xa0\x86\x01\x00\x0a\x36\x30\x30\x30\x30\x30\x30\x30\x30\x30\x0d\x0b\x68\x61\x73\x68\x5f\x7a\x69\x70" + "\x70\x65" + "\x64\x20\x20\x00\x00\x00\x1b\x00\x00\x00\x06\x00\x00\x01\x61\x03\xc0\x01\x00\x04\x01\x62\x03\xc0\x02\x00\x04\x01" + "\x63\x03" + "\xc0\x03\x00\xff\x0c\x0b\x7a\x73\x65\x74\x5f\x7a\x69\x70\x70\x65\x64\x20\x20\x00\x00\x00\x1b\x00\x00\x00\x06\x00" + "\x00\x01" + "\x61\x03\xc0\x01\x00\x04\x01\x62\x03\xc0\x02\x00\x04\x01\x63\x03\xc0\x03\x00\xff\x04\x04\x68\x61\x73\x68\x0b\x01" + "\x62\xc0" + "\x02\x02\x61\x61\xc0\x0a\x01\x63\xc0\x03\x03\x61\x61\x61\xc0\x64\x02\x62\x62\xc0\x14\x02\x63\x63\xc0\x1e\x03\x62" + "\x62\x62" + "\xc1\xc8\x00\x03\x63\x63\x63\xc1\x2c\x01\x03\x64\x64\x64\xc1\x90\x01\x03\x65\x65\x65\x0a\x35\x30\x30\x30\x30\x30" + "\x30\x30" + "\x30\x30\x01\x61\xc0\x01\xff"; + +// hash-ziplist.rdb +inline constexpr const char hash_ziplist_payload[] = + "\x52\x45\x44\x49\x53\x30\x30\x30\x39\xfa\x09\x72\x65\x64\x69\x73\x2d\x76\x65\x72\x0b\x32\x35\x35\x2e\x32\x35\x35" + "\x2e\x32" + "\x35\x35\xfa\x0a\x72\x65\x64\x69\x73\x2d\x62\x69\x74\x73\xc0\x40\xfa\x05\x63\x74\x69\x6d\x65\xc2\xc8\x5c\x96\x60" + "\xfa\x08" + "\x75\x73\x65\x64\x2d\x6d\x65\x6d\xc2\x90\xad\x0c\x00\xfa\x0c\x61\x6f\x66\x2d\x70\x72\x65\x61\x6d\x62\x6c\x65\xc0" + "\x00\xfe" + "\x00\xfb\x01\x00\x0d\x04\x68\x61\x73\x68\x1b\x1b\x00\x00\x00\x16\x00\x00\x00\x04\x00\x00\x02\x66\x31\x04\x02\x76" + "\x31\x04" + "\x02\x66\x32\x04\x02\x76\x32\xff\xff\x4f\x9c\xd1\xfd\x16\x69\x98\x83"; + +// hash-zipmap.rdb +inline constexpr const char hash_zipmap_payload[] = + "\x52\x45\x44\x49\x53\x30\x30\x30\x33\xfe\x00\x09\x04\x68\x61\x73\x68\x10\x02\x02\x66\x31\x02\x00\x76\x31\x02\x66" + "\x32\x02" + "\x00\x76\x32\xff\xff"; + +// list-quicklist.rdb +inline constexpr const char list_quicklist_payload[] = + "\x52\x45\x44\x49\x53\x30\x30\x30\x38\xfa\x09\x72\x65\x64\x69\x73\x2d\x76\x65\x72\x05\x34\x2e\x30\x2e\x39\xfa\x0a" + "\x72\x65" + "\x64\x69\x73\x2d\x62\x69\x74\x73\xc0\x40\xfa\x05\x63\x74\x69\x6d\x65\xc2\x9f\x06\x26\x61\xfa\x08\x75\x73\x65\x64" + "\x2d\x6d" + "\x65\x6d\xc2\x80\x92\x07\x00\xfa\x0c\x61\x6f\x66\x2d\x70\x72\x65\x61\x6d\x62\x6c\x65\xc0\x00\xfe\x00\xfb\x02\x00" + "\x0e\x04" + "\x6c\x69\x73\x74\x01\x0d\x0d\x00\x00\x00\x0a\x00\x00\x00\x01\x00\x00\xf8\xff\x00\x01\x78\xc0\x07\xff\x35\x72\xf8" + "\x54\x1a" + "\xc4\xd7\x40"; + +// dumped from redis-server 7.0, sourced from the 'encodings.rdb' file. +inline constexpr const char encodings_ver10_rdb_payload[] = + "\x52\x45\x44\x49\x53\x30\x30\x31\x30\xfa\x09\x72\x65\x64\x69\x73\x2d\x76\x65\x72\x05\x37\x2e\x30\x2e\x33\xfa\x0a" + "\x72\x65" + "\x64\x69\x73\x2d\x62\x69\x74\x73\xc0\x40\xfa\x05\x63\x74\x69\x6d\x65\xc2\x62\x65\x23\x65\xfa\x08\x75\x73\x65\x64" + "\x2d\x6d" + "\x65\x6d\xc2\x28\x4f\x0e\x00\xfa\x08\x61\x6f\x66\x2d\x62\x61\x73\x65\xc0\x00\xfe\x00\xfb\x0d\x00\x11\x04\x7a\x73" + "\x65\x74" + "\x40\x64\x64\x00\x00\x00\x18\x00\x81\x61\x02\x01\x01\x81\x62\x02\x02\x01\x81\x63\x02\x03\x01\x82\x61\x61\x03\x0a" + "\x01\x82" + "\x62\x62\x03\x14\x01\x82\x63\x63\x03\x1e\x01\x83\x61\x61\x61\x04\x64\x01\x83\x62\x62\x62\x04\xc0\xc8\x02\x83\x63" + "\x63\x63" + "\x04\xc1\x2c\x02\x84\x61\x61\x61\x61\x05\xc3\xe8\x02\x84\x63\x63\x63\x63\x05\xf3\x15\xcd\x5b\x07\x05\x84\x62\x62" + "\x62\x62" + "\x05\xf4\x00\xf2\x05\x2a\x01\x00\x00\x00\x09\xff\x11\x0b\x7a\x73\x65\x74\x5f\x7a\x69\x70\x70\x65\x64\x16\x16\x00" + "\x00\x00" + "\x06\x00\x81\x61\x02\x01\x01\x81\x62\x02\x02\x01\x81\x63\x02\x03\x01\xff\x10\x04\x68\x61\x73\x68\x40\x56\x56\x00" + "\x00\x00" + "\x16\x00\x81\x62\x02\x02\x01\x82\x61\x61\x03\x0a\x01\x81\x63\x02\x03\x01\x83\x61\x61\x61\x04\x64\x01\x82\x62\x62" + "\x03\x14" + "\x01\x82\x63\x63\x03\x1e\x01\x83\x62\x62\x62\x04\xc0\xc8\x02\x83\x63\x63\x63\x04\xc1\x2c\x02\x83\x64\x64\x64\x04" + "\xc1\x90" + "\x02\x83\x65\x65\x65\x04\xf4\x00\xf2\x05\x2a\x01\x00\x00\x00\x09\x81\x61\x02\x01\x01\xff\x0b\x0c\x73\x65\x74\x5f" + "\x7a\x69" + "\x70\x70\x65\x64\x5f\x32\x18\x04\x00\x00\x00\x04\x00\x00\x00\xa0\x86\x01\x00\x40\x0d\x03\x00\xe0\x93\x04\x00\x80" + "\x1a\x06" + "\x00\x12\x04\x6c\x69\x73\x74\x01\x02\xc3\x2b\x40\x61\x1f\x61\x00\x00\x00\x18\x00\x01\x01\x02\x01\x03\x01\x81\x61" + "\x02\x81" + "\x62\x02\x81\x63\x02\xf2\xa0\x86\x01\x04\xf4\x00\xbc\xa0\x65\x01\x20\x1e\x00\x09\xe0\x32\x1d\x01\x09\xff\x02\x03" + "\x73\x65" + "\x74\x08\xc0\x02\xc0\x01\xc2\xa0\x86\x01\x00\x01\x62\x0a\x36\x30\x30\x30\x30\x30\x30\x30\x30\x30\xc0\x03\x01\x61" + "\x01\x63" + "\x00\x06\x6e\x75\x6d\x62\x65\x72\xc0\x0a\x0b\x0c\x73\x65\x74\x5f\x7a\x69\x70\x70\x65\x64\x5f\x31\x10\x02\x00\x00" + "\x00\x04" + "\x00\x00\x00\x01\x00\x02\x00\x03\x00\x04\x00\x10\x0b\x68\x61\x73\x68\x5f\x7a\x69\x70\x70\x65\x64\x16\x16\x00\x00" + "\x00\x06" + "\x00\x81\x61\x02\x01\x01\x81\x62\x02\x02\x01\x81\x63\x02\x03\x01\xff\x00\x0c\x63\x6f\x6d\x70\x72\x65\x73\x73\x69" + "\x62\x6c" + "\x65\xc3\x09\x40\x89\x01\x61\x61\xe0\x7c\x00\x01\x61\x61\x00\x06\x73\x74\x72\x69\x6e\x67\x0b\x48\x65\x6c\x6c\x6f" + "\x20\x57" + "\x6f\x72\x6c\x64\x0b\x0c\x73\x65\x74\x5f\x7a\x69\x70\x70\x65\x64\x5f\x33\x38\x08\x00\x00\x00\x06\x00\x00\x00\x00" + "\xca\x9a" + "\x3b\x00\x00\x00\x00\x00\x94\x35\x77\x00\x00\x00\x00\x00\x5e\xd0\xb2\x00\x00\x00\x00\x00\x28\x6b\xee\x00\x00\x00" + "\x00\x00" + "\xf2\x05\x2a\x01\x00\x00\x00\x00\xbc\xa0\x65\x01\x00\x00\x00\x12\x0b\x6c\x69\x73\x74\x5f\x7a\x69\x70\x70\x65\x64" + "\x01\x02" + "\x25\x25\x00\x00\x00\x08\x00\x01\x01\x02\x01\x03\x01\x81\x61\x02\x81\x62\x02\x81\x63\x02\xf2\xa0\x86\x01\x04\xf4" + "\x00\xbc" + "\xa0\x65\x01\x00\x00\x00\x09\xff\xff\x58\xe7\x62\x56\x52\x9b\xdf\x6c"; + +class ScopedTestRDBFile { + public: + ScopedTestRDBFile(const std::string &name, const char *data, size_t len) : name_(name) { + std::ofstream out_file(name, std::ios::out | std::ios::binary); + if (!out_file) { + EXPECT_TRUE(false); + } + + out_file.write(data, static_cast(len)); + if (!out_file) { + EXPECT_TRUE(false); + } + out_file.close(); + } + + ScopedTestRDBFile(const ScopedTestRDBFile &) = delete; + ScopedTestRDBFile &operator=(const ScopedTestRDBFile &) = delete; + + ~ScopedTestRDBFile() { std::remove(name_.c_str()); } + + private: + std::string name_; +}; diff --git a/tests/cppunit/test_base.h b/tests/cppunit/test_base.h index 043b9329392..5d7dbe76704 100644 --- a/tests/cppunit/test_base.h +++ b/tests/cppunit/test_base.h @@ -59,6 +59,8 @@ class TestBase : public testing::Test { // NOLINT if (ec) { std::cout << "Encounter filesystem error: " << ec << std::endl; } + const char *path = "test.conf"; + unlink(path); } engine::Storage *storage_; diff --git a/tests/testdata/test.conf b/tests/testdata/test.conf deleted file mode 100644 index 00f6a1d39da..00000000000 --- a/tests/testdata/test.conf +++ /dev/null @@ -1,847 +0,0 @@ -################################ GENERAL ##################################### - -# By default kvrocks listens for connections from localhost interface. -# It is possible to listen to just one or multiple interfaces using -# the "bind" configuration directive, followed by one or more IP addresses. -# -# Examples: -# -# bind 192.168.1.100 10.0.0.1 -# bind 127.0.0.1 ::1 -# bind 0.0.0.0 -bind 127.0.0.1 - -# Unix socket. -# -# Specify the path for the unix socket that will be used to listen for -# incoming connections. There is no default, so kvrocks will not listen -# on a unix socket when not specified. -# -# unixsocket /tmp/kvrocks.sock -# unixsocketperm 777 - -# Accept connections on the specified port, default is 6666. -port 6666 - -# Close the connection after a client is idle for N seconds (0 to disable) -timeout 0 - -# The number of worker's threads, increase or decrease would affect the performance. -workers 8 - -# By default, kvrocks does not run as a daemon. Use 'yes' if you need it. -# Note that kvrocks will write a PID file in /var/run/kvrocks.pid when daemonized -daemonize no - -# Kvrocks implements the cluster solution that is similar to the Redis cluster solution. -# You can get cluster information by CLUSTER NODES|SLOTS|INFO command, it also is -# adapted to redis-cli, redis-benchmark, Redis cluster SDK, and Redis cluster proxy. -# But kvrocks doesn't support communicating with each other, so you must set -# cluster topology by CLUSTER SETNODES|SETNODEID commands, more details: #219. -# -# PLEASE NOTE: -# If you enable cluster, kvrocks will encode key with its slot id calculated by -# CRC16 and modulo 16384, encoding key with its slot id makes it efficient to -# migrate keys based on the slot. So if you enabled at first time, cluster mode must -# not be disabled after restarting, and vice versa. That is to say, data is not -# compatible between standalone mode with cluster mode, you must migrate data -# if you want to change mode, otherwise, kvrocks will make data corrupt. -# -# Default: no - -cluster-enabled no - - -# Persist the cluster nodes topology in local file($dir/nodes.conf). This configuration -# takes effect only if the cluster mode was enabled. -# -# If yes, it will try to load the cluster topology from the local file when starting, -# and dump the cluster nodes into the file if it was changed. -# -# Default: yes -# persist-cluster-nodes-enabled yes - -# Set the max number of connected clients at the same time. By default -# this limit is set to 10000 clients. However, if the server is not -# able to configure the process file limit to allow for the specified limit -# the max number of allowed clients is set to the current file limit -# -# Once the limit is reached the server will close all the new connections sending -# an error 'max number of clients reached'. -# -maxclients 10000 - -# Require clients to issue AUTH before processing any other -# commands. This might be useful in environments in which you do not trust -# others with access to the host running kvrocks. -# -# This should stay commented out for backward compatibility and because most -# people do not need auth (e.g. they run their own servers). -# -# Warning: since kvrocks is pretty fast an outside user can try up to -# 150k passwords per second against a good box. This means that you should -# use a very strong password otherwise it will be very easy to break. -# -# requirepass foobared - -# If the master is password protected (using the "masterauth" configuration -# directive below) it is possible to tell the slave to authenticate before -# starting the replication synchronization process. Otherwise, the master will -# refuse the slave request. -# -# masterauth foobared - -# Master-Salve replication would check db name is matched. if not, the slave should -# refuse to sync the db from master. Don't use the default value, set the db-name to identify -# the cluster. -db-name test.db - -# The working directory -# -# The DB will be written inside this directory -# Note that you must specify a directory here, not a file name. -dir /tmp/testdb/ - -# You can configure where to store your server logs by the log-dir. -# If you don't specify one, we will use the above `dir` as our default log directory. -# We also can send logs to stdout/stderr is as simple as: -# -log-dir stdout - -# Log level -# Possible values: info, warning, error, fatal -# Default: info -log-level info - -# You can configure log-retention-days to control whether to enable the log cleaner -# and the maximum retention days that the INFO level logs will be kept. -# -# if set to -1, that means to disable the log cleaner. -# if set to 0, all previous INFO level logs will be immediately removed. -# if set to between 0 to INT_MAX, that means it will retent latest N(log-retention-days) day logs. - -# By default the log-retention-days is -1. -log-retention-days -1 - -# When running in daemonize mode, kvrocks writes a PID file in ${CONFIG_DIR}/kvrocks.pid by -# default. You can specify a custom pid file location here. -# pidfile /var/run/kvrocks.pid -pidfile "" - -# You can configure a slave instance to accept writes or not. Writing against -# a slave instance may be useful to store some ephemeral data (because data -# written on a slave will be easily deleted after resync with the master) but -# may also cause problems if clients are writing to it because of a -# misconfiguration. -slave-read-only yes - -# The slave priority is an integer number published by Kvrocks in the INFO output. -# It is used by Redis Sentinel in order to select a slave to promote into a -# master if the master is no longer working correctly. -# -# A slave with a low priority number is considered better for promotion, so -# for instance if there are three slave with priority 10, 100, 25 Sentinel will -# pick the one with priority 10, that is the lowest. -# -# However a special priority of 0 marks the replica as not able to perform the -# role of master, so a slave with priority of 0 will never be selected by -# Redis Sentinel for promotion. -# -# By default the priority is 100. -slave-priority 100 - -# TCP listen() backlog. -# -# In high requests-per-second environments you need an high backlog in order -# to avoid slow clients connections issues. Note that the Linux kernel -# will silently truncate it to the value of /proc/sys/net/core/somaxconn so -# make sure to raise both the value of somaxconn and tcp_max_syn_backlog -# in order to Get the desired effect. -tcp-backlog 511 - -# If the master is an old version, it may have specified replication threads -# that use 'port + 1' as listening port, but in new versions, we don't use -# extra port to implement replication. In order to allow the new replicas to -# copy old masters, you should indicate that the master uses replication port -# or not. -# If yes, that indicates master uses replication port and replicas will connect -# to 'master's listening port + 1' when synchronization. -# If no, that indicates master doesn't use replication port and replicas will -# connect 'master's listening port' when synchronization. -master-use-repl-port no - -# Currently, master only checks sequence number when replica asks for PSYNC, -# that is not enough since they may have different replication histories even -# the replica asking sequence is in the range of the master current WAL. -# -# We design 'Replication Sequence ID' PSYNC, we add unique replication id for -# every write batch (the operation of each command on the storage engine), so -# the combination of replication id and sequence is unique for write batch. -# The master can identify whether the replica has the same replication history -# by checking replication id and sequence. -# -# By default, it is not enabled since this stricter check may easily lead to -# full synchronization. -use-rsid-psync no - -# Master-Slave replication. Use slaveof to make a kvrocks instance a copy of -# another kvrocks server. A few things to understand ASAP about kvrocks replication. -# -# 1) Kvrocks replication is asynchronous, but you can configure a master to -# stop accepting writes if it appears to be not connected with at least -# a given number of slaves. -# 2) Kvrocks slaves are able to perform a partial resynchronization with the -# master if the replication link is lost for a relatively small amount of -# time. You may want to configure the replication backlog size (see the next -# sections of this file) with a sensible value depending on your needs. -# 3) Replication is automatic and does not need user intervention. After a -# network partition slaves automatically try to reconnect to masters -# and resynchronize with them. -# -# slaveof -# slaveof 127.0.0.1 6379 - -# When a slave loses its connection with the master, or when the replication -# is still in progress, the slave can act in two different ways: -# -# 1) if slave-serve-stale-data is set to 'yes' (the default) the slave will -# still reply to client requests, possibly with out-of-date data, or the -# data set may just be empty if this is the first synchronization. -# -# 2) if slave-serve-stale-data is set to 'no' the slave will reply with -# an error "SYNC with master in progress" to all kinds of commands -# but to INFO and SLAVEOF. -# -slave-serve-stale-data yes - -# To guarantee slave's data safe and serve when it is in full synchronization -# state, slave still keep itself data. But this way needs to occupy much disk -# space, so we provide a way to reduce disk occupation, slave will delete itself -# entire database before fetching files from master during full synchronization. -# If you want to enable this way, you can set 'slave-delete-db-before-fullsync' -# to yes, but you must know that database will be lost if master is down during -# full synchronization, unless you have a backup of database. -# -# This option is similar redis replicas RDB diskless load option: -# repl-diskless-load on-empty-db -# -# Default: no -slave-empty-db-before-fullsync no - -# A Kvrocks master is able to list the address and port of the attached -# replicas in different ways. For example the "INFO replication" section -# offers this information, which is used, among other tools, by -# Redis Sentinel in order to discover replica instances. -# Another place where this info is available is in the output of the -# "ROLE" command of a master. -# -# The listed IP address and port normally reported by a replica is -# obtained in the following way: -# -# IP: The address is auto detected by checking the peer address -# of the socket used by the replica to connect with the master. -# -# Port: The port is communicated by the replica during the replication -# handshake, and is normally the port that the replica is using to -# listen for connections. -# -# However when port forwarding or Network Address Translation (NAT) is -# used, the replica may actually be reachable via different IP and port -# pairs. The following two options can be used by a replica in order to -# report to its master a specific set of IP and port, so that both INFO -# and ROLE will report those values. -# -# There is no need to use both the options if you need to override just -# the port or the IP address. -# -# replica-announce-ip 5.5.5.5 -# replica-announce-port 1234 - -# If replicas need full synchronization with master, master need to create -# checkpoint for feeding replicas, and replicas also stage a checkpoint of -# the master. If we also keep the backup, it maybe occupy extra disk space. -# You can enable 'purge-backup-on-fullsync' if disk is not sufficient, but -# that may cause remote backup copy failing. -# -# Default: no -purge-backup-on-fullsync no - -# The maximum allowed rate (in MB/s) that should be used by replication. -# If the rate exceeds max-replication-mb, replication will slow down. -# Default: 0 (i.e. no limit) -max-replication-mb 0 - -# The maximum allowed aggregated write rate of flush and compaction (in MB/s). -# If the rate exceeds max-io-mb, io will slow down. -# 0 is no limit -# Default: 0 -max-io-mb 0 - -# The maximum allowed space (in GB) that should be used by RocksDB. -# If the total size of the SST files exceeds max_allowed_space, writes to RocksDB will fail. -# Please see: https://github.com/facebook/rocksdb/wiki/Managing-Disk-Space-Utilization -# Default: 0 (i.e. no limit) -max-db-size 0 - -# The maximum backup to keep, server cron would run every minutes to check the num of current -# backup, and purge the old backup if exceed the max backup num to keep. If max-backup-to-keep -# is 0, no backup would be kept. But now, we only support 0 or 1. -max-backup-to-keep 1 - -# The maximum hours to keep the backup. If max-backup-keep-hours is 0, wouldn't purge any backup. -# default: 1 day -max-backup-keep-hours 24 - -# max-bitmap-to-string-mb use to limit the max size of bitmap to string transformation(MB). -# -# Default: 16 -max-bitmap-to-string-mb 16 - -# Whether to enable SCAN-like cursor compatible with Redis. -# If enabled, the cursor will be unsigned 64-bit integers. -# If disabled, the cursor will be a string. -# Default: no -redis-cursor-compatible no - -################################## TLS ################################### - -# By default, TLS/SSL is disabled, i.e. `tls-port` is set to 0. -# To enable it, `tls-port` can be used to define TLS-listening ports. -# tls-port 0 - -# Configure a X.509 certificate and private key to use for authenticating the -# server to connected clients, masters or cluster peers. -# These files should be PEM formatted. -# -# tls-cert-file kvrocks.crt -# tls-key-file kvrocks.key - -# If the key file is encrypted using a passphrase, it can be included here -# as well. -# -# tls-key-file-pass secret - -# Configure a CA certificate(s) bundle or directory to authenticate TLS/SSL -# clients and peers. Kvrocks requires an explicit configuration of at least one -# of these, and will not implicitly use the system wide configuration. -# -# tls-ca-cert-file ca.crt -# tls-ca-cert-dir /etc/ssl/certs - -# By default, clients on a TLS port are required -# to authenticate using valid client side certificates. -# -# If "no" is specified, client certificates are not required and not accepted. -# If "optional" is specified, client certificates are accepted and must be -# valid if provided, but are not required. -# -# tls-auth-clients no -# tls-auth-clients optional - -# By default, only TLSv1.2 and TLSv1.3 are enabled and it is highly recommended -# that older formally deprecated versions are kept disabled to reduce the attack surface. -# You can explicitly specify TLS versions to support. -# Allowed values are case insensitive and include "TLSv1", "TLSv1.1", "TLSv1.2", -# "TLSv1.3" (OpenSSL >= 1.1.1) or any combination. -# To enable only TLSv1.2 and TLSv1.3, use: -# -# tls-protocols "TLSv1.2 TLSv1.3" - -# Configure allowed ciphers. See the ciphers(1ssl) manpage for more information -# about the syntax of this string. -# -# Note: this configuration applies only to <= TLSv1.2. -# -# tls-ciphers DEFAULT:!MEDIUM - -# Configure allowed TLSv1.3 ciphersuites. See the ciphers(1ssl) manpage for more -# information about the syntax of this string, and specifically for TLSv1.3 -# ciphersuites. -# -# tls-ciphersuites TLS_CHACHA20_POLY1305_SHA256 - -# When choosing a cipher, use the server's preference instead of the client -# preference. By default, the server follows the client's preference. -# -# tls-prefer-server-ciphers yes - -# By default, TLS session caching is enabled to allow faster and less expensive -# reconnections by clients that support it. Use the following directive to disable -# caching. -# -# tls-session-caching no - -# Change the default number of TLS sessions cached. A zero value sets the cache -# to unlimited size. The default size is 20480. -# -# tls-session-cache-size 5000 - -# Change the default timeout of cached TLS sessions. The default timeout is 300 -# seconds. -# -# tls-session-cache-timeout 60 - -# By default, a replica does not attempt to establish a TLS connection -# with its master. -# -# Use the following directive to enable TLS on replication links. -# -# tls-replication yes - -################################## SLOW LOG ################################### - -# The Kvrocks Slow Log is a mechanism to log queries that exceeded a specified -# execution time. The execution time does not include the I/O operations -# like talking with the client, sending the reply and so forth, -# but just the time needed to actually execute the command (this is the only -# stage of command execution where the thread is blocked and can not serve -# other requests in the meantime). -# -# You can configure the slow log with two parameters: one tells Kvrocks -# what is the execution time, in microseconds, to exceed in order for the -# command to get logged, and the other parameter is the length of the -# slow log. When a new command is logged the oldest one is removed from the -# queue of logged commands. - -# The following time is expressed in microseconds, so 1000000 is equivalent -# to one second. Note that -1 value disables the slow log, while -# a value of zero forces the logging of every command. -slowlog-log-slower-than 100000 - -# There is no limit to this length. Just be aware that it will consume memory. -# You can reclaim memory used by the slow log with SLOWLOG RESET. -slowlog-max-len 128 - -# If you run kvrocks from upstart or systemd, kvrocks can interact with your -# supervision tree. Options: -# supervised no - no supervision interaction -# supervised upstart - signal upstart by putting kvrocks into SIGSTOP mode -# supervised systemd - signal systemd by writing READY=1 to $NOTIFY_SOCKET -# supervised auto - detect upstart or systemd method based on -# UPSTART_JOB or NOTIFY_SOCKET environment variables -# Note: these supervision methods only signal "process is ready." -# They do not enable continuous liveness pings back to your supervisor. -supervised no - -################################## PERF LOG ################################### - -# The Kvrocks Perf Log is a mechanism to log queries' performance context that -# exceeded a specified execution time. This mechanism uses rocksdb's -# Perf Context and IO Stats Context, Please see: -# https://github.com/facebook/rocksdb/wiki/Perf-Context-and-IO-Stats-Context -# -# This mechanism is enabled when profiling-sample-commands is not empty and -# profiling-sample-ratio greater than 0. -# It is important to note that this mechanism affects performance, but it is -# useful for troubleshooting performance bottlenecks, so it should only be -# enabled when performance problems occur. - -# The name of the commands you want to record. Must be original name of -# commands supported by Kvrocks. Use ',' to separate multiple commands and -# use '*' to record all commands supported by Kvrocks. -# Example: -# - Single command: profiling-sample-commands get -# - Multiple commands: profiling-sample-commands get,mget,hget -# -# Default: empty -# profiling-sample-commands "" - -# Ratio of the samples would be recorded. It is a number between 0 and 100. -# We simply use the rand to determine whether to record the sample or not. -# -# Default: 0 -profiling-sample-ratio 0 - -# There is no limit to this length. Just be aware that it will consume memory. -# You can reclaim memory used by the perf log with PERFLOG RESET. -# -# Default: 256 -profiling-sample-record-max-len 256 - -# profiling-sample-record-threshold-ms use to tell the kvrocks when to record. -# -# Default: 100 millisecond -profiling-sample-record-threshold-ms 100 - -################################## CRON ################################### - -# Compact Scheduler, auto compact at schedule time -# time expression format is the same as crontab(currently only support * and int) -# e.g. compact-cron 0 3 * * * 0 4 * * * -# would compact the db at 3am and 4am everyday -# compact-cron 0 3 * * * - -# The hour range that compaction checker would be active -# e.g. compaction-checker-range 0-7 means compaction checker would be worker between -# 0-7am every day. -compaction-checker-range 0-7 - -# When the compaction checker is triggered, the db will periodically pick the SST file -# with the highest "deleted percentage" (i.e. the percentage of deleted keys in the SST -# file) to compact, in order to free disk space. -# However, if a specific SST file was created more than "force-compact-file-age" seconds -# ago, and its percentage of deleted keys is higher than -# "force-compact-file-min-deleted-percentage", it will be forcely compacted as well. - -# Default: 172800 seconds; Range: [60, INT64_MAX]; -# force-compact-file-age 172800 -# Default: 10 %; Range: [1, 100]; -# force-compact-file-min-deleted-percentage 10 - -# Bgsave scheduler, auto bgsave at scheduled time -# time expression format is the same as crontab(currently only support * and int) -# e.g. bgsave-cron 0 3 * * * 0 4 * * * -# would bgsave the db at 3am and 4am every day - -# Command renaming. -# -# It is possible to change the name of dangerous commands in a shared -# environment. For instance, the KEYS command may be renamed into something -# hard to guess so that it will still be available for internal-use tools -# but not available for general clients. -# -# Example: -# -# rename-command KEYS b840fc02d524045429941cc15f59e41cb7be6c52 -# -# It is also possible to completely kill a command by renaming it into -# an empty string: -# -# rename-command KEYS "" - -################################ MIGRATE ##################################### -# If the network bandwidth is completely consumed by the migration task, -# it will affect the availability of kvrocks. To avoid this situation, -# migrate-speed is adopted to limit the migrating speed. -# Migrating speed is limited by controlling the duration between sending data, -# the duration is calculated by: 1000000 * migrate-pipeline-size / migrate-speed (us). -# Value: [0,INT_MAX], 0 means no limit -# -# Default: 4096 -migrate-speed 4096 - -# In order to reduce data transmission times and improve the efficiency of data migration, -# pipeline is adopted to send multiple data at once. Pipeline size can be set by this option. -# Value: [1, INT_MAX], it can't be 0 -# -# Default: 16 -migrate-pipeline-size 16 - -# In order to reduce the write forbidden time during migrating slot, we will migrate the incremental -# data several times to reduce the amount of incremental data. Until the quantity of incremental -# data is reduced to a certain threshold, slot will be forbidden write. The threshold is set by -# this option. -# Value: [1, INT_MAX], it can't be 0 -# -# Default: 10000 -migrate-sequence-gap 10000 - -################################ ROCKSDB ##################################### - -# Specify the capacity of column family block cache. A larger block cache -# may make requests faster while more keys would be cached. Max Size is 400*1024. -# Default: 4096MB -rocksdb.block_cache_size 4096 - -# A global cache for table-level rows in RocksDB. If almost always point -# lookups, enlarging row cache may improve read performance. Otherwise, -# if we enlarge this value, we can lessen metadata/subkey block cache size. -# -# Default: 0 (disabled) -rocksdb.row_cache_size 0 - -# Number of open files that can be used by the DB. You may need to -# increase this if your database has a large working set. Value -1 means -# files opened are always kept open. You can estimate number of files based -# on target_file_size_base and target_file_size_multiplier for level-based -# compaction. For universal-style compaction, you can usually set it to -1. -# Default: 8096 -rocksdb.max_open_files 8096 - -# Amount of data to build up in memory (backed by an unsorted log -# on disk) before converting to a sorted on-disk file. -# -# Larger values increase performance, especially during bulk loads. -# Up to max_write_buffer_number write buffers may be held in memory -# at the same time, -# so you may wish to adjust this parameter to control memory usage. -# Also, a larger write buffer will result in a longer recovery time -# the next time the database is opened. -# -# Note that write_buffer_size is enforced per column family. -# See db_write_buffer_size for sharing memory across column families. - -# default is 64MB -rocksdb.write_buffer_size 64 - -# Target file size for compaction, target file size for Level N can be calculated -# by target_file_size_base * (target_file_size_multiplier ^ (L-1)) -# -# Default: 128MB -rocksdb.target_file_size_base 128 - -# The maximum number of write buffers that are built up in memory. -# The default and the minimum number is 2, so that when 1 write buffer -# is being flushed to storage, new writes can continue to the other -# write buffer. -# If max_write_buffer_number > 3, writing will be slowed down to -# options.delayed_write_rate if we are writing to the last write buffer -# allowed. -rocksdb.max_write_buffer_number 4 - -# Maximum number of concurrent background jobs (compactions and flushes). -# For backwards compatibility we will set `max_background_jobs = -# max_background_compactions + max_background_flushes` in the case where user -# sets at least one of `max_background_compactions` or `max_background_flushes` -# (we replace -1 by 1 in case one option is unset). -rocksdb.max_background_jobs 4 - -# DEPRECATED: it is automatically decided based on the value of rocksdb.max_background_jobs -# Maximum number of concurrent background compaction jobs, submitted to -# the default LOW priority thread pool. -rocksdb.max_background_compactions -1 - -# DEPRECATED: it is automatically decided based on the value of rocksdb.max_background_jobs -# Maximum number of concurrent background memtable flush jobs, submitted by -# default to the HIGH priority thread pool. If the HIGH priority thread pool -# is configured to have zero threads, flush jobs will share the LOW priority -# thread pool with compaction jobs. -rocksdb.max_background_flushes -1 - -# This value represents the maximum number of threads that will -# concurrently perform a compaction job by breaking it into multiple, -# smaller ones that are run simultaneously. -# Default: 2 -rocksdb.max_sub_compactions 2 - -# In order to limit the size of WALs, RocksDB uses DBOptions::max_total_wal_size -# as the trigger of column family flush. Once WALs exceed this size, RocksDB -# will start forcing the flush of column families to allow deletion of some -# oldest WALs. This config can be useful when column families are updated at -# non-uniform frequencies. If there's no size limit, users may need to keep -# really old WALs when the infrequently-updated column families hasn't flushed -# for a while. -# -# In kvrocks, we use multiple column families to store metadata, subkeys, etc. -# If users always use string type, but use list, hash and other complex data types -# infrequently, there will be a lot of old WALs if we don't set size limit -# (0 by default in rocksdb), because rocksdb will dynamically choose the WAL size -# limit to be [sum of all write_buffer_size * max_write_buffer_number] * 4 if set to 0. -# -# Moreover, you should increase this value if you already set rocksdb.write_buffer_size -# to a big value, to avoid influencing the effect of rocksdb.write_buffer_size and -# rocksdb.max_write_buffer_number. -# -# default is 512MB -rocksdb.max_total_wal_size 512 - -# We implement the replication with rocksdb WAL, it would trigger full sync when the seq was out of range. -# wal_ttl_seconds and wal_size_limit_mb would affect how archived logs will be deleted. -# If WAL_ttl_seconds is not 0, then WAL files will be checked every WAL_ttl_seconds / 2 and those that -# are older than WAL_ttl_seconds will be deleted# -# -# Default: 3 Hours -rocksdb.wal_ttl_seconds 10800 - -# If WAL_ttl_seconds is 0 and WAL_size_limit_MB is not 0, -# WAL files will be checked every 10 min and if total size is greater -# then WAL_size_limit_MB, they will be deleted starting with the -# earliest until size_limit is met. All empty files will be deleted -# Default: 16GB -rocksdb.wal_size_limit_mb 16384 - -# Approximate size of user data packed per block. Note that the -# block size specified here corresponds to uncompressed data. The -# actual size of the unit read from disk may be smaller if -# compression is enabled. -# -# Default: 16KB -rocksdb.block_size 16384 - -# Indicating if we'd put index/filter blocks to the block cache -# -# Default: yes -rocksdb.cache_index_and_filter_blocks yes - -# Specify the compression to use. Only compress level greater -# than 2 to improve performance. -# Accept value: "no", "snappy", "lz4", "zstd", "zlib" -# default snappy -rocksdb.compression snappy - -# If non-zero, we perform bigger reads when doing compaction. If you're -# running RocksDB on spinning disks, you should set this to at least 2MB. -# That way RocksDB's compaction is doing sequential instead of random reads. -# When non-zero, we also force new_table_reader_for_compaction_inputs to -# true. -# -# Default: 2 MB -rocksdb.compaction_readahead_size 2097152 - -# he limited write rate to DB if soft_pending_compaction_bytes_limit or -# level0_slowdown_writes_trigger is triggered. - -# If the value is 0, we will infer a value from `rater_limiter` value -# if it is not empty, or 16MB if `rater_limiter` is empty. Note that -# if users change the rate in `rate_limiter` after DB is opened, -# `delayed_write_rate` won't be adjusted. -# -rocksdb.delayed_write_rate 0 -# If enable_pipelined_write is true, separate write thread queue is -# maintained for WAL write and memtable write. -# -# Default: no -rocksdb.enable_pipelined_write no - -# Soft limit on number of level-0 files. We start slowing down writes at this -# point. A value <0 means that no writing slow down will be triggered by -# number of files in level-0. -# -# Default: 20 -rocksdb.level0_slowdown_writes_trigger 20 - -# Maximum number of level-0 files. We stop writes at this point. -# -# Default: 40 -rocksdb.level0_stop_writes_trigger 40 - -# Number of files to trigger level-0 compaction. -# -# Default: 4 -rocksdb.level0_file_num_compaction_trigger 4 - -# if not zero, dump rocksdb.stats to LOG every stats_dump_period_sec -# -# Default: 0 -rocksdb.stats_dump_period_sec 0 - -# if yes, the auto compaction would be disabled, but the manual compaction remain works -# -# Default: no -rocksdb.disable_auto_compactions no - -# BlobDB(key-value separation) is essentially RocksDB for large-value use cases. -# Since 6.18.0, The new implementation is integrated into the RocksDB core. -# When set, large values (blobs) are written to separate blob files, and only -# pointers to them are stored in SST files. This can reduce write amplification -# for large-value use cases at the cost of introducing a level of indirection -# for reads. Please see: https://github.com/facebook/rocksdb/wiki/BlobDB. -# -# Note that when enable_blob_files is set to yes, BlobDB-related configuration -# items will take effect. -# -# Default: no -rocksdb.enable_blob_files no - -# The size of the smallest value to be stored separately in a blob file. Values -# which have an uncompressed size smaller than this threshold are stored alongside -# the keys in SST files in the usual fashion. -# -# Default: 4096 byte, 0 means that all values are stored in blob files -rocksdb.min_blob_size 4096 - -# The size limit for blob files. When writing blob files, a new file is -# opened once this limit is reached. -# -# Default: 268435456 bytes -rocksdb.blob_file_size 268435456 - -# Enables garbage collection of blobs. Valid blobs residing in blob files -# older than a cutoff get relocated to new files as they are encountered -# during compaction, which makes it possible to clean up blob files once -# they contain nothing but obsolete/garbage blobs. -# See also rocksdb.blob_garbage_collection_age_cutoff below. -# -# Default: yes -rocksdb.enable_blob_garbage_collection yes - -# The percentage cutoff in terms of blob file age for garbage collection. -# Blobs in the oldest N blob files will be relocated when encountered during -# compaction, where N = (garbage_collection_cutoff/100) * number_of_blob_files. -# Note that this value must belong to [0, 100]. -# -# Default: 25 -rocksdb.blob_garbage_collection_age_cutoff 25 - - -# The purpose of the following three options are to dynamically adjust the upper limit of -# the data that each layer can store according to the size of the different -# layers of the LSM. Enabling this option will bring some improvements in -# deletion efficiency and space amplification, but it will lose a certain -# amount of read performance. -# If you want to know more details about Levels' Target Size, you can read RocksDB wiki: -# https://github.com/facebook/rocksdb/wiki/Leveled-Compaction#levels-target-size -# -# Default: yes -rocksdb.level_compaction_dynamic_level_bytes yes - -# The total file size of level-1 sst. -# -# Default: 268435456 bytes -rocksdb.max_bytes_for_level_base 268435456 - -# Multiplication factor for the total file size of L(n+1) layers. -# This option is a double type number in RocksDB, but kvrocks is -# not support the double data type number yet, so we use integer -# number instead of double currently. -# -# Default: 10 -rocksdb.max_bytes_for_level_multiplier 10 - -# This feature only takes effect in Iterators and MultiGet. -# If yes, RocksDB will try to read asynchronously and in parallel as much as possible to hide IO latency. -# In iterators, it will prefetch data asynchronously in the background for each file being iterated on. -# In MultiGet, it will read the necessary data blocks from those files in parallel as much as possible. - -# Default no -rocksdb.read_options.async_io no - -# If yes, the write will be flushed from the operating system -# buffer cache before the write is considered complete. -# If this flag is enabled, writes will be slower. -# If this flag is disabled, and the machine crashes, some recent -# rites may be lost. Note that if it is just the process that -# crashes (i.e., the machine does not reboot), no writes will be -# lost even if sync==false. -# -# Default: no -rocksdb.write_options.sync no - -# If yes, writes will not first go to the write ahead log, -# and the write may get lost after a crash. -# -# Default: no -rocksdb.write_options.disable_wal no - -# If enabled and we need to wait or sleep for the write request, fails -# immediately. -# -# Default: no -rocksdb.write_options.no_slowdown no - -# If enabled, write requests are of lower priority if compaction is -# behind. In this case, no_slowdown = true, the request will be canceled -# immediately. Otherwise, it will be slowed down. -# The slowdown value is determined by RocksDB to guarantee -# it introduces minimum impacts to high priority writes. -# -# Default: no -rocksdb.write_options.low_pri no - -# If enabled, this writebatch will maintain the last insert positions of each -# memtable as hints in concurrent write. It can improve write performance -# in concurrent writes if keys in one writebatch are sequential. -# -# Default: no -rocksdb.write_options.memtable_insert_hint_per_batch no - - -# Support RocksDB auto-tune rate limiter for the background IO -# if enabled, Rate limiter will limit the compaction write if flush write is high -# Please see https://rocksdb.org/blog/2017/12/18/17-auto-tuned-rate-limiter.html -# -# Default: yes -rocksdb.rate_limiter_auto_tuned yes - -################################ NAMESPACE ##################################### -# namespace.test change.me From cfd156525f618739ee902bea53290cd7b5509aa3 Mon Sep 17 00:00:00 2001 From: xq2010 Date: Mon, 16 Oct 2023 15:16:20 +0800 Subject: [PATCH 4/4] fix:Adjust the code according to the review --- src/commands/cmd_server.cc | 15 ++-- src/common/rdb_stream.cc | 5 +- src/common/rdb_stream.h | 12 ++- src/storage/rdb.cc | 131 +++++++++++++------------------ src/storage/rdb.h | 11 ++- src/storage/redis_db.cc | 14 ++++ src/storage/redis_db.h | 1 + tests/cppunit/rdb_stream_test.cc | 2 +- tests/cppunit/rdb_test.cc | 38 ++++----- 9 files changed, 108 insertions(+), 121 deletions(-) diff --git a/src/commands/cmd_server.cc b/src/commands/cmd_server.cc index c5dee98f3fa..14ca00eeca5 100644 --- a/src/commands/cmd_server.cc +++ b/src/commands/cmd_server.cc @@ -1060,8 +1060,8 @@ class CommandRestore : public Commander { ttl_ms_ -= now; } - auto stream_ptr = std::make_shared(args_[3]); - RDB rdb(svr->storage, conn->GetNamespace(), stream_ptr); + auto stream_ptr = std::make_unique(args_[3]); + RDB rdb(svr->storage, conn->GetNamespace(), std::move(stream_ptr)); auto s = rdb.Restore(args_[1], args_[3], ttl_ms_); if (!s.IsOK()) return {Status::RedisExecErr, s.Msg()}; *output = redis::SimpleString("OK"); @@ -1079,10 +1079,9 @@ class CommandRdb : public Commander { public: Status Parse(const std::vector &args) override { CommandParser parser(args, 3); - std::string_view ttl_flag, set_flag; while (parser.Good()) { if (parser.EatEqICase("NX")) { - is_nx_ = true; + overwrite_exist_key_ = false; } else if (parser.EatEqICase("DB")) { db_index_ = GET_OR_RET(parser.TakeInt()); } else { @@ -1099,18 +1098,18 @@ class CommandRdb : public Commander { auto type = args_[1]; auto path = args_[2]; - auto stream_ptr = std::make_shared(path); + auto stream_ptr = std::make_unique(path); GET_OR_RET(stream_ptr->Open()); - RDB rdb(svr->storage, conn->GetNamespace(), stream_ptr); - GET_OR_RET(rdb.LoadRdb(db_index_, is_nx_)); + RDB rdb(svr->storage, conn->GetNamespace(), std::move(stream_ptr)); + GET_OR_RET(rdb.LoadRdb(db_index_, overwrite_exist_key_)); *output = redis::SimpleString("OK"); return Status::OK(); } private: - bool is_nx_ = false; + bool overwrite_exist_key_ = true; // default overwrite exist key uint32_t db_index_ = 0; }; diff --git a/src/common/rdb_stream.cc b/src/common/rdb_stream.cc index b029b728c99..83855ea6d64 100644 --- a/src/common/rdb_stream.cc +++ b/src/common/rdb_stream.cc @@ -22,7 +22,6 @@ #include "fmt/format.h" #include "vendor/crc64.h" -#include "vendor/endianconv.h" StatusOr RdbStringStream::Read(char *buf, size_t n) { if (pos_ + n > input_.size()) { @@ -54,12 +53,12 @@ Status RdbFileStream::Open() { StatusOr RdbFileStream::Read(char *buf, size_t len) { size_t n = 0; while (len) { - size_t read_bytes = max_read_chunk_size_ < len ? max_read_chunk_size_ : len; + size_t read_bytes = std::min(max_read_chunk_size_, len); ifs_.read(buf, static_cast(read_bytes)); if (!ifs_.good()) { return Status(Status::NotOK, fmt::format("read failed: {}:", strerror(errno))); } - check_sum_ = crc64(check_sum_, (const unsigned char *)buf, read_bytes); + check_sum_ = crc64(check_sum_, reinterpret_cast(buf), read_bytes); buf = (char *)buf + read_bytes; len -= read_bytes; total_read_bytes_ += read_bytes; diff --git a/src/common/rdb_stream.h b/src/common/rdb_stream.h index 7cea5ab5df2..842d24a7a6b 100644 --- a/src/common/rdb_stream.h +++ b/src/common/rdb_stream.h @@ -20,12 +20,12 @@ #pragma once -#include - +#include #include #include #include "status.h" +#include "vendor/endianconv.h" class RdbStream { public: @@ -69,7 +69,11 @@ class RdbFileStream : public RdbStream { Status Open(); StatusOr Read(char *buf, size_t len) override; - StatusOr GetCheckSum() const override { return check_sum_; } + StatusOr GetCheckSum() const override { + uint64_t crc = check_sum_; + memrev64ifbe(&crc); + return crc; + } private: std::ifstream ifs_; @@ -77,4 +81,4 @@ class RdbFileStream : public RdbStream { uint64_t check_sum_; size_t total_read_bytes_; size_t max_read_chunk_size_; // maximum single read chunk size -}; \ No newline at end of file +}; diff --git a/src/storage/rdb.cc b/src/storage/rdb.cc index 8433617f9bb..c73c172ef45 100644 --- a/src/storage/rdb.cc +++ b/src/storage/rdb.cc @@ -63,33 +63,35 @@ constexpr const int RDBOpcodeExpireTime = 253; /* Old expire time in seconds. constexpr const int RDBOpcodeSelectDB = 254; /* DB number of the following keys. */ constexpr const int RDBOpcodeEof = 255; /* End of the RDB file. */ -// The current support RDB version -constexpr const int RDBVersion = 10; - -// NOLINTNEXTLINE -#define GET_OR_RETWITHLOG(...) \ - ({ \ - auto &&status = (__VA_ARGS__); \ - if (!status) { \ - LOG(WARNING) << "Short read or unsupported type loading DB. Unrecoverable error, aborting now."; \ - LOG(ERROR) << "Unexpected EOF reading RDB file"; \ - return std::forward(status); \ - } \ - std::forward(status); \ - }).GetValue() +constexpr const int SupportedRDBVersion = 10; // not been tested for version 11, so use this version with caution. +constexpr const int MaxRDBVersion = 11; // The current max rdb version supported by redis. + +constexpr const int RDBCheckSumLen = 8; // rdb check sum length +constexpr const int RestoreRdbVersionLen = 2; // rdb version len in restore string +constexpr const int RestoreFooterLen = RestoreRdbVersionLen + RDBCheckSumLen; // 10 = ver len + checksum len +constexpr const int MinRdbVersionToVerifyChecksum = 5; + +template +T LogWhenError(T &&s) { + if (!s) { + LOG(WARNING) << "Short read or unsupported type loading DB. Unrecoverable error, aborting now."; + LOG(ERROR) << "Unexpected EOF reading RDB file"; + } + return std::forward(s); +} Status RDB::VerifyPayloadChecksum(const std::string_view &payload) { - if (payload.size() < 10) { + if (payload.size() < RestoreFooterLen) { // at least has rdb version and checksum return {Status::NotOK, "invalid payload length"}; } - auto footer = payload.substr(payload.size() - 10); + auto footer = payload.substr(payload.size() - RestoreFooterLen); auto rdb_version = (footer[1] << 8) | footer[0]; // For now, the max redis rdb version is 11 - if (rdb_version > 11) { + if (rdb_version > MaxRDBVersion) { return {Status::NotOK, fmt::format("invalid or unsupported rdb version: {}", rdb_version)}; } auto crc = GET_OR_RET(stream_->GetCheckSum()); - if (memcmp(&crc, footer.data() + 2, 8)) { + if (memcmp(&crc, footer.data() + RestoreRdbVersionLen, RDBCheckSumLen)) { return {Status::NotOK, "incorrect checksum"}; } return Status::OK(); @@ -118,10 +120,10 @@ StatusOr RDB::loadObjectLen(bool *is_encoded) { return (len << 8) | GET_OR_RET(stream_->ReadByte()); default: if (c == RDB32BitLen) { - GET_OR_RET(stream_->Read(reinterpret_cast(&len), 4)); + GET_OR_RET(stream_->Read(reinterpret_cast(&len), sizeof(uint32_t))); return ntohl(len); } else if (c == RDB64BitLen) { - GET_OR_RET(stream_->Read(reinterpret_cast(&len), 8)); + GET_OR_RET(stream_->Read(reinterpret_cast(&len), sizeof(uint64_t))); return ntohu64(len); } else { return {Status::NotOK, fmt::format("Unknown RDB string encoding type {} byte {}", type, c)}; @@ -444,12 +446,9 @@ StatusOr RDB::loadRdbObject(int type, const std::string &key) { elements = GET_OR_RET(LoadListWithQuickList(type)); } return elements; - } else { - return {Status::RedisParseErr, fmt::format("unsupported type: {}", type)}; } - // can't be here - return Status::OK(); + return {Status::RedisParseErr, fmt::format("unsupported type: {}", type)}; } Status RDB::saveRdbObject(int type, const std::string &key, const RedisObjValue &obj, uint64_t ttl_ms) { @@ -511,13 +510,13 @@ Status RDB::saveRdbObject(int type, const std::string &key, const RedisObjValue return db_status.ok() ? Status::OK() : Status{Status::RedisExecErr, db_status.ToString()}; } -StatusOr RDB::loadTime() { +StatusOr RDB::loadExpiredTimeSeconds() { uint32_t t32 = 0; GET_OR_RET(stream_->Read(reinterpret_cast(&t32), 4)); return t32; } -StatusOr RDB::loadMillisecondTime(int rdb_version) { +StatusOr RDB::loadExpiredTimeMilliseconds(int rdb_version) { uint64_t t64 = 0; GET_OR_RET(stream_->Read(reinterpret_cast(&t64), 8)); /* before Redis 5 (RDB version 9), the function @@ -545,20 +544,20 @@ bool RDB::isEmptyRedisObject(const RedisObjValue &value) { } // Load RDB file: copy from redis/src/rdb.c:branch 7.0, 76b9c13d. -Status RDB::LoadRdb(uint32_t db_index, bool is_nx) { +Status RDB::LoadRdb(uint32_t db_index, bool overwrite_exist_key) { char buf[1024] = {0}; - GET_OR_RETWITHLOG(stream_->Read(buf, 9)); + GET_OR_RET(LogWhenError(stream_->Read(buf, 9))); buf[9] = '\0'; if (memcmp(buf, "REDIS", 5) != 0) { LOG(WARNING) << "Wrong signature trying to load DB from file"; - return {Status::NotOK}; + return {Status::NotOK, "Wrong signature trying to load DB from file"}; } auto rdb_ver = std::atoi(buf + 5); - if (rdb_ver < 1 || rdb_ver > RDBVersion) { + if (rdb_ver < 1 || rdb_ver > SupportedRDBVersion) { LOG(WARNING) << "Can't handle RDB format version " << rdb_ver; - return {Status::NotOK}; + return {Status::NotOK, fmt::format("Can't handle RDB format version {}", rdb_ver)}; } uint64_t expire_time = 0; @@ -569,29 +568,29 @@ Status RDB::LoadRdb(uint32_t db_index, bool is_nx) { uint32_t db_id = 0; uint64_t skip_exist_keys = 0; while (true) { - auto type = GET_OR_RETWITHLOG(loadRdbType()); + auto type = GET_OR_RET(LogWhenError(loadRdbType())); if (type == RDBOpcodeExpireTime) { - expire_time = static_cast(GET_OR_RETWITHLOG(loadTime())); + expire_time = static_cast(GET_OR_RET(LogWhenError(loadExpiredTimeSeconds()))); expire_time *= 1000; continue; } else if (type == RDBOpcodeExpireTimeMs) { - expire_time = GET_OR_RETWITHLOG(loadMillisecondTime(rdb_ver)); + expire_time = GET_OR_RET(LogWhenError(loadExpiredTimeMilliseconds(rdb_ver))); continue; - } else if (type == RDBOpcodeFreq) { // LFU frequency: not use in kvrocks - GET_OR_RETWITHLOG(stream_->ReadByte()); // discard the value + } else if (type == RDBOpcodeFreq) { // LFU frequency: not use in kvrocks + GET_OR_RET(LogWhenError(stream_->ReadByte())); // discard the value continue; } else if (type == RDBOpcodeIdle) { // LRU idle time: not use in kvrocks uint64_t discard = 0; - GET_OR_RETWITHLOG(stream_->Read(reinterpret_cast(&discard), 8)); + GET_OR_RET(LogWhenError(stream_->Read(reinterpret_cast(&discard), sizeof(uint64_t)))); continue; } else if (type == RDBOpcodeEof) { break; } else if (type == RDBOpcodeSelectDB) { - db_id = GET_OR_RETWITHLOG(loadObjectLen(nullptr)); + db_id = GET_OR_RET(LogWhenError(loadObjectLen(nullptr))); continue; - } else if (type == RDBOpcodeResizeDB) { // not use in kvrocks, hint redis for hash table resize - GET_OR_RETWITHLOG(loadObjectLen(nullptr)); // db_size - GET_OR_RETWITHLOG(loadObjectLen(nullptr)); // expires_size + } else if (type == RDBOpcodeResizeDB) { // not use in kvrocks, hint redis for hash table resize + GET_OR_RET(LogWhenError(loadObjectLen(nullptr))); // db_size + GET_OR_RET(LogWhenError(loadObjectLen(nullptr))); // expires_size continue; } else if (type == RDBOpcodeAux) { /* AUX: generic string-string fields. Use to add state to RDB @@ -599,8 +598,8 @@ Status RDB::LoadRdb(uint32_t db_index, bool is_nx) { * are required to skip AUX fields they don't understand. * * An AUX field is composed of two strings: key and value. */ - auto key = GET_OR_RETWITHLOG(LoadStringObject()); - auto value = GET_OR_RETWITHLOG(LoadStringObject()); + auto key = GET_OR_RET(LogWhenError(LoadStringObject())); + auto value = GET_OR_RET(LogWhenError(LoadStringObject())); continue; } else if (type == RDBOpcodeModuleAux) { LOG(WARNING) << "RDB module not supported"; @@ -611,12 +610,12 @@ Status RDB::LoadRdb(uint32_t db_index, bool is_nx) { } else { if (!isObjectType(type)) { LOG(WARNING) << "Invalid or Not supported object type: " << type; - return {Status::NotOK, "Invalid or Not supported object type"}; + return {Status::NotOK, fmt::format("Invalid or Not supported object type {}", type)}; } } - auto key = GET_OR_RETWITHLOG(LoadStringObject()); - auto value = GET_OR_RETWITHLOG(loadRdbObject(type, key)); + auto key = GET_OR_RET(LogWhenError(LoadStringObject())); + auto value = GET_OR_RET(LogWhenError(loadRdbObject(type, key))); if (db_index != db_id) { // skip db not match continue; @@ -627,7 +626,7 @@ Status RDB::LoadRdb(uint32_t db_index, bool is_nx) { * (See #8453), we rather not fail when empty key is encountered * in an RDB file, instead we will silently discard it and * continue loading. */ - if (empty_keys_skipped++ < 10) { + if (empty_keys_skipped++ < 10) { // only log 10 empty keys, just as redis does. LOG(WARNING) << "skipping empty key: " << key; } continue; @@ -637,8 +636,9 @@ Status RDB::LoadRdb(uint32_t db_index, bool is_nx) { continue; } - if (is_nx) { // only load not exist key - auto s = exist(key); + if (!overwrite_exist_key) { // only load not exist key + redis::Database redis(storage_, ns_); + auto s = redis.KeyExist(key); if (!s.IsNotFound()) { skip_exist_keys++; // skip it even it's not okay if (!s.ok()) { @@ -657,41 +657,22 @@ Status RDB::LoadRdb(uint32_t db_index, bool is_nx) { } // Verify the checksum if RDB version is >= 5 - if (rdb_ver >= 5) { + if (rdb_ver >= MinRdbVersionToVerifyChecksum) { uint64_t chk_sum = 0; - auto expected = GET_OR_RETWITHLOG(stream_->GetCheckSum()); - GET_OR_RETWITHLOG(stream_->Read(reinterpret_cast(&chk_sum), 8)); + auto expected = GET_OR_RET(LogWhenError(stream_->GetCheckSum())); + GET_OR_RET(LogWhenError(stream_->Read(reinterpret_cast(&chk_sum), RDBCheckSumLen))); if (chk_sum == 0) { LOG(WARNING) << "RDB file was saved with checksum disabled: no check performed."; } else if (chk_sum != expected) { LOG(WARNING) << "Wrong RDB checksum expected: " << chk_sum << " got: " << expected; - return {Status::NotOK, "Wrong RDB checksum"}; + return {Status::NotOK, "All objects were processed and loaded but the checksum is unexpected!"}; } } - std::string skip_info = (is_nx ? ", exist keys skipped: " + std::to_string(skip_exist_keys) : ""); + std::string skip_info = (overwrite_exist_key ? ", exist keys skipped: " + std::to_string(skip_exist_keys) : ""); - if (empty_keys_skipped > 0) { - LOG(INFO) << "Done loading RDB, keys loaded: " << load_keys << ", keys expired:" << expire_keys - << ", empty keys skipped: " << empty_keys_skipped << skip_info; - } else { - LOG(INFO) << "Done loading RDB, keys loaded: " << load_keys << ", keys expired:" << expire_keys << skip_info; - } + LOG(INFO) << "Done loading RDB, keys loaded: " << load_keys << ", keys expired:" << expire_keys + << ", empty keys skipped: " << empty_keys_skipped << skip_info; return Status::OK(); } - -rocksdb::Status RDB::exist(const std::string &key) { - int cnt = 0; - std::vector keys; - keys.emplace_back(key); - redis::Database redis(storage_, ns_); - auto s = redis.Exists(keys, &cnt); - if (!s.ok()) { - return s; - } - if (cnt == 0) { - return rocksdb::Status::NotFound(); - } - return rocksdb::Status::OK(); -} \ No newline at end of file diff --git a/src/storage/rdb.h b/src/storage/rdb.h index 8faabce5cd4..7b4cce31f98 100644 --- a/src/storage/rdb.h +++ b/src/storage/rdb.h @@ -65,7 +65,7 @@ using RedisObjValue = class RDB { public: - explicit RDB(engine::Storage *storage, std::string ns, std::shared_ptr stream) + explicit RDB(engine::Storage *storage, std::string ns, std::unique_ptr stream) : storage_(storage), ns_(std::move(ns)), stream_(std::move(stream)){}; ~RDB() = default; @@ -98,12 +98,12 @@ class RDB { StatusOr> LoadListWithQuickList(int type); // Load rdb - Status LoadRdb(uint32_t db_index, bool is_nx = false); + Status LoadRdb(uint32_t db_index, bool overwrite_exist_key = true); private: engine::Storage *storage_; std::string ns_; - std::shared_ptr stream_; + std::unique_ptr stream_; StatusOr loadLzfString(); StatusOr loadEncodedString(); @@ -114,9 +114,8 @@ class RDB { StatusOr loadRdbType(); StatusOr loadRdbObject(int rdbtype, const std::string &key); Status saveRdbObject(int type, const std::string &key, const RedisObjValue &obj, uint64_t ttl_ms); - StatusOr loadTime(); - StatusOr loadMillisecondTime(int rdb_version); - rocksdb::Status exist(const std::string &key); + StatusOr loadExpiredTimeSeconds(); + StatusOr loadExpiredTimeMilliseconds(int rdb_version); /*0-5 is the basic type of Redis objects and 9-21 is the encoding type of Redis objects. Redis allow basic is 0-7 and 6/7 is for the module type which we don't support here.*/ diff --git a/src/storage/redis_db.cc b/src/storage/redis_db.cc index 9201d386eee..ff6bee4fa16 100644 --- a/src/storage/redis_db.cc +++ b/src/storage/redis_db.cc @@ -614,6 +614,20 @@ rocksdb::Status Database::GetSlotKeysInfo(int slot, std::map *slo return rocksdb::Status::OK(); } +rocksdb::Status Database::KeyExist(const std::string &key) { + int cnt = 0; + std::vector keys; + keys.emplace_back(key); + auto s = Exists(keys, &cnt); + if (!s.ok()) { + return s; + } + if (cnt == 0) { + return rocksdb::Status::NotFound(); + } + return rocksdb::Status::OK(); +} + rocksdb::Status SubKeyScanner::Scan(RedisType type, const Slice &user_key, const std::string &cursor, uint64_t limit, const std::string &subkey_prefix, std::vector *keys, std::vector *values) { diff --git a/src/storage/redis_db.h b/src/storage/redis_db.h index 8b1d7c32399..d81799b377c 100644 --- a/src/storage/redis_db.h +++ b/src/storage/redis_db.h @@ -59,6 +59,7 @@ class Database { [[nodiscard]] rocksdb::Status ClearKeysOfSlot(const rocksdb::Slice &ns, int slot); [[nodiscard]] rocksdb::Status GetSlotKeysInfo(int slot, std::map *slotskeys, std::vector *keys, int count); + [[nodiscard]] rocksdb::Status KeyExist(const std::string &key); protected: engine::Storage *storage_; diff --git a/tests/cppunit/rdb_stream_test.cc b/tests/cppunit/rdb_stream_test.cc index 508f4f69b94..c82a04cfac1 100644 --- a/tests/cppunit/rdb_stream_test.cc +++ b/tests/cppunit/rdb_stream_test.cc @@ -87,4 +87,4 @@ TEST(RdbFileStreamReadTest, ReadRdbLittleChunk) { if (size > 0) { ASSERT_EQ(reader.Read(buf, size).GetValue(), size); } -} \ No newline at end of file +} diff --git a/tests/cppunit/rdb_test.cc b/tests/cppunit/rdb_test.cc index d63060d3804..d3a18355d46 100644 --- a/tests/cppunit/rdb_test.cc +++ b/tests/cppunit/rdb_test.cc @@ -51,11 +51,11 @@ class RDBTest : public TestBase { void TearDown() override { ASSERT_TRUE(clearDBDir(config_->db_dir)); } void loadRdb(const std::string &path) { - auto stream_ptr = std::make_shared(path); + auto stream_ptr = std::make_unique(path); auto s = stream_ptr->Open(); ASSERT_TRUE(s.IsOK()); - RDB rdb(storage_, ns_, stream_ptr); + RDB rdb(storage_, ns_, std::move(stream_ptr)); s = rdb.LoadRdb(0); ASSERT_TRUE(s.IsOK()); } @@ -116,19 +116,9 @@ class RDBTest : public TestBase { } } - rocksdb::Status exists(const std::string &key) { - int cnt = 0; - std::vector keys; - keys.emplace_back(key); + rocksdb::Status keyExist(const std::string &key) { redis::Database redis(storage_, ns_); - auto s = redis.Exists(keys, &cnt); - if (!s.ok()) { - return s; - } - if (cnt == 0) { - return rocksdb::Status::NotFound(); - } - return rocksdb::Status::OK(); + return redis.KeyExist(key); } void flushDB() { @@ -287,36 +277,36 @@ TEST_F(RDBTest, LoadEmptyKeys) { */ // string - rocksdb::Status s = exists("empty_string"); // empty_string not exist in rdb file + rocksdb::Status s = keyExist("empty_string"); // empty_string not exist in rdb file ASSERT_TRUE(s.IsNotFound()); // list - s = exists("list_ziplist"); + s = keyExist("list_ziplist"); ASSERT_TRUE(s.IsNotFound()); - s = exists("list_quicklist"); + s = keyExist("list_quicklist"); ASSERT_TRUE(s.IsNotFound()); - s = exists("list_quicklist_empty_ziplist"); + s = keyExist("list_quicklist_empty_ziplist"); // set - s = exists("set"); + s = keyExist("set"); ASSERT_TRUE(s.IsNotFound()); // hash - s = exists("hash"); + s = keyExist("hash"); ASSERT_TRUE(s.IsNotFound()); - s = exists("hash_ziplist"); + s = keyExist("hash_ziplist"); ASSERT_TRUE(s.IsNotFound()); // zset - s = exists("zset"); + s = keyExist("zset"); ASSERT_TRUE(s.IsNotFound()); - s = exists("zset_ziplist"); + s = keyExist("zset_ziplist"); ASSERT_TRUE(s.IsNotFound()); - s = exists("zset_listpack"); + s = keyExist("zset_listpack"); ASSERT_TRUE(s.IsNotFound()); } \ No newline at end of file