From 9376a4f1eed35c5ba7ac95c1b81a694ebd8d563d Mon Sep 17 00:00:00 2001 From: liuh-80 Date: Tue, 27 Aug 2024 08:01:49 +0000 Subject: [PATCH 01/23] Add debug info --- orchagent/orch.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/orchagent/orch.cpp b/orchagent/orch.cpp index d1cbdb89c8..b38e9cd818 100644 --- a/orchagent/orch.cpp +++ b/orchagent/orch.cpp @@ -248,6 +248,10 @@ void Consumer::execute() { std::deque entries; table->pops(entries); + + // add to sync + SWSS_LOG_ERROR("Table: %s pops: %d", table->getTableName().c_str(), (int)(entries.size())); + update_size = addToSync(entries); } while (update_size != 0); From 96967cf974d9f5fc70a6292c39b7e3a496c3031e Mon Sep 17 00:00:00 2001 From: liuh-80 Date: Wed, 28 Aug 2024 02:09:46 +0000 Subject: [PATCH 02/23] Revert change --- orchagent/orch.cpp | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/orchagent/orch.cpp b/orchagent/orch.cpp index b38e9cd818..875b0265e8 100644 --- a/orchagent/orch.cpp +++ b/orchagent/orch.cpp @@ -242,6 +242,7 @@ void Consumer::execute() // ConsumerBase::execute_impl(); SWSS_LOG_ENTER(); + /* size_t update_size = 0; auto table = static_cast(getSelectable()); do @@ -249,11 +250,17 @@ void Consumer::execute() std::deque entries; table->pops(entries); - // add to sync - SWSS_LOG_ERROR("Table: %s pops: %d", table->getTableName().c_str(), (int)(entries.size())); - update_size = addToSync(entries); } while (update_size != 0); + */ + + // Only handle less than 128 entries + auto table = static_cast(getSelectable()); + std::deque entries; + table->pops(entries); + + // add to sync + addToSync(entries); drain(); } From 229f8838e6efaf5ccd51924c6fef3ca1aaabbfca Mon Sep 17 00:00:00 2001 From: Hua Liu <58683130+liuh-80@users.noreply.github.com> Date: Fri, 30 Aug 2024 13:59:35 +0800 Subject: [PATCH 03/23] Update orch.cpp --- orchagent/orch.cpp | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/orchagent/orch.cpp b/orchagent/orch.cpp index 875b0265e8..708a86280a 100644 --- a/orchagent/orch.cpp +++ b/orchagent/orch.cpp @@ -242,19 +242,6 @@ void Consumer::execute() // ConsumerBase::execute_impl(); SWSS_LOG_ENTER(); - /* - size_t update_size = 0; - auto table = static_cast(getSelectable()); - do - { - std::deque entries; - table->pops(entries); - - update_size = addToSync(entries); - } while (update_size != 0); - */ - - // Only handle less than 128 entries auto table = static_cast(getSelectable()); std::deque entries; table->pops(entries); From a59fc7626d622d96f1d18c9a8708d06bb1b424b0 Mon Sep 17 00:00:00 2001 From: liuh-80 Date: Mon, 2 Sep 2024 06:41:30 +0000 Subject: [PATCH 04/23] Add test case --- tests/mock_tests/consumer_ut.cpp | 98 +++++++++----------- tests/mock_tests/mock_consumerstatetable.cpp | 33 ++++++- 2 files changed, 74 insertions(+), 57 deletions(-) diff --git a/tests/mock_tests/consumer_ut.cpp b/tests/mock_tests/consumer_ut.cpp index 500bf45879..a67b7235c7 100644 --- a/tests/mock_tests/consumer_ut.cpp +++ b/tests/mock_tests/consumer_ut.cpp @@ -1,21 +1,36 @@ + #include "ut_helper.h" #include "mock_orchagent_main.h" #include "mock_table.h" - #include - extern PortsOrch *gPortsOrch; - namespace consumer_test { using namespace std; + class TestOrch : public Orch + { + public: + TestOrch(swss::DBConnector *db, string tableName) + :Orch(db, tableName), + m_notification_count(0) + { + } + + void doTask(Consumer& consumer) + { + std::cout << "TestOrch::doTask " << consumer.m_toSync.size() << std::endl; + m_notification_count += consumer.m_toSync.size(); + consumer.m_toSync.clear(); + } + + long m_notification_count; + }; struct ConsumerTest : public ::testing::Test { shared_ptr m_app_db; shared_ptr m_config_db; shared_ptr m_state_db; - string key = "key"; string f1 = "field1"; string v1a = "value1_a"; @@ -26,10 +41,8 @@ namespace consumer_test string f3 = "field3"; string v3a = "value3_a"; KeyOpFieldsValuesTuple exp_kofv; - unique_ptr consumer; deque kofv_q; - ConsumerTest() { // FIXME: move out from constructor @@ -39,17 +52,14 @@ namespace consumer_test consumer = unique_ptr(new Consumer( new swss::ConsumerStateTable(m_config_db.get(), "CFG_TEST_TABLE", 1, 1), gPortsOrch, "CFG_TEST_TABLE")); } - virtual void SetUp() override { ::testing_db::reset(); } - virtual void TearDown() override { ::testing_db::reset(); } - void validate_syncmap(SyncMap &sync, uint16_t exp_sz, std::string exp_key, KeyOpFieldsValuesTuple exp_kofv) { // verify the content in syncMap @@ -58,7 +68,6 @@ namespace consumer_test while (it != sync.end()) { KeyOpFieldsValuesTuple t = it->second; - string itkey = kfvKey(t); if (itkey == exp_key) { ASSERT_EQ(t, exp_kofv); @@ -71,23 +80,19 @@ namespace consumer_test ASSERT_EQ(sync.size(), exp_sz-1); } }; - TEST_F(ConsumerTest, ConsumerAddToSync_Set) { - // Test case, one set_command auto entry = KeyOpFieldsValuesTuple( { key, SET_COMMAND, { { f1, v1a }, { f2, v2a } } }); - kofv_q.push_back(entry); consumer->addToSync(kofv_q); exp_kofv = entry; validate_syncmap(consumer->m_toSync, 1, key, exp_kofv); } - TEST_F(ConsumerTest, ConsumerAddToSync_Del) { // Test case, one with del_command @@ -95,15 +100,11 @@ namespace consumer_test { key, DEL_COMMAND, { { } } }); - kofv_q.push_back(entry); consumer->addToSync(kofv_q); - exp_kofv = entry; validate_syncmap(consumer->m_toSync, 1, key, exp_kofv); - } - TEST_F(ConsumerTest, ConsumerAddToSync_Set_Del) { // Test case, add SET then DEL @@ -112,50 +113,41 @@ namespace consumer_test SET_COMMAND, { { f1, v1a }, { f2, v2a } } }); - auto entryb = KeyOpFieldsValuesTuple( { key, DEL_COMMAND, { { } } }); - kofv_q.push_back(entrya); kofv_q.push_back(entryb); consumer->addToSync(kofv_q); - // expect only DEL exp_kofv = entryb; validate_syncmap(consumer->m_toSync, 1, key, exp_kofv); } - TEST_F(ConsumerTest, ConsumerAddToSync_Del_Set) { auto entrya = KeyOpFieldsValuesTuple( { key, DEL_COMMAND, { { } } }); - auto entryb = KeyOpFieldsValuesTuple( { key, SET_COMMAND, { { f1, v1a }, { f2, v2a } } }); - // Test case, add DEL then SET, re-try 100 times, order should be kept for (auto x = 0; x < 100; x++) { kofv_q.push_back(entrya); kofv_q.push_back(entryb); consumer->addToSync(kofv_q); - // expect DEL then SET exp_kofv = entrya; validate_syncmap(consumer->m_toSync, 2, key, exp_kofv); - exp_kofv = entryb; validate_syncmap(consumer->m_toSync, 1, key, exp_kofv); } } - TEST_F(ConsumerTest, ConsumerAddToSync_Set_Del_Set_Multi) { // Test5, add SET, DEL then SET, re-try 100 times , order should be kept @@ -164,34 +156,28 @@ namespace consumer_test SET_COMMAND, { { f1, v1a }, { f2, v2a } } }); - auto entryb = KeyOpFieldsValuesTuple( { key, DEL_COMMAND, { { } } }); - auto entryc = KeyOpFieldsValuesTuple( { key, SET_COMMAND, { { f1, v1a }, { f2, v2a } } }); - for (auto x = 0; x < 100; x++) { kofv_q.push_back(entrya); kofv_q.push_back(entryb); kofv_q.push_back(entryc); consumer->addToSync(kofv_q); - // expect DEL then SET exp_kofv = entryb; validate_syncmap(consumer->m_toSync, 2, key, exp_kofv); - exp_kofv = entryc; validate_syncmap(consumer->m_toSync, 1, key, exp_kofv); } } - TEST_F(ConsumerTest, ConsumerAddToSync_Set_Del_Set_Multi_In_Q) { // Test5, add SET, DEL then SET, repeat 100 times in queue, final result and order should be kept @@ -200,18 +186,15 @@ namespace consumer_test SET_COMMAND, { { f1, v1a }, { f2, v2a } } }); - auto entryb = KeyOpFieldsValuesTuple( { key, DEL_COMMAND, { { } } }); - auto entryc = KeyOpFieldsValuesTuple( { key, SET_COMMAND, { { f1, v1a }, { f2, v2a } } }); - for (auto x = 0; x < 100; x++) { kofv_q.push_back(entrya); @@ -219,15 +202,12 @@ namespace consumer_test kofv_q.push_back(entryc); } consumer->addToSync(kofv_q); - // expect DEL then SET exp_kofv = entryb; validate_syncmap(consumer->m_toSync, 2, key, exp_kofv); - exp_kofv = entryc; validate_syncmap(consumer->m_toSync, 1, key, exp_kofv); } - TEST_F(ConsumerTest, ConsumerAddToSync_Del_Set_Setnew) { // Test case, DEL, SET, then SET with different value @@ -235,32 +215,26 @@ namespace consumer_test { key, DEL_COMMAND, { { } } }); - auto entryb = KeyOpFieldsValuesTuple( { key, SET_COMMAND, { { f1, v1a }, { f2, v2a } } }); - auto entryc = KeyOpFieldsValuesTuple( { key, SET_COMMAND, { { f1, v1b }, { f2, v2b } } }); - kofv_q.push_back(entrya); kofv_q.push_back(entryb); kofv_q.push_back(entryc); consumer->addToSync(kofv_q); - // expect DEL then SET with new values exp_kofv = entrya; validate_syncmap(consumer->m_toSync, 2, key, exp_kofv); - exp_kofv = entryc; validate_syncmap(consumer->m_toSync, 1, key, exp_kofv); } - TEST_F(ConsumerTest, ConsumerAddToSync_Del_Set_Setnew1) { // Test case, DEL, SET, then SET with new values and new fields @@ -268,38 +242,31 @@ namespace consumer_test { key, DEL_COMMAND, { { } } }); - auto entryb = KeyOpFieldsValuesTuple( { key, SET_COMMAND, { { f1, v1a }, { f2, v2a } } }); - auto entryc = KeyOpFieldsValuesTuple( { key, SET_COMMAND, { { f1, v1b }, { f3, v3a } } }); - kofv_q.push_back(entrya); kofv_q.push_back(entryb); kofv_q.push_back(entryc); consumer->addToSync(kofv_q); - // expect DEL then SET with new values and new fields exp_kofv = entrya; validate_syncmap(consumer->m_toSync, 2, key, exp_kofv); - exp_kofv = KeyOpFieldsValuesTuple( { key, SET_COMMAND, { { f2, v2a }, { f1, v1b }, { f3, v3a } } }); - validate_syncmap(consumer->m_toSync, 1, key, exp_kofv); } - TEST_F(ConsumerTest, ConsumerAddToSync_Ind_Set_Del) { // Test case, Add individuals by addToSync, SET then DEL @@ -308,18 +275,37 @@ namespace consumer_test SET_COMMAND, { { f1, v1a }, { f2, v2a } } }); - auto entryb = KeyOpFieldsValuesTuple( { key, DEL_COMMAND, { { } } }); - consumer->addToSync(entrya); consumer->addToSync(entryb); - // expect only DEL exp_kofv = entryb; validate_syncmap(consumer->m_toSync, 1, key, exp_kofv); - + } + TEST_F(ConsumerTest, ConsumerPops_notification_count) + { + int consumer_pops_batch_size = 10; + TestOrch test_orch(m_config_db.get(), "CFG_TEST_TABLE"); + Consumer test_consumer( + new swss::ConsumerStateTable(m_config_db.get(), "CFG_TEST_TABLE", consumer_pops_batch_size, 1), &test_orch, "CFG_TEST_TABLE"); + swss::ProducerStateTable producer_table(m_config_db.get(), "CFG_TEST_TABLE"); + m_config_db->flushdb(); + for (int notification_count = 0; notification_count< consumer_pops_batch_size*2; notification_count++) + { + std::vector fields; + FieldValueTuple t("test_field", "test_value"); + fields.push_back(t); + producer_table.set(std::to_string(notification_count), fields); + + cout << "ConsumerPops_notification_count:: add key: " << notification_count << endl; + } + // consumer should pops consumer_pops_batch_size notifications + test_consumer.execute(); + ASSERT_EQ(test_orch.m_notification_count, consumer_pops_batch_size); + test_consumer.execute(); + ASSERT_EQ(test_orch.m_notification_count, consumer_pops_batch_size*2); } } diff --git a/tests/mock_tests/mock_consumerstatetable.cpp b/tests/mock_tests/mock_consumerstatetable.cpp index 822727929a..5f8f8fbdb7 100644 --- a/tests/mock_tests/mock_consumerstatetable.cpp +++ b/tests/mock_tests/mock_consumerstatetable.cpp @@ -1,5 +1,6 @@ -#include "consumerstatetable.h" +#include "consumerstatetable.h" +#include namespace swss { ConsumerStateTable::ConsumerStateTable(DBConnector *db, const std::string &tableName, int popBatchSize, int pri) : @@ -7,4 +8,34 @@ namespace swss TableName_KeySet(tableName) { } + + void ConsumerStateTable::pops(std::deque &vkco, const std::string& /*prefix*/) + { + int count = 0; + swss::Table table(getDbConnector(), getTableName()); + std::vector keys; + table.getKeys(keys); + for (const auto &key: keys) + { + // pop with batch size + if (count < POP_BATCH_SIZE) + { + count++; + } + else + { + break; + } + + KeyOpFieldsValuesTuple kco; + kfvKey(kco) = key; + kfvOp(kco) = SET_COMMAND; + if (!table.get(key, kfvFieldsValues(kco))) + { + continue; + } + table.del(key); + vkco.push_back(kco); + } + } } From 087ae0ee5ea54718153776b263e969a595b6c137 Mon Sep 17 00:00:00 2001 From: Hua Liu <58683130+liuh-80@users.noreply.github.com> Date: Mon, 2 Sep 2024 17:00:57 +0800 Subject: [PATCH 05/23] Update consumer_ut.cpp --- tests/mock_tests/consumer_ut.cpp | 62 +++++++++++++++++++++++++++++++- 1 file changed, 61 insertions(+), 1 deletion(-) diff --git a/tests/mock_tests/consumer_ut.cpp b/tests/mock_tests/consumer_ut.cpp index a67b7235c7..f0008a964b 100644 --- a/tests/mock_tests/consumer_ut.cpp +++ b/tests/mock_tests/consumer_ut.cpp @@ -1,12 +1,15 @@ - #include "ut_helper.h" #include "mock_orchagent_main.h" #include "mock_table.h" + #include + extern PortsOrch *gPortsOrch; + namespace consumer_test { using namespace std; + class TestOrch : public Orch { public: @@ -31,6 +34,7 @@ namespace consumer_test shared_ptr m_app_db; shared_ptr m_config_db; shared_ptr m_state_db; + string key = "key"; string f1 = "field1"; string v1a = "value1_a"; @@ -41,8 +45,10 @@ namespace consumer_test string f3 = "field3"; string v3a = "value3_a"; KeyOpFieldsValuesTuple exp_kofv; + unique_ptr consumer; deque kofv_q; + ConsumerTest() { // FIXME: move out from constructor @@ -52,14 +58,17 @@ namespace consumer_test consumer = unique_ptr(new Consumer( new swss::ConsumerStateTable(m_config_db.get(), "CFG_TEST_TABLE", 1, 1), gPortsOrch, "CFG_TEST_TABLE")); } + virtual void SetUp() override { ::testing_db::reset(); } + virtual void TearDown() override { ::testing_db::reset(); } + void validate_syncmap(SyncMap &sync, uint16_t exp_sz, std::string exp_key, KeyOpFieldsValuesTuple exp_kofv) { // verify the content in syncMap @@ -68,6 +77,7 @@ namespace consumer_test while (it != sync.end()) { KeyOpFieldsValuesTuple t = it->second; + string itkey = kfvKey(t); if (itkey == exp_key) { ASSERT_EQ(t, exp_kofv); @@ -80,19 +90,23 @@ namespace consumer_test ASSERT_EQ(sync.size(), exp_sz-1); } }; + TEST_F(ConsumerTest, ConsumerAddToSync_Set) { + // Test case, one set_command auto entry = KeyOpFieldsValuesTuple( { key, SET_COMMAND, { { f1, v1a }, { f2, v2a } } }); + kofv_q.push_back(entry); consumer->addToSync(kofv_q); exp_kofv = entry; validate_syncmap(consumer->m_toSync, 1, key, exp_kofv); } + TEST_F(ConsumerTest, ConsumerAddToSync_Del) { // Test case, one with del_command @@ -100,11 +114,15 @@ namespace consumer_test { key, DEL_COMMAND, { { } } }); + kofv_q.push_back(entry); consumer->addToSync(kofv_q); + exp_kofv = entry; validate_syncmap(consumer->m_toSync, 1, key, exp_kofv); + } + TEST_F(ConsumerTest, ConsumerAddToSync_Set_Del) { // Test case, add SET then DEL @@ -113,41 +131,50 @@ namespace consumer_test SET_COMMAND, { { f1, v1a }, { f2, v2a } } }); + auto entryb = KeyOpFieldsValuesTuple( { key, DEL_COMMAND, { { } } }); + kofv_q.push_back(entrya); kofv_q.push_back(entryb); consumer->addToSync(kofv_q); + // expect only DEL exp_kofv = entryb; validate_syncmap(consumer->m_toSync, 1, key, exp_kofv); } + TEST_F(ConsumerTest, ConsumerAddToSync_Del_Set) { auto entrya = KeyOpFieldsValuesTuple( { key, DEL_COMMAND, { { } } }); + auto entryb = KeyOpFieldsValuesTuple( { key, SET_COMMAND, { { f1, v1a }, { f2, v2a } } }); + // Test case, add DEL then SET, re-try 100 times, order should be kept for (auto x = 0; x < 100; x++) { kofv_q.push_back(entrya); kofv_q.push_back(entryb); consumer->addToSync(kofv_q); + // expect DEL then SET exp_kofv = entrya; validate_syncmap(consumer->m_toSync, 2, key, exp_kofv); + exp_kofv = entryb; validate_syncmap(consumer->m_toSync, 1, key, exp_kofv); } } + TEST_F(ConsumerTest, ConsumerAddToSync_Set_Del_Set_Multi) { // Test5, add SET, DEL then SET, re-try 100 times , order should be kept @@ -156,28 +183,34 @@ namespace consumer_test SET_COMMAND, { { f1, v1a }, { f2, v2a } } }); + auto entryb = KeyOpFieldsValuesTuple( { key, DEL_COMMAND, { { } } }); + auto entryc = KeyOpFieldsValuesTuple( { key, SET_COMMAND, { { f1, v1a }, { f2, v2a } } }); + for (auto x = 0; x < 100; x++) { kofv_q.push_back(entrya); kofv_q.push_back(entryb); kofv_q.push_back(entryc); consumer->addToSync(kofv_q); + // expect DEL then SET exp_kofv = entryb; validate_syncmap(consumer->m_toSync, 2, key, exp_kofv); + exp_kofv = entryc; validate_syncmap(consumer->m_toSync, 1, key, exp_kofv); } } + TEST_F(ConsumerTest, ConsumerAddToSync_Set_Del_Set_Multi_In_Q) { // Test5, add SET, DEL then SET, repeat 100 times in queue, final result and order should be kept @@ -186,15 +219,18 @@ namespace consumer_test SET_COMMAND, { { f1, v1a }, { f2, v2a } } }); + auto entryb = KeyOpFieldsValuesTuple( { key, DEL_COMMAND, { { } } }); + auto entryc = KeyOpFieldsValuesTuple( { key, SET_COMMAND, { { f1, v1a }, { f2, v2a } } }); + for (auto x = 0; x < 100; x++) { kofv_q.push_back(entrya); @@ -202,12 +238,15 @@ namespace consumer_test kofv_q.push_back(entryc); } consumer->addToSync(kofv_q); + // expect DEL then SET exp_kofv = entryb; validate_syncmap(consumer->m_toSync, 2, key, exp_kofv); + exp_kofv = entryc; validate_syncmap(consumer->m_toSync, 1, key, exp_kofv); } + TEST_F(ConsumerTest, ConsumerAddToSync_Del_Set_Setnew) { // Test case, DEL, SET, then SET with different value @@ -215,26 +254,32 @@ namespace consumer_test { key, DEL_COMMAND, { { } } }); + auto entryb = KeyOpFieldsValuesTuple( { key, SET_COMMAND, { { f1, v1a }, { f2, v2a } } }); + auto entryc = KeyOpFieldsValuesTuple( { key, SET_COMMAND, { { f1, v1b }, { f2, v2b } } }); + kofv_q.push_back(entrya); kofv_q.push_back(entryb); kofv_q.push_back(entryc); consumer->addToSync(kofv_q); + // expect DEL then SET with new values exp_kofv = entrya; validate_syncmap(consumer->m_toSync, 2, key, exp_kofv); + exp_kofv = entryc; validate_syncmap(consumer->m_toSync, 1, key, exp_kofv); } + TEST_F(ConsumerTest, ConsumerAddToSync_Del_Set_Setnew1) { // Test case, DEL, SET, then SET with new values and new fields @@ -242,31 +287,38 @@ namespace consumer_test { key, DEL_COMMAND, { { } } }); + auto entryb = KeyOpFieldsValuesTuple( { key, SET_COMMAND, { { f1, v1a }, { f2, v2a } } }); + auto entryc = KeyOpFieldsValuesTuple( { key, SET_COMMAND, { { f1, v1b }, { f3, v3a } } }); + kofv_q.push_back(entrya); kofv_q.push_back(entryb); kofv_q.push_back(entryc); consumer->addToSync(kofv_q); + // expect DEL then SET with new values and new fields exp_kofv = entrya; validate_syncmap(consumer->m_toSync, 2, key, exp_kofv); + exp_kofv = KeyOpFieldsValuesTuple( { key, SET_COMMAND, { { f2, v2a }, { f1, v1b }, { f3, v3a } } }); + validate_syncmap(consumer->m_toSync, 1, key, exp_kofv); } + TEST_F(ConsumerTest, ConsumerAddToSync_Ind_Set_Del) { // Test case, Add individuals by addToSync, SET then DEL @@ -275,16 +327,21 @@ namespace consumer_test SET_COMMAND, { { f1, v1a }, { f2, v2a } } }); + auto entryb = KeyOpFieldsValuesTuple( { key, DEL_COMMAND, { { } } }); + consumer->addToSync(entrya); consumer->addToSync(entryb); + // expect only DEL exp_kofv = entryb; validate_syncmap(consumer->m_toSync, 1, key, exp_kofv); + } + TEST_F(ConsumerTest, ConsumerPops_notification_count) { int consumer_pops_batch_size = 10; @@ -292,6 +349,7 @@ namespace consumer_test Consumer test_consumer( new swss::ConsumerStateTable(m_config_db.get(), "CFG_TEST_TABLE", consumer_pops_batch_size, 1), &test_orch, "CFG_TEST_TABLE"); swss::ProducerStateTable producer_table(m_config_db.get(), "CFG_TEST_TABLE"); + m_config_db->flushdb(); for (int notification_count = 0; notification_count< consumer_pops_batch_size*2; notification_count++) { @@ -302,9 +360,11 @@ namespace consumer_test cout << "ConsumerPops_notification_count:: add key: " << notification_count << endl; } + // consumer should pops consumer_pops_batch_size notifications test_consumer.execute(); ASSERT_EQ(test_orch.m_notification_count, consumer_pops_batch_size); + test_consumer.execute(); ASSERT_EQ(test_orch.m_notification_count, consumer_pops_batch_size*2); } From 68c65ee1784515aca0ae42944d2b2766ec9e9af7 Mon Sep 17 00:00:00 2001 From: Hua Liu <58683130+liuh-80@users.noreply.github.com> Date: Mon, 2 Sep 2024 17:02:02 +0800 Subject: [PATCH 06/23] Update mock_consumerstatetable.cpp --- tests/mock_tests/mock_consumerstatetable.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/mock_tests/mock_consumerstatetable.cpp b/tests/mock_tests/mock_consumerstatetable.cpp index 5f8f8fbdb7..2764bb10f2 100644 --- a/tests/mock_tests/mock_consumerstatetable.cpp +++ b/tests/mock_tests/mock_consumerstatetable.cpp @@ -1,6 +1,5 @@ - #include "consumerstatetable.h" -#include + namespace swss { ConsumerStateTable::ConsumerStateTable(DBConnector *db, const std::string &tableName, int popBatchSize, int pri) : From 8c206a94fbc27c6849835796c43d0f7c48904f17 Mon Sep 17 00:00:00 2001 From: Hua Liu <58683130+liuh-80@users.noreply.github.com> Date: Wed, 4 Sep 2024 09:21:16 +0800 Subject: [PATCH 07/23] Update dvs_common.py --- tests/dvslib/dvs_common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/dvslib/dvs_common.py b/tests/dvslib/dvs_common.py index 0d81b4cf2e..e854095dc7 100644 --- a/tests/dvslib/dvs_common.py +++ b/tests/dvslib/dvs_common.py @@ -16,7 +16,7 @@ class PollingConfig: strict: If the strict flag is set, reaching the timeout will cause tests to fail. """ - polling_interval: float = 0.01 + polling_interval: float = 0.1 timeout: float = 20.00 strict: bool = True From e30c4292abbe4703ecc96e86a8e9e00979685590 Mon Sep 17 00:00:00 2001 From: Hua Liu <58683130+liuh-80@users.noreply.github.com> Date: Wed, 4 Sep 2024 10:55:38 +0800 Subject: [PATCH 08/23] Update dvs_common.py --- tests/dvslib/dvs_common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/dvslib/dvs_common.py b/tests/dvslib/dvs_common.py index e854095dc7..0d81b4cf2e 100644 --- a/tests/dvslib/dvs_common.py +++ b/tests/dvslib/dvs_common.py @@ -16,7 +16,7 @@ class PollingConfig: strict: If the strict flag is set, reaching the timeout will cause tests to fail. """ - polling_interval: float = 0.1 + polling_interval: float = 0.01 timeout: float = 20.00 strict: bool = True From ad1fc721df384fb931a6bbf25bc47c8e721a0e2d Mon Sep 17 00:00:00 2001 From: Hua Liu <58683130+liuh-80@users.noreply.github.com> Date: Wed, 4 Sep 2024 14:48:00 +0800 Subject: [PATCH 09/23] Update dvs_common.py --- tests/dvslib/dvs_common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/dvslib/dvs_common.py b/tests/dvslib/dvs_common.py index 0d81b4cf2e..f1fb3f0508 100644 --- a/tests/dvslib/dvs_common.py +++ b/tests/dvslib/dvs_common.py @@ -17,7 +17,7 @@ class PollingConfig: """ polling_interval: float = 0.01 - timeout: float = 20.00 + timeout: float = 60.00 strict: bool = True def iterations(self) -> int: From 1cf59b654f5cb3bb0f8953e46c8463772fa652da Mon Sep 17 00:00:00 2001 From: Hua Liu <58683130+liuh-80@users.noreply.github.com> Date: Wed, 4 Sep 2024 17:33:29 +0800 Subject: [PATCH 10/23] Update dvs_common.py --- tests/dvslib/dvs_common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/dvslib/dvs_common.py b/tests/dvslib/dvs_common.py index f1fb3f0508..ad9aa83588 100644 --- a/tests/dvslib/dvs_common.py +++ b/tests/dvslib/dvs_common.py @@ -17,7 +17,7 @@ class PollingConfig: """ polling_interval: float = 0.01 - timeout: float = 60.00 + timeout: float = 120.00 strict: bool = True def iterations(self) -> int: From 21b9d39addf5ef8bbb606b679a91848a2c1d457b Mon Sep 17 00:00:00 2001 From: Hua Liu <58683130+liuh-80@users.noreply.github.com> Date: Tue, 10 Sep 2024 17:43:41 +0800 Subject: [PATCH 11/23] Update orch.cpp --- orchagent/orch.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/orchagent/orch.cpp b/orchagent/orch.cpp index 708a86280a..159a404136 100644 --- a/orchagent/orch.cpp +++ b/orchagent/orch.cpp @@ -244,7 +244,7 @@ void Consumer::execute() auto table = static_cast(getSelectable()); std::deque entries; - table->pops(entries); + table->pops(entries, 128 * 100); // add to sync addToSync(entries); From e262b08fe2ec93ae538967834274213fa1989335 Mon Sep 17 00:00:00 2001 From: Hua Liu <58683130+liuh-80@users.noreply.github.com> Date: Tue, 10 Sep 2024 17:51:04 +0800 Subject: [PATCH 12/23] Update orch.cpp --- orchagent/orch.cpp | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/orchagent/orch.cpp b/orchagent/orch.cpp index 159a404136..cd853875d3 100644 --- a/orchagent/orch.cpp +++ b/orchagent/orch.cpp @@ -242,12 +242,14 @@ void Consumer::execute() // ConsumerBase::execute_impl(); SWSS_LOG_ENTER(); + size_t total_update_size = 0; auto table = static_cast(getSelectable()); - std::deque entries; - table->pops(entries, 128 * 100); - - // add to sync - addToSync(entries); + do + { + std::deque entries; + table->pops(entries); + total_update_size += addToSync(entries); + } while (total_update_size <= 12800); drain(); } From b8396bfbb5785523c08a38a5476a2b200b79ea9c Mon Sep 17 00:00:00 2001 From: Hua Liu <58683130+liuh-80@users.noreply.github.com> Date: Tue, 10 Sep 2024 17:52:01 +0800 Subject: [PATCH 13/23] Update consumer_ut.cpp --- tests/mock_tests/consumer_ut.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/mock_tests/consumer_ut.cpp b/tests/mock_tests/consumer_ut.cpp index f0008a964b..62b811548a 100644 --- a/tests/mock_tests/consumer_ut.cpp +++ b/tests/mock_tests/consumer_ut.cpp @@ -361,11 +361,13 @@ namespace consumer_test cout << "ConsumerPops_notification_count:: add key: " << notification_count << endl; } + /* // consumer should pops consumer_pops_batch_size notifications test_consumer.execute(); ASSERT_EQ(test_orch.m_notification_count, consumer_pops_batch_size); test_consumer.execute(); ASSERT_EQ(test_orch.m_notification_count, consumer_pops_batch_size*2); + */ } } From d0b4886763a692581a13fdf11a89a7072876cd74 Mon Sep 17 00:00:00 2001 From: Hua Liu <58683130+liuh-80@users.noreply.github.com> Date: Wed, 11 Sep 2024 09:16:08 +0800 Subject: [PATCH 14/23] Update orch.cpp --- orchagent/orch.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/orchagent/orch.cpp b/orchagent/orch.cpp index cd853875d3..a250ac3468 100644 --- a/orchagent/orch.cpp +++ b/orchagent/orch.cpp @@ -243,13 +243,15 @@ void Consumer::execute() SWSS_LOG_ENTER(); size_t total_update_size = 0; + size_t update_size = 0; auto table = static_cast(getSelectable()); do { std::deque entries; table->pops(entries); - total_update_size += addToSync(entries); - } while (total_update_size <= 12800); + update_size = addToSync(entries); + total_update_size += update_size; + } while ((update_size != 0) && (total_update_size <= 12800)); drain(); } From d5acaaeb275aeef3ff22b274718e91bde5d85485 Mon Sep 17 00:00:00 2001 From: Hua Liu <58683130+liuh-80@users.noreply.github.com> Date: Wed, 11 Sep 2024 14:51:40 +0800 Subject: [PATCH 15/23] Update dvs_common.py --- tests/dvslib/dvs_common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/dvslib/dvs_common.py b/tests/dvslib/dvs_common.py index ad9aa83588..0d81b4cf2e 100644 --- a/tests/dvslib/dvs_common.py +++ b/tests/dvslib/dvs_common.py @@ -17,7 +17,7 @@ class PollingConfig: """ polling_interval: float = 0.01 - timeout: float = 120.00 + timeout: float = 20.00 strict: bool = True def iterations(self) -> int: From d99ebf66d7f111249de31b507d40d6db43f18ca5 Mon Sep 17 00:00:00 2001 From: Hua Liu <58683130+liuh-80@users.noreply.github.com> Date: Thu, 12 Sep 2024 11:01:53 +0800 Subject: [PATCH 16/23] Update orch.cpp --- orchagent/orch.cpp | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/orchagent/orch.cpp b/orchagent/orch.cpp index a250ac3468..4d7ed3b030 100644 --- a/orchagent/orch.cpp +++ b/orchagent/orch.cpp @@ -13,6 +13,8 @@ #include "zmqconsumerstatetable.h" #include "sai_serialize.h" +#define CONSUMER_POP_MAX_BATCH_COUNT 50 + using namespace swss; int gBatchSize = 0; @@ -242,16 +244,10 @@ void Consumer::execute() // ConsumerBase::execute_impl(); SWSS_LOG_ENTER(); - size_t total_update_size = 0; - size_t update_size = 0; auto table = static_cast(getSelectable()); - do - { - std::deque entries; - table->pops(entries); - update_size = addToSync(entries); - total_update_size += update_size; - } while ((update_size != 0) && (total_update_size <= 12800)); + std::deque entries; + table->pops(entries); + update_size = addToSync(entries); drain(); } @@ -787,7 +783,7 @@ void Orch::addConsumer(DBConnector *db, string tableName, int pri) { if (db->getDbId() == CONFIG_DB || db->getDbId() == STATE_DB || db->getDbId() == CHASSIS_APP_DB) { - addExecutor(new Consumer(new SubscriberStateTable(db, tableName, TableConsumable::DEFAULT_POP_BATCH_SIZE, pri), this, tableName)); + addExecutor(new Consumer(new SubscriberStateTable(db, tableName, TableConsumable::DEFAULT_POP_BATCH_SIZE * CONSUMER_POP_MAX_BATCH_COUNT, pri), this, tableName)); } else { From 317191d690bc1d00adcc0def76a9e832ec26af55 Mon Sep 17 00:00:00 2001 From: Hua Liu <58683130+liuh-80@users.noreply.github.com> Date: Thu, 12 Sep 2024 11:02:49 +0800 Subject: [PATCH 17/23] Update consumer_ut.cpp --- tests/mock_tests/consumer_ut.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/mock_tests/consumer_ut.cpp b/tests/mock_tests/consumer_ut.cpp index 62b811548a..f0008a964b 100644 --- a/tests/mock_tests/consumer_ut.cpp +++ b/tests/mock_tests/consumer_ut.cpp @@ -361,13 +361,11 @@ namespace consumer_test cout << "ConsumerPops_notification_count:: add key: " << notification_count << endl; } - /* // consumer should pops consumer_pops_batch_size notifications test_consumer.execute(); ASSERT_EQ(test_orch.m_notification_count, consumer_pops_batch_size); test_consumer.execute(); ASSERT_EQ(test_orch.m_notification_count, consumer_pops_batch_size*2); - */ } } From 574cf11f9042d9fd02f4c9add2753dc1e35f1700 Mon Sep 17 00:00:00 2001 From: Hua Liu <58683130+liuh-80@users.noreply.github.com> Date: Thu, 12 Sep 2024 13:14:21 +0800 Subject: [PATCH 18/23] Update orch.cpp --- orchagent/orch.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/orchagent/orch.cpp b/orchagent/orch.cpp index 4d7ed3b030..6faaee8843 100644 --- a/orchagent/orch.cpp +++ b/orchagent/orch.cpp @@ -247,7 +247,7 @@ void Consumer::execute() auto table = static_cast(getSelectable()); std::deque entries; table->pops(entries); - update_size = addToSync(entries); + addToSync(entries); drain(); } From 133f057006cf7cb3f078128cd8edac36bb93aa47 Mon Sep 17 00:00:00 2001 From: Hua Liu <58683130+liuh-80@users.noreply.github.com> Date: Thu, 12 Sep 2024 14:51:35 +0800 Subject: [PATCH 19/23] Update orch.cpp --- orchagent/orch.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/orchagent/orch.cpp b/orchagent/orch.cpp index 6faaee8843..2951f374ba 100644 --- a/orchagent/orch.cpp +++ b/orchagent/orch.cpp @@ -783,11 +783,11 @@ void Orch::addConsumer(DBConnector *db, string tableName, int pri) { if (db->getDbId() == CONFIG_DB || db->getDbId() == STATE_DB || db->getDbId() == CHASSIS_APP_DB) { - addExecutor(new Consumer(new SubscriberStateTable(db, tableName, TableConsumable::DEFAULT_POP_BATCH_SIZE * CONSUMER_POP_MAX_BATCH_COUNT, pri), this, tableName)); + addExecutor(new Consumer(new SubscriberStateTable(db, tableName, TableConsumable::DEFAULT_POP_BATCH_SIZE, pri), this, tableName)); } else { - addExecutor(new Consumer(new ConsumerStateTable(db, tableName, gBatchSize, pri), this, tableName)); + addExecutor(new Consumer(new ConsumerStateTable(db, tableName, gBatchSize * CONSUMER_POP_MAX_BATCH_COUNT, pri), this, tableName)); } } From 3620acbea1d11380318cfe1080a3fec837fa0c19 Mon Sep 17 00:00:00 2001 From: Hua Liu <58683130+liuh-80@users.noreply.github.com> Date: Thu, 12 Sep 2024 15:46:32 +0800 Subject: [PATCH 20/23] Update orch.cpp --- orchagent/orch.cpp | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/orchagent/orch.cpp b/orchagent/orch.cpp index 2951f374ba..4a282319d5 100644 --- a/orchagent/orch.cpp +++ b/orchagent/orch.cpp @@ -13,7 +13,7 @@ #include "zmqconsumerstatetable.h" #include "sai_serialize.h" -#define CONSUMER_POP_MAX_BATCH_COUNT 50 +#define CONSUMER_POP_MAX_BATCH_COUNT 10 using namespace swss; @@ -245,9 +245,15 @@ void Consumer::execute() SWSS_LOG_ENTER(); auto table = static_cast(getSelectable()); - std::deque entries; - table->pops(entries); - addToSync(entries); + int batch_count = 0; + size_t update_size = 0; + do + { + std::deque entries; + table->pops(entries); + update_size = addToSync(entries); + batch_count++; + } while ((update_size != 0) && (batch_count < CONSUMER_POP_MAX_BATCH_COUNT)) drain(); } @@ -787,7 +793,7 @@ void Orch::addConsumer(DBConnector *db, string tableName, int pri) } else { - addExecutor(new Consumer(new ConsumerStateTable(db, tableName, gBatchSize * CONSUMER_POP_MAX_BATCH_COUNT, pri), this, tableName)); + addExecutor(new Consumer(new ConsumerStateTable(db, tableName, gBatchSize, pri), this, tableName)); } } From 7c68aeff8f5e5b3e372c8866545eb4df87eabc65 Mon Sep 17 00:00:00 2001 From: Hua Liu <58683130+liuh-80@users.noreply.github.com> Date: Thu, 12 Sep 2024 15:48:23 +0800 Subject: [PATCH 21/23] Update consumer_ut.cpp --- tests/mock_tests/consumer_ut.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/mock_tests/consumer_ut.cpp b/tests/mock_tests/consumer_ut.cpp index f0008a964b..636bd721e2 100644 --- a/tests/mock_tests/consumer_ut.cpp +++ b/tests/mock_tests/consumer_ut.cpp @@ -345,13 +345,14 @@ namespace consumer_test TEST_F(ConsumerTest, ConsumerPops_notification_count) { int consumer_pops_batch_size = 10; + int pops_max_batch_count = 10; TestOrch test_orch(m_config_db.get(), "CFG_TEST_TABLE"); Consumer test_consumer( new swss::ConsumerStateTable(m_config_db.get(), "CFG_TEST_TABLE", consumer_pops_batch_size, 1), &test_orch, "CFG_TEST_TABLE"); swss::ProducerStateTable producer_table(m_config_db.get(), "CFG_TEST_TABLE"); m_config_db->flushdb(); - for (int notification_count = 0; notification_count< consumer_pops_batch_size*2; notification_count++) + for (int notification_count = 0; notification_count< consumer_pops_batch_size*pops_max_batch_count*2; notification_count++) { std::vector fields; FieldValueTuple t("test_field", "test_value"); @@ -363,9 +364,9 @@ namespace consumer_test // consumer should pops consumer_pops_batch_size notifications test_consumer.execute(); - ASSERT_EQ(test_orch.m_notification_count, consumer_pops_batch_size); + ASSERT_EQ(test_orch.m_notification_count, consumer_pops_batch_size*pops_max_batch_count); test_consumer.execute(); - ASSERT_EQ(test_orch.m_notification_count, consumer_pops_batch_size*2); + ASSERT_EQ(test_orch.m_notification_count, consumer_pops_batch_size*pops_max_batch_count*2); } } From e3ae1bf570452130c5c3cefcaefb2082642fe3a6 Mon Sep 17 00:00:00 2001 From: Hua Liu <58683130+liuh-80@users.noreply.github.com> Date: Thu, 12 Sep 2024 16:21:10 +0800 Subject: [PATCH 22/23] Update orch.cpp --- orchagent/orch.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/orchagent/orch.cpp b/orchagent/orch.cpp index 4a282319d5..79e27cb5d4 100644 --- a/orchagent/orch.cpp +++ b/orchagent/orch.cpp @@ -13,7 +13,7 @@ #include "zmqconsumerstatetable.h" #include "sai_serialize.h" -#define CONSUMER_POP_MAX_BATCH_COUNT 10 +#define CONSUMER_POP_MAX_BATCH_COUNT 8 using namespace swss; @@ -253,7 +253,7 @@ void Consumer::execute() table->pops(entries); update_size = addToSync(entries); batch_count++; - } while ((update_size != 0) && (batch_count < CONSUMER_POP_MAX_BATCH_COUNT)) + } while ((update_size != 0) && (batch_count < CONSUMER_POP_MAX_BATCH_COUNT)); drain(); } From ba0cd6d89218261f8f826e138d978036c8fc775c Mon Sep 17 00:00:00 2001 From: Hua Liu <58683130+liuh-80@users.noreply.github.com> Date: Thu, 12 Sep 2024 17:12:56 +0800 Subject: [PATCH 23/23] Update consumer_ut.cpp --- tests/mock_tests/consumer_ut.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/mock_tests/consumer_ut.cpp b/tests/mock_tests/consumer_ut.cpp index 636bd721e2..64ad13836d 100644 --- a/tests/mock_tests/consumer_ut.cpp +++ b/tests/mock_tests/consumer_ut.cpp @@ -345,7 +345,7 @@ namespace consumer_test TEST_F(ConsumerTest, ConsumerPops_notification_count) { int consumer_pops_batch_size = 10; - int pops_max_batch_count = 10; + int pops_max_batch_count = 8; TestOrch test_orch(m_config_db.get(), "CFG_TEST_TABLE"); Consumer test_consumer( new swss::ConsumerStateTable(m_config_db.get(), "CFG_TEST_TABLE", consumer_pops_batch_size, 1), &test_orch, "CFG_TEST_TABLE");