Skip to content

Commit

Permalink
address review comments (#1890) (#1893)
Browse files Browse the repository at this point in the history 
Signed-off-by: Senthil Nathan N <cendhu@gmail.com>
Signed-off-by: David Enyeart <enyeart@us.ibm.com>

Co-authored-by: Senthil Nathan N <cendhu@gmail.com>
  • Loading branch information
@denyeart @cendhu
denyeart and cendhu authored Sep 17, 2020
1 parent a5a6acd commit c5c1105
Show file tree
Hide file tree
Showing 4 changed files with 97 additions and 67 deletions.
16 changes: 3 additions & 13 deletions core/ledger/kvledger/tests/missing_pvtdata_retrieval_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,11 +73,6 @@ func TestGetMissingPvtData(t *testing.T) {

blk, expectedMissingPvtDataInfo := setup(h)

// verify missing pvtdata info
require.Equal(t, uint64(2), blk.Block.Header.Number)
h.verifyBlockAndPvtDataSameAs(2, blk)
h.verifyMissingPvtDataSameAs(int(2), expectedMissingPvtDataInfo)

// commit block 3
h.simulateDataTx("", func(s *simulator) {
s.setPvtdata("cc1", "coll1", "key4", "value4")
Expand Down Expand Up @@ -145,12 +140,7 @@ func TestGetMissingPvtData(t *testing.T) {
env.initLedgerMgmt()
h := env.newTestHelperCreateLgr("ledger1", t)

blk, expectedMissingPvtDataInfo := setup(h)

// verify missing pvtdata info
require.Equal(t, uint64(2), blk.Block.Header.Number)
h.verifyBlockAndPvtDataSameAs(2, blk)
h.verifyMissingPvtDataSameAs(int(2), expectedMissingPvtDataInfo)
_, expectedMissingPvtDataInfo := setup(h)

h.commitPvtDataOfOldBlocks(nil, expectedMissingPvtDataInfo)
for i := 0; i < 5; i++ {
Expand All @@ -163,7 +153,7 @@ func TestGetMissingPvtData(t *testing.T) {

h = env.newTestHelperOpenLgr("ledger1", t)
for i := 0; i < 5; i++ {
h.verifyMissingPvtDataSameAs(int(2), expectedMissingPvtDataInfo)
h.verifyMissingPvtDataSameAs(2, expectedMissingPvtDataInfo)
}

env.closeLedgerMgmt()
Expand All @@ -172,7 +162,7 @@ func TestGetMissingPvtData(t *testing.T) {

h = env.newTestHelperOpenLgr("ledger1", t)
for i := 0; i < 5; i++ {
h.verifyMissingPvtDataSameAs(int(2), ledger.MissingPvtDataInfo{})
h.verifyMissingPvtDataSameAs(2, ledger.MissingPvtDataInfo{})
}
})
}
142 changes: 92 additions & 50 deletions core/ledger/pvtdatastorage/store_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -192,67 +192,109 @@ func TestStoreIteratorError(t *testing.T) {
}

func TestGetMissingDataInfo(t *testing.T) {
ledgerid := "TestGetMissingDataInfoFromPrioDeprioList"
btlPolicy := btltestutil.SampleBTLPolicy(
map[[2]string]uint64{
{"ns-1", "coll-1"}: 0,
{"ns-1", "coll-2"}: 0,
},
)
env := NewTestStoreEnv(t, ledgerid, btlPolicy, pvtDataConf())
defer env.Cleanup()
store := env.TestStore

// construct missing data for block 1
blk1MissingData := make(ledger.TxMissingPvtDataMap)
blk1MissingData.Add(1, "ns-1", "coll-1", true)
blk1MissingData.Add(1, "ns-1", "coll-2", true)

require.NoError(t, store.Commit(0, nil, nil))
require.NoError(t, store.Commit(1, nil, blk1MissingData))

deprioritizedList := ledger.MissingPvtDataInfo{
1: ledger.MissingBlockPvtdataInfo{
1: {
{
Namespace: "ns-1",
Collection: "coll-2",
},
setup := func(ledgerid string, c *PrivateDataConfig) *Store {
btlPolicy := btltestutil.SampleBTLPolicy(
map[[2]string]uint64{
{"ns-1", "coll-1"}: 0,
{"ns-1", "coll-2"}: 0,
},
},
}
require.NoError(t, store.CommitPvtDataOfOldBlocks(nil, deprioritizedList))
)

expectedPrioMissingDataInfo := ledger.MissingPvtDataInfo{
1: ledger.MissingBlockPvtdataInfo{
1: {
{
Namespace: "ns-1",
Collection: "coll-1",
env := NewTestStoreEnv(t, ledgerid, btlPolicy, c)
t.Cleanup(
func() {
defer env.Cleanup()
},
)
store := env.TestStore

// construct missing data for block 1
blk1MissingData := make(ledger.TxMissingPvtDataMap)
blk1MissingData.Add(1, "ns-1", "coll-1", true)
blk1MissingData.Add(1, "ns-1", "coll-2", true)

require.NoError(t, store.Commit(0, nil, nil))
require.NoError(t, store.Commit(1, nil, blk1MissingData))

deprioritizedList := ledger.MissingPvtDataInfo{
1: ledger.MissingBlockPvtdataInfo{
1: {
{
Namespace: "ns-1",
Collection: "coll-2",
},
},
},
},
}
require.NoError(t, store.CommitPvtDataOfOldBlocks(nil, deprioritizedList))

return env.TestStore
}

expectedDeprioMissingDataInfo := ledger.MissingPvtDataInfo{
1: ledger.MissingBlockPvtdataInfo{
1: {
{
Namespace: "ns-1",
Collection: "coll-2",
t.Run("always access deprioritized missing data", func(t *testing.T) {
conf := pvtDataConf()
conf.DeprioritizedDataReconcilerInterval = 0
store := setup("testGetMissingDataInfoFromDeprioList", conf)

expectedDeprioMissingDataInfo := ledger.MissingPvtDataInfo{
1: ledger.MissingBlockPvtdataInfo{
1: {
{
Namespace: "ns-1",
Collection: "coll-2",
},
},
},
},
}
}

assertMissingDataInfo(t, store, expectedPrioMissingDataInfo, 2)
for i := 0; i < 2; i++ {
assertMissingDataInfo(t, store, expectedDeprioMissingDataInfo, 2)
}
})

store.accessDeprioMissingDataAfter = time.Now()
expectedNextAccessDeprioMissingDataTime := time.Now().Add(store.deprioritizedDataReconcilerInterval)
assertMissingDataInfo(t, store, expectedDeprioMissingDataInfo, 2)
t.Run("change the deprioritized missing data access time", func(t *testing.T) {
conf := pvtDataConf()
conf.DeprioritizedDataReconcilerInterval = 300 * time.Minute
store := setup("testGetMissingDataInfoFromPrioAndDeprioList", conf)

expectedPrioMissingDataInfo := ledger.MissingPvtDataInfo{
1: ledger.MissingBlockPvtdataInfo{
1: {
{
Namespace: "ns-1",
Collection: "coll-1",
},
},
},
}

expectedDeprioMissingDataInfo := ledger.MissingPvtDataInfo{
1: ledger.MissingBlockPvtdataInfo{
1: {
{
Namespace: "ns-1",
Collection: "coll-2",
},
},
},
}

for i := 0; i < 3; i++ {
assertMissingDataInfo(t, store, expectedPrioMissingDataInfo, 2)
}

store.accessDeprioMissingDataAfter = time.Now().Add(-time.Second)
lesserThanNextAccessTime := time.Now().Add(store.deprioritizedDataReconcilerInterval).Add(-2 * time.Second)
greaterThanNextAccessTime := time.Now().Add(store.deprioritizedDataReconcilerInterval).Add(2 * time.Second)
assertMissingDataInfo(t, store, expectedDeprioMissingDataInfo, 2)

require.True(t, store.accessDeprioMissingDataAfter.After(lesserThanNextAccessTime))
require.False(t, store.accessDeprioMissingDataAfter.After(greaterThanNextAccessTime))
for i := 0; i < 3; i++ {
assertMissingDataInfo(t, store, expectedPrioMissingDataInfo, 2)
}
})

require.True(t, store.accessDeprioMissingDataAfter.After(expectedNextAccessDeprioMissingDataTime))
assertMissingDataInfo(t, store, expectedPrioMissingDataInfo, 2)
}

func TestExpiryDataNotIncluded(t *testing.T) {
Expand Down
2 changes: 0 additions & 2 deletions internal/peer/node/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ SPDX-License-Identifier: Apache-2.0
package node

import (
"fmt"
"path/filepath"
"time"

Expand Down Expand Up @@ -45,7 +44,6 @@ func ledgerConfig() *ledger.Config {
deprioritizedDataReconcilerInterval := 60 * time.Minute
if viper.IsSet("ledger.pvtdataStore.deprioritizedDataReconcilerInterval") {
deprioritizedDataReconcilerInterval = viper.GetDuration("ledger.pvtdataStore.deprioritizedDataReconcilerInterval")
fmt.Println(deprioritizedDataReconcilerInterval)
}

rootFSPath := filepath.Join(coreconfig.GetPath("peer.fileSystemPath"), "ledgersData")
Expand Down
4 changes: 2 additions & 2 deletions internal/peer/node/config_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,7 @@ func TestLedgerConfig(t *testing.T) {
"ledger.pvtdataStore.collElgProcMaxDbBatchSize": 50000,
"ledger.pvtdataStore.collElgProcDbBatchesInterval": 10000,
"ledger.pvtdataStore.purgeInterval": 1000,
"ledger.pvtdataStore.deprioritizedDataReconcilerInterval": "60m",
"ledger.pvtdataStore.deprioritizedDataReconcilerInterval": "180m",
"ledger.history.enableHistoryDatabase": true,
"ledger.snapshots.rootDir": "/peerfs/snapshots",
},
Expand All @@ -141,7 +141,7 @@ func TestLedgerConfig(t *testing.T) {
MaxBatchSize: 50000,
BatchesInterval: 10000,
PurgeInterval: 1000,
DeprioritizedDataReconcilerInterval: 60 * time.Minute,
DeprioritizedDataReconcilerInterval: 180 * time.Minute,
},
HistoryDBConfig: &ledger.HistoryDBConfig{
Enabled: true,
Expand Down

0 comments on commit c5c1105

Please sign in to comment.