From 4dc59a74d3b31455b3c2545360405a42a71560c1 Mon Sep 17 00:00:00 2001 From: bkiran6398 Date: Wed, 3 Apr 2024 21:50:50 +0530 Subject: [PATCH 1/2] enabled support for couchDB indexes on implicit collections Signed-off-by: bkiran6398 documentation update to specify couchDB index support for implicit collections Signed-off-by: bkiran6398 added support for global index creation for all implicit collections Signed-off-by: bkiran6398 documentation update to brief global implicit collection index creation Signed-off-by: bkiran6398 --- .../kvledger/txmgmt/privacyenabledstate/db.go | 23 ++++++++++++++--- .../txmgmt/privacyenabledstate/db_test.go | 25 +++++++++++++++++-- docs/source/private-data-arch.rst | 11 ++++---- 3 files changed, 47 insertions(+), 12 deletions(-) diff --git a/core/ledger/kvledger/txmgmt/privacyenabledstate/db.go b/core/ledger/kvledger/txmgmt/privacyenabledstate/db.go index 270c618c55b..2fb6460c1fd 100644 --- a/core/ledger/kvledger/txmgmt/privacyenabledstate/db.go +++ b/core/ledger/kvledger/txmgmt/privacyenabledstate/db.go @@ -13,6 +13,7 @@ import ( "github.com/hyperledger/fabric-lib-go/common/flogging" "github.com/hyperledger/fabric-lib-go/common/metrics" "github.com/hyperledger/fabric-lib-go/healthz" + "github.com/hyperledger/fabric/core/chaincode/implicitcollection" "github.com/hyperledger/fabric/core/common/ccprovider" "github.com/hyperledger/fabric/core/ledger" "github.com/hyperledger/fabric/core/ledger/cceventmgmt" @@ -23,6 +24,7 @@ import ( "github.com/hyperledger/fabric/core/ledger/kvledger/txmgmt/statedb/stateleveldb" "github.com/hyperledger/fabric/core/ledger/util" "github.com/pkg/errors" + "github.com/spf13/viper" ) var logger = flogging.MustGetLogger("privacyenabledstate") @@ -33,6 +35,10 @@ const ( hashDataPrefix = "h" ) +const ( + allImplicitCollectionNotation string = "*" +) + // StateDBConfig encapsulates the configuration for stateDB on the ledger. type StateDBConfig struct { // ledger.StateDBConfig is used to configure the stateDB for the ledger. @@ -124,12 +130,13 @@ func (p *DBProvider) Drop(ledgerid string) error { type DB struct { statedb.VersionedDB metadataHint *metadataHint + localMspId string } // NewDB wraps a VersionedDB instance. The public data is managed directly by the wrapped versionedDB. // For managing the hashed data and private data, this implementation creates separate namespaces in the wrapped db func NewDB(vdb statedb.VersionedDB, ledgerid string, metadataHint *metadataHint) (*DB, error) { - return &DB{vdb, metadataHint}, nil + return &DB{vdb, metadataHint, viper.GetString("peer.localMspId")}, nil } // IsBulkOptimizable checks whether the underlying statedb implements statedb.BulkOptimizable @@ -330,9 +337,17 @@ func (s *DB) HandleChaincodeDeploy(chaincodeDefinition *cceventmgmt.ChaincodeDef case indexInfo.hasIndexForCollection: _, ok := collectionConfigMap[indexInfo.collectionName] if !ok { - logger.Errorf("Error processing index for chaincode [%s]: cannot create an index for an undefined collection=[%s]", - chaincodeDefinition.Name, indexInfo.collectionName) - continue + isImplicitCollection, collectionMspId := implicitcollection.MspIDIfImplicitCollection(indexInfo.collectionName) + if !isImplicitCollection { + logger.Errorf("Error processing index for chaincode [%s]: cannot create an index for an undefined collection=[%s]", + chaincodeDefinition.Name, indexInfo.collectionName) + continue + } + if collectionMspId != allImplicitCollectionNotation && collectionMspId != s.localMspId { + logger.Debugf("Skipped processing index of other org implicit collection=[%s] for chaincode [%s] ", + indexInfo.collectionName, chaincodeDefinition.Name) + continue + } } err := indexCapable.ProcessIndexesForChaincodeDeploy(derivePvtDataNs(chaincodeDefinition.Name, indexInfo.collectionName), indexFilesData) if err != nil { diff --git a/core/ledger/kvledger/txmgmt/privacyenabledstate/db_test.go b/core/ledger/kvledger/txmgmt/privacyenabledstate/db_test.go index 4758aa4cf5b..88237f52d43 100644 --- a/core/ledger/kvledger/txmgmt/privacyenabledstate/db_test.go +++ b/core/ledger/kvledger/txmgmt/privacyenabledstate/db_test.go @@ -11,6 +11,7 @@ import ( "encoding/hex" "fmt" "io" + "os" "testing" "github.com/hyperledger/fabric-protos-go/peer" @@ -438,6 +439,9 @@ func createCollectionConfig(collectionName string) *peer.CollectionConfig { func testHandleChainCodeDeploy(t *testing.T, env TestEnv) { env.Init(t) defer env.Cleanup() + // Set local MSP ID for DB to use + err := os.Setenv("CORE_PEER_LOCALMSPID", "testOrgMsp1") + require.NoError(t, err) db := env.GetDBHandle(generateLedgerID(t)) coll1 := createCollectionConfig("collectionMarbles") @@ -451,6 +455,9 @@ func testHandleChainCodeDeploy(t *testing.T, env TestEnv) { {Name: "META-INF/statedb/couchdb/indexes/indexSizeSortName.json", Body: `{"index":{"fields":[{"size":"desc"}]},"ddoc":"indexSizeSortName","name":"indexSizeSortName","type":"json"}`}, {Name: "META-INF/statedb/couchdb/collections/collectionMarbles/indexes/indexCollMarbles.json", Body: `{"index":{"fields":["docType","owner"]},"ddoc":"indexCollectionMarbles", "name":"indexCollectionMarbles","type":"json"}`}, {Name: "META-INF/statedb/couchdb/collections/collectionMarblesPrivateDetails/indexes/indexCollPrivDetails.json", Body: `{"index":{"fields":["docType","price"]},"ddoc":"indexPrivateDetails", "name":"indexPrivateDetails","type":"json"}`}, + {Name: "META-INF/statedb/couchdb/collections/_implicit_org_testOrgMsp1/indexes/indexCreateDate.json", Body: `{"index":{"fields":["create_date"]},"ddoc":"indexCreateDateDoc", "name":"indexCreateDate","type":"json"}`}, + {Name: "META-INF/statedb/couchdb/collections/_implicit_org_testOrgMsp2/indexes/indexUpdateDate.json", Body: `{"index":{"fields":["update_date"]},"ddoc":"indexUpdateDateDoc", "name":"indexUpdateDate","type":"json"}`}, + {Name: "META-INF/statedb/couchdb/collections/_implicit_org_*/indexes/indexDeleteDate.json", Body: `{"index":{"fields":["delete_date"]},"ddoc":"indexDeleteDateDoc", "name":"indexDeleteDate","type":"json"}`}, }, ) @@ -458,8 +465,8 @@ func testHandleChainCodeDeploy(t *testing.T, env TestEnv) { fileEntries, err := ccprovider.ExtractFileEntries(dbArtifactsTarBytes, "couchdb") require.NoError(t, err) - // There should be 3 entries - require.Len(t, fileEntries, 3) + // There should be 6 entries + require.Len(t, fileEntries, 6) // There should be 2 entries for main require.Len(t, fileEntries["META-INF/statedb/couchdb/indexes"], 2) @@ -467,6 +474,15 @@ func testHandleChainCodeDeploy(t *testing.T, env TestEnv) { // There should be 1 entry for collectionMarbles require.Len(t, fileEntries["META-INF/statedb/couchdb/collections/collectionMarbles/indexes"], 1) + // There should be 1 entry for _implicit_org_testOrgMsp1 + require.Len(t, fileEntries["META-INF/statedb/couchdb/collections/_implicit_org_testOrgMsp1/indexes"], 1) + + // There should be 1 entry for _implicit_org_testOrgMsp2 + require.Len(t, fileEntries["META-INF/statedb/couchdb/collections/_implicit_org_testOrgMsp2/indexes"], 1) + + // There should be 1 entry for _implicit_org_* + require.Len(t, fileEntries["META-INF/statedb/couchdb/collections/_implicit_org_*/indexes"], 1) + // Verify the content of the array item expectedJSON := []byte(`{"index":{"fields":["docType","owner"]},"ddoc":"indexCollectionMarbles", "name":"indexCollectionMarbles","type":"json"}`) actualJSON := fileEntries["META-INF/statedb/couchdb/collections/collectionMarbles/indexes"][0].FileContent @@ -474,6 +490,11 @@ func testHandleChainCodeDeploy(t *testing.T, env TestEnv) { // The collection config is added to the chaincodeDef but missing collectionMarblesPrivateDetails. // Hence, the index on collectionMarblesPrivateDetails cannot be created + // + // Index on _implicit_org_testOrgMsp1 will be created as it is implicit collection of this org, + // but _implicit_org_testOrgMsp2 will not be created as it belongs to a different org + // + // Index on _implicit_org_* will be created as it applies to all orgs' implicit collections err = db.HandleChaincodeDeploy(chaincodeDef, dbArtifactsTarBytes) require.NoError(t, err) diff --git a/docs/source/private-data-arch.rst b/docs/source/private-data-arch.rst index 415414cffbf..b4ffeb9e0b5 100644 --- a/docs/source/private-data-arch.rst +++ b/docs/source/private-data-arch.rst @@ -176,9 +176,11 @@ The properties ``requiredPeerCount`` and ``maxPeerCount`` can however be set in can set these properties based on the number of peers that they deploy, as described in the next section. -.. note:: Since implicit private data collections are not explicitly defined, - it is not possible to associate CouchDB indexes with them. Utilize - key-based queries and key-range queries rather than JSON queries. +.. note:: + +CouchDB indexes can be created to organisation specific implicit collections just like explicitly defined collections with indexes directory name `_implicit_org_`. Such indexes are only effective within the respective organization to which the implicit collection belongs. + +To create common indexes for implicit collection of all the organisations, the indexes directory should be named "_implicit_org_*". Private data dissemination -------------------------- @@ -361,9 +363,6 @@ Limitations: chaincode function to make the updates. Note that calls to GetPrivateData() to retrieve individual keys can be made in the same transaction as PutPrivateData() calls, since all peers can validate key reads based on the hashed key version. -* Since implicit private data collections are not explicitly defined, - it is not possible to associate CouchDB indexes with them. - It is therefore not recommended to utilize JSON queries with implicit private data collections. Using Indexes with collections ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From 7b0a4bf67ace17bd61c8ea3dacc0addf70bd1505 Mon Sep 17 00:00:00 2001 From: bkiran6398 Date: Wed, 10 Jul 2024 14:41:28 +0530 Subject: [PATCH 2/2] updated all org implicit collection notation to _implicit_all_orgs Signed-off-by: bkiran6398 --- core/chaincode/implicitcollection/name.go | 7 +++- core/ledger/kvledger/kv_ledger_provider.go | 1 + .../kvledger/txmgmt/privacyenabledstate/db.go | 38 ++++++++----------- 3 files changed, 22 insertions(+), 24 deletions(-) diff --git a/core/chaincode/implicitcollection/name.go b/core/chaincode/implicitcollection/name.go index d6f3d26c05e..a1a9aab60f1 100644 --- a/core/chaincode/implicitcollection/name.go +++ b/core/chaincode/implicitcollection/name.go @@ -11,7 +11,8 @@ import ( ) const ( - prefix = "_implicit_org_" + prefix = "_implicit_org_" + allOrgNotation = "_implicit_all_orgs" ) // NameForOrg constructs the name of the implicit collection for the specified org @@ -31,3 +32,7 @@ func MspIDIfImplicitCollection(collectionName string) (isImplicitCollection bool func IsImplicitCollection(collectionName string) bool { return strings.HasPrefix(collectionName, prefix) } + +func IsAllOrgNotation(collectionName string) bool { + return collectionName == allOrgNotation +} diff --git a/core/ledger/kvledger/kv_ledger_provider.go b/core/ledger/kvledger/kv_ledger_provider.go index 445ee38c19f..86ecb76a8c3 100644 --- a/core/ledger/kvledger/kv_ledger_provider.go +++ b/core/ledger/kvledger/kv_ledger_provider.go @@ -237,6 +237,7 @@ func (p *Provider) initStateDBProvider() error { p.initializer.HealthCheckRegistry, stateDBConfig, sysNamespaces, + p.initializer.MembershipInfoProvider.MyImplicitCollectionName(), ) return err } diff --git a/core/ledger/kvledger/txmgmt/privacyenabledstate/db.go b/core/ledger/kvledger/txmgmt/privacyenabledstate/db.go index 2fb6460c1fd..8efaae31b9f 100644 --- a/core/ledger/kvledger/txmgmt/privacyenabledstate/db.go +++ b/core/ledger/kvledger/txmgmt/privacyenabledstate/db.go @@ -24,7 +24,6 @@ import ( "github.com/hyperledger/fabric/core/ledger/kvledger/txmgmt/statedb/stateleveldb" "github.com/hyperledger/fabric/core/ledger/util" "github.com/pkg/errors" - "github.com/spf13/viper" ) var logger = flogging.MustGetLogger("privacyenabledstate") @@ -35,10 +34,6 @@ const ( hashDataPrefix = "h" ) -const ( - allImplicitCollectionNotation string = "*" -) - // StateDBConfig encapsulates the configuration for stateDB on the ledger. type StateDBConfig struct { // ledger.StateDBConfig is used to configure the stateDB for the ledger. @@ -52,9 +47,10 @@ type StateDBConfig struct { // DBProvider encapsulates other providers such as VersionedDBProvider and // BookeepingProvider which are required to create DB for a channel type DBProvider struct { - VersionedDBProvider statedb.VersionedDBProvider - HealthCheckRegistry ledger.HealthCheckRegistry - bookkeepingProvider *bookkeeping.Provider + VersionedDBProvider statedb.VersionedDBProvider + HealthCheckRegistry ledger.HealthCheckRegistry + bookkeepingProvider *bookkeeping.Provider + myImplicitCollection string } // NewDBProvider constructs an instance of DBProvider @@ -64,6 +60,7 @@ func NewDBProvider( healthCheckRegistry ledger.HealthCheckRegistry, stateDBConf *StateDBConfig, sysNamespaces []string, + myImplicitCollection string, ) (*DBProvider, error) { var vdbProvider statedb.VersionedDBProvider var err error @@ -79,9 +76,10 @@ func NewDBProvider( } dbProvider := &DBProvider{ - VersionedDBProvider: vdbProvider, - HealthCheckRegistry: healthCheckRegistry, - bookkeepingProvider: bookkeeperProvider, + VersionedDBProvider: vdbProvider, + HealthCheckRegistry: healthCheckRegistry, + bookkeepingProvider: bookkeeperProvider, + myImplicitCollection: myImplicitCollection, } err = dbProvider.RegisterHealthChecker() @@ -113,7 +111,7 @@ func (p *DBProvider) GetDBHandle(id string, chInfoProvider channelInfoProvider) if err != nil { return nil, err } - return NewDB(vdb, id, metadataHint) + return NewDB(vdb, id, metadataHint, p.myImplicitCollection) } // Close closes all the VersionedDB instances and releases any resources held by VersionedDBProvider @@ -129,14 +127,14 @@ func (p *DBProvider) Drop(ledgerid string) error { // DB uses a single database to maintain both the public and private data type DB struct { statedb.VersionedDB - metadataHint *metadataHint - localMspId string + metadataHint *metadataHint + myImplicitCollection string } // NewDB wraps a VersionedDB instance. The public data is managed directly by the wrapped versionedDB. // For managing the hashed data and private data, this implementation creates separate namespaces in the wrapped db -func NewDB(vdb statedb.VersionedDB, ledgerid string, metadataHint *metadataHint) (*DB, error) { - return &DB{vdb, metadataHint, viper.GetString("peer.localMspId")}, nil +func NewDB(vdb statedb.VersionedDB, ledgerid string, metadataHint *metadataHint, myImplicitCollection string) (*DB, error) { + return &DB{vdb, metadataHint, myImplicitCollection}, nil } // IsBulkOptimizable checks whether the underlying statedb implements statedb.BulkOptimizable @@ -337,13 +335,7 @@ func (s *DB) HandleChaincodeDeploy(chaincodeDefinition *cceventmgmt.ChaincodeDef case indexInfo.hasIndexForCollection: _, ok := collectionConfigMap[indexInfo.collectionName] if !ok { - isImplicitCollection, collectionMspId := implicitcollection.MspIDIfImplicitCollection(indexInfo.collectionName) - if !isImplicitCollection { - logger.Errorf("Error processing index for chaincode [%s]: cannot create an index for an undefined collection=[%s]", - chaincodeDefinition.Name, indexInfo.collectionName) - continue - } - if collectionMspId != allImplicitCollectionNotation && collectionMspId != s.localMspId { + if !implicitcollection.IsAllOrgNotation(indexInfo.collectionName) && indexInfo.collectionName != s.myImplicitCollection { logger.Debugf("Skipped processing index of other org implicit collection=[%s] for chaincode [%s] ", indexInfo.collectionName, chaincodeDefinition.Name) continue