From 01eeecc88ebb4d89cbb34de3506691c730f0bdb5 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Fri, 25 Mar 2022 11:50:11 +0000 Subject: [PATCH] Initial purge private data integration tests Signed-off-by: James Taylor --- .../chaincode/marbles_private/chaincode.go | 82 +++++++++ integration/pvtdata/data_purge_test.go | 158 ++++++++++++++++++ .../pvtdata/marblechaincodeutil/testutil.go | 19 +++ integration/pvtdata/pvtdata_suite_test.go | 5 + integration/pvtdata/pvtdata_test.go | 40 ++--- 5 files changed, 282 insertions(+), 22 deletions(-) create mode 100644 integration/pvtdata/data_purge_test.go diff --git a/integration/chaincode/marbles_private/chaincode.go b/integration/chaincode/marbles_private/chaincode.go index d3d263d27d4..1617e8e3256 100644 --- a/integration/chaincode/marbles_private/chaincode.go +++ b/integration/chaincode/marbles_private/chaincode.go @@ -61,6 +61,9 @@ func (t *MarblesPrivateChaincode) Invoke(stub shim.ChaincodeStubInterface) pb.Re case "delete": // delete a marble return t.delete(stub, args) + case "purge": + // purge a marble + return t.purge(stub, args) case "getMarblesByRange": // get marbles based on range query return t.getMarblesByRange(stub, args) @@ -377,6 +380,85 @@ func (t *MarblesPrivateChaincode) delete(stub shim.ChaincodeStubInterface, args return shim.Success(nil) } +// ===================================================== +// purge - remove a marble key/value pair from state and +// remove all trace of private details +// ===================================================== +func (t *MarblesPrivateChaincode) purge(stub shim.ChaincodeStubInterface, args []string) pb.Response { + fmt.Println("- start purge marble") + + type marblePurgeTransientInput struct { + Name string `json:"name"` + } + + if len(args) != 0 { + return shim.Error("Incorrect number of arguments. Private marble name must be passed in transient map.") + } + + transMap, err := stub.GetTransient() + if err != nil { + return shim.Error("Error getting transient: " + err.Error()) + } + + marblePurgeJsonBytes, ok := transMap["marble_purge"] + if !ok { + return shim.Error("marble_purge must be a key in the transient map") + } + + if len(marblePurgeJsonBytes) == 0 { + return shim.Error("marble_purge value in the transient map must be a non-empty JSON string") + } + + var marblePurgeInput marblePurgeTransientInput + err = json.Unmarshal(marblePurgeJsonBytes, &marblePurgeInput) + if err != nil { + return shim.Error("Failed to decode JSON of: " + string(marblePurgeJsonBytes)) + } + + if len(marblePurgeInput.Name) == 0 { + return shim.Error("name field must be a non-empty string") + } + + // to maintain the color~name index, we need to read the marble first and get its color + valAsbytes, err := stub.GetPrivateData("collectionMarbles", marblePurgeInput.Name) // get the marble from chaincode state + if err != nil { + return shim.Error("Failed to get state for " + marblePurgeInput.Name) + } else if valAsbytes == nil { + return shim.Error("Marble does not exist: " + marblePurgeInput.Name) + } + + var marbleToPurge marble + err = json.Unmarshal([]byte(valAsbytes), &marbleToPurge) + if err != nil { + return shim.Error("Failed to decode JSON of: " + string(valAsbytes)) + } + + // purge the marble from state + err = stub.PurgePrivateData("collectionMarbles", marblePurgeInput.Name) + if err != nil { + return shim.Error("Failed to purge state:" + err.Error()) + } + + // Also purge the marble from the color~name index + indexName := "color~name" + colorNameIndexKey, err := stub.CreateCompositeKey(indexName, []string{marbleToPurge.Color, marbleToPurge.Name}) + if err != nil { + return shim.Error(err.Error()) + } + err = stub.PurgePrivateData("collectionMarbles", colorNameIndexKey) + if err != nil { + return shim.Error("Failed to purge state:" + err.Error()) + } + + // Finally, purge private details of marble + err = stub.PurgePrivateData("collectionMarblePrivateDetails", marblePurgeInput.Name) + if err != nil { + return shim.Error(err.Error()) + } + + return shim.Success(nil) +} + // =========================================================== // transfer a marble by setting a new owner name on the marble // =========================================================== diff --git a/integration/pvtdata/data_purge_test.go b/integration/pvtdata/data_purge_test.go new file mode 100644 index 00000000000..c0d9fa73dcf --- /dev/null +++ b/integration/pvtdata/data_purge_test.go @@ -0,0 +1,158 @@ +/* +Copyright IBM Corp All Rights Reserved. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package pvtdata + +import ( + "context" + "io/ioutil" + "os" + "path/filepath" + "syscall" + + "github.com/hyperledger/fabric/integration/nwo" + "github.com/hyperledger/fabric/integration/pvtdata/marblechaincodeutil" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/tedsuo/ifrit" +) + +var _ = Describe("Pvtdata purge", func() { + var ( + testDir string + network *nwo.Network + orderer *nwo.Orderer + org2Peer0 *nwo.Peer + process ifrit.Process + cancel context.CancelFunc + chaincode *nwo.Chaincode + ) + + BeforeEach(func() { + var err error + testDir, err = ioutil.TempDir("", "purgedata") + Expect(err).NotTo(HaveOccurred()) + + config := nwo.ThreeOrgRaft() + network = nwo.New(config, testDir, nil, StartPort(), components) + + network.GenerateConfigTree() + network.Bootstrap() + + networkRunner := network.NetworkGroupRunner() + process = ifrit.Invoke(networkRunner) + Eventually(process.Ready(), network.EventuallyTimeout).Should(BeClosed()) + + orderer = network.Orderer("orderer") + network.CreateAndJoinChannel(orderer, "testchannel") + network.UpdateChannelAnchors(orderer, "testchannel") + network.VerifyMembership( + network.PeersWithChannel("testchannel"), + "testchannel", + ) + nwo.EnableCapabilities( + network, + "testchannel", + "Application", "V2_5", + orderer, + network.PeersWithChannel("testchannel")..., + ) + + chaincode = &nwo.Chaincode{ + Name: "marblesp", + Version: "0.0", + Path: components.Build("github.com/hyperledger/fabric/integration/chaincode/marbles_private/cmd"), + Lang: "binary", + PackageFile: filepath.Join(testDir, "purgecc.tar.gz"), + Ctor: `{"Args":[]}`, + SignaturePolicy: `OR ('Org1MSP.member','Org2MSP.member', 'Org3MSP.member')`, + CollectionsConfig: CollectionConfig("collections_config1.json"), + Sequence: "1", + InitRequired: false, + Label: "purgecc_label", + } + + nwo.DeployChaincode(network, "testchannel", orderer, *chaincode) + + org2Peer0 = network.Peer("Org2", "peer0") + + _, cancel = context.WithTimeout(context.Background(), network.EventuallyTimeout) + + marblechaincodeutil.AddMarble(network, orderer, channelID, chaincode.Name, `{"name":"test-marble-0", "color":"blue", "size":35, "owner":"tom", "price":99}`, org2Peer0) + }) + + AfterEach(func() { + cancel() + + if process != nil { + process.Signal(syscall.SIGTERM) + Eventually(process.Wait(), network.EventuallyTimeout).Should(Receive()) + } + if network != nil { + network.Cleanup() + } + os.RemoveAll(testDir) + }) + + // 6. The purge transaction takes effect only if the corresponding capability is set + // - Add a few keys into a collection + // - Issue a purge transaction without setting the new capability + // - [Verify that the purge fails] + PIt("should fail with an error if the purge capability has not been enabled on the channel") + + It("should prevent purged data being included in responses after the purge transaction has been committed", func() { + marblechaincodeutil.AssertPresentInCollectionM(network, channelID, chaincode.Name, `test-marble-0`, org2Peer0) + marblechaincodeutil.AssertPresentInCollectionMPD(network, channelID, chaincode.Name, "test-marble-0", org2Peer0) + + marblechaincodeutil.PurgeMarble(network, orderer, channelID, chaincode.Name, `{"name":"test-marble-0"}`, org2Peer0) + + marblechaincodeutil.AssertDoesNotExistInCollectionM(network, channelID, chaincode.Name, `test-marble-0`, org2Peer0) + marblechaincodeutil.AssertDoesNotExistInCollectionMPD(network, channelID, chaincode.Name, `test-marble-0`, org2Peer0) + }) + + PIt("should prevent purged data being included block event replays after the purge transaction has been committed") + + // 1. User is able to submit a purge transaction that involves more than one keys + PIt("should accept multiple keys for purging in the same transaction") + + // 2. The endorsement policy is evaluated correctly for a purge transaction under + // different endorsement policy settings (e.g., collection level/ key-hash based) + // Note: The endorsement policy level tests need not to be prioritized over other + // behaviour, and they need not to be very exhaustive since they should be covered + // by existing write/delete operations + PIt("should correctly enforce collection level endorsement policies") + PIt("should correctly enforce key-hash based endorsement policies") + PIt("should correctly enforce other endorsement policies (TBC)") + + // 3. Data is purged on an eligible peer + // - Add a few keys into a collection + // - Issue a purge transaction for some of the keys + // - Verify that all the versions of the intended keys are purged while the remaining keys still exist + // - Repeat above to purge all keys to test the corner case + PIt("should remove all purged data from an eligible peer") + + // 4. Data is purged on previously eligible but now ineligible peer + // - Add a few keys into a collection + // - Submit a collection config update to remove an org + // - Issue a purge transaction to delete few keys + // - The removed orgs peer should have purged the historical versions of intended key + PIt("should remove all purged data from a previously eligible peer") + + // 5. A new peer able to reconcile from a purged peer + // - Stop one of the peers of an eligible org + // - Add a few keys into a collection + // - Issue a purge transaction for some of the keys + // - Start the stopped peer and the peer should reconcile the partial available data + PIt("should enable successful peer reconciliation with partial write-sets") + + // 7. Further writes to private data after a purge operation are not purged + // - Add a few keys into a collection + // - Issue a purge transaction + // - Add the purged data back + // - The subsequently added data should not be purged as a + // side-effect of the previous purge operation + PIt("should not remove new data after a previous purge operation") +}) diff --git a/integration/pvtdata/marblechaincodeutil/testutil.go b/integration/pvtdata/marblechaincodeutil/testutil.go index 0bbf7a08de6..e1045889ef1 100644 --- a/integration/pvtdata/marblechaincodeutil/testutil.go +++ b/integration/pvtdata/marblechaincodeutil/testutil.go @@ -57,6 +57,25 @@ func DeleteMarble(n *nwo.Network, orderer *nwo.Orderer, channelID, chaincodeName nwo.WaitUntilEqualLedgerHeight(n, channelID, nwo.GetLedgerHeight(n, peer, channelID), n.Peers...) } +// PurgeMarble invokes marbles_private chaincode to purge a marble +func PurgeMarble(n *nwo.Network, orderer *nwo.Orderer, channelID, chaincodeName, marblePurge string, peer *nwo.Peer) { + marblePurgeBase64 := base64.StdEncoding.EncodeToString([]byte(marblePurge)) + + command := commands.ChaincodeInvoke{ + ChannelID: channelID, + Orderer: n.OrdererAddress(orderer, nwo.ListenPort), + Name: chaincodeName, + Ctor: `{"Args":["purge"]}`, + Transient: fmt.Sprintf(`{"marble_purge":"%s"}`, marblePurgeBase64), + PeerAddresses: []string{ + n.PeerAddress(peer, nwo.ListenPort), + }, + WaitForEvent: true, + } + invokeChaincode(n, peer, command) + nwo.WaitUntilEqualLedgerHeight(n, channelID, nwo.GetLedgerHeight(n, peer, channelID), n.Peers...) +} + // TransferMarble invokes marbles_private chaincode to transfer marble's ownership func TransferMarble(n *nwo.Network, orderer *nwo.Orderer, channelID, chaincodeName, marbleOwner string, peer *nwo.Peer) { marbleOwnerBase64 := base64.StdEncoding.EncodeToString([]byte(marbleOwner)) diff --git a/integration/pvtdata/pvtdata_suite_test.go b/integration/pvtdata/pvtdata_suite_test.go index 4f8b6982b1c..2aa0d7884f3 100644 --- a/integration/pvtdata/pvtdata_suite_test.go +++ b/integration/pvtdata/pvtdata_suite_test.go @@ -8,6 +8,7 @@ package pvtdata import ( "encoding/json" + "path/filepath" "testing" "github.com/hyperledger/fabric/integration" @@ -48,3 +49,7 @@ var _ = SynchronizedAfterSuite(func() { func StartPort() int { return integration.PrivateDataBasePort.StartPortForNode() } + +func CollectionConfig(collConfigFile string) string { + return filepath.Join("testdata", "collection_configs", collConfigFile) +} diff --git a/integration/pvtdata/pvtdata_test.go b/integration/pvtdata/pvtdata_test.go index ee0a6ee7662..4abf964b587 100644 --- a/integration/pvtdata/pvtdata_test.go +++ b/integration/pvtdata/pvtdata_test.go @@ -95,7 +95,7 @@ var _ bool = Describe("PrivateData", func() { // collections_config1.json defines the access as follows: // 1. collectionMarbles - Org1, Org2 have access to this collection // 2. collectionMarblePrivateDetails - Org2 and Org3 have access to this collection - CollectionsConfig: collectionConfig("collections_config1.json"), + CollectionsConfig: CollectionConfig("collections_config1.json"), }, isLegacy: true, } @@ -115,7 +115,7 @@ var _ bool = Describe("PrivateData", func() { Path: "github.com/hyperledger/fabric/integration/chaincode/marbles_private/cmd", Ctor: `{"Args":["init"]}`, Policy: `OR ('Org1MSP.member','Org2MSP.member', 'Org3MSP.member')`, - CollectionsConfig: collectionConfig("collections_config8_high_requiredPeerCount.json"), + CollectionsConfig: CollectionConfig("collections_config8_high_requiredPeerCount.json"), }, isLegacy: true, } @@ -152,7 +152,7 @@ var _ bool = Describe("PrivateData", func() { // collections_config1.json defines the access as follows: // 1. collectionMarbles - Org1, Org2 have access to this collection // 2. collectionMarblePrivateDetails - Org2 and Org3 have access to this collection - CollectionsConfig: collectionConfig("collections_config7.json"), + CollectionsConfig: CollectionConfig("collections_config7.json"), }, isLegacy: true, } @@ -411,7 +411,7 @@ var _ bool = Describe("PrivateData", func() { // collections_config1.json defines the access as follows: // 1. collectionMarbles - Org1, Org2 have access to this collection // 2. collectionMarblePrivateDetails - Org2 and Org3 have access to this collection - CollectionsConfig: collectionConfig("collections_config1.json"), + CollectionsConfig: CollectionConfig("collections_config1.json"), } newLifecycleChaincode = nwo.Chaincode{ @@ -422,7 +422,7 @@ var _ bool = Describe("PrivateData", func() { PackageFile: filepath.Join(network.RootDir, "marbles-pvtdata.tar.gz"), Label: "marbles-private-20", SignaturePolicy: `OR ('Org1MSP.member','Org2MSP.member', 'Org3MSP.member')`, - CollectionsConfig: collectionConfig("collections_config1.json"), + CollectionsConfig: CollectionConfig("collections_config1.json"), Sequence: "1", } org1Peer1 = &nwo.Peer{ @@ -479,7 +479,7 @@ var _ bool = Describe("PrivateData", func() { // 2. collectionMarblePrivateDetails - Org2 and Org3 have access to this collection // the change from collections_config1 - org3 was added to collectionMarbles testChaincode.Version = "1.1" - testChaincode.CollectionsConfig = collectionConfig("collections_config2.json") + testChaincode.CollectionsConfig = CollectionConfig("collections_config2.json") if !testChaincode.isLegacy { testChaincode.Sequence = "2" } @@ -536,7 +536,7 @@ var _ bool = Describe("PrivateData", func() { } nwo.EnableCapabilities(network, channelID, "Application", "V2_0", orderer, network.Peers...) - testChaincode.CollectionsConfig = collectionConfig("short_btl_config.json") + testChaincode.CollectionsConfig = CollectionConfig("short_btl_config.json") deployChaincode(network, orderer, testChaincode) marblechaincodeutil.AddMarble(network, orderer, channelID, testChaincode.Name, `{"name":"marble1", "color":"blue", "size":35, "owner":"tom", "price":99}`, network.Peer("Org2", "peer0")) @@ -571,7 +571,7 @@ var _ bool = Describe("PrivateData", func() { Describe("Org removal from collection", func() { assertOrgRemovalBehavior := func() { By("upgrading chaincode to remove org3 from collectionMarbles") - testChaincode.CollectionsConfig = collectionConfig("collections_config1.json") + testChaincode.CollectionsConfig = CollectionConfig("collections_config1.json") testChaincode.Version = "1.1" if !testChaincode.isLegacy { testChaincode.Sequence = "2" @@ -588,7 +588,7 @@ var _ bool = Describe("PrivateData", func() { isLegacy: false, } nwo.EnableCapabilities(network, channelID, "Application", "V2_0", orderer, network.Peers...) - testChaincode.CollectionsConfig = collectionConfig("collections_config2.json") + testChaincode.CollectionsConfig = CollectionConfig("collections_config2.json") deployChaincode(network, orderer, testChaincode) marblechaincodeutil.AddMarble(network, orderer, channelID, testChaincode.Name, `{"name":"marble1", "color":"blue", "size":35, "owner":"tom", "price":99}`, network.Peer("Org2", "peer0")) assertPvtdataPresencePerCollectionConfig2(network, testChaincode.Name, "marble1") @@ -607,7 +607,7 @@ var _ bool = Describe("PrivateData", func() { deployChaincode(network, orderer, testChaincode) nwo.EnableCapabilities(network, channelID, "Application", "V2_0", orderer, network.Peers...) - newLifecycleChaincode.CollectionsConfig = collectionConfig("short_btl_config.json") + newLifecycleChaincode.CollectionsConfig = CollectionConfig("short_btl_config.json") newLifecycleChaincode.PackageID = "test-package-id" approveChaincodeForMyOrgExpectErr( @@ -627,7 +627,7 @@ var _ bool = Describe("PrivateData", func() { isLegacy: true, } By("setting the collection config endorsement policy to org2 or org3 peers") - testChaincode.CollectionsConfig = collectionConfig("collections_config4.json") + testChaincode.CollectionsConfig = CollectionConfig("collections_config4.json") By("deploying legacy chaincode") deployChaincode(network, orderer, testChaincode) @@ -651,7 +651,7 @@ var _ bool = Describe("PrivateData", func() { When("a peer specified in the chaincode endorsement policy but not in the collection config endorsement policy is used to invoke the chaincode", func() { It("fails validation", func() { By("setting the collection config endorsement policy to org2 or org3 peers") - testChaincode.CollectionsConfig = collectionConfig("collections_config4.json") + testChaincode.CollectionsConfig = CollectionConfig("collections_config4.json") By("deploying new lifecycle chaincode") // set collection endorsement policy to org2 or org3 @@ -686,7 +686,7 @@ var _ bool = Describe("PrivateData", func() { It("successfully invokes the chaincode", func() { // collection config endorsement policy specifies org2 or org3 peers for endorsement By("setting the collection config endorsement policy to use a signature policy") - testChaincode.CollectionsConfig = collectionConfig("collections_config4.json") + testChaincode.CollectionsConfig = CollectionConfig("collections_config4.json") By("setting the chaincode endorsement policy to org1 or org2 peers") testChaincode.SignaturePolicy = `OR ('Org1MSP.member','Org2MSP.member')` @@ -706,7 +706,7 @@ var _ bool = Describe("PrivateData", func() { It("successfully invokes the chaincode", func() { // collection config endorsement policy specifies channel config policy reference /Channel/Application/Readers By("setting the collection config endorsement policy to use a channel config policy reference") - testChaincode.CollectionsConfig = collectionConfig("collections_config5.json") + testChaincode.CollectionsConfig = CollectionConfig("collections_config5.json") By("setting the channel endorsement policy to org1 or org2 peers") testChaincode.SignaturePolicy = `OR ('Org1MSP.member','Org2MSP.member')` @@ -725,7 +725,7 @@ var _ bool = Describe("PrivateData", func() { When("the collection config endorsement policy specifies a semantically wrong, but well formed signature policy", func() { It("fails to invoke the chaincode with an endorsement policy failure", func() { By("setting the collection config endorsement policy to non existent org4 peers") - testChaincode.CollectionsConfig = collectionConfig("collections_config6.json") + testChaincode.CollectionsConfig = CollectionConfig("collections_config6.json") By("deploying new lifecycle chaincode") deployChaincode(network, orderer, testChaincode) @@ -775,7 +775,7 @@ var _ bool = Describe("PrivateData", func() { PackageFile: filepath.Join(network.RootDir, "marbles-pvtdata.tar.gz"), Label: "marbles-private-20", SignaturePolicy: `OR ('Org1MSP.member','Org2MSP.member', 'Org3MSP.member')`, - CollectionsConfig: collectionConfig("collections_config1.json"), + CollectionsConfig: CollectionConfig("collections_config1.json"), Sequence: "1", } @@ -870,7 +870,7 @@ var _ bool = Describe("PrivateData", func() { assertPrivateDataAsExpected(event.BlockAndPvtData.PrivateDataMap, expectedKVWritesMap) By("upgrading chaincode with collections_config1.json where isMemberOnlyRead is false") - testChaincode.CollectionsConfig = collectionConfig("collections_config1.json") + testChaincode.CollectionsConfig = CollectionConfig("collections_config1.json") testChaincode.Version = "1.1" if !testChaincode.isLegacy { testChaincode.Sequence = "2" @@ -912,7 +912,7 @@ var _ bool = Describe("PrivateData", func() { isLegacy: false, } nwo.EnableCapabilities(network, channelID, "Application", "V2_0", orderer, network.Peers...) - testChaincode.CollectionsConfig = collectionConfig("collections_config3.json") + testChaincode.CollectionsConfig = CollectionConfig("collections_config3.json") deployChaincode(network, orderer, testChaincode) By("attempting to invoke chaincode from a user (org1) not in any collection member orgs (org2 and org3)") @@ -1015,10 +1015,6 @@ func testCleanup(network *nwo.Network, process ifrit.Process) { os.RemoveAll(network.RootDir) } -func collectionConfig(collConfigFile string) string { - return filepath.Join("testdata", "collection_configs", collConfigFile) -} - type chaincode struct { nwo.Chaincode isLegacy bool