From bd3971b49199fc78ba91343195476ef702380045 Mon Sep 17 00:00:00 2001 From: Julian Castrence Date: Wed, 25 Aug 2021 14:53:56 -0400 Subject: [PATCH] Private Data Comparison Resolves #2818 Signed-off-by: Julian Castrence --- cmd/ledgerutil/main.go | 2 +- internal/ledgerutil/compare.go | 249 +++++++++++++++++++--------- internal/ledgerutil/compare_test.go | 240 ++++++++++++++++++++++----- 3 files changed, 368 insertions(+), 123 deletions(-) diff --git a/cmd/ledgerutil/main.go b/cmd/ledgerutil/main.go index 92f01bd4bd9..40f0a341a7d 100644 --- a/cmd/ledgerutil/main.go +++ b/cmd/ledgerutil/main.go @@ -68,7 +68,7 @@ func main() { fmt.Print("\nSuccessfully compared snapshots. ") if count == -1 { - fmt.Println("Both snapshot public state hashes were the same. No results were generated.") + fmt.Println("Both snapshot public state and private state hashes were the same. No results were generated.") } else { fmt.Printf("Results saved to %s. Total differences found: %d\n", outputDirPath, count) } diff --git a/internal/ledgerutil/compare.go b/internal/ledgerutil/compare.go index a6e9804fd58..5dcbc28a8cb 100644 --- a/internal/ledgerutil/compare.go +++ b/internal/ledgerutil/compare.go @@ -26,28 +26,31 @@ import ( ) const ( - // AllDiffsByKey - Filename for the json output that contains all differences ordered by key - AllDiffsByKey = "all_diffs_by_key.json" + // AllPubDiffsByKey - Filename for the json output that contains all public differences ordered by key + AllPubDiffsByKey = "all_pub_diffs_by_key.json" + // AllPvtDiffsByKey - Filename for the json output that contains all private differences ordered by key + AllPvtDiffsByKey = "all_pvt_diffs_by_key.json" // FirstDiffsByHeight - Filename for the json output that contains the first n differences ordered by height FirstDiffsByHeight = "first_diffs_by_height.json" ) // Compare - Compares two ledger snapshots and outputs the differences in snapshot records // This function will throw an error if the output directory already exist in the outputDirLoc -// Function will return count of -1 if the public state hashes are the same +// Function will return count of -1 if the public state and private state hashes are the same func Compare(snapshotDir1 string, snapshotDir2 string, outputDirLoc string, firstDiffs int) (count int, outputDirPath string, err error) { - var records diffRecordSlice + // firstRecords - Slice of diffRecords that stores found differences based on block height, used to generate first n differences output file + firstRecords := &firstRecords{records: &diffRecordSlice{}, highestRecord: &diffRecord{}, highestIndex: 0, limit: firstDiffs} // Check the hashes between two files hashPath1 := filepath.Join(snapshotDir1, kvledger.SnapshotSignableMetadataFileName) hashPath2 := filepath.Join(snapshotDir2, kvledger.SnapshotSignableMetadataFileName) - equal, channelName, blockHeight, err := snapshotsComparable(hashPath1, hashPath2) + equalPub, equalPvt, channelName, blockHeight, err := hashesEqual(hashPath1, hashPath2) if err != nil { return 0, "", err } - // Snapshot hashes are the same - if equal { + // Snapshot public and private hashes are the same + if equalPub && equalPvt { return -1, "", nil } @@ -69,32 +72,81 @@ func Compare(snapshotDir1 string, snapshotDir2 string, outputDirLoc string, firs return 0, "", errors.Errorf("%s already exists in %s. Choose a different location or remove the existing results. Aborting compare", outputDirName, outputDirLoc) } - // Create the output files - allOutputFile, err := newJSONFileWriter(filepath.Join(outputDirPath, AllDiffsByKey)) - if err != nil { - return 0, "", err + // Generate all public data differences between snapshots + if !equalPub { + snapshotPubReader1, err := privacyenabledstate.NewSnapshotReader(snapshotDir1, + privacyenabledstate.PubStateDataFileName, privacyenabledstate.PubStateMetadataFileName) + if err != nil { + return 0, "", err + } + snapshotPubReader2, err := privacyenabledstate.NewSnapshotReader(snapshotDir2, + privacyenabledstate.PubStateDataFileName, privacyenabledstate.PubStateMetadataFileName) + if err != nil { + return 0, "", err + } + outputPubFileWriter, err := findAndWriteDifferences(outputDirPath, AllPubDiffsByKey, snapshotPubReader1, snapshotPubReader2, firstDiffs, firstRecords) + if err != nil { + return 0, "", err + } + count += outputPubFileWriter.count } - // Create snapshot readers to read both snapshots - snapshotReader1, err := privacyenabledstate.NewSnapshotReader(snapshotDir1, - privacyenabledstate.PubStateDataFileName, privacyenabledstate.PubStateMetadataFileName) - if err != nil { - return 0, "", err + // Generate all private data differences between snapshots + if !equalPvt { + snapshotPvtReader1, err := privacyenabledstate.NewSnapshotReader(snapshotDir1, + privacyenabledstate.PvtStateHashesFileName, privacyenabledstate.PvtStateHashesMetadataFileName) + if err != nil { + return 0, "", err + } + snapshotPvtReader2, err := privacyenabledstate.NewSnapshotReader(snapshotDir2, + privacyenabledstate.PvtStateHashesFileName, privacyenabledstate.PvtStateHashesMetadataFileName) + if err != nil { + return 0, "", err + } + outputPvtFileWriter, err := findAndWriteDifferences(outputDirPath, AllPvtDiffsByKey, snapshotPvtReader1, snapshotPvtReader2, firstDiffs, firstRecords) + if err != nil { + return 0, "", err + } + count += outputPvtFileWriter.count + } + + // Generate early differences output file + if firstDiffs != 0 { + firstDiffsOutputFileWriter, err := newJSONFileWriter(filepath.Join(outputDirPath, FirstDiffsByHeight)) + if err != nil { + return 0, "", err + } + sort.Sort(*firstRecords.records) + for _, r := range *firstRecords.records { + firstDiffsOutputFileWriter.addRecord(*r) + } + err = firstDiffsOutputFileWriter.close() + if err != nil { + return 0, "", err + } } - snapshotReader2, err := privacyenabledstate.NewSnapshotReader(snapshotDir2, - privacyenabledstate.PubStateDataFileName, privacyenabledstate.PubStateMetadataFileName) + + return count, outputDirPath, nil +} + +// Finds the differing records between two snapshot data files using SnapshotReaders and saves differences +// to an output file. Simultaneously, keep track of the first n differences. +func findAndWriteDifferences(outputDirPath string, outputFilename string, snapshotReader1 *privacyenabledstate.SnapshotReader, snapshotReader2 *privacyenabledstate.SnapshotReader, + firstDiffs int, firstRecords *firstRecords) (outputFileWriter *jsonArrayFileWriter, err error) { + // Create the output file + outputFileWriter, err = newJSONFileWriter(filepath.Join(outputDirPath, outputFilename)) if err != nil { - return 0, "", err + return nil, err } // Read each snapshot record to begin looking for differences namespace1, snapshotRecord1, err := snapshotReader1.Next() if err != nil { - return 0, "", err + return nil, err } namespace2, snapshotRecord2, err := snapshotReader2.Next() if err != nil { - return 0, "", err + return nil, err } // Main snapshot record comparison loop @@ -112,65 +164,65 @@ func Compare(snapshotDir1 string, snapshotDir2 string, outputDirLoc string, firs // Keys are the same but records are different diffRecord, err := newDiffRecord(namespace1, snapshotRecord1, snapshotRecord2) if err != nil { - return 0, "", err + return nil, err } // Add difference to output JSON file - err = allOutputFile.addRecord(*diffRecord) + err = outputFileWriter.addRecord(*diffRecord) if err != nil { - return 0, "", err + return nil, err } if firstDiffs != 0 { - records = append(records, diffRecord) + firstRecords.addRecord(diffRecord) } } // Advance both snapshot readers namespace1, snapshotRecord1, err = snapshotReader1.Next() if err != nil { - return 0, "", err + return nil, err } namespace2, snapshotRecord2, err = snapshotReader2.Next() if err != nil { - return 0, "", err + return nil, err } case 1: // Key 1 is bigger, snapshot 1 is missing a record // Snapshot 2 has the missing record, add missing to output JSON file diffRecord, err := newDiffRecord(namespace2, nil, snapshotRecord2) if err != nil { - return 0, "", err + return nil, err } // Add missing record to output JSON file - err = allOutputFile.addRecord(*diffRecord) + err = outputFileWriter.addRecord(*diffRecord) if err != nil { - return 0, "", err + return nil, err } if firstDiffs != 0 { - records = append(records, diffRecord) + firstRecords.addRecord(diffRecord) } // Advance the second snapshot reader namespace2, snapshotRecord2, err = snapshotReader2.Next() if err != nil { - return 0, "", err + return nil, err } case -1: // Key 2 is bigger, snapshot 2 is missing a record // Snapshot 1 has the missing record, add missing to output JSON file diffRecord, err := newDiffRecord(namespace1, snapshotRecord1, nil) if err != nil { - return 0, "", err + return nil, err } // Add missing record to output JSON file - err = allOutputFile.addRecord(*diffRecord) + err = outputFileWriter.addRecord(*diffRecord) if err != nil { - return 0, "", err + return nil, err } if firstDiffs != 0 { - records = append(records, diffRecord) + firstRecords.addRecord(diffRecord) } // Advance the first snapshot reader namespace1, snapshotRecord1, err = snapshotReader1.Next() if err != nil { - return 0, "", err + return nil, err } default: @@ -186,18 +238,18 @@ func Compare(snapshotDir1 string, snapshotDir2 string, outputDirLoc string, firs // Add missing to output JSON file diffRecord, err := newDiffRecord(namespace1, snapshotRecord1, nil) if err != nil { - return 0, "", err + return nil, err } - err = allOutputFile.addRecord(*diffRecord) + err = outputFileWriter.addRecord(*diffRecord) if err != nil { - return 0, "", err + return nil, err } if firstDiffs != 0 { - records = append(records, diffRecord) + firstRecords.addRecord(diffRecord) } namespace1, snapshotRecord1, err = snapshotReader1.Next() if err != nil { - return 0, "", err + return nil, err } } @@ -206,44 +258,62 @@ func Compare(snapshotDir1 string, snapshotDir2 string, outputDirLoc string, firs // Add missing to output JSON file diffRecord, err := newDiffRecord(namespace2, nil, snapshotRecord2) if err != nil { - return 0, "", err + return nil, err } - err = allOutputFile.addRecord(*diffRecord) + err = outputFileWriter.addRecord(*diffRecord) if err != nil { - return 0, "", err + return nil, err } if firstDiffs != 0 { - records = append(records, diffRecord) + firstRecords.addRecord(diffRecord) } namespace2, snapshotRecord2, err = snapshotReader2.Next() if err != nil { - return 0, "", err + return nil, err } } } - err = allOutputFile.close() + err = outputFileWriter.close() if err != nil { - return 0, "", err + return nil, err } - // Create early differences output file - if firstDiffs != 0 { - firstDiffsOutputFile, err := newJSONFileWriter(filepath.Join(outputDirPath, FirstDiffsByHeight)) - if err != nil { - return 0, "", err - } - sort.Sort(records) - for i := 0; i < firstDiffs && i < len(records); i++ { - firstDiffsOutputFile.addRecord(records[i]) - } - err = firstDiffsOutputFile.close() - if err != nil { - return 0, "", err + return outputFileWriter, nil +} + +// firstRecords is a struct used to hold only the earliest records up to a given limit +type firstRecords struct { + records *diffRecordSlice + highestRecord *diffRecord + highestIndex int + limit int +} + +func (s *firstRecords) addRecord(r *diffRecord) { + if len(*s.records) < s.limit { + *s.records = append(*s.records, r) + s.setHighestRecord() + } else { + if r.lessThan(s.highestRecord) { + (*s.records)[s.highestIndex] = r + s.setHighestRecord() } } +} - return allOutputFile.count, outputDirPath, nil +func (s *firstRecords) setHighestRecord() { + if len(*s.records) == 1 { + s.highestRecord = (*s.records)[0] + s.highestIndex = 0 + return + } + for i, r := range *s.records { + if s.highestRecord.lessThan(r) { + s.highestRecord = r + s.highestIndex = i + } + } } type diffRecordSlice []*diffRecord @@ -257,13 +327,7 @@ func (s diffRecordSlice) Swap(i, j int) { } func (s diffRecordSlice) Less(i, j int) bool { - iBlockNum, iTxNum := s[i].getHeight() - jBlockNum, jTxNum := s[j].getHeight() - - if iBlockNum == jBlockNum { - return iTxNum <= jTxNum - } - return iBlockNum < jBlockNum + return (s[i]).lessThan(s[j]) } // diffRecord represents a diverging record in json @@ -307,11 +371,22 @@ func newDiffRecord(namespace string, record1 *privacyenabledstate.SnapshotRecord } // Get height from a diffRecord -func (d diffRecord) getHeight() (blockNum uint64, txNum uint64) { +func (d *diffRecord) getHeight() (blockNum uint64, txNum uint64) { r := earlierRecord(d.Record1, d.Record2) return r.BlockNum, r.TxNum } +// Returns true if d is an earlier diffRecord than e +func (d *diffRecord) lessThan(e *diffRecord) bool { + dBlockNum, dTxNum := d.getHeight() + eBlockNum, eTxNum := e.getHeight() + + if dBlockNum == eBlockNum { + return dTxNum <= eTxNum + } + return dBlockNum < eBlockNum +} + // snapshotRecord represents the data of a snapshot record in json type snapshotRecord struct { Value string `json:"value"` @@ -319,6 +394,7 @@ type snapshotRecord struct { TxNum uint64 `json:"txNum"` } +// Returns the snapshotRecord with the earlier height func earlierRecord(r1 *snapshotRecord, r2 *snapshotRecord) *snapshotRecord { if r1 == nil { return r2 @@ -405,47 +481,58 @@ func readMetadata(fpath string) (*kvledger.SnapshotSignableMetadata, error) { return &mdata, nil } -// Compares hashes of snapshots -func snapshotsComparable(fpath1 string, fpath2 string) (bool, string, uint64, error) { +// Compares hashes of snapshots to determine if they can be compared, then returns channel name and block height for the output directory name +// Return values: +// equalPub - True if snapshot public data hashes are the same, false otherwise. If true, public differences will not be generated. +// equalPvt - True if snapshot private data hashes are the same, false otherwise. If true, private differences will not be generated. +// chName - Channel name shared between snapshots, used to name output directory. If channel names are not the same, no comparison is made. +// lastBN - Block height shared between snapshots, used to name output directory. If block heights are not the same, no comparison is made. +func hashesEqual(fpath1 string, fpath2 string) (equalPub bool, equalPvt bool, chName string, lastBN uint64, err error) { var mdata1, mdata2 *kvledger.SnapshotSignableMetadata // Read metadata from snapshot metadata filepaths - mdata1, err := readMetadata(fpath1) + mdata1, err = readMetadata(fpath1) if err != nil { - return false, "", 0, err + return false, false, "", 0, err } mdata2, err = readMetadata(fpath2) if err != nil { - return false, "", 0, err + return false, false, "", 0, err } if mdata1.ChannelName != mdata2.ChannelName { - return false, "", 0, errors.Errorf("the supplied snapshots appear to be non-comparable. Channel names do not match."+ + return false, false, "", 0, errors.Errorf("the supplied snapshots appear to be non-comparable. Channel names do not match."+ "\nSnapshot1 channel name: %s\nSnapshot2 channel name: %s", mdata1.ChannelName, mdata2.ChannelName) } if mdata1.LastBlockNumber != mdata2.LastBlockNumber { - return false, "", 0, errors.Errorf("the supplied snapshots appear to be non-comparable. Last block numbers do not match."+ + return false, false, "", 0, errors.Errorf("the supplied snapshots appear to be non-comparable. Last block numbers do not match."+ "\nSnapshot1 last block number: %v\nSnapshot2 last block number: %v", mdata1.LastBlockNumber, mdata2.LastBlockNumber) } if mdata1.LastBlockHashInHex != mdata2.LastBlockHashInHex { - return false, "", 0, errors.Errorf("the supplied snapshots appear to be non-comparable. Last block hashes do not match."+ + return false, false, "", 0, errors.Errorf("the supplied snapshots appear to be non-comparable. Last block hashes do not match."+ "\nSnapshot1 last block hash: %s\nSnapshot2 last block hash: %s", mdata1.LastBlockHashInHex, mdata2.LastBlockHashInHex) } if mdata1.StateDBType != mdata2.StateDBType { - return false, "", 0, errors.Errorf("the supplied snapshots appear to be non-comparable. State db types do not match."+ + return false, false, "", 0, errors.Errorf("the supplied snapshots appear to be non-comparable. State db types do not match."+ "\nSnapshot1 state db type: %s\nSnapshot2 state db type: %s", mdata1.StateDBType, mdata2.StateDBType) } pubDataHash1 := mdata1.FilesAndHashes[privacyenabledstate.PubStateDataFileName] pubMdataHash1 := mdata1.FilesAndHashes[privacyenabledstate.PubStateMetadataFileName] + pvtDataHash1 := mdata1.FilesAndHashes[privacyenabledstate.PvtStateHashesFileName] + pvtMdataHash1 := mdata1.FilesAndHashes[privacyenabledstate.PvtStateHashesMetadataFileName] pubDataHash2 := mdata2.FilesAndHashes[privacyenabledstate.PubStateDataFileName] pubMdataHash2 := mdata2.FilesAndHashes[privacyenabledstate.PubStateMetadataFileName] + pvtDataHash2 := mdata2.FilesAndHashes[privacyenabledstate.PvtStateHashesFileName] + pvtMdataHash2 := mdata2.FilesAndHashes[privacyenabledstate.PvtStateHashesMetadataFileName] - return (pubDataHash1 == pubDataHash2 && pubMdataHash1 == pubMdataHash2), mdata1.ChannelName, mdata1.LastBlockNumber, nil + equalPub = pubDataHash1 == pubDataHash2 && pubMdataHash1 == pubMdataHash2 + equalPvt = pvtDataHash1 == pvtDataHash2 && pvtMdataHash1 == pvtMdataHash2 + return equalPub, equalPvt, mdata1.ChannelName, mdata1.LastBlockNumber, nil } // jsonArrayFileWriter writes a list of diffRecords to a json file diff --git a/internal/ledgerutil/compare_test.go b/internal/ledgerutil/compare_test.go index b88d575ae0a..ff3e94c5656 100644 --- a/internal/ledgerutil/compare_test.go +++ b/internal/ledgerutil/compare_test.go @@ -121,6 +121,20 @@ func TestCompare(t *testing.T) { }, } + samplePvtRecords1 := []*testRecord{ + { + namespace: "_lifecycle$$h_implicit_org_Org1MSP", key: "sk1", value: "#!", + blockNum: 1, txNum: 1, metadata: "md1", + }, + } + + samplePvtRecords2 := []*testRecord{ + { + namespace: "_lifecycle$$h_implicit_org_Org1MSP", key: "sk1", value: "$^", + blockNum: 1, txNum: 1, metadata: "md1", + }, + } + // Signable metadata samples for snapshots sampleSignableMetadata1 := &kvledger.SnapshotSignableMetadata{ ChannelName: "testchannel", @@ -144,8 +158,8 @@ func TestCompare(t *testing.T) { LastBlockHashInHex: "last_block_hash", PreviousBlockHashInHex: "previous_block_hash", FilesAndHashes: map[string]string{ - "private_state_hashes.data": "private_state_hash2", - "private_state_hashes.metadata": "private_state_hash2", + "private_state_hashes.data": "private_state_hash1", + "private_state_hashes.metadata": "private_state_hash1", "public_state.data": "public_state_hash2", "public_state.metadata": "public_state_hash2", "txids.data": "txids_hash2", @@ -170,6 +184,38 @@ func TestCompare(t *testing.T) { StateDBType: "testdatabase2", } + sampleSignableMetadata4 := &kvledger.SnapshotSignableMetadata{ + ChannelName: "testchannel", + LastBlockNumber: sampleRecords1[len(sampleRecords1)-1].blockNum, + LastBlockHashInHex: "last_block_hash", + PreviousBlockHashInHex: "previous_block_hash", + FilesAndHashes: map[string]string{ + "private_state_hashes.data": "private_state_hash2", + "private_state_hashes.metadata": "private_state_hash2", + "public_state.data": "public_state_hash2", + "public_state.metadata": "public_state_hash2", + "txids.data": "txids_hash2", + "txids.metadata": "txids_hash2", + }, + StateDBType: "testdatabase", + } + + sampleSignableMetadata5 := &kvledger.SnapshotSignableMetadata{ + ChannelName: "testchannel", + LastBlockNumber: sampleRecords1[len(sampleRecords1)-1].blockNum, + LastBlockHashInHex: "last_block_hash", + PreviousBlockHashInHex: "previous_block_hash", + FilesAndHashes: map[string]string{ + "private_state_hashes.data": "private_state_hash2", + "private_state_hashes.metadata": "private_state_hash2", + "public_state.data": "public_state_hash1", + "public_state.metadata": "public_state_hash1", + "txids.data": "txids_hash2", + "txids.metadata": "txids_hash2", + }, + StateDBType: "testdatabase", + } + // Expected outputs expectedDifferenceResult := `[ { @@ -293,7 +339,7 @@ func TestCompare(t *testing.T) { } } ]` - expectedAllDiffs := `[ + expectedAllPubDiffs := `[ { "namespace" : "ns1", "key" : "k2", @@ -343,16 +389,37 @@ func TestCompare(t *testing.T) { } } ]` + expectedPvtDiffResult := `[ + { + "namespace" : "_lifecycle$$h_implicit_org_Org1MSP", + "key" : "sk1", + "snapshotrecord1" : { + "value" : "#!", + "blockNum" : 1, + "txNum" : 1 + }, + "snapshotrecord2" : { + "value" : "$^", + "blockNum" : 1, + "txNum" : 1 + } + } + ]` + expectedEmpty := "null" testCases := map[string]struct { inputTestRecords1 []*testRecord + inputTestPvtRecords1 []*testRecord inputSignableMetadata1 *kvledger.SnapshotSignableMetadata inputTestRecords2 []*testRecord + inputTestPvtRecords2 []*testRecord inputSignableMetadata2 *kvledger.SnapshotSignableMetadata - expectedAllOutput string - expectedFirstOutput string - firstN int expectedOutputType string + expectedPubOutput string + expectedPvtOutput string + expectedFirOutput string + expectedError string + firstN int expectedDiffCount int }{ // Snapshots have a single difference in record @@ -361,8 +428,10 @@ func TestCompare(t *testing.T) { inputSignableMetadata1: sampleSignableMetadata1, inputTestRecords2: sampleRecords2, inputSignableMetadata2: sampleSignableMetadata2, - expectedAllOutput: expectedDifferenceResult, expectedOutputType: "json", + expectedPubOutput: expectedDifferenceResult, + expectedPvtOutput: expectedEmpty, + expectedFirOutput: expectedEmpty, expectedDiffCount: 1, }, // Second snapshot is missing a record @@ -371,8 +440,10 @@ func TestCompare(t *testing.T) { inputSignableMetadata1: sampleSignableMetadata1, inputTestRecords2: sampleRecords3, inputSignableMetadata2: sampleSignableMetadata2, - expectedAllOutput: expectedMissingResult1, expectedOutputType: "json", + expectedPubOutput: expectedMissingResult1, + expectedPvtOutput: expectedEmpty, + expectedFirOutput: expectedEmpty, expectedDiffCount: 1, }, // First snapshot is missing a record @@ -381,8 +452,10 @@ func TestCompare(t *testing.T) { inputSignableMetadata1: sampleSignableMetadata2, inputTestRecords2: sampleRecords1, inputSignableMetadata2: sampleSignableMetadata1, - expectedAllOutput: expectedMissingResult2, expectedOutputType: "json", + expectedPubOutput: expectedMissingResult2, + expectedPvtOutput: expectedEmpty, + expectedFirOutput: expectedEmpty, expectedDiffCount: 1, }, // Second snapshot is missing tailing records @@ -391,8 +464,10 @@ func TestCompare(t *testing.T) { inputSignableMetadata1: sampleSignableMetadata1, inputTestRecords2: sampleRecords4, inputSignableMetadata2: sampleSignableMetadata2, - expectedAllOutput: expectedMissingTailResult1, expectedOutputType: "json", + expectedPubOutput: expectedMissingTailResult1, + expectedPvtOutput: expectedEmpty, + expectedFirOutput: expectedEmpty, expectedDiffCount: 2, }, // First snapshot is missing tailing records @@ -401,8 +476,10 @@ func TestCompare(t *testing.T) { inputSignableMetadata1: sampleSignableMetadata2, inputTestRecords2: sampleRecords1, inputSignableMetadata2: sampleSignableMetadata1, - expectedAllOutput: expectedMissingTailResult2, expectedOutputType: "json", + expectedPubOutput: expectedMissingTailResult2, + expectedPvtOutput: expectedEmpty, + expectedFirOutput: expectedEmpty, expectedDiffCount: 2, }, // Snapshots contain the same public state hashes @@ -411,7 +488,6 @@ func TestCompare(t *testing.T) { inputSignableMetadata1: sampleSignableMetadata1, inputTestRecords2: sampleRecords1, inputSignableMetadata2: sampleSignableMetadata1, - expectedAllOutput: "", expectedOutputType: "same-hash", expectedDiffCount: -1, }, @@ -421,8 +497,8 @@ func TestCompare(t *testing.T) { inputSignableMetadata1: sampleSignableMetadata1, inputTestRecords2: sampleRecords2, inputSignableMetadata2: sampleSignableMetadata3, - expectedAllOutput: expectedDiffDatabaseError, expectedOutputType: "error", + expectedError: expectedDiffDatabaseError, expectedDiffCount: 0, }, // Output directory file already exists @@ -440,12 +516,41 @@ func TestCompare(t *testing.T) { inputSignableMetadata1: sampleSignableMetadata1, inputTestRecords2: sampleRecords5, inputSignableMetadata2: sampleSignableMetadata2, - expectedAllOutput: expectedAllDiffs, - expectedFirstOutput: expectedFirstDiffs, - firstN: 3, expectedOutputType: "json", + expectedPubOutput: expectedAllPubDiffs, + expectedPvtOutput: expectedEmpty, + expectedFirOutput: expectedFirstDiffs, + firstN: 3, expectedDiffCount: 4, }, + // Snapshots have a single difference public difference and a single private difference + "pub-and-pvt-difference": { + inputTestRecords1: sampleRecords1, + inputTestPvtRecords1: samplePvtRecords1, + inputSignableMetadata1: sampleSignableMetadata1, + inputTestRecords2: sampleRecords2, + inputTestPvtRecords2: samplePvtRecords2, + inputSignableMetadata2: sampleSignableMetadata4, + expectedOutputType: "json", + expectedPubOutput: expectedDifferenceResult, + expectedPvtOutput: expectedPvtDiffResult, + expectedFirOutput: expectedEmpty, + expectedDiffCount: 2, + }, + // Snapshots have a single private difference only + "pvt-difference-only": { + inputTestRecords1: sampleRecords1, + inputTestPvtRecords1: samplePvtRecords1, + inputSignableMetadata1: sampleSignableMetadata1, + inputTestRecords2: sampleRecords1, + inputTestPvtRecords2: samplePvtRecords2, + inputSignableMetadata2: sampleSignableMetadata5, + expectedOutputType: "json", + expectedPubOutput: expectedEmpty, + expectedPvtOutput: expectedPvtDiffResult, + expectedFirOutput: expectedEmpty, + expectedDiffCount: 1, + }, } // Run test cases individually @@ -463,30 +568,28 @@ func TestCompare(t *testing.T) { defer os.RemoveAll(resultsDir) // Populate temporary directories with sample snapshot data - err = createSnapshot(snapshotDir1, testCase.inputTestRecords1, testCase.inputSignableMetadata1) + err = createSnapshot(snapshotDir1, testCase.inputTestRecords1, testCase.inputTestPvtRecords1, testCase.inputSignableMetadata1) require.NoError(t, err) - err = createSnapshot(snapshotDir2, testCase.inputTestRecords2, testCase.inputSignableMetadata2) + err = createSnapshot(snapshotDir2, testCase.inputTestRecords2, testCase.inputTestPvtRecords2, testCase.inputSignableMetadata2) require.NoError(t, err) // Compare snapshots and check the output - count, allOut, firstOut, err := compareSnapshots(snapshotDir1, snapshotDir2, resultsDir, testCase.firstN) + count, pubOut, pvtOut, firOut, err := compareSnapshots(snapshotDir1, snapshotDir2, resultsDir, testCase.firstN) switch testCase.expectedOutputType { case "error": require.Equal(t, testCase.expectedDiffCount, count) - require.ErrorContains(t, err, testCase.expectedAllOutput) + require.ErrorContains(t, err, testCase.expectedError) case "exists-error": require.NoError(t, err) - count, _, _, err = compareSnapshots(snapshotDir1, snapshotDir2, resultsDir, testCase.firstN) + count, _, _, _, err = compareSnapshots(snapshotDir1, snapshotDir2, resultsDir, testCase.firstN) require.Equal(t, testCase.expectedDiffCount, count) require.ErrorContains(t, err, "testchannel_2_comparison already exists in "+resultsDir+". Choose a different location or remove the existing results. Aborting compare") case "json": require.Equal(t, testCase.expectedDiffCount, count) require.NoError(t, err) - require.JSONEq(t, testCase.expectedAllOutput, allOut) - if firstOut != "" { - require.JSONEq(t, testCase.expectedFirstOutput, firstOut) - } - + require.JSONEq(t, testCase.expectedPubOutput, pubOut) + require.JSONEq(t, testCase.expectedPvtOutput, pvtOut) + require.JSONEq(t, testCase.expectedFirOutput, firOut) case "same-hash": require.Equal(t, testCase.expectedDiffCount, count) require.NoError(t, err) @@ -498,7 +601,8 @@ func TestCompare(t *testing.T) { } // createSnapshot generates a sample snapshot based on the passed in records and metadata -func createSnapshot(dir string, pubStateRecords []*testRecord, signableMetadata *kvledger.SnapshotSignableMetadata) error { +func createSnapshot(dir string, pubStateRecords []*testRecord, pvtStateRecords []*testRecord, + signableMetadata *kvledger.SnapshotSignableMetadata) error { // Generate public state of sample snapshot pubStateWriter, err := privacyenabledstate.NewSnapshotWriter( dir, @@ -529,6 +633,36 @@ func createSnapshot(dir string, pubStateRecords []*testRecord, signableMetadata return err } + // Generate private state of sample snapshot + pvtStateWriter, err := privacyenabledstate.NewSnapshotWriter( + dir, + privacyenabledstate.PvtStateHashesFileName, + privacyenabledstate.PvtStateHashesMetadataFileName, + testNewHashFunc, + ) + if err != nil { + return err + } + defer pvtStateWriter.Close() + + // Add sample records to sample snapshot + for _, sample := range pvtStateRecords { + err = pvtStateWriter.AddData(sample.namespace, &privacyenabledstate.SnapshotRecord{ + Key: []byte(sample.key), + Value: []byte(sample.value), + Metadata: []byte(sample.metadata), + Version: toBytes(sample.blockNum, sample.txNum), + }) + if err != nil { + return err + } + } + + _, _, err = pvtStateWriter.Done() + if err != nil { + return err + } + // Generate the signable metadata files for sample snapshot signableMetadataBytes, err := signableMetadata.ToJSON() if err != nil { @@ -544,35 +678,59 @@ func createSnapshot(dir string, pubStateRecords []*testRecord, signableMetadata } // compareSnapshots calls the Compare tool and extracts the result json -func compareSnapshots(ss1 string, ss2 string, res string, firstN int) (int, string, string, error) { +func compareSnapshots(ss1 string, ss2 string, res string, firstN int) (int, string, string, string, error) { // Run compare tool on snapshots count, opath, err := Compare(ss1, ss2, res, firstN) if err != nil || count == -1 { - return count, "", "", err + return count, "", "", "", err } - // Read results of output - allBytes, err := ioutil.ReadFile(filepath.Join(opath, AllDiffsByKey)) + // Get json as a string from generated output files + pubStr, err := outputFileToString(AllPubDiffsByKey, opath) + if err != nil { + return 0, "", "", "", err + } + pvtStr, err := outputFileToString(AllPvtDiffsByKey, opath) if err != nil { - return 0, "", "", err + return 0, "", "", "", err } - allOut, err := ioutil.ReadAll(bytes.NewReader(allBytes)) + firStr, err := outputFileToString(FirstDiffsByHeight, opath) if err != nil { - return 0, "", "", err + return 0, "", "", "", err } - if firstN != 0 { - firstBytes, err := ioutil.ReadFile(filepath.Join(opath, FirstDiffsByHeight)) + + return count, pubStr, pvtStr, firStr, nil +} + +func outputFileToString(f string, path string) (string, error) { + fpath := filepath.Join(path, f) + exists, err := outputFileExists(fpath) + if err != nil { + return "", err + } + if exists { + b, err := ioutil.ReadFile(fpath) if err != nil { - return 0, "", "", err + return "", err } - firstOut, err := ioutil.ReadAll(bytes.NewReader(firstBytes)) + out, err := ioutil.ReadAll(bytes.NewReader(b)) if err != nil { - return 0, "", "", err + return "", err } - return count, string(allOut), string(firstOut), nil + return string(out), nil } + return "null", nil +} - return count, string(allOut), "", nil +func outputFileExists(f string) (bool, error) { + _, err := os.Stat(f) + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, err + } + return true, nil } // toBytes serializes the Height