Compare Snapshots Utility

FAB-18425

Signed-off-by: Julian Castrence <juliancastrence@ibm.com>

Makefile

@@ -28,11 +28,13 @@
 #   - idemixgen - builds a native idemixgen binary
 #   - integration-test-prereqs - setup prerequisites for integration tests
 #   - integration-test - runs the integration tests
+#   - ledger - builds a native fabric ledger troubleshooting binary
 #   - license - checks go source files for Apache license header
 #   - linter - runs all code checks
 #   - native - ensures all native binaries are available
 #   - orderer - builds a native fabric orderer binary
 #   - orderer-docker[-clean] - ensures the orderer container is available[/cleaned]
+#   - osnadmin - builds a native fabric osnadmin binary
 #   - peer - builds a native fabric peer binary
 #   - peer-docker[-clean] - ensures the peer container is available[/cleaned]
 #   - profile - runs unit tests for all packages in coverprofile mode (slow)
@@ -83,13 +85,14 @@ GO_TAGS ?=
 RELEASE_EXES = orderer $(TOOLS_EXES)
 RELEASE_IMAGES = baseos ccenv orderer peer tools
 RELEASE_PLATFORMS = darwin-amd64 linux-amd64 windows-amd64
-TOOLS_EXES = configtxgen configtxlator cryptogen discover idemixgen osnadmin peer
+TOOLS_EXES = configtxgen configtxlator cryptogen discover idemixgen ledger osnadmin peer
 
 pkgmap.configtxgen    := $(PKGNAME)/cmd/configtxgen
 pkgmap.configtxlator  := $(PKGNAME)/cmd/configtxlator
 pkgmap.cryptogen      := $(PKGNAME)/cmd/cryptogen
 pkgmap.discover       := $(PKGNAME)/cmd/discover
 pkgmap.idemixgen      := $(PKGNAME)/cmd/idemixgen
+pkgmap.ledger		  := $(PKGNAME)/cmd/ledger
 pkgmap.orderer        := $(PKGNAME)/cmd/orderer
 pkgmap.osnadmin       := $(PKGNAME)/cmd/osnadmin
 pkgmap.peer           := $(PKGNAME)/cmd/peer

cmd/ledger/main.go

@@ -0,0 +1,58 @@
+/*
+Copyright IBM Corp. All Rights Reserved.
+
+SPDX-License-Identifier: Apache-2.0
+*/
+
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/hyperledger/fabric/internal/ledger"
+	"gopkg.in/alecthomas/kingpin.v2"
+)
+
+const (
+	resultFilename = "./result.json"
+)
+
+var (
+	app = kingpin.New("ledger", "Ledger Utility Tool")
+
+	compare       = app.Command("compare", "Compare two ledgers via their snapshots.")
+	snapshotPath1 = compare.Arg("snapshotPath1", "File path to first ledger snapshot.").Required().String()
+	snapshotPath2 = compare.Arg("snapshotPath2", "File path to second ledger snapshot.").Required().String()
+
+	troubleshoot = app.Command("troubleshoot", "Identify potentially divergent transactions.")
+
+	args = os.Args[1:]
+)
+
+func main() {
+	kingpin.Version("0.0.1")
+
+	command, err := app.Parse(args)
+	if err != nil {
+		kingpin.Fatalf("parsing arguments: %s. Try --help", err)
+		return
+	}
+
+	switch command {
+
+	case compare.FullCommand():
+
+		count, err := ledger.Compare(*snapshotPath1, *snapshotPath2, resultFilename)
+		if err != nil {
+			fmt.Println(err)
+			return
+		}
+		fmt.Printf("\nSuccessfully compared snapshots. Result saved to %s. Total differences found: %v\n", resultFilename, count)
+
+	case troubleshoot.FullCommand():
+
+		fmt.Println("Command TBD")
+
+	}
+}

cmd/ledger/main_test.go

@@ -0,0 +1,59 @@
+/*
+Copyright IBM Corp. All Rights Reserved.
+
+SPDX-License-Identifier: Apache-2.0
+*/
+
+package main
+
+import (
+	"os/exec"
+	"testing"
+	"time"
+
+	"github.com/onsi/gomega"
+	"github.com/onsi/gomega/gexec"
+)
+
+func TestArguments(t *testing.T) {
+	testCases := map[string]struct {
+		exitCode int
+		args     []string
+	}{
+		"ledger": {
+			exitCode: 0,
+			args:     []string{},
+		},
+		"ledger-help": {
+			exitCode: 0,
+			args:     []string{"--help"},
+		},
+		"compare-help": {
+			exitCode: 0,
+			args:     []string{"compare", "--help"},
+		},
+		"compare": {
+			exitCode: 1,
+			args:     []string{"compare"},
+		},
+		"one-snapshot": {
+			exitCode: 1,
+			args:     []string{"compare, snapshotDir1"},
+		},
+	}
+
+	// Build ledger binary
+	gt := gomega.NewWithT(t)
+	ledger, err := gexec.Build("github.com/hyperledger/fabric/cmd/ledger")
+	gt.Expect(err).NotTo(gomega.HaveOccurred())
+	defer gexec.CleanupBuildArtifacts()
+
+	for testName, testCase := range testCases {
+		t.Run(testName, func(t *testing.T) {
+			cmd := exec.Command(ledger, testCase.args...)
+			session, err := gexec.Start(cmd, nil, nil)
+			gt.Expect(err).NotTo(gomega.HaveOccurred())
+			gt.Eventually(session, 5*time.Second).Should(gexec.Exit(testCase.exitCode))
+		})
+	}
+}

common/ledger/snapshot/file.go

@@ -119,7 +119,7 @@ func (c *FileWriter) Close() error {
 // FileReader reads from a ledger snapshot file. This is expected to be used for loading the ledger snapshot data
 // during bootstrapping a channel from snapshot. The data should be read, using the functions `DecodeXXX`,
 // in the same sequence in which the data was written by the functions `EncodeXXX` in the `FileCreator`.
-// Note that the FileReader does not verifies the hash of stream and it is expected that the hash has been verified
+// Note that the FileReader does not verify the hash of stream and it is expected that the hash has been verified
 // by the consumer. Later, if we decide to perform this, on-the-side, while loading the snapshot data, the FileRedear,
 // like the FileCreator, would take a `hasher` as an input
 type FileReader struct {

core/ledger/kvledger/kv_ledger.go

@@ -47,7 +47,7 @@ var (
 // This implementation provides a key-value based data model
 type kvLedger struct {
 	ledgerID               string
-	bootSnapshotMetadata   *snapshotMetadata
+	bootSnapshotMetadata   *SnapshotMetadata
 	blockStore             *blkstorage.BlockStore
 	pvtdataStore           *pvtdatastorage.Store
 	txmgr                  *txmgr.LockBasedTxMgr
@@ -72,7 +72,7 @@ type kvLedger struct {
 type lgrInitializer struct {
 	ledgerID                 string
 	initializingFromSnapshot bool
-	bootSnapshotMetadata     *snapshotMetadata
+	bootSnapshotMetadata     *SnapshotMetadata
 	blockStore               *blkstorage.BlockStore
 	pvtdataStore             *pvtdatastorage.Store
 	stateDB                  *privacyenabledstate.DB

core/ledger/kvledger/kv_ledger_provider.go

@@ -330,7 +330,7 @@ func (p *Provider) Open(ledgerID string) (ledger.PeerLedger, error) {
 	return p.open(ledgerID, bootSnapshotMetadata, false)
 }
 
-func (p *Provider) open(ledgerID string, bootSnapshotMetadata *snapshotMetadata, initializingFromSnapshot bool) (ledger.PeerLedger, error) {
+func (p *Provider) open(ledgerID string, bootSnapshotMetadata *SnapshotMetadata, initializingFromSnapshot bool) (ledger.PeerLedger, error) {
 	// Get the block store for a chain/ledger
 	blockStore, err := p.blkStoreProvider.Open(ledgerID)
 	if err != nil {
@@ -479,17 +479,17 @@ func (p *Provider) runCleanup(ledgerID string) error {
 	return p.idStore.deleteLedgerID(ledgerID)
 }
 
-func snapshotMetadataFromProto(p *msgs.BootSnapshotMetadata) (*snapshotMetadata, error) {
+func snapshotMetadataFromProto(p *msgs.BootSnapshotMetadata) (*SnapshotMetadata, error) {
 	if p == nil {
 		return nil, nil
 	}
 
-	m := &snapshotMetadataJSONs{
+	m := &SnapshotMetadataJSONs{
 		signableMetadata:   p.SingableMetadata,
 		additionalMetadata: p.AdditionalMetadata,
 	}
 
-	return m.toMetadata()
+	return m.ToMetadata()
 }
 
 //////////////////////////////////////////////////////////////////////

core/ledger/kvledger/snapshot.go

@@ -32,16 +32,16 @@ import (
 )
 
 const (
-	snapshotSignableMetadataFileName   = "_snapshot_signable_metadata.json"
+	SnapshotSignableMetadataFileName   = "_snapshot_signable_metadata.json"
 	snapshotAdditionalMetadataFileName = "_snapshot_additional_metadata.json"
 	jsonFileIndent                     = "    "
 	simpleKeyValueDB                   = "SimpleKeyValueDB"
 )
 
-// snapshotSignableMetadata is used to build a JSON that represents a unique snapshot and
+// SnapshotSignableMetadata is used to build a JSON that represents a unique snapshot and
 // can be signed by the peer. Hashsum of the resultant JSON is intended to be used as a single
 // hash of the snapshot, if need be.
-type snapshotSignableMetadata struct {
+type SnapshotSignableMetadata struct {
 	ChannelName            string            `json:"channel_name"`
 	LastBlockNumber        uint64            `json:"last_block_number"`
 	LastBlockHashInHex     string            `json:"last_block_hash"`
@@ -50,7 +50,7 @@ type snapshotSignableMetadata struct {
 	StateDBType            string            `json:"state_db_type"`
 }
 
-func (m *snapshotSignableMetadata) toJSON() ([]byte, error) {
+func (m *SnapshotSignableMetadata) ToJSON() ([]byte, error) {
 	return json.MarshalIndent(m, "", jsonFileIndent)
 }
 
@@ -59,22 +59,22 @@ type snapshotAdditionalMetadata struct {
 	LastBlockCommitHashInHex string `json:"last_block_commit_hash"`
 }
 
-func (m *snapshotAdditionalMetadata) toJSON() ([]byte, error) {
+func (m *snapshotAdditionalMetadata) ToJSON() ([]byte, error) {
 	return json.MarshalIndent(m, "", jsonFileIndent)
 }
 
-type snapshotMetadata struct {
-	*snapshotSignableMetadata
+type SnapshotMetadata struct {
+	*SnapshotSignableMetadata
 	*snapshotAdditionalMetadata
 }
 
-type snapshotMetadataJSONs struct {
+type SnapshotMetadataJSONs struct {
 	signableMetadata   string
 	additionalMetadata string
 }
 
-func (j *snapshotMetadataJSONs) toMetadata() (*snapshotMetadata, error) {
-	metadata := &snapshotSignableMetadata{}
+func (j *SnapshotMetadataJSONs) ToMetadata() (*SnapshotMetadata, error) {
+	metadata := &SnapshotSignableMetadata{}
 	if err := json.Unmarshal([]byte(j.signableMetadata), metadata); err != nil {
 		return nil, errors.Wrap(err, "error while unmarshalling signable metadata")
 	}
@@ -83,8 +83,8 @@ func (j *snapshotMetadataJSONs) toMetadata() (*snapshotMetadata, error) {
 	if err := json.Unmarshal([]byte(j.additionalMetadata), additionalMetadata); err != nil {
 		return nil, errors.Wrap(err, "error while unmarshalling additional metadata")
 	}
-	return &snapshotMetadata{
-		snapshotSignableMetadata:   metadata,
+	return &SnapshotMetadata{
+		SnapshotSignableMetadata:   metadata,
 		snapshotAdditionalMetadata: additionalMetadata,
 	}, nil
 }
@@ -179,7 +179,7 @@ func (l *kvLedger) generateSnapshotMetadataFiles(
 	if stateDBType != ledger.CouchDB {
 		stateDBType = simpleKeyValueDB
 	}
-	signableMetadata := &snapshotSignableMetadata{
+	signableMetadata := &SnapshotSignableMetadata{
 		ChannelName:            l.ledgerID,
 		LastBlockNumber:        bcInfo.Height - 1,
 		LastBlockHashInHex:     hex.EncodeToString(bcInfo.CurrentBlockHash),
@@ -188,11 +188,11 @@ func (l *kvLedger) generateSnapshotMetadataFiles(
 		StateDBType:            stateDBType,
 	}
 
-	signableMetadataBytes, err := signableMetadata.toJSON()
+	signableMetadataBytes, err := signableMetadata.ToJSON()
 	if err != nil {
 		return errors.Wrap(err, "error while marshelling snapshot metadata to JSON")
 	}
-	if err := fileutil.CreateAndSyncFile(filepath.Join(dir, snapshotSignableMetadataFileName), signableMetadataBytes, 0o444); err != nil {
+	if err := fileutil.CreateAndSyncFile(filepath.Join(dir, SnapshotSignableMetadataFileName), signableMetadataBytes, 0o444); err != nil {
 		return err
 	}
 
@@ -210,7 +210,7 @@ func (l *kvLedger) generateSnapshotMetadataFiles(
 		LastBlockCommitHashInHex: hex.EncodeToString(l.commitHash),
 	}
 
-	additionalMetadataBytes, err := additionalMetadata.toJSON()
+	additionalMetadataBytes, err := additionalMetadata.ToJSON()
 	if err != nil {
 		return errors.Wrap(err, "error while marshalling snapshot additional metadata to JSON")
 	}
@@ -226,7 +226,7 @@ func (p *Provider) CreateFromSnapshot(snapshotDir string) (ledger.PeerLedger, st
 		return nil, "", errors.WithMessagef(err, "error while loading metadata")
 	}
 
-	metadata, err := metadataJSONs.toMetadata()
+	metadata, err := metadataJSONs.ToMetadata()
 	if err != nil {
 		return nil, "", errors.WithMessagef(err, "error while unmarshalling metadata")
 	}
@@ -349,8 +349,8 @@ func (p *Provider) CreateFromSnapshot(snapshotDir string) (ledger.PeerLedger, st
 	return lgr, ledgerID, nil
 }
 
-func loadSnapshotMetadataJSONs(snapshotDir string) (*snapshotMetadataJSONs, error) {
-	signableMetadataFilePath := filepath.Join(snapshotDir, snapshotSignableMetadataFileName)
+func loadSnapshotMetadataJSONs(snapshotDir string) (*SnapshotMetadataJSONs, error) {
+	signableMetadataFilePath := filepath.Join(snapshotDir, SnapshotSignableMetadataFileName)
 	signableMetadataBytes, err := ioutil.ReadFile(signableMetadataFilePath)
 	if err != nil {
 		return nil, err
@@ -360,16 +360,16 @@ func loadSnapshotMetadataJSONs(snapshotDir string) (*snapshotMetadataJSONs, erro
 	if err != nil {
 		return nil, err
 	}
-	return &snapshotMetadataJSONs{
+	return &SnapshotMetadataJSONs{
 		signableMetadata:   string(signableMetadataBytes),
 		additionalMetadata: string(additionalMetadataBytes),
 	}, nil
 }
 
-func verifySnapshot(snapshotDir string, snapshotMetadata *snapshotMetadata, hashProvider ledger.HashProvider) error {
+func verifySnapshot(snapshotDir string, snapshotMetadata *SnapshotMetadata, hashProvider ledger.HashProvider) error {
 	if err := verifyFileHash(
 		snapshotDir,
-		snapshotSignableMetadataFileName,
+		SnapshotSignableMetadataFileName,
 		snapshotMetadata.SnapshotHashInHex,
 		hashProvider,
 	); err != nil {