e2e_test.go

/*
Copyright IBM Corp All Rights Reserved.

SPDX-License-Identifier: Apache-2.0
*/

package e2e

import (
	"bufio"
	"bytes"
	"context"
	"crypto/sha256"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"sort"
	"strings"
	"syscall"
	"time"

	docker "github.com/fsouza/go-dockerclient"
	"github.com/golang/protobuf/proto"
	"github.com/hyperledger/fabric-lib-go/healthz"
	"github.com/hyperledger/fabric-protos-go/orderer/etcdraft"
	"github.com/hyperledger/fabric/integration/channelparticipation"
	"github.com/hyperledger/fabric/integration/nwo"
	"github.com/hyperledger/fabric/integration/nwo/commands"
	"github.com/hyperledger/fabric/integration/nwo/fabricconfig"
	. "github.com/onsi/ginkgo/v2"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
	"github.com/onsi/gomega/gexec"
	"github.com/tedsuo/ifrit"
	ginkgomon "github.com/tedsuo/ifrit/ginkgomon_v2"
)

var _ = Describe("EndToEnd", func() {
	var (
		testDir                     string
		client                      *docker.Client
		network                     *nwo.Network
		chaincode                   nwo.Chaincode
		ordererRunner               *ginkgomon.Runner
		ordererProcess, peerProcess ifrit.Process
	)

	BeforeEach(func() {
		var err error
		testDir, err = os.MkdirTemp("", "e2e")
		Expect(err).NotTo(HaveOccurred())

		client, err = docker.NewClientFromEnv()
		Expect(err).NotTo(HaveOccurred())

		chaincode = nwo.Chaincode{
			Name:            "mycc",
			Version:         "0.0",
			Path:            components.Build("github.com/hyperledger/fabric/integration/chaincode/simple/cmd"),
			Lang:            "binary",
			PackageFile:     filepath.Join(testDir, "simplecc.tar.gz"),
			Ctor:            `{"Args":["init","a","100","b","200"]}`,
			SignaturePolicy: `AND ('Org1MSP.member','Org2MSP.member')`,
			Sequence:        "1",
			InitRequired:    true,
			Label:           "my_prebuilt_chaincode",
		}
	})

	AfterEach(func() {
		if ordererProcess != nil {
			ordererProcess.Signal(syscall.SIGTERM)
			Eventually(ordererProcess.Wait(), network.EventuallyTimeout).Should(Receive())
		}

		if peerProcess != nil {
			peerProcess.Signal(syscall.SIGTERM)
			Eventually(peerProcess.Wait(), network.EventuallyTimeout).Should(Receive())
		}

		if network != nil {
			network.Cleanup()
		}
		os.RemoveAll(testDir)
	})

	Describe("basic etcdraft network with 2 orgs and no docker", func() {
		var (
			metricsReader        *MetricsReader
			runArtifactsFilePath string
		)

		BeforeEach(func() {
			metricsReader = NewMetricsReader()
			go metricsReader.Start()

			network = nwo.New(nwo.BasicEtcdRaft(), testDir, nil, StartPort(), components)
			network.MetricsProvider = "statsd"
			network.StatsdEndpoint = metricsReader.Address()
			network.Channels = append(network.Channels, &nwo.Channel{
				Name:    "another-testchannel",
				Profile: "TwoOrgsAppChannelEtcdRaft",
			})

			runArtifactsFilePath = filepath.Join(testDir, "run-artifacts.txt")
			os.Setenv("RUN_ARTIFACTS_FILE", runArtifactsFilePath)
			for i, e := range network.ExternalBuilders {
				e.PropagateEnvironment = append(e.PropagateEnvironment, "RUN_ARTIFACTS_FILE")
				network.ExternalBuilders[i] = e
			}

			network.GenerateConfigTree()
			for _, peer := range network.PeersWithChannel("testchannel") {
				core := network.ReadPeerConfig(peer)
				core.VM = nil
				network.WritePeerConfig(peer, core)
			}

			network.Bootstrap()

			// Start all the fabric processes
			ordererRunner, ordererProcess, peerProcess = network.StartSingleOrdererNetwork("orderer")
		})

		AfterEach(func() {
			if metricsReader != nil {
				metricsReader.Close()
			}

			// Terminate the processes but defer the network cleanup to the outer
			// AfterEach.
			if ordererProcess != nil {
				ordererProcess.Signal(syscall.SIGTERM)
				Eventually(ordererProcess.Wait(), network.EventuallyTimeout).Should(Receive())
			}

			if peerProcess != nil {
				peerProcess.Signal(syscall.SIGTERM)
				Eventually(peerProcess.Wait(), network.EventuallyTimeout).Should(Receive())
			}

			// Ensure that the temporary directories generated by launched external
			// chaincodes have been cleaned up. This must be done after the peers
			// have been terminated.
			contents, err := os.ReadFile(runArtifactsFilePath)
			Expect(err).NotTo(HaveOccurred())
			scanner := bufio.NewScanner(bytes.NewBuffer(contents))
			for scanner.Scan() {
				Expect(scanner.Text()).NotTo(BeAnExistingFile())
			}
		})

		It("executes a basic etcdraft network with 2 orgs and no docker", func() {
			By("getting the orderer by name")
			orderer := network.Orderer("orderer")

			By("listing channels with osnadmin, no channels yet")
			tlsdir := network.OrdererLocalTLSDir(orderer)
			sess, err := network.Osnadmin(commands.ChannelList{
				OrdererAddress: network.OrdererAddress(orderer, nwo.AdminPort),
				CAFile:         filepath.Join(tlsdir, "ca.crt"),
				ClientCert:     filepath.Join(tlsdir, "server.crt"),
				ClientKey:      filepath.Join(tlsdir, "server.key"),
			})
			Expect(err).NotTo(HaveOccurred())
			Eventually(sess).Should(gexec.Exit(0))
			var channelList channelparticipation.ChannelList
			err = json.Unmarshal(sess.Out.Contents(), &channelList)
			Expect(err).NotTo(HaveOccurred())
			Expect(channelList).To(Equal(channelparticipation.ChannelList{}))

			By("setting up the channel")
			channelparticipation.JoinOrdererJoinPeersAppChannel(network, "testchannel", orderer, ordererRunner)

			By("listing channels with osnadmin")
			sess, err = network.Osnadmin(commands.ChannelList{
				OrdererAddress: network.OrdererAddress(orderer, nwo.AdminPort),
				CAFile:         filepath.Join(tlsdir, "ca.crt"),
				ClientCert:     filepath.Join(tlsdir, "server.crt"),
				ClientKey:      filepath.Join(tlsdir, "server.key"),
			})
			Expect(err).NotTo(HaveOccurred())
			Eventually(sess).Should(gexec.Exit(0))
			err = json.Unmarshal(sess.Out.Contents(), &channelList)
			Expect(err).NotTo(HaveOccurred())
			Expect(channelList).To(Equal(channelparticipation.ChannelList{
				Channels: []channelparticipation.ChannelInfoShort{{
					Name: "testchannel",
					URL:  "/participation/v1/channels/testchannel",
				}},
			}))

			By("enabling capabilities V2_5")
			nwo.EnableCapabilities(network, "testchannel", "Application", "V2_5", orderer, network.Peer("Org1", "peer0"), network.Peer("Org2", "peer0"))

			By("attempting to install unsupported chaincode without docker")
			badCC := chaincode
			badCC.Lang = "unsupported-type"
			badCC.Label = "chaincode-label"
			badCC.PackageFile = filepath.Join(testDir, "unsupported-type.tar.gz")
			nwo.PackageChaincodeBinary(badCC)
			badCC.SetPackageIDFromPackageFile()
			sess, err = network.PeerAdminSession(
				network.Peer("Org1", "peer0"),
				commands.ChaincodeInstall{
					PackageFile: badCC.PackageFile,
					ClientAuth:  network.ClientAuthRequired,
				},
			)
			Expect(err).NotTo(HaveOccurred())
			Eventually(sess, network.EventuallyTimeout).Should(gexec.Exit(1))
			Expect(sess.Err).To(gbytes.Say("docker build is disabled"))

			By("deploying the chaincode")
			nwo.DeployChaincode(network, "testchannel", orderer, chaincode)

			By("ensuring external cc run artifacts exist after deploying")
			contents, err := os.ReadFile(runArtifactsFilePath)
			Expect(err).NotTo(HaveOccurred())
			scanner := bufio.NewScanner(bytes.NewBuffer(contents))
			for scanner.Scan() {
				Expect(scanner.Text()).To(BeADirectory())
			}

			By("getting the client peer by name")
			peer := network.Peer("Org1", "peer0")

			RunQueryInvokeQuery(network, orderer, peer, "testchannel")
			RunRespondWith(network, orderer, peer, "testchannel")

			By("evaluating statsd metrics")
			metricsWriteInterval := 5 * time.Second
			CheckPeerStatsdStreamMetrics(metricsReader, 2*metricsWriteInterval)
			CheckPeerStatsdMetrics("org1_peer0", metricsReader, 2*metricsWriteInterval)
			CheckPeerStatsdMetrics("org2_peer0", metricsReader, 2*metricsWriteInterval)

			By("checking for orderer metrics")
			CheckOrdererStatsdMetrics("ordererorg_orderer", metricsReader, 2*metricsWriteInterval)

			By("setting up another channel from a the same profile")
			channelparticipation.JoinOrdererJoinPeersAppChannel(network, "another-testchannel", orderer, ordererRunner)

			By("listing channels with osnadmin")
			sess, err = network.Osnadmin(commands.ChannelList{
				OrdererAddress: network.OrdererAddress(orderer, nwo.AdminPort),
				CAFile:         filepath.Join(tlsdir, "ca.crt"),
				ClientCert:     filepath.Join(tlsdir, "server.crt"),
				ClientKey:      filepath.Join(tlsdir, "server.key"),
			})
			Expect(err).NotTo(HaveOccurred())
			Eventually(sess).Should(gexec.Exit(0))
			err = json.Unmarshal(sess.Out.Contents(), &channelList)
			Expect(err).NotTo(HaveOccurred())
			sort.Slice(channelList.Channels, func(i, j int) bool {
				return channelList.Channels[i].Name < channelList.Channels[j].Name
			})
			Expect(channelList).To(Equal(channelparticipation.ChannelList{
				Channels: []channelparticipation.ChannelInfoShort{
					{
						Name: "another-testchannel",
						URL:  "/participation/v1/channels/another-testchannel",
					},
					{
						Name: "testchannel",
						URL:  "/participation/v1/channels/testchannel",
					},
				},
			}))
		})
	})

	Describe("basic etcdraft network with docker chaincode builds", func() {
		BeforeEach(func() {
			network = nwo.New(nwo.BasicEtcdRaft(), testDir, client, StartPort(), components)
			network.MetricsProvider = "prometheus"
			network.GenerateConfigTree()
			network.Bootstrap()

			// Start all the fabric processes
			ordererRunner, ordererProcess, peerProcess = network.StartSingleOrdererNetwork("orderer")
		})

		It("builds and executes transactions with docker chaincode", func() {
			chaincodePath, err := filepath.Abs("../chaincode/module")
			Expect(err).NotTo(HaveOccurred())

			// use these two variants of the same chaincode to ensure we test
			// the golang docker build for both module and gopath chaincode
			chaincode := nwo.Chaincode{
				Name:            "mycc",
				Version:         "0.0",
				Path:            chaincodePath,
				Lang:            "golang",
				PackageFile:     filepath.Join(testDir, "modulecc.tar.gz"),
				Ctor:            `{"Args":["init","a","100","b","200"]}`,
				SignaturePolicy: `AND ('Org1MSP.member','Org2MSP.member')`,
				Sequence:        "1",
				InitRequired:    true,
				Label:           "my_module_chaincode",
			}

			gopathChaincode := nwo.Chaincode{
				Name:            "mycc",
				Version:         "0.0",
				Path:            "github.com/hyperledger/fabric/integration/chaincode/simple/cmd",
				Lang:            "golang",
				PackageFile:     filepath.Join(testDir, "simplecc.tar.gz"),
				Ctor:            `{"Args":["init","a","100","b","200"]}`,
				SignaturePolicy: `AND ('Org1MSP.member','Org2MSP.member')`,
				Sequence:        "1",
				InitRequired:    true,
				Label:           "my_simple_chaincode",
			}

			orderer := network.Orderer("orderer")

			By("setting up the channel")
			channelparticipation.JoinOrdererJoinPeersAppChannel(network, "testchannel", orderer, ordererRunner)

			cl := channelparticipation.List(network, orderer)
			channelparticipation.ChannelListMatcher(cl, []string{"testchannel"})

			nwo.EnableCapabilities(network, "testchannel", "Application", "V2_5", orderer, network.Peer("Org1", "peer0"), network.Peer("Org2", "peer0"))

			// package, install, and approve by org1 - module chaincode
			packageInstallApproveChaincode(network, "testchannel", orderer, chaincode, network.Peer("Org1", "peer0"))

			// package, install, and approve by org2 - gopath chaincode, same logic
			packageInstallApproveChaincode(network, "testchannel", orderer, gopathChaincode, network.Peer("Org2", "peer0"))

			testPeers := network.PeersWithChannel("testchannel")
			nwo.CheckCommitReadinessUntilReady(network, "testchannel", chaincode, network.PeerOrgs(), testPeers...)
			nwo.CommitChaincode(network, "testchannel", orderer, chaincode, testPeers[0], testPeers...)
			nwo.InitChaincode(network, "testchannel", orderer, chaincode, testPeers...)

			By("listing the containers after committing the chaincode definition")
			initialContainerFilter := map[string][]string{
				"name": {
					chaincodeContainerNameFilter(network, chaincode),
					chaincodeContainerNameFilter(network, gopathChaincode),
				},
			}

			containers, err := client.ListContainers(docker.ListContainersOptions{Filters: initialContainerFilter})
			Expect(err).NotTo(HaveOccurred())
			Expect(containers).To(HaveLen(2))

			RunQueryInvokeQuery(network, orderer, network.Peer("Org1", "peer0"), "testchannel")

			By("evaluating the operations endpoint and prometheus metrics")
			CheckPeerOperationEndpoints(network, network.Peer("Org2", "peer0"))
			CheckOrdererOperationEndpoints(network, orderer)

			// upgrade chaincode to v2.0 with different label
			chaincode.Version = "1.0"
			chaincode.Sequence = "2"
			chaincode.Label = "my_module_chaincode_updated"
			gopathChaincode.Version = "1.0"
			gopathChaincode.Sequence = "2"
			gopathChaincode.Label = "my_simple_chaincode_updated"

			// package, install, and approve by org1 - module chaincode
			packageInstallApproveChaincode(network, "testchannel", orderer, chaincode, network.Peer("Org1", "peer0"))

			// package, install, and approve by org2 - gopath chaincode, same logic
			packageInstallApproveChaincode(network, "testchannel", orderer, gopathChaincode, network.Peer("Org2", "peer0"))

			nwo.CheckCommitReadinessUntilReady(network, "testchannel", chaincode, network.PeerOrgs(), testPeers...)
			nwo.CommitChaincode(network, "testchannel", orderer, chaincode, testPeers[0], testPeers...)
			nwo.InitChaincode(network, "testchannel", orderer, chaincode, testPeers...)

			By("listing the containers after updating the chaincode definition")
			// expect the containers for the previous package id to be stopped
			containers, err = client.ListContainers(docker.ListContainersOptions{Filters: initialContainerFilter})
			Expect(err).NotTo(HaveOccurred())
			Expect(containers).To(HaveLen(0))
			updatedContainerFilter := map[string][]string{
				"name": {
					chaincodeContainerNameFilter(network, chaincode),
					chaincodeContainerNameFilter(network, gopathChaincode),
				},
			}
			containers, err = client.ListContainers(docker.ListContainersOptions{Filters: updatedContainerFilter})
			Expect(err).NotTo(HaveOccurred())
			Expect(containers).To(HaveLen(2))

			RunQueryInvokeQuery(network, orderer, network.Peer("Org1", "peer0"), "testchannel")

			By("retrieving the local mspid of the peer via simple chaincode")
			sess, err := network.PeerUserSession(network.Peer("Org2", "peer0"), "User1", commands.ChaincodeQuery{
				ChannelID: "testchannel",
				Name:      "mycc",
				Ctor:      `{"Args":["mspid"]}`,
			})
			Expect(err).NotTo(HaveOccurred())
			Eventually(sess, network.EventuallyTimeout).Should(gexec.Exit(0))
			Expect(sess).To(gbytes.Say("Org2MSP"))
		})
	})

	Describe("basic single node etcdraft network with static leader", func() {
		var (
			peerRunners []*ginkgomon.Runner
			processes   map[string]ifrit.Process
		)

		BeforeEach(func() {
			network = nwo.New(nwo.MultiChannelEtcdRaft(), testDir, client, StartPort(), components)
			network.GenerateConfigTree()
			for _, peer := range network.Peers {
				core := network.ReadPeerConfig(peer)
				core.Peer.Gossip.UseLeaderElection = false
				core.Peer.Gossip.OrgLeader = true
				core.Peer.Deliveryclient.ReconnectTotalTimeThreshold = time.Second
				network.WritePeerConfig(peer, core)
			}
			network.Bootstrap()

			ordererRunner = network.OrdererRunner(network.Orderer("orderer"))
			ordererProcess = ifrit.Invoke(ordererRunner)
			Eventually(ordererProcess.Ready(), network.EventuallyTimeout).Should(BeClosed())

			peerRunners = make([]*ginkgomon.Runner, len(network.Peers))
			processes = map[string]ifrit.Process{}
			for i, peer := range network.Peers {
				pr := network.PeerRunner(peer)
				peerRunners[i] = pr
				p := ifrit.Invoke(pr)
				processes[peer.ID()] = p
				Eventually(p.Ready(), network.EventuallyTimeout).Should(BeClosed())
			}
		})

		AfterEach(func() {
			for _, p := range processes {
				p.Signal(syscall.SIGTERM)
				Eventually(p.Wait(), network.EventuallyTimeout).Should(Receive())
			}
		})

		It("creates two channels with two orgs trying to reconfigure and update metadata", func() {
			orderer := network.Orderer("orderer")

			By("Create first channel and deploy the chaincode")
			channelparticipation.JoinOrdererJoinPeersAppChannel(network, "testchannel", orderer, ordererRunner)

			nwo.EnableCapabilities(network, "testchannel", "Application", "V2_5", orderer, network.Peer("Org1", "peer0"), network.Peer("Org2", "peer0"))
			nwo.DeployChaincode(network, "testchannel", orderer, chaincode)
			peer := network.Peer("Org1", "peer0")
			RunQueryInvokeQuery(network, orderer, peer, "testchannel")

			By("Create second channel and deploy chaincode")
			channelparticipation.JoinOrdererJoinPeersAppChannel(network, "testchannel2", orderer, ordererRunner)

			cl := channelparticipation.List(network, orderer)
			channelparticipation.ChannelListMatcher(cl, []string{"testchannel", "testchannel2"})

			nwo.EnableCapabilities(network, "testchannel2", "Application", "V2_5", orderer, network.Peer("Org1", "peer0"), network.Peer("Org2", "peer0"))
			peers := network.PeersWithChannel("testchannel2")
			nwo.ApproveChaincodeForMyOrg(network, "testchannel2", orderer, chaincode, peers...)
			nwo.CheckCommitReadinessUntilReady(network, "testchannel2", chaincode, network.PeerOrgs(), peers...)
			nwo.CommitChaincode(network, "testchannel2", orderer, chaincode, peers[0], peers...)
			nwo.InitChaincode(network, "testchannel2", orderer, chaincode, peers...)
			RunQueryInvokeQuery(network, orderer, peer, "testchannel2")

			By("Update consensus metadata to increase snapshot interval")
			snapDir := path.Join(network.RootDir, "orderers", orderer.ID(), "etcdraft", "snapshot", "testchannel")
			files, err := os.ReadDir(snapDir)
			Expect(err).NotTo(HaveOccurred())
			numOfSnaps := len(files)

			nwo.UpdateConsensusMetadata(network, peer, orderer, "testchannel", func(originalMetadata []byte) []byte {
				metadata := &etcdraft.ConfigMetadata{}
				err := proto.Unmarshal(originalMetadata, metadata)
				Expect(err).NotTo(HaveOccurred())

				// update max in flight messages
				metadata.Options.MaxInflightBlocks = 1000
				metadata.Options.SnapshotIntervalSize = 10 * 1024 * 1024 // 10 MB

				// write metadata back
				newMetadata, err := proto.Marshal(metadata)
				Expect(err).NotTo(HaveOccurred())
				return newMetadata
			})

			// assert that no new snapshot is taken because SnapshotIntervalSize has just enlarged
			files, err = os.ReadDir(snapDir)
			Expect(err).NotTo(HaveOccurred())
			Expect(len(files)).To(Equal(numOfSnaps))

			By("ensuring that static leaders do not give up on retrieving blocks after the orderer goes down")
			ordererProcess.Signal(syscall.SIGTERM)
			Eventually(ordererProcess.Wait(), network.EventuallyTimeout).Should(Receive())
			for _, peerRunner := range peerRunners {
				Eventually(peerRunner.Err(), network.EventuallyTimeout).Should(gbytes.Say("peer is a static leader, ignoring peer.deliveryclient.reconnectTotalTimeThreshold"))
			}
		})
	})

	Describe("single node etcdraft network with remapped orderer endpoints", func() {
		BeforeEach(func() {
			network = nwo.New(nwo.MinimalRaft(), testDir, client, StartPort(), components)
			network.GenerateConfigTree()

			configtxConfig := network.ReadConfigTxConfig()
			ordererEndpoints := configtxConfig.Profiles["OneOrgChannelEtcdRaft"].Orderer.Organizations[0].OrdererEndpoints
			correctOrdererEndpoint := ordererEndpoints[0]
			ordererEndpoints[0] = "127.0.0.1:1"
			network.WriteConfigTxConfig(configtxConfig)

			peer := network.Peer("Org1", "peer0")
			peerConfig := network.ReadPeerConfig(peer)
			peerConfig.Peer.Deliveryclient.AddressOverrides = []*fabricconfig.AddressOverride{
				{
					From:        "127.0.0.1:1",
					To:          correctOrdererEndpoint,
					CACertsFile: network.CACertsBundlePath(),
				},
			}
			network.WritePeerConfig(peer, peerConfig)

			network.Bootstrap()

			// Start all the fabric processes
			ordererRunner, ordererProcess, peerProcess = network.StartSingleOrdererNetwork("orderer")
		})

		It("creates and updates channel", func() {
			orderer := network.Orderer("orderer")
			channelparticipation.JoinOrdererJoinPeersAppChannel(network, "testchannel", orderer, ordererRunner)

			// The below call waits for the config update to commit on the peer, so
			// it will fail if the orderer addresses are wrong.
			nwo.EnableCapabilities(network, "testchannel", "Application", "V2_5", orderer, network.Peer("Org1", "peer0"), network.Peer("Org2", "peer0"))
		})
	})

	Describe("basic etcdraft network with containers being interrupted", func() {
		BeforeEach(func() {
			network = nwo.New(nwo.FullEtcdRaft(), testDir, client, StartPort(), components)

			network.GenerateConfigTree()
			network.Bootstrap()

			// Start all the fabric processes
			ordererRunner, ordererProcess, peerProcess = network.StartSingleOrdererNetwork("orderer")
		})

		It("recreates terminated chaincode containers", func() {
			chaincode := nwo.Chaincode{
				Name:            "mycc",
				Version:         "0.0",
				Path:            "github.com/hyperledger/fabric/integration/chaincode/simple/cmd",
				Lang:            "golang",
				PackageFile:     filepath.Join(testDir, "simplecc.tar.gz"),
				Ctor:            `{"Args":["init","a","100","b","200"]}`,
				SignaturePolicy: `OR ('Org1MSP.peer', 'Org2MSP.peer')`,
				Sequence:        "1",
				InitRequired:    true,
				Label:           "my_simple_chaincode",
			}

			orderer := network.Orderer("orderer")

			By("creating and joining channels")
			channelparticipation.JoinOrdererJoinPeersAppChannel(network, "testchannel", orderer, ordererRunner)

			By("enabling new lifecycle capabilities")
			nwo.EnableCapabilities(network, "testchannel", "Application", "V2_5", orderer, network.Peer("Org1", "peer0"), network.Peer("Org2", "peer0"))
			By("deploying the chaincode")
			nwo.DeployChaincode(network, "testchannel", orderer, chaincode)

			By("querying and invoking chaincode")
			peer := network.Peers[0]
			RunQueryInvokeQuery(network, orderer, peer, "testchannel")

			By("removing chaincode containers from all peers")
			listChaincodeContainers := docker.ListContainersOptions{
				Filters: map[string][]string{
					"name": {chaincodeContainerNameFilter(network, chaincode)},
				},
			}
			ctx := context.Background()
			containers, err := client.ListContainers(listChaincodeContainers)
			Expect(err).NotTo(HaveOccurred())
			Expect(containers).NotTo(BeEmpty())

			var originalContainerIDs []string
			for _, container := range containers {
				originalContainerIDs = append(originalContainerIDs, container.ID)
				err = client.RemoveContainer(docker.RemoveContainerOptions{
					ID:            container.ID,
					RemoveVolumes: true,
					Force:         true,
					Context:       ctx,
				})
				Expect(err).NotTo(HaveOccurred())
			}

			By("invoking chaincode against all peers in test channel")
			for _, peer := range network.Peers {
				sess, err := network.PeerUserSession(peer, "User1", commands.ChaincodeInvoke{
					ChannelID: "testchannel",
					Orderer:   network.OrdererAddress(orderer, nwo.ListenPort),
					Name:      "mycc",
					Ctor:      `{"Args":["invoke","a","b","10"]}`,
					PeerAddresses: []string{
						network.PeerAddress(peer, nwo.ListenPort),
					},
					WaitForEvent: true,
				})
				Expect(err).NotTo(HaveOccurred())
				Eventually(sess, network.EventuallyTimeout).Should(gexec.Exit(0))
				Expect(sess.Err).To(gbytes.Say("Chaincode invoke successful. result: status:200"))
			}

			By("checking successful removals of all old chaincode containers")
			newContainers, err := client.ListContainers(listChaincodeContainers)
			Expect(err).NotTo(HaveOccurred())
			Expect(newContainers).To(HaveLen(len(containers)))

			for _, container := range newContainers {
				Expect(originalContainerIDs).NotTo(ContainElement(container.ID))
			}
		})
	})

	Describe("basic lifecycle operations for chaincode install and update", func() {
		BeforeEach(func() {
			network = nwo.New(nwo.BasicEtcdRaft(), testDir, client, StartPort(), components)

			network.GenerateConfigTree()
			network.Bootstrap()

			// Start all the fabric processes
			ordererRunner, ordererProcess, peerProcess = network.StartSingleOrdererNetwork("orderer")
		})

		It("approves chaincode for org and updates chaincode definition", func() {
			chaincode := nwo.Chaincode{
				Name:            "mycc",
				Version:         "0.0",
				Path:            "github.com/hyperledger/fabric/integration/chaincode/simple/cmd",
				Lang:            "golang",
				PackageFile:     filepath.Join(testDir, "simplecc.tar.gz"),
				Ctor:            `{"Args":["init","a","100","b","200"]}`,
				SignaturePolicy: `OR ('Org1MSP.peer', 'Org2MSP.peer')`,
				Sequence:        "1",
				InitRequired:    true,
				Label:           "my_simple_chaincode",
			}

			orderer := network.Orderer("orderer")

			By("creating and joining channels")
			channelparticipation.JoinOrdererJoinPeersAppChannel(network, "testchannel", orderer, ordererRunner)

			By("enabling new lifecycle capabilities")
			nwo.EnableCapabilities(network, "testchannel", "Application", "V2_5", orderer, network.Peer("Org1", "peer0"), network.Peer("Org2", "peer0"))
			By("deploying the chaincode")
			peers := network.PeersWithChannel("testchannel")
			nwo.PackageAndInstallChaincode(network, chaincode, peers...)

			// approve for each org
			nwo.ApproveChaincodeForMyOrg(network, "testchannel", orderer, chaincode, peers...)
			// commit definition
			nwo.CheckCommitReadinessUntilReady(network, "testchannel", chaincode, network.PeerOrgs(), peers...)

			// update chaincode endorsement policy and commit again
			By("update chaincode's endorsement policy")
			chaincode.SignaturePolicy = `AND ('Org1MSP.peer', 'Org2MSP.peer')`
			nwo.ApproveChaincodeForMyOrg(network, "testchannel", orderer, chaincode, peers...)
			nwo.CheckCommitReadinessUntilReady(network, "testchannel", chaincode, network.PeerOrgs(), peers...)

			// finall commit the chaincode
			By("committing chaincode's definition")
			nwo.CommitChaincode(network, "testchannel", orderer, chaincode, peers[0], peers...)

			if chaincode.PackageID == "" {
				chaincode.SetPackageIDFromPackageFile()
			}

			By("attempting to approve chaincode definition after commit")
			sess, err := network.PeerAdminSession(peers[0], commands.ChaincodeApproveForMyOrg{
				ChannelID:           "testchannel",
				Orderer:             network.OrdererAddress(orderer, nwo.ListenPort),
				Name:                chaincode.Name,
				Version:             chaincode.Version,
				PackageID:           chaincode.PackageID,
				Sequence:            chaincode.Sequence,
				EndorsementPlugin:   chaincode.EndorsementPlugin,
				ValidationPlugin:    chaincode.ValidationPlugin,
				SignaturePolicy:     chaincode.SignaturePolicy,
				ChannelConfigPolicy: chaincode.ChannelConfigPolicy,
				InitRequired:        chaincode.InitRequired,
				CollectionsConfig:   chaincode.CollectionsConfig,
				ClientAuth:          network.ClientAuthRequired,
				WaitForEventTimeout: network.EventuallyTimeout,
			})
			Expect(err).NotTo(HaveOccurred())
			Eventually(sess.Err, network.EventuallyTimeout).Should(gbytes.Say(`Error: proposal failed with status: 500`))
		})
	})
})

func RunQueryInvokeQuery(n *nwo.Network, orderer *nwo.Orderer, peer *nwo.Peer, channel string) {
	By("querying the chaincode")
	sess, err := n.PeerUserSession(peer, "User1", commands.ChaincodeQuery{
		ChannelID: channel,
		Name:      "mycc",
		Ctor:      `{"Args":["query","a"]}`,
	})
	Expect(err).NotTo(HaveOccurred())
	Eventually(sess, n.EventuallyTimeout).Should(gexec.Exit(0))
	Expect(sess).To(gbytes.Say("100"))

	sess, err = n.PeerUserSession(peer, "User1", commands.ChaincodeInvoke{
		ChannelID: channel,
		Orderer:   n.OrdererAddress(orderer, nwo.ListenPort),
		Name:      "mycc",
		Ctor:      `{"Args":["invoke","a","b","10"]}`,
		PeerAddresses: []string{
			n.PeerAddress(n.Peer("Org1", "peer0"), nwo.ListenPort),
			n.PeerAddress(n.Peer("Org2", "peer0"), nwo.ListenPort),
		},
		WaitForEvent: true,
	})
	Expect(err).NotTo(HaveOccurred())
	Eventually(sess, n.EventuallyTimeout).Should(gexec.Exit(0))
	Expect(sess.Err).To(gbytes.Say("Chaincode invoke successful. result: status:200"))

	sess, err = n.PeerUserSession(peer, "User1", commands.ChaincodeQuery{
		ChannelID: channel,
		Name:      "mycc",
		Ctor:      `{"Args":["query","a"]}`,
	})
	Expect(err).NotTo(HaveOccurred())
	Eventually(sess, n.EventuallyTimeout).Should(gexec.Exit(0))
	Expect(sess).To(gbytes.Say("90"))
}

func RunRespondWith(n *nwo.Network, orderer *nwo.Orderer, peer *nwo.Peer, channel string) {
	By("responding with a 300")
	sess, err := n.PeerUserSession(peer, "User1", commands.ChaincodeInvoke{
		ChannelID: channel,
		Orderer:   n.OrdererAddress(orderer, nwo.ListenPort),
		Name:      "mycc",
		Ctor:      `{"Args":["respond","300","response-message","response-payload"]}`,
		PeerAddresses: []string{
			n.PeerAddress(n.Peer("Org1", "peer0"), nwo.ListenPort),
			n.PeerAddress(n.Peer("Org2", "peer0"), nwo.ListenPort),
		},
		WaitForEvent: true,
	})
	Expect(err).NotTo(HaveOccurred())
	Eventually(sess, n.EventuallyTimeout).Should(gexec.Exit(0))
	Expect(sess.Err).To(gbytes.Say("Chaincode invoke successful. result: status:300"))

	By("responding with a 400")
	sess, err = n.PeerUserSession(peer, "User1", commands.ChaincodeInvoke{
		ChannelID: channel,
		Orderer:   n.OrdererAddress(orderer, nwo.ListenPort),
		Name:      "mycc",
		Ctor:      `{"Args":["respond","400","response-message","response-payload"]}`,
		PeerAddresses: []string{
			n.PeerAddress(n.Peer("Org1", "peer0"), nwo.ListenPort),
			n.PeerAddress(n.Peer("Org2", "peer0"), nwo.ListenPort),
		},
		WaitForEvent: true,
	})
	Expect(err).NotTo(HaveOccurred())
	Eventually(sess, n.EventuallyTimeout).Should(gexec.Exit(1))
	Expect(sess.Err).To(gbytes.Say(`Error: endorsement failure during invoke.`))
}

func CheckPeerStatsdMetrics(prefix string, mr *MetricsReader, timeout time.Duration) {
	By("checking for peer statsd metrics")
	Eventually(mr.String, timeout).Should(SatisfyAll(
		ContainSubstring(prefix+".logging.entries_checked.info:"),
		ContainSubstring(prefix+".logging.entries_written.info:"),
		ContainSubstring(prefix+".go.mem.gc_completed_count:"),
		ContainSubstring(prefix+".grpc.server.unary_requests_received.protos_Endorser.ProcessProposal:"),
		ContainSubstring(prefix+".grpc.server.unary_requests_completed.protos_Endorser.ProcessProposal.OK:"),
		ContainSubstring(prefix+".grpc.server.unary_request_duration.protos_Endorser.ProcessProposal.OK:"),
		ContainSubstring(prefix+".ledger.blockchain_height"),
		ContainSubstring(prefix+".ledger.blockstorage_commit_time"),
		ContainSubstring(prefix+".ledger.blockstorage_and_pvtdata_commit_time"),
	))
}

func CheckPeerStatsdStreamMetrics(mr *MetricsReader, timeout time.Duration) {
	By("checking for stream metrics")
	Eventually(mr.String, timeout).Should(SatisfyAll(
		ContainSubstring(".grpc.server.stream_requests_received.protos_Deliver.DeliverFiltered:"),
		ContainSubstring(".grpc.server.stream_requests_completed.protos_Deliver.DeliverFiltered.Unknown:"),
		ContainSubstring(".grpc.server.stream_request_duration.protos_Deliver.DeliverFiltered.Unknown:"),
		ContainSubstring(".grpc.server.stream_messages_received.protos_Deliver.DeliverFiltered"),
		ContainSubstring(".grpc.server.stream_messages_sent.protos_Deliver.DeliverFiltered"),
	))
}

func CheckOrdererStatsdMetrics(prefix string, mr *MetricsReader, timeout time.Duration) {
	Eventually(mr.String, timeout).Should(SatisfyAll(
		ContainSubstring(prefix+".consensus.etcdraft.committed_block_number"),
		ContainSubstring(prefix+".participation.consensus_relation"),
		ContainSubstring(prefix+".grpc.server.stream_request_duration.orderer_AtomicBroadcast.Deliver."),
		ContainSubstring(prefix+".logging.entries_checked.info:"),
		ContainSubstring(prefix+".logging.entries_written.info:"),
		ContainSubstring(prefix+".go.mem.gc_completed_count:"),
		ContainSubstring(prefix+".grpc.server.stream_requests_received.orderer_AtomicBroadcast.Deliver:"),
		ContainSubstring(prefix+".grpc.server.stream_requests_completed.orderer_AtomicBroadcast.Deliver."),
		ContainSubstring(prefix+".grpc.server.stream_messages_received.orderer_AtomicBroadcast.Deliver"),
		ContainSubstring(prefix+".grpc.server.stream_messages_sent.orderer_AtomicBroadcast.Deliver"),
		ContainSubstring(prefix+".ledger.blockchain_height"),
		ContainSubstring(prefix+".ledger.blockstorage_commit_time"),
	))
}

func CheckPeerOperationEndpoints(network *nwo.Network, peer *nwo.Peer) {
	metricsURL := fmt.Sprintf("https://127.0.0.1:%d/metrics", network.PeerPort(peer, nwo.OperationsPort))
	logspecURL := fmt.Sprintf("https://127.0.0.1:%d/logspec", network.PeerPort(peer, nwo.OperationsPort))
	healthURL := fmt.Sprintf("https://127.0.0.1:%d/healthz", network.PeerPort(peer, nwo.OperationsPort))

	authClient, unauthClient := nwo.PeerOperationalClients(network, peer)

	CheckPeerPrometheusMetrics(authClient, metricsURL)
	CheckLogspecOperations(authClient, logspecURL)
	CheckHealthEndpoint(authClient, healthURL)

	By("getting the logspec without a client cert")
	resp, err := unauthClient.Get(logspecURL)
	Expect(err).NotTo(HaveOccurred())
	Expect(resp.StatusCode).To(Equal(http.StatusUnauthorized))

	By("ensuring health checks do not require a client cert")
	CheckHealthEndpoint(unauthClient, healthURL)
}

func CheckOrdererOperationEndpoints(network *nwo.Network, orderer *nwo.Orderer) {
	metricsURL := fmt.Sprintf("https://127.0.0.1:%d/metrics", network.OrdererPort(orderer, nwo.OperationsPort))
	logspecURL := fmt.Sprintf("https://127.0.0.1:%d/logspec", network.OrdererPort(orderer, nwo.OperationsPort))
	healthURL := fmt.Sprintf("https://127.0.0.1:%d/healthz", network.OrdererPort(orderer, nwo.OperationsPort))

	authClient, unauthClient := nwo.OrdererOperationalClients(network, orderer)

	CheckOrdererPrometheusMetrics(authClient, metricsURL)
	CheckLogspecOperations(authClient, logspecURL)
	CheckHealthEndpoint(authClient, healthURL)

	By("getting the logspec without a client cert")
	resp, err := unauthClient.Get(logspecURL)
	Expect(err).NotTo(HaveOccurred())
	Expect(resp.StatusCode).To(Equal(http.StatusUnauthorized))

	By("ensuring health checks do not require a client cert")
	CheckHealthEndpoint(unauthClient, healthURL)
}

func CheckPeerPrometheusMetrics(client *http.Client, url string) {
	By("hitting the prometheus metrics endpoint")
	resp, err := client.Get(url)
	Expect(err).NotTo(HaveOccurred())
	Expect(resp.StatusCode).To(Equal(http.StatusOK))
	resp.Body.Close()

	Eventually(getBody(client, url)).Should(ContainSubstring(`# TYPE grpc_server_stream_request_duration histogram`))

	By("checking for some expected metrics")
	body := getBody(client, url)()
	Expect(body).To(ContainSubstring(`# TYPE go_gc_duration_seconds summary`))
	Expect(body).To(ContainSubstring(`# TYPE grpc_server_stream_request_duration histogram`))
	Expect(body).To(ContainSubstring(`grpc_server_stream_request_duration_count{code="Unknown",method="DeliverFiltered",service="protos_Deliver"}`))
	Expect(body).To(ContainSubstring(`grpc_server_stream_messages_received{method="DeliverFiltered",service="protos_Deliver"}`))
	Expect(body).To(ContainSubstring(`grpc_server_stream_messages_sent{method="DeliverFiltered",service="protos_Deliver"}`))
	Expect(body).To(ContainSubstring(`# TYPE grpc_comm_conn_closed counter`))
	Expect(body).To(ContainSubstring(`# TYPE grpc_comm_conn_opened counter`))
	Expect(body).To(ContainSubstring(`ledger_blockchain_height`))
	Expect(body).To(ContainSubstring(`ledger_blockstorage_commit_time_bucket`))
	Expect(body).To(ContainSubstring(`ledger_blockstorage_and_pvtdata_commit_time_bucket`))
}

func CheckOrdererPrometheusMetrics(client *http.Client, url string) {
	By("hitting the prometheus metrics endpoint")
	resp, err := client.Get(url)
	Expect(err).NotTo(HaveOccurred())
	Expect(resp.StatusCode).To(Equal(http.StatusOK))
	resp.Body.Close()

	Eventually(getBody(client, url)).Should(ContainSubstring(`# TYPE grpc_server_stream_request_duration histogram`))

	By("checking for some expected metrics")
	body := getBody(client, url)()
	Expect(body).To(ContainSubstring(`# TYPE go_gc_duration_seconds summary`))
	Expect(body).To(ContainSubstring(`# TYPE grpc_server_stream_request_duration histogram`))
	Expect(body).To(ContainSubstring(`consensus_etcdraft_committed_block_number`))
	Expect(body).To(ContainSubstring(`participation_consensus_relation`))
	Expect(body).To(ContainSubstring(`# TYPE grpc_comm_conn_closed counter`))
	Expect(body).To(ContainSubstring(`# TYPE grpc_comm_conn_opened counter`))
	Expect(body).To(ContainSubstring(`ledger_blockchain_height`))
	Expect(body).To(ContainSubstring(`ledger_blockstorage_commit_time_bucket`))
}

func CheckLogspecOperations(client *http.Client, logspecURL string) {
	By("getting the logspec")
	resp, err := client.Get(logspecURL)
	Expect(err).NotTo(HaveOccurred())
	Expect(resp.StatusCode).To(Equal(http.StatusOK))
	bodyBytes, err := io.ReadAll(resp.Body)
	resp.Body.Close()
	Expect(err).NotTo(HaveOccurred())
	Expect(string(bodyBytes)).To(MatchJSON(`{"spec":"info"}`))

	updateReq, err := http.NewRequest(http.MethodPut, logspecURL, strings.NewReader(`{"spec":"debug"}`))
	Expect(err).NotTo(HaveOccurred())

	By("setting the logspec")
	resp, err = client.Do(updateReq)
	Expect(err).NotTo(HaveOccurred())
	Expect(resp.StatusCode).To(Equal(http.StatusNoContent))
	resp.Body.Close()

	resp, err = client.Get(logspecURL)
	Expect(err).NotTo(HaveOccurred())
	Expect(resp.StatusCode).To(Equal(http.StatusOK))
	bodyBytes, err = io.ReadAll(resp.Body)
	resp.Body.Close()
	Expect(err).NotTo(HaveOccurred())
	Expect(string(bodyBytes)).To(MatchJSON(`{"spec":"debug"}`))

	By("resetting the logspec")
	updateReq, err = http.NewRequest(http.MethodPut, logspecURL, strings.NewReader(`{"spec":"info"}`))
	Expect(err).NotTo(HaveOccurred())
	resp, err = client.Do(updateReq)
	Expect(err).NotTo(HaveOccurred())
	Expect(resp.StatusCode).To(Equal(http.StatusNoContent))
	resp.Body.Close()
}

func CheckHealthEndpoint(client *http.Client, url string) {
	body := getBody(client, url)()

	var healthStatus healthz.HealthStatus
	err := json.Unmarshal([]byte(body), &healthStatus)
	Expect(err).NotTo(HaveOccurred())
	Expect(healthStatus.Status).To(Equal(healthz.StatusOK))
}

func getBody(client *http.Client, url string) func() string {
	return func() string {
		resp, err := client.Get(url)
		Expect(err).NotTo(HaveOccurred())
		Expect(resp.StatusCode).To(Equal(http.StatusOK))
		bodyBytes, err := io.ReadAll(resp.Body)
		Expect(err).NotTo(HaveOccurred())
		resp.Body.Close()
		return string(bodyBytes)
	}
}

func packageInstallApproveChaincode(network *nwo.Network, channel string, orderer *nwo.Orderer, chaincode nwo.Chaincode, peers ...*nwo.Peer) {
	nwo.PackageChaincode(network, chaincode, peers[0])
	nwo.InstallChaincode(network, chaincode, peers...)
	nwo.ApproveChaincodeForMyOrg(network, channel, orderer, chaincode, peers...)
}

func hashFile(file string) string {
	f, err := os.Open(file)
	Expect(err).NotTo(HaveOccurred())
	defer f.Close()

	h := sha256.New()
	_, err = io.Copy(h, f)
	Expect(err).NotTo(HaveOccurred())

	return fmt.Sprintf("%x", h.Sum(nil))
}

func chaincodeContainerNameFilter(n *nwo.Network, chaincode nwo.Chaincode) string {
	return fmt.Sprintf("^/%s-.*-%s-%s$", n.NetworkID, chaincode.Label, hashFile(chaincode.PackageFile))
}