diff --git a/ci/azure-pipelines.yml b/ci/azure-pipelines.yml index 3399833c08..c8e0c6849f 100644 --- a/ci/azure-pipelines.yml +++ b/ci/azure-pipelines.yml @@ -121,6 +121,7 @@ jobs: Basic-Typescript: CHAINCODE_NAME: basic CHAINCODE_LANGUAGE: typescript + steps: - template: templates/install-deps.yml - script: sudo apt-get install softhsm2 @@ -135,12 +136,17 @@ jobs: vmImage: ubuntu-20.04 strategy: matrix: - Typescript-Java: + CCaaS-Java: CLIENT_LANGUAGE: typescript CHAINCODE_LANGUAGE: java - Typescript-Golang: + CCaaS-Golang: CLIENT_LANGUAGE: typescript CHAINCODE_LANGUAGE: external + K8s-Builder-Java: + CHAINCODE_NAME: basic + CHAINCODE_LANGUAGE: java + CHAINCODE_BUILDER: k8s + steps: - template: templates/install-k8s-deps.yml - script: ../ci/scripts/run-k8s-test-network-basic.sh diff --git a/ci/scripts/run-k8s-test-network-basic.sh b/ci/scripts/run-k8s-test-network-basic.sh index ed544f8657..af87a8a8f0 100755 --- a/ci/scripts/run-k8s-test-network-basic.sh +++ b/ci/scripts/run-k8s-test-network-basic.sh @@ -10,6 +10,7 @@ set -euo pipefail export CONTAINER_CLI=${CONTAINER_CLI:-docker} export CLIENT_LANGUAGE=${CLIENT_LANGUAGE:-typescript} export CHAINCODE_LANGUAGE=${CHAINCODE_LANGUAGE:-java} +export TEST_NETWORK_CHAINCODE_BUILDER=${CHAINCODE_BUILDER:-ccaas} # Fabric version and Docker registry source: use the latest stable tag image from JFrog export FABRIC_VERSION=${FABRIC_VERSION:-2.4} @@ -70,7 +71,7 @@ function createNetwork() { ./network channel create print "Deploying chaincode" - ./network chaincode deploy asset-transfer-basic basic_1.0 $TEST_NETWORK_CHAINCODE_PATH + ./network chaincode deploy $CHAINCODE_NAME $TEST_NETWORK_CHAINCODE_PATH } function stopNetwork() { diff --git a/test-network-k8s/.gitignore b/test-network-k8s/.gitignore index c1f650e1de..944be9b551 100644 --- a/test-network-k8s/.gitignore +++ b/test-network-k8s/.gitignore @@ -4,3 +4,4 @@ network-debug.log build/ .env bin/ +*.tgz diff --git a/test-network-k8s/README.md b/test-network-k8s/README.md index 735f39fcd2..bdfbfc37db 100644 --- a/test-network-k8s/README.md +++ b/test-network-k8s/README.md @@ -45,7 +45,7 @@ Launch the network, create a channel, and deploy the [basic-asset-transfer](../a ./network channel create -./network chaincode deploy asset-transfer-basic basic_1.0 $PWD/../asset-transfer-basic/chaincode-java +./network chaincode deploy asset-transfer-basic ../asset-transfer-basic/chaincode-java ``` Invoke and query chaincode: diff --git a/test-network-k8s/config/org1/core.yaml b/test-network-k8s/config/org1/core.yaml index 27297bd0f1..0200457263 100644 --- a/test-network-k8s/config/org1/core.yaml +++ b/test-network-k8s/config/org1/core.yaml @@ -558,11 +558,18 @@ chaincode: # chaincode. The external builder detection processing will iterate over the # builders in the order specified below. externalBuilders: - - name: ccaas_builder - path: /opt/hyperledger/ccaas_builder - propagateEnvironment: - - CHAINCODE_AS_A_SERVICE_BUILDER_CONFIG - + - name: ccaas_builder + path: /opt/hyperledger/ccaas_builder + propagateEnvironment: + - CHAINCODE_AS_A_SERVICE_BUILDER_CONFIG + + # copied into the /var persistent volume share by "network up" + - name: k8s_builder + path: /var/hyperledger/fabric/external_builders/k8s_builder + propagateEnvironment: + - CORE_PEER_ID + - KUBERNETES_SERVICE_HOST + - KUBERNETES_SERVICE_PORT # The maximum duration to wait for the chaincode build and install process # to complete. diff --git a/test-network-k8s/config/org2/core.yaml b/test-network-k8s/config/org2/core.yaml index 3cd5aeefa6..8d9c1d97c7 100644 --- a/test-network-k8s/config/org2/core.yaml +++ b/test-network-k8s/config/org2/core.yaml @@ -558,11 +558,19 @@ chaincode: # chaincode. The external builder detection processing will iterate over the # builders in the order specified below. externalBuilders: - - name: ccaas_builder - path: /opt/hyperledger/ccaas_builder - propagateEnvironment: - - CHAINCODE_AS_A_SERVICE_BUILDER_CONFIG - + - name: ccaas_builder + path: /opt/hyperledger/ccaas_builder + propagateEnvironment: + - CHAINCODE_AS_A_SERVICE_BUILDER_CONFIG + + # copied into the /var persistent volume share by "network up" + - name: k8s_builder + path: /var/hyperledger/fabric/external_builders/k8s_builder + propagateEnvironment: + - CORE_PEER_ID + - KUBERNETES_SERVICE_HOST + - KUBERNETES_SERVICE_PORT + # The maximum duration to wait for the chaincode build and install process # to complete. installTimeout: 300s diff --git a/test-network-k8s/docs/CHAINCODE_AS_A_SERVICE.md b/test-network-k8s/docs/CHAINCODE_AS_A_SERVICE.md index cc8040c27f..e8a005fa21 100644 --- a/test-network-k8s/docs/CHAINCODE_AS_A_SERVICE.md +++ b/test-network-k8s/docs/CHAINCODE_AS_A_SERVICE.md @@ -28,7 +28,7 @@ network up network channel create ``` ``` -network chaincode deploy asset-transfer-basic basic_1.0 ${PWD} +network chaincode deploy asset-transfer-basic ${PWD} ``` ``` network chaincode metadata asset-transfer-basic @@ -55,7 +55,7 @@ kind load docker-image fabric-samples/asset-transfer-basic/chaincode-java ```shell # Assemble the chaincode package archive -network chaincode package basic_1.0 asset-transfer-basic $PWD/build/asset-transfer.tgz +network chaincode package asset-transfer-basic asset-transfer-basic $PWD/build/asset-transfer.tgz # Determine the ID for the chaincode package CORE_CHAINCODE_ID_NAME=$(network chaincode id $PWD/build/asset-transfer.tgz) diff --git a/test-network-k8s/kube/fabric-builder-role.yaml b/test-network-k8s/kube/fabric-builder-role.yaml new file mode 100644 index 0000000000..886bd91dd8 --- /dev/null +++ b/test-network-k8s/kube/fabric-builder-role.yaml @@ -0,0 +1,23 @@ +# +# Copyright IBM Corp. All Rights Reserved. +# +# SPDX-License-Identifier: Apache-2.0 +# +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: fabric-builder-role +rules: + - apiGroups: + - "" + - apps + resources: + - pods + - deployments + - configmaps + - secrets + verbs: + - get + - watch + - create diff --git a/test-network-k8s/kube/fabric-builder-rolebinding.yaml b/test-network-k8s/kube/fabric-builder-rolebinding.yaml new file mode 100644 index 0000000000..a29a9278b6 --- /dev/null +++ b/test-network-k8s/kube/fabric-builder-rolebinding.yaml @@ -0,0 +1,18 @@ +# +# Copyright IBM Corp. All Rights Reserved. +# +# SPDX-License-Identifier: Apache-2.0 +# +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: fabric-builder-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: fabric-builder-role +subjects: + - namespace: ${NS} + kind: ServiceAccount + name: default \ No newline at end of file diff --git a/test-network-k8s/kube/org1/org1-install-k8s-builder.yaml b/test-network-k8s/kube/org1/org1-install-k8s-builder.yaml new file mode 100644 index 0000000000..867f6ac87e --- /dev/null +++ b/test-network-k8s/kube/org1/org1-install-k8s-builder.yaml @@ -0,0 +1,33 @@ +# +# Copyright IBM Corp. All Rights Reserved. +# +# SPDX-License-Identifier: Apache-2.0 +# +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: org1-install-k8s-builder +spec: + backoffLimit: 0 + completions: 1 + template: + metadata: + name: org1-install-k8s-builder + spec: + restartPolicy: "Never" + containers: + - name: main + image: ghcr.io/hyperledgendary/k8s-fabric-peer:${K8S_CHAINCODE_BUILDER_VERSION} + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - "mkdir -p /mnt/fabric-org1/fabric/external_builders && cp -rv /opt/hyperledger/k8s_builder /mnt/fabric-org1/fabric/external_builders/" + volumeMounts: + - name: fabric-org1-volume + mountPath: /mnt/fabric-org1 + volumes: + - name: fabric-org1-volume + persistentVolumeClaim: + claimName: fabric-org1 \ No newline at end of file diff --git a/test-network-k8s/kube/org2/org2-install-k8s-builder.yaml b/test-network-k8s/kube/org2/org2-install-k8s-builder.yaml new file mode 100644 index 0000000000..79a5531de8 --- /dev/null +++ b/test-network-k8s/kube/org2/org2-install-k8s-builder.yaml @@ -0,0 +1,33 @@ +# +# Copyright IBM Corp. All Rights Reserved. +# +# SPDX-License-Identifier: Apache-2.0 +# +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: org2-install-k8s-builder +spec: + backoffLimit: 0 + completions: 1 + template: + metadata: + name: org2-install-k8s-builder + spec: + restartPolicy: "Never" + containers: + - name: main + image: ghcr.io/hyperledgendary/k8s-fabric-peer:${K8S_CHAINCODE_BUILDER_VERSION} + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - "mkdir -p /mnt/fabric-org2/fabric/external_builders && cp -rv /opt/hyperledger/k8s_builder /mnt/fabric-org2/fabric/external_builders/" + volumeMounts: + - name: fabric-org2-volume + mountPath: /mnt/fabric-org2 + volumes: + - name: fabric-org2-volume + persistentVolumeClaim: + claimName: fabric-org2 \ No newline at end of file diff --git a/test-network-k8s/network b/test-network-k8s/network index 47184e1a4b..450c416b32 100755 --- a/test-network-k8s/network +++ b/test-network-k8s/network @@ -30,8 +30,10 @@ export CLUSTER_NAME=${TEST_NETWORK_KIND_CLUSTER_NAME:-kind} export NS=${TEST_NETWORK_KUBE_NAMESPACE:-${NETWORK_NAME}} export DOMAIN=${TEST_NETWORK_DOMAIN:-vcap.me} export CHANNEL_NAME=${TEST_NETWORK_CHANNEL_NAME:-mychannel} -export ORDERER_TIMEOUT=${TEST_NETWORK_ORDERER_TIMEOUT:-10s} # see https://github.com/hyperledger/fabric/issues/3372 +export ORDERER_TIMEOUT=${TEST_NETWORK_ORDERER_TIMEOUT:-10s} # see https://github.com/hyperledger/fabric/issues/3372 export TEMP_DIR=${PWD}/build +export CHAINCODE_BUILDER=${TEST_NETWORK_CHAINCODE_BUILDER:-ccaas} # see https://github.com/hyperledgendary/fabric-builder-k8s/blob/main/docs/TEST_NETWORK_K8S.md +export K8S_CHAINCODE_BUILDER_VERSION=${TEST_NETWORK_K8S_CHAINCODE_BUILDER_VERSION:-v0.4.0} LOG_FILE=${TEST_NETWORK_LOG_FILE:-network.log} DEBUG_FILE=${TEST_NETWORK_DEBUG_FILE:-network-debug.log} diff --git a/test-network-k8s/scripts/chaincode.sh b/test-network-k8s/scripts/chaincode.sh index 09d3740a64..40f9dd47f3 100755 --- a/test-network-k8s/scripts/chaincode.sh +++ b/test-network-k8s/scripts/chaincode.sh @@ -69,40 +69,63 @@ function chaincode_command_group() { # Convenience routine to "do everything" required to bring up a sample CC. function deploy_chaincode() { local cc_name=$1 - local cc_label=$2 - local cc_folder=$(absolute_path $3) - + local cc_label=$1 + local cc_folder=$(absolute_path $2) local temp_folder=$(mktemp -d) local cc_package=${temp_folder}/${cc_name}.tgz - package_chaincode ${cc_label} ${cc_name} ${cc_package} - - set_chaincode_id ${cc_package} - set_chaincode_image ${cc_folder} + prepare_chaincode_image ${cc_folder} ${cc_name} + package_chaincode ${cc_name} ${cc_label} ${cc_package} - build_chaincode_image ${cc_folder} ${CHAINCODE_IMAGE} - - if [ "${CLUSTER_RUNTIME}" == "kind" ]; then - kind_load_image ${CHAINCODE_IMAGE} + if [ "${CHAINCODE_BUILDER}" == "ccaas" ]; then + set_chaincode_id ${cc_package} + launch_chaincode ${cc_name} ${CHAINCODE_ID} ${CHAINCODE_IMAGE} fi - launch_chaincode ${cc_name} ${CHAINCODE_ID} ${CHAINCODE_IMAGE} activate_chaincode ${cc_name} ${cc_package} } -# Infer a reasonable name for the chaincode image based on the folder path conventions, or -# allow the user to override with TEST_NETWORK_CHAINCODE_IMAGE. -function set_chaincode_image() { +# Prepare a chaincode image for use in a builder package. +# Sets the CHAINCODE_IMAGE environment variable +function prepare_chaincode_image() { local cc_folder=$1 + local cc_name=$2 + + build_chaincode_image ${cc_folder} ${cc_name} - if [ -z "$TEST_NETWORK_CHAINCODE_IMAGE" ]; then - # cc_folder path starting with first index of "fabric-samples" - CHAINCODE_IMAGE=${cc_folder/*fabric-samples/fabric-samples} + if [ "${CLUSTER_RUNTIME}" == "k3s" ]; then + # For rancher / k3s runtimes, bypass the local container registry and load images directly from the image cache. + export CHAINCODE_IMAGE=${cc_name} else - CHAINCODE_IMAGE=${TEST_NETWORK_CHAINCODE_IMAGE} + # For KIND and k8s-builder environments, publish the image to a local docker registry + export CHAINCODE_IMAGE=localhost:${LOCAL_REGISTRY_PORT}/${cc_name} + publish_chaincode_image ${cc_name} ${CHAINCODE_IMAGE} fi } +function build_chaincode_image() { + local cc_folder=$1 + local cc_name=$2 + + push_fn "Building chaincode image ${cc_name}" + + $CONTAINER_CLI build ${CONTAINER_NAMESPACE} -t ${cc_name} ${cc_folder} + + pop_fn +} + +# tag a docker image with a new name and publish to a remote container registry +function publish_chaincode_image() { + local cc_name=$1 + local cc_url=$2 + push_fn "Publishing chaincode image ${cc_url}" + + ${CONTAINER_CLI} tag ${cc_name} ${cc_url} + ${CONTAINER_CLI} push ${cc_url} + + pop_fn +} + # Convenience routine to "do everything other than package and launch" a sample CC. # When debugging a chaincode server, the process must be launched prior to completing # the chaincode lifecycle at the peer. This routine provides a route for packaging @@ -168,36 +191,67 @@ function invoke_chaincode() { sleep 2 } -function build_chaincode_image() { - local cc_folder=$1 - local cc_image=$2 +function package_chaincode() { - push_fn "Building chaincode image ${cc_image}" + if [ "${CHAINCODE_BUILDER}" == "k8s" ]; then + package_k8s_chaincode $@ - $CONTAINER_CLI build ${CONTAINER_NAMESPACE} -t ${cc_image} ${cc_folder} + elif [ "${CHAINCODE_BUILDER}" == "ccaas" ]; then + package_ccaas_chaincode $@ - pop_fn + else + log "Unknown CHAINCODE_BUILDER ${CHAINCODE_BUILDER}" + exit 1 + fi +} + +# The k8s builder expects EXACTLY an IMMUTABLE image digest referencing a SPECIFIC image layer at a container registry. +function package_k8s_chaincode() { + local cc_name=$1 + local cc_label=$2 + local cc_archive=$3 + + local cc_folder=$(dirname $cc_archive) + local archive_name=$(basename $cc_archive) + + push_fn "Packaging k8s chaincode ${cc_archive}" + + mkdir -p ${cc_folder} + + # Find the docker image digest associated with the image at the container registry + local cc_digest=$(${CONTAINER_CLI} inspect --format='{{index .RepoDigests 0}}' ${CHAINCODE_IMAGE} | cut -d'@' -f2) + + cat << IMAGEJSON-EOF > ${cc_folder}/image.json +{ + "name": "${CHAINCODE_IMAGE}", + "digest": "${cc_digest}" } +IMAGEJSON-EOF -function kind_load_image() { - local cc_image=$1 + cat << METADATAJSON-EOF > ${cc_folder}/metadata.json +{ + "type": "k8s", + "label": "${cc_label}" +} +METADATAJSON-EOF - push_fn "Loading chaincode to kind image plane" + tar -C ${cc_folder} -zcf ${cc_folder}/code.tar.gz image.json + tar -C ${cc_folder} -zcf ${cc_archive} code.tar.gz metadata.json - kind load docker-image ${cc_image} + rm ${cc_folder}/code.tar.gz pop_fn } -function package_chaincode() { - local cc_label=$1 - local cc_name=$2 +function package_ccaas_chaincode() { + local cc_name=$1 + local cc_label=$2 local cc_archive=$3 local cc_folder=$(dirname $cc_archive) local archive_name=$(basename $cc_archive) - push_fn "Packaging chaincode ${cc_label}" + push_fn "Packaging ccaas chaincode ${cc_label}" mkdir -p ${cc_folder} diff --git a/test-network-k8s/scripts/fabric_config.sh b/test-network-k8s/scripts/fabric_config.sh index c4a233ca3f..4bc31779ac 100755 --- a/test-network-k8s/scripts/fabric_config.sh +++ b/test-network-k8s/scripts/fabric_config.sh @@ -55,5 +55,23 @@ function load_org_config() { kubectl -n $NS create configmap org1-config --from-file=config/org1 kubectl -n $NS create configmap org2-config --from-file=config/org2 + pop_fn +} + +function apply_k8s_builder_roles() { + push_fn "Applying k8s chaincode builder roles" + + apply_template kube/fabric-builder-role.yaml + apply_template kube/fabric-builder-rolebinding.yaml + + pop_fn +} + +function apply_k8s_builders() { + push_fn "Installing k8s chaincode builders" + + apply_template kube/org1/org1-install-k8s-builder.yaml + apply_template kube/org2/org2-install-k8s-builder.yaml + pop_fn } \ No newline at end of file diff --git a/test-network-k8s/scripts/test_network.sh b/test-network-k8s/scripts/test_network.sh index 4e9b22222e..94aad93182 100755 --- a/test-network-k8s/scripts/test_network.sh +++ b/test-network-k8s/scripts/test_network.sh @@ -164,6 +164,12 @@ function network_up() { init_storage_volumes load_org_config + # Service account permissions for the k8s builder + if [ "${CHAINCODE_BUILDER}" == "k8s" ]; then + apply_k8s_builder_roles + apply_k8s_builders + fi + # Network TLS CAs init_tls_cert_issuers