Add GRPC Reflection Service to Enhance Dynamic Discovery

.github/workflows/build-publish.yaml

@@ -109,6 +109,9 @@ jobs:
           - name: cert-tool
             directory: tools/cert-tool
             file: tools/cert-tool/Dockerfile
+          - name: grpc-reflection
+            directory: tools/grpc-reflection
+            file: tools/grpc-reflection/Dockerfile
           - name: snippet-service
             directory: snippet-service
             file: .tmp/docker/snippet-service/Dockerfile

bundle/Dockerfile

@@ -120,6 +120,13 @@
     -o "/go/bin/$service" \
     ./cmd/service
 
+#grpc-reflection
+ARG service=grpc-reflection
+WORKDIR $root_directory/tools/$service
+RUN go build -ldflags "-linkmode external -extldflags -static -X github.com/plgd-dev/hub/v2/pkg/build.CommitDate=$COMMIT_DATE -X github.com/plgd-dev/hub/v2/pkg/build.CommitHash=$SHORT_COMMIT -X github.com/plgd-dev/hub/v2/pkg/build.BuildDate=$DATE -X github.com/plgd-dev/hub/v2/pkg/build.Version=$VERSION -X github.com/plgd-dev/hub/v2/pkg/build.ReleaseURL=$RELEASE_URL" \
+    -o "/go/bin/$service" \
+    ./cmd/service
+
 #nats
 WORKDIR $root_directory
 RUN apkArch="$(apk --print-arch)"; \
@@ -187,6 +194,8 @@
 COPY --from=build /go/src/github.com/plgd-dev/hub/bundle/nginx /nginx
 COPY --from=build /go/bin/snippet-service /usr/local/bin/snippet-service
 COPY --from=build /go/src/github.com/plgd-dev/hub/snippet-service/config.yaml /configs/snippet-service.yaml
+COPY --from=build /go/bin/grpc-reflection /usr/local/bin/grpc-reflection
+COPY --from=build /go/src/github.com/plgd-dev/hub/tools/grpc-reflection/config.yaml /configs/grpc-reflection.yaml
 
 # install scylla
 RUN curl -sSf get.scylladb.com/server | sudo bash -s -- --scylla-version 5.4
@@ -211,7 +220,7 @@
 ENV OPEN_TELEMETRY_EXPORTER_ENABLED=false
 ENV OPEN_TELEMETRY_EXPORTER_ADDRESS="localhost:4317"
 ENV OPEN_TELEMETRY_EXPORTER_CERT_FILE="/certs/otel/cert.crt"
 ENV OPEN_TELEMETRY_EXPORTER_KEY_FILE="/certs/otel/cert.key"
 ENV OPEN_TELEMETRY_EXPORTER_CA_POOL="/certs/otel/rootca.crt"
 
 # global - certificates generated by cert-tool
@@ -251,11 +260,12 @@
 ENV HTTP_SNIPPET_SERVICE_PORT=9092
 ENV M2M_OAUTH_SERVER_PORT=9079
 ENV HTTP_M2M_OAUTH_SERVER_PORT=9080
+ENV GRPC_REFLECTION_PORT=9078
 
 # OAuth
 ENV DEVICE_PROVIDER=plgd
 ENV DEVICE_OAUTH_SCOPES="offline_access"
 ENV OWNER_CLAIM="sub"
 ENV MOCK_OAUTH_SERVER_ACCESS_TOKEN_LIFETIME="0s"
 
 ENTRYPOINT ["/usr/local/bin/run.sh"]

bundle/nginx/nginx.conf.template

@@ -148,6 +148,19 @@ http {
       proxy_set_header Connection $connection_upgrade;
       proxy_set_header Host $host;
     }
+    location ~ ^(/grpc.reflection.v1alpha.ServerReflection|/grpc.reflection.v1.ServerReflection) {
+      set $upstream_grpc_gateway grpcs://127.0.0.1:REPLACE_GRPC_REFLECTION_PORT;
+      grpc_pass                      $upstream_grpc_gateway;
+      grpc_ssl_certificate           /data/certs/internal/endpoint.crt;
+      grpc_ssl_certificate_key       /data/certs/internal/endpoint.key;
+      grpc_ssl_trusted_certificate   /data/certs/root_ca.crt;
+      grpc_read_timeout       31536000s;
+      grpc_send_timeout       31536000s;
+      client_body_timeout     31536000s;
+      grpc_socket_keepalive   on;
+      proxy_ssl_trusted_certificate   /data/certs/root_ca.crt;
+      proxy_ssl_verify                on;
+    }
     location ~ ^/ {
       set $upstream_http_gateway https://127.0.0.1:REPLACE_HTTP_GATEWAY_PORT;
       proxy_pass                     $upstream_http_gateway;

bundle/run.sh

@@ -28,6 +28,7 @@ export SNIPPET_SERVICE_ADDRESS="localhost:${SNIPPET_SERVICE_PORT}"
 export SNIPPET_SERVICE_HTTP_ADDRESS="localhost:${HTTP_SNIPPET_SERVICE_PORT}"
 export M2M_OAUTH_SERVER_ADDRESS="localhost:${M2M_OAUTH_SERVER_PORT}"
 export M2M_OAUTH_SERVER_HTTP_ADDRESS="localhost:${HTTP_M2M_OAUTH_SERVER_PORT}"
+export GRPC_REFLECTION_ADDRESS="localhost:${GRPC_REFLECTION_PORT}"
 
 export INTERNAL_CERT_DIR_PATH="$CERTIFICATES_PATH/internal"
 export GRPC_INTERNAL_CERT_NAME="endpoint.crt"
@@ -462,6 +463,58 @@ while read -r line; do
   fi
 done < <(yq e '[.. | select(has("keyFile")) | .keyFile]' /configs/cloud2cloud-connector.yaml | sort | uniq)
 
+## snippet-service
+### setup root-cas
+while read -r line; do
+  file=`echo $line | yq e '.[0]' - `
+  mkdir -p `dirname ${file}`
+  if [ "${OVERRIDE_FILES}" = "true" ] || [ ! -f "${file}" ]; then
+    cp $CA_POOL ${file}
+  fi
+done < <(yq e '[.. | select(has("caPool")) | .caPool]' /configs/snippet-service.yaml | sort | uniq)
+### setup certificates
+while read -r line; do
+  file=`echo $line | yq e '.[0]' - `
+  mkdir -p `dirname ${file}`
+  if [ "${OVERRIDE_FILES}" = "true" ] || [ ! -f "${file}" ]; then
+     cp $CERT_FILE ${file}
+  fi
+done < <(yq e '[.. | select(has("certFile")) | .certFile]' /configs/snippet-service.yaml | sort | uniq)
+### setup private keys
+while read -r line; do
+  file=`echo $line | yq e '.[0]' - `
+  mkdir -p `dirname ${file}`
+  if [ "${OVERRIDE_FILES}" = "true" ] || [ ! -f "${file}" ]; then
+     cp $KEY_FILE ${file}
+  fi
+done < <(yq e '[.. | select(has("keyFile")) | .keyFile]' /configs/snippet-service.yaml| sort | uniq)
+
+## grpc-reflection
+### setup root-cas
+while read -r line; do
+  file=`echo $line | yq e '.[0]' - `
+  mkdir -p `dirname ${file}`
+  if [ "${OVERRIDE_FILES}" = "true" ] || [ ! -f "${file}" ]; then
+    cp $CA_POOL ${file}
+  fi
+done < <(yq e '[.. | select(has("caPool")) | .caPool]' /configs/grpc-reflection.yaml | sort | uniq)
+### setup certificates
+while read -r line; do
+  file=`echo $line | yq e '.[0]' - `
+  mkdir -p `dirname ${file}`
+  if [ "${OVERRIDE_FILES}" = "true" ] || [ ! -f "${file}" ]; then
+     cp $CERT_FILE ${file}
+  fi
+done < <(yq e '[.. | select(has("certFile")) | .certFile]' /configs/grpc-reflection.yaml | sort | uniq)
+### setup private keys
+while read -r line; do
+  file=`echo $line | yq e '.[0]' - `
+  mkdir -p `dirname ${file}`
+  if [ "${OVERRIDE_FILES}" = "true" ] || [ ! -f "${file}" ]; then
+     cp $KEY_FILE ${file}
+  fi
+done < <(yq e '[.. | select(has("keyFile")) | .keyFile]' /configs/grpc-reflection.yaml| sort | uniq)
+
 mkdir -p ${OAUTH_KEYS_PATH}
 if [ "${OVERRIDE_FILES}" = "true" ] || [ ! -f "${OAUTH_ID_TOKEN_KEY_PATH}" ]; then
   openssl genrsa -out ${OAUTH_ID_TOKEN_KEY_PATH} 4096
@@ -491,6 +544,7 @@ if [ "${OVERRIDE_FILES}" = "true" ] || [ ! -f "${NGINX_PATH}/nginx.conf" ]; then
   sed -i "s/REPLACE_HTTP_SNIPPET_SERVICE_PORT/$HTTP_SNIPPET_SERVICE_PORT/g" ${NGINX_PATH}/nginx.conf
   sed -i "s/REPLACE_M2M_OAUTH_SERVER_PORT/$M2M_OAUTH_SERVER_PORT/g" ${NGINX_PATH}/nginx.conf
   sed -i "s/REPLACE_HTTP_M2M_OAUTH_SERVER_PORT/$HTTP_M2M_OAUTH_SERVER_PORT/g" ${NGINX_PATH}/nginx.conf
+  sed -i "s/REPLACE_GRPC_REFLECTION_PORT/$GRPC_REFLECTION_PORT/g" ${NGINX_PATH}/nginx.conf
 fi
 
 # nats
@@ -1356,6 +1410,44 @@ while true; do
   sleep 1
 done
 
+# grpc-reflection
+echo "starting grpc-reflection"
+## configuration
+if [ "${OVERRIDE_FILES}" = "true" ] || [ ! -f "/data/grpc-reflection.yaml" ]; then
+cat /configs/grpc-reflection.yaml |
+  yq e "\
+  .hubID = \"${HUB_ID}\" |
+  .log.level = \"${LOG_LEVEL}\" |
+  .apis.grpc.address = \"${GRPC_REFLECTION_ADDRESS}\" |
+  .clients.openTelemetryCollector.grpc.enabled = ${OPEN_TELEMETRY_EXPORTER_ENABLED} |
+  .clients.openTelemetryCollector.grpc.address = \"${OPEN_TELEMETRY_EXPORTER_ADDRESS}\" |
+  .clients.openTelemetryCollector.grpc.tls.caPool = \"${OPEN_TELEMETRY_EXPORTER_CA_POOL}\" |
+  .clients.openTelemetryCollector.grpc.tls.keyFile = \"${OPEN_TELEMETRY_EXPORTER_KEY_FILE}\" |
+  .clients.openTelemetryCollector.grpc.tls.certFile = \"${OPEN_TELEMETRY_EXPORTER_CERT_FILE}\" |
+  .clients.openTelemetryCollector.grpc.tls.useSystemCAPool = true
+" - > /data/grpc-reflection.yaml
+fi
+grpc-reflection --config /data/grpc-reflection.yaml >$LOGS_PATH/grpc-reflection.log 2>&1 &
+status=$?
+grpc_reflection_pid=$!
+if [ $status -ne 0 ]; then
+  echo "Failed to start grpc-reflection: $status"
+  sync
+  cat $LOGS_PATH/grpc-reflection.log
+  exit $status
+fi
+
+# waiting for ca. Without wait, sometimes the service didn't connect.
+i=0
+while true; do
+  i=$((i+1))
+  if openssl s_client -connect ${GRPC_REFLECTION_ADDRESS} -cert ${INTERNAL_CERT_DIR_PATH}/${GRPC_INTERNAL_CERT_NAME} -key ${INTERNAL_CERT_DIR_PATH}/${GRPC_INTERNAL_CERT_KEY_NAME} <<< "Q" 2>/dev/null > /dev/null; then
+    break
+  fi
+  echo "Try to reconnect to grpc-reflection(${GRPC_REFLECTION_ADDRESS}) $i"
+  sleep 1
+done
+
 echo "Open browser at https://${DOMAIN}"
 
 # Naive check runs checks once a minute to see if either of the processes exited.
@@ -1487,4 +1579,11 @@ while sleep 10; do
     cat $LOGS_PATH/snippet-service.log
    exit 1
   fi
+  ps aux |grep $grpc_reflection_pid |grep -q -v grep
+  if [ $? -ne 0 ]; then
+    echo "grpc-reflection has already exited."
+    sync
+    cat $LOGS_PATH/grpc-reflection.log
+   exit 1
+  fi
 done

certificate-authority/Makefile

@@ -49,6 +49,7 @@ GRPCGATEWAY_MODULE_PATH := $(shell go list -m -f '{{.Dir}}' github.com/grpc-ecos
 proto/generate:
 	protoc -I=. -I=$(REPOSITORY_DIRECTORY) -I=$(GOPATH)/src --go_out=$(GOPATH)/src $(WORKING_DIRECTORY)/pb/cert.proto
 	protoc -I=. -I=$(REPOSITORY_DIRECTORY) -I=$(GOPATH)/src --go_out=$(GOPATH)/src $(WORKING_DIRECTORY)/pb/signingRecords.proto
+	protoc -I=. -I=$(REPOSITORY_DIRECTORY) -I=$(GOPATH)/src -I=$(GOOGLEAPIS_PATH) -I=$(GRPCGATEWAY_MODULE_PATH) --go_out=$(GOPATH)/src $(WORKING_DIRECTORY)/pb/service.proto
 	protoc-go-inject-tag -input=$(WORKING_DIRECTORY)/pb/signingRecords.pb.go
 	protoc -I=. -I=$(REPOSITORY_DIRECTORY) -I=$(GOPATH)/src -I=$(GOOGLEAPIS_PATH) -I=$(GRPCGATEWAY_MODULE_PATH) --openapiv2_out=$(REPOSITORY_DIRECTORY) \
 		--openapiv2_opt logtostderr=true \