chore: Terraform template for prover-node (#7846)

Adds a new tf template for prover-node. Updates the node so it no longer
submits proofs, and updates the prover agent so it connects to the
prover-node instead of the node.

---------

Co-authored-by: PhilWindle <philip.windle@gmail.com>

.github/workflows/devnet-deploys.yml

@@ -17,8 +17,9 @@ concurrency:
 
 # Anvil Accounts. Anvil provides 10 pre-funded accounts for the mnemonic we have specified in FORK_MNEMONIC. We are using:
 # 1. The first account (index 0) is used in SEQ_1_PUBLISHER_PRIVATE_KEY
-# 2. The 9th account (index 8) is used in this workflow for deploying contracts etc
-# 3. The 10th account (index 9) is used by the deployed faucet
+# 2. The 3rd account (index 2) is used in PROVER_1_PUBLISHER_PRIVATE_KEY
+# 3. The 9th account (index 8) is used in this workflow for deploying contracts etc
+# 4. The 10th account (index 9) is used by the deployed faucet
 # TODO: Convert all this so we take the provided mnemonic and derive the keys from the above indices
 env:
   DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
@@ -51,6 +52,10 @@ env:
   TF_VAR_NODE_P2P_UDP_PORT: 45000
   TF_VAR_NODE_LB_RULE_PRIORITY: 500
 
+  # Prover Node
+  TF_VAR_PROVER_NODE_LB_RULE_PRIORITY: 6000
+  TF_VAR_PROVER_PRIVATE_KEYS: '["${{ secrets.PROVER_1_PUBLISHER_PRIVATE_KEY }}"]'
+
   # Anvil
   TF_VAR_FORK_MNEMONIC: ${{ secrets.FORK_MNEMONIC }}
   TF_VAR_INFURA_API_KEY: ${{ secrets.INFURA_API_KEY }}
@@ -86,8 +91,6 @@ jobs:
     secrets: inherit
 
   # Set network specific variables as outputs from this job to be referenced in later jobs
-  # The only exception is the network api key which needs to be re-derived in every job as it is a secret
-  # Secrets can't be passed between jobs
   set-network:
     needs: setup
     runs-on: ${{ github.actor }}-x86
@@ -99,8 +102,11 @@ jobs:
       bot_interval: ${{ steps.set_network_vars.outputs.bot_interval }}
       node_tcp_range_start: ${{ steps.set_network_vars.outputs.node_tcp_range_start }}
       node_udp_range_start: ${{ steps.set_network_vars.outputs.node_udp_range_start }}
+      prover_node_tcp_range_start: ${{ steps.set_network_vars.outputs.prover_node_tcp_range_start }}
+      prover_node_udp_range_start: ${{ steps.set_network_vars.outputs.prover_node_udp_range_start }}
       node_lb_priority_range_start: ${{ steps.set_network_vars.outputs.node_lb_priority_range_start }}
       pxe_lb_priority_range_start: ${{ steps.set_network_vars.outputs.pxe_lb_priority_range_start }}
+      prover_node_lb_priority_range_start: ${{ steps.set_network_vars.outputs.prover_node_lb_priority_range_start }}
       faucet_lb_priority: ${{ steps.set_network_vars.outputs.faucet_lb_priority }}
       bot_no_wait: ${{ steps.set_network_vars.outputs.bot_no_wait }}
       max_txs_per_block: ${{ steps.set_network_vars.outputs.max_txs_per_block }}
@@ -119,8 +125,11 @@ jobs:
               echo "bot_interval=30" >> $GITHUB_OUTPUT
               echo "node_tcp_range_start=40100" >> $GITHUB_OUTPUT
               echo "node_udp_range_start=45100" >> $GITHUB_OUTPUT
+              echo "prover_node_tcp_range_start=41100" >> $GITHUB_OUTPUT
+              echo "prover_node_udp_range_start=46100" >> $GITHUB_OUTPUT
               echo "node_lb_priority_range_start=4100" >> $GITHUB_OUTPUT
               echo "pxe_lb_priority_range_start=5100" >> $GITHUB_OUTPUT
+              echo "prover_node_lb_priority_range_start=6100" >> $GITHUB_OUTPUT
               echo "faucet_lb_priority=601" >> $GITHUB_OUTPUT
               echo "bot_no_wait=false" >> $GITHUB_OUTPUT
               echo "max_txs_per_block=64" >> $GITHUB_OUTPUT
@@ -133,8 +142,11 @@ jobs:
               echo "bot_interval=300" >> $GITHUB_OUTPUT
               echo "node_tcp_range_start=40200" >> $GITHUB_OUTPUT
               echo "node_udp_range_start=45200" >> $GITHUB_OUTPUT
+              echo "prover_node_tcp_range_start=41200" >> $GITHUB_OUTPUT
+              echo "prover_node_udp_range_start=46200" >> $GITHUB_OUTPUT
               echo "node_lb_priority_range_start=4200" >> $GITHUB_OUTPUT
               echo "pxe_lb_priority_range_start=5200" >> $GITHUB_OUTPUT
+              echo "prover_node_lb_priority_range_start=6200" >> $GITHUB_OUTPUT
               echo "faucet_lb_priority=602" >> $GITHUB_OUTPUT
               echo "bot_no_wait=true" >> $GITHUB_OUTPUT
               echo "max_txs_per_block=4" >> $GITHUB_OUTPUT
@@ -147,8 +159,11 @@ jobs:
               echo "bot_interval=30" >> $GITHUB_OUTPUT
               echo "node_tcp_range_start=40000" >> $GITHUB_OUTPUT
               echo "node_udp_range_start=45000" >> $GITHUB_OUTPUT
+              echo "prover_node_tcp_range_start=41000" >> $GITHUB_OUTPUT
+              echo "prover_node_udp_range_start=46000" >> $GITHUB_OUTPUT
               echo "node_lb_priority_range_start=4000" >> $GITHUB_OUTPUT
               echo "pxe_lb_priority_range_start=5000" >> $GITHUB_OUTPUT
+              echo "prover_node_lb_priority_range_start=6000" >> $GITHUB_OUTPUT
               echo "faucet_lb_priority=600" >> $GITHUB_OUTPUT
               echo "bot_no_wait=false" >> $GITHUB_OUTPUT
               echo "max_txs_per_block=64" >> $GITHUB_OUTPUT
@@ -344,10 +359,9 @@ jobs:
       TF_VAR_API_KEY: ${{ secrets[needs.set-network.outputs.network_api_key] }}
       TF_VAR_AGENTS_PER_PROVER: ${{ needs.set-network.outputs.agents_per_prover }}
       TF_VAR_BOT_TX_INTERVAL_SECONDS: ${{ needs.set-network.outputs.bot_interval }}
-      TF_VAR_NODE_P2P_TCP_PORT: ${{ needs.set-network.outputs.node_tcp_range_start }}
-      TF_VAR_NODE_P2P_UDP_PORT: ${{ needs.set-network.outputs.node_udp_range_start }}
       TF_VAR_NODE_LB_RULE_PRIORITY: ${{ needs.set-network.outputs.node_lb_priority_range_start }}
       TF_VAR_PXE_LB_RULE_PRIORITY: ${{ needs.set-network.outputs.pxe_lb_priority_range_start }}
+      TF_VAR_PROVER_NODE_LB_RULE_PRIORITY: ${{ needs.set-network.outputs.prover_node_lb_priority_range_start }}
       TF_VAR_BOT_NO_WAIT_FOR_TRANSFERS: ${{ needs.set-network.outputs.bot_no_wait }}
       TF_VAR_SEQ_MAX_TX_PER_BLOCK: ${{ needs.set-network.outputs.max_txs_per_block }}
     steps:
@@ -418,15 +432,17 @@ jobs:
           terraform init -input=false -backend-config="key=${{ env.DEPLOY_TAG }}/bot"
           terraform apply -input=false -auto-approve
 
-      - name: Init Aztec Node Terraform
+      - name: Deploy Aztec Nodes
         working-directory: ./yarn-project/aztec/terraform/node
         run: |
           terraform init -input=false -backend-config="key=${{ env.DEPLOY_TAG }}/aztec-node"
+          terraform apply -input=false -auto-approve -replace="aws_efs_file_system.node_data_store" -var="NODE_P2P_TCP_PORT=${{ needs.set-network.outputs.node_tcp_range_start }}" -var="NODE_P2P_UDP_PORT=${{ needs.set-network.outputs.node_udp_range_start }}"
 
-      - name: Deploy Aztec Nodes
-        working-directory: ./yarn-project/aztec/terraform/node
+      - name: Deploy Aztec Prover Nodes
+        working-directory: ./yarn-project/aztec/terraform/prover-node
         run: |
-          terraform apply -input=false -auto-approve -replace="aws_efs_file_system.node_data_store"
+          terraform init -input=false -backend-config="key=${{ env.DEPLOY_TAG }}/aztec-prover-node"
+          terraform apply -input=false -auto-approve -var="NODE_P2P_TCP_PORT=${{ needs.set-network.outputs.prover_node_tcp_range_start }}" -var="NODE_P2P_UDP_PORT=${{ needs.set-network.outputs.prover_node_udp_range_start }}"
 
       - name: Deploy Provers
         working-directory: ./yarn-project/aztec/terraform/prover
@@ -545,10 +561,9 @@ jobs:
       TF_VAR_API_KEY: ${{ secrets[needs.set-network.outputs.network_api_key] }}
       TF_VAR_AGENTS_PER_PROVER: ${{ needs.set-network.outputs.agents_per_prover }}
       TF_VAR_BOT_TX_INTERVAL_SECONDS: ${{ needs.set-network.outputs.bot_interval }}
-      TF_VAR_NODE_P2P_TCP_PORT: ${{ needs.set-network.outputs.node_tcp_range_start }}
-      TF_VAR_NODE_P2P_UDP_PORT: ${{ needs.set-network.outputs.node_udp_range_start }}
       TF_VAR_NODE_LB_RULE_PRIORITY: ${{ needs.set-network.outputs.node_lb_priority_range_start }}
       TF_VAR_PXE_LB_RULE_PRIORITY: ${{ needs.set-network.outputs.pxe_lb_priority_range_start }}
+      TF_VAR_PROVER_NODE_LB_RULE_PRIORITY: ${{ needs.set-network.outputs.prover_node_lb_priority_range_start }}
       TF_VAR_BOT_NO_WAIT_FOR_TRANSFERS: ${{ needs.set-network.outputs.bot_no_wait }}
       TF_VAR_SEQ_MAX_TX_PER_BLOCK: ${{ needs.set-network.outputs.max_txs_per_block }}
       TF_VAR_PROVING_ENABLED: true
@@ -580,7 +595,13 @@ jobs:
         run: |
           env
           terraform init -input=false -backend-config="key=${{ env.DEPLOY_TAG }}/aztec-node"
-          terraform apply -input=false -auto-approve
+          terraform apply -input=false -auto-approve -var="NODE_P2P_TCP_PORT=${{ needs.set-network.outputs.node_tcp_range_start }}" -var="NODE_P2P_UDP_PORT=${{ needs.set-network.outputs.node_udp_range_start }}"
+
+      - name: Deploy Aztec Prover Nodes
+        working-directory: ./yarn-project/aztec/terraform/prover-node
+        run: |
+          terraform init -input=false -backend-config="key=${{ env.DEPLOY_TAG }}/aztec-prover-node"
+          terraform apply -input=false -auto-approve -var="NODE_P2P_TCP_PORT=${{ needs.set-network.outputs.prover_node_tcp_range_start }}" -var="NODE_P2P_UDP_PORT=${{ needs.set-network.outputs.prover_node_udp_range_start }}"
 
       - name: Deploy Provers
         working-directory: ./yarn-project/aztec/terraform/prover

yarn-project/aztec/terraform/node/main.tf

@@ -163,7 +163,7 @@ resource "aws_ecs_task_definition" "aztec-node" {
     {
       name              = "${var.DEPLOY_TAG}-aztec-node-${count.index + 1}"
       image             = "${var.DOCKERHUB_ACCOUNT}/aztec:${var.IMAGE_TAG}"
-      command           = ["start", "--node", "--archiver", "--sequencer", "--prover"]
+      command           = ["start", "--node", "--archiver", "--sequencer"]
       essential         = true
       memoryReservation = 3776
       portMappings = [
@@ -208,6 +208,10 @@ resource "aws_ecs_task_definition" "aztec-node" {
           name  = "DATA_DIRECTORY"
           value = "${local.data_dir}/node_${count.index + 1}/data"
         },
+        {
+          name  = "IS_DEV_NET"
+          value = "true"
+        },
         {
           name  = "ARCHIVER_POLLING_INTERVAL"
           value = "10000"

yarn-project/aztec/terraform/node/variables.tf

@@ -20,18 +20,15 @@ variable "NODE_P2P_PRIVATE_KEYS" {
 }
 
 variable "L1_CHAIN_ID" {
-  type    = string
-  default = 677692
+  type = string
 }
 
 variable "NODE_P2P_TCP_PORT" {
-  type    = number
-  default = 40000
+  type = number
 }
 
 variable "NODE_P2P_UDP_PORT" {
-  type    = number
-  default = 45000
+  type = number
 }
 
 variable "DOCKERHUB_ACCOUNT" {
@@ -89,6 +86,5 @@ variable "BOOTSTRAP_NODES" {
 }
 
 variable "NODE_LB_RULE_PRIORITY" {
-  type    = number
-  default = 4000
+  type = number
 }