diff --git a/.github/workflows/devnet-deploys.yml b/.github/workflows/devnet-deploys.yml index 11a39129498..03383e3f480 100644 --- a/.github/workflows/devnet-deploys.yml +++ b/.github/workflows/devnet-deploys.yml @@ -17,8 +17,9 @@ concurrency: # Anvil Accounts. Anvil provides 10 pre-funded accounts for the mnemonic we have specified in FORK_MNEMONIC. We are using: # 1. The first account (index 0) is used in SEQ_1_PUBLISHER_PRIVATE_KEY -# 2. The 9th account (index 8) is used in this workflow for deploying contracts etc -# 3. The 10th account (index 9) is used by the deployed faucet +# 2. The 3rd account (index 2) is used in PROVER_1_PUBLISHER_PRIVATE_KEY +# 3. The 9th account (index 8) is used in this workflow for deploying contracts etc +# 4. The 10th account (index 9) is used by the deployed faucet # TODO: Convert all this so we take the provided mnemonic and derive the keys from the above indices env: DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} @@ -51,6 +52,10 @@ env: TF_VAR_NODE_P2P_UDP_PORT: 45000 TF_VAR_NODE_LB_RULE_PRIORITY: 500 + # Prover Node + TF_VAR_PROVER_NODE_LB_RULE_PRIORITY: 6000 + TF_VAR_PROVER_PRIVATE_KEYS: '["${{ secrets.PROVER_1_PUBLISHER_PRIVATE_KEY }}"]' + # Anvil TF_VAR_FORK_MNEMONIC: ${{ secrets.FORK_MNEMONIC }} TF_VAR_INFURA_API_KEY: ${{ secrets.INFURA_API_KEY }} @@ -86,8 +91,6 @@ jobs: secrets: inherit # Set network specific variables as outputs from this job to be referenced in later jobs - # The only exception is the network api key which needs to be re-derived in every job as it is a secret - # Secrets can't be passed between jobs set-network: needs: setup runs-on: ${{ github.actor }}-x86 @@ -99,8 +102,11 @@ jobs: bot_interval: ${{ steps.set_network_vars.outputs.bot_interval }} node_tcp_range_start: ${{ steps.set_network_vars.outputs.node_tcp_range_start }} node_udp_range_start: ${{ steps.set_network_vars.outputs.node_udp_range_start }} + prover_node_tcp_range_start: ${{ steps.set_network_vars.outputs.prover_node_tcp_range_start }} + prover_node_udp_range_start: ${{ steps.set_network_vars.outputs.prover_node_udp_range_start }} node_lb_priority_range_start: ${{ steps.set_network_vars.outputs.node_lb_priority_range_start }} pxe_lb_priority_range_start: ${{ steps.set_network_vars.outputs.pxe_lb_priority_range_start }} + prover_node_lb_priority_range_start: ${{ steps.set_network_vars.outputs.prover_node_lb_priority_range_start }} faucet_lb_priority: ${{ steps.set_network_vars.outputs.faucet_lb_priority }} bot_no_wait: ${{ steps.set_network_vars.outputs.bot_no_wait }} max_txs_per_block: ${{ steps.set_network_vars.outputs.max_txs_per_block }} @@ -119,8 +125,11 @@ jobs: echo "bot_interval=30" >> $GITHUB_OUTPUT echo "node_tcp_range_start=40100" >> $GITHUB_OUTPUT echo "node_udp_range_start=45100" >> $GITHUB_OUTPUT + echo "prover_node_tcp_range_start=41100" >> $GITHUB_OUTPUT + echo "prover_node_udp_range_start=46100" >> $GITHUB_OUTPUT echo "node_lb_priority_range_start=4100" >> $GITHUB_OUTPUT echo "pxe_lb_priority_range_start=5100" >> $GITHUB_OUTPUT + echo "prover_node_lb_priority_range_start=6100" >> $GITHUB_OUTPUT echo "faucet_lb_priority=601" >> $GITHUB_OUTPUT echo "bot_no_wait=false" >> $GITHUB_OUTPUT echo "max_txs_per_block=64" >> $GITHUB_OUTPUT @@ -133,8 +142,11 @@ jobs: echo "bot_interval=300" >> $GITHUB_OUTPUT echo "node_tcp_range_start=40200" >> $GITHUB_OUTPUT echo "node_udp_range_start=45200" >> $GITHUB_OUTPUT + echo "prover_node_tcp_range_start=41200" >> $GITHUB_OUTPUT + echo "prover_node_udp_range_start=46200" >> $GITHUB_OUTPUT echo "node_lb_priority_range_start=4200" >> $GITHUB_OUTPUT echo "pxe_lb_priority_range_start=5200" >> $GITHUB_OUTPUT + echo "prover_node_lb_priority_range_start=6200" >> $GITHUB_OUTPUT echo "faucet_lb_priority=602" >> $GITHUB_OUTPUT echo "bot_no_wait=true" >> $GITHUB_OUTPUT echo "max_txs_per_block=4" >> $GITHUB_OUTPUT @@ -147,8 +159,11 @@ jobs: echo "bot_interval=30" >> $GITHUB_OUTPUT echo "node_tcp_range_start=40000" >> $GITHUB_OUTPUT echo "node_udp_range_start=45000" >> $GITHUB_OUTPUT + echo "prover_node_tcp_range_start=41000" >> $GITHUB_OUTPUT + echo "prover_node_udp_range_start=46000" >> $GITHUB_OUTPUT echo "node_lb_priority_range_start=4000" >> $GITHUB_OUTPUT echo "pxe_lb_priority_range_start=5000" >> $GITHUB_OUTPUT + echo "prover_node_lb_priority_range_start=6000" >> $GITHUB_OUTPUT echo "faucet_lb_priority=600" >> $GITHUB_OUTPUT echo "bot_no_wait=false" >> $GITHUB_OUTPUT echo "max_txs_per_block=64" >> $GITHUB_OUTPUT @@ -344,10 +359,9 @@ jobs: TF_VAR_API_KEY: ${{ secrets[needs.set-network.outputs.network_api_key] }} TF_VAR_AGENTS_PER_PROVER: ${{ needs.set-network.outputs.agents_per_prover }} TF_VAR_BOT_TX_INTERVAL_SECONDS: ${{ needs.set-network.outputs.bot_interval }} - TF_VAR_NODE_P2P_TCP_PORT: ${{ needs.set-network.outputs.node_tcp_range_start }} - TF_VAR_NODE_P2P_UDP_PORT: ${{ needs.set-network.outputs.node_udp_range_start }} TF_VAR_NODE_LB_RULE_PRIORITY: ${{ needs.set-network.outputs.node_lb_priority_range_start }} TF_VAR_PXE_LB_RULE_PRIORITY: ${{ needs.set-network.outputs.pxe_lb_priority_range_start }} + TF_VAR_PROVER_NODE_LB_RULE_PRIORITY: ${{ needs.set-network.outputs.prover_node_lb_priority_range_start }} TF_VAR_BOT_NO_WAIT_FOR_TRANSFERS: ${{ needs.set-network.outputs.bot_no_wait }} TF_VAR_SEQ_MAX_TX_PER_BLOCK: ${{ needs.set-network.outputs.max_txs_per_block }} steps: @@ -418,15 +432,17 @@ jobs: terraform init -input=false -backend-config="key=${{ env.DEPLOY_TAG }}/bot" terraform apply -input=false -auto-approve - - name: Init Aztec Node Terraform + - name: Deploy Aztec Nodes working-directory: ./yarn-project/aztec/terraform/node run: | terraform init -input=false -backend-config="key=${{ env.DEPLOY_TAG }}/aztec-node" + terraform apply -input=false -auto-approve -replace="aws_efs_file_system.node_data_store" -var="NODE_P2P_TCP_PORT=${{ needs.set-network.outputs.node_tcp_range_start }}" -var="NODE_P2P_UDP_PORT=${{ needs.set-network.outputs.node_udp_range_start }}" - - name: Deploy Aztec Nodes - working-directory: ./yarn-project/aztec/terraform/node + - name: Deploy Aztec Prover Nodes + working-directory: ./yarn-project/aztec/terraform/prover-node run: | - terraform apply -input=false -auto-approve -replace="aws_efs_file_system.node_data_store" + terraform init -input=false -backend-config="key=${{ env.DEPLOY_TAG }}/aztec-prover-node" + terraform apply -input=false -auto-approve -var="NODE_P2P_TCP_PORT=${{ needs.set-network.outputs.prover_node_tcp_range_start }}" -var="NODE_P2P_UDP_PORT=${{ needs.set-network.outputs.prover_node_udp_range_start }}" - name: Deploy Provers working-directory: ./yarn-project/aztec/terraform/prover @@ -545,10 +561,9 @@ jobs: TF_VAR_API_KEY: ${{ secrets[needs.set-network.outputs.network_api_key] }} TF_VAR_AGENTS_PER_PROVER: ${{ needs.set-network.outputs.agents_per_prover }} TF_VAR_BOT_TX_INTERVAL_SECONDS: ${{ needs.set-network.outputs.bot_interval }} - TF_VAR_NODE_P2P_TCP_PORT: ${{ needs.set-network.outputs.node_tcp_range_start }} - TF_VAR_NODE_P2P_UDP_PORT: ${{ needs.set-network.outputs.node_udp_range_start }} TF_VAR_NODE_LB_RULE_PRIORITY: ${{ needs.set-network.outputs.node_lb_priority_range_start }} TF_VAR_PXE_LB_RULE_PRIORITY: ${{ needs.set-network.outputs.pxe_lb_priority_range_start }} + TF_VAR_PROVER_NODE_LB_RULE_PRIORITY: ${{ needs.set-network.outputs.prover_node_lb_priority_range_start }} TF_VAR_BOT_NO_WAIT_FOR_TRANSFERS: ${{ needs.set-network.outputs.bot_no_wait }} TF_VAR_SEQ_MAX_TX_PER_BLOCK: ${{ needs.set-network.outputs.max_txs_per_block }} TF_VAR_PROVING_ENABLED: true @@ -580,7 +595,13 @@ jobs: run: | env terraform init -input=false -backend-config="key=${{ env.DEPLOY_TAG }}/aztec-node" - terraform apply -input=false -auto-approve + terraform apply -input=false -auto-approve -var="NODE_P2P_TCP_PORT=${{ needs.set-network.outputs.node_tcp_range_start }}" -var="NODE_P2P_UDP_PORT=${{ needs.set-network.outputs.node_udp_range_start }}" + + - name: Deploy Aztec Prover Nodes + working-directory: ./yarn-project/aztec/terraform/prover-node + run: | + terraform init -input=false -backend-config="key=${{ env.DEPLOY_TAG }}/aztec-prover-node" + terraform apply -input=false -auto-approve -var="NODE_P2P_TCP_PORT=${{ needs.set-network.outputs.prover_node_tcp_range_start }}" -var="NODE_P2P_UDP_PORT=${{ needs.set-network.outputs.prover_node_udp_range_start }}" - name: Deploy Provers working-directory: ./yarn-project/aztec/terraform/prover diff --git a/yarn-project/aztec/terraform/node/main.tf b/yarn-project/aztec/terraform/node/main.tf index e64f28a961d..4bbe9182169 100644 --- a/yarn-project/aztec/terraform/node/main.tf +++ b/yarn-project/aztec/terraform/node/main.tf @@ -163,7 +163,7 @@ resource "aws_ecs_task_definition" "aztec-node" { { name = "${var.DEPLOY_TAG}-aztec-node-${count.index + 1}" image = "${var.DOCKERHUB_ACCOUNT}/aztec:${var.IMAGE_TAG}" - command = ["start", "--node", "--archiver", "--sequencer", "--prover"] + command = ["start", "--node", "--archiver", "--sequencer"] essential = true memoryReservation = 3776 portMappings = [ @@ -208,6 +208,10 @@ resource "aws_ecs_task_definition" "aztec-node" { name = "DATA_DIRECTORY" value = "${local.data_dir}/node_${count.index + 1}/data" }, + { + name = "IS_DEV_NET" + value = "true" + }, { name = "ARCHIVER_POLLING_INTERVAL" value = "10000" diff --git a/yarn-project/aztec/terraform/node/variables.tf b/yarn-project/aztec/terraform/node/variables.tf index fe6b5c88ec8..6a3b24ff9c1 100644 --- a/yarn-project/aztec/terraform/node/variables.tf +++ b/yarn-project/aztec/terraform/node/variables.tf @@ -20,18 +20,15 @@ variable "NODE_P2P_PRIVATE_KEYS" { } variable "L1_CHAIN_ID" { - type = string - default = 677692 + type = string } variable "NODE_P2P_TCP_PORT" { - type = number - default = 40000 + type = number } variable "NODE_P2P_UDP_PORT" { - type = number - default = 45000 + type = number } variable "DOCKERHUB_ACCOUNT" { @@ -89,6 +86,5 @@ variable "BOOTSTRAP_NODES" { } variable "NODE_LB_RULE_PRIORITY" { - type = number - default = 4000 + type = number } diff --git a/yarn-project/aztec/terraform/prover-node/main.tf b/yarn-project/aztec/terraform/prover-node/main.tf new file mode 100644 index 00000000000..36e2cd02291 --- /dev/null +++ b/yarn-project/aztec/terraform/prover-node/main.tf @@ -0,0 +1,379 @@ +terraform { + backend "s3" { + bucket = "aztec-terraform" + region = "eu-west-2" + } + required_providers { + aws = { + source = "hashicorp/aws" + version = "3.74.2" + } + } +} + +# Define provider and region +provider "aws" { + region = "eu-west-2" +} + +data "terraform_remote_state" "setup_iac" { + backend = "s3" + config = { + bucket = "aztec-terraform" + key = "setup/setup-iac" + region = "eu-west-2" + } +} + +data "terraform_remote_state" "aztec2_iac" { + backend = "s3" + config = { + bucket = "aztec-terraform" + key = "aztec2/iac" + region = "eu-west-2" + } +} + +data "terraform_remote_state" "aztec-network_iac" { + backend = "s3" + config = { + bucket = "aztec-terraform" + key = "aztec-network/iac" + region = "eu-west-2" + } +} + +data "terraform_remote_state" "l1_contracts" { + backend = "s3" + config = { + bucket = "aztec-terraform" + key = "${var.DEPLOY_TAG}/l1-contracts" + region = "eu-west-2" + } +} + +# Compute local variables +locals { + prover_private_keys = var.PROVER_PRIVATE_KEYS + node_p2p_private_keys = var.NODE_P2P_PRIVATE_KEYS + node_count = length(local.prover_private_keys) + data_dir = "/usr/src/yarn-project/aztec" +} + +output "node_count" { + value = local.node_count +} + +resource "aws_cloudwatch_log_group" "aztec-prover-node-log-group" { + count = local.node_count + name = "/fargate/service/${var.DEPLOY_TAG}/aztec-prover-node-${count.index + 1}" + retention_in_days = 14 +} + +resource "aws_service_discovery_service" "aztec-prover-node" { + count = local.node_count + name = "${var.DEPLOY_TAG}-aztec-prover-node-${count.index + 1}" + + health_check_custom_config { + failure_threshold = 1 + } + + dns_config { + namespace_id = data.terraform_remote_state.setup_iac.outputs.local_service_discovery_id + + dns_records { + ttl = 60 + type = "A" + } + + dns_records { + ttl = 60 + type = "SRV" + } + + routing_policy = "MULTIVALUE" + } + + # Terraform just fails if this resource changes and you have registered instances. + provisioner "local-exec" { + when = destroy + command = "${path.module}/../servicediscovery-drain.sh ${self.id}" + } +} + +# Configure an EFS filesystem. +resource "aws_efs_file_system" "prover_node_data_store" { + creation_token = "${var.DEPLOY_TAG}-prover-node-data" + throughput_mode = "provisioned" + provisioned_throughput_in_mibps = 20 + + tags = { + Name = "${var.DEPLOY_TAG}-prover-node-data" + } + + lifecycle_policy { + transition_to_ia = "AFTER_14_DAYS" + } +} + +resource "aws_efs_mount_target" "public_az1" { + file_system_id = aws_efs_file_system.prover_node_data_store.id + subnet_id = data.terraform_remote_state.setup_iac.outputs.subnet_az1_id + security_groups = [data.terraform_remote_state.setup_iac.outputs.security_group_public_id] +} + +resource "aws_efs_mount_target" "public_az2" { + file_system_id = aws_efs_file_system.prover_node_data_store.id + subnet_id = data.terraform_remote_state.setup_iac.outputs.subnet_az2_id + security_groups = [data.terraform_remote_state.setup_iac.outputs.security_group_public_id] +} + +# Define task definitions for each node. +resource "aws_ecs_task_definition" "aztec-prover-node" { + count = local.node_count + family = "${var.DEPLOY_TAG}-aztec-prover-node-${count.index + 1}" + requires_compatibilities = ["FARGATE"] + network_mode = "awsvpc" + cpu = "2048" + memory = "4096" + execution_role_arn = data.terraform_remote_state.setup_iac.outputs.ecs_task_execution_role_arn + task_role_arn = data.terraform_remote_state.aztec2_iac.outputs.cloudwatch_logging_ecs_role_arn + + volume { + name = "efs-data-store" + efs_volume_configuration { + root_directory = "/" + file_system_id = aws_efs_file_system.prover_node_data_store.id + } + } + + container_definitions = jsonencode([ + { + name = "init-container" + image = "amazonlinux:latest" + essential = false + command = ["sh", "-c", "mkdir -p ${local.data_dir}/prover_node_${count.index + 1}/data ${local.data_dir}/prover_node_${count.index + 1}/temp"] + mountPoints = [ + { + containerPath = local.data_dir + sourceVolume = "efs-data-store" + } + ] + }, + { + name = "${var.DEPLOY_TAG}-aztec-prover-node-${count.index + 1}" + image = "${var.DOCKERHUB_ACCOUNT}/aztec:${var.IMAGE_TAG}" + command = ["start", "--prover-node", "--archiver"] + essential = true + memoryReservation = 3776 + portMappings = [ + { + containerPort = 80 + }, + { + containerPort = var.NODE_P2P_TCP_PORT + count.index + protocol = "tcp" + }, + { + containerPort = var.NODE_P2P_UDP_PORT + count.index + protocol = "udp" + } + ] + environment = [ + // General + { name = "NODE_ENV", value = "production" }, + { name = "LOG_LEVEL", value = "info" }, + { name = "DEBUG", value = "aztec:*,-json-rpc:json_proxy:*,-aztec:avm_simulator:*" }, + { name = "DEPLOY_TAG", value = var.DEPLOY_TAG }, + { name = "NETWORK_NAME", value = "${var.DEPLOY_TAG}" }, + { name = "ETHEREUM_HOST", value = "https://${var.DEPLOY_TAG}-mainnet-fork.aztec.network:8545/${var.API_KEY}" }, + { name = "L1_CHAIN_ID", value = var.L1_CHAIN_ID }, + { name = "DATA_DIRECTORY", value = "${local.data_dir}/prover_node_${count.index + 1}/data" }, + { name = "DEPLOY_AZTEC_CONTRACTS", value = "false" }, + { name = "IS_DEV_NET", value = "true" }, + + // API + { name = "AZTEC_PORT", value = "80" }, + { name = "API_KEY", value = var.API_KEY }, + { name = "API_PREFIX", value = "/${var.DEPLOY_TAG}/aztec-prover-node-${count.index + 1}/${var.API_KEY}" }, + + // Archiver + { name = "ARCHIVER_POLLING_INTERVAL", value = "10000" }, + + // Aztec node to pull clientivc proofs from (to be replaced with a p2p connection) + { name = "TX_PROVIDER_NODE_URL", value = "http://${var.DEPLOY_TAG}-aztec-node-${count.index + 1}.local/${var.DEPLOY_TAG}/aztec-node-${count.index + 1}/${var.API_KEY}" }, + + // Prover + { name = "PROVER_PUBLISHER_PRIVATE_KEY", value = local.prover_private_keys[count.index] }, + { name = "PROVER_AGENT_ENABLED", value = "false" }, + { name = "PROVER_AGENT_CONCURRENCY", value = "0" }, + { name = "PROVER_REAL_PROOFS", value = tostring(var.PROVING_ENABLED) }, + { name = "BB_WORKING_DIRECTORY", value = "${local.data_dir}/node_${count.index + 1}/temp" }, + { name = "ACVM_WORKING_DIRECTORY", value = "${local.data_dir}/node_${count.index + 1}/temp" }, + + // Metrics + { name = "OTEL_EXPORTER_OTLP_ENDPOINT", value = "http://aztec-otel.local:4318" }, + { name = "OTEL_SERVICE_NAME", value = "${var.DEPLOY_TAG}-aztec-prover-node-${count.index + 1}" }, + + // L1 addresses + { name = "ROLLUP_CONTRACT_ADDRESS", value = data.terraform_remote_state.l1_contracts.outputs.rollup_contract_address }, + { name = "INBOX_CONTRACT_ADDRESS", value = data.terraform_remote_state.l1_contracts.outputs.inbox_contract_address }, + { name = "OUTBOX_CONTRACT_ADDRESS", value = data.terraform_remote_state.l1_contracts.outputs.outbox_contract_address }, + { name = "REGISTRY_CONTRACT_ADDRESS", value = data.terraform_remote_state.l1_contracts.outputs.registry_contract_address }, + { name = "AVAILABILITY_ORACLE_CONTRACT_ADDRESS", value = data.terraform_remote_state.l1_contracts.outputs.availability_oracle_contract_address }, + { name = "FEE_JUICE_CONTRACT_ADDRESS", value = data.terraform_remote_state.l1_contracts.outputs.fee_juice_contract_address }, + { name = "FEE_JUICE_PORTAL_CONTRACT_ADDRESS", value = data.terraform_remote_state.l1_contracts.outputs.FEE_JUICE_PORTAL_CONTRACT_ADDRESS }, + + // P2P (disabled) + { name = "P2P_ENABLED", value = tostring(var.P2P_ENABLED) }, + { name = "BOOTSTRAP_NODES", value = var.BOOTSTRAP_NODES }, + { name = "PEER_ID_PRIVATE_KEY", value = local.node_p2p_private_keys[count.index] }, + { name = "P2P_TCP_LISTEN_ADDR", value = "0.0.0.0:${var.NODE_P2P_TCP_PORT + count.index}" }, + { name = "P2P_UDP_LISTEN_ADDR", value = "0.0.0.0:${var.NODE_P2P_UDP_PORT + count.index}" }, + { name = "P2P_TCP_ANNOUNCE_ADDR", value = ":${var.NODE_P2P_TCP_PORT + count.index}" }, + { name = "P2P_UDP_ANNOUNCE_ADDR", value = ":${var.NODE_P2P_UDP_PORT + count.index}" }, + { name = "P2P_QUERY_FOR_IP", value = "true" }, + { name = "P2P_MIN_PEERS", value = var.P2P_MIN_PEERS }, + { name = "P2P_MAX_PEERS", value = var.P2P_MAX_PEERS }, + { name = "P2P_BLOCK_CHECK_INTERVAL_MS", value = "1000" }, + { name = "P2P_PEER_CHECK_INTERVAL_MS", value = "2000" }, + { name = "P2P_TX_POOL_KEEP_PROVEN_FOR", value = tostring(var.P2P_TX_POOL_KEEP_PROVEN_FOR) }, + ] + mountPoints = [ + { + containerPath = "${local.data_dir}/prover_node_${count.index + 1}" + sourceVolume = "efs-data-store" + } + ] + dependsOn = [ + { + containerName = "init-container" + condition = "COMPLETE" + } + ] + logConfiguration = { + logDriver = "awslogs" + options = { + "awslogs-group" = "/fargate/service/${var.DEPLOY_TAG}/aztec-prover-node-${count.index + 1}" + "awslogs-region" = "eu-west-2" + "awslogs-stream-prefix" = "ecs" + } + } + }]) +} + +resource "aws_ecs_service" "aztec-prover-node" { + count = local.node_count + name = "${var.DEPLOY_TAG}-aztec-prover-node-${count.index + 1}" + cluster = data.terraform_remote_state.setup_iac.outputs.ecs_cluster_id + launch_type = "FARGATE" + desired_count = 1 + deployment_maximum_percent = 100 + deployment_minimum_healthy_percent = 0 + platform_version = "1.4.0" + force_new_deployment = true + + + network_configuration { + assign_public_ip = true + subnets = [ + data.terraform_remote_state.setup_iac.outputs.subnet_az1_id + ] + security_groups = [data.terraform_remote_state.aztec-network_iac.outputs.p2p_security_group_id, data.terraform_remote_state.setup_iac.outputs.security_group_private_id] + } + + load_balancer { + target_group_arn = aws_alb_target_group.aztec-prover-node-http[count.index].arn + container_name = "${var.DEPLOY_TAG}-aztec-prover-node-${count.index + 1}" + container_port = 80 + } + + service_registries { + registry_arn = aws_service_discovery_service.aztec-prover-node[count.index].arn + container_name = "${var.DEPLOY_TAG}-aztec-prover-node-${count.index + 1}" + container_port = 80 + } + + task_definition = aws_ecs_task_definition.aztec-prover-node[count.index].family +} + +# Configure ALB to route /aztec-prover-node to server. +resource "aws_alb_target_group" "aztec-prover-node-http" { + count = local.node_count + name = "${var.DEPLOY_TAG}-node-${count.index + 1}-http-target" + port = 80 + protocol = "HTTP" + target_type = "ip" + vpc_id = data.terraform_remote_state.setup_iac.outputs.vpc_id + deregistration_delay = 5 + + health_check { + path = "/${var.DEPLOY_TAG}/aztec-prover-node-${count.index + 1}/${var.API_KEY}/status" + matcher = "200" + interval = 10 + healthy_threshold = 2 + unhealthy_threshold = 5 + timeout = 5 + } + + tags = { + name = "${var.DEPLOY_TAG}-aztec-prover-node-${count.index + 1}" + } +} + +resource "aws_lb_listener_rule" "api" { + count = local.node_count + listener_arn = data.terraform_remote_state.aztec2_iac.outputs.alb_listener_arn + priority = var.PROVER_NODE_LB_RULE_PRIORITY + count.index + + action { + type = "forward" + target_group_arn = aws_alb_target_group.aztec-prover-node-http[count.index].arn + } + + condition { + path_pattern { + values = ["/${var.DEPLOY_TAG}/aztec-prover-node-${count.index + 1}/${var.API_KEY}*"] + } + } +} + +# resource "aws_security_group_rule" "allow-node-tcp-in" { +# count = local.node_count +# type = "ingress" +# from_port = var.NODE_P2P_TCP_PORT + count.index +# to_port = var.NODE_P2P_TCP_PORT + count.index +# protocol = "tcp" +# cidr_blocks = ["0.0.0.0/0"] +# security_group_id = data.terraform_remote_state.aztec-network_iac.outputs.p2p_security_group_id +# } + +# resource "aws_security_group_rule" "allow-node-tcp-out" { +# count = local.node_count +# type = "egress" +# from_port = var.NODE_P2P_TCP_PORT + count.index +# to_port = var.NODE_P2P_TCP_PORT + count.index +# protocol = "tcp" +# cidr_blocks = ["0.0.0.0/0"] +# security_group_id = data.terraform_remote_state.aztec-network_iac.outputs.p2p_security_group_id +# } + +# resource "aws_security_group_rule" "allow-node-udp-in" { +# count = local.node_count +# type = "ingress" +# from_port = var.NODE_P2P_UDP_PORT +# to_port = var.NODE_P2P_UDP_PORT + count.index +# protocol = "udp" +# cidr_blocks = ["0.0.0.0/0"] +# security_group_id = data.terraform_remote_state.aztec-network_iac.outputs.p2p_security_group_id +# } + +# resource "aws_security_group_rule" "allow-node-udp-out" { +# count = local.node_count +# type = "egress" +# from_port = var.NODE_P2P_UDP_PORT +# to_port = var.NODE_P2P_UDP_PORT + count.index +# protocol = "udp" +# cidr_blocks = ["0.0.0.0/0"] +# security_group_id = data.terraform_remote_state.aztec-network_iac.outputs.p2p_security_group_id +# } diff --git a/yarn-project/aztec/terraform/prover-node/variables.tf b/yarn-project/aztec/terraform/prover-node/variables.tf new file mode 100644 index 00000000000..2c8fc301f23 --- /dev/null +++ b/yarn-project/aztec/terraform/prover-node/variables.tf @@ -0,0 +1,70 @@ +variable "DEPLOY_TAG" { + type = string +} + +variable "IMAGE_TAG" { + type = string + default = "latest" +} + +variable "API_KEY" { + type = string +} + +variable "PROVER_PRIVATE_KEYS" { + type = list(string) +} + +variable "NODE_P2P_PRIVATE_KEYS" { + type = list(string) +} + +variable "L1_CHAIN_ID" { + type = string +} + +variable "NODE_P2P_TCP_PORT" { + type = number +} + +variable "NODE_P2P_UDP_PORT" { + type = number +} + +variable "DOCKERHUB_ACCOUNT" { + type = string +} + +variable "P2P_MIN_PEERS" { + type = string + default = 5 +} + +variable "P2P_MAX_PEERS" { + type = string + default = 100 +} + +variable "P2P_ENABLED" { + type = bool + default = false +} + +variable "P2P_TX_POOL_KEEP_PROVEN_FOR" { + type = number + default = 64 +} + +variable "PROVING_ENABLED" { + type = bool + default = false +} + +variable "BOOTSTRAP_NODES" { + type = string + default = "" +} + +variable "PROVER_NODE_LB_RULE_PRIORITY" { + type = number +} diff --git a/yarn-project/aztec/terraform/prover/main.tf b/yarn-project/aztec/terraform/prover/main.tf index 99d4d770c67..25056aa95b8 100644 --- a/yarn-project/aztec/terraform/prover/main.tf +++ b/yarn-project/aztec/terraform/prover/main.tf @@ -43,17 +43,17 @@ data "terraform_remote_state" "aztec-network_iac" { } } -data "terraform_remote_state" "aztec-network_node" { +data "terraform_remote_state" "aztec-network_prover-node" { backend = "s3" config = { bucket = "aztec-terraform" - key = "${var.DEPLOY_TAG}/aztec-node" + key = "${var.DEPLOY_TAG}/aztec-prover-node" region = "eu-west-2" } } locals { - node_count = data.terraform_remote_state.aztec-network_node.outputs.node_count + node_count = data.terraform_remote_state.aztec-network_prover-node.outputs.node_count agents_per_prover = var.AGENTS_PER_PROVER } @@ -259,7 +259,7 @@ resource "aws_ecs_task_definition" "aztec-proving-agent" { }, { "name": "AZTEC_NODE_URL", - "value": "http://${var.DEPLOY_TAG}-aztec-node-${count.index + 1}.local/${var.DEPLOY_TAG}/aztec-node-${count.index + 1}/${var.API_KEY}" + "value": "http://${var.DEPLOY_TAG}-aztec-prover-node-${count.index + 1}.local/${var.DEPLOY_TAG}/aztec-prover-node-${count.index + 1}/${var.API_KEY}" }, { "name": "PROVER_AGENT_ENABLED", diff --git a/yarn-project/aztec/terraform/pxe/variables.tf b/yarn-project/aztec/terraform/pxe/variables.tf index f154c4c0c99..7ce4470f0c3 100644 --- a/yarn-project/aztec/terraform/pxe/variables.tf +++ b/yarn-project/aztec/terraform/pxe/variables.tf @@ -21,6 +21,5 @@ variable "PROVING_ENABLED" { } variable "PXE_LB_RULE_PRIORITY" { - type = number - default = 5000 + type = number } diff --git a/yarn-project/foundation/src/config/env_var.ts b/yarn-project/foundation/src/config/env_var.ts index 233e89328ac..d4383b4b64b 100644 --- a/yarn-project/foundation/src/config/env_var.ts +++ b/yarn-project/foundation/src/config/env_var.ts @@ -57,7 +57,6 @@ export type EnvVar = | 'SEQ_ALLOWED_TEARDOWN_FN' | 'SEQ_MAX_BLOCK_SIZE_IN_BYTES' | 'ENFORCE_FEES' - | 'SEQ_SKIP_SUBMIT_PROOFS' | 'SEQ_PUBLISHER_PRIVATE_KEY' | 'SEQ_REQUIRED_CONFIRMATIONS' | 'SEQ_PUBLISH_RETRY_INTERVAL_MS'