From 8fe499cac1c24903691fad33aa0960a64a2bef78 Mon Sep 17 00:00:00 2001 From: Geoff Franks Date: Mon, 19 Aug 2024 17:40:12 +0000 Subject: [PATCH 1/3] Disable unproxied container ports and enable TLS for TCP Routing by default, and provide ops-files to disable. Enables TLS encryption for TCP Routes on the hop between tcp-router + app containeris. This behaves the same as the TLS encryption between gorouter and app containers, and relies on the same settings regarding route integrity. This allows us to finally disable the unproxied container ports, and prevent network connections to app ports from hosts other than router or tcp-router. If operators wish to disable TLS for TCP routes, they must first re-enable the unproxied ports by deploying with these ops files (as appropriate): - operations/disable-tls-tcp-routing-stage-1-unproxied-ports.yml - operations/disable-tls-tcp-routing-isolation-segment-stage-1-unproxied-ports.yml - operations/experimental/disable-tls-tcp-routing-windows-stage-1-unproxied-ports.yml Once the unproxied ports are re-enabled, a second deploy can be performed to disable TLS for TCP Routes via these ops files (as appropriate): - operations/disable-tls-tcp-routing-stage-2-tcp-router-and-route-emitter.yml - operations/disable-tls-tcp-routing-isolation-segment-stage-2-route-emitter.yml - operations/experimental/disable-tls-tcp-routing-windows-stage-2-route-emitter.yml Failing to disable this in a two-deploy fashion will result in downtime for TCP Routes. --- cf-deployment.yml | 18 ++++++++++++++++++ ...persistent-isolation-segment-diego-cell.yml | 4 ++++ ...olation-segment-stage-1-unproxied-ports.yml | 4 ++++ ...isolation-segment-stage-2-route-emitter.yml | 4 ++++ ...tls-tcp-routing-stage-1-unproxied-ports.yml | 4 ++++ ...ng-stage-2-tcp-router-and-route-emitter.yml | 8 ++++++++ ...routing-windows-stage-1-unproxied-ports.yml | 4 ++++ ...p-routing-windows-stage-2-route-emitter.yml | 4 ++++ ...ble-nginx-routing-integrity-windows2019.yml | 8 ++++++++ operations/windows2019-cell.yml | 7 +++++++ 10 files changed, 65 insertions(+) create mode 100644 operations/disable-tls-tcp-routing-isolation-segment-stage-1-unproxied-ports.yml create mode 100644 operations/disable-tls-tcp-routing-isolation-segment-stage-2-route-emitter.yml create mode 100644 operations/disable-tls-tcp-routing-stage-1-unproxied-ports.yml create mode 100644 operations/disable-tls-tcp-routing-stage-2-tcp-router-and-route-emitter.yml create mode 100644 operations/experimental/disable-tls-tcp-routing-windows-stage-1-unproxied-ports.yml create mode 100644 operations/experimental/disable-tls-tcp-routing-windows-stage-2-route-emitter.yml diff --git a/cf-deployment.yml b/cf-deployment.yml index e4b5bab36..69dd95210 100644 --- a/cf-deployment.yml +++ b/cf-deployment.yml @@ -1463,6 +1463,11 @@ instance_groups: router_group: default-tcp tls_health_check_cert: ((tcp_router_lb_health_tls.certificate)) tls_health_check_key: ((tcp_router_lb_health_tls.private_key)) + backend_tls: + enabled: true + client_cert: ((tcp_router_backend_tls.certificate)) + client_key: ((tcp_router_backend_tls.private_key)) + ca_cert: ((diego_instance_identity_ca.ca)) uaa: ca_cert: "((uaa_ssl.ca))" tls_port: 8443 @@ -1637,13 +1642,16 @@ instance_groups: containers: proxy: enabled: true + enable_unproxied_port_mappings: false require_and_verify_client_certificates: true trusted_ca_certificates: - ((gorouter_backend_tls.ca)) - ((ssh_proxy_backends_tls.ca)) + - ((tcp_router_backend_tls.ca)) verify_subject_alt_name: - gorouter.service.cf.internal - ssh-proxy.service.cf.internal + - tcp-router.service.cf.internal trusted_ca_certificates: - ((diego_instance_identity_ca.ca)) - ((credhub_tls.ca)) @@ -1682,6 +1690,7 @@ instance_groups: client_key: "((nats_client_cert.private_key))" tcp: enabled: true + enable_tls: true uaa: ca_cert: "((uaa_ssl.ca))" client_secret: "((uaa_clients_tcp_emitter_secret))" @@ -2486,6 +2495,15 @@ variables: common_name: gorouter_lb_health_tls alternative_names: - gorouter.service.cf.internal +- name: tcp_router_backend_tls + type: certificate + options: + ca: service_cf_internal_ca + common_name: tcp-router_backend_tls + alternative_names: + - tcp-router.service.cf.internal + extended_key_usage: + - client_auth - name: tcp_router_lb_health_tls type: certificate options: diff --git a/operations/add-persistent-isolation-segment-diego-cell.yml b/operations/add-persistent-isolation-segment-diego-cell.yml index 33b9cc37c..b8a8ab707 100644 --- a/operations/add-persistent-isolation-segment-diego-cell.yml +++ b/operations/add-persistent-isolation-segment-diego-cell.yml @@ -70,13 +70,16 @@ containers: proxy: enabled: true + enable_unproxied_port_mappings: false require_and_verify_client_certificates: true trusted_ca_certificates: - ((gorouter_backend_tls.ca)) - ((ssh_proxy_backends_tls.ca)) + - ((tcp_router_backend_tls.ca)) verify_subject_alt_name: - gorouter.service.cf.internal - ssh-proxy.service.cf.internal + - tcp-router.service.cf.internal trusted_ca_certificates: - ((diego_instance_identity_ca.ca)) - ((credhub_tls.ca)) @@ -134,6 +137,7 @@ timestamp: "rfc3339" tcp: enabled: true + enable_tls: true uaa: ca_cert: "((uaa_ssl.ca))" client_secret: "((uaa_clients_tcp_emitter_secret))" diff --git a/operations/disable-tls-tcp-routing-isolation-segment-stage-1-unproxied-ports.yml b/operations/disable-tls-tcp-routing-isolation-segment-stage-1-unproxied-ports.yml new file mode 100644 index 000000000..bc1da4fef --- /dev/null +++ b/operations/disable-tls-tcp-routing-isolation-segment-stage-1-unproxied-ports.yml @@ -0,0 +1,4 @@ +--- +- type: replace + path: /instance_groups/name=isolated-diego-cell/jobs/name=rep/properties/containers/proxy/enable_unproxied_port_mappings? + value: true diff --git a/operations/disable-tls-tcp-routing-isolation-segment-stage-2-route-emitter.yml b/operations/disable-tls-tcp-routing-isolation-segment-stage-2-route-emitter.yml new file mode 100644 index 000000000..482e87e5c --- /dev/null +++ b/operations/disable-tls-tcp-routing-isolation-segment-stage-2-route-emitter.yml @@ -0,0 +1,4 @@ +--- +- type: replace + path: /instance_groups/name=isolated-diego-cell/jobs/name=route_emitter/properties/tcp/enable_tls? + value: false diff --git a/operations/disable-tls-tcp-routing-stage-1-unproxied-ports.yml b/operations/disable-tls-tcp-routing-stage-1-unproxied-ports.yml new file mode 100644 index 000000000..f7ecdc78e --- /dev/null +++ b/operations/disable-tls-tcp-routing-stage-1-unproxied-ports.yml @@ -0,0 +1,4 @@ +--- +- type: replace + path: /instance_groups/name=diego-cell/jobs/name=rep/properties/containers/proxy/enable_unproxied_port_mappings? + value: true diff --git a/operations/disable-tls-tcp-routing-stage-2-tcp-router-and-route-emitter.yml b/operations/disable-tls-tcp-routing-stage-2-tcp-router-and-route-emitter.yml new file mode 100644 index 000000000..7c87c7711 --- /dev/null +++ b/operations/disable-tls-tcp-routing-stage-2-tcp-router-and-route-emitter.yml @@ -0,0 +1,8 @@ +--- +- type: replace + path: /instance_groups/name=tcp-router/jobs/name=tcp_router/properties/tcp_router/backend_tls?/enabled + value: false + +- type: replace + path: /instance_groups/name=diego-cell/jobs/name=route_emitter/properties/tcp/enable_tls? + value: false diff --git a/operations/experimental/disable-tls-tcp-routing-windows-stage-1-unproxied-ports.yml b/operations/experimental/disable-tls-tcp-routing-windows-stage-1-unproxied-ports.yml new file mode 100644 index 000000000..fffa15a3a --- /dev/null +++ b/operations/experimental/disable-tls-tcp-routing-windows-stage-1-unproxied-ports.yml @@ -0,0 +1,4 @@ +--- +- type: replace + path: /instance_groups/name=windows2019-cell/jobs/name=rep_windows/properties/containers/proxy/enable_unproxied_port_mappings? + value: true diff --git a/operations/experimental/disable-tls-tcp-routing-windows-stage-2-route-emitter.yml b/operations/experimental/disable-tls-tcp-routing-windows-stage-2-route-emitter.yml new file mode 100644 index 000000000..b39d40f21 --- /dev/null +++ b/operations/experimental/disable-tls-tcp-routing-windows-stage-2-route-emitter.yml @@ -0,0 +1,4 @@ +--- +- type: replace + path: /instance_groups/name=windows2019-cell/jobs/name=route_emitter_windows/properties/tcp/enable_tls? + value: false diff --git a/operations/experimental/enable-nginx-routing-integrity-windows2019.yml b/operations/experimental/enable-nginx-routing-integrity-windows2019.yml index 4cb792d01..99308d47e 100644 --- a/operations/experimental/enable-nginx-routing-integrity-windows2019.yml +++ b/operations/experimental/enable-nginx-routing-integrity-windows2019.yml @@ -1,6 +1,9 @@ - type: replace path: /instance_groups/name=windows2019-cell/jobs/name=rep_windows/properties/containers?/proxy/enabled value: true +- type: replace + path: /instance_groups/name=windows2019-cell/jobs/name=rep_windows/properties/containers?/proxy/enable_unproxied_port_mappings + value: false - type: replace path: /instance_groups/name=windows2019-cell/jobs/name=rep_windows/properties/containers?/proxy/require_and_verify_client_certificates value: true @@ -9,11 +12,16 @@ value: - ((gorouter_backend_tls.ca)) - ((ssh_proxy_backends_tls.ca)) + ((tcp_router_backend_tls.ca)) - type: replace path: /instance_groups/name=windows2019-cell/jobs/name=rep_windows/properties/containers?/proxy/verify_subject_alt_name value: - gorouter.service.cf.internal - ssh-proxy.service.cf.internal + - tcp-router.service.cf.internal +- type: replace + path: /instance_groups/name=windows2019-cell/jobs/name=route_emitter_windows/properties/tcp?/enable_tls + value: true - type: replace path: /instance_groups/name=windows2019-cell/jobs/- value: diff --git a/operations/windows2019-cell.yml b/operations/windows2019-cell.yml index 8c994d447..0f175145c 100644 --- a/operations/windows2019-cell.yml +++ b/operations/windows2019-cell.yml @@ -86,6 +86,13 @@ client_cert: ((nats_client_cert.certificate)) client_key: ((nats_client_cert.private_key)) enabled: true + internal_routes: + enabled: true + uaa: + ca_cert: "((uaa_ssl.ca))" + client_secret: "((uaa_clients_tcp_emitter_secret))" + tcp: + enabled: true logging: format: timestamp: rfc3339 From 070f54bb59afeaa3e9063c86085a452f97142b46 Mon Sep 17 00:00:00 2001 From: Geoff Franks Date: Fri, 23 Aug 2024 20:34:29 +0000 Subject: [PATCH 2/3] update readmes for new tls for tcp route disabling ops files --- operations/README.md | 4 ++++ operations/experimental/README.md | 2 ++ 2 files changed, 6 insertions(+) diff --git a/operations/README.md b/operations/README.md index 79ea334c5..672d31649 100644 --- a/operations/README.md +++ b/operations/README.md @@ -39,6 +39,10 @@ This is the README for Ops-files. To learn more about `cf-deployment`, go to the | [`disable-router-tls-termination.yml`](disable-router-tls-termination.yml) | Eliminates keys related to performing TLS termination within the gorouter job. | Useful for deployments where TLS termination is performed prior to the gorouter - for instance, on AWS, such termination is commonly done at the ELB. This also eliminates the need to specify `((router_ssl.certificate))` and `((router_ssl.private_key))` in the var files. | **NO** | | [`disable-http2.yml`](disable-http2.yml) | Prevent gorouter from accepting and forwarding HTTP/2 requests. | | **NO** | | [`disable-dynamic-asgs.yml`](disable-dynamic-asgs.yml) | Disable dynamic updates for security groups. | | **NO** | +| [`disable-tls-tcp-routing-stage-1-unproxied-ports.yml`](disable-tls-tcp-routing-stage-1-unproxied-ports.yml) | Stage 1 deployment for disabling TLS for TCP Routes on. See [configuring TCP routes](https://docs.cloudfoundry.org/adminguide/enabling-tcp-routing.html#tls-tcp-routes) for more info. | | **NO ** | +| [`disable-tls-tcp-routing-stage-2-tcp-router-and-route-emitter.yml`](disable-tls-tcp-routing-stage-2-tcp-router-and-route-emitter.yml) | Stage 2 deployment for disabling TLS for TCP Routes on. See [configuring TCP routes](https://docs.cloudfoundry.org/adminguide/enabling-tcp-routing.html#tls-tcp-routes) for more info. | | **NO ** | +| [`disable-tls-tcp-routing-isolation-segment-stage-1-unproxied-ports.yml`](disable-tls-tcp-routing-isolation-segment-stage-1-unproxied-ports.yml) | Stage 1 deployment for disabling TLS for TCP Routes on isolation segments. See [configuring TCP routes](https://docs.cloudfoundry.org/adminguide/enabling-tcp-routing.html#tls-tcp-routes) for more info. | | **NO ** | +| [`disable-tls-tcp-routing-isolation-segment-stage-2-route-emitter.yml`](disable-tls-tcp-routing-isolation-segment-stage-2-route-emitter.yml) | Stage 2 deployment for disabling TLS for TCP Routes on isolation segments. See [configuring TCP routes](https://docs.cloudfoundry.org/adminguide/enabling-tcp-routing.html#tls-tcp-routes) for more info. | | **NO ** | | [`enable-cc-rate-limiting.yml`](enable-cc-rate-limiting.yml) | Enable rate limiting for UAA-authenticated endpoints. | Introduces variables `cc_rate_limiter_general_limit` and `cc_rate_limiter_unauthenticated_limit` | **NO** | | [`enable-cc-v2-rate-limiting.yml`](enable-cc-rate-limiting.yml) | Enable V2 API rate limiting for UAA-authenticated endpoints. | Introduces variables `cc_v2_rate_limiter_general_limit`, `cc_v2_rate_limiter_admin_limit` and `cc_v2_rate_limiter_reset_interval_in_minutes` | **NO** | | [`enable-cpu-throttling.yml`](enable-cpu-throttling.yml) | Configure Garden containers with CPU entitlement. | This ops file requires `set-cpu-weight.yml`. | **YES** | diff --git a/operations/experimental/README.md b/operations/experimental/README.md index 94661f930..a3188ca11 100644 --- a/operations/experimental/README.md +++ b/operations/experimental/README.md @@ -22,6 +22,8 @@ This is the README for Experimental Ops-files. To learn more about `cf-deploymen | [`colocate-smoke-tests-on-cc-worker.yml`](colocate-smoke-tests-on-cc-worker.yml) | Colocate the smoke_tests job on the cc-worker instance | A number of other operations files reference this instance group and may be incompatible with this operations file. Use `find ./operations/ -name "*.yml" | xargs grep "/instance_groups/name=smoke-tests"` to locate said files. | **YES** | | [`disable-interpolate-service-bindings.yml`](disable-interpolate-service-bindings.yml) | Disables the interpolation of CredHub service credentials by Cloud Controller. | | **YES** | | [`disable-cf-credhub.yml`](disable-cf-credhub.yml) | Completely removes the CF CredHub instances, UAA clients, credentials and certificates. Can be used to save cost if you don't use CredHub to store service credentials. | | **YES** | +| [`disable-tls-tcp-routing-windows-stage-1-unproxied-ports.yml`](disable-tls-tcp-routing-windows-stage-1-unproxied-ports.yml) | Stage 1 deployment for disabling TLS for TCP Routes on Windows Diego Cells. See [configuring TCP routes](https://docs.cloudfoundry.org/adminguide/enabling-tcp-routing.html#tls-tcp-routes) for more info. | | **NO ** | +| [`disable-tls-tcp-routing-windows-stage-2-route-emitter.yml`](disable-tls-tcp-routing-windows-stage-2-route-emitter.yml) | Stage 2 deployment for disabling TLS for TCP Routes on Windows Diego Cells. See [configuring TCP routes](https://docs.cloudfoundry.org/adminguide/enabling-tcp-routing.html#tls-tcp-routes) for more info. | | **NO ** | | [`enable-app-log-rate-limiting.yml`](enable-app-log-rate-limiting.yml) | Enable rate limiting for number of logs generated by the application. | Introduces variable `app_log_rate_limit`. | **NO** | | [`enable-app-log-rate-limiting-windows2019.yml`](enable-app-log-rate-limiting-windows2019.yml) | Enable rate limiting for number of logs generated by the application. | Introduces variable `app_log_rate_limit`. Requires `../windows2019-cell.yml` | **NO** | | [`enable-bpm-garden.yml`](enable-bpm-garden.yml) | Enables the [BOSH Process Manager](https://github.com/cloudfoundry-incubator/bpm-release) for Garden. | This ops file **cannot** be deployed in conjunction with `enable-oci-phase-1.yml`. | **NO** | From 93f1f9db275ed0342de1c1da4ac0add889e19821 Mon Sep 17 00:00:00 2001 From: Geoff Franks Date: Mon, 26 Aug 2024 14:14:52 +0000 Subject: [PATCH 3/3] Add unit tests for new opsfiles --- units/tests/experimental_test/operations.yml | 9 +++++++++ units/tests/standard_test/operations.yml | 13 ++++++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/units/tests/experimental_test/operations.yml b/units/tests/experimental_test/operations.yml index aaa7333a4..ca42cfda8 100644 --- a/units/tests/experimental_test/operations.yml +++ b/units/tests/experimental_test/operations.yml @@ -76,3 +76,12 @@ use-mysql-version-8.0.yml: pathvalidator: path: /instance_groups/name=database/jobs/name=pxc-mysql/properties/mysql_version? expectedvalue: "8.0" +disable-tls-tcp-routing-windows-stage-1-unproxied-ports.yml: + ops: + - ../windows2019-cell.yml + - enable-nginx-routing-integrity-windows2019.yml +disable-tls-tcp-routing-windows-stage-2-route-emitter.yml: + ops: + - ../windows2019-cell.yml + - enable-nginx-routing-integrity-windows2019.yml + - disable-tls-tcp-routing-windows-stage-1-unproxied-ports.yml diff --git a/units/tests/standard_test/operations.yml b/units/tests/standard_test/operations.yml index aa9287fb0..bcb4656af 100644 --- a/units/tests/standard_test/operations.yml +++ b/units/tests/standard_test/operations.yml @@ -154,4 +154,15 @@ use-trusted-ca-cert-for-apps.yml: windows2019-cell.yml: {} use-cflinuxfs4-compat.yml: ops: - - use-cflinuxfs4-compat.yml \ No newline at end of file + - use-cflinuxfs4-compat.yml +disable-tls-tcp-routing-isolation-segment-stage-1-unproxied-ports.yml: + ops: + - add-persistent-isolation-segment-diego-cell.yml +disable-tls-tcp-routing-isolation-segment-stage-2-route-emitter.yml: + ops: + - add-persistent-isolation-segment-diego-cell.yml + - disable-tls-tcp-routing-isolation-segment-stage-1-unproxied-ports.yml +disable-tls-tcp-routing-stage-1-unproxied-ports.yml: {} +disable-tls-tcp-routing-stage-2-tcp-router-and-route-emitter.yml: + ops: + - disable-tls-tcp-routing-stage-1-unproxied-ports.yml