Add Option to Use Host Network and Configure Ports

cmd/controller/main.go

@@ -87,6 +87,7 @@ const (
 	defaultResync                = 30 * time.Second
 	podNamespace                 = "pod-namespace"
 	leaderElectionFlag           = "leader-election"
+	httpPort                     = "http-port"
 )
 
 var (
@@ -171,7 +172,9 @@ func main() {
 	agonesInformerFactory := externalversions.NewSharedInformerFactory(agonesClient, defaultResync)
 	kubeInformerFactory := informers.NewSharedInformerFactory(kubeClient, defaultResync)
 
-	server := &httpServer{}
+	server := &httpServer{
+		Port: ctlConf.HTTPPort,
+	}
 	var rs []runner
 	var health healthcheck.Handler
 
@@ -272,6 +275,7 @@ func parseEnvFlags() config {
 	viper.SetDefault(allocationBatchWaitTime, 500*time.Millisecond)
 	viper.SetDefault(podNamespace, "agones-system")
 	viper.SetDefault(leaderElectionFlag, false)
+	viper.SetDefault(httpPort, "8080")
 
 	viper.SetDefault(projectIDFlag, "")
 	viper.SetDefault(numWorkersFlag, 64)
@@ -307,6 +311,7 @@ func parseEnvFlags() config {
 	pflag.String(logLevelFlag, viper.GetString(logLevelFlag), "Agones Log level")
 	pflag.Duration(allocationBatchWaitTime, viper.GetDuration(allocationBatchWaitTime), "Flag to configure the waiting period between allocations batches")
 	pflag.String(podNamespace, viper.GetString(podNamespace), "namespace of current pod")
+	pflag.String(httpPort, viper.GetString(httpPort), "Port for the HTTP server. Defaults to 8080, can also use HTTP_PORT env variable")
 	pflag.Bool(leaderElectionFlag, viper.GetBool(leaderElectionFlag), "Flag to enable/disable leader election for controller pod")
 	cloudproduct.BindFlags()
 	runtime.FeaturesBindFlags()
@@ -340,6 +345,7 @@ func parseEnvFlags() config {
 	runtime.Must(viper.BindEnv(allocationBatchWaitTime))
 	runtime.Must(viper.BindEnv(podNamespace))
 	runtime.Must(viper.BindEnv(leaderElectionFlag))
+	runtime.Must(viper.BindEnv(httpPort))
 	runtime.Must(viper.BindPFlags(pflag.CommandLine))
 	runtime.Must(cloudproduct.BindEnv())
 	runtime.Must(runtime.FeaturesBindEnv())
@@ -401,6 +407,7 @@ func parseEnvFlags() config {
 		AllocationBatchWaitTime: viper.GetDuration(allocationBatchWaitTime),
 		PodNamespace:            viper.GetString(podNamespace),
 		LeaderElection:          viper.GetBool(leaderElectionFlag),
+		HTTPPort:                viper.GetString(httpPort),
 	}
 }
 
@@ -454,6 +461,7 @@ type config struct {
 	AllocationBatchWaitTime time.Duration
 	PodNamespace            string
 	LeaderElection          bool
+	HTTPPort                string
 }
 
 // validate ensures the ctlConfig data is valid.
@@ -549,6 +557,7 @@ type runner interface {
 
 type httpServer struct {
 	http.ServeMux
+	Port string
 }
 
 func whenLeader(ctx context.Context, cancel context.CancelFunc, logger *logrus.Entry, doLeaderElection bool, kubeClient *kubernetes.Clientset, namespace string, start func(_ context.Context)) {
@@ -604,7 +613,7 @@ func whenLeader(ctx context.Context, cancel context.CancelFunc, logger *logrus.E
 func (h *httpServer) Run(_ context.Context, _ int) error {
 	logger.Info("Starting http server...")
 	srv := &http.Server{
-		Addr:    ":8080",
+		Addr:    ":" + h.Port,
 		Handler: h,
 	}
 	defer srv.Close() // nolint: errcheck
@@ -613,7 +622,7 @@ func (h *httpServer) Run(_ context.Context, _ int) error {
 		if err == http.ErrServerClosed {
 			logger.WithError(err).Info("http server closed")
 		} else {
-			wrappedErr := errors.Wrap(err, "Could not listen on :8080")
+			wrappedErr := errors.Wrap(err, "Could not listen on :"+h.Port)
 			runtime.HandleError(logger.WithError(wrappedErr), wrappedErr)
 		}
 	}

cmd/extensions/main.go

@@ -68,6 +68,8 @@ const (
 	apiServerSustainedQPSFlag    = "api-server-qps"
 	apiServerBurstQPSFlag        = "api-server-qps-burst"
 	readinessShutdownDuration    = "readiness-shutdown-duration"
+	httpPort                     = "http-port"
+	webhookPort                  = "webhook-port"
 )
 
 var (
@@ -138,7 +140,7 @@ func main() {
 		logger.WithError(err).Fatal("Could not initialize cloud product")
 	}
 	// https server and the items that share the Mux for routing
-	httpsServer := https.NewServer(ctlConf.CertFile, ctlConf.KeyFile)
+	httpsServer := https.NewServer(ctlConf.CertFile, ctlConf.KeyFile, ctlConf.WebhookPort)
 	cancelTLS, err := httpsServer.WatchForCertificateChanges()
 	if err != nil {
 		logger.WithError(err).Fatal("Got an error while watching certificate changes")
@@ -150,7 +152,9 @@ func main() {
 	agonesInformerFactory := externalversions.NewSharedInformerFactory(agonesClient, defaultResync)
 	kubeInformerFactory := informers.NewSharedInformerFactory(kubeClient, defaultResync)
 
-	server := &httpServer{}
+	server := &httpServer{
+		Port: ctlConf.HTTPPort,
+	}
 	var health healthcheck.Handler
 
 	// Stackdriver metrics
@@ -249,6 +253,8 @@ func parseEnvFlags() config {
 	viper.SetDefault(logDirFlag, "")
 	viper.SetDefault(logLevelFlag, "Info")
 	viper.SetDefault(logSizeLimitMBFlag, 10000) // 10 GB, will be split into 100 MB chunks
+	viper.SetDefault(httpPort, "8080")
+	viper.SetDefault(webhookPort, "8081")
 
 	pflag.String(keyFileFlag, viper.GetString(keyFileFlag), "Optional. Path to the key file")
 	pflag.String(certFileFlag, viper.GetString(certFileFlag), "Optional. Path to the crt file")
@@ -262,6 +268,8 @@ func parseEnvFlags() config {
 	pflag.Int32(numWorkersFlag, 64, "Number of controller workers per resource type")
 	pflag.Int32(apiServerSustainedQPSFlag, 100, "Maximum sustained queries per second to send to the API server")
 	pflag.Int32(apiServerBurstQPSFlag, 200, "Maximum burst queries per second to send to the API server")
+	pflag.String(httpPort, viper.GetString(httpPort), "Port for the HTTP server. Defaults to 8080, can also use HTTP_PORT env variable")
+	pflag.String(webhookPort, viper.GetString(webhookPort), "Port for the Webhook. Defaults to 8081, can also use WEBHOOK_PORT env variable")
 	pflag.String(logDirFlag, viper.GetString(logDirFlag), "If set, store logs in a given directory.")
 	pflag.Int32(logSizeLimitMBFlag, 1000, "Log file size limit in MB")
 	pflag.String(logLevelFlag, viper.GetString(logLevelFlag), "Agones Log level")
@@ -288,6 +296,8 @@ func parseEnvFlags() config {
 	runtime.Must(viper.BindEnv(logLevelFlag))
 	runtime.Must(viper.BindEnv(logDirFlag))
 	runtime.Must(viper.BindEnv(logSizeLimitMBFlag))
+	runtime.Must(viper.BindEnv(httpPort))
+	runtime.Must(viper.BindEnv(webhookPort))
 	runtime.Must(viper.BindEnv(allocationBatchWaitTime))
 	runtime.Must(viper.BindPFlags(pflag.CommandLine))
 	runtime.Must(viper.BindEnv(readinessShutdownDuration))
@@ -311,6 +321,8 @@ func parseEnvFlags() config {
 		LogDir:                    viper.GetString(logDirFlag),
 		LogLevel:                  viper.GetString(logLevelFlag),
 		LogSizeLimitMB:            int(viper.GetInt32(logSizeLimitMBFlag)),
+		HTTPPort:                  viper.GetString(httpPort),
+		WebhookPort:               viper.GetString(webhookPort),
 		AllocationBatchWaitTime:   viper.GetDuration(allocationBatchWaitTime),
 		ReadinessShutdownDuration: viper.GetDuration(readinessShutdownDuration),
 	}
@@ -333,6 +345,8 @@ type config struct {
 	LogDir                    string
 	LogLevel                  string
 	LogSizeLimitMB            int
+	HTTPPort                  string
+	WebhookPort               string
 	AllocationBatchWaitTime   time.Duration
 	ReadinessShutdownDuration time.Duration
 }
@@ -343,12 +357,13 @@ type runner interface {
 
 type httpServer struct {
 	http.ServeMux
+	Port string
 }
 
 func (h *httpServer) Run(_ context.Context, _ int) error {
 	logger.Info("Starting http server...")
 	srv := &http.Server{
-		Addr:    ":8080",
+		Addr:    ":" + h.Port,
 		Handler: h,
 	}
 	defer srv.Close() // nolint: errcheck
@@ -357,7 +372,7 @@ func (h *httpServer) Run(_ context.Context, _ int) error {
 		if err == http.ErrServerClosed {
 			logger.WithError(err).Info("http server closed")
 		} else {
-			wrappedErr := errors.Wrap(err, "Could not listen on :8080")
+			wrappedErr := errors.Wrap(err, "Could not listen on :"+h.Port)
 			runtime.HandleError(logger.WithError(wrappedErr), wrappedErr)
 		}
 	}

install/helm/agones/templates/controller.yaml

@@ -48,7 +48,7 @@ spec:
 {{- end }}
 {{- if and (.Values.agones.metrics.prometheusServiceDiscovery) (.Values.agones.metrics.prometheusEnabled) }}
         prometheus.io/scrape: "true"
-        prometheus.io/port: "8080"
+        prometheus.io/port: {{ .Values.agones.controller.http.port | quote }}
         prometheus.io/path: "/metrics"
 {{- end }}
 {{- if .Values.agones.controller.annotations }}
@@ -99,6 +99,10 @@ spec:
       priorityClassName: {{ .Values.agones.priorityClassName }}
       {{- end }}
       serviceAccountName: {{ .Values.agones.serviceaccount.controller.name }}
+      {{- if .Values.agones.controller.hostNetwork }}
+      hostNetwork: true
+      dnsPolicy: ClusterFirstWithHostNet
+      {{- end }}
       containers:
       - name: agones-controller
         image: "{{ .Values.agones.image.registry }}/{{ .Values.agones.image.controller.name}}:{{ default .Values.agones.image.tag .Values.agones.image.controller.tag }}"
@@ -180,11 +184,11 @@ spec:
         - name: LEADER_ELECTION
           value: "true"
         {{- end }}
+        - name: HTTP_PORT
+          value: {{ .Values.agones.controller.http.port | quote }}
         ports:
-        - name: webhooks
-          containerPort: 8081
         - name: http
-          containerPort: 8080
+          containerPort: {{ .Values.agones.controller.http.port }}
         livenessProbe:
           httpGet:
             path: /live

install/helm/agones/templates/extensions-deployment.yaml

@@ -41,7 +41,7 @@ spec:
 {{- end }}
 {{- if and (.Values.agones.metrics.prometheusServiceDiscovery) (.Values.agones.metrics.prometheusEnabled) }}
         prometheus.io/scrape: "true"
-        prometheus.io/port: "8080"
+        prometheus.io/port: {{ .Values.agones.extensions.http.port | quote }}
         prometheus.io/path: "/metrics"
 {{- end }}
 {{- if .Values.agones.extensions.annotations }}
@@ -93,6 +93,10 @@ spec:
       {{- end }}
       serviceAccountName: {{ .Values.agones.serviceaccount.controller.name }}
       terminationGracePeriodSeconds: {{ mul .Values.agones.extensions.readiness.periodSeconds .Values.agones.extensions.readiness.failureThreshold 3 }}
+      {{- if .Values.agones.extensions.hostNetwork }}
+      hostNetwork: true
+      dnsPolicy: ClusterFirstWithHostNet
+      {{- end }}
       containers:
       - name: agones-extensions
         image: "{{ .Values.agones.image.registry }}/{{ .Values.agones.image.extensions.name}}:{{ default .Values.agones.image.tag .Values.agones.image.extensions.tag }}"
@@ -107,7 +111,7 @@ spec:
         - name: STACKDRIVER_EXPORTER
           value: {{ .Values.agones.metrics.stackdriverEnabled | quote }}
         - name: STACKDRIVER_LABELS
-          value: {{ .Values.agones.metrics.stackdriverLabels | quote }}       
+          value: {{ .Values.agones.metrics.stackdriverLabels | quote }}
         - name: GCP_PROJECT_ID
           value: {{ .Values.agones.metrics.stackdriverProjectID | quote }}
         - name: NUM_WORKERS
@@ -142,11 +146,15 @@ spec:
           value: "agones-extensions"
         - name: READINESS_SHUTDOWN_DURATION
           value: {{ mul .Values.agones.extensions.readiness.periodSeconds .Values.agones.extensions.readiness.failureThreshold 2 }}s
+        - name: WEBHOOK_PORT
+          value: {{ .Values.agones.extensions.webhooks.port | quote }}
+        - name: HTTP_PORT
+          value: {{ .Values.agones.extensions.http.port | quote }}
         ports:
         - name: webhooks
-          containerPort: 8081
+          containerPort: {{ .Values.agones.extensions.webhooks.port }}
         - name: http
-          containerPort: 8080
+          containerPort: {{ .Values.agones.extensions.http.port }}
         livenessProbe:
           httpGet:
             path: /live
@@ -158,7 +166,7 @@ spec:
         readinessProbe:
           httpGet:
             path: /ready
-            port: 8080
+            port: {{ .Values.agones.extensions.http.port }}
           initialDelaySeconds: {{ .Values.agones.extensions.readiness.initialDelaySeconds }}
           periodSeconds: {{ .Values.agones.extensions.readiness.periodSeconds }}
           failureThreshold: {{ .Values.agones.extensions.readiness.failureThreshold }}

install/helm/agones/templates/extensions-metrics-service.yaml

@@ -28,5 +28,5 @@ spec:
     agones.dev/role: extensions
   ports:
     - name: metrics
-      port: {{ .Values.agones.controller.http.port }}
+      port: {{ .Values.agones.extensions.http.port }}
       targetPort: http

install/helm/agones/templates/service.yaml

@@ -29,8 +29,8 @@ spec:
     agones.dev/role: extensions
   ports:
     - name: webhooks
-      port: 443
+      port: {{ .Values.agones.extensions.webhooks.port }}
       targetPort: webhooks
     - name: web
-      port: {{ .Values.agones.controller.http.port }}
+      port: {{ .Values.agones.extensions.http.port }}
       targetPort: http

install/helm/agones/values.yaml

@@ -55,6 +55,15 @@ agones:
       #   memory: 256Mi
     nodeSelector: {}
     annotations: {}
+  # Determines if the Agones controller should operate in hostNetwork mode.
+  #
+  # This setting is necessary for certain managed Kubernetes clusters (e.g., AWS EKS) that use custom
+  # CNI plugins (such as Calico or Cilium) because the AWS-managed control plane cannot communicate
+  # with pod IP CIDRs.
+  #
+  # Note: The default port may conflicts with others on the host network. Therefore, if
+  # running in hostNetwork mode, you should change `http.port` to an available port.
+    hostNetwork: false
     tolerations:
     - key: "agones.dev/agones-system"
       operator: "Equal"
@@ -100,6 +109,15 @@ agones:
       #   memory: 256Mi
     nodeSelector: {}
     annotations: {}
+  # Determines if the Agones extensions should operate in hostNetwork mode.
+  #
+  # This setting is necessary for certain managed Kubernetes clusters (e.g., AWS EKS) that use custom
+  # CNI plugins (such as Calico or Cilium) because the AWS-managed control plane cannot communicate
+  # with pod IP CIDRs.
+  #
+  # Note: The default port may conflicts with others on the host network. Therefore, if
+  # running in hostNetwork mode, you should change `http.port` and `webhooks.port` to an available port.
+    hostNetwork: false
     tolerations:
     - key: "agones.dev/agones-system"
       operator: "Equal"
@@ -125,6 +143,8 @@ agones:
     numWorkers: 100
     apiServerQPS: 400
     apiServerQPSBurst: 500
+    webhooks:
+      port: 8081
     http:
       port: 8080
     healthCheck:

install/yaml/install.yaml

@@ -16932,7 +16932,7 @@ spec:
     agones.dev/role: extensions
   ports:
     - name: webhooks
-      port: 443
+      port: 8081
       targetPort: webhooks
     - name: web
       port: 8080
@@ -17126,9 +17126,9 @@ spec:
           value: "agones-controller"
         - name: LEADER_ELECTION
           value: "true"
+        - name: HTTP_PORT
+          value: "8080"
         ports:
-        - name: webhooks
-          containerPort: 8081
         - name: http
           containerPort: 8080
         livenessProbe:
@@ -17237,7 +17237,7 @@ spec:
         - name: STACKDRIVER_EXPORTER
           value: "false"
         - name: STACKDRIVER_LABELS
-          value: ""       
+          value: ""
         - name: GCP_PROJECT_ID
           value: ""
         - name: NUM_WORKERS
@@ -17270,6 +17270,10 @@ spec:
           value: "agones-extensions"
         - name: READINESS_SHUTDOWN_DURATION
           value: 18s
+        - name: WEBHOOK_PORT
+          value: "8081"
+        - name: HTTP_PORT
+          value: "8080"
         ports:
         - name: webhooks
           containerPort: 8081