kube-state-metrics: support for exposing single cluster metrics

Signed-off-by: Iceber Gu <wei.cai-nat@daocloud.io>
clusterpedia-io · Aug 7, 2023 · d086ad2 · d086ad2
1 parent 2e33b49
commit d086ad2
Show file tree

Hide file tree

Showing 17 changed files with 3,172 additions and 63 deletions.
diff --git a/go.mod b/go.mod
@@ -6,6 +6,7 @@ require (
 	github.com/DATA-DOG/go-sqlmock v1.5.0
 	github.com/clusterpedia-io/api v0.0.0
 	github.com/go-sql-driver/mysql v1.6.0
+	github.com/gorilla/mux v1.8.0
 	github.com/jackc/pgconn v1.13.0
 	github.com/jackc/pgerrcode v0.0.0-20220416144525-469b46aa5efa
 	github.com/jackc/pgx/v4 v4.17.2

diff --git a/go.sum b/go.sum
@@ -270,6 +270,8 @@ github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
+github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
+github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
 github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw=

diff --git a/pkg/kube_state_metrics/landing_page/landing_page.css b/pkg/kube_state_metrics/landing_page/landing_page.css
@@ -0,0 +1,18 @@
+body {
+  font-family: -apple-system,BlinkMacSystemFont,Segoe UI,Roboto,Helvetica Neue,Arial,Noto Sans,Liberation Sans,sans-serif,Apple Color Emoji,Segoe UI Emoji,Segoe UI Symbol,Noto Color Emoji;
+  margin: 0;
+}
+header {
+  background-color: {{.HeaderColor}};
+  color: #fff;
+  font-size: 1rem;
+  padding: 1rem;
+}
+main {
+  padding: 1rem;
+}
+label {
+  display: inline-block;
+  width: {{.Form.Width}}em;
+}
+{{.ExtraCSS}}
diff --git a/pkg/kube_state_metrics/landing_page/landing_page.html b/pkg/kube_state_metrics/landing_page/landing_page.html
@@ -0,0 +1,35 @@
+<html lang="en">
+  <head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>{{.Name}}</title>
+    <style>{{.CSS}}</style>
+  </head>
+  <body>
+    <header>
+      <h1>{{.Name}}</h1>
+    </header>
+    <main>
+      {{if .Description}}<h2>{{.Description}}</h2>{{end}}
+      {{if .Version}}<div>Version: {{.Version}}</div>{{end}}
+      <div>
+        <ul>
+          {{ range .Links }}
+          <li><a href="{{ .Address }}">{{.Text}}</a>{{if .Description}}: {{.Description}}{{end}}</li>
+          {{ end }}
+        </ul>
+      </div>
+      {{ if .Form.Action }}
+      <div>
+      <form action="{{ .Form.Action}}">
+      {{ range .Form.Inputs }}
+      <label>{{ .Label }}:</label>&nbsp;<input type="{{ .Type }}" name="{{ .Name }}" placeholder="{{ .Placeholder }}" value="{{ .Value }}"><br>
+      {{ end }}
+      <input type="submit" value="Submit">
+      </form>
+      </div>
+      {{ end }}
+      {{ .ExtraHTML }}
+    </main>
+  </body>
+</html>
diff --git a/pkg/kube_state_metrics/metrics_handler.go b/pkg/kube_state_metrics/metrics_handler.go
@@ -6,6 +6,7 @@ import (
 	"net/http"
 	"strings"
 
+	"github.com/gorilla/mux"
 	"github.com/prometheus/common/expfmt"
 	"k8s.io/klog/v2"
 	metricsstore "k8s.io/kube-state-metrics/v2/pkg/metrics_store"
@@ -34,8 +35,14 @@ func NewMetricsHandler(getter ClusterMetricsWriterListGetter, diableGZIPEncoding
 // metrics to the response body.
 func (m *MetricsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	var writers metricsstore.MetricsWriterList
-	for _, wl := range m.getter.GetMetricsWriterList() {
-		writers = append(writers, wl...)
+
+	clusterWriters := m.getter.GetMetricsWriterList()
+	if cluster := mux.Vars(r)["cluster"]; cluster != "" {
+		writers = clusterWriters[cluster]
+	} else {
+		for _, wl := range clusterWriters {
+			writers = append(writers, wl...)
+		}
 	}
 
 	resHeader := w.Header()

diff --git a/pkg/kube_state_metrics/server.go b/pkg/kube_state_metrics/server.go
@@ -1,13 +1,22 @@
 package kubestatemetrics
 
 import (
+	"bytes"
+	_ "embed"
+	"fmt"
+	"net/http"
+	"text/template"
+	"time"
+
+	"github.com/gorilla/mux"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/prometheus/exporter-toolkit/web"
 	"k8s.io/klog/v2"
 
 	"github.com/clusterpedia-io/clusterpedia/pkg/metrics"
+	"github.com/clusterpedia-io/clusterpedia/pkg/version"
 )
 
 type ServerConfig struct {
@@ -26,24 +35,112 @@ func RunServer(config ServerConfig, getter ClusterMetricsWriterListGetter) {
 		}, []string{"method"},
 	)
 
-	handlers := []metrics.Handler{
-		{
-			LandingName: "Metrics",
-			Path:        "/metrics",
-			Handler: promhttp.InstrumentHandlerDuration(durationVec,
-				NewMetricsHandler(getter, config.DisableGZIPEncoding),
-			),
-		},
+	server := &http.Server{
+		Handler:           buildMetricsServer(config, getter, durationVec),
+		ReadHeaderTimeout: 5 * time.Second,
+	}
+
+	flags := &web.FlagConfig{
+		WebListenAddresses: &[]string{config.Endpoint},
+		WebSystemdSocket:   new(bool),
+		WebConfigFile:      &config.TLSConfig,
 	}
-	server, flags := metrics.BuildMetricsServer(
-		config.Endpoint,
-		config.TLSConfig,
-		"Clusterpedia kube-state-metrics",
-		"Metrics for Kubernetes' state managed by Clusterpedia",
-		handlers,
-	)
 
 	klog.Info("Kube State Metrics Server is running...")
 	// TODO(iceber): handle error
 	_ = web.ListenAndServe(server, flags, metrics.Logger)
 }
+
+func buildMetricsServer(config ServerConfig, getter ClusterMetricsWriterListGetter, durationObserver prometheus.ObserverVec) *mux.Router {
+	handler := promhttp.InstrumentHandlerDuration(durationObserver,
+		NewMetricsHandler(getter, config.DisableGZIPEncoding),
+	)
+	mux := mux.NewRouter()
+	mux.Handle("/metrics", handler)
+	mux.Handle("/clusters/{cluster}/metrics", handler)
+
+	// Add index
+	landingConfig := web.LandingConfig{
+		Name:        "Clusterpedia kube-state-metrics",
+		Description: "Metrics for Kubernetes' state managed by Clusterpedia",
+		Version:     version.Get().String(),
+		Links: []web.LandingLinks{
+			{
+				Text:    "Metrics",
+				Address: "/metrics",
+			},
+		},
+	}
+	landingPage, err := NewLandingPage(landingConfig, getter)
+	if err != nil {
+		klog.ErrorS(err, "failed to create landing page")
+	}
+	mux.Handle("/", landingPage)
+	return mux
+}
+
+var (
+	//go:embed landing_page/landing_page.html
+	landingPagehtmlContent string
+	//go:embed landing_page/landing_page.css
+	landingPagecssContent string
+)
+
+func NewLandingPage(c web.LandingConfig, getter ClusterMetricsWriterListGetter) (*LandingPageHandler, error) {
+	length := 0
+	for _, input := range c.Form.Inputs {
+		inputLength := len(input.Label)
+		if inputLength > length {
+			length = inputLength
+		}
+	}
+	c.Form.Width = (float64(length) + 1) / 2
+	if c.CSS == "" {
+		if c.HeaderColor == "" {
+			// Default to Prometheus orange.
+			c.HeaderColor = "#e6522c"
+		}
+		cssTemplate := template.Must(template.New("landing css").Parse(landingPagecssContent))
+		var buf bytes.Buffer
+		if err := cssTemplate.Execute(&buf, c); err != nil {
+			return nil, err
+		}
+		c.CSS = buf.String()
+	}
+	return &LandingPageHandler{
+		config:   c,
+		template: template.Must(template.New("landing page").Parse(landingPagehtmlContent)),
+		getter:   getter,
+	}, nil
+}
+
+type LandingPageHandler struct {
+	config   web.LandingConfig
+	template *template.Template
+
+	getter ClusterMetricsWriterListGetter
+}
+
+func (h *LandingPageHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	clusters := h.getter.GetMetricsWriterList()
+
+	config := h.config
+	config.Links = append(make([]web.LandingLinks, 0, len(config.Links)+len(clusters)),
+		h.config.Links...,
+	)
+	for cluster := range clusters {
+		config.Links = append(config.Links, web.LandingLinks{
+			Text:    cluster + " Metrics",
+			Address: fmt.Sprintf("/clusters/%s/metrics", cluster),
+		})
+	}
+
+	var buf bytes.Buffer
+	if err := h.template.Execute(&buf, config); err != nil {
+		_, _ = w.Write([]byte(err.Error()))
+		return
+	}
+
+	_, _ = w.Write(buf.Bytes())
+	w.Header().Add("Content-Type", "text/html; charset=UTF-8")
+}
diff --git a/pkg/metrics/server.go b/pkg/metrics/server.go
@@ -19,67 +19,44 @@ type Config struct {
 }
 
 func RunServer(config Config) {
-	handlers := []Handler{
-		{
-			LandingName: "Metrics",
-			Path:        "metrics",
-			Handler: promhttp.HandlerFor(registry, promhttp.HandlerOpts{
-				ErrorLog:           Logger,
-				DisableCompression: config.DisableGZIPEncoding,
-			}),
-		},
+	server := &http.Server{
+		Handler:           buildMetricsServer(config),
+		ReadHeaderTimeout: 6 * time.Second,
+	}
+
+	flags := &web.FlagConfig{
+		WebListenAddresses: &[]string{config.Endpoint},
+		WebSystemdSocket:   new(bool),
+		WebConfigFile:      &config.TLSConfig,
 	}
-	server, flags := BuildMetricsServer(
-		config.Endpoint,
-		config.TLSConfig,
-		"clusterpedia clustersynchro manager",
-		"Self-metrics for clusterpedia clustersynchro manager",
-		handlers,
-	)
 
 	klog.Info("Metrics Server is running...")
 	_ = web.ListenAndServe(server, flags, Logger)
 }
 
-type Handler struct {
-	LandingName string
-	Path        string
-	Handler     http.Handler
-}
-
-func BuildMetricsServer(endpoint, tlsConfigFile, name, desc string, handlers []Handler) (*http.Server, *web.FlagConfig) {
-	var links []web.LandingLinks
+func buildMetricsServer(config Config) *http.ServeMux {
 	mux := http.NewServeMux()
-	for _, handler := range handlers {
-		mux.Handle(handler.Path, handler.Handler)
-		if handler.LandingName != "" {
-			links = append(links, web.LandingLinks{
-				Address: handler.Path,
-				Text:    handler.LandingName,
-			})
-		}
-	}
+	mux.Handle("/metrics", promhttp.HandlerFor(registry, promhttp.HandlerOpts{
+		ErrorLog:           Logger,
+		DisableCompression: config.DisableGZIPEncoding,
+	}))
 
 	// Add index
 	landingConfig := web.LandingConfig{
-		Name:        name,
-		Description: desc,
+		Name:        "clusterpedia clustersynchro manager",
+		Description: "Self-metrics for clusterpedia clustersynchro manager",
 		Version:     version.Get().String(),
-		Links:       links,
+		Links: []web.LandingLinks{
+			{
+				Text:    "Metrics",
+				Address: "/metrics",
+			},
+		},
 	}
 	landingPage, err := web.NewLandingPage(landingConfig)
 	if err != nil {
 		klog.ErrorS(err, "failed to create landing page")
 	}
 	mux.Handle("/", landingPage)
-
-	return &http.Server{
-			Handler:           mux,
-			ReadHeaderTimeout: 5 * time.Second,
-		},
-		&web.FlagConfig{
-			WebListenAddresses: &[]string{endpoint},
-			WebSystemdSocket:   new(bool),
-			WebConfigFile:      &tlsConfigFile,
-		}
+	return mux
 }
diff --git a/vendor/github.com/gorilla/mux/AUTHORS b/vendor/github.com/gorilla/mux/AUTHORS
diff --git a/vendor/github.com/gorilla/mux/LICENSE b/vendor/github.com/gorilla/mux/LICENSE