From bc4e6c58dc4b3bb27008f1dd7983228b60e9bc30 Mon Sep 17 00:00:00 2001 From: Vlad <13818348+walldiss@users.noreply.github.com> Date: Thu, 19 Sep 2024 19:23:49 +0200 Subject: [PATCH 1/2] ensure libp2p metrics are collected to prometheus --- cmd/flags_misc.go | 26 +++++++++++++------------- nodebuilder/p2p/config.go | 3 +++ nodebuilder/p2p/host.go | 5 ++++- nodebuilder/p2p/metrics.go | 15 ++++++++++++--- 4 files changed, 32 insertions(+), 17 deletions(-) diff --git a/cmd/flags_misc.go b/cmd/flags_misc.go index 332d2f8022..93c56ce8ca 100644 --- a/cmd/flags_misc.go +++ b/cmd/flags_misc.go @@ -149,12 +149,12 @@ func ParseMiscFlags(ctx context.Context, cmd *cobra.Command) (context.Context, e } } - ok, err := cmd.Flags().GetBool(pprofFlag) + enablePprof, err := cmd.Flags().GetBool(pprofFlag) if err != nil { panic(err) } - if ok { + if enablePprof { // TODO(@Wondertan): Eventually, this should be registered on http server in RPC // by passing the http.Server with preregistered pprof handlers to the node. // Node should not register pprof itself. @@ -174,12 +174,12 @@ func ParseMiscFlags(ctx context.Context, cmd *cobra.Command) (context.Context, e }() } - ok, err = cmd.Flags().GetBool(pyroscopeFlag) + enablePyro, err := cmd.Flags().GetBool(pyroscopeFlag) if err != nil { panic(err) } - if ok { + if enablePyro { ctx = WithNodeOptions(ctx, nodebuilder.WithPyroscope( cmd.Flag(pyroscopeEndpoint).Value.String(), @@ -188,12 +188,12 @@ func ParseMiscFlags(ctx context.Context, cmd *cobra.Command) (context.Context, e ) } - ok, err = cmd.Flags().GetBool(tracingFlag) + enableTracing, err := cmd.Flags().GetBool(tracingFlag) if err != nil { panic(err) } - if ok { + if enableTracing { opts := []otlptracehttp.Option{ otlptracehttp.WithCompression(otlptracehttp.GzipCompression), otlptracehttp.WithEndpoint(cmd.Flag(tracingEndpointFlag).Value.String()), @@ -205,11 +205,11 @@ func ParseMiscFlags(ctx context.Context, cmd *cobra.Command) (context.Context, e } pyroOpts := make([]otelpyroscope.Option, 0) - ok, err = cmd.Flags().GetBool(pyroscopeTracing) + enablePyroTracing, err := cmd.Flags().GetBool(pyroscopeTracing) if err != nil { panic(err) } - if ok { + if enablePyroTracing { pyroOpts = append(pyroOpts, otelpyroscope.WithAppName("celestia.da-node"), otelpyroscope.WithPyroscopeURL(cmd.Flag(pyroscopeEndpoint).Value.String()), @@ -222,12 +222,12 @@ func ParseMiscFlags(ctx context.Context, cmd *cobra.Command) (context.Context, e ctx = WithNodeOptions(ctx, nodebuilder.WithTraces(opts, pyroOpts)) } - ok, err = cmd.Flags().GetBool(metricsFlag) + enableMetrics, err := cmd.Flags().GetBool(metricsFlag) if err != nil { panic(err) } - if ok { + if enableMetrics { opts := []otlpmetrichttp.Option{ otlpmetrichttp.WithCompression(otlpmetrichttp.GzipCompression), otlpmetrichttp.WithEndpoint(cmd.Flag(metricsEndpointFlag).Value.String()), @@ -241,13 +241,13 @@ func ParseMiscFlags(ctx context.Context, cmd *cobra.Command) (context.Context, e ctx = WithNodeOptions(ctx, nodebuilder.WithMetrics(opts, NodeType(ctx))) } - ok, err = cmd.Flags().GetBool(p2pMetrics) + enablep2pMetrics, err := cmd.Flags().GetBool(p2pMetrics) if err != nil { panic(err) } - if ok { - if metricsEnabled, _ := cmd.Flags().GetBool(metricsFlag); !metricsEnabled { + if enablep2pMetrics { + if !enableMetrics { log.Error("--p2p.metrics used without --metrics being enabled") } else { ctx = WithNodeOptions(ctx, modp2p.WithMetrics()) diff --git a/nodebuilder/p2p/config.go b/nodebuilder/p2p/config.go index 5276ff9acb..47e9ed0c0f 100644 --- a/nodebuilder/p2p/config.go +++ b/nodebuilder/p2p/config.go @@ -34,6 +34,9 @@ type Config struct { // Allowlist for IPColocation PubSub parameter, a list of string CIDRs IPColocationWhitelist []string + + // enableMetrics enables prometheus metrics collection in libp2p + enableMetrics bool } // DefaultConfig returns default configuration for P2P subsystem. diff --git a/nodebuilder/p2p/host.go b/nodebuilder/p2p/host.go index 913591b37d..394d5c2742 100644 --- a/nodebuilder/p2p/host.go +++ b/nodebuilder/p2p/host.go @@ -104,7 +104,10 @@ func host(params hostParams) (HostBase, error) { libp2p.DefaultMuxers, } - if params.Registry != nil { + if params.Cfg.enableMetrics { + if params.Registry == nil { + panic("metrics enabled but no prometheus registry provided") + } opts = append(opts, libp2p.PrometheusRegisterer(params.Registry)) } else { opts = append(opts, libp2p.DisableMetrics()) diff --git a/nodebuilder/p2p/metrics.go b/nodebuilder/p2p/metrics.go index 606e5d39f9..f42bbdad5f 100644 --- a/nodebuilder/p2p/metrics.go +++ b/nodebuilder/p2p/metrics.go @@ -18,8 +18,9 @@ import ( func WithMetrics() fx.Option { return fx.Options( fx.Provide(resourceManagerOpt(traceReporter)), - fx.Invoke(prometheusMetrics), + fx.Provide(prometheusMetrics), fx.Invoke(enableBitswapMetrics), + fx.Invoke(enableMetrics), ) } @@ -32,12 +33,16 @@ const ( peerIDLabel = "peer_id" ) +func enableMetrics(cfg *Config) { + cfg.enableMetrics = true +} + // prometheusMetrics option sets up native libp2p metrics up func prometheusMetrics(lifecycle fx.Lifecycle, peerID peer.ID, nodeType node.Type, network Network, -) error { +) (prometheus.Registerer, error) { reg := prometheus.NewRegistry() labels := prometheus.Labels{ networkLabel: network.String(), @@ -45,6 +50,10 @@ func prometheusMetrics(lifecycle fx.Lifecycle, peerIDLabel: peerID.String(), } wrapped := prometheus.WrapRegistererWith(labels, reg) + // Set the default global registerer to the wrapped one with labels. This way all the metrics + // registered with the default registerer will be labeled with the provided labels. It is important + // because unlike libp2p metrics, bitswap metrics are registered with the default global registerer. + prometheus.DefaultRegisterer = wrapped mux := http.NewServeMux() handler := promhttp.HandlerFor(reg, promhttp.HandlerOpts{Registry: wrapped}) @@ -72,5 +81,5 @@ func prometheusMetrics(lifecycle fx.Lifecycle, return promHTTPServer.Shutdown(ctx) }, }) - return nil + return wrapped, nil } From e756b0fa62ab4f23484c5cc009f78cc7b5e80bfa Mon Sep 17 00:00:00 2001 From: Vlad <13818348+walldiss@users.noreply.github.com> Date: Thu, 19 Sep 2024 20:01:38 +0200 Subject: [PATCH 2/2] enforce metrics order through fx --- nodebuilder/p2p/bitswap.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/nodebuilder/p2p/bitswap.go b/nodebuilder/p2p/bitswap.go index 995800376e..bfc7f760d7 100644 --- a/nodebuilder/p2p/bitswap.go +++ b/nodebuilder/p2p/bitswap.go @@ -13,6 +13,7 @@ import ( routinghelpers "github.com/libp2p/go-libp2p-routing-helpers" hst "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/protocol" + "github.com/prometheus/client_golang/prometheus" "go.uber.org/fx" "github.com/celestiaorg/celestia-node/share/eds" @@ -29,6 +30,7 @@ const ( // dataExchange provides a constructor for IPFS block's DataExchange over BitSwap. func dataExchange(params bitSwapParams) exchange.Interface { + fmt.Println("START BITSWAP") prefix := protocolID(params.Net) net := network.NewFromIpfsHost(params.Host, &routinghelpers.Null{}, network.Prefix(prefix)) @@ -83,6 +85,9 @@ type bitSwapParams struct { Net Network Host hst.Host Bs blockstore.Blockstore + // Registerer is unused, it is in dependency graph to ensure that prometheus metrics are enabled before bitswap + // is started. + Registerer prometheus.Registerer `optional:"true"` } func protocolID(network Network) protocol.ID {