From 6141347b7953658d5082bac9e9a5d5941d9d0101 Mon Sep 17 00:00:00 2001 From: Danny Kopping Date: Tue, 9 May 2023 20:26:03 +0200 Subject: [PATCH] Adding more buckets to loki_memcache_request_duration_seconds histogram (#9431) **What this PR does / why we need it**: Users using extstore (with NVMe drives) may experience latencies higher than the ~250ms max as previously defined. --------- Signed-off-by: Danny Kopping --- CHANGELOG.md | 1 + pkg/storage/chunk/cache/memcached.go | 9 +++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index becd094a79d4..10ecbb73edb1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ * [8732](https://github.com/grafana/loki/pull/8732) **abaguas**: azure: respect retry config before cancelling the context * [9206](https://github.com/grafana/loki/pull/9206) **dannykopping**: Ruler: log rule evaluation detail. * [9184](https://github.com/grafana/loki/pull/9184) **periklis**: Bump dskit to introduce IPv6 support for memberlist +* [9431](https://github.com/grafana/loki/pull/9431) **dannykopping**: Add more buckets to `loki_memcache_request_duration_seconds` metric; latencies can increase if using memcached with NVMe ##### Fixes diff --git a/pkg/storage/chunk/cache/memcached.go b/pkg/storage/chunk/cache/memcached.go index 4256baf8b0d6..fe8a3edc4cfe 100644 --- a/pkg/storage/chunk/cache/memcached.go +++ b/pkg/storage/chunk/cache/memcached.go @@ -65,8 +65,13 @@ func NewMemcached(cfg MemcachedConfig, client MemcachedClient, name string, reg Namespace: "loki", Name: "memcache_request_duration_seconds", Help: "Total time spent in seconds doing memcache requests.", - // Memcached requests are very quick: smallest bucket is 16us, biggest is 1s - Buckets: prometheus.ExponentialBuckets(0.000016, 4, 8), + // 16us, 64us, 256us, 1.024ms, 4.096ms, 16.384ms, 65.536ms, 150ms, 250ms, 500ms, 1s + Buckets: append(prometheus.ExponentialBuckets(0.000016, 4, 7), []float64{ + (150 * time.Millisecond).Seconds(), + (250 * time.Millisecond).Seconds(), + (500 * time.Millisecond).Seconds(), + (time.Second).Seconds(), + }...), ConstLabels: prometheus.Labels{"name": name}, }, []string{"method", "status_code"}), ),