Skip to content

Commit

Permalink
Merge branch 'stempler-feat/counter-only-relabels (#128)'
Browse files Browse the repository at this point in the history
  • Loading branch information
martin-helmich committed Jan 5, 2021
2 parents 969f1c8 + 2f9c03f commit 1caf11e
Show file tree
Hide file tree
Showing 8 changed files with 111 additions and 21 deletions.
3 changes: 3 additions & 0 deletions README.adoc
Original file line number Diff line number Diff line change
Expand Up @@ -486,6 +486,9 @@ namespace "app1" {
split = 2
separator = " " // <1>
// if enabled, only include label in response count metric (default is false)
only_counter = false
match "^/users/[0-9]+" {
replacement = "/users/:id"
}
Expand Down
1 change: 1 addition & 0 deletions config/struct_relabel.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ type RelabelConfig struct {
Matches []RelabelValueMatch `hcl:"match"`
Split int `hcl:"split"`
Separator string `hcl:"separator"`
OnlyCounter bool `hcl:"only_counter" yaml:"only_counter"`

WhitelistExists bool
WhitelistMap map[string]interface{}
Expand Down
2 changes: 2 additions & 0 deletions example-config.hcl
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,8 @@ namespace "nginx" {
relabel "user" {
from = "remote_user"
// whitelist = ["-", "user1", "user2"]

only_counter = true
}

relabel "request_uri" {
Expand Down
37 changes: 37 additions & 0 deletions features/only_counter_relabel.feature
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
Feature: Config file allows relabeling that only apply to the request counter

Scenario: Labels are added request counter
Given a running exporter listening with configuration file "test-configuration-only-counter-relabel.hcl"
When the following HTTP request is logged to "access.log"
"""
172.17.0.1 - - [23/Jun/2016:16:04:20 +0000] "GET /users HTTP/1.1" 200 612 "-" "curl/7.29.0" "-" 10 10
172.17.0.1 - foo [23/Jun/2016:16:04:20 +0000] "GET /groups HTTP/1.1" 200 518 "-" "curl/7.29.0" "-" 10 10
"""
Then the exporter should report value 1 for metric nginx_http_response_count_total{method="GET",path="/groups",status="200",user="foo"}
And the exporter should report value 1 for metric nginx_http_response_count_total{method="GET",path="/users",status="200",user="other"}

Scenario: Labels are not add added to size counter
Given a running exporter listening with configuration file "test-configuration-only-counter-relabel.hcl"
When the following HTTP request is logged to "access.log"
"""
172.17.0.1 - - [23/Jun/2016:16:04:20 +0000] "GET /users HTTP/1.1" 200 612 "-" "curl/7.29.0" "-" 10 10
172.17.0.1 - foo [23/Jun/2016:16:04:20 +0000] "GET /groups HTTP/1.1" 200 518 "-" "curl/7.29.0" "-" 10 10
"""
Then the exporter should report value 518 for metric nginx_http_response_size_bytes{method="GET",status="200",user="foo"}
And the exporter should report value 612 for metric nginx_http_response_size_bytes{method="GET",status="200",user="other"}

Scenario: Labels are not add added to histograms or summaries
Given a running exporter listening with configuration file "test-configuration-only-counter-relabel.hcl"
When the following HTTP request is logged to "access.log"
"""
172.17.0.1 - - [23/Jun/2016:16:04:20 +0000] "GET /users HTTP/1.1" 200 612 "-" "curl/7.29.0" "-" 10 10
172.17.0.1 - foo [23/Jun/2016:16:04:20 +0000] "GET /groups HTTP/1.1" 200 518 "-" "curl/7.29.0" "-" 10 10
"""
Then the exporter should report value 1 for metric nginx_http_upstream_time_seconds_hist_count{method="GET",status="200",user="foo"}
And the exporter should report value 1 for metric nginx_http_upstream_time_seconds_hist_count{method="GET",status="200",user="other"}
And the exporter should report value 1 for metric nginx_http_upstream_time_seconds_count{method="GET",status="200",user="foo"}
And the exporter should report value 1 for metric nginx_http_upstream_time_seconds_count{method="GET",status="200",user="other"}
And the exporter should report value 1 for metric nginx_http_response_time_seconds_hist_count{method="GET",status="200",user="foo"}
And the exporter should report value 1 for metric nginx_http_response_time_seconds_hist_count{method="GET",status="200",user="other"}
And the exporter should report value 1 for metric nginx_http_response_time_seconds_count{method="GET",status="200",user="foo"}
And the exporter should report value 1 for metric nginx_http_response_time_seconds_count{method="GET",status="200",user="other"}
21 changes: 21 additions & 0 deletions features/test-configuration-only-counter-relabel.hcl
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
port = 4040
enable_experimental = true

namespace "nginx" {
source {
files = [".behave-sandbox/access.log"]
}
format = "$remote_addr - $remote_user [$time_local] \"$request\" $status $body_bytes_sent \"$http_referer\" \"$http_user_agent\" \"$http_x_forwarded_for\" $request_time $upstream_response_time"

relabel "user" {
from = "remote_user"
whitelist = ["foo", "bar"]
}

relabel "path" {
from = "request"
split = 2

only_counter = true
}
}
11 changes: 0 additions & 11 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -318,7 +318,6 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R
github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
github.com/prometheus/procfs v0.1.3 h1:F0+tqvhOksq22sc6iCHF5WGlWjdwj92p0udFh1VFBS8=
github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
github.com/prometheus/procfs v0.2.0 h1:wH4vA7pcjKuZzjF7lM8awk4fnuJO6idemZXoKnULUx4=
github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
Expand All @@ -327,7 +326,6 @@ github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6So
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
github.com/ryanuber/columnize v2.1.0+incompatible h1:j1Wcmh8OrK4Q7GXY+V7SVSY8nUWQxHW5TkBe7YUl+2s=
github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E=
github.com/satyrius/gonx v1.3.1-0.20180709120835-47c52b995fe5 h1:nIAK+9DnhpSebWeiIqvPr0rqSDC3j9r1I2bp0OJAYVE=
Expand All @@ -336,7 +334,6 @@ github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUt
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
Expand All @@ -351,16 +348,13 @@ github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3
github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=
github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926 h1:G3dpKMzFDjgEh2q1Z7zUUtKa8ViPtH+ocF0bE0g00O8=
github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
Expand Down Expand Up @@ -448,14 +442,11 @@ golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1 h1:ogLJMz+qpzav7lGMh10LMvAkM/fAoGlaiiHYiFYdm80=
golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e h1:AyodaIpKjppX+cBfTASF2E1US3H2JFBj920Ot3rtDjs=
golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
Expand Down Expand Up @@ -499,7 +490,6 @@ google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLY
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
google.golang.org/protobuf v1.21.0 h1:qdOKuR/EIArgaWNjetjgTzgVTAZ+S/WXVrq9HW9zimw=
google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM=
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
Expand All @@ -525,7 +515,6 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
Expand Down
43 changes: 33 additions & 10 deletions main.go
Original file line number Diff line number Diff line change
Expand Up @@ -89,23 +89,30 @@ func (m *Metrics) Init(cfg *config.NamespaceConfig) {
cfg.MustCompile()

labels := cfg.OrderedLabelNames
counterLabels := labels

for i := range cfg.RelabelConfigs {
labels = append(labels, cfg.RelabelConfigs[i].TargetLabel)
if !cfg.RelabelConfigs[i].OnlyCounter {
labels = append(labels, cfg.RelabelConfigs[i].TargetLabel)
}
counterLabels = append(counterLabels, cfg.RelabelConfigs[i].TargetLabel)
}

for _, r := range relabeling.DefaultRelabelings {
if !inLabels(r.TargetLabel, labels) {
labels = append(labels, r.TargetLabel)
}
if !inLabels(r.TargetLabel, counterLabels) {
counterLabels = append(counterLabels, r.TargetLabel)
}
}

m.countTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: cfg.NamespacePrefix,
ConstLabels: cfg.NamespaceLabels,
Name: "http_response_count_total",
Help: "Amount of processed HTTP requests",
}, labels)
}, counterLabels)

m.responseBytesTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: cfg.NamespacePrefix,
Expand Down Expand Up @@ -326,13 +333,22 @@ func processNamespace(nsCfg config.NamespaceConfig, metrics *Metrics) {
}
}

// determine once if there are any relabeling configurations for only the response counter
hasCounterOnlyLabels := false
for _, r := range nsCfg.RelabelConfigs {
if r.OnlyCounter {
hasCounterOnlyLabels = true
break
}
}

for _, f := range followers {
go processSource(nsCfg, f, parser, metrics)
go processSource(nsCfg, f, parser, metrics, hasCounterOnlyLabels)
}

}

func processSource(nsCfg config.NamespaceConfig, t tail.Follower, parser *gonx.Parser, metrics *Metrics) {
func processSource(nsCfg config.NamespaceConfig, t tail.Follower, parser *gonx.Parser, metrics *Metrics, hasCounterOnlyLabels bool) {
relabelings := relabeling.NewRelabelings(nsCfg.RelabelConfigs)
relabelings = append(relabelings, relabeling.DefaultRelabelings...)
relabelings = relabeling.UniqueRelabelings(relabelings)
Expand Down Expand Up @@ -370,24 +386,31 @@ func processSource(nsCfg config.NamespaceConfig, t tail.Follower, parser *gonx.P
}
}

var notCounterValues []string
if hasCounterOnlyLabels {
notCounterValues = relabeling.StripOnlyCounterValues(labelValues, relabelings)
} else {
notCounterValues = labelValues
}

metrics.countTotal.WithLabelValues(labelValues...).Inc()

if bytes, ok := floatFromFields(fields, "body_bytes_sent"); ok {
metrics.responseBytesTotal.WithLabelValues(labelValues...).Add(bytes)
metrics.responseBytesTotal.WithLabelValues(notCounterValues...).Add(bytes)
}

if bytes, ok := floatFromFields(fields, "request_length"); ok {
metrics.requestBytesTotal.WithLabelValues(labelValues...).Add(bytes)
metrics.requestBytesTotal.WithLabelValues(notCounterValues...).Add(bytes)
}

if upstreamTime, ok := floatFromFields(fields, "upstream_response_time"); ok {
metrics.upstreamSeconds.WithLabelValues(labelValues...).Observe(upstreamTime)
metrics.upstreamSecondsHist.WithLabelValues(labelValues...).Observe(upstreamTime)
metrics.upstreamSeconds.WithLabelValues(notCounterValues...).Observe(upstreamTime)
metrics.upstreamSecondsHist.WithLabelValues(notCounterValues...).Observe(upstreamTime)
}

if responseTime, ok := floatFromFields(fields, "request_time"); ok {
metrics.responseSeconds.WithLabelValues(labelValues...).Observe(responseTime)
metrics.responseSecondsHist.WithLabelValues(labelValues...).Observe(responseTime)
metrics.responseSeconds.WithLabelValues(notCounterValues...).Observe(responseTime)
metrics.responseSecondsHist.WithLabelValues(notCounterValues...).Observe(responseTime)
}
}
}
Expand Down
14 changes: 14 additions & 0 deletions relabeling/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,3 +38,17 @@ func UniqueRelabelings(relabelings []*Relabeling) []*Relabeling {
}
return result
}

// StripOnlyCounterValues strips all values that are associated to relabelings only intended for the request counter
func StripOnlyCounterValues(values []string, relabelings []*Relabeling) []string {
result := make([]string, 0, len(values))
offset := len(values) - len(relabelings)
for i := range values {
if i >= offset && relabelings[i-offset].OnlyCounter {
// skip if relabeling and only enabled for counter
continue
}
result = append(result, values[i])
}
return result
}

0 comments on commit 1caf11e

Please sign in to comment.