From 778e2bae6239d7b913c615724d97dfe0b39fb922 Mon Sep 17 00:00:00 2001 From: mmta Date: Tue, 16 Jul 2019 00:20:45 +0700 Subject: [PATCH 1/2] fix: support multiple ES filter terms in dpluger --- internal/pkg/dpluger/es5client.go | 17 ++++++++++------- internal/pkg/dpluger/es6client.go | 17 ++++++++++------- internal/pkg/dpluger/es7client.go | 17 ++++++++++------- 3 files changed, 30 insertions(+), 21 deletions(-) diff --git a/internal/pkg/dpluger/es5client.go b/internal/pkg/dpluger/es5client.go index 376b0136..a766e95a 100644 --- a/internal/pkg/dpluger/es5client.go +++ b/internal/pkg/dpluger/es5client.go @@ -85,16 +85,19 @@ func (es *es5Client) Collect(plugin Plugin, confFile, sidSource, esFilter string size := 1000 c.init(plugin.Name, confFile) terms := elastic5.NewTermsAggregation().Field(sidSource).Size(size) - var query elastic5.Query + query := elastic5.NewBoolQuery() if esFilter != "" { - s := strings.Split(esFilter, "=") - if len(s) != 2 { - err = errors.New("Cannot split the ES filter term") - return + coll := strings.Split(esFilter, ";") + for _, v := range coll { + s := strings.Split(v, "=") + if len(s) != 2 { + err = errors.New("Cannot split the ES filter term") + return + } + query = query.Must(elastic5.NewTermQuery(s[0], s[1])) } - query = elastic5.NewTermsQuery(s[0], s[1]) } else { - query = elastic5.NewMatchAllQuery() + query = query.Must(elastic5.NewMatchAllQuery()) } ctx := context.Background() diff --git a/internal/pkg/dpluger/es6client.go b/internal/pkg/dpluger/es6client.go index 773515aa..99751579 100644 --- a/internal/pkg/dpluger/es6client.go +++ b/internal/pkg/dpluger/es6client.go @@ -87,16 +87,19 @@ func (es *es6Client) Collect(plugin Plugin, confFile, sidSource, esFilter string size := 1000 c.init(plugin.Name, confFile) terms := elastic6.NewTermsAggregation().Field(sidSource).Size(size) - var query elastic6.Query + query := elastic6.NewBoolQuery() if esFilter != "" { - s := strings.Split(esFilter, "=") - if len(s) != 2 { - err = errors.New("Cannot split the ES filter term") - return + coll := strings.Split(esFilter, ";") + for _, v := range coll { + s := strings.Split(v, "=") + if len(s) != 2 { + err = errors.New("Cannot split the ES filter term") + return + } + query = query.Must(elastic6.NewTermQuery(s[0], s[1])) } - query = elastic6.NewTermsQuery(s[0], s[1]) } else { - query = elastic6.NewMatchAllQuery() + query = query.Must(elastic6.NewMatchAllQuery()) } ctx := context.Background() diff --git a/internal/pkg/dpluger/es7client.go b/internal/pkg/dpluger/es7client.go index 24c2457f..28576878 100644 --- a/internal/pkg/dpluger/es7client.go +++ b/internal/pkg/dpluger/es7client.go @@ -86,16 +86,19 @@ func (es *es7Client) Collect(plugin Plugin, confFile, sidSource, esFilter string size := 1000 c.init(plugin.Name, confFile) terms := elastic7.NewTermsAggregation().Field(sidSource).Size(size) - var query elastic7.Query + query := elastic7.NewBoolQuery() if esFilter != "" { - s := strings.Split(esFilter, "=") - if len(s) != 2 { - err = errors.New("Cannot split the ES filter term") - return + coll := strings.Split(esFilter, ";") + for _, v := range coll { + s := strings.Split(v, "=") + if len(s) != 2 { + err = errors.New("Cannot split the ES filter term") + return + } + query = query.Must(elastic7.NewTermQuery(s[0], s[1])) } - query = elastic7.NewTermsQuery(s[0], s[1]) } else { - query = elastic7.NewMatchAllQuery() + query = query.Must(elastic7.NewMatchAllQuery()) } ctx := context.Background() From cfb1f2ceaf7165379ea63f51901c8e22f4247ae6 Mon Sep 17 00:00:00 2001 From: mmta Date: Tue, 16 Jul 2019 00:22:37 +0700 Subject: [PATCH 2/2] fix: add note about multiple terms in generated cf --- internal/pkg/dpluger/dpluger.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/pkg/dpluger/dpluger.go b/internal/pkg/dpluger/dpluger.go index 8be58530..8d13f736 100644 --- a/internal/pkg/dpluger/dpluger.go +++ b/internal/pkg/dpluger/dpluger.go @@ -111,7 +111,7 @@ func CreateConfig(confFile, address, index, name, typ string) error { plugin.IdentifierField = getStaticText("LOGSTASH_IDENTIFYING_FIELD") + " (example: [application] or [fields][log_type] etc)" plugin.IdentifierValue = getStaticText("IDENTIFYING_FIELD_VALUE") + " (example: suricata)" plugin.IdentifierFilter = getStaticText("ADDITIONAL_FILTER") + " (example: and [alert])" - plugin.ESCollectionFilter = getStaticText("ES_TERM_FILTER") + " (example: type=http will only collect SIDs from documents whose type field is http)" + plugin.ESCollectionFilter = getStaticText("ES_TERM_FILTER") + " (example: type=http will only collect SIDs from documents whose type field is http). Separate multiple term with ; character" plugin.Fields.Timestamp = defMappingText plugin.Fields.TimestampFormat = getStaticText("TIMESTAMP_FORMAT") + " (example: ISO8601)" plugin.Fields.Title = defMappingText