Merge pull request #1187 from eveningcafe/master

add_alertmanager_resolve_timeout

CHANGELOG.md

@@ -4,7 +4,7 @@
 - TBD
 
 ## New features
-- TBD
+- [Alertmanager] Add alertmanager resolve timeout configuration option - [#1187](https://github.com/jertel/elastalert2/pull/1187) - @eveningcafe
 
 ## Other changes
 - [Docs] Clarify docs to state that alert_time_limit should not be 0 - [#1208](https://github.com/jertel/elastalert2/pull/1208) - @jertel

docs/source/ruletypes.rst

@@ -1828,7 +1828,9 @@ Optional:
 ``alertmanager_ignore_ssl_errors``: By default ElastAlert 2 will verify SSL certificate. Set this option to ``True`` if you want to ignore SSL errors.
 
 ``alertmanager_timeout``: You can specify a timeout value, in seconds, for making communicating with Alertmanager. The default is 10. If a timeout occurs, the alert will be retried next time ElastAlert 2 cycles.
-
+``
+``alertmanager_resolve_time``: Optionally provide an automatic resolution timeframe. If no further alerts arrive within this time period alertmanager will automatically mark the alert as resolved. If not defined it will use Alertmanager's default behavior.
+``
 ``alertmanager_basic_auth_login``: Basic authentication username.
 
 ``alertmanager_basic_auth_password``: Basic authentication password.
@@ -1842,6 +1844,8 @@ Example usage::
   alertmanager_alertname: "Title"
   alertmanager_annotations:
     severity: "error"
+  alertmanager_resolve_time:
+    minutes: 10
   alertmanager_labels:
     source: "elastalert"
   alertmanager_fields:

elastalert/alerters/alertmanager.py

@@ -1,6 +1,7 @@
 import json
 import warnings
 
+from datetime import datetime, timedelta, timezone
 import requests
 from requests import RequestException
 from requests.auth import HTTPBasicAuth
@@ -30,7 +31,10 @@ def __init__(self, rule):
         self.timeout = self.rule.get('alertmanager_timeout', 10)
         self.alertmanager_basic_auth_login = self.rule.get('alertmanager_basic_auth_login', None)
         self.alertmanager_basic_auth_password = self.rule.get('alertmanager_basic_auth_password', None)
-
+        if 'alertmanager_resolve_time' in rule:
+            self.resolve_timeout = timedelta(**rule['alertmanager_resolve_time'])
+        else:
+            self.resolve_timeout = None
 
     @staticmethod
     def _json_or_string(obj):
@@ -53,11 +57,16 @@ def alert(self, matches):
         self.annotations.update({
             self.title_labelname: self.create_title(matches),
             self.body_labelname: self.create_alert_body(matches)})
+
         payload = {
             'annotations': self.annotations,
             'labels': self.labels
         }
 
+        if self.resolve_timeout is not None:
+            end_time = self.now() + self.resolve_timeout
+            payload['endsAt'] = end_time.isoformat()
+
         for host in self.hosts:
             try:
                 url = '{}/api/{}/alerts'.format(host, self.api_version)
@@ -87,3 +96,6 @@ def alert(self, matches):
 
     def get_info(self):
         return {'type': 'alertmanager'}
+
+    def now(self):
+        return datetime.nowdatetime.now(timezone.utc)

elastalert/schema.yaml

@@ -329,6 +329,7 @@ properties:
   alertmanager_ca_certs: {type: [boolean, string]}
   alertmanager_ignore_ssl_errors: {type: boolean}
   alertmanager_timeout: {type: integer}
+  alertmanager_resolve_time: *timedelta
   alertmanager_basic_auth_login: {type: string}
   alertmanager_basic_auth_password: {type: string}
   alertmanager_labels:

tests/alerters/alertmanager_test.py

@@ -1,3 +1,4 @@
+import datetime
 import json
 import logging
 import pytest
@@ -386,3 +387,66 @@ def test_alertmanager_basic_auth():
         auth=HTTPBasicAuth('user', 'password')
     )
     assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data'])
+
+
+def test_alertmanager_resolve_timeout(caplog):
+    caplog.set_level(logging.INFO)
+    rule = {
+        'name': 'Test Alertmanager Rule',
+        'type': 'any',
+        'alertmanager_hosts': ['http://alertmanager:9093'],
+        'alertmanager_alertname': 'Title',
+        'alertmanager_annotations': {'severity': 'error'},
+        'alertmanager_resolve_time': {'minutes': 10},
+        'alertmanager_labels': {'source': 'elastalert'},
+        'alertmanager_fields': {'msg': 'message', 'log': '@log_name'},
+        'alert_subject_args': ['message', '@log_name'],
+        'alert': []
+    }
+    rules_loader = FileRulesLoader({})
+    rules_loader.load_modules(rule)
+
+    alert = AlertmanagerAlerter(rule)
+    match = {
+        '@timestamp': '2021-01-01T00:00:00',
+        'somefield': 'foobarbaz',
+        'message': 'Quit 123',
+        '@log_name': 'mysqld.general'
+    }
+    with mock.patch('requests.post') as mock_post_request:
+        alert.now = mock.Mock(return_value=datetime.datetime(2023, 7, 14, tzinfo=datetime.timezone.utc))
+        alert.alert([match])
+
+    expected_data = [
+        {
+            'annotations':
+            {
+                'severity': 'error',
+                'summary': 'Test Alertmanager Rule',
+                'description': 'Test Alertmanager Rule\n\n' +
+                '@log_name: mysqld.general\n' +
+                '@timestamp: 2021-01-01T00:00:00\n' +
+                'message: Quit 123\nsomefield: foobarbaz\n'
+            },
+            'endsAt': '2023-07-14T00:10:00+00:00',
+            'labels': {
+                'source': 'elastalert',
+                'msg': 'Quit 123',
+                'log': 'mysqld.general',
+                'alertname': 'Title',
+                'elastalert_rule': 'Test Alertmanager Rule'
+            }
+        }
+    ]
+
+    mock_post_request.assert_called_once_with(
+        'http://alertmanager:9093/api/v1/alerts',
+        data=mock.ANY,
+        headers={'content-type': 'application/json'},
+        proxies=None,
+        verify=True,
+        timeout=10,
+        auth=None
+    )
+    assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data'])
+    assert ('elastalert', logging.INFO, "Alert sent to Alertmanager") == caplog.record_tuples[0]