From a15d4d0280366bc0ae3331d82058f80e0e2e2ebb Mon Sep 17 00:00:00 2001 From: liubo Date: Thu, 13 Feb 2020 12:45:52 +0800 Subject: [PATCH] Support multiple monitor nodes (#1137) * Support multiple monitor nodes --- roles/check_config_static/tasks/main.yml | 6 +- roles/grafana/templates/data_source.json.j2 | 17 +++++- roles/prometheus/templates/prometheus.yml.j2 | 59 +++++++++++++------- rolling_update_monitor.yml | 4 +- scripts/grafana-config-copy.py | 3 +- start.yml | 4 +- 6 files changed, 63 insertions(+), 30 deletions(-) diff --git a/roles/check_config_static/tasks/main.yml b/roles/check_config_static/tasks/main.yml index e1a5709f9..8b03d3dbe 100644 --- a/roles/check_config_static/tasks/main.yml +++ b/roles/check_config_static/tasks/main.yml @@ -1,9 +1,9 @@ --- # Common Tasks -- name: Ensure only one monitoring host exists - fail: msg="One monitoring host should be specified." - when: groups['monitoring_servers'] | length != 1 +- name: Ensure monitoring_servers exists + fail: msg="monitoring_servers should be specified." + when: groups['monitoring_servers'] | length < 1 - name: Ensure monitored_servers exists fail: msg="monitored_servers should be specified." diff --git a/roles/grafana/templates/data_source.json.j2 b/roles/grafana/templates/data_source.json.j2 index 787529312..5c800eeb7 100644 --- a/roles/grafana/templates/data_source.json.j2 +++ b/roles/grafana/templates/data_source.json.j2 @@ -1,9 +1,20 @@ -{% set metric_host = hostvars[groups.monitoring_servers[0]].ansible_host | default(hostvars[groups.monitoring_servers[0]].inventory_hostname) if groups.monitoring_servers else "" -%} -{% set metric_port = hostvars[groups.monitoring_servers[0]].prometheus_port if groups.monitoring_servers else ''-%} +{% if groups.monitoring_servers | length == groups.grafana_servers | length -%} + {% set index = [] -%} + {% for host in groups.monitoring_servers -%} + {% if inventory_hostname == hostvars[host].inventory_hostname -%} + {% set _ = index.append(loop.index0) -%} + {% endif -%} + {% endfor -%} + {% set metric_host = hostvars[groups.monitoring_servers[index.0]].ansible_host | default(hostvars[groups.monitoring_servers[index.0]].inventory_hostname) -%} + {% set metric_port = hostvars[groups.monitoring_servers[index.0]].prometheus_port -%} +{% else -%} + {% set metric_host = hostvars[groups.monitoring_servers[0]].ansible_host | default(hostvars[groups.monitoring_servers[0]].inventory_hostname) -%} + {% set metric_port = hostvars[groups.monitoring_servers[0]].prometheus_port -%} +{% endif -%} { "name":"{{ cluster_name }}", "type":"prometheus", "access":"proxy", "url":"http://{{ metric_host }}:{{ metric_port }}/", "basicAuth":false -} \ No newline at end of file +} diff --git a/roles/prometheus/templates/prometheus.yml.j2 b/roles/prometheus/templates/prometheus.yml.j2 index 796bb8fa2..5994b4cba 100644 --- a/roles/prometheus/templates/prometheus.yml.j2 +++ b/roles/prometheus/templates/prometheus.yml.j2 @@ -27,37 +27,49 @@ rule_files: - 'lightning.rules.yml' {% endif %} -{% set alertmanager_host = hostvars[groups.alertmanager_servers[0]].ansible_host | default(hostvars[groups.alertmanager_servers[0]].inventory_hostname) - if groups.get('alertmanager_servers', []) else '' -%} -{% set alertmanager_port = hostvars[groups.alertmanager_servers[0]].alertmanager_port if alertmanager_host else '' -%} - -{% if alertmanager_target|default("") %} +{% if alertmanager_target|default("") -%} alerting: alertmanagers: - static_configs: - targets: - '{{ alertmanager_target }}' -{% elif alertmanager_host %} +{% elif groups.alertmanager_servers -%} +{% if groups.monitoring_servers | length == groups.alertmanager_servers | length -%} + {% set index = [] -%} + {% for host in groups.monitoring_servers -%} + {% if inventory_hostname == hostvars[host].inventory_hostname -%} + {% set _ = index.append(loop.index0) -%} + {% endif -%} + {% endfor -%} + {% set alertmanager_host = hostvars[groups.alertmanager_servers[index.0]].ansible_host | default(hostvars[groups.alertmanager_servers[index.0]].inventory_hostname) -%} + {% set alertmanager_port = hostvars[groups.alertmanager_servers[index.0]].alertmanager_port -%} alerting: alertmanagers: - static_configs: - targets: - '{{ alertmanager_host }}:{{ alertmanager_port }}' -{% else %} +{% endif -%} +{% else -%} # alerting: # alertmanagers: # - static_configs: # - targets: # - 'alertmanager_host:9093' -{% endif %} +{% endif -%} -{% set pushgateway_host = hostvars[groups.monitoring_servers[0]].ansible_host | default(hostvars[groups.monitoring_servers[0]].inventory_hostname) - if groups.get('monitoring_servers', []) else '' -%} -{% set pushgateway_port = hostvars[groups.monitoring_servers[0]].pushgateway_port if pushgateway_host else '' -%} +{% set pushgateway_addrs = [] -%} +{% for host in groups.monitoring_servers -%} + {% set pushgateway_ip = hostvars[host].ansible_host | default(hostvars[host].inventory_hostname) -%} + {% set pushgateway_port = hostvars[host].pushgateway_port -%} + {% set _ = pushgateway_addrs.append("%s:%s" % (pushgateway_ip, pushgateway_port)) -%} +{% endfor -%} -{% set grafana_host = hostvars[groups.grafana_servers[0]].ansible_host | default(hostvars[groups.grafana_servers[0]].inventory_hostname) - if groups.get('grafana_servers', []) else '' -%} -{% set grafana_port = hostvars[groups.grafana_servers[0]].grafana_port if grafana_host else '' -%} +{% set grafana_addrs = [] -%} +{% for host in groups.grafana_servers -%} + {% set grafana_ip = hostvars[host].ansible_host | default(hostvars[host].inventory_hostname) -%} + {% set grafana_port = hostvars[host].grafana_port -%} + {% set _ = grafana_addrs.append("%s:%s" % (grafana_ip, grafana_port)) -%} +{% endfor -%} {% set blackbox_host = hostvars[groups.monitored_servers[0]].ansible_host | default(hostvars[groups.monitored_servers[0]].inventory_hostname) if groups.get('monitored_servers', []) else '' -%} {% set blackbox_port = hostvars[groups.monitored_servers[0]].blackbox_exporter_port if blackbox_host else '' -%} @@ -105,12 +117,15 @@ alerting: {% endfor -%} scrape_configs: -{% if pushgateway_host %} +{% if pushgateway_addrs %} - job_name: 'overwritten-cluster' scrape_interval: 15s honor_labels: true # don't overwrite job & instance labels static_configs: - - targets: ['{{ pushgateway_host }}:{{ pushgateway_port }}'] + - targets: +{% for pushgateway_addr in pushgateway_addrs %} + - '{{ pushgateway_addr }}' +{% endfor %} - job_name: "blackbox_exporter_http" scrape_interval: 30s @@ -119,7 +134,9 @@ scrape_configs: module: [http_2xx] static_configs: - targets: - - 'http://{{ pushgateway_host }}:{{ pushgateway_port }}/metrics' +{% for pushgateway_addr in pushgateway_addrs %} + - 'http://{{ pushgateway_addr }}/metrics' +{% endfor %} relabel_configs: - source_labels: [__address__] target_label: __param_target @@ -281,11 +298,15 @@ scrape_configs: labels: group: 'pd' - targets: - - '{{ pushgateway_host }}:{{ pushgateway_port }}' +{% for pushgateway_addr in pushgateway_addrs %} + - '{{ pushgateway_addr }}' +{% endfor %} labels: group: 'pushgateway' - targets: - - '{{ grafana_host }}:{{ grafana_port }}' +{% for grafana_addr in grafana_addrs %} + - '{{ grafana_addr }}' +{% endfor %} labels: group: 'grafana' - targets: diff --git a/rolling_update_monitor.yml b/rolling_update_monitor.yml index 57296bd1d..48e4143fd 100644 --- a/rolling_update_monitor.yml +++ b/rolling_update_monitor.yml @@ -395,7 +395,7 @@ - name: import grafana dashboards - prepare config delegate_to: localhost - template: src=grafana.dest.json.j2 dest={{ playbook_dir }}/scripts/dests.json + template: src=grafana.dest.json.j2 dest={{ playbook_dir }}/scripts/dests-{{ inventory_hostname }}.json vars: - ansible_become: false - ansible_connection: local @@ -426,7 +426,7 @@ - name: import grafana dashboards - run import script delegate_to: localhost - shell: "python grafana-config-copy.py" + shell: "python grafana-config-copy.py dests-{{ inventory_hostname }}.json" args: chdir: "{{ playbook_dir }}/scripts" vars: diff --git a/scripts/grafana-config-copy.py b/scripts/grafana-config-copy.py index 104bf6f25..0fdf9751f 100755 --- a/scripts/grafana-config-copy.py +++ b/scripts/grafana-config-copy.py @@ -3,6 +3,7 @@ from __future__ import print_function, \ unicode_literals +import sys import urllib import urllib2 import base64 @@ -23,7 +24,7 @@ ] if not dests: - with open("./dests.json") as fp: + with open(sys.argv[1]) as fp: dests = json.load(fp) src = dict( diff --git a/start.yml b/start.yml index c7f31aa71..a620e3d42 100644 --- a/start.yml +++ b/start.yml @@ -454,7 +454,7 @@ - name: import grafana dashboards - prepare config delegate_to: localhost - template: src=grafana.dest.json.j2 dest={{ playbook_dir }}/scripts/dests.json + template: src=grafana.dest.json.j2 dest={{ playbook_dir }}/scripts/dests-{{ inventory_hostname }}.json vars: - ansible_become: false - ansible_connection: local @@ -485,7 +485,7 @@ - name: import grafana dashboards - run import script delegate_to: localhost - shell: "python grafana-config-copy.py" + shell: "python grafana-config-copy.py dests-{{ inventory_hostname }}.json" args: chdir: "{{ playbook_dir }}/scripts" vars: