## 创建相关目录

mkdir -p /data/alertmanager/{bin,conf,logs,data,templates}

## 下载二进制包，并

wget https://github.com/prometheus/alertmanager/releases/download/v0.21.0/alertmanager-0.21.0.linux-amd64.tar.gz

tar xvf alertmanager-0.21.0.linux-amd64.tar.gz

mv alertmanager-0.21.0.linux-amd64/{alertmanager,amtool} /data/alertmanager/bin/

mv alertmanager-0.21.0.linux-amd64/alertmanager.yml /data/alertmanager/conf/

# 目录结构

/data/alertmanager/

├── bin

│   ├── alertmanager

│   └── amtool

├── conf

│   └── alertmanager.yml

├── data

├── logs

└── templates

## 加入systemd启动脚本

cat <<EOF >/lib/systemd/system/alertmanager.service

[Unit]

Description=alertmanager

Documentation=https://prometheus.io/

After=network.target

StartLimitIntervalSec=0

[Service]

Type=simple

User=prometheus

ExecStart=/data/alertmanager/bin/alertmanager --storage.path="/data/alertmanager/data/" \

--config.file=/usr/local/alertmanager/alertmanager.yml \

--web.external-url=http://192.168.1.220 # 此处可以写域名，需要做proxy。

Restart=always

RestartSec=1

# Restart=on-failure

[Install]

WantedBy=multi-user.target

EOF

## 启动

systemctl enable alertmanager

systemctl start alertmanager

## Alertmanager 配置文件

global:

  resolve_timeout: 5m

  # smtp配置

  smtp_from: "123456789@qq.com"

  smtp_smarthost: 'smtp.qq.com:465'

  smtp_auth_username: "123456789@qq.com"

  smtp_auth_password: "auth_pass"

  smtp_require_tls: true

# email、企业微信的模板配置存放位置，钉钉的模板会单独讲如果配置。

templates:

  - '/data/alertmanager/templates/*.tmpl'

# 路由分组

route:

  receiver: ops

  group_wait: 30s # 在组内等待所配置的时间，如果同组内，30秒内出现相同报警，在一个组内出现。

  group_interval: 5m # 如果组内内容不变化，合并为一条警报信息，5m后发送。

  repeat_interval: 24h # 发送报警间隔，如果指定时间内没有修复，则重新发送报警。

  group_by: [alertname]  # 报警分组

  routes:

      - match:

          team: operations

        group_by: [env,dc]

        receiver: 'ops'

      - match_re:

          service: nginx|apache

        receiver: 'web'

      - match_re:

          service: hbase|spark

        receiver: 'hadoop'

      - match_re:

          service: mysql|mongodb

        receiver: 'db'

# 接收器

# 抑制测试配置

      - receiver: ops

        group_wait: 10s

        match:

          status: 'High'

# ops

      - receiver: ops # 路由和标签，根据match来指定发送目标，如果 rule的lable 包含 alertname， 使用 ops 来发送

        group_wait: 10s

        match:

          team: operations

# web

      - receiver: db # 路由和标签，根据match来指定发送目标，如果 rule的lable 包含 alertname， 使用 db 来发送

        group_wait: 10s

        match:

          team: db

# 接收器指定发送人以及发送渠道

receivers:

# ops分组的定义

- name: ops

  email_configs:

  - to: '9935226@qq.com,10000@qq.com'

    send_resolved: true

    headers:

      subject: "[operations] 报警邮件"

      from: "警报中心"

      to: "小煜狼皇"

  # 钉钉配置

  webhook_configs:

  - url: http://localhost:8070/dingtalk/ops/send

    # 企业微信配置

  wechat_configs:

  - corp_id: 'ww5421dksajhdasjkhj'

    api_url: 'https://qyapi.weixin.qq.com/cgi-bin/'

    send_resolved: true

    to_party: '2'

    agent_id: '1000002'

    api_secret: 'Tm1kkEE3RGqVhv5hO-khdakjsdkjsahjkdksahjkdsahkj'

# web

- name: web

  email_configs:

  - to: '9935226@qq.com'

    send_resolved: true

    headers: { Subject: "[web] 报警邮件"} # 接收邮件的标题

  webhook_configs:

  - url: http://localhost:8070/dingtalk/web/send

  - url: http://localhost:8070/dingtalk/ops/send

# db

- name: db

  email_configs:

  - to: '9935226@qq.com'

    send_resolved: true

    headers: { Subject: "[db] 报警邮件"} # 接收邮件的标题

  webhook_configs:

  - url: http://localhost:8070/dingtalk/db/send

  - url: http://localhost:8070/dingtalk/ops/send

# hadoop

- name: hadoop

  email_configs:

  - to: '9935226@qq.com'

    send_resolved: true

    headers: { Subject: "[hadoop] 报警邮件"} # 接收邮件的标题

  webhook_configs:

  - url: http://localhost:8070/dingtalk/hadoop/send

  - url: http://localhost:8070/dingtalk/ops/send

# 抑制器配置

inhibit_rules: # 抑制规则

  - source_match: # 源标签警报触发时抑制含有目标标签的警报，在当前警报匹配 status: 'High'

      status: 'High'  # 此处的抑制匹配一定在最上面的route中配置不然，会提示找不key。

    target_match:

      status: 'Warning' # 目标标签值正则匹配，可以是正则表达式如: ".*MySQL.*"

    equal: ['alertname','operations', 'instance'] # 确保这个配置下的标签内容相同才会抑制，也就是说警报中必须有这三个标签值才会被抑制。

route:

  receiver: admin # 默认的接收器名称

  group_wait: 30s # 在组内等待所配置的时间，如果同组内，30秒内出现相同报警，在一个组内出现。

  group_interval: 5m # 如果组内内容不变化，5m后发送。

  repeat_interval: 24h # 发送报警间隔，如果指定时间内没有修复，则重新发送报警

  group_by: [alertname,cluster]  # 报警分组，根据 prometheus 的 lables 进行报警分组，这些警报会合并为一个通知发送给接收器，也就是警报分组。

  routes:

      - match:

          team: ops

        group_by: [env,dc]

        receiver: 'ops'

      - match_re:

          service: nginx|apache

        receiver: 'web'

      - match_re:

          service: mysql|mongodb

        receiver: 'db'

      - match_re:

          service: hbase|spark

        receiver: 'hadoop'

inhibit_rules: # 抑制规则

  - source_match: # 源标签警报触发时抑制含有目标标签的警报，在当前警报匹配 status: 'High'

      status: 'High'  # 此处的抑制匹配一定在最上面的route中配置不然，会提示找不key。

    target_match:

      status: 'Warning' # 目标标签值正则匹配，可以是正则表达式如: ".*MySQL.*"

    equal: ['alertname','operations', 'instance'] # 确保这个配置下的标签内容相同才会抑制，也就是说警报中必须有这三个标签值才会被抑制。

global:

  resolve_timeout: 5m

  # smtp配置

  smtp_from: "1234567890@qq.com" # 发送邮件主题

  smtp_smarthost: 'smtp.qq.com:465' # 邮箱服务器的SMTP主机配置

  smtp_auth_username: "1234567890@qq.com" # 登录用户名

  smtp_auth_password: "auth_pass" # 此处的auth password是邮箱的第三方登录授权密码，而非用户密码，尽量用QQ来测试。

  smtp_require_tls: false # 有些邮箱需要开启此配置，这里使用的是163邮箱，仅做测试，不需要开启此功能。

route:

  receiver: ops

  group_wait: 30s # 在组内等待所配置的时间，如果同组内，30秒内出现相同报警，在一个组内出现。

  group_interval: 5m # 如果组内内容不变化，合并为一条警报信息，5m后发送。

  repeat_interval: 24h # 发送报警间隔，如果指定时间内没有修复，则重新发送报警。

  group_by: [alertname]  # 报警分组

  routes:

      - match:

          team: operations

        group_by: [env,dc]

        receiver: 'ops'

      - receiver: ops # 路由和标签，根据match来指定发送目标，如果 rule的lable 包含 alertname， 使用 ops 来发送

        group_wait: 10s

        match:

          team: operations

# 接收器指定发送人以及发送渠道

receivers:

# ops分组的定义

- name: ops

  email_configs:

  - to: '9935226@qq.com,xxxxx@qq.com' # 如果想发送多个人就以 ','做分割，写多个邮件人即可。

    send_resolved: true

    headers:

      from: "警报中心"

      subject: "[operations] 报警邮件"

      to: "小煜狼皇"

cat wechat.tmpl

## wechat模板

{{ define "wechat.default.message" }}

{{ if gt (len .Alerts.Firing) 0 -}}

Alerts Firing:

{{ range .Alerts }}

警报级别：{{ .Labels.status }}

警报类型：{{ .Labels.alertname }}

故障主机: {{ .Labels.instance }}

警报主题: {{ .Annotations.summary }}

警报详情: {{ .Annotations.description }}

⏱ : {{ (.StartsAt.Add 28800e9).Format "2006-01-02 15:04:05" }}

{{- end }}

{{- end }}

{{ if gt (len .Alerts.Resolved) 0 -}}

Alerts Resolved:

{{ range .Alerts }}

警报级别：{{ .Labels.status }}

警报类型：{{ .Labels.alertname }}

故障主机: {{ .Labels.instance }}

警报主题: {{ .Annotations.summary }}

警报详情: {{ .Annotations.description }}

⏱ : {{ (.StartsAt.Add 28800e9).Format "2006-01-02 15:04:05" }}

⏲ : {{ (.EndsAt.Add 28800e9).Format "2006-01-02 15:04:05" }}

{{- end }}

{{- end }}

{{- end }}

mkdir -p /etc/prometheus-webhook-dingtalk/template/

cd /etc/prometheus-webhook-dingtalk/

wget https://github.com/timonwong/prometheus-webhook-dingtalk/releases/download/v1.4.0/prometheus-webhook-dingtalk-1.4.0.linux-amd64.tar.gz

tar xf prometheus-webhook-dingtalk-1.4.0.linux-amd64.tar.gz

mv prometheus-webhook-dingtalk-1.4.0.linux-amd64/* /etc/prometheus-webhook-dingtalk/

mv prometheus-webhook-dingtalk /bin/

cat <<EOF> /lib/systemd/system/prometheus-webhook-dingtalk.service

[Unit]

Description=prometheus-webhook-dingding

Documentation=https://prometheus.io/

After=network.target

[Service]

Type=simple

User=prometheus

ExecStart=/bin/prometheus-webhook-dingtalk --web.listen-address=":8070" --web.enable-ui --config.file="/etc/prometheus-webhook-dingtalk/config.yml"

Restart=on-failure

[Install]

WantedBy=multi-user.target

EOF

## 启动服务

systemctl enable prometheus-webhook-dingtalk.service

systemctl start prometheus-webhook-dingtalk.service

## Request timeout

# timeout: 5s

## Customizable templates path

## Customizable templates path

templates:

        # - contrib/templates/legacy/template.tmpl

  # 自定义模板路径

  - /etc/prometheus-webhook-dingtalk/template/default.tmpl

## 你还可以使用' default_message '覆盖默认模板

## 下面的示例使用v0.3.0中的“legacy”模板

# default_message:

#   title: '{{ template "legacy.title" . }}'

#   text: '{{ template "legacy.content" . }}'

## Targets, previously was known as "profiles"

# 定义的webhook，钉钉创建的webhook token

targets:

# 如果有多个分组就可以在这里定义多个接口

  ops:

    url: https://oapi.dingtalk.com/robot/send?access_token=a4feed2322222222222222222222222

  web:

    url: https://oapi.dingtalk.com/robot/send?access_token=a4feed2325c1333333333333333333333

cd /etc/prometheus-webhook-dingtalk/template

cat default.tmpl

{{ define "__subject" }}[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .GroupLabels.SortedPairs.Values | join " " }} {{ if gt (len .CommonLabels) (len .GroupLabels) }}({{ with .CommonLabels.Remove .GroupLabels.Names }}{{ .Values | join " " }}{{ end }}){{ end }}{{ end }}

{{ define "__alertmanagerURL" }}{{ .ExternalURL }}/#/alerts?receiver={{ .Receiver }}{{ end }}

{{ define "__text_alert_list" }}{{ range . }}

**Labels**

{{ range .Labels.SortedPairs }}> - {{ .Name }}: {{ .Value | markdown | html }}

{{ end }}

**Annotations**

{{ range .Annotations.SortedPairs }}> - {{ .Name }}: {{ .Value | markdown | html }}

{{ end }}

**Source:** [{{ .GeneratorURL }}]({{ .GeneratorURL }})

{{ end }}{{ end }}

{{/* Firing */}}

{{ define "default.__text_alert_list" }}{{ range . }}

**Trigger Time:** {{ dateInZone "2006.01.02 15:04:05" (.StartsAt) "Asia/Shanghai" }}

**Summary:** {{ .Annotations.summary }}

**Description:** {{ .Annotations.description }}

**Graph:** [ ]({{ .GeneratorURL }})

**Details:**

{{ range .Labels.SortedPairs }}{{ if and (ne (.Name) "severity") (ne (.Name) "summary") }}> - {{ .Name }}: {{ .Value | markdown | html }}

{{ end }}{{ end }}

{{ end }}{{ end }}

{{/* Resolved */}}

{{ define "default.__text_resolved_list" }}{{ range . }}

**Trigger Time:** {{ dateInZone "2006.01.02 15:04:05" (.StartsAt) "Asia/Shanghai" }}

**Resolved Time:** {{ dateInZone "2006.01.02 15:04:05" (.EndsAt) "Asia/Shanghai" }}

**Summary:** {{ .Annotations.summary }}

**Graph:** [ ]({{ .GeneratorURL }})

**Details:**

{{ range .Labels.SortedPairs }}{{ if and (ne (.Name) "severity") (ne (.Name) "summary") }}> - {{ .Name }}: {{ .Value | markdown | html }}

{{ end }}{{ end }}

{{ end }}{{ end }}

{{/* Default */}}

{{ define "default.title" }}{{ template "__subject" . }}{{ end }}

{{ define "default.content" }}#### \[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}\] **[{{ index .GroupLabels "alertname" }}]({{ template "__alertmanagerURL" . }})**

{{ if gt (len .Alerts.Firing) 0 -}}

![Firing-img](https://is3-ssl.mzstatic.com/image/thumb/Purple20/v4/e0/23/cf/e023cf56-0623-0cdf-afce-97ae90eabfda/mzl.uplmrpgi.png/320x0w.jpg)

**Alerts Firing**

{{ template "default.__text_alert_list" .Alerts.Firing }}

{{- end }}

{{ if gt (len .Alerts.Resolved) 0 -}}

![Resolved-img](https://is3-ssl.mzstatic.com/image/thumb/Purple18/v4/41/72/99/4172990a-f666-badf-9726-6204a320c16e/mzl.dypdixoy.png/320x0w.png)

**Alerts Resolved**

{{ template "default.__text_resolved_list" .Alerts.Resolved }}

{{- end }}

{{- end }}

{{/* Legacy */}}

{{ define "legacy.title" }}{{ template "__subject" . }}{{ end }}

{{ define "legacy.content" }}#### \[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}\] **[{{ index .GroupLabels "alertname" }}]({{ template "__alertmanagerURL" . }})**

{{ template "__text_alert_list" .Alerts.Firing }}

{{- end }}

{{/* Following names for compatibility */}}

{{ define "ding.link.title" }}{{ template "default.title" . }}{{ end }}

{{ define "ding.link.content" }}{{ template "default.content" . }}{{ end }}

# 接收器指定发送人以及发送渠道

receivers:

# ops分组的定义

- name: ops

  email_configs:

  - to: '9935226@qq.com,10000@qq.com'

    send_resolved: true

    headers: { Subject: "[operations] 报警邮件"} # 接收邮件的标题

  # 钉钉配置

  webhook_configs:

  - url: http://localhost:8070/dingtalk/ops/send # 这里是在钉钉开源组件中的接口，如果单独定义的receiver需要对应你的分组与钉钉机器人的webhook token

    # 企业微信配置

  wechat_configs:

  - corp_id: 'ww5421dksajhdasjkhj'

    api_url: 'https://qyapi.weixin.qq.com/cgi-bin/'

    send_resolved: true

    to_party: '2'

    agent_id: '1000002'

    api_secret: 'Tm1kkEE3RGqVhv5hO-khdakjsdkjsahjkdksahjkdsahkj'

# web

- name: web

  email_configs:

  - to: '9935226@qq.com'

    send_resolved: true

    headers: { Subject: "[web] 报警邮件"} # 接收邮件的标题

  webhook_configs:

  - url: http://localhost:8070/dingtalk/web/send

receivers:

  - name: slack_and_email

    # Slack

    slack_configs:

      - api_url: '<THE_WEBHOOK_URL>'

        channel: '#general'

      - api_url: '<ANOTHER_WEBHOOK_URL>'

        channel: '#alerts'

    # Email

    email_configs:

      - to: 'k4nz@example.com'

route:

  receiver: slack_and_email

# 首先，我们定义多个不同的 receiver 信息

receivers:

  - name: slack

    slack_configs:

      - api_url: THE_WEBHOOK_URL

        channel: '#general'

  - name: email

    email_configs:

      - to: 'k4nz@example.com'

# 然后，在 route 中通过 continue 参数，是告警消息进行多个匹配

route:

 receiver: slack  # Fallback，必须设置 receiver 字段

 routes:

   - match:

       severity: page

     receiver: slack

     continue: true

   - match:

       severity: page

     receiver: pagerduty

通常告警消息的 Lable 匹配 match 之后，不会继续向下匹配。通过 continue: true 能够使告警消息继续向下匹配。