Alertmanager的使用

DevOps
留言

关联博客:

Grafana配置告警

prom相关

prom这台机器,/data/alermanager

cat debug.json

[{“labels”:{“alertname”:”TestAlert3”,”severity”:”testtg”}}]

http -v POST http://localhost:9093/api/v2/alerts < debug.json

cat debugopenobserver.json 

1
2
3
4
5
6
7
8
9
10
11
12
13
[{
    
    "labels": {
        "alertname": "testing",    
        "stream": "hello",
        "organization": "myteam",
        "alerttype": "warning",
        "severity": "page" 
    },
    "annotations": {
        "summary": "testing testing"  
    }    
}]

http -v POST http://172.22.0.89:9093/api/v2/alerts < debug.json

cat debug.json

1
2
3
4
5
6
7
8
9
10
11
12
13
[{
    
    "labels": {
        "alertname": "testing4",    
        "stream": "hello",
        "alerttype": "warning",
        "severity": "page",
        "business": "team2"
    },
    "annotations": {
        "summary": "sending for xxx"  
    }    
}]

ubuntu@prometheus-01:/data/alermanager$ cat alertmanager.yml

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
global:
  resolve_timeout: 5m
  smtp_smarthost: 'smtp.163.com:465'
  smtp_from: 'xxxxx@163.com' # 用于发送告警邮件的邮箱
  smtp_auth_username: 'xxxx@163.com'
  smtp_auth_password: 'xxxxxx'    #此处为邮箱的授权密码，非邮箱登录密码
  smtp_require_tls: false
  smtp_hello: '163.com'
   #企业微信相关配置
  wechat_api_url: 'https://qyapi.weixin.qq.com/cgi-bin/'
  wechat_api_secret: 'YpZ-xxxxx'  #应用的secret，在应用的配置页面可以看到
  wechat_api_corp_id: 'xxxxx'   #企业id，在企业的配置页面可以看到

route:
  group_by: ['alertname']
  group_wait: 0s
  group_interval: 1m
  repeat_interval: 15m
  receiver: 'default-receiver'
  routes: # 子路由
   - receiver: 'xxxonly'
     match:
       business: team2
     continue: false # 
   - receiver: 'k8smail'
     continue: true
   - receiver: 'slack'
     continue: true
   - receiver: 'wechat'
     continue: true
   - receiver: 'telegram'
     continue: true

   - receiver: 'critical-notifications'
     continue: true
     match:
       severity_level: critical



receivers:
- name: 'default-receiver'
  email_configs:
  - to: 'xxxx@xxx.io' # 接收报警邮件的邮箱(实际不会收到)
    send_resolved: true

- name: 'critical-notifications'
  email_configs:
  - to: 'xxx@xxx.io' # 接收报警邮件的邮箱
    send_resolved: true
  - to: 'xxx@163.com' # 接收报警邮件的邮箱
    send_resolved: true
- name: 'k8smail'
  email_configs:
  - to: 'xxxxx@163.com' # 接收报警邮件的邮箱
    send_resolved: true
  - to: 'xxxx@outlook.com' # 接收报警邮件的邮箱
    send_resolved: true
- name: 'slack'
  slack_configs:
   - send_resolved: true
     api_url: https://hooks.slack.com/services/xxxxx  #这里填你安装webhook app应该的时候给的url
     channel: '#monitoring'   #要发往那个频道
     text: "{{ range .Alerts }} {{ .Annotations.description}}\n {{end}} @{{ .CommonAnnotations.username}} <{{.CommonAnnotations.link}}| click here>"
     title: "{{.CommonAnnotations.summary}}"
     title_link: "{{.CommonAnnotations.link}}"
- name: 'cjonly'
  slack_configs:
   - send_resolved: true
     api_url: https://hooks.slack.com/servicesxxxx
     channel: '#alert-notice'   #要发往那个频道
     text: "{{ range .Alerts }} {{ .Annotations.description}}\n {{end}} @{{ .CommonAnnotations.username}} <{{.CommonAnnotations.link}}| click here>"
     title: "{{.CommonAnnotations.summary}}"
     title_link: "{{.CommonAnnotations.link}}"
- name: 'wechat'
  wechat_configs:
  - send_resolved: true
    corp_id: 'xxxx'
    to_user: '@all'
   # to_party: ' PartyID1 | PartyID2 '
    message: '{{ template "wechat.default.message" . }}'
  #  message: '触发告警，请查看'
    agent_id: '1000002'  #应用的AgentId，在应用的配置页面可以看到
    api_secret: 'YpZ-xxxxxx'  #应用的secret，在应用的配置页面可以看到

- name: "pagerduty"
  pagerduty_configs:
    - service_key: "xxxx"
- name: 'telegram'
  telegram_configs:
  - bot_token: xxx:xxxx
    api_url: https://api.telegram.org
    chat_id: -xxxx
    parse_mode: 'HTML'
    message: '{{ template "telegram.message". }}'

inhibit_rules:
  - source_match:
      severity: 'critical'
    target_match:
      severity: 'warning'
    equal: ['alertname', 'dev', 'instance']




templates:
- 'template/telegram.tmpl' # path to our telegram.tmpl

这个名字没法编辑,创建后改不了…

https://cloud.tencent.com/document/product/1263/74219?cps_key=1d358d18a7a17b4a6df8d67a62fd3d3d

参考这篇,再来一遍

20250814

暴露一个公网能访问的地址

1
2
3
[program:alertmanager2]
directory = /data/alermanager
command = ./alertmanager --web.external-url=https://alert.your-domain.app

即启动命令加一个 –web.external-url=https://alert.your-domain.app

文章目录