告警管理

apiVersion: monitoring.openshift.io/v1
kind: AlertingRule
metadata:
  name: example
  namespace: openshift-monitoring (1)
spec:
  groups:
  - name: example-rules
    rules:
    - alert: ExampleAlert (2)
      for: 1m (3)
      expr: vector(1) (4)
      labels:
        severity: warning (5)
      annotations:
        message: This is an example alert. (6)

$ oc apply -f example-alerting-rule.yaml

apiVersion: monitoring.openshift.io/v1
kind: AlertRelabelConfig
metadata:
  name: watchdog
  namespace: openshift-monitoring (1)
spec:
  configs:
  - sourceLabels: [alertname,severity] (2)
    regex: "Watchdog;none" (3)
    targetLabel: severity (4)
    replacement: critical (5)
    action: Replace (6)

$ oc apply -f example-modified-alerting-rule.yaml

apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: example-alert
  namespace: ns1
spec:
  groups:
  - name: example
    rules:
    - alert: VersionAlert (1)
      for: 1m (2)
      expr: version{job="prometheus-example-app"} == 0 (3)
      labels:
        severity: warning (4)
      annotations:
        message: This is an example alert. (5)

$ oc apply -f example-app-alerting-rule.yaml

$ oc -n <project> get prometheusrule

$ oc -n <project> get prometheusrule <rule> -o yaml

apiVersion: monitoring.coreos.com/v1beta1
kind: AlertmanagerConfig
metadata:
  name: example-routing
  namespace: ns1
spec:
  route:
    receiver: default
    groupBy: [job]
  receivers:
  - name: default
    webhookConfigs:
    - url: https://example.org/post

$ oc apply -f example-app-alert-routing.yaml

global:
  resolve_timeout: 5m
route:
  group_wait: 30s (1)
  group_interval: 5m (2)
  repeat_interval: 12h (3)
  receiver: default
  routes:
  - matchers:
    - "alertname=Watchdog"
    repeat_interval: 2m
    receiver: watchdog
  - matchers:
    - "service=<your_service>" (4)
    routes:
    - matchers:
      - <your_matching_rules> (5)
      receiver: <receiver> (6)
receivers:
- name: default
- name: watchdog
- name: <receiver>
  <receiver_configuration> (7)

global:
  resolve_timeout: 5m
route:
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 12h
  receiver: default
  routes:
  - matchers:
    - "alertname=Watchdog"
    repeat_interval: 2m
    receiver: watchdog
  - matchers:
    - "service=example-app"
    routes:
    - matchers:
      - "severity=critical"
      receiver: team-frontend-page
receivers:
- name: default
- name: watchdog
- name: team-frontend-page
  pagerduty_configs:
  - service_key: "<your_key>"

$ oc -n openshift-user-workload-monitoring get secret alertmanager-user-workload --template='{{ index .data "alertmanager.yaml" }}' | base64 --decode > alertmanager.yaml

route:
  receiver: Default
  group_by:
  - name: Default
  routes:
  - matchers:
    - "service = prometheus-example-monitor" (1)
    receiver: <receiver> (2)
receivers:
- name: Default
- name: <receiver>
  <receiver_configuration> (3)

$ oc -n openshift-user-workload-monitoring create secret generic alertmanager-user-workload --from-file=alertmanager.yaml --dry-run=client -o=yaml |  oc -n openshift-user-workload-monitoring replace secret --filename=-