Alertmanager

# Arch Linux
sudo pacman -S alertmanager

# Enable
sudo systemctl enable --now alertmanager

# /etc/alertmanager/alertmanager.yml
global:
  resolve_timeout: 5m

route:
  receiver: 'default'
  group_by: ['alertname', 'severity']
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 4h
  routes:
    - match:
        severity: critical
      receiver: 'pagerduty'
    - match:
        severity: warning
      receiver: 'slack'

receivers:
  - name: 'default'
    email_configs:
      - to: 'alerts@example.com'

  - name: 'slack'
    slack_configs:
      - api_url: 'https://hooks.slack.com/services/xxx/yyy/zzz'
        channel: '#alerts'
        send_resolved: true

  - name: 'pagerduty'
    pagerduty_configs:
      - service_key: 'your-pagerduty-key'

# Check config
amtool check-config {alertmanager-config}

# Query alerts
amtool alert query
amtool alert query alertname=HighMemory

# Create silence
amtool silence add alertname=HighMemory --duration=1h --author="evan" --comment="Maintenance"

# List silences
amtool silence query

# Expire silence
amtool silence expire SILENCE_ID