groups:
  - name: onlineornot
    rules:
      - alert: OnlineOrNotMonitorDown
        expr: oon_monitor_status{state="DOWN"} == 1
        labels:
          severity: critical
        annotations:
          summary: '{{ $labels.name }} is down'
          description: 'OnlineOrNot has confirmed that {{ $labels.name }} ({{ $labels.check_type }}) is down.'
          onlineornot_url: 'https://onlineornot.com/app/{{ if eq $labels.check_type "HEARTBEAT" }}heartbeats{{ else }}checks{{ end }}/{{ $labels.monitor_id }}'

      - alert: OnlineOrNotMultiRegionFailure
        expr: count by (monitor_id, check_type, name) (sum by (monitor_id, check_type, name, region) (increase(oon_check_results_total{outcome="failing", region!=""}[15m])) > 0) >= 2
        labels:
          severity: warning
        annotations:
          summary: '{{ $labels.name }} is failing in {{ $value }} regions'
          description: '{{ $labels.name }} ({{ $labels.check_type }}) had failing results in {{ $value }} regions in the last 15 minutes. This alert needs per_region=true in the scrape config.'
          onlineornot_url: 'https://onlineornot.com/app/checks/{{ $labels.monitor_id }}'

      - alert: OnlineOrNotResultsStale
        expr: (time() - oon_monitor_last_result_timestamp_seconds > 3 * oon_check_interval_seconds) unless on (monitor_id, check_type) (oon_monitor_status{state=~"PAUSED|MAINTENANCE|PENDING"} == 1)
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: 'No recent results for {{ $labels.name }}'
          description: '{{ $labels.name }} ({{ $labels.check_type }}) has not reported a result for more than 3 check intervals.'
          onlineornot_url: 'https://onlineornot.com/app/checks/{{ $labels.monitor_id }}'

      - alert: OnlineOrNotScrapeFailing
        expr: up{job="onlineornot"} == 0
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: 'Prometheus cannot scrape OnlineOrNot'
          description: 'Scraping {{ $labels.instance }} has failed for 5 minutes. Check that the API token is valid, has not expired and can read checks or heartbeats.'
