| Rule |
State |
Error |
Last Evaluation |
Evaluation Time |
| alert: BlackboxProbeFailed
expr: probe_success{job!~"node_pve01|blackbox_tls_pve01"}
== 0
for: 10m
labels:
severity: critical
annotations:
description: |-
Probe failed
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Blackbox probe failed (instance {{ $labels.instance }})
|
ok
|
|
7.263s ago
|
256.8us |
| alert: BlackboxConfigurationReloadFailure
expr: blackbox_exporter_config_last_reload_successful
!= 1
labels:
severity: warning
annotations:
description: |-
Blackbox configuration reload failure
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Blackbox configuration reload failure (instance {{ $labels.instance }})
|
ok
|
|
7.263s ago
|
49.19us |
| alert: BlackboxSslCertificateWillExpireSoon
expr: 3
<= round((last_over_time(probe_ssl_earliest_cert_expiry[10m]) - time()) / 86400,
0.1) < 20
labels:
severity: warning
annotations:
description: |-
SSL certificate expires in less than 20 days
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Blackbox SSL certificate will expire soon (instance {{ $labels.instance
}})
|
ok
|
|
7.263s ago
|
104us |
| alert: BlackboxSslCertificateWillExpireSoon
expr: 0
<= round((last_over_time(probe_ssl_earliest_cert_expiry[10m]) - time()) / 86400,
0.1) < 3
labels:
severity: critical
annotations:
description: |-
SSL certificate expires in less than 3 days
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Blackbox SSL certificate will expire soon (instance {{ $labels.instance
}})
|
ok
|
|
7.263s ago
|
71.91us |
| alert: BlackboxSslCertificateExpired
expr: round((last_over_time(probe_ssl_earliest_cert_expiry[10m])
- time()) / 86400, 0.1) < 0
labels:
severity: critical
annotations:
description: |-
SSL certificate has expired already
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Blackbox SSL certificate expired (instance {{ $labels.instance }})
|
ok
|
|
7.263s ago
|
55.37us |
| alert: BlackboxProbeSlowHttp
expr: avg_over_time(probe_http_duration_seconds[1m])
> 1
for: 1m
labels:
severity: warning
annotations:
description: |-
HTTP request took more than 1s
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Blackbox probe slow HTTP (instance {{ $labels.instance }})
|
ok
|
|
7.263s ago
|
75.4us |
| alert: BlackboxProbeSlowPing
expr: avg_over_time(probe_icmp_duration_seconds[1m])
> 1
for: 1m
labels:
severity: warning
annotations:
description: |-
Blackbox ping took more than 1s
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Blackbox probe slow ping (instance {{ $labels.instance }})
|
ok
|
|
7.263s ago
|
223.6us |
|
6.681s ago |
113.6ms |
| Rule |
State |
Error |
Last Evaluation |
Evaluation Time |
| alert: HostOutOfMemory
expr: (node_memory_MemAvailable_bytes
/ node_memory_MemTotal_bytes * 100 < 10) * on (instance) group_left (nodename)
node_uname_info{nodename=~".+"}
for: 2m
labels:
severity: warning
annotations:
description: |-
Node memory is filling up (< 10% left)
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Host out of memory (instance {{ $labels.instance }})
|
ok
|
|
6.681s ago
|
743.7us |
| alert: HostMemoryUnderMemoryPressure
expr: (rate(node_vmstat_pgmajfault[1m])
> 1000) * on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
for: 2m
labels:
severity: warning
annotations:
description: |-
The node is under heavy memory pressure. High rate of major page faults
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Host memory under memory pressure (instance {{ $labels.instance }})
|
ok
|
|
6.68s ago
|
525.1us |
| alert: HostUnusualNetworkThroughputIn
expr: (sum
by (instance) (rate(node_network_receive_bytes_total[2m])) / 1024 / 1024 > 100)
* on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
for: 5m
labels:
severity: warning
annotations:
description: |-
Host network interfaces are probably receiving too much data (> 100 MB/s)
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Host unusual network throughput in (instance {{ $labels.instance }})
|
ok
|
|
6.68s ago
|
6.394ms |
| alert: HostUnusualNetworkThroughputOut
expr: (sum
by (instance) (rate(node_network_transmit_bytes_total[2m])) / 1024 / 1024 > 100)
* on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
for: 5m
labels:
severity: warning
annotations:
description: |-
Host network interfaces are probably sending too much data (> 100 MB/s)
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Host unusual network throughput out (instance {{ $labels.instance }})
|
ok
|
|
6.674s ago
|
6.873ms |
| alert: HostOutOfDiskSpace
expr: ((node_filesystem_avail_bytes
* 100) / node_filesystem_size_bytes < 10 and on (instance, device, mountpoint)
node_filesystem_readonly == 0) * on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
for: 2m
labels:
severity: warning
annotations:
description: |-
Disk is almost full (< 10% left)
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Host out of disk space (instance {{ $labels.instance }})
|
ok
|
|
6.667s ago
|
3.234ms |
| alert: HostDiskWillFillIn24Hours
expr: ((node_filesystem_avail_bytes
* 100) / node_filesystem_size_bytes < 10 and on (instance, device, mountpoint)
predict_linear(node_filesystem_avail_bytes{fstype!~"tmpfs"}[1h], 24 * 3600)
< 0 and on (instance, device, mountpoint) node_filesystem_readonly == 0) * on
(instance) group_left (nodename) node_uname_info{nodename=~".+"}
for: 2m
labels:
severity: warning
annotations:
description: |-
Filesystem is predicted to run out of space within the next 24 hours at current write rate
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Host disk will fill in 24 hours (instance {{ $labels.instance }})
|
ok
|
|
6.664s ago
|
5.535ms |
| alert: HostOutOfInodes
expr: (node_filesystem_files_free{fstype!="msdosfs"}
/ node_filesystem_files{fstype!="msdosfs"} * 100 < 10 and on (instance,
device, mountpoint) node_filesystem_readonly == 0) * on (instance) group_left (nodename)
node_uname_info{nodename=~".+"}
for: 2m
labels:
severity: warning
annotations:
description: |-
Disk is almost running out of available inodes (< 10% left)
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Host out of inodes (instance {{ $labels.instance }})
|
ok
|
|
6.658s ago
|
2.553ms |
| alert: HostFilesystemDeviceError
expr: node_filesystem_device_error
== 1
for: 2m
labels:
severity: critical
annotations:
description: |-
{{ $labels.instance }}: Device error with the {{ $labels.mountpoint }} filesystem
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Host filesystem device error (instance {{ $labels.instance }})
|
ok
|
|
6.656s ago
|
549.6us |
| alert: HostInodesWillFillIn24Hours
expr: (node_filesystem_files_free{fstype!="msdosfs"}
/ node_filesystem_files{fstype!="msdosfs"} * 100 < 10 and predict_linear(node_filesystem_files_free{fstype!="msdosfs"}[1h],
24 * 3600) < 0 and on (instance, device, mountpoint) node_filesystem_readonly{fstype!="msdosfs"}
== 0) * on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
for: 2m
labels:
severity: warning
annotations:
description: |-
Filesystem is predicted to run out of inodes within the next 24 hours at current write rate
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Host inodes will fill in 24 hours (instance {{ $labels.instance }})
|
ok
|
|
6.655s ago
|
5.171ms |
| alert: HostUnusualDiskReadLatency
expr: (rate(node_disk_read_time_seconds_total[1m])
/ rate(node_disk_reads_completed_total[1m]) > 0.1 and rate(node_disk_reads_completed_total[1m])
> 0) * on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
for: 2m
labels:
severity: warning
annotations:
description: |-
Disk latency is growing (read operations > 100ms)
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Host unusual disk read latency (instance {{ $labels.instance }})
|
ok
|
|
6.65s ago
|
3.747ms |
| alert: HostUnusualDiskWriteLatency
expr: (rate(node_disk_write_time_seconds_total[1m])
/ rate(node_disk_writes_completed_total[1m]) > 0.1 and rate(node_disk_writes_completed_total[1m])
> 0) * on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
for: 2m
labels:
severity: warning
annotations:
description: |-
Disk latency is growing (write operations > 100ms)
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Host unusual disk write latency (instance {{ $labels.instance }})
|
ok
|
|
6.647s ago
|
3.785ms |
| alert: HostCpuStealNoisyNeighbor
expr: (avg
by (instance) (rate(node_cpu_seconds_total{mode="steal"}[5m])) * 100 >
10) * on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
for: 10m
labels:
severity: warning
annotations:
description: |-
CPU steal is > 10%. A noisy neighbor is killing VM performances or a spot instance may be out of credit.
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Host CPU steal noisy neighbor (instance {{ $labels.instance }})
|
ok
|
|
6.643s ago
|
2.858ms |
| alert: HostUnusualDiskIo
expr: (rate(node_disk_io_time_seconds_total[1m])
> 0.5) * on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
for: 5m
labels:
severity: warning
annotations:
description: |-
Time spent in IO is too high on {{ $labels.instance }}. Check storage for issues.
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Host unusual disk IO (instance {{ $labels.instance }})
|
ok
|
|
6.64s ago
|
1.781ms |
| alert: HostSwapIsFillingUp
expr: ((1
- (node_memory_SwapFree_bytes / node_memory_SwapTotal_bytes)) * 100 > 80) * on
(instance) group_left (nodename) node_uname_info{nodename=~".+"}
for: 2m
labels:
severity: warning
annotations:
description: |-
Swap is filling up (>80%)
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Host swap is filling up (instance {{ $labels.instance }})
|
ok
|
|
6.638s ago
|
424.7us |
| alert: HostSystemdServiceCrashed
expr: (node_systemd_unit_state{state="failed"}
== 1) * on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
for: 10m
labels:
severity: warning
annotations:
description: |-
systemd service crashed
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Host systemd service crashed (instance {{ $labels.instance }})
|
ok
|
|
6.638s ago
|
22.28ms |
| alert: HostPhysicalComponentTooHot
expr: ((node_hwmon_temp_celsius
* ignoring (label) group_left (instance, job, node, sensor) node_hwmon_sensor_label{label!="tctl"}
> 75)) * on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
for: 5m
labels:
severity: warning
annotations:
description: |-
Physical hardware component too hot
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Host physical component too hot (instance {{ $labels.instance }})
|
ok
|
|
6.616s ago
|
2.016ms |
| alert: HostNodeOvertemperatureAlarm
expr: ((node_hwmon_temp_crit_alarm_celsius
== 1) or (node_hwmon_temp_alarm == 1)) * on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
labels:
severity: critical
annotations:
description: |-
Physical node temperature alarm triggered
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Host node overtemperature alarm (instance {{ $labels.instance }})
|
ok
|
|
6.615s ago
|
963.2us |
| alert: HostRaidArrayGotInactive
expr: (node_md_state{state="inactive"}
> 0) * on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
labels:
severity: critical
annotations:
description: |-
RAID array {{ $labels.device }} is in a degraded state due to one or more disk failures. The number of spare drives is insufficient to fix the issue automatically.
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Host RAID array got inactive (instance {{ $labels.instance }})
|
ok
|
|
6.614s ago
|
534us |
| alert: HostRaidDiskFailure
expr: (node_md_disks{state="failed"}
> 0) * on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
for: 2m
labels:
severity: warning
annotations:
description: |-
At least one device in RAID array on {{ $labels.instance }} failed. Array {{ $labels.md_device }} needs attention and possibly a disk swap
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Host RAID disk failure (instance {{ $labels.instance }})
|
ok
|
|
6.613s ago
|
375.9us |
| alert: HostKernelVersionDeviations
expr: (count(sum
by (kernel) (label_replace(node_uname_info, "kernel", "$1", "release",
"([0-9]+.[0-9]+.[0-9]+).*"))) > 1) * on (instance) group_left (nodename)
node_uname_info{nodename=~".+"}
for: 6h
labels:
severity: warning
annotations:
description: |-
Different kernel versions are running
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Host kernel version deviations (instance {{ $labels.instance }})
|
ok
|
|
6.613s ago
|
403.1us |
| alert: HostOomKillDetected
expr: (increase(node_vmstat_oom_kill[1m])
> 0) * on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
labels:
severity: warning
annotations:
description: |-
OOM kill detected
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Host OOM kill detected (instance {{ $labels.instance }})
|
ok
|
|
6.613s ago
|
348.8us |
| alert: HostEdacCorrectableErrorsDetected
expr: (increase(node_edac_correctable_errors_total[1m])
> 0) * on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
labels:
severity: info
annotations:
description: |-
Host {{ $labels.instance }} has had {{ printf "%.0f" $value }} correctable memory errors reported by EDAC in the last 5 minutes.
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Host EDAC Correctable Errors detected (instance {{ $labels.instance }})
|
ok
|
|
6.613s ago
|
359.5us |
| alert: HostEdacUncorrectableErrorsDetected
expr: (node_edac_uncorrectable_errors_total
> 0) * on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
labels:
severity: warning
annotations:
description: |-
Host {{ $labels.instance }} has had {{ printf "%.0f" $value }} uncorrectable memory errors reported by EDAC in the last 5 minutes.
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Host EDAC Uncorrectable Errors detected (instance {{ $labels.instance }})
|
ok
|
|
6.612s ago
|
335.5us |
| alert: HostNetworkReceiveErrors
expr: (rate(node_network_receive_errs_total[2m])
/ rate(node_network_receive_packets_total[2m]) > 0.01) * on (instance) group_left
(nodename) node_uname_info{nodename=~".+"}
for: 2m
labels:
severity: warning
annotations:
description: |-
Host {{ $labels.instance }} interface {{ $labels.device }} has encountered {{ printf "%.0f" $value }} receive errors in the last two minutes.
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Host Network Receive Errors (instance {{ $labels.instance }})
|
ok
|
|
6.612s ago
|
14.13ms |
| alert: HostNetworkTransmitErrors
expr: (rate(node_network_transmit_errs_total{device!~"^g09n03abbtesta|^g09n03amobrtra|^g09n03bbbtestb"}[2m])
/ rate(node_network_transmit_packets_total[2m]) > 0.01) * on (instance) group_left
(nodename) node_uname_info{nodename=~".+"}
for: 2m
labels:
severity: warning
annotations:
description: |-
Host {{ $labels.instance }} interface {{ $labels.device }} has encountered {{ printf "%.0f" $value }} transmit errors in the last two minutes.
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Host Network Transmit Errors (instance {{ $labels.instance }})
|
ok
|
|
6.598s ago
|
12.82ms |
| alert: HostNetworkInterfaceSaturated
expr: ((rate(node_network_receive_bytes_total{device!~"^tap.*|^vnet.*|^veth.*|^tun.*|^vp.*"}[1m])
+ rate(node_network_transmit_bytes_total{device!~"^tap.*|^vnet.*|^veth.*|^tun.*|^vp.*"}[1m]))
/ node_network_speed_bytes{device!~"^tap.*|^vnet.*|^veth.*|^tun.*|^vp.*"}
> 0.8 < 10000) * on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
for: 1m
labels:
severity: warning
annotations:
description: |-
The network interface "{{ $labels.device }}" on "{{ $labels.instance }}" is getting overloaded.
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Host Network Interface Saturated (instance {{ $labels.instance }})
|
ok
|
|
6.585s ago
|
12.24ms |
| alert: HostNetworkBondDegraded
expr: ((node_bonding_active
- node_bonding_slaves) != 0) * on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
for: 2m
labels:
severity: warning
annotations:
description: |-
Bond "{{ $labels.device }}" degraded on "{{ $labels.instance }}".
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Host Network Bond Degraded (instance {{ $labels.instance }})
|
ok
|
|
6.573s ago
|
314.8us |
| alert: HostConntrackLimit
expr: (node_nf_conntrack_entries
/ node_nf_conntrack_entries_limit > 0.8) * on (instance) group_left (nodename)
node_uname_info{nodename=~".+"}
for: 5m
labels:
severity: warning
annotations:
description: |-
The number of conntrack is approaching limit
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Host conntrack limit (instance {{ $labels.instance }})
|
ok
|
|
6.573s ago
|
397.5us |
| alert: HostClockSkew
expr: ((node_timex_offset_seconds
> 0.05 and deriv(node_timex_offset_seconds[5m]) >= 0) or (node_timex_offset_seconds
< -0.05 and deriv(node_timex_offset_seconds[5m]) <= 0)) * on (instance) group_left
(nodename) node_uname_info{nodename=~".+"}
for: 10m
labels:
severity: warning
annotations:
description: |-
Clock skew detected. Clock is out of sync. Ensure NTP is configured correctly on this host.
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Host clock skew (instance {{ $labels.instance }})
|
ok
|
|
6.572s ago
|
551.3us |
| alert: HostClockNotSynchronising
expr: (min_over_time(node_timex_sync_status[1m])
== 0 and node_timex_maxerror_seconds >= 16) * on (instance) group_left (nodename)
node_uname_info{nodename=~".+"}
for: 2m
labels:
severity: warning
annotations:
description: |-
Clock not synchronising. Ensure NTP is configured on this host.
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Host clock not synchronising (instance {{ $labels.instance }})
|
ok
|
|
6.572s ago
|
393us |
| alert: HostRequiresReboot
expr: (node_reboot_required
> 0) * on (instance) group_left (nodename) node_uname_info{nodename=~".+"}
for: 4h
labels:
severity: info
annotations:
description: |-
{{ $labels.instance }} requires a reboot.
VALUE = {{ $value }}
LABELS = {{ $labels }}
summary: Host requires reboot (instance {{ $labels.instance }})
|
ok
|
|
6.572s ago
|
864.5us |
|
3.406s ago |
497.5us |