DEV-616: increased vol_count to mitigate disk size problem

group_vars/stage_prodnso/prometheus.yml

@@ -1,5 +1,5 @@
 ---
 prometheus_lvm_hcloudvol_size: 30
-prometheus_lvm_hcloudvol_count: 2
+prometheus_lvm_hcloudvol_count: 3
 
 prometheus_tsdb_rentention_time: '90d'

templates/prometheus/config/prometheus/alert.rules.j2

@@ -90,6 +90,15 @@ groups:
       summary: "Disk Space Usage (instance {{ '{{' }} $labels.instance {{ '}}' }})"
       description: "Disk Space on Drive is used more than {{ prometheus_alert_diskspaceusage_warning }}%\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS: {{ '{{' }} $labels {{ '}}' }}"
 
+  - alert: DiskSpaceUsage mounted volumes
+    expr: 100.0 - 100 * (node_filesystem_free_bytes{env="{{ stage }}",device=~"/dev/mapper/.*"} / node_filesystem_size_bytes{device!="/dev/loop1",env="{{ stage }}",device=~"/dev/mapper/.*"}) > {{ prometheus_alert_diskspaceusage_warning }}
+    for: 10m
+    labels:
+      severity: warning
+    annotations:
+      summary: "Disk Space Usage (instance {{ '{{' }} $labels.instance {{ '}}' }})"
+      description: "Disk Space on Drive is used more than {{ prometheus_alert_diskspaceusage_warning }}%\n  VALUE = {{ '{{' }} $value {{ '}}' }}\n  LABELS: {{ '{{' }} $labels {{ '}}' }}"
+
   - alert: software_raid_disks_active
     expr: node_md_disks_active != 2
     for: 2m