DEV-878: added promeRule to monitor s3 capa

prometheusRules/s3_capacity_monitoring.yaml

@@ -0,0 +1,26 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  labels:
+    prometheus: kube-prometheus-stack-prometheus
+    role: alert-rules
+    release: kube-prometheus-stack
+  name: s3-capacity
+spec:
+  groups:
+    - name: "s3_monitoring.rules"
+      rules:
+        - alert: s3_cluster_capacity_warning
+          for: 10m
+          labels:
+            severity: warning
+          expr: avg(minio_cluster_capacity_usable_free_bytes{namespace="{{ .Release.Name }}") * 100 / avg(minio_cluster_capacity_usable_total_bytes{namespace="{{ .Release.Name }}"}) < 15 
+          annotations:
+            message: there is less than 15% left. plz check for deletion or increase capacity
+        - alert: s3_cluster_capacity_critical
+          for: 10m
+          labels:
+            severity: critical
+          expr: avg(minio_cluster_capacity_usable_free_bytes{namespace="{{ .Release.Name }}"}) * 100 / avg(minio_cluster_capacity_usable_total_bytes{namespace="{{ .Release.Name }}"}) < 10
+          annotations:
+            message: there is less than 10% left. plz check for deletion or increase capacity

templates/netpol_egress-miniopods2prometheus.yaml

@@ -0,0 +1,23 @@
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  labels:
+    argocd.argoproj.io/instance: nsodev
+  name: egress-miniopods2prometheus # to display metric stuff within s3-console 
+spec:
+  egress:
+  - ports:
+    - port: 9090
+      protocol: TCP
+    to:
+    - namespaceSelector:
+        matchLabels:
+          kubernetes.io/metadata.name: monitoring
+      podSelector:
+        matchLabels:
+          prometheus: kube-prometheus-stack-prometheus
+  podSelector:
+    matchLabels:
+      v1.min.io/tenant: {{ .Values.tenant.tenant.name }}
+  policyTypes:
+  - Egress