Add remaining kubernetesResources alert rules

helmfiles/values/prometheus.yaml.gotmpl

@@ -55,6 +55,63 @@ defaultRules:
     # always firing
     kubernetesResources: false
 
+additionalPrometheusRulesMap:
+   # We disable the creating of kubernetesResources rules above because
+   # of the always firing KubeCPUOvercommit and KubeMemOvercommit rules
+   # we need to manually add those which are fine here
+  kubernetes-resources:
+    groups:
+    - name: kubernetes-resources
+      rules:
+      - alert: KubeCPUOvercommit
+        annotations:
+          message: Cluster has overcommitted CPU resource requests for Namespaces.
+          runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecpuovercommit
+        expr: |-
+          sum(kube_resourcequota{job="kube-state-metrics", type="hard", resource="cpu"})
+            /
+          sum(node:node_num_cpu:sum)
+            > 1.5
+        for: 5m
+        labels:
+          severity: warning
+      - alert: KubeMemOvercommit
+        annotations:
+          message: Cluster has overcommitted memory resource requests for Namespaces.
+          runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubememovercommit
+        expr: |-
+          sum(kube_resourcequota{job="kube-state-metrics", type="hard", resource="memory"})
+            /
+          sum(node_memory_MemTotal_bytes{job="node-exporter"})
+            > 1.5
+        for: 5m
+        labels:
+          severity: warning
+      - alert: KubeQuotaExceeded
+        annotations:
+          message: Namespace {{`{{ $labels.namespace }}`}} is using {{`{{ printf "%0.0f" $value }}`}}% of its {{`{{ $labels.resource }}`}} quota.
+          runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubequotaexceeded
+        expr: |-
+          100 * kube_resourcequota{job="kube-state-metrics", type="used"}
+            / ignoring(instance, job, type)
+          (kube_resourcequota{job="kube-state-metrics", type="hard"} > 0)
+            > 90
+        for: 15m
+        labels:
+          severity: warning
+      - alert: CPUThrottlingHigh
+        annotations:
+          message: '{{`{{ printf "%0.0f" $value }}`}}% throttling of CPU in namespace {{`{{ $labels.namespace }}`}} for container {{`{{ $labels.container_name }}`}} in pod {{`{{ $labels.pod_name }}`}}.'
+          runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-cputhrottlinghigh
+        expr: |-
+          100 * sum(increase(container_cpu_cfs_throttled_periods_total{container_name!="", }[5m])) by (container_name, pod_name, namespace)
+            /
+          sum(increase(container_cpu_cfs_periods_total{}[5m])) by (container_name, pod_name, namespace)
+            > 25
+        for: 15m
+        labels:
+          severity: warning
+
 grafana:
   adminPassword: "{{ requiredEnv "GRAFANA_ADMIN_PASSWORD" }}"