Add Prometheus config for monitoring Kubernetes Ingress

* Allow Kubernetes Ingress resources to be probed via Blackbox
Exporter (if present) if annotated `prometheus.io/probe: "true"`
* Fix probes of Services via Blackbox Exporter. Require Blackbox
Exporter to be deployed in the same `monitoring` namespace, be
named `blackbox-exporter`, and use port 8080

CHANGES.md

@@ -14,6 +14,8 @@ Notable changes between versions.
 ### Addons
 
 * Configure Prometheus to allow a custom scrape query parameter ([#1095](https://github.com/poseidon/typhoon/pull/1095))
+* Configure Prometheus to probe Kubernetes Ingress via `blackbox-exporter` ([#1096](https://github.com/poseidon/typhoon/pull/1096))
+* Fix Prometheus Service probes to use `blackbox-exporter`, not `blackbox` ([#1096](https://github.com/poseidon/typhoon/pull/1096))
 
 ## v1.23.0
 

addons/prometheus/config.yaml

@@ -220,11 +220,45 @@ data:
         action: drop
         regex: etcd_(debugging|disk|request|server).*
 
-    # Example scrape config for probing services via the Blackbox Exporter.
+    # Example scrape config for pods
     #
-    # The relabeling allows the actual service scrape endpoint to be configured
-    # via the following annotations:
+    # The relabeling allows the actual pod scrape endpoint to be configured via the
+    # following annotations:
+    #
+    # * `prometheus.io/scrape`: Only scrape pods that have a value of `true`
+    # * `prometheus.io/path`: If the metrics path is not `/metrics` override this.
+    # * `prometheus.io/port`: Scrape the pod on the indicated port instead of the
+    # pod's declared ports (default is a port-free target if none are declared).
+    - job_name: 'kubernetes-pods'
+
+      kubernetes_sd_configs:
+      - role: pod
+
+      relabel_configs:
+      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
+        action: keep
+        regex: true
+      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
+        action: replace
+        target_label: __metrics_path__
+        regex: (.+)
+      - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
+        action: replace
+        regex: ([^:]+)(?::\d+)?;(\d+)
+        replacement: $1:$2
+        target_label: __address__
+      - action: labelmap
+        regex: __meta_kubernetes_pod_label_(.+)
+      - source_labels: [__meta_kubernetes_namespace]
+        action: replace
+        target_label: kubernetes_namespace
+      - source_labels: [__meta_kubernetes_pod_name]
+        action: replace
+        target_label: kubernetes_pod_name
+
+    # Example scrape config for probing Services via the Blackbox Exporter.
     #
+    # Relabeling allows service scraping to be configured via annotations:
     # * `prometheus.io/probe`: Only probe services that have a value of `true`
     - job_name: 'kubernetes-services'
 
@@ -242,7 +276,7 @@ data:
       - source_labels: [__address__]
         target_label: __param_target
       - target_label: __address__
-        replacement: blackbox
+        replacement: blackbox-exporter:8080
       - source_labels: [__param_target]
         target_label: instance
       - action: labelmap
@@ -252,41 +286,36 @@ data:
       - source_labels: [__meta_kubernetes_service_name]
         target_label: job
 
-    # Example scrape config for pods
+    # Example scrape config for probing Ingresses via a Blackbox Exporter.
     #
-    # The relabeling allows the actual pod scrape endpoint to be configured via the
-    # following annotations:
-    #
-    # * `prometheus.io/scrape`: Only scrape pods that have a value of `true`
-    # * `prometheus.io/path`: If the metrics path is not `/metrics` override this.
-    # * `prometheus.io/port`: Scrape the pod on the indicated port instead of the
-    # pod's declared ports (default is a port-free target if none are declared).
-    - job_name: 'kubernetes-pods'
+    # Relabeling allows service scraping to be configured via annotations:
+    # * `prometheus.io/probe`: Only probe ingresses that have a value of `true`
+    - job_name: 'kubernetes-ingresses'
+      metrics_path: /probe
+      params:
+        module: [http_2xx]
 
       kubernetes_sd_configs:
-      - role: pod
+      - role: ingress
 
       relabel_configs:
-      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
+      - source_labels: [__meta_kubernetes_ingress_annotation_prometheus_io_probe]
         action: keep
         regex: true
-      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
-        action: replace
-        target_label: __metrics_path__
-        regex: (.+)
-      - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
-        action: replace
-        regex: ([^:]+)(?::\d+)?;(\d+)
-        replacement: $1:$2
-        target_label: __address__
+      - source_labels: [__meta_kubernetes_ingress_scheme, __address__, __meta_kubernetes_ingress_path]
+        regex: (.+);(.+);(.+)
+        replacement: ${1}://${2}${3}
+        target_label: __param_target
+      - target_label: __address__
+        replacement: blackbox-exporter:8080
+      - source_labels: [__param_target]
+        target_label: instance
       - action: labelmap
-        regex: __meta_kubernetes_pod_label_(.+)
+        regex: __meta_kubernetes_ingress_label_(.+)
       - source_labels: [__meta_kubernetes_namespace]
-        action: replace
-        target_label: kubernetes_namespace
-      - source_labels: [__meta_kubernetes_pod_name]
-        action: replace
-        target_label: kubernetes_pod_name
+        target_label: namespace
+      - source_labels: [__meta_kubernetes_service_name]
+        target_label: job
 
     # Rule files
     rule_files:

addons/prometheus/rbac/cluster-role.yaml

@@ -10,6 +10,17 @@ rules:
   - services
   - endpoints
   - pods
-  verbs: ["get", "list", "watch"]
+  verbs:
+  - get
+  - list
+  - watch
 - nonResourceURLs: ["/metrics"]
   verbs: ["get"]
+- apiGroups:
+  - networking.k8s.io
+  resources:
+  - ingresses
+  verbs:
+  - get
+  - list
+  - watch