From 34444fdcd2d4d02ccf69b491cf3244d550c0caf2 Mon Sep 17 00:00:00 2001 From: Danijel Simeunovic Date: Wed, 7 Oct 2026 13:34:38 +0200 Subject: [PATCH] chore(prometheus): tighten retention and drop high-cardinality series - Cap retention at 7d / 6GB so WAL cannot fill the 8Gi PV - Raise kyverno/traefik scrape interval 15s -> 60s - Drop request-duration histogram buckets and go_*/process_* runtime metrics from both jobs Co-Authored-By: Claude Opus 4.7 (1M context) --- infra/values/base/prometheus-values.yaml | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/infra/values/base/prometheus-values.yaml b/infra/values/base/prometheus-values.yaml index e30119c..5f82624 100644 --- a/infra/values/base/prometheus-values.yaml +++ b/infra/values/base/prometheus-values.yaml @@ -7,6 +7,9 @@ server: strategy: type: Recreate + retention: 7d + retentionSize: 6GB + resources: requests: cpu: 150m @@ -21,7 +24,7 @@ server: extraScrapeConfigs: | - job_name: kyverno - scrape_interval: 15s + scrape_interval: 60s metrics_path: /metrics kubernetes_sd_configs: - role: endpoints @@ -38,9 +41,16 @@ extraScrapeConfigs: | target_label: pod - source_labels: [__meta_kubernetes_namespace] target_label: namespace + metric_relabel_configs: + - source_labels: [__name__] + regex: 'kyverno_(http_requests_duration_seconds|controller_.+_duration_seconds|admission_review_duration_seconds)_bucket' + action: drop + - source_labels: [__name__] + regex: 'go_.*|process_.*' + action: drop - job_name: traefik - scrape_interval: 15s + scrape_interval: 60s metrics_path: /metrics kubernetes_sd_configs: - role: endpoints @@ -57,6 +67,13 @@ extraScrapeConfigs: | target_label: pod - source_labels: [__meta_kubernetes_namespace] target_label: namespace + metric_relabel_configs: + - source_labels: [__name__] + regex: 'traefik_(router|service|entrypoint)_request_duration_seconds_bucket' + action: drop + - source_labels: [__name__] + regex: 'go_.*|process_.*' + action: drop alertmanager: enabled: false