From 0fd93a62f2d5300cb915f48af960114b163726cf Mon Sep 17 00:00:00 2001 From: Dmitriy Kononov Date: Mon, 18 Nov 2019 12:34:05 +0600 Subject: [PATCH] adding IGNORE_PODS --- README.md | 1 + config/custom-environment-variables.yaml | 1 + config/default.yaml | 2 ++ src/monitors/longnotready.ts | 16 ++++++++++++++++ 4 files changed, 20 insertions(+) diff --git a/README.md b/README.md index bd2924d..984e738 100644 --- a/README.md +++ b/README.md @@ -104,6 +104,7 @@ Additionally, the following environment variables can be used: - `TICK_RATE`: How often to update in milliseconds. (Default to 15000 or 15s) - `FLOOD_EXPIRE`: Repeat notification after this many milliseconds has passed after status returned to normal. (Default to 60000 or 60s) - `NOT_READY_MIN_TIME`: Time to wait after pod become not ready before notifying. (Default to 60000 or 60s) +- `IGNORE_PODS`: Comma separated list with the name prefix of the pods to ignore. Example: `kube-proxy-gke,prometheus-prometheus-node-exporter` (Default "") - `METRICS_CPU`: Enable/disable metric alerting on cpu (Default true) - `METRICS_MEMORY`: Enable/disable metric alerting on memory (Default true) - `METRICS_PERCENT`: Set percentage threshold on metric alerts (Default 80) diff --git a/config/custom-environment-variables.yaml b/config/custom-environment-variables.yaml index 1eff74f..326c9b8 100644 --- a/config/custom-environment-variables.yaml +++ b/config/custom-environment-variables.yaml @@ -24,6 +24,7 @@ metrics_requests: __format: json flood_expire: FLOOD_EXPIRE not_ready_min_time: NOT_READY_MIN_TIME +ignore_pods: IGNORE_PODS slack_url: SLACK_URL slack_channel: SLACK_CHANNEL diff --git a/config/default.yaml b/config/default.yaml index 19e21de..6b8927a 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -21,6 +21,8 @@ interval: 15000 flood_expire: 60000 not_ready_min_time: 60000 +ignore_pods: "" + metrics_cpu: true metrics_memory: true metrics_alert: 80 # percent diff --git a/src/monitors/longnotready.ts b/src/monitors/longnotready.ts index 5933f69..8ab0417 100644 --- a/src/monitors/longnotready.ts +++ b/src/monitors/longnotready.ts @@ -10,11 +10,13 @@ import { class PodLongNotReady extends EventEmitter { minimumTime: number; + ignorePods: string; alerted: { [key: string]: KubernetesPod }; constructor() { super(); this.minimumTime = parseInt(config.get('not_ready_min_time'), 10); + this.ignorePods = config.get('ignore_pods'); this.alerted = {}; } @@ -31,9 +33,23 @@ class PodLongNotReady extends EventEmitter { async check() { let pods = await kube.getWatchedPods(); + const ignorePods = this.ignorePods.split(',') for (let pod of pods) { let messageProps: Partial = {}; + + // Ignore pod if exists in ingnore_pods array + let skipPod = false; + for (let ingnorePod of ignorePods) { + if (ingnorePod.length > 0 && pod.metadata.name.startsWith(ingnorePod.trim())) { + skipPod = true; + break; + } + } + if (skipPod) { + continue; + } + let annotations = pod.metadata.annotations; if (annotations) { // Ignore pod if the annotation is set and evaluates to true