0

I am working to build an alert rule that is basically "When number of running pods is greater than cluster_pod_capacity - single_node_capacity"

sum(kubelet_running_pod_count)
>
sum(kube_node_status_capacity_pods) - ( sum(kube_node_status_capacity_pods) / count(kube_node_labels{label_region="node"})

This is not accurate though, as the metrics for kubelet_running_pod_count and kube_node_status_capacity_pods do not have any label which identifies the system as a node.

Here are sample outputs from each of the commands

kubelet_running_pod_count

kubelet_running_pod_count{endpoint="https-metrics",instance="10.200.2.100:10250",job="kubelet",namespace="kube-system",service="kubelet"}   5
kubelet_running_pod_count{endpoint="https-metrics",instance="10.200.2.42:10250",job="kubelet",namespace="kube-system",service="kubelet"}    24
kubelet_running_pod_count{endpoint="https-metrics",instance="10.200.2.90:10250",job="kubelet",namespace="kube-system",service="kubelet"}    3
kubelet_running_pod_count{endpoint="https-metrics",instance="10.200.4.54:10250",job="kubelet",namespace="kube-system",service="kubelet"}    111
kubelet_running_pod_count{endpoint="https-metrics",instance="10.200.4.90:10250",job="kubelet",namespace="kube-system",service="kubelet"}    6
kubelet_running_pod_count{endpoint="https-metrics",instance="10.200.6.11:10250",job="kubelet",namespace="kube-system",service="kubelet"}    92
kubelet_running_pod_count{endpoint="https-metrics",instance="10.200.6.187:10250",job="kubelet",namespace="kube-system",service="kubelet"}   2
kubelet_running_pod_count{endpoint="https-metrics",instance="10.200.6.18:10250",job="kubelet",namespace="kube-system",service="kubelet"}    3

kube_node_status_capacity_pods

kube_node_status_capacity_pods{endpoint="kube-state-metrics",instance="10.1.8.75:8080",job="kube-state",namespace="prometheus-prod",node="ip-10-200-2-100.ap-southeast-2.compute.internal",pod="prometheus-prod-exporter-kube-state-67cd655ffc-s57b5",service="prometheus-prod-exporter-kube-state"}    40
kube_node_status_capacity_pods{endpoint="kube-state-metrics",instance="10.1.8.75:8080",job="kube-state",namespace="prometheus-prod",node="ip-10-200-2-42.ap-southeast-2.compute.internal",pod="prometheus-prod-exporter-kube-state-67cd655ffc-s57b5",service="prometheus-prod-exporter-kube-state"} 160
kube_node_status_capacity_pods{endpoint="kube-state-metrics",instance="10.1.8.75:8080",job="kube-state",namespace="prometheus-prod",node="ip-10-200-2-90.ap-southeast-2.compute.internal",pod="prometheus-prod-exporter-kube-state-67cd655ffc-s57b5",service="prometheus-prod-exporter-kube-state"} 20
kube_node_status_capacity_pods{endpoint="kube-state-metrics",instance="10.1.8.75:8080",job="kube-state",namespace="prometheus-prod",node="ip-10-200-4-54.ap-southeast-2.compute.internal",pod="prometheus-prod-exporter-kube-state-67cd655ffc-s57b5",service="prometheus-prod-exporter-kube-state"} 160
kube_node_status_capacity_pods{endpoint="kube-state-metrics",instance="10.1.8.75:8080",job="kube-state",namespace="prometheus-prod",node="ip-10-200-4-90.ap-southeast-2.compute.internal",pod="prometheus-prod-exporter-kube-state-67cd655ffc-s57b5",service="prometheus-prod-exporter-kube-state"} 40
kube_node_status_capacity_pods{endpoint="kube-state-metrics",instance="10.1.8.75:8080",job="kube-state",namespace="prometheus-prod",node="ip-10-200-6-11.ap-southeast-2.compute.internal",pod="prometheus-prod-exporter-kube-state-67cd655ffc-s57b5",service="prometheus-prod-exporter-kube-state"} 160
kube_node_status_capacity_pods{endpoint="kube-state-metrics",instance="10.1.8.75:8080",job="kube-state",namespace="prometheus-prod",node="ip-10-200-6-18.ap-southeast-2.compute.internal",pod="prometheus-prod-exporter-kube-state-67cd655ffc-s57b5",service="prometheus-prod-exporter-kube-state"} 40
kube_node_status_capacity_pods{endpoint="kube-state-metrics",instance="10.1.8.75:8080",job="kube-state",namespace="prometheus-prod",node="ip-10-200-6-187.ap-southeast-2.compute.internal",pod="prometheus-prod-exporter-kube-state-67cd655ffc-s57b5",service="prometheus-prod-exporter-kube-state"}    20

kube_node_labels (some identifying data redacted)

kube_node_labels{endpoint="kube-state-metrics",instance="10.1.8.75:8080",job="kube-state",label_beta_kubernetes_io_arch="amd64",label_beta_kubernetes_io_instance_type="m4.4xlarge",label_beta_kubernetes_io_os="linux",label_failure_domain_beta_kubernetes_io_region="ap-southeast-2",label_failure_domain_beta_kubernetes_io_zone="ap-southeast-2b",label_kubernetes_io_hostname="ip-10-200-4-54.ap-southeast-2.compute.internal",label_logging_es_node="forbidden",label_logging_infra_fluentd="allowed",label_node_role_kubernetes_io_compute="true",label_puppetname="node2.cluster",label_region="node",label_registry="allowed",label_router="forbidden",label_zone="ap-southeast-2b",namespace="prometheus-prod",node="ip-10-200-4-54.ap-southeast-2.compute.internal",pod="prometheus-prod-exporter-kube-state-67cd655ffc-s57b5",service="prometheus-prod-exporter-kube-state"}   1
kube_node_labels{endpoint="kube-state-metrics",instance="10.1.8.75:8080",job="kube-state",label_beta_kubernetes_io_arch="amd64",label_beta_kubernetes_io_instance_type="m4.xlarge",label_beta_kubernetes_io_os="linux",label_failure_domain_beta_kubernetes_io_region="ap-southeast-2",label_failure_domain_beta_kubernetes_io_zone="ap-southeast-2b",label_kubernetes_io_hostname="ip-10-200-4-90.ap-southeast-2.compute.internal",label_logging_es_node="forbidden",label_logging_infra_fluentd="allowed",label_node_role_kubernetes_io_master="true",label_puppetname="master2.cluster",label_region="master",label_registry="forbidden",label_router="forbidden",label_zone="ap-southeast-2b",namespace="prometheus-prod",node="ip-10-200-4-90.ap-southeast-2.compute.internal",pod="prometheus-prod-exporter-kube-state-67cd655ffc-s57b5",service="prometheus-prod-exporter-kube-state"}   1
kube_node_labels{endpoint="kube-state-metrics",instance="10.1.8.75:8080",job="kube-state",label_beta_kubernetes_io_arch="amd64",label_beta_kubernetes_io_instance_type="m5.4xlarge",label_beta_kubernetes_io_os="linux",label_failure_domain_beta_kubernetes_io_region="ap-southeast-2",label_failure_domain_beta_kubernetes_io_zone="ap-southeast-2a",label_kubernetes_io_hostname="ip-10-200-2-42.ap-southeast-2.compute.internal",label_logging_es_node="forbidden",label_logging_infra_fluentd="allowed",label_node_role_kubernetes_io_compute="true",label_puppetname="node1.cluster",label_region="node",label_registry="allowed",label_router="forbidden",label_zone="ap-southeast-2a",namespace="prometheus-prod",node="ip-10-200-2-42.ap-southeast-2.compute.internal",pod="prometheus-prod-exporter-kube-state-67cd655ffc-s57b5",service="prometheus-prod-exporter-kube-state"}   1
kube_node_labels{endpoint="kube-state-metrics",instance="10.1.8.75:8080",job="kube-state",label_beta_kubernetes_io_arch="amd64",label_beta_kubernetes_io_instance_type="m5.4xlarge",label_beta_kubernetes_io_os="linux",label_failure_domain_beta_kubernetes_io_region="ap-southeast-2",label_failure_domain_beta_kubernetes_io_zone="ap-southeast-2c",label_kubernetes_io_hostname="ip-10-200-6-11.ap-southeast-2.compute.internal",label_logging_es_node="forbidden",label_logging_infra_fluentd="allowed",label_node_role_kubernetes_io_compute="true",label_puppetname="node3.cluster",label_region="node",label_registry="allowed",label_router="forbidden",label_zone="ap-southeast-2c",namespace="prometheus-prod",node="ip-10-200-6-11.ap-southeast-2.compute.internal",pod="prometheus-prod-exporter-kube-state-67cd655ffc-s57b5",service="prometheus-prod-exporter-kube-state"}   1
kube_node_labels{endpoint="kube-state-metrics",instance="10.1.8.75:8080",job="kube-state",label_beta_kubernetes_io_arch="amd64",label_beta_kubernetes_io_instance_type="m5.xlarge",label_beta_kubernetes_io_os="linux",label_failure_domain_beta_kubernetes_io_region="ap-southeast-2",label_failure_domain_beta_kubernetes_io_zone="ap-southeast-2a",label_kubernetes_io_hostname="ip-10-200-2-100.ap-southeast-2.compute.internal",label_logging_es_node="forbidden",label_logging_infra_fluentd="allowed",label_node_role_kubernetes_io_master="true",label_puppetname="master1.cluster",label_region="master",label_registry="forbidden",label_router="forbidden",label_zone="ap-southeast-2a",namespace="prometheus-prod",node="ip-10-200-2-100.ap-southeast-2.compute.internal",pod="prometheus-prod-exporter-kube-state-67cd655ffc-s57b5",service="prometheus-prod-exporter-kube-state"} 1
kube_node_labels{endpoint="kube-state-metrics",instance="10.1.8.75:8080",job="kube-state",label_beta_kubernetes_io_arch="amd64",label_beta_kubernetes_io_instance_type="m5.xlarge",label_beta_kubernetes_io_os="linux",label_failure_domain_beta_kubernetes_io_region="ap-southeast-2",label_failure_domain_beta_kubernetes_io_zone="ap-southeast-2c",label_kubernetes_io_hostname="ip-10-200-6-18.ap-southeast-2.compute.internal",label_logging_es_node="forbidden",label_logging_infra_fluentd="allowed",label_node_role_kubernetes_io_master="true",label_puppetname="master3.cluster",label_region="master",label_registry="forbidden",label_router="forbidden",label_zone="ap-southeast-2c",namespace="prometheus-prod",node="ip-10-200-6-18.ap-southeast-2.compute.internal",pod="prometheus-prod-exporter-kube-state-67cd655ffc-s57b5",service="prometheus-prod-exporter-kube-state"}   1
kube_node_labels{endpoint="kube-state-metrics",instance="10.1.8.75:8080",job="kube-state",label_beta_kubernetes_io_arch="amd64",label_beta_kubernetes_io_instance_type="t2.medium",label_beta_kubernetes_io_os="linux",label_failure_domain_beta_kubernetes_io_region="ap-southeast-2",label_failure_domain_beta_kubernetes_io_zone="ap-southeast-2a",label_kubernetes_io_hostname="ip-10-200-2-90.ap-southeast-2.compute.internal",label_logging_es_node="forbidden",label_logging_infra_fluentd="allowed",label_node_role_kubernetes_io_compute="true",label_puppetname="lb1.cluster",label_region="router",label_registry="forbidden",label_router="allowed",label_zone="ap-southeast-2a",namespace="prometheus-prod",node="ip-10-200-2-90.ap-southeast-2.compute.internal",pod="prometheus-prod-exporter-kube-state-67cd655ffc-s57b5",service="prometheus-prod-exporter-kube-state"}    1
kube_node_labels{endpoint="kube-state-metrics",instance="10.1.8.75:8080",job="kube-state",label_beta_kubernetes_io_arch="amd64",label_beta_kubernetes_io_instance_type="t2.medium",label_beta_kubernetes_io_os="linux",label_failure_domain_beta_kubernetes_io_region="ap-southeast-2",label_failure_domain_beta_kubernetes_io_zone="ap-southeast-2c",label_kubernetes_io_hostname="ip-10-200-6-187.ap-southeast-2.compute.internal",label_logging_es_node="forbidden",label_logging_infra_fluentd="allowed",label_node_role_kubernetes_io_compute="true",label_puppetname="lb2.cluster",label_region="router",label_registry="forbidden",label_router="allowed",label_zone="ap-southeast-2c",namespace="prometheus-prod",node="ip-10-200-6-187.ap-southeast-2.compute.internal",pod="prometheus-prod-exporter-kube-state-67cd655ffc-s57b5",service="prometheus-prod-exporter-kube-state"}  1

I suppose what I am looking to figure out, how can I have the subqueries reference the labels on each node?

For reference, we are running OpenShift 3.9, with Prometheus being installed from the older coreos/kube-prometheus charts (until we can get upgraded to OS 3.11)

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.