【云原生】云原生监控方案（六）：集成k8s监控组件kube-state-metrics的使用

官方仓库地址：https://github.com/kubernetes/kube-state-metrics

下载地址

兼容性

https://github.com/kubernetes/kube-state-metrics#versioning

kube-state-metrics	Kubernetes client-go Version
v2.12.0	v1.29
v2.13.0	v1.30
v2.14.0	v1.31
v2.15.0	v1.32
v2.16.0	v1.32
main	v1.32

部署

rbac.yaml

组件使用的集群权限

apiVersion: v1
automountServiceAccountToken: false
kind: ServiceAccount
metadata:
  name: kube-state-metrics
  namespace: monitoring
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: kube-state-metrics
rules:
- apiGroups:
  - ""
  resources:
  - configmaps
  - secrets
  - nodes
  - pods
  - services
  - serviceaccounts
  - resourcequotas
  - replicationcontrollers
  - limitranges
  - persistentvolumeclaims
  - persistentvolumes
  - namespaces
  - endpoints
  verbs:
  - list
  - watch
- apiGroups:
  - apps
  resources:
  - statefulsets
  - daemonsets
  - deployments
  - replicasets
  verbs:
  - list
  - watch
- apiGroups:
  - batch
  resources:
  - cronjobs
  - jobs
  verbs:
  - list
  - watch
- apiGroups:
  - autoscaling
  resources:
  - horizontalpodautoscalers
  verbs:
  - list
  - watch
- apiGroups:
  - authentication.k8s.io
  resources:
  - tokenreviews
  verbs:
  - create
- apiGroups:
  - authorization.k8s.io
  resources:
  - subjectaccessreviews
  verbs:
  - create
- apiGroups:
  - policy
  resources:
  - poddisruptionbudgets
  verbs:
  - list
  - watch
- apiGroups:
  - certificates.k8s.io
  resources:
  - certificatesigningrequests
  verbs:
  - list
  - watch
- apiGroups:
  - discovery.k8s.io
  resources:
  - endpointslices
  verbs:
  - list
  - watch
- apiGroups:
  - storage.k8s.io
  resources:
  - storageclasses
  - volumeattachments
  verbs:
  - list
  - watch
- apiGroups:
  - admissionregistration.k8s.io
  resources:
  - mutatingwebhookconfigurations
  - validatingwebhookconfigurations
  verbs:
  - list
  - watch
- apiGroups:
  - networking.k8s.io
  resources:
  - networkpolicies
  - ingressclasses
  - ingresses
  verbs:
  - list
  - watch
- apiGroups:
  - coordination.k8s.io
  resources:
  - leases
  verbs:
  - list
  - watch
- apiGroups:
  - rbac.authorization.k8s.io
  resources:
  - clusterrolebindings
  - clusterroles
  - rolebindings
  - roles
  verbs:
  - list
  - watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: kube-state-metrics
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: kube-state-metrics
subjects:
- kind: ServiceAccount
  name: kube-state-metrics
  namespace: monitoring

kube-state-metrics.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: kube-state-metrics
  namespace: monitoring
spec:
  replicas: 1
  selector:
    matchLabels:
      app: kube-state-metrics
  template:
    metadata:
      labels:
        app: kube-state-metrics
    spec:
      automountServiceAccountToken: true
      containers:
        #- image: registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.16.0
      - image: swr.cn-north-4.myhuaweicloud.com/ddn-k8s/registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.15.0
        livenessProbe:
          httpGet:
            path: /livez
            port: http-metrics
          initialDelaySeconds: 5
          timeoutSeconds: 5
        name: kube-state-metrics
        ports:
        - containerPort: 8080
          name: http-metrics
        - containerPort: 8081
          name: telemetry
        readinessProbe:
          httpGet:
            path: /readyz
            port: telemetry
          initialDelaySeconds: 5
          timeoutSeconds: 5
        securityContext:
          allowPrivilegeEscalation: false
          capabilities:
            drop:
            - ALL
          readOnlyRootFilesystem: true
          runAsNonRoot: true
          runAsUser: 65534
          seccompProfile:
            type: RuntimeDefault
      nodeSelector:
        kubernetes.io/os: linux
      serviceAccountName: kube-state-metrics
---
apiVersion: v1
kind: Service
metadata:
  labels:
    app: kube-state-metrics
  name: kube-state-metrics
  namespace: monitoring
spec:
  clusterIP: None
  ports:
  - name: http-metrics
    port: 8080
    targetPort: http-metrics
  - name: telemetry
    port: 8081
    targetPort: telemetry
  selector:
    app: kube-state-metrics
---
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: kube-state-metrics
  namespace: monitoring
spec:
  endpoints:
  - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
    honorLabels: true
    interval: 30s
    metricRelabelings:
    - action: drop
      regex: kube_endpoint_address_not_ready|kube_endpoint_address_available
      sourceLabels:
      - __name__
    port: http-metrics
    relabelings:
    - action: labeldrop
      regex: (pod|service|endpoint|namespace)
    scheme: http
    scrapeTimeout: 30s
  - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
    interval: 30s
    port: telemetry
    scheme: http
  jobLabel: app.kubernetes.io/name
  selector:
    matchLabels:
      app: kube-state-metrics

kubeStateMetrics-prometheusRule.yaml

apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  labels:
    prometheus: prometheus
    role: alert-rules
  name: kube-state-metrics-rules
  namespace: monitoring
spec:
  groups:
  - name: kube-state-metrics
    rules:
    - alert: 列表操作错误
      annotations:
        description: kube-state-metrics 在列表操作中出现错误的频率过高，这可能会导致它无法正确或根本无法暴露 Kubernetes 对象的指标。
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kube-state-metrics/kubestatemetricslisterrors
        summary: kube-state-metrics 在列表操作中出现错误。
      expr: |
        (sum(rate(kube_state_metrics_list_total{job="kube-state-metrics",result="error"}[5m])) by (cluster)
          /
        sum(rate(kube_state_metrics_list_total{job="kube-state-metrics"}[5m])) by (cluster))
        > 0.01
      for: 15m
      labels:
        severity: critical
    - alert: 监控操作错误
      annotations:
        description: kube-state-metrics 在监控操作中出现错误的频率过高，这可能会导致它无法正确或根本无法暴露 Kubernetes 对象的指标。
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kube-state-metrics/kubestatemetricswatcherrors
        summary: kube-state-metrics 在监控操作中出现错误。
      expr: |
        (sum(rate(kube_state_metrics_watch_total{job="kube-state-metrics",result="error"}[5m])) by (cluster)
          /
        sum(rate(kube_state_metrics_watch_total{job="kube-state-metrics"}[5m])) by (cluster))
        > 0.01
      for: 15m
      labels:
        severity: critical
    - alert: 分片配置不一致
      annotations:
        description: kube-state-metrics 的分片配置不一致，可能会导致某些 Kubernetes 对象被重复暴露或完全不暴露。
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kube-state-metrics/kubestatemetricsshardingmismatch
        summary: kube-state-metrics 的分片配置不一致。
      expr: |
        stdvar (kube_state_metrics_total_shards{job="kube-state-metrics"}) by (cluster) != 0
      for: 15m
      labels:
        severity: critical
    - alert: 分片缺失
      annotations:
        description: kube-state-metrics 的分片缺失，某些 Kubernetes 对象未被暴露。
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kube-state-metrics/kubestatemetricsshardsmissing
        summary: kube-state-metrics 的分片缺失。
      expr: |
        2^max(kube_state_metrics_total_shards{job="kube-state-metrics"}) by (cluster) - 1
          -
        sum( 2 ^ max by (cluster, shard_ordinal) (kube_state_metrics_shard_ordinal{job="kube-state-metrics"}) ) by (cluster)
        != 0
      for: 15m
      labels:
        severity: critical