Ceph+K8s 中间件实战 | 第 15 天：K8s 中间件统一监控与运维巡检总结（Prometheus + Grafana 全景）

中间件 Exporter ─┐

node-exporter ───┤──→ Prometheus Server ──→ Alertmanager ──→ 钉钉/邮件

kube-state-metrics┘ │

└──→ Grafana Dashboard（可视化展示）

│

Ceph RBD PVC（持久化 TSDB + Dashboard 配置）

# 添加 Prometheus Community 官方 Helm 仓库

helm repo add prometheus-community https://prometheus-community.github.io/helm-charts

helm repo add bitnami https://charts.bitnami.com/bitnami

helm repo update

# 查看可用版本

helm search repo prometheus-community/kube-prometheus-stack –versions | head -10

# monitoring-values.yaml

# === Prometheus 配置 ===

prometheus:

prometheusSpec:

retention: 30d

storageSpec:

volumeClaimTemplate:

spec:

storageClassName: ceph-rbd-sc # 指向第 2 天创建的 Ceph RBD StorageClass

accessModes: ["ReadWriteOnce"]

resources:

requests:

storage: 50Gi

# 启用 ServiceMonitor 自动发现，覆盖所有带 monitoring 标签的 Service

serviceMonitorSelectorNilUsesHelmValues: false

podMonitorSelectorNilUsesHelmValues: false

resources:

requests:

cpu: 500m

memory: 1Gi

limits:

cpu: 2000m

memory: 4Gi

# === Grafana 配置 ===

grafana:

adminPassword: "Grafana@2026#Secure"

persistence:

enabled: true

storageClassName: ceph-rbd-sc # Grafana Dashboard 持久化到 Ceph

accessModes: ["ReadWriteOnce"]

size: 10Gi

resources:

requests:

cpu: 100m

memory: 256Mi

limits:

cpu: 500m

memory: 512Mi

# 预装常用 Dashboard

dashboardProviders:

dashboardproviders.yaml:

apiVersion: 1

providers:

– name: 'default'

orgId: 1

folder: 'Middleware'

type: file

disableDeletion: false

editable: true

options:

path: /var/lib/grafana/dashboards/default

# === Alertmanager 配置 ===

alertmanager:

alertmanagerSpec:

storage:

volumeClaimTemplate:

spec:

storageClassName: ceph-rbd-sc

accessModes: ["ReadWriteOnce"]

resources:

requests:

storage: 5Gi

# === node-exporter ===

nodeExporter:

enabled: true

# === kube-state-metrics ===

kubeStateMetrics:

enabled: true

# 创建监控命名空间

kubectl create namespace monitoring

# 部署 kube-prometheus-stack

helm install kube-prometheus-stack prometheus-community/kube-prometheus-stack

–namespace monitoring

–version 65.x.x

-f monitoring-values.yaml

# 等待所有 Pod 就绪

kubectl get pods -n monitoring -w

# 查看 Pod 状态（全部应为 Running）

kubectl get pods -n monitoring

# 查看 Service

kubectl get svc -n monitoring

# 查看 PVC — 确认已绑定 Ceph RBD 卷

kubectl get pvc -n monitoring

# 预期输出示例：

# NAME STATUS VOLUME CAPACITY STORAGECLASS

# kube-prometheus-stack-grafana Bound pvc-xxxxx-ceph-rbd 10Gi ceph-rbd-sc

# kube-prometheus-stack-prometheus Bound pvc-yyyyy-ceph-rbd 50Gi ceph-rbd-sc

# kube-prometheus-stack-alertmanager Bound pvc-zzzzz-ceph-rbd 5Gi ceph-rbd-sc

# mysql-servicemonitor.yaml

apiVersion: monitoring.coreos.com/v1

kind: ServiceMonitor

metadata:

name: mysql-monitor

namespace: database

labels:

release: kube-prometheus-stack # 必须匹配 Helm release 名称

spec:

selector:

matchLabels:

app.kubernetes.io/name: mysql

endpoints:

– port: metrics # MySQL exporter 暴露的端口名

interval: 30s

scrapeTimeout: 10s

path: /metrics

# clickhouse-servicemonitor.yaml

apiVersion: monitoring.coreos.com/v1

kind: ServiceMonitor

metadata:

name: clickhouse-monitor

namespace: database

labels:

release: kube-prometheus-stack

spec:

selector:

matchLabels:

app.kubernetes.io/name: clickhouse

endpoints:

– port: http

interval: 15s

path: /metrics

# ClickHouse Prometheus 端点默认在 9362 端口

# 如使用 HTTPS 则需要配置 scheme: https + tlsConfig

# 应用所有中间件的 ServiceMonitor（假设已提前编写到统一目录）

kubectl apply -f ./servicemonitors/

# 验证 ServiceMonitor 已被 Prometheus 识别

kubectl get servicemonitor -A

# 查看 Prometheus 采集目标

kubectl port-forward -n monitoring svc/kube-prometheus-stack-prometheus 9090:9090 &

curl -s http://localhost:9090/api/v1/targets | python3 -m json.tool | grep '"health"'

# 端口转发访问 Grafana

kubectl port-forward -n monitoring svc/kube-prometheus-stack-grafana 3000:80 &

# 浏览器访问 http://localhost:3000

# 用户名: admin

# 密码: Grafana@2026#Secure（在 values.yaml 中设置的）

# 在 Ceph 管理节点启用 Prometheus 模块

ceph mgr module enable prometheus

# 验证指标端点（默认 9283 端口）

curl -s http://<ceph-mgr-ip>:9283/metrics | head -20

# 在 Grafana 中导入 Ceph Dashboard（模板 ID: 2842 / 7330）

# 端口转发到 Prometheus

kubectl port-forward -n monitoring svc/kube-prometheus-stack-prometheus 9090:9090 &

# 验证所有 Target 健康状态

curl -s http://localhost:9090/api/v1/targets |

python3 -c "

import sys, json

data = json.load(sys.stdin)

up = sum(1 for t in data['data']['activeTargets'] if t['health']=='up')

total = len(data['data']['activeTargets'])

print(f'Active Targets: {up}/{total} UP')

for t in data['data']['activeTargets']:

status = '✅' if t['health']=='up' else '❌'

print(f'{status} {t["labels"].get("job","unknown")} → {t["scrapeUrl"]}')

"

# 常用 PromQL 查询示例

# 查询 Pod CPU 使用率

curl -s -G http://localhost:9090/api/v1/query

–data-urlencode 'query=sum(rate(container_cpu_usage_seconds_total{container!=""}[5m])) by (pod)' | python3 -m json.tool

# 检查 Grafana Pod 状态

kubectl get pod -n monitoring -l app.kubernetes.io/name=grafana

# 检查 Grafana 数据源配置

kubectl exec -n monitoring deployment/kube-prometheus-stack-grafana —

curl -s -u admin:'Grafana@2026#Secure' http://localhost:3000/api/datasources | python3 -m json.tool

# 检查 PVC 持久化是否生效（Grafana Dashboard 存储在 Ceph RBD 上）

kubectl get pvc -n monitoring -l app.kubernetes.io/name=grafana

# === 每日巡检脚本 monitoring-daily-check.sh ===

#!/bin/bash

NS=monitoring

echo "===== K8s 中间件监控每日巡检 ====="

echo "[$(date)] 巡检开始"

echo ""

echo "— 1. 监控命名空间 Pod 状态 —"

kubectl get pods -n $NS –no-headers |

awk '{print $1, $3, $4}' | column -t

echo ""

echo "— 2. PVC 绑定状态（Ceph RBD 持久化） —"

kubectl get pvc -n $NS –no-headers |

awk '{print $1, $2, $4}' | column -t

echo ""

echo "— 3. Prometheus 采集目标统计 —"

kubectl exec -n $NS prometheus-kube-prometheus-stack-prometheus-0 —

curl -s http://localhost:9090/api/v1/targets |

python3 -c "

import sys, json

data = json.load(sys.stdin)

targets = data['data']['activeTargets']

up = [t for t in targets if t['health']=='up']

down = [t for t in targets if t['health']!='up']

print(f'UP: {len(up)}/{len(targets)}')

if down:

print('DOWN targets:')

for t in down:

print(f' ❌ {t["labels"].get("job")}: {t["lastError"]}')

"

echo ""

echo "— 4. Alertmanager 告警统计 —"

kubectl exec -n $NS alertmanager-kube-prometheus-stack-alertmanager-0 —

amtool –alertmanager.url=http://localhost:9093 alert query 2>/dev/null | head -20

echo ""

echo "— 5. Ceph 存储池容量 —"

kubectl exec -n $NS prometheus-kube-prometheus-stack-prometheus-0 —

curl -s 'http://localhost:9090/api/v1/query?query=ceph_cluster_available_bytes' |

python3 -m json.tool 2>/dev/null || echo "Ceph 指标未接入"

echo ""

echo "[$(date)] 巡检结束"

# 查询各中间件关键指标（通过 PromQL）

# MySQL 连接数

curl -s -G http://localhost:9090/api/v1/query

–data-urlencode 'query=mysql_global_status_threads_connected'

# Redis 内存使用

curl -s -G http://localhost:9090/api/v1/query

–data-urlencode 'query=redis_memory_used_bytes'

# MongoDB 连接数

curl -s -G http://localhost:9090/api/v1/query

–data-urlencode 'query=mongodb_connections'

# Kafka 消息滞后

curl -s -G http://localhost:9090/api/v1/query

–data-urlencode 'query=kafka_consumergroup_lag'

# ClickHouse 查询数

curl -s -G http://localhost:9090/api/v1/query

–data-urlencode 'query=clickhouse_query_count'

# Elasticsearch 集群健康状态

curl -s -G http://localhost:9090/api/v1/query

–data-urlencode 'query=elasticsearch_cluster_health_status'

# 查看已配置的告警规则

kubectl exec -n $NS prometheus-kube-prometheus-stack-prometheus-0 —

curl -s http://localhost:9090/api/v1/rules |

python3 -c "

import sys, json

data = json.load(sys.stdin)

groups = data['data']['groups']

total = 0

firing = 0

for g in groups:

for r in g['rules']:

total += 1

if r.get('state') == 'firing':

firing += 1

print(f'🔥 FIRING: {r["name"]}')

print(f'nTotal rules: {total}, Firing: {firing}')

"