Ceph+K8s 中间件实战 | 第 18 天：K8s 部署 RabbitMQ 集群（Ceph RBD + 高可用架构 + 登录巡检）

Producer Pod ──AMQP──▶ Headless Service ──▶ RabbitMQ-0/1/2 任一节点

└─▶ Erlang Cluster (25672)

├─ 队列镜像 (Quorum/Mirrored)

├─ Raft 共识 (Quorum Queue)

└─ MySQL 元数据同步 (可选)

# 添加 Bitnami Chart 仓库

helm repo add bitnami https://charts.bitnami.com/bitnami

helm repo update

# 查看 RabbitMQ Chart 最新版本

helm search repo bitnami/rabbitmq

# bitnami/rabbitmq 16.x.x 3.13.x RabbitMQ is an open-source message broker

# 拉取 Chart 到本地查看 values 默认值

helm pull bitnami/rabbitmq –version 16.9.0

tar -xf rabbitmq-16.9.0.tgz

cat rabbitmq/values.yaml | head -80

# 通过 kubectl exec 进入 MySQL Pod 创建用户与库

kubectl exec -it mysql-0 -n middleware — mysql -uroot -p"$MYSQL_ROOT_PASSWORD" -e "

CREATE DATABASE IF NOT EXISTS rabbitmq CHARACTER SET utf8mb4;

CREATE USER IF NOT EXISTS 'rabbitmq_user'@'%' IDENTIFIED BY 'Rmq@2026Secure!';

GRANT ALL PRIVILEGES ON rabbitmq.* TO 'rabbitmq_user'@'%';

FLUSH PRIVILEGES;

"

# rabbitmq-values.yaml

# RabbitMQ 集群高可用配置（Ceph RBD 持久化）

# — 集群拓扑 —

replicaCount: 3

auth:

# 管理员账号密码（生产用随机字符串）

username: admin

password: "RabbitMQ@2026#Secure"

existingPasswordSecret: ""

# — Erlang Cookie（节点间信任）—

rabbitmq:

# 统一 secret，写入所有 Pod 的 /var/lib/rabbitmq/.erlang.cookie

erlangCookie: "SharedErlangCookie_2026!@#Secure"

# 自定义配置（挂载到 /etc/rabbitmq/rabbitmq.conf）

configuration: |

cluster_formation.peer_discovery_backend = rabbit_peer_discovery_k8s

cluster_formation.k8s.service_host = rabbitmq-headless.middleware.svc.cluster.local

quorum_queue_type = classic

disk_free_limit.absolute = 1GB

vm_memory_high_watermark.relative = 0.5

# 启用 management 插件（15672 端口）

plugins:

– rabbitmq_management

– rabbitmq_management_agent

# — Ceph RBD 持久化（核心）—

volumeClaimTemplates:

data:

storageClass: ceph-rbd # 复用第 2 天创建的 StorageClass

accessModes:

– ReadWriteOnce # RBD 块存储仅支持 RWO

resources:

requests:

storage: 20Gi

reclaimPolicy: Retain # 保留卷，便于数据恢复

# — 持久化开关 —

persistence:

enabled: true

existingClaim: "" # 空 = 使用 volumeClaimTemplates 自动创建

# — 资源限制 —

resources:

requests:

cpu: 500m

memory: 1Gi

limits:

cpu: "2"

memory: 4Gi

# — 高可用与调优 —

extraConfiguration: |

# Quorum Queue 推荐配置

quorum_queue_type = classic

# 磁盘预留

disk_free_limit.absolute = 1GB

# Erlang 内存高水位

vm_memory_high_watermark.relative = 0.5

vm_memory_high_watermark_paging_ratio = 0.5

# 心跳间隔

heartbeat = 60

# Channel 与 Connection 上限

channel_max = 2048

connection_high_watermark.relative = 0.9

# — Network Policies（生产建议开启）—

networkPolicy:

enabled: false

allowIngressFromAnywhere: true

allowEgressToAnywhere: true

# — Ingress（可选，仅调试）—

ingress:

enabled: false

# — Metrics（Prometheus 抓取）—

metrics:

enabled: true

serviceMonitor:

enabled: true

namespace: monitoring

labels:

release: prometheus

# 创建独立命名空间

kubectl create namespace middleware –dry-run=client -o yaml | kubectl apply -f –

# 执行安装（–wait 等待 Pod Ready，–timeout 防超时）

helm install rabbitmq bitnami/rabbitmq

–namespace middleware

-f rabbitmq-values.yaml

–version 16.9.0

–set image.registry=registry.cn-hangzhou.aliyuncs.com

–set image.repository=bitnami/rabbitmq

–wait

–timeout 10m0s

–debug

# Pod 状态（应 3/3 Running 且 READY 为 3/3）

kubectl get pods -n middleware -l app.kubernetes.io/instance=rabbitmq

# PVC 绑定（应全部 Bound，storageClass 为 ceph-rbd）

kubectl get pvc -n middleware -o wide | grep rabbitmq

# Service 列表（Headless 供集群内连接，NodePort/ClusterIP 供外部）

kubectl get svc -n middleware | grep rabbitmq

# 查看 Ceph 后端实际 RBD 镜像

rbd ls ceph/rabbitmq-data –id admin

rbd du ceph/rabbitmq-data-rabbitmq-0

# 进入 rabbitmq-0 节点

kubectl exec -it rabbitmq-0 -n middleware — bash

# 列出所有集群节点（应看到 rabbitmq-0/1/2）

rabbitmqctl cluster_status

# 列出队列及其类型（quorum/classic）

rabbitmqctl list_queues name type durable messages state

# 列出所有 vhost

rabbitmqctl list_vhosts

# 列出用户与权限

rabbitmqctl list_users

rabbitmqctl list_permissions

# 查看节点运行状态

rabbitmqctl status | head -40

# 通过 kubectl port-forward 暴露 15672

kubectl port-forward -n middleware svc/rabbitmq 15672:15672 &

# 登录获取 cookie（curl -c 保存 session）

curl -c /tmp/cookie -X POST http://localhost:15672/api/login

-H "Content-Type: application/json"

-d '{"username":"admin","password":"RabbitMQ@2026#Secure"}'

# 列出所有队列（JSON）

curl -b /tmp/cookie http://localhost:15672/api/queues

-H "Accept: application/json" | python3 -m json.tool

# 列出所有连接

curl -b /tmp/cookie http://localhost:15672/api/connections

-H "Accept: application/json" | python3 -m json.tool

# 创建测试队列（持久化 + Quorum）

curl -b /tmp/cookie -X PUT http://localhost:15672/api/queues/%2F/test-queue

-H "Content-Type: application/json"

-d '{"durable":true,"type":"quorum","auto_delete":false}'

import pika

# 连接参数

params = pika.ConnectionParameters(

host='rabbitmq.middleware.svc.cluster.local',

port=5672,

credentials=pika.PlainCredentials('admin', 'RabbitMQ@2026#Secure'),

heartbeat=60,

connection_attempts=3,

)

conn = pika.BlockingConnection(params)

ch = conn.channel()

ch.queue_declare(queue='test-queue', durable=True)

# 发布消息

for i in range(10):

ch.basic_publish(

exchange='',

routing_key='test-queue',

body=f'hello rabbitmq message {i}',

properties=pika.BasicProperties(delivery_mode=2), # 2 = persistent

)

print('Published 10 messages')

conn.close()

# ① 节点存活

kubectl get pods -n middleware -l app.kubernetes.io/instance=rabbitmq

-o custom-columns=NAME:.metadata.name,READY:.status.containerStatuses[*].ready,RESTARTS:.status.containerStatuses[*].restartCount

# ② 集群节点列表

kubectl exec -n middleware rabbitmq-0 — rabbitmqctl cluster_status | grep -E "Nodes|Running"

# ③ 队列健康（消息堆积告警阈值）

kubectl exec -n middleware rabbitmq-0 — rabbitmqctl list_queues name messages ready unacknowledged

–formatter pretty_table

# ④ Erlang 进程内存与磁盘水位

kubectl exec -n middleware rabbitmq-0 — rabbitmqctl status |

grep -E "memory|disk|uptime" -A2

# 抓取 RabbitMQ metrics endpoint（Pod 内暴露 9419）

kubectl exec -n middleware rabbitmq-0 — curl -s http://localhost:9419/metrics |

grep -E "^rabbitmq_messages_(published|delivered|redelivered|unacknowledged|ready|unacked)"

# Grafana 查询示例：过去 1 小时消息吞吐

rabbitmq_messages_published_total[1h] – rabbitmq_messages_published_total[1h offset 1h]

# 磁盘使用率

node_filesystem_avail_bytes{mountpoint="/var/lib/rabbitmq"} /

node_filesystem_size_bytes{mountpoint="/var/lib/rabbitmq"}

# ① PVC 容量（应小于 80% 触发扩容）

kubectl get pvc -n middleware -o json | jq '.items[] | {name:.metadata.name, cap:.status.capacity.storage, sc:.spec.storageClassName}'

# ② Ceph 后端占用

ceph df | grep rabbitmq

rbd du ceph/rabbitmq-data-rabbitmq-0

# ③ 错误日志（近 100 行 ERROR）

kubectl logs -n middleware rabbitmq-0 –tail=100 | grep -iE "error|warn|refused|timeout"

# ④ 连接数突增告警

rabbitmqctl list_connections | wc -l

# ① 全量备份队列元数据（含 vhost / user / policy）

kubectl exec -n middleware rabbitmq-0 —

rabbitmqctl dump_policies /tmp/policies.json

kubectl cp rabbitmq-0:/tmp/policies.json ./policies-backup.json -n middleware

# ② Ceph 快照（Ceph CSI 支持）

cat > /tmp/rmq-snapshot.yaml <<EOF

apiVersion: snapshot.storage.k8s.io/v1

kind: VolumeSnapshot

metadata:

name: rabbitmq-0-snap-20261019

namespace: middleware

spec:

source:

persistentVolumeClaimName: data-rabbitmq-0

volumeSnapshotClassName: ceph-rbd-snapshotclass

EOF

kubectl apply -f /tmp/rmq-snapshot.yaml

# ③ 验证快照状态

kubectl get volumesnapshot -n middleware

ceph rbd snap ls ceph/rabbitmq-data-rabbitmq-0