Ceph+K8s 中间件实战 | 第 14 天：K8s 部署 ClickHouse 列式数据库集群（Ceph RBD + 架构设计）

客户端写入 → Distributed 表（路由分发）

→ 分片1: 本地表写入 → MerkleTree 同步 → 副本2

→ 分片2: 本地表写入 → MerkleTree 同步 → 副本2

查询请求 → Distributed 表 → 聚合各分片局部结果 → 合并返回

# 添加 Bitnami Chart 仓库

helm repo add bitnami https://charts.bitnami.com/bitnami

helm repo update

# 查看 ClickHouse Chart 可用版本

helm search repo bitnami/clickhouse –versions | head -10

# 拉取默认 values 以便自定义

helm show values bitnami/clickhouse > /tmp/clickhouse-default-values.yaml

# /tmp/clickhouse-values.yaml

## ClickHouse 集群配置

##

shards: 2

replicaCount: 2

## ClickHouse 认证配置

##

auth:

username: "admin"

password: "ClickHouse@2026"

existingSecret: ""

## ClickHouse Keeper（内置协调服务，替代外部 ZooKeeper）

##

keeper:

enabled: true

replicaCount: 3

auth:

client:

enabled: false

persistence:

enabled: true

storageClass: "ceph-rbd"

accessModes:

– ReadWriteOnce

size: 10Gi

## ClickHouse 持久化存储（核心数据卷）

##

persistence:

enabled: true

storageClass: "ceph-rbd"

accessModes:

– ReadWriteOnce

size: 100Gi

## 资源限制（按需调整）

##

resources:

requests:

cpu: "1000m"

memory: "2Gi"

limits:

cpu: "4000m"

memory: "8Gi"

## Service 配置

##

service:

type: ClusterIP

ports:

http: 8123

tcp: 9000

mysql: 9004

## 集群配置（分布式表 DDL）

##

clickhouse:

configmap:

## 远程服务器定义（自动由 Chart 根据 shards/replicas 生成）

remote_servers: ""

## 日志级别

##

logging:

level: "information"

# 创建独立命名空间

kubectl create namespace middleware

# 部署 ClickHouse 集群

helm install clickhouse bitnami/clickhouse

–namespace middleware

–version 6.2.0

–values /tmp/clickhouse-values.yaml

–timeout 10m

# 观察 Pod 启动过程

kubectl get pods -n middleware -w

# 查看 ClickHouse 与 Keeper Pod 状态

kubectl get pods -n middleware -l app.kubernetes.io/instance=clickhouse

# 预期输出：

# NAME READY STATUS RESTARTS AGE

# clickhouse-shard0-0 1/1 Running 0 3m

# clickhouse-shard0-1 1/1 Running 0 3m

# clickhouse-shard1-0 1/1 Running 0 3m

# clickhouse-shard1-1 1/1 Running 0 3m

# clickhouse-keeper-0 1/1 Running 0 4m

# clickhouse-keeper-1 1/1 Running 0 3m

# clickhouse-keeper-2 1/1 Running 0 2m

# 查看持久化卷声明（应全部 Bound）

kubectl get pvc -n middleware -l app.kubernetes.io/instance=clickhouse

# 查看 Service

kubectl get svc -n middleware -l app.kubernetes.io/instance=clickhouse

# 查看 StatefulSet

kubectl get sts -n middleware -l app.kubernetes.io/instance=clickhouse

# 转发 ClickHouse TCP 端口（原生协议 9000）

kubectl port-forward -n middleware svc/clickhouse 9000:9000 &

# 使用 clickhouse-client 连接

clickhouse-client –host 127.0.0.1 –port 9000

–user admin –password 'ClickHouse@2026'

–query "SELECT version()"

# 预期输出：24.x.x

— 进入 clickhouse-client 交互模式

— kubectl exec -it -n middleware clickhouse-shard0-0 — clickhouse-client –user admin –password 'ClickHouse@2026'

— 查看集群分片与副本信息

SELECT

shard_num,

replica_num,

host_name,

port

FROM system.clusters

WHERE cluster = 'default';

— 预期输出：4 行（2 分片 × 2 副本）

— 在每个分片创建本地表（使用 ReplicatedMergeTree 引擎）

CREATE TABLE IF NOT EXISTS events_local ON CLUSTER 'default'

(

event_time DateTime,

event_id UInt64,

user_id String,

event_type String

)

ENGINE = ReplicatedMergeTree(

'/clickhouse/tables/{shard}/events_local',

'{replica}'

)

PARTITION BY toYYYYMM(event_time)

ORDER BY (event_id, event_time);

— 创建分布式表（写入与查询的统一入口）

CREATE TABLE IF NOT EXISTS events_distributed ON CLUSTER 'default'

AS events_local

ENGINE = Distributed('default', 'default', 'events_local', rand());

— 插入测试数据

INSERT INTO events_distributed VALUES

(now(), 1, 'user_001', 'click'),

(now(), 2, 'user_002', 'view'),

(now(), 3, 'user_003', 'purchase');

— 查询验证

SELECT count() FROM events_distributed;

— 预期：3

— 验证各分片数据分布

SELECT

hostName() AS node,

count() AS rows

FROM events_distributed

GROUP BY node;

# 检查所有 Pod 运行状态

kubectl get pods -n middleware -l app.kubernetes.io/instance=clickhouse

# 检查 Keeper 集群状态（Raft 共识）

kubectl exec -n middleware clickhouse-keeper-0 —

clickhouse-keeper-client –host clickhouse-keeper-0 –port 9181

–query "ruok"

# 预期：imok

# 检查 PVC 绑定状态

kubectl get pvc -n middleware | grep clickhouse

# 一键巡检脚本

kubectl exec -it -n middleware clickhouse-shard0-0 —

clickhouse-client –user admin –password 'ClickHouse@2026' –multiquery << 'SQL'

— 1. 查看活跃副本同步状态

SELECT database, table, is_leader, is_readonly, is_session_expired

FROM system.replicas;

— 2. 查看合并队列积压

SELECT database, table, count() AS pending_merges

FROM system.merges

GROUP BY database, table;

— 3. 查看磁盘使用情况

SELECT

name,

path,

formatReadableSize(free_space) AS free,

formatReadableSize(total_space) AS total

FROM system.disks;

— 4. 查看最近慢查询

SELECT

query_duration_ms,

formatReadableSize(read_bytes) AS read_bytes,

substring(query, 1, 80) AS query

FROM system.query_log

WHERE type = 'QueryFinish'

ORDER BY query_duration_ms DESC

LIMIT 5;

SQL

# 查看 ClickHouse 服务日志

kubectl logs -n middleware clickhouse-shard0-0 –tail=50

# 查看 Keeper 日志

kubectl logs -n middleware clickhouse-keeper-0 –tail=30

# 查看 HTTP 接口指标（Prometheus 格式）

kubectl exec -n middleware clickhouse-shard0-0 —

curl -s http://admin:ClickHouse%402026@localhost:8123/metrics | head -20

# 查看各分片数据分区大小

kubectl exec -n middleware clickhouse-shard0-0 —

clickhouse-client –user admin –password 'ClickHouse@2026'

–query "

SELECT

database,

table,

partition,

formatReadableSize(sum(bytes_on_disk)) AS size

FROM system.parts

WHERE active = 1

GROUP BY database, table, partition

ORDER BY size DESC

LIMIT 10

"

# 查看 Ceph RBD 卷的实际使用量

kubectl exec -n middleware clickhouse-shard0-0 — df -h /bitnami/clickhouse/data