【存储工程】存储混沌工程

# 1. 创建一个用于测试的环回设备
dd if=/dev/zero of=test_disk.img bs=1M count=1024
LOOP_DEV=$(losetup --find --show test_disk.img)
echo "环回设备: $LOOP_DEV"

# 2. 获取设备大小（以扇区为单位）
SECTORS=$(blockdev --getsz "$LOOP_DEV")

# 3. 创建 dm-flakey 设备
# 格式: <start> <length> flakey <dev> <offset> <up_interval> <down_interval> [<features>]
# 下面的配置：每 30 秒正常，之后 5 秒返回错误
echo "0 $SECTORS flakey $LOOP_DEV 0 30 5" | dmsetup create flakey-test

# 4. 在 dm-flakey 设备上创建文件系统并挂载
mkfs.ext4 /dev/mapper/flakey-test
mkdir -p /mnt/flakey-test
mount /dev/mapper/flakey-test /mnt/flakey-test

# 只丢弃写入，读取仍然正常（模拟写故障）
echo "0 $SECTORS flakey $LOOP_DEV 0 30 5 1 drop_writes" | dmsetup create flakey-write-only

# 在 down_interval 期间对写入数据进行比特翻转（模拟静默损坏）
echo "0 $SECTORS flakey $LOOP_DEV 0 30 5 1 corrupt_bio_byte 1 w 0 0" | dmsetup create flakey-corrupt

umount /mnt/flakey-test
dmsetup remove flakey-test
losetup -d "$LOOP_DEV"
rm test_disk.img

# 加载 scsi_debug 模块，创建一个 1GB 的虚拟 SCSI 磁盘
# num_tgts: 目标数量  dev_size_mb: 设备大小
modprobe scsi_debug num_tgts=1 dev_size_mb=1024

# 查看创建的设备
lsscsi | grep scsi_debug
# 输出示例: [2:0:0:0] disk Linux scsi_debug 0191 /dev/sdb

# 注入介质错误（Medium Error）：让特定 LBA 范围不可读
# 通过 sysfs 接口注入
echo "0+100" > /sys/bus/pseudo/drivers/scsi_debug/medium_error_start
echo "100" > /sys/bus/pseudo/drivers/scsi_debug/medium_error_count

# 模拟命令超时
echo "5" > /sys/bus/pseudo/drivers/scsi_debug/every_nth
echo "1" > /sys/bus/pseudo/drivers/scsi_debug/opts  # 1 = SDEBUG_OPT_TIMEOUT

#!/bin/bash
# 实验：dm-flakey 下 SQLite 崩溃恢复测试

set -euo pipefail

IMG_FILE="sqlite_flakey_test.img"
DM_NAME="sqlite-flakey"
MOUNT_POINT="/mnt/sqlite-flakey"

# 创建测试环境
dd if=/dev/zero of="$IMG_FILE" bs=1M count=512
LOOP_DEV=$(losetup --find --show "$IMG_FILE")
SECTORS=$(blockdev --getsz "$LOOP_DEV")

# 初始阶段：创建正常设备，建库并写入基线数据
echo "0 $SECTORS linear $LOOP_DEV 0" | dmsetup create "$DM_NAME"
mkfs.ext4 -q /dev/mapper/"$DM_NAME"
mkdir -p "$MOUNT_POINT"
mount /dev/mapper/"$DM_NAME" "$MOUNT_POINT"

# 创建数据库并写入 10000 行基线数据
sqlite3 "$MOUNT_POINT/test.db" <<EOF
PRAGMA journal_mode=WAL;
CREATE TABLE data(id INTEGER PRIMARY KEY, value TEXT, checksum TEXT);
BEGIN;
$(for i in $(seq 1 10000); do
    echo "INSERT INTO data VALUES($i, 'baseline_row_$i', hex(randomblob(16)));"
done)
COMMIT;
SELECT count(*) FROM data;
EOF

echo "基线数据写入完成"
sync

# 切换到 flakey 模式：30 秒正常，5 秒故障
umount "$MOUNT_POINT"
dmsetup remove "$DM_NAME"
echo "0 $SECTORS flakey $LOOP_DEV 0 30 5 1 drop_writes" | dmsetup create "$DM_NAME"
mount /dev/mapper/"$DM_NAME" "$MOUNT_POINT"

# 在 flakey 模式下持续写入
echo "开始在 flakey 模式下写入..."
for batch in $(seq 1 5); do
    sqlite3 "$MOUNT_POINT/test.db" <<EOF 2>&1 || true
BEGIN;
$(for i in $(seq 1 1000); do
    idx=$((10000 + (batch - 1) * 1000 + i))
    echo "INSERT OR REPLACE INTO data VALUES($idx, 'flakey_row_$idx', hex(randomblob(16)));"
done)
COMMIT;
EOF
    echo "  批次 $batch 完成（或失败）"
    sleep 8  # 等待经过一个 down_interval
done

# 恢复到正常模式，检查数据库完整性
umount "$MOUNT_POINT" 2>/dev/null || true
dmsetup remove "$DM_NAME"
echo "0 $SECTORS linear $LOOP_DEV 0" | dmsetup create "$DM_NAME"
mount /dev/mapper/"$DM_NAME" "$MOUNT_POINT"

echo "=== 数据库完整性检查 ==="
sqlite3 "$MOUNT_POINT/test.db" "PRAGMA integrity_check;"
echo "=== 实际行数 ==="
sqlite3 "$MOUNT_POINT/test.db" "SELECT count(*) FROM data;"

# 清理
umount "$MOUNT_POINT"
dmsetup remove "$DM_NAME"
losetup -d "$LOOP_DEV"
rm "$IMG_FILE"

# 创建环回设备
dd if=/dev/zero of=slow_disk.img bs=1M count=1024
LOOP_DEV=$(losetup --find --show slow_disk.img)
SECTORS=$(blockdev --getsz "$LOOP_DEV")

# 创建 dm-delay 设备
# 格式: <start> <length> delay <dev> <offset> <read_delay_ms> [<write_dev> <write_offset> <write_delay_ms>]
# 读延迟 50ms，写延迟 100ms
echo "0 $SECTORS delay $LOOP_DEV 0 50 $LOOP_DEV 0 100" | dmsetup create slow-disk

# 格式化并挂载
mkfs.ext4 -q /dev/mapper/slow-disk
mkdir -p /mnt/slow-disk
mount /dev/mapper/slow-disk /mnt/slow-disk

# 用 fio 验证延迟注入效果
fio --name=latency-test \
    --filename=/mnt/slow-disk/testfile \
    --size=64M \
    --bs=4k \
    --rw=randread \
    --direct=1 \
    --ioengine=libaio \
    --iodepth=1 \
    --numjobs=1 \
    --runtime=10 \
    --time_based \
    --group_reporting

# 对 eth0 接口添加延迟：平均 50ms，标准差 20ms，相关性 25%
tc qdisc add dev eth0 root netem delay 50ms 20ms 25%

# 更复杂的场景：延迟 + 丢包 + 包重排序
tc qdisc add dev eth0 root netem \
    delay 50ms 20ms 25% \
    loss 0.1% \
    reorder 5% 50%

# 只对特定端口生效（例如 iSCSI 3260 端口）
tc qdisc add dev eth0 root handle 1: prio
tc qdisc add dev eth0 parent 1:3 handle 30: netem delay 50ms 20ms
tc filter add dev eth0 protocol ip parent 1:0 prio 3 u32 \
    match ip dport 3260 0xffff flowid 1:3

# 查看当前 qdisc 配置
tc qdisc show dev eth0

# 删除 qdisc 规则
tc qdisc del dev eth0 root

# 确认 cgroup v2 已挂载
mount | grep cgroup2

# 创建 cgroup
mkdir -p /sys/fs/cgroup/io-throttle

# 查看目标设备的 major:minor 号
ls -la /dev/sda
# 假设是 8:0

# 设置 I/O 带宽限制：读 10MB/s，写 5MB/s
echo "8:0 rbps=10485760 wbps=5242880" > /sys/fs/cgroup/io-throttle/io.max

# 设置 IOPS 限制：读 100 IOPS，写 50 IOPS
echo "8:0 riops=100 wiops=50" > /sys/fs/cgroup/io-throttle/io.max

# 同时限制带宽和 IOPS
echo "8:0 rbps=10485760 wbps=5242880 riops=100 wiops=50" > /sys/fs/cgroup/io-throttle/io.max

# 将目标进程加入 cgroup
echo $PID > /sys/fs/cgroup/io-throttle/cgroup.procs

# 查看当前 I/O 统计
cat /sys/fs/cgroup/io-throttle/io.stat

# 安装 libfiu（Ubuntu/Debian）
apt-get install -y fiu-utils libfiu-dev

# 基本用法：让 open() 系统调用以一定概率失败
# fiu-run 会预加载 libfiu 的拦截库
fiu-run -c "enable name=posix/io/rw/read,failinfo=5" -- cat /etc/hostname

# 更精细的控制：通过 fiu-ctrl 动态启用/禁用故障点
# 1. 启动目标程序，开启 fiu 控制通道
fiu-run -x -- my_storage_app &
APP_PID=$!

# 2. 在运行时启用特定故障点
fiu-ctrl -c "enable name=posix/io/rw/write,probability=0.01" $APP_PID

# 3. 观察一段时间后禁用故障
fiu-ctrl -c "disable name=posix/io/rw/write" $APP_PID

#!/usr/bin/env python3
"""
对指定文件的指定偏移量进行比特翻转，模拟静默数据损坏。
注意：此脚本会直接修改文件内容，仅用于测试环境。
"""
import os
import sys
import random

def flip_bits(filepath: str, offset: int, num_bits: int = 1) -> None:
    """在文件的指定偏移量处翻转指定数量的比特。"""
    with open(filepath, "r+b") as f:
        f.seek(offset)
        original_byte = f.read(1)
        if not original_byte:
            print(f"错误：偏移量 {offset} 超出文件大小", file=sys.stderr)
            sys.exit(1)

        byte_val = original_byte[0]
        for _ in range(num_bits):
            bit_pos = random.randint(0, 7)
            byte_val ^= (1 << bit_pos)

        f.seek(offset)
        f.write(bytes([byte_val]))

    print(f"文件: {filepath}")
    print(f"偏移量: {offset}")
    print(f"原始字节: 0x{original_byte[0]:02x} -> 修改后: 0x{byte_val:02x}")

if __name__ == "__main__":
    if len(sys.argv) < 3:
        print(f"用法: {sys.argv[0]} <文件路径> <偏移量> [翻转比特数]")
        sys.exit(1)

    filepath = sys.argv[1]
    offset = int(sys.argv[2])
    num_bits = int(sys.argv[3]) if len(sys.argv) > 3 else 1
    flip_bits(filepath, offset, num_bits)

# 在 ZFS 池中写入测试文件
dd if=/dev/urandom of=/zpool/test/datafile bs=1M count=100
md5sum /zpool/test/datafile  # 记录校验和

# 绕过 ZFS 直接在底层设备上翻转比特
# 先找到文件在底层设备上的物理位置
zdb -ddddd zpool/test datafile

# 对底层设备进行比特翻转
python3 flip_bits.py /dev/sdb 1048576 4

# 读取文件，观察 ZFS 是否检测到损坏
cat /zpool/test/datafile > /dev/null
# 检查 ZFS 事件日志
zpool events | grep -i "checksum"

# 执行 scrub 主动检测
zpool scrub zpool
zpool status -v zpool

# IOChaos 示例：对数据库容器的数据目录注入延迟和错误
apiVersion: chaos-mesh.org/v1alpha1
kind: IOChaos
metadata:
  name: db-io-delay
  namespace: chaos-testing
spec:
  action: latency
  mode: one
  selector:
    namespaces:
      - production
    labelSelectors:
      app: tikv
  volumePath: /var/lib/tikv
  path: "/var/lib/tikv/db/**"
  delay: "100ms"
  percent: 50         # 50% 的 I/O 操作受影响
  duration: "5m"
  methods:
    - read
    - write

# 1. 部署 Chaos Mesh（假设已有 Kubernetes 集群）
# helm repo add chaos-mesh https://charts.chaos-mesh.org
# helm install chaos-mesh chaos-mesh/chaos-mesh -n chaos-mesh --create-namespace

# 2. 定义稳态假设（通过 Prometheus 查询验证）
# - TiDB QPS > 1000
# - TiDB P99 延迟 < 100ms
# - TiKV region leader 均匀分布

# 3. IOChaos 实验：TiKV 写延迟注入
---
apiVersion: chaos-mesh.org/v1alpha1
kind: IOChaos
metadata:
  name: tikv-write-latency
  namespace: chaos-testing
spec:
  action: latency
  mode: one
  selector:
    namespaces:
      - tidb-cluster
    labelSelectors:
      app.kubernetes.io/component: tikv
  volumePath: /var/lib/tikv
  path: "/var/lib/tikv/**"
  delay: "200ms"
  percent: 100
  duration: "10m"
  methods:
    - write
    - fsync

# 4. IOChaos 实验：TiKV I/O 错误注入
---
apiVersion: chaos-mesh.org/v1alpha1
kind: IOChaos
metadata:
  name: tikv-io-error
  namespace: chaos-testing
spec:
  action: fault
  mode: one
  selector:
    namespaces:
      - tidb-cluster
    labelSelectors:
      app.kubernetes.io/component: tikv
  volumePath: /var/lib/tikv
  path: "/var/lib/tikv/db/**.sst"   # 只影响 SST 文件
  errno: 5                           # EIO (I/O error)
  percent: 10                         # 10% 的读操作返回错误
  duration: "5m"
  methods:
    - read

# 5. 使用 Workflow 编排多阶段实验
---
apiVersion: chaos-mesh.org/v1alpha1
kind: Workflow
metadata:
  name: tikv-io-chaos-workflow
  namespace: chaos-testing
spec:
  entry: io-chaos-sequence
  templates:
    - name: io-chaos-sequence
      templateType: Serial
      deadline: "30m"
      children:
        - io-latency-phase
        - recovery-check
        - io-error-phase
        - final-check
    - name: io-latency-phase
      templateType: IOChaos
      deadline: "10m"
      ioChaos:
        action: latency
        mode: one
        selector:
          namespaces:
            - tidb-cluster
          labelSelectors:
            app.kubernetes.io/component: tikv
        volumePath: /var/lib/tikv
        path: "/var/lib/tikv/**"
        delay: "200ms"
        percent: 100
        methods:
          - write
    - name: recovery-check
      templateType: Suspend
      deadline: "5m"   # 等待 5 分钟观察恢复
    - name: io-error-phase
      templateType: IOChaos
      deadline: "5m"
      ioChaos:
        action: fault
        mode: one
        selector:
          namespaces:
            - tidb-cluster
          labelSelectors:
            app.kubernetes.io/component: tikv
        volumePath: /var/lib/tikv
        path: "/var/lib/tikv/**"
        errno: 5
        percent: 5
        methods:
          - read
    - name: final-check
      templateType: Suspend
      deadline: "5m"

# ChaosExperiment：磁盘填充
apiVersion: litmuschaos.io/v1alpha1
kind: ChaosExperiment
metadata:
  name: disk-fill
  namespace: litmus
spec:
  definition:
    scope: Namespaced
    permissions:
      - apiGroups: [""]
        resources: ["pods"]
        verbs: ["create", "delete", "get", "list"]
    image: litmuschaos/go-runner:latest
    args:
      - -c
      - ./experiments -name disk-fill
    command:
      - /bin/bash
    env:
      - name: FILL_PERCENTAGE
        value: "90"
      - name: TOTAL_CHAOS_DURATION
        value: "300"   # 持续 5 分钟
      - name: TARGET_CONTAINER
        value: "tikv"
      - name: CONTAINER_PATH
        value: "/var/lib/tikv"
---
# ChaosEngine：绑定实验到目标应用
apiVersion: litmuschaos.io/v1alpha1
kind: ChaosEngine
metadata:
  name: tikv-disk-fill
  namespace: tidb-cluster
spec:
  appinfo:
    appns: tidb-cluster
    applabel: "app.kubernetes.io/component=tikv"
    appkind: deployment
  engineState: active
  chaosServiceAccount: litmus-admin
  experiments:
    - name: disk-fill
      spec:
        probe:
          - name: tikv-health-check
            type: httpProbe
            httpProbe/inputs:
              url: "http://tikv-svc:20180/status"
              method:
                get:
                  criteria: "=="
                  responseCode: "200"
            mode: Continuous
            runProperties:
              probeTimeout: 5s
              interval: 10s
              retry: 3
          - name: data-integrity-check
            type: cmdProbe
            cmdProbe/inputs:
              command: "mysql -h tidb-svc -e 'SELECT count(*) FROM test.verify_table'"
              comparator:
                type: int
                criteria: ">="
                value: "1000"
            mode: Edge
            runProperties:
              probeTimeout: 10s
              interval: 30s

apiVersion: litmuschaos.io/v1alpha1
kind: ChaosEngine
metadata:
  name: node-io-stress-test
  namespace: litmus
spec:
  appinfo:
    appns: tidb-cluster
    applabel: "app.kubernetes.io/component=tikv"
    appkind: deployment
  engineState: active
  chaosServiceAccount: litmus-admin
  experiments:
    - name: node-io-stress
      spec:
        components:
          env:
            - name: FILESYSTEM_UTILIZATION_PERCENTAGE
              value: "80"
            - name: NUMBER_OF_WORKERS
              value: "4"
            - name: TOTAL_CHAOS_DURATION
              value: "600"     # 持续 10 分钟
            - name: VOLUME_MOUNT_PATH
              value: "/var/lib/tikv"
        probe:
          - name: p99-latency-check
            type: promProbe
            promProbe/inputs:
              endpoint: "http://prometheus:9090"
              query: "histogram_quantile(0.99, sum(rate(tikv_grpc_msg_duration_seconds_bucket[1m])) by (le))"
              comparator:
                type: float
                criteria: "<="
                value: "0.5"    # P99 延迟不超过 500ms
            mode: Continuous
            runProperties:
              probeTimeout: 5s
              interval: 15s

#!/usr/bin/env python3
"""
基于 MAD（中位数绝对偏差）的慢盘检测算法。
MAD 比标准差更能抵抗离群值的干扰。
"""
import numpy as np
from dataclasses import dataclass

@dataclass
class DiskLatencyStats:
    disk_id: str
    p50_us: float   # 中位数延迟（微秒）
    p99_us: float   # P99 延迟（微秒）
    p999_us: float  # P999 延迟（微秒）
    iops: float     # 当前 IOPS

def detect_slow_disks(
    stats_list: list[DiskLatencyStats],
    threshold_multiplier: float = 3.0
) -> list[tuple[str, str]]:
    """
    使用 MAD 方法检测慢盘。
    返回 (disk_id, reason) 的列表。

    MAD = median(|Xi - median(X)|)
    异常阈值 = median(X) + threshold_multiplier * 1.4826 * MAD
    1.4826 是将 MAD 转换为标准差估计的常数（正态分布下）。
    """
    if len(stats_list) < 3:
        return []  # 样本太少，无法做有意义的比较

    slow_disks = []

    for metric_name, get_metric in [
        ("P50", lambda s: s.p50_us),
        ("P99", lambda s: s.p99_us),
        ("P999", lambda s: s.p999_us),
    ]:
        values = np.array([get_metric(s) for s in stats_list])
        median_val = np.median(values)
        mad = np.median(np.abs(values - median_val))

        if mad == 0:
            continue  # 所有磁盘延迟相同，跳过

        threshold = median_val + threshold_multiplier * 1.4826 * mad

        for i, stats in enumerate(stats_list):
            metric_val = get_metric(stats)
            if metric_val > threshold:
                deviation = (metric_val - median_val) / (1.4826 * mad)
                slow_disks.append((
                    stats.disk_id,
                    f"{metric_name} 延迟 {metric_val:.0f}us "
                    f"超过阈值 {threshold:.0f}us "
                    f"（偏离 {deviation:.1f} 倍 MAD）"
                ))

    return slow_disks


# 使用示例
if __name__ == "__main__":
    disk_stats = [
        DiskLatencyStats("sda", 120, 450, 2100, 5000),
        DiskLatencyStats("sdb", 115, 430, 1980, 5200),
        DiskLatencyStats("sdc", 125, 470, 2050, 4800),
        DiskLatencyStats("sdd", 118, 440, 2000, 5100),
        DiskLatencyStats("sde", 350, 1500, 8500, 2000),  # 慢盘
        DiskLatencyStats("sdf", 122, 460, 2080, 4900),
        DiskLatencyStats("sdg", 130, 480, 2150, 4700),
        DiskLatencyStats("sdh", 119, 435, 1990, 5050),
    ]

    results = detect_slow_disks(disk_stats)
    for disk_id, reason in results:
        print(f"[慢盘告警] {disk_id}: {reason}")

#!/usr/bin/env python3
"""
基于指数加权移动平均（EWMA）的纵向慢盘检测。
"""
from collections import deque

class EWMADetector:
    """
    使用 EWMA（Exponentially Weighted Moving Average）跟踪延迟趋势，
    当当前延迟显著偏离历史趋势时触发告警。
    """

    def __init__(self, alpha: float = 0.1, sigma_threshold: float = 3.0):
        """
        alpha: EWMA 衰减因子，越小越平滑
        sigma_threshold: 告警阈值（偏离几倍标准差）
        """
        self.alpha = alpha
        self.sigma_threshold = sigma_threshold
        self.ewma: float | None = None
        self.ewma_var: float = 0.0  # EWMA 方差
        self.sample_count: int = 0

    def update(self, latency_us: float) -> tuple[bool, str]:
        """
        输入一个新的延迟样本，返回 (是否异常, 原因描述)。
        """
        self.sample_count += 1

        if self.ewma is None:
            self.ewma = latency_us
            return False, ""

        # 更新 EWMA 均值
        diff = latency_us - self.ewma
        self.ewma = self.alpha * latency_us + (1 - self.alpha) * self.ewma

        # 更新 EWMA 方差
        self.ewma_var = (1 - self.alpha) * (self.ewma_var + self.alpha * diff * diff)
        ewma_std = self.ewma_var ** 0.5

        if self.sample_count < 30:
            return False, ""  # 预热阶段不告警

        if ewma_std > 0 and abs(diff) > self.sigma_threshold * ewma_std:
            return True, (
                f"当前延迟 {latency_us:.0f}us 偏离 EWMA {self.ewma:.0f}us "
                f"达 {abs(diff)/ewma_std:.1f} 倍标准差"
            )

        return False, ""

experiment:
  id: "CHAOS-STORAGE-2025-042"
  name: "TiKV 单节点慢盘对 TiDB P99 延迟的影响"
  date: "2025-10-01"
  owner: "storage-team"

hypothesis:
  statement: >
    当一个 TiKV 节点的磁盘写延迟增加 200ms 时，
    TiDB 的 P99 查询延迟不超过 500ms，
    且所有已确认的写入事务不丢失。
  pass_criteria:
    - "tidb_p99_latency_ms <= 500"
    - "tikv_write_loss_count == 0"
    - "data_integrity_check == pass"

injection:
  tool: "Chaos Mesh IOChaos"
  target: "tikv-2 Pod"
  type: "latency"
  parameters:
    delay: "200ms"
    methods: ["write", "fsync"]
    percent: 100
    duration: "10m"

blast_radius:
  affected_pods: 1
  affected_nodes: 1
  affected_regions: "~1/3 of total regions"
  rollback_mechanism: "IOChaos 自动删除，FUSE 卸载"
  rollback_timeout: "10m"

baseline:
  collection_period: "2025-10-01T10:00:00Z to 2025-10-01T10:30:00Z"
  tidb_p99_latency_ms: 12.5
  tidb_qps: 15000
  tikv_leader_distribution: "均匀（每节点约 1/3）"

results:
  injection_period: "2025-10-01T10:30:00Z to 2025-10-01T10:40:00Z"
  tidb_p99_latency_ms: 380
  tidb_qps: 12000
  leader_transfer_triggered: true
  leader_transfer_duration_s: 15
  data_integrity_check: "pass"
  hypothesis_result: "PASS"

findings:
  - "TiKV 的 Raft leader 在检测到慢盘后 15 秒触发 leader 转移"
  - "leader 转移期间 TiDB P99 峰值达到 380ms，仍在 500ms 以内"
  - "QPS 下降 20%，主要因为 leader 转移期间的短暂不可用"

action_items:
  - "优化 TiKV 慢盘检测灵敏度，目标从 15s 降低到 5s"
  - "将此实验加入月度混沌测试计划"

# .github/workflows/chaos-test.yml
name: Storage Chaos Tests

on:
  schedule:
    - cron: "0 2 * * 1"   # 每周一凌晨 2 点
  workflow_dispatch:        # 手动触发

jobs:
  setup-cluster:
    runs-on: self-hosted
    outputs:
      cluster-name: ${{ steps.create.outputs.name }}
    steps:
      - name: 创建测试集群
        id: create
        run: |
          CLUSTER_NAME="chaos-test-$(date +%Y%m%d%H%M)"
          kind create cluster --name "$CLUSTER_NAME" --config kind-config.yaml
          echo "name=$CLUSTER_NAME" >> "$GITHUB_OUTPUT"

      - name: 部署存储系统
        run: |
          helm install tidb-operator pingcap/tidb-operator -n tidb-admin --create-namespace
          kubectl apply -f tidb-cluster.yaml
          kubectl wait --for=condition=ready pod -l app.kubernetes.io/component=tikv \
            --timeout=600s -n tidb-cluster

      - name: 部署 Chaos Mesh
        run: |
          helm install chaos-mesh chaos-mesh/chaos-mesh -n chaos-mesh --create-namespace
          kubectl wait --for=condition=ready pod -l app.kubernetes.io/component=controller-manager \
            --timeout=300s -n chaos-mesh

  run-chaos-tests:
    needs: setup-cluster
    runs-on: self-hosted
    strategy:
      fail-fast: false
      matrix:
        scenario:
          - name: "disk-latency-50ms"
            manifest: "chaos/io-latency-50ms.yaml"
            duration: 300
          - name: "disk-latency-200ms"
            manifest: "chaos/io-latency-200ms.yaml"
            duration: 300
          - name: "disk-io-error"
            manifest: "chaos/io-error-5pct.yaml"
            duration: 180
          - name: "disk-fill-90pct"
            manifest: "chaos/disk-fill-90.yaml"
            duration: 300
    steps:
      - uses: actions/checkout@v4

      - name: 采集基线数据
        run: |
          ./scripts/collect-baseline.sh \
            --duration 120 \
            --output "baseline-${{ matrix.scenario.name }}.json"

      - name: 执行混沌实验
        run: |
          kubectl apply -f "${{ matrix.scenario.manifest }}"
          sleep "${{ matrix.scenario.duration }}"

      - name: 验证稳态假设
        run: |
          ./scripts/verify-steady-state.sh \
            --baseline "baseline-${{ matrix.scenario.name }}.json" \
            --scenario "${{ matrix.scenario.name }}"

      - name: 清理混沌实验
        if: always()
        run: kubectl delete -f "${{ matrix.scenario.manifest }}" --ignore-not-found

      - name: 等待恢复并验证
        run: |
          sleep 120  # 等待系统恢复
          ./scripts/verify-recovery.sh

  cleanup:
    needs: [setup-cluster, run-chaos-tests]
    if: always()
    runs-on: self-hosted
    steps:
      - name: 删除测试集群
        run: kind delete cluster --name "${{ needs.setup-cluster.outputs.cluster-name }}"

#!/bin/bash
# collect-and-compare.sh
# 混沌实验前后指标采集与对比

set -euo pipefail

PROMETHEUS_URL="${PROMETHEUS_URL:-http://prometheus:9090}"
BASELINE_DURATION="${BASELINE_DURATION:-300}"  # 基线采集时长（秒）

query_prometheus() {
    local query="$1"
    local time="$2"
    curl -s "${PROMETHEUS_URL}/api/v1/query" \
        --data-urlencode "query=${query}" \
        --data-urlencode "time=${time}" | \
        python3 -c "import sys,json; d=json.load(sys.stdin); print(d['data']['result'][0]['value'][1] if d['data']['result'] else 'N/A')"
}

echo "=== 基线采集（故障注入前） ==="
BASELINE_END=$(date +%s)
BASELINE_START=$((BASELINE_END - BASELINE_DURATION))

BASELINE_P99=$(query_prometheus \
    'histogram_quantile(0.99, sum(rate(tikv_grpc_msg_duration_seconds_bucket{type="kv_prewrite"}[5m])) by (le))' \
    "$BASELINE_END")
BASELINE_QPS=$(query_prometheus \
    'sum(rate(tikv_grpc_msg_duration_seconds_count{type="kv_prewrite"}[5m]))' \
    "$BASELINE_END")

echo "  P99 延迟: ${BASELINE_P99}s"
echo "  QPS: ${BASELINE_QPS}"

echo ""
echo "=== 等待故障注入期结束 ==="
echo "  （由外部脚本控制故障注入）"

echo ""
echo "=== 故障期间指标 ==="
CHAOS_END=$(date +%s)
CHAOS_P99=$(query_prometheus \
    'histogram_quantile(0.99, sum(rate(tikv_grpc_msg_duration_seconds_bucket{type="kv_prewrite"}[5m])) by (le))' \
    "$CHAOS_END")
CHAOS_QPS=$(query_prometheus \
    'sum(rate(tikv_grpc_msg_duration_seconds_count{type="kv_prewrite"}[5m]))' \
    "$CHAOS_END")

echo "  P99 延迟: ${CHAOS_P99}s"
echo "  QPS: ${CHAOS_QPS}"

echo ""
echo "=== 对比 ==="
python3 -c "
baseline_p99 = float('${BASELINE_P99}')
chaos_p99 = float('${CHAOS_P99}')
baseline_qps = float('${BASELINE_QPS}')
chaos_qps = float('${CHAOS_QPS}')

p99_ratio = chaos_p99 / baseline_p99 if baseline_p99 > 0 else float('inf')
qps_ratio = chaos_qps / baseline_qps if baseline_qps > 0 else 0

print(f'  P99 延迟变化: {p99_ratio:.2f}x (基线 {baseline_p99*1000:.1f}ms -> 故障期 {chaos_p99*1000:.1f}ms)')
print(f'  QPS 变化: {qps_ratio:.2f}x (基线 {baseline_qps:.0f} -> 故障期 {chaos_qps:.0f})')
print()

if p99_ratio > 10:
    print('  [FAIL] P99 延迟上升超过 10 倍')
elif p99_ratio > 5:
    print('  [WARN] P99 延迟上升超过 5 倍')
else:
    print('  [PASS] P99 延迟变化在可接受范围内')

if qps_ratio < 0.5:
    print('  [FAIL] QPS 下降超过 50%')
elif qps_ratio < 0.8:
    print('  [WARN] QPS 下降超过 20%')
else:
    print('  [PASS] QPS 变化在可接受范围内')
"