【网络工程】负载均衡工程实践：会话保持、灰度发布与容量规划

# Envoy 配置
clusters:
  - name: app_cluster
    type: STRICT_DNS
    lb_policy: RING_HASH
    ring_hash_lb_config:
      minimum_ring_size: 1024
    load_assignment:
      cluster_name: app_cluster
      endpoints:
        - lb_endpoints:
            - endpoint:
                address:
                  socket_address:
                    address: 10.0.1.1
                    port_value: 8080
            - endpoint:
                address:
                  socket_address:
                    address: 10.0.1.2
                    port_value: 8080

route_config:
  virtual_hosts:
    - name: app
      domains: ["*"]
      routes:
        - match:
            prefix: "/"
          route:
            cluster: app_cluster
            hash_policy:
              - cookie:
                  name: "session_affinity"
                  ttl: 3600s    # Cookie 有效期
                  path: "/"

# Flask + Redis Session 示例
from flask import Flask, session
from flask_session import Session
import redis

app = Flask(__name__)
app.config['SESSION_TYPE'] = 'redis'
app.config['SESSION_REDIS'] = redis.Redis(
    host='redis-cluster.internal',
    port=6379,
    db=0,
    socket_timeout=1,        # 超时要短，避免阻塞请求
    socket_connect_timeout=1,
    retry_on_timeout=True
)
app.config['SESSION_KEY_PREFIX'] = 'session:'
app.config['PERMANENT_SESSION_LIFETIME'] = 3600  # 1小时
Session(app)

@app.route('/login', methods=['POST'])
def login():
    session['user_id'] = request.form['user_id']
    session['login_time'] = time.time()
    return jsonify({"status": "ok"})

@app.route('/profile')
def profile():
    user_id = session.get('user_id')
    if not user_id:
        return jsonify({"error": "not logged in"}), 401
    return jsonify({"user_id": user_id})

# Envoy 权重路由
route_config:
  virtual_hosts:
    - name: app
      domains: ["*"]
      routes:
        - match:
            prefix: "/"
          route:
            weighted_clusters:
              clusters:
                - name: stable
                  weight: 95
                - name: canary
                  weight: 5
              total_weight: 100

# Envoy 基于 Header 的灰度
route_config:
  virtual_hosts:
    - name: app
      domains: ["*"]
      routes:
        # 内部测试流量
        - match:
            prefix: "/"
            headers:
              - name: "x-canary"
                exact_match: "true"
          route:
            cluster: canary
        # 特定用户
        - match:
            prefix: "/"
            headers:
              - name: "x-user-group"
                exact_match: "beta-tester"
          route:
            cluster: canary
        # 默认走稳定版
        - match:
            prefix: "/"
          route:
            cluster: stable

#!/bin/bash
# canary_deploy.sh — 灰度发布自动化脚本
# 用法: ./canary_deploy.sh <new_version> <canary_weight>

NEW_VERSION=$1
CANARY_WEIGHT=${2:-5}  # 默认 5%

echo "=== 灰度发布 v${NEW_VERSION} (${CANARY_WEIGHT}%) ==="

# Step 1: 部署 Canary 实例
echo "Step 1: 部署 Canary 实例..."
kubectl set image deployment/app-canary \
    app=myapp:${NEW_VERSION} --record

kubectl rollout status deployment/app-canary --timeout=120s
if [ $? -ne 0 ]; then
    echo "Canary 部署失败，中止"
    kubectl rollout undo deployment/app-canary
    exit 1
fi

# Step 2: 设置流量权重
echo "Step 2: 设置 Canary 流量权重: ${CANARY_WEIGHT}%..."
kubectl patch virtualservice app-vs --type=json -p="[{
    \"op\": \"replace\",
    \"path\": \"/spec/http/0/route\",
    \"value\": [
        {\"destination\": {\"host\": \"app-stable\"}, \"weight\": $((100 - CANARY_WEIGHT))},
        {\"destination\": {\"host\": \"app-canary\"}, \"weight\": ${CANARY_WEIGHT}}
    ]
}]"

# Step 3: 监控 Canary 指标
echo "Step 3: 监控 Canary 指标 (300s)..."
MONITOR_DURATION=300
INTERVAL=30
ELAPSED=0

while [ $ELAPSED -lt $MONITOR_DURATION ]; do
    # 查询 Canary 的错误率
    ERROR_RATE=$(curl -s "http://prometheus:9090/api/v1/query" \
        --data-urlencode "query=rate(http_requests_total{deployment=\"canary\",code=~\"5..\"}[1m]) / rate(http_requests_total{deployment=\"canary\"}[1m]) * 100" \
        | jq -r '.data.result[0].value[1] // "0"')

    # 查询 Canary 的 P99 延迟
    P99_LATENCY=$(curl -s "http://prometheus:9090/api/v1/query" \
        --data-urlencode "query=histogram_quantile(0.99, rate(http_request_duration_seconds_bucket{deployment=\"canary\"}[1m]))" \
        | jq -r '.data.result[0].value[1] // "0"')

    printf "  [%3ds] 错误率: %.2f%%, P99延迟: %.3fs\n" \
        $ELAPSED "$ERROR_RATE" "$P99_LATENCY"

    # 如果错误率 > 1% 或 P99 > 2s，自动回滚
    if (( $(echo "$ERROR_RATE > 1.0" | bc -l) )); then
        echo "错误率超过阈值，自动回滚！"
        kubectl patch virtualservice app-vs --type=json -p="[{
            \"op\": \"replace\",
            \"path\": \"/spec/http/0/route\",
            \"value\": [{\"destination\": {\"host\": \"app-stable\"}, \"weight\": 100}]
        }]"
        kubectl rollout undo deployment/app-canary
        exit 1
    fi

    sleep $INTERVAL
    ELAPSED=$((ELAPSED + INTERVAL))
done

echo "Step 4: Canary 验证通过，逐步扩大流量..."
for weight in 25 50 75 100; do
    echo "  扩大到 ${weight}%..."
    kubectl patch virtualservice app-vs --type=json -p="[{
        \"op\": \"replace\",
        \"path\": \"/spec/http/0/route\",
        \"value\": [
            {\"destination\": {\"host\": \"app-stable\"}, \"weight\": $((100 - weight))},
            {\"destination\": {\"host\": \"app-canary\"}, \"weight\": ${weight}}
        ]
    }]"
    sleep 60
done

echo "=== 灰度发布完成 ==="

# 检查当前 LB 连接使用情况
# AWS NLB
aws cloudwatch get-metric-statistics \
    --namespace AWS/NetworkELB \
    --metric-name ActiveFlowCount \
    --dimensions Name=LoadBalancer,Value=net/my-nlb/1234567890 \
    --start-time "$(date -u -d '1 hour ago' +%Y-%m-%dT%H:%M:%S)" \
    --end-time "$(date -u +%Y-%m-%dT%H:%M:%S)" \
    --period 60 \
    --statistics Maximum

# 阿里云 CLB
aliyun slb DescribeLoadBalancerAttribute \
    --LoadBalancerId lb-xxxxx \
    --RegionId cn-beijing

# Keepalived 配置 — LB-1 (Master)
# /etc/keepalived/keepalived.conf
global_defs {
    router_id LB_1
    vrrp_skip_check_adv_addr
    vrrp_garp_interval 0
    vrrp_gna_interval 0
}

vrrp_script check_nginx {
    script "/etc/keepalived/check_nginx.sh"
    interval 2
    weight -20        # 检查失败时降低优先级 20
    fall 3            # 连续失败 3 次判定不健康
    rise 2            # 连续成功 2 次判定恢复
}

vrrp_instance VI_1 {
    state MASTER
    interface eth0
    virtual_router_id 51
    priority 100                  # Master 优先级 100
    advert_int 1                  # VRRP 通告间隔 1 秒
    authentication {
        auth_type PASS
        auth_pass secret123
    }
    virtual_ipaddress {
        10.0.0.100/24 dev eth0    # VIP
    }
    track_script {
        check_nginx
    }
    notify_master "/etc/keepalived/notify.sh MASTER"
    notify_backup "/etc/keepalived/notify.sh BACKUP"
    notify_fault  "/etc/keepalived/notify.sh FAULT"
}

#!/bin/bash
# /etc/keepalived/check_nginx.sh
# 检查 Nginx 是否正常运行

# 检查进程
if ! pgrep -x nginx > /dev/null; then
    exit 1
fi

# 检查端口
if ! ss -tlnp | grep -q ':80 '; then
    exit 1
fi

# 检查 HTTP 响应
HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" \
    --connect-timeout 2 --max-time 5 http://127.0.0.1/health)
if [ "$HTTP_CODE" != "200" ]; then
    exit 1
fi

exit 0

#!/bin/bash
# /etc/keepalived/notify.sh
# VRRP 状态变化通知

STATE=$1
TIMESTAMP=$(date '+%Y-%m-%d %H:%M:%S')

case $STATE in
    MASTER)
        echo "$TIMESTAMP: 转为 MASTER，接管 VIP" >> /var/log/keepalived_state.log
        # 发送告警
        curl -X POST "https://alerts.example.com/webhook" \
            -d "{\"msg\": \"LB 主备切换: $(hostname) 成为 MASTER\"}"
        ;;
    BACKUP)
        echo "$TIMESTAMP: 转为 BACKUP，释放 VIP" >> /var/log/keepalived_state.log
        ;;
    FAULT)
        echo "$TIMESTAMP: 进入 FAULT 状态" >> /var/log/keepalived_state.log
        curl -X POST "https://alerts.example.com/webhook" \
            -d "{\"msg\": \"LB 故障: $(hostname) 进入 FAULT 状态\"}"
        ;;
esac

# LB-1 的 Keepalived 配置（双 VIP）
# VIP-1: LB-1 是 Master
vrrp_instance VI_1 {
    state MASTER
    interface eth0
    virtual_router_id 51
    priority 100
    virtual_ipaddress {
        10.0.0.100/24    # VIP-1
    }
    track_script {
        check_nginx
    }
}

# VIP-2: LB-1 是 Backup
vrrp_instance VI_2 {
    state BACKUP
    interface eth0
    virtual_router_id 52
    priority 90
    virtual_ipaddress {
        10.0.0.101/24    # VIP-2
    }
    track_script {
        check_nginx
    }
}

# 使用 MetalLB 提供裸金属环境的 LoadBalancer
# metallb-config.yaml
apiVersion: metallb.io/v1beta1
kind: IPAddressPool
metadata:
  name: lb-pool
  namespace: metallb-system
spec:
  addresses:
    - 10.0.0.100-10.0.0.110

---
apiVersion: metallb.io/v1beta1
kind: L2Advertisement
metadata:
  name: lb-l2
  namespace: metallb-system
spec:
  ipAddressPools:
    - lb-pool

---
# Ingress Controller 部署（高可用）
apiVersion: apps/v1
kind: Deployment
metadata:
  name: nginx-ingress
  namespace: ingress-nginx
spec:
  replicas: 3    # 多副本
  selector:
    matchLabels:
      app: nginx-ingress
  template:
    metadata:
      labels:
        app: nginx-ingress
    spec:
      # Pod 反亲和：确保分布在不同节点
      affinity:
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            - labelSelector:
                matchExpressions:
                  - key: app
                    operator: In
                    values: ["nginx-ingress"]
              topologyKey: kubernetes.io/hostname
      containers:
        - name: nginx-ingress
          image: registry.k8s.io/ingress-nginx/controller:v1.9.0
          resources:
            requests:
              cpu: "500m"
              memory: "256Mi"
            limits:
              cpu: "2000m"
              memory: "1Gi"
          ports:
            - containerPort: 80
            - containerPort: 443

# 验证 Ingress Controller 的分布
kubectl get pods -n ingress-nginx -o wide
# NAME                             READY   NODE
# nginx-ingress-xxx-abc   1/1     node-1
# nginx-ingress-xxx-def   1/1     node-2
# nginx-ingress-xxx-ghi   1/1     node-3

# 检查 Service 的 Endpoints
kubectl get endpoints -n ingress-nginx nginx-ingress
# NAME            ENDPOINTS
# nginx-ingress   10.244.1.5:80,10.244.2.8:80,10.244.3.3:80

# Prometheus 告警规则 — LB 监控
groups:
  - name: lb_alerts
    rules:
      # LB 后端全部不可用
      - alert: LBNoHealthyBackends
        expr: |
          nginx_upstream_peers{state="up"} == 0
        for: 30s
        labels:
          severity: critical
        annotations:
          summary: "LB upstream {{ $labels.upstream }} 无健康后端"

      # LB 高错误率
      - alert: LBHighErrorRate
        expr: |
          rate(nginx_http_requests_total{status=~"5.."}[5m])
          / rate(nginx_http_requests_total[5m]) > 0.05
        for: 2m
        labels:
          severity: warning
        annotations:
          summary: "LB 5xx 错误率超过 5%"

      # LB 连接数接近上限
      - alert: LBHighConnectionCount
        expr: |
          nginx_connections_active / nginx_connections_active_limit > 0.8
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "LB 活跃连接数超过 80%"

      # LB 响应延迟异常
      - alert: LBHighLatency
        expr: |
          histogram_quantile(0.99,
            rate(nginx_http_request_duration_seconds_bucket[5m])
          ) > 2.0
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "LB P99 延迟超过 2 秒"

      # VRRP 主备切换
      - alert: VRRPFailover
        expr: |
          changes(keepalived_vrrp_state[5m]) > 0
        labels:
          severity: critical
        annotations:
          summary: "Keepalived VRRP 发生主备切换"

#!/bin/bash
# lb_health_report.sh — LB 健康巡检报告

echo "=========================================="
echo "LB 健康巡检报告 - $(date '+%Y-%m-%d %H:%M:%S')"
echo "=========================================="

# 1. Nginx 进程状态
echo -e "\n--- Nginx 进程 ---"
nginx_workers=$(pgrep -c nginx)
echo "Worker 进程数: $nginx_workers"
nginx -t 2>&1 | tail -1

# 2. 连接状态
echo -e "\n--- 连接统计 ---"
curl -s http://127.0.0.1/nginx_status 2>/dev/null || echo "status 模块未启用"

# 3. Upstream 健康状态
echo -e "\n--- Upstream 状态 ---"
curl -s http://127.0.0.1/upstream_status 2>/dev/null | \
    python3 -c "
import sys, json
try:
    data = json.load(sys.stdin)
    for name, servers in data.items():
        up = sum(1 for s in servers if s.get('status') == 'up')
        total = len(servers)
        status = 'OK' if up == total else 'DEGRADED' if up > 0 else 'DOWN'
        print(f'  {name}: {up}/{total} healthy [{status}]')
except:
    print('  无法解析 upstream 状态')
"

# 4. 系统资源
echo -e "\n--- 系统资源 ---"
echo "CPU: $(top -bn1 | grep 'Cpu(s)' | awk '{print $2}')% 使用"
echo "内存: $(free -m | awk '/Mem:/{printf "%d/%dMB (%.1f%%)", $3, $2, $3/$2*100}')"
echo "网络连接: $(ss -s | grep 'TCP:' | awk '{print $2}')"

# 5. Keepalived 状态
echo -e "\n--- VRRP 状态 ---"
if systemctl is-active keepalived > /dev/null 2>&1; then
    ip addr show | grep "10.0.0.100" > /dev/null && \
        echo "VIP 10.0.0.100: 本机持有 (MASTER)" || \
        echo "VIP 10.0.0.100: 非本机 (BACKUP)"
else
    echo "Keepalived 未运行"
fi

# 6. 连接排队
echo -e "\n--- 连接排队 ---"
echo "SYN Backlog 溢出: $(cat /proc/net/netstat | awk '/TcpExt/{getline; print $22}')"
echo "Accept Queue 溢出: $(ss -tlnp | head -5)"

echo -e "\n=========================================="
echo "巡检完成"

# 症状：配置了 Cookie 会话保持，但用户仍然被分配到不同后端
# 排查步骤：

# 1. 检查 LB 是否设置了 Cookie
curl -v http://app.example.com/ 2>&1 | grep -i "set-cookie"
# 应该看到: Set-Cookie: SERVERID=s1; Path=/; HttpOnly

# 2. 检查客户端是否发送了 Cookie
curl -v -b "SERVERID=s1" http://app.example.com/ 2>&1 | grep -i cookie
# 应该看到: Cookie: SERVERID=s1

# 3. 检查是否有中间层剥离了 Cookie
# CDN、WAF 可能会修改或删除特定 Cookie
curl -H "Host: app.example.com" http://cdn-ip/debug/headers

# 4. 检查 Cookie 的 Path/Domain 是否正确
# Path=/ 才能在所有路径生效
# Domain 需要匹配请求的域名

# 症状：配置了 5% 灰度，但实际流量远高于 5%
# 常见原因：

# 1. 哈希分布不均
# split_clients 使用的变量基数太小
# 例如只有 10 个不同的 client IP，5% = 0.5 个 IP → 实际是 0% 或 10%

# 2. DNS 缓存
# 如果用 DNS 做灰度，缓存可能导致比例偏差

# 3. 采样误差
# 流量太小时，统计偏差大
# 1000 个请求中 5% = 50 个，实际可能在 30-70 之间波动

# 验证实际流量比例
tail -10000 /var/log/nginx/access.log | \
    awk '{print $NF}' | sort | uniq -c | sort -rn
# 其中 $NF 是日志中标识版本的字段

# 判断瓶颈在 LB 还是后端

# 1. 检查 LB CPU
top -bn1 | head -5
# 如果 CPU > 80%，LB 可能是瓶颈

# 2. 检查 LB 连接数
ss -s
# 如果 established 接近系统限制，需要扩容

# 3. 检查 LB 到后端的延迟
# 在 LB 上执行：
curl -o /dev/null -s -w "connect: %{time_connect}s\nttfb: %{time_starttransfer}s\ntotal: %{time_total}s\n" \
    http://10.0.1.1:8080/health

# 4. 对比 LB 入口延迟和后端延迟
# 如果 LB 入口延迟 >> 后端延迟，说明 LB 是瓶颈
# 检查 Nginx upstream_response_time vs request_time
tail -1000 /var/log/nginx/access.log | \
    awk '{
        req_time = $(NF-1)
        ups_time = $NF
        lb_overhead = req_time - ups_time
        if (lb_overhead > 0.1) print "HIGH LB overhead:", lb_overhead, "s"
    }'