【网络工程】CDN 故障调试：缓存命中率、回源异常与头分析

# 获取 CDN 响应头（不下载 body）
$ curl -sI https://www.example.com/api/products

HTTP/2 200
date: Sat, 19 Jul 2025 08:30:15 GMT
content-type: application/json
content-length: 4523
cache-control: public, max-age=3600, s-maxage=7200
age: 1847
x-cache: HIT
x-cache-hits: 42
x-served-by: cache-hkg17927-HKG
x-timer: S1721377815.234521,VS0,VE0
via: 1.1 varnish (Fastly)
cf-cache-status: HIT
cf-ray: 8a1234567890-HKG

#!/bin/bash
# cdn-header-check.sh — 批量分析 CDN 响应头

URL="${1:-https://www.example.com/}"
ITERATIONS="${2:-10}"

echo "=== CDN Header Analysis: $URL ==="
echo "Iterations: $ITERATIONS"
echo ""

declare -A cache_stats
total_age=0
age_count=0

for i in $(seq 1 $ITERATIONS); do
    headers=$(curl -sI "$URL" 2>/dev/null)
    
    # 提取缓存状态
    cf_status=$(echo "$headers" | grep -i "cf-cache-status" | awk '{print $2}' | tr -d '\r')
    x_cache=$(echo "$headers" | grep -i "x-cache:" | awk '{print $2}' | tr -d '\r')
    cache_status="${cf_status:-$x_cache}"
    cache_status="${cache_status:-UNKNOWN}"
    
    # 统计缓存状态分布
    cache_stats[$cache_status]=$(( ${cache_stats[$cache_status]:-0} + 1 ))
    
    # 提取 Age
    age=$(echo "$headers" | grep -i "^age:" | awk '{print $2}' | tr -d '\r')
    if [ -n "$age" ]; then
        total_age=$((total_age + age))
        age_count=$((age_count + 1))
    fi
    
    # 提取服务节点
    served_by=$(echo "$headers" | grep -i "x-served-by\|cf-ray\|x-amz-cf-pop" | head -1 | tr -d '\r')
    
    printf "  #%02d: cache=%s age=%s node=%s\n" "$i" "$cache_status" "${age:-N/A}" "$served_by"
    
    sleep 0.5
done

echo ""
echo "--- Cache Status Distribution ---"
for status in "${!cache_stats[@]}"; do
    count=${cache_stats[$status]}
    pct=$((count * 100 / ITERATIONS))
    printf "  %-15s %3d/%d (%d%%)\n" "$status" "$count" "$ITERATIONS" "$pct"
done

if [ $age_count -gt 0 ]; then
    avg_age=$((total_age / age_count))
    echo ""
    echo "--- Age Statistics ---"
    echo "  Average Age: ${avg_age}s"
fi

// custom-cache-key.js — 自定义 Cache Key 逻辑
export default {
  async fetch(request, env) {
    const url = new URL(request.url);
    
    // 定义需要保留的 Query String 参数
    const keepParams = new Set(['id', 'page', 'category', 'v']);
    
    // 定义需要忽略的追踪参数
    const ignoreParams = new Set([
      'utm_source', 'utm_medium', 'utm_campaign', 'utm_content',
      'fbclid', 'gclid', 'msclkid', 'ref', '_ga'
    ]);
    
    // 构建干净的 Cache Key
    const cacheUrl = new URL(url.pathname, url.origin);
    const sortedParams = [...url.searchParams.entries()]
      .filter(([key]) => !ignoreParams.has(key))
      .sort(([a], [b]) => a.localeCompare(b));
    
    sortedParams.forEach(([key, value]) => {
      cacheUrl.searchParams.set(key, value);
    });
    
    // 使用自定义 Cache Key 查找缓存
    const cacheKey = new Request(cacheUrl.toString());
    const cache = caches.default;
    
    let response = await cache.match(cacheKey);
    if (response) {
      // 添加调试头
      response = new Response(response.body, response);
      response.headers.set('X-Cache-Key', cacheUrl.toString());
      response.headers.set('X-Cache', 'HIT');
      return response;
    }
    
    // 缓存未命中，回源
    response = await fetch(request);
    
    // 缓存响应
    if (response.ok) {
      const cloned = response.clone();
      event.waitUntil(cache.put(cacheKey, cloned));
    }
    
    response = new Response(response.body, response);
    response.headers.set('X-Cache-Key', cacheUrl.toString());
    response.headers.set('X-Cache', 'MISS');
    return response;
  }
};

# cdn_metrics.py — CDN 缓存命中率监控
import time
from collections import Counter
from datetime import datetime

class CDNMetrics:
    """CDN 缓存指标采集器"""
    
    def __init__(self):
        self.cache_stats = Counter()    # {status: count}
        self.latency_samples = []       # [(timestamp, latency_ms)]
        self.origin_errors = Counter()  # {status_code: count}
        self.pop_distribution = Counter()  # {pop_id: count}
    
    def record_request(self, cache_status, latency_ms, pop_id, origin_status=None):
        """记录一次请求的指标"""
        self.cache_stats[cache_status] += 1
        self.latency_samples.append((time.time(), latency_ms))
        self.pop_distribution[pop_id] += 1
        
        if origin_status and origin_status >= 500:
            self.origin_errors[origin_status] += 1
    
    def cache_hit_ratio(self):
        """计算缓存命中率"""
        total = sum(self.cache_stats.values())
        if total == 0:
            return 0.0
        
        hits = self.cache_stats.get('HIT', 0) + \
               self.cache_stats.get('STALE', 0) + \
               self.cache_stats.get('REVALIDATED', 0) + \
               self.cache_stats.get('UPDATING', 0)
        
        return hits / total * 100
    
    def report(self):
        """生成指标报告"""
        total = sum(self.cache_stats.values())
        chr_value = self.cache_hit_ratio()
        
        print(f"=== CDN Metrics Report ===")
        print(f"Time: {datetime.now().isoformat()}")
        print(f"Total Requests: {total}")
        print(f"Cache Hit Ratio: {chr_value:.1f}%")
        print(f"")
        print(f"Cache Status Breakdown:")
        for status, count in self.cache_stats.most_common():
            pct = count / total * 100
            print(f"  {status:15s} {count:8d} ({pct:.1f}%)")
        
        if self.latency_samples:
            latencies = [l for _, l in self.latency_samples]
            latencies.sort()
            p50 = latencies[len(latencies) // 2]
            p99 = latencies[int(len(latencies) * 0.99)]
            print(f"")
            print(f"Latency:")
            print(f"  P50: {p50:.0f}ms")
            print(f"  P99: {p99:.0f}ms")
        
        if self.origin_errors:
            print(f"")
            print(f"Origin Errors:")
            for code, count in self.origin_errors.most_common():
                print(f"  HTTP {code}: {count}")
        
        if self.pop_distribution:
            print(f"")
            print(f"Top PoPs:")
            for pop, count in self.pop_distribution.most_common(5):
                pct = count / total * 100
                print(f"  {pop:20s} {count:8d} ({pct:.1f}%)")

# prometheus/rules/cdn-alerts.yml
groups:
  - name: cdn_cache
    rules:
      # 缓存命中率低于 80% 告警
      - alert: CDNCacheHitRatioLow
        expr: |
          sum(rate(cdn_requests_total{cache_status=~"HIT|STALE|REVALIDATED"}[5m]))
          /
          sum(rate(cdn_requests_total[5m]))
          < 0.80
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "CDN 缓存命中率低: {{ $value | humanizePercentage }}"
          description: "过去 15 分钟 CDN 缓存命中率低于 80%，请检查 Cache-Control 配置"
      
      # 回源 5xx 错误率高
      - alert: CDNOrigin5xxHigh
        expr: |
          sum(rate(cdn_origin_requests_total{status=~"5.."}[5m]))
          /
          sum(rate(cdn_origin_requests_total[5m]))
          > 0.05
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "CDN 回源 5xx 错误率高: {{ $value | humanizePercentage }}"
          description: "回源 5xx 错误率超过 5%，源站可能异常"
      
      # 缓存 BYPASS 比例异常
      - alert: CDNCacheBypassHigh
        expr: |
          sum(rate(cdn_requests_total{cache_status="BYPASS"}[5m]))
          /
          sum(rate(cdn_requests_total[5m]))
          > 0.30
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "CDN BYPASS 比例异常: {{ $value | humanizePercentage }}"

# 1. 测试源站可达性（模拟 CDN 边缘视角）
# 使用 CDN 提供的诊断工具或从多个地区测试

# TCP 连接测试
$ nc -zv origin.example.com 443 -w 5
Connection to origin.example.com 443 port [tcp/https] succeeded!

# TLS 握手测试
$ echo | openssl s_client -servername origin.example.com \
    -connect origin.example.com:443 -brief
CONNECTION ESTABLISHED
Protocol version: TLSv1.3
Ciphersuite: TLS_AES_256_GCM_SHA384

# 2. 测量源站响应时间
$ curl -o /dev/null -s -w "
  连接: %{time_connect}s
  TLS:  %{time_appconnect}s
  TTFB: %{time_starttransfer}s
  总计: %{time_total}s
" https://origin.example.com/api/health

# 如果 TTFB > 10s → 源站处理太慢

# 3. 持续监控源站响应时间
$ while true; do
    ttfb=$(curl -o /dev/null -s -w "%{time_starttransfer}" \
        https://origin.example.com/api/health)
    echo "$(date '+%H:%M:%S') TTFB: ${ttfb}s"
    sleep 5
done

#!/bin/bash
# cdn-consistency-check.sh — 检查 CDN 内容一致性

URL="${1:-https://www.example.com/page}"

echo "=== CDN Content Consistency Check ==="
echo "URL: $URL"
echo ""

# 1. 获取源站原始内容的 ETag/Last-Modified
echo "--- Origin Response ---"
origin_headers=$(curl -sI -H "Cache-Control: no-cache" "$URL")
origin_etag=$(echo "$origin_headers" | grep -i "etag" | awk '{print $2}' | tr -d '\r"')
origin_lm=$(echo "$origin_headers" | grep -i "last-modified" | cut -d: -f2- | tr -d '\r')
origin_cl=$(echo "$origin_headers" | grep -i "content-length" | awk '{print $2}' | tr -d '\r')
echo "  ETag: $origin_etag"
echo "  Last-Modified: $origin_lm"
echo "  Content-Length: $origin_cl"
echo ""

# 2. 多次请求检查缓存一致性
echo "--- Cache Consistency (10 requests) ---"
for i in $(seq 1 10); do
    resp=$(curl -sI "$URL")
    
    etag=$(echo "$resp" | grep -i "etag" | awk '{print $2}' | tr -d '\r"')
    age=$(echo "$resp" | grep -i "^age:" | awk '{print $2}' | tr -d '\r')
    cache=$(echo "$resp" | grep -i "cf-cache-status\|x-cache" | head -1 | awk '{print $2}' | tr -d '\r')
    pop=$(echo "$resp" | grep -i "cf-ray\|x-served-by\|x-amz-cf-pop" | head -1 | tr -d '\r')
    cl=$(echo "$resp" | grep -i "content-length" | awk '{print $2}' | tr -d '\r')
    
    match="✓"
    if [ "$etag" != "$origin_etag" ] && [ -n "$origin_etag" ] && [ -n "$etag" ]; then
        match="✗ ETag MISMATCH"
    fi
    if [ "$cl" != "$origin_cl" ] && [ -n "$origin_cl" ] && [ -n "$cl" ]; then
        match="✗ Size MISMATCH"
    fi
    
    printf "  #%02d cache=%-12s age=%-6s etag=%-20s size=%-8s %s\n" \
        "$i" "$cache" "${age:-N/A}" "${etag:-N/A}" "${cl:-N/A}" "$match"
    
    sleep 0.3
done

# 完整的 CDN 性能诊断
$ curl -o /dev/null -s -w @- https://www.example.com/ << 'EOF'
==== CDN Performance Breakdown ====
DNS Lookup:      %{time_namelookup}s
TCP Connect:     %{time_connect}s
TLS Handshake:   %{time_appconnect}s
TTFB:            %{time_starttransfer}s
Total:           %{time_total}s
Download Size:   %{size_download} bytes
Download Speed:  %{speed_download} bytes/s
Remote IP:       %{remote_ip}
HTTP Code:       %{http_code}
=====================================
EOF

# 解读指标：
# DNS Lookup:   > 50ms → DNS 解析慢，检查 DNS 配置
# TCP Connect:  > 100ms → 到 CDN PoP 距离远，检查 Anycast
# TLS Handshake: > 200ms → TLS 配置不优，检查 TLS 版本/OCSP
# TTFB:         > 500ms → 可能回源了，检查缓存状态
# Total:        > 1s → 整体偏慢，需要逐项分析

#!/bin/bash
# cdn-performance-map.sh — 多地域 CDN 性能测试

URL="${1:-https://www.example.com/}"

# 使用不同 DNS 解析器模拟不同地区
declare -A RESOLVERS=(
    ["Cloudflare"]="1.1.1.1"
    ["Google"]="8.8.8.8"
    ["Quad9"]="9.9.9.9"
)

echo "=== Multi-Region CDN Performance ==="
echo "URL: $URL"
echo ""

printf "%-15s %-15s %-8s %-8s %-8s %-8s %-8s %-15s\n" \
    "Resolver" "IP" "DNS" "TCP" "TLS" "TTFB" "Total" "Cache"

for name in "${!RESOLVERS[@]}"; do
    resolver=${RESOLVERS[$name]}
    
    result=$(curl -o /dev/null -s --resolve "$(echo $URL | awk -F/ '{print $3}'):443:$(dig +short @$resolver $(echo $URL | awk -F/ '{print $3}') | head -1)" \
        -w "%{time_namelookup} %{time_connect} %{time_appconnect} %{time_starttransfer} %{time_total} %{remote_ip}" \
        "$URL" 2>/dev/null)
    
    cache=$(curl -sI "$URL" 2>/dev/null | grep -i "cf-cache-status\|x-cache" | head -1 | awk '{print $2}' | tr -d '\r')
    
    read dns tcp tls ttfb total ip <<< "$result"
    
    printf "%-15s %-15s %-8s %-8s %-8s %-8s %-8s %-15s\n" \
        "$name" "$ip" \
        "$(echo "$dns * 1000" | bc)ms" \
        "$(echo "$tcp * 1000" | bc)ms" \
        "$(echo "$tls * 1000" | bc)ms" \
        "$(echo "$ttfb * 1000" | bc)ms" \
        "$(echo "$total * 1000" | bc)ms" \
        "${cache:-N/A}"
done

# 从 CDN 日志中分析缓存命中率（假设日志格式已解析为 TSV）
# 字段: timestamp client_ip cache_status status_code url edge_pop ttfb

# 1. 缓存命中率
echo "=== Cache Hit Ratio ==="
awk -F'\t' '{stats[$3]++; total++} END {
    for (s in stats) printf "  %-15s %8d (%5.1f%%)\n", s, stats[s], stats[s]/total*100
}' cdn_access.log

# 2. 回源慢请求 Top 10
echo "=== Slowest Origin Requests ==="
awk -F'\t' '$3 == "MISS" && $7 > 1000 {print $7"ms\t"$5}' cdn_access.log | \
    sort -rn | head -10

# 3. 5xx 错误的 URL 分布
echo "=== 5xx Errors by URL ==="
awk -F'\t' '$4 >= 500 {urls[$5]++} END {
    for (u in urls) print urls[u], u
}' cdn_access.log | sort -rn | head -20

# 4. 各 PoP 节点的命中率
echo "=== Cache Hit Ratio by PoP ==="
awk -F'\t' '{
    pop_total[$6]++
    if ($3 == "HIT" || $3 == "STALE" || $3 == "REVALIDATED")
        pop_hit[$6]++
} END {
    for (p in pop_total) {
        hit = pop_hit[p]+0
        ratio = hit / pop_total[p] * 100
        printf "  %-20s %8d total, %5.1f%% hit\n", p, pop_total[p], ratio
    }
}' cdn_access.log | sort -t',' -k2 -rn

# === CDN 调试命令速查表 ===

# 1. 查看完整响应头
curl -sI https://www.example.com/

# 2. 跳过缓存直接回源
curl -H "Cache-Control: no-cache" -H "Pragma: no-cache" https://www.example.com/

# 3. 查看请求经过的所有代理
curl -sI https://www.example.com/ | grep -i via

# 4. 检查特定 CDN 的 PoP 节点
curl -sI https://www.example.com/ | grep -i "cf-ray\|x-served-by\|x-amz-cf-pop"

# 5. 测试特定 PoP 节点（通过 DNS 解析到特定 IP）
curl -sI --resolve www.example.com:443:104.16.1.1 https://www.example.com/

# 6. 查看 CDN 解析结果
dig www.example.com +short
dig www.example.com +trace

# 7. 查看 CNAME 链
dig www.example.com CNAME +short
# www.example.com.cdn.cloudflare.net.

# 8. HTTP/2 帧级别调试
curl -v --http2 https://www.example.com/ 2>&1 | grep -E "^[<>*]"

# 9. TLS 握手详情
openssl s_client -servername www.example.com \
    -connect www.example.com:443 -brief

# 10. 测量 CDN vs 源站的性能差异
echo "CDN:"
curl -o /dev/null -s -w "TTFB: %{time_starttransfer}s Total: %{time_total}s\n" \
    https://www.example.com/

echo "Origin (bypass CDN):"
curl -o /dev/null -s -w "TTFB: %{time_starttransfer}s Total: %{time_total}s\n" \
    https://origin.example.com/