【网络工程】HTTP 调试方法论：curl、DevTools 与 mitmproxy

# -v (verbose) 显示完整的请求/响应头和 TLS 握手信息
curl -v https://api.example.com/health

# 输出示例：
# * Trying 93.184.216.34:443...
# * Connected to api.example.com (93.184.216.34) port 443
# * ALPN: offers h2,http/1.1
# * TLSv1.3 (OUT), TLS handshake, Client hello (1):
# * TLSv1.3 (IN), TLS handshake, Server hello (2):
# * TLSv1.3 (IN), TLS handshake, Certificate (11):
# * TLSv1.3 (IN), TLS handshake, CERT verify (15):
# * SSL connection using TLSv1.3 / TLS_AES_256_GCM_SHA384
# * ALPN: server accepted h2
# > GET /health HTTP/2
# > Host: api.example.com
# > User-Agent: curl/8.4.0
# > Accept: */*
# >
# < HTTP/2 200
# < content-type: application/json
# < cache-control: no-store
# < x-request-id: abc-123
# <
# {"status":"healthy"}

# -I 只获取响应头（HEAD 请求）
curl -I https://example.com

# -s 静默模式 + -o /dev/null 丢弃 body，只看头
curl -s -o /dev/null -D - https://example.com

# 指定 HTTP 方法
curl -X POST https://api.example.com/users \
    -H "Content-Type: application/json" \
    -d '{"name":"test","email":"test@example.com"}'

# 发送表单数据
curl -X POST https://example.com/login \
    -d "username=admin&password=secret"

# 上传文件
curl -X POST https://api.example.com/upload \
    -F "file=@/path/to/document.pdf" \
    -F "description=test upload"

# 指定自定义头
curl -H "Authorization: Bearer eyJ..." \
     -H "X-Request-ID: debug-001" \
     -H "Accept: application/json" \
     https://api.example.com/users/me

# 指定 Cookie
curl -b "session=abc123; lang=zh-CN" https://example.com

# 保存和发送 Cookie（模拟浏览器的 Cookie 行为）
curl -c cookies.txt https://example.com/login -d "user=admin&pass=secret"
curl -b cookies.txt https://example.com/dashboard

# 输出完整的字节流（十六进制 + ASCII）
curl --trace trace.log https://example.com

# --trace-ascii 只看 ASCII 部分（更可读）
curl --trace-ascii - https://example.com 2>&1 | head -40

# 输出示例：
# == Info:   Trying 93.184.216.34:443...
# == Info: Connected to example.com (93.184.216.34) port 443
# => Send SSL data, 512 bytes (0x200)
# 0000: 16 03 01 01 fb 01 00 01 f7 03 03 ...
# <= Recv SSL data, 155 bytes (0x9b)
# ...
# => Send header, 78 bytes (0x4e)
# 0000: GET / HTTP/2
# 0014: Host: example.com
# 0027: User-Agent: curl/8.4.0
# 003f: Accept: */*
# <= Recv header, 13 bytes (0xd)
# 0000: HTTP/2 200

# --trace-time 添加时间戳（分析延迟）
curl --trace-ascii - --trace-time https://example.com 2>&1 | head -20
# 09:15:32.456789 == Info: Trying 93.184.216.34:443...
# 09:15:32.478123 == Info: Connected ...
# 09:15:32.534567 => Send SSL data ...

# 将 api.example.com 强制解析到指定 IP
curl --resolve api.example.com:443:10.0.1.50 \
     https://api.example.com/health

# 同时指定多个域名
curl --resolve api.example.com:443:10.0.1.50 \
     --resolve cdn.example.com:443:10.0.2.100 \
     https://api.example.com/page

# 配合 --connect-to 重定向连接（不改变 Host 和 SNI）
curl --connect-to api.example.com:443:backend-01.internal:443 \
     https://api.example.com/health

# 实际用途：
# 1. 灰度发布时测试特定后端
# 2. CDN 回源问题排查，直连源站
# 3. DNS 切换前验证新服务器
# 4. 多数据中心环境验证特定节点

# 创建 timing 模板文件
cat > curl-timing.txt << 'EOF'
    dns_lookup:  %{time_namelookup}s\n
   tcp_connect:  %{time_connect}s\n
   tls_handshake: %{time_appconnect}s\n
  time_redirect:  %{time_redirect}s\n
time_pretransfer:  %{time_pretransfer}s\n
  starttransfer:  %{time_starttransfer}s\n
     total_time:  %{time_total}s\n
  download_speed:  %{speed_download} bytes/s\n
     http_code:  %{http_code}\n
   remote_addr:  %{remote_ip}:%{remote_port}\n
    ssl_verify:  %{ssl_verify_result}\n
      num_redirects: %{num_redirects}\n
EOF

# 使用 timing 模板
curl -s -o /dev/null -w "@curl-timing.txt" https://example.com

# 输出示例：
#     dns_lookup:  0.004123s        ← DNS 解析耗时
#    tcp_connect:  0.025456s        ← TCP 三次握手完成
#    tls_handshake: 0.078912s       ← TLS 握手完成
#   time_redirect:  0.000000s      ← 重定向耗时
# time_pretransfer:  0.079034s     ← 准备发送请求
#   starttransfer:  0.156789s      ← 收到第一个字节（TTFB）
#      total_time:  0.234567s      ← 请求总耗时

# 计算各阶段耗时：
# DNS 解析:     time_namelookup                     = 4.1ms
# TCP 连接:     time_connect - time_namelookup       = 21.3ms
# TLS 握手:     time_appconnect - time_connect       = 53.5ms
# 服务端处理:   time_starttransfer - time_appconnect = 77.9ms
# 内容传输:     time_total - time_starttransfer      = 77.8ms

# 批量测试脚本 — 多次请求取统计值
#!/bin/bash
URL="${1:-https://example.com}"
N="${2:-10}"

echo "Testing $URL ($N requests)..."
echo "---"

for i in $(seq 1 $N); do
    curl -s -o /dev/null \
        -w "%{time_namelookup} %{time_connect} %{time_appconnect} %{time_starttransfer} %{time_total}" \
        "$URL"
    echo ""
done | awk '{
    dns+=$1; tcp+=$2; tls+=$3; ttfb+=$4; total+=$5; n++
} END {
    printf "Average over %d requests:\n", n
    printf "  DNS:   %.3fms\n", (dns/n)*1000
    printf "  TCP:   %.3fms\n", ((tcp-dns)/n)*1000
    printf "  TLS:   %.3fms\n", ((tls-tcp)/n)*1000
    printf "  TTFB:  %.3fms\n", ((ttfb-tls)/n)*1000
    printf "  Total: %.3fms\n", (total/n)*1000
}'

# 查看服务端证书链
curl -v --head https://example.com 2>&1 | grep -A 20 "Server certificate"

# 指定 TLS 版本
curl --tlsv1.2 --tls-max 1.2 https://example.com  # 强制 TLS 1.2
curl --tlsv1.3 https://example.com                  # 要求 TLS 1.3

# 指定密码套件
curl --ciphers ECDHE-RSA-AES128-GCM-SHA256 https://example.com

# 使用客户端证书（mTLS）
curl --cert client.pem --key client-key.pem \
     --cacert ca.pem \
     https://mtls.example.com/api

# 跳过证书验证（仅调试用！）
curl -k https://self-signed.example.com

# 使用指定 CA 证书
curl --cacert /path/to/custom-ca.pem https://internal.example.com

# 强制使用 HTTP/2
curl --http2 -v https://example.com 2>&1 | grep "ALPN\|HTTP/2"

# 使用 HTTP/1.1（禁用 HTTP/2）
curl --http1.1 https://example.com

# 查看 ALPN 协商结果
curl -v https://example.com 2>&1 | grep "ALPN"
# * ALPN: offers h2,http/1.1
# * ALPN: server accepted h2

# 明文 HTTP/2（h2c，仅限本地调试）
curl --http2-prior-knowledge http://localhost:8080/api

# 安装
pip install mitmproxy

# 三种模式:
mitmproxy    # 终端交互 UI
mitmweb      # Web 界面（推荐初学者）
mitmdump     # 命令行模式（适合脚本集成）

# 启动（默认监听 8080 端口）
mitmproxy -p 8080

# 配置系统代理或应用代理
export http_proxy=http://127.0.0.1:8080
export https_proxy=http://127.0.0.1:8080

# curl 通过 mitmproxy
curl -x http://127.0.0.1:8080 https://api.example.com/health

# 安装 CA 证书（HTTPS 拦截需要）
# 启动 mitmproxy 后访问 http://mitm.it 下载对应平台的证书

# slow_api_detector.py — 检测慢 API 响应
import mitmproxy.http
import time

class SlowAPIDetector:
    def __init__(self):
        self.threshold_ms = 500  # 超过 500ms 视为慢请求

    def request(self, flow: mitmproxy.http.HTTPFlow):
        flow.metadata["start_time"] = time.time()

    def response(self, flow: mitmproxy.http.HTTPFlow):
        start = flow.metadata.get("start_time", 0)
        duration_ms = (time.time() - start) * 1000

        if duration_ms > self.threshold_ms:
            print(f"[SLOW] {duration_ms:.0f}ms "
                  f"{flow.request.method} {flow.request.pretty_url} "
                  f"→ {flow.response.status_code}")

addons = [SlowAPIDetector()]

# response_modifier.py — 修改响应内容（模拟故障）
import mitmproxy.http
import json

class ResponseModifier:
    def response(self, flow: mitmproxy.http.HTTPFlow):
        # 注入延迟头以帮助调试
        flow.response.headers["X-Debug-Via"] = "mitmproxy"

        # 模拟 API 降级：将特定 API 的响应改为错误
        if "/api/payment" in flow.request.pretty_url:
            flow.response.status_code = 503
            flow.response.text = json.dumps({
                "error": "Service temporarily unavailable",
                "retry_after": 30
            })

        # 移除安全头进行测试
        if flow.request.pretty_url.endswith("/test-no-csp"):
            flow.response.headers.pop("Content-Security-Policy", None)

addons = [ResponseModifier()]

# 运行脚本
mitmdump -s slow_api_detector.py
mitmdump -s response_modifier.py

# 同时使用多个脚本
mitmdump -s slow_api_detector.py -s response_modifier.py

# 录制流量到文件
mitmdump -w traffic.flow

# 回放录制的流量
mitmdump -c traffic.flow

# 服务端回放（将录制的响应作为 mock 返回）
mitmdump --server-replay traffic.flow \
    --server-replay-nopop \
    --server-replay-kill-extra

# 用途：
# 1. 录制生产环境流量 → 本地回放调试
# 2. 接口测试：录制正常响应 → 作为 mock server
# 3. 对比分析：录制修改前后的流量差异

# 步骤 1: 发送带追踪 ID 的请求
REQUEST_ID="debug-$(date +%s)-$(( RANDOM % 1000 ))"
curl -v \
    -H "X-Request-ID: $REQUEST_ID" \
    -H "X-Debug: true" \
    https://api.example.com/orders/123

# 步骤 2: 在各层日志中搜索该 ID

# Nginx access log
grep "$REQUEST_ID" /var/log/nginx/access.log

# 应用日志
grep "$REQUEST_ID" /var/log/app/application.log

# 上游服务日志
grep "$REQUEST_ID" /var/log/upstream/service.log

# 步骤 3: 对比各层的时间戳，找到延迟瓶颈
# Nginx:    10:00:00.100 → 10:00:00.500  (400ms, 代理处理)
# App:      10:00:00.110 → 10:00:00.480  (370ms, 应用处理)
# Database: 10:00:00.120 → 10:00:00.450  (330ms, 数据库查询!)
#
# 结论: 延迟主要来自数据库查询

# 检查各层添加的响应头
curl -s -D - -o /dev/null https://api.example.com/orders/123 | \
    grep -iE "^(x-|via|server|age|cf-)"

# 常见的调试响应头：
# X-Request-ID: abc123             ← 请求追踪 ID
# X-Response-Time: 0.234           ← 服务端处理时间
# Via: 1.1 proxy1, 1.1 proxy2     ← 经过的代理链
# X-Cache: HIT from cdn-edge-01   ← CDN 缓存命中
# Age: 3600                       ← 缓存已存在时间
# Server: nginx/1.24.0            ← 服务端软件
# CF-Ray: 8a1234567890abcd-SJC    ← Cloudflare 请求 ID

# 查看证书链完整信息
openssl s_client -connect example.com:443 -showcerts 2>/dev/null | \
    openssl x509 -noout -text | \
    grep -A 2 "Issuer:\|Subject:\|Not After\|DNS:"

# 检查证书过期时间
echo | openssl s_client -connect example.com:443 2>/dev/null | \
    openssl x509 -noout -dates
# notBefore=Jan  1 00:00:00 2025 GMT
# notAfter=Mar 31 23:59:59 2025 GMT

# 验证证书链完整性
openssl s_client -connect example.com:443 \
    -CAfile /etc/ssl/certs/ca-certificates.crt 2>&1 | \
    grep "Verify return code"
# Verify return code: 0 (ok)

# 常见错误码:
# 10 — 证书过期
# 18 — 自签名证书
# 19 — CA 未知（证书链不完整）
# 20 — 无法找到本地颁发者证书
# 21 — 证书链过长

# 测试特定协议和密码套件
openssl s_client -connect example.com:443 \
    -tls1_2 -cipher ECDHE-RSA-AES128-GCM-SHA256

# 跟踪完整的重定向链
curl -v -L https://example.com 2>&1 | grep -E "< HTTP|< location"

# 使用 -w 获取重定向统计
curl -s -o /dev/null -L -w \
    "Redirects: %{num_redirects}\nFinal URL: %{url_effective}\nTotal: %{time_total}s\n" \
    http://example.com

# 最大重定向次数（防止无限循环）
curl -L --max-redirs 5 http://example.com

# 常见重定向问题：
# 1. HTTP → HTTPS 重定向未设置 HSTS
# 2. www → non-www 重定向循环
# 3. Trailing slash 重定向浪费一次 RTT
# 4. 过长的重定向链（>3 次）增加延迟

# 验证缓存头
curl -s -D - -o /dev/null https://example.com/style.css | \
    grep -iE "^(cache-control|etag|last-modified|age|expires|vary):"

# 条件请求验证
ETAG=$(curl -sI https://example.com/style.css | grep -i etag | tr -d '\r')
curl -v -H "If-None-Match: ${ETAG#*: }" https://example.com/style.css
# 期望: HTTP 304 Not Modified

# Last-Modified 条件请求
LM=$(curl -sI https://example.com/style.css | grep -i "last-modified" | cut -d: -f2-)
curl -v -H "If-Modified-Since:$LM" https://example.com/style.css
# 期望: HTTP 304 Not Modified

# 验证 CDN 缓存命中
for i in 1 2 3; do
    echo "--- Request $i ---"
    curl -sI https://cdn.example.com/image.jpg | \
        grep -iE "^(x-cache|age|cf-cache-status):"
    sleep 1
done
# 第 1 次: X-Cache: MISS, Age: 0
# 第 2 次: X-Cache: HIT,  Age: 1
# 第 3 次: X-Cache: HIT,  Age: 2

# 验证 HTTP/2 支持
curl -v --http2 https://example.com 2>&1 | grep "HTTP/2"

# 使用 nghttp 查看帧级别信息
nghttp -v https://example.com 2>&1 | head -30
# [  0.023] Connected
# [  0.045] recv SETTINGS frame
# [  0.045] recv WINDOW_UPDATE frame
# [  0.045] send HEADERS frame
# [  0.067] recv HEADERS frame
# [  0.067] recv DATA frame

# 查看 HTTP/2 连接信息
curl -v --http2 https://example.com 2>&1 | \
    grep -E "ALPN|HTTP/2|stream"

# 检测服务端是否支持 Server Push（已基本废弃）
nghttp -v https://example.com 2>&1 | grep "PUSH_PROMISE"