【网络工程】网络延迟分析方法论：从现象到瓶颈定位

curl -w "\
    DNS:        %{time_namelookup}s\n\
    TCP:        %{time_connect}s\n\
    TLS:        %{time_appconnect}s\n\
    TTFB:       %{time_starttransfer}s\n\
    Total:      %{time_total}s\n\
    Size:       %{size_download} bytes\n\
    Speed:      %{speed_download} bytes/s\n" \
  -o /dev/null -s https://example.com/api/data

#!/bin/bash
# latency-breakdown.sh — 批量延迟分解测量
URL="${1:?Usage: $0 <url> [count]}"
COUNT="${2:-20}"

TMPFILE=$(mktemp)
FORMAT='%{time_namelookup} %{time_connect} %{time_appconnect} %{time_starttransfer} %{time_total}\n'

for i in $(seq 1 "$COUNT"); do
    curl -w "$FORMAT" -o /dev/null -s "$URL" >> "$TMPFILE"
    sleep 0.5
done

echo "=== Latency Breakdown (${COUNT} samples) ==="
echo "Phase          P50      P95      P99      Max"
echo "-----          ---      ---      ---      ---"

awk '{
    dns[NR]   = $1 * 1000;
    tcp[NR]   = ($2 - $1) * 1000;
    tls[NR]   = ($3 - $2) * 1000;
    server[NR]= ($4 - $3) * 1000;
    xfer[NR]  = ($5 - $4) * 1000;
    total[NR] = $5 * 1000;
    n = NR;
}
function percentile(arr, n, p,    k, sorted) {
    for (i=1; i<=n; i++) sorted[i] = arr[i];
    asort(sorted);
    k = int(n * p / 100 + 0.5);
    if (k < 1) k = 1;
    return sorted[k];
}
END {
    printf "DNS          %7.1f  %7.1f  %7.1f  %7.1f ms\n", \
        percentile(dns,n,50), percentile(dns,n,95), percentile(dns,n,99), percentile(dns,n,100);
    printf "TCP          %7.1f  %7.1f  %7.1f  %7.1f ms\n", \
        percentile(tcp,n,50), percentile(tcp,n,95), percentile(tcp,n,99), percentile(tcp,n,100);
    printf "TLS          %7.1f  %7.1f  %7.1f  %7.1f ms\n", \
        percentile(tls,n,50), percentile(tls,n,95), percentile(tls,n,99), percentile(tls,n,100);
    printf "Server       %7.1f  %7.1f  %7.1f  %7.1f ms\n", \
        percentile(server,n,50), percentile(server,n,95), percentile(server,n,99), percentile(server,n,100);
    printf "Transfer     %7.1f  %7.1f  %7.1f  %7.1f ms\n", \
        percentile(xfer,n,50), percentile(xfer,n,95), percentile(xfer,n,99), percentile(xfer,n,100);
    printf "Total        %7.1f  %7.1f  %7.1f  %7.1f ms\n", \
        percentile(total,n,50), percentile(total,n,95), percentile(total,n,99), percentile(total,n,100);
}' "$TMPFILE"

rm -f "$TMPFILE"

chmod +x latency-breakdown.sh
./latency-breakdown.sh https://api.example.com/health 50

tshark -r capture.pcap \
  -Y "tcp.flags.syn==1 && tcp.flags.ack==1" \
  -T fields \
  -e frame.time_relative \
  -e ip.src \
  -e tcp.analysis.ack_rtt \
  | sort -t$'\t' -k3 -n

# 重传统计
tshark -r capture.pcap -Y "tcp.analysis.retransmission" \
  -T fields -e frame.time_relative -e ip.dst -e tcp.stream \
  | awk '{streams[$3]++} END {
    for (s in streams) printf "Stream %s: %d retransmissions\n", s, streams[s]
  }' | sort -t: -k2 -rn | head -10

# 增大 TCP 接收缓冲区
sysctl -w net.ipv4.tcp_rmem="4096 1048576 16777216"
sysctl -w net.core.rmem_max=16777216

# 确保窗口缩放已启用（默认启用）
sysctl net.ipv4.tcp_window_scaling

# ICMP traceroute（默认在 Windows，Linux 需要 -I）
traceroute -I example.com

# UDP traceroute（Linux 默认）
traceroute example.com

# TCP traceroute（穿透防火墙能力最强）
traceroute -T -p 443 example.com

# 基本用法
mtr -rw -c 100 example.com

# TCP 模式（推荐，穿透防火墙）
mtr -rw -c 100 -T -P 443 example.com

# 报告模式（非交互，适合脚本）
mtr -rw -c 200 --json example.com > mtr-report.json

# TCP SYN ping（最准确的 RTT 测量）
hping3 -S -p 443 -c 20 example.com

# 指定间隔和数据大小
hping3 -S -p 443 -c 100 -i u100000 example.com    # 100ms 间隔

# 测量到不同端口的延迟（检测端口级别的路由差异）
hping3 -S -p 80 -c 10 example.com
hping3 -S -p 443 -c 10 example.com
hping3 -S -p 8080 -c 10 example.com

#!/bin/bash
# path-monitor.sh — 路径变化检测
TARGET="${1:?Usage: $0 <target>}"
PREV_PATH=""

while true; do
    CURRENT_PATH=$(traceroute -n -q 1 -w 2 "$TARGET" 2>/dev/null \
      | awk '{print $2}' | grep -v '*' | tr '\n' ' ')

    if [ -n "$PREV_PATH" ] && [ "$CURRENT_PATH" != "$PREV_PATH" ]; then
        echo "[$(date '+%Y-%m-%d %H:%M:%S')] Path changed!"
        echo "  Old: $PREV_PATH"
        echo "  New: $CURRENT_PATH"
    fi

    PREV_PATH="$CURRENT_PATH"
    sleep 60
done

# 精确测量 DNS 解析时间
dig +stats example.com | grep "Query time"
;; Query time: 23 msec

# 测试不同 DNS 服务器的响应时间
for dns in 8.8.8.8 1.1.1.1 223.5.5.5 119.29.29.29; do
    time=$(dig @$dns +stats example.com 2>/dev/null | grep "Query time" | awk '{print $4}')
    echo "$dns: ${time}ms"
done

# 查看本地 DNS 缓存（systemd-resolved）
resolvectl statistics

# 强制不使用缓存查询（测量完整解析链路延迟）
dig +trace +stats example.com

# 查看每一级解析的延迟
dig +trace example.com 2>&1 | grep -E "(Query time|Received)"

# 检查是否有 AAAA 超时
dig AAAA example.com +time=2

# 如果 IPv6 不可用，在 /etc/gai.conf 中禁用
echo "precedence ::ffff:0:0/96 100" >> /etc/gai.conf

# 查看当前 search domain
cat /etc/resolv.conf | grep search

# 高并发服务中，设置 ndots:1 减少无效查询
# /etc/resolv.conf
options ndots:1

# 测量完整 TLS 握手
openssl s_client -connect example.com:443 -servername example.com \
  < /dev/null 2>&1 | grep -E "(Protocol|Cipher|Verify)"

# 用 curl 精确分解
curl -w "TCP: %{time_connect}s\nTLS: %{time_appconnect}s\nDelta: $(echo '%{time_appconnect} - %{time_connect}' | bc)s\n" \
  -o /dev/null -s https://example.com

#!/bin/bash
# tls-latency-check.sh — TLS 延迟诊断
HOST="${1:?Usage: $0 <hostname>}"

echo "=== TLS Version ==="
echo | openssl s_client -connect "$HOST:443" -servername "$HOST" 2>/dev/null \
  | grep "Protocol  :"

echo ""
echo "=== Certificate Chain Length ==="
echo | openssl s_client -connect "$HOST:443" -servername "$HOST" -showcerts 2>/dev/null \
  | grep -c "BEGIN CERTIFICATE"

echo ""
echo "=== OCSP Stapling ==="
echo | openssl s_client -connect "$HOST:443" -servername "$HOST" -status 2>/dev/null \
  | grep -A 2 "OCSP Response Status"

echo ""
echo "=== Session Resumption ==="
# 第一次连接，保存 session
openssl s_client -connect "$HOST:443" -servername "$HOST" \
  -sess_out /tmp/tls_session 2>/dev/null < /dev/null > /dev/null

# 第二次连接，尝试恢复
REUSED=$(openssl s_client -connect "$HOST:443" -servername "$HOST" \
  -sess_in /tmp/tls_session 2>/dev/null < /dev/null | grep "Reused")

if [ -n "$REUSED" ]; then
    echo "Session resumption: SUPPORTED"
else
    echo "Session resumption: NOT SUPPORTED"
fi

rm -f /tmp/tls_session

tshark -r capture.pcap \
  -Y "http.request || http.response" \
  -T fields \
  -e frame.time_relative \
  -e http.request.method \
  -e http.request.uri \
  -e http.response.code \
  -e http.time

package middleware

import (
    "context"
    "fmt"
    "net/http"
    "strings"
    "sync"
    "time"
)

type spanKey struct{}

type Span struct {
    Name     string
    Start    time.Time
    Duration time.Duration
    children []*Span
    mu       sync.Mutex
}

func NewSpan(name string) *Span {
    return &Span{Name: name, Start: time.Now()}
}

func (s *Span) End() {
    s.Duration = time.Since(s.Start)
}

func (s *Span) Child(name string) *Span {
    child := NewSpan(name)
    s.mu.Lock()
    s.children = append(s.children, child)
    s.mu.Unlock()
    return child
}

func (s *Span) String() string {
    var b strings.Builder
    s.format(&b, 0)
    return b.String()
}

func (s *Span) format(b *strings.Builder, depth int) {
    indent := strings.Repeat("  ", depth)
    fmt.Fprintf(b, "%s%s: %v\n", indent, s.Name, s.Duration)
    for _, child := range s.children {
        child.format(b, depth+1)
    }
}

func LatencyTrace(next http.Handler) http.Handler {
    return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
        root := NewSpan("request")
        ctx := context.WithValue(r.Context(), spanKey{}, root)
        next.ServeHTTP(w, r.WithContext(ctx))
        root.End()

        if root.Duration > 200*time.Millisecond {
            fmt.Printf("[SLOW REQUEST] %s %s\n%s",
                r.Method, r.URL.Path, root.String())
        }
    })
}

func SpanFromContext(ctx context.Context) *Span {
    if s, ok := ctx.Value(spanKey{}).(*Span); ok {
        return s
    }
    return NewSpan("unknown")
}

func TraceDB(ctx context.Context, query string) func() {
    span := SpanFromContext(ctx).Child("db: " + query)
    return func() { span.End() }
}

func handleGetUser(w http.ResponseWriter, r *http.Request) {
    // 数据库查询
    done := middleware.TraceDB(r.Context(), "SELECT * FROM users")
    user, err := db.QueryUser(r.Context(), userID)
    done()

    // 外部 API 调用
    span := middleware.SpanFromContext(r.Context()).Child("external-api")
    profile, err := fetchProfile(r.Context(), user.ID)
    span.End()

    // 序列化
    span = middleware.SpanFromContext(r.Context()).Child("json-marshal")
    data, _ := json.Marshal(response)
    span.End()

    w.Write(data)
}

#!/usr/bin/env bpftrace
// net-latency.bt — 追踪 TCP 收包到应用读取的延迟

kprobe:tcp_v4_rcv
{
    @recv_time[arg0] = nsecs;
}

kretprobe:tcp_recvmsg
/@ recv_time[arg0]/
{
    $latency_us = (nsecs - @recv_time[arg0]) / 1000;
    @latency_hist = hist($latency_us);

    if ($latency_us > 1000) {
        printf("HIGH LATENCY: %d us, pid=%d comm=%s\n",
            $latency_us, pid, comm);
    }

    delete(@recv_time[arg0]);
}

END
{
    print(@latency_hist);
}

# 查看 softirq 处理时间分布
sudo bpftrace -e '
tracepoint:irq:softirq_entry /args->vec == 3/ { @start[tid] = nsecs; }
tracepoint:irq:softirq_exit /args->vec == 3 && @start[tid]/ {
    @us = hist((nsecs - @start[tid]) / 1000);
    delete(@start[tid]);
}'

# 查看网卡中断分布
cat /proc/interrupts | grep eth0

# 查看 RPS 配置
for f in /sys/class/net/eth0/queues/rx-*/rps_cpus; do
    echo "$f: $(cat $f)"
done

# 查看 softirq 在各 CPU 上的分布
awk '/NET_RX/ {for(i=2;i<=NF;i++) printf "CPU%d: %s  ", i-2, $i; print ""}' \
  /proc/softirqs

# 为 eth0 的 rx-0 队列启用 RPS，分散到 CPU 0-7
echo "ff" > /sys/class/net/eth0/queues/rx-0/rps_cpus

# 设置中断亲和（将 eth0 的 IRQ 绑定到 CPU 2）
IRQ=$(grep eth0 /proc/interrupts | awk '{print $1}' | tr -d ':')
echo 4 > /proc/irq/$IRQ/smp_affinity   # 0x4 = CPU 2

package metrics

import (
    "github.com/prometheus/client_golang/prometheus"
)

var (
    HTTPRequestDuration = prometheus.NewHistogramVec(
        prometheus.HistogramOpts{
            Name:    "http_request_duration_seconds",
            Help:    "HTTP request duration in seconds",
            Buckets: []float64{
                0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10,
            },
        },
        []string{"method", "path", "status"},
    )

    DNSResolveDuration = prometheus.NewHistogram(
        prometheus.HistogramOpts{
            Name:    "dns_resolve_duration_seconds",
            Help:    "DNS resolution duration in seconds",
            Buckets: []float64{0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 5},
        },
    )

    TCPConnectDuration = prometheus.NewHistogramVec(
        prometheus.HistogramOpts{
            Name:    "tcp_connect_duration_seconds",
            Help:    "TCP connection establishment duration",
            Buckets: []float64{0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.5, 1},
        },
        []string{"remote_addr"},
    )

    TLSHandshakeDuration = prometheus.NewHistogram(
        prometheus.HistogramOpts{
            Name:    "tls_handshake_duration_seconds",
            Help:    "TLS handshake duration in seconds",
            Buckets: []float64{0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1},
        },
    )
)

func init() {
    prometheus.MustRegister(
        HTTPRequestDuration,
        DNSResolveDuration,
        TCPConnectDuration,
        TLSHandshakeDuration,
    )
}

# blackbox.yml
modules:
  http_2xx:
    prober: http
    timeout: 15s
    http:
      valid_http_versions: ["HTTP/1.1", "HTTP/2.0"]
      valid_status_codes: [200]
      method: GET
      preferred_ip_protocol: ip4
      ip_protocol_fallback: false
  
  tcp_connect:
    prober: tcp
    timeout: 10s
    tcp:
      preferred_ip_protocol: ip4
  
  icmp_rtt:
    prober: icmp
    timeout: 5s
    icmp:
      preferred_ip_protocol: ip4

# prometheus.yml
scrape_configs:
  - job_name: 'blackbox-http'
    metrics_path: /probe
    params:
      module: [http_2xx]
    static_configs:
      - targets:
        - https://api.example.com/health
        - https://web.example.com
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: blackbox-exporter:9115

# alerting-rules.yml
groups:
  - name: latency_alerts
    rules:
      - alert: HighP99Latency
        expr: |
          histogram_quantile(0.99,
            rate(http_request_duration_seconds_bucket{path!~"/health.*"}[5m])
          ) > 2
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "P99 延迟超过 2 秒"
          description: "{{ $labels.path }} 的 P99 延迟为 {{ $value | humanizeDuration }}"

      - alert: DNSResolutionSlow
        expr: probe_dns_lookup_time_seconds > 0.5
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "DNS 解析延迟超过 500ms"
          description: "目标 {{ $labels.instance }} 的 DNS 解析耗时 {{ $value | humanizeDuration }}"

      - alert: TLSHandshakeSlow
        expr: |
          probe_http_duration_seconds{phase="tls"} > 1
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "TLS 握手延迟超过 1 秒"

# 1. curl 延迟分解
curl -w "DNS:%{time_namelookup} TCP:%{time_connect} TLS:%{time_appconnect} TTFB:%{time_starttransfer} Total:%{time_total}\n" \
  -o /dev/null -s https://api.example.com/data
# DNS:0.002 TCP:0.015 TLS:0.048 TTFB:0.065 Total:0.072
# → 正常请求没问题

# 2. 多次采样发现偶发慢请求
for i in $(seq 1 50); do
    curl -w "%{time_starttransfer}\n" -o /dev/null -s https://api.example.com/data
done | sort -n | tail -5
# 0.065
# 0.071
# 0.068
# 1.856   ← 偶发
# 2.134   ← 偶发

# 3. 慢请求的 TTFB 高 → 瓶颈在服务端
# 4. 检查服务端日志
grep "duration_ms.*[0-9]\{4,\}" /var/log/app/access.log | tail -20
# → 发现慢请求都在查询同一个数据库表

# 5. 检查数据库
mysql -e "SHOW PROCESSLIST" | grep -v Sleep
# → 发现大量 Waiting for table metadata lock

# 6. 根因：一个长事务持有表锁
mysql -e "SELECT * FROM information_schema.INNODB_TRX ORDER BY trx_started"

# 1. mtr 路径分析
mtr -rw -c 50 -T -P 443 bj-server.example.com
# HOST                        Loss%  Snt  Last   Avg   Best  Wrst  StDev
# 1. gw-gz.isp.net             0.0%  50   0.5    0.6   0.3   1.2   0.2
# 2. core-gz.isp.net           0.0%  50   2.1    2.3   1.8   4.5   0.5
# 3. backbone-gz.isp.net       0.0%  50   5.2    5.5   4.8   8.1   0.7
# 4. peer-sh.isp.net           0.0%  50  45.3   46.1  44.2  52.3   1.8  ← 广州→上海
# 5. backbone-sh.isp.net       0.0%  50  46.8   47.2  45.1  53.6   1.9
# 6. peer-us.transit.net       0.0%  50 185.2  186.8 183.1 195.4   2.8  ← 上海→美国!
# 7. peer-us2.transit.net      0.0%  50 186.1  187.5 184.2 196.8   2.9
# 8. peer-bj.isp.net           0.0%  50 278.3  280.1 275.4 295.2   4.2  ← 美国→北京
# 9. bj-server.example.com     0.0%  50 279.8  281.5 276.8 297.1   4.5

# 2. 发现流量从上海绕道美国再回北京!
# 这是 BGP 路由问题——ISP 的对等互联策略导致的

# 3. 验证：直接 traceroute 看路径
traceroute -T -p 443 bj-server.example.com

# 1. curl 分解
curl -w "TCP:%{time_connect} TLS:%{time_appconnect}\n" \
  -o /dev/null -s https://slow.example.com
# TCP:0.015 TLS:2.134
# TLS 握手耗时 2.119 秒！

# 2. openssl 诊断
echo | openssl s_client -connect slow.example.com:443 \
  -servername slow.example.com -status 2>&1 | head -30
# Protocol: TLSv1.2         ← 还在用 TLS 1.2
# OCSP Response Status: ...没有响应...  ← 没有 OCSP Stapling

# 3. 检查证书链
echo | openssl s_client -connect slow.example.com:443 \
  -servername slow.example.com -showcerts 2>/dev/null \
  | grep -c "BEGIN CERTIFICATE"
# 5                          ← 证书链有 5 级，过长

# 4. 检查 OCSP 响应
openssl s_client -connect slow.example.com:443 \
  -servername slow.example.com -status 2>&1 | grep "OCSP"
# OCSP response: no response sent  ← 客户端需要自行查询 OCSP