Kubernetes 网络可观测性：从 tcpdump 到 Hubble 到 Pixie

# 找到容器 PID
PID=$(crictl inspect --output go-template \
  --template '{{.info.pid}}' <CONTAINER_ID>)

# 进入 netns 抓包
nsenter -t $PID -n tcpdump -i eth0 -nn -w capture.pcap

# BPF 过滤表达式缩小范围
nsenter -t $PID -n tcpdump -i eth0 -nn \
  'tcp port 8080 and (tcp[tcpflags] & (tcp-syn|tcp-rst) != 0)'

kubectl debug -it my-app-pod \
  --image=nicolaka/netshoot \
  --target=my-app \
  -- tcpdump -i eth0 -nn -c 100

# 解码 HTTP 请求/响应
nsenter -t $PID -n tshark -i eth0 -Y 'http' \
  -T fields -e http.request.method -e http.request.uri \
  -e http.response.code -e http.time

# 解码 DNS 查询
nsenter -t $PID -n tshark -i eth0 -Y 'dns' \
  -T fields -e dns.qry.name -e dns.resp.addr -e dns.time

# TCP 连接（含进程信息）
nsenter -t $PID -n ss -tnpi

# TIME_WAIT 数量（端口耗尽前兆）
nsenter -t $PID -n ss -tan state time-wait | wc -l

# socket 内存和拥塞窗口
nsenter -t $PID -n ss -tnmi
# 输出：cubic wscale:7,7 rto:204 rtt:1.5/0.75 cwnd:10

# 按目的地址汇总连接数
nsenter -t $PID -n ss -tn | awk '{print $5}' | \
  cut -d: -f1 | sort | uniq -c | sort -rn | head -20

# 按状态过滤
conntrack -L -p tcp --state ESTABLISHED

# 表容量和使用量
sysctl net.netfilter.nf_conntrack_count
sysctl net.netfilter.nf_conntrack_max

# 实时监听事件
conntrack -E

# TCP 重传相关指标
nstat -az | grep -i retrans
# TcpRetransSegs     142    0.0
# TcpTimeouts        23     0.0

# 每秒刷新观察趋势
nstat -az -d 1 | grep TcpRetransSegs

# NAT 表 Service 转发规则
iptables -t nat -L KUBE-SERVICES -v -n --line-numbers

# 某个 Service 的后端选择
iptables -t nat -L KUBE-SVC-XXXXXXXXXXXXXXXX -v -n

# filter 表 NetworkPolicy 规则
iptables -t filter -L KUBE-NWPLCY-XXXXXXXX -v -n

# 找 DROP 规则及命中计数
iptables -t filter -L -v -n | grep -i drop

# 1. 确认 Service ClusterIP
kubectl get svc my-service -o wide

# 2. 追踪 iptables 链
iptables -t nat -L KUBE-SERVICES -v -n | grep "10.96.100.200"

# 3. 检查后端 Endpoint（pkts 列为 0 说明流量未到达）
iptables -t nat -L KUBE-SVC-XXXXXXX -v -n

# 4. conntrack 确认 DNAT
conntrack -L -d 10.96.100.200

# 5. Pod 内抓包
nsenter -t $PID -n tcpdump -i eth0 -nn 'host 10.96.100.200'

cilium install --set hubble.enabled=true \
  --set hubble.relay.enabled=true \
  --set hubble.ui.enabled=true \
  --set hubble.metrics.enableOpenMetrics=true

# 验证状态
cilium hubble port-forward &
hubble status

# 按 namespace 过滤
hubble observe --namespace default

# 只看被丢弃的流量
hubble observe --verdict DROPPED

# 组合过滤：某 Pod 的 HTTP 丢弃流量
hubble observe --pod default/my-app \
  --protocol http --verdict DROPPED

# JSON 输出便于后续处理
hubble observe --output json | jq '.flow.source.namespace'

# DNS 查询与解析失败
hubble observe --protocol dns --verdict DROPPED

# 策略判定详情
hubble observe --verdict DROPPED --output json | \
  jq '{src: .flow.source.labels, dst: .flow.destination.labels,
       policy: .flow.drop_reason_desc}'

# 全集群观察
hubble observe --server localhost:4245

# Hubble UI
kubectl port-forward -n kube-system svc/hubble-ui 12000:80

hubble:
  metrics:
    enabled:
      - dns
      - drop
      - tcp
      - flow
      - httpV2:exemplars=true;labelsContext=source_ip,source_namespace,destination_ip,destination_namespace
    serviceMonitor:
      enabled: true

helm install tetragon cilium/tetragon -n kube-system

# 查看进程级事件
kubectl exec -n kube-system ds/tetragon -c tetragon -- \
  tetra getevents -o compact
# 输出：
# connect  default/my-app /usr/bin/curl  -> 10.96.0.1:443
# sendmsg  default/my-app /usr/bin/python -> 10.96.100.200:8080
# accept   default/backend /usr/bin/node  <- 10.244.1.5:42316

apiVersion: cilium.io/v1alpha1
kind: TracingPolicy
metadata:
  name: tcp-connect-monitor
spec:
  kprobes:
    - call: "tcp_v4_connect"
      syscall: false
      args:
        - index: 0
          type: "sock"
      selectors:
        - matchNamespaces:
            - namespace: default
              operator: In
              values: ["default", "production"]
      return: true
      returnArg:
        index: 0
        type: "int"

bpftrace -e '
tracepoint:tcp:tcp_retransmit_skb {
    printf("retransmit: %s:%d -> %s:%d state=%d\n",
        ntop(args->saddr), args->sport,
        ntop(args->daddr), args->dport,
        args->state);
}'

bpftrace -e '
kprobe:nf_conntrack_hash_check_insert {
    printf("conntrack insert: comm=%s pid=%d\n", comm, pid);
}
kprobe:nf_ct_delete {
    printf("conntrack delete: comm=%s pid=%d\n", comm, pid);
}'

bpftrace -e '
kprobe:tcp_v4_connect {
    @start[tid] = nsecs;
}
kretprobe:tcp_v4_connect /@start[tid]/ {
    $dur = (nsecs - @start[tid]) / 1000;
    printf("connect latency: %d us, comm=%s\n", $dur, comm);
    @latency = hist($dur);
    delete(@start[tid]);
}'

# 安装与部署
bash -c "$(curl -fsSL https://withpixie.ai/install.sh)"
px deploy
px get viziers

# 查看 HTTP 错误
import px

df = px.DataFrame(table='http_events', start_time='-5m')
df = df[df.ctx['namespace'] == 'default']
df.resp_status = df.resp_status.astype('int64')
df = df[['time_', 'source', 'destination', 'req_method',
         'req_path', 'resp_status', 'latency']]
df = df[df.resp_status >= 400]
px.display(df, 'HTTP Errors')

# 按 Service 聚合延迟 P99
import px

df = px.DataFrame(table='http_events', start_time='-10m')
df.service = df.ctx['service']
df = df.groupby('service').agg(
    count=('latency', px.count),
    p50=('latency', px.quantiles, 0.5),
    p99=('latency', px.quantiles, 0.99),
    error_rate=('resp_status', lambda x: px.mean(x >= 400)),
)
px.display(df, 'Service Latency Summary')

kube_pod_status_phase{phase="Running"}
kube_endpoint_address_available
kube_endpoint_address_not_ready
kube_networkpolicy_labels

groups:
  - name: kubernetes-network
    rules:
      - alert: ConntrackTableNearlyFull
        expr: >
          node_nf_conntrack_entries / node_nf_conntrack_entries_limit > 0.8
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "conntrack 表使用率超过 80%"

      - alert: HighTCPRetransmissions
        expr: >
          rate(node_netstat_Tcp_RetransSegs[5m]) /
          rate(node_netstat_Tcp_OutSegs[5m]) > 0.01
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "TCP 重传率超过 1%"

      - alert: HubbleHighDropRate
        expr: rate(hubble_drop_total[5m]) > 100
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "Hubble 检测到高丢包率"
          description: "丢包原因：{{ $labels.reason }}"

      - alert: HighDNSFailureRate
        expr: >
          rate(hubble_dns_responses_total{rcode!="No Error"}[5m]) /
          rate(hubble_dns_responses_total[5m]) > 0.05
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "DNS 解析失败率超过 5%"

apiVersion: v1
kind: ConfigMap
metadata:
  name: tempo-config
  namespace: monitoring
data:
  tempo.yaml: |
    server:
      http_listen_port: 3200
    distributor:
      receivers:
        otlp:
          protocols:
            grpc:
              endpoint: "0.0.0.0:4317"
    storage:
      trace:
        backend: local
        local:
          path: /var/tempo/traces
    metrics_generator:
      processor:
        service_graphs:
          dimensions: [k8s.namespace.name, k8s.pod.name]

# Trace 中找到慢请求的 Pod，再查 Hubble 对应的网络流
hubble observe --pod default/order-svc-xxx \
  --since "2026-04-03T10:15:00Z" \
  --until "2026-04-03T10:15:05Z"

hubble observe --namespace default --verdict DROPPED --last 100

# 汇总丢包原因
hubble observe --namespace default --verdict DROPPED -o json | \
  jq -r '.flow.drop_reason_desc' | sort | uniq -c | sort -rn

# Tetragon 查看异常连接的进程
kubectl exec -n kube-system ds/tetragon -c tetragon -- \
  tetra getevents --namespace default --process-name my-app

# Pixie 查看应用层协议
px run px/http_data -- --start_time=-5m --namespace=default

PID=$(crictl inspect --output go-template \
  --template '{{.info.pid}}' <CONTAINER_ID>)

nsenter -t $PID -n tcpdump -i eth0 -nn -w debug.pcap 'tcp port 8080'

# tshark 分析重传
tshark -r debug.pcap -Y 'tcp.analysis.retransmission' \
  -T fields -e frame.time -e ip.src -e ip.dst -e tcp.analysis.rto

# Cilium + Hubble
cilium install \
  --set hubble.enabled=true \
  --set hubble.relay.enabled=true \
  --set hubble.ui.enabled=true \
  --set hubble.metrics.enableOpenMetrics=true \
  --set hubble.metrics.enabled="{dns,drop,tcp,flow,httpV2:exemplars=true;labelsContext=source_ip,source_namespace,destination_ip,destination_namespace}"

cilium status --wait

# Prometheus + Grafana
helm repo add prometheus-community \
  https://prometheus-community.github.io/helm-charts
helm install kube-prom prometheus-community/kube-prometheus-stack \
  -n monitoring --create-namespace \
  --set grafana.enabled=true \
  --set prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues=false

apiVersion: apps/v1
kind: Deployment
metadata:
  name: http-server
spec:
  replicas: 2
  selector:
    matchLabels:
      app: http-server
  template:
    metadata:
      labels:
        app: http-server
    spec:
      containers:
        - name: server
          image: nginx:alpine
          ports:
            - containerPort: 80
---
apiVersion: v1
kind: Service
metadata:
  name: http-server
spec:
  selector:
    app: http-server
  ports:
    - port: 80
      targetPort: 80
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: http-client
spec:
  replicas: 1
  selector:
    matchLabels:
      app: http-client
  template:
    metadata:
      labels:
        app: http-client
    spec:
      containers:
        - name: client
          image: curlimages/curl:latest
          command: ["/bin/sh", "-c"]
          args:
            - |
              while true; do
                curl -s -o /dev/null -w "%{http_code} %{time_total}s\n" \
                  http://http-server/
                sleep 1
              done

# deny-policy.yaml -- http-client 标签不匹配，流量被拒绝
apiVersion: cilium.io/v2
kind: CiliumNetworkPolicy
metadata:
  name: restrict-http-server
spec:
  endpointSelector:
    matchLabels:
      app: http-server
  ingress:
    - fromEndpoints:
        - matchLabels:
            app: http-client-authorized
      toPorts:
        - ports:
            - port: "80"
              protocol: TCP

# 应用策略
kubectl apply -f deny-policy.yaml

# 排查步骤一：Hubble 观察丢包
hubble observe --namespace default --verdict DROPPED
# 输出：http-client -> http-server DROPPED Policy denied

# 排查步骤二：查看详细原因
hubble observe --namespace default --verdict DROPPED -o json | \
  jq '{src: .flow.source.labels, dst: .flow.destination.labels,
       reason: .flow.drop_reason_desc}'

# 排查步骤三：确认策略内容
kubectl describe cnp restrict-http-server

# 排查步骤四：修复
kubectl label pod -l app=http-client \
  app=http-client-authorized --overwrite

# 排查步骤五：验证
hubble observe --namespace default --last 10
# 状态变为 FORWARDED

kubectl port-forward -n monitoring svc/kube-prom-grafana 3000:80
# 导入 Hubble Dashboard（ID: 16611）

name: Network Policy Drops
condition: rate(hubble_drop_total{reason="POLICY_DENIED"}[5m]) > 10
for: 2m
annotations:
  summary: "NetworkPolicy 拒绝了大量流量"
  runbook: "hubble observe --verdict DROPPED 查看详情"

hubble:
  eventBufferCapacity: 4096
  metrics:
    enabled:
      - dns
      - drop
      - "tcp:sourceContext=pod;destinationContext=pod"
      - "httpV2:exemplars=true;sourceContext=pod;destinationContext=pod"

# 查看 eBPF 程序 CPU 使用
bpftool prog show

# 监控 Hubble 资源
kubectl top pod -n kube-system -l k8s-app=hubble-relay