【Linux 网络子系统深度拆解】网络丢包定位：从 drop_monitor 到 kfree_skb 追踪

/* include/trace/events/skb.h:24 */
TRACE_EVENT(kfree_skb,
    TP_PROTO(struct sk_buff *skb, void *location,
             enum skb_drop_reason reason),
    TP_ARGS(skb, location, reason),
    TP_STRUCT__entry(
        __field(void *,             skbaddr)
        __field(void *,             location)    /* 丢包的内核函数地址 */
        __field(unsigned short,     protocol)    /* 协议号（ETH_P_IP 等） */
        __field(enum skb_drop_reason, reason)    /* 丢弃原因枚举 */
    ),
    ...
);

/* include/trace/events/skb.h:54 */
TRACE_EVENT(consume_skb,
    TP_PROTO(struct sk_buff *skb, void *location),
    TP_ARGS(skb, location),
    TP_STRUCT__entry(
        __field(void *, skbaddr)
        __field(void *, location)
    ),
    ...
);

/* include/net/dropreason.h */
enum skb_drop_reason_subsys {
    SKB_DROP_REASON_SUBSYS_CORE,              /* 核心丢包原因 */
    SKB_DROP_REASON_SUBSYS_MAC80211_UNUSABLE, /* Wi-Fi 不可用帧 */
    SKB_DROP_REASON_SUBSYS_MAC80211_MONITOR,  /* Wi-Fi 监控帧 */
    SKB_DROP_REASON_SUBSYS_OPENVSWITCH,       /* Open vSwitch */
    SKB_DROP_REASON_SUBSYS_NUM,
};

void drop_reasons_register_subsys(enum skb_drop_reason_subsys subsys,
                                  const struct drop_reason_list *list);
void drop_reasons_unregister_subsys(enum skb_drop_reason_subsys subsys);

# 安装
apt install dropwatch    # Debian/Ubuntu
yum install dropwatch    # RHEL/CentOS

# 启动交互模式
dropwatch -l kas

# 在交互模式中
> start
# 输出示例：
# Initalizing kallsyms db
# 1 drops at tcp_v4_rcv+0x3c (sobject: netfilter)
# 5 drops at __udp4_lib_rcv+0x2a1 (sobject: NO_SOCKET)
# 2 drops at nf_hook_slow+0x4b (sobject: netfilter)
> stop
> exit

# 列出网卡支持的 trap 类型
devlink trap list pci/0000:03:00.0

# 启用特定 trap
devlink trap set pci/0000:03:00.0 trap source_mac_is_multicast action trap

# 查看 trap 统计
devlink trap show pci/0000:03:00.0

bpftrace -e '
tracepoint:skb:kfree_skb {
    @by_reason[args->reason] = count();
    @by_location[ksym(args->location)] = count();
}
interval:s:10 {
    printf("\n--- Drop reasons ---\n");
    print(@by_reason);
    printf("\n--- Drop locations ---\n");
    print(@by_location);
    clear(@by_reason);
    clear(@by_location);
}
'

bpftrace -e '
#include <linux/skbuff.h>
tracepoint:skb:kfree_skb {
    $skb = (struct sk_buff *)args->skbaddr;
    if ($skb->dev != 0) {
        $dev = $skb->dev;
        @by_dev[str($dev->name)] = count();
    }
    @by_reason[args->reason] = count();
}
'

# 追踪 SOCKET_RCVBUFF 丢包的调用栈
bpftrace -e '
tracepoint:skb:kfree_skb
/args->reason == 14/    /* SKB_DROP_REASON_SOCKET_RCVBUFF */
{
    printf("SOCKET_RCVBUFF drop at %s protocol=0x%x\n",
        ksym(args->location), args->protocol);
    print(kstack);
}
'

bpftrace -e '
#include <linux/skbuff.h>
#include <linux/ip.h>
tracepoint:skb:kfree_skb
/args->reason == 6/    /* SKB_DROP_REASON_NETFILTER_DROP */
{
    $skb = (struct sk_buff *)args->skbaddr;
    $iph = (struct iphdr *)($skb->head + $skb->network_header);
    printf("NF_DROP: %s -> %s proto=%d\n",
        ntop(AF_INET, &$iph->saddr),
        ntop(AF_INET, &$iph->daddr),
        $iph->protocol);
}
'

# 记录 60 秒的丢包事件及调用栈
perf record -a -g --call-graph dwarf \
    -e 'skb:kfree_skb' -- sleep 60

# 查看丢包栈
perf script | head -200

# 按丢包函数统计
perf report --sort comm,dso,symbol

# 按 drop_reason 统计
perf script -F event,trace | grep 'reason:' | \
    awk -F'reason: ' '{print $2}' | sort | uniq -c | sort -rn

# 启用 kfree_skb tracepoint
echo 1 > /sys/kernel/debug/tracing/events/skb/kfree_skb/enable

# 实时查看
cat /sys/kernel/debug/tracing/trace_pipe
# 输出示例：
# <idle>-0  [003] ..s1 12345.678: kfree_skb: skbaddr=0xffff... protocol=2048
#     location=tcp_v4_rcv+0x1a3 reason: TCP_INVALID_SEQUENCE

# 过滤特定原因（通过 filter 文件）
echo 'reason == 2' > /sys/kernel/debug/tracing/events/skb/kfree_skb/filter

# 关闭
echo 0 > /sys/kernel/debug/tracing/events/skb/kfree_skb/enable

# TCP 相关统计
nstat -az | grep -i 'drop\|error\|fail\|overflow\|prune\|retrans'

# 关键计数器解读
# TcpExtListenDrops     - SYN 队列满丢弃
# TcpExtListenOverflows - Accept 队列满溢出
# TcpExtTCPBacklogDrop  - socket backlog 满丢弃
# TcpExtTCPMinTTLDrop   - TTL 太小丢弃
# TcpExtTCPRcvQDrop     - 接收队列满丢弃
# TcpExtTCPOFODrop      - OFO 队列丢弃
# TcpExtTCPRetransFail  - 重传失败
# TcpExtPFMemallocDrop  - 内存压力丢弃

# UDP 丢包
cat /proc/net/snmp | grep Udp
# Udp: InDatagrams NoPorts InErrors OutDatagrams RcvbufErrors SndbufErrors
# InErrors = 校验和错误等
# RcvbufErrors = 接收缓冲区满丢弃
# NoPorts = 无监听端口

# 网卡级丢包
cat /proc/net/dev | awk 'NR>2{print $1, "rx_drop="$5, "tx_drop="$13}'

# ethtool 统计
ethtool -S eth0 | grep -i 'drop\|error\|discard\|miss\|overflow'

bpftrace -e '
tracepoint:skb:kfree_skb /args->reason == 67/ {  /* FULL_RING */
    @by_cpu[cpu] = count();
}
interval:s:5 { print(@by_cpu); clear(@by_cpu); }
'

# 增大 ring buffer
ethtool -G eth0 rx 4096

# 检查当前 ring buffer 大小
ethtool -g eth0

bpftrace -e '
tracepoint:skb:kfree_skb /args->reason == 14/ {  /* SOCKET_RCVBUFF */
    printf("RCVBUF full at %s\n", ksym(args->location));
    @stacks = count();
}
'

# 增大默认接收缓冲区
sysctl -w net.core.rmem_default=262144
sysctl -w net.core.rmem_max=16777216

# 或应用层设置 SO_RCVBUF

bpftrace -e '
tracepoint:skb:kfree_skb /args->reason == 6/ {  /* NETFILTER_DROP */
    printf("NF drop at %s\n", ksym(args->location));
    print(kstack(5));
}
'

bpftrace -e '
tracepoint:skb:kfree_skb
/args->reason >= 33 && args->reason <= 44/  /* TCP_OLD_DATA..TCP_OFO_DROP */
{
    @tcp_drop_reason[args->reason] = count();
}
interval:s:10 { print(@tcp_drop_reason); clear(@tcp_drop_reason); }
'

bpftrace -e '
tracepoint:skb:kfree_skb /args->reason == 57/ {  /* QDISC_DROP */
    printf("qdisc drop at %s\n", ksym(args->location));
    @by_cpu[cpu] = count();
}
'

# 增大 txqueuelen
ip link set dev eth0 txqueuelen 10000

# 或更换 qdisc
tc qdisc replace dev eth0 root fq_codel limit 10240

# 一行命令：统计 10 秒内的丢包原因
bpftrace -e '
tracepoint:skb:kfree_skb {
    @[args->reason] = count();
}
interval:s:10 { exit(); }
' 2>/dev/null

perf stat -a -e 'skb:kfree_skb' -- sleep 10