内核网络栈：sk_buff、NAPI 与 XDP

struct sk_buff {
    struct sk_buff *next, *prev;
    union { struct net_device *dev; unsigned long dev_scratch; };
    char cb[48] __aligned(8);        /* 各层私有控制块 */
    unsigned int len, data_len;      /* 线性区 + 分片长度 */
    __u16 mac_len, hdr_len, protocol;
    sk_buff_data_t tail, end, head;  /* 用偏移量定位，节省指针 */
    unsigned char *data;
    ...
};

struct sk_buff *alloc_skb(unsigned int size, gfp_t priority);   /* 普通分配 */
struct sk_buff *__dev_alloc_skb(unsigned int len, gfp_t mask);  /* 软中断中分配 */
void kfree_skb(struct sk_buff *skb);      /* 引用计数归零时真正回收 */
void consume_skb(struct sk_buff *skb);    /* 消费一个引用 */

struct skb_shared_info {
    __u8 nr_frags;               /* frag 数量 */
    __u8 tx_flags;
    unsigned short gso_size;     /* GSO 分段大小 */
    unsigned short gso_segs;
    struct sk_buff *frag_list;   /* 分片链表 */
    skb_frag_t frags[MAX_SKB_FRAGS];
};

struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t mask);  /* 共享数据，仅复制头部 */
struct sk_buff *skb_copy(const struct sk_buff *skb, gfp_t mask); /* 数据也复制 */
struct sk_buff *skb_share_check(struct sk_buff *skb, gfp_t mask);

void netif_napi_add(struct net_device *dev, struct napi_struct *napi,
                    int (*poll)(struct napi_struct *, int), int weight);
void napi_schedule(struct napi_struct *n);      /* 中断上下文中调度 */
int (*poll)(struct napi_struct *napi, int budget);  /* 轮询回调，返回处理报文数 */

int dev_queue_xmit(struct sk_buff *skb);          /* 交给设备（经 qdisc） */
int dev_hard_start_xmit(struct sk_buff *skb, struct net_device *dev,
                        struct netdev_queue *txq); /* 直接调用驱动发送 */

struct softnet_data {
    struct list_head poll_list;           /* 待轮询的 NAPI 实例 */
    struct sk_buff_head input_pkt_queue;  /* backlog 队列 */
    struct napi_struct backlog;
    unsigned int dropped;                 /* 因 backlog 满丢弃的计数 */
    unsigned int time_squeeze;            /* 因预算耗尽被迫退出的次数 */
    ...
};

sysctl net.core.netdev_budget          # 单次软中断处理的最大报文数（默认 300）
sysctl net.core.netdev_budget_usecs    # 单次软中断的最大耗时（默认 2000）
sysctl net.core.netdev_max_backlog     # per-CPU backlog 队列上限（默认 1000）

gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb);

nstat -az | grep -E 'GRO|IpInDelivers'
ethtool -k eth0 | grep -E 'gro|gso'
ethtool -K eth0 gro off    # 需要抓包精确分析时关闭

echo f > /sys/class/net/eth0/queues/rx-0/rps_cpus   # 让 8 核都参与收包
cat /sys/class/net/eth0/queues/rx-0/rps_cpus

sysctl net.core.rps_sock_flow_entries=32768      # 全局流表大小
echo 4096 > /sys/class/net/eth0/queues/rx-0/rps_flow_cnt

echo f > /sys/class/net/eth0/queues/tx-0/xps_cpus   # CPU 0-3 使用 tx 队列 0

static unsigned int my_hook(void *priv, struct sk_buff *skb,
                            const struct nf_hook_state *state)
{
    /* 返回 NF_ACCEPT 继续，NF_DROP 丢弃，NF_STOLEN 接管 */
    return NF_ACCEPT;
}

static const struct nf_hook_ops my_ops = {
    .hook = my_hook,
    .pf = NFPROTO_IPV4,
    .hooknum = NF_INET_PRE_ROUTING,
    .priority = NF_IP_PRI_FIRST,
};
nf_register_net_hook(net, &my_ops);

iptables -t filter -L -n -v --line-numbers    # 查看规则与计数器
nft list ruleset                              # nftables 等价操作
nft add rule ip filter input tcp dport 22 accept
iptables-save > rules.v4                      # 迁移到 nftables
iptables-restore-translate -f rules.v4 > rules.nft

conntrack -L                                   # 查看当前连接跟踪表
conntrack -L -p tcp --state ESTABLISHED
conntrack -C                                   # 当前条目数
conntrack -S                                   # 统计信息（插入失败、无效包等）
sysctl net.netfilter.nf_conntrack_max          # 表容量上限
sysctl net.netfilter.nf_conntrack_tcp_timeout_established

sysctl -w net.netfilter.nf_conntrack_max=1048576            # 提升表容量（每条约 300 字节）
sysctl -w net.netfilter.nf_conntrack_tcp_timeout_established=3600  # 加快条目回收
# options nf_conntrack hashsize=262144                       # 提升哈希桶，重启生效

struct tcp_sock {
    struct inet_connection_sock inet_conn;
    u32 rcv_nxt, snd_nxt;        /* 期望接收 / 下一个发送序号 */
    u32 snd_una, snd_wnd;        /* 未确认序号 / 发送窗口 */
    u32 snd_cwnd, snd_ssthresh;  /* 拥塞窗口 / 慢启动阈值 */
    u32 srtt_us, mdev_us;        /* RTT 估计值 */
    u32 rcv_wnd, rcv_ssthresh;
    struct tcp_congestion_ops *ca_ops;   /* 拥塞控制算法 */
    ...
};

int tcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t size);
int tcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,
                int nonblock, int flags, int *addr_len);

ss -tnm      # 查看 socket 收发队列积压
ss -s        # 全局汇总
ss -tni      # 显示 cwnd、rtt、retrans 等 TCP 内部信息

struct tcp_congestion_ops {
    struct list_head list;
    u32 (*ssthresh)(struct sock *sk);
    void (*cong_avoid)(struct sock *sk, u32 ack, u32 acked);
    void (*set_state)(struct sock *sk, u8 new_state);
    void (*cwnd_event)(struct sock *sk, enum tcp_ca_event ev);
    void (*pkts_acked)(struct sock *sk, const struct ack_sample *sample);
    u32 (*undo_cwnd)(struct sock *sk);
    char name[TCP_CA_NAME_MAX];
};

sysctl net.ipv4.tcp_available_congestion_control    # cubic reno bbr
sysctl -w net.ipv4.tcp_congestion_control=bbr       # 切换全局默认算法
tc qdisc replace dev eth0 root fq                   # BBR 需配合 fq 队列
# setsockopt(fd, IPPROTO_TCP, TCP_CONGESTION, "bbr", 4);  # 单连接指定

int flag = 1;
setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &flag, sizeof(flag));  /* 禁用 Nagle */

int cork = 1;
setsockopt(fd, IPPROTO_TCP, TCP_CORK, &cork, sizeof(cork));     /* 攒包 */
/* ... 多次 write ... */
cork = 0;
setsockopt(fd, IPPROTO_TCP, TCP_CORK, &cork, sizeof(cork));     /* flush */

SEC("xdp")
int xdp_drop_udp(struct xdp_md *ctx)
{
    void *data = (void *)(long)ctx->data;
    void *data_end = (void *)(long)ctx->data_end;
    struct ethhdr *eth = data;
    if ((void *)(eth + 1) > data_end) return XDP_PASS;
    struct iphdr *ip = (void *)(eth + 1);
    if ((void *)(ip + 1) > data_end) return XDP_PASS;
    if (ip->protocol == IPPROTO_UDP) return XDP_DROP;
    return XDP_PASS;
}

ip link set dev eth0 xdp obj prog.o sec xdp           # native：驱动直接调用，性能最高
ip link set dev eth0 xdpgeneric obj prog.o sec xdp    # generic：skb 之后执行，全兼容
ip link set dev eth0 xdpoffload obj prog.o sec xdp    # offload：卸载到智能网卡
ip link show eth0 ; ip link set dev eth0 xdp off      # 查看与卸载

int fd = socket(AF_XDP, SOCK_RAW, 0);
struct sockaddr_xdp sxdp = {
    .sxdp_family = AF_XDP,
    .sxdp_ifindex = ifindex,
    .sxdp_queue_id = 0,
    .sxdp_flags = XDP_FLAGS_UPDATE_IF_NOEXIST,
};
bind(fd, (struct sockaddr *)&sxdp, sizeof(sxdp));

ip netns add ns1 ; ip netns add ns2                # 创建两个网络命名空间
ip netns exec ns1 ip link show                     # 在命名空间内查看设备
ip netns exec ns1 ip link set lo up

ip link add veth1 type veth peer name veth1-peer   # 创建 veth 对
ip link set veth1-peer netns ns1                   # 一端放入 ns1
ip netns exec ns1 ip addr add 10.0.0.1/24 dev veth1-peer
ip netns exec ns1 ip link set veth1-peer up ; ip link set veth1 up

ip link add br0 type bridge ; ip link set br0 up   # 创建网桥并接入端口
ip link set veth1 master br0
brctl show                                         # 旧工具等价查看

tc qdisc show dev eth0                             # 查看当前 qdisc
tc qdisc replace dev eth0 root fq_codel            # 缓解 bufferbloat
tc qdisc add dev eth0 root tbf rate 100mbit burst 32kbit latency 400ms
tc qdisc add dev eth0 root netem delay 100ms loss 1%

cat /proc/net/softnet_stat
# 第 1 列：处理的报文总数
# 第 2 列：因 backlog 满而丢弃的报文数（dropped）
# 第 3 列：time_squeeze，预算耗尽被迫退出次数
# 第 9 列：CPU 冲突次数（collision）
# 第 14 列：收到的 RPS 报文数

netstat -s | grep -i -E 'drop|overflow|error|retrans'
nstat -az | grep -E 'Drop|Prune|Backlog|Retrans'
ss -s                                    # socket 汇总
# 关键指标：TcpExtListenDrops / ListenOverflows（accept 队列满）
#           TcpRetransSegs（重传次数）
#           UdpInErrors / RcvbufErrors（UDP 接收缓冲区溢出）

dropwatch -l kas                          # 交互式输入 start 查看 kfree_skb 调用位置
perf trace -e net:* sleep 5               # 追踪网络 tracepoint
perf trace -e skb:kfree_skb -a sleep 5
perf record -e skb:kfree_skb -a -g -- sleep 10 ; perf report   # 按调用栈统计

echo 1 > /sys/kernel/debug/tracing/events/skb/kfree_skb/enable
cat /sys/kernel/debug/tracing/trace_pipe
bpftrace -e 'tracepoint:skb:kfree_skb { @[kstack] = count(); }'   # 丢包原因分布

# ============================================================
# 完整可运行示例：内核网络栈健康检查与丢包定位（需 root）
# ============================================================
set -u
IFACE="${1:-eth0}"
echo "=== 目标网卡: $IFACE ==="

echo "=== 1. 网卡驱动丢包统计 ==="
ethtool -S "$IFACE" 2>/dev/null | grep -iE 'drop|error|no_buffer|missed' || echo "  (无匹配)"

echo "=== 2. ring buffer 大小 ==="
ethtool -g "$IFACE" 2>/dev/null | sed -n '1,10p'

echo "=== 3. 每 CPU 软中断统计（processed / dropped / squeeze）==="
awk '{ printf "CPU%-3d %-12d %-12d %-12d\n", NR-1, strtonum("0x"$1), strtonum("0x"$2), strtonum("0x"$3) }' \
    /proc/net/softnet_stat

echo "=== 4. 关键 sysctl 参数 ==="
for k in net.core.netdev_max_backlog net.core.netdev_budget net.core.netdev_budget_usecs \
         net.core.rmem_max net.core.rmem_default net.core.wmem_max net.core.somaxconn \
         net.ipv4.tcp_max_syn_backlog net.ipv4.tcp_congestion_control; do
    printf "  %-38s = %s\n" "$k" "$(sysctl -n "$k" 2>/dev/null || echo n/a)"
done

echo "=== 5. conntrack 表使用情况 ==="
if [ -r /proc/sys/net/netfilter/nf_conntrack_count ]; then
    cnt=$(cat /proc/sys/net/netfilter/nf_conntrack_count)
    max=$(cat /proc/sys/net/netfilter/nf_conntrack_max)
    echo "  条目数: $cnt / $max ($(( cnt * 100 / max ))%)"
else
    echo "  (conntrack 未启用)"
fi

echo "=== 6. qdisc 队列与丢包 ==="
tc -s qdisc show dev "$IFACE" 2>/dev/null | grep -E 'qdisc|dropped|backlog' || echo "  (无)"

echo "=== 7. 协议层错误统计 ==="
nstat -az 2>/dev/null | grep -E 'Drop|Prune|Overflow|Retrans|RcvbufErrors' || \
    netstat -s 2>/dev/null | grep -iE 'drop|overflow'

echo "=== 8. socket 收发队列积压（Recv-Q > 0 的 TCP 连接）==="
ss -s 2>/dev/null
ss -tnm 2>/dev/null | awk 'NR>1 && $2+0 > 0 { print "  " $0 }' | head -10

echo "=== 9. 仍无法定位时追踪丢包调用栈 ==="
echo "  perf record -e skb:kfree_skb -a -g -- sleep 10 && perf report"
echo "  bpftrace -e 'tracepoint:skb:kfree_skb { @[kstack] = count(); }'"