TCP发送接收过程（三） -- 学习netfilter和iptables

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
// linux-4.18/net/core/dev.c
static int __netif_receive_skb_core(struct sk_buff *skb, bool pfmemalloc)
{
    struct packet_type *ptype, *pt_prev;
    rx_handler_func_t *rx_handler;
    struct net_device *orig_dev;
    bool deliver_exact = false;
    int ret = NET_RX_DROP;
    __be16 type;

    net_timestamp_check(!netdev_tstamp_prequeue, skb);

    // 定义 tracepoint:net:netif_receive_skb，可以通过eBPF追踪
    trace_netif_receive_skb(skb);
    ...

    // &ptype_all、&skb->dev->ptype_all 这里会将数据送入抓包点。
    // tcpdump就是从这个入口获取包的
    list_for_each_entry_rcu(ptype, &ptype_all, list) {
        if (pt_prev)
            // 从数据包中取出协议信息，然后遍历注册在这个协议上的回调函数列表
            ret = deliver_skb(skb, pt_prev, orig_dev);
        pt_prev = ptype;
    }

    list_for_each_entry_rcu(ptype, &skb->dev->ptype_all, list) {
        if (pt_prev)
            // 从数据包中取出协议信息，然后遍历注册在这个协议上的回调函数列表
            ret = deliver_skb(skb, pt_prev, orig_dev);
        pt_prev = ptype;
    }
    ...
}

1
2
3
4
5
6
7
8
9
10
11
// linux-4.18/net/core/dev.c
static inline int deliver_skb(struct sk_buff *skb,
                  struct packet_type *pt_prev,
                  struct net_device *orig_dev)
{
    if (unlikely(skb_orphan_frags_rx(skb, GFP_ATOMIC)))
        return -ENOMEM;
    refcount_inc(&skb->users);
    // 协议层注册的处理函数，对于ip包来讲，就会进入到ip_rcv（如果是arp包的话，会进入到arp_rcv）
    return pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
}

1
2
3
4
5
6
7
8
9
10
11
12
13
// linux-4.18/include/linux/netdevice.h
struct packet_type {
    __be16          type;   /* This is really htons(ether_type). */
    struct net_device *dev; /* NULL is wildcarded here	     */
    int             (*func) (struct sk_buff *,
                        struct net_device *,
                        struct packet_type *,
                        struct net_device *);
    bool            (*id_match)(struct packet_type *ptype,
                        struct sock *sk);
    void            *af_packet_priv;
    struct list_head    list;
};

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
// linux-4.18/net/ipv4/af_inet.c
static int __init inet_init(void)
{
    ...
    rc = proto_register(&tcp_prot, 1);
    if (rc)
        goto out;

    rc = proto_register(&udp_prot, 1);
    if (rc)
        goto out_unregister_tcp_proto;
    ...
    // 注册ip网络层的处理函数为 ip_rcv
    dev_add_pack(&ip_packet_type);
    ...
}

// ip_packet_type结构如下
static struct packet_type ip_packet_type __read_mostly = {
    .type = cpu_to_be16(ETH_P_IP),
    .func = ip_rcv,
};

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
// linux-4.18/net/ipv4/ip_input.c
int ip_rcv(struct sk_buff *skb, struct net_device *dev, struct packet_type *pt, struct net_device *orig_dev)
{
    const struct iphdr *iph;
    struct net *net;
    u32 len;
    ...

    // 这里就是一个 netfilter 的hook了，类型为 NF_INET_PRE_ROUTING
    return NF_HOOK(NFPROTO_IPV4, NF_INET_PRE_ROUTING,
               net, NULL, skb, dev, NULL,
               ip_rcv_finish);

csum_error:
    __IP_INC_STATS(net, IPSTATS_MIB_CSUMERRORS);
inhdr_error:
    __IP_INC_STATS(net, IPSTATS_MIB_INHDRERRORS);
drop:
    kfree_skb(skb);
out:
    return NET_RX_DROP;
}

1
2
3
4
5
6
7
8
9
// linux-4.18/include/uapi/linux/netfilter.h
enum nf_inet_hooks {
    NF_INET_PRE_ROUTING,
    NF_INET_LOCAL_IN,
    NF_INET_FORWARD,
    NF_INET_LOCAL_OUT,
    NF_INET_POST_ROUTING,
    NF_INET_NUMHOOKS
};

1
2
3
4
5
6
7
8
9
10
11
12
13
// linux-4.18/include/uapi/linux/netfilter_ipv4.h
/* IP Hooks */
/* After promisc drops, checksum checks. */
#define NF_IP_PRE_ROUTING	0
/* If the packet is destined for this box. */
#define NF_IP_LOCAL_IN		1
/* If the packet is destined for another interface. */
#define NF_IP_FORWARD		2
/* Packets coming from a local process. */
#define NF_IP_LOCAL_OUT		3
/* Packets about to hit the wire. */
#define NF_IP_POST_ROUTING	4
#define NF_IP_NUMHOOKS		5

1
2
3
4
5
6
7
8
9
10
11
12
13
// linux-4.18/include/uapi/linux/netfilter_ipv6.h
/* IP6 Hooks */
/* After promisc drops, checksum checks. */
#define NF_IP6_PRE_ROUTING	0
/* If the packet is destined for this box. */
#define NF_IP6_LOCAL_IN		1
/* If the packet is destined for another interface. */
#define NF_IP6_FORWARD		2
/* Packets coming from a local process. */
#define NF_IP6_LOCAL_OUT		3
/* Packets about to hit the wire. */
#define NF_IP6_POST_ROUTING	4
#define NF_IP6_NUMHOOKS		5

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
// linux-4.18/net/ipv4/ip_input.c
static int ip_rcv_finish(struct net *net, struct sock *sk, struct sk_buff *skb)
{
    const struct iphdr *iph = ip_hdr(skb);
    int (*edemux)(struct sk_buff *skb);
    struct net_device *dev = skb->dev;
    struct rtable *rt;
    int err;
    ...
    // 处理input，rt_dst_alloc 时(route.c)，input函数注册为了 ip_local_deliver
    return dst_input(skb);

    ...
drop:
    kfree_skb(skb);
    return NET_RX_DROP;

drop_error:
    if (err == -EXDEV)
        __NET_INC_STATS(net, LINUX_MIB_IPRPFILTER);
    goto drop;
}

1
2
3
4
5
6
// linux-4.18/include/net/dst.h
/* Input packet from network to transport.  */
static inline int dst_input(struct sk_buff *skb)
{
    return skb_dst(skb)->input(skb);
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
// linux-4.18/net/ipv4/ip_input.c
int ip_local_deliver(struct sk_buff *skb)
{
    /*
     *	Reassemble IP fragments.
     */
    struct net *net = dev_net(skb->dev);

    if (ip_is_fragment(ip_hdr(skb))) {
        if (ip_defrag(net, skb, IP_DEFRAG_LOCAL_DELIVER))
            return 0;
    }

    // netfilter hook: NF_INET_LOCAL_IN
    return NF_HOOK(NFPROTO_IPV4, NF_INET_LOCAL_IN,
               net, NULL, skb, skb->dev, NULL,
               ip_local_deliver_finish);
}

1
2
3
4
5
6
7
8
9
10
11
static int ip_local_deliver_finish(struct net *net, struct sock *sk, struct sk_buff *skb)
{
    ...
    const struct net_protocol *ipprot;
    ...
    ipprot = rcu_dereference(inet_protos[protocol]);
    ...
    // 对于TCP这里是tcp_v4_rcv，对应struct net_protocol tcp_protocol等协议初始化注册的接口
    ret = ipprot->handler(skb);
    ...
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
// linux-4.18/net/socket.c
int __sys_sendto(int fd, void __user *buff, size_t len, unsigned int flags,
         struct sockaddr __user *addr,  int addr_len)
{
    struct socket *sock;
    struct sockaddr_storage address;
    ...
    // 根据fd找到socket
    sock = sockfd_lookup_light(fd, &err, &fput_needed);
    if (!sock)
        goto out;

    ...
    // 调用到 sock_sendmsg
    err = sock_sendmsg(sock, &msg);
    ...
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
// linux-4.18/net/socket.c
int sock_sendmsg(struct socket *sock, struct msghdr *msg)
{
    // 安全相关校验，暂不关注
    int err = security_socket_sendmsg(sock, msg,
                      msg_data_left(msg));

    return err ?: sock_sendmsg_nosec(sock, msg);
}

static inline int sock_sendmsg_nosec(struct socket *sock, struct msghdr *msg)
{
    // socket注册的相应 sendmsg 函数
    int ret = sock->ops->sendmsg(sock, msg, msg_data_left(msg));
    BUG_ON(ret == -EIOCBQUEUED);
    return ret;
}

1
2
3
4
5
6
7
// linux-4.18/net/ipv4/af_inet.c
const struct proto_ops inet_stream_ops = {
    .family		   = PF_INET,
    ...
    .sendmsg	   = inet_sendmsg,
    ...
}

1
2
3
4
5
6
7
8
int inet_sendmsg(struct socket *sock, struct msghdr *msg, size_t size)
{
    struct sock *sk = sock->sk;
    ...
    // sk_prot 的定义：#define sk_prot	__sk_common.skc_prot（对应的结构是：struct proto *skc_prot; 这里会进行协议接口区分）
    // 对于TCP协议的，此处注册为 tcp_sendmsg 接口，具体可见下面小节的梳理
    return sk->sk_prot->sendmsg(sk, msg, size);
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
// linux-4.18/net/ipv4/af_inet.c
static struct inet_protosw inetsw_array[] =
{
    {
        .type =       SOCK_STREAM,
        .protocol =   IPPROTO_TCP,
        .prot =       &tcp_prot,
        .ops =        &inet_stream_ops,
        .flags =      INET_PROTOSW_PERMANENT |
                  INET_PROTOSW_ICSK,
    },
    {
        .type =       SOCK_DGRAM,
        .protocol =   IPPROTO_UDP,
        .prot =       &udp_prot,
        .ops =        &inet_dgram_ops,
        .flags =      INET_PROTOSW_PERMANENT,
    },
    ...
};

1
2
3
4
5
6
7
8
9
10
11
12
13
14
// linux-4.18/net/ipv4/tcp_ipv4.c
// 这里是定义一个`struct proto`实例，并初始化各种操作接口，用于指代TCP协议，网络初始化时会进行注册
struct proto tcp_prot = {
    .name           = "TCP",
    .owner          = THIS_MODULE,
    .close          = tcp_close,
    .pre_connect    = tcp_v4_pre_connect,
    .connect        = tcp_v4_connect,
    ...
    .recvmsg        = tcp_recvmsg,
    .sendmsg        = tcp_sendmsg,
    ...
}
EXPORT_SYMBOL(tcp_prot);

1
2
3
4
5
6
7
8
9
10
11
12
13
struct proto udp_prot = {
    .name           = "UDP",
    .owner          = THIS_MODULE,
    .close          = udp_lib_close,
    .pre_connect    = udp_pre_connect,
    .connect        = ip4_datagram_connect,
    .disconnect     = udp_disconnect,
    ...
    .sendmsg        = udp_sendmsg,
    .recvmsg        = udp_recvmsg,
    ...
}
EXPORT_SYMBOL(udp_prot);

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
// linux-4.18/net/ipv4/tcp.c
int tcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)
{
    int ret;

    lock_sock(sk);
    ret = tcp_sendmsg_locked(sk, msg, size);
    release_sock(sk);

    return ret;
}

// linux-4.18/net/ipv4/tcp.c
int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size)
{
    struct tcp_sock *tp = tcp_sk(sk);
    struct ubuf_info *uarg = NULL;
    struct sk_buff *skb;
    ...
    while (msg_data_left(msg)) {
        int copy = 0;

        // 获取发送队列
        skb = tcp_write_queue_tail(sk);
        if (skb)
            copy = size_goal - skb->len;
        ...
        // 只有满足下面2个条件之一，内核才会真正启动发送数据包
        // 这里判断的是未发送的数据数据是否已经超过最大窗口的一半
        if (forced_push(tp)) {
            tcp_mark_push(tp, skb);
            __tcp_push_pending_frames(sk, mss_now, TCP_NAGLE_PUSH);
        } else if (skb == tcp_send_head(sk))
            tcp_push_one(sk, mss_now);
        ...
    }
    ...
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
// linux-4.18/net/ipv4/tcp_output.c
static bool tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle,
               int push_one, gfp_t gfp)
{
    struct tcp_sock *tp = tcp_sk(sk);
    struct sk_buff *skb;
    ...
    while ((skb = tcp_send_head(sk))) {
        // 真正开启发送
        if (unlikely(tcp_transmit_skb(sk, skb, 1, gfp)))
            break;
        ...
    }
}

// linux-4.18/net/ipv4/tcp_output.c
static int tcp_transmit_skb(struct sock *sk, struct sk_buff *skb, int clone_it,
                gfp_t gfp_mask)
{
    return __tcp_transmit_skb(sk, skb, clone_it, gfp_mask,
                  tcp_sk(sk)->rcv_nxt);
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
// linux-4.18/net/ipv4/tcp_output.c
static int __tcp_transmit_skb(struct sock *sk, struct sk_buff *skb,
                  int clone_it, gfp_t gfp_mask, u32 rcv_nxt)
{
    const struct inet_connection_sock *icsk = inet_csk(sk);
    struct inet_sock *inet;
    struct tcp_sock *tp;
    struct tcp_skb_cb *tcb;
    ...
    tp = tcp_sk(sk);

    // 1.克隆新 skb 出来
    if (clone_it) {
        TCP_SKB_CB(skb)->tx.in_flight = TCP_SKB_CB(skb)->end_seq
            - tp->snd_una;
        oskb = skb;
        ...
        skb = skb_clone(oskb, gfp_mask);
    }
    ...
    // 2.封装 TCP 头
    /* Build TCP header and checksum it. */
    th = (struct tcphdr *)skb->data;
    th->source		= inet->inet_sport;
    th->dest		= inet->inet_dport;
    th->seq			= htonl(tcb->seq);
    ...
    // 3.调用网络层发送接口
    err = icsk->icsk_af_ops->queue_xmit(sk, skb, &inet->cork.fl);
    ...
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
// linux-5.10.10/net/ipv4/tcp_ipv4.c
static int tcp_v4_init_sock(struct sock *sk)
{
    struct inet_connection_sock *icsk = inet_csk(sk);

    tcp_init_sock(sk);

    icsk->icsk_af_ops = &ipv4_specific;

#ifdef CONFIG_TCP_MD5SIG
    tcp_sk(sk)->af_specific = &tcp_sock_ipv4_specific;
#endif

    return 0;
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
// linux-4.18/net/ipv4/ip_output.c
int ip_queue_xmit(struct sock *sk, struct sk_buff *skb, struct flowi *fl)
{
    struct inet_sock *inet = inet_sk(sk);
    struct net *net = sock_net(sk);
    ...
    // 检查 socket 中是否有缓存的路由表；没有时会查找路由项并缓存到socket中
    rt = (struct rtable *)__sk_dst_check(sk, 0);
    ...
    // 为 skb 设置路由表
    skb_dst_set_noref(skb, &rt->dst);
    ...
    // 设置 IP 头
    iph = ip_hdr(skb);
    ...
    // 发送
    res = ip_local_out(net, sk, skb);
    ...
}

1
2
3
4
5
6
7
8
9
10
11
12
13
// linux-4.18/net/ipv4/ip_output.c
int ip_local_out(struct net *net, struct sock *sk, struct sk_buff *skb)
{
    int err;

    err = __ip_local_out(net, sk, skb);
    if (likely(err == 1))
        err = dst_output(net, sk, skb);

    return err;
}
// 可通过kprobe跟踪到
EXPORT_SYMBOL_GPL(ip_local_out);

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
// linux-4.18/net/ipv4/ip_output.c
int __ip_local_out(struct net *net, struct sock *sk, struct sk_buff *skb)
{
    struct iphdr *iph = ip_hdr(skb);

    iph->tot_len = htons(skb->len);
    ip_send_check(iph);

    /* if egress device is enslaved to an L3 master device pass the
     * skb to its handler for processing
     */
    skb = l3mdev_ip_out(sk, skb);
    if (unlikely(!skb))
        return 0;

    skb->protocol = htons(ETH_P_IP);

    // 这里是netfilter hook：NF_INET_LOCAL_OUT
    return nf_hook(NFPROTO_IPV4, NF_INET_LOCAL_OUT,
               net, sk, skb, NULL, skb_dst(skb)->dev,
               dst_output);
}

1
2
3
4
5
/* Output packet to network from transport.  */
static inline int dst_output(struct net *net, struct sock *sk, struct sk_buff *skb)
{
    return skb_dst(skb)->output(net, sk, skb);
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
// linux-4.18/net/ipv4/ip_output.c
int ip_output(struct net *net, struct sock *sk, struct sk_buff *skb)
{
    struct net_device *dev = skb_dst(skb)->dev;

    IP_UPD_PO_STATS(net, IPSTATS_MIB_OUT, skb->len);

    skb->dev = dev;
    skb->protocol = htons(ETH_P_IP);

    return NF_HOOK_COND(NFPROTO_IPV4, NF_INET_POST_ROUTING,
                net, sk, skb, NULL, dev,
                ip_finish_output,
                !(IPCB(skb)->flags & IPSKB_REROUTED));
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
// linux-4.18/net/ipv4/ip_output.c
static int ip_finish_output(struct net *net, struct sock *sk, struct sk_buff *skb)
{
    unsigned int mtu;
    int ret;
    ...
    mtu = ip_skb_dst_mtu(sk, skb);
    if (skb_is_gso(skb))
        return ip_finish_output_gso(net, sk, skb, mtu);

    if (skb->len > mtu || (IPCB(skb)->flags & IPSKB_FRAG_PMTU))
        // ip分片
        return ip_fragment(net, sk, skb, mtu, ip_finish_output2);

    return ip_finish_output2(net, sk, skb);
}