#define TCP_NAGLE_OFF        1    /* Nagle's algo is disabled */

#define TCP_NAGLE_CORK        2    /* Socket is corked        */

#define TCP_NAGLE_PUSH        4    /* Cork is overridden for already queued data */

case TCP_CORK:

        /* When set indicates to always queue non-full frames.

         * Later the user clears this option and we transmit

         * any pending partial frames in the queue.  This is

         * meant to be used alongside sendfile() to get properly

         * filled frames when the user (for example) must write

         * out headers with a write() call first and then use

         * sendfile to send out the data parts.

         *

         * TCP_CORK can be set together with TCP_NODELAY and it is

         * stronger than TCP_NODELAY.

         */

        if (val) {

            tp->nonagle |= TCP_NAGLE_CORK;

        } else {

            tp->nonagle &= ~TCP_NAGLE_CORK;

            if (tp->nonagle&TCP_NAGLE_OFF)

                tp->nonagle |= TCP_NAGLE_PUSH;

            tcp_push_pending_frames(sk);

        }

        break;

case TCP_NODELAY:

        if (val) {

            /* TCP_NODELAY is weaker than TCP_CORK, so that

             * this option on corked socket is remembered, but

             * it is not activated until cork is cleared.

             *

             * However, when TCP_NODELAY is set we make

             * an explicit push, which overrides even TCP_CORK

             * for currently queued segments.

             */

            tp->nonagle |= TCP_NAGLE_OFF|TCP_NAGLE_PUSH;

            tcp_push_pending_frames(sk);

        } else {

            tp->nonagle &= ~TCP_NAGLE_OFF;

        }

int tcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)

{

    //size_goal表示GSO支持的大小，为mss_now的整数倍，不支持GSO时则相等

    mss_now = tcp_send_mss(sk, &size_goal, flags);

    // 把msg的用户态数据，按照GSO支持的最大大小，尽量copy到一个skb中

    //skb_entail(sk,skb)到发送队列

    //还有数据没copy，但是当前skb已经满了，可以发送了

    if (forced_push(tp)) {    //超过最大窗口的一半没有设置push了

        tcp_mark_push(tp, skb);    //设置push标记，更新pushed_seq

        __tcp_push_pending_frames(sk, mss_now, TCP_NAGLE_PUSH);    //调用tcp_write_xmit马上发送

    } else if (skb == tcp_send_head(sk))    //第一个包，直接发送

        tcp_push_one(sk, mss_now);

    else{

        //说明发送队列前面还有skb等待发送，且距离之前push的包还不是非常久, 则只是继续放到队列中，继续开始创建下一个skb copy

        continue

    }

out:

    //最后的包调用tcp_push发送

    tcp_push(sk, flags, mss_now, tp->nonagle, size_goal);

    ...

}

static void skb_entail(struct sock *sk, struct sk_buff *skb)

{

    ...

    tcp_add_write_queue_tail(sk, skb);

    if (tp->nonagle & TCP_NAGLE_PUSH)

        tp->nonagle &= ~TCP_NAGLE_PUSH;    //创建新的skb放入发送队列，立刻清楚push选项

}

static void tcp_push(struct sock *sk, int flags, int mss_now,

             int nonagle, int size_goal)

{

    struct tcp_sock *tp = tcp_sk(sk);

    struct sk_buff *skb;

    if (!tcp_send_head(sk))

        return;

    skb = tcp_write_queue_tail(sk);

    if (!(flags & MSG_MORE) || forced_push(tp))

        tcp_mark_push(tp, skb);

    tcp_mark_urg(tp, flags);

    if (tcp_should_autocork(sk, skb, size_goal)) {

        //利用tsq机制延后发送

        /* avoid atomic op if TSQ_THROTTLED bit is already set */

        if (!test_bit(TSQ_THROTTLED, &tp->tsq_flags)) {

            NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPAUTOCORKING);

            set_bit(TSQ_THROTTLED, &tp->tsq_flags);

        }

        /* It is possible TX completion already happened

         * before we set TSQ_THROTTLED.

         */

        if (atomic_read(&sk->sk_wmem_alloc) > skb->truesize)

            return;

    }

    if (flags & MSG_MORE)    //应用程序标记了很快有新的数据到来，则标记cork，不发送小包

        nonagle = TCP_NAGLE_CORK;

    __tcp_push_pending_frames(sk, mss_now, nonagle);    //最终调用tcp_write_xmit

}

/* If a not yet filled skb is pushed, do not send it if

 * we have data packets in Qdisc or NIC queues :

 * Because TX completion will happen shortly, it gives a chance

 * to coalesce future sendmsg() payload into this skb, without

 * need for a timer, and with no latency trade off.

 * As packets containing data payload have a bigger truesize

 * than pure acks (dataless) packets, the last checks prevent

 * autocorking if we only have an ACK in Qdisc/NIC queues,

 * or if TX completion was delayed after we processed ACK packet.

 */

static bool tcp_should_autocork(struct sock *sk, struct sk_buff *skb,

                int size_goal)

{

    return skb->len < size_goal &&    //不到最大GSO size

           sysctl_tcp_autocorking &&    //默认开启

           skb != tcp_write_queue_head(sk) &&    //发送队列前面还有其他skb

           atomic_read(&sk->sk_wmem_alloc) > skb->truesize;    //qdisc中有数据， 说明网卡发送后完成中断释放内存，会很快有新的数据到来

}

static bool tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle,

               int push_one, gfp_t gfp)

{

    max_segs = tcp_tso_segs(sk, mss_now);    //当前tso支持的最大segs数量

    while ((skb = tcp_send_head(sk))) {    //遍历发送队列

        tso_segs = tcp_init_tso_segs(skb, mss_now);    //skb->len/mss，重新设置tcp_gso_segs，因为在tcp_sendmsg中被清零了

        ...

        if (tso_segs == 1) {//tso_segs=1表示无需tso分段

            /* 根据nagle算法，计算是否需要推迟发送数据 */

            if (unlikely(!tcp_nagle_test(tp, skb, mss_now,

                             (tcp_skb_is_last(sk, skb) ?

                              nonagle : TCP_NAGLE_PUSH))))    //last skb就直接发送

                break;    //推迟发送

        } else {    //tso分段

            if (!push_one &&    //不只一个skb

                tcp_tso_should_defer(sk, skb, &is_cwnd_limited, //如果发送窗口剩余不多，并且预计下一个ack将很快到来(意味着可用窗口会增加)，则推迟发送

                         max_segs))

                break;    //可以推迟

        }

        //不用推迟发送，马上发送

        limit = mss_now;

        ...

        if (tcp_small_queue_check(sk, skb, 0))    //tsq检查，qdisc是否达到限制

            break;

        if (unlikely(tcp_transmit_skb(sk, skb, 1, gfp)))    //发送，如果包被qdisc丢了，则退出循环，不继续发送了

            break;

        tcp_event_new_data_sent(sk, skb);//更新sk_send_head和packets_out

        /* 更新struct tcp_sock中的snd_sml字段。记录非全尺寸发送的最后一个字节序号，主要用来做nagle测试

         */

        tcp_minshall_update(tp, mss_now, skb);

        sent_pkts += tcp_skb_pcount(skb);

        if (push_one)    //只发一个skb的则退出循环

            break;

    }

    ...

    //没有数据包inflight，并且有数据等待发送，则准备尝试0窗口探测

    return !tp->packets_out && tcp_send_head(sk);

}

/* Return true if the Nagle test allows this packet to be

 * sent now.

 */

static inline bool tcp_nagle_test(const struct tcp_sock *tp, const struct sk_buff *skb,

                  unsigned int cur_mss, int nonagle)

{

    /* Nagle rule does not apply to frames, which sit in the middle of the

     * write_queue (they have no chances to get new data).

     *

     * This is implemented in the callers, where they modify the 'nonagle'

     * argument based upon the location of SKB in the send queue.

     */

    if (nonagle & TCP_NAGLE_PUSH)

        return true;

    /* Don't use the nagle rule for urgent data (or for the final FIN). */

    if (tcp_urg_mode(tp) || (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN))

        return true;

    if (!tcp_nagle_check(skb->len < cur_mss, tp, nonagle))

        return true;

    //skb->len < cur_mss且设置了TCP_NAGLE_CORK， 或者上一个发送的小包还未被确认， 则推迟发送

    return false;

}

static bool tcp_nagle_check(bool partial, const struct tcp_sock *tp,

                int nonagle)

{

    return partial &&    //skb->len < mss, 也就是说>=mss就直接发送

        ((nonagle & TCP_NAGLE_CORK) ||    //设置了cork则使用nagle

         (!nonagle && tp->packets_out && tcp_minshall_check(tp)));    //有inflight数据且上一个发送的小包还没被确认则进入nagle

}

/* Minshall's variant of the Nagle send check. */

static bool tcp_minshall_check(const struct tcp_sock *tp)

{

    return after(tp->snd_sml, tp->snd_una) &&    //上一个发送的小包还没确认

        !after(tp->snd_sml, tp->snd_nxt);    //没有回绕

}

static void tcp_minshall_update(struct tcp_sock *tp, unsigned int mss_now,

                const struct sk_buff *skb)

{

    if (skb->len < tcp_skb_pcount(skb) * mss_now)

        tp->snd_sml = TCP_SKB_CB(skb)->end_seq;

}

static bool tcp_tso_should_defer(struct sock *sk, struct sk_buff *skb,

                 bool *is_cwnd_limited, u32 max_segs)

{

    const struct inet_connection_sock *icsk = inet_csk(sk);

    u32 age, send_win, cong_win, limit, in_flight;

    struct tcp_sock *tp = tcp_sk(sk);

    struct skb_mstamp now;

    struct sk_buff *head;

    int win_divisor;

    if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)

        goto send_now;

    if (icsk->icsk_ca_state >= TCP_CA_Recovery)

        goto send_now;

    /* Avoid bursty behavior by allowing defer

     * only if the last write was recent.

     */

    if ((s32)(tcp_time_stamp - tp->lsndtime) > 0)

        goto send_now;

    in_flight = tcp_packets_in_flight(tp);

    BUG_ON(tcp_skb_pcount(skb) <= 1 || (tp->snd_cwnd <= in_flight));

    send_win = tcp_wnd_end(tp) - TCP_SKB_CB(skb)->seq;    //发送窗口

    /* From in_flight test above, we know that cwnd > in_flight.  */

    cong_win = (tp->snd_cwnd - in_flight) * tp->mss_cache;    //拥塞窗口

    limit = min(send_win, cong_win);    //最大发送窗口剩余

    /* If a full-sized TSO skb can be sent, do it. */

    if (limit >= max_segs * tp->mss_cache)    //支持最大尺寸的tso发送

        goto send_now;

    /* Middle in queue won't get any more data, full sendable already? */

    if ((skb != tcp_write_queue_tail(sk)) && (limit >= skb->len))    //不是发送队列的最后一个，且满足发送窗口

        goto send_now;    //直接发送，不会有数据被添加到这个skb了

    win_divisor = ACCESS_ONCE(sysctl_tcp_tso_win_divisor);

    if (win_divisor) {

        u32 chunk = min(tp->snd_wnd, tp->snd_cwnd * tp->mss_cache);

        /* If at least some fraction of a window is available,

         * just use it.

         */

        chunk /= win_divisor;

        if (limit >= chunk)    //剩余的窗口大于总窗口的比例， 默认1/3

            goto send_now;

    } else {

        /* Different approach, try not to defer past a single

         * ACK.  Receiver should ACK every other full sized

         * frame, so if we have space for more than 3 frames

         * then send now.

         */

        if (limit > tcp_max_tso_deferred_mss(tp) * tp->mss_cache)

            goto send_now;

    }

    head = tcp_write_queue_head(sk);

    skb_mstamp_get(&now);

    age = skb_mstamp_us_delta(&now, &head->skb_mstamp);    //最早的未确认包的距离现在的时间

    /* If next ACK is likely to come too late (half srtt), do not defer */

    if (age < (tp->srtt_us >> 4))    // 也就是说下一个ack的到来很可能大于1/2的srtt，直接发送

        goto send_now;

    /* Ok, it looks like it is advisable to defer. */

    //当前skb的收到cwnd限制

    if (cong_win < send_win && cong_win <= skb->len)

        *is_cwnd_limited = true;

    //可以推迟发送了

    return true;

send_now:

    return false;

}