/* Note: skb->sk can be different from sk, in case of tunnels */

 int ip_queue_xmit(struct sock *sk, struct sk_buff *skb, struct flowi *fl)

 {

     struct inet_sock *inet = inet_sk(sk);

     struct net *net = sock_net(sk);

     struct ip_options_rcu *inet_opt;

     struct flowi4 *fl4;

     struct rtable *rt;

     struct iphdr *iph;

     int res;

     /* Skip all of this if the packet is already routed,

      * f.e. by something like SCTP.

      */

     rcu_read_lock();

     inet_opt = rcu_dereference(inet->inet_opt);

     fl4 = &fl->u.ip4;

     /* 获取skb中的路由缓存 */

     rt = skb_rtable(skb);

     /* skb中有缓存则跳转处理 */

     if (rt)

         goto packet_routed;

     /* Make sure we can route this packet. */

     /* 检查控制块中的路由缓存 */

     rt = (struct rtable *)__sk_dst_check(sk, );

     /* 缓存过期 */

     if (!rt) {

         __be32 daddr;

         /* Use correct destination address if we have options. */

         /* 目的地址 */

         daddr = inet->inet_daddr;

         /* 严格路由选项 */

         if (inet_opt && inet_opt->opt.srr)

             daddr = inet_opt->opt.faddr;

         /* If this fails, retransmit mechanism of transport layer will

          * keep trying until route appears or the connection times

          * itself out.

          */

         /* 查找路由缓存 */

         rt = ip_route_output_ports(net, fl4, sk,

                        daddr, inet->inet_saddr,

                        inet->inet_dport,

                        inet->inet_sport,

                        sk->sk_protocol,

                        RT_CONN_FLAGS(sk),

                        sk->sk_bound_dev_if);

         /* 失败 */

         if (IS_ERR(rt))

             goto no_route;

         /* 设置控制块的路由缓存 */

         sk_setup_caps(sk, &rt->dst);

     }

     /* 将路由设置到skb中 */

     skb_dst_set_noref(skb, &rt->dst);

 packet_routed:

     /* 严格路由选项    &&使用网关，无路由 */

     if (inet_opt && inet_opt->opt.is_strictroute && rt->rt_uses_gateway)

         goto no_route;

     /* OK, we know where to send it, allocate and build IP header. */

     /* 加入ip头 */

     skb_push(skb, sizeof(struct iphdr) + (inet_opt ? inet_opt->opt.optlen : ));

     skb_reset_network_header(skb);

     /* 构造ip头 */

     iph = ip_hdr(skb);

     *((__be16 *)iph) = htons(( << ) | ( << ) | (inet->tos & 0xff));

     if (ip_dont_fragment(sk, &rt->dst) && !skb->ignore_df)

         iph->frag_off = htons(IP_DF);

     else

         iph->frag_off = ;

     iph->ttl      = ip_select_ttl(inet, &rt->dst);

     iph->protocol = sk->sk_protocol;

     ip_copy_addrs(iph, fl4);

     /* Transport layer set skb->h.foo itself. */

     /* 构造ip选项 */

     if (inet_opt && inet_opt->opt.optlen) {

         iph->ihl += inet_opt->opt.optlen >> ;

         ip_options_build(skb, &inet_opt->opt, inet->inet_daddr, rt, );

     }

     /* 设置id */

     ip_select_ident_segs(net, skb, sk,

                  skb_shinfo(skb)->gso_segs ?: );

     /* TODO : should we use skb->sk here instead of sk ? */

     /* QOS等级 */

     skb->priority = sk->sk_priority;

     skb->mark = sk->sk_mark;

     /* 输出 */

     res = ip_local_out(net, sk, skb);

     rcu_read_unlock();

     return res;

 no_route:

     /* 无路由处理 */

     rcu_read_unlock();

     IP_INC_STATS(net, IPSTATS_MIB_OUTNOROUTES);

     kfree_skb(skb);

     return -EHOSTUNREACH;

 }

 int ip_build_and_send_pkt(struct sk_buff *skb, const struct sock *sk,

               __be32 saddr, __be32 daddr, struct ip_options_rcu *opt)

 {

     struct inet_sock *inet = inet_sk(sk);

     struct rtable *rt = skb_rtable(skb);

     struct net *net = sock_net(sk);

     struct iphdr *iph;

     /* Build the IP header. */

     /* 构造ip头 */

     skb_push(skb, sizeof(struct iphdr) + (opt ? opt->opt.optlen : ));

     skb_reset_network_header(skb);

     iph = ip_hdr(skb);

     iph->version  = ;

     iph->ihl      = ;

     iph->tos      = inet->tos;

     iph->ttl      = ip_select_ttl(inet, &rt->dst);

     iph->daddr    = (opt && opt->opt.srr ? opt->opt.faddr : daddr);

     iph->saddr    = saddr;

     iph->protocol = sk->sk_protocol;

     /* 分片与否 */

     if (ip_dont_fragment(sk, &rt->dst)) {

         iph->frag_off = htons(IP_DF);

         iph->id = ;

     } else {

         iph->frag_off = ;

         __ip_select_ident(net, iph, );

     }

     /* 选项 */

     if (opt && opt->opt.optlen) {

         iph->ihl += opt->opt.optlen>>;

         ip_options_build(skb, &opt->opt, daddr, rt, );

     }

     /* QOS优先级 */

     skb->priority = sk->sk_priority;

     skb->mark = sk->sk_mark;

     /* Send it out. */

     /* 输出 */

     return ip_local_out(net, skb->sk, skb);

 }

 void ip_send_unicast_reply(struct sock *sk, struct sk_buff *skb,

                const struct ip_options *sopt,

                __be32 daddr, __be32 saddr,

                const struct ip_reply_arg *arg,

                unsigned int len)

 {

     struct ip_options_data replyopts;

     struct ipcm_cookie ipc;

     struct flowi4 fl4;

     struct rtable *rt = skb_rtable(skb);

     struct net *net = sock_net(sk);

     struct sk_buff *nskb;

     int err;

     int oif;

     /* 获取ip选项 */

     if (__ip_options_echo(&replyopts.opt.opt, skb, sopt))

         return;

     ipc.addr = daddr;

     ipc.opt = NULL;

     ipc.tx_flags = ;

     ipc.ttl = ;

     ipc.tos = -;

     /* 选项存在 */

     if (replyopts.opt.opt.optlen) {

         ipc.opt = &replyopts.opt;

         /* 源路由存在，设置下一跳ip地址为目的地址 */

         if (replyopts.opt.opt.srr)

             daddr = replyopts.opt.opt.faddr;

     }

     /* 输出接口设置 */

     oif = arg->bound_dev_if;

     if (!oif && netif_index_is_l3_master(net, skb->skb_iif))

         oif = skb->skb_iif;

     /* 查路由 */

     flowi4_init_output(&fl4, oif,

                IP4_REPLY_MARK(net, skb->mark),

                RT_TOS(arg->tos),

                RT_SCOPE_UNIVERSE, ip_hdr(skb)->protocol,

                ip_reply_arg_flowi_flags(arg),

                daddr, saddr,

                tcp_hdr(skb)->source, tcp_hdr(skb)->dest,

                arg->uid);

     security_skb_classify_flow(skb, flowi4_to_flowi(&fl4));

     rt = ip_route_output_key(net, &fl4);

     if (IS_ERR(rt))

         return;

     /* 根据skb更新sk的属性 */

     inet_sk(sk)->tos = arg->tos;

     sk->sk_priority = skb->priority;

     sk->sk_protocol = ip_hdr(skb)->protocol;

     sk->sk_bound_dev_if = arg->bound_dev_if;

     sk->sk_sndbuf = sysctl_wmem_default;

     sk->sk_mark = fl4.flowi4_mark;

     /* 数据追加到前一个skb或者新建skb后添加到发送队列 */

     err = ip_append_data(sk, &fl4, ip_reply_glue_bits, arg->iov->iov_base,

                  len, , &ipc, &rt, MSG_DONTWAIT);

     if (unlikely(err)) {

         ip_flush_pending_frames(sk);

         goto out;

     }

     /* 如果发送队列有skb，则计算校验和，发送 */

     nskb = skb_peek(&sk->sk_write_queue);

     if (nskb) {

         if (arg->csumoffset >= )

             *((__sum16 *)skb_transport_header(nskb) +

               arg->csumoffset) = csum_fold(csum_add(nskb->csum,

                                 arg->csum));

         nskb->ip_summed = CHECKSUM_NONE;

         /* 发送数据包 */

         ip_push_pending_frames(sk, &fl4);

     }

 out:

     ip_rt_put(rt);

 }