2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
6 * The Internet Protocol (IP) output module.
8 * Version: $Id: ip_output.c,v 1.100 2002/02/01 22:01:03 davem Exp $
11 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
12 * Donald Becker, <becker@super.org>
13 * Alan Cox, <Alan.Cox@linux.org>
15 * Stefan Becker, <stefanb@yello.ping.de>
16 * Jorge Cwik, <jorge@laser.satlink.net>
17 * Arnt Gulbrandsen, <agulbra@nvg.unit.no>
18 * Hirokazu Takahashi, <taka@valinux.co.jp>
20 * See ip_input.c for original log
23 * Alan Cox : Missing nonblock feature in ip_build_xmit.
24 * Mike Kilburn : htons() missing in ip_build_xmit.
25 * Bradford Johnson: Fix faulty handling of some frames when
27 * Alexander Demenshin: Missing sk/skb free in ip_queue_xmit
28 * (in case if packet not accepted by
29 * output firewall rules)
30 * Mike McLagan : Routing by source
31 * Alexey Kuznetsov: use new route cache
32 * Andi Kleen: Fix broken PMTU recovery and remove
33 * some redundant tests.
34 * Vitaly E. Lavrov : Transparent proxy revived after year coma.
35 * Andi Kleen : Replace ip_reply with ip_send_reply.
36 * Andi Kleen : Split fast and slow ip_build_xmit path
37 * for decreased register pressure on x86
38 * and more readibility.
39 * Marc Boucher : When call_out_firewall returns FW_QUEUE,
40 * silently drop skb instead of failing with -EPERM.
41 * Detlev Wengorz : Copy protocol for fragments.
42 * Hirokazu Takahashi: HW checksumming for outgoing UDP
44 * Hirokazu Takahashi: sendfile() on UDP works now.
47 #include <asm/uaccess.h>
48 #include <asm/system.h>
49 #include <linux/module.h>
50 #include <linux/types.h>
51 #include <linux/kernel.h>
53 #include <linux/string.h>
54 #include <linux/errno.h>
55 #include <linux/highmem.h>
57 #include <linux/socket.h>
58 #include <linux/sockios.h>
60 #include <linux/inet.h>
61 #include <linux/netdevice.h>
62 #include <linux/etherdevice.h>
63 #include <linux/proc_fs.h>
64 #include <linux/stat.h>
65 #include <linux/init.h>
69 #include <net/protocol.h>
70 #include <net/route.h>
72 #include <linux/skbuff.h>
76 #include <net/checksum.h>
77 #include <net/inetpeer.h>
78 #include <linux/igmp.h>
79 #include <linux/netfilter_ipv4.h>
80 #include <linux/netfilter_bridge.h>
81 #include <linux/mroute.h>
82 #include <linux/netlink.h>
83 #include <linux/tcp.h>
85 int sysctl_ip_default_ttl __read_mostly
= IPDEFTTL
;
87 /* Generate a checksum for an outgoing IP datagram. */
88 __inline__
void ip_send_check(struct iphdr
*iph
)
91 iph
->check
= ip_fast_csum((unsigned char *)iph
, iph
->ihl
);
94 int __ip_local_out(struct sk_buff
*skb
)
96 struct iphdr
*iph
= ip_hdr(skb
);
98 iph
->tot_len
= htons(skb
->len
);
100 return nf_hook(PF_INET
, NF_INET_LOCAL_OUT
, skb
, NULL
, skb
->dst
->dev
,
104 int ip_local_out(struct sk_buff
*skb
)
108 err
= __ip_local_out(skb
);
109 if (likely(err
== 1))
110 err
= dst_output(skb
);
114 EXPORT_SYMBOL_GPL(ip_local_out
);
116 /* dev_loopback_xmit for use with netfilter. */
117 static int ip_dev_loopback_xmit(struct sk_buff
*newskb
)
119 skb_reset_mac_header(newskb
);
120 __skb_pull(newskb
, skb_network_offset(newskb
));
121 newskb
->pkt_type
= PACKET_LOOPBACK
;
122 newskb
->ip_summed
= CHECKSUM_UNNECESSARY
;
123 BUG_TRAP(newskb
->dst
);
128 static inline int ip_select_ttl(struct inet_sock
*inet
, struct dst_entry
*dst
)
130 int ttl
= inet
->uc_ttl
;
133 ttl
= dst_metric(dst
, RTAX_HOPLIMIT
);
138 * Add an ip header to a skbuff and send it out.
141 int ip_build_and_send_pkt(struct sk_buff
*skb
, struct sock
*sk
,
142 __be32 saddr
, __be32 daddr
, struct ip_options
*opt
)
144 struct inet_sock
*inet
= inet_sk(sk
);
145 struct rtable
*rt
= (struct rtable
*)skb
->dst
;
148 /* Build the IP header. */
149 skb_push(skb
, sizeof(struct iphdr
) + (opt
? opt
->optlen
: 0));
150 skb_reset_network_header(skb
);
154 iph
->tos
= inet
->tos
;
155 if (ip_dont_fragment(sk
, &rt
->u
.dst
))
156 iph
->frag_off
= htons(IP_DF
);
159 iph
->ttl
= ip_select_ttl(inet
, &rt
->u
.dst
);
160 iph
->daddr
= rt
->rt_dst
;
161 iph
->saddr
= rt
->rt_src
;
162 iph
->protocol
= sk
->sk_protocol
;
163 ip_select_ident(iph
, &rt
->u
.dst
, sk
);
165 if (opt
&& opt
->optlen
) {
166 iph
->ihl
+= opt
->optlen
>>2;
167 ip_options_build(skb
, opt
, daddr
, rt
, 0);
170 skb
->priority
= sk
->sk_priority
;
173 return ip_local_out(skb
);
176 EXPORT_SYMBOL_GPL(ip_build_and_send_pkt
);
178 static inline int ip_finish_output2(struct sk_buff
*skb
)
180 struct dst_entry
*dst
= skb
->dst
;
181 struct rtable
*rt
= (struct rtable
*)dst
;
182 struct net_device
*dev
= dst
->dev
;
183 unsigned int hh_len
= LL_RESERVED_SPACE(dev
);
185 if (rt
->rt_type
== RTN_MULTICAST
)
186 IP_INC_STATS(IPSTATS_MIB_OUTMCASTPKTS
);
187 else if (rt
->rt_type
== RTN_BROADCAST
)
188 IP_INC_STATS(IPSTATS_MIB_OUTBCASTPKTS
);
190 /* Be paranoid, rather than too clever. */
191 if (unlikely(skb_headroom(skb
) < hh_len
&& dev
->header_ops
)) {
192 struct sk_buff
*skb2
;
194 skb2
= skb_realloc_headroom(skb
, LL_RESERVED_SPACE(dev
));
200 skb_set_owner_w(skb2
, skb
->sk
);
206 return neigh_hh_output(dst
->hh
, skb
);
207 else if (dst
->neighbour
)
208 return dst
->neighbour
->output(skb
);
211 printk(KERN_DEBUG
"ip_finish_output2: No header cache and no neighbour!\n");
216 static inline int ip_skb_dst_mtu(struct sk_buff
*skb
)
218 struct inet_sock
*inet
= skb
->sk
? inet_sk(skb
->sk
) : NULL
;
220 return (inet
&& inet
->pmtudisc
== IP_PMTUDISC_PROBE
) ?
221 skb
->dst
->dev
->mtu
: dst_mtu(skb
->dst
);
224 static int ip_finish_output(struct sk_buff
*skb
)
226 #if defined(CONFIG_NETFILTER) && defined(CONFIG_XFRM)
227 /* Policy lookup after SNAT yielded a new policy */
228 if (skb
->dst
->xfrm
!= NULL
) {
229 IPCB(skb
)->flags
|= IPSKB_REROUTED
;
230 return dst_output(skb
);
233 if (skb
->len
> ip_skb_dst_mtu(skb
) && !skb_is_gso(skb
))
234 return ip_fragment(skb
, ip_finish_output2
);
236 return ip_finish_output2(skb
);
239 int ip_mc_output(struct sk_buff
*skb
)
241 struct sock
*sk
= skb
->sk
;
242 struct rtable
*rt
= (struct rtable
*)skb
->dst
;
243 struct net_device
*dev
= rt
->u
.dst
.dev
;
246 * If the indicated interface is up and running, send the packet.
248 IP_INC_STATS(IPSTATS_MIB_OUTREQUESTS
);
251 skb
->protocol
= htons(ETH_P_IP
);
254 * Multicasts are looped back for other local users
257 if (rt
->rt_flags
&RTCF_MULTICAST
) {
258 if ((!sk
|| inet_sk(sk
)->mc_loop
)
259 #ifdef CONFIG_IP_MROUTE
260 /* Small optimization: do not loopback not local frames,
261 which returned after forwarding; they will be dropped
262 by ip_mr_input in any case.
263 Note, that local frames are looped back to be delivered
266 This check is duplicated in ip_mr_input at the moment.
268 && ((rt
->rt_flags
&RTCF_LOCAL
) || !(IPCB(skb
)->flags
&IPSKB_FORWARDED
))
271 struct sk_buff
*newskb
= skb_clone(skb
, GFP_ATOMIC
);
273 NF_HOOK(PF_INET
, NF_INET_POST_ROUTING
, newskb
,
275 ip_dev_loopback_xmit
);
278 /* Multicasts with ttl 0 must not go beyond the host */
280 if (ip_hdr(skb
)->ttl
== 0) {
286 if (rt
->rt_flags
&RTCF_BROADCAST
) {
287 struct sk_buff
*newskb
= skb_clone(skb
, GFP_ATOMIC
);
289 NF_HOOK(PF_INET
, NF_INET_POST_ROUTING
, newskb
, NULL
,
290 newskb
->dev
, ip_dev_loopback_xmit
);
293 return NF_HOOK_COND(PF_INET
, NF_INET_POST_ROUTING
, skb
, NULL
, skb
->dev
,
295 !(IPCB(skb
)->flags
& IPSKB_REROUTED
));
298 int ip_output(struct sk_buff
*skb
)
300 struct net_device
*dev
= skb
->dst
->dev
;
302 IP_INC_STATS(IPSTATS_MIB_OUTREQUESTS
);
305 skb
->protocol
= htons(ETH_P_IP
);
307 return NF_HOOK_COND(PF_INET
, NF_INET_POST_ROUTING
, skb
, NULL
, dev
,
309 !(IPCB(skb
)->flags
& IPSKB_REROUTED
));
312 int ip_queue_xmit(struct sk_buff
*skb
, int ipfragok
)
314 struct sock
*sk
= skb
->sk
;
315 struct inet_sock
*inet
= inet_sk(sk
);
316 struct ip_options
*opt
= inet
->opt
;
320 /* Skip all of this if the packet is already routed,
321 * f.e. by something like SCTP.
323 rt
= (struct rtable
*) skb
->dst
;
327 /* Make sure we can route this packet. */
328 rt
= (struct rtable
*)__sk_dst_check(sk
, 0);
332 /* Use correct destination address if we have options. */
338 struct flowi fl
= { .oif
= sk
->sk_bound_dev_if
,
341 .saddr
= inet
->saddr
,
342 .tos
= RT_CONN_FLAGS(sk
) } },
343 .proto
= sk
->sk_protocol
,
345 { .sport
= inet
->sport
,
346 .dport
= inet
->dport
} } };
348 /* If this fails, retransmit mechanism of transport layer will
349 * keep trying until route appears or the connection times
352 security_sk_classify_flow(sk
, &fl
);
353 if (ip_route_output_flow(&rt
, &fl
, sk
, 0))
356 sk_setup_caps(sk
, &rt
->u
.dst
);
358 skb
->dst
= dst_clone(&rt
->u
.dst
);
361 if (opt
&& opt
->is_strictroute
&& rt
->rt_dst
!= rt
->rt_gateway
)
364 /* OK, we know where to send it, allocate and build IP header. */
365 skb_push(skb
, sizeof(struct iphdr
) + (opt
? opt
->optlen
: 0));
366 skb_reset_network_header(skb
);
368 *((__be16
*)iph
) = htons((4 << 12) | (5 << 8) | (inet
->tos
& 0xff));
369 if (ip_dont_fragment(sk
, &rt
->u
.dst
) && !ipfragok
)
370 iph
->frag_off
= htons(IP_DF
);
373 iph
->ttl
= ip_select_ttl(inet
, &rt
->u
.dst
);
374 iph
->protocol
= sk
->sk_protocol
;
375 iph
->saddr
= rt
->rt_src
;
376 iph
->daddr
= rt
->rt_dst
;
377 /* Transport layer set skb->h.foo itself. */
379 if (opt
&& opt
->optlen
) {
380 iph
->ihl
+= opt
->optlen
>> 2;
381 ip_options_build(skb
, opt
, inet
->daddr
, rt
, 0);
384 ip_select_ident_more(iph
, &rt
->u
.dst
, sk
,
385 (skb_shinfo(skb
)->gso_segs
?: 1) - 1);
387 skb
->priority
= sk
->sk_priority
;
389 return ip_local_out(skb
);
392 IP_INC_STATS(IPSTATS_MIB_OUTNOROUTES
);
394 return -EHOSTUNREACH
;
398 static void ip_copy_metadata(struct sk_buff
*to
, struct sk_buff
*from
)
400 to
->pkt_type
= from
->pkt_type
;
401 to
->priority
= from
->priority
;
402 to
->protocol
= from
->protocol
;
403 dst_release(to
->dst
);
404 to
->dst
= dst_clone(from
->dst
);
406 to
->mark
= from
->mark
;
408 /* Copy the flags to each fragment. */
409 IPCB(to
)->flags
= IPCB(from
)->flags
;
411 #ifdef CONFIG_NET_SCHED
412 to
->tc_index
= from
->tc_index
;
415 #if defined(CONFIG_NETFILTER_XT_TARGET_TRACE) || \
416 defined(CONFIG_NETFILTER_XT_TARGET_TRACE_MODULE)
417 to
->nf_trace
= from
->nf_trace
;
419 #if defined(CONFIG_IP_VS) || defined(CONFIG_IP_VS_MODULE)
420 to
->ipvs_property
= from
->ipvs_property
;
422 skb_copy_secmark(to
, from
);
426 * This IP datagram is too large to be sent in one piece. Break it up into
427 * smaller pieces (each of size equal to IP header plus
428 * a block of the data of the original IP data part) that will yet fit in a
429 * single device frame, and queue such a frame for sending.
432 int ip_fragment(struct sk_buff
*skb
, int (*output
)(struct sk_buff
*))
437 struct net_device
*dev
;
438 struct sk_buff
*skb2
;
439 unsigned int mtu
, hlen
, left
, len
, ll_rs
, pad
;
441 __be16 not_last_frag
;
442 struct rtable
*rt
= (struct rtable
*)skb
->dst
;
448 * Point into the IP datagram header.
453 if (unlikely((iph
->frag_off
& htons(IP_DF
)) && !skb
->local_df
)) {
454 IP_INC_STATS(IPSTATS_MIB_FRAGFAILS
);
455 icmp_send(skb
, ICMP_DEST_UNREACH
, ICMP_FRAG_NEEDED
,
456 htonl(ip_skb_dst_mtu(skb
)));
462 * Setup starting values.
466 mtu
= dst_mtu(&rt
->u
.dst
) - hlen
; /* Size of data space */
467 IPCB(skb
)->flags
|= IPSKB_FRAG_COMPLETE
;
469 /* When frag_list is given, use it. First, check its validity:
470 * some transformers could create wrong frag_list or break existing
471 * one, it is not prohibited. In this case fall back to copying.
473 * LATER: this step can be merged to real generation of fragments,
474 * we can switch to copy when see the first bad fragment.
476 if (skb_shinfo(skb
)->frag_list
) {
477 struct sk_buff
*frag
;
478 int first_len
= skb_pagelen(skb
);
480 if (first_len
- hlen
> mtu
||
481 ((first_len
- hlen
) & 7) ||
482 (iph
->frag_off
& htons(IP_MF
|IP_OFFSET
)) ||
486 for (frag
= skb_shinfo(skb
)->frag_list
; frag
; frag
= frag
->next
) {
487 /* Correct geometry. */
488 if (frag
->len
> mtu
||
489 ((frag
->len
& 7) && frag
->next
) ||
490 skb_headroom(frag
) < hlen
)
493 /* Partially cloned skb? */
494 if (skb_shared(frag
))
501 frag
->destructor
= sock_wfree
;
502 skb
->truesize
-= frag
->truesize
;
506 /* Everything is OK. Generate! */
510 frag
= skb_shinfo(skb
)->frag_list
;
511 skb_shinfo(skb
)->frag_list
= NULL
;
512 skb
->data_len
= first_len
- skb_headlen(skb
);
513 skb
->len
= first_len
;
514 iph
->tot_len
= htons(first_len
);
515 iph
->frag_off
= htons(IP_MF
);
519 /* Prepare header of the next frame,
520 * before previous one went down. */
522 frag
->ip_summed
= CHECKSUM_NONE
;
523 skb_reset_transport_header(frag
);
524 __skb_push(frag
, hlen
);
525 skb_reset_network_header(frag
);
526 memcpy(skb_network_header(frag
), iph
, hlen
);
528 iph
->tot_len
= htons(frag
->len
);
529 ip_copy_metadata(frag
, skb
);
531 ip_options_fragment(frag
);
532 offset
+= skb
->len
- hlen
;
533 iph
->frag_off
= htons(offset
>>3);
534 if (frag
->next
!= NULL
)
535 iph
->frag_off
|= htons(IP_MF
);
536 /* Ready, complete checksum */
543 IP_INC_STATS(IPSTATS_MIB_FRAGCREATES
);
553 IP_INC_STATS(IPSTATS_MIB_FRAGOKS
);
562 IP_INC_STATS(IPSTATS_MIB_FRAGFAILS
);
567 left
= skb
->len
- hlen
; /* Space per frame */
568 ptr
= raw
+ hlen
; /* Where to start from */
570 /* for bridged IP traffic encapsulated inside f.e. a vlan header,
571 * we need to make room for the encapsulating header
573 pad
= nf_bridge_pad(skb
);
574 ll_rs
= LL_RESERVED_SPACE_EXTRA(rt
->u
.dst
.dev
, pad
);
578 * Fragment the datagram.
581 offset
= (ntohs(iph
->frag_off
) & IP_OFFSET
) << 3;
582 not_last_frag
= iph
->frag_off
& htons(IP_MF
);
585 * Keep copying data until we run out.
590 /* IF: it doesn't fit, use 'mtu' - the data space left */
593 /* IF: we are not sending upto and including the packet end
594 then align the next start on an eight byte boundary */
602 if ((skb2
= alloc_skb(len
+hlen
+ll_rs
, GFP_ATOMIC
)) == NULL
) {
603 NETDEBUG(KERN_INFO
"IP: frag: no memory for new fragment!\n");
609 * Set up data on packet
612 ip_copy_metadata(skb2
, skb
);
613 skb_reserve(skb2
, ll_rs
);
614 skb_put(skb2
, len
+ hlen
);
615 skb_reset_network_header(skb2
);
616 skb2
->transport_header
= skb2
->network_header
+ hlen
;
619 * Charge the memory for the fragment to any owner
624 skb_set_owner_w(skb2
, skb
->sk
);
627 * Copy the packet header into the new buffer.
630 skb_copy_from_linear_data(skb
, skb_network_header(skb2
), hlen
);
633 * Copy a block of the IP datagram.
635 if (skb_copy_bits(skb
, ptr
, skb_transport_header(skb2
), len
))
640 * Fill in the new header fields.
643 iph
->frag_off
= htons((offset
>> 3));
645 /* ANK: dirty, but effective trick. Upgrade options only if
646 * the segment to be fragmented was THE FIRST (otherwise,
647 * options are already fixed) and make it ONCE
648 * on the initial skb, so that all the following fragments
649 * will inherit fixed options.
652 ip_options_fragment(skb
);
655 * Added AC : If we are fragmenting a fragment that's not the
656 * last fragment then keep MF on each bit
658 if (left
> 0 || not_last_frag
)
659 iph
->frag_off
|= htons(IP_MF
);
664 * Put this fragment into the sending queue.
666 iph
->tot_len
= htons(len
+ hlen
);
674 IP_INC_STATS(IPSTATS_MIB_FRAGCREATES
);
677 IP_INC_STATS(IPSTATS_MIB_FRAGOKS
);
682 IP_INC_STATS(IPSTATS_MIB_FRAGFAILS
);
686 EXPORT_SYMBOL(ip_fragment
);
689 ip_generic_getfrag(void *from
, char *to
, int offset
, int len
, int odd
, struct sk_buff
*skb
)
691 struct iovec
*iov
= from
;
693 if (skb
->ip_summed
== CHECKSUM_PARTIAL
) {
694 if (memcpy_fromiovecend(to
, iov
, offset
, len
) < 0)
698 if (csum_partial_copy_fromiovecend(to
, iov
, offset
, len
, &csum
) < 0)
700 skb
->csum
= csum_block_add(skb
->csum
, csum
, odd
);
706 csum_page(struct page
*page
, int offset
, int copy
)
711 csum
= csum_partial(kaddr
+ offset
, copy
, 0);
716 static inline int ip_ufo_append_data(struct sock
*sk
,
717 int getfrag(void *from
, char *to
, int offset
, int len
,
718 int odd
, struct sk_buff
*skb
),
719 void *from
, int length
, int hh_len
, int fragheaderlen
,
720 int transhdrlen
, int mtu
,unsigned int flags
)
725 /* There is support for UDP fragmentation offload by network
726 * device, so create one single skb packet containing complete
729 if ((skb
= skb_peek_tail(&sk
->sk_write_queue
)) == NULL
) {
730 skb
= sock_alloc_send_skb(sk
,
731 hh_len
+ fragheaderlen
+ transhdrlen
+ 20,
732 (flags
& MSG_DONTWAIT
), &err
);
737 /* reserve space for Hardware header */
738 skb_reserve(skb
, hh_len
);
740 /* create space for UDP/IP header */
741 skb_put(skb
,fragheaderlen
+ transhdrlen
);
743 /* initialize network header pointer */
744 skb_reset_network_header(skb
);
746 /* initialize protocol header pointer */
747 skb
->transport_header
= skb
->network_header
+ fragheaderlen
;
749 skb
->ip_summed
= CHECKSUM_PARTIAL
;
751 sk
->sk_sndmsg_off
= 0;
754 err
= skb_append_datato_frags(sk
,skb
, getfrag
, from
,
755 (length
- transhdrlen
));
757 /* specify the length of each IP datagram fragment*/
758 skb_shinfo(skb
)->gso_size
= mtu
- fragheaderlen
;
759 skb_shinfo(skb
)->gso_type
= SKB_GSO_UDP
;
760 __skb_queue_tail(&sk
->sk_write_queue
, skb
);
764 /* There is not enough support do UFO ,
765 * so follow normal path
772 * ip_append_data() and ip_append_page() can make one large IP datagram
773 * from many pieces of data. Each pieces will be holded on the socket
774 * until ip_push_pending_frames() is called. Each piece can be a page
777 * Not only UDP, other transport protocols - e.g. raw sockets - can use
778 * this interface potentially.
780 * LATER: length must be adjusted by pad at tail, when it is required.
782 int ip_append_data(struct sock
*sk
,
783 int getfrag(void *from
, char *to
, int offset
, int len
,
784 int odd
, struct sk_buff
*skb
),
785 void *from
, int length
, int transhdrlen
,
786 struct ipcm_cookie
*ipc
, struct rtable
*rt
,
789 struct inet_sock
*inet
= inet_sk(sk
);
792 struct ip_options
*opt
= NULL
;
799 unsigned int maxfraglen
, fragheaderlen
;
800 int csummode
= CHECKSUM_NONE
;
805 if (skb_queue_empty(&sk
->sk_write_queue
)) {
811 if (inet
->cork
.opt
== NULL
) {
812 inet
->cork
.opt
= kmalloc(sizeof(struct ip_options
) + 40, sk
->sk_allocation
);
813 if (unlikely(inet
->cork
.opt
== NULL
))
816 memcpy(inet
->cork
.opt
, opt
, sizeof(struct ip_options
)+opt
->optlen
);
817 inet
->cork
.flags
|= IPCORK_OPT
;
818 inet
->cork
.addr
= ipc
->addr
;
820 dst_hold(&rt
->u
.dst
);
821 inet
->cork
.fragsize
= mtu
= inet
->pmtudisc
== IP_PMTUDISC_PROBE
?
823 dst_mtu(rt
->u
.dst
.path
);
825 inet
->cork
.length
= 0;
826 sk
->sk_sndmsg_page
= NULL
;
827 sk
->sk_sndmsg_off
= 0;
828 if ((exthdrlen
= rt
->u
.dst
.header_len
) != 0) {
830 transhdrlen
+= exthdrlen
;
834 if (inet
->cork
.flags
& IPCORK_OPT
)
835 opt
= inet
->cork
.opt
;
839 mtu
= inet
->cork
.fragsize
;
841 hh_len
= LL_RESERVED_SPACE(rt
->u
.dst
.dev
);
843 fragheaderlen
= sizeof(struct iphdr
) + (opt
? opt
->optlen
: 0);
844 maxfraglen
= ((mtu
- fragheaderlen
) & ~7) + fragheaderlen
;
846 if (inet
->cork
.length
+ length
> 0xFFFF - fragheaderlen
) {
847 ip_local_error(sk
, EMSGSIZE
, rt
->rt_dst
, inet
->dport
, mtu
-exthdrlen
);
852 * transhdrlen > 0 means that this is the first fragment and we wish
853 * it won't be fragmented in the future.
856 length
+ fragheaderlen
<= mtu
&&
857 rt
->u
.dst
.dev
->features
& NETIF_F_V4_CSUM
&&
859 csummode
= CHECKSUM_PARTIAL
;
861 inet
->cork
.length
+= length
;
862 if (((length
> mtu
) && (sk
->sk_protocol
== IPPROTO_UDP
)) &&
863 (rt
->u
.dst
.dev
->features
& NETIF_F_UFO
)) {
865 err
= ip_ufo_append_data(sk
, getfrag
, from
, length
, hh_len
,
866 fragheaderlen
, transhdrlen
, mtu
,
873 /* So, what's going on in the loop below?
875 * We use calculated fragment length to generate chained skb,
876 * each of segments is IP fragment ready for sending to network after
877 * adding appropriate IP header.
880 if ((skb
= skb_peek_tail(&sk
->sk_write_queue
)) == NULL
)
884 /* Check if the remaining data fits into current packet. */
885 copy
= mtu
- skb
->len
;
887 copy
= maxfraglen
- skb
->len
;
890 unsigned int datalen
;
891 unsigned int fraglen
;
892 unsigned int fraggap
;
893 unsigned int alloclen
;
894 struct sk_buff
*skb_prev
;
898 fraggap
= skb_prev
->len
- maxfraglen
;
903 * If remaining data exceeds the mtu,
904 * we know we need more fragment(s).
906 datalen
= length
+ fraggap
;
907 if (datalen
> mtu
- fragheaderlen
)
908 datalen
= maxfraglen
- fragheaderlen
;
909 fraglen
= datalen
+ fragheaderlen
;
911 if ((flags
& MSG_MORE
) &&
912 !(rt
->u
.dst
.dev
->features
&NETIF_F_SG
))
915 alloclen
= datalen
+ fragheaderlen
;
917 /* The last fragment gets additional space at tail.
918 * Note, with MSG_MORE we overallocate on fragments,
919 * because we have no idea what fragment will be
922 if (datalen
== length
+ fraggap
)
923 alloclen
+= rt
->u
.dst
.trailer_len
;
926 skb
= sock_alloc_send_skb(sk
,
927 alloclen
+ hh_len
+ 15,
928 (flags
& MSG_DONTWAIT
), &err
);
931 if (atomic_read(&sk
->sk_wmem_alloc
) <=
933 skb
= sock_wmalloc(sk
,
934 alloclen
+ hh_len
+ 15, 1,
936 if (unlikely(skb
== NULL
))
943 * Fill in the control structures
945 skb
->ip_summed
= csummode
;
947 skb_reserve(skb
, hh_len
);
950 * Find where to start putting bytes.
952 data
= skb_put(skb
, fraglen
);
953 skb_set_network_header(skb
, exthdrlen
);
954 skb
->transport_header
= (skb
->network_header
+
956 data
+= fragheaderlen
;
959 skb
->csum
= skb_copy_and_csum_bits(
960 skb_prev
, maxfraglen
,
961 data
+ transhdrlen
, fraggap
, 0);
962 skb_prev
->csum
= csum_sub(skb_prev
->csum
,
965 pskb_trim_unique(skb_prev
, maxfraglen
);
968 copy
= datalen
- transhdrlen
- fraggap
;
969 if (copy
> 0 && getfrag(from
, data
+ transhdrlen
, offset
, copy
, fraggap
, skb
) < 0) {
976 length
-= datalen
- fraggap
;
979 csummode
= CHECKSUM_NONE
;
982 * Put the packet on the pending queue.
984 __skb_queue_tail(&sk
->sk_write_queue
, skb
);
991 if (!(rt
->u
.dst
.dev
->features
&NETIF_F_SG
)) {
995 if (getfrag(from
, skb_put(skb
, copy
),
996 offset
, copy
, off
, skb
) < 0) {
997 __skb_trim(skb
, off
);
1002 int i
= skb_shinfo(skb
)->nr_frags
;
1003 skb_frag_t
*frag
= &skb_shinfo(skb
)->frags
[i
-1];
1004 struct page
*page
= sk
->sk_sndmsg_page
;
1005 int off
= sk
->sk_sndmsg_off
;
1008 if (page
&& (left
= PAGE_SIZE
- off
) > 0) {
1011 if (page
!= frag
->page
) {
1012 if (i
== MAX_SKB_FRAGS
) {
1017 skb_fill_page_desc(skb
, i
, page
, sk
->sk_sndmsg_off
, 0);
1018 frag
= &skb_shinfo(skb
)->frags
[i
];
1020 } else if (i
< MAX_SKB_FRAGS
) {
1021 if (copy
> PAGE_SIZE
)
1023 page
= alloc_pages(sk
->sk_allocation
, 0);
1028 sk
->sk_sndmsg_page
= page
;
1029 sk
->sk_sndmsg_off
= 0;
1031 skb_fill_page_desc(skb
, i
, page
, 0, 0);
1032 frag
= &skb_shinfo(skb
)->frags
[i
];
1037 if (getfrag(from
, page_address(frag
->page
)+frag
->page_offset
+frag
->size
, offset
, copy
, skb
->len
, skb
) < 0) {
1041 sk
->sk_sndmsg_off
+= copy
;
1044 skb
->data_len
+= copy
;
1045 skb
->truesize
+= copy
;
1046 atomic_add(copy
, &sk
->sk_wmem_alloc
);
1055 inet
->cork
.length
-= length
;
1056 IP_INC_STATS(IPSTATS_MIB_OUTDISCARDS
);
1060 ssize_t
ip_append_page(struct sock
*sk
, struct page
*page
,
1061 int offset
, size_t size
, int flags
)
1063 struct inet_sock
*inet
= inet_sk(sk
);
1064 struct sk_buff
*skb
;
1066 struct ip_options
*opt
= NULL
;
1071 unsigned int maxfraglen
, fragheaderlen
, fraggap
;
1076 if (flags
&MSG_PROBE
)
1079 if (skb_queue_empty(&sk
->sk_write_queue
))
1083 if (inet
->cork
.flags
& IPCORK_OPT
)
1084 opt
= inet
->cork
.opt
;
1086 if (!(rt
->u
.dst
.dev
->features
&NETIF_F_SG
))
1089 hh_len
= LL_RESERVED_SPACE(rt
->u
.dst
.dev
);
1090 mtu
= inet
->cork
.fragsize
;
1092 fragheaderlen
= sizeof(struct iphdr
) + (opt
? opt
->optlen
: 0);
1093 maxfraglen
= ((mtu
- fragheaderlen
) & ~7) + fragheaderlen
;
1095 if (inet
->cork
.length
+ size
> 0xFFFF - fragheaderlen
) {
1096 ip_local_error(sk
, EMSGSIZE
, rt
->rt_dst
, inet
->dport
, mtu
);
1100 if ((skb
= skb_peek_tail(&sk
->sk_write_queue
)) == NULL
)
1103 inet
->cork
.length
+= size
;
1104 if ((sk
->sk_protocol
== IPPROTO_UDP
) &&
1105 (rt
->u
.dst
.dev
->features
& NETIF_F_UFO
)) {
1106 skb_shinfo(skb
)->gso_size
= mtu
- fragheaderlen
;
1107 skb_shinfo(skb
)->gso_type
= SKB_GSO_UDP
;
1114 if (skb_is_gso(skb
))
1118 /* Check if the remaining data fits into current packet. */
1119 len
= mtu
- skb
->len
;
1121 len
= maxfraglen
- skb
->len
;
1124 struct sk_buff
*skb_prev
;
1128 fraggap
= skb_prev
->len
- maxfraglen
;
1130 alloclen
= fragheaderlen
+ hh_len
+ fraggap
+ 15;
1131 skb
= sock_wmalloc(sk
, alloclen
, 1, sk
->sk_allocation
);
1132 if (unlikely(!skb
)) {
1138 * Fill in the control structures
1140 skb
->ip_summed
= CHECKSUM_NONE
;
1142 skb_reserve(skb
, hh_len
);
1145 * Find where to start putting bytes.
1147 skb_put(skb
, fragheaderlen
+ fraggap
);
1148 skb_reset_network_header(skb
);
1149 skb
->transport_header
= (skb
->network_header
+
1152 skb
->csum
= skb_copy_and_csum_bits(skb_prev
,
1154 skb_transport_header(skb
),
1156 skb_prev
->csum
= csum_sub(skb_prev
->csum
,
1158 pskb_trim_unique(skb_prev
, maxfraglen
);
1162 * Put the packet on the pending queue.
1164 __skb_queue_tail(&sk
->sk_write_queue
, skb
);
1168 i
= skb_shinfo(skb
)->nr_frags
;
1171 if (skb_can_coalesce(skb
, i
, page
, offset
)) {
1172 skb_shinfo(skb
)->frags
[i
-1].size
+= len
;
1173 } else if (i
< MAX_SKB_FRAGS
) {
1175 skb_fill_page_desc(skb
, i
, page
, offset
, len
);
1181 if (skb
->ip_summed
== CHECKSUM_NONE
) {
1183 csum
= csum_page(page
, offset
, len
);
1184 skb
->csum
= csum_block_add(skb
->csum
, csum
, skb
->len
);
1188 skb
->data_len
+= len
;
1189 skb
->truesize
+= len
;
1190 atomic_add(len
, &sk
->sk_wmem_alloc
);
1197 inet
->cork
.length
-= size
;
1198 IP_INC_STATS(IPSTATS_MIB_OUTDISCARDS
);
1202 static void ip_cork_release(struct inet_sock
*inet
)
1204 inet
->cork
.flags
&= ~IPCORK_OPT
;
1205 kfree(inet
->cork
.opt
);
1206 inet
->cork
.opt
= NULL
;
1207 if (inet
->cork
.rt
) {
1208 ip_rt_put(inet
->cork
.rt
);
1209 inet
->cork
.rt
= NULL
;
1214 * Combined all pending IP fragments on the socket as one IP datagram
1215 * and push them out.
1217 int ip_push_pending_frames(struct sock
*sk
)
1219 struct sk_buff
*skb
, *tmp_skb
;
1220 struct sk_buff
**tail_skb
;
1221 struct inet_sock
*inet
= inet_sk(sk
);
1222 struct ip_options
*opt
= NULL
;
1223 struct rtable
*rt
= inet
->cork
.rt
;
1229 if ((skb
= __skb_dequeue(&sk
->sk_write_queue
)) == NULL
)
1231 tail_skb
= &(skb_shinfo(skb
)->frag_list
);
1233 /* move skb->data to ip header from ext header */
1234 if (skb
->data
< skb_network_header(skb
))
1235 __skb_pull(skb
, skb_network_offset(skb
));
1236 while ((tmp_skb
= __skb_dequeue(&sk
->sk_write_queue
)) != NULL
) {
1237 __skb_pull(tmp_skb
, skb_network_header_len(skb
));
1238 *tail_skb
= tmp_skb
;
1239 tail_skb
= &(tmp_skb
->next
);
1240 skb
->len
+= tmp_skb
->len
;
1241 skb
->data_len
+= tmp_skb
->len
;
1242 skb
->truesize
+= tmp_skb
->truesize
;
1243 __sock_put(tmp_skb
->sk
);
1244 tmp_skb
->destructor
= NULL
;
1248 /* Unless user demanded real pmtu discovery (IP_PMTUDISC_DO), we allow
1249 * to fragment the frame generated here. No matter, what transforms
1250 * how transforms change size of the packet, it will come out.
1252 if (inet
->pmtudisc
< IP_PMTUDISC_DO
)
1255 /* DF bit is set when we want to see DF on outgoing frames.
1256 * If local_df is set too, we still allow to fragment this frame
1258 if (inet
->pmtudisc
>= IP_PMTUDISC_DO
||
1259 (skb
->len
<= dst_mtu(&rt
->u
.dst
) &&
1260 ip_dont_fragment(sk
, &rt
->u
.dst
)))
1263 if (inet
->cork
.flags
& IPCORK_OPT
)
1264 opt
= inet
->cork
.opt
;
1266 if (rt
->rt_type
== RTN_MULTICAST
)
1269 ttl
= ip_select_ttl(inet
, &rt
->u
.dst
);
1271 iph
= (struct iphdr
*)skb
->data
;
1275 iph
->ihl
+= opt
->optlen
>>2;
1276 ip_options_build(skb
, opt
, inet
->cork
.addr
, rt
, 0);
1278 iph
->tos
= inet
->tos
;
1280 ip_select_ident(iph
, &rt
->u
.dst
, sk
);
1282 iph
->protocol
= sk
->sk_protocol
;
1283 iph
->saddr
= rt
->rt_src
;
1284 iph
->daddr
= rt
->rt_dst
;
1286 skb
->priority
= sk
->sk_priority
;
1287 skb
->dst
= dst_clone(&rt
->u
.dst
);
1289 if (iph
->protocol
== IPPROTO_ICMP
)
1290 icmp_out_count(((struct icmphdr
*)
1291 skb_transport_header(skb
))->type
);
1293 /* Netfilter gets whole the not fragmented skb. */
1294 err
= ip_local_out(skb
);
1297 err
= inet
->recverr
? net_xmit_errno(err
) : 0;
1303 ip_cork_release(inet
);
1307 IP_INC_STATS(IPSTATS_MIB_OUTDISCARDS
);
1312 * Throw away all pending data on the socket.
1314 void ip_flush_pending_frames(struct sock
*sk
)
1316 struct sk_buff
*skb
;
1318 while ((skb
= __skb_dequeue_tail(&sk
->sk_write_queue
)) != NULL
)
1321 ip_cork_release(inet_sk(sk
));
1326 * Fetch data from kernel space and fill in checksum if needed.
1328 static int ip_reply_glue_bits(void *dptr
, char *to
, int offset
,
1329 int len
, int odd
, struct sk_buff
*skb
)
1333 csum
= csum_partial_copy_nocheck(dptr
+offset
, to
, len
, 0);
1334 skb
->csum
= csum_block_add(skb
->csum
, csum
, odd
);
1339 * Generic function to send a packet as reply to another packet.
1340 * Used to send TCP resets so far. ICMP should use this function too.
1342 * Should run single threaded per socket because it uses the sock
1343 * structure to pass arguments.
1345 * LATER: switch from ip_build_xmit to ip_append_*
1347 void ip_send_reply(struct sock
*sk
, struct sk_buff
*skb
, struct ip_reply_arg
*arg
,
1350 struct inet_sock
*inet
= inet_sk(sk
);
1352 struct ip_options opt
;
1355 struct ipcm_cookie ipc
;
1357 struct rtable
*rt
= (struct rtable
*)skb
->dst
;
1359 if (ip_options_echo(&replyopts
.opt
, skb
))
1362 daddr
= ipc
.addr
= rt
->rt_src
;
1365 if (replyopts
.opt
.optlen
) {
1366 ipc
.opt
= &replyopts
.opt
;
1369 daddr
= replyopts
.opt
.faddr
;
1373 struct flowi fl
= { .oif
= arg
->bound_dev_if
,
1376 .saddr
= rt
->rt_spec_dst
,
1377 .tos
= RT_TOS(ip_hdr(skb
)->tos
) } },
1378 /* Not quite clean, but right. */
1380 { .sport
= tcp_hdr(skb
)->dest
,
1381 .dport
= tcp_hdr(skb
)->source
} },
1382 .proto
= sk
->sk_protocol
};
1383 security_skb_classify_flow(skb
, &fl
);
1384 if (ip_route_output_key(&rt
, &fl
))
1388 /* And let IP do all the hard work.
1390 This chunk is not reenterable, hence spinlock.
1391 Note that it uses the fact, that this function is called
1392 with locally disabled BH and that sk cannot be already spinlocked.
1395 inet
->tos
= ip_hdr(skb
)->tos
;
1396 sk
->sk_priority
= skb
->priority
;
1397 sk
->sk_protocol
= ip_hdr(skb
)->protocol
;
1398 sk
->sk_bound_dev_if
= arg
->bound_dev_if
;
1399 ip_append_data(sk
, ip_reply_glue_bits
, arg
->iov
->iov_base
, len
, 0,
1400 &ipc
, rt
, MSG_DONTWAIT
);
1401 if ((skb
= skb_peek(&sk
->sk_write_queue
)) != NULL
) {
1402 if (arg
->csumoffset
>= 0)
1403 *((__sum16
*)skb_transport_header(skb
) +
1404 arg
->csumoffset
) = csum_fold(csum_add(skb
->csum
,
1406 skb
->ip_summed
= CHECKSUM_NONE
;
1407 ip_push_pending_frames(sk
);
1415 void __init
ip_init(void)
1420 #if defined(CONFIG_IP_MULTICAST) && defined(CONFIG_PROC_FS)
1421 igmp_mc_proc_init();
1425 EXPORT_SYMBOL(ip_generic_getfrag
);
1426 EXPORT_SYMBOL(ip_queue_xmit
);
1427 EXPORT_SYMBOL(ip_send_check
);