?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Committer: Michael Beasley <mike@snafu.setup>
[mikesnafu-overlay.git] / net / ipv4 / inet_lro.c
blob4a4d49fca1f2518a12eebbfbae02211eace06a19
1 /*
2 * linux/net/ipv4/inet_lro.c
3 *
4 * Large Receive Offload (ipv4 / tcp)
5 *
6 * (C) Copyright IBM Corp. 2007
7 *
8 * Authors:
9 * Jan-Bernd Themann <themann@de.ibm.com>
10 * Christoph Raisch <raisch@de.ibm.com>
11 *
12 *
13 * This program is free software; you can redistribute it and/or modify
14 * it under the terms of the GNU General Public License as published by
15 * the Free Software Foundation; either version 2, or (at your option)
16 * any later version.
17 *
18 * This program is distributed in the hope that it will be useful,
19 * but WITHOUT ANY WARRANTY; without even the implied warranty of
20 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 * GNU General Public License for more details.
22 *
23 * You should have received a copy of the GNU General Public License
24 * along with this program; if not, write to the Free Software
25 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
26 */
27
28
29 #include <linux/module.h>
30 #include <linux/if_vlan.h>
31 #include <linux/inet_lro.h>
32
33 MODULE_LICENSE("GPL");
34 MODULE_AUTHOR("Jan-Bernd Themann <themann@de.ibm.com>");
35 MODULE_DESCRIPTION("Large Receive Offload (ipv4 / tcp)");
36
37 #define TCP_HDR_LEN(tcph) (tcph->doff << 2)
38 #define IP_HDR_LEN(iph) (iph->ihl << 2)
39 #define TCP_PAYLOAD_LENGTH(iph, tcph) \
40 (ntohs(iph->tot_len) - IP_HDR_LEN(iph) - TCP_HDR_LEN(tcph))
41
42 #define IPH_LEN_WO_OPTIONS 5
43 #define TCPH_LEN_WO_OPTIONS 5
44 #define TCPH_LEN_W_TIMESTAMP 8
45
46 #define LRO_MAX_PG_HLEN 64
47
48 #define LRO_INC_STATS(lro_mgr, attr) { lro_mgr->stats.attr++; }
49
50 /*
51 * Basic tcp checks whether packet is suitable for LRO
52 */
53
54 static int lro_tcp_ip_check(struct iphdr *iph, struct tcphdr *tcph,
55 int len, struct net_lro_desc *lro_desc)
56 {
57 /* check ip header: don't aggregate padded frames */
58 if (ntohs(iph->tot_len) != len)
59 return -1;
60
61 if (TCP_PAYLOAD_LENGTH(iph, tcph) == 0)
62 return -1;
63
64 if (iph->ihl != IPH_LEN_WO_OPTIONS)
65 return -1;
66
67 if (tcph->cwr || tcph->ece || tcph->urg || !tcph->ack
68 || tcph->rst || tcph->syn || tcph->fin)
69 return -1;
70
71 if (INET_ECN_is_ce(ipv4_get_dsfield(iph)))
72 return -1;
73
74 if (tcph->doff != TCPH_LEN_WO_OPTIONS
75 && tcph->doff != TCPH_LEN_W_TIMESTAMP)
76 return -1;
77
78 /* check tcp options (only timestamp allowed) */
79 if (tcph->doff == TCPH_LEN_W_TIMESTAMP) {
80 __be32 *topt = (__be32 *)(tcph + 1);
81
82 if (*topt != htonl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16)
83 | (TCPOPT_TIMESTAMP << 8)
84 | TCPOLEN_TIMESTAMP))
85 return -1;
86
87 /* timestamp should be in right order */
88 topt++;
89 if (lro_desc && after(ntohl(lro_desc->tcp_rcv_tsval),
90 ntohl(*topt)))
91 return -1;
92
93 /* timestamp reply should not be zero */
94 topt++;
95 if (*topt == 0)
96 return -1;
97 }
98
99 return 0;
100 }
101
102 static void lro_update_tcp_ip_header(struct net_lro_desc *lro_desc)
103 {
104 struct iphdr *iph = lro_desc->iph;
105 struct tcphdr *tcph = lro_desc->tcph;
106 __be32 *p;
107 __wsum tcp_hdr_csum;
108
109 tcph->ack_seq = lro_desc->tcp_ack;
110 tcph->window = lro_desc->tcp_window;
111
112 if (lro_desc->tcp_saw_tstamp) {
113 p = (__be32 *)(tcph + 1);
114 *(p+2) = lro_desc->tcp_rcv_tsecr;
115 }
116
117 iph->tot_len = htons(lro_desc->ip_tot_len);
118
119 iph->check = 0;
120 iph->check = ip_fast_csum((u8 *)lro_desc->iph, iph->ihl);
121
122 tcph->check = 0;
123 tcp_hdr_csum = csum_partial((u8 *)tcph, TCP_HDR_LEN(tcph), 0);
124 lro_desc->data_csum = csum_add(lro_desc->data_csum, tcp_hdr_csum);
125 tcph->check = csum_tcpudp_magic(iph->saddr, iph->daddr,
126 lro_desc->ip_tot_len -
127 IP_HDR_LEN(iph), IPPROTO_TCP,
128 lro_desc->data_csum);
129 }
130
131 static __wsum lro_tcp_data_csum(struct iphdr *iph, struct tcphdr *tcph, int len)
132 {
133 __wsum tcp_csum;
134 __wsum tcp_hdr_csum;
135 __wsum tcp_ps_hdr_csum;
136
137 tcp_csum = ~csum_unfold(tcph->check);
138 tcp_hdr_csum = csum_partial((u8 *)tcph, TCP_HDR_LEN(tcph), tcp_csum);
139
140 tcp_ps_hdr_csum = csum_tcpudp_nofold(iph->saddr, iph->daddr,
141 len + TCP_HDR_LEN(tcph),
142 IPPROTO_TCP, 0);
143
144 return csum_sub(csum_sub(tcp_csum, tcp_hdr_csum),
145 tcp_ps_hdr_csum);
146 }
147
148 static void lro_init_desc(struct net_lro_desc *lro_desc, struct sk_buff *skb,
149 struct iphdr *iph, struct tcphdr *tcph,
150 u16 vlan_tag, struct vlan_group *vgrp)
151 {
152 int nr_frags;
153 __be32 *ptr;
154 u32 tcp_data_len = TCP_PAYLOAD_LENGTH(iph, tcph);
155
156 nr_frags = skb_shinfo(skb)->nr_frags;
157 lro_desc->parent = skb;
158 lro_desc->next_frag = &(skb_shinfo(skb)->frags[nr_frags]);
159 lro_desc->iph = iph;
160 lro_desc->tcph = tcph;
161 lro_desc->tcp_next_seq = ntohl(tcph->seq) + tcp_data_len;
162 lro_desc->tcp_ack = tcph->ack_seq;
163 lro_desc->tcp_window = tcph->window;
164
165 lro_desc->pkt_aggr_cnt = 1;
166 lro_desc->ip_tot_len = ntohs(iph->tot_len);
167
168 if (tcph->doff == 8) {
169 ptr = (__be32 *)(tcph+1);
170 lro_desc->tcp_saw_tstamp = 1;
171 lro_desc->tcp_rcv_tsval = *(ptr+1);
172 lro_desc->tcp_rcv_tsecr = *(ptr+2);
173 }
174
175 lro_desc->mss = tcp_data_len;
176 lro_desc->vgrp = vgrp;
177 lro_desc->vlan_tag = vlan_tag;
178 lro_desc->active = 1;
179
180 lro_desc->data_csum = lro_tcp_data_csum(iph, tcph,
181 tcp_data_len);
182 }
183
184 static inline void lro_clear_desc(struct net_lro_desc *lro_desc)
185 {
186 memset(lro_desc, 0, sizeof(struct net_lro_desc));
187 }
188
189 static void lro_add_common(struct net_lro_desc *lro_desc, struct iphdr *iph,
190 struct tcphdr *tcph, int tcp_data_len)
191 {
192 struct sk_buff *parent = lro_desc->parent;
193 __be32 *topt;
194
195 lro_desc->pkt_aggr_cnt++;
196 lro_desc->ip_tot_len += tcp_data_len;
197 lro_desc->tcp_next_seq += tcp_data_len;
198 lro_desc->tcp_window = tcph->window;
199 lro_desc->tcp_ack = tcph->ack_seq;
200
201 /* don't update tcp_rcv_tsval, would not work with PAWS */
202 if (lro_desc->tcp_saw_tstamp) {
203 topt = (__be32 *) (tcph + 1);
204 lro_desc->tcp_rcv_tsecr = *(topt + 2);
205 }
206
207 lro_desc->data_csum = csum_block_add(lro_desc->data_csum,
208 lro_tcp_data_csum(iph, tcph,
209 tcp_data_len),
210 parent->len);
211
212 parent->len += tcp_data_len;
213 parent->data_len += tcp_data_len;
214 if (tcp_data_len > lro_desc->mss)
215 lro_desc->mss = tcp_data_len;
216 }
217
218 static void lro_add_packet(struct net_lro_desc *lro_desc, struct sk_buff *skb,
219 struct iphdr *iph, struct tcphdr *tcph)
220 {
221 struct sk_buff *parent = lro_desc->parent;
222 int tcp_data_len = TCP_PAYLOAD_LENGTH(iph, tcph);
223
224 lro_add_common(lro_desc, iph, tcph, tcp_data_len);
225
226 skb_pull(skb, (skb->len - tcp_data_len));
227 parent->truesize += skb->truesize;
228
229 if (lro_desc->last_skb)
230 lro_desc->last_skb->next = skb;
231 else
232 skb_shinfo(parent)->frag_list = skb;
233
234 lro_desc->last_skb = skb;
235 }
236
237 static void lro_add_frags(struct net_lro_desc *lro_desc,
238 int len, int hlen, int truesize,
239 struct skb_frag_struct *skb_frags,
240 struct iphdr *iph, struct tcphdr *tcph)
241 {
242 struct sk_buff *skb = lro_desc->parent;
243 int tcp_data_len = TCP_PAYLOAD_LENGTH(iph, tcph);
244
245 lro_add_common(lro_desc, iph, tcph, tcp_data_len);
246
247 skb->truesize += truesize;
248
249 skb_frags[0].page_offset += hlen;
250 skb_frags[0].size -= hlen;
251
252 while (tcp_data_len > 0) {
253 *(lro_desc->next_frag) = *skb_frags;
254 tcp_data_len -= skb_frags->size;
255 lro_desc->next_frag++;
256 skb_frags++;
257 skb_shinfo(skb)->nr_frags++;
258 }
259 }
260
261 static int lro_check_tcp_conn(struct net_lro_desc *lro_desc,
262 struct iphdr *iph,
263 struct tcphdr *tcph)
264 {
265 if ((lro_desc->iph->saddr != iph->saddr)
266 || (lro_desc->iph->daddr != iph->daddr)
267 || (lro_desc->tcph->source != tcph->source)
268 || (lro_desc->tcph->dest != tcph->dest))
269 return -1;
270 return 0;
271 }
272
273 static struct net_lro_desc *lro_get_desc(struct net_lro_mgr *lro_mgr,
274 struct net_lro_desc *lro_arr,
275 struct iphdr *iph,
276 struct tcphdr *tcph)
277 {
278 struct net_lro_desc *lro_desc = NULL;
279 struct net_lro_desc *tmp;
280 int max_desc = lro_mgr->max_desc;
281 int i;
282
283 for (i = 0; i < max_desc; i++) {
284 tmp = &lro_arr[i];
285 if (tmp->active)
286 if (!lro_check_tcp_conn(tmp, iph, tcph)) {
287 lro_desc = tmp;
288 goto out;
289 }
290 }
291
292 for (i = 0; i < max_desc; i++) {
293 if (!lro_arr[i].active) {
294 lro_desc = &lro_arr[i];
295 goto out;
296 }
297 }
298
299 LRO_INC_STATS(lro_mgr, no_desc);
300 out:
301 return lro_desc;
302 }
303
304 static void lro_flush(struct net_lro_mgr *lro_mgr,
305 struct net_lro_desc *lro_desc)
306 {
307 if (lro_desc->pkt_aggr_cnt > 1)
308 lro_update_tcp_ip_header(lro_desc);
309
310 skb_shinfo(lro_desc->parent)->gso_size = lro_desc->mss;
311
312 if (lro_desc->vgrp) {
313 if (lro_mgr->features & LRO_F_NAPI)
314 vlan_hwaccel_receive_skb(lro_desc->parent,
315 lro_desc->vgrp,
316 lro_desc->vlan_tag);
317 else
318 vlan_hwaccel_rx(lro_desc->parent,
319 lro_desc->vgrp,
320 lro_desc->vlan_tag);
321
322 } else {
323 if (lro_mgr->features & LRO_F_NAPI)
324 netif_receive_skb(lro_desc->parent);
325 else
326 netif_rx(lro_desc->parent);
327 }
328
329 LRO_INC_STATS(lro_mgr, flushed);
330 lro_clear_desc(lro_desc);
331 }
332
333 static int __lro_proc_skb(struct net_lro_mgr *lro_mgr, struct sk_buff *skb,
334 struct vlan_group *vgrp, u16 vlan_tag, void *priv)
335 {
336 struct net_lro_desc *lro_desc;
337 struct iphdr *iph;
338 struct tcphdr *tcph;
339 u64 flags;
340 int vlan_hdr_len = 0;
341
342 if (!lro_mgr->get_skb_header
343 || lro_mgr->get_skb_header(skb, (void *)&iph, (void *)&tcph,
344 &flags, priv))
345 goto out;
346
347 if (!(flags & LRO_IPV4) || !(flags & LRO_TCP))
348 goto out;
349
350 lro_desc = lro_get_desc(lro_mgr, lro_mgr->lro_arr, iph, tcph);
351 if (!lro_desc)
352 goto out;
353
354 if ((skb->protocol == htons(ETH_P_8021Q))
355 && !(lro_mgr->features & LRO_F_EXTRACT_VLAN_ID))
356 vlan_hdr_len = VLAN_HLEN;
357
358 if (!lro_desc->active) { /* start new lro session */
359 if (lro_tcp_ip_check(iph, tcph, skb->len - vlan_hdr_len, NULL))
360 goto out;
361
362 skb->ip_summed = lro_mgr->ip_summed_aggr;
363 lro_init_desc(lro_desc, skb, iph, tcph, vlan_tag, vgrp);
364 LRO_INC_STATS(lro_mgr, aggregated);
365 return 0;
366 }
367
368 if (lro_desc->tcp_next_seq != ntohl(tcph->seq))
369 goto out2;
370
371 if (lro_tcp_ip_check(iph, tcph, skb->len, lro_desc))
372 goto out2;
373
374 lro_add_packet(lro_desc, skb, iph, tcph);
375 LRO_INC_STATS(lro_mgr, aggregated);
376
377 if ((lro_desc->pkt_aggr_cnt >= lro_mgr->max_aggr) ||
378 lro_desc->parent->len > (0xFFFF - lro_mgr->dev->mtu))
379 lro_flush(lro_mgr, lro_desc);
380
381 return 0;
382
383 out2: /* send aggregated SKBs to stack */
384 lro_flush(lro_mgr, lro_desc);
385
386 out: /* Original SKB has to be posted to stack */
387 skb->ip_summed = lro_mgr->ip_summed;
388 return 1;
389 }
390
391
392 static struct sk_buff *lro_gen_skb(struct net_lro_mgr *lro_mgr,
393 struct skb_frag_struct *frags,
394 int len, int true_size,
395 void *mac_hdr,
396 int hlen, __wsum sum,
397 u32 ip_summed)
398 {
399 struct sk_buff *skb;
400 struct skb_frag_struct *skb_frags;
401 int data_len = len;
402 int hdr_len = min(len, hlen);
403
404 skb = netdev_alloc_skb(lro_mgr->dev, hlen + lro_mgr->frag_align_pad);
405 if (!skb)
406 return NULL;
407
408 skb_reserve(skb, lro_mgr->frag_align_pad);
409 skb->len = len;
410 skb->data_len = len - hdr_len;
411 skb->truesize += true_size;
412 skb->tail += hdr_len;
413
414 memcpy(skb->data, mac_hdr, hdr_len);
415
416 skb_frags = skb_shinfo(skb)->frags;
417 while (data_len > 0) {
418 *skb_frags = *frags;
419 data_len -= frags->size;
420 skb_frags++;
421 frags++;
422 skb_shinfo(skb)->nr_frags++;
423 }
424
425 skb_shinfo(skb)->frags[0].page_offset += hdr_len;
426 skb_shinfo(skb)->frags[0].size -= hdr_len;
427
428 skb->ip_summed = ip_summed;
429 skb->csum = sum;
430 skb->protocol = eth_type_trans(skb, lro_mgr->dev);
431 return skb;
432 }
433
434 static struct sk_buff *__lro_proc_segment(struct net_lro_mgr *lro_mgr,
435 struct skb_frag_struct *frags,
436 int len, int true_size,
437 struct vlan_group *vgrp,
438 u16 vlan_tag, void *priv, __wsum sum)
439 {
440 struct net_lro_desc *lro_desc;
441 struct iphdr *iph;
442 struct tcphdr *tcph;
443 struct sk_buff *skb;
444 u64 flags;
445 void *mac_hdr;
446 int mac_hdr_len;
447 int hdr_len = LRO_MAX_PG_HLEN;
448 int vlan_hdr_len = 0;
449
450 if (!lro_mgr->get_frag_header
451 || lro_mgr->get_frag_header(frags, (void *)&mac_hdr, (void *)&iph,
452 (void *)&tcph, &flags, priv)) {
453 mac_hdr = page_address(frags->page) + frags->page_offset;
454 goto out1;
455 }
456
457 if (!(flags & LRO_IPV4) || !(flags & LRO_TCP))
458 goto out1;
459
460 hdr_len = (int)((void *)(tcph) + TCP_HDR_LEN(tcph) - mac_hdr);
461 mac_hdr_len = (int)((void *)(iph) - mac_hdr);
462
463 lro_desc = lro_get_desc(lro_mgr, lro_mgr->lro_arr, iph, tcph);
464 if (!lro_desc)
465 goto out1;
466
467 if (!lro_desc->active) { /* start new lro session */
468 if (lro_tcp_ip_check(iph, tcph, len - mac_hdr_len, NULL))
469 goto out1;
470
471 skb = lro_gen_skb(lro_mgr, frags, len, true_size, mac_hdr,
472 hdr_len, 0, lro_mgr->ip_summed_aggr);
473 if (!skb)
474 goto out;
475
476 if ((skb->protocol == htons(ETH_P_8021Q))
477 && !(lro_mgr->features & LRO_F_EXTRACT_VLAN_ID))
478 vlan_hdr_len = VLAN_HLEN;
479
480 iph = (void *)(skb->data + vlan_hdr_len);
481 tcph = (void *)((u8 *)skb->data + vlan_hdr_len
482 + IP_HDR_LEN(iph));
483
484 lro_init_desc(lro_desc, skb, iph, tcph, 0, NULL);
485 LRO_INC_STATS(lro_mgr, aggregated);
486 return NULL;
487 }
488
489 if (lro_desc->tcp_next_seq != ntohl(tcph->seq))
490 goto out2;
491
492 if (lro_tcp_ip_check(iph, tcph, len - mac_hdr_len, lro_desc))
493 goto out2;
494
495 lro_add_frags(lro_desc, len, hdr_len, true_size, frags, iph, tcph);
496 LRO_INC_STATS(lro_mgr, aggregated);
497
498 if ((skb_shinfo(lro_desc->parent)->nr_frags >= lro_mgr->max_aggr) ||
499 lro_desc->parent->len > (0xFFFF - lro_mgr->dev->mtu))
500 lro_flush(lro_mgr, lro_desc);
501
502 return NULL;
503
504 out2: /* send aggregated packets to the stack */
505 lro_flush(lro_mgr, lro_desc);
506
507 out1: /* Original packet has to be posted to the stack */
508 skb = lro_gen_skb(lro_mgr, frags, len, true_size, mac_hdr,
509 hdr_len, sum, lro_mgr->ip_summed);
510 out:
511 return skb;
512 }
513
514 void lro_receive_skb(struct net_lro_mgr *lro_mgr,
515 struct sk_buff *skb,
516 void *priv)
517 {
518 if (__lro_proc_skb(lro_mgr, skb, NULL, 0, priv)) {
519 if (lro_mgr->features & LRO_F_NAPI)
520 netif_receive_skb(skb);
521 else
522 netif_rx(skb);
523 }
524 }
525 EXPORT_SYMBOL(lro_receive_skb);
526
527 void lro_vlan_hwaccel_receive_skb(struct net_lro_mgr *lro_mgr,
528 struct sk_buff *skb,
529 struct vlan_group *vgrp,
530 u16 vlan_tag,
531 void *priv)
532 {
533 if (__lro_proc_skb(lro_mgr, skb, vgrp, vlan_tag, priv)) {
534 if (lro_mgr->features & LRO_F_NAPI)
535 vlan_hwaccel_receive_skb(skb, vgrp, vlan_tag);
536 else
537 vlan_hwaccel_rx(skb, vgrp, vlan_tag);
538 }
539 }
540 EXPORT_SYMBOL(lro_vlan_hwaccel_receive_skb);
541
542 void lro_receive_frags(struct net_lro_mgr *lro_mgr,
543 struct skb_frag_struct *frags,
544 int len, int true_size, void *priv, __wsum sum)
545 {
546 struct sk_buff *skb;
547
548 skb = __lro_proc_segment(lro_mgr, frags, len, true_size, NULL, 0,
549 priv, sum);
550 if (!skb)
551 return;
552
553 if (lro_mgr->features & LRO_F_NAPI)
554 netif_receive_skb(skb);
555 else
556 netif_rx(skb);
557 }
558 EXPORT_SYMBOL(lro_receive_frags);
559
560 void lro_vlan_hwaccel_receive_frags(struct net_lro_mgr *lro_mgr,
561 struct skb_frag_struct *frags,
562 int len, int true_size,
563 struct vlan_group *vgrp,
564 u16 vlan_tag, void *priv, __wsum sum)
565 {
566 struct sk_buff *skb;
567
568 skb = __lro_proc_segment(lro_mgr, frags, len, true_size, vgrp,
569 vlan_tag, priv, sum);
570 if (!skb)
571 return;
572
573 if (lro_mgr->features & LRO_F_NAPI)
574 vlan_hwaccel_receive_skb(skb, vgrp, vlan_tag);
575 else
576 vlan_hwaccel_rx(skb, vgrp, vlan_tag);
577 }
578 EXPORT_SYMBOL(lro_vlan_hwaccel_receive_frags);
579
580 void lro_flush_all(struct net_lro_mgr *lro_mgr)
581 {
582 int i;
583 struct net_lro_desc *lro_desc = lro_mgr->lro_arr;
584
585 for (i = 0; i < lro_mgr->max_desc; i++) {
586 if (lro_desc[i].active)
587 lro_flush(lro_mgr, &lro_desc[i]);
588 }
589 }
590 EXPORT_SYMBOL(lro_flush_all);
591
592 void lro_flush_pkt(struct net_lro_mgr *lro_mgr,
593 struct iphdr *iph, struct tcphdr *tcph)
594 {
595 struct net_lro_desc *lro_desc;
596
597 lro_desc = lro_get_desc(lro_mgr, lro_mgr->lro_arr, iph, tcph);
598 if (lro_desc->active)
599 lro_flush(lro_mgr, lro_desc);
600 }
601 EXPORT_SYMBOL(lro_flush_pkt);
mah project overlay