kernel/drivers/net/myri10ge/myri10ge_lro.c

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21
  22 /*
  23  * Copyright 2007-2009 Myricom, Inc.  All rights reserved.
  24  * Use is subject to license terms.
  25  */
  26
  27 static const char __idstring[] =
  28         "@(#)$Id: myri10ge_lro.c,v 1.7 2009-06-29 13:47:22 gallatin Exp $";
  29
  30 #include "myri10ge_var.h"
  31
  32 #define IP_OFFMASK 0x1fff
  33 #define TCPOPT_TIMESTAMP 8
  34 #define TCPOLEN_TIMESTAMP 10
  35 #define TCPOLEN_TSTAMP_APPA 12
  36
  37
  38 /*
  39  * Assume len is a multiple of 4. Note that "raw" must be
  40  * suitably aligned. In practice, it will always enter algned on
  41  * at least a 4 bytes bounday, due to the alignment of our rx buffers.
  42  */
  43 uint16_t
  44 myri10ge_csum_generic(uint16_t *raw, int len)
  45 {
  46         uint32_t csum;
  47         csum = 0;
  48         while (len > 0) {
  49                 csum += *raw;
  50                 raw++;
  51                 csum += *raw;
  52                 raw++;
  53                 len -= 4;
  54         }
  55         csum = (csum >> 16) + (csum & 0xffff);
  56         csum = (csum >> 16) + (csum & 0xffff);
  57         return ((uint16_t)csum);
  58 }
  59
  60 static uint16_t
  61 myri10ge_in_pseudo(unsigned int a, unsigned int b,
  62     unsigned int c)
  63 {
  64         uint64_t csum;
  65
  66         csum = (uint64_t)a + b + c;
  67         csum = (csum >> 16) + (csum & 0xffff);
  68         csum = (csum >> 16) + (csum & 0xffff);
  69         return ((uint16_t)csum);
  70 }
  71
  72 void
  73 myri10ge_lro_flush(struct myri10ge_slice_state *ss, struct lro_entry *lro,
  74         struct myri10ge_mblk_list *mbl)
  75 {
  76         struct ip *ip;
  77         struct tcphdr *tcp;
  78         uint32_t *ts_ptr;
  79         uint32_t tcplen, tcp_csum;
  80
  81         if (lro->append_cnt) {
  82                 /*
  83                  * incorporate the new len into the ip header and
  84                  * re-calculate the checksum
  85                  */
  86                 ip = lro->ip;
  87                 ip->ip_len = htons(lro->len - ETHERNET_HEADER_SIZE);
  88                 ip->ip_sum = 0;
  89                 ip->ip_sum = 0xffff ^
  90                     myri10ge_csum_generic((uint16_t *)ip, sizeof (*ip));
  91                 /* incorporate the latest ack into the tcp header */
  92                 tcp = (struct tcphdr *)(ip + 1);
  93                 tcp->th_ack = lro->ack_seq;
  94                 tcp->th_win = lro->window;
  95                 tcp->th_flags = lro->flags;
  96                 /* incorporate latest timestamp into the tcp header */
  97                 if (lro->timestamp) {
  98                         ts_ptr = (uint32_t *)(tcp + 1);
  99                         ts_ptr[1] = htonl(lro->tsval);
 100                         ts_ptr[2] = lro->tsecr;
 101                 }
 102                 /*
 103                  * update checksum in tcp header by re-calculating the
 104                  * tcp pseudoheader checksum, and adding it to the checksum
 105                  * of the tcp payload data
 106                  */
 107                 tcp->th_sum = 0;
 108                 tcplen = lro->len - sizeof (*ip) - ETHERNET_HEADER_SIZE;
 109                 tcp_csum = lro->data_csum;
 110                 tcp_csum += myri10ge_in_pseudo(ip->ip_src.s_addr,
 111                     ip->ip_dst.s_addr, htons(tcplen + IPPROTO_TCP));
 112                 tcp_csum += myri10ge_csum_generic((uint16_t *)tcp,
 113                     tcp->th_off << 2);
 114                 tcp_csum = (tcp_csum & 0xffff) + (tcp_csum >> 16);
 115                 tcp_csum = (tcp_csum & 0xffff) + (tcp_csum >> 16);
 116                 tcp->th_sum = 0xffff ^ tcp_csum;
 117         }
 118
 119         mac_hcksum_set(lro->m_head, 0, 0, 0,
 120             0, HCK_IPV4_HDRCKSUM_OK | HCK_FULLCKSUM_OK);
 121
 122         mbl->cnt += lro->append_cnt;
 123         myri10ge_mbl_append(ss, mbl, lro->m_head);
 124         MYRI10GE_SLICE_STAT_INC(lro_flushed);
 125         MYRI10GE_SLICE_STAT_ADD(lro_queued, lro->append_cnt + 1);
 126         lro->m_head = NULL;
 127         lro->timestamp = 0;
 128         lro->append_cnt = 0;
 129         lro->next = ss->lro_free;
 130         ss->lro_free = lro;
 131 }
 132
 133 int
 134 myri10ge_lro_rx(struct myri10ge_slice_state *ss, mblk_t *m_head,
 135                 uint32_t csum, struct myri10ge_mblk_list *mbl)
 136 {
 137         struct ether_header *eh;
 138         struct ip *ip;
 139         struct tcphdr *tcp;
 140         uint32_t *ts_ptr;
 141         struct lro_entry *lro, *curr;
 142         int hlen, ip_len, tcp_hdr_len, tcp_data_len;
 143         int opt_bytes, trim;
 144         int tot_len = MBLKL(m_head);
 145         uint32_t seq, tmp_csum;
 146
 147         eh = (struct ether_header *)(void *)m_head->b_rptr;
 148         if (eh->ether_type != htons(ETHERTYPE_IP))
 149                 return (EINVAL);
 150         ip = (struct ip *)(void *)(eh + 1);
 151         if (ip->ip_p != IPPROTO_TCP)
 152                 return (EINVAL);
 153
 154         /* ensure there are no options */
 155         if ((ip->ip_hl << 2) != sizeof (*ip))
 156                 return (EINVAL);
 157
 158         /* .. and the packet is not fragmented */
 159         if (ip->ip_off & htons(IP_MF|IP_OFFMASK))
 160                 return (EINVAL);
 161
 162         /* verify that the IP header checksum is correct */
 163         tmp_csum = myri10ge_csum_generic((uint16_t *)ip, sizeof (*ip));
 164         if (unlikely((tmp_csum ^ 0xffff) != 0)) {
 165                 MYRI10GE_SLICE_STAT_INC(lro_bad_csum);
 166                 return (EINVAL);
 167         }
 168
 169         /* find the TCP header */
 170         tcp = (struct tcphdr *)(ip + 1);
 171
 172         /* ensure no bits set besides ack or psh */
 173         if ((tcp->th_flags & ~(TH_ACK | TH_PUSH)) != 0)
 174                 return (EINVAL);
 175
 176         /*
 177          * check for timestamps. Since the only option we handle are
 178          * timestamps, we only have to handle the simple case of
 179          * aligned timestamps
 180          */
 181
 182         opt_bytes = (tcp->th_off << 2) - sizeof (*tcp);
 183         tcp_hdr_len =  sizeof (*tcp) + opt_bytes;
 184         ts_ptr = (uint32_t *)(tcp + 1);
 185         if (opt_bytes != 0) {
 186                 if (unlikely(opt_bytes != TCPOLEN_TSTAMP_APPA) ||
 187                     (*ts_ptr !=  ntohl(TCPOPT_NOP<<24|TCPOPT_NOP<<16|
 188                     TCPOPT_TIMESTAMP<<8|TCPOLEN_TIMESTAMP)))
 189                         return (EINVAL);
 190         }
 191
 192         ip_len = ntohs(ip->ip_len);
 193         tcp_data_len = ip_len - (tcp->th_off << 2) - sizeof (*ip);
 194
 195         /*
 196          * If frame is padded beyond the end of the IP packet,
 197          * then we must trim the extra bytes off the end.
 198          */
 199         trim = tot_len - (ip_len + ETHERNET_HEADER_SIZE);
 200         if (trim != 0) {
 201                 if (trim < 0) {
 202                         /* truncated packet */
 203                         return (EINVAL);
 204                 }
 205                 m_head->b_wptr -= trim;
 206                 tot_len -= trim;
 207         }
 208
 209         /* Verify TCP checksum */
 210         csum = ntohs((uint16_t)csum);
 211         tmp_csum = csum + myri10ge_in_pseudo(ip->ip_src.s_addr,
 212             ip->ip_dst.s_addr, htons(tcp_hdr_len + tcp_data_len + IPPROTO_TCP));
 213         tmp_csum = (tmp_csum & 0xffff) + (tmp_csum >> 16);
 214         tmp_csum = (tmp_csum & 0xffff) + (tmp_csum >> 16);
 215         if (tmp_csum != 0xffff) {
 216                 MYRI10GE_SLICE_STAT_INC(lro_bad_csum);
 217                 return (EINVAL);
 218         }
 219
 220         hlen = ip_len + ETHERNET_HEADER_SIZE - tcp_data_len;
 221         seq = ntohl(tcp->th_seq);
 222
 223         for (lro = ss->lro_active; lro != NULL; lro = lro->next) {
 224                 if (lro->source_port == tcp->th_sport &&
 225                     lro->dest_port == tcp->th_dport &&
 226                     lro->source_ip == ip->ip_src.s_addr &&
 227                     lro->dest_ip == ip->ip_dst.s_addr) {
 228                         /* Try to append it */
 229
 230                         if (unlikely(seq != lro->next_seq)) {
 231                                 /* out of order packet */
 232                                 if (ss->lro_active == lro) {
 233                                         ss->lro_active = lro->next;
 234                                 } else {
 235                                         curr = ss->lro_active;
 236                                         while (curr->next != lro)
 237                                                 curr = curr->next;
 238                                         curr->next = lro->next;
 239                                 }
 240                                 myri10ge_lro_flush(ss, lro, mbl);
 241                                 return (EINVAL);
 242                         }
 243
 244                         if (opt_bytes) {
 245                                 uint32_t tsval = ntohl(*(ts_ptr + 1));
 246                                 /* make sure timestamp values are increasing */
 247                                 if (unlikely(lro->tsval > tsval ||
 248                                     *(ts_ptr + 2) == 0)) {
 249                                         return (-8);
 250                                 }
 251                                 lro->tsval = tsval;
 252                                 lro->tsecr = *(ts_ptr + 2);
 253                         }
 254
 255                         lro->next_seq += tcp_data_len;
 256                         lro->ack_seq = tcp->th_ack;
 257                         lro->window = tcp->th_win;
 258                         lro->flags |= tcp->th_flags;
 259                         lro->append_cnt++;
 260                         if (tcp_data_len == 0) {
 261                                 freeb(m_head);
 262                                 return (0);
 263                         }
 264                         /*
 265                          * subtract off the checksum of the tcp header
 266                          * from the hardware checksum, and add it to
 267                          * the stored tcp data checksum.  Byteswap
 268                          * the checksum if the total length so far is
 269                          * odd
 270                          */
 271                         tmp_csum = myri10ge_csum_generic((uint16_t *)tcp,
 272                             tcp_hdr_len);
 273                         csum = csum + (tmp_csum ^ 0xffff);
 274                         csum = (csum & 0xffff) + (csum >> 16);
 275                         csum = (csum & 0xffff) + (csum >> 16);
 276                         if (lro->len & 0x1) {
 277                                 /* Odd number of bytes so far, flip bytes */
 278                                 csum = ((csum << 8) | (csum >> 8)) & 0xffff;
 279                         }
 280                         csum = csum + lro->data_csum;
 281                         csum = (csum & 0xffff) + (csum >> 16);
 282                         csum = (csum & 0xffff) + (csum >> 16);
 283                         lro->data_csum = csum;
 284
 285                         lro->len += tcp_data_len;
 286
 287                         /*
 288                          * adjust mblk so that rptr points to
 289                          * the first byte of the payload
 290                          */
 291                         m_head->b_rptr += hlen;
 292                         /* append mbuf chain */
 293                         lro->m_tail->b_cont = m_head;
 294                         /* advance the last pointer */
 295                         lro->m_tail = m_head;
 296                         /* flush packet if required */
 297                         if (lro->len > (65535 - myri10ge_mtu) ||
 298                             (lro->append_cnt + 1) == myri10ge_lro_max_aggr) {
 299                                 if (ss->lro_active == lro) {
 300                                         ss->lro_active = lro->next;
 301                                 } else {
 302                                         curr = ss->lro_active;
 303                                         while (curr->next != lro)
 304                                                 curr = curr->next;
 305                                         curr->next = lro->next;
 306                                 }
 307                                 myri10ge_lro_flush(ss, lro, mbl);
 308                         }
 309                         return (0);
 310                 }
 311         }
 312
 313         if (ss->lro_free == NULL)
 314                 return (ENOMEM);
 315
 316         /* start a new chain */
 317         lro = ss->lro_free;
 318         ss->lro_free = lro->next;
 319         lro->next = ss->lro_active;
 320         ss->lro_active = lro;
 321         lro->source_port = tcp->th_sport;
 322         lro->dest_port = tcp->th_dport;
 323         lro->source_ip = ip->ip_src.s_addr;
 324         lro->dest_ip = ip->ip_dst.s_addr;
 325         lro->next_seq = seq + tcp_data_len;
 326         lro->mss = (uint16_t)tcp_data_len;
 327         lro->ack_seq = tcp->th_ack;
 328         lro->window = tcp->th_win;
 329         lro->flags = tcp->th_flags;
 330
 331         /*
 332          * save the checksum of just the TCP payload by
 333          * subtracting off the checksum of the TCP header from
 334          * the entire hardware checksum
 335          * Since IP header checksum is correct, checksum over
 336          * the IP header is -0.  Substracting -0 is unnecessary.
 337          */
 338         tmp_csum = myri10ge_csum_generic((uint16_t *)tcp, tcp_hdr_len);
 339         csum = csum + (tmp_csum ^ 0xffff);
 340         csum = (csum & 0xffff) + (csum >> 16);
 341         csum = (csum & 0xffff) + (csum >> 16);
 342         lro->data_csum = csum;
 343         lro->ip = ip;
 344
 345         /* record timestamp if it is present */
 346         if (opt_bytes) {
 347                 lro->timestamp = 1;
 348                 lro->tsval = ntohl(*(ts_ptr + 1));
 349                 lro->tsecr = *(ts_ptr + 2);
 350         }
 351         lro->len = tot_len;
 352         lro->m_head = m_head;
 353         lro->m_tail = m_head;
 354         return (0);
 355 }
 356
 357 /*
 358  *  This file uses MyriGE driver indentation.
 359  *
 360  * Local Variables:
 361  * c-file-style:"sun"
 362  * tab-width:8
 363  * End:
 364  */