src/trafgen.c

   1 /*
   2  * netsniff-ng - the packet sniffing beast
   3  * By Daniel Borkmann <daniel@netsniff-ng.org>
   4  * Copyright 2011 - 2013 Daniel Borkmann <dborkma@tik.ee.ethz.ch>,
   5  * Swiss federal institute of technology (ETH Zurich)
   6  * Subject to the GPL, version 2.
   7  *
   8  * A high-performance network traffic generator that uses the zero-copy
   9  * kernelspace TX_RING for network I/O. On comodity Gigabit hardware up
  10  * to 1,488,095 pps 64 Byte pps have been achieved with 2 trafgen instances
  11  * bound to different CPUs from the userspace and turned off pause frames,
  12  * ask Ronald from NST (Network Security Toolkit) for more details. ;-)
  13  * So, this line-rate result is the very same as pktgen from kernelspace!
  14  *
  15  *   Who can now hold the fords when the King of the Nine Riders comes? And
  16  *   other armies will come. I am too late. All is lost. I tarried on the
  17  *   way. All is lost. Even if my errand is performed, no one will ever
  18  *   know. There will be no one I can tell. It will be in vain.
  19  *
  20  *     -- The Lord of the Rings, Frodo thinking,
  21  *        Chapter 'The Stairs of Cirith Ungol'.
  22  */
  23
  24 #include <stdio.h>
  25 #include <string.h>
  26 #include <getopt.h>
  27 #include <ctype.h>
  28 #include <stdbool.h>
  29 #include <sys/socket.h>
  30 #include <sys/types.h>
  31 #include <sys/fsuid.h>
  32 #include <sys/stat.h>
  33 #include <sys/time.h>
  34 #include <sys/wait.h>
  35 #include <sys/mman.h>
  36 #include <net/ethernet.h>
  37 #include <netinet/in.h>
  38 #include <netinet/ip.h>
  39 #include <linux/icmp.h>
  40 #include <arpa/inet.h>
  41 #include <signal.h>
  42 #include <stdint.h>
  43 #include <stdlib.h>
  44 #include <fcntl.h>
  45 #include <time.h>
  46 #include <poll.h>
  47 #include <netdb.h>
  48 #include <math.h>
  49
  50 #include "xmalloc.h"
  51 #include "die.h"
  52 #include "mac80211.h"
  53 #include "xutils.h"
  54 #include "xio.h"
  55 #include "built_in.h"
  56 #include "trafgen_conf.h"
  57 #include "tprintf.h"
  58 #include "ring_tx.h"
  59 #include "csum.h"
  60
  61 struct ctx {
  62         bool rand, rfraw, jumbo_support, verbose, smoke_test;
  63         unsigned long kpull, num, gap, reserve_size, cpus;
  64         struct sockaddr_in dest;
  65         char *device, *device_trans, *rhost;
  66 };
  67
  68 struct cpu_stats {
  69         unsigned long tv_sec, tv_usec;
  70         unsigned long long tx_packets, tx_bytes;
  71         unsigned long long cf_packets, cf_bytes;
  72         unsigned long long cd_packets;
  73         sig_atomic_t state;
  74 };
  75
  76 sig_atomic_t sigint = 0;
  77
  78 struct packet *packets = NULL;
  79 size_t plen = 0;
  80
  81 struct packet_dyn *packet_dyn = NULL;
  82 size_t dlen = 0;
  83
  84 static const char *short_options = "d:c:n:t:vJhS:rk:i:o:VRsP:";
  85 static const struct option long_options[] = {
  86         {"dev",                 required_argument,      NULL, 'd'},
  87         {"out",                 required_argument,      NULL, 'o'},
  88         {"in",                  required_argument,      NULL, 'i'},
  89         {"conf",                required_argument,      NULL, 'c'},
  90         {"num",                 required_argument,      NULL, 'n'},
  91         {"gap",                 required_argument,      NULL, 't'},
  92         {"cpus",                required_argument,      NULL, 'P'},
  93         {"ring-size",           required_argument,      NULL, 'S'},
  94         {"kernel-pull",         required_argument,      NULL, 'k'},
  95         {"smoke-test",          required_argument,      NULL, 's'},
  96         {"jumbo-support",       no_argument,            NULL, 'J'},
  97         {"rfraw",               no_argument,            NULL, 'R'},
  98         {"rand",                no_argument,            NULL, 'r'},
  99         {"verbose",             no_argument,            NULL, 'V'},
 100         {"version",             no_argument,            NULL, 'v'},
 101         {"help",                no_argument,            NULL, 'h'},
 102         {NULL, 0, NULL, 0}
 103 };
 104
 105 static int sock;
 106
 107 static struct itimerval itimer;
 108
 109 static unsigned long interval = TX_KERNEL_PULL_INT;
 110
 111 static struct cpu_stats *stats;
 112
 113 #define CPU_STATS_STATE_CFG     1
 114 #define CPU_STATS_STATE_CHK     2
 115 #define CPU_STATS_STATE_RES     4
 116
 117 #define set_system_socket_memory(vals) \
 118         do { \
 119                 if ((vals[0] = get_system_socket_mem(sock_rmem_max)) < SMEM_SUG_MAX) \
 120                         set_system_socket_mem(sock_rmem_max, SMEM_SUG_MAX); \
 121                 if ((vals[1] = get_system_socket_mem(sock_rmem_def)) < SMEM_SUG_DEF) \
 122                         set_system_socket_mem(sock_rmem_def, SMEM_SUG_DEF); \
 123                 if ((vals[2] = get_system_socket_mem(sock_wmem_max)) < SMEM_SUG_MAX) \
 124                         set_system_socket_mem(sock_wmem_max, SMEM_SUG_MAX); \
 125                 if ((vals[3] = get_system_socket_mem(sock_wmem_def)) < SMEM_SUG_DEF) \
 126                         set_system_socket_mem(sock_wmem_def, SMEM_SUG_DEF); \
 127         } while (0)
 128
 129 #define reset_system_socket_memory(vals) \
 130         do { \
 131                 set_system_socket_mem(sock_rmem_max, vals[0]); \
 132                 set_system_socket_mem(sock_rmem_def, vals[1]); \
 133                 set_system_socket_mem(sock_wmem_max, vals[2]); \
 134                 set_system_socket_mem(sock_wmem_def, vals[3]); \
 135         } while (0)
 136
 137 #ifndef ICMP_FILTER
 138 # define ICMP_FILTER    1
 139
 140 struct icmp_filter {
 141         __u32   data;
 142 };
 143 #endif
 144
 145 static void signal_handler(int number)
 146 {
 147         switch (number) {
 148         case SIGINT:
 149                 sigint = 1;
 150         case SIGHUP:
 151         default:
 152                 break;
 153         }
 154 }
 155
 156 static void timer_elapsed(int number)
 157 {
 158         itimer.it_interval.tv_sec = 0;
 159         itimer.it_interval.tv_usec = interval;
 160
 161         itimer.it_value.tv_sec = 0;
 162         itimer.it_value.tv_usec = interval;
 163
 164         pull_and_flush_tx_ring(sock);
 165         setitimer(ITIMER_REAL, &itimer, NULL);
 166 }
 167
 168 static void header(void)
 169 {
 170         printf("%s%s%s\n", colorize_start(bold), "trafgen " VERSION_STRING, colorize_end());
 171 }
 172
 173 static void help(void)
 174 {
 175         printf("\ntrafgen %s, multithreaded zero-copy network packet generator\n", VERSION_STRING);
 176         puts("http://www.netsniff-ng.org\n\n"
 177              "Usage: trafgen [options]\n"
 178              "Options:\n"
 179              "  -o|-d|--out|--dev <netdev>        Networking Device i.e., eth0\n"
 180              "  -i|-c|--in|--conf <cfg-file>      Packet configuration file\n"
 181              "  -J|--jumbo-support                Support 64KB Super Jumbo Frames (def: 2048B)\n"
 182              "  -R|--rfraw                        Inject raw 802.11 frames\n"
 183              "  -s|--smoke-test <ipv4-receiver>   Test if machine survived packet\n"
 184              "  -n|--num <uint>                   Number of packets until exit (def: 0)\n"
 185              "  -r|--rand                         Randomize packet selection (def: round robin)\n"
 186              "  -P|--cpus <uint>                  Specify number of forks(<= CPUs) (def: #CPUs)\n"
 187              "  -t|--gap <uint>                   Interpacket gap in us (approx)\n"
 188              "  -S|--ring-size <size>             Manually set mmap size (KB/MB/GB): e.g.\'10MB\'\n"
 189              "  -k|--kernel-pull <uint>           Kernel batch interval in us (def: 10us)\n"
 190              "  -V|--verbose                      Be more verbose\n"
 191              "  -v|--version                      Show version\n"
 192              "  -h|--help                         Guess what?!\n\n"
 193              "Examples:\n"
 194              "  See trafgen.txf for configuration file examples.\n"
 195              "  trafgen --dev eth0 --conf trafgen.cfg\n"
 196              "  trafgen --dev eth0 --conf trafgen.cfg --smoke-test 10.0.0.1\n"
 197              "  trafgen --dev wlan0 --rfraw --conf beacon-test.txf -V --cpus 2\n"
 198              "  trafgen --dev eth0 --conf trafgen.cfg --rand --gap 1000\n"
 199              "  trafgen --dev eth0 --conf trafgen.cfg --rand --num 1400000 -k1000\n\n"
 200              "Arbitrary packet config examples (trafgen.cfg):\n"
 201              "  Run packet on  all CPUs:              { fill(0xff, 64) csum16(0, 64) }\n"
 202              "  Run packet only on CPU1:    cpu(1):   { rnd(64), 0b11001100, 0xaa }\n"
 203              "  Run packet only on CPU1-2:  cpu(1:2): { drnd(64),'a',csum16(1, 8),'b',42 }\n\n"
 204              "Note:\n"
 205              "  Smoke test example: machine A, 10.0.0.2 (trafgen) is directly\n"
 206              "  connected to machine B (test kernel), 10.0.0.1. If ICMP reply fails\n"
 207              "  we assume the kernel crashed, thus we print the packet and quit.\n\n"
 208              "  This tool is targeted for network developers! You should\n"
 209              "  be aware of what you are doing and what these options above\n"
 210              "  mean! Only use this tool in an isolated LAN that you own!\n\n"
 211              "Please report bugs to <bugs@netsniff-ng.org>\n"
 212              "Copyright (C) 2011-2013 Daniel Borkmann <dborkma@tik.ee.ethz.ch>,\n"
 213              "Swiss federal institute of technology (ETH Zurich)\n"
 214              "License: GNU GPL version 2.0\n"
 215              "This is free software: you are free to change and redistribute it.\n"
 216              "There is NO WARRANTY, to the extent permitted by law.\n");
 217         die();
 218 }
 219
 220 static void version(void)
 221 {
 222         printf("\ntrafgen %s, multithreaded zero-copy network packet generator\n", VERSION_STRING);
 223         puts("http://www.netsniff-ng.org\n\n"
 224              "Please report bugs to <bugs@netsniff-ng.org>\n"
 225              "Copyright (C) 2011-2013 Daniel Borkmann <dborkma@tik.ee.ethz.ch>,\n"
 226              "Swiss federal institute of technology (ETH Zurich)\n"
 227              "License: GNU GPL version 2.0\n"
 228              "This is free software: you are free to change and redistribute it.\n"
 229              "There is NO WARRANTY, to the extent permitted by law.\n");
 230         die();
 231 }
 232
 233 static void apply_counter(int counter_id)
 234 {
 235         int j, i = counter_id;
 236         size_t counter_max = packet_dyn[i].clen;
 237
 238         for (j = 0; j < counter_max; ++j) {
 239                 uint8_t val;
 240                 struct counter *counter = &packet_dyn[i].cnt[j];
 241
 242                 val = counter->val - counter->min;
 243
 244                 switch (counter->type) {
 245                 case TYPE_INC:
 246                         val = (val + counter->inc) % (counter->max - counter->min + 1);
 247                         break;
 248                 case TYPE_DEC:
 249                         val = (val - counter->inc) % (counter->min - counter->max + 1);
 250                         break;
 251                 default:
 252                         bug();
 253                 }
 254
 255                 counter->val = val + counter->min;
 256                 packets[i].payload[counter->off] = val;
 257         }
 258 }
 259
 260 static void apply_randomizer(int rand_id)
 261 {
 262         int j, i = rand_id;
 263         size_t rand_max = packet_dyn[i].rlen;
 264
 265         for (j = 0; j < rand_max; ++j) {
 266                 uint8_t val = (uint8_t) rand();
 267                 struct randomizer *randomizer = &packet_dyn[i].rnd[j];
 268
 269                 packets[i].payload[randomizer->off] = val;
 270         }
 271 }
 272
 273 static void apply_csum16(int csum_id)
 274 {
 275         int j, i = csum_id;
 276         size_t csum_max = packet_dyn[i].slen;
 277
 278         for (j = 0; j < csum_max; ++j) {
 279                 uint16_t sum;
 280                 uint8_t *psum;
 281                 struct csum16 *csum = &packet_dyn[i].csum[j];
 282
 283                 sum = htons(calc_csum(packets[i].payload + csum->from, csum->to - csum->from, 0));
 284                 psum = (uint8_t *) &sum;
 285
 286                 packets[i].payload[csum->off]     = psum[0];
 287                 packets[i].payload[csum->off + 1] = psum[1];
 288         }
 289 }
 290
 291 static struct cpu_stats *setup_shared_var(unsigned long cpus)
 292 {
 293         int fd;
 294         char zbuff[cpus * sizeof(struct cpu_stats)], file[256];
 295         struct cpu_stats *buff;
 296
 297         memset(zbuff, 0, sizeof(zbuff));
 298         slprintf(file, sizeof(file), ".tmp_mmap.%u", (unsigned int) rand());
 299
 300         fd = creat(file, S_IRUSR | S_IWUSR);
 301         bug_on(fd < 0);
 302         close(fd);
 303
 304         fd = open_or_die_m(file, O_RDWR | O_CREAT | O_TRUNC,
 305                            S_IRUSR | S_IWUSR);
 306         write_or_die(fd, zbuff, sizeof(zbuff));
 307
 308         buff = (void *) mmap(0, sizeof(zbuff), PROT_READ | PROT_WRITE,
 309                              MAP_SHARED, fd, 0);
 310         if (buff == (void *) -1)
 311                 panic("Cannot setup shared variable!\n");
 312
 313         close(fd);
 314         unlink(file);
 315
 316         memset(buff, 0, sizeof(zbuff));
 317
 318         return buff;
 319 }
 320
 321 static void destroy_shared_var(void *buff, unsigned long cpus)
 322 {
 323         munmap(buff, cpus * sizeof(struct cpu_stats));
 324 }
 325
 326 static void dump_trafgen_snippet(uint8_t *payload, size_t len)
 327 {
 328         int i;
 329
 330         printf("{");
 331         for (i = 0; i < len; ++i) {
 332                 if (i % 15 == 0)
 333                         printf("\n  ");
 334                 printf("0x%02x, ", payload[i]);
 335         }
 336         printf("\n}\n");
 337         fflush(stdout);
 338 }
 339
 340 static inline unsigned short csum(unsigned short *buf, int nwords)
 341 {
 342         unsigned long sum;
 343
 344         for (sum = 0; nwords > 0; nwords--)
 345                 sum += *buf++;
 346         sum = (sum >> 16) + (sum & 0xffff);
 347         sum += (sum >> 16);
 348
 349         return ~sum;
 350 }
 351
 352 static int xmit_smoke_setup(struct ctx *ctx)
 353 {
 354         int icmp_sock, ret, ttl = 64;
 355         struct icmp_filter filter;
 356
 357         icmp_sock = socket(AF_INET, SOCK_RAW, IPPROTO_ICMP);
 358         if (icmp_sock < 0)
 359                 panic("Cannot get a ICMP socket: %s!\n", strerror(errno));
 360
 361         filter.data = ~(1 << ICMP_ECHOREPLY);
 362
 363         ret = setsockopt(icmp_sock, SOL_RAW, ICMP_FILTER, &filter, sizeof(filter));
 364         if (ret < 0)
 365                 panic("Cannot install filter!\n");
 366
 367         ret = setsockopt(icmp_sock, SOL_IP, IP_TTL, &ttl, sizeof(ttl));
 368         if (ret < 0)
 369                 panic("Cannot set TTL!\n");
 370
 371         memset(&ctx->dest, 0, sizeof(ctx->dest));
 372         ctx->dest.sin_family = AF_INET;
 373         ctx->dest.sin_port = 0;
 374
 375         ret = inet_aton(ctx->rhost, &ctx->dest.sin_addr);
 376         if (ret < 0)
 377                 panic("Cannot resolv address!\n");
 378
 379         return icmp_sock;
 380 }
 381
 382 static int xmit_smoke_probe(int icmp_sock, struct ctx *ctx)
 383 {
 384         int ret, i, probes = 5;
 385         short ident, cnt = 1;
 386         uint8_t outpack[512], *data;
 387         struct icmphdr *icmp;
 388         struct iphdr *ip;
 389         size_t len = sizeof(*icmp) + 56;
 390         struct sockaddr_in from;
 391         socklen_t from_len;
 392         struct pollfd fds = {
 393                 .fd = icmp_sock,
 394                 .events = POLLIN,
 395         };
 396
 397         while (probes-- > 0) {
 398                 ident = htons((short) rand());
 399
 400                 memset(outpack, 0, sizeof(outpack));
 401                 icmp = (void *) outpack;
 402                 icmp->type = ICMP_ECHO;
 403                 icmp->code = 0;
 404                 icmp->checksum = 0;
 405                 icmp->un.echo.id = ident;
 406                 icmp->un.echo.sequence = htons(cnt++);
 407
 408                 data = ((uint8_t *) outpack + sizeof(*icmp));
 409                 for (i = 0; i < 56; ++i)
 410                         data[i] = (uint8_t) rand();
 411
 412                 icmp->checksum = csum((unsigned short *) outpack,
 413                                       len / sizeof(unsigned short));
 414
 415                 ret = sendto(icmp_sock, outpack, len, MSG_DONTWAIT,
 416                              (struct sockaddr *) &ctx->dest, sizeof(ctx->dest));
 417                 if (unlikely(ret != len))
 418                         panic("Cannot send out probe: %s!\n", strerror(errno));
 419
 420                 ret = poll(&fds, 1, 500);
 421                 if (ret < 0)
 422                         panic("Poll failed!\n");
 423
 424                 if (fds.revents & POLLIN) {
 425                         ret = recvfrom(icmp_sock, outpack, sizeof(outpack), 0,
 426                                        (struct sockaddr *) &from, &from_len);
 427                         if (unlikely(ret <= 0))
 428                                 panic("Probe receive failed!\n");
 429                         if (unlikely(from_len != sizeof(ctx->dest)))
 430                                 continue;
 431                         if (unlikely(memcmp(&from, &ctx->dest, sizeof(ctx->dest))))
 432                                 continue;
 433                         if (unlikely(ret < sizeof(*ip) + sizeof(*icmp)))
 434                                 continue;
 435                         ip = (void *) outpack;
 436                         if (unlikely(ip->ihl * 4 + sizeof(*icmp) > ret))
 437                                 continue;
 438                         icmp = (void *) outpack + ip->ihl * 4;
 439                         if (unlikely(icmp->un.echo.id != ident))
 440                                 continue;
 441
 442                         return 0;
 443                 }
 444         }
 445
 446         return -1;
 447 }
 448
 449 static void xmit_slowpath_or_die(struct ctx *ctx, int cpu)
 450 {
 451         int ret, icmp_sock = -1;
 452         unsigned long num = 1, i = 0;
 453         struct timeval start, end, diff;
 454         unsigned long long tx_bytes = 0, tx_packets = 0;
 455         struct packet_dyn *pktd;
 456         struct sockaddr_ll saddr = {
 457                 .sll_family = PF_PACKET,
 458                 .sll_halen = ETH_ALEN,
 459                 .sll_ifindex = device_ifindex(ctx->device),
 460         };
 461
 462         if (ctx->num > 0)
 463                 num = ctx->num;
 464
 465         if (ctx->smoke_test)
 466                 icmp_sock = xmit_smoke_setup(ctx);
 467
 468         bug_on(gettimeofday(&start, NULL));
 469
 470         while (likely(sigint == 0) && likely(num > 0)) {
 471                 pktd = &packet_dyn[i];
 472                 if (pktd->clen + pktd->rlen + pktd->slen) {
 473                         apply_counter(i);
 474                         apply_randomizer(i);
 475                         apply_csum16(i);
 476                 }
 477 retry:
 478                 ret = sendto(sock, packets[i].payload, packets[i].len, 0,
 479                              (struct sockaddr *) &saddr, sizeof(saddr));
 480                 if (unlikely(ret < 0)) {
 481                         if (errno == ENOBUFS) {
 482                                 sched_yield();
 483                                 goto retry;
 484                         }
 485
 486                         panic("Sendto error: %s!\n", strerror(errno));
 487                 }
 488
 489                 tx_bytes += packets[i].len;
 490                 tx_packets++;
 491
 492                 if (ctx->smoke_test) {
 493                         ret = xmit_smoke_probe(icmp_sock, ctx);
 494                         if (unlikely(ret < 0)) {
 495                                 printf("%sSmoke test alert:%s\n", colorize_start(bold), colorize_end());
 496                                 printf("  Remote host seems to be unresponsive to ICMP pings!\n");
 497                                 printf("  Last instance was packet%lu, trafgen snippet:\n\n", i);
 498
 499                                 dump_trafgen_snippet(packets[i].payload, packets[i].len);
 500                                 break;
 501                         }
 502                 }
 503
 504                 if (!ctx->rand) {
 505                         i++;
 506                         if (i >= plen)
 507                                 i = 0;
 508                 } else
 509                         i = rand() % plen;
 510
 511                 if (ctx->num > 0)
 512                         num--;
 513
 514                 if (ctx->gap > 0)
 515                         usleep(ctx->gap);
 516         }
 517
 518         bug_on(gettimeofday(&end, NULL));
 519         diff = tv_subtract(end, start);
 520
 521         if (ctx->smoke_test)
 522                 close(icmp_sock);
 523
 524         stats[cpu].tx_packets = tx_packets;
 525         stats[cpu].tx_bytes = tx_bytes;
 526         stats[cpu].tv_sec = diff.tv_sec;
 527         stats[cpu].tv_usec = diff.tv_usec;
 528
 529         stats[cpu].state |= CPU_STATS_STATE_RES;
 530 }
 531
 532 static void xmit_fastpath_or_die(struct ctx *ctx, int cpu)
 533 {
 534         int ifindex = device_ifindex(ctx->device);
 535         uint8_t *out = NULL;
 536         unsigned int it = 0;
 537         unsigned long num = 1, i = 0, size;
 538         struct ring tx_ring;
 539         struct frame_map *hdr;
 540         struct timeval start, end, diff;
 541         struct packet_dyn *pktd;
 542         unsigned long long tx_bytes = 0, tx_packets = 0;
 543
 544         fmemset(&tx_ring, 0, sizeof(tx_ring));
 545
 546         size = ring_size(ctx->device, ctx->reserve_size);
 547
 548         set_sock_prio(sock, 512);
 549         set_packet_loss_discard(sock);
 550
 551         setup_tx_ring_layout(sock, &tx_ring, size, ctx->jumbo_support);
 552         create_tx_ring(sock, &tx_ring, ctx->verbose);
 553         mmap_tx_ring(sock, &tx_ring);
 554         alloc_tx_ring_frames(&tx_ring);
 555         bind_tx_ring(sock, &tx_ring, ifindex);
 556
 557         if (ctx->kpull)
 558                 interval = ctx->kpull;
 559         if (ctx->num > 0)
 560                 num = ctx->num;
 561
 562         itimer.it_interval.tv_sec = 0;
 563         itimer.it_interval.tv_usec = interval;
 564
 565         itimer.it_value.tv_sec = 0;
 566         itimer.it_value.tv_usec = interval;
 567
 568         setitimer(ITIMER_REAL, &itimer, NULL);
 569
 570         bug_on(gettimeofday(&start, NULL));
 571
 572         while (likely(sigint == 0) && likely(num > 0)) {
 573                 while (user_may_pull_from_tx(tx_ring.frames[it].iov_base) && likely(num > 0)) {
 574                         hdr = tx_ring.frames[it].iov_base;
 575
 576                         /* Kernel assumes: data = ph.raw + po->tp_hdrlen -
 577                          *                        sizeof(struct sockaddr_ll); */
 578                         out = ((uint8_t *) hdr) + TPACKET2_HDRLEN - sizeof(struct sockaddr_ll);
 579
 580                         hdr->tp_h.tp_snaplen = packets[i].len;
 581                         hdr->tp_h.tp_len = packets[i].len;
 582
 583                         pktd = &packet_dyn[i];
 584                         if (pktd->clen + pktd->rlen + pktd->slen) {
 585                                 apply_counter(i);
 586                                 apply_randomizer(i);
 587                                 apply_csum16(i);
 588                         }
 589
 590                         fmemcpy(out, packets[i].payload, packets[i].len);
 591
 592                         tx_bytes += packets[i].len;
 593                         tx_packets++;
 594
 595                         if (!ctx->rand) {
 596                                 i++;
 597                                 if (i >= plen)
 598                                         i = 0;
 599                         } else
 600                                 i = rand() % plen;
 601
 602                         kernel_may_pull_from_tx(&hdr->tp_h);
 603
 604                         it++;
 605                         if (it >= tx_ring.layout.tp_frame_nr)
 606                                 it = 0;
 607
 608                         if (ctx->num > 0)
 609                                 num--;
 610
 611                         if (unlikely(sigint == 1))
 612                                 break;
 613                 }
 614         }
 615
 616         bug_on(gettimeofday(&end, NULL));
 617         diff = tv_subtract(end, start);
 618
 619         destroy_tx_ring(sock, &tx_ring);
 620
 621         stats[cpu].tx_packets = tx_packets;
 622         stats[cpu].tx_bytes = tx_bytes;
 623         stats[cpu].tv_sec = diff.tv_sec;
 624         stats[cpu].tv_usec = diff.tv_usec;
 625
 626         stats[cpu].state |= CPU_STATS_STATE_RES;
 627 }
 628
 629 static inline void __set_state(int cpu, sig_atomic_t s)
 630 {
 631         stats[cpu].state = s;
 632 }
 633
 634 static inline sig_atomic_t __get_state(int cpu)
 635 {
 636         return stats[cpu].state;
 637 }
 638
 639 static unsigned long __wait_and_sum_others(struct ctx *ctx, int cpu)
 640 {
 641         int i;
 642         unsigned long total;
 643
 644         for (i = 0, total = plen; i < ctx->cpus; i++) {
 645                 if (i == cpu)
 646                         continue;
 647
 648                 while ((__get_state(i) & CPU_STATS_STATE_CFG) == 0 &&
 649                        sigint == 0)
 650                         sched_yield();
 651
 652                 total += stats[i].cf_packets;
 653         }
 654
 655         return total;
 656 }
 657
 658 static void __correct_global_delta(struct ctx *ctx, int cpu, unsigned long orig)
 659 {
 660         int i, cpu_sel;
 661         unsigned long total;
 662         long long delta_correction = 0;
 663
 664         for (i = 0, total = ctx->num; i < ctx->cpus; i++) {
 665                 if (i == cpu)
 666                         continue;
 667
 668                 while ((__get_state(i) & CPU_STATS_STATE_CHK) == 0 &&
 669                        sigint == 0)
 670                         sched_yield();
 671
 672                 total += stats[i].cd_packets;
 673         }
 674
 675         if (total > orig)
 676                 delta_correction = -1 * ((long long) total - orig);
 677         if (total < orig)
 678                 delta_correction = +1 * ((long long) orig - total);
 679
 680         for (cpu_sel = -1, i = 0; i < ctx->cpus; i++) {
 681                 if (stats[i].cd_packets > 0) {
 682                         if ((long long) stats[i].cd_packets +
 683                             delta_correction > 0) {
 684                                 cpu_sel = i;
 685                                 break;
 686                         }
 687                 }
 688         }
 689
 690         if (cpu == cpu_sel)
 691                 ctx->num += delta_correction;
 692 }
 693
 694 static void __set_state_cf(int cpu, unsigned long p, unsigned long b,
 695                            sig_atomic_t s)
 696 {
 697         stats[cpu].cf_packets = p;
 698         stats[cpu].cf_bytes = b;
 699         stats[cpu].state = s;
 700 }
 701
 702 static void __set_state_cd(int cpu, unsigned long p, sig_atomic_t s)
 703 {
 704         stats[cpu].cd_packets = p;
 705         stats[cpu].state = s;
 706 }
 707
 708 static int xmit_packet_precheck(struct ctx *ctx, int cpu)
 709 {
 710         int i;
 711         unsigned long plen_total, orig = ctx->num;
 712         size_t mtu, total_len = 0;
 713
 714         bug_on(plen != dlen);
 715
 716         for (i = 0; i < plen; ++i)
 717                 total_len += packets[i].len;
 718
 719         __set_state_cf(cpu, plen, total_len, CPU_STATS_STATE_CFG);
 720         plen_total = __wait_and_sum_others(ctx, cpu);
 721
 722         if (orig > 0) {
 723                 ctx->num = (unsigned long) nearbyint((1.0 * plen / plen_total) * orig);
 724
 725                 __set_state_cd(cpu, ctx->num, CPU_STATS_STATE_CHK |
 726                                CPU_STATS_STATE_CFG);
 727                 __correct_global_delta(ctx, cpu, orig);
 728         }
 729
 730         if (plen == 0) {
 731                 __set_state(cpu, CPU_STATS_STATE_RES);
 732                 return -1;
 733         }
 734
 735         for (mtu = device_mtu(ctx->device), i = 0; i < plen; ++i) {
 736                 if (packets[i].len > mtu + 14)
 737                         panic("Device MTU < than packet%d's size!\n", i);
 738                 if (packets[i].len <= 14)
 739                         panic("Packet%d's size too short!\n", i);
 740         }
 741
 742         return 0;
 743 }
 744
 745 static void main_loop(struct ctx *ctx, char *confname, bool slow, int cpu)
 746 {
 747         compile_packets(confname, ctx->verbose, cpu);
 748         if (xmit_packet_precheck(ctx, cpu) < 0)
 749                 return;
 750
 751         if (cpu == 0) {
 752                 int i;
 753                 size_t total_len = 0, total_pkts = 0;
 754
 755                 for (i = 0; i < ctx->cpus; ++i) {
 756                         total_len  += stats[i].cf_bytes;
 757                         total_pkts += stats[i].cf_packets;
 758                 }
 759
 760                 printf("%6zu packets to schedule\n", total_pkts);
 761                 printf("%6zu bytes in total\n", total_len);
 762                 printf("Running! Hang up with ^C!\n\n");
 763                 fflush(stdout);
 764         }
 765
 766         sock = pf_socket();
 767
 768         if (slow)
 769                 xmit_slowpath_or_die(ctx, cpu);
 770         else
 771                 xmit_fastpath_or_die(ctx, cpu);
 772
 773         close(sock);
 774
 775         cleanup_packets();
 776 }
 777
 778 int main(int argc, char **argv)
 779 {
 780         bool slow = false;
 781         int c, opt_index, i, j, vals[4] = {0}, irq;
 782         char *confname = NULL, *ptr;
 783         unsigned long cpus_tmp;
 784         unsigned long long tx_packets, tx_bytes;
 785         struct ctx ctx;
 786
 787         setfsuid(getuid());
 788         setfsgid(getgid());
 789
 790         srand(time(NULL));
 791         fmemset(&ctx, 0, sizeof(ctx));
 792         ctx.cpus = get_number_cpus_online();
 793
 794         while ((c = getopt_long(argc, argv, short_options, long_options,
 795                                 &opt_index)) != EOF) {
 796                 switch (c) {
 797                 case 'h':
 798                         help();
 799                         break;
 800                 case 'v':
 801                         version();
 802                         break;
 803                 case 'V':
 804                         ctx.verbose = true;
 805                         break;
 806                 case 'P':
 807                         cpus_tmp = strtoul(optarg, NULL, 0);
 808                         if (cpus_tmp > 0 && cpus_tmp < ctx.cpus)
 809                                 ctx.cpus = cpus_tmp;
 810                         break;
 811                 case 'd':
 812                 case 'o':
 813                         ctx.device = xstrndup(optarg, IFNAMSIZ);
 814                         break;
 815                 case 'r':
 816                         ctx.rand = true;
 817                         break;
 818                 case 's':
 819                         slow = true;
 820                         ctx.cpus = 1;
 821                         ctx.smoke_test = true;
 822                         ctx.rhost = xstrdup(optarg);
 823                         break;
 824                 case 'R':
 825                         ctx.rfraw = true;
 826                         break;
 827                 case 'J':
 828                         ctx.jumbo_support = true;
 829                         break;
 830                 case 'c':
 831                 case 'i':
 832                         confname = xstrdup(optarg);
 833                         break;
 834                 case 'k':
 835                         ctx.kpull = strtoul(optarg, NULL, 0);
 836                         break;
 837                 case 'n':
 838                         ctx.num = strtoul(optarg, NULL, 0);
 839                         break;
 840                 case 't':
 841                         slow = true;
 842                         ctx.gap = strtoul(optarg, NULL, 0);
 843                         if (ctx.gap > 0)
 844                                 /* Fall back to single core to not
 845                                  * mess up correct timing. We are slow
 846                                  * anyway!
 847                                  */
 848                                 ctx.cpus = 1;
 849                         break;
 850                 case 'S':
 851                         ptr = optarg;
 852                         ctx.reserve_size = 0;
 853
 854                         for (j = i = strlen(optarg); i > 0; --i) {
 855                                 if (!isdigit(optarg[j - i]))
 856                                         break;
 857                                 ptr++;
 858                         }
 859
 860                         if (!strncmp(ptr, "KB", strlen("KB")))
 861                                 ctx.reserve_size = 1 << 10;
 862                         else if (!strncmp(ptr, "MB", strlen("MB")))
 863                                 ctx.reserve_size = 1 << 20;
 864                         else if (!strncmp(ptr, "GB", strlen("GB")))
 865                                 ctx.reserve_size = 1 << 30;
 866                         else
 867                                 panic("Syntax error in ring size param!\n");
 868                         *ptr = 0;
 869
 870                         ctx.reserve_size *= strtol(optarg, NULL, 0);
 871                         break;
 872                 case '?':
 873                         switch (optopt) {
 874                         case 'd':
 875                         case 'c':
 876                         case 'n':
 877                         case 'S':
 878                         case 's':
 879                         case 'P':
 880                         case 'o':
 881                         case 'i':
 882                         case 'k':
 883                         case 't':
 884                                 panic("Option -%c requires an argument!\n",
 885                                       optopt);
 886                         default:
 887                                 if (isprint(optopt))
 888                                         whine("Unknown option character "
 889                                               "`0x%X\'!\n", optopt);
 890                                 die();
 891                         }
 892                 default:
 893                         break;
 894                 }
 895         }
 896
 897         if (argc < 5)
 898                 help();
 899         if (ctx.device == NULL)
 900                 panic("No networking device given!\n");
 901         if (confname == NULL)
 902                 panic("No configuration file given!\n");
 903         if (device_mtu(ctx.device) == 0)
 904                 panic("This is no networking device!\n");
 905         if (!ctx.rfraw && device_up_and_running(ctx.device) == 0)
 906                 panic("Networking device not running!\n");
 907
 908         register_signal(SIGINT, signal_handler);
 909         register_signal(SIGHUP, signal_handler);
 910         register_signal_f(SIGALRM, timer_elapsed, SA_SIGINFO);
 911
 912         header();
 913
 914         set_system_socket_memory(vals);
 915
 916         if (ctx.rfraw) {
 917                 ctx.device_trans = xstrdup(ctx.device);
 918                 xfree(ctx.device);
 919
 920                 enter_rfmon_mac80211(ctx.device_trans, &ctx.device);
 921                 sleep(0);
 922         }
 923
 924         irq = device_irq_number(ctx.device);
 925         device_reset_irq_affinity(irq);
 926
 927         if (ctx.num > 0 && ctx.num <= ctx.cpus)
 928                 ctx.cpus = 1;
 929
 930         stats = setup_shared_var(ctx.cpus);
 931
 932         for (i = 0; i < ctx.cpus; i++) {
 933                 pid_t pid = fork();
 934
 935                 switch (pid) {
 936                 case 0:
 937                         cpu_affinity(i);
 938                         main_loop(&ctx, confname, slow, i);
 939
 940                         goto thread_out;
 941                 case -1:
 942                         panic("Cannot fork processes!\n");
 943                 }
 944         }
 945
 946         for (i = 0; i < ctx.cpus; i++) {
 947                 int status;
 948
 949                 wait(&status);
 950         }
 951
 952         if (ctx.rfraw)
 953                 leave_rfmon_mac80211(ctx.device_trans, ctx.device);
 954
 955         reset_system_socket_memory(vals);
 956
 957         for (i = 0, tx_packets = tx_bytes = 0; i < ctx.cpus; i++) {
 958                 while ((__get_state(i) & CPU_STATS_STATE_RES) == 0)
 959                         sched_yield();
 960
 961                 tx_packets += stats[i].tx_packets;
 962                 tx_bytes   += stats[i].tx_bytes;
 963         }
 964
 965         fflush(stdout);
 966         printf("\n");
 967         printf("\r%12llu packets outgoing\n", tx_packets);
 968         printf("\r%12llu bytes outgoing\n", tx_bytes);
 969         for (i = 0; i < ctx.cpus; i++) {
 970                 printf("\r%12lu sec, %lu usec on CPU%d (%llu packets)\n",
 971                        stats[i].tv_sec, stats[i].tv_usec, i,
 972                        stats[i].tx_packets);
 973         }
 974
 975 thread_out:
 976         destroy_shared_var(stats, ctx.cpus);
 977
 978         free(ctx.device);
 979         free(ctx.device_trans);
 980         free(ctx.rhost);
 981         free(confname);
 982
 983         return 0;
 984 }