src/trafgen.c

   1 /*
   2  * netsniff-ng - the packet sniffing beast
   3  * By Daniel Borkmann <daniel@netsniff-ng.org>
   4  * Copyright 2011 - 2013 Daniel Borkmann <dborkma@tik.ee.ethz.ch>,
   5  * Swiss federal institute of technology (ETH Zurich)
   6  * Subject to the GPL, version 2.
   7  *
   8  * A high-performance network traffic generator that uses the zero-copy
   9  * kernelspace TX_RING for network I/O. On comodity Gigabit hardware up
  10  * to 1,488,095 pps 64 Byte pps have been achieved with 2 trafgen instances
  11  * bound to different CPUs from the userspace and turned off pause frames,
  12  * ask Ronald from NST (Network Security Toolkit) for more details. ;-)
  13  * So, this line-rate result is the very same as pktgen from kernelspace!
  14  *
  15  *   Who can now hold the fords when the King of the Nine Riders comes? And
  16  *   other armies will come. I am too late. All is lost. I tarried on the
  17  *   way. All is lost. Even if my errand is performed, no one will ever
  18  *   know. There will be no one I can tell. It will be in vain.
  19  *
  20  *     -- The Lord of the Rings, Frodo thinking,
  21  *        Chapter 'The Stairs of Cirith Ungol'.
  22  */
  23
  24 #include <stdio.h>
  25 #include <string.h>
  26 #include <getopt.h>
  27 #include <ctype.h>
  28 #include <stdbool.h>
  29 #include <sys/socket.h>
  30 #include <sys/types.h>
  31 #include <sys/fsuid.h>
  32 #include <sys/stat.h>
  33 #include <sys/time.h>
  34 #include <sys/wait.h>
  35 #include <sys/mman.h>
  36 #include <net/ethernet.h>
  37 #include <netinet/in.h>
  38 #include <netinet/ip.h>
  39 #include <linux/icmp.h>
  40 #include <arpa/inet.h>
  41 #include <signal.h>
  42 #include <stdint.h>
  43 #include <stdlib.h>
  44 #include <fcntl.h>
  45 #include <time.h>
  46 #include <poll.h>
  47 #include <netdb.h>
  48 #include <math.h>
  49
  50 #include "xmalloc.h"
  51 #include "die.h"
  52 #include "mac80211.h"
  53 #include "xutils.h"
  54 #include "xio.h"
  55 #include "built_in.h"
  56 #include "trafgen_conf.h"
  57 #include "tprintf.h"
  58 #include "ring_tx.h"
  59 #include "csum.h"
  60
  61 struct ctx {
  62         bool rand, rfraw, jumbo_support, verbose, smoke_test;
  63         unsigned long kpull, num, gap, reserve_size, cpus;
  64         struct sockaddr_in dest;
  65         char *device, *device_trans, *rhost;
  66 };
  67
  68 struct cpu_stats {
  69         unsigned long tv_sec, tv_usec;
  70         unsigned long long tx_packets, tx_bytes;
  71         unsigned long long cf_packets, cf_bytes;
  72         unsigned long long cd_packets;
  73         sig_atomic_t state;
  74 };
  75
  76 sig_atomic_t sigint = 0;
  77
  78 struct packet *packets = NULL;
  79 size_t plen = 0;
  80
  81 struct packet_dyn *packet_dyn = NULL;
  82 size_t dlen = 0;
  83
  84 static const char *short_options = "d:c:n:t:vJhS:rk:i:o:VRsP:e";
  85 static const struct option long_options[] = {
  86         {"dev",                 required_argument,      NULL, 'd'},
  87         {"out",                 required_argument,      NULL, 'o'},
  88         {"in",                  required_argument,      NULL, 'i'},
  89         {"conf",                required_argument,      NULL, 'c'},
  90         {"num",                 required_argument,      NULL, 'n'},
  91         {"gap",                 required_argument,      NULL, 't'},
  92         {"cpus",                required_argument,      NULL, 'P'},
  93         {"ring-size",           required_argument,      NULL, 'S'},
  94         {"kernel-pull",         required_argument,      NULL, 'k'},
  95         {"smoke-test",          required_argument,      NULL, 's'},
  96         {"jumbo-support",       no_argument,            NULL, 'J'},
  97         {"rfraw",               no_argument,            NULL, 'R'},
  98         {"rand",                no_argument,            NULL, 'r'},
  99         {"verbose",             no_argument,            NULL, 'V'},
 100         {"version",             no_argument,            NULL, 'v'},
 101         {"example",             no_argument,            NULL, 'e'},
 102         {"help",                no_argument,            NULL, 'h'},
 103         {NULL, 0, NULL, 0}
 104 };
 105
 106 static int sock;
 107
 108 static struct itimerval itimer;
 109
 110 static unsigned long interval = TX_KERNEL_PULL_INT;
 111
 112 static struct cpu_stats *stats;
 113
 114 #define CPU_STATS_STATE_CFG     1
 115 #define CPU_STATS_STATE_CHK     2
 116 #define CPU_STATS_STATE_RES     4
 117
 118 #define set_system_socket_memory(vals) \
 119         do { \
 120                 if ((vals[0] = get_system_socket_mem(sock_rmem_max)) < SMEM_SUG_MAX) \
 121                         set_system_socket_mem(sock_rmem_max, SMEM_SUG_MAX); \
 122                 if ((vals[1] = get_system_socket_mem(sock_rmem_def)) < SMEM_SUG_DEF) \
 123                         set_system_socket_mem(sock_rmem_def, SMEM_SUG_DEF); \
 124                 if ((vals[2] = get_system_socket_mem(sock_wmem_max)) < SMEM_SUG_MAX) \
 125                         set_system_socket_mem(sock_wmem_max, SMEM_SUG_MAX); \
 126                 if ((vals[3] = get_system_socket_mem(sock_wmem_def)) < SMEM_SUG_DEF) \
 127                         set_system_socket_mem(sock_wmem_def, SMEM_SUG_DEF); \
 128         } while (0)
 129
 130 #define reset_system_socket_memory(vals) \
 131         do { \
 132                 set_system_socket_mem(sock_rmem_max, vals[0]); \
 133                 set_system_socket_mem(sock_rmem_def, vals[1]); \
 134                 set_system_socket_mem(sock_wmem_max, vals[2]); \
 135                 set_system_socket_mem(sock_wmem_def, vals[3]); \
 136         } while (0)
 137
 138 #ifndef ICMP_FILTER
 139 # define ICMP_FILTER    1
 140
 141 struct icmp_filter {
 142         __u32   data;
 143 };
 144 #endif
 145
 146 static void signal_handler(int number)
 147 {
 148         switch (number) {
 149         case SIGINT:
 150                 sigint = 1;
 151         case SIGHUP:
 152         default:
 153                 break;
 154         }
 155 }
 156
 157 static void timer_elapsed(int number)
 158 {
 159         itimer.it_interval.tv_sec = 0;
 160         itimer.it_interval.tv_usec = interval;
 161
 162         itimer.it_value.tv_sec = 0;
 163         itimer.it_value.tv_usec = interval;
 164
 165         pull_and_flush_tx_ring(sock);
 166         setitimer(ITIMER_REAL, &itimer, NULL);
 167 }
 168
 169 static void header(void)
 170 {
 171         printf("%s%s%s\n", colorize_start(bold), "trafgen " VERSION_STRING, colorize_end());
 172 }
 173
 174 static void help(void)
 175 {
 176         printf("\ntrafgen %s, multithreaded zero-copy network packet generator\n", VERSION_STRING);
 177         puts("http://www.netsniff-ng.org\n\n"
 178              "Usage: trafgen [options]\n"
 179              "Options:\n"
 180              "  -o|-d|--out|--dev <netdev>        Networking Device i.e., eth0\n"
 181              "  -i|-c|--in|--conf <cfg-file>      Packet configuration file\n"
 182              "  -J|--jumbo-support                Support 64KB Super Jumbo Frames (def: 2048B)\n"
 183              "  -R|--rfraw                        Inject raw 802.11 frames\n"
 184              "  -s|--smoke-test <ipv4-receiver>   Test if machine survived packet\n"
 185              "  -n|--num <uint>                   Number of packets until exit (def: 0)\n"
 186              "  -r|--rand                         Randomize packet selection (def: round robin)\n"
 187              "  -P|--cpus <uint>                  Specify number of forks(<= CPUs) (def: #CPUs)\n"
 188              "  -t|--gap <uint>                   Interpacket gap in us (approx)\n"
 189              "  -S|--ring-size <size>             Manually set mmap size (KB/MB/GB): e.g.\'10MB\'\n"
 190              "  -k|--kernel-pull <uint>           Kernel batch interval in us (def: 10us)\n"
 191              "  -V|--verbose                      Be more verbose\n"
 192              "  -v|--version                      Show version\n"
 193              "  -e|--example                      Show built-in packet config example\n"
 194              "  -h|--help                         Guess what?!\n\n"
 195              "Examples:\n"
 196              "  See trafgen.txf for configuration file examples.\n"
 197              "  trafgen --dev eth0 --conf trafgen.cfg\n"
 198              "  trafgen --dev eth0 --conf trafgen.cfg --smoke-test 10.0.0.1\n"
 199              "  trafgen --dev wlan0 --rfraw --conf beacon-test.txf -V --cpus 2\n"
 200              "  trafgen --dev eth0 --conf trafgen.cfg --rand --gap 1000\n"
 201              "  trafgen --dev eth0 --conf trafgen.cfg --rand --num 1400000 -k1000\n\n"
 202              "Arbitrary packet config examples (e.g. trafgen -e > trafgen.cfg):\n"
 203              "  Run packet on  all CPUs:              { fill(0xff, 64) csum16(0, 64) }\n"
 204              "  Run packet only on CPU1:    cpu(1):   { rnd(64), 0b11001100, 0xaa }\n"
 205              "  Run packet only on CPU1-2:  cpu(1:2): { drnd(64),'a',csum16(1, 8),'b',42 }\n\n"
 206              "Note:\n"
 207              "  Smoke test example: machine A, 10.0.0.2 (trafgen) is directly\n"
 208              "  connected to machine B (test kernel), 10.0.0.1. If ICMP reply fails\n"
 209              "  we assume the kernel crashed, thus we print the packet and quit.\n"
 210              "  In case you find a ping-of-death, please mention trafgen in your\n"
 211              "  commit message of the fix!\n\n"
 212              "  This tool is targeted for network developers! You should\n"
 213              "  be aware of what you are doing and what these options above\n"
 214              "  mean! Only use this tool in an isolated LAN that you own!\n\n"
 215              "Please report bugs to <bugs@netsniff-ng.org>\n"
 216              "Copyright (C) 2011-2013 Daniel Borkmann <dborkma@tik.ee.ethz.ch>,\n"
 217              "Swiss federal institute of technology (ETH Zurich)\n"
 218              "License: GNU GPL version 2.0\n"
 219              "This is free software: you are free to change and redistribute it.\n"
 220              "There is NO WARRANTY, to the extent permitted by law.\n");
 221         die();
 222 }
 223
 224 static void example(void)
 225 {
 226         puts("{\n"
 227              "  # MAC Destination\n"
 228              "  drnd(6),\n"
 229              "  # MAC Source\n"
 230              "  drnd(6),\n"
 231              "  # Protocol\n"
 232              "  0x08,0x00,\n"
 233              "  # IP header bits'n'pieces\n"
 234              "  0x45,0x00,0x00,0x34,0xaf,0xf1,0x00,0x00,0x37,0x06,\n"
 235              "  # Dynamic IP Checksum\n"
 236              "  csumip(14, 34),\n"
 237              "  # Source IP\n"
 238              "  drnd(4),\n"
 239              "  # Dest IP\n"
 240              "  drnd(4),\n"
 241              "  # TCP Proto\n"
 242              "  drnd(32),\n"
 243              "}");
 244         die();
 245 }
 246
 247 static void version(void)
 248 {
 249         printf("\ntrafgen %s, multithreaded zero-copy network packet generator\n", VERSION_STRING);
 250         puts("http://www.netsniff-ng.org\n\n"
 251              "Please report bugs to <bugs@netsniff-ng.org>\n"
 252              "Copyright (C) 2011-2013 Daniel Borkmann <dborkma@tik.ee.ethz.ch>,\n"
 253              "Swiss federal institute of technology (ETH Zurich)\n"
 254              "License: GNU GPL version 2.0\n"
 255              "This is free software: you are free to change and redistribute it.\n"
 256              "There is NO WARRANTY, to the extent permitted by law.\n");
 257         die();
 258 }
 259
 260 static void apply_counter(int counter_id)
 261 {
 262         int j, i = counter_id;
 263         size_t counter_max = packet_dyn[i].clen;
 264
 265         for (j = 0; j < counter_max; ++j) {
 266                 uint8_t val;
 267                 struct counter *counter = &packet_dyn[i].cnt[j];
 268
 269                 val = counter->val - counter->min;
 270
 271                 switch (counter->type) {
 272                 case TYPE_INC:
 273                         val = (val + counter->inc) % (counter->max - counter->min + 1);
 274                         break;
 275                 case TYPE_DEC:
 276                         val = (val - counter->inc) % (counter->min - counter->max + 1);
 277                         break;
 278                 default:
 279                         bug();
 280                 }
 281
 282                 counter->val = val + counter->min;
 283                 packets[i].payload[counter->off] = val;
 284         }
 285 }
 286
 287 static void apply_randomizer(int rand_id)
 288 {
 289         int j, i = rand_id;
 290         size_t rand_max = packet_dyn[i].rlen;
 291
 292         for (j = 0; j < rand_max; ++j) {
 293                 uint8_t val = (uint8_t) rand();
 294                 struct randomizer *randomizer = &packet_dyn[i].rnd[j];
 295
 296                 packets[i].payload[randomizer->off] = val;
 297         }
 298 }
 299
 300 static void apply_csum16(int csum_id)
 301 {
 302         int j, i = csum_id;
 303         size_t csum_max = packet_dyn[i].slen;
 304
 305         for (j = 0; j < csum_max; ++j) {
 306                 uint16_t sum;
 307                 uint8_t *psum;
 308                 struct csum16 *csum = &packet_dyn[i].csum[j];
 309
 310                 packets[i].payload[csum->off]     = 0;
 311                 packets[i].payload[csum->off + 1] = 0;
 312
 313                 if (csum->to >= packets[i].len)
 314                         csum->to = packets[i].len - 1;
 315
 316                 sum = htons(calc_csum(packets[i].payload + csum->from,
 317                                       csum->to - csum->from, 0));
 318                 psum = (uint8_t *) &sum;
 319
 320                 packets[i].payload[csum->off]     = psum[0];
 321                 packets[i].payload[csum->off + 1] = psum[1];
 322         }
 323 }
 324
 325 static struct cpu_stats *setup_shared_var(unsigned long cpus)
 326 {
 327         int fd;
 328         char zbuff[cpus * sizeof(struct cpu_stats)], file[256];
 329         struct cpu_stats *buff;
 330
 331         memset(zbuff, 0, sizeof(zbuff));
 332         slprintf(file, sizeof(file), ".tmp_mmap.%u", (unsigned int) rand());
 333
 334         fd = creat(file, S_IRUSR | S_IWUSR);
 335         bug_on(fd < 0);
 336         close(fd);
 337
 338         fd = open_or_die_m(file, O_RDWR | O_CREAT | O_TRUNC,
 339                            S_IRUSR | S_IWUSR);
 340         write_or_die(fd, zbuff, sizeof(zbuff));
 341
 342         buff = (void *) mmap(0, sizeof(zbuff), PROT_READ | PROT_WRITE,
 343                              MAP_SHARED, fd, 0);
 344         if (buff == (void *) -1)
 345                 panic("Cannot setup shared variable!\n");
 346
 347         close(fd);
 348         unlink(file);
 349
 350         memset(buff, 0, sizeof(zbuff));
 351
 352         return buff;
 353 }
 354
 355 static void destroy_shared_var(void *buff, unsigned long cpus)
 356 {
 357         munmap(buff, cpus * sizeof(struct cpu_stats));
 358 }
 359
 360 static void dump_trafgen_snippet(uint8_t *payload, size_t len)
 361 {
 362         int i;
 363
 364         printf("{");
 365         for (i = 0; i < len; ++i) {
 366                 if (i % 15 == 0)
 367                         printf("\n  ");
 368                 printf("0x%02x, ", payload[i]);
 369         }
 370         printf("\n}\n");
 371         fflush(stdout);
 372 }
 373
 374 static inline unsigned short csum(unsigned short *buf, int nwords)
 375 {
 376         unsigned long sum;
 377
 378         for (sum = 0; nwords > 0; nwords--)
 379                 sum += *buf++;
 380         sum = (sum >> 16) + (sum & 0xffff);
 381         sum += (sum >> 16);
 382
 383         return ~sum;
 384 }
 385
 386 static int xmit_smoke_setup(struct ctx *ctx)
 387 {
 388         int icmp_sock, ret, ttl = 64;
 389         struct icmp_filter filter;
 390
 391         icmp_sock = socket(AF_INET, SOCK_RAW, IPPROTO_ICMP);
 392         if (icmp_sock < 0)
 393                 panic("Cannot get a ICMP socket: %s!\n", strerror(errno));
 394
 395         filter.data = ~(1 << ICMP_ECHOREPLY);
 396
 397         ret = setsockopt(icmp_sock, SOL_RAW, ICMP_FILTER, &filter, sizeof(filter));
 398         if (ret < 0)
 399                 panic("Cannot install filter!\n");
 400
 401         ret = setsockopt(icmp_sock, SOL_IP, IP_TTL, &ttl, sizeof(ttl));
 402         if (ret < 0)
 403                 panic("Cannot set TTL!\n");
 404
 405         memset(&ctx->dest, 0, sizeof(ctx->dest));
 406         ctx->dest.sin_family = AF_INET;
 407         ctx->dest.sin_port = 0;
 408
 409         ret = inet_aton(ctx->rhost, &ctx->dest.sin_addr);
 410         if (ret < 0)
 411                 panic("Cannot resolv address!\n");
 412
 413         return icmp_sock;
 414 }
 415
 416 static int xmit_smoke_probe(int icmp_sock, struct ctx *ctx)
 417 {
 418         int ret, i, probes = 5;
 419         short ident, cnt = 1;
 420         uint8_t outpack[512], *data;
 421         struct icmphdr *icmp;
 422         struct iphdr *ip;
 423         size_t len = sizeof(*icmp) + 56;
 424         struct sockaddr_in from;
 425         socklen_t from_len;
 426         struct pollfd fds = {
 427                 .fd = icmp_sock,
 428                 .events = POLLIN,
 429         };
 430
 431         while (probes-- > 0) {
 432                 ident = htons((short) rand());
 433
 434                 memset(outpack, 0, sizeof(outpack));
 435                 icmp = (void *) outpack;
 436                 icmp->type = ICMP_ECHO;
 437                 icmp->code = 0;
 438                 icmp->checksum = 0;
 439                 icmp->un.echo.id = ident;
 440                 icmp->un.echo.sequence = htons(cnt++);
 441
 442                 data = ((uint8_t *) outpack + sizeof(*icmp));
 443                 for (i = 0; i < 56; ++i)
 444                         data[i] = (uint8_t) rand();
 445
 446                 icmp->checksum = csum((unsigned short *) outpack,
 447                                       len / sizeof(unsigned short));
 448
 449                 ret = sendto(icmp_sock, outpack, len, MSG_DONTWAIT,
 450                              (struct sockaddr *) &ctx->dest, sizeof(ctx->dest));
 451                 if (unlikely(ret != len))
 452                         panic("Cannot send out probe: %s!\n", strerror(errno));
 453
 454                 ret = poll(&fds, 1, 500);
 455                 if (ret < 0)
 456                         panic("Poll failed!\n");
 457
 458                 if (fds.revents & POLLIN) {
 459                         ret = recvfrom(icmp_sock, outpack, sizeof(outpack), 0,
 460                                        (struct sockaddr *) &from, &from_len);
 461                         if (unlikely(ret <= 0))
 462                                 panic("Probe receive failed!\n");
 463                         if (unlikely(from_len != sizeof(ctx->dest)))
 464                                 continue;
 465                         if (unlikely(memcmp(&from, &ctx->dest, sizeof(ctx->dest))))
 466                                 continue;
 467                         if (unlikely(ret < sizeof(*ip) + sizeof(*icmp)))
 468                                 continue;
 469                         ip = (void *) outpack;
 470                         if (unlikely(ip->ihl * 4 + sizeof(*icmp) > ret))
 471                                 continue;
 472                         icmp = (void *) outpack + ip->ihl * 4;
 473                         if (unlikely(icmp->un.echo.id != ident))
 474                                 continue;
 475
 476                         return 0;
 477                 }
 478         }
 479
 480         return -1;
 481 }
 482
 483 static void xmit_slowpath_or_die(struct ctx *ctx, int cpu)
 484 {
 485         int ret, icmp_sock = -1;
 486         unsigned long num = 1, i = 0;
 487         struct timeval start, end, diff;
 488         unsigned long long tx_bytes = 0, tx_packets = 0;
 489         struct packet_dyn *pktd;
 490         struct sockaddr_ll saddr = {
 491                 .sll_family = PF_PACKET,
 492                 .sll_halen = ETH_ALEN,
 493                 .sll_ifindex = device_ifindex(ctx->device),
 494         };
 495
 496         if (ctx->num > 0)
 497                 num = ctx->num;
 498
 499         if (ctx->smoke_test)
 500                 icmp_sock = xmit_smoke_setup(ctx);
 501
 502         bug_on(gettimeofday(&start, NULL));
 503
 504         while (likely(sigint == 0) && likely(num > 0)) {
 505                 pktd = &packet_dyn[i];
 506                 if (pktd->clen + pktd->rlen + pktd->slen) {
 507                         apply_counter(i);
 508                         apply_randomizer(i);
 509                         apply_csum16(i);
 510                 }
 511 retry:
 512                 ret = sendto(sock, packets[i].payload, packets[i].len, 0,
 513                              (struct sockaddr *) &saddr, sizeof(saddr));
 514                 if (unlikely(ret < 0)) {
 515                         if (errno == ENOBUFS) {
 516                                 sched_yield();
 517                                 goto retry;
 518                         }
 519
 520                         panic("Sendto error: %s!\n", strerror(errno));
 521                 }
 522
 523                 tx_bytes += packets[i].len;
 524                 tx_packets++;
 525
 526                 if (ctx->smoke_test) {
 527                         ret = xmit_smoke_probe(icmp_sock, ctx);
 528                         if (unlikely(ret < 0)) {
 529                                 printf("%sSmoke test alert:%s\n", colorize_start(bold), colorize_end());
 530                                 printf("  Remote host seems to be unresponsive to ICMP pings!\n");
 531                                 printf("  Last instance was packet%lu, trafgen snippet:\n\n", i);
 532
 533                                 dump_trafgen_snippet(packets[i].payload, packets[i].len);
 534                                 break;
 535                         }
 536                 }
 537
 538                 if (!ctx->rand) {
 539                         i++;
 540                         if (i >= plen)
 541                                 i = 0;
 542                 } else
 543                         i = rand() % plen;
 544
 545                 if (ctx->num > 0)
 546                         num--;
 547
 548                 if (ctx->gap > 0)
 549                         usleep(ctx->gap);
 550         }
 551
 552         bug_on(gettimeofday(&end, NULL));
 553         diff = tv_subtract(end, start);
 554
 555         if (ctx->smoke_test)
 556                 close(icmp_sock);
 557
 558         stats[cpu].tx_packets = tx_packets;
 559         stats[cpu].tx_bytes = tx_bytes;
 560         stats[cpu].tv_sec = diff.tv_sec;
 561         stats[cpu].tv_usec = diff.tv_usec;
 562
 563         stats[cpu].state |= CPU_STATS_STATE_RES;
 564 }
 565
 566 static void xmit_fastpath_or_die(struct ctx *ctx, int cpu)
 567 {
 568         int ifindex = device_ifindex(ctx->device);
 569         uint8_t *out = NULL;
 570         unsigned int it = 0;
 571         unsigned long num = 1, i = 0, size;
 572         struct ring tx_ring;
 573         struct frame_map *hdr;
 574         struct timeval start, end, diff;
 575         struct packet_dyn *pktd;
 576         unsigned long long tx_bytes = 0, tx_packets = 0;
 577
 578         fmemset(&tx_ring, 0, sizeof(tx_ring));
 579
 580         size = ring_size(ctx->device, ctx->reserve_size);
 581
 582         set_sock_prio(sock, 512);
 583         set_packet_loss_discard(sock);
 584
 585         setup_tx_ring_layout(sock, &tx_ring, size, ctx->jumbo_support);
 586         create_tx_ring(sock, &tx_ring, ctx->verbose);
 587         mmap_tx_ring(sock, &tx_ring);
 588         alloc_tx_ring_frames(&tx_ring);
 589         bind_tx_ring(sock, &tx_ring, ifindex);
 590
 591         if (ctx->kpull)
 592                 interval = ctx->kpull;
 593         if (ctx->num > 0)
 594                 num = ctx->num;
 595
 596         itimer.it_interval.tv_sec = 0;
 597         itimer.it_interval.tv_usec = interval;
 598
 599         itimer.it_value.tv_sec = 0;
 600         itimer.it_value.tv_usec = interval;
 601
 602         setitimer(ITIMER_REAL, &itimer, NULL);
 603
 604         bug_on(gettimeofday(&start, NULL));
 605
 606         while (likely(sigint == 0) && likely(num > 0)) {
 607                 while (user_may_pull_from_tx(tx_ring.frames[it].iov_base) && likely(num > 0)) {
 608                         hdr = tx_ring.frames[it].iov_base;
 609
 610                         /* Kernel assumes: data = ph.raw + po->tp_hdrlen -
 611                          *                        sizeof(struct sockaddr_ll); */
 612                         out = ((uint8_t *) hdr) + TPACKET2_HDRLEN - sizeof(struct sockaddr_ll);
 613
 614                         hdr->tp_h.tp_snaplen = packets[i].len;
 615                         hdr->tp_h.tp_len = packets[i].len;
 616
 617                         pktd = &packet_dyn[i];
 618                         if (pktd->clen + pktd->rlen + pktd->slen) {
 619                                 apply_counter(i);
 620                                 apply_randomizer(i);
 621                                 apply_csum16(i);
 622                         }
 623
 624                         fmemcpy(out, packets[i].payload, packets[i].len);
 625
 626                         tx_bytes += packets[i].len;
 627                         tx_packets++;
 628
 629                         if (!ctx->rand) {
 630                                 i++;
 631                                 if (i >= plen)
 632                                         i = 0;
 633                         } else
 634                                 i = rand() % plen;
 635
 636                         kernel_may_pull_from_tx(&hdr->tp_h);
 637
 638                         it++;
 639                         if (it >= tx_ring.layout.tp_frame_nr)
 640                                 it = 0;
 641
 642                         if (ctx->num > 0)
 643                                 num--;
 644
 645                         if (unlikely(sigint == 1))
 646                                 break;
 647                 }
 648         }
 649
 650         bug_on(gettimeofday(&end, NULL));
 651         diff = tv_subtract(end, start);
 652
 653         destroy_tx_ring(sock, &tx_ring);
 654
 655         stats[cpu].tx_packets = tx_packets;
 656         stats[cpu].tx_bytes = tx_bytes;
 657         stats[cpu].tv_sec = diff.tv_sec;
 658         stats[cpu].tv_usec = diff.tv_usec;
 659
 660         stats[cpu].state |= CPU_STATS_STATE_RES;
 661 }
 662
 663 static inline void __set_state(int cpu, sig_atomic_t s)
 664 {
 665         stats[cpu].state = s;
 666 }
 667
 668 static inline sig_atomic_t __get_state(int cpu)
 669 {
 670         return stats[cpu].state;
 671 }
 672
 673 static unsigned long __wait_and_sum_others(struct ctx *ctx, int cpu)
 674 {
 675         int i;
 676         unsigned long total;
 677
 678         for (i = 0, total = plen; i < ctx->cpus; i++) {
 679                 if (i == cpu)
 680                         continue;
 681
 682                 while ((__get_state(i) & CPU_STATS_STATE_CFG) == 0 &&
 683                        sigint == 0)
 684                         sched_yield();
 685
 686                 total += stats[i].cf_packets;
 687         }
 688
 689         return total;
 690 }
 691
 692 static void __correct_global_delta(struct ctx *ctx, int cpu, unsigned long orig)
 693 {
 694         int i, cpu_sel;
 695         unsigned long total;
 696         long long delta_correction = 0;
 697
 698         for (i = 0, total = ctx->num; i < ctx->cpus; i++) {
 699                 if (i == cpu)
 700                         continue;
 701
 702                 while ((__get_state(i) & CPU_STATS_STATE_CHK) == 0 &&
 703                        sigint == 0)
 704                         sched_yield();
 705
 706                 total += stats[i].cd_packets;
 707         }
 708
 709         if (total > orig)
 710                 delta_correction = -1 * ((long long) total - orig);
 711         if (total < orig)
 712                 delta_correction = +1 * ((long long) orig - total);
 713
 714         for (cpu_sel = -1, i = 0; i < ctx->cpus; i++) {
 715                 if (stats[i].cd_packets > 0) {
 716                         if ((long long) stats[i].cd_packets +
 717                             delta_correction > 0) {
 718                                 cpu_sel = i;
 719                                 break;
 720                         }
 721                 }
 722         }
 723
 724         if (cpu == cpu_sel)
 725                 ctx->num += delta_correction;
 726 }
 727
 728 static void __set_state_cf(int cpu, unsigned long p, unsigned long b,
 729                            sig_atomic_t s)
 730 {
 731         stats[cpu].cf_packets = p;
 732         stats[cpu].cf_bytes = b;
 733         stats[cpu].state = s;
 734 }
 735
 736 static void __set_state_cd(int cpu, unsigned long p, sig_atomic_t s)
 737 {
 738         stats[cpu].cd_packets = p;
 739         stats[cpu].state = s;
 740 }
 741
 742 static int xmit_packet_precheck(struct ctx *ctx, int cpu)
 743 {
 744         int i;
 745         unsigned long plen_total, orig = ctx->num;
 746         size_t mtu, total_len = 0;
 747
 748         bug_on(plen != dlen);
 749
 750         for (i = 0; i < plen; ++i)
 751                 total_len += packets[i].len;
 752
 753         __set_state_cf(cpu, plen, total_len, CPU_STATS_STATE_CFG);
 754         plen_total = __wait_and_sum_others(ctx, cpu);
 755
 756         if (orig > 0) {
 757                 ctx->num = (unsigned long) nearbyint((1.0 * plen / plen_total) * orig);
 758
 759                 __set_state_cd(cpu, ctx->num, CPU_STATS_STATE_CHK |
 760                                CPU_STATS_STATE_CFG);
 761                 __correct_global_delta(ctx, cpu, orig);
 762         }
 763
 764         if (plen == 0) {
 765                 __set_state(cpu, CPU_STATS_STATE_RES);
 766                 return -1;
 767         }
 768
 769         for (mtu = device_mtu(ctx->device), i = 0; i < plen; ++i) {
 770                 if (packets[i].len > mtu + 14)
 771                         panic("Device MTU < than packet%d's size!\n", i);
 772                 if (packets[i].len <= 14)
 773                         panic("Packet%d's size too short!\n", i);
 774         }
 775
 776         return 0;
 777 }
 778
 779 static void main_loop(struct ctx *ctx, char *confname, bool slow, int cpu)
 780 {
 781         compile_packets(confname, ctx->verbose, cpu);
 782         if (xmit_packet_precheck(ctx, cpu) < 0)
 783                 return;
 784
 785         if (cpu == 0) {
 786                 int i;
 787                 size_t total_len = 0, total_pkts = 0;
 788
 789                 for (i = 0; i < ctx->cpus; ++i) {
 790                         total_len  += stats[i].cf_bytes;
 791                         total_pkts += stats[i].cf_packets;
 792                 }
 793
 794                 printf("%6zu packets to schedule\n", total_pkts);
 795                 printf("%6zu bytes in total\n", total_len);
 796                 printf("Running! Hang up with ^C!\n\n");
 797                 fflush(stdout);
 798         }
 799
 800         sock = pf_socket();
 801
 802         if (slow)
 803                 xmit_slowpath_or_die(ctx, cpu);
 804         else
 805                 xmit_fastpath_or_die(ctx, cpu);
 806
 807         close(sock);
 808
 809         cleanup_packets();
 810 }
 811
 812 int main(int argc, char **argv)
 813 {
 814         bool slow = false;
 815         int c, opt_index, i, j, vals[4] = {0}, irq;
 816         char *confname = NULL, *ptr;
 817         unsigned long cpus_tmp;
 818         unsigned long long tx_packets, tx_bytes;
 819         struct ctx ctx;
 820
 821         setfsuid(getuid());
 822         setfsgid(getgid());
 823
 824         srand(time(NULL));
 825         fmemset(&ctx, 0, sizeof(ctx));
 826         ctx.cpus = get_number_cpus_online();
 827
 828         while ((c = getopt_long(argc, argv, short_options, long_options,
 829                                 &opt_index)) != EOF) {
 830                 switch (c) {
 831                 case 'h':
 832                         help();
 833                         break;
 834                 case 'v':
 835                         version();
 836                         break;
 837                 case 'e':
 838                         example();
 839                         break;
 840                 case 'V':
 841                         ctx.verbose = true;
 842                         break;
 843                 case 'P':
 844                         cpus_tmp = strtoul(optarg, NULL, 0);
 845                         if (cpus_tmp > 0 && cpus_tmp < ctx.cpus)
 846                                 ctx.cpus = cpus_tmp;
 847                         break;
 848                 case 'd':
 849                 case 'o':
 850                         ctx.device = xstrndup(optarg, IFNAMSIZ);
 851                         break;
 852                 case 'r':
 853                         ctx.rand = true;
 854                         break;
 855                 case 's':
 856                         slow = true;
 857                         ctx.cpus = 1;
 858                         ctx.smoke_test = true;
 859                         ctx.rhost = xstrdup(optarg);
 860                         break;
 861                 case 'R':
 862                         ctx.rfraw = true;
 863                         break;
 864                 case 'J':
 865                         ctx.jumbo_support = true;
 866                         break;
 867                 case 'c':
 868                 case 'i':
 869                         confname = xstrdup(optarg);
 870                         break;
 871                 case 'k':
 872                         ctx.kpull = strtoul(optarg, NULL, 0);
 873                         break;
 874                 case 'n':
 875                         ctx.num = strtoul(optarg, NULL, 0);
 876                         break;
 877                 case 't':
 878                         slow = true;
 879                         ctx.gap = strtoul(optarg, NULL, 0);
 880                         if (ctx.gap > 0)
 881                                 /* Fall back to single core to not
 882                                  * mess up correct timing. We are slow
 883                                  * anyway!
 884                                  */
 885                                 ctx.cpus = 1;
 886                         break;
 887                 case 'S':
 888                         ptr = optarg;
 889                         ctx.reserve_size = 0;
 890
 891                         for (j = i = strlen(optarg); i > 0; --i) {
 892                                 if (!isdigit(optarg[j - i]))
 893                                         break;
 894                                 ptr++;
 895                         }
 896
 897                         if (!strncmp(ptr, "KB", strlen("KB")))
 898                                 ctx.reserve_size = 1 << 10;
 899                         else if (!strncmp(ptr, "MB", strlen("MB")))
 900                                 ctx.reserve_size = 1 << 20;
 901                         else if (!strncmp(ptr, "GB", strlen("GB")))
 902                                 ctx.reserve_size = 1 << 30;
 903                         else
 904                                 panic("Syntax error in ring size param!\n");
 905                         *ptr = 0;
 906
 907                         ctx.reserve_size *= strtol(optarg, NULL, 0);
 908                         break;
 909                 case '?':
 910                         switch (optopt) {
 911                         case 'd':
 912                         case 'c':
 913                         case 'n':
 914                         case 'S':
 915                         case 's':
 916                         case 'P':
 917                         case 'o':
 918                         case 'i':
 919                         case 'k':
 920                         case 't':
 921                                 panic("Option -%c requires an argument!\n",
 922                                       optopt);
 923                         default:
 924                                 if (isprint(optopt))
 925                                         whine("Unknown option character "
 926                                               "`0x%X\'!\n", optopt);
 927                                 die();
 928                         }
 929                 default:
 930                         break;
 931                 }
 932         }
 933
 934         if (argc < 5)
 935                 help();
 936         if (ctx.device == NULL)
 937                 panic("No networking device given!\n");
 938         if (confname == NULL)
 939                 panic("No configuration file given!\n");
 940         if (device_mtu(ctx.device) == 0)
 941                 panic("This is no networking device!\n");
 942         if (!ctx.rfraw && device_up_and_running(ctx.device) == 0)
 943                 panic("Networking device not running!\n");
 944
 945         register_signal(SIGINT, signal_handler);
 946         register_signal(SIGHUP, signal_handler);
 947         register_signal_f(SIGALRM, timer_elapsed, SA_SIGINFO);
 948
 949         header();
 950
 951         set_system_socket_memory(vals);
 952
 953         if (ctx.rfraw) {
 954                 ctx.device_trans = xstrdup(ctx.device);
 955                 xfree(ctx.device);
 956
 957                 enter_rfmon_mac80211(ctx.device_trans, &ctx.device);
 958                 sleep(0);
 959         }
 960
 961         irq = device_irq_number(ctx.device);
 962         device_reset_irq_affinity(irq);
 963
 964         if (ctx.num > 0 && ctx.num <= ctx.cpus)
 965                 ctx.cpus = 1;
 966
 967         stats = setup_shared_var(ctx.cpus);
 968
 969         for (i = 0; i < ctx.cpus; i++) {
 970                 pid_t pid = fork();
 971
 972                 switch (pid) {
 973                 case 0:
 974                         cpu_affinity(i);
 975                         main_loop(&ctx, confname, slow, i);
 976
 977                         goto thread_out;
 978                 case -1:
 979                         panic("Cannot fork processes!\n");
 980                 }
 981         }
 982
 983         for (i = 0; i < ctx.cpus; i++) {
 984                 int status;
 985
 986                 wait(&status);
 987                 if (WEXITSTATUS(status) == EXIT_FAILURE)
 988                         die();
 989         }
 990
 991         if (ctx.rfraw)
 992                 leave_rfmon_mac80211(ctx.device_trans, ctx.device);
 993
 994         reset_system_socket_memory(vals);
 995
 996         for (i = 0, tx_packets = tx_bytes = 0; i < ctx.cpus; i++) {
 997                 while ((__get_state(i) & CPU_STATS_STATE_RES) == 0)
 998                         sched_yield();
 999
1000                 tx_packets += stats[i].tx_packets;
1001                 tx_bytes   += stats[i].tx_bytes;
1002         }
1003
1004         fflush(stdout);
1005         printf("\n");
1006         printf("\r%12llu packets outgoing\n", tx_packets);
1007         printf("\r%12llu bytes outgoing\n", tx_bytes);
1008         for (i = 0; i < ctx.cpus; i++) {
1009                 printf("\r%12lu sec, %lu usec on CPU%d (%llu packets)\n",
1010                        stats[i].tv_sec, stats[i].tv_usec, i,
1011                        stats[i].tx_packets);
1012         }
1013
1014 thread_out:
1015         destroy_shared_var(stats, ctx.cpus);
1016
1017         free(ctx.device);
1018         free(ctx.device_trans);
1019         free(ctx.rhost);
1020         free(confname);
1021
1022         return 0;
1023 }