?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
fix race condition in cor_announce_send_stop
[cor.git] / samples / bpf / xdpsock_user.c
bloba15480010828fbe53c71692af71d4c958166979c
1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 2017 - 2018 Intel Corporation. */
3
4 #include <asm/barrier.h>
5 #include <errno.h>
6 #include <getopt.h>
7 #include <libgen.h>
8 #include <linux/bpf.h>
9 #include <linux/compiler.h>
10 #include <linux/if_link.h>
11 #include <linux/if_xdp.h>
12 #include <linux/if_ether.h>
13 #include <locale.h>
14 #include <net/ethernet.h>
15 #include <net/if.h>
16 #include <poll.h>
17 #include <pthread.h>
18 #include <signal.h>
19 #include <stdbool.h>
20 #include <stdio.h>
21 #include <stdlib.h>
22 #include <string.h>
23 #include <sys/mman.h>
24 #include <sys/resource.h>
25 #include <sys/socket.h>
26 #include <sys/types.h>
27 #include <time.h>
28 #include <unistd.h>
29
30 #include "libbpf.h"
31 #include "xsk.h"
32 #include "xdpsock.h"
33 #include <bpf/bpf.h>
34
35 #ifndef SOL_XDP
36 #define SOL_XDP 283
37 #endif
38
39 #ifndef AF_XDP
40 #define AF_XDP 44
41 #endif
42
43 #ifndef PF_XDP
44 #define PF_XDP AF_XDP
45 #endif
46
47 #define NUM_FRAMES (4 * 1024)
48 #define BATCH_SIZE 64
49
50 #define DEBUG_HEXDUMP 0
51
52 typedef __u64 u64;
53 typedef __u32 u32;
54
55 static unsigned long prev_time;
56
57 enum benchmark_type {
58 BENCH_RXDROP = 0,
59 BENCH_TXONLY = 1,
60 BENCH_L2FWD = 2,
61 };
62
63 static enum benchmark_type opt_bench = BENCH_RXDROP;
64 static u32 opt_xdp_flags = XDP_FLAGS_UPDATE_IF_NOEXIST;
65 static const char *opt_if = "";
66 static int opt_ifindex;
67 static int opt_queue;
68 static int opt_poll;
69 static int opt_interval = 1;
70 static u32 opt_xdp_bind_flags = XDP_USE_NEED_WAKEUP;
71 static u32 opt_umem_flags;
72 static int opt_unaligned_chunks;
73 static int opt_mmap_flags;
74 static u32 opt_xdp_bind_flags;
75 static int opt_xsk_frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE;
76 static int opt_timeout = 1000;
77 static bool opt_need_wakeup = true;
78 static u32 opt_num_xsks = 1;
79 static u32 prog_id;
80
81 struct xsk_umem_info {
82 struct xsk_ring_prod fq;
83 struct xsk_ring_cons cq;
84 struct xsk_umem *umem;
85 void *buffer;
86 };
87
88 struct xsk_socket_info {
89 struct xsk_ring_cons rx;
90 struct xsk_ring_prod tx;
91 struct xsk_umem_info *umem;
92 struct xsk_socket *xsk;
93 unsigned long rx_npkts;
94 unsigned long tx_npkts;
95 unsigned long prev_rx_npkts;
96 unsigned long prev_tx_npkts;
97 u32 outstanding_tx;
98 };
99
100 static int num_socks;
101 struct xsk_socket_info *xsks[MAX_SOCKS];
102
103 static unsigned long get_nsecs(void)
104 {
105 struct timespec ts;
106
107 clock_gettime(CLOCK_MONOTONIC, &ts);
108 return ts.tv_sec * 1000000000UL + ts.tv_nsec;
109 }
110
111 static void print_benchmark(bool running)
112 {
113 const char *bench_str = "INVALID";
114
115 if (opt_bench == BENCH_RXDROP)
116 bench_str = "rxdrop";
117 else if (opt_bench == BENCH_TXONLY)
118 bench_str = "txonly";
119 else if (opt_bench == BENCH_L2FWD)
120 bench_str = "l2fwd";
121
122 printf("%s:%d %s ", opt_if, opt_queue, bench_str);
123 if (opt_xdp_flags & XDP_FLAGS_SKB_MODE)
124 printf("xdp-skb ");
125 else if (opt_xdp_flags & XDP_FLAGS_DRV_MODE)
126 printf("xdp-drv ");
127 else
128 printf(" ");
129
130 if (opt_poll)
131 printf("poll() ");
132
133 if (running) {
134 printf("running...");
135 fflush(stdout);
136 }
137 }
138
139 static void dump_stats(void)
140 {
141 unsigned long now = get_nsecs();
142 long dt = now - prev_time;
143 int i;
144
145 prev_time = now;
146
147 for (i = 0; i < num_socks && xsks[i]; i++) {
148 char *fmt = "%-15s %'-11.0f %'-11lu\n";
149 double rx_pps, tx_pps;
150
151 rx_pps = (xsks[i]->rx_npkts - xsks[i]->prev_rx_npkts) *
152 1000000000. / dt;
153 tx_pps = (xsks[i]->tx_npkts - xsks[i]->prev_tx_npkts) *
154 1000000000. / dt;
155
156 printf("\n sock%d@", i);
157 print_benchmark(false);
158 printf("\n");
159
160 printf("%-15s %-11s %-11s %-11.2f\n", "", "pps", "pkts",
161 dt / 1000000000.);
162 printf(fmt, "rx", rx_pps, xsks[i]->rx_npkts);
163 printf(fmt, "tx", tx_pps, xsks[i]->tx_npkts);
164
165 xsks[i]->prev_rx_npkts = xsks[i]->rx_npkts;
166 xsks[i]->prev_tx_npkts = xsks[i]->tx_npkts;
167 }
168 }
169
170 static void *poller(void *arg)
171 {
172 (void)arg;
173 for (;;) {
174 sleep(opt_interval);
175 dump_stats();
176 }
177
178 return NULL;
179 }
180
181 static void remove_xdp_program(void)
182 {
183 u32 curr_prog_id = 0;
184
185 if (bpf_get_link_xdp_id(opt_ifindex, &curr_prog_id, opt_xdp_flags)) {
186 printf("bpf_get_link_xdp_id failed\n");
187 exit(EXIT_FAILURE);
188 }
189 if (prog_id == curr_prog_id)
190 bpf_set_link_xdp_fd(opt_ifindex, -1, opt_xdp_flags);
191 else if (!curr_prog_id)
192 printf("couldn't find a prog id on a given interface\n");
193 else
194 printf("program on interface changed, not removing\n");
195 }
196
197 static void int_exit(int sig)
198 {
199 struct xsk_umem *umem = xsks[0]->umem->umem;
200 int i;
201
202 dump_stats();
203 for (i = 0; i < num_socks; i++)
204 xsk_socket__delete(xsks[i]->xsk);
205 (void)xsk_umem__delete(umem);
206 remove_xdp_program();
207
208 exit(EXIT_SUCCESS);
209 }
210
211 static void __exit_with_error(int error, const char *file, const char *func,
212 int line)
213 {
214 fprintf(stderr, "%s:%s:%i: errno: %d/\"%s\"\n", file, func,
215 line, error, strerror(error));
216 dump_stats();
217 remove_xdp_program();
218 exit(EXIT_FAILURE);
219 }
220
221 #define exit_with_error(error) __exit_with_error(error, __FILE__, __func__, \
222 __LINE__)
223
224 static const char pkt_data[] =
225 "\x3c\xfd\xfe\x9e\x7f\x71\xec\xb1\xd7\x98\x3a\xc0\x08\x00\x45\x00"
226 "\x00\x2e\x00\x00\x00\x00\x40\x11\x88\x97\x05\x08\x07\x08\xc8\x14"
227 "\x1e\x04\x10\x92\x10\x92\x00\x1a\x6d\xa3\x34\x33\x1f\x69\x40\x6b"
228 "\x54\x59\xb6\x14\x2d\x11\x44\xbf\xaf\xd9\xbe\xaa";
229
230 static void swap_mac_addresses(void *data)
231 {
232 struct ether_header *eth = (struct ether_header *)data;
233 struct ether_addr *src_addr = (struct ether_addr *)&eth->ether_shost;
234 struct ether_addr *dst_addr = (struct ether_addr *)&eth->ether_dhost;
235 struct ether_addr tmp;
236
237 tmp = *src_addr;
238 *src_addr = *dst_addr;
239 *dst_addr = tmp;
240 }
241
242 static void hex_dump(void *pkt, size_t length, u64 addr)
243 {
244 const unsigned char *address = (unsigned char *)pkt;
245 const unsigned char *line = address;
246 size_t line_size = 32;
247 unsigned char c;
248 char buf[32];
249 int i = 0;
250
251 if (!DEBUG_HEXDUMP)
252 return;
253
254 sprintf(buf, "addr=%llu", addr);
255 printf("length = %zu\n", length);
256 printf("%s | ", buf);
257 while (length-- > 0) {
258 printf("%02X ", *address++);
259 if (!(++i % line_size) || (length == 0 && i % line_size)) {
260 if (length == 0) {
261 while (i++ % line_size)
262 printf("__ ");
263 }
264 printf(" | "); /* right close */
265 while (line < address) {
266 c = *line++;
267 printf("%c", (c < 33 || c == 255) ? 0x2E : c);
268 }
269 printf("\n");
270 if (length > 0)
271 printf("%s | ", buf);
272 }
273 }
274 printf("\n");
275 }
276
277 static size_t gen_eth_frame(struct xsk_umem_info *umem, u64 addr)
278 {
279 memcpy(xsk_umem__get_data(umem->buffer, addr), pkt_data,
280 sizeof(pkt_data) - 1);
281 return sizeof(pkt_data) - 1;
282 }
283
284 static struct xsk_umem_info *xsk_configure_umem(void *buffer, u64 size)
285 {
286 struct xsk_umem_info *umem;
287 struct xsk_umem_config cfg = {
288 .fill_size = XSK_RING_PROD__DEFAULT_NUM_DESCS,
289 .comp_size = XSK_RING_CONS__DEFAULT_NUM_DESCS,
290 .frame_size = opt_xsk_frame_size,
291 .frame_headroom = XSK_UMEM__DEFAULT_FRAME_HEADROOM,
292 .flags = opt_umem_flags
293 };
294 int ret;
295
296 umem = calloc(1, sizeof(*umem));
297 if (!umem)
298 exit_with_error(errno);
299
300 ret = xsk_umem__create(&umem->umem, buffer, size, &umem->fq, &umem->cq,
301 &cfg);
302 if (ret)
303 exit_with_error(-ret);
304
305 umem->buffer = buffer;
306 return umem;
307 }
308
309 static void xsk_populate_fill_ring(struct xsk_umem_info *umem)
310 {
311 int ret, i;
312 u32 idx;
313
314 ret = xsk_ring_prod__reserve(&umem->fq,
315 XSK_RING_PROD__DEFAULT_NUM_DESCS, &idx);
316 if (ret != XSK_RING_PROD__DEFAULT_NUM_DESCS)
317 exit_with_error(-ret);
318 for (i = 0; i < XSK_RING_PROD__DEFAULT_NUM_DESCS; i++)
319 *xsk_ring_prod__fill_addr(&umem->fq, idx++) =
320 i * opt_xsk_frame_size;
321 xsk_ring_prod__submit(&umem->fq, XSK_RING_PROD__DEFAULT_NUM_DESCS);
322 }
323
324 static struct xsk_socket_info *xsk_configure_socket(struct xsk_umem_info *umem,
325 bool rx, bool tx)
326 {
327 struct xsk_socket_config cfg;
328 struct xsk_socket_info *xsk;
329 struct xsk_ring_cons *rxr;
330 struct xsk_ring_prod *txr;
331 int ret;
332
333 xsk = calloc(1, sizeof(*xsk));
334 if (!xsk)
335 exit_with_error(errno);
336
337 xsk->umem = umem;
338 cfg.rx_size = XSK_RING_CONS__DEFAULT_NUM_DESCS;
339 cfg.tx_size = XSK_RING_PROD__DEFAULT_NUM_DESCS;
340 if (opt_num_xsks > 1)
341 cfg.libbpf_flags = XSK_LIBBPF_FLAGS__INHIBIT_PROG_LOAD;
342 else
343 cfg.libbpf_flags = 0;
344 cfg.xdp_flags = opt_xdp_flags;
345 cfg.bind_flags = opt_xdp_bind_flags;
346
347 rxr = rx ? &xsk->rx : NULL;
348 txr = tx ? &xsk->tx : NULL;
349 ret = xsk_socket__create(&xsk->xsk, opt_if, opt_queue, umem->umem,
350 rxr, txr, &cfg);
351 if (ret)
352 exit_with_error(-ret);
353
354 ret = bpf_get_link_xdp_id(opt_ifindex, &prog_id, opt_xdp_flags);
355 if (ret)
356 exit_with_error(-ret);
357
358 return xsk;
359 }
360
361 static struct option long_options[] = {
362 {"rxdrop", no_argument, 0, 'r'},
363 {"txonly", no_argument, 0, 't'},
364 {"l2fwd", no_argument, 0, 'l'},
365 {"interface", required_argument, 0, 'i'},
366 {"queue", required_argument, 0, 'q'},
367 {"poll", no_argument, 0, 'p'},
368 {"xdp-skb", no_argument, 0, 'S'},
369 {"xdp-native", no_argument, 0, 'N'},
370 {"interval", required_argument, 0, 'n'},
371 {"zero-copy", no_argument, 0, 'z'},
372 {"copy", no_argument, 0, 'c'},
373 {"frame-size", required_argument, 0, 'f'},
374 {"no-need-wakeup", no_argument, 0, 'm'},
375 {"unaligned", no_argument, 0, 'u'},
376 {"shared-umem", no_argument, 0, 'M'},
377 {"force", no_argument, 0, 'F'},
378 {0, 0, 0, 0}
379 };
380
381 static void usage(const char *prog)
382 {
383 const char *str =
384 " Usage: %s [OPTIONS]\n"
385 " Options:\n"
386 " -r, --rxdrop Discard all incoming packets (default)\n"
387 " -t, --txonly Only send packets\n"
388 " -l, --l2fwd MAC swap L2 forwarding\n"
389 " -i, --interface=n Run on interface n\n"
390 " -q, --queue=n Use queue n (default 0)\n"
391 " -p, --poll Use poll syscall\n"
392 " -S, --xdp-skb=n Use XDP skb-mod\n"
393 " -N, --xdp-native=n Enforce XDP native mode\n"
394 " -n, --interval=n Specify statistics update interval (default 1 sec).\n"
395 " -z, --zero-copy Force zero-copy mode.\n"
396 " -c, --copy Force copy mode.\n"
397 " -m, --no-need-wakeup Turn off use of driver need wakeup flag.\n"
398 " -f, --frame-size=n Set the frame size (must be a power of two in aligned mode, default is %d).\n"
399 " -u, --unaligned Enable unaligned chunk placement\n"
400 " -M, --shared-umem Enable XDP_SHARED_UMEM\n"
401 " -F, --force Force loading the XDP prog\n"
402 "\n";
403 fprintf(stderr, str, prog, XSK_UMEM__DEFAULT_FRAME_SIZE);
404 exit(EXIT_FAILURE);
405 }
406
407 static void parse_command_line(int argc, char **argv)
408 {
409 int option_index, c;
410
411 opterr = 0;
412
413 for (;;) {
414 c = getopt_long(argc, argv, "Frtli:q:psSNn:czf:muM",
415 long_options, &option_index);
416 if (c == -1)
417 break;
418
419 switch (c) {
420 case 'r':
421 opt_bench = BENCH_RXDROP;
422 break;
423 case 't':
424 opt_bench = BENCH_TXONLY;
425 break;
426 case 'l':
427 opt_bench = BENCH_L2FWD;
428 break;
429 case 'i':
430 opt_if = optarg;
431 break;
432 case 'q':
433 opt_queue = atoi(optarg);
434 break;
435 case 'p':
436 opt_poll = 1;
437 break;
438 case 'S':
439 opt_xdp_flags |= XDP_FLAGS_SKB_MODE;
440 opt_xdp_bind_flags |= XDP_COPY;
441 break;
442 case 'N':
443 opt_xdp_flags |= XDP_FLAGS_DRV_MODE;
444 break;
445 case 'n':
446 opt_interval = atoi(optarg);
447 break;
448 case 'z':
449 opt_xdp_bind_flags |= XDP_ZEROCOPY;
450 break;
451 case 'c':
452 opt_xdp_bind_flags |= XDP_COPY;
453 break;
454 case 'u':
455 opt_umem_flags |= XDP_UMEM_UNALIGNED_CHUNK_FLAG;
456 opt_unaligned_chunks = 1;
457 opt_mmap_flags = MAP_HUGETLB;
458 break;
459 case 'F':
460 opt_xdp_flags &= ~XDP_FLAGS_UPDATE_IF_NOEXIST;
461 break;
462 case 'f':
463 opt_xsk_frame_size = atoi(optarg);
464 break;
465 case 'm':
466 opt_need_wakeup = false;
467 opt_xdp_bind_flags &= ~XDP_USE_NEED_WAKEUP;
468 break;
469 case 'M':
470 opt_num_xsks = MAX_SOCKS;
471 break;
472 default:
473 usage(basename(argv[0]));
474 }
475 }
476
477 opt_ifindex = if_nametoindex(opt_if);
478 if (!opt_ifindex) {
479 fprintf(stderr, "ERROR: interface \"%s\" does not exist\n",
480 opt_if);
481 usage(basename(argv[0]));
482 }
483
484 if ((opt_xsk_frame_size & (opt_xsk_frame_size - 1)) &&
485 !opt_unaligned_chunks) {
486 fprintf(stderr, "--frame-size=%d is not a power of two\n",
487 opt_xsk_frame_size);
488 usage(basename(argv[0]));
489 }
490 }
491
492 static void kick_tx(struct xsk_socket_info *xsk)
493 {
494 int ret;
495
496 ret = sendto(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, 0);
497 if (ret >= 0 || errno == ENOBUFS || errno == EAGAIN || errno == EBUSY)
498 return;
499 exit_with_error(errno);
500 }
501
502 static inline void complete_tx_l2fwd(struct xsk_socket_info *xsk,
503 struct pollfd *fds)
504 {
505 struct xsk_umem_info *umem = xsk->umem;
506 u32 idx_cq = 0, idx_fq = 0;
507 unsigned int rcvd;
508 size_t ndescs;
509
510 if (!xsk->outstanding_tx)
511 return;
512
513 if (!opt_need_wakeup || xsk_ring_prod__needs_wakeup(&xsk->tx))
514 kick_tx(xsk);
515
516 ndescs = (xsk->outstanding_tx > BATCH_SIZE) ? BATCH_SIZE :
517 xsk->outstanding_tx;
518
519 /* re-add completed Tx buffers */
520 rcvd = xsk_ring_cons__peek(&umem->cq, ndescs, &idx_cq);
521 if (rcvd > 0) {
522 unsigned int i;
523 int ret;
524
525 ret = xsk_ring_prod__reserve(&umem->fq, rcvd, &idx_fq);
526 while (ret != rcvd) {
527 if (ret < 0)
528 exit_with_error(-ret);
529 if (xsk_ring_prod__needs_wakeup(&umem->fq))
530 ret = poll(fds, num_socks, opt_timeout);
531 ret = xsk_ring_prod__reserve(&umem->fq, rcvd, &idx_fq);
532 }
533
534 for (i = 0; i < rcvd; i++)
535 *xsk_ring_prod__fill_addr(&umem->fq, idx_fq++) =
536 *xsk_ring_cons__comp_addr(&umem->cq, idx_cq++);
537
538 xsk_ring_prod__submit(&xsk->umem->fq, rcvd);
539 xsk_ring_cons__release(&xsk->umem->cq, rcvd);
540 xsk->outstanding_tx -= rcvd;
541 xsk->tx_npkts += rcvd;
542 }
543 }
544
545 static inline void complete_tx_only(struct xsk_socket_info *xsk)
546 {
547 unsigned int rcvd;
548 u32 idx;
549
550 if (!xsk->outstanding_tx)
551 return;
552
553 if (!opt_need_wakeup || xsk_ring_prod__needs_wakeup(&xsk->tx))
554 kick_tx(xsk);
555
556 rcvd = xsk_ring_cons__peek(&xsk->umem->cq, BATCH_SIZE, &idx);
557 if (rcvd > 0) {
558 xsk_ring_cons__release(&xsk->umem->cq, rcvd);
559 xsk->outstanding_tx -= rcvd;
560 xsk->tx_npkts += rcvd;
561 }
562 }
563
564 static void rx_drop(struct xsk_socket_info *xsk, struct pollfd *fds)
565 {
566 unsigned int rcvd, i;
567 u32 idx_rx = 0, idx_fq = 0;
568 int ret;
569
570 rcvd = xsk_ring_cons__peek(&xsk->rx, BATCH_SIZE, &idx_rx);
571 if (!rcvd) {
572 if (xsk_ring_prod__needs_wakeup(&xsk->umem->fq))
573 ret = poll(fds, num_socks, opt_timeout);
574 return;
575 }
576
577 ret = xsk_ring_prod__reserve(&xsk->umem->fq, rcvd, &idx_fq);
578 while (ret != rcvd) {
579 if (ret < 0)
580 exit_with_error(-ret);
581 if (xsk_ring_prod__needs_wakeup(&xsk->umem->fq))
582 ret = poll(fds, num_socks, opt_timeout);
583 ret = xsk_ring_prod__reserve(&xsk->umem->fq, rcvd, &idx_fq);
584 }
585
586 for (i = 0; i < rcvd; i++) {
587 u64 addr = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx)->addr;
588 u32 len = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx++)->len;
589 u64 orig = xsk_umem__extract_addr(addr);
590
591 addr = xsk_umem__add_offset_to_addr(addr);
592 char *pkt = xsk_umem__get_data(xsk->umem->buffer, addr);
593
594 hex_dump(pkt, len, addr);
595 *xsk_ring_prod__fill_addr(&xsk->umem->fq, idx_fq++) = orig;
596 }
597
598 xsk_ring_prod__submit(&xsk->umem->fq, rcvd);
599 xsk_ring_cons__release(&xsk->rx, rcvd);
600 xsk->rx_npkts += rcvd;
601 }
602
603 static void rx_drop_all(void)
604 {
605 struct pollfd fds[MAX_SOCKS] = {};
606 int i, ret;
607
608 for (i = 0; i < num_socks; i++) {
609 fds[i].fd = xsk_socket__fd(xsks[i]->xsk);
610 fds[i].events = POLLIN;
611 }
612
613 for (;;) {
614 if (opt_poll) {
615 ret = poll(fds, num_socks, opt_timeout);
616 if (ret <= 0)
617 continue;
618 }
619
620 for (i = 0; i < num_socks; i++)
621 rx_drop(xsks[i], fds);
622 }
623 }
624
625 static void tx_only(struct xsk_socket_info *xsk, u32 frame_nb)
626 {
627 u32 idx;
628
629 if (xsk_ring_prod__reserve(&xsk->tx, BATCH_SIZE, &idx) == BATCH_SIZE) {
630 unsigned int i;
631
632 for (i = 0; i < BATCH_SIZE; i++) {
633 xsk_ring_prod__tx_desc(&xsk->tx, idx + i)->addr =
634 (frame_nb + i) << XSK_UMEM__DEFAULT_FRAME_SHIFT;
635 xsk_ring_prod__tx_desc(&xsk->tx, idx + i)->len =
636 sizeof(pkt_data) - 1;
637 }
638
639 xsk_ring_prod__submit(&xsk->tx, BATCH_SIZE);
640 xsk->outstanding_tx += BATCH_SIZE;
641 frame_nb += BATCH_SIZE;
642 frame_nb %= NUM_FRAMES;
643 }
644
645 complete_tx_only(xsk);
646 }
647
648 static void tx_only_all(void)
649 {
650 struct pollfd fds[MAX_SOCKS] = {};
651 u32 frame_nb[MAX_SOCKS] = {};
652 int i, ret;
653
654 for (i = 0; i < num_socks; i++) {
655 fds[0].fd = xsk_socket__fd(xsks[i]->xsk);
656 fds[0].events = POLLOUT;
657 }
658
659 for (;;) {
660 if (opt_poll) {
661 ret = poll(fds, num_socks, opt_timeout);
662 if (ret <= 0)
663 continue;
664
665 if (!(fds[0].revents & POLLOUT))
666 continue;
667 }
668
669 for (i = 0; i < num_socks; i++)
670 tx_only(xsks[i], frame_nb[i]);
671 }
672 }
673
674 static void l2fwd(struct xsk_socket_info *xsk, struct pollfd *fds)
675 {
676 unsigned int rcvd, i;
677 u32 idx_rx = 0, idx_tx = 0;
678 int ret;
679
680 complete_tx_l2fwd(xsk, fds);
681
682 rcvd = xsk_ring_cons__peek(&xsk->rx, BATCH_SIZE, &idx_rx);
683 if (!rcvd) {
684 if (xsk_ring_prod__needs_wakeup(&xsk->umem->fq))
685 ret = poll(fds, num_socks, opt_timeout);
686 return;
687 }
688
689 ret = xsk_ring_prod__reserve(&xsk->tx, rcvd, &idx_tx);
690 while (ret != rcvd) {
691 if (ret < 0)
692 exit_with_error(-ret);
693 if (xsk_ring_prod__needs_wakeup(&xsk->tx))
694 kick_tx(xsk);
695 ret = xsk_ring_prod__reserve(&xsk->tx, rcvd, &idx_tx);
696 }
697
698 for (i = 0; i < rcvd; i++) {
699 u64 addr = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx)->addr;
700 u32 len = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx++)->len;
701 u64 orig = addr;
702
703 addr = xsk_umem__add_offset_to_addr(addr);
704 char *pkt = xsk_umem__get_data(xsk->umem->buffer, addr);
705
706 swap_mac_addresses(pkt);
707
708 hex_dump(pkt, len, addr);
709 xsk_ring_prod__tx_desc(&xsk->tx, idx_tx)->addr = orig;
710 xsk_ring_prod__tx_desc(&xsk->tx, idx_tx++)->len = len;
711 }
712
713 xsk_ring_prod__submit(&xsk->tx, rcvd);
714 xsk_ring_cons__release(&xsk->rx, rcvd);
715
716 xsk->rx_npkts += rcvd;
717 xsk->outstanding_tx += rcvd;
718 }
719
720 static void l2fwd_all(void)
721 {
722 struct pollfd fds[MAX_SOCKS] = {};
723 int i, ret;
724
725 for (i = 0; i < num_socks; i++) {
726 fds[i].fd = xsk_socket__fd(xsks[i]->xsk);
727 fds[i].events = POLLOUT | POLLIN;
728 }
729
730 for (;;) {
731 if (opt_poll) {
732 ret = poll(fds, num_socks, opt_timeout);
733 if (ret <= 0)
734 continue;
735 }
736
737 for (i = 0; i < num_socks; i++)
738 l2fwd(xsks[i], fds);
739 }
740 }
741
742 static void load_xdp_program(char **argv, struct bpf_object **obj)
743 {
744 struct bpf_prog_load_attr prog_load_attr = {
745 .prog_type = BPF_PROG_TYPE_XDP,
746 };
747 char xdp_filename[256];
748 int prog_fd;
749
750 snprintf(xdp_filename, sizeof(xdp_filename), "%s_kern.o", argv[0]);
751 prog_load_attr.file = xdp_filename;
752
753 if (bpf_prog_load_xattr(&prog_load_attr, obj, &prog_fd))
754 exit(EXIT_FAILURE);
755 if (prog_fd < 0) {
756 fprintf(stderr, "ERROR: no program found: %s\n",
757 strerror(prog_fd));
758 exit(EXIT_FAILURE);
759 }
760
761 if (bpf_set_link_xdp_fd(opt_ifindex, prog_fd, opt_xdp_flags) < 0) {
762 fprintf(stderr, "ERROR: link set xdp fd failed\n");
763 exit(EXIT_FAILURE);
764 }
765 }
766
767 static void enter_xsks_into_map(struct bpf_object *obj)
768 {
769 struct bpf_map *map;
770 int i, xsks_map;
771
772 map = bpf_object__find_map_by_name(obj, "xsks_map");
773 xsks_map = bpf_map__fd(map);
774 if (xsks_map < 0) {
775 fprintf(stderr, "ERROR: no xsks map found: %s\n",
776 strerror(xsks_map));
777 exit(EXIT_FAILURE);
778 }
779
780 for (i = 0; i < num_socks; i++) {
781 int fd = xsk_socket__fd(xsks[i]->xsk);
782 int key, ret;
783
784 key = i;
785 ret = bpf_map_update_elem(xsks_map, &key, &fd, 0);
786 if (ret) {
787 fprintf(stderr, "ERROR: bpf_map_update_elem %d\n", i);
788 exit(EXIT_FAILURE);
789 }
790 }
791 }
792
793 int main(int argc, char **argv)
794 {
795 struct rlimit r = {RLIM_INFINITY, RLIM_INFINITY};
796 bool rx = false, tx = false;
797 struct xsk_umem_info *umem;
798 struct bpf_object *obj;
799 pthread_t pt;
800 int i, ret;
801 void *bufs;
802
803 parse_command_line(argc, argv);
804
805 if (setrlimit(RLIMIT_MEMLOCK, &r)) {
806 fprintf(stderr, "ERROR: setrlimit(RLIMIT_MEMLOCK) \"%s\"\n",
807 strerror(errno));
808 exit(EXIT_FAILURE);
809 }
810
811 if (opt_num_xsks > 1)
812 load_xdp_program(argv, &obj);
813
814 /* Reserve memory for the umem. Use hugepages if unaligned chunk mode */
815 bufs = mmap(NULL, NUM_FRAMES * opt_xsk_frame_size,
816 PROT_READ | PROT_WRITE,
817 MAP_PRIVATE | MAP_ANONYMOUS | opt_mmap_flags, -1, 0);
818 if (bufs == MAP_FAILED) {
819 printf("ERROR: mmap failed\n");
820 exit(EXIT_FAILURE);
821 }
822
823 /* Create sockets... */
824 umem = xsk_configure_umem(bufs, NUM_FRAMES * opt_xsk_frame_size);
825 if (opt_bench == BENCH_RXDROP || opt_bench == BENCH_L2FWD) {
826 rx = true;
827 xsk_populate_fill_ring(umem);
828 }
829 if (opt_bench == BENCH_L2FWD || opt_bench == BENCH_TXONLY)
830 tx = true;
831 for (i = 0; i < opt_num_xsks; i++)
832 xsks[num_socks++] = xsk_configure_socket(umem, rx, tx);
833
834 if (opt_bench == BENCH_TXONLY)
835 for (i = 0; i < NUM_FRAMES; i++)
836 gen_eth_frame(umem, i * opt_xsk_frame_size);
837
838 if (opt_num_xsks > 1 && opt_bench != BENCH_TXONLY)
839 enter_xsks_into_map(obj);
840
841 signal(SIGINT, int_exit);
842 signal(SIGTERM, int_exit);
843 signal(SIGABRT, int_exit);
844
845 setlocale(LC_ALL, "");
846
847 ret = pthread_create(&pt, NULL, poller, NULL);
848 if (ret)
849 exit_with_error(ret);
850
851 prev_time = get_nsecs();
852
853 if (opt_bench == BENCH_RXDROP)
854 rx_drop_all();
855 else if (opt_bench == BENCH_TXONLY)
856 tx_only_all();
857 else
858 l2fwd_all();
859
860 return 0;
861 }
connection oriented routing