ebpf/ebpf_rss.c

   1 /*
   2  * eBPF RSS loader
   3  *
   4  * Developed by Daynix Computing LTD (http://www.daynix.com)
   5  *
   6  * Authors:
   7  *  Andrew Melnychenko <andrew@daynix.com>
   8  *  Yuri Benditovich <yuri.benditovich@daynix.com>
   9  *
  10  * This work is licensed under the terms of the GNU GPL, version 2.  See
  11  * the COPYING file in the top-level directory.
  12  */
  13
  14 #include "qemu/osdep.h"
  15 #include "qemu/error-report.h"
  16 #include "qapi/qapi-types-misc.h"
  17 #include "qapi/qapi-commands-ebpf.h"
  18
  19 #include <bpf/libbpf.h>
  20 #include <bpf/bpf.h>
  21
  22 #include "hw/virtio/virtio-net.h" /* VIRTIO_NET_RSS_MAX_TABLE_LEN */
  23
  24 #include "ebpf/ebpf_rss.h"
  25 #include "ebpf/rss.bpf.skeleton.h"
  26 #include "ebpf/ebpf.h"
  27
  28 void ebpf_rss_init(struct EBPFRSSContext *ctx)
  29 {
  30     if (ctx != NULL) {
  31         ctx->obj = NULL;
  32         ctx->program_fd = -1;
  33         ctx->map_configuration = -1;
  34         ctx->map_toeplitz_key = -1;
  35         ctx->map_indirections_table = -1;
  36
  37         ctx->mmap_configuration = NULL;
  38         ctx->mmap_toeplitz_key = NULL;
  39         ctx->mmap_indirections_table = NULL;
  40     }
  41 }
  42
  43 bool ebpf_rss_is_loaded(struct EBPFRSSContext *ctx)
  44 {
  45     return ctx != NULL && (ctx->obj != NULL || ctx->program_fd != -1);
  46 }
  47
  48 static bool ebpf_rss_mmap(struct EBPFRSSContext *ctx)
  49 {
  50     if (!ebpf_rss_is_loaded(ctx)) {
  51         return false;
  52     }
  53
  54     ctx->mmap_configuration = mmap(NULL, qemu_real_host_page_size(),
  55                                    PROT_READ | PROT_WRITE, MAP_SHARED,
  56                                    ctx->map_configuration, 0);
  57     if (ctx->mmap_configuration == MAP_FAILED) {
  58         return false;
  59     }
  60     ctx->mmap_toeplitz_key = mmap(NULL, qemu_real_host_page_size(),
  61                                    PROT_READ | PROT_WRITE, MAP_SHARED,
  62                                    ctx->map_toeplitz_key, 0);
  63     if (ctx->mmap_toeplitz_key == MAP_FAILED) {
  64         goto toeplitz_fail;
  65     }
  66     ctx->mmap_indirections_table = mmap(NULL, qemu_real_host_page_size(),
  67                                    PROT_READ | PROT_WRITE, MAP_SHARED,
  68                                    ctx->map_indirections_table, 0);
  69     if (ctx->mmap_indirections_table == MAP_FAILED) {
  70         goto indirection_fail;
  71     }
  72
  73     return true;
  74
  75 indirection_fail:
  76     munmap(ctx->mmap_toeplitz_key, qemu_real_host_page_size());
  77     ctx->mmap_toeplitz_key = NULL;
  78 toeplitz_fail:
  79     munmap(ctx->mmap_configuration, qemu_real_host_page_size());
  80     ctx->mmap_configuration = NULL;
  81
  82     ctx->mmap_indirections_table = NULL;
  83     return false;
  84 }
  85
  86 static void ebpf_rss_munmap(struct EBPFRSSContext *ctx)
  87 {
  88     if (!ebpf_rss_is_loaded(ctx)) {
  89         return;
  90     }
  91
  92     munmap(ctx->mmap_indirections_table, qemu_real_host_page_size());
  93     munmap(ctx->mmap_toeplitz_key, qemu_real_host_page_size());
  94     munmap(ctx->mmap_configuration, qemu_real_host_page_size());
  95
  96     ctx->mmap_configuration = NULL;
  97     ctx->mmap_toeplitz_key = NULL;
  98     ctx->mmap_indirections_table = NULL;
  99 }
 100
 101 bool ebpf_rss_load(struct EBPFRSSContext *ctx)
 102 {
 103     struct rss_bpf *rss_bpf_ctx;
 104
 105     if (ebpf_rss_is_loaded(ctx)) {
 106         return false;
 107     }
 108
 109     rss_bpf_ctx = rss_bpf__open();
 110     if (rss_bpf_ctx == NULL) {
 111         goto error;
 112     }
 113
 114     bpf_program__set_type(rss_bpf_ctx->progs.tun_rss_steering_prog, BPF_PROG_TYPE_SOCKET_FILTER);
 115
 116     if (rss_bpf__load(rss_bpf_ctx)) {
 117         goto error;
 118     }
 119
 120     ctx->obj = rss_bpf_ctx;
 121     ctx->program_fd = bpf_program__fd(
 122             rss_bpf_ctx->progs.tun_rss_steering_prog);
 123     ctx->map_configuration = bpf_map__fd(
 124             rss_bpf_ctx->maps.tap_rss_map_configurations);
 125     ctx->map_indirections_table = bpf_map__fd(
 126             rss_bpf_ctx->maps.tap_rss_map_indirection_table);
 127     ctx->map_toeplitz_key = bpf_map__fd(
 128             rss_bpf_ctx->maps.tap_rss_map_toeplitz_key);
 129
 130     if (!ebpf_rss_mmap(ctx)) {
 131         goto error;
 132     }
 133
 134     return true;
 135 error:
 136     rss_bpf__destroy(rss_bpf_ctx);
 137     ctx->obj = NULL;
 138     ctx->program_fd = -1;
 139     ctx->map_configuration = -1;
 140     ctx->map_toeplitz_key = -1;
 141     ctx->map_indirections_table = -1;
 142
 143     return false;
 144 }
 145
 146 bool ebpf_rss_load_fds(struct EBPFRSSContext *ctx, int program_fd,
 147                        int config_fd, int toeplitz_fd, int table_fd)
 148 {
 149     if (ebpf_rss_is_loaded(ctx)) {
 150         return false;
 151     }
 152
 153     if (program_fd < 0 || config_fd < 0 || toeplitz_fd < 0 || table_fd < 0) {
 154         return false;
 155     }
 156
 157     ctx->program_fd = program_fd;
 158     ctx->map_configuration = config_fd;
 159     ctx->map_toeplitz_key = toeplitz_fd;
 160     ctx->map_indirections_table = table_fd;
 161
 162     if (!ebpf_rss_mmap(ctx)) {
 163         ctx->program_fd = -1;
 164         ctx->map_configuration = -1;
 165         ctx->map_toeplitz_key = -1;
 166         ctx->map_indirections_table = -1;
 167         return false;
 168     }
 169
 170     return true;
 171 }
 172
 173 static bool ebpf_rss_set_config(struct EBPFRSSContext *ctx,
 174                                 struct EBPFRSSConfig *config)
 175 {
 176     if (!ebpf_rss_is_loaded(ctx)) {
 177         return false;
 178     }
 179
 180     memcpy(ctx->mmap_configuration, config, sizeof(*config));
 181     return true;
 182 }
 183
 184 static bool ebpf_rss_set_indirections_table(struct EBPFRSSContext *ctx,
 185                                             uint16_t *indirections_table,
 186                                             size_t len)
 187 {
 188     char *cursor = ctx->mmap_indirections_table;
 189
 190     if (!ebpf_rss_is_loaded(ctx) || indirections_table == NULL ||
 191        len > VIRTIO_NET_RSS_MAX_TABLE_LEN) {
 192         return false;
 193     }
 194
 195     for (size_t i = 0; i < len; i++) {
 196         *(uint16_t *)cursor = indirections_table[i];
 197         cursor += 8;
 198     }
 199
 200     return true;
 201 }
 202
 203 static bool ebpf_rss_set_toepliz_key(struct EBPFRSSContext *ctx,
 204                                      uint8_t *toeplitz_key)
 205 {
 206     /* prepare toeplitz key */
 207     uint8_t toe[VIRTIO_NET_RSS_MAX_KEY_SIZE] = {};
 208
 209     if (!ebpf_rss_is_loaded(ctx) || toeplitz_key == NULL) {
 210         return false;
 211     }
 212     memcpy(toe, toeplitz_key, VIRTIO_NET_RSS_MAX_KEY_SIZE);
 213     *(uint32_t *)toe = ntohl(*(uint32_t *)toe);
 214
 215     memcpy(ctx->mmap_toeplitz_key, toe, VIRTIO_NET_RSS_MAX_KEY_SIZE);
 216     return true;
 217 }
 218
 219 bool ebpf_rss_set_all(struct EBPFRSSContext *ctx, struct EBPFRSSConfig *config,
 220                       uint16_t *indirections_table, uint8_t *toeplitz_key)
 221 {
 222     if (!ebpf_rss_is_loaded(ctx) || config == NULL ||
 223         indirections_table == NULL || toeplitz_key == NULL) {
 224         return false;
 225     }
 226
 227     if (!ebpf_rss_set_config(ctx, config)) {
 228         return false;
 229     }
 230
 231     if (!ebpf_rss_set_indirections_table(ctx, indirections_table,
 232                                       config->indirections_len)) {
 233         return false;
 234     }
 235
 236     if (!ebpf_rss_set_toepliz_key(ctx, toeplitz_key)) {
 237         return false;
 238     }
 239
 240     return true;
 241 }
 242
 243 void ebpf_rss_unload(struct EBPFRSSContext *ctx)
 244 {
 245     if (!ebpf_rss_is_loaded(ctx)) {
 246         return;
 247     }
 248
 249     ebpf_rss_munmap(ctx);
 250
 251     if (ctx->obj) {
 252         rss_bpf__destroy(ctx->obj);
 253     } else {
 254         close(ctx->program_fd);
 255         close(ctx->map_configuration);
 256         close(ctx->map_toeplitz_key);
 257         close(ctx->map_indirections_table);
 258     }
 259
 260     ctx->obj = NULL;
 261     ctx->program_fd = -1;
 262     ctx->map_configuration = -1;
 263     ctx->map_toeplitz_key = -1;
 264     ctx->map_indirections_table = -1;
 265 }
 266
 267 ebpf_binary_init(EBPF_PROGRAMID_RSS, rss_bpf__elf_bytes)