ebpf/ebpf_rss.c

   1 /*
   2  * eBPF RSS loader
   3  *
   4  * Developed by Daynix Computing LTD (http://www.daynix.com)
   5  *
   6  * Authors:
   7  *  Andrew Melnychenko <andrew@daynix.com>
   8  *  Yuri Benditovich <yuri.benditovich@daynix.com>
   9  *
  10  * This work is licensed under the terms of the GNU GPL, version 2.  See
  11  * the COPYING file in the top-level directory.
  12  */
  13
  14 #include "qemu/osdep.h"
  15 #include "qemu/error-report.h"
  16 #include "qapi/qapi-types-misc.h"
  17 #include "qapi/qapi-commands-ebpf.h"
  18
  19 #include <bpf/libbpf.h>
  20 #include <bpf/bpf.h>
  21
  22 #include "hw/virtio/virtio-net.h" /* VIRTIO_NET_RSS_MAX_TABLE_LEN */
  23
  24 #include "ebpf/ebpf_rss.h"
  25 #include "ebpf/rss.bpf.skeleton.h"
  26 #include "ebpf/ebpf.h"
  27
  28 #include "trace.h"
  29
  30 void ebpf_rss_init(struct EBPFRSSContext *ctx)
  31 {
  32     if (ctx != NULL) {
  33         ctx->obj = NULL;
  34         ctx->program_fd = -1;
  35         ctx->map_configuration = -1;
  36         ctx->map_toeplitz_key = -1;
  37         ctx->map_indirections_table = -1;
  38
  39         ctx->mmap_configuration = NULL;
  40         ctx->mmap_toeplitz_key = NULL;
  41         ctx->mmap_indirections_table = NULL;
  42     }
  43 }
  44
  45 bool ebpf_rss_is_loaded(struct EBPFRSSContext *ctx)
  46 {
  47     return ctx != NULL && (ctx->obj != NULL || ctx->program_fd != -1);
  48 }
  49
  50 static bool ebpf_rss_mmap(struct EBPFRSSContext *ctx)
  51 {
  52     if (!ebpf_rss_is_loaded(ctx)) {
  53         return false;
  54     }
  55
  56     ctx->mmap_configuration = mmap(NULL, qemu_real_host_page_size(),
  57                                    PROT_READ | PROT_WRITE, MAP_SHARED,
  58                                    ctx->map_configuration, 0);
  59     if (ctx->mmap_configuration == MAP_FAILED) {
  60         trace_ebpf_error("eBPF RSS", "can not mmap eBPF configuration array");
  61         return false;
  62     }
  63     ctx->mmap_toeplitz_key = mmap(NULL, qemu_real_host_page_size(),
  64                                    PROT_READ | PROT_WRITE, MAP_SHARED,
  65                                    ctx->map_toeplitz_key, 0);
  66     if (ctx->mmap_toeplitz_key == MAP_FAILED) {
  67         trace_ebpf_error("eBPF RSS", "can not mmap eBPF toeplitz key");
  68         goto toeplitz_fail;
  69     }
  70     ctx->mmap_indirections_table = mmap(NULL, qemu_real_host_page_size(),
  71                                    PROT_READ | PROT_WRITE, MAP_SHARED,
  72                                    ctx->map_indirections_table, 0);
  73     if (ctx->mmap_indirections_table == MAP_FAILED) {
  74         trace_ebpf_error("eBPF RSS", "can not mmap eBPF indirection table");
  75         goto indirection_fail;
  76     }
  77
  78     return true;
  79
  80 indirection_fail:
  81     munmap(ctx->mmap_toeplitz_key, qemu_real_host_page_size());
  82     ctx->mmap_toeplitz_key = NULL;
  83 toeplitz_fail:
  84     munmap(ctx->mmap_configuration, qemu_real_host_page_size());
  85     ctx->mmap_configuration = NULL;
  86
  87     ctx->mmap_indirections_table = NULL;
  88     return false;
  89 }
  90
  91 static void ebpf_rss_munmap(struct EBPFRSSContext *ctx)
  92 {
  93     if (!ebpf_rss_is_loaded(ctx)) {
  94         return;
  95     }
  96
  97     munmap(ctx->mmap_indirections_table, qemu_real_host_page_size());
  98     munmap(ctx->mmap_toeplitz_key, qemu_real_host_page_size());
  99     munmap(ctx->mmap_configuration, qemu_real_host_page_size());
 100
 101     ctx->mmap_configuration = NULL;
 102     ctx->mmap_toeplitz_key = NULL;
 103     ctx->mmap_indirections_table = NULL;
 104 }
 105
 106 bool ebpf_rss_load(struct EBPFRSSContext *ctx)
 107 {
 108     struct rss_bpf *rss_bpf_ctx;
 109
 110     if (ebpf_rss_is_loaded(ctx)) {
 111         return false;
 112     }
 113
 114     rss_bpf_ctx = rss_bpf__open();
 115     if (rss_bpf_ctx == NULL) {
 116         trace_ebpf_error("eBPF RSS", "can not open eBPF RSS object");
 117         goto error;
 118     }
 119
 120     bpf_program__set_type(rss_bpf_ctx->progs.tun_rss_steering_prog, BPF_PROG_TYPE_SOCKET_FILTER);
 121
 122     if (rss_bpf__load(rss_bpf_ctx)) {
 123         trace_ebpf_error("eBPF RSS", "can not load RSS program");
 124         goto error;
 125     }
 126
 127     ctx->obj = rss_bpf_ctx;
 128     ctx->program_fd = bpf_program__fd(
 129             rss_bpf_ctx->progs.tun_rss_steering_prog);
 130     ctx->map_configuration = bpf_map__fd(
 131             rss_bpf_ctx->maps.tap_rss_map_configurations);
 132     ctx->map_indirections_table = bpf_map__fd(
 133             rss_bpf_ctx->maps.tap_rss_map_indirection_table);
 134     ctx->map_toeplitz_key = bpf_map__fd(
 135             rss_bpf_ctx->maps.tap_rss_map_toeplitz_key);
 136
 137     if (!ebpf_rss_mmap(ctx)) {
 138         goto error;
 139     }
 140
 141     return true;
 142 error:
 143     rss_bpf__destroy(rss_bpf_ctx);
 144     ctx->obj = NULL;
 145     ctx->program_fd = -1;
 146     ctx->map_configuration = -1;
 147     ctx->map_toeplitz_key = -1;
 148     ctx->map_indirections_table = -1;
 149
 150     return false;
 151 }
 152
 153 bool ebpf_rss_load_fds(struct EBPFRSSContext *ctx, int program_fd,
 154                        int config_fd, int toeplitz_fd, int table_fd)
 155 {
 156     if (ebpf_rss_is_loaded(ctx)) {
 157         return false;
 158     }
 159
 160     if (program_fd < 0 || config_fd < 0 || toeplitz_fd < 0 || table_fd < 0) {
 161         return false;
 162     }
 163
 164     ctx->program_fd = program_fd;
 165     ctx->map_configuration = config_fd;
 166     ctx->map_toeplitz_key = toeplitz_fd;
 167     ctx->map_indirections_table = table_fd;
 168
 169     if (!ebpf_rss_mmap(ctx)) {
 170         ctx->program_fd = -1;
 171         ctx->map_configuration = -1;
 172         ctx->map_toeplitz_key = -1;
 173         ctx->map_indirections_table = -1;
 174         return false;
 175     }
 176
 177     return true;
 178 }
 179
 180 static bool ebpf_rss_set_config(struct EBPFRSSContext *ctx,
 181                                 struct EBPFRSSConfig *config)
 182 {
 183     if (!ebpf_rss_is_loaded(ctx)) {
 184         return false;
 185     }
 186
 187     memcpy(ctx->mmap_configuration, config, sizeof(*config));
 188     return true;
 189 }
 190
 191 static bool ebpf_rss_set_indirections_table(struct EBPFRSSContext *ctx,
 192                                             uint16_t *indirections_table,
 193                                             size_t len)
 194 {
 195     char *cursor = ctx->mmap_indirections_table;
 196
 197     if (!ebpf_rss_is_loaded(ctx) || indirections_table == NULL ||
 198        len > VIRTIO_NET_RSS_MAX_TABLE_LEN) {
 199         return false;
 200     }
 201
 202     for (size_t i = 0; i < len; i++) {
 203         *(uint16_t *)cursor = indirections_table[i];
 204         cursor += 8;
 205     }
 206
 207     return true;
 208 }
 209
 210 static bool ebpf_rss_set_toepliz_key(struct EBPFRSSContext *ctx,
 211                                      uint8_t *toeplitz_key)
 212 {
 213     /* prepare toeplitz key */
 214     uint8_t toe[VIRTIO_NET_RSS_MAX_KEY_SIZE] = {};
 215
 216     if (!ebpf_rss_is_loaded(ctx) || toeplitz_key == NULL) {
 217         return false;
 218     }
 219     memcpy(toe, toeplitz_key, VIRTIO_NET_RSS_MAX_KEY_SIZE);
 220     *(uint32_t *)toe = ntohl(*(uint32_t *)toe);
 221
 222     memcpy(ctx->mmap_toeplitz_key, toe, VIRTIO_NET_RSS_MAX_KEY_SIZE);
 223     return true;
 224 }
 225
 226 bool ebpf_rss_set_all(struct EBPFRSSContext *ctx, struct EBPFRSSConfig *config,
 227                       uint16_t *indirections_table, uint8_t *toeplitz_key)
 228 {
 229     if (!ebpf_rss_is_loaded(ctx) || config == NULL ||
 230         indirections_table == NULL || toeplitz_key == NULL) {
 231         return false;
 232     }
 233
 234     if (!ebpf_rss_set_config(ctx, config)) {
 235         return false;
 236     }
 237
 238     if (!ebpf_rss_set_indirections_table(ctx, indirections_table,
 239                                       config->indirections_len)) {
 240         return false;
 241     }
 242
 243     if (!ebpf_rss_set_toepliz_key(ctx, toeplitz_key)) {
 244         return false;
 245     }
 246
 247     return true;
 248 }
 249
 250 void ebpf_rss_unload(struct EBPFRSSContext *ctx)
 251 {
 252     if (!ebpf_rss_is_loaded(ctx)) {
 253         return;
 254     }
 255
 256     ebpf_rss_munmap(ctx);
 257
 258     if (ctx->obj) {
 259         rss_bpf__destroy(ctx->obj);
 260     } else {
 261         close(ctx->program_fd);
 262         close(ctx->map_configuration);
 263         close(ctx->map_toeplitz_key);
 264         close(ctx->map_indirections_table);
 265     }
 266
 267     ctx->obj = NULL;
 268     ctx->program_fd = -1;
 269     ctx->map_configuration = -1;
 270     ctx->map_toeplitz_key = -1;
 271     ctx->map_indirections_table = -1;
 272 }
 273
 274 ebpf_binary_init(EBPF_PROGRAMID_RSS, rss_bpf__elf_bytes)