tools/perf/util/bpf-loader.c

   1 /*
   2  * bpf-loader.c
   3  *
   4  * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com>
   5  * Copyright (C) 2015 Huawei Inc.
   6  */
   7
   8 #include <bpf/libbpf.h>
   9 #include <linux/err.h>
  10 #include "perf.h"
  11 #include "debug.h"
  12 #include "bpf-loader.h"
  13 #include "probe-event.h"
  14 #include "probe-finder.h" // for MAX_PROBES
  15 #include "llvm-utils.h"
  16
  17 #define DEFINE_PRINT_FN(name, level) \
  18 static int libbpf_##name(const char *fmt, ...)  \
  19 {                                               \
  20         va_list args;                           \
  21         int ret;                                \
  22                                                 \
  23         va_start(args, fmt);                    \
  24         ret = veprintf(level, verbose, pr_fmt(fmt), args);\
  25         va_end(args);                           \
  26         return ret;                             \
  27 }
  28
  29 DEFINE_PRINT_FN(warning, 1)
  30 DEFINE_PRINT_FN(info, 1)
  31 DEFINE_PRINT_FN(debug, 1)
  32
  33 struct bpf_prog_priv {
  34         struct perf_probe_event pev;
  35 };
  36
  37 static bool libbpf_initialized;
  38
  39 struct bpf_object *
  40 bpf__prepare_load_buffer(void *obj_buf, size_t obj_buf_sz, const char *name)
  41 {
  42         struct bpf_object *obj;
  43
  44         if (!libbpf_initialized) {
  45                 libbpf_set_print(libbpf_warning,
  46                                  libbpf_info,
  47                                  libbpf_debug);
  48                 libbpf_initialized = true;
  49         }
  50
  51         obj = bpf_object__open_buffer(obj_buf, obj_buf_sz, name);
  52         if (IS_ERR(obj)) {
  53                 pr_debug("bpf: failed to load buffer\n");
  54                 return ERR_PTR(-EINVAL);
  55         }
  56
  57         return obj;
  58 }
  59
  60 struct bpf_object *bpf__prepare_load(const char *filename, bool source)
  61 {
  62         struct bpf_object *obj;
  63
  64         if (!libbpf_initialized) {
  65                 libbpf_set_print(libbpf_warning,
  66                                  libbpf_info,
  67                                  libbpf_debug);
  68                 libbpf_initialized = true;
  69         }
  70
  71         if (source) {
  72                 int err;
  73                 void *obj_buf;
  74                 size_t obj_buf_sz;
  75
  76                 err = llvm__compile_bpf(filename, &obj_buf, &obj_buf_sz);
  77                 if (err)
  78                         return ERR_PTR(-BPF_LOADER_ERRNO__COMPILE);
  79                 obj = bpf_object__open_buffer(obj_buf, obj_buf_sz, filename);
  80                 free(obj_buf);
  81         } else
  82                 obj = bpf_object__open(filename);
  83
  84         if (IS_ERR(obj)) {
  85                 pr_debug("bpf: failed to load %s\n", filename);
  86                 return obj;
  87         }
  88
  89         return obj;
  90 }
  91
  92 void bpf__clear(void)
  93 {
  94         struct bpf_object *obj, *tmp;
  95
  96         bpf_object__for_each_safe(obj, tmp) {
  97                 bpf__unprobe(obj);
  98                 bpf_object__close(obj);
  99         }
 100 }
 101
 102 static void
 103 bpf_prog_priv__clear(struct bpf_program *prog __maybe_unused,
 104                      void *_priv)
 105 {
 106         struct bpf_prog_priv *priv = _priv;
 107
 108         cleanup_perf_probe_events(&priv->pev, 1);
 109         free(priv);
 110 }
 111
 112 static int
 113 config__exec(const char *value, struct perf_probe_event *pev)
 114 {
 115         pev->uprobes = true;
 116         pev->target = strdup(value);
 117         if (!pev->target)
 118                 return -ENOMEM;
 119         return 0;
 120 }
 121
 122 static int
 123 config__module(const char *value, struct perf_probe_event *pev)
 124 {
 125         pev->uprobes = false;
 126         pev->target = strdup(value);
 127         if (!pev->target)
 128                 return -ENOMEM;
 129         return 0;
 130 }
 131
 132 static struct {
 133         const char *key;
 134         const char *usage;
 135         const char *desc;
 136         int (*func)(const char *, struct perf_probe_event *);
 137 } bpf_config_terms[] = {
 138         {
 139                 .key    = "exec",
 140                 .usage  = "exec=<full path of file>",
 141                 .desc   = "Set uprobe target",
 142                 .func   = config__exec,
 143         },
 144         {
 145                 .key    = "module",
 146                 .usage  = "module=<module name>    ",
 147                 .desc   = "Set kprobe module",
 148                 .func   = config__module,
 149         }
 150 };
 151
 152 static int
 153 do_config(const char *key, const char *value,
 154           struct perf_probe_event *pev)
 155 {
 156         unsigned int i;
 157
 158         pr_debug("config bpf program: %s=%s\n", key, value);
 159         for (i = 0; i < ARRAY_SIZE(bpf_config_terms); i++)
 160                 if (strcmp(key, bpf_config_terms[i].key) == 0)
 161                         return bpf_config_terms[i].func(value, pev);
 162
 163         pr_debug("BPF: ERROR: invalid config option in object: %s=%s\n",
 164                  key, value);
 165
 166         pr_debug("\nHint: Currently valid options are:\n");
 167         for (i = 0; i < ARRAY_SIZE(bpf_config_terms); i++)
 168                 pr_debug("\t%s:\t%s\n", bpf_config_terms[i].usage,
 169                          bpf_config_terms[i].desc);
 170         pr_debug("\n");
 171
 172         return -BPF_LOADER_ERRNO__CONFIG_TERM;
 173 }
 174
 175 static const char *
 176 parse_config_kvpair(const char *config_str, struct perf_probe_event *pev)
 177 {
 178         char *text = strdup(config_str);
 179         char *sep, *line;
 180         const char *main_str = NULL;
 181         int err = 0;
 182
 183         if (!text) {
 184                 pr_debug("No enough memory: dup config_str failed\n");
 185                 return ERR_PTR(-ENOMEM);
 186         }
 187
 188         line = text;
 189         while ((sep = strchr(line, ';'))) {
 190                 char *equ;
 191
 192                 *sep = '\0';
 193                 equ = strchr(line, '=');
 194                 if (!equ) {
 195                         pr_warning("WARNING: invalid config in BPF object: %s\n",
 196                                    line);
 197                         pr_warning("\tShould be 'key=value'.\n");
 198                         goto nextline;
 199                 }
 200                 *equ = '\0';
 201
 202                 err = do_config(line, equ + 1, pev);
 203                 if (err)
 204                         break;
 205 nextline:
 206                 line = sep + 1;
 207         }
 208
 209         if (!err)
 210                 main_str = config_str + (line - text);
 211         free(text);
 212
 213         return err ? ERR_PTR(err) : main_str;
 214 }
 215
 216 static int
 217 parse_config(const char *config_str, struct perf_probe_event *pev)
 218 {
 219         int err;
 220         const char *main_str = parse_config_kvpair(config_str, pev);
 221
 222         if (IS_ERR(main_str))
 223                 return PTR_ERR(main_str);
 224
 225         err = parse_perf_probe_command(main_str, pev);
 226         if (err < 0) {
 227                 pr_debug("bpf: '%s' is not a valid config string\n",
 228                          config_str);
 229                 /* parse failed, don't need clear pev. */
 230                 return -BPF_LOADER_ERRNO__CONFIG;
 231         }
 232         return 0;
 233 }
 234
 235 static int
 236 config_bpf_program(struct bpf_program *prog)
 237 {
 238         struct perf_probe_event *pev = NULL;
 239         struct bpf_prog_priv *priv = NULL;
 240         const char *config_str;
 241         int err;
 242
 243         config_str = bpf_program__title(prog, false);
 244         if (IS_ERR(config_str)) {
 245                 pr_debug("bpf: unable to get title for program\n");
 246                 return PTR_ERR(config_str);
 247         }
 248
 249         priv = calloc(sizeof(*priv), 1);
 250         if (!priv) {
 251                 pr_debug("bpf: failed to alloc priv\n");
 252                 return -ENOMEM;
 253         }
 254         pev = &priv->pev;
 255
 256         pr_debug("bpf: config program '%s'\n", config_str);
 257         err = parse_config(config_str, pev);
 258         if (err)
 259                 goto errout;
 260
 261         if (pev->group && strcmp(pev->group, PERF_BPF_PROBE_GROUP)) {
 262                 pr_debug("bpf: '%s': group for event is set and not '%s'.\n",
 263                          config_str, PERF_BPF_PROBE_GROUP);
 264                 err = -BPF_LOADER_ERRNO__GROUP;
 265                 goto errout;
 266         } else if (!pev->group)
 267                 pev->group = strdup(PERF_BPF_PROBE_GROUP);
 268
 269         if (!pev->group) {
 270                 pr_debug("bpf: strdup failed\n");
 271                 err = -ENOMEM;
 272                 goto errout;
 273         }
 274
 275         if (!pev->event) {
 276                 pr_debug("bpf: '%s': event name is missing. Section name should be 'key=value'\n",
 277                          config_str);
 278                 err = -BPF_LOADER_ERRNO__EVENTNAME;
 279                 goto errout;
 280         }
 281         pr_debug("bpf: config '%s' is ok\n", config_str);
 282
 283         err = bpf_program__set_private(prog, priv, bpf_prog_priv__clear);
 284         if (err) {
 285                 pr_debug("Failed to set priv for program '%s'\n", config_str);
 286                 goto errout;
 287         }
 288
 289         return 0;
 290
 291 errout:
 292         if (pev)
 293                 clear_perf_probe_event(pev);
 294         free(priv);
 295         return err;
 296 }
 297
 298 static int bpf__prepare_probe(void)
 299 {
 300         static int err = 0;
 301         static bool initialized = false;
 302
 303         /*
 304          * Make err static, so if init failed the first, bpf__prepare_probe()
 305          * fails each time without calling init_probe_symbol_maps multiple
 306          * times.
 307          */
 308         if (initialized)
 309                 return err;
 310
 311         initialized = true;
 312         err = init_probe_symbol_maps(false);
 313         if (err < 0)
 314                 pr_debug("Failed to init_probe_symbol_maps\n");
 315         probe_conf.max_probes = MAX_PROBES;
 316         return err;
 317 }
 318
 319 int bpf__probe(struct bpf_object *obj)
 320 {
 321         int err = 0;
 322         struct bpf_program *prog;
 323         struct bpf_prog_priv *priv;
 324         struct perf_probe_event *pev;
 325
 326         err = bpf__prepare_probe();
 327         if (err) {
 328                 pr_debug("bpf__prepare_probe failed\n");
 329                 return err;
 330         }
 331
 332         bpf_object__for_each_program(prog, obj) {
 333                 err = config_bpf_program(prog);
 334                 if (err)
 335                         goto out;
 336
 337                 err = bpf_program__get_private(prog, (void **)&priv);
 338                 if (err || !priv)
 339                         goto out;
 340                 pev = &priv->pev;
 341
 342                 err = convert_perf_probe_events(pev, 1);
 343                 if (err < 0) {
 344                         pr_debug("bpf_probe: failed to convert perf probe events");
 345                         goto out;
 346                 }
 347
 348                 err = apply_perf_probe_events(pev, 1);
 349                 if (err < 0) {
 350                         pr_debug("bpf_probe: failed to apply perf probe events");
 351                         goto out;
 352                 }
 353         }
 354 out:
 355         return err < 0 ? err : 0;
 356 }
 357
 358 #define EVENTS_WRITE_BUFSIZE  4096
 359 int bpf__unprobe(struct bpf_object *obj)
 360 {
 361         int err, ret = 0;
 362         struct bpf_program *prog;
 363         struct bpf_prog_priv *priv;
 364
 365         bpf_object__for_each_program(prog, obj) {
 366                 int i;
 367
 368                 err = bpf_program__get_private(prog, (void **)&priv);
 369                 if (err || !priv)
 370                         continue;
 371
 372                 for (i = 0; i < priv->pev.ntevs; i++) {
 373                         struct probe_trace_event *tev = &priv->pev.tevs[i];
 374                         char name_buf[EVENTS_WRITE_BUFSIZE];
 375                         struct strfilter *delfilter;
 376
 377                         snprintf(name_buf, EVENTS_WRITE_BUFSIZE,
 378                                  "%s:%s", tev->group, tev->event);
 379                         name_buf[EVENTS_WRITE_BUFSIZE - 1] = '\0';
 380
 381                         delfilter = strfilter__new(name_buf, NULL);
 382                         if (!delfilter) {
 383                                 pr_debug("Failed to create filter for unprobing\n");
 384                                 ret = -ENOMEM;
 385                                 continue;
 386                         }
 387
 388                         err = del_perf_probe_events(delfilter);
 389                         strfilter__delete(delfilter);
 390                         if (err) {
 391                                 pr_debug("Failed to delete %s\n", name_buf);
 392                                 ret = err;
 393                                 continue;
 394                         }
 395                 }
 396         }
 397         return ret;
 398 }
 399
 400 int bpf__load(struct bpf_object *obj)
 401 {
 402         int err;
 403
 404         err = bpf_object__load(obj);
 405         if (err) {
 406                 pr_debug("bpf: load objects failed\n");
 407                 return err;
 408         }
 409         return 0;
 410 }
 411
 412 int bpf__foreach_tev(struct bpf_object *obj,
 413                      bpf_prog_iter_callback_t func,
 414                      void *arg)
 415 {
 416         struct bpf_program *prog;
 417         int err;
 418
 419         bpf_object__for_each_program(prog, obj) {
 420                 struct probe_trace_event *tev;
 421                 struct perf_probe_event *pev;
 422                 struct bpf_prog_priv *priv;
 423                 int i, fd;
 424
 425                 err = bpf_program__get_private(prog,
 426                                 (void **)&priv);
 427                 if (err || !priv) {
 428                         pr_debug("bpf: failed to get private field\n");
 429                         return -BPF_LOADER_ERRNO__INTERNAL;
 430                 }
 431
 432                 pev = &priv->pev;
 433                 for (i = 0; i < pev->ntevs; i++) {
 434                         tev = &pev->tevs[i];
 435
 436                         fd = bpf_program__fd(prog);
 437                         if (fd < 0) {
 438                                 pr_debug("bpf: failed to get file descriptor\n");
 439                                 return fd;
 440                         }
 441
 442                         err = (*func)(tev, fd, arg);
 443                         if (err) {
 444                                 pr_debug("bpf: call back failed, stop iterate\n");
 445                                 return err;
 446                         }
 447                 }
 448         }
 449         return 0;
 450 }
 451
 452 #define ERRNO_OFFSET(e)         ((e) - __BPF_LOADER_ERRNO__START)
 453 #define ERRCODE_OFFSET(c)       ERRNO_OFFSET(BPF_LOADER_ERRNO__##c)
 454 #define NR_ERRNO        (__BPF_LOADER_ERRNO__END - __BPF_LOADER_ERRNO__START)
 455
 456 static const char *bpf_loader_strerror_table[NR_ERRNO] = {
 457         [ERRCODE_OFFSET(CONFIG)]        = "Invalid config string",
 458         [ERRCODE_OFFSET(GROUP)]         = "Invalid group name",
 459         [ERRCODE_OFFSET(EVENTNAME)]     = "No event name found in config string",
 460         [ERRCODE_OFFSET(INTERNAL)]      = "BPF loader internal error",
 461         [ERRCODE_OFFSET(COMPILE)]       = "Error when compiling BPF scriptlet",
 462         [ERRCODE_OFFSET(CONFIG_TERM)]   = "Invalid config term in config string",
 463 };
 464
 465 static int
 466 bpf_loader_strerror(int err, char *buf, size_t size)
 467 {
 468         char sbuf[STRERR_BUFSIZE];
 469         const char *msg;
 470
 471         if (!buf || !size)
 472                 return -1;
 473
 474         err = err > 0 ? err : -err;
 475
 476         if (err >= __LIBBPF_ERRNO__START)
 477                 return libbpf_strerror(err, buf, size);
 478
 479         if (err >= __BPF_LOADER_ERRNO__START && err < __BPF_LOADER_ERRNO__END) {
 480                 msg = bpf_loader_strerror_table[ERRNO_OFFSET(err)];
 481                 snprintf(buf, size, "%s", msg);
 482                 buf[size - 1] = '\0';
 483                 return 0;
 484         }
 485
 486         if (err >= __BPF_LOADER_ERRNO__END)
 487                 snprintf(buf, size, "Unknown bpf loader error %d", err);
 488         else
 489                 snprintf(buf, size, "%s",
 490                          strerror_r(err, sbuf, sizeof(sbuf)));
 491
 492         buf[size - 1] = '\0';
 493         return -1;
 494 }
 495
 496 #define bpf__strerror_head(err, buf, size) \
 497         char sbuf[STRERR_BUFSIZE], *emsg;\
 498         if (!size)\
 499                 return 0;\
 500         if (err < 0)\
 501                 err = -err;\
 502         bpf_loader_strerror(err, sbuf, sizeof(sbuf));\
 503         emsg = sbuf;\
 504         switch (err) {\
 505         default:\
 506                 scnprintf(buf, size, "%s", emsg);\
 507                 break;
 508
 509 #define bpf__strerror_entry(val, fmt...)\
 510         case val: {\
 511                 scnprintf(buf, size, fmt);\
 512                 break;\
 513         }
 514
 515 #define bpf__strerror_end(buf, size)\
 516         }\
 517         buf[size - 1] = '\0';
 518
 519 int bpf__strerror_prepare_load(const char *filename, bool source,
 520                                int err, char *buf, size_t size)
 521 {
 522         size_t n;
 523         int ret;
 524
 525         n = snprintf(buf, size, "Failed to load %s%s: ",
 526                          filename, source ? " from source" : "");
 527         if (n >= size) {
 528                 buf[size - 1] = '\0';
 529                 return 0;
 530         }
 531         buf += n;
 532         size -= n;
 533
 534         ret = bpf_loader_strerror(err, buf, size);
 535         buf[size - 1] = '\0';
 536         return ret;
 537 }
 538
 539 int bpf__strerror_probe(struct bpf_object *obj __maybe_unused,
 540                         int err, char *buf, size_t size)
 541 {
 542         bpf__strerror_head(err, buf, size);
 543         case BPF_LOADER_ERRNO__CONFIG_TERM: {
 544                 scnprintf(buf, size, "%s (add -v to see detail)", emsg);
 545                 break;
 546         }
 547         bpf__strerror_entry(EEXIST, "Probe point exist. Try use 'perf probe -d \"*\"'");
 548         bpf__strerror_entry(EACCES, "You need to be root");
 549         bpf__strerror_entry(EPERM, "You need to be root, and /proc/sys/kernel/kptr_restrict should be 0");
 550         bpf__strerror_entry(ENOENT, "You need to check probing points in BPF file");
 551         bpf__strerror_end(buf, size);
 552         return 0;
 553 }
 554
 555 int bpf__strerror_load(struct bpf_object *obj,
 556                        int err, char *buf, size_t size)
 557 {
 558         bpf__strerror_head(err, buf, size);
 559         case LIBBPF_ERRNO__KVER: {
 560                 unsigned int obj_kver = bpf_object__get_kversion(obj);
 561                 unsigned int real_kver;
 562
 563                 if (fetch_kernel_version(&real_kver, NULL, 0)) {
 564                         scnprintf(buf, size, "Unable to fetch kernel version");
 565                         break;
 566                 }
 567
 568                 if (obj_kver != real_kver) {
 569                         scnprintf(buf, size,
 570                                   "'version' ("KVER_FMT") doesn't match running kernel ("KVER_FMT")",
 571                                   KVER_PARAM(obj_kver),
 572                                   KVER_PARAM(real_kver));
 573                         break;
 574                 }
 575
 576                 scnprintf(buf, size, "Failed to load program for unknown reason");
 577                 break;
 578         }
 579         bpf__strerror_end(buf, size);
 580         return 0;
 581 }