kernel/tracepoint.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Copyright (C) 2008-2014 Mathieu Desnoyers
   4  */
   5 #include <linux/module.h>
   6 #include <linux/mutex.h>
   7 #include <linux/types.h>
   8 #include <linux/jhash.h>
   9 #include <linux/list.h>
  10 #include <linux/rcupdate.h>
  11 #include <linux/tracepoint.h>
  12 #include <linux/err.h>
  13 #include <linux/slab.h>
  14 #include <linux/sched/signal.h>
  15 #include <linux/sched/task.h>
  16 #include <linux/static_key.h>
  17
  18 enum tp_func_state {
  19         TP_FUNC_0,
  20         TP_FUNC_1,
  21         TP_FUNC_2,
  22         TP_FUNC_N,
  23 };
  24
  25 extern tracepoint_ptr_t __start___tracepoints_ptrs[];
  26 extern tracepoint_ptr_t __stop___tracepoints_ptrs[];
  27
  28 DEFINE_SRCU(tracepoint_srcu);
  29 EXPORT_SYMBOL_GPL(tracepoint_srcu);
  30
  31 enum tp_transition_sync {
  32         TP_TRANSITION_SYNC_1_0_1,
  33         TP_TRANSITION_SYNC_N_2_1,
  34
  35         _NR_TP_TRANSITION_SYNC,
  36 };
  37
  38 struct tp_transition_snapshot {
  39         unsigned long rcu;
  40         unsigned long srcu;
  41         bool ongoing;
  42 };
  43
  44 /* Protected by tracepoints_mutex */
  45 static struct tp_transition_snapshot tp_transition_snapshot[_NR_TP_TRANSITION_SYNC];
  46
  47 static void tp_rcu_get_state(enum tp_transition_sync sync)
  48 {
  49         struct tp_transition_snapshot *snapshot = &tp_transition_snapshot[sync];
  50
  51         /* Keep the latest get_state snapshot. */
  52         snapshot->rcu = get_state_synchronize_rcu();
  53         snapshot->srcu = start_poll_synchronize_srcu(&tracepoint_srcu);
  54         snapshot->ongoing = true;
  55 }
  56
  57 static void tp_rcu_cond_sync(enum tp_transition_sync sync)
  58 {
  59         struct tp_transition_snapshot *snapshot = &tp_transition_snapshot[sync];
  60
  61         if (!snapshot->ongoing)
  62                 return;
  63         cond_synchronize_rcu(snapshot->rcu);
  64         if (!poll_state_synchronize_srcu(&tracepoint_srcu, snapshot->srcu))
  65                 synchronize_srcu(&tracepoint_srcu);
  66         snapshot->ongoing = false;
  67 }
  68
  69 /* Set to 1 to enable tracepoint debug output */
  70 static const int tracepoint_debug;
  71
  72 #ifdef CONFIG_MODULES
  73 /*
  74  * Tracepoint module list mutex protects the local module list.
  75  */
  76 static DEFINE_MUTEX(tracepoint_module_list_mutex);
  77
  78 /* Local list of struct tp_module */
  79 static LIST_HEAD(tracepoint_module_list);
  80 #endif /* CONFIG_MODULES */
  81
  82 /*
  83  * tracepoints_mutex protects the builtin and module tracepoints.
  84  * tracepoints_mutex nests inside tracepoint_module_list_mutex.
  85  */
  86 static DEFINE_MUTEX(tracepoints_mutex);
  87
  88 static struct rcu_head *early_probes;
  89 static bool ok_to_free_tracepoints;
  90
  91 /*
  92  * Note about RCU :
  93  * It is used to delay the free of multiple probes array until a quiescent
  94  * state is reached.
  95  */
  96 struct tp_probes {
  97         struct rcu_head rcu;
  98         struct tracepoint_func probes[];
  99 };
 100
 101 /* Called in removal of a func but failed to allocate a new tp_funcs */
 102 static void tp_stub_func(void)
 103 {
 104         return;
 105 }
 106
 107 static inline void *allocate_probes(int count)
 108 {
 109         struct tp_probes *p  = kmalloc(struct_size(p, probes, count),
 110                                        GFP_KERNEL);
 111         return p == NULL ? NULL : p->probes;
 112 }
 113
 114 static void srcu_free_old_probes(struct rcu_head *head)
 115 {
 116         kfree(container_of(head, struct tp_probes, rcu));
 117 }
 118
 119 static void rcu_free_old_probes(struct rcu_head *head)
 120 {
 121         call_srcu(&tracepoint_srcu, head, srcu_free_old_probes);
 122 }
 123
 124 static __init int release_early_probes(void)
 125 {
 126         struct rcu_head *tmp;
 127
 128         ok_to_free_tracepoints = true;
 129
 130         while (early_probes) {
 131                 tmp = early_probes;
 132                 early_probes = tmp->next;
 133                 call_rcu(tmp, rcu_free_old_probes);
 134         }
 135
 136         return 0;
 137 }
 138
 139 /* SRCU is initialized at core_initcall */
 140 postcore_initcall(release_early_probes);
 141
 142 static inline void release_probes(struct tracepoint_func *old)
 143 {
 144         if (old) {
 145                 struct tp_probes *tp_probes = container_of(old,
 146                         struct tp_probes, probes[0]);
 147
 148                 /*
 149                  * We can't free probes if SRCU is not initialized yet.
 150                  * Postpone the freeing till after SRCU is initialized.
 151                  */
 152                 if (unlikely(!ok_to_free_tracepoints)) {
 153                         tp_probes->rcu.next = early_probes;
 154                         early_probes = &tp_probes->rcu;
 155                         return;
 156                 }
 157
 158                 /*
 159                  * Tracepoint probes are protected by both sched RCU and SRCU,
 160                  * by calling the SRCU callback in the sched RCU callback we
 161                  * cover both cases. So let us chain the SRCU and sched RCU
 162                  * callbacks to wait for both grace periods.
 163                  */
 164                 call_rcu(&tp_probes->rcu, rcu_free_old_probes);
 165         }
 166 }
 167
 168 static void debug_print_probes(struct tracepoint_func *funcs)
 169 {
 170         int i;
 171
 172         if (!tracepoint_debug || !funcs)
 173                 return;
 174
 175         for (i = 0; funcs[i].func; i++)
 176                 printk(KERN_DEBUG "Probe %d : %p\n", i, funcs[i].func);
 177 }
 178
 179 static struct tracepoint_func *
 180 func_add(struct tracepoint_func **funcs, struct tracepoint_func *tp_func,
 181          int prio)
 182 {
 183         struct tracepoint_func *old, *new;
 184         int iter_probes;        /* Iterate over old probe array. */
 185         int nr_probes = 0;      /* Counter for probes */
 186         int pos = -1;           /* Insertion position into new array */
 187
 188         if (WARN_ON(!tp_func->func))
 189                 return ERR_PTR(-EINVAL);
 190
 191         debug_print_probes(*funcs);
 192         old = *funcs;
 193         if (old) {
 194                 /* (N -> N+1), (N != 0, 1) probes */
 195                 for (iter_probes = 0; old[iter_probes].func; iter_probes++) {
 196                         if (old[iter_probes].func == tp_stub_func)
 197                                 continue;       /* Skip stub functions. */
 198                         if (old[iter_probes].func == tp_func->func &&
 199                             old[iter_probes].data == tp_func->data)
 200                                 return ERR_PTR(-EEXIST);
 201                         nr_probes++;
 202                 }
 203         }
 204         /* + 2 : one for new probe, one for NULL func */
 205         new = allocate_probes(nr_probes + 2);
 206         if (new == NULL)
 207                 return ERR_PTR(-ENOMEM);
 208         if (old) {
 209                 nr_probes = 0;
 210                 for (iter_probes = 0; old[iter_probes].func; iter_probes++) {
 211                         if (old[iter_probes].func == tp_stub_func)
 212                                 continue;
 213                         /* Insert before probes of lower priority */
 214                         if (pos < 0 && old[iter_probes].prio < prio)
 215                                 pos = nr_probes++;
 216                         new[nr_probes++] = old[iter_probes];
 217                 }
 218                 if (pos < 0)
 219                         pos = nr_probes++;
 220                 /* nr_probes now points to the end of the new array */
 221         } else {
 222                 pos = 0;
 223                 nr_probes = 1; /* must point at end of array */
 224         }
 225         new[pos] = *tp_func;
 226         new[nr_probes].func = NULL;
 227         *funcs = new;
 228         debug_print_probes(*funcs);
 229         return old;
 230 }
 231
 232 static void *func_remove(struct tracepoint_func **funcs,
 233                 struct tracepoint_func *tp_func)
 234 {
 235         int nr_probes = 0, nr_del = 0, i;
 236         struct tracepoint_func *old, *new;
 237
 238         old = *funcs;
 239
 240         if (!old)
 241                 return ERR_PTR(-ENOENT);
 242
 243         debug_print_probes(*funcs);
 244         /* (N -> M), (N > 1, M >= 0) probes */
 245         if (tp_func->func) {
 246                 for (nr_probes = 0; old[nr_probes].func; nr_probes++) {
 247                         if ((old[nr_probes].func == tp_func->func &&
 248                              old[nr_probes].data == tp_func->data) ||
 249                             old[nr_probes].func == tp_stub_func)
 250                                 nr_del++;
 251                 }
 252         }
 253
 254         /*
 255          * If probe is NULL, then nr_probes = nr_del = 0, and then the
 256          * entire entry will be removed.
 257          */
 258         if (nr_probes - nr_del == 0) {
 259                 /* N -> 0, (N > 1) */
 260                 *funcs = NULL;
 261                 debug_print_probes(*funcs);
 262                 return old;
 263         } else {
 264                 int j = 0;
 265                 /* N -> M, (N > 1, M > 0) */
 266                 /* + 1 for NULL */
 267                 new = allocate_probes(nr_probes - nr_del + 1);
 268                 if (new) {
 269                         for (i = 0; old[i].func; i++) {
 270                                 if ((old[i].func != tp_func->func ||
 271                                      old[i].data != tp_func->data) &&
 272                                     old[i].func != tp_stub_func)
 273                                         new[j++] = old[i];
 274                         }
 275                         new[nr_probes - nr_del].func = NULL;
 276                         *funcs = new;
 277                 } else {
 278                         /*
 279                          * Failed to allocate, replace the old function
 280                          * with calls to tp_stub_func.
 281                          */
 282                         for (i = 0; old[i].func; i++) {
 283                                 if (old[i].func == tp_func->func &&
 284                                     old[i].data == tp_func->data)
 285                                         WRITE_ONCE(old[i].func, tp_stub_func);
 286                         }
 287                         *funcs = old;
 288                 }
 289         }
 290         debug_print_probes(*funcs);
 291         return old;
 292 }
 293
 294 /*
 295  * Count the number of functions (enum tp_func_state) in a tp_funcs array.
 296  */
 297 static enum tp_func_state nr_func_state(const struct tracepoint_func *tp_funcs)
 298 {
 299         if (!tp_funcs)
 300                 return TP_FUNC_0;
 301         if (!tp_funcs[1].func)
 302                 return TP_FUNC_1;
 303         if (!tp_funcs[2].func)
 304                 return TP_FUNC_2;
 305         return TP_FUNC_N;       /* 3 or more */
 306 }
 307
 308 static void tracepoint_update_call(struct tracepoint *tp, struct tracepoint_func *tp_funcs)
 309 {
 310         void *func = tp->iterator;
 311
 312         /* Synthetic events do not have static call sites */
 313         if (!tp->static_call_key)
 314                 return;
 315         if (nr_func_state(tp_funcs) == TP_FUNC_1)
 316                 func = tp_funcs[0].func;
 317         __static_call_update(tp->static_call_key, tp->static_call_tramp, func);
 318 }
 319
 320 /*
 321  * Add the probe function to a tracepoint.
 322  */
 323 static int tracepoint_add_func(struct tracepoint *tp,
 324                                struct tracepoint_func *func, int prio,
 325                                bool warn)
 326 {
 327         struct tracepoint_func *old, *tp_funcs;
 328         int ret;
 329
 330         if (tp->regfunc && !static_key_enabled(&tp->key)) {
 331                 ret = tp->regfunc();
 332                 if (ret < 0)
 333                         return ret;
 334         }
 335
 336         tp_funcs = rcu_dereference_protected(tp->funcs,
 337                         lockdep_is_held(&tracepoints_mutex));
 338         old = func_add(&tp_funcs, func, prio);
 339         if (IS_ERR(old)) {
 340                 WARN_ON_ONCE(warn && PTR_ERR(old) != -ENOMEM);
 341                 return PTR_ERR(old);
 342         }
 343
 344         /*
 345          * rcu_assign_pointer has as smp_store_release() which makes sure
 346          * that the new probe callbacks array is consistent before setting
 347          * a pointer to it.  This array is referenced by __DO_TRACE from
 348          * include/linux/tracepoint.h using rcu_dereference_sched().
 349          */
 350         switch (nr_func_state(tp_funcs)) {
 351         case TP_FUNC_1:         /* 0->1 */
 352                 /*
 353                  * Make sure new static func never uses old data after a
 354                  * 1->0->1 transition sequence.
 355                  */
 356                 tp_rcu_cond_sync(TP_TRANSITION_SYNC_1_0_1);
 357                 /* Set static call to first function */
 358                 tracepoint_update_call(tp, tp_funcs);
 359                 /* Both iterator and static call handle NULL tp->funcs */
 360                 rcu_assign_pointer(tp->funcs, tp_funcs);
 361                 static_key_enable(&tp->key);
 362                 break;
 363         case TP_FUNC_2:         /* 1->2 */
 364                 /* Set iterator static call */
 365                 tracepoint_update_call(tp, tp_funcs);
 366                 /*
 367                  * Iterator callback installed before updating tp->funcs.
 368                  * Requires ordering between RCU assign/dereference and
 369                  * static call update/call.
 370                  */
 371                 fallthrough;
 372         case TP_FUNC_N:         /* N->N+1 (N>1) */
 373                 rcu_assign_pointer(tp->funcs, tp_funcs);
 374                 /*
 375                  * Make sure static func never uses incorrect data after a
 376                  * N->...->2->1 (N>1) transition sequence.
 377                  */
 378                 if (tp_funcs[0].data != old[0].data)
 379                         tp_rcu_get_state(TP_TRANSITION_SYNC_N_2_1);
 380                 break;
 381         default:
 382                 WARN_ON_ONCE(1);
 383                 break;
 384         }
 385
 386         release_probes(old);
 387         return 0;
 388 }
 389
 390 /*
 391  * Remove a probe function from a tracepoint.
 392  * Note: only waiting an RCU period after setting elem->call to the empty
 393  * function insures that the original callback is not used anymore. This insured
 394  * by preempt_disable around the call site.
 395  */
 396 static int tracepoint_remove_func(struct tracepoint *tp,
 397                 struct tracepoint_func *func)
 398 {
 399         struct tracepoint_func *old, *tp_funcs;
 400
 401         tp_funcs = rcu_dereference_protected(tp->funcs,
 402                         lockdep_is_held(&tracepoints_mutex));
 403         old = func_remove(&tp_funcs, func);
 404         if (WARN_ON_ONCE(IS_ERR(old)))
 405                 return PTR_ERR(old);
 406
 407         if (tp_funcs == old)
 408                 /* Failed allocating new tp_funcs, replaced func with stub */
 409                 return 0;
 410
 411         switch (nr_func_state(tp_funcs)) {
 412         case TP_FUNC_0:         /* 1->0 */
 413                 /* Removed last function */
 414                 if (tp->unregfunc && static_key_enabled(&tp->key))
 415                         tp->unregfunc();
 416
 417                 static_key_disable(&tp->key);
 418                 /* Set iterator static call */
 419                 tracepoint_update_call(tp, tp_funcs);
 420                 /* Both iterator and static call handle NULL tp->funcs */
 421                 rcu_assign_pointer(tp->funcs, NULL);
 422                 /*
 423                  * Make sure new static func never uses old data after a
 424                  * 1->0->1 transition sequence.
 425                  */
 426                 tp_rcu_get_state(TP_TRANSITION_SYNC_1_0_1);
 427                 break;
 428         case TP_FUNC_1:         /* 2->1 */
 429                 rcu_assign_pointer(tp->funcs, tp_funcs);
 430                 /*
 431                  * Make sure static func never uses incorrect data after a
 432                  * N->...->2->1 (N>2) transition sequence. If the first
 433                  * element's data has changed, then force the synchronization
 434                  * to prevent current readers that have loaded the old data
 435                  * from calling the new function.
 436                  */
 437                 if (tp_funcs[0].data != old[0].data)
 438                         tp_rcu_get_state(TP_TRANSITION_SYNC_N_2_1);
 439                 tp_rcu_cond_sync(TP_TRANSITION_SYNC_N_2_1);
 440                 /* Set static call to first function */
 441                 tracepoint_update_call(tp, tp_funcs);
 442                 break;
 443         case TP_FUNC_2:         /* N->N-1 (N>2) */
 444                 fallthrough;
 445         case TP_FUNC_N:
 446                 rcu_assign_pointer(tp->funcs, tp_funcs);
 447                 /*
 448                  * Make sure static func never uses incorrect data after a
 449                  * N->...->2->1 (N>2) transition sequence.
 450                  */
 451                 if (tp_funcs[0].data != old[0].data)
 452                         tp_rcu_get_state(TP_TRANSITION_SYNC_N_2_1);
 453                 break;
 454         default:
 455                 WARN_ON_ONCE(1);
 456                 break;
 457         }
 458         release_probes(old);
 459         return 0;
 460 }
 461
 462 /**
 463  * tracepoint_probe_register_prio_may_exist -  Connect a probe to a tracepoint with priority
 464  * @tp: tracepoint
 465  * @probe: probe handler
 466  * @data: tracepoint data
 467  * @prio: priority of this function over other registered functions
 468  *
 469  * Same as tracepoint_probe_register_prio() except that it will not warn
 470  * if the tracepoint is already registered.
 471  */
 472 int tracepoint_probe_register_prio_may_exist(struct tracepoint *tp, void *probe,
 473                                              void *data, int prio)
 474 {
 475         struct tracepoint_func tp_func;
 476         int ret;
 477
 478         mutex_lock(&tracepoints_mutex);
 479         tp_func.func = probe;
 480         tp_func.data = data;
 481         tp_func.prio = prio;
 482         ret = tracepoint_add_func(tp, &tp_func, prio, false);
 483         mutex_unlock(&tracepoints_mutex);
 484         return ret;
 485 }
 486 EXPORT_SYMBOL_GPL(tracepoint_probe_register_prio_may_exist);
 487
 488 /**
 489  * tracepoint_probe_register_prio -  Connect a probe to a tracepoint with priority
 490  * @tp: tracepoint
 491  * @probe: probe handler
 492  * @data: tracepoint data
 493  * @prio: priority of this function over other registered functions
 494  *
 495  * Returns 0 if ok, error value on error.
 496  * Note: if @tp is within a module, the caller is responsible for
 497  * unregistering the probe before the module is gone. This can be
 498  * performed either with a tracepoint module going notifier, or from
 499  * within module exit functions.
 500  */
 501 int tracepoint_probe_register_prio(struct tracepoint *tp, void *probe,
 502                                    void *data, int prio)
 503 {
 504         struct tracepoint_func tp_func;
 505         int ret;
 506
 507         mutex_lock(&tracepoints_mutex);
 508         tp_func.func = probe;
 509         tp_func.data = data;
 510         tp_func.prio = prio;
 511         ret = tracepoint_add_func(tp, &tp_func, prio, true);
 512         mutex_unlock(&tracepoints_mutex);
 513         return ret;
 514 }
 515 EXPORT_SYMBOL_GPL(tracepoint_probe_register_prio);
 516
 517 /**
 518  * tracepoint_probe_register -  Connect a probe to a tracepoint
 519  * @tp: tracepoint
 520  * @probe: probe handler
 521  * @data: tracepoint data
 522  *
 523  * Returns 0 if ok, error value on error.
 524  * Note: if @tp is within a module, the caller is responsible for
 525  * unregistering the probe before the module is gone. This can be
 526  * performed either with a tracepoint module going notifier, or from
 527  * within module exit functions.
 528  */
 529 int tracepoint_probe_register(struct tracepoint *tp, void *probe, void *data)
 530 {
 531         return tracepoint_probe_register_prio(tp, probe, data, TRACEPOINT_DEFAULT_PRIO);
 532 }
 533 EXPORT_SYMBOL_GPL(tracepoint_probe_register);
 534
 535 /**
 536  * tracepoint_probe_unregister -  Disconnect a probe from a tracepoint
 537  * @tp: tracepoint
 538  * @probe: probe function pointer
 539  * @data: tracepoint data
 540  *
 541  * Returns 0 if ok, error value on error.
 542  */
 543 int tracepoint_probe_unregister(struct tracepoint *tp, void *probe, void *data)
 544 {
 545         struct tracepoint_func tp_func;
 546         int ret;
 547
 548         mutex_lock(&tracepoints_mutex);
 549         tp_func.func = probe;
 550         tp_func.data = data;
 551         ret = tracepoint_remove_func(tp, &tp_func);
 552         mutex_unlock(&tracepoints_mutex);
 553         return ret;
 554 }
 555 EXPORT_SYMBOL_GPL(tracepoint_probe_unregister);
 556
 557 static void for_each_tracepoint_range(
 558                 tracepoint_ptr_t *begin, tracepoint_ptr_t *end,
 559                 void (*fct)(struct tracepoint *tp, void *priv),
 560                 void *priv)
 561 {
 562         tracepoint_ptr_t *iter;
 563
 564         if (!begin)
 565                 return;
 566         for (iter = begin; iter < end; iter++)
 567                 fct(tracepoint_ptr_deref(iter), priv);
 568 }
 569
 570 #ifdef CONFIG_MODULES
 571 bool trace_module_has_bad_taint(struct module *mod)
 572 {
 573         return mod->taints & ~((1 << TAINT_OOT_MODULE) | (1 << TAINT_CRAP) |
 574                                 (1 << TAINT_UNSIGNED_MODULE) | (1 << TAINT_TEST) |
 575                                 (1 << TAINT_LIVEPATCH));
 576 }
 577
 578 static BLOCKING_NOTIFIER_HEAD(tracepoint_notify_list);
 579
 580 /**
 581  * register_tracepoint_module_notifier - register tracepoint coming/going notifier
 582  * @nb: notifier block
 583  *
 584  * Notifiers registered with this function are called on module
 585  * coming/going with the tracepoint_module_list_mutex held.
 586  * The notifier block callback should expect a "struct tp_module" data
 587  * pointer.
 588  */
 589 int register_tracepoint_module_notifier(struct notifier_block *nb)
 590 {
 591         struct tp_module *tp_mod;
 592         int ret;
 593
 594         mutex_lock(&tracepoint_module_list_mutex);
 595         ret = blocking_notifier_chain_register(&tracepoint_notify_list, nb);
 596         if (ret)
 597                 goto end;
 598         list_for_each_entry(tp_mod, &tracepoint_module_list, list)
 599                 (void) nb->notifier_call(nb, MODULE_STATE_COMING, tp_mod);
 600 end:
 601         mutex_unlock(&tracepoint_module_list_mutex);
 602         return ret;
 603 }
 604 EXPORT_SYMBOL_GPL(register_tracepoint_module_notifier);
 605
 606 /**
 607  * unregister_tracepoint_module_notifier - unregister tracepoint coming/going notifier
 608  * @nb: notifier block
 609  *
 610  * The notifier block callback should expect a "struct tp_module" data
 611  * pointer.
 612  */
 613 int unregister_tracepoint_module_notifier(struct notifier_block *nb)
 614 {
 615         struct tp_module *tp_mod;
 616         int ret;
 617
 618         mutex_lock(&tracepoint_module_list_mutex);
 619         ret = blocking_notifier_chain_unregister(&tracepoint_notify_list, nb);
 620         if (ret)
 621                 goto end;
 622         list_for_each_entry(tp_mod, &tracepoint_module_list, list)
 623                 (void) nb->notifier_call(nb, MODULE_STATE_GOING, tp_mod);
 624 end:
 625         mutex_unlock(&tracepoint_module_list_mutex);
 626         return ret;
 627
 628 }
 629 EXPORT_SYMBOL_GPL(unregister_tracepoint_module_notifier);
 630
 631 /*
 632  * Ensure the tracer unregistered the module's probes before the module
 633  * teardown is performed. Prevents leaks of probe and data pointers.
 634  */
 635 static void tp_module_going_check_quiescent(struct tracepoint *tp, void *priv)
 636 {
 637         WARN_ON_ONCE(tp->funcs);
 638 }
 639
 640 static int tracepoint_module_coming(struct module *mod)
 641 {
 642         struct tp_module *tp_mod;
 643
 644         if (!mod->num_tracepoints)
 645                 return 0;
 646
 647         /*
 648          * We skip modules that taint the kernel, especially those with different
 649          * module headers (for forced load), to make sure we don't cause a crash.
 650          * Staging, out-of-tree, unsigned GPL, and test modules are fine.
 651          */
 652         if (trace_module_has_bad_taint(mod))
 653                 return 0;
 654
 655         tp_mod = kmalloc(sizeof(struct tp_module), GFP_KERNEL);
 656         if (!tp_mod)
 657                 return -ENOMEM;
 658         tp_mod->mod = mod;
 659
 660         mutex_lock(&tracepoint_module_list_mutex);
 661         list_add_tail(&tp_mod->list, &tracepoint_module_list);
 662         blocking_notifier_call_chain(&tracepoint_notify_list,
 663                         MODULE_STATE_COMING, tp_mod);
 664         mutex_unlock(&tracepoint_module_list_mutex);
 665         return 0;
 666 }
 667
 668 static void tracepoint_module_going(struct module *mod)
 669 {
 670         struct tp_module *tp_mod;
 671
 672         if (!mod->num_tracepoints)
 673                 return;
 674
 675         mutex_lock(&tracepoint_module_list_mutex);
 676         list_for_each_entry(tp_mod, &tracepoint_module_list, list) {
 677                 if (tp_mod->mod == mod) {
 678                         blocking_notifier_call_chain(&tracepoint_notify_list,
 679                                         MODULE_STATE_GOING, tp_mod);
 680                         list_del(&tp_mod->list);
 681                         kfree(tp_mod);
 682                         /*
 683                          * Called the going notifier before checking for
 684                          * quiescence.
 685                          */
 686                         for_each_tracepoint_range(mod->tracepoints_ptrs,
 687                                 mod->tracepoints_ptrs + mod->num_tracepoints,
 688                                 tp_module_going_check_quiescent, NULL);
 689                         break;
 690                 }
 691         }
 692         /*
 693          * In the case of modules that were tainted at "coming", we'll simply
 694          * walk through the list without finding it. We cannot use the "tainted"
 695          * flag on "going", in case a module taints the kernel only after being
 696          * loaded.
 697          */
 698         mutex_unlock(&tracepoint_module_list_mutex);
 699 }
 700
 701 static int tracepoint_module_notify(struct notifier_block *self,
 702                 unsigned long val, void *data)
 703 {
 704         struct module *mod = data;
 705         int ret = 0;
 706
 707         switch (val) {
 708         case MODULE_STATE_COMING:
 709                 ret = tracepoint_module_coming(mod);
 710                 break;
 711         case MODULE_STATE_LIVE:
 712                 break;
 713         case MODULE_STATE_GOING:
 714                 tracepoint_module_going(mod);
 715                 break;
 716         case MODULE_STATE_UNFORMED:
 717                 break;
 718         }
 719         return notifier_from_errno(ret);
 720 }
 721
 722 static struct notifier_block tracepoint_module_nb = {
 723         .notifier_call = tracepoint_module_notify,
 724         .priority = 0,
 725 };
 726
 727 static __init int init_tracepoints(void)
 728 {
 729         int ret;
 730
 731         ret = register_module_notifier(&tracepoint_module_nb);
 732         if (ret)
 733                 pr_warn("Failed to register tracepoint module enter notifier\n");
 734
 735         return ret;
 736 }
 737 __initcall(init_tracepoints);
 738
 739 /**
 740  * for_each_tracepoint_in_module - iteration on all tracepoints in a module
 741  * @mod: module
 742  * @fct: callback
 743  * @priv: private data
 744  */
 745 void for_each_tracepoint_in_module(struct module *mod,
 746                                    void (*fct)(struct tracepoint *tp,
 747                                     struct module *mod, void *priv),
 748                                    void *priv)
 749 {
 750         tracepoint_ptr_t *begin, *end, *iter;
 751
 752         lockdep_assert_held(&tracepoint_module_list_mutex);
 753
 754         if (!mod)
 755                 return;
 756
 757         begin = mod->tracepoints_ptrs;
 758         end = mod->tracepoints_ptrs + mod->num_tracepoints;
 759
 760         for (iter = begin; iter < end; iter++)
 761                 fct(tracepoint_ptr_deref(iter), mod, priv);
 762 }
 763
 764 /**
 765  * for_each_module_tracepoint - iteration on all tracepoints in all modules
 766  * @fct: callback
 767  * @priv: private data
 768  */
 769 void for_each_module_tracepoint(void (*fct)(struct tracepoint *tp,
 770                                  struct module *mod, void *priv),
 771                                 void *priv)
 772 {
 773         struct tp_module *tp_mod;
 774
 775         mutex_lock(&tracepoint_module_list_mutex);
 776         list_for_each_entry(tp_mod, &tracepoint_module_list, list)
 777                 for_each_tracepoint_in_module(tp_mod->mod, fct, priv);
 778         mutex_unlock(&tracepoint_module_list_mutex);
 779 }
 780 #endif /* CONFIG_MODULES */
 781
 782 /**
 783  * for_each_kernel_tracepoint - iteration on all kernel tracepoints
 784  * @fct: callback
 785  * @priv: private data
 786  */
 787 void for_each_kernel_tracepoint(void (*fct)(struct tracepoint *tp, void *priv),
 788                 void *priv)
 789 {
 790         for_each_tracepoint_range(__start___tracepoints_ptrs,
 791                 __stop___tracepoints_ptrs, fct, priv);
 792 }
 793 EXPORT_SYMBOL_GPL(for_each_kernel_tracepoint);
 794
 795 #ifdef CONFIG_HAVE_SYSCALL_TRACEPOINTS
 796
 797 /* NB: reg/unreg are called while guarded with the tracepoints_mutex */
 798 static int sys_tracepoint_refcount;
 799
 800 int syscall_regfunc(void)
 801 {
 802         struct task_struct *p, *t;
 803
 804         if (!sys_tracepoint_refcount) {
 805                 read_lock(&tasklist_lock);
 806                 for_each_process_thread(p, t) {
 807                         set_task_syscall_work(t, SYSCALL_TRACEPOINT);
 808                 }
 809                 read_unlock(&tasklist_lock);
 810         }
 811         sys_tracepoint_refcount++;
 812
 813         return 0;
 814 }
 815
 816 void syscall_unregfunc(void)
 817 {
 818         struct task_struct *p, *t;
 819
 820         sys_tracepoint_refcount--;
 821         if (!sys_tracepoint_refcount) {
 822                 read_lock(&tasklist_lock);
 823                 for_each_process_thread(p, t) {
 824                         clear_task_syscall_work(t, SYSCALL_TRACEPOINT);
 825                 }
 826                 read_unlock(&tasklist_lock);
 827         }
 828 }
 829 #endif