4 * This provides a low-level interface to the hardware's Debug Store
5 * feature that is used for branch trace store (BTS) and
6 * precise-event based sampling (PEBS).
9 * - per-thread and per-cpu allocation of BTS and PEBS
10 * - buffer memory allocation (optional)
11 * - buffer overflow handling
15 * - get_task_struct on all parameter tasks
16 * - current is allowed to trace parameter tasks
19 * Copyright (C) 2007-2008 Intel Corporation.
20 * Markus Metzger <markus.t.metzger@intel.com>, 2007-2008
26 #include <linux/errno.h>
27 #include <linux/string.h>
28 #include <linux/slab.h>
29 #include <linux/sched.h>
34 * The configuration for a particular DS hardware implementation.
36 struct ds_configuration
{
37 /* the size of the DS structure in bytes */
38 unsigned char sizeof_ds
;
39 /* the size of one pointer-typed field in the DS structure in bytes;
40 this covers the first 8 fields related to buffer management. */
41 unsigned char sizeof_field
;
42 /* the size of a BTS/PEBS record in bytes */
43 unsigned char sizeof_rec
[2];
45 static struct ds_configuration ds_cfg
;
49 * Debug Store (DS) save area configuration (see Intel64 and IA32
50 * Architectures Software Developer's Manual, section 18.5)
52 * The DS configuration consists of the following fields; different
53 * architetures vary in the size of those fields.
54 * - double-word aligned base linear address of the BTS buffer
55 * - write pointer into the BTS buffer
56 * - end linear address of the BTS buffer (one byte beyond the end of
58 * - interrupt pointer into BTS buffer
59 * (interrupt occurs when write pointer passes interrupt pointer)
60 * - double-word aligned base linear address of the PEBS buffer
61 * - write pointer into the PEBS buffer
62 * - end linear address of the PEBS buffer (one byte beyond the end of
64 * - interrupt pointer into PEBS buffer
65 * (interrupt occurs when write pointer passes interrupt pointer)
66 * - value to which counter is reset following counter overflow
68 * Later architectures use 64bit pointers throughout, whereas earlier
69 * architectures use 32bit pointers in 32bit mode.
72 * We compute the base address for the first 8 fields based on:
73 * - the field size stored in the DS configuration
74 * - the relative field position
75 * - an offset giving the start of the respective region
77 * This offset is further used to index various arrays holding
78 * information for BTS and PEBS at the respective index.
80 * On later 32bit processors, we only access the lower 32bit of the
81 * 64bit pointer fields. The upper halves will be zeroed out.
88 ds_interrupt_threshold
,
96 static inline unsigned long ds_get(const unsigned char *base
,
97 enum ds_qualifier qual
, enum ds_field field
)
99 base
+= (ds_cfg
.sizeof_field
* (field
+ (4 * qual
)));
100 return *(unsigned long *)base
;
103 static inline void ds_set(unsigned char *base
, enum ds_qualifier qual
,
104 enum ds_field field
, unsigned long value
)
106 base
+= (ds_cfg
.sizeof_field
* (field
+ (4 * qual
)));
107 (*(unsigned long *)base
) = value
;
112 * Locking is done only for allocating BTS or PEBS resources and for
113 * guarding context and buffer memory allocation.
115 * Most functions require the current task to own the ds context part
116 * they are going to access. All the locking is done when validating
117 * access to the context.
119 static spinlock_t ds_lock
= __SPIN_LOCK_UNLOCKED(ds_lock
);
122 * Validate that the current task is allowed to access the BTS/PEBS
123 * buffer of the parameter task.
125 * Returns 0, if access is granted; -Eerrno, otherwise.
127 static inline int ds_validate_access(struct ds_context
*context
,
128 enum ds_qualifier qual
)
133 if (context
->owner
[qual
] == current
)
141 * We either support (system-wide) per-cpu or per-thread allocation.
142 * We distinguish the two based on the task_struct pointer, where a
143 * NULL pointer indicates per-cpu allocation for the current cpu.
145 * Allocations are use-counted. As soon as resources are allocated,
146 * further allocations must be of the same type (per-cpu or
147 * per-thread). We model this by counting allocations (i.e. the number
148 * of tracers of a certain type) for one type negatively:
150 * >0 number of per-thread tracers
151 * <0 number of per-cpu tracers
153 * The below functions to get and put tracers and to check the
154 * allocation type require the ds_lock to be held by the caller.
156 * Tracers essentially gives the number of ds contexts for a certain
157 * type of allocation.
161 static inline void get_tracer(struct task_struct
*task
)
163 tracers
+= (task
? 1 : -1);
166 static inline void put_tracer(struct task_struct
*task
)
168 tracers
-= (task
? 1 : -1);
171 static inline int check_tracer(struct task_struct
*task
)
173 return (task
? (tracers
>= 0) : (tracers
<= 0));
178 * The DS context is either attached to a thread or to a cpu:
179 * - in the former case, the thread_struct contains a pointer to the
181 * - in the latter case, we use a static array of per-cpu context
184 * Contexts are use-counted. They are allocated on first access and
185 * deallocated when the last user puts the context.
187 * We distinguish between an allocating and a non-allocating get of a
189 * - the allocating get is used for requesting BTS/PEBS resources. It
190 * requires the caller to hold the global ds_lock.
191 * - the non-allocating get is used for all other cases. A
192 * non-existing context indicates an error. It acquires and releases
193 * the ds_lock itself for obtaining the context.
195 * A context and its DS configuration are allocated and deallocated
196 * together. A context always has a DS configuration of the
199 static DEFINE_PER_CPU(struct ds_context
*, system_context
);
201 #define this_system_context per_cpu(system_context, smp_processor_id())
204 * Returns the pointer to the parameter task's context or to the
205 * system-wide context, if task is NULL.
207 * Increases the use count of the returned context, if not NULL.
209 static inline struct ds_context
*ds_get_context(struct task_struct
*task
)
211 struct ds_context
*context
;
215 context
= (task
? task
->thread
.ds_ctx
: this_system_context
);
219 spin_unlock(&ds_lock
);
225 * Same as ds_get_context, but allocates the context and it's DS
226 * structure, if necessary; returns NULL; if out of memory.
228 * pre: requires ds_lock to be held
230 static inline struct ds_context
*ds_alloc_context(struct task_struct
*task
)
232 struct ds_context
**p_context
=
233 (task
? &task
->thread
.ds_ctx
: &this_system_context
);
234 struct ds_context
*context
= *p_context
;
237 spin_unlock(&ds_lock
);
239 context
= kzalloc(sizeof(*context
), GFP_KERNEL
);
246 context
->ds
= kzalloc(ds_cfg
.sizeof_ds
, GFP_KERNEL
);
255 * Check for race - another CPU could have allocated
264 *p_context
= context
;
266 context
->this = p_context
;
267 context
->task
= task
;
270 set_tsk_thread_flag(task
, TIF_DS_AREA_MSR
);
272 if (!task
|| (task
== current
))
273 wrmsr(MSR_IA32_DS_AREA
, (unsigned long)context
->ds
, 0);
284 * Decreases the use count of the parameter context, if not NULL.
285 * Deallocates the context, if the use count reaches zero.
287 static inline void ds_put_context(struct ds_context
*context
)
294 if (--context
->count
)
297 *(context
->this) = NULL
;
300 clear_tsk_thread_flag(context
->task
, TIF_DS_AREA_MSR
);
302 if (!context
->task
|| (context
->task
== current
))
303 wrmsrl(MSR_IA32_DS_AREA
, 0);
305 put_tracer(context
->task
);
307 /* free any leftover buffers from tracers that did not
308 * deallocate them properly. */
309 kfree(context
->buffer
[ds_bts
]);
310 kfree(context
->buffer
[ds_pebs
]);
314 spin_unlock(&ds_lock
);
319 * Handle a buffer overflow
321 * task: the task whose buffers are overflowing;
322 * NULL for a buffer overflow on the current cpu
323 * context: the ds context
324 * qual: the buffer type
326 static void ds_overflow(struct task_struct
*task
, struct ds_context
*context
,
327 enum ds_qualifier qual
)
332 if (context
->callback
[qual
])
333 (*context
->callback
[qual
])(task
);
335 /* todo: do some more overflow handling */
340 * Allocate a non-pageable buffer of the parameter size.
341 * Checks the memory and the locked memory rlimit.
343 * Returns the buffer, if successful;
344 * NULL, if out of memory or rlimit exceeded.
346 * size: the requested buffer size in bytes
347 * pages (out): if not NULL, contains the number of pages reserved
349 static inline void *ds_allocate_buffer(size_t size
, unsigned int *pages
)
351 unsigned long rlim
, vm
, pgsz
;
354 pgsz
= PAGE_ALIGN(size
) >> PAGE_SHIFT
;
356 rlim
= current
->signal
->rlim
[RLIMIT_AS
].rlim_cur
>> PAGE_SHIFT
;
357 vm
= current
->mm
->total_vm
+ pgsz
;
361 rlim
= current
->signal
->rlim
[RLIMIT_MEMLOCK
].rlim_cur
>> PAGE_SHIFT
;
362 vm
= current
->mm
->locked_vm
+ pgsz
;
366 buffer
= kzalloc(size
, GFP_KERNEL
);
370 current
->mm
->total_vm
+= pgsz
;
371 current
->mm
->locked_vm
+= pgsz
;
379 static int ds_request(struct task_struct
*task
, void *base
, size_t size
,
380 ds_ovfl_callback_t ovfl
, enum ds_qualifier qual
)
382 struct ds_context
*context
;
383 unsigned long buffer
, adj
;
384 const unsigned long alignment
= (1 << 3);
387 if (!ds_cfg
.sizeof_ds
)
390 /* we require some space to do alignment adjustments below */
391 if (size
< (alignment
+ ds_cfg
.sizeof_rec
[qual
]))
394 /* buffer overflow notification is not yet implemented */
402 context
= ds_alloc_context(task
);
407 if (!check_tracer(task
))
411 if (context
->owner
[qual
] == current
)
414 if (context
->owner
[qual
] != NULL
)
416 context
->owner
[qual
] = current
;
418 spin_unlock(&ds_lock
);
423 base
= ds_allocate_buffer(size
, &context
->pages
[qual
]);
427 context
->buffer
[qual
] = base
;
431 context
->callback
[qual
] = ovfl
;
433 /* adjust the buffer address and size to meet alignment
435 * - buffer is double-word aligned
436 * - size is multiple of record size
438 * We checked the size at the very beginning; we have enough
439 * space to do the adjustment.
441 buffer
= (unsigned long)base
;
443 adj
= ALIGN(buffer
, alignment
) - buffer
;
447 size
/= ds_cfg
.sizeof_rec
[qual
];
448 size
*= ds_cfg
.sizeof_rec
[qual
];
450 ds_set(context
->ds
, qual
, ds_buffer_base
, buffer
);
451 ds_set(context
->ds
, qual
, ds_index
, buffer
);
452 ds_set(context
->ds
, qual
, ds_absolute_maximum
, buffer
+ size
);
455 /* todo: select a suitable interrupt threshold */
457 ds_set(context
->ds
, qual
,
458 ds_interrupt_threshold
, buffer
+ size
+ 1);
460 /* we keep the context until ds_release */
464 context
->owner
[qual
] = NULL
;
465 ds_put_context(context
);
469 spin_unlock(&ds_lock
);
470 ds_put_context(context
);
474 int ds_request_bts(struct task_struct
*task
, void *base
, size_t size
,
475 ds_ovfl_callback_t ovfl
)
477 return ds_request(task
, base
, size
, ovfl
, ds_bts
);
480 int ds_request_pebs(struct task_struct
*task
, void *base
, size_t size
,
481 ds_ovfl_callback_t ovfl
)
483 return ds_request(task
, base
, size
, ovfl
, ds_pebs
);
486 static int ds_release(struct task_struct
*task
, enum ds_qualifier qual
)
488 struct ds_context
*context
;
491 context
= ds_get_context(task
);
492 error
= ds_validate_access(context
, qual
);
496 kfree(context
->buffer
[qual
]);
497 context
->buffer
[qual
] = NULL
;
499 current
->mm
->total_vm
-= context
->pages
[qual
];
500 current
->mm
->locked_vm
-= context
->pages
[qual
];
501 context
->pages
[qual
] = 0;
502 context
->owner
[qual
] = NULL
;
505 * we put the context twice:
506 * once for the ds_get_context
507 * once for the corresponding ds_request
509 ds_put_context(context
);
511 ds_put_context(context
);
515 int ds_release_bts(struct task_struct
*task
)
517 return ds_release(task
, ds_bts
);
520 int ds_release_pebs(struct task_struct
*task
)
522 return ds_release(task
, ds_pebs
);
525 static int ds_get_index(struct task_struct
*task
, size_t *pos
,
526 enum ds_qualifier qual
)
528 struct ds_context
*context
;
529 unsigned long base
, index
;
532 context
= ds_get_context(task
);
533 error
= ds_validate_access(context
, qual
);
537 base
= ds_get(context
->ds
, qual
, ds_buffer_base
);
538 index
= ds_get(context
->ds
, qual
, ds_index
);
540 error
= ((index
- base
) / ds_cfg
.sizeof_rec
[qual
]);
544 ds_put_context(context
);
548 int ds_get_bts_index(struct task_struct
*task
, size_t *pos
)
550 return ds_get_index(task
, pos
, ds_bts
);
553 int ds_get_pebs_index(struct task_struct
*task
, size_t *pos
)
555 return ds_get_index(task
, pos
, ds_pebs
);
558 static int ds_get_end(struct task_struct
*task
, size_t *pos
,
559 enum ds_qualifier qual
)
561 struct ds_context
*context
;
562 unsigned long base
, end
;
565 context
= ds_get_context(task
);
566 error
= ds_validate_access(context
, qual
);
570 base
= ds_get(context
->ds
, qual
, ds_buffer_base
);
571 end
= ds_get(context
->ds
, qual
, ds_absolute_maximum
);
573 error
= ((end
- base
) / ds_cfg
.sizeof_rec
[qual
]);
577 ds_put_context(context
);
581 int ds_get_bts_end(struct task_struct
*task
, size_t *pos
)
583 return ds_get_end(task
, pos
, ds_bts
);
586 int ds_get_pebs_end(struct task_struct
*task
, size_t *pos
)
588 return ds_get_end(task
, pos
, ds_pebs
);
591 static int ds_access(struct task_struct
*task
, size_t index
,
592 const void **record
, enum ds_qualifier qual
)
594 struct ds_context
*context
;
595 unsigned long base
, idx
;
601 context
= ds_get_context(task
);
602 error
= ds_validate_access(context
, qual
);
606 base
= ds_get(context
->ds
, qual
, ds_buffer_base
);
607 idx
= base
+ (index
* ds_cfg
.sizeof_rec
[qual
]);
610 if (idx
> ds_get(context
->ds
, qual
, ds_absolute_maximum
))
613 *record
= (const void *)idx
;
614 error
= ds_cfg
.sizeof_rec
[qual
];
616 ds_put_context(context
);
620 int ds_access_bts(struct task_struct
*task
, size_t index
, const void **record
)
622 return ds_access(task
, index
, record
, ds_bts
);
625 int ds_access_pebs(struct task_struct
*task
, size_t index
, const void **record
)
627 return ds_access(task
, index
, record
, ds_pebs
);
630 static int ds_write(struct task_struct
*task
, const void *record
, size_t size
,
631 enum ds_qualifier qual
, int force
)
633 struct ds_context
*context
;
640 context
= ds_get_context(task
);
645 error
= ds_validate_access(context
, qual
);
652 unsigned long base
, index
, end
, write_end
, int_th
;
653 unsigned long write_size
, adj_write_size
;
656 * write as much as possible without producing an
657 * overflow interrupt.
659 * interrupt_threshold must either be
660 * - bigger than absolute_maximum or
661 * - point to a record between buffer_base and absolute_maximum
663 * index points to a valid record.
665 base
= ds_get(context
->ds
, qual
, ds_buffer_base
);
666 index
= ds_get(context
->ds
, qual
, ds_index
);
667 end
= ds_get(context
->ds
, qual
, ds_absolute_maximum
);
668 int_th
= ds_get(context
->ds
, qual
, ds_interrupt_threshold
);
670 write_end
= min(end
, int_th
);
672 /* if we are already beyond the interrupt threshold,
673 * we fill the entire buffer */
674 if (write_end
<= index
)
677 if (write_end
<= index
)
680 write_size
= min((unsigned long) size
, write_end
- index
);
681 memcpy((void *)index
, record
, write_size
);
683 record
= (const char *)record
+ write_size
;
687 adj_write_size
= write_size
/ ds_cfg
.sizeof_rec
[qual
];
688 adj_write_size
*= ds_cfg
.sizeof_rec
[qual
];
690 /* zero out trailing bytes */
691 memset((char *)index
+ write_size
, 0,
692 adj_write_size
- write_size
);
693 index
+= adj_write_size
;
697 ds_set(context
->ds
, qual
, ds_index
, index
);
700 ds_overflow(task
, context
, qual
);
704 ds_put_context(context
);
708 int ds_write_bts(struct task_struct
*task
, const void *record
, size_t size
)
710 return ds_write(task
, record
, size
, ds_bts
, /* force = */ 0);
713 int ds_write_pebs(struct task_struct
*task
, const void *record
, size_t size
)
715 return ds_write(task
, record
, size
, ds_pebs
, /* force = */ 0);
718 int ds_unchecked_write_bts(struct task_struct
*task
,
719 const void *record
, size_t size
)
721 return ds_write(task
, record
, size
, ds_bts
, /* force = */ 1);
724 int ds_unchecked_write_pebs(struct task_struct
*task
,
725 const void *record
, size_t size
)
727 return ds_write(task
, record
, size
, ds_pebs
, /* force = */ 1);
730 static int ds_reset_or_clear(struct task_struct
*task
,
731 enum ds_qualifier qual
, int clear
)
733 struct ds_context
*context
;
734 unsigned long base
, end
;
737 context
= ds_get_context(task
);
738 error
= ds_validate_access(context
, qual
);
742 base
= ds_get(context
->ds
, qual
, ds_buffer_base
);
743 end
= ds_get(context
->ds
, qual
, ds_absolute_maximum
);
746 memset((void *)base
, 0, end
- base
);
748 ds_set(context
->ds
, qual
, ds_index
, base
);
752 ds_put_context(context
);
756 int ds_reset_bts(struct task_struct
*task
)
758 return ds_reset_or_clear(task
, ds_bts
, /* clear = */ 0);
761 int ds_reset_pebs(struct task_struct
*task
)
763 return ds_reset_or_clear(task
, ds_pebs
, /* clear = */ 0);
766 int ds_clear_bts(struct task_struct
*task
)
768 return ds_reset_or_clear(task
, ds_bts
, /* clear = */ 1);
771 int ds_clear_pebs(struct task_struct
*task
)
773 return ds_reset_or_clear(task
, ds_pebs
, /* clear = */ 1);
776 int ds_get_pebs_reset(struct task_struct
*task
, u64
*value
)
778 struct ds_context
*context
;
784 context
= ds_get_context(task
);
785 error
= ds_validate_access(context
, ds_pebs
);
789 *value
= *(u64
*)(context
->ds
+ (ds_cfg
.sizeof_field
* 8));
793 ds_put_context(context
);
797 int ds_set_pebs_reset(struct task_struct
*task
, u64 value
)
799 struct ds_context
*context
;
802 context
= ds_get_context(task
);
803 error
= ds_validate_access(context
, ds_pebs
);
807 *(u64
*)(context
->ds
+ (ds_cfg
.sizeof_field
* 8)) = value
;
811 ds_put_context(context
);
815 static const struct ds_configuration ds_cfg_var
= {
816 .sizeof_ds
= sizeof(long) * 12,
817 .sizeof_field
= sizeof(long),
818 .sizeof_rec
[ds_bts
] = sizeof(long) * 3,
820 .sizeof_rec
[ds_pebs
] = sizeof(long) * 10
822 .sizeof_rec
[ds_pebs
] = sizeof(long) * 18
825 static const struct ds_configuration ds_cfg_64
= {
828 .sizeof_rec
[ds_bts
] = 8 * 3,
830 .sizeof_rec
[ds_pebs
] = 8 * 10
832 .sizeof_rec
[ds_pebs
] = 8 * 18
837 ds_configure(const struct ds_configuration
*cfg
)
842 void __cpuinit
ds_init_intel(struct cpuinfo_x86
*c
)
846 switch (c
->x86_model
) {
848 case 0xE: /* Pentium M */
849 ds_configure(&ds_cfg_var
);
851 case 0xF: /* Core2 */
852 case 0x1C: /* Atom */
853 ds_configure(&ds_cfg_64
);
856 /* sorry, don't know about them */
861 switch (c
->x86_model
) {
864 case 0x2: /* Netburst */
865 ds_configure(&ds_cfg_var
);
868 /* sorry, don't know about them */
873 /* sorry, don't know about them */
878 void ds_free(struct ds_context
*context
)
880 /* This is called when the task owning the parameter context
881 * is dying. There should not be any user of that context left
882 * to disturb us, anymore. */
883 unsigned long leftovers
= context
->count
;
885 ds_put_context(context
);