4 * @remark Copyright 2002 OProfile authors
5 * @remark Read the file COPYING
7 * @author John Levon <levon@movementarian.org>
8 * @author Barry Kasindorf <barry.kasindorf@amd.com>
10 * Each CPU has a local buffer that stores PC value/event
11 * pairs. We also log context switches when we notice them.
12 * Eventually each CPU's buffer is processed into the global
13 * event buffer by sync_buffer().
15 * We use a local buffer for two reasons: an NMI or similar
16 * interrupt cannot synchronise, and high sampling rates
17 * would lead to catastrophic global synchronisation if
18 * a global buffer was used.
21 #include <linux/sched.h>
22 #include <linux/oprofile.h>
23 #include <linux/vmalloc.h>
24 #include <linux/errno.h>
26 #include "event_buffer.h"
27 #include "cpu_buffer.h"
28 #include "buffer_sync.h"
31 DEFINE_PER_CPU(struct oprofile_cpu_buffer
, cpu_buffer
);
33 static void wq_sync_buffer(struct work_struct
*work
);
35 #define DEFAULT_TIMER_EXPIRE (HZ / 10)
36 static int work_enabled
;
38 void free_cpu_buffers(void)
42 for_each_possible_cpu(i
) {
43 vfree(per_cpu(cpu_buffer
, i
).buffer
);
44 per_cpu(cpu_buffer
, i
).buffer
= NULL
;
48 unsigned long oprofile_get_cpu_buffer_size(void)
50 return fs_cpu_buffer_size
;
53 void oprofile_cpu_buffer_inc_smpl_lost(void)
55 struct oprofile_cpu_buffer
*cpu_buf
56 = &__get_cpu_var(cpu_buffer
);
58 cpu_buf
->sample_lost_overflow
++;
61 int alloc_cpu_buffers(void)
65 unsigned long buffer_size
= fs_cpu_buffer_size
;
67 for_each_possible_cpu(i
) {
68 struct oprofile_cpu_buffer
*b
= &per_cpu(cpu_buffer
, i
);
70 b
->buffer
= vmalloc_node(sizeof(struct op_sample
) * buffer_size
,
76 b
->last_is_kernel
= -1;
78 b
->buffer_size
= buffer_size
;
81 b
->sample_received
= 0;
82 b
->sample_lost_overflow
= 0;
83 b
->backtrace_aborted
= 0;
84 b
->sample_invalid_eip
= 0;
86 INIT_DELAYED_WORK(&b
->work
, wq_sync_buffer
);
95 void start_cpu_work(void)
101 for_each_online_cpu(i
) {
102 struct oprofile_cpu_buffer
*b
= &per_cpu(cpu_buffer
, i
);
105 * Spread the work by 1 jiffy per cpu so they dont all
108 schedule_delayed_work_on(i
, &b
->work
, DEFAULT_TIMER_EXPIRE
+ i
);
112 void end_cpu_work(void)
118 for_each_online_cpu(i
) {
119 struct oprofile_cpu_buffer
*b
= &per_cpu(cpu_buffer
, i
);
121 cancel_delayed_work(&b
->work
);
124 flush_scheduled_work();
127 /* Resets the cpu buffer to a sane state. */
128 void cpu_buffer_reset(struct oprofile_cpu_buffer
*cpu_buf
)
131 * reset these to invalid values; the next sample collected
132 * will populate the buffer with proper values to initialize
135 cpu_buf
->last_is_kernel
= -1;
136 cpu_buf
->last_task
= NULL
;
139 /* compute number of available slots in cpu_buffer queue */
140 static unsigned long nr_available_slots(struct oprofile_cpu_buffer
const *b
)
142 unsigned long head
= b
->head_pos
;
143 unsigned long tail
= b
->tail_pos
;
146 return (tail
- head
) - 1;
148 return tail
+ (b
->buffer_size
- head
) - 1;
151 static void increment_head(struct oprofile_cpu_buffer
*b
)
153 unsigned long new_head
= b
->head_pos
+ 1;
156 * Ensure anything written to the slot before we increment is
161 if (new_head
< b
->buffer_size
)
162 b
->head_pos
= new_head
;
168 add_sample(struct oprofile_cpu_buffer
*cpu_buf
,
169 unsigned long pc
, unsigned long event
)
171 struct op_sample
*entry
= &cpu_buf
->buffer
[cpu_buf
->head_pos
];
173 entry
->event
= event
;
174 increment_head(cpu_buf
);
178 add_code(struct oprofile_cpu_buffer
*buffer
, unsigned long value
)
180 add_sample(buffer
, ESCAPE_CODE
, value
);
183 /* This must be safe from any context. It's safe writing here
184 * because of the head/tail separation of the writer and reader
187 * is_kernel is needed because on some architectures you cannot
188 * tell if you are in kernel or user space simply by looking at
189 * pc. We tag this in the buffer by generating kernel enter/exit
190 * events whenever is_kernel changes
192 static int log_sample(struct oprofile_cpu_buffer
*cpu_buf
, unsigned long pc
,
193 int is_kernel
, unsigned long event
)
195 struct task_struct
*task
;
197 cpu_buf
->sample_received
++;
199 if (pc
== ESCAPE_CODE
) {
200 cpu_buf
->sample_invalid_eip
++;
204 if (nr_available_slots(cpu_buf
) < 3) {
205 cpu_buf
->sample_lost_overflow
++;
209 is_kernel
= !!is_kernel
;
213 /* notice a switch from user->kernel or vice versa */
214 if (cpu_buf
->last_is_kernel
!= is_kernel
) {
215 cpu_buf
->last_is_kernel
= is_kernel
;
216 add_code(cpu_buf
, is_kernel
);
219 /* notice a task switch */
220 if (cpu_buf
->last_task
!= task
) {
221 cpu_buf
->last_task
= task
;
222 add_code(cpu_buf
, (unsigned long)task
);
225 add_sample(cpu_buf
, pc
, event
);
229 static int oprofile_begin_trace(struct oprofile_cpu_buffer
*cpu_buf
)
231 if (nr_available_slots(cpu_buf
) < 4) {
232 cpu_buf
->sample_lost_overflow
++;
236 add_code(cpu_buf
, CPU_TRACE_BEGIN
);
237 cpu_buf
->tracing
= 1;
241 static void oprofile_end_trace(struct oprofile_cpu_buffer
*cpu_buf
)
243 cpu_buf
->tracing
= 0;
246 void oprofile_add_ext_sample(unsigned long pc
, struct pt_regs
* const regs
,
247 unsigned long event
, int is_kernel
)
249 struct oprofile_cpu_buffer
*cpu_buf
= &__get_cpu_var(cpu_buffer
);
251 if (!backtrace_depth
) {
252 log_sample(cpu_buf
, pc
, is_kernel
, event
);
256 if (!oprofile_begin_trace(cpu_buf
))
260 * if log_sample() fail we can't backtrace since we lost the
261 * source of this event
263 if (log_sample(cpu_buf
, pc
, is_kernel
, event
))
264 oprofile_ops
.backtrace(regs
, backtrace_depth
);
265 oprofile_end_trace(cpu_buf
);
268 void oprofile_add_sample(struct pt_regs
* const regs
, unsigned long event
)
270 int is_kernel
= !user_mode(regs
);
271 unsigned long pc
= profile_pc(regs
);
273 oprofile_add_ext_sample(pc
, regs
, event
, is_kernel
);
276 #ifdef CONFIG_OPROFILE_IBS
278 #define MAX_IBS_SAMPLE_SIZE 14
280 void oprofile_add_ibs_sample(struct pt_regs
* const regs
,
281 unsigned int * const ibs_sample
, int ibs_code
)
283 int is_kernel
= !user_mode(regs
);
284 struct oprofile_cpu_buffer
*cpu_buf
= &__get_cpu_var(cpu_buffer
);
285 struct task_struct
*task
;
287 cpu_buf
->sample_received
++;
289 if (nr_available_slots(cpu_buf
) < MAX_IBS_SAMPLE_SIZE
) {
290 /* we can't backtrace since we lost the source of this event */
291 cpu_buf
->sample_lost_overflow
++;
295 /* notice a switch from user->kernel or vice versa */
296 if (cpu_buf
->last_is_kernel
!= is_kernel
) {
297 cpu_buf
->last_is_kernel
= is_kernel
;
298 add_code(cpu_buf
, is_kernel
);
301 /* notice a task switch */
304 if (cpu_buf
->last_task
!= task
) {
305 cpu_buf
->last_task
= task
;
306 add_code(cpu_buf
, (unsigned long)task
);
310 add_code(cpu_buf
, ibs_code
);
311 add_sample(cpu_buf
, ibs_sample
[0], ibs_sample
[1]);
312 add_sample(cpu_buf
, ibs_sample
[2], ibs_sample
[3]);
313 add_sample(cpu_buf
, ibs_sample
[4], ibs_sample
[5]);
315 if (ibs_code
== IBS_OP_BEGIN
) {
316 add_sample(cpu_buf
, ibs_sample
[6], ibs_sample
[7]);
317 add_sample(cpu_buf
, ibs_sample
[8], ibs_sample
[9]);
318 add_sample(cpu_buf
, ibs_sample
[10], ibs_sample
[11]);
322 oprofile_ops
.backtrace(regs
, backtrace_depth
);
327 void oprofile_add_pc(unsigned long pc
, int is_kernel
, unsigned long event
)
329 struct oprofile_cpu_buffer
*cpu_buf
= &__get_cpu_var(cpu_buffer
);
330 log_sample(cpu_buf
, pc
, is_kernel
, event
);
333 void oprofile_add_trace(unsigned long pc
)
335 struct oprofile_cpu_buffer
*cpu_buf
= &__get_cpu_var(cpu_buffer
);
337 if (!cpu_buf
->tracing
)
340 if (nr_available_slots(cpu_buf
) < 1) {
341 cpu_buf
->tracing
= 0;
342 cpu_buf
->sample_lost_overflow
++;
347 * broken frame can give an eip with the same value as an
348 * escape code, abort the trace if we get it
350 if (pc
== ESCAPE_CODE
) {
351 cpu_buf
->tracing
= 0;
352 cpu_buf
->backtrace_aborted
++;
356 add_sample(cpu_buf
, pc
, 0);
360 * This serves to avoid cpu buffer overflow, and makes sure
361 * the task mortuary progresses
363 * By using schedule_delayed_work_on and then schedule_delayed_work
364 * we guarantee this will stay on the correct cpu
366 static void wq_sync_buffer(struct work_struct
*work
)
368 struct oprofile_cpu_buffer
*b
=
369 container_of(work
, struct oprofile_cpu_buffer
, work
.work
);
370 if (b
->cpu
!= smp_processor_id()) {
371 printk(KERN_DEBUG
"WQ on CPU%d, prefer CPU%d\n",
372 smp_processor_id(), b
->cpu
);
374 if (!cpu_online(b
->cpu
)) {
375 cancel_delayed_work(&b
->work
);
381 /* don't re-add the work if we're shutting down */
383 schedule_delayed_work(&b
->work
, DEFAULT_TIMER_EXPIRE
);