4 * @remark Copyright 2002 OProfile authors
5 * @remark Read the file COPYING
7 * @author John Levon <levon@movementarian.org>
9 * Each CPU has a local buffer that stores PC value/event
10 * pairs. We also log context switches when we notice them.
11 * Eventually each CPU's buffer is processed into the global
12 * event buffer by sync_buffer().
14 * We use a local buffer for two reasons: an NMI or similar
15 * interrupt cannot synchronise, and high sampling rates
16 * would lead to catastrophic global synchronisation if
17 * a global buffer was used.
20 #include <linux/sched.h>
21 #include <linux/oprofile.h>
22 #include <linux/vmalloc.h>
23 #include <linux/errno.h>
25 #include "event_buffer.h"
26 #include "cpu_buffer.h"
27 #include "buffer_sync.h"
30 DEFINE_PER_CPU(struct oprofile_cpu_buffer
, cpu_buffer
);
32 static void wq_sync_buffer(struct work_struct
*work
);
34 #define DEFAULT_TIMER_EXPIRE (HZ / 10)
35 static int work_enabled
;
37 void free_cpu_buffers(void)
41 for_each_online_cpu(i
) {
42 vfree(per_cpu(cpu_buffer
, i
).buffer
);
43 per_cpu(cpu_buffer
, i
).buffer
= NULL
;
47 int alloc_cpu_buffers(void)
51 unsigned long buffer_size
= fs_cpu_buffer_size
;
53 for_each_online_cpu(i
) {
54 struct oprofile_cpu_buffer
*b
= &per_cpu(cpu_buffer
, i
);
56 b
->buffer
= vmalloc_node(sizeof(struct op_sample
) * buffer_size
,
62 b
->last_is_kernel
= -1;
64 b
->buffer_size
= buffer_size
;
67 b
->sample_received
= 0;
68 b
->sample_lost_overflow
= 0;
69 b
->backtrace_aborted
= 0;
70 b
->sample_invalid_eip
= 0;
72 INIT_DELAYED_WORK(&b
->work
, wq_sync_buffer
);
81 void start_cpu_work(void)
87 for_each_online_cpu(i
) {
88 struct oprofile_cpu_buffer
*b
= &per_cpu(cpu_buffer
, i
);
91 * Spread the work by 1 jiffy per cpu so they dont all
94 schedule_delayed_work_on(i
, &b
->work
, DEFAULT_TIMER_EXPIRE
+ i
);
98 void end_cpu_work(void)
104 for_each_online_cpu(i
) {
105 struct oprofile_cpu_buffer
*b
= &per_cpu(cpu_buffer
, i
);
107 cancel_delayed_work(&b
->work
);
110 flush_scheduled_work();
113 /* Resets the cpu buffer to a sane state. */
114 void cpu_buffer_reset(struct oprofile_cpu_buffer
* cpu_buf
)
116 /* reset these to invalid values; the next sample
117 * collected will populate the buffer with proper
118 * values to initialize the buffer
120 cpu_buf
->last_is_kernel
= -1;
121 cpu_buf
->last_task
= NULL
;
124 /* compute number of available slots in cpu_buffer queue */
125 static unsigned long nr_available_slots(struct oprofile_cpu_buffer
const * b
)
127 unsigned long head
= b
->head_pos
;
128 unsigned long tail
= b
->tail_pos
;
131 return (tail
- head
) - 1;
133 return tail
+ (b
->buffer_size
- head
) - 1;
136 static void increment_head(struct oprofile_cpu_buffer
* b
)
138 unsigned long new_head
= b
->head_pos
+ 1;
140 /* Ensure anything written to the slot before we
141 * increment is visible */
144 if (new_head
< b
->buffer_size
)
145 b
->head_pos
= new_head
;
151 add_sample(struct oprofile_cpu_buffer
* cpu_buf
,
152 unsigned long pc
, unsigned long event
)
154 struct op_sample
* entry
= &cpu_buf
->buffer
[cpu_buf
->head_pos
];
156 entry
->event
= event
;
157 increment_head(cpu_buf
);
161 add_code(struct oprofile_cpu_buffer
* buffer
, unsigned long value
)
163 add_sample(buffer
, ESCAPE_CODE
, value
);
166 /* This must be safe from any context. It's safe writing here
167 * because of the head/tail separation of the writer and reader
170 * is_kernel is needed because on some architectures you cannot
171 * tell if you are in kernel or user space simply by looking at
172 * pc. We tag this in the buffer by generating kernel enter/exit
173 * events whenever is_kernel changes
175 static int log_sample(struct oprofile_cpu_buffer
* cpu_buf
, unsigned long pc
,
176 int is_kernel
, unsigned long event
)
178 struct task_struct
* task
;
180 cpu_buf
->sample_received
++;
182 if (pc
== ESCAPE_CODE
) {
183 cpu_buf
->sample_invalid_eip
++;
187 if (nr_available_slots(cpu_buf
) < 3) {
188 cpu_buf
->sample_lost_overflow
++;
192 is_kernel
= !!is_kernel
;
196 /* notice a switch from user->kernel or vice versa */
197 if (cpu_buf
->last_is_kernel
!= is_kernel
) {
198 cpu_buf
->last_is_kernel
= is_kernel
;
199 add_code(cpu_buf
, is_kernel
);
202 /* notice a task switch */
203 if (cpu_buf
->last_task
!= task
) {
204 cpu_buf
->last_task
= task
;
205 add_code(cpu_buf
, (unsigned long)task
);
208 add_sample(cpu_buf
, pc
, event
);
212 static int oprofile_begin_trace(struct oprofile_cpu_buffer
* cpu_buf
)
214 if (nr_available_slots(cpu_buf
) < 4) {
215 cpu_buf
->sample_lost_overflow
++;
219 add_code(cpu_buf
, CPU_TRACE_BEGIN
);
220 cpu_buf
->tracing
= 1;
224 static void oprofile_end_trace(struct oprofile_cpu_buffer
* cpu_buf
)
226 cpu_buf
->tracing
= 0;
229 void oprofile_add_ext_sample(unsigned long pc
, struct pt_regs
* const regs
,
230 unsigned long event
, int is_kernel
)
232 struct oprofile_cpu_buffer
*cpu_buf
= &__get_cpu_var(cpu_buffer
);
234 if (!backtrace_depth
) {
235 log_sample(cpu_buf
, pc
, is_kernel
, event
);
239 if (!oprofile_begin_trace(cpu_buf
))
242 /* if log_sample() fail we can't backtrace since we lost the source
244 if (log_sample(cpu_buf
, pc
, is_kernel
, event
))
245 oprofile_ops
.backtrace(regs
, backtrace_depth
);
246 oprofile_end_trace(cpu_buf
);
249 void oprofile_add_sample(struct pt_regs
* const regs
, unsigned long event
)
251 int is_kernel
= !user_mode(regs
);
252 unsigned long pc
= profile_pc(regs
);
254 oprofile_add_ext_sample(pc
, regs
, event
, is_kernel
);
257 void oprofile_add_pc(unsigned long pc
, int is_kernel
, unsigned long event
)
259 struct oprofile_cpu_buffer
*cpu_buf
= &__get_cpu_var(cpu_buffer
);
260 log_sample(cpu_buf
, pc
, is_kernel
, event
);
263 void oprofile_add_trace(unsigned long pc
)
265 struct oprofile_cpu_buffer
*cpu_buf
= &__get_cpu_var(cpu_buffer
);
267 if (!cpu_buf
->tracing
)
270 if (nr_available_slots(cpu_buf
) < 1) {
271 cpu_buf
->tracing
= 0;
272 cpu_buf
->sample_lost_overflow
++;
276 /* broken frame can give an eip with the same value as an escape code,
277 * abort the trace if we get it */
278 if (pc
== ESCAPE_CODE
) {
279 cpu_buf
->tracing
= 0;
280 cpu_buf
->backtrace_aborted
++;
284 add_sample(cpu_buf
, pc
, 0);
288 * This serves to avoid cpu buffer overflow, and makes sure
289 * the task mortuary progresses
291 * By using schedule_delayed_work_on and then schedule_delayed_work
292 * we guarantee this will stay on the correct cpu
294 static void wq_sync_buffer(struct work_struct
*work
)
296 struct oprofile_cpu_buffer
* b
=
297 container_of(work
, struct oprofile_cpu_buffer
, work
.work
);
298 if (b
->cpu
!= smp_processor_id()) {
299 printk("WQ on CPU%d, prefer CPU%d\n",
300 smp_processor_id(), b
->cpu
);
304 /* don't re-add the work if we're shutting down */
306 schedule_delayed_work(&b
->work
, DEFAULT_TIMER_EXPIRE
);