2 * Copyright (C) 2006 Jens Axboe <axboe@kernel.dk>
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
13 * You should have received a copy of the GNU General Public License
14 * along with this program; if not, write to the Free Software
15 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
18 #include <linux/kernel.h>
19 #include <linux/blkdev.h>
20 #include <linux/blktrace_api.h>
21 #include <linux/percpu.h>
22 #include <linux/init.h>
23 #include <linux/mutex.h>
24 #include <linux/debugfs.h>
25 #include <linux/time.h>
26 #include <asm/uaccess.h>
28 static DEFINE_PER_CPU(unsigned long long, blk_trace_cpu_offset
) = { 0, };
29 static unsigned int blktrace_seq __read_mostly
= 1;
32 * Send out a notify message.
34 static void trace_note(struct blk_trace
*bt
, pid_t pid
, int action
,
35 const void *data
, size_t len
)
37 struct blk_io_trace
*t
;
39 t
= relay_reserve(bt
->rchan
, sizeof(*t
) + len
);
41 const int cpu
= smp_processor_id();
43 t
->magic
= BLK_IO_TRACE_MAGIC
| BLK_IO_TRACE_VERSION
;
44 t
->time
= cpu_clock(cpu
) - per_cpu(blk_trace_cpu_offset
, cpu
);
50 memcpy((void *) t
+ sizeof(*t
), data
, len
);
55 * Send out a notify for this process, if we haven't done so since a trace
58 static void trace_note_tsk(struct blk_trace
*bt
, struct task_struct
*tsk
)
60 tsk
->btrace_seq
= blktrace_seq
;
61 trace_note(bt
, tsk
->pid
, BLK_TN_PROCESS
, tsk
->comm
, sizeof(tsk
->comm
));
64 static void trace_note_time(struct blk_trace
*bt
)
71 words
[0] = now
.tv_sec
;
72 words
[1] = now
.tv_nsec
;
74 local_irq_save(flags
);
75 trace_note(bt
, 0, BLK_TN_TIMESTAMP
, words
, sizeof(words
));
76 local_irq_restore(flags
);
79 static int act_log_check(struct blk_trace
*bt
, u32 what
, sector_t sector
,
82 if (((bt
->act_mask
<< BLK_TC_SHIFT
) & what
) == 0)
84 if (sector
< bt
->start_lba
|| sector
> bt
->end_lba
)
86 if (bt
->pid
&& pid
!= bt
->pid
)
93 * Data direction bit lookup
95 static u32 ddir_act
[2] __read_mostly
= { BLK_TC_ACT(BLK_TC_READ
), BLK_TC_ACT(BLK_TC_WRITE
) };
98 * Bio action bits of interest
100 static u32 bio_act
[9] __read_mostly
= { 0, BLK_TC_ACT(BLK_TC_BARRIER
), BLK_TC_ACT(BLK_TC_SYNC
), 0, BLK_TC_ACT(BLK_TC_AHEAD
), 0, 0, 0, BLK_TC_ACT(BLK_TC_META
) };
103 * More could be added as needed, taking care to increment the decrementer
104 * to get correct indexing
106 #define trace_barrier_bit(rw) \
107 (((rw) & (1 << BIO_RW_BARRIER)) >> (BIO_RW_BARRIER - 0))
108 #define trace_sync_bit(rw) \
109 (((rw) & (1 << BIO_RW_SYNC)) >> (BIO_RW_SYNC - 1))
110 #define trace_ahead_bit(rw) \
111 (((rw) & (1 << BIO_RW_AHEAD)) << (2 - BIO_RW_AHEAD))
112 #define trace_meta_bit(rw) \
113 (((rw) & (1 << BIO_RW_META)) >> (BIO_RW_META - 3))
116 * The worker for the various blk_add_trace*() types. Fills out a
117 * blk_io_trace structure and places it in a per-cpu subbuffer.
119 void __blk_add_trace(struct blk_trace
*bt
, sector_t sector
, int bytes
,
120 int rw
, u32 what
, int error
, int pdu_len
, void *pdu_data
)
122 struct task_struct
*tsk
= current
;
123 struct blk_io_trace
*t
;
125 unsigned long *sequence
;
129 if (unlikely(bt
->trace_state
!= Blktrace_running
))
132 what
|= ddir_act
[rw
& WRITE
];
133 what
|= bio_act
[trace_barrier_bit(rw
)];
134 what
|= bio_act
[trace_sync_bit(rw
)];
135 what
|= bio_act
[trace_ahead_bit(rw
)];
136 what
|= bio_act
[trace_meta_bit(rw
)];
139 if (unlikely(act_log_check(bt
, what
, sector
, pid
)))
143 * A word about the locking here - we disable interrupts to reserve
144 * some space in the relay per-cpu buffer, to prevent an irq
145 * from coming in and stepping on our toes. Once reserved, it's
146 * enough to get preemption disabled to prevent read of this data
147 * before we are through filling it. get_cpu()/put_cpu() does this
150 local_irq_save(flags
);
152 if (unlikely(tsk
->btrace_seq
!= blktrace_seq
))
153 trace_note_tsk(bt
, tsk
);
155 t
= relay_reserve(bt
->rchan
, sizeof(*t
) + pdu_len
);
157 cpu
= smp_processor_id();
158 sequence
= per_cpu_ptr(bt
->sequence
, cpu
);
160 t
->magic
= BLK_IO_TRACE_MAGIC
| BLK_IO_TRACE_VERSION
;
161 t
->sequence
= ++(*sequence
);
162 t
->time
= cpu_clock(cpu
) - per_cpu(blk_trace_cpu_offset
, cpu
);
170 t
->pdu_len
= pdu_len
;
173 memcpy((void *) t
+ sizeof(*t
), pdu_data
, pdu_len
);
176 local_irq_restore(flags
);
179 EXPORT_SYMBOL_GPL(__blk_add_trace
);
181 static struct dentry
*blk_tree_root
;
182 static struct mutex blk_tree_mutex
;
183 static unsigned int root_users
;
185 static inline void blk_remove_root(void)
188 debugfs_remove(blk_tree_root
);
189 blk_tree_root
= NULL
;
193 static void blk_remove_tree(struct dentry
*dir
)
195 mutex_lock(&blk_tree_mutex
);
197 if (--root_users
== 0)
199 mutex_unlock(&blk_tree_mutex
);
202 static struct dentry
*blk_create_tree(const char *blk_name
)
204 struct dentry
*dir
= NULL
;
206 mutex_lock(&blk_tree_mutex
);
208 if (!blk_tree_root
) {
209 blk_tree_root
= debugfs_create_dir("block", NULL
);
214 dir
= debugfs_create_dir(blk_name
, blk_tree_root
);
221 mutex_unlock(&blk_tree_mutex
);
225 static void blk_trace_cleanup(struct blk_trace
*bt
)
227 relay_close(bt
->rchan
);
228 debugfs_remove(bt
->dropped_file
);
229 blk_remove_tree(bt
->dir
);
230 free_percpu(bt
->sequence
);
234 static int blk_trace_remove(struct request_queue
*q
)
236 struct blk_trace
*bt
;
238 bt
= xchg(&q
->blk_trace
, NULL
);
242 if (bt
->trace_state
== Blktrace_setup
||
243 bt
->trace_state
== Blktrace_stopped
)
244 blk_trace_cleanup(bt
);
249 static int blk_dropped_open(struct inode
*inode
, struct file
*filp
)
251 filp
->private_data
= inode
->i_private
;
256 static ssize_t
blk_dropped_read(struct file
*filp
, char __user
*buffer
,
257 size_t count
, loff_t
*ppos
)
259 struct blk_trace
*bt
= filp
->private_data
;
262 snprintf(buf
, sizeof(buf
), "%u\n", atomic_read(&bt
->dropped
));
264 return simple_read_from_buffer(buffer
, count
, ppos
, buf
, strlen(buf
));
267 static const struct file_operations blk_dropped_fops
= {
268 .owner
= THIS_MODULE
,
269 .open
= blk_dropped_open
,
270 .read
= blk_dropped_read
,
274 * Keep track of how many times we encountered a full subbuffer, to aid
275 * the user space app in telling how many lost events there were.
277 static int blk_subbuf_start_callback(struct rchan_buf
*buf
, void *subbuf
,
278 void *prev_subbuf
, size_t prev_padding
)
280 struct blk_trace
*bt
;
282 if (!relay_buf_full(buf
))
285 bt
= buf
->chan
->private_data
;
286 atomic_inc(&bt
->dropped
);
290 static int blk_remove_buf_file_callback(struct dentry
*dentry
)
292 debugfs_remove(dentry
);
296 static struct dentry
*blk_create_buf_file_callback(const char *filename
,
297 struct dentry
*parent
,
299 struct rchan_buf
*buf
,
302 return debugfs_create_file(filename
, mode
, parent
, buf
,
303 &relay_file_operations
);
306 static struct rchan_callbacks blk_relay_callbacks
= {
307 .subbuf_start
= blk_subbuf_start_callback
,
308 .create_buf_file
= blk_create_buf_file_callback
,
309 .remove_buf_file
= blk_remove_buf_file_callback
,
313 * Setup everything required to start tracing
315 int do_blk_trace_setup(struct request_queue
*q
, struct block_device
*bdev
,
316 struct blk_user_trace_setup
*buts
)
318 struct blk_trace
*old_bt
, *bt
= NULL
;
319 struct dentry
*dir
= NULL
;
320 char b
[BDEVNAME_SIZE
];
323 if (!buts
->buf_size
|| !buts
->buf_nr
)
326 strcpy(buts
->name
, bdevname(bdev
, b
));
329 * some device names have larger paths - convert the slashes
330 * to underscores for this to work as expected
332 for (i
= 0; i
< strlen(buts
->name
); i
++)
333 if (buts
->name
[i
] == '/')
337 bt
= kzalloc(sizeof(*bt
), GFP_KERNEL
);
341 bt
->sequence
= alloc_percpu(unsigned long);
346 dir
= blk_create_tree(buts
->name
);
351 bt
->dev
= bdev
->bd_dev
;
352 atomic_set(&bt
->dropped
, 0);
355 bt
->dropped_file
= debugfs_create_file("dropped", 0444, dir
, bt
, &blk_dropped_fops
);
356 if (!bt
->dropped_file
)
359 bt
->rchan
= relay_open("trace", dir
, buts
->buf_size
,
360 buts
->buf_nr
, &blk_relay_callbacks
, bt
);
364 bt
->act_mask
= buts
->act_mask
;
366 bt
->act_mask
= (u16
) -1;
368 bt
->start_lba
= buts
->start_lba
;
369 bt
->end_lba
= buts
->end_lba
;
374 bt
->trace_state
= Blktrace_setup
;
377 old_bt
= xchg(&q
->blk_trace
, bt
);
379 (void) xchg(&q
->blk_trace
, old_bt
);
386 blk_remove_tree(dir
);
388 if (bt
->dropped_file
)
389 debugfs_remove(bt
->dropped_file
);
390 free_percpu(bt
->sequence
);
392 relay_close(bt
->rchan
);
398 static int blk_trace_setup(struct request_queue
*q
, struct block_device
*bdev
,
401 struct blk_user_trace_setup buts
;
404 ret
= copy_from_user(&buts
, arg
, sizeof(buts
));
408 ret
= do_blk_trace_setup(q
, bdev
, &buts
);
412 if (copy_to_user(arg
, &buts
, sizeof(buts
)))
418 static int blk_trace_startstop(struct request_queue
*q
, int start
)
420 struct blk_trace
*bt
;
423 if ((bt
= q
->blk_trace
) == NULL
)
427 * For starting a trace, we can transition from a setup or stopped
428 * trace. For stopping a trace, the state must be running
432 if (bt
->trace_state
== Blktrace_setup
||
433 bt
->trace_state
== Blktrace_stopped
) {
436 bt
->trace_state
= Blktrace_running
;
442 if (bt
->trace_state
== Blktrace_running
) {
443 bt
->trace_state
= Blktrace_stopped
;
444 relay_flush(bt
->rchan
);
453 * blk_trace_ioctl: - handle the ioctls associated with tracing
454 * @bdev: the block device
455 * @cmd: the ioctl cmd
456 * @arg: the argument data, if any
459 int blk_trace_ioctl(struct block_device
*bdev
, unsigned cmd
, char __user
*arg
)
461 struct request_queue
*q
;
464 q
= bdev_get_queue(bdev
);
468 mutex_lock(&bdev
->bd_mutex
);
472 ret
= blk_trace_setup(q
, bdev
, arg
);
477 ret
= blk_trace_startstop(q
, start
);
479 case BLKTRACETEARDOWN
:
480 ret
= blk_trace_remove(q
);
487 mutex_unlock(&bdev
->bd_mutex
);
492 * blk_trace_shutdown: - stop and cleanup trace structures
493 * @q: the request queue associated with the device
496 void blk_trace_shutdown(struct request_queue
*q
)
499 blk_trace_startstop(q
, 0);
505 * Average offset over two calls to cpu_clock() with a gettimeofday()
508 static void blk_check_time(unsigned long long *t
, int this_cpu
)
510 unsigned long long a
, b
;
513 a
= cpu_clock(this_cpu
);
514 do_gettimeofday(&tv
);
515 b
= cpu_clock(this_cpu
);
517 *t
= tv
.tv_sec
* 1000000000 + tv
.tv_usec
* 1000;
522 * calibrate our inter-CPU timings
524 static void blk_trace_check_cpu_time(void *data
)
526 unsigned long long *t
;
527 int this_cpu
= get_cpu();
529 t
= &per_cpu(blk_trace_cpu_offset
, this_cpu
);
532 * Just call it twice, hopefully the second call will be cache hot
533 * and a little more precise
535 blk_check_time(t
, this_cpu
);
536 blk_check_time(t
, this_cpu
);
541 static void blk_trace_set_ht_offsets(void)
543 #if defined(CONFIG_SCHED_SMT)
547 * now make sure HT siblings have the same time offset
550 for_each_online_cpu(cpu
) {
551 unsigned long long *cpu_off
, *sibling_off
;
553 for_each_cpu_mask(i
, cpu_sibling_map
[cpu
]) {
557 cpu_off
= &per_cpu(blk_trace_cpu_offset
, cpu
);
558 sibling_off
= &per_cpu(blk_trace_cpu_offset
, i
);
559 *sibling_off
= *cpu_off
;
566 static __init
int blk_trace_init(void)
568 mutex_init(&blk_tree_mutex
);
569 on_each_cpu(blk_trace_check_cpu_time
, NULL
, 1, 1);
570 blk_trace_set_ht_offsets();
575 module_init(blk_trace_init
);