2 * Copyright (C) 2006 Jens Axboe <axboe@kernel.dk>
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
13 * You should have received a copy of the GNU General Public License
14 * along with this program; if not, write to the Free Software
15 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
18 #include <linux/kernel.h>
19 #include <linux/blkdev.h>
20 #include <linux/blktrace_api.h>
21 #include <linux/percpu.h>
22 #include <linux/init.h>
23 #include <linux/mutex.h>
24 #include <linux/debugfs.h>
25 #include <linux/time.h>
26 #include <asm/uaccess.h>
28 static unsigned int blktrace_seq __read_mostly
= 1;
31 * Send out a notify message.
33 static void trace_note(struct blk_trace
*bt
, pid_t pid
, int action
,
34 const void *data
, size_t len
)
36 struct blk_io_trace
*t
;
38 t
= relay_reserve(bt
->rchan
, sizeof(*t
) + len
);
40 const int cpu
= smp_processor_id();
42 t
->magic
= BLK_IO_TRACE_MAGIC
| BLK_IO_TRACE_VERSION
;
43 t
->time
= ktime_to_ns(ktime_get());
49 memcpy((void *) t
+ sizeof(*t
), data
, len
);
54 * Send out a notify for this process, if we haven't done so since a trace
57 static void trace_note_tsk(struct blk_trace
*bt
, struct task_struct
*tsk
)
59 tsk
->btrace_seq
= blktrace_seq
;
60 trace_note(bt
, tsk
->pid
, BLK_TN_PROCESS
, tsk
->comm
, sizeof(tsk
->comm
));
63 static void trace_note_time(struct blk_trace
*bt
)
70 words
[0] = now
.tv_sec
;
71 words
[1] = now
.tv_nsec
;
73 local_irq_save(flags
);
74 trace_note(bt
, 0, BLK_TN_TIMESTAMP
, words
, sizeof(words
));
75 local_irq_restore(flags
);
78 void __trace_note_message(struct blk_trace
*bt
, const char *fmt
, ...)
85 local_irq_save(flags
);
86 buf
= per_cpu_ptr(bt
->msg_data
, smp_processor_id());
88 n
= vscnprintf(buf
, BLK_TN_MAX_MSG
, fmt
, args
);
91 trace_note(bt
, 0, BLK_TN_MESSAGE
, buf
, n
);
92 local_irq_restore(flags
);
94 EXPORT_SYMBOL_GPL(__trace_note_message
);
96 static int act_log_check(struct blk_trace
*bt
, u32 what
, sector_t sector
,
99 if (((bt
->act_mask
<< BLK_TC_SHIFT
) & what
) == 0)
101 if (sector
< bt
->start_lba
|| sector
> bt
->end_lba
)
103 if (bt
->pid
&& pid
!= bt
->pid
)
110 * Data direction bit lookup
112 static u32 ddir_act
[2] __read_mostly
= { BLK_TC_ACT(BLK_TC_READ
), BLK_TC_ACT(BLK_TC_WRITE
) };
115 * Bio action bits of interest
117 static u32 bio_act
[17] __read_mostly
= {
118 [1] = BLK_TC_ACT(BLK_TC_BARRIER
),
119 [2] = BLK_TC_ACT(BLK_TC_SYNC
),
120 [4] = BLK_TC_ACT(BLK_TC_AHEAD
),
121 [8] = BLK_TC_ACT(BLK_TC_META
),
122 [16] = BLK_TC_ACT(BLK_TC_DISCARD
)
126 * More could be added as needed, taking care to increment the decrementer
127 * to get correct indexing
129 #define trace_barrier_bit(rw) \
130 (((rw) & (1 << BIO_RW_BARRIER)) >> (BIO_RW_BARRIER - 0))
131 #define trace_sync_bit(rw) \
132 (((rw) & (1 << BIO_RW_SYNC)) >> (BIO_RW_SYNC - 1))
133 #define trace_ahead_bit(rw) \
134 (((rw) & (1 << BIO_RW_AHEAD)) << (2 - BIO_RW_AHEAD))
135 #define trace_meta_bit(rw) \
136 (((rw) & (1 << BIO_RW_META)) >> (BIO_RW_META - 3))
137 #define trace_discard_bit(rw) \
138 (((rw) & (1 << BIO_RW_DISCARD)) >> (BIO_RW_DISCARD - 4))
141 * The worker for the various blk_add_trace*() types. Fills out a
142 * blk_io_trace structure and places it in a per-cpu subbuffer.
144 void __blk_add_trace(struct blk_trace
*bt
, sector_t sector
, int bytes
,
145 int rw
, u32 what
, int error
, int pdu_len
, void *pdu_data
)
147 struct task_struct
*tsk
= current
;
148 struct blk_io_trace
*t
;
150 unsigned long *sequence
;
154 if (unlikely(bt
->trace_state
!= Blktrace_running
))
157 what
|= ddir_act
[rw
& WRITE
];
158 what
|= bio_act
[trace_barrier_bit(rw
)];
159 what
|= bio_act
[trace_sync_bit(rw
)];
160 what
|= bio_act
[trace_ahead_bit(rw
)];
161 what
|= bio_act
[trace_meta_bit(rw
)];
162 what
|= bio_act
[trace_discard_bit(rw
)];
165 if (unlikely(act_log_check(bt
, what
, sector
, pid
)))
169 * A word about the locking here - we disable interrupts to reserve
170 * some space in the relay per-cpu buffer, to prevent an irq
171 * from coming in and stepping on our toes.
173 local_irq_save(flags
);
175 if (unlikely(tsk
->btrace_seq
!= blktrace_seq
))
176 trace_note_tsk(bt
, tsk
);
178 t
= relay_reserve(bt
->rchan
, sizeof(*t
) + pdu_len
);
180 cpu
= smp_processor_id();
181 sequence
= per_cpu_ptr(bt
->sequence
, cpu
);
183 t
->magic
= BLK_IO_TRACE_MAGIC
| BLK_IO_TRACE_VERSION
;
184 t
->sequence
= ++(*sequence
);
185 t
->time
= ktime_to_ns(ktime_get());
193 t
->pdu_len
= pdu_len
;
196 memcpy((void *) t
+ sizeof(*t
), pdu_data
, pdu_len
);
199 local_irq_restore(flags
);
202 EXPORT_SYMBOL_GPL(__blk_add_trace
);
204 static struct dentry
*blk_tree_root
;
205 static DEFINE_MUTEX(blk_tree_mutex
);
206 static unsigned int root_users
;
208 static inline void blk_remove_root(void)
211 debugfs_remove(blk_tree_root
);
212 blk_tree_root
= NULL
;
216 static void blk_remove_tree(struct dentry
*dir
)
218 mutex_lock(&blk_tree_mutex
);
220 if (--root_users
== 0)
222 mutex_unlock(&blk_tree_mutex
);
225 static struct dentry
*blk_create_tree(const char *blk_name
)
227 struct dentry
*dir
= NULL
;
230 mutex_lock(&blk_tree_mutex
);
232 if (!blk_tree_root
) {
233 blk_tree_root
= debugfs_create_dir("block", NULL
);
239 dir
= debugfs_create_dir(blk_name
, blk_tree_root
);
243 /* Delete root only if we created it */
249 mutex_unlock(&blk_tree_mutex
);
253 static void blk_trace_cleanup(struct blk_trace
*bt
)
255 relay_close(bt
->rchan
);
256 debugfs_remove(bt
->msg_file
);
257 debugfs_remove(bt
->dropped_file
);
258 blk_remove_tree(bt
->dir
);
259 free_percpu(bt
->sequence
);
260 free_percpu(bt
->msg_data
);
264 int blk_trace_remove(struct request_queue
*q
)
266 struct blk_trace
*bt
;
268 bt
= xchg(&q
->blk_trace
, NULL
);
272 if (bt
->trace_state
== Blktrace_setup
||
273 bt
->trace_state
== Blktrace_stopped
)
274 blk_trace_cleanup(bt
);
278 EXPORT_SYMBOL_GPL(blk_trace_remove
);
280 static int blk_dropped_open(struct inode
*inode
, struct file
*filp
)
282 filp
->private_data
= inode
->i_private
;
287 static ssize_t
blk_dropped_read(struct file
*filp
, char __user
*buffer
,
288 size_t count
, loff_t
*ppos
)
290 struct blk_trace
*bt
= filp
->private_data
;
293 snprintf(buf
, sizeof(buf
), "%u\n", atomic_read(&bt
->dropped
));
295 return simple_read_from_buffer(buffer
, count
, ppos
, buf
, strlen(buf
));
298 static const struct file_operations blk_dropped_fops
= {
299 .owner
= THIS_MODULE
,
300 .open
= blk_dropped_open
,
301 .read
= blk_dropped_read
,
304 static int blk_msg_open(struct inode
*inode
, struct file
*filp
)
306 filp
->private_data
= inode
->i_private
;
311 static ssize_t
blk_msg_write(struct file
*filp
, const char __user
*buffer
,
312 size_t count
, loff_t
*ppos
)
315 struct blk_trace
*bt
;
317 if (count
> BLK_TN_MAX_MSG
)
320 msg
= kmalloc(count
, GFP_KERNEL
);
324 if (copy_from_user(msg
, buffer
, count
)) {
329 bt
= filp
->private_data
;
330 __trace_note_message(bt
, "%s", msg
);
336 static const struct file_operations blk_msg_fops
= {
337 .owner
= THIS_MODULE
,
338 .open
= blk_msg_open
,
339 .write
= blk_msg_write
,
343 * Keep track of how many times we encountered a full subbuffer, to aid
344 * the user space app in telling how many lost events there were.
346 static int blk_subbuf_start_callback(struct rchan_buf
*buf
, void *subbuf
,
347 void *prev_subbuf
, size_t prev_padding
)
349 struct blk_trace
*bt
;
351 if (!relay_buf_full(buf
))
354 bt
= buf
->chan
->private_data
;
355 atomic_inc(&bt
->dropped
);
359 static int blk_remove_buf_file_callback(struct dentry
*dentry
)
361 debugfs_remove(dentry
);
365 static struct dentry
*blk_create_buf_file_callback(const char *filename
,
366 struct dentry
*parent
,
368 struct rchan_buf
*buf
,
371 return debugfs_create_file(filename
, mode
, parent
, buf
,
372 &relay_file_operations
);
375 static struct rchan_callbacks blk_relay_callbacks
= {
376 .subbuf_start
= blk_subbuf_start_callback
,
377 .create_buf_file
= blk_create_buf_file_callback
,
378 .remove_buf_file
= blk_remove_buf_file_callback
,
382 * Setup everything required to start tracing
384 int do_blk_trace_setup(struct request_queue
*q
, char *name
, dev_t dev
,
385 struct blk_user_trace_setup
*buts
)
387 struct blk_trace
*old_bt
, *bt
= NULL
;
388 struct dentry
*dir
= NULL
;
391 if (!buts
->buf_size
|| !buts
->buf_nr
)
394 strcpy(buts
->name
, name
);
397 * some device names have larger paths - convert the slashes
398 * to underscores for this to work as expected
400 for (i
= 0; i
< strlen(buts
->name
); i
++)
401 if (buts
->name
[i
] == '/')
405 bt
= kzalloc(sizeof(*bt
), GFP_KERNEL
);
409 bt
->sequence
= alloc_percpu(unsigned long);
413 bt
->msg_data
= __alloc_percpu(BLK_TN_MAX_MSG
);
418 dir
= blk_create_tree(buts
->name
);
424 atomic_set(&bt
->dropped
, 0);
427 bt
->dropped_file
= debugfs_create_file("dropped", 0444, dir
, bt
, &blk_dropped_fops
);
428 if (!bt
->dropped_file
)
431 bt
->msg_file
= debugfs_create_file("msg", 0222, dir
, bt
, &blk_msg_fops
);
435 bt
->rchan
= relay_open("trace", dir
, buts
->buf_size
,
436 buts
->buf_nr
, &blk_relay_callbacks
, bt
);
440 bt
->act_mask
= buts
->act_mask
;
442 bt
->act_mask
= (u16
) -1;
444 bt
->start_lba
= buts
->start_lba
;
445 bt
->end_lba
= buts
->end_lba
;
450 bt
->trace_state
= Blktrace_setup
;
453 old_bt
= xchg(&q
->blk_trace
, bt
);
455 (void) xchg(&q
->blk_trace
, old_bt
);
462 blk_remove_tree(dir
);
465 debugfs_remove(bt
->msg_file
);
466 if (bt
->dropped_file
)
467 debugfs_remove(bt
->dropped_file
);
468 free_percpu(bt
->sequence
);
469 free_percpu(bt
->msg_data
);
471 relay_close(bt
->rchan
);
477 int blk_trace_setup(struct request_queue
*q
, char *name
, dev_t dev
,
480 struct blk_user_trace_setup buts
;
483 ret
= copy_from_user(&buts
, arg
, sizeof(buts
));
487 ret
= do_blk_trace_setup(q
, name
, dev
, &buts
);
491 if (copy_to_user(arg
, &buts
, sizeof(buts
)))
496 EXPORT_SYMBOL_GPL(blk_trace_setup
);
498 int blk_trace_startstop(struct request_queue
*q
, int start
)
500 struct blk_trace
*bt
;
503 if ((bt
= q
->blk_trace
) == NULL
)
507 * For starting a trace, we can transition from a setup or stopped
508 * trace. For stopping a trace, the state must be running
512 if (bt
->trace_state
== Blktrace_setup
||
513 bt
->trace_state
== Blktrace_stopped
) {
516 bt
->trace_state
= Blktrace_running
;
522 if (bt
->trace_state
== Blktrace_running
) {
523 bt
->trace_state
= Blktrace_stopped
;
524 relay_flush(bt
->rchan
);
531 EXPORT_SYMBOL_GPL(blk_trace_startstop
);
534 * blk_trace_ioctl: - handle the ioctls associated with tracing
535 * @bdev: the block device
536 * @cmd: the ioctl cmd
537 * @arg: the argument data, if any
540 int blk_trace_ioctl(struct block_device
*bdev
, unsigned cmd
, char __user
*arg
)
542 struct request_queue
*q
;
544 char b
[BDEVNAME_SIZE
];
546 q
= bdev_get_queue(bdev
);
550 mutex_lock(&bdev
->bd_mutex
);
555 ret
= blk_trace_setup(q
, b
, bdev
->bd_dev
, arg
);
560 ret
= blk_trace_startstop(q
, start
);
562 case BLKTRACETEARDOWN
:
563 ret
= blk_trace_remove(q
);
570 mutex_unlock(&bdev
->bd_mutex
);
575 * blk_trace_shutdown: - stop and cleanup trace structures
576 * @q: the request queue associated with the device
579 void blk_trace_shutdown(struct request_queue
*q
)
582 blk_trace_startstop(q
, 0);