2 * This program is free software; you can redistribute it and/or modify
3 * it under the terms of the GNU General Public License as published by
4 * the Free Software Foundation; either version 2 of the License, or
5 * (at your option) any later version.
7 * This program is distributed in the hope that it will be useful,
8 * but WITHOUT ANY WARRANTY; without even the implied warranty of
9 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 * GNU General Public License for more details.
12 * You should have received a copy of the GNU General Public License
13 * along with this program; if not, write to the Free Software
14 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
16 * Copyright (C) IBM Corporation, 2005
17 * Jeff Muizelaar, 2006, 2007
18 * Pekka Paalanen, 2008 <pq@iki.fi>
20 * Derived from the read-mod example from relay-examples by Tom Zanussi.
22 #include <linux/module.h>
23 #include <linux/relay.h>
24 #include <linux/debugfs.h>
25 #include <linux/proc_fs.h>
27 #include <linux/version.h>
28 #include <linux/kallsyms.h>
29 #include <asm/pgtable.h>
30 #include <linux/mmiotrace.h>
31 #include <asm/e820.h> /* for ISA_START_ADDRESS */
32 #include <asm/atomic.h>
37 /* This app's relay channel files will appear in /debug/mmio-trace */
38 #define APP_DIR "mmio-trace"
39 /* the marker injection file in /proc */
40 #define MARKER_FILE "mmio-marker"
42 #define MODULE_NAME "mmiotrace"
47 enum reason_type type
;
51 /* Accessed per-cpu. */
52 static struct trap_reason pf_reason
[NR_CPUS
];
53 static struct mm_io_header_rw cpu_trace
[NR_CPUS
];
55 /* Access to this is not per-cpu. */
56 static atomic_t dropped
[NR_CPUS
];
58 static struct file_operations mmio_fops
= {
62 static const size_t subbuf_size
= 256*1024;
63 static struct rchan
*chan
;
64 static struct dentry
*dir
;
65 static struct proc_dir_entry
*proc_marker_file
;
67 /* module parameters */
68 static unsigned int n_subbufs
= 32*4;
69 static unsigned long filter_offset
;
70 static int nommiotrace
;
74 module_param(n_subbufs
, uint
, 0);
75 module_param(filter_offset
, ulong
, 0);
76 module_param(nommiotrace
, bool, 0);
77 module_param(ISA_trace
, bool, 0);
78 module_param(trace_pc
, bool, 0);
80 MODULE_PARM_DESC(n_subbufs
, "Number of 256kB buffers, default 128.");
81 MODULE_PARM_DESC(filter_offset
, "Start address of traced mappings.");
82 MODULE_PARM_DESC(nommiotrace
, "Disable actual MMIO tracing.");
83 MODULE_PARM_DESC(ISA_trace
, "Do not exclude the low ISA range.");
84 MODULE_PARM_DESC(trace_pc
, "Record address of faulting instructions.");
86 static void record_timestamp(struct mm_io_header
*header
)
91 header
->sec
= now
.tv_sec
;
92 header
->nsec
= now
.tv_nsec
;
96 * Write callback for the /proc entry:
97 * Read a marker and write it to the mmio trace log
99 static int write_marker(struct file
*file
, const char __user
*buffer
,
100 unsigned long count
, void *data
)
103 struct mm_io_header
*headp
;
104 int len
= (count
> 65535) ? 65535 : count
;
106 event
= kzalloc(sizeof(*headp
) + len
, GFP_KERNEL
);
110 headp
= (struct mm_io_header
*)event
;
111 headp
->type
= MMIO_MAGIC
| (MMIO_MARKER
<< MMIO_OPCODE_SHIFT
);
112 headp
->data_len
= len
;
113 record_timestamp(headp
);
115 if (copy_from_user(event
+ sizeof(*headp
), buffer
, len
)) {
120 relay_write(chan
, event
, sizeof(*headp
) + len
);
125 static void print_pte(unsigned long address
)
128 pte_t
*pte
= lookup_address(address
, &level
);
131 printk(KERN_ERR
"Error in %s: no pte for page 0x%08lx\n",
132 __FUNCTION__
, address
);
136 if (level
== PG_LEVEL_2M
) {
137 printk(KERN_EMERG MODULE_NAME
": 4MB pages are not "
138 "currently supported: %lx\n",
142 printk(KERN_DEBUG MODULE_NAME
": pte for 0x%lx: 0x%lx 0x%lx\n",
143 address
, pte_val(*pte
),
144 pte_val(*pte
) & _PAGE_PRESENT
);
148 * For some reason the pre/post pairs have been called in an
149 * unmatched order. Report and die.
151 static void die_kmmio_nesting_error(struct pt_regs
*regs
, unsigned long addr
)
153 const unsigned long cpu
= smp_processor_id();
154 printk(KERN_EMERG MODULE_NAME
": unexpected fault for address: %lx, "
155 "last fault for address: %lx\n",
156 addr
, pf_reason
[cpu
].addr
);
159 print_symbol(KERN_EMERG
"faulting EIP is at %s\n", regs
->ip
);
160 print_symbol(KERN_EMERG
"last faulting EIP was at %s\n",
163 "eax: %08lx ebx: %08lx ecx: %08lx edx: %08lx\n",
164 regs
->ax
, regs
->bx
, regs
->cx
, regs
->dx
);
166 "esi: %08lx edi: %08lx ebp: %08lx esp: %08lx\n",
167 regs
->si
, regs
->di
, regs
->bp
, regs
->sp
);
169 print_symbol(KERN_EMERG
"faulting RIP is at %s\n", regs
->ip
);
170 print_symbol(KERN_EMERG
"last faulting RIP was at %s\n",
172 printk(KERN_EMERG
"rax: %016lx rcx: %016lx rdx: %016lx\n",
173 regs
->ax
, regs
->cx
, regs
->dx
);
174 printk(KERN_EMERG
"rsi: %016lx rdi: %016lx "
175 "rbp: %016lx rsp: %016lx\n",
176 regs
->si
, regs
->di
, regs
->bp
, regs
->sp
);
181 static void pre(struct kmmio_probe
*p
, struct pt_regs
*regs
,
184 const unsigned long cpu
= smp_processor_id();
185 const unsigned long instptr
= instruction_pointer(regs
);
186 const enum reason_type type
= get_ins_type(instptr
);
188 /* it doesn't make sense to have more than one active trace per cpu */
189 if (pf_reason
[cpu
].active_traces
)
190 die_kmmio_nesting_error(regs
, addr
);
192 pf_reason
[cpu
].active_traces
++;
194 pf_reason
[cpu
].type
= type
;
195 pf_reason
[cpu
].addr
= addr
;
196 pf_reason
[cpu
].ip
= instptr
;
198 cpu_trace
[cpu
].header
.type
= MMIO_MAGIC
;
199 cpu_trace
[cpu
].header
.pid
= 0;
200 cpu_trace
[cpu
].header
.data_len
= sizeof(struct mm_io_rw
);
201 cpu_trace
[cpu
].rw
.address
= addr
;
204 * Only record the program counter when requested.
205 * It may taint clean-room reverse engineering.
208 cpu_trace
[cpu
].rw
.pc
= instptr
;
210 cpu_trace
[cpu
].rw
.pc
= 0;
212 record_timestamp(&cpu_trace
[cpu
].header
);
216 cpu_trace
[cpu
].header
.type
|=
217 (MMIO_READ
<< MMIO_OPCODE_SHIFT
) |
218 (get_ins_mem_width(instptr
) << MMIO_WIDTH_SHIFT
);
221 cpu_trace
[cpu
].header
.type
|=
222 (MMIO_WRITE
<< MMIO_OPCODE_SHIFT
) |
223 (get_ins_mem_width(instptr
) << MMIO_WIDTH_SHIFT
);
224 cpu_trace
[cpu
].rw
.value
= get_ins_reg_val(instptr
, regs
);
227 cpu_trace
[cpu
].header
.type
|=
228 (MMIO_WRITE
<< MMIO_OPCODE_SHIFT
) |
229 (get_ins_mem_width(instptr
) << MMIO_WIDTH_SHIFT
);
230 cpu_trace
[cpu
].rw
.value
= get_ins_imm_val(instptr
);
234 unsigned char *ip
= (unsigned char *)instptr
;
235 cpu_trace
[cpu
].header
.type
|=
236 (MMIO_UNKNOWN_OP
<< MMIO_OPCODE_SHIFT
);
237 cpu_trace
[cpu
].rw
.value
= (*ip
) << 16 |
244 static void post(struct kmmio_probe
*p
, unsigned long condition
,
245 struct pt_regs
*regs
)
247 const unsigned long cpu
= smp_processor_id();
249 /* this should always return the active_trace count to 0 */
250 pf_reason
[cpu
].active_traces
--;
251 if (pf_reason
[cpu
].active_traces
) {
252 printk(KERN_EMERG MODULE_NAME
": unexpected post handler");
256 switch (pf_reason
[cpu
].type
) {
258 cpu_trace
[cpu
].rw
.value
= get_ins_reg_val(pf_reason
[cpu
].ip
,
264 relay_write(chan
, &cpu_trace
[cpu
], sizeof(struct mm_io_header_rw
));
268 * subbuf_start() relay callback.
270 * Defined so that we know when events are dropped due to the buffer-full
273 static int subbuf_start_handler(struct rchan_buf
*buf
, void *subbuf
,
274 void *prev_subbuf
, size_t prev_padding
)
276 unsigned int cpu
= buf
->cpu
;
277 atomic_t
*drop
= &dropped
[cpu
];
279 if (relay_buf_full(buf
)) {
280 if (atomic_inc_return(drop
) == 1) {
281 printk(KERN_ERR MODULE_NAME
": cpu %d buffer full!\n",
285 } else if ((count
= atomic_read(drop
))) {
286 printk(KERN_ERR MODULE_NAME
287 ": cpu %d buffer no longer full, "
288 "missed %d events.\n",
290 atomic_sub(count
, drop
);
296 /* file_create() callback. Creates relay file in debugfs. */
297 static struct dentry
*create_buf_file_handler(const char *filename
,
298 struct dentry
*parent
,
300 struct rchan_buf
*buf
,
303 struct dentry
*buf_file
;
305 mmio_fops
.read
= relay_file_operations
.read
;
306 mmio_fops
.open
= relay_file_operations
.open
;
307 mmio_fops
.poll
= relay_file_operations
.poll
;
308 mmio_fops
.mmap
= relay_file_operations
.mmap
;
309 mmio_fops
.release
= relay_file_operations
.release
;
310 mmio_fops
.splice_read
= relay_file_operations
.splice_read
;
312 buf_file
= debugfs_create_file(filename
, mode
, parent
, buf
,
318 /* file_remove() default callback. Removes relay file in debugfs. */
319 static int remove_buf_file_handler(struct dentry
*dentry
)
321 debugfs_remove(dentry
);
325 static struct rchan_callbacks relay_callbacks
= {
326 .subbuf_start
= subbuf_start_handler
,
327 .create_buf_file
= create_buf_file_handler
,
328 .remove_buf_file
= remove_buf_file_handler
,
332 * create_channel - creates channel /debug/APP_DIR/cpuXXX
333 * Returns channel on success, NULL otherwise
335 static struct rchan
*create_channel(unsigned size
, unsigned n
)
337 return relay_open("cpu", dir
, size
, n
, &relay_callbacks
, NULL
);
340 /* destroy_channel - destroys channel /debug/APP_DIR/cpuXXX */
341 static void destroy_channel(void)
350 struct list_head list
;
351 struct kmmio_probe probe
;
353 static LIST_HEAD(trace_list
);
354 static DEFINE_SPINLOCK(trace_list_lock
);
356 static void do_ioremap_trace_core(unsigned long offset
, unsigned long size
,
359 struct remap_trace
*trace
= kmalloc(sizeof(*trace
), GFP_KERNEL
);
360 struct mm_io_header_map event
= {
363 (MMIO_PROBE
<< MMIO_OPCODE_SHIFT
),
367 .data_len
= sizeof(struct mm_io_map
)
371 .addr
= (unsigned long)addr
,
376 record_timestamp(&event
.header
);
378 *trace
= (struct remap_trace
) {
380 .addr
= (unsigned long)addr
,
383 .post_handler
= post
,
387 relay_write(chan
, &event
, sizeof(event
));
388 spin_lock(&trace_list_lock
);
389 list_add_tail(&trace
->list
, &trace_list
);
390 spin_unlock(&trace_list_lock
);
392 register_kmmio_probe(&trace
->probe
);
395 static void ioremap_trace_core(unsigned long offset
, unsigned long size
,
398 if ((filter_offset
) && (offset
!= filter_offset
))
401 /* Don't trace the low PCI/ISA area, it's always mapped.. */
402 if (!ISA_trace
&& (offset
< ISA_END_ADDRESS
) &&
403 (offset
+ size
> ISA_START_ADDRESS
)) {
404 printk(KERN_NOTICE MODULE_NAME
": Ignoring map of low "
405 "PCI/ISA area (0x%lx-0x%lx)\n",
406 offset
, offset
+ size
);
409 do_ioremap_trace_core(offset
, size
, addr
);
412 void __iomem
*ioremap_cache_trace(unsigned long offset
, unsigned long size
)
414 void __iomem
*p
= ioremap_cache(offset
, size
);
415 printk(KERN_DEBUG MODULE_NAME
": ioremap_cache(0x%lx, 0x%lx) = %p\n",
417 ioremap_trace_core(offset
, size
, p
);
420 EXPORT_SYMBOL(ioremap_cache_trace
);
422 void __iomem
*ioremap_nocache_trace(unsigned long offset
, unsigned long size
)
424 void __iomem
*p
= ioremap_nocache(offset
, size
);
425 printk(KERN_DEBUG MODULE_NAME
": ioremap_nocache(0x%lx, 0x%lx) = %p\n",
427 ioremap_trace_core(offset
, size
, p
);
430 EXPORT_SYMBOL(ioremap_nocache_trace
);
432 void iounmap_trace(volatile void __iomem
*addr
)
434 struct mm_io_header_map event
= {
437 (MMIO_UNPROBE
<< MMIO_OPCODE_SHIFT
),
441 .data_len
= sizeof(struct mm_io_map
)
445 .addr
= (unsigned long)addr
,
450 struct remap_trace
*trace
;
451 struct remap_trace
*tmp
;
452 printk(KERN_DEBUG MODULE_NAME
": Unmapping %p.\n", addr
);
453 record_timestamp(&event
.header
);
455 spin_lock(&trace_list_lock
);
456 list_for_each_entry_safe(trace
, tmp
, &trace_list
, list
) {
457 if ((unsigned long)addr
== trace
->probe
.addr
) {
459 unregister_kmmio_probe(&trace
->probe
);
460 list_del(&trace
->list
);
465 spin_unlock(&trace_list_lock
);
466 relay_write(chan
, &event
, sizeof(event
));
469 EXPORT_SYMBOL(iounmap_trace
);
471 static void clear_trace_list(void)
473 struct remap_trace
*trace
;
474 struct remap_trace
*tmp
;
476 spin_lock(&trace_list_lock
);
477 list_for_each_entry_safe(trace
, tmp
, &trace_list
, list
) {
478 printk(KERN_WARNING MODULE_NAME
": purging non-iounmapped "
479 "trace @0x%08lx, size 0x%lx.\n",
480 trace
->probe
.addr
, trace
->probe
.len
);
482 unregister_kmmio_probe(&trace
->probe
);
483 list_del(&trace
->list
);
487 spin_unlock(&trace_list_lock
);
490 static int __init
init(void)
495 dir
= debugfs_create_dir(APP_DIR
, NULL
);
497 printk(KERN_ERR MODULE_NAME
498 ": Couldn't create relay app directory.\n");
502 chan
= create_channel(subbuf_size
, n_subbufs
);
505 printk(KERN_ERR MODULE_NAME
506 ": relay app channel creation failed\n");
512 proc_marker_file
= create_proc_entry(MARKER_FILE
, 0, NULL
);
513 if (proc_marker_file
)
514 proc_marker_file
->write_proc
= write_marker
;
516 printk(KERN_DEBUG MODULE_NAME
": loaded.\n");
518 printk(KERN_DEBUG MODULE_NAME
": MMIO tracing disabled.\n");
520 printk(KERN_WARNING MODULE_NAME
521 ": Warning! low ISA range will be traced.\n");
525 static void __exit
cleanup(void)
527 printk(KERN_DEBUG MODULE_NAME
": unload...\n");
530 remove_proc_entry(MARKER_FILE
, NULL
);
537 module_exit(cleanup
);
538 MODULE_LICENSE("GPL");