2 * User-space Probes (UProbes)
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software
16 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
18 * Copyright (C) IBM Corporation, 2008-2012
22 * Copyright (C) 2011-2012 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
25 #include <linux/kernel.h>
26 #include <linux/highmem.h>
27 #include <linux/pagemap.h> /* read_mapping_page */
28 #include <linux/slab.h>
29 #include <linux/sched.h>
30 #include <linux/rmap.h> /* anon_vma_prepare */
31 #include <linux/mmu_notifier.h> /* set_pte_at_notify */
32 #include <linux/swap.h> /* try_to_free_swap */
33 #include <linux/ptrace.h> /* user_enable_single_step */
34 #include <linux/kdebug.h> /* notifier mechanism */
36 #include <linux/uprobes.h>
38 #define UINSNS_PER_PAGE (PAGE_SIZE/UPROBE_XOL_SLOT_BYTES)
39 #define MAX_UPROBE_XOL_SLOTS UINSNS_PER_PAGE
41 static struct srcu_struct uprobes_srcu
;
42 static struct rb_root uprobes_tree
= RB_ROOT
;
44 static DEFINE_SPINLOCK(uprobes_treelock
); /* serialize rbtree access */
46 #define UPROBES_HASH_SZ 13
48 /* serialize (un)register */
49 static struct mutex uprobes_mutex
[UPROBES_HASH_SZ
];
51 #define uprobes_hash(v) (&uprobes_mutex[((unsigned long)(v)) % UPROBES_HASH_SZ])
53 /* serialize uprobe->pending_list */
54 static struct mutex uprobes_mmap_mutex
[UPROBES_HASH_SZ
];
55 #define uprobes_mmap_hash(v) (&uprobes_mmap_mutex[((unsigned long)(v)) % UPROBES_HASH_SZ])
58 * uprobe_events allows us to skip the uprobe_mmap if there are no uprobe
59 * events active at this time. Probably a fine grained per inode count is
62 static atomic_t uprobe_events
= ATOMIC_INIT(0);
65 * Maintain a temporary per vma info that can be used to search if a vma
66 * has already been handled. This structure is introduced since extending
67 * vm_area_struct wasnt recommended.
70 struct list_head probe_list
;
76 struct rb_node rb_node
; /* node in the rb tree */
78 struct rw_semaphore consumer_rwsem
;
79 struct list_head pending_list
;
80 struct uprobe_consumer
*consumers
;
81 struct inode
*inode
; /* Also hold a ref to inode */
84 struct arch_uprobe arch
;
88 * valid_vma: Verify if the specified vma is an executable vma
89 * Relax restrictions while unregistering: vm_flags might have
90 * changed after breakpoint was inserted.
91 * - is_register: indicates if we are in register context.
92 * - Return 1 if the specified virtual address is in an
95 static bool valid_vma(struct vm_area_struct
*vma
, bool is_register
)
103 if ((vma
->vm_flags
& (VM_READ
|VM_WRITE
|VM_EXEC
|VM_SHARED
)) == (VM_READ
|VM_EXEC
))
109 static loff_t
vma_address(struct vm_area_struct
*vma
, loff_t offset
)
113 vaddr
= vma
->vm_start
+ offset
;
114 vaddr
-= vma
->vm_pgoff
<< PAGE_SHIFT
;
120 * __replace_page - replace page in vma by new page.
121 * based on replace_page in mm/ksm.c
123 * @vma: vma that holds the pte pointing to page
124 * @page: the cowed page we are replacing by kpage
125 * @kpage: the modified page we replace page by
127 * Returns 0 on success, -EFAULT on failure.
129 static int __replace_page(struct vm_area_struct
*vma
, struct page
*page
, struct page
*kpage
)
131 struct mm_struct
*mm
= vma
->vm_mm
;
140 addr
= page_address_in_vma(page
, vma
);
144 pgd
= pgd_offset(mm
, addr
);
145 if (!pgd_present(*pgd
))
148 pud
= pud_offset(pgd
, addr
);
149 if (!pud_present(*pud
))
152 pmd
= pmd_offset(pud
, addr
);
153 if (!pmd_present(*pmd
))
156 ptep
= pte_offset_map_lock(mm
, pmd
, addr
, &ptl
);
161 page_add_new_anon_rmap(kpage
, vma
, addr
);
163 if (!PageAnon(page
)) {
164 dec_mm_counter(mm
, MM_FILEPAGES
);
165 inc_mm_counter(mm
, MM_ANONPAGES
);
168 flush_cache_page(vma
, addr
, pte_pfn(*ptep
));
169 ptep_clear_flush(vma
, addr
, ptep
);
170 set_pte_at_notify(mm
, addr
, ptep
, mk_pte(kpage
, vma
->vm_page_prot
));
172 page_remove_rmap(page
);
173 if (!page_mapped(page
))
174 try_to_free_swap(page
);
176 pte_unmap_unlock(ptep
, ptl
);
184 * is_swbp_insn - check if instruction is breakpoint instruction.
185 * @insn: instruction to be checked.
186 * Default implementation of is_swbp_insn
187 * Returns true if @insn is a breakpoint instruction.
189 bool __weak
is_swbp_insn(uprobe_opcode_t
*insn
)
191 return *insn
== UPROBE_SWBP_INSN
;
196 * Expect the breakpoint instruction to be the smallest size instruction for
197 * the architecture. If an arch has variable length instruction and the
198 * breakpoint instruction is not of the smallest length instruction
199 * supported by that architecture then we need to modify read_opcode /
200 * write_opcode accordingly. This would never be a problem for archs that
201 * have fixed length instructions.
205 * write_opcode - write the opcode at a given virtual address.
206 * @auprobe: arch breakpointing information.
207 * @mm: the probed process address space.
208 * @vaddr: the virtual address to store the opcode.
209 * @opcode: opcode to be written at @vaddr.
211 * Called with mm->mmap_sem held (for read and with a reference to
214 * For mm @mm, write the opcode at @vaddr.
215 * Return 0 (success) or a negative errno.
217 static int write_opcode(struct arch_uprobe
*auprobe
, struct mm_struct
*mm
,
218 unsigned long vaddr
, uprobe_opcode_t opcode
)
220 struct page
*old_page
, *new_page
;
221 struct address_space
*mapping
;
222 void *vaddr_old
, *vaddr_new
;
223 struct vm_area_struct
*vma
;
224 struct uprobe
*uprobe
;
228 /* Read the page with vaddr into memory */
229 ret
= get_user_pages(NULL
, mm
, vaddr
, 1, 0, 0, &old_page
, &vma
);
236 * We are interested in text pages only. Our pages of interest
237 * should be mapped for read and execute only. We desist from
238 * adding probes in write mapped pages since the breakpoints
239 * might end up in the file copy.
241 if (!valid_vma(vma
, is_swbp_insn(&opcode
)))
244 uprobe
= container_of(auprobe
, struct uprobe
, arch
);
245 mapping
= uprobe
->inode
->i_mapping
;
246 if (mapping
!= vma
->vm_file
->f_mapping
)
249 addr
= vma_address(vma
, uprobe
->offset
);
250 if (vaddr
!= (unsigned long)addr
)
254 new_page
= alloc_page_vma(GFP_HIGHUSER_MOVABLE
, vma
, vaddr
);
258 __SetPageUptodate(new_page
);
261 * lock page will serialize against do_wp_page()'s
262 * PageAnon() handling
265 /* copy the page now that we've got it stable */
266 vaddr_old
= kmap_atomic(old_page
);
267 vaddr_new
= kmap_atomic(new_page
);
269 memcpy(vaddr_new
, vaddr_old
, PAGE_SIZE
);
271 /* poke the new insn in, ASSUMES we don't cross page boundary */
273 BUG_ON(vaddr
+ UPROBE_SWBP_INSN_SIZE
> PAGE_SIZE
);
274 memcpy(vaddr_new
+ vaddr
, &opcode
, UPROBE_SWBP_INSN_SIZE
);
276 kunmap_atomic(vaddr_new
);
277 kunmap_atomic(vaddr_old
);
279 ret
= anon_vma_prepare(vma
);
284 ret
= __replace_page(vma
, old_page
, new_page
);
285 unlock_page(new_page
);
288 unlock_page(old_page
);
289 page_cache_release(new_page
);
298 * read_opcode - read the opcode at a given virtual address.
299 * @mm: the probed process address space.
300 * @vaddr: the virtual address to read the opcode.
301 * @opcode: location to store the read opcode.
303 * Called with mm->mmap_sem held (for read and with a reference to
306 * For mm @mm, read the opcode at @vaddr and store it in @opcode.
307 * Return 0 (success) or a negative errno.
309 static int read_opcode(struct mm_struct
*mm
, unsigned long vaddr
, uprobe_opcode_t
*opcode
)
315 ret
= get_user_pages(NULL
, mm
, vaddr
, 1, 0, 1, &page
, NULL
);
320 vaddr_new
= kmap_atomic(page
);
322 memcpy(opcode
, vaddr_new
+ vaddr
, UPROBE_SWBP_INSN_SIZE
);
323 kunmap_atomic(vaddr_new
);
331 static int is_swbp_at_addr(struct mm_struct
*mm
, unsigned long vaddr
)
333 uprobe_opcode_t opcode
;
336 if (current
->mm
== mm
) {
338 result
= __copy_from_user_inatomic(&opcode
, (void __user
*)vaddr
,
342 if (likely(result
== 0))
346 result
= read_opcode(mm
, vaddr
, &opcode
);
350 if (is_swbp_insn(&opcode
))
357 * set_swbp - store breakpoint at a given address.
358 * @auprobe: arch specific probepoint information.
359 * @mm: the probed process address space.
360 * @vaddr: the virtual address to insert the opcode.
362 * For mm @mm, store the breakpoint instruction at @vaddr.
363 * Return 0 (success) or a negative errno.
365 int __weak
set_swbp(struct arch_uprobe
*auprobe
, struct mm_struct
*mm
, unsigned long vaddr
)
369 result
= is_swbp_at_addr(mm
, vaddr
);
376 return write_opcode(auprobe
, mm
, vaddr
, UPROBE_SWBP_INSN
);
380 * set_orig_insn - Restore the original instruction.
381 * @mm: the probed process address space.
382 * @auprobe: arch specific probepoint information.
383 * @vaddr: the virtual address to insert the opcode.
384 * @verify: if true, verify existance of breakpoint instruction.
386 * For mm @mm, restore the original opcode (opcode) at @vaddr.
387 * Return 0 (success) or a negative errno.
390 set_orig_insn(struct arch_uprobe
*auprobe
, struct mm_struct
*mm
, unsigned long vaddr
, bool verify
)
395 result
= is_swbp_at_addr(mm
, vaddr
);
402 return write_opcode(auprobe
, mm
, vaddr
, *(uprobe_opcode_t
*)auprobe
->insn
);
405 static int match_uprobe(struct uprobe
*l
, struct uprobe
*r
)
407 if (l
->inode
< r
->inode
)
410 if (l
->inode
> r
->inode
)
413 if (l
->offset
< r
->offset
)
416 if (l
->offset
> r
->offset
)
422 static struct uprobe
*__find_uprobe(struct inode
*inode
, loff_t offset
)
424 struct uprobe u
= { .inode
= inode
, .offset
= offset
};
425 struct rb_node
*n
= uprobes_tree
.rb_node
;
426 struct uprobe
*uprobe
;
430 uprobe
= rb_entry(n
, struct uprobe
, rb_node
);
431 match
= match_uprobe(&u
, uprobe
);
433 atomic_inc(&uprobe
->ref
);
446 * Find a uprobe corresponding to a given inode:offset
447 * Acquires uprobes_treelock
449 static struct uprobe
*find_uprobe(struct inode
*inode
, loff_t offset
)
451 struct uprobe
*uprobe
;
454 spin_lock_irqsave(&uprobes_treelock
, flags
);
455 uprobe
= __find_uprobe(inode
, offset
);
456 spin_unlock_irqrestore(&uprobes_treelock
, flags
);
461 static struct uprobe
*__insert_uprobe(struct uprobe
*uprobe
)
463 struct rb_node
**p
= &uprobes_tree
.rb_node
;
464 struct rb_node
*parent
= NULL
;
470 u
= rb_entry(parent
, struct uprobe
, rb_node
);
471 match
= match_uprobe(uprobe
, u
);
478 p
= &parent
->rb_left
;
480 p
= &parent
->rb_right
;
485 rb_link_node(&uprobe
->rb_node
, parent
, p
);
486 rb_insert_color(&uprobe
->rb_node
, &uprobes_tree
);
487 /* get access + creation ref */
488 atomic_set(&uprobe
->ref
, 2);
494 * Acquire uprobes_treelock.
495 * Matching uprobe already exists in rbtree;
496 * increment (access refcount) and return the matching uprobe.
498 * No matching uprobe; insert the uprobe in rb_tree;
499 * get a double refcount (access + creation) and return NULL.
501 static struct uprobe
*insert_uprobe(struct uprobe
*uprobe
)
506 spin_lock_irqsave(&uprobes_treelock
, flags
);
507 u
= __insert_uprobe(uprobe
);
508 spin_unlock_irqrestore(&uprobes_treelock
, flags
);
510 /* For now assume that the instruction need not be single-stepped */
511 uprobe
->flags
|= UPROBE_SKIP_SSTEP
;
516 static void put_uprobe(struct uprobe
*uprobe
)
518 if (atomic_dec_and_test(&uprobe
->ref
))
522 static struct uprobe
*alloc_uprobe(struct inode
*inode
, loff_t offset
)
524 struct uprobe
*uprobe
, *cur_uprobe
;
526 uprobe
= kzalloc(sizeof(struct uprobe
), GFP_KERNEL
);
530 uprobe
->inode
= igrab(inode
);
531 uprobe
->offset
= offset
;
532 init_rwsem(&uprobe
->consumer_rwsem
);
533 INIT_LIST_HEAD(&uprobe
->pending_list
);
535 /* add to uprobes_tree, sorted on inode:offset */
536 cur_uprobe
= insert_uprobe(uprobe
);
538 /* a uprobe exists for this inode:offset combination */
544 atomic_inc(&uprobe_events
);
550 static void handler_chain(struct uprobe
*uprobe
, struct pt_regs
*regs
)
552 struct uprobe_consumer
*uc
;
554 if (!(uprobe
->flags
& UPROBE_RUN_HANDLER
))
557 down_read(&uprobe
->consumer_rwsem
);
558 for (uc
= uprobe
->consumers
; uc
; uc
= uc
->next
) {
559 if (!uc
->filter
|| uc
->filter(uc
, current
))
560 uc
->handler(uc
, regs
);
562 up_read(&uprobe
->consumer_rwsem
);
565 /* Returns the previous consumer */
566 static struct uprobe_consumer
*
567 consumer_add(struct uprobe
*uprobe
, struct uprobe_consumer
*uc
)
569 down_write(&uprobe
->consumer_rwsem
);
570 uc
->next
= uprobe
->consumers
;
571 uprobe
->consumers
= uc
;
572 up_write(&uprobe
->consumer_rwsem
);
578 * For uprobe @uprobe, delete the consumer @uc.
579 * Return true if the @uc is deleted successfully
582 static bool consumer_del(struct uprobe
*uprobe
, struct uprobe_consumer
*uc
)
584 struct uprobe_consumer
**con
;
587 down_write(&uprobe
->consumer_rwsem
);
588 for (con
= &uprobe
->consumers
; *con
; con
= &(*con
)->next
) {
595 up_write(&uprobe
->consumer_rwsem
);
601 __copy_insn(struct address_space
*mapping
, struct vm_area_struct
*vma
, char *insn
,
602 unsigned long nbytes
, unsigned long offset
)
604 struct file
*filp
= vma
->vm_file
;
613 idx
= (unsigned long)(offset
>> PAGE_CACHE_SHIFT
);
614 off1
= offset
&= ~PAGE_MASK
;
617 * Ensure that the page that has the original instruction is
618 * populated and in page-cache.
620 page
= read_mapping_page(mapping
, idx
, filp
);
622 return PTR_ERR(page
);
624 vaddr
= kmap_atomic(page
);
625 memcpy(insn
, vaddr
+ off1
, nbytes
);
626 kunmap_atomic(vaddr
);
627 page_cache_release(page
);
633 copy_insn(struct uprobe
*uprobe
, struct vm_area_struct
*vma
, unsigned long addr
)
635 struct address_space
*mapping
;
636 unsigned long nbytes
;
640 nbytes
= PAGE_SIZE
- addr
;
641 mapping
= uprobe
->inode
->i_mapping
;
643 /* Instruction at end of binary; copy only available bytes */
644 if (uprobe
->offset
+ MAX_UINSN_BYTES
> uprobe
->inode
->i_size
)
645 bytes
= uprobe
->inode
->i_size
- uprobe
->offset
;
647 bytes
= MAX_UINSN_BYTES
;
649 /* Instruction at the page-boundary; copy bytes in second page */
650 if (nbytes
< bytes
) {
651 if (__copy_insn(mapping
, vma
, uprobe
->arch
.insn
+ nbytes
,
652 bytes
- nbytes
, uprobe
->offset
+ nbytes
))
657 return __copy_insn(mapping
, vma
, uprobe
->arch
.insn
, bytes
, uprobe
->offset
);
661 * How mm->uprobes_state.count gets updated
662 * uprobe_mmap() increments the count if
663 * - it successfully adds a breakpoint.
664 * - it cannot add a breakpoint, but sees that there is a underlying
665 * breakpoint (via a is_swbp_at_addr()).
667 * uprobe_munmap() decrements the count if
668 * - it sees a underlying breakpoint, (via is_swbp_at_addr)
669 * (Subsequent uprobe_unregister wouldnt find the breakpoint
670 * unless a uprobe_mmap kicks in, since the old vma would be
671 * dropped just after uprobe_munmap.)
673 * uprobe_register increments the count if:
674 * - it successfully adds a breakpoint.
676 * uprobe_unregister decrements the count if:
677 * - it sees a underlying breakpoint and removes successfully.
678 * (via is_swbp_at_addr)
679 * (Subsequent uprobe_munmap wouldnt find the breakpoint
680 * since there is no underlying breakpoint after the
681 * breakpoint removal.)
684 install_breakpoint(struct uprobe
*uprobe
, struct mm_struct
*mm
,
685 struct vm_area_struct
*vma
, loff_t vaddr
)
691 * If probe is being deleted, unregister thread could be done with
692 * the vma-rmap-walk through. Adding a probe now can be fatal since
693 * nobody will be able to cleanup. Also we could be from fork or
694 * mremap path, where the probe might have already been inserted.
695 * Hence behave as if probe already existed.
697 if (!uprobe
->consumers
)
700 addr
= (unsigned long)vaddr
;
702 if (!(uprobe
->flags
& UPROBE_COPY_INSN
)) {
703 ret
= copy_insn(uprobe
, vma
, addr
);
707 if (is_swbp_insn((uprobe_opcode_t
*)uprobe
->arch
.insn
))
710 ret
= arch_uprobe_analyze_insn(&uprobe
->arch
, mm
);
714 uprobe
->flags
|= UPROBE_COPY_INSN
;
718 * Ideally, should be updating the probe count after the breakpoint
719 * has been successfully inserted. However a thread could hit the
720 * breakpoint we just inserted even before the probe count is
721 * incremented. If this is the first breakpoint placed, breakpoint
722 * notifier might ignore uprobes and pass the trap to the thread.
723 * Hence increment before and decrement on failure.
725 atomic_inc(&mm
->uprobes_state
.count
);
726 ret
= set_swbp(&uprobe
->arch
, mm
, addr
);
728 atomic_dec(&mm
->uprobes_state
.count
);
734 remove_breakpoint(struct uprobe
*uprobe
, struct mm_struct
*mm
, loff_t vaddr
)
736 if (!set_orig_insn(&uprobe
->arch
, mm
, (unsigned long)vaddr
, true))
737 atomic_dec(&mm
->uprobes_state
.count
);
741 * There could be threads that have hit the breakpoint and are entering the
742 * notifier code and trying to acquire the uprobes_treelock. The thread
743 * calling delete_uprobe() that is removing the uprobe from the rb_tree can
744 * race with these threads and might acquire the uprobes_treelock compared
745 * to some of the breakpoint hit threads. In such a case, the breakpoint
746 * hit threads will not find the uprobe. The current unregistering thread
747 * waits till all other threads have hit a breakpoint, to acquire the
748 * uprobes_treelock before the uprobe is removed from the rbtree.
750 static void delete_uprobe(struct uprobe
*uprobe
)
754 synchronize_srcu(&uprobes_srcu
);
755 spin_lock_irqsave(&uprobes_treelock
, flags
);
756 rb_erase(&uprobe
->rb_node
, &uprobes_tree
);
757 spin_unlock_irqrestore(&uprobes_treelock
, flags
);
760 atomic_dec(&uprobe_events
);
763 static struct vma_info
*
764 __find_next_vma_info(struct address_space
*mapping
, struct list_head
*head
,
765 struct vma_info
*vi
, loff_t offset
, bool is_register
)
767 struct prio_tree_iter iter
;
768 struct vm_area_struct
*vma
;
769 struct vma_info
*tmpvi
;
774 pgoff
= offset
>> PAGE_SHIFT
;
776 vma_prio_tree_foreach(vma
, &iter
, &mapping
->i_mmap
, pgoff
, pgoff
) {
777 if (!valid_vma(vma
, is_register
))
781 vaddr
= vma_address(vma
, offset
);
783 list_for_each_entry(tmpvi
, head
, probe_list
) {
784 if (tmpvi
->mm
== vma
->vm_mm
&& tmpvi
->vaddr
== vaddr
) {
791 * Another vma needs a probe to be installed. However skip
792 * installing the probe if the vma is about to be unlinked.
794 if (!existing_vma
&& atomic_inc_not_zero(&vma
->vm_mm
->mm_users
)) {
797 list_add(&vi
->probe_list
, head
);
807 * Iterate in the rmap prio tree and find a vma where a probe has not
810 static struct vma_info
*
811 find_next_vma_info(struct address_space
*mapping
, struct list_head
*head
,
812 loff_t offset
, bool is_register
)
814 struct vma_info
*vi
, *retvi
;
816 vi
= kzalloc(sizeof(struct vma_info
), GFP_KERNEL
);
818 return ERR_PTR(-ENOMEM
);
820 mutex_lock(&mapping
->i_mmap_mutex
);
821 retvi
= __find_next_vma_info(mapping
, head
, vi
, offset
, is_register
);
822 mutex_unlock(&mapping
->i_mmap_mutex
);
830 static int register_for_each_vma(struct uprobe
*uprobe
, bool is_register
)
832 struct list_head try_list
;
833 struct vm_area_struct
*vma
;
834 struct address_space
*mapping
;
835 struct vma_info
*vi
, *tmpvi
;
836 struct mm_struct
*mm
;
840 mapping
= uprobe
->inode
->i_mapping
;
841 INIT_LIST_HEAD(&try_list
);
846 vi
= find_next_vma_info(mapping
, &try_list
, uprobe
->offset
, is_register
);
856 down_write(&mm
->mmap_sem
);
857 vma
= find_vma(mm
, (unsigned long)vi
->vaddr
);
858 if (!vma
|| !valid_vma(vma
, is_register
)) {
859 list_del(&vi
->probe_list
);
861 up_write(&mm
->mmap_sem
);
865 vaddr
= vma_address(vma
, uprobe
->offset
);
866 if (vma
->vm_file
->f_mapping
->host
!= uprobe
->inode
||
867 vaddr
!= vi
->vaddr
) {
868 list_del(&vi
->probe_list
);
870 up_write(&mm
->mmap_sem
);
876 ret
= install_breakpoint(uprobe
, mm
, vma
, vi
->vaddr
);
878 remove_breakpoint(uprobe
, mm
, vi
->vaddr
);
880 up_write(&mm
->mmap_sem
);
883 if (ret
&& ret
== -EEXIST
)
890 list_for_each_entry_safe(vi
, tmpvi
, &try_list
, probe_list
) {
891 list_del(&vi
->probe_list
);
898 static int __uprobe_register(struct uprobe
*uprobe
)
900 return register_for_each_vma(uprobe
, true);
903 static void __uprobe_unregister(struct uprobe
*uprobe
)
905 if (!register_for_each_vma(uprobe
, false))
906 delete_uprobe(uprobe
);
908 /* TODO : cant unregister? schedule a worker thread */
912 * uprobe_register - register a probe
913 * @inode: the file in which the probe has to be placed.
914 * @offset: offset from the start of the file.
915 * @uc: information on howto handle the probe..
917 * Apart from the access refcount, uprobe_register() takes a creation
918 * refcount (thro alloc_uprobe) if and only if this @uprobe is getting
919 * inserted into the rbtree (i.e first consumer for a @inode:@offset
920 * tuple). Creation refcount stops uprobe_unregister from freeing the
921 * @uprobe even before the register operation is complete. Creation
922 * refcount is released when the last @uc for the @uprobe
925 * Return errno if it cannot successully install probes
926 * else return 0 (success)
928 int uprobe_register(struct inode
*inode
, loff_t offset
, struct uprobe_consumer
*uc
)
930 struct uprobe
*uprobe
;
933 if (!inode
|| !uc
|| uc
->next
)
936 if (offset
> i_size_read(inode
))
940 mutex_lock(uprobes_hash(inode
));
941 uprobe
= alloc_uprobe(inode
, offset
);
943 if (uprobe
&& !consumer_add(uprobe
, uc
)) {
944 ret
= __uprobe_register(uprobe
);
946 uprobe
->consumers
= NULL
;
947 __uprobe_unregister(uprobe
);
949 uprobe
->flags
|= UPROBE_RUN_HANDLER
;
953 mutex_unlock(uprobes_hash(inode
));
960 * uprobe_unregister - unregister a already registered probe.
961 * @inode: the file in which the probe has to be removed.
962 * @offset: offset from the start of the file.
963 * @uc: identify which probe if multiple probes are colocated.
965 void uprobe_unregister(struct inode
*inode
, loff_t offset
, struct uprobe_consumer
*uc
)
967 struct uprobe
*uprobe
;
972 uprobe
= find_uprobe(inode
, offset
);
976 mutex_lock(uprobes_hash(inode
));
978 if (consumer_del(uprobe
, uc
)) {
979 if (!uprobe
->consumers
) {
980 __uprobe_unregister(uprobe
);
981 uprobe
->flags
&= ~UPROBE_RUN_HANDLER
;
985 mutex_unlock(uprobes_hash(inode
));
991 * Of all the nodes that correspond to the given inode, return the node
992 * with the least offset.
994 static struct rb_node
*find_least_offset_node(struct inode
*inode
)
996 struct uprobe u
= { .inode
= inode
, .offset
= 0};
997 struct rb_node
*n
= uprobes_tree
.rb_node
;
998 struct rb_node
*close_node
= NULL
;
999 struct uprobe
*uprobe
;
1003 uprobe
= rb_entry(n
, struct uprobe
, rb_node
);
1004 match
= match_uprobe(&u
, uprobe
);
1006 if (uprobe
->inode
== inode
)
1022 * For a given inode, build a list of probes that need to be inserted.
1024 static void build_probe_list(struct inode
*inode
, struct list_head
*head
)
1026 struct uprobe
*uprobe
;
1027 unsigned long flags
;
1030 spin_lock_irqsave(&uprobes_treelock
, flags
);
1032 n
= find_least_offset_node(inode
);
1034 for (; n
; n
= rb_next(n
)) {
1035 uprobe
= rb_entry(n
, struct uprobe
, rb_node
);
1036 if (uprobe
->inode
!= inode
)
1039 list_add(&uprobe
->pending_list
, head
);
1040 atomic_inc(&uprobe
->ref
);
1043 spin_unlock_irqrestore(&uprobes_treelock
, flags
);
1047 * Called from mmap_region.
1048 * called with mm->mmap_sem acquired.
1050 * Return -ve no if we fail to insert probes and we cannot
1052 * Return 0 otherwise. i.e:
1054 * - successful insertion of probes
1055 * - (or) no possible probes to be inserted.
1056 * - (or) insertion of probes failed but we can bail-out.
1058 int uprobe_mmap(struct vm_area_struct
*vma
)
1060 struct list_head tmp_list
;
1061 struct uprobe
*uprobe
, *u
;
1062 struct inode
*inode
;
1065 if (!atomic_read(&uprobe_events
) || !valid_vma(vma
, true))
1068 inode
= vma
->vm_file
->f_mapping
->host
;
1072 INIT_LIST_HEAD(&tmp_list
);
1073 mutex_lock(uprobes_mmap_hash(inode
));
1074 build_probe_list(inode
, &tmp_list
);
1079 list_for_each_entry_safe(uprobe
, u
, &tmp_list
, pending_list
) {
1082 list_del(&uprobe
->pending_list
);
1084 vaddr
= vma_address(vma
, uprobe
->offset
);
1086 if (vaddr
< vma
->vm_start
|| vaddr
>= vma
->vm_end
) {
1091 ret
= install_breakpoint(uprobe
, vma
->vm_mm
, vma
, vaddr
);
1093 /* Ignore double add: */
1094 if (ret
== -EEXIST
) {
1097 if (!is_swbp_at_addr(vma
->vm_mm
, vaddr
))
1101 * Unable to insert a breakpoint, but
1102 * breakpoint lies underneath. Increment the
1105 atomic_inc(&vma
->vm_mm
->uprobes_state
.count
);
1114 mutex_unlock(uprobes_mmap_hash(inode
));
1117 atomic_sub(count
, &vma
->vm_mm
->uprobes_state
.count
);
1123 * Called in context of a munmap of a vma.
1125 void uprobe_munmap(struct vm_area_struct
*vma
, unsigned long start
, unsigned long end
)
1127 struct list_head tmp_list
;
1128 struct uprobe
*uprobe
, *u
;
1129 struct inode
*inode
;
1131 if (!atomic_read(&uprobe_events
) || !valid_vma(vma
, false))
1134 if (!atomic_read(&vma
->vm_mm
->uprobes_state
.count
))
1137 inode
= vma
->vm_file
->f_mapping
->host
;
1141 INIT_LIST_HEAD(&tmp_list
);
1142 mutex_lock(uprobes_mmap_hash(inode
));
1143 build_probe_list(inode
, &tmp_list
);
1145 list_for_each_entry_safe(uprobe
, u
, &tmp_list
, pending_list
) {
1148 list_del(&uprobe
->pending_list
);
1149 vaddr
= vma_address(vma
, uprobe
->offset
);
1151 if (vaddr
>= start
&& vaddr
< end
) {
1153 * An unregister could have removed the probe before
1154 * unmap. So check before we decrement the count.
1156 if (is_swbp_at_addr(vma
->vm_mm
, vaddr
) == 1)
1157 atomic_dec(&vma
->vm_mm
->uprobes_state
.count
);
1161 mutex_unlock(uprobes_mmap_hash(inode
));
1164 /* Slot allocation for XOL */
1165 static int xol_add_vma(struct xol_area
*area
)
1167 struct mm_struct
*mm
;
1170 area
->page
= alloc_page(GFP_HIGHUSER
);
1177 down_write(&mm
->mmap_sem
);
1178 if (mm
->uprobes_state
.xol_area
)
1183 /* Try to map as high as possible, this is only a hint. */
1184 area
->vaddr
= get_unmapped_area(NULL
, TASK_SIZE
- PAGE_SIZE
, PAGE_SIZE
, 0, 0);
1185 if (area
->vaddr
& ~PAGE_MASK
) {
1190 ret
= install_special_mapping(mm
, area
->vaddr
, PAGE_SIZE
,
1191 VM_EXEC
|VM_MAYEXEC
|VM_DONTCOPY
|VM_IO
, &area
->page
);
1195 smp_wmb(); /* pairs with get_xol_area() */
1196 mm
->uprobes_state
.xol_area
= area
;
1200 up_write(&mm
->mmap_sem
);
1202 __free_page(area
->page
);
1207 static struct xol_area
*get_xol_area(struct mm_struct
*mm
)
1209 struct xol_area
*area
;
1211 area
= mm
->uprobes_state
.xol_area
;
1212 smp_read_barrier_depends(); /* pairs with wmb in xol_add_vma() */
1218 * xol_alloc_area - Allocate process's xol_area.
1219 * This area will be used for storing instructions for execution out of
1222 * Returns the allocated area or NULL.
1224 static struct xol_area
*xol_alloc_area(void)
1226 struct xol_area
*area
;
1228 area
= kzalloc(sizeof(*area
), GFP_KERNEL
);
1229 if (unlikely(!area
))
1232 area
->bitmap
= kzalloc(BITS_TO_LONGS(UINSNS_PER_PAGE
) * sizeof(long), GFP_KERNEL
);
1237 init_waitqueue_head(&area
->wq
);
1238 if (!xol_add_vma(area
))
1242 kfree(area
->bitmap
);
1245 return get_xol_area(current
->mm
);
1249 * uprobe_clear_state - Free the area allocated for slots.
1251 void uprobe_clear_state(struct mm_struct
*mm
)
1253 struct xol_area
*area
= mm
->uprobes_state
.xol_area
;
1258 put_page(area
->page
);
1259 kfree(area
->bitmap
);
1264 * uprobe_reset_state - Free the area allocated for slots.
1266 void uprobe_reset_state(struct mm_struct
*mm
)
1268 mm
->uprobes_state
.xol_area
= NULL
;
1269 atomic_set(&mm
->uprobes_state
.count
, 0);
1273 * - search for a free slot.
1275 static unsigned long xol_take_insn_slot(struct xol_area
*area
)
1277 unsigned long slot_addr
;
1281 slot_nr
= find_first_zero_bit(area
->bitmap
, UINSNS_PER_PAGE
);
1282 if (slot_nr
< UINSNS_PER_PAGE
) {
1283 if (!test_and_set_bit(slot_nr
, area
->bitmap
))
1286 slot_nr
= UINSNS_PER_PAGE
;
1289 wait_event(area
->wq
, (atomic_read(&area
->slot_count
) < UINSNS_PER_PAGE
));
1290 } while (slot_nr
>= UINSNS_PER_PAGE
);
1292 slot_addr
= area
->vaddr
+ (slot_nr
* UPROBE_XOL_SLOT_BYTES
);
1293 atomic_inc(&area
->slot_count
);
1299 * xol_get_insn_slot - If was not allocated a slot, then
1301 * Returns the allocated slot address or 0.
1303 static unsigned long xol_get_insn_slot(struct uprobe
*uprobe
, unsigned long slot_addr
)
1305 struct xol_area
*area
;
1306 unsigned long offset
;
1309 area
= get_xol_area(current
->mm
);
1311 area
= xol_alloc_area();
1315 current
->utask
->xol_vaddr
= xol_take_insn_slot(area
);
1318 * Initialize the slot if xol_vaddr points to valid
1321 if (unlikely(!current
->utask
->xol_vaddr
))
1324 current
->utask
->vaddr
= slot_addr
;
1325 offset
= current
->utask
->xol_vaddr
& ~PAGE_MASK
;
1326 vaddr
= kmap_atomic(area
->page
);
1327 memcpy(vaddr
+ offset
, uprobe
->arch
.insn
, MAX_UINSN_BYTES
);
1328 kunmap_atomic(vaddr
);
1330 return current
->utask
->xol_vaddr
;
1334 * xol_free_insn_slot - If slot was earlier allocated by
1335 * @xol_get_insn_slot(), make the slot available for
1336 * subsequent requests.
1338 static void xol_free_insn_slot(struct task_struct
*tsk
)
1340 struct xol_area
*area
;
1341 unsigned long vma_end
;
1342 unsigned long slot_addr
;
1344 if (!tsk
->mm
|| !tsk
->mm
->uprobes_state
.xol_area
|| !tsk
->utask
)
1347 slot_addr
= tsk
->utask
->xol_vaddr
;
1349 if (unlikely(!slot_addr
|| IS_ERR_VALUE(slot_addr
)))
1352 area
= tsk
->mm
->uprobes_state
.xol_area
;
1353 vma_end
= area
->vaddr
+ PAGE_SIZE
;
1354 if (area
->vaddr
<= slot_addr
&& slot_addr
< vma_end
) {
1355 unsigned long offset
;
1358 offset
= slot_addr
- area
->vaddr
;
1359 slot_nr
= offset
/ UPROBE_XOL_SLOT_BYTES
;
1360 if (slot_nr
>= UINSNS_PER_PAGE
)
1363 clear_bit(slot_nr
, area
->bitmap
);
1364 atomic_dec(&area
->slot_count
);
1365 if (waitqueue_active(&area
->wq
))
1368 tsk
->utask
->xol_vaddr
= 0;
1373 * uprobe_get_swbp_addr - compute address of swbp given post-swbp regs
1374 * @regs: Reflects the saved state of the task after it has hit a breakpoint
1376 * Return the address of the breakpoint instruction.
1378 unsigned long __weak
uprobe_get_swbp_addr(struct pt_regs
*regs
)
1380 return instruction_pointer(regs
) - UPROBE_SWBP_INSN_SIZE
;
1384 * Called with no locks held.
1385 * Called in context of a exiting or a exec-ing thread.
1387 void uprobe_free_utask(struct task_struct
*t
)
1389 struct uprobe_task
*utask
= t
->utask
;
1391 if (t
->uprobe_srcu_id
!= -1)
1392 srcu_read_unlock_raw(&uprobes_srcu
, t
->uprobe_srcu_id
);
1397 if (utask
->active_uprobe
)
1398 put_uprobe(utask
->active_uprobe
);
1400 xol_free_insn_slot(t
);
1406 * Called in context of a new clone/fork from copy_process.
1408 void uprobe_copy_process(struct task_struct
*t
)
1411 t
->uprobe_srcu_id
= -1;
1415 * Allocate a uprobe_task object for the task.
1416 * Called when the thread hits a breakpoint for the first time.
1419 * - pointer to new uprobe_task on success
1422 static struct uprobe_task
*add_utask(void)
1424 struct uprobe_task
*utask
;
1426 utask
= kzalloc(sizeof *utask
, GFP_KERNEL
);
1427 if (unlikely(!utask
))
1430 utask
->active_uprobe
= NULL
;
1431 current
->utask
= utask
;
1435 /* Prepare to single-step probed instruction out of line. */
1437 pre_ssout(struct uprobe
*uprobe
, struct pt_regs
*regs
, unsigned long vaddr
)
1439 if (xol_get_insn_slot(uprobe
, vaddr
) && !arch_uprobe_pre_xol(&uprobe
->arch
, regs
))
1446 * If we are singlestepping, then ensure this thread is not connected to
1447 * non-fatal signals until completion of singlestep. When xol insn itself
1448 * triggers the signal, restart the original insn even if the task is
1449 * already SIGKILL'ed (since coredump should report the correct ip). This
1450 * is even more important if the task has a handler for SIGSEGV/etc, The
1451 * _same_ instruction should be repeated again after return from the signal
1452 * handler, and SSTEP can never finish in this case.
1454 bool uprobe_deny_signal(void)
1456 struct task_struct
*t
= current
;
1457 struct uprobe_task
*utask
= t
->utask
;
1459 if (likely(!utask
|| !utask
->active_uprobe
))
1462 WARN_ON_ONCE(utask
->state
!= UTASK_SSTEP
);
1464 if (signal_pending(t
)) {
1465 spin_lock_irq(&t
->sighand
->siglock
);
1466 clear_tsk_thread_flag(t
, TIF_SIGPENDING
);
1467 spin_unlock_irq(&t
->sighand
->siglock
);
1469 if (__fatal_signal_pending(t
) || arch_uprobe_xol_was_trapped(t
)) {
1470 utask
->state
= UTASK_SSTEP_TRAPPED
;
1471 set_tsk_thread_flag(t
, TIF_UPROBE
);
1472 set_tsk_thread_flag(t
, TIF_NOTIFY_RESUME
);
1480 * Avoid singlestepping the original instruction if the original instruction
1481 * is a NOP or can be emulated.
1483 static bool can_skip_sstep(struct uprobe
*uprobe
, struct pt_regs
*regs
)
1485 if (arch_uprobe_skip_sstep(&uprobe
->arch
, regs
))
1488 uprobe
->flags
&= ~UPROBE_SKIP_SSTEP
;
1492 static struct uprobe
*find_active_uprobe(unsigned long bp_vaddr
, int *is_swbp
)
1494 struct mm_struct
*mm
= current
->mm
;
1495 struct uprobe
*uprobe
= NULL
;
1496 struct vm_area_struct
*vma
;
1498 down_read(&mm
->mmap_sem
);
1499 vma
= find_vma(mm
, bp_vaddr
);
1500 if (vma
&& vma
->vm_start
<= bp_vaddr
) {
1501 if (valid_vma(vma
, false)) {
1502 struct inode
*inode
;
1505 inode
= vma
->vm_file
->f_mapping
->host
;
1506 offset
= bp_vaddr
- vma
->vm_start
;
1507 offset
+= (vma
->vm_pgoff
<< PAGE_SHIFT
);
1508 uprobe
= find_uprobe(inode
, offset
);
1512 *is_swbp
= is_swbp_at_addr(mm
, bp_vaddr
);
1517 srcu_read_unlock_raw(&uprobes_srcu
, current
->uprobe_srcu_id
);
1518 current
->uprobe_srcu_id
= -1;
1519 up_read(&mm
->mmap_sem
);
1525 * Run handler and ask thread to singlestep.
1526 * Ensure all non-fatal signals cannot interrupt thread while it singlesteps.
1528 static void handle_swbp(struct pt_regs
*regs
)
1530 struct uprobe_task
*utask
;
1531 struct uprobe
*uprobe
;
1532 unsigned long bp_vaddr
;
1533 int uninitialized_var(is_swbp
);
1535 bp_vaddr
= uprobe_get_swbp_addr(regs
);
1536 uprobe
= find_active_uprobe(bp_vaddr
, &is_swbp
);
1540 /* No matching uprobe; signal SIGTRAP. */
1541 send_sig(SIGTRAP
, current
, 0);
1544 * Either we raced with uprobe_unregister() or we can't
1545 * access this memory. The latter is only possible if
1546 * another thread plays with our ->mm. In both cases
1547 * we can simply restart. If this vma was unmapped we
1548 * can pretend this insn was not executed yet and get
1549 * the (correct) SIGSEGV after restart.
1551 instruction_pointer_set(regs
, bp_vaddr
);
1556 utask
= current
->utask
;
1558 utask
= add_utask();
1559 /* Cannot allocate; re-execute the instruction. */
1563 utask
->active_uprobe
= uprobe
;
1564 handler_chain(uprobe
, regs
);
1565 if (uprobe
->flags
& UPROBE_SKIP_SSTEP
&& can_skip_sstep(uprobe
, regs
))
1568 utask
->state
= UTASK_SSTEP
;
1569 if (!pre_ssout(uprobe
, regs
, bp_vaddr
)) {
1570 user_enable_single_step(current
);
1576 utask
->active_uprobe
= NULL
;
1577 utask
->state
= UTASK_RUNNING
;
1580 if (!(uprobe
->flags
& UPROBE_SKIP_SSTEP
))
1583 * cannot singlestep; cannot skip instruction;
1584 * re-execute the instruction.
1586 instruction_pointer_set(regs
, bp_vaddr
);
1593 * Perform required fix-ups and disable singlestep.
1594 * Allow pending signals to take effect.
1596 static void handle_singlestep(struct uprobe_task
*utask
, struct pt_regs
*regs
)
1598 struct uprobe
*uprobe
;
1600 uprobe
= utask
->active_uprobe
;
1601 if (utask
->state
== UTASK_SSTEP_ACK
)
1602 arch_uprobe_post_xol(&uprobe
->arch
, regs
);
1603 else if (utask
->state
== UTASK_SSTEP_TRAPPED
)
1604 arch_uprobe_abort_xol(&uprobe
->arch
, regs
);
1609 utask
->active_uprobe
= NULL
;
1610 utask
->state
= UTASK_RUNNING
;
1611 user_disable_single_step(current
);
1612 xol_free_insn_slot(current
);
1614 spin_lock_irq(¤t
->sighand
->siglock
);
1615 recalc_sigpending(); /* see uprobe_deny_signal() */
1616 spin_unlock_irq(¤t
->sighand
->siglock
);
1620 * On breakpoint hit, breakpoint notifier sets the TIF_UPROBE flag. (and on
1621 * subsequent probe hits on the thread sets the state to UTASK_BP_HIT) and
1622 * allows the thread to return from interrupt.
1624 * On singlestep exception, singlestep notifier sets the TIF_UPROBE flag and
1625 * also sets the state to UTASK_SSTEP_ACK and allows the thread to return from
1628 * While returning to userspace, thread notices the TIF_UPROBE flag and calls
1629 * uprobe_notify_resume().
1631 void uprobe_notify_resume(struct pt_regs
*regs
)
1633 struct uprobe_task
*utask
;
1635 utask
= current
->utask
;
1636 if (!utask
|| utask
->state
== UTASK_BP_HIT
)
1639 handle_singlestep(utask
, regs
);
1643 * uprobe_pre_sstep_notifier gets called from interrupt context as part of
1644 * notifier mechanism. Set TIF_UPROBE flag and indicate breakpoint hit.
1646 int uprobe_pre_sstep_notifier(struct pt_regs
*regs
)
1648 struct uprobe_task
*utask
;
1650 if (!current
->mm
|| !atomic_read(¤t
->mm
->uprobes_state
.count
))
1651 /* task is currently not uprobed */
1654 utask
= current
->utask
;
1656 utask
->state
= UTASK_BP_HIT
;
1658 set_thread_flag(TIF_UPROBE
);
1659 current
->uprobe_srcu_id
= srcu_read_lock_raw(&uprobes_srcu
);
1665 * uprobe_post_sstep_notifier gets called in interrupt context as part of notifier
1666 * mechanism. Set TIF_UPROBE flag and indicate completion of singlestep.
1668 int uprobe_post_sstep_notifier(struct pt_regs
*regs
)
1670 struct uprobe_task
*utask
= current
->utask
;
1672 if (!current
->mm
|| !utask
|| !utask
->active_uprobe
)
1673 /* task is currently not uprobed */
1676 utask
->state
= UTASK_SSTEP_ACK
;
1677 set_thread_flag(TIF_UPROBE
);
1681 static struct notifier_block uprobe_exception_nb
= {
1682 .notifier_call
= arch_uprobe_exception_notify
,
1683 .priority
= INT_MAX
-1, /* notified after kprobes, kgdb */
1686 static int __init
init_uprobes(void)
1690 for (i
= 0; i
< UPROBES_HASH_SZ
; i
++) {
1691 mutex_init(&uprobes_mutex
[i
]);
1692 mutex_init(&uprobes_mmap_mutex
[i
]);
1694 init_srcu_struct(&uprobes_srcu
);
1696 return register_die_notifier(&uprobe_exception_nb
);
1698 module_init(init_uprobes
);
1700 static void __exit
exit_uprobes(void)
1703 module_exit(exit_uprobes
);