4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License, Version 1.0 only
6 * (the "License"). You may not use this file except in compliance
9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 * or http://www.opensolaris.org/os/licensing.
11 * See the License for the specific language governing permissions
12 * and limitations under the License.
14 * When distributing Covered Code, include this CDDL HEADER in each
15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 * If applicable, add the following below this CDDL HEADER, with the
17 * fields enclosed by brackets "[]" replaced with your own identifying
18 * information: Portions Copyright [yyyy] [name of copyright owner]
23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
27 #include <sys/types.h>
28 #include <sys/t_lock.h>
29 #include <sys/param.h>
31 #include <sys/debug.h>
32 #include <sys/inline.h>
35 #include <sys/regset.h>
36 #include <sys/sysmacros.h>
37 #include <sys/systm.h>
38 #include <sys/prsystm.h>
40 #include <sys/signal.h>
42 #include <sys/cpuvar.h>
44 #include <sys/fault.h>
45 #include <sys/syscall.h>
46 #include <sys/procfs.h>
47 #include <sys/cmn_err.h>
48 #include <sys/stack.h>
49 #include <sys/watchpoint.h>
50 #include <sys/copyops.h>
51 #include <sys/schedctl.h>
58 * Copy ops vector for watchpoints.
60 static int watch_copyin(const void *, void *, size_t);
61 static int watch_xcopyin(const void *, void *, size_t);
62 static int watch_copyout(const void *, void *, size_t);
63 static int watch_xcopyout(const void *, void *, size_t);
64 static int watch_copyinstr(const char *, char *, size_t, size_t *);
65 static int watch_copyoutstr(const char *, char *, size_t, size_t *);
66 static int watch_fuword8(const void *, uint8_t *);
67 static int watch_fuword16(const void *, uint16_t *);
68 static int watch_fuword32(const void *, uint32_t *);
69 static int watch_suword8(void *, uint8_t);
70 static int watch_suword16(void *, uint16_t);
71 static int watch_suword32(void *, uint32_t);
72 static int watch_physio(int (*)(struct buf
*), struct buf
*,
73 dev_t
, int, void (*)(struct buf
*), struct uio
*);
75 static int watch_fuword64(const void *, uint64_t *);
76 static int watch_suword64(void *, uint64_t);
79 struct copyops watch_copyops
= {
106 * Map the 'rw' argument to a protection flag.
109 rw_to_prot(enum seg_rw rw
)
119 return (PROT_NONE
); /* can't happen */
124 * Map the 'rw' argument to an index into an array of exec/write/read things.
125 * The index follows the precedence order: exec .. write .. read
128 rw_to_index(enum seg_rw rw
)
131 default: /* default case "can't happen" */
142 * Map an index back to a seg_rw.
144 static enum seg_rw S_rw
[4] = {
154 #define sum(a) (a[X] + a[W] + a[R])
157 * Common code for pr_mappage() and pr_unmappage().
160 pr_do_mappage(caddr_t addr
, size_t size
, int mapin
, enum seg_rw rw
, int kernel
)
163 struct as
*as
= p
->p_as
;
164 char *eaddr
= addr
+ size
;
165 int prot_rw
= rw_to_prot(rw
);
166 int xrw
= rw_to_index(rw
);
168 struct watched_page
*pwp
;
169 struct watched_page tpw
;
177 if (avl_numnodes(&as
->a_wpage
) == 0)
181 * as->a_wpage can only be changed while the process is totally stopped.
182 * Don't grab p_lock here. Holding p_lock while grabbing the address
183 * space lock leads to deadlocks with the clock thread. Note that if an
184 * as_fault() is servicing a fault to a watched page on behalf of an
185 * XHAT provider, watchpoint will be temporarily cleared (and wp_prot
186 * will be set to wp_oprot). Since this is done while holding as writer
187 * lock, we need to grab as lock (reader lock is good enough).
189 * p_maplock prevents simultaneous execution of this function. Under
190 * normal circumstances, holdwatch() will stop all other threads, so the
191 * lock isn't really needed. But there may be multiple threads within
192 * stop() when SWATCHOK is set, so we need to handle multiple threads
193 * at once. See holdwatch() for the details of this dance.
196 mutex_enter(&p
->p_maplock
);
197 AS_LOCK_ENTER(as
, RW_READER
);
199 tpw
.wp_vaddr
= (caddr_t
)((uintptr_t)addr
& (uintptr_t)PAGEMASK
);
200 if ((pwp
= avl_find(&as
->a_wpage
, &tpw
, &where
)) == NULL
)
201 pwp
= avl_nearest(&as
->a_wpage
, where
, AVL_AFTER
);
203 for (; pwp
!= NULL
&& pwp
->wp_vaddr
< eaddr
;
204 pwp
= AVL_NEXT(&as
->a_wpage
, pwp
)) {
207 * If the requested protection has not been
208 * removed, we need not remap this page.
211 if (kernel
|| (prot
& PROT_USER
))
215 * If the requested access does not exist in the page's
216 * original protections, we need not remap this page.
217 * If the page does not exist yet, we can't test it.
219 if ((prot
= pwp
->wp_oprot
) != 0) {
220 if (!(kernel
|| (prot
& PROT_USER
)))
222 if (!(prot
& prot_rw
))
228 * Before mapping the page in, ensure that
229 * all other lwps are held in the kernel.
231 if (p
->p_mapcnt
== 0) {
233 * Release as lock while in holdwatch()
234 * in case other threads need to grab it.
237 mutex_exit(&p
->p_maplock
);
238 if (holdwatch() != 0) {
240 * We stopped in holdwatch().
241 * Start all over again because the
242 * watched page list may have changed.
246 mutex_enter(&p
->p_maplock
);
247 AS_LOCK_ENTER(as
, RW_READER
);
252 addr
= pwp
->wp_vaddr
;
261 pwp
->wp_flags
|= WP_NOWATCH
;
262 if (pwp
->wp_kmap
[X
] + pwp
->wp_umap
[X
])
263 /* cannot have exec-only protection */
264 prot
|= PROT_READ
|PROT_EXEC
;
265 if (pwp
->wp_kmap
[R
] + pwp
->wp_umap
[R
])
267 if (pwp
->wp_kmap
[W
] + pwp
->wp_umap
[W
])
268 /* cannot have write-only protection */
269 prot
|= PROT_READ
|PROT_WRITE
;
270 #if 0 /* damned broken mmu feature! */
271 if (sum(pwp
->wp_umap
) == 0)
275 ASSERT(pwp
->wp_flags
& WP_NOWATCH
);
277 ASSERT(pwp
->wp_kmap
[xrw
] != 0);
280 ASSERT(pwp
->wp_umap
[xrw
] != 0);
283 if (sum(pwp
->wp_kmap
) + sum(pwp
->wp_umap
) == 0)
284 pwp
->wp_flags
&= ~WP_NOWATCH
;
286 if (pwp
->wp_kmap
[X
] + pwp
->wp_umap
[X
])
287 /* cannot have exec-only protection */
288 prot
|= PROT_READ
|PROT_EXEC
;
289 if (pwp
->wp_kmap
[R
] + pwp
->wp_umap
[R
])
291 if (pwp
->wp_kmap
[W
] + pwp
->wp_umap
[W
])
292 /* cannot have write-only protection */
293 prot
|= PROT_READ
|PROT_WRITE
;
294 #if 0 /* damned broken mmu feature! */
295 if (sum(pwp
->wp_umap
) == 0)
302 if (pwp
->wp_oprot
!= 0) { /* if page exists */
305 int err
, retrycnt
= 0;
308 AS_LOCK_ENTER(as
, RW_WRITER
);
310 seg
= as_segat(as
, addr
);
312 SEGOP_GETPROT(seg
, addr
, 0, &oprot
);
314 err
= SEGOP_SETPROT(seg
, addr
, PAGESIZE
, prot
);
315 if (err
== IE_RETRY
) {
316 ASSERT(retrycnt
== 0);
326 * When all pages are mapped back to their normal state,
327 * continue the other lwps.
330 ASSERT(p
->p_mapcnt
> 0);
332 if (p
->p_mapcnt
== 0) {
333 mutex_exit(&p
->p_maplock
);
334 mutex_enter(&p
->p_lock
);
336 mutex_exit(&p
->p_lock
);
337 mutex_enter(&p
->p_maplock
);
341 AS_LOCK_ENTER(as
, RW_READER
);
345 mutex_exit(&p
->p_maplock
);
351 * Restore the original page protections on an address range.
352 * If 'kernel' is non-zero, just do it for the kernel.
353 * pr_mappage() returns non-zero if it actually changed anything.
355 * pr_mappage() and pr_unmappage() must be executed in matched pairs,
356 * but pairs may be nested within other pairs. The reference counts
357 * sort it all out. See pr_do_mappage(), above.
360 pr_mappage(const caddr_t addr
, size_t size
, enum seg_rw rw
, int kernel
)
362 return (pr_do_mappage(addr
, size
, 1, rw
, kernel
));
366 * Set the modified page protections on a watched page.
367 * Inverse of pr_mappage().
368 * Needs to be called only if pr_mappage() returned non-zero.
371 pr_unmappage(const caddr_t addr
, size_t size
, enum seg_rw rw
, int kernel
)
373 (void) pr_do_mappage(addr
, size
, 0, rw
, kernel
);
377 * Function called by an lwp after it resumes from stop().
383 struct as
*as
= curproc
->p_as
;
384 struct watched_page
*pwp
, *next
;
390 if (p
->p_wprot
== NULL
)
393 ASSERT(MUTEX_NOT_HELD(&curproc
->p_lock
));
395 AS_LOCK_ENTER(as
, RW_WRITER
);
398 while (pwp
!= NULL
) {
400 vaddr
= pwp
->wp_vaddr
;
403 ASSERT(pwp
->wp_flags
& WP_SETPROT
);
404 if ((seg
= as_segat(as
, vaddr
)) != NULL
&&
405 !(pwp
->wp_flags
& WP_NOWATCH
)) {
407 err
= SEGOP_SETPROT(seg
, vaddr
, PAGESIZE
, prot
);
408 if (err
== IE_RETRY
) {
409 ASSERT(retrycnt
== 0);
417 if (pwp
->wp_read
+ pwp
->wp_write
+ pwp
->wp_exec
== 0) {
419 * No watched areas remain in this page.
420 * Free the watched_page structure.
422 avl_remove(&as
->a_wpage
, pwp
);
423 kmem_free(pwp
, sizeof (struct watched_page
));
425 pwp
->wp_flags
&= ~WP_SETPROT
;
437 /* Must be called with as lock held */
439 pr_is_watchpage_as(caddr_t addr
, enum seg_rw rw
, struct as
*as
)
441 register struct watched_page
*pwp
;
442 struct watched_page tpw
;
456 * as->a_wpage can only be modified while the process is totally
457 * stopped. We need, and should use, no locks here.
459 if (as
!= &kas
&& avl_numnodes(&as
->a_wpage
) != 0) {
460 tpw
.wp_vaddr
= (caddr_t
)((uintptr_t)addr
& (uintptr_t)PAGEMASK
);
461 pwp
= avl_find(&as
->a_wpage
, &tpw
, NULL
);
463 ASSERT(addr
>= pwp
->wp_vaddr
&&
464 addr
< pwp
->wp_vaddr
+ PAGESIZE
);
465 if (pwp
->wp_oprot
!= 0) {
469 rv
= ((prot
& (PROT_USER
|PROT_READ
))
470 != (PROT_USER
|PROT_READ
));
473 rv
= ((prot
& (PROT_USER
|PROT_WRITE
))
474 != (PROT_USER
|PROT_WRITE
));
477 rv
= ((prot
& (PROT_USER
|PROT_EXEC
))
478 != (PROT_USER
|PROT_EXEC
));
493 * trap() calls here to determine if a fault is in a watched page.
494 * We return nonzero if this is true and the load/store would fail.
497 pr_is_watchpage(caddr_t addr
, enum seg_rw rw
)
499 struct as
*as
= curproc
->p_as
;
502 if ((as
== &kas
) || avl_numnodes(&as
->a_wpage
) == 0)
505 /* Grab the lock because of XHAT (see comment in pr_mappage()) */
506 AS_LOCK_ENTER(as
, RW_READER
);
507 rv
= pr_is_watchpage_as(addr
, rw
, as
);
516 * trap() calls here to determine if a fault is a watchpoint.
519 pr_is_watchpoint(caddr_t
*paddr
, int *pta
, size_t size
, size_t *plen
,
523 caddr_t addr
= *paddr
;
524 caddr_t eaddr
= addr
+ size
;
525 register struct watched_area
*pwa
;
526 struct watched_area twa
;
542 * p->p_warea is protected by p->p_lock.
544 mutex_enter(&p
->p_lock
);
548 * This loop is somewhat complicated because the fault region can span
549 * multiple watched areas. For example:
552 * +-----------------+
554 * +-------+--------+----+---+------------+
555 * | prot not right | | prot correct |
556 * +----------------+ +----------------+
560 * We start at the area greater than or equal to the starting address.
561 * As long as some portion of the fault region overlaps the current
562 * area, we continue checking permissions until we find an appropriate
567 twa
.wa_eaddr
= eaddr
;
569 for (pwa
= pr_find_watched_area(p
, &twa
, NULL
);
570 pwa
!= NULL
&& eaddr
> pwa
->wa_vaddr
&& addr
< pwa
->wa_eaddr
;
571 pwa
= AVL_NEXT(&p
->p_warea
, pwa
)) {
575 if (pwa
->wa_flags
& WA_READ
)
579 if (pwa
->wa_flags
& WA_WRITE
)
583 if (pwa
->wa_flags
& WA_EXEC
)
592 * If protections didn't match, check the next watched
596 if (addr
< pwa
->wa_vaddr
)
597 addr
= pwa
->wa_vaddr
;
598 len
= pwa
->wa_eaddr
- addr
;
599 if (pwa
->wa_flags
& WA_TRAPAFTER
)
605 mutex_exit(&p
->p_lock
);
615 * Set up to perform a single-step at user level for the
616 * case of a trapafter watchpoint. Called from trap().
619 do_watch_step(caddr_t vaddr
, size_t sz
, enum seg_rw rw
,
620 int watchcode
, greg_t pc
)
622 register klwp_t
*lwp
= ttolwp(curthread
);
623 struct lwp_watch
*pw
= &lwp
->lwp_watch
[rw_to_index(rw
)];
626 * Check to see if we are already performing this special
627 * watchpoint single-step. We must not do pr_mappage() twice.
630 /* special check for two read traps on the same instruction */
631 if (rw
== S_READ
&& pw
->wpaddr
!= NULL
&&
632 !(pw
->wpaddr
<= vaddr
&& vaddr
< pw
->wpaddr
+ pw
->wpsize
)) {
633 ASSERT(lwp
->lwp_watchtrap
!= 0);
634 pw
++; /* use the extra S_READ struct */
637 if (pw
->wpaddr
!= NULL
) {
638 ASSERT(lwp
->lwp_watchtrap
!= 0);
639 ASSERT(pw
->wpaddr
<= vaddr
&& vaddr
< pw
->wpaddr
+ pw
->wpsize
);
640 if (pw
->wpcode
== 0) {
641 pw
->wpcode
= watchcode
;
645 int mapped
= pr_mappage(vaddr
, sz
, rw
, 0);
647 lwp
->lwp_watchtrap
= 1;
650 pw
->wpcode
= watchcode
;
651 pw
->wpmapped
= mapped
;
657 * Undo the effects of do_watch_step().
658 * Called from trap() after the single-step is finished.
659 * Also called from issig_forreal() and stop() with a NULL
660 * argument to avoid having these things set more than once.
663 undo_watch_step(k_siginfo_t
*sip
)
665 register klwp_t
*lwp
= ttolwp(curthread
);
668 if (lwp
->lwp_watchtrap
) {
669 struct lwp_watch
*pw
= lwp
->lwp_watch
;
672 for (i
= 0; i
< 4; i
++, pw
++) {
673 if (pw
->wpaddr
== NULL
)
676 pr_unmappage(pw
->wpaddr
, pw
->wpsize
, S_rw
[i
],
678 if (pw
->wpcode
!= 0) {
680 sip
->si_signo
= SIGTRAP
;
681 sip
->si_code
= pw
->wpcode
;
682 sip
->si_addr
= pw
->wpaddr
;
683 sip
->si_trapafter
= 1;
684 sip
->si_pc
= (caddr_t
)pw
->wppc
;
693 lwp
->lwp_watchtrap
= 0;
700 * Handle a watchpoint that occurs while doing copyin()
701 * or copyout() in a system call.
702 * Return non-zero if the fault or signal is cleared
703 * by a debugger while the lwp is stopped.
706 sys_watchpoint(caddr_t addr
, int watchcode
, int ta
)
708 extern greg_t
getuserpc(void); /* XXX header file */
710 register proc_t
*p
= ttoproc(curthread
);
711 register klwp_t
*lwp
= ttolwp(curthread
);
712 register sigqueue_t
*sqp
;
715 /* assert no locks are held */
716 /* ASSERT(curthread->t_nlocks == 0); */
718 sqp
= kmem_zalloc(sizeof (sigqueue_t
), KM_SLEEP
);
719 sqp
->sq_info
.si_signo
= SIGTRAP
;
720 sqp
->sq_info
.si_code
= watchcode
;
721 sqp
->sq_info
.si_addr
= addr
;
722 sqp
->sq_info
.si_trapafter
= ta
;
723 sqp
->sq_info
.si_pc
= (caddr_t
)getuserpc();
725 mutex_enter(&p
->p_lock
);
727 /* this will be tested and cleared by the caller */
728 lwp
->lwp_sysabort
= 0;
730 if (prismember(&p
->p_fltmask
, FLTWATCH
)) {
731 lwp
->lwp_curflt
= (uchar_t
)FLTWATCH
;
732 lwp
->lwp_siginfo
= sqp
->sq_info
;
733 stop(PR_FAULTED
, FLTWATCH
);
734 if (lwp
->lwp_curflt
== 0) {
735 mutex_exit(&p
->p_lock
);
736 kmem_free(sqp
, sizeof (sigqueue_t
));
743 * post the SIGTRAP signal.
744 * Block all other signals so we only stop showing SIGTRAP.
746 if (signal_is_blocked(curthread
, SIGTRAP
) ||
747 sigismember(&p
->p_ignore
, SIGTRAP
)) {
748 /* SIGTRAP is blocked or ignored, forget the rest. */
749 mutex_exit(&p
->p_lock
);
750 kmem_free(sqp
, sizeof (sigqueue_t
));
753 sigdelq(p
, curthread
, SIGTRAP
);
754 sigaddqa(p
, curthread
, sqp
);
755 schedctl_finish_sigblock(curthread
);
756 smask
= curthread
->t_hold
;
757 sigfillset(&curthread
->t_hold
);
758 sigdiffset(&curthread
->t_hold
, &cantmask
);
759 sigdelset(&curthread
->t_hold
, SIGTRAP
);
760 mutex_exit(&p
->p_lock
);
762 rval
= ((ISSIG_FAST(curthread
, lwp
, p
, FORREAL
))? 0 : 1);
764 /* restore the original signal mask */
765 mutex_enter(&p
->p_lock
);
766 curthread
->t_hold
= smask
;
767 mutex_exit(&p
->p_lock
);
773 * Wrappers for the copyin()/copyout() functions to deal
774 * with watchpoints that fire while in system calls.
778 watch_xcopyin(const void *uaddr
, void *kaddr
, size_t count
)
780 klwp_t
*lwp
= ttolwp(curthread
);
781 caddr_t watch_uaddr
= (caddr_t
)uaddr
;
782 caddr_t watch_kaddr
= (caddr_t
)kaddr
;
788 while (count
&& error
== 0) {
794 if ((part
= PAGESIZE
-
795 (((uintptr_t)uaddr
) & PAGEOFFSET
)) > count
)
798 if (!pr_is_watchpage(watch_uaddr
, S_READ
))
802 watchcode
= pr_is_watchpoint(&vaddr
, &ta
,
804 if (watchcode
&& ta
== 0)
805 part
= vaddr
- watch_uaddr
;
809 * Copy the initial part, up to a watched address, if any.
812 mapped
= pr_mappage(watch_uaddr
, part
, S_READ
, 1);
816 copyin_noerr(watch_uaddr
, watch_kaddr
, part
);
819 pr_unmappage(watch_uaddr
, part
, S_READ
, 1);
825 * If trapafter was specified, then copy through the
826 * watched area before taking the watchpoint trap.
828 while (count
&& watchcode
&& ta
&& len
> part
&& error
== 0) {
830 if ((part
= PAGESIZE
) > count
)
834 mapped
= pr_mappage(watch_uaddr
, part
, S_READ
, 1);
838 copyin_noerr(watch_uaddr
, watch_kaddr
, part
);
841 pr_unmappage(watch_uaddr
, part
, S_READ
, 1);
848 /* if we hit a watched address, do the watchpoint logic */
850 (!sys_watchpoint(vaddr
, watchcode
, ta
) ||
851 lwp
->lwp_sysabort
)) {
852 lwp
->lwp_sysabort
= 0;
862 watch_copyin(const void *kaddr
, void *uaddr
, size_t count
)
864 return (watch_xcopyin(kaddr
, uaddr
, count
) ? -1 : 0);
869 watch_xcopyout(const void *kaddr
, void *uaddr
, size_t count
)
871 klwp_t
*lwp
= ttolwp(curthread
);
872 caddr_t watch_uaddr
= (caddr_t
)uaddr
;
873 caddr_t watch_kaddr
= (caddr_t
)kaddr
;
877 while (count
&& error
== 0) {
885 if ((part
= PAGESIZE
-
886 (((uintptr_t)uaddr
) & PAGEOFFSET
)) > count
)
889 if (!pr_is_watchpage(watch_uaddr
, S_WRITE
))
893 watchcode
= pr_is_watchpoint(&vaddr
, &ta
,
894 part
, &len
, S_WRITE
);
897 part
= vaddr
- watch_uaddr
;
899 len
+= vaddr
- watch_uaddr
;
907 * Copy the initial part, up to a watched address, if any.
910 mapped
= pr_mappage(watch_uaddr
, part
, S_WRITE
, 1);
914 copyout_noerr(watch_kaddr
, watch_uaddr
, part
);
917 pr_unmappage(watch_uaddr
, part
, S_WRITE
, 1);
924 * If trapafter was specified, then copy through the
925 * watched area before taking the watchpoint trap.
927 while (count
&& watchcode
&& ta
&& len
> part
&& error
== 0) {
929 if ((part
= PAGESIZE
) > count
)
933 mapped
= pr_mappage(watch_uaddr
, part
, S_WRITE
, 1);
937 copyout_noerr(watch_kaddr
, watch_uaddr
, part
);
940 pr_unmappage(watch_uaddr
, part
, S_WRITE
, 1);
946 /* if we hit a watched address, do the watchpoint logic */
948 (!sys_watchpoint(vaddr
, watchcode
, ta
) ||
949 lwp
->lwp_sysabort
)) {
950 lwp
->lwp_sysabort
= 0;
960 watch_copyout(const void *kaddr
, void *uaddr
, size_t count
)
962 return (watch_xcopyout(kaddr
, uaddr
, count
) ? -1 : 0);
972 klwp_t
*lwp
= ttolwp(curthread
);
977 if ((resid
= maxlength
) == 0)
978 return (ENAMETOOLONG
);
980 while (resid
&& error
== 0) {
989 if ((part
= PAGESIZE
-
990 (((uintptr_t)uaddr
) & PAGEOFFSET
)) > resid
)
993 if (!pr_is_watchpage((caddr_t
)uaddr
, S_READ
))
996 vaddr
= (caddr_t
)uaddr
;
997 watchcode
= pr_is_watchpoint(&vaddr
, &ta
,
1001 part
= vaddr
- uaddr
;
1003 len
+= vaddr
- uaddr
;
1011 * Copy the initial part, up to a watched address, if any.
1014 mapped
= pr_mappage((caddr_t
)uaddr
, part
, S_READ
, 1);
1018 error
= copyinstr_noerr(uaddr
, kaddr
, part
,
1022 pr_unmappage((caddr_t
)uaddr
, part
, S_READ
, 1);
1026 if (error
== ENAMETOOLONG
&& resid
> 0)
1028 if (error
!= 0 || (watchcode
&&
1029 (uaddr
< vaddr
|| kaddr
[-1] == '\0')))
1030 break; /* didn't reach the watched area */
1034 * If trapafter was specified, then copy through the
1035 * watched area before taking the watchpoint trap.
1037 while (resid
&& watchcode
&& ta
&& len
> part
&& error
== 0 &&
1038 size
== part
&& kaddr
[-1] != '\0') {
1040 if ((part
= PAGESIZE
) > resid
)
1044 mapped
= pr_mappage((caddr_t
)uaddr
, part
, S_READ
, 1);
1048 error
= copyinstr_noerr(uaddr
, kaddr
, part
,
1052 pr_unmappage((caddr_t
)uaddr
, part
, S_READ
, 1);
1056 if (error
== ENAMETOOLONG
&& resid
> 0)
1060 /* if we hit a watched address, do the watchpoint logic */
1062 (!sys_watchpoint(vaddr
, watchcode
, ta
) ||
1063 lwp
->lwp_sysabort
)) {
1064 lwp
->lwp_sysabort
= 0;
1069 if (error
== 0 && part
!= 0 &&
1070 (size
< part
|| kaddr
[-1] == '\0'))
1074 if (error
!= EFAULT
&& lencopied
)
1075 *lencopied
= maxlength
- resid
;
1086 klwp_t
*lwp
= ttolwp(curthread
);
1091 if ((resid
= maxlength
) == 0)
1092 return (ENAMETOOLONG
);
1094 while (resid
&& error
== 0) {
1103 if ((part
= PAGESIZE
-
1104 (((uintptr_t)uaddr
) & PAGEOFFSET
)) > resid
)
1107 if (!pr_is_watchpage(uaddr
, S_WRITE
)) {
1111 watchcode
= pr_is_watchpoint(&vaddr
, &ta
,
1112 part
, &len
, S_WRITE
);
1113 if (watchcode
&& ta
== 0)
1114 part
= vaddr
- uaddr
;
1118 * Copy the initial part, up to a watched address, if any.
1121 mapped
= pr_mappage(uaddr
, part
, S_WRITE
, 1);
1125 error
= copyoutstr_noerr(kaddr
, uaddr
, part
,
1129 pr_unmappage(uaddr
, part
, S_WRITE
, 1);
1133 if (error
== ENAMETOOLONG
&& resid
> 0)
1135 if (error
!= 0 || (watchcode
&&
1136 (uaddr
< vaddr
|| kaddr
[-1] == '\0')))
1137 break; /* didn't reach the watched area */
1141 * If trapafter was specified, then copy through the
1142 * watched area before taking the watchpoint trap.
1144 while (resid
&& watchcode
&& ta
&& len
> part
&& error
== 0 &&
1145 size
== part
&& kaddr
[-1] != '\0') {
1147 if ((part
= PAGESIZE
) > resid
)
1151 mapped
= pr_mappage(uaddr
, part
, S_WRITE
, 1);
1155 error
= copyoutstr_noerr(kaddr
, uaddr
, part
,
1159 pr_unmappage(uaddr
, part
, S_WRITE
, 1);
1163 if (error
== ENAMETOOLONG
&& resid
> 0)
1167 /* if we hit a watched address, do the watchpoint logic */
1169 (!sys_watchpoint(vaddr
, watchcode
, ta
) ||
1170 lwp
->lwp_sysabort
)) {
1171 lwp
->lwp_sysabort
= 0;
1176 if (error
== 0 && part
!= 0 &&
1177 (size
< part
|| kaddr
[-1] == '\0'))
1181 if (error
!= EFAULT
&& lencopied
)
1182 *lencopied
= maxlength
- resid
;
1186 typedef int (*fuword_func
)(const void *, void *);
1189 * Generic form of watch_fuword8(), watch_fuword16(), etc.
1192 watch_fuword(const void *addr
, void *dst
, fuword_func func
, size_t size
)
1194 klwp_t
*lwp
= ttolwp(curthread
);
1204 vaddr
= (caddr_t
)addr
;
1205 watchcode
= pr_is_watchpoint(&vaddr
, &ta
, size
, NULL
, S_READ
);
1206 if (watchcode
== 0 || ta
!= 0) {
1207 mapped
= pr_mappage((caddr_t
)addr
, size
, S_READ
, 1);
1214 pr_unmappage((caddr_t
)addr
, size
, S_READ
, 1);
1217 (!sys_watchpoint(vaddr
, watchcode
, ta
) ||
1218 lwp
->lwp_sysabort
)) {
1219 lwp
->lwp_sysabort
= 0;
1223 if (watchcode
== 0 || ta
!= 0)
1231 watch_fuword8(const void *addr
, uint8_t *dst
)
1233 return (watch_fuword(addr
, dst
, (fuword_func
)fuword8_noerr
,
1238 watch_fuword16(const void *addr
, uint16_t *dst
)
1240 return (watch_fuword(addr
, dst
, (fuword_func
)fuword16_noerr
,
1245 watch_fuword32(const void *addr
, uint32_t *dst
)
1247 return (watch_fuword(addr
, dst
, (fuword_func
)fuword32_noerr
,
1253 watch_fuword64(const void *addr
, uint64_t *dst
)
1255 return (watch_fuword(addr
, dst
, (fuword_func
)fuword64_noerr
,
1262 watch_suword8(void *addr
, uint8_t value
)
1264 klwp_t
*lwp
= ttolwp(curthread
);
1274 vaddr
= (caddr_t
)addr
;
1275 watchcode
= pr_is_watchpoint(&vaddr
, &ta
, sizeof (value
), NULL
,
1277 if (watchcode
== 0 || ta
!= 0) {
1278 mapped
= pr_mappage((caddr_t
)addr
, sizeof (value
),
1283 suword8_noerr(addr
, value
);
1286 pr_unmappage((caddr_t
)addr
, sizeof (value
),
1290 (!sys_watchpoint(vaddr
, watchcode
, ta
) ||
1291 lwp
->lwp_sysabort
)) {
1292 lwp
->lwp_sysabort
= 0;
1296 if (watchcode
== 0 || ta
!= 0)
1304 watch_suword16(void *addr
, uint16_t value
)
1306 klwp_t
*lwp
= ttolwp(curthread
);
1316 vaddr
= (caddr_t
)addr
;
1317 watchcode
= pr_is_watchpoint(&vaddr
, &ta
, sizeof (value
), NULL
,
1319 if (watchcode
== 0 || ta
!= 0) {
1320 mapped
= pr_mappage((caddr_t
)addr
, sizeof (value
),
1325 suword16_noerr(addr
, value
);
1328 pr_unmappage((caddr_t
)addr
, sizeof (value
),
1332 (!sys_watchpoint(vaddr
, watchcode
, ta
) ||
1333 lwp
->lwp_sysabort
)) {
1334 lwp
->lwp_sysabort
= 0;
1338 if (watchcode
== 0 || ta
!= 0)
1346 watch_suword32(void *addr
, uint32_t value
)
1348 klwp_t
*lwp
= ttolwp(curthread
);
1358 vaddr
= (caddr_t
)addr
;
1359 watchcode
= pr_is_watchpoint(&vaddr
, &ta
, sizeof (value
), NULL
,
1361 if (watchcode
== 0 || ta
!= 0) {
1362 mapped
= pr_mappage((caddr_t
)addr
, sizeof (value
),
1367 suword32_noerr(addr
, value
);
1370 pr_unmappage((caddr_t
)addr
, sizeof (value
),
1374 (!sys_watchpoint(vaddr
, watchcode
, ta
) ||
1375 lwp
->lwp_sysabort
)) {
1376 lwp
->lwp_sysabort
= 0;
1380 if (watchcode
== 0 || ta
!= 0)
1389 watch_suword64(void *addr
, uint64_t value
)
1391 klwp_t
*lwp
= ttolwp(curthread
);
1401 vaddr
= (caddr_t
)addr
;
1402 watchcode
= pr_is_watchpoint(&vaddr
, &ta
, sizeof (value
), NULL
,
1404 if (watchcode
== 0 || ta
!= 0) {
1405 mapped
= pr_mappage((caddr_t
)addr
, sizeof (value
),
1410 suword64_noerr(addr
, value
);
1413 pr_unmappage((caddr_t
)addr
, sizeof (value
),
1417 (!sys_watchpoint(vaddr
, watchcode
, ta
) ||
1418 lwp
->lwp_sysabort
)) {
1419 lwp
->lwp_sysabort
= 0;
1423 if (watchcode
== 0 || ta
!= 0)
1432 * Check for watched addresses in the given address space.
1433 * Return 1 if this is true, otherwise 0.
1436 pr_is_watched(caddr_t base
, size_t len
, int rw
)
1438 caddr_t saddr
= (caddr_t
)((uintptr_t)base
& (uintptr_t)PAGEMASK
);
1439 caddr_t eaddr
= base
+ len
;
1442 for (paddr
= saddr
; paddr
< eaddr
; paddr
+= PAGESIZE
) {
1443 if (pr_is_watchpage(paddr
, rw
))
1451 * Wrapper for the physio() function.
1452 * Splits one uio operation with multiple iovecs into uio operations with
1453 * only one iovecs to do the watchpoint handling separately for each iovecs.
1456 watch_physio(int (*strat
)(struct buf
*), struct buf
*bp
, dev_t dev
,
1457 int rw
, void (*mincnt
)(struct buf
*), struct uio
*uio
)
1466 if (uio
->uio_segflg
== UIO_SYSSPACE
)
1467 return (default_physio(strat
, bp
, dev
, rw
, mincnt
, uio
));
1469 seg_rw
= (rw
== B_READ
) ? S_WRITE
: S_READ
;
1471 while (uio
->uio_iovcnt
> 0) {
1472 if (uio
->uio_resid
== 0) {
1474 * Make sure to return the uio structure with the
1475 * same values as default_physio() does.
1483 len
= MIN(iov
->iov_len
, uio
->uio_resid
);
1485 auio
.uio_iovcnt
= 1;
1487 auio
.uio_resid
= len
;
1488 auio
.uio_loffset
= uio
->uio_loffset
;
1489 auio
.uio_llimit
= uio
->uio_llimit
;
1490 auio
.uio_fmode
= uio
->uio_fmode
;
1491 auio
.uio_extflg
= uio
->uio_extflg
;
1492 auio
.uio_segflg
= uio
->uio_segflg
;
1494 base
= iov
->iov_base
;
1496 if (!pr_is_watched(base
, len
, seg_rw
)) {
1498 * The given memory references don't cover a
1501 error
= default_physio(strat
, bp
, dev
, rw
, mincnt
,
1504 /* Update uio with values from auio. */
1505 len
-= auio
.uio_resid
;
1506 uio
->uio_resid
-= len
;
1507 uio
->uio_loffset
+= len
;
1510 * Return if an error occurred or not all data
1513 if (auio
.uio_resid
|| error
)
1518 int mapped
, watchcode
, ta
;
1519 caddr_t vaddr
= base
;
1520 klwp_t
*lwp
= ttolwp(curthread
);
1522 watchcode
= pr_is_watchpoint(&vaddr
, &ta
, len
,
1525 if (watchcode
== 0 || ta
!= 0) {
1527 * Do the io if the given memory references
1528 * don't cover a watched area (watchcode=0)
1529 * or if WA_TRAPAFTER was specified.
1531 mapped
= pr_mappage(base
, len
, seg_rw
, 1);
1532 error
= default_physio(strat
, bp
, dev
, rw
,
1535 pr_unmappage(base
, len
, seg_rw
, 1);
1537 len
-= auio
.uio_resid
;
1538 uio
->uio_resid
-= len
;
1539 uio
->uio_loffset
+= len
;
1543 * If we hit a watched address, do the watchpoint logic.
1546 (!sys_watchpoint(vaddr
, watchcode
, ta
) ||
1547 lwp
->lwp_sysabort
)) {
1548 lwp
->lwp_sysabort
= 0;
1553 * Check for errors from default_physio().
1555 if (watchcode
== 0 || ta
!= 0) {
1556 if (auio
.uio_resid
|| error
)
1568 wa_compare(const void *a
, const void *b
)
1570 const watched_area_t
*pa
= a
;
1571 const watched_area_t
*pb
= b
;
1573 if (pa
->wa_vaddr
< pb
->wa_vaddr
)
1575 else if (pa
->wa_vaddr
> pb
->wa_vaddr
)
1582 wp_compare(const void *a
, const void *b
)
1584 const watched_page_t
*pa
= a
;
1585 const watched_page_t
*pb
= b
;
1587 if (pa
->wp_vaddr
< pb
->wp_vaddr
)
1589 else if (pa
->wp_vaddr
> pb
->wp_vaddr
)
1596 * Given an address range, finds the first watched area which overlaps some or
1600 pr_find_watched_area(proc_t
*p
, watched_area_t
*pwa
, avl_index_t
*where
)
1602 caddr_t vaddr
= pwa
->wa_vaddr
;
1603 caddr_t eaddr
= pwa
->wa_eaddr
;
1604 watched_area_t
*wap
;
1605 avl_index_t real_where
;
1607 /* First, check if there is an exact match. */
1608 wap
= avl_find(&p
->p_warea
, pwa
, &real_where
);
1611 /* Check to see if we overlap with the previous area. */
1613 wap
= avl_nearest(&p
->p_warea
, real_where
, AVL_BEFORE
);
1615 (vaddr
>= wap
->wa_eaddr
|| eaddr
<= wap
->wa_vaddr
))
1619 /* Try the next area. */
1621 wap
= avl_nearest(&p
->p_warea
, real_where
, AVL_AFTER
);
1623 (vaddr
>= wap
->wa_eaddr
|| eaddr
<= wap
->wa_vaddr
))
1628 *where
= real_where
;
1634 watch_enable(kthread_id_t t
)
1636 t
->t_proc_flag
|= TP_WATCHPT
;
1637 install_copyops(t
, &watch_copyops
);
1641 watch_disable(kthread_id_t t
)
1643 t
->t_proc_flag
&= ~TP_WATCHPT
;
1648 copyin_nowatch(const void *uaddr
, void *kaddr
, size_t len
)
1652 watched
= watch_disable_addr(uaddr
, len
, S_READ
);
1653 ret
= copyin(uaddr
, kaddr
, len
);
1655 watch_enable_addr(uaddr
, len
, S_READ
);
1661 copyout_nowatch(const void *kaddr
, void *uaddr
, size_t len
)
1665 watched
= watch_disable_addr(uaddr
, len
, S_WRITE
);
1666 ret
= copyout(kaddr
, uaddr
, len
);
1668 watch_enable_addr(uaddr
, len
, S_WRITE
);
1675 fuword64_nowatch(const void *addr
, uint64_t *value
)
1679 watched
= watch_disable_addr(addr
, sizeof (*value
), S_READ
);
1680 ret
= fuword64(addr
, value
);
1682 watch_enable_addr(addr
, sizeof (*value
), S_READ
);
1689 fuword32_nowatch(const void *addr
, uint32_t *value
)
1693 watched
= watch_disable_addr(addr
, sizeof (*value
), S_READ
);
1694 ret
= fuword32(addr
, value
);
1696 watch_enable_addr(addr
, sizeof (*value
), S_READ
);
1703 suword64_nowatch(void *addr
, uint64_t value
)
1707 watched
= watch_disable_addr(addr
, sizeof (value
), S_WRITE
);
1708 ret
= suword64(addr
, value
);
1710 watch_enable_addr(addr
, sizeof (value
), S_WRITE
);
1717 suword32_nowatch(void *addr
, uint32_t value
)
1721 watched
= watch_disable_addr(addr
, sizeof (value
), S_WRITE
);
1722 ret
= suword32(addr
, value
);
1724 watch_enable_addr(addr
, sizeof (value
), S_WRITE
);
1730 watch_disable_addr(const void *addr
, size_t len
, enum seg_rw rw
)
1732 if (pr_watch_active(curproc
))
1733 return (pr_mappage((caddr_t
)addr
, len
, rw
, 1));
1738 watch_enable_addr(const void *addr
, size_t len
, enum seg_rw rw
)
1740 if (pr_watch_active(curproc
))
1741 pr_unmappage((caddr_t
)addr
, len
, rw
, 1);