1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License, Version 1.0 only 6 * (the "License"). You may not use this file except in compliance 7 * with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #include <sys/types.h> 28 #include <sys/t_lock.h> 29 #include <sys/param.h> 30 #include <sys/cred.h> 31 #include <sys/debug.h> 32 #include <sys/inline.h> 33 #include <sys/kmem.h> 34 #include <sys/proc.h> 35 #include <sys/regset.h> 36 #include <sys/sysmacros.h> 37 #include <sys/systm.h> 38 #include <sys/prsystm.h> 39 #include <sys/buf.h> 40 #include <sys/signal.h> 41 #include <sys/user.h> 42 #include <sys/cpuvar.h> 43 44 #include <sys/fault.h> 45 #include <sys/syscall.h> 46 #include <sys/procfs.h> 47 #include <sys/cmn_err.h> 48 #include <sys/stack.h> 49 #include <sys/watchpoint.h> 50 #include <sys/copyops.h> 51 #include <sys/schedctl.h> 52 53 #include <sys/mman.h> 54 #include <vm/as.h> 55 #include <vm/seg.h> 56 57 /* 58 * Copy ops vector for watchpoints. 59 */ 60 static int watch_copyin(const void *, void *, size_t); 61 static int watch_xcopyin(const void *, void *, size_t); 62 static int watch_copyout(const void *, void *, size_t); 63 static int watch_xcopyout(const void *, void *, size_t); 64 static int watch_copyinstr(const char *, char *, size_t, size_t *); 65 static int watch_copyoutstr(const char *, char *, size_t, size_t *); 66 static int watch_fuword8(const void *, uint8_t *); 67 static int watch_fuword16(const void *, uint16_t *); 68 static int watch_fuword32(const void *, uint32_t *); 69 static int watch_suword8(void *, uint8_t); 70 static int watch_suword16(void *, uint16_t); 71 static int watch_suword32(void *, uint32_t); 72 static int watch_physio(int (*)(struct buf *), struct buf *, 73 dev_t, int, void (*)(struct buf *), struct uio *); 74 #ifdef _LP64 75 static int watch_fuword64(const void *, uint64_t *); 76 static int watch_suword64(void *, uint64_t); 77 #endif 78 79 struct copyops watch_copyops = { 80 watch_copyin, 81 watch_xcopyin, 82 watch_copyout, 83 watch_xcopyout, 84 watch_copyinstr, 85 watch_copyoutstr, 86 watch_fuword8, 87 watch_fuword16, 88 watch_fuword32, 89 #ifdef _LP64 90 watch_fuword64, 91 #else 92 NULL, 93 #endif 94 watch_suword8, 95 watch_suword16, 96 watch_suword32, 97 #ifdef _LP64 98 watch_suword64, 99 #else 100 NULL, 101 #endif 102 watch_physio 103 }; 104 105 /* 106 * Map the 'rw' argument to a protection flag. 107 */ 108 static int 109 rw_to_prot(enum seg_rw rw) 110 { 111 switch (rw) { 112 case S_EXEC: 113 return (PROT_EXEC); 114 case S_READ: 115 return (PROT_READ); 116 case S_WRITE: 117 return (PROT_WRITE); 118 default: 119 return (PROT_NONE); /* can't happen */ 120 } 121 } 122 123 /* 124 * Map the 'rw' argument to an index into an array of exec/write/read things. 125 * The index follows the precedence order: exec .. write .. read 126 */ 127 static int 128 rw_to_index(enum seg_rw rw) 129 { 130 switch (rw) { 131 default: /* default case "can't happen" */ 132 case S_EXEC: 133 return (0); 134 case S_WRITE: 135 return (1); 136 case S_READ: 137 return (2); 138 } 139 } 140 141 /* 142 * Map an index back to a seg_rw. 143 */ 144 static enum seg_rw S_rw[4] = { 145 S_EXEC, 146 S_WRITE, 147 S_READ, 148 S_READ, 149 }; 150 151 #define X 0 152 #define W 1 153 #define R 2 154 #define sum(a) (a[X] + a[W] + a[R]) 155 156 /* 157 * Common code for pr_mappage() and pr_unmappage(). 158 */ 159 static int 160 pr_do_mappage(caddr_t addr, size_t size, int mapin, enum seg_rw rw, int kernel) 161 { 162 proc_t *p = curproc; 163 struct as *as = p->p_as; 164 char *eaddr = addr + size; 165 int prot_rw = rw_to_prot(rw); 166 int xrw = rw_to_index(rw); 167 int rv = 0; 168 struct watched_page *pwp; 169 struct watched_page tpw; 170 avl_index_t where; 171 uint_t prot; 172 173 ASSERT(as != &kas); 174 175 startover: 176 ASSERT(rv == 0); 177 if (avl_numnodes(&as->a_wpage) == 0) 178 return (0); 179 180 /* 181 * as->a_wpage can only be changed while the process is totally stopped. 182 * Don't grab p_lock here. Holding p_lock while grabbing the address 183 * space lock leads to deadlocks with the clock thread. 184 * 185 * p_maplock prevents simultaneous execution of this function. Under 186 * normal circumstances, holdwatch() will stop all other threads, so the 187 * lock isn't really needed. But there may be multiple threads within 188 * stop() when SWATCHOK is set, so we need to handle multiple threads 189 * at once. See holdwatch() for the details of this dance. 190 */ 191 192 mutex_enter(&p->p_maplock); 193 194 tpw.wp_vaddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); 195 if ((pwp = avl_find(&as->a_wpage, &tpw, &where)) == NULL) 196 pwp = avl_nearest(&as->a_wpage, where, AVL_AFTER); 197 198 for (; pwp != NULL && pwp->wp_vaddr < eaddr; 199 pwp = AVL_NEXT(&as->a_wpage, pwp)) { 200 201 /* 202 * If the requested protection has not been 203 * removed, we need not remap this page. 204 */ 205 prot = pwp->wp_prot; 206 if (kernel || (prot & PROT_USER)) 207 if (prot & prot_rw) 208 continue; 209 /* 210 * If the requested access does not exist in the page's 211 * original protections, we need not remap this page. 212 * If the page does not exist yet, we can't test it. 213 */ 214 if ((prot = pwp->wp_oprot) != 0) { 215 if (!(kernel || (prot & PROT_USER))) 216 continue; 217 if (!(prot & prot_rw)) 218 continue; 219 } 220 221 if (mapin) { 222 /* 223 * Before mapping the page in, ensure that 224 * all other lwps are held in the kernel. 225 */ 226 if (p->p_mapcnt == 0) { 227 mutex_exit(&p->p_maplock); 228 if (holdwatch() != 0) { 229 /* 230 * We stopped in holdwatch(). 231 * Start all over again because the 232 * watched page list may have changed. 233 */ 234 goto startover; 235 } 236 mutex_enter(&p->p_maplock); 237 } 238 p->p_mapcnt++; 239 } 240 241 addr = pwp->wp_vaddr; 242 rv++; 243 244 prot = pwp->wp_prot; 245 if (mapin) { 246 if (kernel) 247 pwp->wp_kmap[xrw]++; 248 else 249 pwp->wp_umap[xrw]++; 250 pwp->wp_flags |= WP_NOWATCH; 251 if (pwp->wp_kmap[X] + pwp->wp_umap[X]) 252 /* cannot have exec-only protection */ 253 prot |= PROT_READ|PROT_EXEC; 254 if (pwp->wp_kmap[R] + pwp->wp_umap[R]) 255 prot |= PROT_READ; 256 if (pwp->wp_kmap[W] + pwp->wp_umap[W]) 257 /* cannot have write-only protection */ 258 prot |= PROT_READ|PROT_WRITE; 259 #if 0 /* damned broken mmu feature! */ 260 if (sum(pwp->wp_umap) == 0) 261 prot &= ~PROT_USER; 262 #endif 263 } else { 264 ASSERT(pwp->wp_flags & WP_NOWATCH); 265 if (kernel) { 266 ASSERT(pwp->wp_kmap[xrw] != 0); 267 --pwp->wp_kmap[xrw]; 268 } else { 269 ASSERT(pwp->wp_umap[xrw] != 0); 270 --pwp->wp_umap[xrw]; 271 } 272 if (sum(pwp->wp_kmap) + sum(pwp->wp_umap) == 0) 273 pwp->wp_flags &= ~WP_NOWATCH; 274 else { 275 if (pwp->wp_kmap[X] + pwp->wp_umap[X]) 276 /* cannot have exec-only protection */ 277 prot |= PROT_READ|PROT_EXEC; 278 if (pwp->wp_kmap[R] + pwp->wp_umap[R]) 279 prot |= PROT_READ; 280 if (pwp->wp_kmap[W] + pwp->wp_umap[W]) 281 /* cannot have write-only protection */ 282 prot |= PROT_READ|PROT_WRITE; 283 #if 0 /* damned broken mmu feature! */ 284 if (sum(pwp->wp_umap) == 0) 285 prot &= ~PROT_USER; 286 #endif 287 } 288 } 289 290 291 if (pwp->wp_oprot != 0) { /* if page exists */ 292 struct seg *seg; 293 uint_t oprot; 294 int err, retrycnt = 0; 295 296 AS_LOCK_ENTER(as, RW_WRITER); 297 retry: 298 seg = as_segat(as, addr); 299 ASSERT(seg != NULL); 300 SEGOP_GETPROT(seg, addr, 0, &oprot); 301 if (prot != oprot) { 302 err = SEGOP_SETPROT(seg, addr, PAGESIZE, prot); 303 if (err == IE_RETRY) { 304 ASSERT(retrycnt == 0); 305 retrycnt++; 306 goto retry; 307 } 308 } 309 AS_LOCK_EXIT(as); 310 } 311 312 /* 313 * When all pages are mapped back to their normal state, 314 * continue the other lwps. 315 */ 316 if (!mapin) { 317 ASSERT(p->p_mapcnt > 0); 318 p->p_mapcnt--; 319 if (p->p_mapcnt == 0) { 320 mutex_exit(&p->p_maplock); 321 mutex_enter(&p->p_lock); 322 continuelwps(p); 323 mutex_exit(&p->p_lock); 324 mutex_enter(&p->p_maplock); 325 } 326 } 327 } 328 329 mutex_exit(&p->p_maplock); 330 331 return (rv); 332 } 333 334 /* 335 * Restore the original page protections on an address range. 336 * If 'kernel' is non-zero, just do it for the kernel. 337 * pr_mappage() returns non-zero if it actually changed anything. 338 * 339 * pr_mappage() and pr_unmappage() must be executed in matched pairs, 340 * but pairs may be nested within other pairs. The reference counts 341 * sort it all out. See pr_do_mappage(), above. 342 */ 343 static int 344 pr_mappage(const caddr_t addr, size_t size, enum seg_rw rw, int kernel) 345 { 346 return (pr_do_mappage(addr, size, 1, rw, kernel)); 347 } 348 349 /* 350 * Set the modified page protections on a watched page. 351 * Inverse of pr_mappage(). 352 * Needs to be called only if pr_mappage() returned non-zero. 353 */ 354 static void 355 pr_unmappage(const caddr_t addr, size_t size, enum seg_rw rw, int kernel) 356 { 357 (void) pr_do_mappage(addr, size, 0, rw, kernel); 358 } 359 360 /* 361 * Function called by an lwp after it resumes from stop(). 362 */ 363 void 364 setallwatch(void) 365 { 366 proc_t *p = curproc; 367 struct as *as = curproc->p_as; 368 struct watched_page *pwp, *next; 369 struct seg *seg; 370 caddr_t vaddr; 371 uint_t prot; 372 int err, retrycnt; 373 374 if (p->p_wprot == NULL) 375 return; 376 377 ASSERT(MUTEX_NOT_HELD(&curproc->p_lock)); 378 379 AS_LOCK_ENTER(as, RW_WRITER); 380 381 pwp = p->p_wprot; 382 while (pwp != NULL) { 383 384 vaddr = pwp->wp_vaddr; 385 retrycnt = 0; 386 retry: 387 ASSERT(pwp->wp_flags & WP_SETPROT); 388 if ((seg = as_segat(as, vaddr)) != NULL && 389 !(pwp->wp_flags & WP_NOWATCH)) { 390 prot = pwp->wp_prot; 391 err = SEGOP_SETPROT(seg, vaddr, PAGESIZE, prot); 392 if (err == IE_RETRY) { 393 ASSERT(retrycnt == 0); 394 retrycnt++; 395 goto retry; 396 } 397 } 398 399 next = pwp->wp_list; 400 401 if (pwp->wp_read + pwp->wp_write + pwp->wp_exec == 0) { 402 /* 403 * No watched areas remain in this page. 404 * Free the watched_page structure. 405 */ 406 avl_remove(&as->a_wpage, pwp); 407 kmem_free(pwp, sizeof (struct watched_page)); 408 } else { 409 pwp->wp_flags &= ~WP_SETPROT; 410 } 411 412 pwp = next; 413 } 414 p->p_wprot = NULL; 415 416 AS_LOCK_EXIT(as); 417 } 418 419 420 421 int 422 pr_is_watchpage_as(caddr_t addr, enum seg_rw rw, struct as *as) 423 { 424 register struct watched_page *pwp; 425 struct watched_page tpw; 426 uint_t prot; 427 int rv = 0; 428 429 switch (rw) { 430 case S_READ: 431 case S_WRITE: 432 case S_EXEC: 433 break; 434 default: 435 return (0); 436 } 437 438 /* 439 * as->a_wpage can only be modified while the process is totally 440 * stopped. We need, and should use, no locks here. 441 */ 442 if (as != &kas && avl_numnodes(&as->a_wpage) != 0) { 443 tpw.wp_vaddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); 444 pwp = avl_find(&as->a_wpage, &tpw, NULL); 445 if (pwp != NULL) { 446 ASSERT(addr >= pwp->wp_vaddr && 447 addr < pwp->wp_vaddr + PAGESIZE); 448 if (pwp->wp_oprot != 0) { 449 prot = pwp->wp_prot; 450 switch (rw) { 451 case S_READ: 452 rv = ((prot & (PROT_USER|PROT_READ)) 453 != (PROT_USER|PROT_READ)); 454 break; 455 case S_WRITE: 456 rv = ((prot & (PROT_USER|PROT_WRITE)) 457 != (PROT_USER|PROT_WRITE)); 458 break; 459 case S_EXEC: 460 rv = ((prot & (PROT_USER|PROT_EXEC)) 461 != (PROT_USER|PROT_EXEC)); 462 break; 463 default: 464 /* can't happen! */ 465 break; 466 } 467 } 468 } 469 } 470 471 return (rv); 472 } 473 474 475 /* 476 * trap() calls here to determine if a fault is in a watched page. 477 * We return nonzero if this is true and the load/store would fail. 478 */ 479 int 480 pr_is_watchpage(caddr_t addr, enum seg_rw rw) 481 { 482 struct as *as = curproc->p_as; 483 484 if ((as == &kas) || avl_numnodes(&as->a_wpage) == 0) 485 return (0); 486 487 return (pr_is_watchpage_as(addr, rw, as)); 488 } 489 490 491 492 /* 493 * trap() calls here to determine if a fault is a watchpoint. 494 */ 495 int 496 pr_is_watchpoint(caddr_t *paddr, int *pta, size_t size, size_t *plen, 497 enum seg_rw rw) 498 { 499 proc_t *p = curproc; 500 caddr_t addr = *paddr; 501 caddr_t eaddr = addr + size; 502 register struct watched_area *pwa; 503 struct watched_area twa; 504 int rv = 0; 505 int ta = 0; 506 size_t len = 0; 507 508 switch (rw) { 509 case S_READ: 510 case S_WRITE: 511 case S_EXEC: 512 break; 513 default: 514 *pta = 0; 515 return (0); 516 } 517 518 /* 519 * p->p_warea is protected by p->p_lock. 520 */ 521 mutex_enter(&p->p_lock); 522 523 /* BEGIN CSTYLED */ 524 /* 525 * This loop is somewhat complicated because the fault region can span 526 * multiple watched areas. For example: 527 * 528 * addr eaddr 529 * +-----------------+ 530 * | fault region | 531 * +-------+--------+----+---+------------+ 532 * | prot not right | | prot correct | 533 * +----------------+ +----------------+ 534 * wa_vaddr wa_eaddr 535 * wa_vaddr wa_eaddr 536 * 537 * We start at the area greater than or equal to the starting address. 538 * As long as some portion of the fault region overlaps the current 539 * area, we continue checking permissions until we find an appropriate 540 * match. 541 */ 542 /* END CSTYLED */ 543 twa.wa_vaddr = addr; 544 twa.wa_eaddr = eaddr; 545 546 for (pwa = pr_find_watched_area(p, &twa, NULL); 547 pwa != NULL && eaddr > pwa->wa_vaddr && addr < pwa->wa_eaddr; 548 pwa = AVL_NEXT(&p->p_warea, pwa)) { 549 550 switch (rw) { 551 case S_READ: 552 if (pwa->wa_flags & WA_READ) 553 rv = TRAP_RWATCH; 554 break; 555 case S_WRITE: 556 if (pwa->wa_flags & WA_WRITE) 557 rv = TRAP_WWATCH; 558 break; 559 case S_EXEC: 560 if (pwa->wa_flags & WA_EXEC) 561 rv = TRAP_XWATCH; 562 break; 563 default: 564 /* can't happen */ 565 break; 566 } 567 568 /* 569 * If protections didn't match, check the next watched 570 * area 571 */ 572 if (rv != 0) { 573 if (addr < pwa->wa_vaddr) 574 addr = pwa->wa_vaddr; 575 len = pwa->wa_eaddr - addr; 576 if (pwa->wa_flags & WA_TRAPAFTER) 577 ta = 1; 578 break; 579 } 580 } 581 582 mutex_exit(&p->p_lock); 583 584 *paddr = addr; 585 *pta = ta; 586 if (plen != NULL) 587 *plen = len; 588 return (rv); 589 } 590 591 /* 592 * Set up to perform a single-step at user level for the 593 * case of a trapafter watchpoint. Called from trap(). 594 */ 595 void 596 do_watch_step(caddr_t vaddr, size_t sz, enum seg_rw rw, 597 int watchcode, greg_t pc) 598 { 599 register klwp_t *lwp = ttolwp(curthread); 600 struct lwp_watch *pw = &lwp->lwp_watch[rw_to_index(rw)]; 601 602 /* 603 * Check to see if we are already performing this special 604 * watchpoint single-step. We must not do pr_mappage() twice. 605 */ 606 607 /* special check for two read traps on the same instruction */ 608 if (rw == S_READ && pw->wpaddr != NULL && 609 !(pw->wpaddr <= vaddr && vaddr < pw->wpaddr + pw->wpsize)) { 610 ASSERT(lwp->lwp_watchtrap != 0); 611 pw++; /* use the extra S_READ struct */ 612 } 613 614 if (pw->wpaddr != NULL) { 615 ASSERT(lwp->lwp_watchtrap != 0); 616 ASSERT(pw->wpaddr <= vaddr && vaddr < pw->wpaddr + pw->wpsize); 617 if (pw->wpcode == 0) { 618 pw->wpcode = watchcode; 619 pw->wppc = pc; 620 } 621 } else { 622 int mapped = pr_mappage(vaddr, sz, rw, 0); 623 prstep(lwp, 1); 624 lwp->lwp_watchtrap = 1; 625 pw->wpaddr = vaddr; 626 pw->wpsize = sz; 627 pw->wpcode = watchcode; 628 pw->wpmapped = mapped; 629 pw->wppc = pc; 630 } 631 } 632 633 /* 634 * Undo the effects of do_watch_step(). 635 * Called from trap() after the single-step is finished. 636 * Also called from issig_forreal() and stop() with a NULL 637 * argument to avoid having these things set more than once. 638 */ 639 int 640 undo_watch_step(k_siginfo_t *sip) 641 { 642 register klwp_t *lwp = ttolwp(curthread); 643 int fault = 0; 644 645 if (lwp->lwp_watchtrap) { 646 struct lwp_watch *pw = lwp->lwp_watch; 647 int i; 648 649 for (i = 0; i < 4; i++, pw++) { 650 if (pw->wpaddr == NULL) 651 continue; 652 if (pw->wpmapped) 653 pr_unmappage(pw->wpaddr, pw->wpsize, S_rw[i], 654 0); 655 if (pw->wpcode != 0) { 656 if (sip != NULL) { 657 sip->si_signo = SIGTRAP; 658 sip->si_code = pw->wpcode; 659 sip->si_addr = pw->wpaddr; 660 sip->si_trapafter = 1; 661 sip->si_pc = (caddr_t)pw->wppc; 662 } 663 fault = FLTWATCH; 664 pw->wpcode = 0; 665 } 666 pw->wpaddr = NULL; 667 pw->wpsize = 0; 668 pw->wpmapped = 0; 669 } 670 lwp->lwp_watchtrap = 0; 671 } 672 673 return (fault); 674 } 675 676 /* 677 * Handle a watchpoint that occurs while doing copyin() 678 * or copyout() in a system call. 679 * Return non-zero if the fault or signal is cleared 680 * by a debugger while the lwp is stopped. 681 */ 682 static int 683 sys_watchpoint(caddr_t addr, int watchcode, int ta) 684 { 685 extern greg_t getuserpc(void); /* XXX header file */ 686 k_sigset_t smask; 687 register proc_t *p = ttoproc(curthread); 688 register klwp_t *lwp = ttolwp(curthread); 689 register sigqueue_t *sqp; 690 int rval; 691 692 /* assert no locks are held */ 693 /* ASSERT(curthread->t_nlocks == 0); */ 694 695 sqp = kmem_zalloc(sizeof (sigqueue_t), KM_SLEEP); 696 sqp->sq_info.si_signo = SIGTRAP; 697 sqp->sq_info.si_code = watchcode; 698 sqp->sq_info.si_addr = addr; 699 sqp->sq_info.si_trapafter = ta; 700 sqp->sq_info.si_pc = (caddr_t)getuserpc(); 701 702 mutex_enter(&p->p_lock); 703 704 /* this will be tested and cleared by the caller */ 705 lwp->lwp_sysabort = 0; 706 707 if (prismember(&p->p_fltmask, FLTWATCH)) { 708 lwp->lwp_curflt = (uchar_t)FLTWATCH; 709 lwp->lwp_siginfo = sqp->sq_info; 710 stop(PR_FAULTED, FLTWATCH); 711 if (lwp->lwp_curflt == 0) { 712 mutex_exit(&p->p_lock); 713 kmem_free(sqp, sizeof (sigqueue_t)); 714 return (1); 715 } 716 lwp->lwp_curflt = 0; 717 } 718 719 /* 720 * post the SIGTRAP signal. 721 * Block all other signals so we only stop showing SIGTRAP. 722 */ 723 if (signal_is_blocked(curthread, SIGTRAP) || 724 sigismember(&p->p_ignore, SIGTRAP)) { 725 /* SIGTRAP is blocked or ignored, forget the rest. */ 726 mutex_exit(&p->p_lock); 727 kmem_free(sqp, sizeof (sigqueue_t)); 728 return (0); 729 } 730 sigdelq(p, curthread, SIGTRAP); 731 sigaddqa(p, curthread, sqp); 732 schedctl_finish_sigblock(curthread); 733 smask = curthread->t_hold; 734 sigfillset(&curthread->t_hold); 735 sigdiffset(&curthread->t_hold, &cantmask); 736 sigdelset(&curthread->t_hold, SIGTRAP); 737 mutex_exit(&p->p_lock); 738 739 rval = ((ISSIG_FAST(curthread, lwp, p, FORREAL))? 0 : 1); 740 741 /* restore the original signal mask */ 742 mutex_enter(&p->p_lock); 743 curthread->t_hold = smask; 744 mutex_exit(&p->p_lock); 745 746 return (rval); 747 } 748 749 /* 750 * Wrappers for the copyin()/copyout() functions to deal 751 * with watchpoints that fire while in system calls. 752 */ 753 754 static int 755 watch_xcopyin(const void *uaddr, void *kaddr, size_t count) 756 { 757 klwp_t *lwp = ttolwp(curthread); 758 caddr_t watch_uaddr = (caddr_t)uaddr; 759 caddr_t watch_kaddr = (caddr_t)kaddr; 760 int error = 0; 761 label_t ljb; 762 size_t part; 763 int mapped; 764 765 while (count && error == 0) { 766 int watchcode; 767 caddr_t vaddr; 768 size_t len; 769 int ta; 770 771 if ((part = PAGESIZE - 772 (((uintptr_t)uaddr) & PAGEOFFSET)) > count) 773 part = count; 774 775 if (!pr_is_watchpage(watch_uaddr, S_READ)) 776 watchcode = 0; 777 else { 778 vaddr = watch_uaddr; 779 watchcode = pr_is_watchpoint(&vaddr, &ta, 780 part, &len, S_READ); 781 if (watchcode && ta == 0) 782 part = vaddr - watch_uaddr; 783 } 784 785 /* 786 * Copy the initial part, up to a watched address, if any. 787 */ 788 if (part != 0) { 789 mapped = pr_mappage(watch_uaddr, part, S_READ, 1); 790 if (on_fault(&ljb)) 791 error = EFAULT; 792 else 793 copyin_noerr(watch_uaddr, watch_kaddr, part); 794 no_fault(); 795 if (mapped) 796 pr_unmappage(watch_uaddr, part, S_READ, 1); 797 watch_uaddr += part; 798 watch_kaddr += part; 799 count -= part; 800 } 801 /* 802 * If trapafter was specified, then copy through the 803 * watched area before taking the watchpoint trap. 804 */ 805 while (count && watchcode && ta && len > part && error == 0) { 806 len -= part; 807 if ((part = PAGESIZE) > count) 808 part = count; 809 if (part > len) 810 part = len; 811 mapped = pr_mappage(watch_uaddr, part, S_READ, 1); 812 if (on_fault(&ljb)) 813 error = EFAULT; 814 else 815 copyin_noerr(watch_uaddr, watch_kaddr, part); 816 no_fault(); 817 if (mapped) 818 pr_unmappage(watch_uaddr, part, S_READ, 1); 819 watch_uaddr += part; 820 watch_kaddr += part; 821 count -= part; 822 } 823 824 error: 825 /* if we hit a watched address, do the watchpoint logic */ 826 if (watchcode && 827 (!sys_watchpoint(vaddr, watchcode, ta) || 828 lwp->lwp_sysabort)) { 829 lwp->lwp_sysabort = 0; 830 error = EFAULT; 831 break; 832 } 833 } 834 835 return (error); 836 } 837 838 static int 839 watch_copyin(const void *kaddr, void *uaddr, size_t count) 840 { 841 return (watch_xcopyin(kaddr, uaddr, count) ? -1 : 0); 842 } 843 844 845 static int 846 watch_xcopyout(const void *kaddr, void *uaddr, size_t count) 847 { 848 klwp_t *lwp = ttolwp(curthread); 849 caddr_t watch_uaddr = (caddr_t)uaddr; 850 caddr_t watch_kaddr = (caddr_t)kaddr; 851 int error = 0; 852 label_t ljb; 853 854 while (count && error == 0) { 855 int watchcode; 856 caddr_t vaddr; 857 size_t part; 858 size_t len; 859 int ta; 860 int mapped; 861 862 if ((part = PAGESIZE - 863 (((uintptr_t)uaddr) & PAGEOFFSET)) > count) 864 part = count; 865 866 if (!pr_is_watchpage(watch_uaddr, S_WRITE)) 867 watchcode = 0; 868 else { 869 vaddr = watch_uaddr; 870 watchcode = pr_is_watchpoint(&vaddr, &ta, 871 part, &len, S_WRITE); 872 if (watchcode) { 873 if (ta == 0) 874 part = vaddr - watch_uaddr; 875 else { 876 len += vaddr - watch_uaddr; 877 if (part > len) 878 part = len; 879 } 880 } 881 } 882 883 /* 884 * Copy the initial part, up to a watched address, if any. 885 */ 886 if (part != 0) { 887 mapped = pr_mappage(watch_uaddr, part, S_WRITE, 1); 888 if (on_fault(&ljb)) 889 error = EFAULT; 890 else 891 copyout_noerr(watch_kaddr, watch_uaddr, part); 892 no_fault(); 893 if (mapped) 894 pr_unmappage(watch_uaddr, part, S_WRITE, 1); 895 watch_uaddr += part; 896 watch_kaddr += part; 897 count -= part; 898 } 899 900 /* 901 * If trapafter was specified, then copy through the 902 * watched area before taking the watchpoint trap. 903 */ 904 while (count && watchcode && ta && len > part && error == 0) { 905 len -= part; 906 if ((part = PAGESIZE) > count) 907 part = count; 908 if (part > len) 909 part = len; 910 mapped = pr_mappage(watch_uaddr, part, S_WRITE, 1); 911 if (on_fault(&ljb)) 912 error = EFAULT; 913 else 914 copyout_noerr(watch_kaddr, watch_uaddr, part); 915 no_fault(); 916 if (mapped) 917 pr_unmappage(watch_uaddr, part, S_WRITE, 1); 918 watch_uaddr += part; 919 watch_kaddr += part; 920 count -= part; 921 } 922 923 /* if we hit a watched address, do the watchpoint logic */ 924 if (watchcode && 925 (!sys_watchpoint(vaddr, watchcode, ta) || 926 lwp->lwp_sysabort)) { 927 lwp->lwp_sysabort = 0; 928 error = EFAULT; 929 break; 930 } 931 } 932 933 return (error); 934 } 935 936 static int 937 watch_copyout(const void *kaddr, void *uaddr, size_t count) 938 { 939 return (watch_xcopyout(kaddr, uaddr, count) ? -1 : 0); 940 } 941 942 static int 943 watch_copyinstr( 944 const char *uaddr, 945 char *kaddr, 946 size_t maxlength, 947 size_t *lencopied) 948 { 949 klwp_t *lwp = ttolwp(curthread); 950 size_t resid; 951 int error = 0; 952 label_t ljb; 953 954 if ((resid = maxlength) == 0) 955 return (ENAMETOOLONG); 956 957 while (resid && error == 0) { 958 int watchcode; 959 caddr_t vaddr; 960 size_t part; 961 size_t len; 962 size_t size; 963 int ta; 964 int mapped; 965 966 if ((part = PAGESIZE - 967 (((uintptr_t)uaddr) & PAGEOFFSET)) > resid) 968 part = resid; 969 970 if (!pr_is_watchpage((caddr_t)uaddr, S_READ)) 971 watchcode = 0; 972 else { 973 vaddr = (caddr_t)uaddr; 974 watchcode = pr_is_watchpoint(&vaddr, &ta, 975 part, &len, S_READ); 976 if (watchcode) { 977 if (ta == 0) 978 part = vaddr - uaddr; 979 else { 980 len += vaddr - uaddr; 981 if (part > len) 982 part = len; 983 } 984 } 985 } 986 987 /* 988 * Copy the initial part, up to a watched address, if any. 989 */ 990 if (part != 0) { 991 mapped = pr_mappage((caddr_t)uaddr, part, S_READ, 1); 992 if (on_fault(&ljb)) 993 error = EFAULT; 994 else 995 error = copyinstr_noerr(uaddr, kaddr, part, 996 &size); 997 no_fault(); 998 if (mapped) 999 pr_unmappage((caddr_t)uaddr, part, S_READ, 1); 1000 uaddr += size; 1001 kaddr += size; 1002 resid -= size; 1003 if (error == ENAMETOOLONG && resid > 0) 1004 error = 0; 1005 if (error != 0 || (watchcode && 1006 (uaddr < vaddr || kaddr[-1] == '\0'))) 1007 break; /* didn't reach the watched area */ 1008 } 1009 1010 /* 1011 * If trapafter was specified, then copy through the 1012 * watched area before taking the watchpoint trap. 1013 */ 1014 while (resid && watchcode && ta && len > part && error == 0 && 1015 size == part && kaddr[-1] != '\0') { 1016 len -= part; 1017 if ((part = PAGESIZE) > resid) 1018 part = resid; 1019 if (part > len) 1020 part = len; 1021 mapped = pr_mappage((caddr_t)uaddr, part, S_READ, 1); 1022 if (on_fault(&ljb)) 1023 error = EFAULT; 1024 else 1025 error = copyinstr_noerr(uaddr, kaddr, part, 1026 &size); 1027 no_fault(); 1028 if (mapped) 1029 pr_unmappage((caddr_t)uaddr, part, S_READ, 1); 1030 uaddr += size; 1031 kaddr += size; 1032 resid -= size; 1033 if (error == ENAMETOOLONG && resid > 0) 1034 error = 0; 1035 } 1036 1037 /* if we hit a watched address, do the watchpoint logic */ 1038 if (watchcode && 1039 (!sys_watchpoint(vaddr, watchcode, ta) || 1040 lwp->lwp_sysabort)) { 1041 lwp->lwp_sysabort = 0; 1042 error = EFAULT; 1043 break; 1044 } 1045 1046 if (error == 0 && part != 0 && 1047 (size < part || kaddr[-1] == '\0')) 1048 break; 1049 } 1050 1051 if (error != EFAULT && lencopied) 1052 *lencopied = maxlength - resid; 1053 return (error); 1054 } 1055 1056 static int 1057 watch_copyoutstr( 1058 const char *kaddr, 1059 char *uaddr, 1060 size_t maxlength, 1061 size_t *lencopied) 1062 { 1063 klwp_t *lwp = ttolwp(curthread); 1064 size_t resid; 1065 int error = 0; 1066 label_t ljb; 1067 1068 if ((resid = maxlength) == 0) 1069 return (ENAMETOOLONG); 1070 1071 while (resid && error == 0) { 1072 int watchcode; 1073 caddr_t vaddr; 1074 size_t part; 1075 size_t len; 1076 size_t size; 1077 int ta; 1078 int mapped; 1079 1080 if ((part = PAGESIZE - 1081 (((uintptr_t)uaddr) & PAGEOFFSET)) > resid) 1082 part = resid; 1083 1084 if (!pr_is_watchpage(uaddr, S_WRITE)) { 1085 watchcode = 0; 1086 } else { 1087 vaddr = uaddr; 1088 watchcode = pr_is_watchpoint(&vaddr, &ta, 1089 part, &len, S_WRITE); 1090 if (watchcode && ta == 0) 1091 part = vaddr - uaddr; 1092 } 1093 1094 /* 1095 * Copy the initial part, up to a watched address, if any. 1096 */ 1097 if (part != 0) { 1098 mapped = pr_mappage(uaddr, part, S_WRITE, 1); 1099 if (on_fault(&ljb)) 1100 error = EFAULT; 1101 else 1102 error = copyoutstr_noerr(kaddr, uaddr, part, 1103 &size); 1104 no_fault(); 1105 if (mapped) 1106 pr_unmappage(uaddr, part, S_WRITE, 1); 1107 uaddr += size; 1108 kaddr += size; 1109 resid -= size; 1110 if (error == ENAMETOOLONG && resid > 0) 1111 error = 0; 1112 if (error != 0 || (watchcode && 1113 (uaddr < vaddr || kaddr[-1] == '\0'))) 1114 break; /* didn't reach the watched area */ 1115 } 1116 1117 /* 1118 * If trapafter was specified, then copy through the 1119 * watched area before taking the watchpoint trap. 1120 */ 1121 while (resid && watchcode && ta && len > part && error == 0 && 1122 size == part && kaddr[-1] != '\0') { 1123 len -= part; 1124 if ((part = PAGESIZE) > resid) 1125 part = resid; 1126 if (part > len) 1127 part = len; 1128 mapped = pr_mappage(uaddr, part, S_WRITE, 1); 1129 if (on_fault(&ljb)) 1130 error = EFAULT; 1131 else 1132 error = copyoutstr_noerr(kaddr, uaddr, part, 1133 &size); 1134 no_fault(); 1135 if (mapped) 1136 pr_unmappage(uaddr, part, S_WRITE, 1); 1137 uaddr += size; 1138 kaddr += size; 1139 resid -= size; 1140 if (error == ENAMETOOLONG && resid > 0) 1141 error = 0; 1142 } 1143 1144 /* if we hit a watched address, do the watchpoint logic */ 1145 if (watchcode && 1146 (!sys_watchpoint(vaddr, watchcode, ta) || 1147 lwp->lwp_sysabort)) { 1148 lwp->lwp_sysabort = 0; 1149 error = EFAULT; 1150 break; 1151 } 1152 1153 if (error == 0 && part != 0 && 1154 (size < part || kaddr[-1] == '\0')) 1155 break; 1156 } 1157 1158 if (error != EFAULT && lencopied) 1159 *lencopied = maxlength - resid; 1160 return (error); 1161 } 1162 1163 typedef void (*fuword_func)(const void *, void *); 1164 1165 /* 1166 * Generic form of watch_fuword8(), watch_fuword16(), etc. 1167 */ 1168 static int 1169 watch_fuword(const void *addr, void *dst, fuword_func func, size_t size) 1170 { 1171 klwp_t *lwp = ttolwp(curthread); 1172 int watchcode; 1173 caddr_t vaddr; 1174 int mapped; 1175 int rv = 0; 1176 int ta; 1177 label_t ljb; 1178 1179 for (;;) { 1180 1181 vaddr = (caddr_t)addr; 1182 watchcode = pr_is_watchpoint(&vaddr, &ta, size, NULL, S_READ); 1183 if (watchcode == 0 || ta != 0) { 1184 mapped = pr_mappage((caddr_t)addr, size, S_READ, 1); 1185 if (on_fault(&ljb)) 1186 rv = -1; 1187 else 1188 (*func)(addr, dst); 1189 no_fault(); 1190 if (mapped) 1191 pr_unmappage((caddr_t)addr, size, S_READ, 1); 1192 } 1193 if (watchcode && 1194 (!sys_watchpoint(vaddr, watchcode, ta) || 1195 lwp->lwp_sysabort)) { 1196 lwp->lwp_sysabort = 0; 1197 rv = -1; 1198 break; 1199 } 1200 if (watchcode == 0 || ta != 0) 1201 break; 1202 } 1203 1204 return (rv); 1205 } 1206 1207 static int 1208 watch_fuword8(const void *addr, uint8_t *dst) 1209 { 1210 return (watch_fuword(addr, dst, (fuword_func)fuword8_noerr, 1211 sizeof (*dst))); 1212 } 1213 1214 static int 1215 watch_fuword16(const void *addr, uint16_t *dst) 1216 { 1217 return (watch_fuword(addr, dst, (fuword_func)fuword16_noerr, 1218 sizeof (*dst))); 1219 } 1220 1221 static int 1222 watch_fuword32(const void *addr, uint32_t *dst) 1223 { 1224 return (watch_fuword(addr, dst, (fuword_func)fuword32_noerr, 1225 sizeof (*dst))); 1226 } 1227 1228 #ifdef _LP64 1229 static int 1230 watch_fuword64(const void *addr, uint64_t *dst) 1231 { 1232 return (watch_fuword(addr, dst, (fuword_func)fuword64_noerr, 1233 sizeof (*dst))); 1234 } 1235 #endif 1236 1237 1238 static int 1239 watch_suword8(void *addr, uint8_t value) 1240 { 1241 klwp_t *lwp = ttolwp(curthread); 1242 int watchcode; 1243 caddr_t vaddr; 1244 int mapped; 1245 int rv = 0; 1246 int ta; 1247 label_t ljb; 1248 1249 for (;;) { 1250 1251 vaddr = (caddr_t)addr; 1252 watchcode = pr_is_watchpoint(&vaddr, &ta, sizeof (value), NULL, 1253 S_WRITE); 1254 if (watchcode == 0 || ta != 0) { 1255 mapped = pr_mappage((caddr_t)addr, sizeof (value), 1256 S_WRITE, 1); 1257 if (on_fault(&ljb)) 1258 rv = -1; 1259 else 1260 suword8_noerr(addr, value); 1261 no_fault(); 1262 if (mapped) 1263 pr_unmappage((caddr_t)addr, sizeof (value), 1264 S_WRITE, 1); 1265 } 1266 if (watchcode && 1267 (!sys_watchpoint(vaddr, watchcode, ta) || 1268 lwp->lwp_sysabort)) { 1269 lwp->lwp_sysabort = 0; 1270 rv = -1; 1271 break; 1272 } 1273 if (watchcode == 0 || ta != 0) 1274 break; 1275 } 1276 1277 return (rv); 1278 } 1279 1280 static int 1281 watch_suword16(void *addr, uint16_t value) 1282 { 1283 klwp_t *lwp = ttolwp(curthread); 1284 int watchcode; 1285 caddr_t vaddr; 1286 int mapped; 1287 int rv = 0; 1288 int ta; 1289 label_t ljb; 1290 1291 for (;;) { 1292 1293 vaddr = (caddr_t)addr; 1294 watchcode = pr_is_watchpoint(&vaddr, &ta, sizeof (value), NULL, 1295 S_WRITE); 1296 if (watchcode == 0 || ta != 0) { 1297 mapped = pr_mappage((caddr_t)addr, sizeof (value), 1298 S_WRITE, 1); 1299 if (on_fault(&ljb)) 1300 rv = -1; 1301 else 1302 suword16_noerr(addr, value); 1303 no_fault(); 1304 if (mapped) 1305 pr_unmappage((caddr_t)addr, sizeof (value), 1306 S_WRITE, 1); 1307 } 1308 if (watchcode && 1309 (!sys_watchpoint(vaddr, watchcode, ta) || 1310 lwp->lwp_sysabort)) { 1311 lwp->lwp_sysabort = 0; 1312 rv = -1; 1313 break; 1314 } 1315 if (watchcode == 0 || ta != 0) 1316 break; 1317 } 1318 1319 return (rv); 1320 } 1321 1322 static int 1323 watch_suword32(void *addr, uint32_t value) 1324 { 1325 klwp_t *lwp = ttolwp(curthread); 1326 int watchcode; 1327 caddr_t vaddr; 1328 int mapped; 1329 int rv = 0; 1330 int ta; 1331 label_t ljb; 1332 1333 for (;;) { 1334 1335 vaddr = (caddr_t)addr; 1336 watchcode = pr_is_watchpoint(&vaddr, &ta, sizeof (value), NULL, 1337 S_WRITE); 1338 if (watchcode == 0 || ta != 0) { 1339 mapped = pr_mappage((caddr_t)addr, sizeof (value), 1340 S_WRITE, 1); 1341 if (on_fault(&ljb)) 1342 rv = -1; 1343 else 1344 suword32_noerr(addr, value); 1345 no_fault(); 1346 if (mapped) 1347 pr_unmappage((caddr_t)addr, sizeof (value), 1348 S_WRITE, 1); 1349 } 1350 if (watchcode && 1351 (!sys_watchpoint(vaddr, watchcode, ta) || 1352 lwp->lwp_sysabort)) { 1353 lwp->lwp_sysabort = 0; 1354 rv = -1; 1355 break; 1356 } 1357 if (watchcode == 0 || ta != 0) 1358 break; 1359 } 1360 1361 return (rv); 1362 } 1363 1364 #ifdef _LP64 1365 static int 1366 watch_suword64(void *addr, uint64_t value) 1367 { 1368 klwp_t *lwp = ttolwp(curthread); 1369 int watchcode; 1370 caddr_t vaddr; 1371 int mapped; 1372 int rv = 0; 1373 int ta; 1374 label_t ljb; 1375 1376 for (;;) { 1377 1378 vaddr = (caddr_t)addr; 1379 watchcode = pr_is_watchpoint(&vaddr, &ta, sizeof (value), NULL, 1380 S_WRITE); 1381 if (watchcode == 0 || ta != 0) { 1382 mapped = pr_mappage((caddr_t)addr, sizeof (value), 1383 S_WRITE, 1); 1384 if (on_fault(&ljb)) 1385 rv = -1; 1386 else 1387 suword64_noerr(addr, value); 1388 no_fault(); 1389 if (mapped) 1390 pr_unmappage((caddr_t)addr, sizeof (value), 1391 S_WRITE, 1); 1392 } 1393 if (watchcode && 1394 (!sys_watchpoint(vaddr, watchcode, ta) || 1395 lwp->lwp_sysabort)) { 1396 lwp->lwp_sysabort = 0; 1397 rv = -1; 1398 break; 1399 } 1400 if (watchcode == 0 || ta != 0) 1401 break; 1402 } 1403 1404 return (rv); 1405 } 1406 #endif /* _LP64 */ 1407 1408 /* 1409 * Check for watched addresses in the given address space. 1410 * Return 1 if this is true, otherwise 0. 1411 */ 1412 static int 1413 pr_is_watched(caddr_t base, size_t len, int rw) 1414 { 1415 caddr_t saddr = (caddr_t)((uintptr_t)base & (uintptr_t)PAGEMASK); 1416 caddr_t eaddr = base + len; 1417 caddr_t paddr; 1418 1419 for (paddr = saddr; paddr < eaddr; paddr += PAGESIZE) { 1420 if (pr_is_watchpage(paddr, rw)) 1421 return (1); 1422 } 1423 1424 return (0); 1425 } 1426 1427 /* 1428 * Wrapper for the physio() function. 1429 * Splits one uio operation with multiple iovecs into uio operations with 1430 * only one iovecs to do the watchpoint handling separately for each iovecs. 1431 */ 1432 static int 1433 watch_physio(int (*strat)(struct buf *), struct buf *bp, dev_t dev, 1434 int rw, void (*mincnt)(struct buf *), struct uio *uio) 1435 { 1436 struct uio auio; 1437 struct iovec *iov; 1438 caddr_t base; 1439 size_t len; 1440 int seg_rw; 1441 int error = 0; 1442 1443 if (uio->uio_segflg == UIO_SYSSPACE) 1444 return (default_physio(strat, bp, dev, rw, mincnt, uio)); 1445 1446 seg_rw = (rw == B_READ) ? S_WRITE : S_READ; 1447 1448 while (uio->uio_iovcnt > 0) { 1449 if (uio->uio_resid == 0) { 1450 /* 1451 * Make sure to return the uio structure with the 1452 * same values as default_physio() does. 1453 */ 1454 uio->uio_iov++; 1455 uio->uio_iovcnt--; 1456 continue; 1457 } 1458 1459 iov = uio->uio_iov; 1460 len = MIN(iov->iov_len, uio->uio_resid); 1461 1462 auio.uio_iovcnt = 1; 1463 auio.uio_iov = iov; 1464 auio.uio_resid = len; 1465 auio.uio_loffset = uio->uio_loffset; 1466 auio.uio_llimit = uio->uio_llimit; 1467 auio.uio_fmode = uio->uio_fmode; 1468 auio.uio_extflg = uio->uio_extflg; 1469 auio.uio_segflg = uio->uio_segflg; 1470 1471 base = iov->iov_base; 1472 1473 if (!pr_is_watched(base, len, seg_rw)) { 1474 /* 1475 * The given memory references don't cover a 1476 * watched page. 1477 */ 1478 error = default_physio(strat, bp, dev, rw, mincnt, 1479 &auio); 1480 1481 /* Update uio with values from auio. */ 1482 len -= auio.uio_resid; 1483 uio->uio_resid -= len; 1484 uio->uio_loffset += len; 1485 1486 /* 1487 * Return if an error occurred or not all data 1488 * was copied. 1489 */ 1490 if (auio.uio_resid || error) 1491 break; 1492 uio->uio_iov++; 1493 uio->uio_iovcnt--; 1494 } else { 1495 int mapped, watchcode, ta; 1496 caddr_t vaddr = base; 1497 klwp_t *lwp = ttolwp(curthread); 1498 1499 watchcode = pr_is_watchpoint(&vaddr, &ta, len, 1500 NULL, seg_rw); 1501 1502 if (watchcode == 0 || ta != 0) { 1503 /* 1504 * Do the io if the given memory references 1505 * don't cover a watched area (watchcode=0) 1506 * or if WA_TRAPAFTER was specified. 1507 */ 1508 mapped = pr_mappage(base, len, seg_rw, 1); 1509 error = default_physio(strat, bp, dev, rw, 1510 mincnt, &auio); 1511 if (mapped) 1512 pr_unmappage(base, len, seg_rw, 1); 1513 1514 len -= auio.uio_resid; 1515 uio->uio_resid -= len; 1516 uio->uio_loffset += len; 1517 } 1518 1519 /* 1520 * If we hit a watched address, do the watchpoint logic. 1521 */ 1522 if (watchcode && 1523 (!sys_watchpoint(vaddr, watchcode, ta) || 1524 lwp->lwp_sysabort)) { 1525 lwp->lwp_sysabort = 0; 1526 return (EFAULT); 1527 } 1528 1529 /* 1530 * Check for errors from default_physio(). 1531 */ 1532 if (watchcode == 0 || ta != 0) { 1533 if (auio.uio_resid || error) 1534 break; 1535 uio->uio_iov++; 1536 uio->uio_iovcnt--; 1537 } 1538 } 1539 } 1540 1541 return (error); 1542 } 1543 1544 int 1545 wa_compare(const void *a, const void *b) 1546 { 1547 const watched_area_t *pa = a; 1548 const watched_area_t *pb = b; 1549 1550 if (pa->wa_vaddr < pb->wa_vaddr) 1551 return (-1); 1552 else if (pa->wa_vaddr > pb->wa_vaddr) 1553 return (1); 1554 else 1555 return (0); 1556 } 1557 1558 int 1559 wp_compare(const void *a, const void *b) 1560 { 1561 const watched_page_t *pa = a; 1562 const watched_page_t *pb = b; 1563 1564 if (pa->wp_vaddr < pb->wp_vaddr) 1565 return (-1); 1566 else if (pa->wp_vaddr > pb->wp_vaddr) 1567 return (1); 1568 else 1569 return (0); 1570 } 1571 1572 /* 1573 * Given an address range, finds the first watched area which overlaps some or 1574 * all of the range. 1575 */ 1576 watched_area_t * 1577 pr_find_watched_area(proc_t *p, watched_area_t *pwa, avl_index_t *where) 1578 { 1579 caddr_t vaddr = pwa->wa_vaddr; 1580 caddr_t eaddr = pwa->wa_eaddr; 1581 watched_area_t *wap; 1582 avl_index_t real_where; 1583 1584 /* First, check if there is an exact match. */ 1585 wap = avl_find(&p->p_warea, pwa, &real_where); 1586 1587 1588 /* Check to see if we overlap with the previous area. */ 1589 if (wap == NULL) { 1590 wap = avl_nearest(&p->p_warea, real_where, AVL_BEFORE); 1591 if (wap != NULL && 1592 (vaddr >= wap->wa_eaddr || eaddr <= wap->wa_vaddr)) 1593 wap = NULL; 1594 } 1595 1596 /* Try the next area. */ 1597 if (wap == NULL) { 1598 wap = avl_nearest(&p->p_warea, real_where, AVL_AFTER); 1599 if (wap != NULL && 1600 (vaddr >= wap->wa_eaddr || eaddr <= wap->wa_vaddr)) 1601 wap = NULL; 1602 } 1603 1604 if (where) 1605 *where = real_where; 1606 1607 return (wap); 1608 } 1609 1610 void 1611 watch_enable(kthread_id_t t) 1612 { 1613 t->t_proc_flag |= TP_WATCHPT; 1614 install_copyops(t, &watch_copyops); 1615 } 1616 1617 void 1618 watch_disable(kthread_id_t t) 1619 { 1620 t->t_proc_flag &= ~TP_WATCHPT; 1621 remove_copyops(t); 1622 } 1623 1624 int 1625 copyin_nowatch(const void *uaddr, void *kaddr, size_t len) 1626 { 1627 int watched, ret; 1628 1629 watched = watch_disable_addr(uaddr, len, S_READ); 1630 ret = copyin(uaddr, kaddr, len); 1631 if (watched) 1632 watch_enable_addr(uaddr, len, S_READ); 1633 1634 return (ret); 1635 } 1636 1637 int 1638 copyout_nowatch(const void *kaddr, void *uaddr, size_t len) 1639 { 1640 int watched, ret; 1641 1642 watched = watch_disable_addr(uaddr, len, S_WRITE); 1643 ret = copyout(kaddr, uaddr, len); 1644 if (watched) 1645 watch_enable_addr(uaddr, len, S_WRITE); 1646 1647 return (ret); 1648 } 1649 1650 #ifdef _LP64 1651 int 1652 fuword64_nowatch(const void *addr, uint64_t *value) 1653 { 1654 int watched, ret; 1655 1656 watched = watch_disable_addr(addr, sizeof (*value), S_READ); 1657 ret = fuword64(addr, value); 1658 if (watched) 1659 watch_enable_addr(addr, sizeof (*value), S_READ); 1660 1661 return (ret); 1662 } 1663 #endif 1664 1665 int 1666 fuword32_nowatch(const void *addr, uint32_t *value) 1667 { 1668 int watched, ret; 1669 1670 watched = watch_disable_addr(addr, sizeof (*value), S_READ); 1671 ret = fuword32(addr, value); 1672 if (watched) 1673 watch_enable_addr(addr, sizeof (*value), S_READ); 1674 1675 return (ret); 1676 } 1677 1678 #ifdef _LP64 1679 int 1680 suword64_nowatch(void *addr, uint64_t value) 1681 { 1682 int watched, ret; 1683 1684 watched = watch_disable_addr(addr, sizeof (value), S_WRITE); 1685 ret = suword64(addr, value); 1686 if (watched) 1687 watch_enable_addr(addr, sizeof (value), S_WRITE); 1688 1689 return (ret); 1690 } 1691 #endif 1692 1693 int 1694 suword32_nowatch(void *addr, uint32_t value) 1695 { 1696 int watched, ret; 1697 1698 watched = watch_disable_addr(addr, sizeof (value), S_WRITE); 1699 ret = suword32(addr, value); 1700 if (watched) 1701 watch_enable_addr(addr, sizeof (value), S_WRITE); 1702 1703 return (ret); 1704 } 1705 1706 int 1707 watch_disable_addr(const void *addr, size_t len, enum seg_rw rw) 1708 { 1709 if (pr_watch_active(curproc)) 1710 return (pr_mappage((caddr_t)addr, len, rw, 1)); 1711 return (0); 1712 } 1713 1714 void 1715 watch_enable_addr(const void *addr, size_t len, enum seg_rw rw) 1716 { 1717 if (pr_watch_active(curproc)) 1718 pr_unmappage((caddr_t)addr, len, rw, 1); 1719 } 1720