usr/src/uts/intel/ia32/ml/lock_prim.s

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  23  * Use is subject to license terms.
  24  */
  25
  26 #pragma ident   "%Z%%M% %I%     %E% SMI"
  27
  28 #if defined(lint) || defined(__lint)
  29 #include <sys/types.h>
  30 #include <sys/thread.h>
  31 #include <sys/cpuvar.h>
  32 #include <vm/page.h>
  33 #else   /* __lint */
  34 #include "assym.h"
  35 #endif  /* __lint */
  36
  37 #include <sys/mutex_impl.h>
  38 #include <sys/asm_linkage.h>
  39 #include <sys/asm_misc.h>
  40 #include <sys/regset.h>
  41 #include <sys/rwlock_impl.h>
  42 #include <sys/lockstat.h>
  43
  44 /*
  45  * lock_try(lp), ulock_try(lp)
  46  *      - returns non-zero on success.
  47  *      - doesn't block interrupts so don't use this to spin on a lock.
  48  *
  49  * ulock_try() is for a lock in the user address space.
  50  */
  51
  52 #if defined(lint) || defined(__lint)
  53
  54 /* ARGSUSED */
  55 int
  56 lock_try(lock_t *lp)
  57 { return (0); }
  58
  59 /* ARGSUSED */
  60 int
  61 lock_spin_try(lock_t *lp)
  62 { return (0); }
  63
  64 /* ARGSUSED */
  65 int
  66 ulock_try(lock_t *lp)
  67 { return (0); }
  68
  69 #else   /* __lint */
  70         .globl  kernelbase
  71
  72 #if defined(__amd64)
  73
  74         ENTRY(lock_try)
  75         movb    $-1, %dl
  76         movzbq  %dl, %rax
  77         xchgb   %dl, (%rdi)
  78         xorb    %dl, %al
  79 .lock_try_lockstat_patch_point:
  80         ret
  81         testb   %al, %al
  82         jnz     0f
  83         ret
  84 0:
  85         movq    %gs:CPU_THREAD, %rdx    /* rdx = thread addr */
  86         movq    %rdi, %rsi              /* rsi = lock addr */
  87         movl    $LS_LOCK_TRY_ACQUIRE, %edi /* edi = event */
  88         jmp     lockstat_wrapper
  89         SET_SIZE(lock_try)
  90
  91         ENTRY(lock_spin_try)
  92         movb    $-1, %dl
  93         movzbq  %dl, %rax
  94         xchgb   %dl, (%rdi)
  95         xorb    %dl, %al
  96         ret
  97         SET_SIZE(lock_spin_try)
  98
  99         ENTRY(ulock_try)
 100 #ifdef DEBUG
 101         movq    kernelbase(%rip), %rax
 102         cmpq    %rax, %rdi              /* test uaddr < kernelbase */
 103         jb      ulock_pass              /*      uaddr < kernelbase, proceed */
 104
 105         movq    %rdi, %r12              /* preserve lock ptr for debugging */
 106         leaq    .ulock_panic_msg(%rip), %rdi
 107         pushq   %rbp                    /* align stack properly */
 108         movq    %rsp, %rbp
 109         xorl    %eax, %eax              /* clear for varargs */
 110         call    panic
 111
 112 #endif /* DEBUG */
 113
 114 ulock_pass:
 115         movl    $1, %eax
 116         xchgb   %al, (%rdi)
 117         xorb    $1, %al
 118         ret
 119         SET_SIZE(ulock_try)
 120
 121 #else
 122
 123         ENTRY(lock_try)
 124         movl    $1,%edx
 125         movl    4(%esp),%ecx            /* ecx = lock addr */
 126         xorl    %eax,%eax
 127         xchgb   %dl, (%ecx)             /* using dl will avoid partial */
 128         testb   %dl,%dl                 /* stalls on P6 ? */
 129         setz    %al
 130 .lock_try_lockstat_patch_point:
 131         ret
 132         movl    %gs:CPU_THREAD, %edx    /* edx = thread addr */
 133         testl   %eax, %eax
 134         jz      0f
 135         movl    $LS_LOCK_TRY_ACQUIRE, %eax
 136         jmp     lockstat_wrapper
 137 0:
 138         ret
 139         SET_SIZE(lock_try)
 140
 141         ENTRY(lock_spin_try)
 142         movl    $-1,%edx
 143         movl    4(%esp),%ecx            /* ecx = lock addr */
 144         xorl    %eax,%eax
 145         xchgb   %dl, (%ecx)             /* using dl will avoid partial */
 146         testb   %dl,%dl                 /* stalls on P6 ? */
 147         setz    %al
 148         ret
 149         SET_SIZE(lock_spin_try)
 150
 151         ENTRY(ulock_try)
 152 #ifdef DEBUG
 153         movl    kernelbase, %eax
 154         cmpl    %eax, 4(%esp)           /* test uaddr < kernelbase */
 155         jb      ulock_pass              /* uaddr < kernelbase, proceed */
 156
 157         pushl   $.ulock_panic_msg
 158         call    panic
 159
 160 #endif /* DEBUG */
 161
 162 ulock_pass:
 163         movl    $1,%eax
 164         movl    4(%esp),%ecx
 165         xchgb   %al, (%ecx)
 166         xorb    $1, %al
 167         ret
 168         SET_SIZE(ulock_try)
 169
 170 #endif  /* !__amd64 */
 171
 172 #ifdef DEBUG
 173         .data
 174 .ulock_panic_msg:
 175         .string "ulock_try: Argument is above kernelbase"
 176         .text
 177 #endif  /* DEBUG */
 178
 179 #endif  /* __lint */
 180
 181 /*
 182  * lock_clear(lp)
 183  *      - unlock lock without changing interrupt priority level.
 184  */
 185
 186 #if defined(lint) || defined(__lint)
 187
 188 /* ARGSUSED */
 189 void
 190 lock_clear(lock_t *lp)
 191 {}
 192
 193 /* ARGSUSED */
 194 void
 195 ulock_clear(lock_t *lp)
 196 {}
 197
 198 #else   /* __lint */
 199
 200 #if defined(__amd64)
 201
 202         ENTRY(lock_clear)
 203         movb    $0, (%rdi)
 204 .lock_clear_lockstat_patch_point:
 205         ret
 206         movq    %rdi, %rsi                      /* rsi = lock addr */
 207         movq    %gs:CPU_THREAD, %rdx            /* rdx = thread addr */
 208         movl    $LS_LOCK_CLEAR_RELEASE, %edi    /* edi = event */
 209         jmp     lockstat_wrapper
 210         SET_SIZE(lock_clear)
 211
 212         ENTRY(ulock_clear)
 213 #ifdef DEBUG
 214         movq    kernelbase(%rip), %rcx
 215         cmpq    %rcx, %rdi              /* test uaddr < kernelbase */
 216         jb      ulock_clr               /*       uaddr < kernelbase, proceed */
 217
 218         leaq    .ulock_clear_msg(%rip), %rdi
 219         pushq   %rbp                    /* align stack properly */
 220         movq    %rsp, %rbp
 221         xorl    %eax, %eax              /* clear for varargs */
 222         call    panic
 223 #endif
 224
 225 ulock_clr:
 226         movb    $0, (%rdi)
 227         ret
 228         SET_SIZE(ulock_clear)
 229
 230 #else
 231
 232         ENTRY(lock_clear)
 233         movl    4(%esp), %eax
 234         movb    $0, (%eax)
 235 .lock_clear_lockstat_patch_point:
 236         ret
 237         movl    %gs:CPU_THREAD, %edx            /* edx = thread addr */
 238         movl    %eax, %ecx                      /* ecx = lock pointer */
 239         movl    $LS_LOCK_CLEAR_RELEASE, %eax
 240         jmp     lockstat_wrapper
 241         SET_SIZE(lock_clear)
 242
 243         ENTRY(ulock_clear)
 244 #ifdef DEBUG
 245         movl    kernelbase, %ecx
 246         cmpl    %ecx, 4(%esp)           /* test uaddr < kernelbase */
 247         jb      ulock_clr               /* uaddr < kernelbase, proceed */
 248
 249         pushl   $.ulock_clear_msg
 250         call    panic
 251 #endif
 252
 253 ulock_clr:
 254         movl    4(%esp),%eax
 255         xorl    %ecx,%ecx
 256         movb    %cl, (%eax)
 257         ret
 258         SET_SIZE(ulock_clear)
 259
 260 #endif  /* !__amd64 */
 261
 262 #ifdef DEBUG
 263         .data
 264 .ulock_clear_msg:
 265         .string "ulock_clear: Argument is above kernelbase"
 266         .text
 267 #endif  /* DEBUG */
 268
 269
 270 #endif  /* __lint */
 271
 272 /*
 273  * lock_set_spl(lock_t *lp, int new_pil, u_short *old_pil)
 274  * Drops lp, sets pil to new_pil, stores old pil in *old_pil.
 275  */
 276
 277 #if defined(lint) || defined(__lint)
 278
 279 /* ARGSUSED */
 280 void
 281 lock_set_spl(lock_t *lp, int new_pil, u_short *old_pil)
 282 {}
 283
 284 #else   /* __lint */
 285
 286 #if defined(__amd64)
 287
 288         ENTRY(lock_set_spl)
 289         pushq   %rbp
 290         movq    %rsp, %rbp
 291         subq    $32, %rsp
 292         movl    %esi, 8(%rsp)           /* save priority level */
 293         movq    %rdx, 16(%rsp)          /* save old pil ptr */
 294         movq    %rdi, 24(%rsp)          /* save lock pointer */
 295         movl    %esi, %edi              /* pass priority level */
 296         call    splr                    /* raise priority level */
 297         movq    24(%rsp), %rdi          /* rdi = lock addr */
 298         movb    $-1, %dl
 299         xchgb   %dl, (%rdi)             /* try to set lock */
 300         testb   %dl, %dl                /* did we get the lock? ... */
 301         jnz     .lss_miss               /* ... no, go to C for the hard case */
 302         movq    16(%rsp), %rdx          /* rdx = old pil addr */
 303         movw    %ax, (%rdx)             /* store old pil */
 304         leave
 305 .lock_set_spl_lockstat_patch_point:
 306         ret
 307         movq    %rdi, %rsi              /* rsi = lock addr */
 308         movq    %gs:CPU_THREAD, %rdx    /* rdx = thread addr */
 309         movl    $LS_LOCK_SET_SPL_ACQUIRE, %edi
 310         jmp     lockstat_wrapper
 311 .lss_miss:
 312         movl    8(%rsp), %esi           /* new_pil */
 313         movq    16(%rsp), %rdx          /* old_pil_addr */
 314         movl    %eax, %ecx              /* original pil */
 315         leave                           /* unwind stack */
 316         jmp     lock_set_spl_spin
 317         SET_SIZE(lock_set_spl)
 318
 319 #else
 320
 321         ENTRY(lock_set_spl)
 322         movl    8(%esp), %eax           /* get priority level */
 323         pushl   %eax
 324         call    splr                    /* raise priority level */
 325         movl    8(%esp), %ecx           /* ecx = lock addr */
 326         movl    $-1, %edx
 327         addl    $4, %esp
 328         xchgb   %dl, (%ecx)             /* try to set lock */
 329         testb   %dl, %dl                /* did we get the lock? ... */
 330         movl    12(%esp), %edx          /* edx = olp pil addr (ZF unaffected) */
 331         jnz     .lss_miss               /* ... no, go to C for the hard case */
 332         movw    %ax, (%edx)             /* store old pil */
 333 .lock_set_spl_lockstat_patch_point:
 334         ret
 335         movl    %gs:CPU_THREAD, %edx    /* edx = thread addr*/
 336         movl    $LS_LOCK_SET_SPL_ACQUIRE, %eax
 337         jmp     lockstat_wrapper
 338 .lss_miss:
 339         pushl   %eax                    /* original pil */
 340         pushl   %edx                    /* old_pil addr */
 341         pushl   16(%esp)                /* new_pil */
 342         pushl   %ecx                    /* lock addr */
 343         call    lock_set_spl_spin
 344         addl    $16, %esp
 345         ret
 346         SET_SIZE(lock_set_spl)
 347
 348 #endif  /* !__amd64 */
 349
 350 #endif  /* __lint */
 351
 352 /*
 353  * void
 354  * lock_init(lp)
 355  */
 356
 357 #if defined(__lint)
 358
 359 /* ARGSUSED */
 360 void
 361 lock_init(lock_t *lp)
 362 {}
 363
 364 #else   /* __lint */
 365
 366 #if defined(__amd64)
 367
 368         ENTRY(lock_init)
 369         movb    $0, (%rdi)
 370         ret
 371         SET_SIZE(lock_init)
 372
 373 #else
 374
 375         ENTRY(lock_init)
 376         movl    4(%esp), %eax
 377         movb    $0, (%eax)
 378         ret
 379         SET_SIZE(lock_init)
 380
 381 #endif  /* !__amd64 */
 382
 383 #endif  /* __lint */
 384
 385 /*
 386  * void
 387  * lock_set(lp)
 388  */
 389
 390 #if defined(lint) || defined(__lint)
 391
 392 /* ARGSUSED */
 393 void
 394 lock_set(lock_t *lp)
 395 {}
 396
 397 #else   /* __lint */
 398
 399 #if defined(__amd64)
 400
 401         ENTRY(lock_set)
 402         movb    $-1, %dl
 403         xchgb   %dl, (%rdi)             /* try to set lock */
 404         testb   %dl, %dl                /* did we get it? */
 405         jnz     lock_set_spin           /* no, go to C for the hard case */
 406 .lock_set_lockstat_patch_point:
 407         ret
 408         movq    %rdi, %rsi              /* rsi = lock addr */
 409         movq    %gs:CPU_THREAD, %rdx    /* rdx = thread addr */
 410         movl    $LS_LOCK_SET_ACQUIRE, %edi
 411         jmp     lockstat_wrapper
 412         SET_SIZE(lock_set)
 413
 414 #else
 415
 416         ENTRY(lock_set)
 417         movl    4(%esp), %ecx           /* ecx = lock addr */
 418         movl    $-1, %edx
 419         xchgb   %dl, (%ecx)             /* try to set lock */
 420         testb   %dl, %dl                /* did we get it? */
 421         jnz     lock_set_spin           /* no, go to C for the hard case */
 422 .lock_set_lockstat_patch_point:
 423         ret
 424         movl    %gs:CPU_THREAD, %edx    /* edx = thread addr */
 425         movl    $LS_LOCK_SET_ACQUIRE, %eax
 426         jmp     lockstat_wrapper
 427         SET_SIZE(lock_set)
 428
 429 #endif  /* !__amd64 */
 430
 431 #endif  /* __lint */
 432
 433 /*
 434  * lock_clear_splx(lp, s)
 435  */
 436
 437 #if defined(lint) || defined(__lint)
 438
 439 /* ARGSUSED */
 440 void
 441 lock_clear_splx(lock_t *lp, int s)
 442 {}
 443
 444 #else   /* __lint */
 445
 446 #if defined(__amd64)
 447
 448         ENTRY(lock_clear_splx)
 449         movb    $0, (%rdi)              /* clear lock */
 450 .lock_clear_splx_lockstat_patch_point:
 451         jmp     0f
 452 0:
 453         movl    %esi, %edi              /* arg for splx */
 454         jmp     splx                    /* let splx do its thing */
 455 .lock_clear_splx_lockstat:
 456         pushq   %rbp                    /* align stack properly */
 457         movq    %rsp, %rbp
 458         subq    $16, %rsp               /* space to save args across splx */
 459         movq    %rdi, 8(%rsp)           /* save lock ptr across splx call */
 460         movl    %esi, %edi              /* arg for splx */
 461         call    splx                    /* lower the priority */
 462         movq    8(%rsp), %rsi           /* rsi = lock ptr */
 463         leave                           /* unwind stack */
 464         movq    %gs:CPU_THREAD, %rdx    /* rdx = thread addr */
 465         movl    $LS_LOCK_CLEAR_SPLX_RELEASE, %edi
 466         jmp     lockstat_wrapper
 467         SET_SIZE(lock_clear_splx)
 468
 469 #else
 470
 471         ENTRY(lock_clear_splx)
 472         movl    4(%esp), %eax           /* eax = lock addr */
 473         movb    $0, (%eax)              /* clear lock */
 474 .lock_clear_splx_lockstat_patch_point:
 475         jmp     0f
 476 0:
 477         movl    8(%esp), %edx           /* edx = desired pil */
 478         movl    %edx, 4(%esp)           /* set spl arg up for splx */
 479         jmp     splx                    /* let splx do it's thing */
 480 .lock_clear_splx_lockstat:
 481         movl    8(%esp), %edx           /* edx = desired pil */
 482         pushl   %ebp                    /* set up stack frame */
 483         movl    %esp, %ebp
 484         pushl   %edx
 485         call    splx
 486         leave                           /* unwind stack */
 487         movl    4(%esp), %ecx           /* ecx = lock pointer */
 488         movl    %gs:CPU_THREAD, %edx    /* edx = thread addr */
 489         movl    $LS_LOCK_CLEAR_SPLX_RELEASE, %eax
 490         jmp     lockstat_wrapper
 491         SET_SIZE(lock_clear_splx)
 492
 493 #endif  /* !__amd64 */
 494
 495 #if defined(__GNUC_AS__)
 496 #define LOCK_CLEAR_SPLX_LOCKSTAT_PATCH_VAL      \
 497         (.lock_clear_splx_lockstat - .lock_clear_splx_lockstat_patch_point - 2)
 498
 499 #define LOCK_CLEAR_SPLX_LOCKSTAT_PATCH_POINT    \
 500         (.lock_clear_splx_lockstat_patch_point + 1)
 501 #else
 502 #define LOCK_CLEAR_SPLX_LOCKSTAT_PATCH_VAL      \
 503         [.lock_clear_splx_lockstat - .lock_clear_splx_lockstat_patch_point - 2]
 504
 505 #define LOCK_CLEAR_SPLX_LOCKSTAT_PATCH_POINT    \
 506         [.lock_clear_splx_lockstat_patch_point + 1]
 507 #endif
 508
 509 #endif  /* __lint */
 510
 511 /*
 512  * mutex_enter() and mutex_exit().
 513  *
 514  * These routines handle the simple cases of mutex_enter() (adaptive
 515  * lock, not held) and mutex_exit() (adaptive lock, held, no waiters).
 516  * If anything complicated is going on we punt to mutex_vector_enter().
 517  *
 518  * mutex_tryenter() is similar to mutex_enter() but returns zero if
 519  * the lock cannot be acquired, nonzero on success.
 520  *
 521  * If mutex_exit() gets preempted in the window between checking waiters
 522  * and clearing the lock, we can miss wakeups.  Disabling preemption
 523  * in the mutex code is prohibitively expensive, so instead we detect
 524  * mutex preemption by examining the trapped PC in the interrupt path.
 525  * If we interrupt a thread in mutex_exit() that has not yet cleared
 526  * the lock, cmnint() resets its PC back to the beginning of
 527  * mutex_exit() so it will check again for waiters when it resumes.
 528  *
 529  * The lockstat code below is activated when the lockstat driver
 530  * calls lockstat_hot_patch() to hot-patch the kernel mutex code.
 531  * Note that we don't need to test lockstat_event_mask here -- we won't
 532  * patch this code in unless we're gathering ADAPTIVE_HOLD lockstats.
 533  */
 534 #if defined(lint) || defined(__lint)
 535
 536 /* ARGSUSED */
 537 void
 538 mutex_enter(kmutex_t *lp)
 539 {}
 540
 541 /* ARGSUSED */
 542 int
 543 mutex_tryenter(kmutex_t *lp)
 544 { return (0); }
 545
 546 /* ARGSUSED */
 547 int
 548 mutex_adaptive_tryenter(mutex_impl_t *lp)
 549 { return (0); }
 550
 551 /* ARGSUSED */
 552 void
 553 mutex_exit(kmutex_t *lp)
 554 {}
 555
 556 #else
 557
 558 #if defined(__amd64)
 559
 560         ENTRY_NP(mutex_enter)
 561         movq    %gs:CPU_THREAD, %rdx            /* rdx = thread ptr */
 562         xorl    %eax, %eax                      /* rax = 0 (unheld adaptive) */
 563         lock
 564         cmpxchgq %rdx, (%rdi)
 565         jnz     mutex_vector_enter
 566 .mutex_enter_lockstat_patch_point:
 567 #if defined(OPTERON_WORKAROUND_6323525)
 568 .mutex_enter_6323525_patch_point:
 569         ret                                     /* nop space for lfence */
 570         nop
 571         nop
 572 .mutex_enter_lockstat_6323525_patch_point:      /* new patch point if lfence */
 573         nop
 574 #else   /* OPTERON_WORKAROUND_6323525 */
 575         ret
 576 #endif  /* OPTERON_WORKAROUND_6323525 */
 577         movq    %rdi, %rsi
 578         movl    $LS_MUTEX_ENTER_ACQUIRE, %edi
 579 /*
 580  * expects %rdx=thread, %rsi=lock, %edi=lockstat event
 581  */
 582         ALTENTRY(lockstat_wrapper)
 583         incb    T_LOCKSTAT(%rdx)                /* curthread->t_lockstat++ */
 584         leaq    lockstat_probemap(%rip), %rax
 585         movl    (%rax, %rdi, DTRACE_IDSIZE), %eax
 586         testl   %eax, %eax                      /* check for non-zero probe */
 587         jz      1f
 588         pushq   %rbp                            /* align stack properly */
 589         movq    %rsp, %rbp
 590         movl    %eax, %edi
 591         call    *lockstat_probe
 592         leave                                   /* unwind stack */
 593 1:
 594         movq    %gs:CPU_THREAD, %rdx            /* reload thread ptr */
 595         decb    T_LOCKSTAT(%rdx)                /* curthread->t_lockstat-- */
 596         movl    $1, %eax                        /* return success if tryenter */
 597         ret
 598         SET_SIZE(lockstat_wrapper)
 599         SET_SIZE(mutex_enter)
 600
 601 /*
 602  * expects %rcx=thread, %rdx=arg, %rsi=lock, %edi=lockstat event
 603  */
 604         ENTRY(lockstat_wrapper_arg)
 605         incb    T_LOCKSTAT(%rcx)                /* curthread->t_lockstat++ */
 606         leaq    lockstat_probemap(%rip), %rax
 607         movl    (%rax, %rdi, DTRACE_IDSIZE), %eax
 608         testl   %eax, %eax                      /* check for non-zero probe */
 609         jz      1f
 610         pushq   %rbp                            /* align stack properly */
 611         movq    %rsp, %rbp
 612         movl    %eax, %edi
 613         call    *lockstat_probe
 614         leave                                   /* unwind stack */
 615 1:
 616         movq    %gs:CPU_THREAD, %rdx            /* reload thread ptr */
 617         decb    T_LOCKSTAT(%rdx)                /* curthread->t_lockstat-- */
 618         movl    $1, %eax                        /* return success if tryenter */
 619         ret
 620         SET_SIZE(lockstat_wrapper_arg)
 621
 622
 623         ENTRY(mutex_tryenter)
 624         movq    %gs:CPU_THREAD, %rdx            /* rdx = thread ptr */
 625         xorl    %eax, %eax                      /* rax = 0 (unheld adaptive) */
 626         lock
 627         cmpxchgq %rdx, (%rdi)
 628         jnz     mutex_vector_tryenter
 629         not     %eax                            /* return success (nonzero) */
 630 #if defined(OPTERON_WORKAROUND_6323525)
 631 .mutex_tryenter_lockstat_patch_point:
 632 .mutex_tryenter_6323525_patch_point:
 633         ret                                     /* nop space for lfence */
 634         nop
 635         nop
 636 .mutex_tryenter_lockstat_6323525_patch_point:   /* new patch point if lfence */
 637         nop
 638 #else   /* OPTERON_WORKAROUND_6323525 */
 639 .mutex_tryenter_lockstat_patch_point:
 640         ret
 641 #endif  /* OPTERON_WORKAROUND_6323525 */
 642         movq    %rdi, %rsi
 643         movl    $LS_MUTEX_ENTER_ACQUIRE, %edi
 644         jmp     lockstat_wrapper
 645         SET_SIZE(mutex_tryenter)
 646
 647         ENTRY(mutex_adaptive_tryenter)
 648         movq    %gs:CPU_THREAD, %rdx            /* rdx = thread ptr */
 649         xorl    %eax, %eax                      /* rax = 0 (unheld adaptive) */
 650         lock
 651         cmpxchgq %rdx, (%rdi)
 652         jnz     0f
 653         not     %eax                            /* return success (nonzero) */
 654 #if defined(OPTERON_WORKAROUND_6323525)
 655 .mutex_atryenter_6323525_patch_point:
 656         ret                                     /* nop space for lfence */
 657         nop
 658         nop
 659         nop
 660 #else   /* OPTERON_WORKAROUND_6323525 */
 661         ret
 662 #endif  /* OPTERON_WORKAROUND_6323525 */
 663 0:
 664         xorl    %eax, %eax                      /* return failure */
 665         ret
 666         SET_SIZE(mutex_adaptive_tryenter)
 667
 668         .globl  mutex_owner_running_critical_start
 669
 670         ENTRY(mutex_owner_running)
 671 mutex_owner_running_critical_start:
 672         movq    (%rdi), %r11            /* get owner field */
 673         andq    $MUTEX_THREAD, %r11     /* remove waiters bit */
 674         cmpq    $0, %r11                /* if free, skip */
 675         je      1f                      /* go return 0 */
 676         movq    T_CPU(%r11), %r8        /* get owner->t_cpu */
 677         movq    CPU_THREAD(%r8), %r9    /* get t_cpu->cpu_thread */
 678 .mutex_owner_running_critical_end:
 679         cmpq    %r11, %r9       /* owner == running thread? */
 680         je      2f              /* yes, go return cpu */
 681 1:
 682         xorq    %rax, %rax      /* return 0 */
 683         ret
 684 2:
 685         movq    %r8, %rax               /* return cpu */
 686         ret
 687         SET_SIZE(mutex_owner_running)
 688
 689         .globl  mutex_owner_running_critical_size
 690         .type   mutex_owner_running_critical_size, @object
 691         .align  CPTRSIZE
 692 mutex_owner_running_critical_size:
 693         .quad   .mutex_owner_running_critical_end - mutex_owner_running_critical_start
 694         SET_SIZE(mutex_owner_running_critical_size)
 695
 696         .globl  mutex_exit_critical_start
 697
 698         ENTRY(mutex_exit)
 699 mutex_exit_critical_start:              /* If interrupted, restart here */
 700         movq    %gs:CPU_THREAD, %rdx
 701         cmpq    %rdx, (%rdi)
 702         jne     mutex_vector_exit               /* wrong type or wrong owner */
 703         movq    $0, (%rdi)                      /* clear owner AND lock */
 704 .mutex_exit_critical_end:
 705 .mutex_exit_lockstat_patch_point:
 706         ret
 707         movq    %rdi, %rsi
 708         movl    $LS_MUTEX_EXIT_RELEASE, %edi
 709         jmp     lockstat_wrapper
 710         SET_SIZE(mutex_exit)
 711
 712         .globl  mutex_exit_critical_size
 713         .type   mutex_exit_critical_size, @object
 714         .align  CPTRSIZE
 715 mutex_exit_critical_size:
 716         .quad   .mutex_exit_critical_end - mutex_exit_critical_start
 717         SET_SIZE(mutex_exit_critical_size)
 718
 719 #else
 720
 721         ENTRY_NP(mutex_enter)
 722         movl    %gs:CPU_THREAD, %edx            /* edx = thread ptr */
 723         movl    4(%esp), %ecx                   /* ecx = lock ptr */
 724         xorl    %eax, %eax                      /* eax = 0 (unheld adaptive) */
 725         lock
 726         cmpxchgl %edx, (%ecx)
 727         jnz     mutex_vector_enter
 728 #if defined(OPTERON_WORKAROUND_6323525)
 729 .mutex_enter_lockstat_patch_point:
 730 .mutex_enter_6323525_patch_point:
 731         ret                                     /* nop space for lfence */
 732         nop
 733         nop
 734 .mutex_enter_lockstat_6323525_patch_point:      /* new patch point if lfence */
 735         nop
 736 #else   /* OPTERON_WORKAROUND_6323525 */
 737 .mutex_enter_lockstat_patch_point:
 738         ret
 739 #endif  /* OPTERON_WORKAROUND_6323525 */
 740         movl    $LS_MUTEX_ENTER_ACQUIRE, %eax
 741         ALTENTRY(lockstat_wrapper)      /* expects edx=thread, ecx=lock, */
 742                                         /*   eax=lockstat event */
 743         pushl   %ebp                            /* buy a frame */
 744         movl    %esp, %ebp
 745         incb    T_LOCKSTAT(%edx)                /* curthread->t_lockstat++ */
 746         pushl   %edx                            /* save thread pointer   */
 747         movl    $lockstat_probemap, %edx
 748         movl    (%edx, %eax, DTRACE_IDSIZE), %eax
 749         testl   %eax, %eax                      /* check for non-zero probe */
 750         jz      1f
 751         pushl   %ecx                            /* push lock */
 752         pushl   %eax                            /* push probe ID */
 753         call    *lockstat_probe
 754         addl    $8, %esp
 755 1:
 756         popl    %edx                            /* restore thread pointer */
 757         decb    T_LOCKSTAT(%edx)                /* curthread->t_lockstat-- */
 758         movl    $1, %eax                        /* return success if tryenter */
 759         popl    %ebp                            /* pop off frame */
 760         ret
 761         SET_SIZE(lockstat_wrapper)
 762         SET_SIZE(mutex_enter)
 763
 764         ENTRY(lockstat_wrapper_arg)     /* expects edx=thread, ecx=lock, */
 765                                         /* eax=lockstat event, pushed arg */
 766         incb    T_LOCKSTAT(%edx)                /* curthread->t_lockstat++ */
 767         pushl   %edx                            /* save thread pointer   */
 768         movl    $lockstat_probemap, %edx
 769         movl    (%edx, %eax, DTRACE_IDSIZE), %eax
 770         testl   %eax, %eax                      /* check for non-zero probe */
 771         jz      1f
 772         pushl   %ebp                            /* save %ebp */
 773         pushl   8(%esp)                         /* push arg1 */
 774         movl    %ebp, 12(%esp)                  /* fake up the stack frame */
 775         movl    %esp, %ebp                      /* fake up base pointer */
 776         addl    $12, %ebp                       /* adjust faked base pointer */
 777         pushl   %ecx                            /* push lock */
 778         pushl   %eax                            /* push probe ID */
 779         call    *lockstat_probe
 780         addl    $12, %esp                       /* adjust for arguments */
 781         popl    %ebp                            /* pop frame */
 782 1:
 783         popl    %edx                            /* restore thread pointer */
 784         decb    T_LOCKSTAT(%edx)                /* curthread->t_lockstat-- */
 785         movl    $1, %eax                        /* return success if tryenter */
 786         addl    $4, %esp                        /* pop argument */
 787         ret
 788         SET_SIZE(lockstat_wrapper_arg)
 789
 790
 791         ENTRY(mutex_tryenter)
 792         movl    %gs:CPU_THREAD, %edx            /* edx = thread ptr */
 793         movl    4(%esp), %ecx                   /* ecx = lock ptr */
 794         xorl    %eax, %eax                      /* eax = 0 (unheld adaptive) */
 795         lock
 796         cmpxchgl %edx, (%ecx)
 797         jnz     mutex_vector_tryenter
 798         movl    %ecx, %eax
 799 #if defined(OPTERON_WORKAROUND_6323525)
 800 .mutex_tryenter_lockstat_patch_point:
 801 .mutex_tryenter_6323525_patch_point:
 802         ret                                     /* nop space for lfence */
 803         nop
 804         nop
 805 .mutex_tryenter_lockstat_6323525_patch_point:   /* new patch point if lfence */
 806         nop
 807 #else   /* OPTERON_WORKAROUND_6323525 */
 808 .mutex_tryenter_lockstat_patch_point:
 809         ret
 810 #endif  /* OPTERON_WORKAROUND_6323525 */
 811         movl    $LS_MUTEX_ENTER_ACQUIRE, %eax
 812         jmp     lockstat_wrapper
 813         SET_SIZE(mutex_tryenter)
 814
 815         ENTRY(mutex_adaptive_tryenter)
 816         movl    %gs:CPU_THREAD, %edx            /* edx = thread ptr */
 817         movl    4(%esp), %ecx                   /* ecx = lock ptr */
 818         xorl    %eax, %eax                      /* eax = 0 (unheld adaptive) */
 819         lock
 820         cmpxchgl %edx, (%ecx)
 821         jnz     0f
 822         movl    %ecx, %eax
 823 #if defined(OPTERON_WORKAROUND_6323525)
 824 .mutex_atryenter_6323525_patch_point:
 825         ret                                     /* nop space for lfence */
 826         nop
 827         nop
 828         nop
 829 #else   /* OPTERON_WORKAROUND_6323525 */
 830         ret
 831 #endif  /* OPTERON_WORKAROUND_6323525 */
 832 0:
 833         xorl    %eax, %eax
 834         ret
 835         SET_SIZE(mutex_adaptive_tryenter)
 836
 837         .globl  mutex_owner_running_critical_start
 838
 839         ENTRY(mutex_owner_running)
 840 mutex_owner_running_critical_start:
 841         movl    4(%esp), %eax           /* get owner field */
 842         movl    (%eax), %eax
 843         andl    $MUTEX_THREAD, %eax     /* remove waiters bit */
 844         cmpl    $0, %eax                /* if free, skip */
 845         je      1f                      /* go return 0 */
 846         movl    T_CPU(%eax), %ecx       /* get owner->t_cpu */
 847         movl    CPU_THREAD(%ecx), %edx  /* get t_cpu->cpu_thread */
 848 .mutex_owner_running_critical_end:
 849         cmpl    %eax, %edx      /* owner == running thread? */
 850         je      2f              /* yes, go return cpu */
 851 1:
 852         xorl    %eax, %eax      /* return 0 */
 853         ret
 854 2:
 855         movl    %ecx, %eax      /* return cpu */
 856         ret
 857
 858         SET_SIZE(mutex_owner_running)
 859
 860         .globl  mutex_owner_running_critical_size
 861         .type   mutex_owner_running_critical_size, @object
 862         .align  CPTRSIZE
 863 mutex_owner_running_critical_size:
 864         .long   .mutex_owner_running_critical_end - mutex_owner_running_critical_start
 865         SET_SIZE(mutex_owner_running_critical_size)
 866
 867         .globl  mutex_exit_critical_start
 868
 869         ENTRY(mutex_exit)
 870 mutex_exit_critical_start:              /* If interrupted, restart here */
 871         movl    %gs:CPU_THREAD, %edx
 872         movl    4(%esp), %ecx
 873         cmpl    %edx, (%ecx)
 874         jne     mutex_vector_exit               /* wrong type or wrong owner */
 875         movl    $0, (%ecx)                      /* clear owner AND lock */
 876 .mutex_exit_critical_end:
 877 .mutex_exit_lockstat_patch_point:
 878         ret
 879         movl    $LS_MUTEX_EXIT_RELEASE, %eax
 880         jmp     lockstat_wrapper
 881         SET_SIZE(mutex_exit)
 882
 883         .globl  mutex_exit_critical_size
 884         .type   mutex_exit_critical_size, @object
 885         .align  CPTRSIZE
 886 mutex_exit_critical_size:
 887         .long   .mutex_exit_critical_end - mutex_exit_critical_start
 888         SET_SIZE(mutex_exit_critical_size)
 889
 890 #endif  /* !__amd64 */
 891
 892 #endif  /* __lint */
 893
 894 /*
 895  * rw_enter() and rw_exit().
 896  *
 897  * These routines handle the simple cases of rw_enter (write-locking an unheld
 898  * lock or read-locking a lock that's neither write-locked nor write-wanted)
 899  * and rw_exit (no waiters or not the last reader).  If anything complicated
 900  * is going on we punt to rw_enter_sleep() and rw_exit_wakeup(), respectively.
 901  */
 902 #if defined(lint) || defined(__lint)
 903
 904 /* ARGSUSED */
 905 void
 906 rw_enter(krwlock_t *lp, krw_t rw)
 907 {}
 908
 909 /* ARGSUSED */
 910 void
 911 rw_exit(krwlock_t *lp)
 912 {}
 913
 914 #else   /* __lint */
 915
 916 #if defined(__amd64)
 917
 918         ENTRY(rw_enter)
 919         movq    %gs:CPU_THREAD, %rdx            /* rdx = thread ptr */
 920         cmpl    $RW_WRITER, %esi
 921         je      .rw_write_enter
 922         incl    T_KPRI_REQ(%rdx)                /* THREAD_KPRI_REQUEST() */
 923         movq    (%rdi), %rax                    /* rax = old rw_wwwh value */
 924         testl   $RW_WRITE_LOCKED|RW_WRITE_WANTED, %eax
 925         jnz     rw_enter_sleep
 926         leaq    RW_READ_LOCK(%rax), %rdx        /* rdx = new rw_wwwh value */
 927         lock
 928         cmpxchgq %rdx, (%rdi)                   /* try to grab read lock */
 929         jnz     rw_enter_sleep
 930 .rw_read_enter_lockstat_patch_point:
 931         ret
 932         movq    %gs:CPU_THREAD, %rcx            /* rcx = thread ptr */
 933         movq    %rdi, %rsi                      /* rsi = lock ptr */
 934         movl    $LS_RW_ENTER_ACQUIRE, %edi
 935         movl    $RW_READER, %edx
 936         jmp     lockstat_wrapper_arg
 937 .rw_write_enter:
 938         orq     $RW_WRITE_LOCKED, %rdx          /* rdx = write-locked value */
 939         xorl    %eax, %eax                      /* rax = unheld value */
 940         lock
 941         cmpxchgq %rdx, (%rdi)                   /* try to grab write lock */
 942         jnz     rw_enter_sleep
 943
 944 #if defined(OPTERON_WORKAROUND_6323525)
 945 .rw_write_enter_lockstat_patch_point:
 946 .rw_write_enter_6323525_patch_point:
 947         ret
 948         nop
 949         nop
 950 .rw_write_enter_lockstat_6323525_patch_point:
 951         nop
 952 #else   /* OPTERON_WORKAROUND_6323525 */
 953 .rw_write_enter_lockstat_patch_point:
 954         ret
 955 #endif  /* OPTERON_WORKAROUND_6323525 */
 956
 957         movq    %gs:CPU_THREAD, %rcx            /* rcx = thread ptr */
 958         movq    %rdi, %rsi                      /* rsi = lock ptr */
 959         movl    $LS_RW_ENTER_ACQUIRE, %edi
 960         movl    $RW_WRITER, %edx
 961         jmp     lockstat_wrapper_arg
 962         SET_SIZE(rw_enter)
 963
 964         ENTRY(rw_exit)
 965         movq    (%rdi), %rax                    /* rax = old rw_wwwh value */
 966         cmpl    $RW_READ_LOCK, %eax             /* single-reader, no waiters? */
 967         jne     .rw_not_single_reader
 968         xorl    %edx, %edx                      /* rdx = new value (unheld) */
 969 .rw_read_exit:
 970         lock
 971         cmpxchgq %rdx, (%rdi)                   /* try to drop read lock */
 972         jnz     rw_exit_wakeup
 973         movq    %gs:CPU_THREAD, %rcx            /* rcx = thread ptr */
 974         decl    T_KPRI_REQ(%rcx)                /* THREAD_KPRI_RELEASE() */
 975 .rw_read_exit_lockstat_patch_point:
 976         ret
 977         movq    %rdi, %rsi                      /* rsi = lock ptr */
 978         movl    $LS_RW_EXIT_RELEASE, %edi
 979         movl    $RW_READER, %edx
 980         jmp     lockstat_wrapper_arg
 981 .rw_not_single_reader:
 982         testl   $RW_WRITE_LOCKED, %eax  /* write-locked or write-wanted? */
 983         jnz     .rw_write_exit
 984         leaq    -RW_READ_LOCK(%rax), %rdx       /* rdx = new value */
 985         cmpl    $RW_READ_LOCK, %edx
 986         jge     .rw_read_exit           /* not last reader, safe to drop */
 987         jmp     rw_exit_wakeup                  /* last reader with waiters */
 988 .rw_write_exit:
 989         movq    %gs:CPU_THREAD, %rax            /* rax = thread ptr */
 990         xorl    %edx, %edx                      /* rdx = new value (unheld) */
 991         orq     $RW_WRITE_LOCKED, %rax          /* eax = write-locked value */
 992         lock
 993         cmpxchgq %rdx, (%rdi)                   /* try to drop read lock */
 994         jnz     rw_exit_wakeup
 995 .rw_write_exit_lockstat_patch_point:
 996         ret
 997         movq    %gs:CPU_THREAD, %rcx            /* rcx = thread ptr */
 998         movq    %rdi, %rsi                      /* rsi - lock ptr */
 999         movl    $LS_RW_EXIT_RELEASE, %edi
1000         movl    $RW_WRITER, %edx
1001         jmp     lockstat_wrapper_arg
1002         SET_SIZE(rw_exit)
1003
1004 #else
1005
1006         ENTRY(rw_enter)
1007         movl    %gs:CPU_THREAD, %edx            /* edx = thread ptr */
1008         movl    4(%esp), %ecx                   /* ecx = lock ptr */
1009         cmpl    $RW_WRITER, 8(%esp)
1010         je      .rw_write_enter
1011         incl    T_KPRI_REQ(%edx)                /* THREAD_KPRI_REQUEST() */
1012         movl    (%ecx), %eax                    /* eax = old rw_wwwh value */
1013         testl   $RW_WRITE_LOCKED|RW_WRITE_WANTED, %eax
1014         jnz     rw_enter_sleep
1015         leal    RW_READ_LOCK(%eax), %edx        /* edx = new rw_wwwh value */
1016         lock
1017         cmpxchgl %edx, (%ecx)                   /* try to grab read lock */
1018         jnz     rw_enter_sleep
1019 .rw_read_enter_lockstat_patch_point:
1020         ret
1021         movl    %gs:CPU_THREAD, %edx            /* edx = thread ptr */
1022         movl    $LS_RW_ENTER_ACQUIRE, %eax
1023         pushl   $RW_READER
1024         jmp     lockstat_wrapper_arg
1025 .rw_write_enter:
1026         orl     $RW_WRITE_LOCKED, %edx          /* edx = write-locked value */
1027         xorl    %eax, %eax                      /* eax = unheld value */
1028         lock
1029         cmpxchgl %edx, (%ecx)                   /* try to grab write lock */
1030         jnz     rw_enter_sleep
1031
1032 #if defined(OPTERON_WORKAROUND_6323525)
1033 .rw_write_enter_lockstat_patch_point:
1034 .rw_write_enter_6323525_patch_point:
1035         ret
1036         nop
1037         nop
1038 .rw_write_enter_lockstat_6323525_patch_point:
1039         nop
1040 #else   /* OPTERON_WORKAROUND_6323525 */
1041 .rw_write_enter_lockstat_patch_point:
1042         ret
1043 #endif  /* OPTERON_WORKAROUND_6323525 */
1044
1045         movl    %gs:CPU_THREAD, %edx            /* edx = thread ptr */
1046         movl    $LS_RW_ENTER_ACQUIRE, %eax
1047         pushl   $RW_WRITER
1048         jmp     lockstat_wrapper_arg
1049         SET_SIZE(rw_enter)
1050
1051         ENTRY(rw_exit)
1052         movl    4(%esp), %ecx                   /* ecx = lock ptr */
1053         movl    (%ecx), %eax                    /* eax = old rw_wwwh value */
1054         cmpl    $RW_READ_LOCK, %eax             /* single-reader, no waiters? */
1055         jne     .rw_not_single_reader
1056         xorl    %edx, %edx                      /* edx = new value (unheld) */
1057 .rw_read_exit:
1058         lock
1059         cmpxchgl %edx, (%ecx)                   /* try to drop read lock */
1060         jnz     rw_exit_wakeup
1061         movl    %gs:CPU_THREAD, %edx            /* edx = thread ptr */
1062         decl    T_KPRI_REQ(%edx)                /* THREAD_KPRI_RELEASE() */
1063 .rw_read_exit_lockstat_patch_point:
1064         ret
1065         movl    $LS_RW_EXIT_RELEASE, %eax
1066         pushl   $RW_READER
1067         jmp     lockstat_wrapper_arg
1068 .rw_not_single_reader:
1069         testl   $RW_WRITE_LOCKED, %eax  /* write-locked or write-wanted? */
1070         jnz     .rw_write_exit
1071         leal    -RW_READ_LOCK(%eax), %edx       /* edx = new value */
1072         cmpl    $RW_READ_LOCK, %edx
1073         jge     .rw_read_exit           /* not last reader, safe to drop */
1074         jmp     rw_exit_wakeup                  /* last reader with waiters */
1075 .rw_write_exit:
1076         movl    %gs:CPU_THREAD, %eax            /* eax = thread ptr */
1077         xorl    %edx, %edx                      /* edx = new value (unheld) */
1078         orl     $RW_WRITE_LOCKED, %eax          /* eax = write-locked value */
1079         lock
1080         cmpxchgl %edx, (%ecx)                   /* try to drop read lock */
1081         jnz     rw_exit_wakeup
1082 .rw_write_exit_lockstat_patch_point:
1083         ret
1084         movl    %gs:CPU_THREAD, %edx            /* edx = thread ptr */
1085         movl    $LS_RW_EXIT_RELEASE, %eax
1086         pushl   $RW_WRITER
1087         jmp     lockstat_wrapper_arg
1088         SET_SIZE(rw_exit)
1089
1090 #endif  /* !__amd64 */
1091
1092 #endif  /* __lint */
1093
1094 #if defined(OPTERON_WORKAROUND_6323525)
1095 #if defined(lint) || defined(__lint)
1096
1097 int     workaround_6323525_patched;
1098
1099 void
1100 patch_workaround_6323525(void)
1101 {}
1102
1103 #else   /* lint */
1104
1105 /*
1106  * If it is necessary to patch the lock enter routines with the lfence
1107  * workaround, workaround_6323525_patched is set to a non-zero value so that
1108  * the lockstat_hat_patch routine can patch to the new location of the 'ret'
1109  * instruction.
1110  */
1111         DGDEF3(workaround_6323525_patched, 4, 4)
1112         .long   0
1113
1114 #if defined(__amd64)
1115
1116 #define HOT_MUTEX_PATCH(srcaddr, dstaddr, size) \
1117         movq    $size, %rbx;                    \
1118         movq    $dstaddr, %r13;                 \
1119         addq    %rbx, %r13;                     \
1120         movq    $srcaddr, %r12;                 \
1121         addq    %rbx, %r12;                     \
1122 0:                                              \
1123         decq    %r13;                           \
1124         decq    %r12;                           \
1125         movzbl  (%r12), %esi;                   \
1126         movq    $1, %rdx;                       \
1127         movq    %r13, %rdi;                     \
1128         call    hot_patch_kernel_text;          \
1129         decq    %rbx;                           \
1130         testq   %rbx, %rbx;                     \
1131         jg      0b;
1132
1133 /*
1134  * patch_workaround_6323525: provide workaround for 6323525
1135  *
1136  * The workaround is to place a fencing instruction (lfence) between the
1137  * mutex operation and the subsequent read-modify-write instruction.
1138  *
1139  * This routine hot patches the lfence instruction on top of the space
1140  * reserved by nops in the lock enter routines.
1141  */
1142         ENTRY_NP(patch_workaround_6323525)
1143         pushq   %rbp
1144         movq    %rsp, %rbp
1145         pushq   %r12
1146         pushq   %r13
1147         pushq   %rbx
1148
1149         /*
1150          * lockstat_hot_patch() to use the alternate lockstat workaround
1151          * 6323525 patch points (points past the lfence instruction to the
1152          * new ret) when workaround_6323525_patched is set.
1153          */
1154         movl    $1, workaround_6323525_patched
1155
1156         /*
1157          * patch ret/nop/nop/nop to lfence/ret at the end of the lock enter
1158          * routines. The 4 bytes are patched in reverse order so that the
1159          * the existing ret is overwritten last. This provides lock enter
1160          * sanity during the intermediate patching stages.
1161          */
1162         HOT_MUTEX_PATCH(_lfence_insn, .mutex_enter_6323525_patch_point, 4)
1163         HOT_MUTEX_PATCH(_lfence_insn, .mutex_tryenter_6323525_patch_point, 4)
1164         HOT_MUTEX_PATCH(_lfence_insn, .mutex_atryenter_6323525_patch_point, 4)
1165         HOT_MUTEX_PATCH(_lfence_insn, .rw_write_enter_6323525_patch_point, 4)
1166
1167         popq    %rbx
1168         popq    %r13
1169         popq    %r12
1170         movq    %rbp, %rsp
1171         popq    %rbp
1172         ret
1173 _lfence_insn:
1174         lfence
1175         ret
1176         SET_SIZE(patch_workaround_6323525)
1177
1178
1179 #else   /* __amd64 */
1180
1181 #define HOT_MUTEX_PATCH(srcaddr, dstaddr, size) \
1182         movl    $size, %ebx;                    \
1183         movl    $srcaddr, %esi;                 \
1184         addl    %ebx, %esi;                     \
1185         movl    $dstaddr, %edi;                 \
1186         addl    %ebx, %edi;                     \
1187 0:                                              \
1188         decl    %esi;                           \
1189         decl    %edi;                           \
1190         pushl   $1;                             \
1191         movzbl  (%esi), %eax;                   \
1192         pushl   %eax;                           \
1193         pushl   %edi;                           \
1194         call    hot_patch_kernel_text;          \
1195         addl    $12, %esp;                      \
1196         decl    %ebx;                           \
1197         testl   %ebx, %ebx;                     \
1198         jg      0b;
1199
1200
1201         /* see comments above */
1202         ENTRY_NP(patch_workaround_6323525)
1203         pushl   %ebp
1204         movl    %esp, %ebp
1205         pushl   %ebx
1206         pushl   %esi
1207         pushl   %edi
1208
1209         movl    $1, workaround_6323525_patched
1210
1211         HOT_MUTEX_PATCH(_lfence_insn, .mutex_enter_6323525_patch_point, 4)
1212         HOT_MUTEX_PATCH(_lfence_insn, .mutex_tryenter_6323525_patch_point, 4)
1213         HOT_MUTEX_PATCH(_lfence_insn, .mutex_atryenter_6323525_patch_point, 4)
1214         HOT_MUTEX_PATCH(_lfence_insn, .rw_write_enter_6323525_patch_point, 4)
1215
1216         popl    %edi
1217         popl    %esi
1218         popl    %ebx
1219         movl    %ebp, %esp
1220         popl    %ebp
1221         ret
1222 _lfence_insn:
1223         .byte   0xf, 0xae, 0xe8         / [lfence instruction]
1224         ret
1225         SET_SIZE(patch_workaround_6323525)
1226
1227 #endif  /* !__amd64 */
1228 #endif  /* !lint */
1229 #endif  /* OPTERON_WORKAROUND_6323525 */
1230
1231
1232 #if defined(lint) || defined(__lint)
1233
1234 void
1235 lockstat_hot_patch(void)
1236 {}
1237
1238 #else
1239
1240 #if defined(__amd64)
1241
1242 #define HOT_PATCH(addr, event, active_instr, normal_instr, len) \
1243         movq    $normal_instr, %rsi;            \
1244         movq    $active_instr, %rdi;            \
1245         leaq    lockstat_probemap(%rip), %rax;  \
1246         movl    _MUL(event, DTRACE_IDSIZE)(%rax), %eax; \
1247         testl   %eax, %eax;                     \
1248         jz      9f;                             \
1249         movq    %rdi, %rsi;                     \
1250 9:                                              \
1251         movq    $len, %rdx;                     \
1252         movq    $addr, %rdi;                    \
1253         call    hot_patch_kernel_text
1254
1255 #else
1256
1257 #define HOT_PATCH(addr, event, active_instr, normal_instr, len) \
1258         movl    $normal_instr, %ecx;            \
1259         movl    $active_instr, %edx;            \
1260         movl    $lockstat_probemap, %eax;       \
1261         movl    _MUL(event, DTRACE_IDSIZE)(%eax), %eax; \
1262         testl   %eax, %eax;                     \
1263         jz      . + 4;                          \
1264         movl    %edx, %ecx;                     \
1265         pushl   $len;                           \
1266         pushl   %ecx;                           \
1267         pushl   $addr;                          \
1268         call    hot_patch_kernel_text;          \
1269         addl    $12, %esp;
1270
1271 #endif  /* !__amd64 */
1272
1273         ENTRY(lockstat_hot_patch)
1274 #if defined(__amd64)
1275         pushq   %rbp                    /* align stack properly */
1276         movq    %rsp, %rbp
1277 #endif  /* __amd64 */
1278
1279 #if defined(OPTERON_WORKAROUND_6323525)
1280         cmpl    $0, workaround_6323525_patched
1281         je      1f
1282         HOT_PATCH(.mutex_enter_lockstat_6323525_patch_point,
1283                 LS_MUTEX_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1284         HOT_PATCH(.mutex_tryenter_lockstat_6323525_patch_point,
1285                 LS_MUTEX_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1286         HOT_PATCH(.rw_write_enter_lockstat_6323525_patch_point,
1287                 LS_RW_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1288         jmp     2f
1289 1:
1290         HOT_PATCH(.mutex_enter_lockstat_patch_point,
1291                 LS_MUTEX_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1292         HOT_PATCH(.mutex_tryenter_lockstat_patch_point,
1293                 LS_MUTEX_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1294         HOT_PATCH(.rw_write_enter_lockstat_patch_point,
1295                 LS_RW_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1296 2:
1297 #else   /* OPTERON_WORKAROUND_6323525 */
1298         HOT_PATCH(.mutex_enter_lockstat_patch_point,
1299                 LS_MUTEX_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1300         HOT_PATCH(.mutex_tryenter_lockstat_patch_point,
1301                 LS_MUTEX_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1302         HOT_PATCH(.rw_write_enter_lockstat_patch_point,
1303                 LS_RW_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1304 #endif  /* !OPTERON_WORKAROUND_6323525 */
1305         HOT_PATCH(.mutex_exit_lockstat_patch_point,
1306                 LS_MUTEX_EXIT_RELEASE, NOP_INSTR, RET_INSTR, 1)
1307         HOT_PATCH(.rw_read_enter_lockstat_patch_point,
1308                 LS_RW_ENTER_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1309         HOT_PATCH(.rw_write_exit_lockstat_patch_point,
1310                 LS_RW_EXIT_RELEASE, NOP_INSTR, RET_INSTR, 1)
1311         HOT_PATCH(.rw_read_exit_lockstat_patch_point,
1312                 LS_RW_EXIT_RELEASE, NOP_INSTR, RET_INSTR, 1)
1313         HOT_PATCH(.lock_set_lockstat_patch_point,
1314                 LS_LOCK_SET_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1315         HOT_PATCH(.lock_try_lockstat_patch_point,
1316                 LS_LOCK_TRY_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1317         HOT_PATCH(.lock_clear_lockstat_patch_point,
1318                 LS_LOCK_CLEAR_RELEASE, NOP_INSTR, RET_INSTR, 1)
1319         HOT_PATCH(.lock_set_spl_lockstat_patch_point,
1320                 LS_LOCK_SET_SPL_ACQUIRE, NOP_INSTR, RET_INSTR, 1)
1321
1322         HOT_PATCH(LOCK_CLEAR_SPLX_LOCKSTAT_PATCH_POINT,
1323                 LS_LOCK_CLEAR_SPLX_RELEASE,
1324                 LOCK_CLEAR_SPLX_LOCKSTAT_PATCH_VAL, 0, 1);
1325 #if defined(__amd64)
1326         leave                   /* unwind stack */
1327 #endif  /* __amd64 */
1328         ret
1329         SET_SIZE(lockstat_hot_patch)
1330
1331 #endif  /* __lint */
1332
1333 #if defined(lint) || defined(__lint)
1334
1335 /* XX64 membar_*() should be inlines */
1336
1337 void
1338 membar_sync(void)
1339 {}
1340
1341 void
1342 membar_enter(void)
1343 {}
1344
1345 void
1346 membar_exit(void)
1347 {}
1348
1349 void
1350 membar_producer(void)
1351 {}
1352
1353 void
1354 membar_consumer(void)
1355 {}
1356
1357 #else   /* __lint */
1358
1359 #if defined(__amd64)
1360
1361         ENTRY(membar_enter)
1362         ALTENTRY(membar_exit)
1363         ALTENTRY(membar_sync)
1364         mfence                  /* lighter weight than lock; xorq $0,(%rsp) */
1365         ret
1366         SET_SIZE(membar_sync)
1367         SET_SIZE(membar_exit)
1368         SET_SIZE(membar_enter)
1369
1370         ENTRY(membar_producer)
1371         sfence
1372         ret
1373         SET_SIZE(membar_producer)
1374
1375         ENTRY(membar_consumer)
1376         lfence
1377         ret
1378         SET_SIZE(membar_consumer)
1379
1380 #else
1381
1382         ENTRY(membar_enter)
1383         ALTENTRY(membar_exit)
1384         ALTENTRY(membar_sync)
1385         lock
1386         xorl    $0, (%esp)
1387         ret
1388         SET_SIZE(membar_sync)
1389         SET_SIZE(membar_exit)
1390         SET_SIZE(membar_enter)
1391
1392 /*
1393  * On machines that support sfence and lfence, these
1394  * memory barriers can be more precisely implemented
1395  * without causing the whole world to stop
1396  */
1397         ENTRY(membar_producer)
1398         .globl  _patch_sfence_ret
1399 _patch_sfence_ret:                      /* c.f. membar #StoreStore */
1400         lock
1401         xorl    $0, (%esp)
1402         ret
1403         SET_SIZE(membar_producer)
1404
1405         ENTRY(membar_consumer)
1406         .globl  _patch_lfence_ret
1407 _patch_lfence_ret:                      /* c.f. membar #LoadLoad */
1408         lock
1409         xorl    $0, (%esp)
1410         ret
1411         SET_SIZE(membar_consumer)
1412
1413 #endif  /* !__amd64 */
1414
1415 #endif  /* __lint */
1416
1417 /*
1418  * thread_onproc()
1419  * Set thread in onproc state for the specified CPU.
1420  * Also set the thread lock pointer to the CPU's onproc lock.
1421  * Since the new lock isn't held, the store ordering is important.
1422  * If not done in assembler, the compiler could reorder the stores.
1423  */
1424 #if defined(lint) || defined(__lint)
1425
1426 void
1427 thread_onproc(kthread_id_t t, cpu_t *cp)
1428 {
1429         t->t_state = TS_ONPROC;
1430         t->t_lockp = &cp->cpu_thread_lock;
1431 }
1432
1433 #else   /* __lint */
1434
1435 #if defined(__amd64)
1436
1437         ENTRY(thread_onproc)
1438         addq    $CPU_THREAD_LOCK, %rsi  /* pointer to disp_lock while running */
1439         movl    $ONPROC_THREAD, T_STATE(%rdi)   /* set state to TS_ONPROC */
1440         movq    %rsi, T_LOCKP(%rdi)     /* store new lock pointer */
1441         ret
1442         SET_SIZE(thread_onproc)
1443
1444 #else
1445
1446         ENTRY(thread_onproc)
1447         movl    4(%esp), %eax
1448         movl    8(%esp), %ecx
1449         addl    $CPU_THREAD_LOCK, %ecx  /* pointer to disp_lock while running */
1450         movl    $ONPROC_THREAD, T_STATE(%eax)   /* set state to TS_ONPROC */
1451         movl    %ecx, T_LOCKP(%eax)     /* store new lock pointer */
1452         ret
1453         SET_SIZE(thread_onproc)
1454
1455 #endif  /* !__amd64 */
1456
1457 #endif  /* __lint */
1458
1459 /*
1460  * mutex_delay_default(void)
1461  * Spins for approx a few hundred processor cycles and returns to caller.
1462  */
1463
1464 #if defined(lint) || defined(__lint)
1465
1466 void
1467 mutex_delay_default(void)
1468 {}
1469
1470 #else   /* __lint */
1471
1472 #if defined(__amd64)
1473
1474         ENTRY(mutex_delay_default)
1475         movq    $92,%r11
1476 0:      decq    %r11
1477         jg      0b
1478         ret
1479         SET_SIZE(mutex_delay_default)
1480
1481 #else
1482
1483         ENTRY(mutex_delay_default)
1484         push    %ebp
1485         movl    %esp,%ebp
1486         andl    $-16,%esp
1487         push    %ebx
1488         movl    $93,%ebx
1489 0:      decl    %ebx
1490         jg      0b
1491         pop     %ebx
1492         leave
1493         ret
1494         SET_SIZE(mutex_delay_default)
1495
1496 #endif  /* !__amd64 */
1497 #endif  /* __lint */