arch/x86/kernel/platform/i86pc/ml/mpcore.s

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
  23  */
  24 /*
  25  * Copyright (c) 2010, Intel Corporation.
  26  * All rights reserved.
  27  *
  28  * Copyright 2018 Joyent, Inc.
  29  */
  30
  31 #include <sys/asm_linkage.h>
  32 #include <sys/asm_misc.h>
  33 #include <sys/regset.h>
  34 #include <sys/privregs.h>
  35 #include <sys/x86_archext.h>
  36
  37 #include <sys/segments.h>
  38 #include "assym.h"
  39
  40 /*
  41  *      Our assumptions:
  42  *              - We are running in real mode.
  43  *              - Interrupts are disabled.
  44  *              - Selectors are equal (cs == ds == ss) for all real mode code
  45  *              - The GDT, IDT, ktss and page directory has been built for us
  46  *
  47  *      Our actions:
  48  *      Start CPU:
  49  *              - We start using our GDT by loading correct values in the
  50  *                selector registers (cs=KCS_SEL, ds=es=ss=KDS_SEL, fs=KFS_SEL,
  51  *                gs=KGS_SEL).
  52  *              - We change over to using our IDT.
  53  *              - We load the default LDT into the hardware LDT register.
  54  *              - We load the default TSS into the hardware task register.
  55  *              - call mp_startup(void) indirectly through the T_PC
  56  *      Stop CPU:
  57  *              - Put CPU into halted state with interrupts disabled
  58  *
  59  */
  60
  61
  62 #if defined(__amd64)
  63
  64         ENTRY_NP(real_mode_start_cpu)
  65
  66         /*
  67          * NOTE:  The GNU assembler automatically does the right thing to
  68          *        generate data size operand prefixes based on the code size
  69          *        generation mode (e.g. .code16, .code32, .code64) and as such
  70          *        prefixes need not be used on instructions EXCEPT in the case
  71          *        of address prefixes for code for which the reference is not
  72          *        automatically of the default operand size.
  73          */
  74         .code16
  75         cli
  76         movw            %cs, %ax
  77         movw            %ax, %ds        /* load cs into ds */
  78         movw            %ax, %ss        /* and into ss */
  79
  80         /*
  81          * Helps in debugging by giving us the fault address.
  82          *
  83          * Remember to patch a hlt (0xf4) at cmntrap to get a good stack.
  84          */
  85         movl            $0xffc, %esp
  86         movl            %cr0, %eax
  87
  88         /*
  89          * Enable protected-mode, write protect, and alignment mask
  90          */
  91         orl             $(CR0_PE|CR0_WP|CR0_AM), %eax
  92         movl            %eax, %cr0
  93
  94         /*
  95          * Do a jmp immediately after writing to cr0 when enabling protected
  96          * mode to clear the real mode prefetch queue (per Intel's docs)
  97          */
  98         jmp             pestart
  99
 100 pestart:
 101         /*
 102          * 16-bit protected mode is now active, so prepare to turn on long
 103          * mode.
 104          *
 105          * Note that we currently assume that if we're attempting to run a
 106          * kernel compiled with (__amd64) #defined, the target CPU has long
 107          * mode support.
 108          */
 109
 110 #if 0
 111         /*
 112          * If there's a chance this might not be true, the following test should
 113          * be done, with the no_long_mode branch then doing something
 114          * appropriate:
 115          */
 116
 117         movl            $0x80000000, %eax       /* get largest extended CPUID */
 118         cpuid
 119         cmpl            $0x80000000, %eax       /* check if > 0x80000000 */
 120         jbe             no_long_mode            /* nope, no long mode */
 121         movl            $0x80000001, %eax
 122         cpuid                                   /* get extended feature flags */
 123         btl             $29, %edx               /* check for long mode */
 124         jnc             no_long_mode            /* long mode not supported */
 125 #endif
 126
 127         /*
 128          * Add any initial cr4 bits
 129          */
 130         movl            %cr4, %eax
 131         addr32 orl      CR4OFF, %eax
 132
 133         /*
 134          * Enable PAE mode (CR4.PAE)
 135          */
 136         orl             $CR4_PAE, %eax
 137         movl            %eax, %cr4
 138
 139         /*
 140          * Point cr3 to the 64-bit long mode page tables.
 141          *
 142          * Note that these MUST exist in 32-bit space, as we don't have
 143          * a way to load %cr3 with a 64-bit base address for the page tables
 144          * until the CPU is actually executing in 64-bit long mode.
 145          */
 146         addr32 movl     CR3OFF, %eax
 147         movl            %eax, %cr3
 148
 149         /*
 150          * Set long mode enable in EFER (EFER.LME = 1)
 151          */
 152         movl    $MSR_AMD_EFER, %ecx
 153         rdmsr
 154         orl     $AMD_EFER_LME, %eax
 155         wrmsr
 156
 157         /*
 158          * Finally, turn on paging (CR0.PG = 1) to activate long mode.
 159          */
 160         movl    %cr0, %eax
 161         orl     $CR0_PG, %eax
 162         movl    %eax, %cr0
 163
 164         /*
 165          * The instruction after enabling paging in CR0 MUST be a branch.
 166          */
 167         jmp     long_mode_active
 168
 169 long_mode_active:
 170         /*
 171          * Long mode is now active but since we're still running with the
 172          * original 16-bit CS we're actually in 16-bit compatability mode.
 173          *
 174          * We have to load an intermediate GDT and IDT here that we know are
 175          * in 32-bit space before we can use the kernel's GDT and IDT, which
 176          * may be in the 64-bit address space, and since we're in compatability
 177          * mode, we only have access to 16 and 32-bit instructions at the
 178          * moment.
 179          */
 180         addr32 lgdtl    TEMPGDTOFF      /* load temporary GDT */
 181         addr32 lidtl    TEMPIDTOFF      /* load temporary IDT */
 182
 183         /*
 184          * Do a far transfer to 64-bit mode.  Set the CS selector to a 64-bit
 185          * long mode selector (CS.L=1) in the temporary 32-bit GDT and jump
 186          * to the real mode platter address of long_mode 64 as until the 64-bit
 187          * CS is in place we don't have access to 64-bit instructions and thus
 188          * can't reference a 64-bit %rip.
 189          */
 190         pushl           $TEMP_CS64_SEL
 191         addr32 pushl    LM64OFF
 192         lretl
 193
 194         .globl  long_mode_64
 195 long_mode_64:
 196         .code64
 197         /*
 198          * We are now running in long mode with a 64-bit CS (EFER.LMA=1,
 199          * CS.L=1) so we now have access to 64-bit instructions.
 200          *
 201          * First, set the 64-bit GDT base.
 202          */
 203         .globl  rm_platter_pa
 204         movl    rm_platter_pa, %eax
 205         lgdtq   GDTROFF(%rax)           /* load 64-bit GDT */
 206
 207         /*
 208          * Save the CPU number in %r11; get the value here since it's saved in
 209          * the real mode platter.
 210          */
 211         movl    CPUNOFF(%rax), %r11d
 212
 213         /*
 214          * Add rm_platter_pa to %rsp to point it to the same location as seen
 215          * from 64-bit mode.
 216          */
 217         addq    %rax, %rsp
 218
 219         /*
 220          * Now do an lretq to load CS with the appropriate selector for the
 221          * kernel's 64-bit GDT and to start executing 64-bit setup code at the
 222          * virtual address where boot originally loaded this code rather than
 223          * the copy in the real mode platter's rm_code array as we've been
 224          * doing so far.
 225          */
 226         pushq   $KCS_SEL
 227         pushq   $kernel_cs_code
 228         lretq
 229         .globl real_mode_start_cpu_end
 230 real_mode_start_cpu_end:
 231         nop
 232
 233 kernel_cs_code:
 234         /*
 235          * Complete the balance of the setup we need to before executing
 236          * 64-bit kernel code (namely init rsp, TSS, LGDT, FS and GS).
 237          */
 238         .globl  rm_platter_va
 239         movq    rm_platter_va, %rax
 240         lidtq   IDTROFF(%rax)
 241
 242         movw    $KDS_SEL, %ax
 243         movw    %ax, %ds
 244         movw    %ax, %es
 245         movw    %ax, %ss
 246
 247         movw    $KTSS_SEL, %ax          /* setup kernel TSS */
 248         ltr     %ax
 249
 250         xorw    %ax, %ax                /* clear LDTR */
 251         lldt    %ax
 252
 253         /*
 254          * Set GS to the address of the per-cpu structure as contained in
 255          * cpu[cpu_number].
 256          *
 257          * Unfortunately there's no way to set the 64-bit gsbase with a mov,
 258          * so we have to stuff the low 32 bits in %eax and the high 32 bits in
 259          * %edx, then call wrmsr.
 260          */
 261         leaq    cpu(%rip), %rdi
 262         movl    (%rdi, %r11, 8), %eax
 263         movl    4(%rdi, %r11, 8), %edx
 264         movl    $MSR_AMD_GSBASE, %ecx
 265         wrmsr
 266
 267         /*
 268          * Init FS and KernelGSBase.
 269          *
 270          * Based on code in mlsetup(), set them both to 8G (which shouldn't be
 271          * valid until some 64-bit processes run); this will then cause an
 272          * exception in any code that tries to index off them before they are
 273          * properly setup.
 274          */
 275         xorl    %eax, %eax              /* low 32 bits = 0 */
 276         movl    $2, %edx                /* high 32 bits = 2 */
 277         movl    $MSR_AMD_FSBASE, %ecx
 278         wrmsr
 279
 280         movl    $MSR_AMD_KGSBASE, %ecx
 281         wrmsr
 282
 283         /*
 284          * Init %rsp to the exception stack set in tss_ist1 and create a legal
 285          * AMD64 ABI stack frame
 286          */
 287         movq    %gs:CPU_TSS, %rax
 288         movq    TSS_IST1(%rax), %rsp
 289         pushq   $0              /* null return address */
 290         pushq   $0              /* null frame pointer terminates stack trace */
 291         movq    %rsp, %rbp      /* stack aligned on 16-byte boundary */
 292
 293         movq    %cr0, %rax
 294         andq    $~(CR0_TS|CR0_EM), %rax /* clear emulate math chip bit */
 295         orq     $(CR0_MP|CR0_NE), %rax
 296         movq    %rax, %cr0              /* set machine status word */
 297
 298         /*
 299          * Before going any further, enable usage of page table NX bit if
 300          * that's how our page tables are set up.
 301          */
 302         bt      $X86FSET_NX, x86_featureset(%rip)
 303         jnc     1f
 304         movl    $MSR_AMD_EFER, %ecx
 305         rdmsr
 306         orl     $AMD_EFER_NXE, %eax
 307         wrmsr
 308 1:
 309
 310         /*
 311          * Complete the rest of the setup and call mp_startup().
 312          */
 313         movq    %gs:CPU_THREAD, %rax    /* get thread ptr */
 314         call    *T_PC(%rax)             /* call mp_startup_boot */
 315         /* not reached */
 316         int     $20                     /* whoops, returned somehow! */
 317
 318         SET_SIZE(real_mode_start_cpu)
 319
 320 #elif defined(__i386)
 321
 322         ENTRY_NP(real_mode_start_cpu)
 323
 324 #if !defined(__GNUC_AS__)
 325
 326         cli
 327         D16 movw        %cs, %eax
 328         movw            %eax, %ds       /* load cs into ds */
 329         movw            %eax, %ss       /* and into ss */
 330
 331         /*
 332          * Helps in debugging by giving us the fault address.
 333          *
 334          * Remember to patch a hlt (0xf4) at cmntrap to get a good stack.
 335          */
 336         D16 movl        $0xffc, %esp
 337
 338         D16 A16 lgdt    %cs:GDTROFF
 339         D16 A16 lidt    %cs:IDTROFF
 340         D16 A16 movl    %cs:CR4OFF, %eax        /* set up CR4, if desired */
 341         D16 andl        %eax, %eax
 342         D16 A16 je      no_cr4
 343
 344         D16 movl        %eax, %ecx
 345         D16 movl        %cr4, %eax
 346         D16 orl         %ecx, %eax
 347         D16 movl        %eax, %cr4
 348 no_cr4:
 349         D16 A16 movl    %cs:CR3OFF, %eax
 350         A16 movl        %eax, %cr3
 351         movl            %cr0, %eax
 352
 353         /*
 354          * Enable protected-mode, paging, write protect, and alignment mask
 355          */
 356         D16 orl         $[CR0_PG|CR0_PE|CR0_WP|CR0_AM], %eax
 357         movl            %eax, %cr0
 358         jmp             pestart
 359
 360 pestart:
 361         D16 pushl       $KCS_SEL
 362         D16 pushl       $kernel_cs_code
 363         D16 lret
 364         .globl real_mode_start_cpu_end
 365 real_mode_start_cpu_end:
 366         nop
 367
 368         .globl  kernel_cs_code
 369 kernel_cs_code:
 370         /*
 371          * At this point we are with kernel's cs and proper eip.
 372          *
 373          * We will be executing not from the copy in real mode platter,
 374          * but from the original code where boot loaded us.
 375          *
 376          * By this time GDT and IDT are loaded as is cr3.
 377          */
 378         movw    $KFS_SEL,%eax
 379         movw    %eax,%fs
 380         movw    $KGS_SEL,%eax
 381         movw    %eax,%gs
 382         movw    $KDS_SEL,%eax
 383         movw    %eax,%ds
 384         movw    %eax,%es
 385         movl    %gs:CPU_TSS,%esi
 386         movw    %eax,%ss
 387         movl    TSS_ESP0(%esi),%esp
 388         movw    $KTSS_SEL,%ax
 389         ltr     %ax
 390         xorw    %ax, %ax                /* clear LDTR */
 391         lldt    %ax
 392         movl    %cr0,%edx
 393         andl    $-1![CR0_TS|CR0_EM],%edx  /* clear emulate math chip bit */
 394         orl     $[CR0_MP|CR0_NE],%edx
 395         movl    %edx,%cr0                 /* set machine status word */
 396
 397         /*
 398          * Before going any further, enable usage of page table NX bit if
 399          * that's how our page tables are set up.
 400          */
 401         bt      $X86FSET_NX, x86_featureset
 402         jnc     1f
 403         movl    %cr4, %ecx
 404         andl    $CR4_PAE, %ecx
 405         jz      1f
 406         movl    $MSR_AMD_EFER, %ecx
 407         rdmsr
 408         orl     $AMD_EFER_NXE, %eax
 409         wrmsr
 410 1:
 411         movl    %gs:CPU_THREAD, %eax    /* get thread ptr */
 412         call    *T_PC(%eax)             /* call mp_startup */
 413         /* not reached */
 414         int     $20                     /* whoops, returned somehow! */
 415
 416 #else
 417
 418         cli
 419         mov             %cs, %ax
 420         mov             %eax, %ds       /* load cs into ds */
 421         mov             %eax, %ss       /* and into ss */
 422
 423         /*
 424          * Helps in debugging by giving us the fault address.
 425          *
 426          * Remember to patch a hlt (0xf4) at cmntrap to get a good stack.
 427          */
 428         D16 mov         $0xffc, %esp
 429
 430         D16 A16 lgdtl   %cs:GDTROFF
 431         D16 A16 lidtl   %cs:IDTROFF
 432         D16 A16 mov     %cs:CR4OFF, %eax        /* set up CR4, if desired */
 433         D16 and         %eax, %eax
 434         D16 A16 je      no_cr4
 435
 436         D16 mov         %eax, %ecx
 437         D16 mov         %cr4, %eax
 438         D16 or          %ecx, %eax
 439         D16 mov         %eax, %cr4
 440 no_cr4:
 441         D16 A16 mov     %cs:CR3OFF, %eax
 442         A16 mov         %eax, %cr3
 443         mov             %cr0, %eax
 444
 445         /*
 446          * Enable protected-mode, paging, write protect, and alignment mask
 447          */
 448         D16 or          $(CR0_PG|CR0_PE|CR0_WP|CR0_AM), %eax
 449         mov             %eax, %cr0
 450         jmp             pestart
 451
 452 pestart:
 453         D16 pushl       $KCS_SEL
 454         D16 pushl       $kernel_cs_code
 455         D16 lret
 456         .globl real_mode_start_cpu_end
 457 real_mode_start_cpu_end:
 458         nop
 459         .globl  kernel_cs_code
 460 kernel_cs_code:
 461         /*
 462          * At this point we are with kernel's cs and proper eip.
 463          *
 464          * We will be executing not from the copy in real mode platter,
 465          * but from the original code where boot loaded us.
 466          *
 467          * By this time GDT and IDT are loaded as is cr3.
 468          */
 469         mov     $KFS_SEL, %ax
 470         mov     %eax, %fs
 471         mov     $KGS_SEL, %ax
 472         mov     %eax, %gs
 473         mov     $KDS_SEL, %ax
 474         mov     %eax, %ds
 475         mov     %eax, %es
 476         mov     %gs:CPU_TSS, %esi
 477         mov     %eax, %ss
 478         mov     TSS_ESP0(%esi), %esp
 479         mov     $(KTSS_SEL), %ax
 480         ltr     %ax
 481         xorw    %ax, %ax                /* clear LDTR */
 482         lldt    %ax
 483         mov     %cr0, %edx
 484         and     $~(CR0_TS|CR0_EM), %edx /* clear emulate math chip bit */
 485         or      $(CR0_MP|CR0_NE), %edx
 486         mov     %edx, %cr0              /* set machine status word */
 487
 488         /*
 489          * Before going any farther, enable usage of page table NX bit if
 490          * that's how our page tables are set up.  (PCIDE is enabled later on).
 491          */
 492         bt      $X86FSET_NX, x86_featureset
 493         jnc     1f
 494         movl    %cr4, %ecx
 495         andl    $CR4_PAE, %ecx
 496         jz      1f
 497         movl    $MSR_AMD_EFER, %ecx
 498         rdmsr
 499         orl     $AMD_EFER_NXE, %eax
 500         wrmsr
 501 1:
 502         mov     %gs:CPU_THREAD, %eax    /* get thread ptr */
 503         call    *T_PC(%eax)             /* call mp_startup */
 504         /* not reached */
 505         int     $20                     /* whoops, returned somehow! */
 506 #endif
 507
 508         SET_SIZE(real_mode_start_cpu)
 509
 510 #endif  /* __amd64 */
 511
 512 #if defined(__amd64)
 513
 514         ENTRY_NP(real_mode_stop_cpu_stage1)
 515
 516 #if !defined(__GNUC_AS__)
 517
 518         /*
 519          * For vulcan as we need to do a .code32 and mentally invert the
 520          * meaning of the addr16 and data16 prefixes to get 32-bit access when
 521          * generating code to be executed in 16-bit mode (sigh...)
 522          */
 523         .code32
 524         cli
 525         movw            %cs, %ax
 526         movw            %ax, %ds        /* load cs into ds */
 527         movw            %ax, %ss        /* and into ss */
 528
 529         /*
 530          * Jump to the stage 2 code in the rm_platter_va->rm_cpu_halt_code
 531          */
 532         movw            $CPUHALTCODEOFF, %ax
 533         .byte           0xff, 0xe0      /* jmp *%ax */
 534
 535 #else   /* __GNUC_AS__ */
 536
 537         /*
 538          * NOTE:  The GNU assembler automatically does the right thing to
 539          *        generate data size operand prefixes based on the code size
 540          *        generation mode (e.g. .code16, .code32, .code64) and as such
 541          *        prefixes need not be used on instructions EXCEPT in the case
 542          *        of address prefixes for code for which the reference is not
 543          *        automatically of the default operand size.
 544          */
 545         .code16
 546         cli
 547         movw            %cs, %ax
 548         movw            %ax, %ds        /* load cs into ds */
 549         movw            %ax, %ss        /* and into ss */
 550
 551         /*
 552          * Jump to the stage 2 code in the rm_platter_va->rm_cpu_halt_code
 553          */
 554         movw            $CPUHALTCODEOFF, %ax
 555         jmp             *%ax
 556
 557 #endif  /* !__GNUC_AS__ */
 558
 559         .globl real_mode_stop_cpu_stage1_end
 560 real_mode_stop_cpu_stage1_end:
 561         nop
 562
 563         SET_SIZE(real_mode_stop_cpu_stage1)
 564
 565 #elif defined(__i386)
 566
 567         ENTRY_NP(real_mode_stop_cpu_stage1)
 568
 569 #if !defined(__GNUC_AS__)
 570
 571         cli
 572         D16 movw        %cs, %eax
 573         movw            %eax, %ds       /* load cs into ds */
 574         movw            %eax, %ss       /* and into ss */
 575
 576         /*
 577          * Jump to the stage 2 code in the rm_platter_va->rm_cpu_halt_code
 578          */
 579         movw            $CPUHALTCODEOFF, %ax
 580         .byte           0xff, 0xe0      /* jmp *%ax */
 581
 582 #else   /* __GNUC_AS__ */
 583
 584         cli
 585         mov             %cs, %ax
 586         mov             %eax, %ds       /* load cs into ds */
 587         mov             %eax, %ss       /* and into ss */
 588
 589         /*
 590          * Jump to the stage 2 code in the rm_platter_va->rm_cpu_halt_code
 591          */
 592         movw            $CPUHALTCODEOFF, %ax
 593         jmp             *%ax
 594
 595 #endif  /* !__GNUC_AS__ */
 596
 597         .globl real_mode_stop_cpu_stage1_end
 598 real_mode_stop_cpu_stage1_end:
 599         nop
 600
 601         SET_SIZE(real_mode_stop_cpu_stage1)
 602
 603 #endif  /* __amd64 */
 604
 605         ENTRY_NP(real_mode_stop_cpu_stage2)
 606
 607         movw            $0xdead, %ax
 608         movw            %ax, CPUHALTEDOFF
 609
 610 real_mode_stop_cpu_loop:
 611         /*
 612          * Put CPU into halted state.
 613          * Only INIT, SMI, NMI could break the loop.
 614          */
 615         hlt
 616         jmp             real_mode_stop_cpu_loop
 617
 618         .globl real_mode_stop_cpu_stage2_end
 619 real_mode_stop_cpu_stage2_end:
 620         nop
 621
 622         SET_SIZE(real_mode_stop_cpu_stage2)
 623