1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #pragma ident "%Z%%M% %I% %E% SMI" 28 29 /* 30 * Copyright (c) 1992 Terrence R. Lambert. 31 * Copyright (c) 1990 The Regents of the University of California. 32 * All rights reserved. 33 * 34 * This code is derived from software contributed to Berkeley by 35 * William Jolitz. 36 * 37 * Redistribution and use in source and binary forms, with or without 38 * modification, are permitted provided that the following conditions 39 * are met: 40 * 1. Redistributions of source code must retain the above copyright 41 * notice, this list of conditions and the following disclaimer. 42 * 2. Redistributions in binary form must reproduce the above copyright 43 * notice, this list of conditions and the following disclaimer in the 44 * documentation and/or other materials provided with the distribution. 45 * 3. All advertising materials mentioning features or use of this software 46 * must display the following acknowledgement: 47 * This product includes software developed by the University of 48 * California, Berkeley and its contributors. 49 * 4. Neither the name of the University nor the names of its contributors 50 * may be used to endorse or promote products derived from this software 51 * without specific prior written permission. 52 * 53 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 54 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 55 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 56 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 57 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 58 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 59 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 60 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 61 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 62 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 63 * SUCH DAMAGE. 64 * 65 * from: @(#)machdep.c 7.4 (Berkeley) 6/3/91 66 */ 67 68 #include <sys/types.h> 69 #include <sys/sysmacros.h> 70 #include <sys/tss.h> 71 #include <sys/segments.h> 72 #include <sys/trap.h> 73 #include <sys/cpuvar.h> 74 #include <sys/bootconf.h> 75 #include <sys/x86_archext.h> 76 #include <sys/controlregs.h> 77 #include <sys/archsystm.h> 78 #include <sys/machsystm.h> 79 #include <sys/kobj.h> 80 #include <sys/cmn_err.h> 81 #include <sys/reboot.h> 82 #include <sys/kdi.h> 83 #include <sys/mach_mmu.h> 84 #include <sys/systm.h> 85 86 #ifdef __xpv 87 #include <sys/hypervisor.h> 88 #include <vm/as.h> 89 #endif 90 91 #include <sys/promif.h> 92 #include <sys/bootinfo.h> 93 #include <vm/kboot_mmu.h> 94 #include <vm/hat_pte.h> 95 96 /* 97 * cpu0 and default tables and structures. 98 */ 99 user_desc_t *gdt0; 100 #if !defined(__xpv) 101 desctbr_t gdt0_default_r; 102 #endif 103 104 gate_desc_t *idt0; /* interrupt descriptor table */ 105 #if defined(__i386) 106 desctbr_t idt0_default_r; /* describes idt0 in IDTR format */ 107 #endif 108 109 struct tss *ktss0; /* kernel task state structure */ 110 111 #if defined(__i386) 112 struct tss *dftss0; /* #DF double-fault exception */ 113 #endif /* __i386 */ 114 115 user_desc_t zero_udesc; /* base zero user desc native procs */ 116 user_desc_t null_udesc; /* null user descriptor */ 117 system_desc_t null_sdesc; /* null system descriptor */ 118 119 #if defined(__amd64) 120 user_desc_t zero_u32desc; /* 32-bit compatibility procs */ 121 #endif /* __amd64 */ 122 123 #if defined(__amd64) 124 user_desc_t ucs_on; 125 user_desc_t ucs_off; 126 user_desc_t ucs32_on; 127 user_desc_t ucs32_off; 128 #endif /* __amd64 */ 129 130 #pragma align 16(dblfault_stack0) 131 char dblfault_stack0[DEFAULTSTKSZ]; 132 133 extern void fast_null(void); 134 extern hrtime_t get_hrtime(void); 135 extern hrtime_t gethrvtime(void); 136 extern hrtime_t get_hrestime(void); 137 extern uint64_t getlgrp(void); 138 139 void (*(fasttable[]))(void) = { 140 fast_null, /* T_FNULL routine */ 141 fast_null, /* T_FGETFP routine (initially null) */ 142 fast_null, /* T_FSETFP routine (initially null) */ 143 (void (*)())get_hrtime, /* T_GETHRTIME */ 144 (void (*)())gethrvtime, /* T_GETHRVTIME */ 145 (void (*)())get_hrestime, /* T_GETHRESTIME */ 146 (void (*)())getlgrp /* T_GETLGRP */ 147 }; 148 149 /* 150 * Structure containing pre-computed descriptors to allow us to temporarily 151 * interpose on a standard handler. 152 */ 153 struct interposing_handler { 154 int ih_inum; 155 gate_desc_t ih_interp_desc; 156 gate_desc_t ih_default_desc; 157 }; 158 159 /* 160 * The brand infrastructure interposes on two handlers, and we use one as a 161 * NULL signpost. 162 */ 163 static struct interposing_handler brand_tbl[3]; 164 165 /* 166 * software prototypes for default local descriptor table 167 */ 168 169 /* 170 * Routines for loading segment descriptors in format the hardware 171 * can understand. 172 */ 173 174 #if defined(__amd64) 175 176 /* 177 * In long mode we have the new L or long mode attribute bit 178 * for code segments. Only the conforming bit in type is used along 179 * with descriptor priority and present bits. Default operand size must 180 * be zero when in long mode. In 32-bit compatibility mode all fields 181 * are treated as in legacy mode. For data segments while in long mode 182 * only the present bit is loaded. 183 */ 184 void 185 set_usegd(user_desc_t *dp, uint_t lmode, void *base, size_t size, 186 uint_t type, uint_t dpl, uint_t gran, uint_t defopsz) 187 { 188 ASSERT(lmode == SDP_SHORT || lmode == SDP_LONG); 189 190 /* 191 * 64-bit long mode. 192 */ 193 if (lmode == SDP_LONG) 194 dp->usd_def32 = 0; /* 32-bit operands only */ 195 else 196 /* 197 * 32-bit compatibility mode. 198 */ 199 dp->usd_def32 = defopsz; /* 0 = 16, 1 = 32-bit ops */ 200 201 dp->usd_long = lmode; /* 64-bit mode */ 202 dp->usd_type = type; 203 dp->usd_dpl = dpl; 204 dp->usd_p = 1; 205 dp->usd_gran = gran; /* 0 = bytes, 1 = pages */ 206 207 dp->usd_lobase = (uintptr_t)base; 208 dp->usd_midbase = (uintptr_t)base >> 16; 209 dp->usd_hibase = (uintptr_t)base >> (16 + 8); 210 dp->usd_lolimit = size; 211 dp->usd_hilimit = (uintptr_t)size >> 16; 212 } 213 214 #elif defined(__i386) 215 216 /* 217 * Install user segment descriptor for code and data. 218 */ 219 void 220 set_usegd(user_desc_t *dp, void *base, size_t size, uint_t type, 221 uint_t dpl, uint_t gran, uint_t defopsz) 222 { 223 dp->usd_lolimit = size; 224 dp->usd_hilimit = (uintptr_t)size >> 16; 225 226 dp->usd_lobase = (uintptr_t)base; 227 dp->usd_midbase = (uintptr_t)base >> 16; 228 dp->usd_hibase = (uintptr_t)base >> (16 + 8); 229 230 dp->usd_type = type; 231 dp->usd_dpl = dpl; 232 dp->usd_p = 1; 233 dp->usd_def32 = defopsz; /* 0 = 16, 1 = 32 bit operands */ 234 dp->usd_gran = gran; /* 0 = bytes, 1 = pages */ 235 } 236 237 #endif /* __i386 */ 238 239 /* 240 * Install system segment descriptor for LDT and TSS segments. 241 */ 242 243 #if defined(__amd64) 244 245 void 246 set_syssegd(system_desc_t *dp, void *base, size_t size, uint_t type, 247 uint_t dpl) 248 { 249 dp->ssd_lolimit = size; 250 dp->ssd_hilimit = (uintptr_t)size >> 16; 251 252 dp->ssd_lobase = (uintptr_t)base; 253 dp->ssd_midbase = (uintptr_t)base >> 16; 254 dp->ssd_hibase = (uintptr_t)base >> (16 + 8); 255 dp->ssd_hi64base = (uintptr_t)base >> (16 + 8 + 8); 256 257 dp->ssd_type = type; 258 dp->ssd_zero1 = 0; /* must be zero */ 259 dp->ssd_zero2 = 0; 260 dp->ssd_dpl = dpl; 261 dp->ssd_p = 1; 262 dp->ssd_gran = 0; /* force byte units */ 263 } 264 265 void * 266 get_ssd_base(system_desc_t *dp) 267 { 268 uintptr_t base; 269 270 base = (uintptr_t)dp->ssd_lobase | 271 (uintptr_t)dp->ssd_midbase << 16 | 272 (uintptr_t)dp->ssd_hibase << (16 + 8) | 273 (uintptr_t)dp->ssd_hi64base << (16 + 8 + 8); 274 return ((void *)base); 275 } 276 277 #elif defined(__i386) 278 279 void 280 set_syssegd(system_desc_t *dp, void *base, size_t size, uint_t type, 281 uint_t dpl) 282 { 283 dp->ssd_lolimit = size; 284 dp->ssd_hilimit = (uintptr_t)size >> 16; 285 286 dp->ssd_lobase = (uintptr_t)base; 287 dp->ssd_midbase = (uintptr_t)base >> 16; 288 dp->ssd_hibase = (uintptr_t)base >> (16 + 8); 289 290 dp->ssd_type = type; 291 dp->ssd_zero = 0; /* must be zero */ 292 dp->ssd_dpl = dpl; 293 dp->ssd_p = 1; 294 dp->ssd_gran = 0; /* force byte units */ 295 } 296 297 void * 298 get_ssd_base(system_desc_t *dp) 299 { 300 uintptr_t base; 301 302 base = (uintptr_t)dp->ssd_lobase | 303 (uintptr_t)dp->ssd_midbase << 16 | 304 (uintptr_t)dp->ssd_hibase << (16 + 8); 305 return ((void *)base); 306 } 307 308 #endif /* __i386 */ 309 310 /* 311 * Install gate segment descriptor for interrupt, trap, call and task gates. 312 */ 313 314 #if defined(__amd64) 315 316 void 317 set_gatesegd(gate_desc_t *dp, void (*func)(void), selector_t sel, 318 uint_t type, uint_t dpl) 319 { 320 dp->sgd_looffset = (uintptr_t)func; 321 dp->sgd_hioffset = (uintptr_t)func >> 16; 322 dp->sgd_hi64offset = (uintptr_t)func >> (16 + 16); 323 324 dp->sgd_selector = (uint16_t)sel; 325 326 /* 327 * For 64 bit native we use the IST stack mechanism 328 * for double faults. All other traps use the CPL = 0 329 * (tss_rsp0) stack. 330 */ 331 #if !defined(__xpv) 332 if (type == T_DBLFLT) 333 dp->sgd_ist = 1; 334 else 335 #endif 336 dp->sgd_ist = 0; 337 338 dp->sgd_type = type; 339 dp->sgd_dpl = dpl; 340 dp->sgd_p = 1; 341 } 342 343 #elif defined(__i386) 344 345 void 346 set_gatesegd(gate_desc_t *dp, void (*func)(void), selector_t sel, 347 uint_t type, uint_t dpl) 348 { 349 dp->sgd_looffset = (uintptr_t)func; 350 dp->sgd_hioffset = (uintptr_t)func >> 16; 351 352 dp->sgd_selector = (uint16_t)sel; 353 dp->sgd_stkcpy = 0; /* always zero bytes */ 354 dp->sgd_type = type; 355 dp->sgd_dpl = dpl; 356 dp->sgd_p = 1; 357 } 358 359 #endif /* __i386 */ 360 361 /* 362 * Updates a single user descriptor in the the GDT of the current cpu. 363 * Caller is responsible for preventing cpu migration. 364 */ 365 366 void 367 gdt_update_usegd(uint_t sidx, user_desc_t *udp) 368 { 369 #if defined(__xpv) 370 371 uint64_t dpa = CPU->cpu_m.mcpu_gdtpa + sizeof (*udp) * sidx; 372 373 if (HYPERVISOR_update_descriptor(pa_to_ma(dpa), *(uint64_t *)udp)) 374 panic("gdt_update_usegd: HYPERVISOR_update_descriptor"); 375 376 #else /* __xpv */ 377 378 CPU->cpu_gdt[sidx] = *udp; 379 380 #endif /* __xpv */ 381 } 382 383 /* 384 * Writes single descriptor pointed to by udp into a processes 385 * LDT entry pointed to by ldp. 386 */ 387 int 388 ldt_update_segd(user_desc_t *ldp, user_desc_t *udp) 389 { 390 #if defined(__xpv) 391 392 uint64_t dpa; 393 394 dpa = mmu_ptob(hat_getpfnum(kas.a_hat, (caddr_t)ldp)) | 395 ((uintptr_t)ldp & PAGEOFFSET); 396 397 /* 398 * The hypervisor is a little more restrictive about what it 399 * supports in the LDT. 400 */ 401 if (HYPERVISOR_update_descriptor(pa_to_ma(dpa), *(uint64_t *)udp) != 0) 402 return (EINVAL); 403 404 #else /* __xpv */ 405 406 *ldp = *udp; 407 408 #endif /* __xpv */ 409 return (0); 410 } 411 412 #if defined(__xpv) 413 414 /* 415 * Converts hw format gate descriptor into pseudo-IDT format for the hypervisor. 416 * Returns true if a valid entry was written. 417 */ 418 int 419 xen_idt_to_trap_info(uint_t vec, gate_desc_t *sgd, void *ti_arg) 420 { 421 trap_info_t *ti = ti_arg; /* XXPV Aargh - segments.h comment */ 422 423 /* 424 * skip holes in the IDT 425 */ 426 if (GATESEG_GETOFFSET(sgd) == 0) 427 return (0); 428 429 ASSERT(sgd->sgd_type == SDT_SYSIGT); 430 ti->vector = vec; 431 TI_SET_DPL(ti, sgd->sgd_dpl); 432 433 /* 434 * Is this an interrupt gate? 435 */ 436 if (sgd->sgd_type == SDT_SYSIGT) { 437 /* LINTED */ 438 TI_SET_IF(ti, 1); 439 } 440 ti->cs = sgd->sgd_selector; 441 #if defined(__amd64) 442 ti->cs |= SEL_KPL; /* force into ring 3. see KCS_SEL */ 443 #endif 444 ti->address = GATESEG_GETOFFSET(sgd); 445 return (1); 446 } 447 448 /* 449 * Convert a single hw format gate descriptor and write it into our virtual IDT. 450 */ 451 void 452 xen_idt_write(gate_desc_t *sgd, uint_t vec) 453 { 454 trap_info_t trapinfo[2]; 455 456 bzero(trapinfo, sizeof (trapinfo)); 457 if (xen_idt_to_trap_info(vec, sgd, &trapinfo[0]) == 0) 458 return; 459 if (xen_set_trap_table(trapinfo) != 0) 460 panic("xen_idt_write: xen_set_trap_table() failed"); 461 } 462 463 #endif /* __xpv */ 464 465 #if defined(__amd64) 466 467 /* 468 * Build kernel GDT. 469 */ 470 471 static void 472 init_gdt_common(user_desc_t *gdt) 473 { 474 int i; 475 476 /* 477 * 64-bit kernel code segment. 478 */ 479 set_usegd(&gdt[GDT_KCODE], SDP_LONG, NULL, 0, SDT_MEMERA, SEL_KPL, 480 SDP_PAGES, SDP_OP32); 481 482 /* 483 * 64-bit kernel data segment. The limit attribute is ignored in 64-bit 484 * mode, but we set it here to 0xFFFF so that we can use the SYSRET 485 * instruction to return from system calls back to 32-bit applications. 486 * SYSRET doesn't update the base, limit, or attributes of %ss or %ds 487 * descriptors. We therefore must ensure that the kernel uses something, 488 * though it will be ignored by hardware, that is compatible with 32-bit 489 * apps. For the same reason we must set the default op size of this 490 * descriptor to 32-bit operands. 491 */ 492 set_usegd(&gdt[GDT_KDATA], SDP_LONG, NULL, -1, SDT_MEMRWA, 493 SEL_KPL, SDP_PAGES, SDP_OP32); 494 gdt[GDT_KDATA].usd_def32 = 1; 495 496 /* 497 * 64-bit user code segment. 498 */ 499 set_usegd(&gdt[GDT_UCODE], SDP_LONG, NULL, 0, SDT_MEMERA, SEL_UPL, 500 SDP_PAGES, SDP_OP32); 501 502 /* 503 * 32-bit user code segment. 504 */ 505 set_usegd(&gdt[GDT_U32CODE], SDP_SHORT, NULL, -1, SDT_MEMERA, 506 SEL_UPL, SDP_PAGES, SDP_OP32); 507 508 /* 509 * See gdt_ucode32() and gdt_ucode_native(). 510 */ 511 ucs_on = ucs_off = gdt[GDT_UCODE]; 512 ucs_off.usd_p = 0; /* forces #np fault */ 513 514 ucs32_on = ucs32_off = gdt[GDT_U32CODE]; 515 ucs32_off.usd_p = 0; /* forces #np fault */ 516 517 /* 518 * 32 and 64 bit data segments can actually share the same descriptor. 519 * In long mode only the present bit is checked but all other fields 520 * are loaded. But in compatibility mode all fields are interpreted 521 * as in legacy mode so they must be set correctly for a 32-bit data 522 * segment. 523 */ 524 set_usegd(&gdt[GDT_UDATA], SDP_SHORT, NULL, -1, SDT_MEMRWA, SEL_UPL, 525 SDP_PAGES, SDP_OP32); 526 527 #if !defined(__xpv) 528 529 /* 530 * The 64-bit kernel has no default LDT. By default, the LDT descriptor 531 * in the GDT is 0. 532 */ 533 534 /* 535 * Kernel TSS 536 */ 537 set_syssegd((system_desc_t *)&gdt[GDT_KTSS], ktss0, 538 sizeof (*ktss0) - 1, SDT_SYSTSS, SEL_KPL); 539 540 #endif /* !__xpv */ 541 542 /* 543 * Initialize fs and gs descriptors for 32 bit processes. 544 * Only attributes and limits are initialized, the effective 545 * base address is programmed via fsbase/gsbase. 546 */ 547 set_usegd(&gdt[GDT_LWPFS], SDP_SHORT, NULL, -1, SDT_MEMRWA, 548 SEL_UPL, SDP_PAGES, SDP_OP32); 549 set_usegd(&gdt[GDT_LWPGS], SDP_SHORT, NULL, -1, SDT_MEMRWA, 550 SEL_UPL, SDP_PAGES, SDP_OP32); 551 552 /* 553 * Initialize the descriptors set aside for brand usage. 554 * Only attributes and limits are initialized. 555 */ 556 for (i = GDT_BRANDMIN; i <= GDT_BRANDMAX; i++) 557 set_usegd(&gdt0[i], SDP_SHORT, NULL, -1, SDT_MEMRWA, 558 SEL_UPL, SDP_PAGES, SDP_OP32); 559 560 /* 561 * Initialize convenient zero base user descriptors for clearing 562 * lwp private %fs and %gs descriptors in GDT. See setregs() for 563 * an example. 564 */ 565 set_usegd(&zero_udesc, SDP_LONG, 0, 0, SDT_MEMRWA, SEL_UPL, 566 SDP_BYTES, SDP_OP32); 567 set_usegd(&zero_u32desc, SDP_SHORT, 0, -1, SDT_MEMRWA, SEL_UPL, 568 SDP_PAGES, SDP_OP32); 569 } 570 571 #if defined(__xpv) 572 573 static user_desc_t * 574 init_gdt(void) 575 { 576 uint64_t gdtpa; 577 ulong_t ma[1]; /* XXPV should be a memory_t */ 578 ulong_t addr; 579 580 #if !defined(__lint) 581 /* 582 * Our gdt is never larger than a single page. 583 */ 584 ASSERT((sizeof (*gdt0) * NGDT) <= PAGESIZE); 585 #endif 586 gdt0 = (user_desc_t *)BOP_ALLOC(bootops, (caddr_t)GDT_VA, 587 PAGESIZE, PAGESIZE); 588 bzero(gdt0, PAGESIZE); 589 590 init_gdt_common(gdt0); 591 592 /* 593 * XXX Since we never invoke kmdb until after the kernel takes 594 * over the descriptor tables why not have it use the kernel's 595 * selectors? 596 */ 597 if (boothowto & RB_DEBUG) { 598 set_usegd(&gdt0[GDT_B32DATA], SDP_LONG, NULL, -1, SDT_MEMRWA, 599 SEL_KPL, SDP_PAGES, SDP_OP32); 600 set_usegd(&gdt0[GDT_B64CODE], SDP_LONG, NULL, -1, SDT_MEMERA, 601 SEL_KPL, SDP_PAGES, SDP_OP32); 602 } 603 604 /* 605 * Clear write permission for page containing the gdt and install it. 606 */ 607 gdtpa = pfn_to_pa(va_to_pfn(gdt0)); 608 ma[0] = (ulong_t)(pa_to_ma(gdtpa) >> PAGESHIFT); 609 kbm_read_only((uintptr_t)gdt0, gdtpa); 610 xen_set_gdt(ma, NGDT); 611 612 /* 613 * Reload the segment registers to use the new GDT. 614 * On 64-bit, fixup KCS_SEL to be in ring 3. 615 * See KCS_SEL in segments.h. 616 */ 617 load_segment_registers((KCS_SEL | SEL_KPL), KFS_SEL, KGS_SEL, KDS_SEL); 618 619 /* 620 * setup %gs for kernel 621 */ 622 xen_set_segment_base(SEGBASE_GS_KERNEL, (ulong_t)&cpus[0]); 623 624 /* 625 * XX64 We should never dereference off "other gsbase" or 626 * "fsbase". So, we should arrange to point FSBASE and 627 * KGSBASE somewhere truly awful e.g. point it at the last 628 * valid address below the hole so that any attempts to index 629 * off them cause an exception. 630 * 631 * For now, point it at 8G -- at least it should be unmapped 632 * until some 64-bit processes run. 633 */ 634 addr = 0x200000000ul; 635 xen_set_segment_base(SEGBASE_FS, addr); 636 xen_set_segment_base(SEGBASE_GS_USER, addr); 637 xen_set_segment_base(SEGBASE_GS_USER_SEL, 0); 638 639 return (gdt0); 640 } 641 642 #else /* __xpv */ 643 644 static user_desc_t * 645 init_gdt(void) 646 { 647 desctbr_t r_bgdt, r_gdt; 648 user_desc_t *bgdt; 649 650 #if !defined(__lint) 651 /* 652 * Our gdt is never larger than a single page. 653 */ 654 ASSERT((sizeof (*gdt0) * NGDT) <= PAGESIZE); 655 #endif 656 gdt0 = (user_desc_t *)BOP_ALLOC(bootops, (caddr_t)GDT_VA, 657 PAGESIZE, PAGESIZE); 658 bzero(gdt0, PAGESIZE); 659 660 init_gdt_common(gdt0); 661 662 /* 663 * Copy in from boot's gdt to our gdt. 664 * Entry 0 is the null descriptor by definition. 665 */ 666 rd_gdtr(&r_bgdt); 667 bgdt = (user_desc_t *)r_bgdt.dtr_base; 668 if (bgdt == NULL) 669 panic("null boot gdt"); 670 671 gdt0[GDT_B32DATA] = bgdt[GDT_B32DATA]; 672 gdt0[GDT_B32CODE] = bgdt[GDT_B32CODE]; 673 gdt0[GDT_B16CODE] = bgdt[GDT_B16CODE]; 674 gdt0[GDT_B16DATA] = bgdt[GDT_B16DATA]; 675 gdt0[GDT_B64CODE] = bgdt[GDT_B64CODE]; 676 677 /* 678 * Install our new GDT 679 */ 680 r_gdt.dtr_limit = (sizeof (*gdt0) * NGDT) - 1; 681 r_gdt.dtr_base = (uintptr_t)gdt0; 682 wr_gdtr(&r_gdt); 683 684 /* 685 * Reload the segment registers to use the new GDT 686 */ 687 load_segment_registers(KCS_SEL, KFS_SEL, KGS_SEL, KDS_SEL); 688 689 /* 690 * setup %gs for kernel 691 */ 692 wrmsr(MSR_AMD_GSBASE, (uint64_t)&cpus[0]); 693 694 /* 695 * XX64 We should never dereference off "other gsbase" or 696 * "fsbase". So, we should arrange to point FSBASE and 697 * KGSBASE somewhere truly awful e.g. point it at the last 698 * valid address below the hole so that any attempts to index 699 * off them cause an exception. 700 * 701 * For now, point it at 8G -- at least it should be unmapped 702 * until some 64-bit processes run. 703 */ 704 wrmsr(MSR_AMD_FSBASE, 0x200000000ul); 705 wrmsr(MSR_AMD_KGSBASE, 0x200000000ul); 706 return (gdt0); 707 } 708 709 #endif /* __xpv */ 710 711 #elif defined(__i386) 712 713 static void 714 init_gdt_common(user_desc_t *gdt) 715 { 716 int i; 717 718 /* 719 * Text and data for both kernel and user span entire 32 bit 720 * address space. 721 */ 722 723 /* 724 * kernel code segment. 725 */ 726 set_usegd(&gdt[GDT_KCODE], NULL, -1, SDT_MEMERA, SEL_KPL, SDP_PAGES, 727 SDP_OP32); 728 729 /* 730 * kernel data segment. 731 */ 732 set_usegd(&gdt[GDT_KDATA], NULL, -1, SDT_MEMRWA, SEL_KPL, SDP_PAGES, 733 SDP_OP32); 734 735 /* 736 * user code segment. 737 */ 738 set_usegd(&gdt[GDT_UCODE], NULL, -1, SDT_MEMERA, SEL_UPL, SDP_PAGES, 739 SDP_OP32); 740 741 /* 742 * user data segment. 743 */ 744 set_usegd(&gdt[GDT_UDATA], NULL, -1, SDT_MEMRWA, SEL_UPL, SDP_PAGES, 745 SDP_OP32); 746 747 #if !defined(__xpv) 748 749 /* 750 * TSS for T_DBLFLT (double fault) handler 751 */ 752 set_syssegd((system_desc_t *)&gdt[GDT_DBFLT], dftss0, 753 sizeof (*dftss0) - 1, SDT_SYSTSS, SEL_KPL); 754 755 /* 756 * TSS for kernel 757 */ 758 set_syssegd((system_desc_t *)&gdt[GDT_KTSS], ktss0, 759 sizeof (*ktss0) - 1, SDT_SYSTSS, SEL_KPL); 760 761 #endif /* !__xpv */ 762 763 /* 764 * %gs selector for kernel 765 */ 766 set_usegd(&gdt[GDT_GS], &cpus[0], sizeof (struct cpu) -1, SDT_MEMRWA, 767 SEL_KPL, SDP_BYTES, SDP_OP32); 768 769 /* 770 * Initialize lwp private descriptors. 771 * Only attributes and limits are initialized, the effective 772 * base address is programmed via fsbase/gsbase. 773 */ 774 set_usegd(&gdt[GDT_LWPFS], NULL, (size_t)-1, SDT_MEMRWA, SEL_UPL, 775 SDP_PAGES, SDP_OP32); 776 set_usegd(&gdt[GDT_LWPGS], NULL, (size_t)-1, SDT_MEMRWA, SEL_UPL, 777 SDP_PAGES, SDP_OP32); 778 779 /* 780 * Initialize the descriptors set aside for brand usage. 781 * Only attributes and limits are initialized. 782 */ 783 for (i = GDT_BRANDMIN; i <= GDT_BRANDMAX; i++) 784 set_usegd(&gdt0[i], NULL, (size_t)-1, SDT_MEMRWA, SEL_UPL, 785 SDP_PAGES, SDP_OP32); 786 /* 787 * Initialize convenient zero base user descriptor for clearing 788 * lwp private %fs and %gs descriptors in GDT. See setregs() for 789 * an example. 790 */ 791 set_usegd(&zero_udesc, NULL, -1, SDT_MEMRWA, SEL_UPL, 792 SDP_BYTES, SDP_OP32); 793 } 794 795 #if defined(__xpv) 796 797 static user_desc_t * 798 init_gdt(void) 799 { 800 uint64_t gdtpa; 801 ulong_t ma[1]; /* XXPV should be a memory_t */ 802 803 #if !defined(__lint) 804 /* 805 * Our gdt is never larger than a single page. 806 */ 807 ASSERT((sizeof (*gdt0) * NGDT) <= PAGESIZE); 808 #endif 809 gdt0 = (user_desc_t *)BOP_ALLOC(bootops, (caddr_t)GDT_VA, 810 PAGESIZE, PAGESIZE); 811 bzero(gdt0, PAGESIZE); 812 813 init_gdt_common(gdt0); 814 gdtpa = pfn_to_pa(va_to_pfn(gdt0)); 815 816 /* 817 * XXX Since we never invoke kmdb until after the kernel takes 818 * over the descriptor tables why not have it use the kernel's 819 * selectors? 820 */ 821 if (boothowto & RB_DEBUG) { 822 set_usegd(&gdt0[GDT_B32DATA], NULL, -1, SDT_MEMRWA, SEL_KPL, 823 SDP_PAGES, SDP_OP32); 824 set_usegd(&gdt0[GDT_B32CODE], NULL, -1, SDT_MEMERA, SEL_KPL, 825 SDP_PAGES, SDP_OP32); 826 } 827 828 /* 829 * Clear write permission for page containing the gdt and install it. 830 */ 831 ma[0] = (ulong_t)(pa_to_ma(gdtpa) >> PAGESHIFT); 832 kbm_read_only((uintptr_t)gdt0, gdtpa); 833 xen_set_gdt(ma, NGDT); 834 835 /* 836 * Reload the segment registers to use the new GDT 837 */ 838 load_segment_registers( 839 KCS_SEL, KDS_SEL, KDS_SEL, KFS_SEL, KGS_SEL, KDS_SEL); 840 841 return (gdt0); 842 } 843 844 #else /* __xpv */ 845 846 static user_desc_t * 847 init_gdt(void) 848 { 849 desctbr_t r_bgdt, r_gdt; 850 user_desc_t *bgdt; 851 852 #if !defined(__lint) 853 /* 854 * Our gdt is never larger than a single page. 855 */ 856 ASSERT((sizeof (*gdt0) * NGDT) <= PAGESIZE); 857 #endif 858 /* 859 * XXX this allocation belongs in our caller, not here. 860 */ 861 gdt0 = (user_desc_t *)BOP_ALLOC(bootops, (caddr_t)GDT_VA, 862 PAGESIZE, PAGESIZE); 863 bzero(gdt0, PAGESIZE); 864 865 init_gdt_common(gdt0); 866 867 /* 868 * Copy in from boot's gdt to our gdt entries. 869 * Entry 0 is null descriptor by definition. 870 */ 871 rd_gdtr(&r_bgdt); 872 bgdt = (user_desc_t *)r_bgdt.dtr_base; 873 if (bgdt == NULL) 874 panic("null boot gdt"); 875 876 gdt0[GDT_B32DATA] = bgdt[GDT_B32DATA]; 877 gdt0[GDT_B32CODE] = bgdt[GDT_B32CODE]; 878 gdt0[GDT_B16CODE] = bgdt[GDT_B16CODE]; 879 gdt0[GDT_B16DATA] = bgdt[GDT_B16DATA]; 880 881 /* 882 * Install our new GDT 883 */ 884 r_gdt.dtr_limit = (sizeof (*gdt0) * NGDT) - 1; 885 r_gdt.dtr_base = (uintptr_t)gdt0; 886 wr_gdtr(&r_gdt); 887 888 /* 889 * Reload the segment registers to use the new GDT 890 */ 891 load_segment_registers( 892 KCS_SEL, KDS_SEL, KDS_SEL, KFS_SEL, KGS_SEL, KDS_SEL); 893 894 return (gdt0); 895 } 896 897 #endif /* __xpv */ 898 #endif /* __i386 */ 899 900 /* 901 * Build kernel IDT. 902 * 903 * Note that for amd64 we pretty much require every gate to be an interrupt 904 * gate which blocks interrupts atomically on entry; that's because of our 905 * dependency on using 'swapgs' every time we come into the kernel to find 906 * the cpu structure. If we get interrupted just before doing that, %cs could 907 * be in kernel mode (so that the trap prolog doesn't do a swapgs), but 908 * %gsbase is really still pointing at something in userland. Bad things will 909 * ensue. We also use interrupt gates for i386 as well even though this is not 910 * required for some traps. 911 * 912 * Perhaps they should have invented a trap gate that does an atomic swapgs? 913 */ 914 static void 915 init_idt_common(gate_desc_t *idt) 916 { 917 set_gatesegd(&idt[T_ZERODIV], &div0trap, KCS_SEL, SDT_SYSIGT, TRP_KPL); 918 set_gatesegd(&idt[T_SGLSTP], &dbgtrap, KCS_SEL, SDT_SYSIGT, TRP_KPL); 919 set_gatesegd(&idt[T_NMIFLT], &nmiint, KCS_SEL, SDT_SYSIGT, TRP_KPL); 920 set_gatesegd(&idt[T_BPTFLT], &brktrap, KCS_SEL, SDT_SYSIGT, TRP_UPL); 921 set_gatesegd(&idt[T_OVFLW], &ovflotrap, KCS_SEL, SDT_SYSIGT, TRP_UPL); 922 set_gatesegd(&idt[T_BOUNDFLT], &boundstrap, KCS_SEL, SDT_SYSIGT, 923 TRP_KPL); 924 set_gatesegd(&idt[T_ILLINST], &invoptrap, KCS_SEL, SDT_SYSIGT, TRP_KPL); 925 set_gatesegd(&idt[T_NOEXTFLT], &ndptrap, KCS_SEL, SDT_SYSIGT, TRP_KPL); 926 927 /* 928 * double fault handler. 929 * 930 * Note that on the hypervisor a guest does not receive #df faults. 931 * Instead a failsafe event is injected into the guest if its selectors 932 * and/or stack is in a broken state. See xen_failsafe_callback. 933 */ 934 #if !defined(__xpv) 935 #if defined(__amd64) 936 937 set_gatesegd(&idt[T_DBLFLT], &syserrtrap, KCS_SEL, SDT_SYSIGT, TRP_KPL); 938 939 #elif defined(__i386) 940 941 /* 942 * task gate required. 943 */ 944 set_gatesegd(&idt[T_DBLFLT], NULL, DFTSS_SEL, SDT_SYSTASKGT, TRP_KPL); 945 946 #endif /* __i386 */ 947 #endif /* !__xpv */ 948 949 /* 950 * T_EXTOVRFLT coprocessor-segment-overrun not supported. 951 */ 952 953 set_gatesegd(&idt[T_TSSFLT], &invtsstrap, KCS_SEL, SDT_SYSIGT, TRP_KPL); 954 set_gatesegd(&idt[T_SEGFLT], &segnptrap, KCS_SEL, SDT_SYSIGT, TRP_KPL); 955 set_gatesegd(&idt[T_STKFLT], &stktrap, KCS_SEL, SDT_SYSIGT, TRP_KPL); 956 set_gatesegd(&idt[T_GPFLT], &gptrap, KCS_SEL, SDT_SYSIGT, TRP_KPL); 957 set_gatesegd(&idt[T_PGFLT], &pftrap, KCS_SEL, SDT_SYSIGT, TRP_KPL); 958 set_gatesegd(&idt[T_EXTERRFLT], &ndperr, KCS_SEL, SDT_SYSIGT, TRP_KPL); 959 set_gatesegd(&idt[T_ALIGNMENT], &achktrap, KCS_SEL, SDT_SYSIGT, 960 TRP_KPL); 961 set_gatesegd(&idt[T_MCE], &mcetrap, KCS_SEL, SDT_SYSIGT, TRP_KPL); 962 set_gatesegd(&idt[T_SIMDFPE], &xmtrap, KCS_SEL, SDT_SYSIGT, TRP_KPL); 963 964 /* 965 * install "int80" handler at, well, 0x80. 966 */ 967 set_gatesegd(&idt0[T_INT80], &sys_int80, KCS_SEL, SDT_SYSIGT, TRP_UPL); 968 969 /* 970 * install fast trap handler at 210. 971 */ 972 set_gatesegd(&idt[T_FASTTRAP], &fasttrap, KCS_SEL, SDT_SYSIGT, TRP_UPL); 973 974 /* 975 * System call handler. 976 */ 977 #if defined(__amd64) 978 set_gatesegd(&idt[T_SYSCALLINT], &sys_syscall_int, KCS_SEL, SDT_SYSIGT, 979 TRP_UPL); 980 981 #elif defined(__i386) 982 set_gatesegd(&idt[T_SYSCALLINT], &sys_call, KCS_SEL, SDT_SYSIGT, 983 TRP_UPL); 984 #endif /* __i386 */ 985 986 /* 987 * Install the DTrace interrupt handler for the pid provider. 988 */ 989 set_gatesegd(&idt[T_DTRACE_RET], &dtrace_ret, KCS_SEL, 990 SDT_SYSIGT, TRP_UPL); 991 992 /* 993 * Prepare interposing descriptors for the branded "int80" 994 * and syscall handlers and cache copies of the default 995 * descriptors. 996 */ 997 brand_tbl[0].ih_inum = T_INT80; 998 brand_tbl[0].ih_default_desc = idt0[T_INT80]; 999 set_gatesegd(&(brand_tbl[0].ih_interp_desc), &brand_sys_int80, KCS_SEL, 1000 SDT_SYSIGT, TRP_UPL); 1001 1002 brand_tbl[1].ih_inum = T_SYSCALLINT; 1003 brand_tbl[1].ih_default_desc = idt0[T_SYSCALLINT]; 1004 1005 #if defined(__amd64) 1006 set_gatesegd(&(brand_tbl[1].ih_interp_desc), &brand_sys_syscall_int, 1007 KCS_SEL, SDT_SYSIGT, TRP_UPL); 1008 #elif defined(__i386) 1009 set_gatesegd(&(brand_tbl[1].ih_interp_desc), &brand_sys_call, 1010 KCS_SEL, SDT_SYSIGT, TRP_UPL); 1011 #endif /* __i386 */ 1012 1013 brand_tbl[2].ih_inum = 0; 1014 } 1015 1016 #if defined(__xpv) 1017 1018 static void 1019 init_idt(gate_desc_t *idt) 1020 { 1021 /* 1022 * currently nothing extra for the hypervisor 1023 */ 1024 init_idt_common(idt); 1025 } 1026 1027 #else /* __xpv */ 1028 1029 static void 1030 init_idt(gate_desc_t *idt) 1031 { 1032 char ivctname[80]; 1033 void (*ivctptr)(void); 1034 int i; 1035 1036 /* 1037 * Initialize entire table with 'reserved' trap and then overwrite 1038 * specific entries. T_EXTOVRFLT (9) is unsupported and reserved 1039 * since it can only be generated on a 386 processor. 15 is also 1040 * unsupported and reserved. 1041 */ 1042 for (i = 0; i < NIDT; i++) 1043 set_gatesegd(&idt[i], &resvtrap, KCS_SEL, SDT_SYSIGT, TRP_KPL); 1044 1045 /* 1046 * 20-31 reserved 1047 */ 1048 for (i = 20; i < 32; i++) 1049 set_gatesegd(&idt[i], &invaltrap, KCS_SEL, SDT_SYSIGT, TRP_KPL); 1050 1051 /* 1052 * interrupts 32 - 255 1053 */ 1054 for (i = 32; i < 256; i++) { 1055 (void) snprintf(ivctname, sizeof (ivctname), "ivct%d", i); 1056 ivctptr = (void (*)(void))kobj_getsymvalue(ivctname, 0); 1057 if (ivctptr == NULL) 1058 panic("kobj_getsymvalue(%s) failed", ivctname); 1059 1060 set_gatesegd(&idt[i], ivctptr, KCS_SEL, SDT_SYSIGT, TRP_KPL); 1061 } 1062 1063 /* 1064 * Now install the common ones. Note that it will overlay some 1065 * entries installed above like T_SYSCALLINT, T_FASTTRAP etc. 1066 */ 1067 init_idt_common(idt); 1068 } 1069 1070 #endif /* __xpv */ 1071 1072 /* 1073 * The kernel does not deal with LDTs unless a user explicitly creates 1074 * one. Under normal circumstances, the LDTR contains 0. Any process attempting 1075 * to reference the LDT will therefore cause a #gp. System calls made via the 1076 * obsolete lcall mechanism are emulated by the #gp fault handler. 1077 */ 1078 static void 1079 init_ldt(void) 1080 { 1081 #if defined(__xpv) 1082 xen_set_ldt(NULL, 0); 1083 #else 1084 wr_ldtr(0); 1085 #endif 1086 } 1087 1088 #if !defined(__xpv) 1089 #if defined(__amd64) 1090 1091 static void 1092 init_tss(void) 1093 { 1094 /* 1095 * tss_rsp0 is dynamically filled in by resume() on each context switch. 1096 * All exceptions but #DF will run on the thread stack. 1097 * Set up the double fault stack here. 1098 */ 1099 ktss0->tss_ist1 = 1100 (uint64_t)&dblfault_stack0[sizeof (dblfault_stack0)]; 1101 1102 /* 1103 * Set I/O bit map offset equal to size of TSS segment limit 1104 * for no I/O permission map. This will force all user I/O 1105 * instructions to generate #gp fault. 1106 */ 1107 ktss0->tss_bitmapbase = sizeof (*ktss0); 1108 1109 /* 1110 * Point %tr to descriptor for ktss0 in gdt. 1111 */ 1112 wr_tsr(KTSS_SEL); 1113 } 1114 1115 #elif defined(__i386) 1116 1117 static void 1118 init_tss(void) 1119 { 1120 /* 1121 * ktss0->tss_esp dynamically filled in by resume() on each 1122 * context switch. 1123 */ 1124 ktss0->tss_ss0 = KDS_SEL; 1125 ktss0->tss_eip = (uint32_t)_start; 1126 ktss0->tss_ds = ktss0->tss_es = ktss0->tss_ss = KDS_SEL; 1127 ktss0->tss_cs = KCS_SEL; 1128 ktss0->tss_fs = KFS_SEL; 1129 ktss0->tss_gs = KGS_SEL; 1130 ktss0->tss_ldt = ULDT_SEL; 1131 1132 /* 1133 * Initialize double fault tss. 1134 */ 1135 dftss0->tss_esp0 = (uint32_t)&dblfault_stack0[sizeof (dblfault_stack0)]; 1136 dftss0->tss_ss0 = KDS_SEL; 1137 1138 /* 1139 * tss_cr3 will get initialized in hat_kern_setup() once our page 1140 * tables have been setup. 1141 */ 1142 dftss0->tss_eip = (uint32_t)syserrtrap; 1143 dftss0->tss_esp = (uint32_t)&dblfault_stack0[sizeof (dblfault_stack0)]; 1144 dftss0->tss_cs = KCS_SEL; 1145 dftss0->tss_ds = KDS_SEL; 1146 dftss0->tss_es = KDS_SEL; 1147 dftss0->tss_ss = KDS_SEL; 1148 dftss0->tss_fs = KFS_SEL; 1149 dftss0->tss_gs = KGS_SEL; 1150 1151 /* 1152 * Set I/O bit map offset equal to size of TSS segment limit 1153 * for no I/O permission map. This will force all user I/O 1154 * instructions to generate #gp fault. 1155 */ 1156 ktss0->tss_bitmapbase = sizeof (*ktss0); 1157 1158 /* 1159 * Point %tr to descriptor for ktss0 in gdt. 1160 */ 1161 wr_tsr(KTSS_SEL); 1162 } 1163 1164 #endif /* __i386 */ 1165 #endif /* !__xpv */ 1166 1167 #if defined(__xpv) 1168 1169 void 1170 init_desctbls(void) 1171 { 1172 uint_t vec; 1173 user_desc_t *gdt; 1174 1175 /* 1176 * Setup and install our GDT. 1177 */ 1178 gdt = init_gdt(); 1179 1180 /* 1181 * Store static pa of gdt to speed up pa_to_ma() translations 1182 * on lwp context switches. 1183 */ 1184 ASSERT(IS_P2ALIGNED((uintptr_t)gdt, PAGESIZE)); 1185 CPU->cpu_gdt = gdt; 1186 CPU->cpu_m.mcpu_gdtpa = pfn_to_pa(va_to_pfn(gdt)); 1187 1188 /* 1189 * Setup and install our IDT. 1190 */ 1191 #if !defined(__lint) 1192 ASSERT(NIDT * sizeof (*idt0) <= PAGESIZE); 1193 #endif 1194 idt0 = (gate_desc_t *)BOP_ALLOC(bootops, (caddr_t)IDT_VA, 1195 PAGESIZE, PAGESIZE); 1196 init_idt(idt0); 1197 for (vec = 0; vec < NIDT; vec++) 1198 xen_idt_write(&idt0[vec], vec); 1199 1200 CPU->cpu_idt = idt0; 1201 1202 /* 1203 * set default kernel stack 1204 */ 1205 xen_stack_switch(KDS_SEL, 1206 (ulong_t)&dblfault_stack0[sizeof (dblfault_stack0)]); 1207 1208 xen_init_callbacks(); 1209 1210 init_ldt(); 1211 } 1212 1213 #else /* __xpv */ 1214 1215 void 1216 init_desctbls(void) 1217 { 1218 user_desc_t *gdt; 1219 desctbr_t idtr; 1220 1221 /* 1222 * Allocate IDT and TSS structures on unique pages for better 1223 * performance in virtual machines. 1224 */ 1225 #if !defined(__lint) 1226 ASSERT(NIDT * sizeof (*idt0) <= PAGESIZE); 1227 #endif 1228 idt0 = (gate_desc_t *)BOP_ALLOC(bootops, (caddr_t)IDT_VA, 1229 PAGESIZE, PAGESIZE); 1230 #if !defined(__lint) 1231 ASSERT(sizeof (*ktss0) <= PAGESIZE); 1232 #endif 1233 ktss0 = (struct tss *)BOP_ALLOC(bootops, (caddr_t)KTSS_VA, 1234 PAGESIZE, PAGESIZE); 1235 1236 #if defined(__i386) 1237 #if !defined(__lint) 1238 ASSERT(sizeof (*dftss0) <= PAGESIZE); 1239 #endif 1240 dftss0 = (struct tss *)BOP_ALLOC(bootops, (caddr_t)DFTSS_VA, 1241 PAGESIZE, PAGESIZE); 1242 #endif 1243 1244 /* 1245 * Setup and install our GDT. 1246 */ 1247 gdt = init_gdt(); 1248 ASSERT(IS_P2ALIGNED((uintptr_t)gdt, PAGESIZE)); 1249 CPU->cpu_gdt = gdt; 1250 1251 /* 1252 * Setup and install our IDT. 1253 */ 1254 init_idt(idt0); 1255 1256 idtr.dtr_base = (uintptr_t)idt0; 1257 idtr.dtr_limit = (NIDT * sizeof (*idt0)) - 1; 1258 wr_idtr(&idtr); 1259 CPU->cpu_idt = idt0; 1260 1261 #if defined(__i386) 1262 /* 1263 * We maintain a description of idt0 in convenient IDTR format 1264 * for #pf's on some older pentium processors. See pentium_pftrap(). 1265 */ 1266 idt0_default_r = idtr; 1267 #endif /* __i386 */ 1268 1269 init_tss(); 1270 CPU->cpu_tss = ktss0; 1271 init_ldt(); 1272 } 1273 1274 #endif /* __xpv */ 1275 1276 /* 1277 * In the early kernel, we need to set up a simple GDT to run on. 1278 * 1279 * XXPV Can dboot use this too? See dboot_gdt.s 1280 */ 1281 void 1282 init_boot_gdt(user_desc_t *bgdt) 1283 { 1284 #if defined(__amd64) 1285 set_usegd(&bgdt[GDT_B32DATA], SDP_LONG, NULL, -1, SDT_MEMRWA, SEL_KPL, 1286 SDP_PAGES, SDP_OP32); 1287 set_usegd(&bgdt[GDT_B64CODE], SDP_LONG, NULL, -1, SDT_MEMERA, SEL_KPL, 1288 SDP_PAGES, SDP_OP32); 1289 #elif defined(__i386) 1290 set_usegd(&bgdt[GDT_B32DATA], NULL, -1, SDT_MEMRWA, SEL_KPL, 1291 SDP_PAGES, SDP_OP32); 1292 set_usegd(&bgdt[GDT_B32CODE], NULL, -1, SDT_MEMERA, SEL_KPL, 1293 SDP_PAGES, SDP_OP32); 1294 #endif /* __i386 */ 1295 } 1296 1297 /* 1298 * Enable interpositioning on the system call path by rewriting the 1299 * sys{call|enter} MSRs and the syscall-related entries in the IDT to use 1300 * the branded entry points. 1301 */ 1302 void 1303 brand_interpositioning_enable(void) 1304 { 1305 gate_desc_t *idt = CPU->cpu_idt; 1306 int i; 1307 1308 ASSERT(curthread->t_preempt != 0 || getpil() >= DISP_LEVEL); 1309 1310 for (i = 0; brand_tbl[i].ih_inum; i++) { 1311 idt[brand_tbl[i].ih_inum] = brand_tbl[i].ih_interp_desc; 1312 #if defined(__xpv) 1313 xen_idt_write(&idt[brand_tbl[i].ih_inum], 1314 brand_tbl[i].ih_inum); 1315 #endif 1316 } 1317 1318 #if defined(__amd64) 1319 #if defined(__xpv) 1320 1321 /* 1322 * Currently the hypervisor only supports 64-bit syscalls via 1323 * syscall instruction. The 32-bit syscalls are handled by 1324 * interrupt gate above. 1325 */ 1326 xen_set_callback(brand_sys_syscall, CALLBACKTYPE_syscall, 1327 CALLBACKF_mask_events); 1328 1329 #else 1330 1331 if (x86_feature & X86_ASYSC) { 1332 wrmsr(MSR_AMD_LSTAR, (uintptr_t)brand_sys_syscall); 1333 wrmsr(MSR_AMD_CSTAR, (uintptr_t)brand_sys_syscall32); 1334 } 1335 1336 #endif 1337 #endif /* __amd64 */ 1338 1339 if (x86_feature & X86_SEP) 1340 wrmsr(MSR_INTC_SEP_EIP, (uintptr_t)brand_sys_sysenter); 1341 } 1342 1343 /* 1344 * Disable interpositioning on the system call path by rewriting the 1345 * sys{call|enter} MSRs and the syscall-related entries in the IDT to use 1346 * the standard entry points, which bypass the interpositioning hooks. 1347 */ 1348 void 1349 brand_interpositioning_disable(void) 1350 { 1351 gate_desc_t *idt = CPU->cpu_idt; 1352 int i; 1353 1354 ASSERT(curthread->t_preempt != 0 || getpil() >= DISP_LEVEL); 1355 1356 for (i = 0; brand_tbl[i].ih_inum; i++) { 1357 idt[brand_tbl[i].ih_inum] = brand_tbl[i].ih_default_desc; 1358 #if defined(__xpv) 1359 xen_idt_write(&idt[brand_tbl[i].ih_inum], 1360 brand_tbl[i].ih_inum); 1361 #endif 1362 } 1363 1364 #if defined(__amd64) 1365 #if defined(__xpv) 1366 1367 /* 1368 * See comment above in brand_interpositioning_enable. 1369 */ 1370 xen_set_callback(sys_syscall, CALLBACKTYPE_syscall, 1371 CALLBACKF_mask_events); 1372 1373 #else 1374 1375 if (x86_feature & X86_ASYSC) { 1376 wrmsr(MSR_AMD_LSTAR, (uintptr_t)sys_syscall); 1377 wrmsr(MSR_AMD_CSTAR, (uintptr_t)sys_syscall32); 1378 } 1379 1380 #endif 1381 #endif /* __amd64 */ 1382 1383 if (x86_feature & X86_SEP) 1384 wrmsr(MSR_INTC_SEP_EIP, (uintptr_t)sys_sysenter); 1385 } 1386