1 /*-
2 * SPDX-License-Identifier: BSD-4-Clause
3 *
4 * Copyright (c) 2003 Peter Wemm.
5 * Copyright (c) 1992 Terrence R. Lambert.
6 * Copyright (c) 1982, 1987, 1990 The Regents of the University of California.
7 * All rights reserved.
8 *
9 * This code is derived from software contributed to Berkeley by
10 * William Jolitz.
11 *
12 * Redistribution and use in source and binary forms, with or without
13 * modification, are permitted provided that the following conditions
14 * are met:
15 * 1. Redistributions of source code must retain the above copyright
16 * notice, this list of conditions and the following disclaimer.
17 * 2. Redistributions in binary form must reproduce the above copyright
18 * notice, this list of conditions and the following disclaimer in the
19 * documentation and/or other materials provided with the distribution.
20 * 3. All advertising materials mentioning features or use of this software
21 * must display the following acknowledgement:
22 * This product includes software developed by the University of
23 * California, Berkeley and its contributors.
24 * 4. Neither the name of the University nor the names of its contributors
25 * may be used to endorse or promote products derived from this software
26 * without specific prior written permission.
27 *
28 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
29 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
30 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
31 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
32 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
33 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
34 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
35 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
36 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
37 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
38 * SUCH DAMAGE.
39 */
40
41 #include "opt_atpic.h"
42 #include "opt_cpu.h"
43 #include "opt_ddb.h"
44 #include "opt_inet.h"
45 #include "opt_isa.h"
46 #include "opt_kstack_pages.h"
47 #include "opt_maxmem.h"
48 #include "opt_pci.h"
49 #include "opt_platform.h"
50 #include "opt_sched.h"
51
52 #include <sys/param.h>
53 #include <sys/proc.h>
54 #include <sys/systm.h>
55 #include <sys/asan.h>
56 #include <sys/bio.h>
57 #include <sys/buf.h>
58 #include <sys/bus.h>
59 #include <sys/callout.h>
60 #include <sys/cons.h>
61 #include <sys/cpu.h>
62 #include <sys/csan.h>
63 #include <sys/efi.h>
64 #include <sys/eventhandler.h>
65 #include <sys/exec.h>
66 #include <sys/imgact.h>
67 #include <sys/kdb.h>
68 #include <sys/kernel.h>
69 #include <sys/ktr.h>
70 #include <sys/linker.h>
71 #include <sys/lock.h>
72 #include <sys/malloc.h>
73 #include <sys/memrange.h>
74 #include <sys/msan.h>
75 #include <sys/msgbuf.h>
76 #include <sys/mutex.h>
77 #include <sys/pcpu.h>
78 #include <sys/ptrace.h>
79 #include <sys/reboot.h>
80 #include <sys/reg.h>
81 #include <sys/rwlock.h>
82 #include <sys/sched.h>
83 #include <sys/signalvar.h>
84 #include <sys/smp.h>
85 #include <sys/syscallsubr.h>
86 #include <sys/sysctl.h>
87 #include <sys/sysent.h>
88 #include <sys/sysproto.h>
89 #include <sys/ucontext.h>
90 #include <sys/vmmeter.h>
91
92 #include <vm/vm.h>
93 #include <vm/vm_param.h>
94 #include <vm/vm_extern.h>
95 #include <vm/vm_kern.h>
96 #include <vm/vm_page.h>
97 #include <vm/vm_map.h>
98 #include <vm/vm_object.h>
99 #include <vm/vm_pager.h>
100 #include <vm/vm_phys.h>
101 #include <vm/vm_dumpset.h>
102
103 #ifdef DDB
104 #ifndef KDB
105 #error KDB must be enabled in order for DDB to work!
106 #endif
107 #include <ddb/ddb.h>
108 #include <ddb/db_sym.h>
109 #endif
110
111 #include <net/netisr.h>
112
113 #include <dev/smbios/smbios.h>
114
115 #include <machine/clock.h>
116 #include <machine/cpu.h>
117 #include <machine/cputypes.h>
118 #include <machine/frame.h>
119 #include <machine/intr_machdep.h>
120 #include <x86/mca.h>
121 #include <machine/md_var.h>
122 #include <machine/metadata.h>
123 #include <machine/pc/bios.h>
124 #include <machine/pcb.h>
125 #include <machine/proc.h>
126 #include <machine/sigframe.h>
127 #include <machine/specialreg.h>
128 #include <machine/trap.h>
129 #include <machine/tss.h>
130 #include <x86/ucode.h>
131 #include <x86/ifunc.h>
132 #include <machine/smp.h>
133 #ifdef FDT
134 #include <x86/fdt.h>
135 #endif
136
137 #ifdef DEV_ATPIC
138 #include <x86/isa/icu.h>
139 #else
140 #include <x86/apicvar.h>
141 #endif
142
143 #include <isa/isareg.h>
144 #include <isa/rtc.h>
145 #include <x86/init.h>
146
147 #ifndef SMP
148 #error amd64 requires options SMP
149 #endif
150
151 /* Sanity check for __curthread() */
152 CTASSERT(offsetof(struct pcpu, pc_curthread) == 0);
153
154 /*
155 * The PTI trampoline stack needs enough space for a hardware trapframe and a
156 * couple of scratch registers, as well as the trapframe left behind after an
157 * iret fault.
158 */
159 CTASSERT(PC_PTI_STACK_SZ * sizeof(register_t) >= 2 * sizeof(struct pti_frame) -
160 offsetof(struct pti_frame, pti_rip));
161
162 extern u_int64_t hammer_time(u_int64_t, u_int64_t);
163
164 static void cpu_startup(void *);
165 SYSINIT(cpu, SI_SUB_CPU, SI_ORDER_FIRST, cpu_startup, NULL);
166
167 /* Probe 8254 PIT and TSC. */
168 static void native_clock_source_init(void);
169
170 /* Preload data parse function */
171 static void native_parse_preload_data(u_int64_t);
172
173 /* Native function to fetch and parse the e820 map */
174 static void native_parse_memmap(vm_paddr_t *, int *);
175
176 /* Default init_ops implementation. */
177 struct init_ops init_ops = {
178 .parse_preload_data = native_parse_preload_data,
179 .early_clock_source_init = native_clock_source_init,
180 .early_delay = i8254_delay,
181 .parse_memmap = native_parse_memmap,
182 };
183
184 /*
185 * Physical address of the EFI System Table. Stashed from the metadata hints
186 * passed into the kernel and used by the EFI code to call runtime services.
187 */
188 vm_paddr_t efi_systbl_phys;
189
190 /*
191 * Bitmap of extra EFI memory region types that should be preserved and mapped
192 * during runtime services calls.
193 */
194 uint32_t efi_map_regs;
195
196 /* Intel ICH registers */
197 #define ICH_PMBASE 0x400
198 #define ICH_SMI_EN ICH_PMBASE + 0x30
199
200 int _udatasel, _ucodesel, _ucode32sel, _ufssel, _ugssel;
201
202 int cold = 1;
203
204 long Maxmem = 0;
205 long realmem = 0;
206 int late_console = 1;
207 int lass_enabled = 0;
208
209 int ia32_splitlock = 0;
210 SYSCTL_INT(_hw, OID_AUTO, splitlock, CTLFLAG_RD,
211 &ia32_splitlock, 0,
212 "splitlock prevention supported");
213 int ia32_splitlock_force = 1;
214 SYSCTL_INT(_hw, OID_AUTO, splitlock_force, CTLFLAG_RDTUN | CTLFLAG_NOFETCH,
215 &ia32_splitlock_force, 0,
216 "splitlock prevention enabled by default");
217
218 int __read_frequently fred = 0;
219 SYSCTL_INT(_hw, OID_AUTO, fred, CTLFLAG_RDTUN | CTLFLAG_NOFETCH,
220 &fred, 0,
221 "FRED is used");
222
223 struct kva_md_info kmi;
224
225 struct region_descriptor r_idt;
226 struct pcpu *__pcpu;
227 struct pcpu temp_bsp_pcpu;
228
229 struct mtx icu_lock;
230
231 struct mem_range_softc mem_range_softc;
232
233 struct mtx dt_lock; /* lock for GDT and LDT */
234
235 void (*vmm_suspend_p)(void);
236 void (*vmm_resume_p)(void);
237
238 bool efi_boot;
239
240 static void
cpu_startup(void * dummy)241 cpu_startup(void *dummy)
242 {
243 uintmax_t memsize;
244 char *sysenv;
245
246 /*
247 * On MacBooks, we need to disallow the legacy USB circuit to
248 * generate an SMI# because this can cause several problems,
249 * namely: incorrect CPU frequency detection and failure to
250 * start the APs.
251 * We do this by disabling a bit in the SMI_EN (SMI Control and
252 * Enable register) of the Intel ICH LPC Interface Bridge.
253 */
254 sysenv = kern_getenv("smbios.system.product");
255 if (sysenv != NULL) {
256 if (strncmp(sysenv, "MacBook1,1", 10) == 0 ||
257 strncmp(sysenv, "MacBook3,1", 10) == 0 ||
258 strncmp(sysenv, "MacBook4,1", 10) == 0 ||
259 strncmp(sysenv, "MacBookPro1,1", 13) == 0 ||
260 strncmp(sysenv, "MacBookPro1,2", 13) == 0 ||
261 strncmp(sysenv, "MacBookPro3,1", 13) == 0 ||
262 strncmp(sysenv, "MacBookPro4,1", 13) == 0 ||
263 strncmp(sysenv, "Macmini1,1", 10) == 0) {
264 if (bootverbose)
265 printf("Disabling LEGACY_USB_EN bit on "
266 "Intel ICH.\n");
267 outl(ICH_SMI_EN, inl(ICH_SMI_EN) & ~0x8);
268 }
269 freeenv(sysenv);
270 }
271
272 /*
273 * Good {morning,afternoon,evening,night}.
274 */
275 startrtclock();
276 printcpuinfo();
277
278 /*
279 * Display physical memory if SMBIOS reports reasonable amount.
280 */
281 memsize = 0;
282 sysenv = kern_getenv("smbios.memory.enabled");
283 if (sysenv != NULL) {
284 memsize = (uintmax_t)strtoul(sysenv, (char **)NULL, 10) << 10;
285 freeenv(sysenv);
286 }
287 if (memsize < ptoa((uintmax_t)vm_free_count()))
288 memsize = ptoa((uintmax_t)Maxmem);
289 printf("real memory = %ju (%ju MB)\n", memsize, memsize >> 20);
290 realmem = atop(memsize);
291
292 /*
293 * Display any holes after the first chunk of extended memory.
294 */
295 if (bootverbose) {
296 int indx;
297
298 printf("Physical memory chunk(s):\n");
299 for (indx = 0; phys_avail[indx + 1] != 0; indx += 2) {
300 vm_paddr_t size;
301
302 size = phys_avail[indx + 1] - phys_avail[indx];
303 printf(
304 "0x%016jx - 0x%016jx, %ju bytes (%ju pages)\n",
305 (uintmax_t)phys_avail[indx],
306 (uintmax_t)phys_avail[indx + 1] - 1,
307 (uintmax_t)size, (uintmax_t)size / PAGE_SIZE);
308 }
309 }
310
311 vm_ksubmap_init(&kmi);
312
313 printf("avail memory = %ju (%ju MB)\n",
314 ptoa((uintmax_t)vm_free_count()),
315 ptoa((uintmax_t)vm_free_count()) / 1048576);
316 #ifdef DEV_PCI
317 if (bootverbose && intel_graphics_stolen_base != 0)
318 printf("intel stolen mem: base %#jx size %ju MB\n",
319 (uintmax_t)intel_graphics_stolen_base,
320 (uintmax_t)intel_graphics_stolen_size / 1024 / 1024);
321 #endif
322
323 /*
324 * Set up buffers, so they can be used to read disk labels.
325 */
326 bufinit();
327 vm_pager_bufferinit();
328
329 cpu_setregs();
330 }
331
332 static void
late_ifunc_resolve(void * dummy __unused)333 late_ifunc_resolve(void *dummy __unused)
334 {
335 link_elf_late_ireloc();
336 }
337 SYSINIT(late_ifunc_resolve, SI_SUB_CPU, SI_ORDER_ANY, late_ifunc_resolve, NULL);
338
339 void
cpu_setregs(void)340 cpu_setregs(void)
341 {
342 register_t cr0;
343
344 TSENTER();
345 cr0 = rcr0();
346 cr0 |= CR0_MP | CR0_NE | CR0_TS | CR0_WP | CR0_AM;
347 TSENTER2("load_cr0");
348 load_cr0(cr0);
349 TSEXIT2("load_cr0");
350 TSEXIT();
351 }
352
353 /*
354 * Initialize amd64 and configure to run kernel
355 */
356
357 /*
358 * Initialize segments & interrupt table
359 */
360 static struct gate_descriptor idt0[NIDT];
361 struct gate_descriptor *idt = &idt0[0]; /* interrupt descriptor table */
362
363 static char dblfault_stack[DBLFAULT_STACK_SIZE] __aligned(64);
364 static char mce0_stack[MCE_STACK_SIZE] __aligned(16);
365 static char nmi0_stack[NMI_STACK_SIZE] __aligned(64);
366 static char dbg0_stack[DBG_STACK_SIZE] __aligned(16);
367 CTASSERT(sizeof(struct nmi_pcpu) == 16);
368
369 /*
370 * Software prototypes -- in more palatable form.
371 *
372 * Keep GUFS32, GUGS32, GUCODE32 and GUDATA at the same
373 * slots as corresponding segments for i386 kernel.
374 */
375 struct soft_segment_descriptor gdt_segs[] = {
376 [GNULL_SEL] = { /* 0 Null Descriptor */
377 .ssd_base = 0x0,
378 .ssd_limit = 0x0,
379 .ssd_type = 0,
380 .ssd_dpl = 0,
381 .ssd_p = 0,
382 .ssd_long = 0,
383 .ssd_def32 = 0,
384 .ssd_gran = 0 },
385 [GNULL2_SEL] = { /* 1 Null Descriptor */
386 .ssd_base = 0x0,
387 .ssd_limit = 0x0,
388 .ssd_type = 0,
389 .ssd_dpl = 0,
390 .ssd_p = 0,
391 .ssd_long = 0,
392 .ssd_def32 = 0,
393 .ssd_gran = 0 },
394 [GUFS32_SEL] = { /* 2 32 bit %gs Descriptor for user */
395 .ssd_base = 0x0,
396 .ssd_limit = 0xfffff,
397 .ssd_type = SDT_MEMRWA,
398 .ssd_dpl = SEL_UPL,
399 .ssd_p = 1,
400 .ssd_long = 0,
401 .ssd_def32 = 1,
402 .ssd_gran = 1 },
403 [GUGS32_SEL] = { /* 3 32 bit %fs Descriptor for user */
404 .ssd_base = 0x0,
405 .ssd_limit = 0xfffff,
406 .ssd_type = SDT_MEMRWA,
407 .ssd_dpl = SEL_UPL,
408 .ssd_p = 1,
409 .ssd_long = 0,
410 .ssd_def32 = 1,
411 .ssd_gran = 1 },
412 [GCODE_SEL] = { /* 4 Code Descriptor for kernel */
413 .ssd_base = 0x0,
414 .ssd_limit = 0xfffff,
415 .ssd_type = SDT_MEMERA,
416 .ssd_dpl = SEL_KPL,
417 .ssd_p = 1,
418 .ssd_long = 1,
419 .ssd_def32 = 0,
420 .ssd_gran = 1 },
421 [GDATA_SEL] = { /* 5 Data Descriptor for kernel */
422 .ssd_base = 0x0,
423 .ssd_limit = 0xfffff,
424 .ssd_type = SDT_MEMRWA,
425 .ssd_dpl = SEL_KPL,
426 .ssd_p = 1,
427 .ssd_long = 1,
428 .ssd_def32 = 0,
429 .ssd_gran = 1 },
430 [GUCODE32_SEL] = { /* 6 32 bit Code Descriptor for user */
431 .ssd_base = 0x0,
432 .ssd_limit = 0xfffff,
433 .ssd_type = SDT_MEMERA,
434 .ssd_dpl = SEL_UPL,
435 .ssd_p = 1,
436 .ssd_long = 0,
437 .ssd_def32 = 1,
438 .ssd_gran = 1 },
439 [GUDATA_SEL] = { /* 7 32/64 bit Data Descriptor for user */
440 .ssd_base = 0x0,
441 .ssd_limit = 0xfffff,
442 .ssd_type = SDT_MEMRWA,
443 .ssd_dpl = SEL_UPL,
444 .ssd_p = 1,
445 .ssd_long = 0,
446 .ssd_def32 = 1,
447 .ssd_gran = 1 },
448 [GUCODE_SEL] = { /* 8 64 bit Code Descriptor for user */
449 .ssd_base = 0x0,
450 .ssd_limit = 0xfffff,
451 .ssd_type = SDT_MEMERA,
452 .ssd_dpl = SEL_UPL,
453 .ssd_p = 1,
454 .ssd_long = 1,
455 .ssd_def32 = 0,
456 .ssd_gran = 1 },
457 [GPROC0_SEL] = { /* 9 Proc 0 TSS Descriptor */
458 .ssd_base = 0x0,
459 .ssd_limit = sizeof(struct amd64tss) + IOPERM_BITMAP_SIZE - 1,
460 .ssd_type = SDT_SYSTSS,
461 .ssd_dpl = SEL_KPL,
462 .ssd_p = 1,
463 .ssd_long = 0,
464 .ssd_def32 = 0,
465 .ssd_gran = 0 },
466 [GPROC0_SEL + 1] = { /* 10 Proc 0 TSS descriptor, double size */
467 .ssd_base = 0x0,
468 .ssd_limit = 0x0,
469 .ssd_type = 0,
470 .ssd_dpl = 0,
471 .ssd_p = 0,
472 .ssd_long = 0,
473 .ssd_def32 = 0,
474 .ssd_gran = 0 },
475 [GUSERLDT_SEL] = { /* 11 LDT Descriptor */
476 .ssd_base = 0x0,
477 .ssd_limit = 0x0,
478 .ssd_type = 0,
479 .ssd_dpl = 0,
480 .ssd_p = 0,
481 .ssd_long = 0,
482 .ssd_def32 = 0,
483 .ssd_gran = 0 },
484 [GUSERLDT_SEL + 1] = { /* 12 LDT Descriptor, double size */
485 .ssd_base = 0x0,
486 .ssd_limit = 0x0,
487 .ssd_type = 0,
488 .ssd_dpl = 0,
489 .ssd_p = 0,
490 .ssd_long = 0,
491 .ssd_def32 = 0,
492 .ssd_gran = 0 },
493 };
494 _Static_assert(nitems(gdt_segs) == NGDT, "Stale NGDT");
495
496 void
setidt(int idx,inthand_t * func,int typ,int dpl,int ist)497 setidt(int idx, inthand_t *func, int typ, int dpl, int ist)
498 {
499 struct gate_descriptor *ip;
500
501 if (fred)
502 return;
503
504 ip = idt + idx;
505 ip->gd_looffset = (uintptr_t)func;
506 ip->gd_selector = GSEL(GCODE_SEL, SEL_KPL);
507 ip->gd_ist = ist;
508 ip->gd_xx = 0;
509 ip->gd_type = typ;
510 ip->gd_dpl = dpl;
511 ip->gd_p = 1;
512 ip->gd_hioffset = ((uintptr_t)func)>>16 ;
513 }
514
515 extern inthand_t
516 IDTVEC(div), IDTVEC(dbg), IDTVEC(nmi), IDTVEC(bpt), IDTVEC(ofl),
517 IDTVEC(bnd), IDTVEC(ill), IDTVEC(dna), IDTVEC(fpusegm),
518 IDTVEC(tss), IDTVEC(missing), IDTVEC(stk), IDTVEC(prot),
519 IDTVEC(page), IDTVEC(mchk), IDTVEC(rsvd), IDTVEC(fpu), IDTVEC(align),
520 IDTVEC(xmm), IDTVEC(dblfault),
521 IDTVEC(div_pti), IDTVEC(bpt_pti),
522 IDTVEC(ofl_pti), IDTVEC(bnd_pti), IDTVEC(ill_pti), IDTVEC(dna_pti),
523 IDTVEC(fpusegm_pti), IDTVEC(tss_pti), IDTVEC(missing_pti),
524 IDTVEC(stk_pti), IDTVEC(prot_pti), IDTVEC(page_pti),
525 IDTVEC(rsvd_pti), IDTVEC(fpu_pti), IDTVEC(align_pti),
526 IDTVEC(xmm_pti),
527 #ifdef KDTRACE_HOOKS
528 IDTVEC(dtrace_ret), IDTVEC(dtrace_ret_pti),
529 #endif
530 #ifdef XENHVM
531 IDTVEC(xen_intr_upcall), IDTVEC(xen_intr_upcall_pti),
532 #endif
533 IDTVEC(fast_syscall), IDTVEC(fast_syscall32),
534 IDTVEC(fast_syscall_pti);
535
536 #ifdef DDB
537 /*
538 * Display the index and function name of any IDT entries that don't use
539 * the default 'rsvd' entry point.
540 */
DB_SHOW_COMMAND_FLAGS(idt,db_show_idt,DB_CMD_MEMSAFE)541 DB_SHOW_COMMAND_FLAGS(idt, db_show_idt, DB_CMD_MEMSAFE)
542 {
543 struct gate_descriptor *ip;
544 int idx;
545 uintptr_t func;
546
547 ip = idt;
548 for (idx = 0; idx < NIDT && !db_pager_quit; idx++) {
549 func = ((long)ip->gd_hioffset << 16 | ip->gd_looffset);
550 if (func != (uintptr_t)&IDTVEC(rsvd)) {
551 db_printf("%3d\t", idx);
552 db_printsym(func, DB_STGY_PROC);
553 db_printf("\n");
554 }
555 ip++;
556 }
557 }
558
559 /* Show privileged registers. */
DB_SHOW_COMMAND_FLAGS(sysregs,db_show_sysregs,DB_CMD_MEMSAFE)560 DB_SHOW_COMMAND_FLAGS(sysregs, db_show_sysregs, DB_CMD_MEMSAFE)
561 {
562 struct {
563 uint16_t limit;
564 uint64_t base;
565 } __packed idtr, gdtr;
566 uint16_t ldt, tr;
567
568 __asm __volatile("sidt %0" : "=m" (idtr));
569 db_printf("idtr\t0x%016lx/%04x\n",
570 (u_long)idtr.base, (u_int)idtr.limit);
571 __asm __volatile("sgdt %0" : "=m" (gdtr));
572 db_printf("gdtr\t0x%016lx/%04x\n",
573 (u_long)gdtr.base, (u_int)gdtr.limit);
574 __asm __volatile("sldt %0" : "=r" (ldt));
575 db_printf("ldtr\t0x%04x\n", ldt);
576 __asm __volatile("str %0" : "=r" (tr));
577 db_printf("tr\t0x%04x\n", tr);
578 db_printf("cr0\t0x%016lx\n", rcr0());
579 db_printf("cr2\t0x%016lx\n", rcr2());
580 db_printf("cr3\t0x%016lx\n", rcr3());
581 db_printf("cr4\t0x%016lx\n", rcr4());
582 if (rcr4() & CR4_XSAVE)
583 db_printf("xcr0\t0x%016lx\n", rxcr(0));
584 db_printf("EFER\t0x%016lx\n", rdmsr(MSR_EFER));
585 if (cpu_feature2 & (CPUID2_VMX | CPUID2_SMX))
586 db_printf("FEATURES_CTL\t%016lx\n",
587 rdmsr(MSR_IA32_FEATURE_CONTROL));
588 db_printf("DEBUG_CTL\t0x%016lx\n", rdmsr(MSR_DEBUGCTLMSR));
589 db_printf("PAT\t0x%016lx\n", rdmsr(MSR_PAT));
590 db_printf("GSBASE\t0x%016lx\n", rdmsr(MSR_GSBASE));
591 }
592
DB_SHOW_COMMAND_FLAGS(dbregs,db_show_dbregs,DB_CMD_MEMSAFE)593 DB_SHOW_COMMAND_FLAGS(dbregs, db_show_dbregs, DB_CMD_MEMSAFE)
594 {
595
596 db_printf("dr0\t0x%016lx\n", rdr0());
597 db_printf("dr1\t0x%016lx\n", rdr1());
598 db_printf("dr2\t0x%016lx\n", rdr2());
599 db_printf("dr3\t0x%016lx\n", rdr3());
600 db_printf("dr6\t0x%016lx\n", rdr6());
601 db_printf("dr7\t0x%016lx\n", rdr7());
602 }
603 #endif
604
605 void
sdtossd(struct user_segment_descriptor * sd,struct soft_segment_descriptor * ssd)606 sdtossd(struct user_segment_descriptor *sd, struct soft_segment_descriptor *ssd)
607 {
608
609 ssd->ssd_base = (sd->sd_hibase << 24) | sd->sd_lobase;
610 ssd->ssd_limit = (sd->sd_hilimit << 16) | sd->sd_lolimit;
611 ssd->ssd_type = sd->sd_type;
612 ssd->ssd_dpl = sd->sd_dpl;
613 ssd->ssd_p = sd->sd_p;
614 ssd->ssd_long = sd->sd_long;
615 ssd->ssd_def32 = sd->sd_def32;
616 ssd->ssd_gran = sd->sd_gran;
617 }
618
619 void
ssdtosd(struct soft_segment_descriptor * ssd,struct user_segment_descriptor * sd)620 ssdtosd(struct soft_segment_descriptor *ssd, struct user_segment_descriptor *sd)
621 {
622
623 sd->sd_lobase = (ssd->ssd_base) & 0xffffff;
624 sd->sd_hibase = (ssd->ssd_base >> 24) & 0xff;
625 sd->sd_lolimit = (ssd->ssd_limit) & 0xffff;
626 sd->sd_hilimit = (ssd->ssd_limit >> 16) & 0xf;
627 sd->sd_type = ssd->ssd_type;
628 sd->sd_dpl = ssd->ssd_dpl;
629 sd->sd_p = ssd->ssd_p;
630 sd->sd_long = ssd->ssd_long;
631 sd->sd_def32 = ssd->ssd_def32;
632 sd->sd_gran = ssd->ssd_gran;
633 }
634
635 void
ssdtosyssd(struct soft_segment_descriptor * ssd,struct system_segment_descriptor * sd)636 ssdtosyssd(struct soft_segment_descriptor *ssd, struct system_segment_descriptor *sd)
637 {
638
639 sd->sd_lobase = (ssd->ssd_base) & 0xffffff;
640 sd->sd_hibase = (ssd->ssd_base >> 24) & 0xfffffffffful;
641 sd->sd_lolimit = (ssd->ssd_limit) & 0xffff;
642 sd->sd_hilimit = (ssd->ssd_limit >> 16) & 0xf;
643 sd->sd_type = ssd->ssd_type;
644 sd->sd_dpl = ssd->ssd_dpl;
645 sd->sd_p = ssd->ssd_p;
646 sd->sd_gran = ssd->ssd_gran;
647 }
648
649 u_int basemem;
650
651 static int
add_physmap_entry(uint64_t base,uint64_t length,vm_paddr_t * physmap,int * physmap_idxp)652 add_physmap_entry(uint64_t base, uint64_t length, vm_paddr_t *physmap,
653 int *physmap_idxp)
654 {
655 int i, insert_idx, physmap_idx;
656
657 physmap_idx = *physmap_idxp;
658
659 if (length == 0)
660 return (1);
661
662 /*
663 * Find insertion point while checking for overlap. Start off by
664 * assuming the new entry will be added to the end.
665 *
666 * NB: physmap_idx points to the next free slot.
667 */
668 insert_idx = physmap_idx;
669 for (i = 0; i < physmap_idx; i += 2) {
670 if (base < physmap[i + 1]) {
671 if (base + length <= physmap[i]) {
672 insert_idx = i;
673 break;
674 }
675 if (boothowto & RB_VERBOSE)
676 printf(
677 "Overlapping memory regions, ignoring second region\n");
678 return (1);
679 }
680 }
681
682 /* See if we can prepend to the next entry. */
683 if (insert_idx < physmap_idx && base + length == physmap[insert_idx]) {
684 physmap[insert_idx] = base;
685 return (1);
686 }
687
688 /* See if we can append to the previous entry. */
689 if (insert_idx > 0 && base == physmap[insert_idx - 1]) {
690 physmap[insert_idx - 1] += length;
691 return (1);
692 }
693
694 if (physmap_idx == PHYS_AVAIL_ENTRIES) {
695 printf(
696 "Too many segments in the physical address map, giving up\n");
697 return (0);
698 }
699
700 /*
701 * Move the last 'N' entries down to make room for the new
702 * entry if needed.
703 */
704 for (i = physmap_idx; i > insert_idx; i -= 2) {
705 physmap[i] = physmap[i - 2];
706 physmap[i + 1] = physmap[i - 1];
707 }
708
709 physmap_idx += 2;
710 *physmap_idxp = physmap_idx;
711
712 /* Insert the new entry. */
713 physmap[insert_idx] = base;
714 physmap[insert_idx + 1] = base + length;
715 return (1);
716 }
717
718 void
bios_add_smap_entries(struct bios_smap * smapbase,u_int32_t smapsize,vm_paddr_t * physmap,int * physmap_idx)719 bios_add_smap_entries(struct bios_smap *smapbase, u_int32_t smapsize,
720 vm_paddr_t *physmap, int *physmap_idx)
721 {
722 struct bios_smap *smap, *smapend;
723
724 smapend = (struct bios_smap *)((uintptr_t)smapbase + smapsize);
725
726 for (smap = smapbase; smap < smapend; smap++) {
727 if (boothowto & RB_VERBOSE)
728 printf("SMAP type=%02x base=%016lx len=%016lx\n",
729 smap->type, smap->base, smap->length);
730
731 if (smap->type != SMAP_TYPE_MEMORY)
732 continue;
733
734 if (!add_physmap_entry(smap->base, smap->length, physmap,
735 physmap_idx))
736 break;
737 }
738 }
739
740 static void
add_efi_map_entries(struct efi_map_header * efihdr,vm_paddr_t * physmap,int * physmap_idx)741 add_efi_map_entries(struct efi_map_header *efihdr, vm_paddr_t *physmap,
742 int *physmap_idx)
743 {
744 struct efi_md *map, *p;
745 const char *type;
746 size_t efisz;
747 int ndesc, i;
748
749 static const char *types[] = {
750 "Reserved",
751 "LoaderCode",
752 "LoaderData",
753 "BootServicesCode",
754 "BootServicesData",
755 "RuntimeServicesCode",
756 "RuntimeServicesData",
757 "ConventionalMemory",
758 "UnusableMemory",
759 "ACPIReclaimMemory",
760 "ACPIMemoryNVS",
761 "MemoryMappedIO",
762 "MemoryMappedIOPortSpace",
763 "PalCode",
764 "PersistentMemory"
765 };
766
767 /*
768 * Memory map data provided by UEFI via the GetMemoryMap
769 * Boot Services API.
770 */
771 efisz = (sizeof(struct efi_map_header) + 0xf) & ~0xf;
772 map = (struct efi_md *)((uint8_t *)efihdr + efisz);
773
774 if (efihdr->descriptor_size == 0)
775 return;
776 ndesc = efihdr->memory_size / efihdr->descriptor_size;
777
778 if (boothowto & RB_VERBOSE)
779 printf("%23s %12s %12s %8s %4s\n",
780 "Type", "Physical", "Virtual", "#Pages", "Attr");
781
782 TUNABLE_INT_FETCH("machdep.efirt.regs", &efi_map_regs);
783 for (i = 0, p = map; i < ndesc; i++,
784 p = efi_next_descriptor(p, efihdr->descriptor_size)) {
785 if (boothowto & RB_VERBOSE) {
786 if (p->md_type < nitems(types))
787 type = types[p->md_type];
788 else
789 type = "<INVALID>";
790 printf("%23s %012lx %012lx %08lx ", type, p->md_phys,
791 p->md_virt, p->md_pages);
792 if (p->md_attr & EFI_MD_ATTR_UC)
793 printf("UC ");
794 if (p->md_attr & EFI_MD_ATTR_WC)
795 printf("WC ");
796 if (p->md_attr & EFI_MD_ATTR_WT)
797 printf("WT ");
798 if (p->md_attr & EFI_MD_ATTR_WB)
799 printf("WB ");
800 if (p->md_attr & EFI_MD_ATTR_UCE)
801 printf("UCE ");
802 if (p->md_attr & EFI_MD_ATTR_WP)
803 printf("WP ");
804 if (p->md_attr & EFI_MD_ATTR_RP)
805 printf("RP ");
806 if (p->md_attr & EFI_MD_ATTR_XP)
807 printf("XP ");
808 if (p->md_attr & EFI_MD_ATTR_NV)
809 printf("NV ");
810 if (p->md_attr & EFI_MD_ATTR_MORE_RELIABLE)
811 printf("MORE_RELIABLE ");
812 if (p->md_attr & EFI_MD_ATTR_RO)
813 printf("RO ");
814 if (p->md_attr & EFI_MD_ATTR_RT)
815 printf("RUNTIME");
816 printf("\n");
817 }
818
819 switch (p->md_type) {
820 case EFI_MD_TYPE_BS_CODE:
821 case EFI_MD_TYPE_BS_DATA:
822 if (EFI_MAP_BOOTTYPE_ALLOWED(p->md_type))
823 continue;
824 /* FALLTHROUGH */
825 case EFI_MD_TYPE_CODE:
826 case EFI_MD_TYPE_DATA:
827 case EFI_MD_TYPE_FREE:
828 /*
829 * We're allowed to use any entry with these types.
830 */
831 break;
832 default:
833 continue;
834 }
835
836 if (!add_physmap_entry(p->md_phys, p->md_pages * EFI_PAGE_SIZE,
837 physmap, physmap_idx))
838 break;
839 }
840 }
841
842 static void
native_parse_memmap(vm_paddr_t * physmap,int * physmap_idx)843 native_parse_memmap(vm_paddr_t *physmap, int *physmap_idx)
844 {
845 struct bios_smap *smap;
846 struct efi_map_header *efihdr;
847
848 efihdr = (struct efi_map_header *)preload_search_info(preload_kmdp,
849 MODINFO_METADATA | MODINFOMD_EFI_MAP);
850 smap = (struct bios_smap *)preload_search_info(preload_kmdp,
851 MODINFO_METADATA | MODINFOMD_SMAP);
852 if (efihdr == NULL && smap == NULL)
853 panic("No BIOS smap or EFI map info from loader!");
854
855 if (efihdr != NULL) {
856 add_efi_map_entries(efihdr, physmap, physmap_idx);
857 strlcpy(bootmethod, "UEFI", sizeof(bootmethod));
858 } else {
859 /*
860 * Memory map from INT 15:E820.
861 *
862 * subr_module.c says:
863 * "Consumer may safely assume that size value precedes data."
864 * ie: an int32_t immediately precedes smap.
865 */
866 u_int32_t size = *((u_int32_t *)smap - 1);
867
868 bios_add_smap_entries(smap, size, physmap, physmap_idx);
869 strlcpy(bootmethod, "BIOS", sizeof(bootmethod));
870 }
871 }
872
873 #define PAGES_PER_GB (1024 * 1024 * 1024 / PAGE_SIZE)
874
875 /*
876 * Populate the (physmap) array with base/bound pairs describing the
877 * available physical memory in the system, then test this memory and
878 * build the phys_avail array describing the actually-available memory.
879 *
880 * Total memory size may be set by the kernel environment variable
881 * hw.physmem or the compile-time define MAXMEM.
882 *
883 * XXX first should be vm_paddr_t.
884 */
885 static void
getmemsize(u_int64_t first)886 getmemsize(u_int64_t first)
887 {
888 int i, physmap_idx, pa_indx, da_indx;
889 vm_paddr_t pa, physmap[PHYS_AVAIL_ENTRIES];
890 u_long physmem_start, physmem_tunable, memtest;
891 pt_entry_t *pte;
892 quad_t dcons_addr, dcons_size;
893 int page_counter;
894
895 TSENTER();
896 /*
897 * Tell the physical memory allocator about pages used to store
898 * the kernel and preloaded data. See kmem_bootstrap_free().
899 */
900 vm_phys_early_add_seg((vm_paddr_t)kernphys, trunc_page(first));
901
902 bzero(physmap, sizeof(physmap));
903 physmap_idx = 0;
904
905 init_ops.parse_memmap(physmap, &physmap_idx);
906 physmap_idx -= 2;
907
908 /*
909 * Find the 'base memory' segment for SMP
910 */
911 basemem = 0;
912 for (i = 0; i <= physmap_idx; i += 2) {
913 if (physmap[i] <= 0xA0000) {
914 basemem = physmap[i + 1] / 1024;
915 break;
916 }
917 }
918 if (basemem == 0 || basemem > 640) {
919 if (bootverbose)
920 printf(
921 "Memory map doesn't contain a basemem segment, faking it");
922 basemem = 640;
923 }
924
925 /*
926 * Maxmem isn't the "maximum memory", it's one larger than the
927 * highest page of the physical address space. It should be
928 * called something like "Maxphyspage". We may adjust this
929 * based on ``hw.physmem'' and the results of the memory test.
930 */
931 Maxmem = atop(physmap[physmap_idx + 1]);
932
933 #ifdef MAXMEM
934 Maxmem = MAXMEM / 4;
935 #endif
936
937 if (TUNABLE_ULONG_FETCH("hw.physmem", &physmem_tunable))
938 Maxmem = atop(physmem_tunable);
939
940 /*
941 * The boot memory test is disabled by default, as it takes a
942 * significant amount of time on large-memory systems, and is
943 * unfriendly to virtual machines as it unnecessarily touches all
944 * pages.
945 *
946 * A general name is used as the code may be extended to support
947 * additional tests beyond the current "page present" test.
948 */
949 memtest = 0;
950 TUNABLE_ULONG_FETCH("hw.memtest.tests", &memtest);
951
952 /*
953 * Don't allow MAXMEM or hw.physmem to extend the amount of memory
954 * in the system.
955 */
956 if (Maxmem > atop(physmap[physmap_idx + 1]))
957 Maxmem = atop(physmap[physmap_idx + 1]);
958
959 if (atop(physmap[physmap_idx + 1]) != Maxmem &&
960 (boothowto & RB_VERBOSE))
961 printf("Physical memory use set to %ldK\n", Maxmem * 4);
962
963 /* call pmap initialization to make new kernel address space */
964 pmap_bootstrap(&first);
965
966 /*
967 * Size up each available chunk of physical memory.
968 *
969 * XXX Some BIOSes corrupt low 64KB between suspend and resume.
970 * By default, mask off the first 16 pages unless we appear to be
971 * running in a VM.
972 */
973 physmem_start = (vm_guest > VM_GUEST_NO ? 1 : 16) << PAGE_SHIFT;
974 TUNABLE_ULONG_FETCH("hw.physmem.start", &physmem_start);
975 if (physmap[0] < physmem_start) {
976 if (physmem_start < PAGE_SIZE)
977 physmap[0] = PAGE_SIZE;
978 else if (physmem_start >= physmap[1])
979 physmap[0] = round_page(physmap[1] - PAGE_SIZE);
980 else
981 physmap[0] = round_page(physmem_start);
982 }
983 pa_indx = 0;
984 da_indx = 1;
985 phys_avail[pa_indx++] = physmap[0];
986 phys_avail[pa_indx] = physmap[0];
987 dump_avail[da_indx] = physmap[0];
988 pte = CMAP1;
989
990 /*
991 * Get dcons buffer address
992 */
993 if (getenv_quad("dcons.addr", &dcons_addr) == 0 ||
994 getenv_quad("dcons.size", &dcons_size) == 0)
995 dcons_addr = 0;
996
997 /*
998 * physmap is in bytes, so when converting to page boundaries,
999 * round up the start address and round down the end address.
1000 */
1001 page_counter = 0;
1002 if (memtest != 0)
1003 printf("Testing system memory");
1004 for (i = 0; i <= physmap_idx; i += 2) {
1005 vm_paddr_t end;
1006
1007 end = ptoa((vm_paddr_t)Maxmem);
1008 if (physmap[i + 1] < end)
1009 end = trunc_page(physmap[i + 1]);
1010 for (pa = round_page(physmap[i]); pa < end; pa += PAGE_SIZE) {
1011 int *ptr = (int *)CADDR1;
1012 int tmp;
1013 bool full, page_bad;
1014
1015 full = false;
1016 /*
1017 * block out kernel memory as not available.
1018 */
1019 if (pa >= (vm_paddr_t)kernphys && pa < first)
1020 goto do_dump_avail;
1021
1022 /*
1023 * block out dcons buffer
1024 */
1025 if (dcons_addr > 0
1026 && pa >= trunc_page(dcons_addr)
1027 && pa < dcons_addr + dcons_size)
1028 goto do_dump_avail;
1029
1030 page_bad = false;
1031 if (memtest == 0)
1032 goto skip_memtest;
1033
1034 /*
1035 * Print a "." every GB to show we're making
1036 * progress.
1037 */
1038 page_counter++;
1039 if ((page_counter % PAGES_PER_GB) == 0)
1040 printf(".");
1041
1042 /*
1043 * map page into kernel: valid, read/write,non-cacheable
1044 */
1045 *pte = pa | PG_V | PG_RW | PG_NC_PWT | PG_NC_PCD;
1046 invltlb();
1047
1048 tmp = *(int *)ptr;
1049 /*
1050 * Test for alternating 1's and 0's
1051 */
1052 *(volatile int *)ptr = 0xaaaaaaaa;
1053 if (*(volatile int *)ptr != 0xaaaaaaaa)
1054 page_bad = true;
1055 /*
1056 * Test for alternating 0's and 1's
1057 */
1058 *(volatile int *)ptr = 0x55555555;
1059 if (*(volatile int *)ptr != 0x55555555)
1060 page_bad = true;
1061 /*
1062 * Test for all 1's
1063 */
1064 *(volatile int *)ptr = 0xffffffff;
1065 if (*(volatile int *)ptr != 0xffffffff)
1066 page_bad = true;
1067 /*
1068 * Test for all 0's
1069 */
1070 *(volatile int *)ptr = 0x0;
1071 if (*(volatile int *)ptr != 0x0)
1072 page_bad = true;
1073 /*
1074 * Restore original value.
1075 */
1076 *(int *)ptr = tmp;
1077
1078 skip_memtest:
1079 /*
1080 * Adjust array of valid/good pages.
1081 */
1082 if (page_bad == true)
1083 continue;
1084 /*
1085 * If this good page is a continuation of the
1086 * previous set of good pages, then just increase
1087 * the end pointer. Otherwise start a new chunk.
1088 * Note that "end" points one higher than end,
1089 * making the range >= start and < end.
1090 * If we're also doing a speculative memory
1091 * test and we at or past the end, bump up Maxmem
1092 * so that we keep going. The first bad page
1093 * will terminate the loop.
1094 */
1095 if (phys_avail[pa_indx] == pa) {
1096 phys_avail[pa_indx] += PAGE_SIZE;
1097 } else {
1098 pa_indx++;
1099 if (pa_indx == PHYS_AVAIL_ENTRIES) {
1100 printf(
1101 "Too many holes in the physical address space, giving up\n");
1102 pa_indx--;
1103 full = true;
1104 goto do_dump_avail;
1105 }
1106 phys_avail[pa_indx++] = pa; /* start */
1107 phys_avail[pa_indx] = pa + PAGE_SIZE; /* end */
1108 }
1109 physmem++;
1110 do_dump_avail:
1111 if (dump_avail[da_indx] == pa) {
1112 dump_avail[da_indx] += PAGE_SIZE;
1113 } else {
1114 da_indx++;
1115 if (da_indx == PHYS_AVAIL_ENTRIES) {
1116 da_indx--;
1117 goto do_next;
1118 }
1119 dump_avail[da_indx++] = pa; /* start */
1120 dump_avail[da_indx] = pa + PAGE_SIZE; /* end */
1121 }
1122 do_next:
1123 if (full)
1124 break;
1125 }
1126 }
1127 *pte = 0;
1128 invltlb();
1129 if (memtest != 0)
1130 printf("\n");
1131
1132 /*
1133 * XXX
1134 * The last chunk must contain at least one page plus the message
1135 * buffer to avoid complicating other code (message buffer address
1136 * calculation, etc.).
1137 */
1138 while (phys_avail[pa_indx - 1] + PAGE_SIZE +
1139 round_page(msgbufsize) >= phys_avail[pa_indx]) {
1140 physmem -= atop(phys_avail[pa_indx] - phys_avail[pa_indx - 1]);
1141 phys_avail[pa_indx--] = 0;
1142 phys_avail[pa_indx--] = 0;
1143 }
1144
1145 Maxmem = atop(phys_avail[pa_indx]);
1146
1147 /* Trim off space for the message buffer. */
1148 phys_avail[pa_indx] -= round_page(msgbufsize);
1149
1150 /* Map the message buffer. */
1151 msgbufp = PHYS_TO_DMAP(phys_avail[pa_indx]);
1152 TSEXIT();
1153 }
1154
1155 static void
native_parse_preload_data(u_int64_t modulep)1156 native_parse_preload_data(u_int64_t modulep)
1157 {
1158 char *envp;
1159 #ifdef DDB
1160 vm_offset_t ksym_start;
1161 vm_offset_t ksym_end;
1162 #endif
1163
1164 preload_metadata = (caddr_t)(uintptr_t)(modulep + KERNBASE);
1165 preload_bootstrap_relocate(KERNBASE);
1166 preload_initkmdp(true);
1167 boothowto = MD_FETCH(preload_kmdp, MODINFOMD_HOWTO, int);
1168 envp = MD_FETCH(preload_kmdp, MODINFOMD_ENVP, char *);
1169 if (envp != NULL)
1170 envp += KERNBASE;
1171 init_static_kenv(envp, 0);
1172 #ifdef DDB
1173 ksym_start = MD_FETCH(preload_kmdp, MODINFOMD_SSYM, uintptr_t);
1174 ksym_end = MD_FETCH(preload_kmdp, MODINFOMD_ESYM, uintptr_t);
1175 db_fetch_ksymtab(ksym_start, ksym_end, 0);
1176 #endif
1177 efi_systbl_phys = MD_FETCH(preload_kmdp, MODINFOMD_FW_HANDLE,
1178 vm_paddr_t);
1179 }
1180
1181 static void
native_clock_source_init(void)1182 native_clock_source_init(void)
1183 {
1184 i8254_init();
1185 }
1186
1187 static void
amd64_kdb_init(void)1188 amd64_kdb_init(void)
1189 {
1190 kdb_init();
1191 #ifdef KDB
1192 if (boothowto & RB_KDB)
1193 kdb_enter(KDB_WHY_BOOTFLAGS, "Boot flags requested debugger");
1194 #endif
1195 }
1196
1197 /* Set up the fast syscall stuff */
1198 void
amd64_conf_fast_syscall(void)1199 amd64_conf_fast_syscall(void)
1200 {
1201 uint64_t msr;
1202
1203 msr = rdmsr(MSR_EFER) | EFER_SCE;
1204 wrmsr(MSR_EFER, msr);
1205 if (!fred) {
1206 wrmsr(MSR_LSTAR, pti ? (u_int64_t)IDTVEC(fast_syscall_pti) :
1207 (u_int64_t)IDTVEC(fast_syscall));
1208 wrmsr(MSR_CSTAR, (u_int64_t)IDTVEC(fast_syscall32));
1209 }
1210 msr = ((u_int64_t)GSEL(GCODE_SEL, SEL_KPL) << 32) |
1211 ((u_int64_t)GSEL(GUCODE32_SEL, SEL_UPL) << 48);
1212 wrmsr(MSR_STAR, msr);
1213 wrmsr(MSR_SF_MASK, PSL_NT | PSL_T | PSL_I | PSL_C | PSL_D | PSL_AC);
1214 }
1215
1216 void
amd64_bsp_pcpu_init1(struct pcpu * pc)1217 amd64_bsp_pcpu_init1(struct pcpu *pc)
1218 {
1219 struct user_segment_descriptor *gdt;
1220
1221 PCPU_SET(prvspace, pc);
1222 gdt = *PCPU_PTR(gdt);
1223 PCPU_SET(curthread, &thread0);
1224 PCPU_SET(tssp, PCPU_PTR(common_tss));
1225 PCPU_SET(tss, (struct system_segment_descriptor *)&gdt[GPROC0_SEL]);
1226 PCPU_SET(ldt, (struct system_segment_descriptor *)&gdt[GUSERLDT_SEL]);
1227 PCPU_SET(fs32p, &gdt[GUFS32_SEL]);
1228 PCPU_SET(gs32p, &gdt[GUGS32_SEL]);
1229 PCPU_SET(ucr3_load_mask, PMAP_UCR3_NOMASK);
1230 PCPU_SET(smp_tlb_gen, 1);
1231 }
1232
1233 void
amd64_bsp_pcpu_init2(uint64_t rsp0)1234 amd64_bsp_pcpu_init2(uint64_t rsp0)
1235 {
1236
1237 PCPU_SET(rsp0, rsp0);
1238 PCPU_SET(pti_rsp0, STACKALIGN((vm_offset_t)PCPU_PTR(pti_stack) +
1239 PC_PTI_STACK_SZ * sizeof(uint64_t)));
1240 PCPU_SET(curpcb, thread0.td_pcb);
1241 }
1242
1243 void
amd64_bsp_ist_init(struct pcpu * pc)1244 amd64_bsp_ist_init(struct pcpu *pc)
1245 {
1246 struct nmi_pcpu *np;
1247 struct amd64tss *tssp;
1248
1249 tssp = &pc->pc_common_tss;
1250
1251 /* Doublefault stack space, runs on ist1 for IDT. */
1252 if (fred) {
1253 wrmsr(MSR_FRED_RSP2, (uint64_t)&dblfault_stack[
1254 sizeof(dblfault_stack)]);
1255 } else {
1256 np = ((struct nmi_pcpu *)&dblfault_stack[sizeof(
1257 dblfault_stack)]) - 1;
1258 np->np_pcpu = (register_t)pc;
1259 tssp->tss_ist1 = (long)np;
1260 }
1261
1262 /*
1263 * NMI stack.
1264 */
1265 if (fred) {
1266 wrmsr(MSR_FRED_RSP1, (uint64_t)&nmi0_stack[
1267 sizeof(nmi0_stack)]);
1268 } else {
1269 /*
1270 * Runs on ist2 for IDT. The pcpu pointer is stored
1271 * just above the start of the ist2 stack.
1272 */
1273 np = ((struct nmi_pcpu *)&nmi0_stack[sizeof(nmi0_stack)]) - 1;
1274 np->np_pcpu = (register_t)pc;
1275 tssp->tss_ist2 = (long)np;
1276 }
1277
1278 if (!fred) {
1279 /*
1280 * MC# stack for IDT, runs on ist3. The pcpu pointer
1281 * is stored just above the start of the ist3 stack.
1282 */
1283 np = ((struct nmi_pcpu *)&mce0_stack[sizeof(mce0_stack)]) - 1;
1284 np->np_pcpu = (register_t)pc;
1285 tssp->tss_ist3 = (long)np;
1286
1287 /*
1288 * DB# stack for IDT, runs on ist4.
1289 */
1290 np = ((struct nmi_pcpu *)&dbg0_stack[sizeof(dbg0_stack)]) - 1;
1291 np->np_pcpu = (register_t)pc;
1292 tssp->tss_ist4 = (long)np;
1293 }
1294 }
1295
1296 /*
1297 * Calculate the kernel load address by inspecting page table created by loader.
1298 * The assumptions:
1299 * - kernel is mapped at KERNBASE, backed by contiguous phys memory
1300 * aligned at 2M, below 4G (the latter is important for AP startup)
1301 * - there is a 2M hole at KERNBASE (KERNSTART = KERNBASE + 2M)
1302 * - kernel is mapped with 2M superpages
1303 * - all participating memory, i.e. kernel, modules, metadata,
1304 * page table is accessible by pre-created 1:1 mapping
1305 * (right now loader creates 1:1 mapping for lower 4G, and all
1306 * memory is from there)
1307 * - there is a usable memory block right after the end of the
1308 * mapped kernel and all modules/metadata, pointed to by
1309 * physfree, for early allocations
1310 *
1311 * The memory block after the end of the kernel is important, loader
1312 * must ensure that no critical data structures are put there. Among
1313 * them is the trampoline page table, which must not be overwritten by
1314 * the allocations until pmap_bootstrap() switches %cr3 to the initial
1315 * version of the kernel page table. Size of the block is controlled
1316 * by the 'staging_slop' command for loader.efi.
1317 */
1318 vm_paddr_t __nosanitizeaddress __nosanitizememory
amd64_loadaddr(void)1319 amd64_loadaddr(void)
1320 {
1321 pml4_entry_t *pml4e;
1322 pdp_entry_t *pdpe;
1323 pd_entry_t *pde;
1324 uint64_t cr3;
1325
1326 cr3 = rcr3();
1327 pml4e = (pml4_entry_t *)cr3 + pmap_pml4e_index(KERNSTART);
1328 pdpe = (pdp_entry_t *)(*pml4e & PG_FRAME) + pmap_pdpe_index(KERNSTART);
1329 pde = (pd_entry_t *)(*pdpe & PG_FRAME) + pmap_pde_index(KERNSTART);
1330 return (*pde & PG_FRAME);
1331 }
1332
1333 u_int64_t
hammer_time(u_int64_t modulep,u_int64_t physfree)1334 hammer_time(u_int64_t modulep, u_int64_t physfree)
1335 {
1336 int gsel_tss, x;
1337 struct pcpu *pc;
1338 uint64_t rsp0;
1339 char *env;
1340 struct user_segment_descriptor *gdt;
1341 struct region_descriptor r_gdt;
1342 size_t kstack0_sz;
1343
1344 TSRAW(&thread0, TS_ENTER, __func__, NULL);
1345
1346 kernphys = amd64_loadaddr();
1347
1348 physfree += kernphys;
1349
1350 /* Initializes preload_kmdp */
1351 init_ops.parse_preload_data(modulep);
1352
1353 efi_boot = preload_search_info(preload_kmdp, MODINFO_METADATA |
1354 MODINFOMD_EFI_MAP) != NULL;
1355
1356 if (!efi_boot) {
1357 /* Tell the bios to warmboot next time */
1358 atomic_store_short((u_short *)0x472, 0x1234);
1359 }
1360
1361 physfree += ucode_load_bsp(physfree - kernphys + KERNSTART);
1362 physfree = roundup2(physfree, PAGE_SIZE);
1363
1364 identify_cpu1();
1365 identify_hypervisor();
1366 identify_hypervisor_smbios();
1367 identify_cpu_fixup_bsp();
1368 identify_cpu2();
1369 initializecpucache();
1370
1371 /*
1372 * Check for pti, pcid, and invpcid before ifuncs are
1373 * resolved, to correctly select the implementation for
1374 * pmap_activate_sw_mode().
1375 */
1376 pti = pti_get_default();
1377 TUNABLE_INT_FETCH("vm.pmap.pti", &pti);
1378 TUNABLE_INT_FETCH("vm.pmap.pcid_enabled", &pmap_pcid_enabled);
1379 if ((cpu_feature2 & CPUID2_PCID) == 0)
1380 pmap_pcid_enabled = 0;
1381 invpcid_works = (cpu_stdext_feature & CPUID_STDEXT_INVPCID) != 0;
1382
1383 /*
1384 * Now we can do small core initialization, after the PCID
1385 * CPU features and user knobs are evaluated.
1386 */
1387 TUNABLE_INT_FETCH("vm.pmap.pcid_invlpg_workaround",
1388 &pmap_pcid_invlpg_workaround_uena);
1389 cpu_init_small_core();
1390
1391 if ((cpu_feature2 & CPUID2_XSAVE) != 0) {
1392 use_xsave = 1;
1393 TUNABLE_INT_FETCH("hw.use_xsave", &use_xsave);
1394 }
1395
1396 if ((cpu_stdext_feature4 & (CPUID_STDEXT4_FRED | CPUID_STDEXT4_LKGS)) ==
1397 (CPUID_STDEXT4_FRED | CPUID_STDEXT4_LKGS) &&
1398 (cpu_stdext_feature & CPUID_STDEXT_FSGSBASE) != 0 && !pti) {
1399 fred = 1;
1400 TUNABLE_INT_FETCH("hw.fred", &fred);
1401 }
1402
1403 sched_instance_select();
1404
1405 link_elf_ireloc();
1406
1407 /*
1408 * This may be done better later if it gets more high level
1409 * components in it. If so just link td->td_proc here.
1410 */
1411 proc_linkup0(&proc0, &thread0);
1412
1413 /* Init basic tunables, hz etc */
1414 init_param1();
1415
1416 thread0.td_kstack = (char *)physfree - kernphys + KERNSTART;
1417 thread0.td_kstack_pages = kstack_pages;
1418 kstack0_sz = ptoa(kstack_pages);
1419 bzero(thread0.td_kstack, kstack0_sz);
1420 cpu_thread_new_kstack(&thread0);
1421 physfree += kstack0_sz;
1422
1423 /*
1424 * Initialize enough of thread0 for delayed invalidation to
1425 * work very early. Rely on thread0.td_base_pri
1426 * zero-initialization, it is reset to PVM at proc0_init().
1427 */
1428 pmap_thread_init_invl_gen(&thread0);
1429
1430 pc = &temp_bsp_pcpu;
1431 pcpu_init(pc, 0, sizeof(struct pcpu));
1432 gdt = &temp_bsp_pcpu.pc_gdt[0];
1433
1434 /*
1435 * make gdt memory segments
1436 */
1437 for (x = 0; x < NGDT; x++) {
1438 if (x != GPROC0_SEL && x != (GPROC0_SEL + 1) &&
1439 x != GUSERLDT_SEL && x != (GUSERLDT_SEL + 1))
1440 ssdtosd(&gdt_segs[x], &gdt[x]);
1441 }
1442 gdt_segs[GPROC0_SEL].ssd_base = (uintptr_t)&pc->pc_common_tss;
1443 ssdtosyssd(&gdt_segs[GPROC0_SEL],
1444 (struct system_segment_descriptor *)&gdt[GPROC0_SEL]);
1445
1446 r_gdt.rd_limit = NGDT * sizeof(gdt[0]) - 1;
1447 r_gdt.rd_base = (long)gdt;
1448 lgdt(&r_gdt);
1449
1450 wrmsr(MSR_FSBASE, 0); /* User value */
1451 wrmsr(MSR_GSBASE, (u_int64_t)pc);
1452 wrmsr(MSR_KGSBASE, 0); /* User value while in the kernel */
1453
1454 dpcpu_init((void *)(physfree - kernphys + KERNSTART), 0);
1455 physfree += DPCPU_SIZE;
1456 amd64_bsp_pcpu_init1(pc);
1457 /* Non-late cninit() and printf() can be moved up to here. */
1458
1459 /*
1460 * Initialize mutexes.
1461 *
1462 * icu_lock: in order to allow an interrupt to occur in a critical
1463 * section, to set pcpu->ipending (etc...) properly, we
1464 * must be able to get the icu lock, so it can't be
1465 * under witness.
1466 */
1467 mutex_init();
1468 mtx_init(&icu_lock, "icu", NULL, MTX_SPIN | MTX_NOWITNESS);
1469 mtx_init(&dt_lock, "descriptor tables", NULL, MTX_DEF);
1470
1471 /* exceptions */
1472 for (x = 0; x < NIDT; x++)
1473 setidt(x, pti ? &IDTVEC(rsvd_pti) : &IDTVEC(rsvd), SDT_SYSIGT,
1474 SEL_KPL, 0);
1475 setidt(IDT_DE, pti ? &IDTVEC(div_pti) : &IDTVEC(div), SDT_SYSIGT,
1476 SEL_KPL, 0);
1477 setidt(IDT_DB, &IDTVEC(dbg), SDT_SYSIGT, SEL_KPL, 4);
1478 setidt(IDT_NMI, &IDTVEC(nmi), SDT_SYSIGT, SEL_KPL, 2);
1479 setidt(IDT_BP, pti ? &IDTVEC(bpt_pti) : &IDTVEC(bpt), SDT_SYSIGT,
1480 SEL_UPL, 0);
1481 setidt(IDT_OF, pti ? &IDTVEC(ofl_pti) : &IDTVEC(ofl), SDT_SYSIGT,
1482 SEL_UPL, 0);
1483 setidt(IDT_BR, pti ? &IDTVEC(bnd_pti) : &IDTVEC(bnd), SDT_SYSIGT,
1484 SEL_KPL, 0);
1485 setidt(IDT_UD, pti ? &IDTVEC(ill_pti) : &IDTVEC(ill), SDT_SYSIGT,
1486 SEL_KPL, 0);
1487 setidt(IDT_NM, pti ? &IDTVEC(dna_pti) : &IDTVEC(dna), SDT_SYSIGT,
1488 SEL_KPL, 0);
1489 setidt(IDT_DF, &IDTVEC(dblfault), SDT_SYSIGT, SEL_KPL, 1);
1490 setidt(IDT_FPUGP, pti ? &IDTVEC(fpusegm_pti) : &IDTVEC(fpusegm),
1491 SDT_SYSIGT, SEL_KPL, 0);
1492 setidt(IDT_TS, pti ? &IDTVEC(tss_pti) : &IDTVEC(tss), SDT_SYSIGT,
1493 SEL_KPL, 0);
1494 setidt(IDT_NP, pti ? &IDTVEC(missing_pti) : &IDTVEC(missing),
1495 SDT_SYSIGT, SEL_KPL, 0);
1496 setidt(IDT_SS, pti ? &IDTVEC(stk_pti) : &IDTVEC(stk), SDT_SYSIGT,
1497 SEL_KPL, 0);
1498 setidt(IDT_GP, pti ? &IDTVEC(prot_pti) : &IDTVEC(prot), SDT_SYSIGT,
1499 SEL_KPL, 0);
1500 setidt(IDT_PF, pti ? &IDTVEC(page_pti) : &IDTVEC(page), SDT_SYSIGT,
1501 SEL_KPL, 0);
1502 setidt(IDT_MF, pti ? &IDTVEC(fpu_pti) : &IDTVEC(fpu), SDT_SYSIGT,
1503 SEL_KPL, 0);
1504 setidt(IDT_AC, pti ? &IDTVEC(align_pti) : &IDTVEC(align), SDT_SYSIGT,
1505 SEL_KPL, 0);
1506 setidt(IDT_MC, &IDTVEC(mchk), SDT_SYSIGT, SEL_KPL, 3);
1507 setidt(IDT_XF, pti ? &IDTVEC(xmm_pti) : &IDTVEC(xmm), SDT_SYSIGT,
1508 SEL_KPL, 0);
1509 #ifdef KDTRACE_HOOKS
1510 setidt(IDT_DTRACE_RET, pti ? &IDTVEC(dtrace_ret_pti) :
1511 &IDTVEC(dtrace_ret), SDT_SYSIGT, SEL_UPL, 0);
1512 #endif
1513 #ifdef XENHVM
1514 setidt(IDT_EVTCHN, pti ? &IDTVEC(xen_intr_upcall_pti) :
1515 &IDTVEC(xen_intr_upcall), SDT_SYSIGT, SEL_KPL, 0);
1516 #endif
1517 if (!fred) {
1518 r_idt.rd_limit = sizeof(idt0) - 1;
1519 r_idt.rd_base = (long) idt;
1520 lidt(&r_idt);
1521 }
1522
1523 TUNABLE_INT_FETCH("hw.ibrs_disable", &hw_ibrs_disable);
1524 TUNABLE_INT_FETCH("machdep.mitigations.ibrs.disable", &hw_ibrs_disable);
1525
1526 TUNABLE_INT_FETCH("hw.spec_store_bypass_disable", &hw_ssb_disable);
1527 TUNABLE_INT_FETCH("machdep.mitigations.ssb.disable", &hw_ssb_disable);
1528
1529 TUNABLE_INT_FETCH("machdep.syscall_ret_flush_l1d",
1530 &syscall_ret_l1d_flush_mode);
1531
1532 TUNABLE_INT_FETCH("hw.mds_disable", &hw_mds_disable);
1533 TUNABLE_INT_FETCH("machdep.mitigations.mds.disable", &hw_mds_disable);
1534
1535 TUNABLE_INT_FETCH("machdep.mitigations.taa.enable", &x86_taa_enable);
1536
1537 TUNABLE_INT_FETCH("machdep.mitigations.rngds.enable",
1538 &x86_rngds_mitg_enable);
1539
1540 TUNABLE_INT_FETCH("machdep.mitigations.zenbleed.enable",
1541 &zenbleed_enable);
1542 zenbleed_sanitize_enable();
1543
1544 finishidentcpu(); /* Final stage of CPU initialization */
1545
1546 invlpgb_works = (amd_extended_feature_extensions &
1547 AMDFEID_INVLPGB) != 0;
1548 TUNABLE_INT_FETCH("vm.pmap.invlpgb_works", &invlpgb_works);
1549 if (invlpgb_works)
1550 invlpgb_maxcnt = cpu_procinfo3 & AMDID_INVLPGB_MAXCNT;
1551
1552 /*
1553 * Initialize the clock before the console so that console
1554 * initialization can use DELAY().
1555 */
1556 clock_init();
1557
1558 initializecpu(); /* Initialize CPU registers */
1559
1560 amd64_bsp_ist_init(pc);
1561
1562 /* Set the IO permission bitmap (empty due to tss seg limit) */
1563 pc->pc_common_tss.tss_iobase = sizeof(struct amd64tss) +
1564 IOPERM_BITMAP_SIZE;
1565
1566 gsel_tss = GSEL(GPROC0_SEL, SEL_KPL);
1567 ltr(gsel_tss);
1568
1569 amd64_conf_fast_syscall();
1570
1571 /*
1572 * We initialize the PCB pointer early so that exception
1573 * handlers will work.
1574 */
1575 thread0.td_pcb = get_pcb_td(&thread0);
1576
1577 /*
1578 * The console and kdb should be initialized even earlier than here,
1579 * but some console drivers don't work until after getmemsize().
1580 * Default to late console initialization to support these drivers.
1581 * This loses mainly printf()s in getmemsize() and early debugging.
1582 */
1583 TUNABLE_INT_FETCH("debug.late_console", &late_console);
1584 if (!late_console) {
1585 cninit();
1586 amd64_kdb_init();
1587 }
1588
1589 getmemsize(physfree);
1590 init_param2(physmem);
1591
1592 /* now running on new page tables, configured,and u/iom is accessible */
1593
1594 #ifdef DEV_PCI
1595 /* This call might adjust phys_avail[]. */
1596 pci_early_quirks();
1597 #endif
1598
1599 if (late_console)
1600 cninit();
1601
1602 /*
1603 * Dump the boot metadata. We have to wait for cninit() since console
1604 * output is required. If it's grossly incorrect the kernel will never
1605 * make it this far.
1606 */
1607 if (getenv_is_true("debug.dump_modinfo_at_boot"))
1608 preload_dump();
1609
1610 if (fred)
1611 amd64_cpu_init_fred();
1612
1613 #ifdef DEV_ISA
1614 #ifdef DEV_ATPIC
1615 elcr_probe();
1616 atpic_startup();
1617 #else
1618 /* Reset and mask the atpics and leave them shut down. */
1619 atpic_reset();
1620
1621 /*
1622 * Point the ICU spurious interrupt vectors at the APIC spurious
1623 * interrupt handler.
1624 */
1625 setidt(IDT_IO_INTS + 7, IDTVEC(spuriousint), SDT_SYSIGT, SEL_KPL, 0);
1626 setidt(IDT_IO_INTS + 15, IDTVEC(spuriousint), SDT_SYSIGT, SEL_KPL, 0);
1627 #endif
1628 #else
1629 #error "have you forgotten the isa device?"
1630 #endif
1631
1632 if (late_console)
1633 amd64_kdb_init();
1634
1635 msgbufinit(msgbufp, msgbufsize);
1636 fpuinit();
1637
1638 /* make an initial tss so cpu can get interrupt stack on syscall! */
1639 rsp0 = (uintptr_t)thread0.td_md.md_stack_base;
1640 /* Ensure the stack is aligned to 16 bytes */
1641 rsp0 = STACKALIGN(rsp0);
1642 PCPU_PTR(common_tss)->tss_rsp0 = rsp0;
1643 amd64_bsp_pcpu_init2(rsp0);
1644
1645 /* transfer to user mode */
1646
1647 _ucodesel = GSEL(GUCODE_SEL, SEL_UPL);
1648 _udatasel = GSEL(GUDATA_SEL, SEL_UPL);
1649 _ucode32sel = GSEL(GUCODE32_SEL, SEL_UPL);
1650 _ufssel = GSEL(GUFS32_SEL, SEL_UPL);
1651 _ugssel = GSEL(GUGS32_SEL, SEL_UPL);
1652
1653 load_ds(_udatasel);
1654 load_es(_udatasel);
1655 load_fs(_ufssel);
1656
1657 /* setup proc 0's pcb */
1658 thread0.td_pcb->pcb_flags = 0;
1659
1660 amd64_init_splitlock();
1661 amd64_cpu_init_msr_memctl();
1662
1663 env = kern_getenv("kernelname");
1664 if (env != NULL)
1665 strlcpy(kernelname, env, sizeof(kernelname));
1666
1667 kcsan_cpu_init(0);
1668
1669 #ifdef FDT
1670 x86_init_fdt();
1671 #endif
1672
1673 kasan_init();
1674 kmsan_init();
1675
1676 TSEXIT();
1677
1678 /* Location of kernel stack for locore */
1679 return ((uintptr_t)thread0.td_md.md_stack_base);
1680 }
1681
1682 void
cpu_pcpu_init(struct pcpu * pcpu,int cpuid,size_t size)1683 cpu_pcpu_init(struct pcpu *pcpu, int cpuid, size_t size)
1684 {
1685
1686 pcpu->pc_acpi_id = 0xffffffff;
1687 }
1688
1689 static int
smap_sysctl_handler(SYSCTL_HANDLER_ARGS)1690 smap_sysctl_handler(SYSCTL_HANDLER_ARGS)
1691 {
1692 struct bios_smap *smapbase;
1693 struct bios_smap_xattr smap;
1694 uint32_t *smapattr;
1695 int count, error, i;
1696
1697 /* Retrieve the system memory map from the loader. */
1698 smapbase = (struct bios_smap *)preload_search_info(preload_kmdp,
1699 MODINFO_METADATA | MODINFOMD_SMAP);
1700 if (smapbase == NULL)
1701 return (0);
1702 smapattr = (uint32_t *)preload_search_info(preload_kmdp,
1703 MODINFO_METADATA | MODINFOMD_SMAP_XATTR);
1704 count = *((uint32_t *)smapbase - 1) / sizeof(*smapbase);
1705 error = 0;
1706 for (i = 0; i < count; i++) {
1707 smap.base = smapbase[i].base;
1708 smap.length = smapbase[i].length;
1709 smap.type = smapbase[i].type;
1710 if (smapattr != NULL)
1711 smap.xattr = smapattr[i];
1712 else
1713 smap.xattr = 0;
1714 error = SYSCTL_OUT(req, &smap, sizeof(smap));
1715 }
1716 return (error);
1717 }
1718 SYSCTL_PROC(_machdep, OID_AUTO, smap,
1719 CTLTYPE_OPAQUE | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0,
1720 smap_sysctl_handler, "S,bios_smap_xattr",
1721 "Raw BIOS SMAP data");
1722
1723 static int
efi_map_sysctl_handler(SYSCTL_HANDLER_ARGS)1724 efi_map_sysctl_handler(SYSCTL_HANDLER_ARGS)
1725 {
1726 struct efi_map_header *efihdr;
1727 uint32_t efisize;
1728
1729 efihdr = (struct efi_map_header *)preload_search_info(preload_kmdp,
1730 MODINFO_METADATA | MODINFOMD_EFI_MAP);
1731 if (efihdr == NULL)
1732 return (0);
1733 efisize = *((uint32_t *)efihdr - 1);
1734 return (SYSCTL_OUT(req, efihdr, efisize));
1735 }
1736 SYSCTL_PROC(_machdep, OID_AUTO, efi_map,
1737 CTLTYPE_OPAQUE | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0,
1738 efi_map_sysctl_handler, "S,efi_map_header",
1739 "Raw EFI Memory Map");
1740
1741 static int
efi_arch_sysctl_handler(SYSCTL_HANDLER_ARGS)1742 efi_arch_sysctl_handler(SYSCTL_HANDLER_ARGS)
1743 {
1744 char *arch;
1745
1746 arch = (char *)preload_search_info(preload_kmdp,
1747 MODINFO_METADATA | MODINFOMD_EFI_ARCH);
1748 if (arch == NULL)
1749 return (0);
1750
1751 return (SYSCTL_OUT_STR(req, arch));
1752 }
1753 SYSCTL_PROC(_machdep, OID_AUTO, efi_arch,
1754 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0,
1755 efi_arch_sysctl_handler, "A", "EFI Firmware Architecture");
1756
1757 void
spinlock_enter(void)1758 spinlock_enter(void)
1759 {
1760 struct thread *td;
1761 register_t flags;
1762
1763 td = curthread;
1764 if (td->td_md.md_spinlock_count == 0) {
1765 flags = intr_disable();
1766 td->td_md.md_spinlock_count = 1;
1767 td->td_md.md_saved_flags = flags;
1768 critical_enter();
1769 } else
1770 td->td_md.md_spinlock_count++;
1771 }
1772
1773 void
spinlock_exit(void)1774 spinlock_exit(void)
1775 {
1776 struct thread *td;
1777 register_t flags;
1778
1779 td = curthread;
1780 flags = td->td_md.md_saved_flags;
1781 td->td_md.md_spinlock_count--;
1782 if (td->td_md.md_spinlock_count == 0) {
1783 critical_exit();
1784 intr_restore(flags);
1785 }
1786 }
1787
1788 /*
1789 * Construct a PCB from a trapframe. This is called from kdb_trap() where
1790 * we want to start a backtrace from the function that caused us to enter
1791 * the debugger. We have the context in the trapframe, but base the trace
1792 * on the PCB. The PCB doesn't have to be perfect, as long as it contains
1793 * enough for a backtrace.
1794 */
1795 void
makectx(struct trapframe * tf,struct pcb * pcb)1796 makectx(struct trapframe *tf, struct pcb *pcb)
1797 {
1798
1799 pcb->pcb_r12 = tf->tf_r12;
1800 pcb->pcb_r13 = tf->tf_r13;
1801 pcb->pcb_r14 = tf->tf_r14;
1802 pcb->pcb_r15 = tf->tf_r15;
1803 pcb->pcb_rbp = tf->tf_rbp;
1804 pcb->pcb_rbx = tf->tf_rbx;
1805 pcb->pcb_rip = tf->tf_rip;
1806 pcb->pcb_rsp = tf->tf_rsp;
1807 }
1808
1809 /*
1810 * The pcb_flags is only modified by current thread, or by other threads
1811 * when current thread is stopped. However, current thread may change it
1812 * from the interrupt context in cpu_switch(), or in the trap handler.
1813 * When we read-modify-write pcb_flags from C sources, compiler may generate
1814 * code that is not atomic regarding the interrupt handler. If a trap or
1815 * interrupt happens and any flag is modified from the handler, it can be
1816 * clobbered with the cached value later. Therefore, we implement setting
1817 * and clearing flags with single-instruction functions, which do not race
1818 * with possible modification of the flags from the trap or interrupt context,
1819 * because traps and interrupts are executed only on instruction boundary.
1820 */
1821 void
set_pcb_flags_raw(struct pcb * pcb,const u_int flags)1822 set_pcb_flags_raw(struct pcb *pcb, const u_int flags)
1823 {
1824
1825 __asm __volatile("orl %1,%0"
1826 : "=m" (pcb->pcb_flags) : "ir" (flags), "m" (pcb->pcb_flags)
1827 : "cc", "memory");
1828
1829 }
1830
1831 /*
1832 * The support for RDFSBASE, WRFSBASE and similar instructions for %gs
1833 * base requires that kernel saves MSR_FSBASE and MSR_{K,}GSBASE into
1834 * pcb if user space modified the bases. We must save on the context
1835 * switch or if the return to usermode happens through the doreti.
1836 *
1837 * Tracking of both events is performed by the pcb flag PCB_FULL_IRET,
1838 * which have a consequence that the base MSRs must be saved each time
1839 * the PCB_FULL_IRET flag is set. We disable interrupts to sync with
1840 * context switches.
1841 */
1842 static void
set_pcb_flags_fsgsbase(struct pcb * pcb,const u_int flags)1843 set_pcb_flags_fsgsbase(struct pcb *pcb, const u_int flags)
1844 {
1845 register_t r;
1846
1847 if (curpcb == pcb &&
1848 (flags & PCB_FULL_IRET) != 0 &&
1849 (pcb->pcb_flags & PCB_FULL_IRET) == 0) {
1850 r = intr_disable();
1851 if ((pcb->pcb_flags & PCB_FULL_IRET) == 0) {
1852 pcb->pcb_fsbase = rdfsbase();
1853 pcb->pcb_gsbase = rdmsr(MSR_KGSBASE);
1854 }
1855 set_pcb_flags_raw(pcb, flags);
1856 intr_restore(r);
1857 } else {
1858 set_pcb_flags_raw(pcb, flags);
1859 }
1860 }
1861
1862 DEFINE_IFUNC(, void, set_pcb_flags, (struct pcb *, const u_int))
1863 {
1864
1865 return ((cpu_stdext_feature & CPUID_STDEXT_FSGSBASE) != 0 ?
1866 set_pcb_flags_fsgsbase : set_pcb_flags_raw);
1867 }
1868
1869 void
clear_pcb_flags(struct pcb * pcb,const u_int flags)1870 clear_pcb_flags(struct pcb *pcb, const u_int flags)
1871 {
1872
1873 __asm __volatile("andl %1,%0"
1874 : "=m" (pcb->pcb_flags) : "ir" (~flags), "m" (pcb->pcb_flags)
1875 : "cc", "memory");
1876 }
1877
1878 extern const char wrmsr_early_safe_gp_handler[];
1879
1880 /*
1881 * What about FRED? wrmsr_early_safe_start() is used before we
1882 * switched CPU to the FRED mode. We use IDT to catch #GP from MSR
1883 * write even if BSP is switched to the FRED mode later.
1884 */
1885 void
wrmsr_early_safe_start(void)1886 wrmsr_early_safe_start(void)
1887 {
1888 struct region_descriptor efi_idt;
1889 struct gate_descriptor *gpf_descr;
1890 int i;
1891
1892 efi_idt.rd_limit = 32 * sizeof(idt0[0]);
1893 efi_idt.rd_base = (uintptr_t)idt0;
1894 lidt(&efi_idt);
1895
1896 /* Setup handler for all possible exceptions. */
1897 for (i = 0; i < 32; i++) {
1898 gpf_descr = &idt0[i];
1899 gpf_descr->gd_looffset =
1900 (uintptr_t)wrmsr_early_safe_gp_handler;
1901 gpf_descr->gd_hioffset =
1902 (uintptr_t)wrmsr_early_safe_gp_handler >> 16;
1903 gpf_descr->gd_selector = rcs();
1904 gpf_descr->gd_type = SDT_SYSTGT;
1905 gpf_descr->gd_p = 1;
1906 }
1907 }
1908
1909 void
wrmsr_early_safe_end(void)1910 wrmsr_early_safe_end(void)
1911 {
1912 }
1913
1914 int
safe_read(vm_offset_t addr,char * valp)1915 safe_read(vm_offset_t addr, char *valp)
1916 {
1917 struct uio uio;
1918 struct iovec iov;
1919
1920 iov.iov_base = valp;
1921 iov.iov_len = 1;
1922 uio.uio_offset = addr;
1923 uio.uio_iov = &iov;
1924 uio.uio_iovcnt = 1;
1925 uio.uio_resid = 1;
1926 uio.uio_segflg = UIO_SYSSPACE;
1927 uio.uio_rw = UIO_READ;
1928 uio.uio_td = NULL;
1929 return (uiomove_mem(UIO_MEM_KMEM, &uio));
1930 }
1931
1932 static void
enable_splitlock_ac_wrmsr(void)1933 enable_splitlock_ac_wrmsr(void)
1934 {
1935 MPASS(ia32_splitlock);
1936 wrmsr(MSR_MEMORY_CTL, PCPU_GET(msr_memctl) | MSR_MEMORY_CTL_SPLITLOCK);
1937 }
1938
1939 static void
enable_splitlock_ac_wrmsrimm(void)1940 enable_splitlock_ac_wrmsrimm(void)
1941 {
1942 MPASS(ia32_splitlock);
1943 wrmsr_imm(MSR_MEMORY_CTL, PCPU_GET(msr_memctl) |
1944 MSR_MEMORY_CTL_SPLITLOCK);
1945 }
1946
1947 DEFINE_IFUNC(, void, enable_splitlock_ac, (void))
1948 {
1949 if ((cpu_stdext_feature5 & CPUID_STDEXT5_MSR_IMM) != 0)
1950 return (enable_splitlock_ac_wrmsrimm);
1951 return (enable_splitlock_ac_wrmsr);
1952 }
1953
1954 void
enable_splitlock(struct thread * td)1955 enable_splitlock(struct thread *td)
1956 {
1957 MPASS(td == curthread);
1958 td->td_md.md_td_flags |= TDF_MD_SPLITLOCK_AC;
1959 critical_enter();
1960 enable_splitlock_ac();
1961 critical_exit();
1962 }
1963
1964 static void
disable_splitlock_ac_wrmsr(void)1965 disable_splitlock_ac_wrmsr(void)
1966 {
1967 MPASS(ia32_splitlock);
1968 wrmsr(MSR_MEMORY_CTL, PCPU_GET(msr_memctl) & ~MSR_MEMORY_CTL_SPLITLOCK);
1969 }
1970
1971 static void
disable_splitlock_ac_wrmsrimm(void)1972 disable_splitlock_ac_wrmsrimm(void)
1973 {
1974 MPASS(ia32_splitlock);
1975 wrmsr_imm(MSR_MEMORY_CTL, PCPU_GET(msr_memctl) &
1976 ~MSR_MEMORY_CTL_SPLITLOCK);
1977 }
1978
1979 DEFINE_IFUNC(, void, disable_splitlock_ac, (void))
1980 {
1981 if ((cpu_stdext_feature5 & CPUID_STDEXT5_MSR_IMM) != 0)
1982 return (disable_splitlock_ac_wrmsrimm);
1983 return (disable_splitlock_ac_wrmsr);
1984 }
1985
1986 void
disable_splitlock(struct thread * td)1987 disable_splitlock(struct thread *td)
1988 {
1989 MPASS(td == curthread);
1990 td->td_md.md_td_flags &= ~TDF_MD_SPLITLOCK_AC;
1991 critical_enter();
1992 disable_splitlock_ac();
1993 critical_exit();
1994 }
1995
1996 #ifdef KDB
1997
1998 /*
1999 * Provide inb() and outb() as functions. They are normally only available as
2000 * inline functions, thus cannot be called from the debugger.
2001 */
2002
2003 /* silence compiler warnings */
2004 u_char inb_(u_short);
2005 void outb_(u_short, u_char);
2006
2007 u_char
inb_(u_short port)2008 inb_(u_short port)
2009 {
2010 return inb(port);
2011 }
2012
2013 void
outb_(u_short port,u_char data)2014 outb_(u_short port, u_char data)
2015 {
2016 outb(port, data);
2017 }
2018
2019 #endif /* KDB */
2020
2021 #undef memset
2022 #undef memmove
2023 #undef memcpy
2024
2025 void *memset_std(void *buf, int c, size_t len);
2026 void *memset_erms(void *buf, int c, size_t len);
2027 void *memmove_std(void * _Nonnull dst, const void * _Nonnull src,
2028 size_t len);
2029 void *memmove_erms(void * _Nonnull dst, const void * _Nonnull src,
2030 size_t len);
2031 void *memcpy_std(void * _Nonnull dst, const void * _Nonnull src,
2032 size_t len);
2033 void *memcpy_erms(void * _Nonnull dst, const void * _Nonnull src,
2034 size_t len);
2035
2036 #ifdef KCSAN
2037 /*
2038 * These fail to build as ifuncs when used with KCSAN.
2039 */
2040 void *
memset(void * buf,int c,size_t len)2041 memset(void *buf, int c, size_t len)
2042 {
2043
2044 return (memset_std(buf, c, len));
2045 }
2046
2047 void *
memmove(void * _Nonnull dst,const void * _Nonnull src,size_t len)2048 memmove(void * _Nonnull dst, const void * _Nonnull src, size_t len)
2049 {
2050
2051 return (memmove_std(dst, src, len));
2052 }
2053
2054 void *
memcpy(void * _Nonnull dst,const void * _Nonnull src,size_t len)2055 memcpy(void * _Nonnull dst, const void * _Nonnull src, size_t len)
2056 {
2057
2058 return (memcpy_std(dst, src, len));
2059 }
2060 #else
2061 DEFINE_IFUNC(, void *, memset, (void *, int, size_t))
2062 {
2063
2064 return ((cpu_stdext_feature & CPUID_STDEXT_ERMS) != 0 ?
2065 memset_erms : memset_std);
2066 }
2067
2068 DEFINE_IFUNC(, void *, memmove, (void * _Nonnull, const void * _Nonnull,
2069 size_t))
2070 {
2071
2072 return ((cpu_stdext_feature & CPUID_STDEXT_ERMS) != 0 ?
2073 memmove_erms : memmove_std);
2074 }
2075
2076 DEFINE_IFUNC(, void *, memcpy, (void * _Nonnull, const void * _Nonnull,size_t))
2077 {
2078
2079 return ((cpu_stdext_feature & CPUID_STDEXT_ERMS) != 0 ?
2080 memcpy_erms : memcpy_std);
2081 }
2082 #endif
2083
2084 void pagezero_std(void *addr);
2085 void pagezero_erms(void *addr);
2086 DEFINE_IFUNC(, void , pagezero, (void *))
2087 {
2088
2089 return ((cpu_stdext_feature & CPUID_STDEXT_ERMS) != 0 ?
2090 pagezero_erms : pagezero_std);
2091 }
2092