1 /*-
2 * SPDX-License-Identifier: BSD-4-Clause
3 *
4 * Copyright (c) 2003 Peter Wemm.
5 * Copyright (c) 1992 Terrence R. Lambert.
6 * Copyright (c) 1982, 1987, 1990 The Regents of the University of California.
7 * All rights reserved.
8 *
9 * This code is derived from software contributed to Berkeley by
10 * William Jolitz.
11 *
12 * Redistribution and use in source and binary forms, with or without
13 * modification, are permitted provided that the following conditions
14 * are met:
15 * 1. Redistributions of source code must retain the above copyright
16 * notice, this list of conditions and the following disclaimer.
17 * 2. Redistributions in binary form must reproduce the above copyright
18 * notice, this list of conditions and the following disclaimer in the
19 * documentation and/or other materials provided with the distribution.
20 * 3. All advertising materials mentioning features or use of this software
21 * must display the following acknowledgement:
22 * This product includes software developed by the University of
23 * California, Berkeley and its contributors.
24 * 4. Neither the name of the University nor the names of its contributors
25 * may be used to endorse or promote products derived from this software
26 * without specific prior written permission.
27 *
28 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
29 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
30 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
31 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
32 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
33 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
34 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
35 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
36 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
37 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
38 * SUCH DAMAGE.
39 */
40
41 #include "opt_atpic.h"
42 #include "opt_cpu.h"
43 #include "opt_ddb.h"
44 #include "opt_inet.h"
45 #include "opt_isa.h"
46 #include "opt_kstack_pages.h"
47 #include "opt_maxmem.h"
48 #include "opt_pci.h"
49 #include "opt_platform.h"
50 #include "opt_sched.h"
51
52 #include <sys/param.h>
53 #include <sys/proc.h>
54 #include <sys/systm.h>
55 #include <sys/asan.h>
56 #include <sys/bio.h>
57 #include <sys/buf.h>
58 #include <sys/bus.h>
59 #include <sys/callout.h>
60 #include <sys/cons.h>
61 #include <sys/cpu.h>
62 #include <sys/csan.h>
63 #include <sys/efi.h>
64 #include <sys/eventhandler.h>
65 #include <sys/exec.h>
66 #include <sys/imgact.h>
67 #include <sys/kdb.h>
68 #include <sys/kernel.h>
69 #include <sys/ktr.h>
70 #include <sys/linker.h>
71 #include <sys/lock.h>
72 #include <sys/malloc.h>
73 #include <sys/memrange.h>
74 #include <sys/msan.h>
75 #include <sys/msgbuf.h>
76 #include <sys/mutex.h>
77 #include <sys/pcpu.h>
78 #include <sys/ptrace.h>
79 #include <sys/reboot.h>
80 #include <sys/reg.h>
81 #include <sys/rwlock.h>
82 #include <sys/sched.h>
83 #include <sys/signalvar.h>
84 #include <sys/smp.h>
85 #include <sys/syscallsubr.h>
86 #include <sys/sysctl.h>
87 #include <sys/sysent.h>
88 #include <sys/sysproto.h>
89 #include <sys/ucontext.h>
90 #include <sys/vmmeter.h>
91
92 #include <vm/vm.h>
93 #include <vm/vm_param.h>
94 #include <vm/vm_extern.h>
95 #include <vm/vm_kern.h>
96 #include <vm/vm_page.h>
97 #include <vm/vm_map.h>
98 #include <vm/vm_object.h>
99 #include <vm/vm_pager.h>
100 #include <vm/vm_phys.h>
101 #include <vm/vm_dumpset.h>
102
103 #ifdef DDB
104 #ifndef KDB
105 #error KDB must be enabled in order for DDB to work!
106 #endif
107 #include <ddb/ddb.h>
108 #include <ddb/db_sym.h>
109 #endif
110
111 #include <net/netisr.h>
112
113 #include <dev/smbios/smbios.h>
114
115 #include <machine/clock.h>
116 #include <machine/cpu.h>
117 #include <machine/cputypes.h>
118 #include <machine/frame.h>
119 #include <machine/intr_machdep.h>
120 #include <x86/mca.h>
121 #include <machine/md_var.h>
122 #include <machine/metadata.h>
123 #include <machine/pc/bios.h>
124 #include <machine/pcb.h>
125 #include <machine/proc.h>
126 #include <machine/sigframe.h>
127 #include <machine/specialreg.h>
128 #include <machine/trap.h>
129 #include <machine/tss.h>
130 #include <x86/ucode.h>
131 #include <x86/ifunc.h>
132 #include <machine/smp.h>
133 #ifdef FDT
134 #include <x86/fdt.h>
135 #endif
136
137 #ifdef DEV_ATPIC
138 #include <x86/isa/icu.h>
139 #else
140 #include <x86/apicvar.h>
141 #endif
142
143 #include <isa/isareg.h>
144 #include <isa/rtc.h>
145 #include <x86/init.h>
146
147 #ifndef SMP
148 #error amd64 requires options SMP
149 #endif
150
151 /* Sanity check for __curthread() */
152 CTASSERT(offsetof(struct pcpu, pc_curthread) == 0);
153
154 /*
155 * The PTI trampoline stack needs enough space for a hardware trapframe and a
156 * couple of scratch registers, as well as the trapframe left behind after an
157 * iret fault.
158 */
159 CTASSERT(PC_PTI_STACK_SZ * sizeof(register_t) >= 2 * sizeof(struct pti_frame) -
160 offsetof(struct pti_frame, pti_rip));
161
162 extern u_int64_t hammer_time(u_int64_t, u_int64_t);
163
164 static void cpu_startup(void *);
165 SYSINIT(cpu, SI_SUB_CPU, SI_ORDER_FIRST, cpu_startup, NULL);
166
167 /* Probe 8254 PIT and TSC. */
168 static void native_clock_source_init(void);
169
170 /* Preload data parse function */
171 static void native_parse_preload_data(u_int64_t);
172
173 /* Native function to fetch and parse the e820 map */
174 static void native_parse_memmap(vm_paddr_t *, int *);
175
176 /* Default init_ops implementation. */
177 struct init_ops init_ops = {
178 .parse_preload_data = native_parse_preload_data,
179 .early_clock_source_init = native_clock_source_init,
180 .early_delay = i8254_delay,
181 .parse_memmap = native_parse_memmap,
182 };
183
184 /*
185 * Physical address of the EFI System Table. Stashed from the metadata hints
186 * passed into the kernel and used by the EFI code to call runtime services.
187 */
188 vm_paddr_t efi_systbl_phys;
189
190 /*
191 * Bitmap of extra EFI memory region types that should be preserved and mapped
192 * during runtime services calls.
193 */
194 uint32_t efi_map_regs;
195
196 /* Intel ICH registers */
197 #define ICH_PMBASE 0x400
198 #define ICH_SMI_EN ICH_PMBASE + 0x30
199
200 int _udatasel, _ucodesel, _ucode32sel, _ufssel, _ugssel;
201
202 int cold = 1;
203
204 long Maxmem = 0;
205 long realmem = 0;
206 int late_console = 1;
207 int lass_enabled = 0;
208
209 int __read_frequently fred = 0;
210 SYSCTL_INT(_hw, OID_AUTO, fred, CTLFLAG_RDTUN | CTLFLAG_NOFETCH,
211 &fred, 0,
212 "FRED is used");
213
214 struct kva_md_info kmi;
215
216 struct region_descriptor r_idt;
217 struct pcpu *__pcpu;
218 struct pcpu temp_bsp_pcpu;
219
220 struct mtx icu_lock;
221
222 struct mem_range_softc mem_range_softc;
223
224 struct mtx dt_lock; /* lock for GDT and LDT */
225
226 void (*vmm_suspend_p)(void);
227 void (*vmm_resume_p)(void);
228
229 bool efi_boot;
230
231 static void
cpu_startup(void * dummy)232 cpu_startup(void *dummy)
233 {
234 uintmax_t memsize;
235 char *sysenv;
236
237 /*
238 * On MacBooks, we need to disallow the legacy USB circuit to
239 * generate an SMI# because this can cause several problems,
240 * namely: incorrect CPU frequency detection and failure to
241 * start the APs.
242 * We do this by disabling a bit in the SMI_EN (SMI Control and
243 * Enable register) of the Intel ICH LPC Interface Bridge.
244 */
245 sysenv = kern_getenv("smbios.system.product");
246 if (sysenv != NULL) {
247 if (strncmp(sysenv, "MacBook1,1", 10) == 0 ||
248 strncmp(sysenv, "MacBook3,1", 10) == 0 ||
249 strncmp(sysenv, "MacBook4,1", 10) == 0 ||
250 strncmp(sysenv, "MacBookPro1,1", 13) == 0 ||
251 strncmp(sysenv, "MacBookPro1,2", 13) == 0 ||
252 strncmp(sysenv, "MacBookPro3,1", 13) == 0 ||
253 strncmp(sysenv, "MacBookPro4,1", 13) == 0 ||
254 strncmp(sysenv, "Macmini1,1", 10) == 0) {
255 if (bootverbose)
256 printf("Disabling LEGACY_USB_EN bit on "
257 "Intel ICH.\n");
258 outl(ICH_SMI_EN, inl(ICH_SMI_EN) & ~0x8);
259 }
260 freeenv(sysenv);
261 }
262
263 /*
264 * Good {morning,afternoon,evening,night}.
265 */
266 startrtclock();
267 printcpuinfo();
268
269 /*
270 * Display physical memory if SMBIOS reports reasonable amount.
271 */
272 memsize = 0;
273 sysenv = kern_getenv("smbios.memory.enabled");
274 if (sysenv != NULL) {
275 memsize = (uintmax_t)strtoul(sysenv, (char **)NULL, 10) << 10;
276 freeenv(sysenv);
277 }
278 if (memsize < ptoa((uintmax_t)vm_free_count()))
279 memsize = ptoa((uintmax_t)Maxmem);
280 printf("real memory = %ju (%ju MB)\n", memsize, memsize >> 20);
281 realmem = atop(memsize);
282
283 /*
284 * Display any holes after the first chunk of extended memory.
285 */
286 if (bootverbose) {
287 int indx;
288
289 printf("Physical memory chunk(s):\n");
290 for (indx = 0; phys_avail[indx + 1] != 0; indx += 2) {
291 vm_paddr_t size;
292
293 size = phys_avail[indx + 1] - phys_avail[indx];
294 printf(
295 "0x%016jx - 0x%016jx, %ju bytes (%ju pages)\n",
296 (uintmax_t)phys_avail[indx],
297 (uintmax_t)phys_avail[indx + 1] - 1,
298 (uintmax_t)size, (uintmax_t)size / PAGE_SIZE);
299 }
300 }
301
302 vm_ksubmap_init(&kmi);
303
304 printf("avail memory = %ju (%ju MB)\n",
305 ptoa((uintmax_t)vm_free_count()),
306 ptoa((uintmax_t)vm_free_count()) / 1048576);
307 #ifdef DEV_PCI
308 if (bootverbose && intel_graphics_stolen_base != 0)
309 printf("intel stolen mem: base %#jx size %ju MB\n",
310 (uintmax_t)intel_graphics_stolen_base,
311 (uintmax_t)intel_graphics_stolen_size / 1024 / 1024);
312 #endif
313
314 /*
315 * Set up buffers, so they can be used to read disk labels.
316 */
317 bufinit();
318 vm_pager_bufferinit();
319
320 cpu_setregs();
321 }
322
323 static void
late_ifunc_resolve(void * dummy __unused)324 late_ifunc_resolve(void *dummy __unused)
325 {
326 link_elf_late_ireloc();
327 }
328 SYSINIT(late_ifunc_resolve, SI_SUB_CPU, SI_ORDER_ANY, late_ifunc_resolve, NULL);
329
330 void
cpu_setregs(void)331 cpu_setregs(void)
332 {
333 register_t cr0;
334
335 TSENTER();
336 cr0 = rcr0();
337 cr0 |= CR0_MP | CR0_NE | CR0_TS | CR0_WP | CR0_AM;
338 TSENTER2("load_cr0");
339 load_cr0(cr0);
340 TSEXIT2("load_cr0");
341 TSEXIT();
342 }
343
344 /*
345 * Initialize amd64 and configure to run kernel
346 */
347
348 /*
349 * Initialize segments & interrupt table
350 */
351 static struct gate_descriptor idt0[NIDT];
352 struct gate_descriptor *idt = &idt0[0]; /* interrupt descriptor table */
353
354 static char dblfault_stack[DBLFAULT_STACK_SIZE] __aligned(64);
355 static char mce0_stack[MCE_STACK_SIZE] __aligned(16);
356 static char nmi0_stack[NMI_STACK_SIZE] __aligned(64);
357 static char dbg0_stack[DBG_STACK_SIZE] __aligned(16);
358 CTASSERT(sizeof(struct nmi_pcpu) == 16);
359
360 /*
361 * Software prototypes -- in more palatable form.
362 *
363 * Keep GUFS32, GUGS32, GUCODE32 and GUDATA at the same
364 * slots as corresponding segments for i386 kernel.
365 */
366 struct soft_segment_descriptor gdt_segs[] = {
367 [GNULL_SEL] = { /* 0 Null Descriptor */
368 .ssd_base = 0x0,
369 .ssd_limit = 0x0,
370 .ssd_type = 0,
371 .ssd_dpl = 0,
372 .ssd_p = 0,
373 .ssd_long = 0,
374 .ssd_def32 = 0,
375 .ssd_gran = 0 },
376 [GNULL2_SEL] = { /* 1 Null Descriptor */
377 .ssd_base = 0x0,
378 .ssd_limit = 0x0,
379 .ssd_type = 0,
380 .ssd_dpl = 0,
381 .ssd_p = 0,
382 .ssd_long = 0,
383 .ssd_def32 = 0,
384 .ssd_gran = 0 },
385 [GUFS32_SEL] = { /* 2 32 bit %gs Descriptor for user */
386 .ssd_base = 0x0,
387 .ssd_limit = 0xfffff,
388 .ssd_type = SDT_MEMRWA,
389 .ssd_dpl = SEL_UPL,
390 .ssd_p = 1,
391 .ssd_long = 0,
392 .ssd_def32 = 1,
393 .ssd_gran = 1 },
394 [GUGS32_SEL] = { /* 3 32 bit %fs Descriptor for user */
395 .ssd_base = 0x0,
396 .ssd_limit = 0xfffff,
397 .ssd_type = SDT_MEMRWA,
398 .ssd_dpl = SEL_UPL,
399 .ssd_p = 1,
400 .ssd_long = 0,
401 .ssd_def32 = 1,
402 .ssd_gran = 1 },
403 [GCODE_SEL] = { /* 4 Code Descriptor for kernel */
404 .ssd_base = 0x0,
405 .ssd_limit = 0xfffff,
406 .ssd_type = SDT_MEMERA,
407 .ssd_dpl = SEL_KPL,
408 .ssd_p = 1,
409 .ssd_long = 1,
410 .ssd_def32 = 0,
411 .ssd_gran = 1 },
412 [GDATA_SEL] = { /* 5 Data Descriptor for kernel */
413 .ssd_base = 0x0,
414 .ssd_limit = 0xfffff,
415 .ssd_type = SDT_MEMRWA,
416 .ssd_dpl = SEL_KPL,
417 .ssd_p = 1,
418 .ssd_long = 1,
419 .ssd_def32 = 0,
420 .ssd_gran = 1 },
421 [GUCODE32_SEL] = { /* 6 32 bit Code Descriptor for user */
422 .ssd_base = 0x0,
423 .ssd_limit = 0xfffff,
424 .ssd_type = SDT_MEMERA,
425 .ssd_dpl = SEL_UPL,
426 .ssd_p = 1,
427 .ssd_long = 0,
428 .ssd_def32 = 1,
429 .ssd_gran = 1 },
430 [GUDATA_SEL] = { /* 7 32/64 bit Data Descriptor for user */
431 .ssd_base = 0x0,
432 .ssd_limit = 0xfffff,
433 .ssd_type = SDT_MEMRWA,
434 .ssd_dpl = SEL_UPL,
435 .ssd_p = 1,
436 .ssd_long = 0,
437 .ssd_def32 = 1,
438 .ssd_gran = 1 },
439 [GUCODE_SEL] = { /* 8 64 bit Code Descriptor for user */
440 .ssd_base = 0x0,
441 .ssd_limit = 0xfffff,
442 .ssd_type = SDT_MEMERA,
443 .ssd_dpl = SEL_UPL,
444 .ssd_p = 1,
445 .ssd_long = 1,
446 .ssd_def32 = 0,
447 .ssd_gran = 1 },
448 [GPROC0_SEL] = { /* 9 Proc 0 TSS Descriptor */
449 .ssd_base = 0x0,
450 .ssd_limit = sizeof(struct amd64tss) + IOPERM_BITMAP_SIZE - 1,
451 .ssd_type = SDT_SYSTSS,
452 .ssd_dpl = SEL_KPL,
453 .ssd_p = 1,
454 .ssd_long = 0,
455 .ssd_def32 = 0,
456 .ssd_gran = 0 },
457 [GPROC0_SEL + 1] = { /* 10 Proc 0 TSS descriptor, double size */
458 .ssd_base = 0x0,
459 .ssd_limit = 0x0,
460 .ssd_type = 0,
461 .ssd_dpl = 0,
462 .ssd_p = 0,
463 .ssd_long = 0,
464 .ssd_def32 = 0,
465 .ssd_gran = 0 },
466 [GUSERLDT_SEL] = { /* 11 LDT Descriptor */
467 .ssd_base = 0x0,
468 .ssd_limit = 0x0,
469 .ssd_type = 0,
470 .ssd_dpl = 0,
471 .ssd_p = 0,
472 .ssd_long = 0,
473 .ssd_def32 = 0,
474 .ssd_gran = 0 },
475 [GUSERLDT_SEL + 1] = { /* 12 LDT Descriptor, double size */
476 .ssd_base = 0x0,
477 .ssd_limit = 0x0,
478 .ssd_type = 0,
479 .ssd_dpl = 0,
480 .ssd_p = 0,
481 .ssd_long = 0,
482 .ssd_def32 = 0,
483 .ssd_gran = 0 },
484 };
485 _Static_assert(nitems(gdt_segs) == NGDT, "Stale NGDT");
486
487 void
setidt(int idx,inthand_t * func,int typ,int dpl,int ist)488 setidt(int idx, inthand_t *func, int typ, int dpl, int ist)
489 {
490 struct gate_descriptor *ip;
491
492 if (fred)
493 return;
494
495 ip = idt + idx;
496 ip->gd_looffset = (uintptr_t)func;
497 ip->gd_selector = GSEL(GCODE_SEL, SEL_KPL);
498 ip->gd_ist = ist;
499 ip->gd_xx = 0;
500 ip->gd_type = typ;
501 ip->gd_dpl = dpl;
502 ip->gd_p = 1;
503 ip->gd_hioffset = ((uintptr_t)func)>>16 ;
504 }
505
506 extern inthand_t
507 IDTVEC(div), IDTVEC(dbg), IDTVEC(nmi), IDTVEC(bpt), IDTVEC(ofl),
508 IDTVEC(bnd), IDTVEC(ill), IDTVEC(dna), IDTVEC(fpusegm),
509 IDTVEC(tss), IDTVEC(missing), IDTVEC(stk), IDTVEC(prot),
510 IDTVEC(page), IDTVEC(mchk), IDTVEC(rsvd), IDTVEC(fpu), IDTVEC(align),
511 IDTVEC(xmm), IDTVEC(dblfault),
512 IDTVEC(div_pti), IDTVEC(bpt_pti),
513 IDTVEC(ofl_pti), IDTVEC(bnd_pti), IDTVEC(ill_pti), IDTVEC(dna_pti),
514 IDTVEC(fpusegm_pti), IDTVEC(tss_pti), IDTVEC(missing_pti),
515 IDTVEC(stk_pti), IDTVEC(prot_pti), IDTVEC(page_pti),
516 IDTVEC(rsvd_pti), IDTVEC(fpu_pti), IDTVEC(align_pti),
517 IDTVEC(xmm_pti),
518 #ifdef KDTRACE_HOOKS
519 IDTVEC(dtrace_ret), IDTVEC(dtrace_ret_pti),
520 #endif
521 #ifdef XENHVM
522 IDTVEC(xen_intr_upcall), IDTVEC(xen_intr_upcall_pti),
523 #endif
524 IDTVEC(fast_syscall), IDTVEC(fast_syscall32),
525 IDTVEC(fast_syscall_pti);
526
527 #ifdef DDB
528 /*
529 * Display the index and function name of any IDT entries that don't use
530 * the default 'rsvd' entry point.
531 */
DB_SHOW_COMMAND_FLAGS(idt,db_show_idt,DB_CMD_MEMSAFE)532 DB_SHOW_COMMAND_FLAGS(idt, db_show_idt, DB_CMD_MEMSAFE)
533 {
534 struct gate_descriptor *ip;
535 int idx;
536 uintptr_t func;
537
538 ip = idt;
539 for (idx = 0; idx < NIDT && !db_pager_quit; idx++) {
540 func = ((long)ip->gd_hioffset << 16 | ip->gd_looffset);
541 if (func != (uintptr_t)&IDTVEC(rsvd)) {
542 db_printf("%3d\t", idx);
543 db_printsym(func, DB_STGY_PROC);
544 db_printf("\n");
545 }
546 ip++;
547 }
548 }
549
550 /* Show privileged registers. */
DB_SHOW_COMMAND_FLAGS(sysregs,db_show_sysregs,DB_CMD_MEMSAFE)551 DB_SHOW_COMMAND_FLAGS(sysregs, db_show_sysregs, DB_CMD_MEMSAFE)
552 {
553 struct {
554 uint16_t limit;
555 uint64_t base;
556 } __packed idtr, gdtr;
557 uint16_t ldt, tr;
558
559 __asm __volatile("sidt %0" : "=m" (idtr));
560 db_printf("idtr\t0x%016lx/%04x\n",
561 (u_long)idtr.base, (u_int)idtr.limit);
562 __asm __volatile("sgdt %0" : "=m" (gdtr));
563 db_printf("gdtr\t0x%016lx/%04x\n",
564 (u_long)gdtr.base, (u_int)gdtr.limit);
565 __asm __volatile("sldt %0" : "=r" (ldt));
566 db_printf("ldtr\t0x%04x\n", ldt);
567 __asm __volatile("str %0" : "=r" (tr));
568 db_printf("tr\t0x%04x\n", tr);
569 db_printf("cr0\t0x%016lx\n", rcr0());
570 db_printf("cr2\t0x%016lx\n", rcr2());
571 db_printf("cr3\t0x%016lx\n", rcr3());
572 db_printf("cr4\t0x%016lx\n", rcr4());
573 if (rcr4() & CR4_XSAVE)
574 db_printf("xcr0\t0x%016lx\n", rxcr(0));
575 db_printf("EFER\t0x%016lx\n", rdmsr(MSR_EFER));
576 if (cpu_feature2 & (CPUID2_VMX | CPUID2_SMX))
577 db_printf("FEATURES_CTL\t%016lx\n",
578 rdmsr(MSR_IA32_FEATURE_CONTROL));
579 db_printf("DEBUG_CTL\t0x%016lx\n", rdmsr(MSR_DEBUGCTLMSR));
580 db_printf("PAT\t0x%016lx\n", rdmsr(MSR_PAT));
581 db_printf("GSBASE\t0x%016lx\n", rdmsr(MSR_GSBASE));
582 }
583
DB_SHOW_COMMAND_FLAGS(dbregs,db_show_dbregs,DB_CMD_MEMSAFE)584 DB_SHOW_COMMAND_FLAGS(dbregs, db_show_dbregs, DB_CMD_MEMSAFE)
585 {
586
587 db_printf("dr0\t0x%016lx\n", rdr0());
588 db_printf("dr1\t0x%016lx\n", rdr1());
589 db_printf("dr2\t0x%016lx\n", rdr2());
590 db_printf("dr3\t0x%016lx\n", rdr3());
591 db_printf("dr6\t0x%016lx\n", rdr6());
592 db_printf("dr7\t0x%016lx\n", rdr7());
593 }
594 #endif
595
596 void
sdtossd(struct user_segment_descriptor * sd,struct soft_segment_descriptor * ssd)597 sdtossd(struct user_segment_descriptor *sd, struct soft_segment_descriptor *ssd)
598 {
599
600 ssd->ssd_base = (sd->sd_hibase << 24) | sd->sd_lobase;
601 ssd->ssd_limit = (sd->sd_hilimit << 16) | sd->sd_lolimit;
602 ssd->ssd_type = sd->sd_type;
603 ssd->ssd_dpl = sd->sd_dpl;
604 ssd->ssd_p = sd->sd_p;
605 ssd->ssd_long = sd->sd_long;
606 ssd->ssd_def32 = sd->sd_def32;
607 ssd->ssd_gran = sd->sd_gran;
608 }
609
610 void
ssdtosd(struct soft_segment_descriptor * ssd,struct user_segment_descriptor * sd)611 ssdtosd(struct soft_segment_descriptor *ssd, struct user_segment_descriptor *sd)
612 {
613
614 sd->sd_lobase = (ssd->ssd_base) & 0xffffff;
615 sd->sd_hibase = (ssd->ssd_base >> 24) & 0xff;
616 sd->sd_lolimit = (ssd->ssd_limit) & 0xffff;
617 sd->sd_hilimit = (ssd->ssd_limit >> 16) & 0xf;
618 sd->sd_type = ssd->ssd_type;
619 sd->sd_dpl = ssd->ssd_dpl;
620 sd->sd_p = ssd->ssd_p;
621 sd->sd_long = ssd->ssd_long;
622 sd->sd_def32 = ssd->ssd_def32;
623 sd->sd_gran = ssd->ssd_gran;
624 }
625
626 void
ssdtosyssd(struct soft_segment_descriptor * ssd,struct system_segment_descriptor * sd)627 ssdtosyssd(struct soft_segment_descriptor *ssd, struct system_segment_descriptor *sd)
628 {
629
630 sd->sd_lobase = (ssd->ssd_base) & 0xffffff;
631 sd->sd_hibase = (ssd->ssd_base >> 24) & 0xfffffffffful;
632 sd->sd_lolimit = (ssd->ssd_limit) & 0xffff;
633 sd->sd_hilimit = (ssd->ssd_limit >> 16) & 0xf;
634 sd->sd_type = ssd->ssd_type;
635 sd->sd_dpl = ssd->ssd_dpl;
636 sd->sd_p = ssd->ssd_p;
637 sd->sd_gran = ssd->ssd_gran;
638 }
639
640 u_int basemem;
641
642 static int
add_physmap_entry(uint64_t base,uint64_t length,vm_paddr_t * physmap,int * physmap_idxp)643 add_physmap_entry(uint64_t base, uint64_t length, vm_paddr_t *physmap,
644 int *physmap_idxp)
645 {
646 int i, insert_idx, physmap_idx;
647
648 physmap_idx = *physmap_idxp;
649
650 if (length == 0)
651 return (1);
652
653 /*
654 * Find insertion point while checking for overlap. Start off by
655 * assuming the new entry will be added to the end.
656 *
657 * NB: physmap_idx points to the next free slot.
658 */
659 insert_idx = physmap_idx;
660 for (i = 0; i < physmap_idx; i += 2) {
661 if (base < physmap[i + 1]) {
662 if (base + length <= physmap[i]) {
663 insert_idx = i;
664 break;
665 }
666 if (boothowto & RB_VERBOSE)
667 printf(
668 "Overlapping memory regions, ignoring second region\n");
669 return (1);
670 }
671 }
672
673 /* See if we can prepend to the next entry. */
674 if (insert_idx < physmap_idx && base + length == physmap[insert_idx]) {
675 physmap[insert_idx] = base;
676 return (1);
677 }
678
679 /* See if we can append to the previous entry. */
680 if (insert_idx > 0 && base == physmap[insert_idx - 1]) {
681 physmap[insert_idx - 1] += length;
682 return (1);
683 }
684
685 if (physmap_idx == PHYS_AVAIL_ENTRIES) {
686 printf(
687 "Too many segments in the physical address map, giving up\n");
688 return (0);
689 }
690
691 /*
692 * Move the last 'N' entries down to make room for the new
693 * entry if needed.
694 */
695 for (i = physmap_idx; i > insert_idx; i -= 2) {
696 physmap[i] = physmap[i - 2];
697 physmap[i + 1] = physmap[i - 1];
698 }
699
700 physmap_idx += 2;
701 *physmap_idxp = physmap_idx;
702
703 /* Insert the new entry. */
704 physmap[insert_idx] = base;
705 physmap[insert_idx + 1] = base + length;
706 return (1);
707 }
708
709 void
bios_add_smap_entries(struct bios_smap * smapbase,u_int32_t smapsize,vm_paddr_t * physmap,int * physmap_idx)710 bios_add_smap_entries(struct bios_smap *smapbase, u_int32_t smapsize,
711 vm_paddr_t *physmap, int *physmap_idx)
712 {
713 struct bios_smap *smap, *smapend;
714
715 smapend = (struct bios_smap *)((uintptr_t)smapbase + smapsize);
716
717 for (smap = smapbase; smap < smapend; smap++) {
718 if (boothowto & RB_VERBOSE)
719 printf("SMAP type=%02x base=%016lx len=%016lx\n",
720 smap->type, smap->base, smap->length);
721
722 if (smap->type != SMAP_TYPE_MEMORY)
723 continue;
724
725 if (!add_physmap_entry(smap->base, smap->length, physmap,
726 physmap_idx))
727 break;
728 }
729 }
730
731 static void
add_efi_map_entries(struct efi_map_header * efihdr,vm_paddr_t * physmap,int * physmap_idx)732 add_efi_map_entries(struct efi_map_header *efihdr, vm_paddr_t *physmap,
733 int *physmap_idx)
734 {
735 struct efi_md *map, *p;
736 const char *type;
737 size_t efisz;
738 int ndesc, i;
739
740 static const char *types[] = {
741 "Reserved",
742 "LoaderCode",
743 "LoaderData",
744 "BootServicesCode",
745 "BootServicesData",
746 "RuntimeServicesCode",
747 "RuntimeServicesData",
748 "ConventionalMemory",
749 "UnusableMemory",
750 "ACPIReclaimMemory",
751 "ACPIMemoryNVS",
752 "MemoryMappedIO",
753 "MemoryMappedIOPortSpace",
754 "PalCode",
755 "PersistentMemory"
756 };
757
758 /*
759 * Memory map data provided by UEFI via the GetMemoryMap
760 * Boot Services API.
761 */
762 efisz = (sizeof(struct efi_map_header) + 0xf) & ~0xf;
763 map = (struct efi_md *)((uint8_t *)efihdr + efisz);
764
765 if (efihdr->descriptor_size == 0)
766 return;
767 ndesc = efihdr->memory_size / efihdr->descriptor_size;
768
769 if (boothowto & RB_VERBOSE)
770 printf("%23s %12s %12s %8s %4s\n",
771 "Type", "Physical", "Virtual", "#Pages", "Attr");
772
773 TUNABLE_INT_FETCH("machdep.efirt.regs", &efi_map_regs);
774 for (i = 0, p = map; i < ndesc; i++,
775 p = efi_next_descriptor(p, efihdr->descriptor_size)) {
776 if (boothowto & RB_VERBOSE) {
777 if (p->md_type < nitems(types))
778 type = types[p->md_type];
779 else
780 type = "<INVALID>";
781 printf("%23s %012lx %012lx %08lx ", type, p->md_phys,
782 p->md_virt, p->md_pages);
783 if (p->md_attr & EFI_MD_ATTR_UC)
784 printf("UC ");
785 if (p->md_attr & EFI_MD_ATTR_WC)
786 printf("WC ");
787 if (p->md_attr & EFI_MD_ATTR_WT)
788 printf("WT ");
789 if (p->md_attr & EFI_MD_ATTR_WB)
790 printf("WB ");
791 if (p->md_attr & EFI_MD_ATTR_UCE)
792 printf("UCE ");
793 if (p->md_attr & EFI_MD_ATTR_WP)
794 printf("WP ");
795 if (p->md_attr & EFI_MD_ATTR_RP)
796 printf("RP ");
797 if (p->md_attr & EFI_MD_ATTR_XP)
798 printf("XP ");
799 if (p->md_attr & EFI_MD_ATTR_NV)
800 printf("NV ");
801 if (p->md_attr & EFI_MD_ATTR_MORE_RELIABLE)
802 printf("MORE_RELIABLE ");
803 if (p->md_attr & EFI_MD_ATTR_RO)
804 printf("RO ");
805 if (p->md_attr & EFI_MD_ATTR_RT)
806 printf("RUNTIME");
807 printf("\n");
808 }
809
810 switch (p->md_type) {
811 case EFI_MD_TYPE_BS_CODE:
812 case EFI_MD_TYPE_BS_DATA:
813 if (EFI_MAP_BOOTTYPE_ALLOWED(p->md_type))
814 continue;
815 /* FALLTHROUGH */
816 case EFI_MD_TYPE_CODE:
817 case EFI_MD_TYPE_DATA:
818 case EFI_MD_TYPE_FREE:
819 /*
820 * We're allowed to use any entry with these types.
821 */
822 break;
823 default:
824 continue;
825 }
826
827 if (!add_physmap_entry(p->md_phys, p->md_pages * EFI_PAGE_SIZE,
828 physmap, physmap_idx))
829 break;
830 }
831 }
832
833 static void
native_parse_memmap(vm_paddr_t * physmap,int * physmap_idx)834 native_parse_memmap(vm_paddr_t *physmap, int *physmap_idx)
835 {
836 struct bios_smap *smap;
837 struct efi_map_header *efihdr;
838
839 efihdr = (struct efi_map_header *)preload_search_info(preload_kmdp,
840 MODINFO_METADATA | MODINFOMD_EFI_MAP);
841 smap = (struct bios_smap *)preload_search_info(preload_kmdp,
842 MODINFO_METADATA | MODINFOMD_SMAP);
843 if (efihdr == NULL && smap == NULL)
844 panic("No BIOS smap or EFI map info from loader!");
845
846 if (efihdr != NULL) {
847 add_efi_map_entries(efihdr, physmap, physmap_idx);
848 strlcpy(bootmethod, "UEFI", sizeof(bootmethod));
849 } else {
850 /*
851 * Memory map from INT 15:E820.
852 *
853 * subr_module.c says:
854 * "Consumer may safely assume that size value precedes data."
855 * ie: an int32_t immediately precedes smap.
856 */
857 u_int32_t size = *((u_int32_t *)smap - 1);
858
859 bios_add_smap_entries(smap, size, physmap, physmap_idx);
860 strlcpy(bootmethod, "BIOS", sizeof(bootmethod));
861 }
862 }
863
864 #define PAGES_PER_GB (1024 * 1024 * 1024 / PAGE_SIZE)
865
866 /*
867 * Populate the (physmap) array with base/bound pairs describing the
868 * available physical memory in the system, then test this memory and
869 * build the phys_avail array describing the actually-available memory.
870 *
871 * Total memory size may be set by the kernel environment variable
872 * hw.physmem or the compile-time define MAXMEM.
873 *
874 * XXX first should be vm_paddr_t.
875 */
876 static void
getmemsize(u_int64_t first)877 getmemsize(u_int64_t first)
878 {
879 int i, physmap_idx, pa_indx, da_indx;
880 vm_paddr_t pa, physmap[PHYS_AVAIL_ENTRIES];
881 u_long physmem_start, physmem_tunable, memtest;
882 pt_entry_t *pte;
883 quad_t dcons_addr, dcons_size;
884 int page_counter;
885
886 TSENTER();
887 /*
888 * Tell the physical memory allocator about pages used to store
889 * the kernel and preloaded data. See kmem_bootstrap_free().
890 */
891 vm_phys_early_add_seg((vm_paddr_t)kernphys, trunc_page(first));
892
893 bzero(physmap, sizeof(physmap));
894 physmap_idx = 0;
895
896 init_ops.parse_memmap(physmap, &physmap_idx);
897 physmap_idx -= 2;
898
899 /*
900 * Find the 'base memory' segment for SMP
901 */
902 basemem = 0;
903 for (i = 0; i <= physmap_idx; i += 2) {
904 if (physmap[i] <= 0xA0000) {
905 basemem = physmap[i + 1] / 1024;
906 break;
907 }
908 }
909 if (basemem == 0 || basemem > 640) {
910 if (bootverbose)
911 printf(
912 "Memory map doesn't contain a basemem segment, faking it");
913 basemem = 640;
914 }
915
916 /*
917 * Maxmem isn't the "maximum memory", it's one larger than the
918 * highest page of the physical address space. It should be
919 * called something like "Maxphyspage". We may adjust this
920 * based on ``hw.physmem'' and the results of the memory test.
921 */
922 Maxmem = atop(physmap[physmap_idx + 1]);
923
924 #ifdef MAXMEM
925 Maxmem = MAXMEM / 4;
926 #endif
927
928 if (TUNABLE_ULONG_FETCH("hw.physmem", &physmem_tunable))
929 Maxmem = atop(physmem_tunable);
930
931 /*
932 * The boot memory test is disabled by default, as it takes a
933 * significant amount of time on large-memory systems, and is
934 * unfriendly to virtual machines as it unnecessarily touches all
935 * pages.
936 *
937 * A general name is used as the code may be extended to support
938 * additional tests beyond the current "page present" test.
939 */
940 memtest = 0;
941 TUNABLE_ULONG_FETCH("hw.memtest.tests", &memtest);
942
943 /*
944 * Don't allow MAXMEM or hw.physmem to extend the amount of memory
945 * in the system.
946 */
947 if (Maxmem > atop(physmap[physmap_idx + 1]))
948 Maxmem = atop(physmap[physmap_idx + 1]);
949
950 if (atop(physmap[physmap_idx + 1]) != Maxmem &&
951 (boothowto & RB_VERBOSE))
952 printf("Physical memory use set to %ldK\n", Maxmem * 4);
953
954 /* call pmap initialization to make new kernel address space */
955 pmap_bootstrap(&first);
956
957 /*
958 * Size up each available chunk of physical memory.
959 *
960 * XXX Some BIOSes corrupt low 64KB between suspend and resume.
961 * By default, mask off the first 16 pages unless we appear to be
962 * running in a VM.
963 */
964 physmem_start = (vm_guest > VM_GUEST_NO ? 1 : 16) << PAGE_SHIFT;
965 TUNABLE_ULONG_FETCH("hw.physmem.start", &physmem_start);
966 if (physmap[0] < physmem_start) {
967 if (physmem_start < PAGE_SIZE)
968 physmap[0] = PAGE_SIZE;
969 else if (physmem_start >= physmap[1])
970 physmap[0] = round_page(physmap[1] - PAGE_SIZE);
971 else
972 physmap[0] = round_page(physmem_start);
973 }
974 pa_indx = 0;
975 da_indx = 1;
976 phys_avail[pa_indx++] = physmap[0];
977 phys_avail[pa_indx] = physmap[0];
978 dump_avail[da_indx] = physmap[0];
979 pte = CMAP1;
980
981 /*
982 * Get dcons buffer address
983 */
984 if (getenv_quad("dcons.addr", &dcons_addr) == 0 ||
985 getenv_quad("dcons.size", &dcons_size) == 0)
986 dcons_addr = 0;
987
988 /*
989 * physmap is in bytes, so when converting to page boundaries,
990 * round up the start address and round down the end address.
991 */
992 page_counter = 0;
993 if (memtest != 0)
994 printf("Testing system memory");
995 for (i = 0; i <= physmap_idx; i += 2) {
996 vm_paddr_t end;
997
998 end = ptoa((vm_paddr_t)Maxmem);
999 if (physmap[i + 1] < end)
1000 end = trunc_page(physmap[i + 1]);
1001 for (pa = round_page(physmap[i]); pa < end; pa += PAGE_SIZE) {
1002 int *ptr = (int *)CADDR1;
1003 int tmp;
1004 bool full, page_bad;
1005
1006 full = false;
1007 /*
1008 * block out kernel memory as not available.
1009 */
1010 if (pa >= (vm_paddr_t)kernphys && pa < first)
1011 goto do_dump_avail;
1012
1013 /*
1014 * block out dcons buffer
1015 */
1016 if (dcons_addr > 0
1017 && pa >= trunc_page(dcons_addr)
1018 && pa < dcons_addr + dcons_size)
1019 goto do_dump_avail;
1020
1021 page_bad = false;
1022 if (memtest == 0)
1023 goto skip_memtest;
1024
1025 /*
1026 * Print a "." every GB to show we're making
1027 * progress.
1028 */
1029 page_counter++;
1030 if ((page_counter % PAGES_PER_GB) == 0)
1031 printf(".");
1032
1033 /*
1034 * map page into kernel: valid, read/write,non-cacheable
1035 */
1036 *pte = pa | PG_V | PG_RW | PG_NC_PWT | PG_NC_PCD;
1037 invltlb();
1038
1039 tmp = *(int *)ptr;
1040 /*
1041 * Test for alternating 1's and 0's
1042 */
1043 *(volatile int *)ptr = 0xaaaaaaaa;
1044 if (*(volatile int *)ptr != 0xaaaaaaaa)
1045 page_bad = true;
1046 /*
1047 * Test for alternating 0's and 1's
1048 */
1049 *(volatile int *)ptr = 0x55555555;
1050 if (*(volatile int *)ptr != 0x55555555)
1051 page_bad = true;
1052 /*
1053 * Test for all 1's
1054 */
1055 *(volatile int *)ptr = 0xffffffff;
1056 if (*(volatile int *)ptr != 0xffffffff)
1057 page_bad = true;
1058 /*
1059 * Test for all 0's
1060 */
1061 *(volatile int *)ptr = 0x0;
1062 if (*(volatile int *)ptr != 0x0)
1063 page_bad = true;
1064 /*
1065 * Restore original value.
1066 */
1067 *(int *)ptr = tmp;
1068
1069 skip_memtest:
1070 /*
1071 * Adjust array of valid/good pages.
1072 */
1073 if (page_bad == true)
1074 continue;
1075 /*
1076 * If this good page is a continuation of the
1077 * previous set of good pages, then just increase
1078 * the end pointer. Otherwise start a new chunk.
1079 * Note that "end" points one higher than end,
1080 * making the range >= start and < end.
1081 * If we're also doing a speculative memory
1082 * test and we at or past the end, bump up Maxmem
1083 * so that we keep going. The first bad page
1084 * will terminate the loop.
1085 */
1086 if (phys_avail[pa_indx] == pa) {
1087 phys_avail[pa_indx] += PAGE_SIZE;
1088 } else {
1089 pa_indx++;
1090 if (pa_indx == PHYS_AVAIL_ENTRIES) {
1091 printf(
1092 "Too many holes in the physical address space, giving up\n");
1093 pa_indx--;
1094 full = true;
1095 goto do_dump_avail;
1096 }
1097 phys_avail[pa_indx++] = pa; /* start */
1098 phys_avail[pa_indx] = pa + PAGE_SIZE; /* end */
1099 }
1100 physmem++;
1101 do_dump_avail:
1102 if (dump_avail[da_indx] == pa) {
1103 dump_avail[da_indx] += PAGE_SIZE;
1104 } else {
1105 da_indx++;
1106 if (da_indx == PHYS_AVAIL_ENTRIES) {
1107 da_indx--;
1108 goto do_next;
1109 }
1110 dump_avail[da_indx++] = pa; /* start */
1111 dump_avail[da_indx] = pa + PAGE_SIZE; /* end */
1112 }
1113 do_next:
1114 if (full)
1115 break;
1116 }
1117 }
1118 *pte = 0;
1119 invltlb();
1120 if (memtest != 0)
1121 printf("\n");
1122
1123 /*
1124 * XXX
1125 * The last chunk must contain at least one page plus the message
1126 * buffer to avoid complicating other code (message buffer address
1127 * calculation, etc.).
1128 */
1129 while (phys_avail[pa_indx - 1] + PAGE_SIZE +
1130 round_page(msgbufsize) >= phys_avail[pa_indx]) {
1131 physmem -= atop(phys_avail[pa_indx] - phys_avail[pa_indx - 1]);
1132 phys_avail[pa_indx--] = 0;
1133 phys_avail[pa_indx--] = 0;
1134 }
1135
1136 Maxmem = atop(phys_avail[pa_indx]);
1137
1138 /* Trim off space for the message buffer. */
1139 phys_avail[pa_indx] -= round_page(msgbufsize);
1140
1141 /* Map the message buffer. */
1142 msgbufp = PHYS_TO_DMAP(phys_avail[pa_indx]);
1143 TSEXIT();
1144 }
1145
1146 static void
native_parse_preload_data(u_int64_t modulep)1147 native_parse_preload_data(u_int64_t modulep)
1148 {
1149 char *envp;
1150 #ifdef DDB
1151 vm_offset_t ksym_start;
1152 vm_offset_t ksym_end;
1153 #endif
1154
1155 preload_metadata = (caddr_t)(uintptr_t)(modulep + KERNBASE);
1156 preload_bootstrap_relocate(KERNBASE);
1157 preload_initkmdp(true);
1158 boothowto = MD_FETCH(preload_kmdp, MODINFOMD_HOWTO, int);
1159 envp = MD_FETCH(preload_kmdp, MODINFOMD_ENVP, char *);
1160 if (envp != NULL)
1161 envp += KERNBASE;
1162 init_static_kenv(envp, 0);
1163 #ifdef DDB
1164 ksym_start = MD_FETCH(preload_kmdp, MODINFOMD_SSYM, uintptr_t);
1165 ksym_end = MD_FETCH(preload_kmdp, MODINFOMD_ESYM, uintptr_t);
1166 db_fetch_ksymtab(ksym_start, ksym_end, 0);
1167 #endif
1168 efi_systbl_phys = MD_FETCH(preload_kmdp, MODINFOMD_FW_HANDLE,
1169 vm_paddr_t);
1170 }
1171
1172 static void
native_clock_source_init(void)1173 native_clock_source_init(void)
1174 {
1175 i8254_init();
1176 }
1177
1178 static void
amd64_kdb_init(void)1179 amd64_kdb_init(void)
1180 {
1181 kdb_init();
1182 #ifdef KDB
1183 if (boothowto & RB_KDB)
1184 kdb_enter(KDB_WHY_BOOTFLAGS, "Boot flags requested debugger");
1185 #endif
1186 }
1187
1188 /* Set up the fast syscall stuff */
1189 void
amd64_conf_fast_syscall(void)1190 amd64_conf_fast_syscall(void)
1191 {
1192 uint64_t msr;
1193
1194 msr = rdmsr(MSR_EFER) | EFER_SCE;
1195 wrmsr(MSR_EFER, msr);
1196 if (!fred) {
1197 wrmsr(MSR_LSTAR, pti ? (u_int64_t)IDTVEC(fast_syscall_pti) :
1198 (u_int64_t)IDTVEC(fast_syscall));
1199 wrmsr(MSR_CSTAR, (u_int64_t)IDTVEC(fast_syscall32));
1200 }
1201 msr = ((u_int64_t)GSEL(GCODE_SEL, SEL_KPL) << 32) |
1202 ((u_int64_t)GSEL(GUCODE32_SEL, SEL_UPL) << 48);
1203 wrmsr(MSR_STAR, msr);
1204 wrmsr(MSR_SF_MASK, PSL_NT | PSL_T | PSL_I | PSL_C | PSL_D | PSL_AC);
1205 }
1206
1207 void
amd64_bsp_pcpu_init1(struct pcpu * pc)1208 amd64_bsp_pcpu_init1(struct pcpu *pc)
1209 {
1210 struct user_segment_descriptor *gdt;
1211
1212 PCPU_SET(prvspace, pc);
1213 gdt = *PCPU_PTR(gdt);
1214 PCPU_SET(curthread, &thread0);
1215 PCPU_SET(tssp, PCPU_PTR(common_tss));
1216 PCPU_SET(tss, (struct system_segment_descriptor *)&gdt[GPROC0_SEL]);
1217 PCPU_SET(ldt, (struct system_segment_descriptor *)&gdt[GUSERLDT_SEL]);
1218 PCPU_SET(fs32p, &gdt[GUFS32_SEL]);
1219 PCPU_SET(gs32p, &gdt[GUGS32_SEL]);
1220 PCPU_SET(ucr3_load_mask, PMAP_UCR3_NOMASK);
1221 PCPU_SET(smp_tlb_gen, 1);
1222 }
1223
1224 void
amd64_bsp_pcpu_init2(uint64_t rsp0)1225 amd64_bsp_pcpu_init2(uint64_t rsp0)
1226 {
1227
1228 PCPU_SET(rsp0, rsp0);
1229 PCPU_SET(pti_rsp0, STACKALIGN((vm_offset_t)PCPU_PTR(pti_stack) +
1230 PC_PTI_STACK_SZ * sizeof(uint64_t)));
1231 PCPU_SET(curpcb, thread0.td_pcb);
1232 }
1233
1234 void
amd64_bsp_ist_init(struct pcpu * pc)1235 amd64_bsp_ist_init(struct pcpu *pc)
1236 {
1237 struct nmi_pcpu *np;
1238 struct amd64tss *tssp;
1239
1240 tssp = &pc->pc_common_tss;
1241
1242 /* Doublefault stack space, runs on ist1 for IDT. */
1243 if (fred) {
1244 wrmsr(MSR_FRED_RSP2, (uint64_t)&dblfault_stack[
1245 sizeof(dblfault_stack)]);
1246 } else {
1247 np = ((struct nmi_pcpu *)&dblfault_stack[sizeof(
1248 dblfault_stack)]) - 1;
1249 np->np_pcpu = (register_t)pc;
1250 tssp->tss_ist1 = (long)np;
1251 }
1252
1253 /*
1254 * NMI stack.
1255 */
1256 if (fred) {
1257 wrmsr(MSR_FRED_RSP1, (uint64_t)&nmi0_stack[
1258 sizeof(nmi0_stack)]);
1259 } else {
1260 /*
1261 * Runs on ist2 for IDT. The pcpu pointer is stored
1262 * just above the start of the ist2 stack.
1263 */
1264 np = ((struct nmi_pcpu *)&nmi0_stack[sizeof(nmi0_stack)]) - 1;
1265 np->np_pcpu = (register_t)pc;
1266 tssp->tss_ist2 = (long)np;
1267 }
1268
1269 if (!fred) {
1270 /*
1271 * MC# stack for IDT, runs on ist3. The pcpu pointer
1272 * is stored just above the start of the ist3 stack.
1273 */
1274 np = ((struct nmi_pcpu *)&mce0_stack[sizeof(mce0_stack)]) - 1;
1275 np->np_pcpu = (register_t)pc;
1276 tssp->tss_ist3 = (long)np;
1277
1278 /*
1279 * DB# stack for IDT, runs on ist4.
1280 */
1281 np = ((struct nmi_pcpu *)&dbg0_stack[sizeof(dbg0_stack)]) - 1;
1282 np->np_pcpu = (register_t)pc;
1283 tssp->tss_ist4 = (long)np;
1284 }
1285 }
1286
1287 /*
1288 * Calculate the kernel load address by inspecting page table created by loader.
1289 * The assumptions:
1290 * - kernel is mapped at KERNBASE, backed by contiguous phys memory
1291 * aligned at 2M, below 4G (the latter is important for AP startup)
1292 * - there is a 2M hole at KERNBASE (KERNSTART = KERNBASE + 2M)
1293 * - kernel is mapped with 2M superpages
1294 * - all participating memory, i.e. kernel, modules, metadata,
1295 * page table is accessible by pre-created 1:1 mapping
1296 * (right now loader creates 1:1 mapping for lower 4G, and all
1297 * memory is from there)
1298 * - there is a usable memory block right after the end of the
1299 * mapped kernel and all modules/metadata, pointed to by
1300 * physfree, for early allocations
1301 *
1302 * The memory block after the end of the kernel is important, loader
1303 * must ensure that no critical data structures are put there. Among
1304 * them is the trampoline page table, which must not be overwritten by
1305 * the allocations until pmap_bootstrap() switches %cr3 to the initial
1306 * version of the kernel page table. Size of the block is controlled
1307 * by the 'staging_slop' command for loader.efi.
1308 */
1309 vm_paddr_t __nosanitizeaddress __nosanitizememory
amd64_loadaddr(void)1310 amd64_loadaddr(void)
1311 {
1312 pml4_entry_t *pml4e;
1313 pdp_entry_t *pdpe;
1314 pd_entry_t *pde;
1315 uint64_t cr3;
1316
1317 cr3 = rcr3();
1318 pml4e = (pml4_entry_t *)cr3 + pmap_pml4e_index(KERNSTART);
1319 pdpe = (pdp_entry_t *)(*pml4e & PG_FRAME) + pmap_pdpe_index(KERNSTART);
1320 pde = (pd_entry_t *)(*pdpe & PG_FRAME) + pmap_pde_index(KERNSTART);
1321 return (*pde & PG_FRAME);
1322 }
1323
1324 u_int64_t
hammer_time(u_int64_t modulep,u_int64_t physfree)1325 hammer_time(u_int64_t modulep, u_int64_t physfree)
1326 {
1327 int gsel_tss, x;
1328 struct pcpu *pc;
1329 uint64_t rsp0;
1330 char *env;
1331 struct user_segment_descriptor *gdt;
1332 struct region_descriptor r_gdt;
1333 size_t kstack0_sz;
1334
1335 TSRAW(&thread0, TS_ENTER, __func__, NULL);
1336
1337 kernphys = amd64_loadaddr();
1338
1339 physfree += kernphys;
1340
1341 /* Initializes preload_kmdp */
1342 init_ops.parse_preload_data(modulep);
1343
1344 efi_boot = preload_search_info(preload_kmdp, MODINFO_METADATA |
1345 MODINFOMD_EFI_MAP) != NULL;
1346
1347 if (!efi_boot) {
1348 /* Tell the bios to warmboot next time */
1349 atomic_store_short((u_short *)0x472, 0x1234);
1350 }
1351
1352 physfree += ucode_load_bsp(physfree - kernphys + KERNSTART);
1353 physfree = roundup2(physfree, PAGE_SIZE);
1354
1355 identify_cpu1();
1356 identify_hypervisor();
1357 identify_hypervisor_smbios();
1358 identify_cpu_fixup_bsp();
1359 identify_cpu2();
1360 initializecpucache();
1361
1362 /*
1363 * Check for pti, pcid, and invpcid before ifuncs are
1364 * resolved, to correctly select the implementation for
1365 * pmap_activate_sw_mode().
1366 */
1367 pti = pti_get_default();
1368 TUNABLE_INT_FETCH("vm.pmap.pti", &pti);
1369 TUNABLE_INT_FETCH("vm.pmap.pcid_enabled", &pmap_pcid_enabled);
1370 if ((cpu_feature2 & CPUID2_PCID) == 0)
1371 pmap_pcid_enabled = 0;
1372 invpcid_works = (cpu_stdext_feature & CPUID_STDEXT_INVPCID) != 0;
1373
1374 /*
1375 * Now we can do small core initialization, after the PCID
1376 * CPU features and user knobs are evaluated.
1377 */
1378 TUNABLE_INT_FETCH("vm.pmap.pcid_invlpg_workaround",
1379 &pmap_pcid_invlpg_workaround_uena);
1380 cpu_init_small_core();
1381
1382 if ((cpu_feature2 & CPUID2_XSAVE) != 0) {
1383 use_xsave = 1;
1384 TUNABLE_INT_FETCH("hw.use_xsave", &use_xsave);
1385 }
1386
1387 if ((cpu_stdext_feature4 & (CPUID_STDEXT4_FRED | CPUID_STDEXT4_LKGS)) ==
1388 (CPUID_STDEXT4_FRED | CPUID_STDEXT4_LKGS) &&
1389 (cpu_stdext_feature & CPUID_STDEXT_FSGSBASE) != 0 && !pti) {
1390 fred = 1;
1391 TUNABLE_INT_FETCH("hw.fred", &fred);
1392 }
1393
1394 sched_instance_select();
1395
1396 link_elf_ireloc();
1397
1398 /*
1399 * This may be done better later if it gets more high level
1400 * components in it. If so just link td->td_proc here.
1401 */
1402 proc_linkup0(&proc0, &thread0);
1403
1404 /* Init basic tunables, hz etc */
1405 init_param1();
1406
1407 thread0.td_kstack = (char *)physfree - kernphys + KERNSTART;
1408 thread0.td_kstack_pages = kstack_pages;
1409 kstack0_sz = ptoa(kstack_pages);
1410 bzero(thread0.td_kstack, kstack0_sz);
1411 cpu_thread_new_kstack(&thread0);
1412 physfree += kstack0_sz;
1413
1414 /*
1415 * Initialize enough of thread0 for delayed invalidation to
1416 * work very early. Rely on thread0.td_base_pri
1417 * zero-initialization, it is reset to PVM at proc0_init().
1418 */
1419 pmap_thread_init_invl_gen(&thread0);
1420
1421 pc = &temp_bsp_pcpu;
1422 pcpu_init(pc, 0, sizeof(struct pcpu));
1423 gdt = &temp_bsp_pcpu.pc_gdt[0];
1424
1425 /*
1426 * make gdt memory segments
1427 */
1428 for (x = 0; x < NGDT; x++) {
1429 if (x != GPROC0_SEL && x != (GPROC0_SEL + 1) &&
1430 x != GUSERLDT_SEL && x != (GUSERLDT_SEL + 1))
1431 ssdtosd(&gdt_segs[x], &gdt[x]);
1432 }
1433 gdt_segs[GPROC0_SEL].ssd_base = (uintptr_t)&pc->pc_common_tss;
1434 ssdtosyssd(&gdt_segs[GPROC0_SEL],
1435 (struct system_segment_descriptor *)&gdt[GPROC0_SEL]);
1436
1437 r_gdt.rd_limit = NGDT * sizeof(gdt[0]) - 1;
1438 r_gdt.rd_base = (long)gdt;
1439 lgdt(&r_gdt);
1440
1441 wrmsr(MSR_FSBASE, 0); /* User value */
1442 wrmsr(MSR_GSBASE, (u_int64_t)pc);
1443 wrmsr(MSR_KGSBASE, 0); /* User value while in the kernel */
1444
1445 dpcpu_init((void *)(physfree - kernphys + KERNSTART), 0);
1446 physfree += DPCPU_SIZE;
1447 amd64_bsp_pcpu_init1(pc);
1448 /* Non-late cninit() and printf() can be moved up to here. */
1449
1450 /*
1451 * Initialize mutexes.
1452 *
1453 * icu_lock: in order to allow an interrupt to occur in a critical
1454 * section, to set pcpu->ipending (etc...) properly, we
1455 * must be able to get the icu lock, so it can't be
1456 * under witness.
1457 */
1458 mutex_init();
1459 mtx_init(&icu_lock, "icu", NULL, MTX_SPIN | MTX_NOWITNESS);
1460 mtx_init(&dt_lock, "descriptor tables", NULL, MTX_DEF);
1461
1462 /* exceptions */
1463 for (x = 0; x < NIDT; x++)
1464 setidt(x, pti ? &IDTVEC(rsvd_pti) : &IDTVEC(rsvd), SDT_SYSIGT,
1465 SEL_KPL, 0);
1466 setidt(IDT_DE, pti ? &IDTVEC(div_pti) : &IDTVEC(div), SDT_SYSIGT,
1467 SEL_KPL, 0);
1468 setidt(IDT_DB, &IDTVEC(dbg), SDT_SYSIGT, SEL_KPL, 4);
1469 setidt(IDT_NMI, &IDTVEC(nmi), SDT_SYSIGT, SEL_KPL, 2);
1470 setidt(IDT_BP, pti ? &IDTVEC(bpt_pti) : &IDTVEC(bpt), SDT_SYSIGT,
1471 SEL_UPL, 0);
1472 setidt(IDT_OF, pti ? &IDTVEC(ofl_pti) : &IDTVEC(ofl), SDT_SYSIGT,
1473 SEL_UPL, 0);
1474 setidt(IDT_BR, pti ? &IDTVEC(bnd_pti) : &IDTVEC(bnd), SDT_SYSIGT,
1475 SEL_KPL, 0);
1476 setidt(IDT_UD, pti ? &IDTVEC(ill_pti) : &IDTVEC(ill), SDT_SYSIGT,
1477 SEL_KPL, 0);
1478 setidt(IDT_NM, pti ? &IDTVEC(dna_pti) : &IDTVEC(dna), SDT_SYSIGT,
1479 SEL_KPL, 0);
1480 setidt(IDT_DF, &IDTVEC(dblfault), SDT_SYSIGT, SEL_KPL, 1);
1481 setidt(IDT_FPUGP, pti ? &IDTVEC(fpusegm_pti) : &IDTVEC(fpusegm),
1482 SDT_SYSIGT, SEL_KPL, 0);
1483 setidt(IDT_TS, pti ? &IDTVEC(tss_pti) : &IDTVEC(tss), SDT_SYSIGT,
1484 SEL_KPL, 0);
1485 setidt(IDT_NP, pti ? &IDTVEC(missing_pti) : &IDTVEC(missing),
1486 SDT_SYSIGT, SEL_KPL, 0);
1487 setidt(IDT_SS, pti ? &IDTVEC(stk_pti) : &IDTVEC(stk), SDT_SYSIGT,
1488 SEL_KPL, 0);
1489 setidt(IDT_GP, pti ? &IDTVEC(prot_pti) : &IDTVEC(prot), SDT_SYSIGT,
1490 SEL_KPL, 0);
1491 setidt(IDT_PF, pti ? &IDTVEC(page_pti) : &IDTVEC(page), SDT_SYSIGT,
1492 SEL_KPL, 0);
1493 setidt(IDT_MF, pti ? &IDTVEC(fpu_pti) : &IDTVEC(fpu), SDT_SYSIGT,
1494 SEL_KPL, 0);
1495 setidt(IDT_AC, pti ? &IDTVEC(align_pti) : &IDTVEC(align), SDT_SYSIGT,
1496 SEL_KPL, 0);
1497 setidt(IDT_MC, &IDTVEC(mchk), SDT_SYSIGT, SEL_KPL, 3);
1498 setidt(IDT_XF, pti ? &IDTVEC(xmm_pti) : &IDTVEC(xmm), SDT_SYSIGT,
1499 SEL_KPL, 0);
1500 #ifdef KDTRACE_HOOKS
1501 setidt(IDT_DTRACE_RET, pti ? &IDTVEC(dtrace_ret_pti) :
1502 &IDTVEC(dtrace_ret), SDT_SYSIGT, SEL_UPL, 0);
1503 #endif
1504 #ifdef XENHVM
1505 setidt(IDT_EVTCHN, pti ? &IDTVEC(xen_intr_upcall_pti) :
1506 &IDTVEC(xen_intr_upcall), SDT_SYSIGT, SEL_KPL, 0);
1507 #endif
1508 if (!fred) {
1509 r_idt.rd_limit = sizeof(idt0) - 1;
1510 r_idt.rd_base = (long) idt;
1511 lidt(&r_idt);
1512 }
1513
1514 TUNABLE_INT_FETCH("hw.ibrs_disable", &hw_ibrs_disable);
1515 TUNABLE_INT_FETCH("machdep.mitigations.ibrs.disable", &hw_ibrs_disable);
1516
1517 TUNABLE_INT_FETCH("hw.spec_store_bypass_disable", &hw_ssb_disable);
1518 TUNABLE_INT_FETCH("machdep.mitigations.ssb.disable", &hw_ssb_disable);
1519
1520 TUNABLE_INT_FETCH("machdep.syscall_ret_flush_l1d",
1521 &syscall_ret_l1d_flush_mode);
1522
1523 TUNABLE_INT_FETCH("hw.mds_disable", &hw_mds_disable);
1524 TUNABLE_INT_FETCH("machdep.mitigations.mds.disable", &hw_mds_disable);
1525
1526 TUNABLE_INT_FETCH("machdep.mitigations.taa.enable", &x86_taa_enable);
1527
1528 TUNABLE_INT_FETCH("machdep.mitigations.rngds.enable",
1529 &x86_rngds_mitg_enable);
1530
1531 TUNABLE_INT_FETCH("machdep.mitigations.zenbleed.enable",
1532 &zenbleed_enable);
1533 zenbleed_sanitize_enable();
1534
1535 finishidentcpu(); /* Final stage of CPU initialization */
1536
1537 invlpgb_works = (amd_extended_feature_extensions &
1538 AMDFEID_INVLPGB) != 0;
1539 TUNABLE_INT_FETCH("vm.pmap.invlpgb_works", &invlpgb_works);
1540 if (invlpgb_works)
1541 invlpgb_maxcnt = cpu_procinfo3 & AMDID_INVLPGB_MAXCNT;
1542
1543 /*
1544 * Initialize the clock before the console so that console
1545 * initialization can use DELAY().
1546 */
1547 clock_init();
1548
1549 initializecpu(); /* Initialize CPU registers */
1550
1551 amd64_bsp_ist_init(pc);
1552
1553 /* Set the IO permission bitmap (empty due to tss seg limit) */
1554 pc->pc_common_tss.tss_iobase = sizeof(struct amd64tss) +
1555 IOPERM_BITMAP_SIZE;
1556
1557 gsel_tss = GSEL(GPROC0_SEL, SEL_KPL);
1558 ltr(gsel_tss);
1559
1560 amd64_conf_fast_syscall();
1561
1562 /*
1563 * We initialize the PCB pointer early so that exception
1564 * handlers will work.
1565 */
1566 thread0.td_pcb = get_pcb_td(&thread0);
1567
1568 /*
1569 * The console and kdb should be initialized even earlier than here,
1570 * but some console drivers don't work until after getmemsize().
1571 * Default to late console initialization to support these drivers.
1572 * This loses mainly printf()s in getmemsize() and early debugging.
1573 */
1574 TUNABLE_INT_FETCH("debug.late_console", &late_console);
1575 if (!late_console) {
1576 cninit();
1577 amd64_kdb_init();
1578 }
1579
1580 getmemsize(physfree);
1581 init_param2(physmem);
1582
1583 /* now running on new page tables, configured,and u/iom is accessible */
1584
1585 #ifdef DEV_PCI
1586 /* This call might adjust phys_avail[]. */
1587 pci_early_quirks();
1588 #endif
1589
1590 if (late_console)
1591 cninit();
1592
1593 /*
1594 * Dump the boot metadata. We have to wait for cninit() since console
1595 * output is required. If it's grossly incorrect the kernel will never
1596 * make it this far.
1597 */
1598 if (getenv_is_true("debug.dump_modinfo_at_boot"))
1599 preload_dump();
1600
1601 if (fred)
1602 amd64_cpu_init_fred();
1603
1604 #ifdef DEV_ISA
1605 #ifdef DEV_ATPIC
1606 elcr_probe();
1607 atpic_startup();
1608 #else
1609 /* Reset and mask the atpics and leave them shut down. */
1610 atpic_reset();
1611
1612 /*
1613 * Point the ICU spurious interrupt vectors at the APIC spurious
1614 * interrupt handler.
1615 */
1616 setidt(IDT_IO_INTS + 7, IDTVEC(spuriousint), SDT_SYSIGT, SEL_KPL, 0);
1617 setidt(IDT_IO_INTS + 15, IDTVEC(spuriousint), SDT_SYSIGT, SEL_KPL, 0);
1618 #endif
1619 #else
1620 #error "have you forgotten the isa device?"
1621 #endif
1622
1623 if (late_console)
1624 amd64_kdb_init();
1625
1626 msgbufinit(msgbufp, msgbufsize);
1627 fpuinit();
1628
1629 /* make an initial tss so cpu can get interrupt stack on syscall! */
1630 rsp0 = (uintptr_t)thread0.td_md.md_stack_base;
1631 /* Ensure the stack is aligned to 16 bytes */
1632 rsp0 = STACKALIGN(rsp0);
1633 PCPU_PTR(common_tss)->tss_rsp0 = rsp0;
1634 amd64_bsp_pcpu_init2(rsp0);
1635
1636 /* transfer to user mode */
1637
1638 _ucodesel = GSEL(GUCODE_SEL, SEL_UPL);
1639 _udatasel = GSEL(GUDATA_SEL, SEL_UPL);
1640 _ucode32sel = GSEL(GUCODE32_SEL, SEL_UPL);
1641 _ufssel = GSEL(GUFS32_SEL, SEL_UPL);
1642 _ugssel = GSEL(GUGS32_SEL, SEL_UPL);
1643
1644 load_ds(_udatasel);
1645 load_es(_udatasel);
1646 load_fs(_ufssel);
1647
1648 /* setup proc 0's pcb */
1649 thread0.td_pcb->pcb_flags = 0;
1650
1651 env = kern_getenv("kernelname");
1652 if (env != NULL)
1653 strlcpy(kernelname, env, sizeof(kernelname));
1654
1655 kcsan_cpu_init(0);
1656
1657 #ifdef FDT
1658 x86_init_fdt();
1659 #endif
1660
1661 kasan_init();
1662 kmsan_init();
1663
1664 TSEXIT();
1665
1666 /* Location of kernel stack for locore */
1667 return ((uintptr_t)thread0.td_md.md_stack_base);
1668 }
1669
1670 void
cpu_pcpu_init(struct pcpu * pcpu,int cpuid,size_t size)1671 cpu_pcpu_init(struct pcpu *pcpu, int cpuid, size_t size)
1672 {
1673
1674 pcpu->pc_acpi_id = 0xffffffff;
1675 }
1676
1677 static int
smap_sysctl_handler(SYSCTL_HANDLER_ARGS)1678 smap_sysctl_handler(SYSCTL_HANDLER_ARGS)
1679 {
1680 struct bios_smap *smapbase;
1681 struct bios_smap_xattr smap;
1682 uint32_t *smapattr;
1683 int count, error, i;
1684
1685 /* Retrieve the system memory map from the loader. */
1686 smapbase = (struct bios_smap *)preload_search_info(preload_kmdp,
1687 MODINFO_METADATA | MODINFOMD_SMAP);
1688 if (smapbase == NULL)
1689 return (0);
1690 smapattr = (uint32_t *)preload_search_info(preload_kmdp,
1691 MODINFO_METADATA | MODINFOMD_SMAP_XATTR);
1692 count = *((uint32_t *)smapbase - 1) / sizeof(*smapbase);
1693 error = 0;
1694 for (i = 0; i < count; i++) {
1695 smap.base = smapbase[i].base;
1696 smap.length = smapbase[i].length;
1697 smap.type = smapbase[i].type;
1698 if (smapattr != NULL)
1699 smap.xattr = smapattr[i];
1700 else
1701 smap.xattr = 0;
1702 error = SYSCTL_OUT(req, &smap, sizeof(smap));
1703 }
1704 return (error);
1705 }
1706 SYSCTL_PROC(_machdep, OID_AUTO, smap,
1707 CTLTYPE_OPAQUE | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0,
1708 smap_sysctl_handler, "S,bios_smap_xattr",
1709 "Raw BIOS SMAP data");
1710
1711 static int
efi_map_sysctl_handler(SYSCTL_HANDLER_ARGS)1712 efi_map_sysctl_handler(SYSCTL_HANDLER_ARGS)
1713 {
1714 struct efi_map_header *efihdr;
1715 uint32_t efisize;
1716
1717 efihdr = (struct efi_map_header *)preload_search_info(preload_kmdp,
1718 MODINFO_METADATA | MODINFOMD_EFI_MAP);
1719 if (efihdr == NULL)
1720 return (0);
1721 efisize = *((uint32_t *)efihdr - 1);
1722 return (SYSCTL_OUT(req, efihdr, efisize));
1723 }
1724 SYSCTL_PROC(_machdep, OID_AUTO, efi_map,
1725 CTLTYPE_OPAQUE | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0,
1726 efi_map_sysctl_handler, "S,efi_map_header",
1727 "Raw EFI Memory Map");
1728
1729 static int
efi_arch_sysctl_handler(SYSCTL_HANDLER_ARGS)1730 efi_arch_sysctl_handler(SYSCTL_HANDLER_ARGS)
1731 {
1732 char *arch;
1733
1734 arch = (char *)preload_search_info(preload_kmdp,
1735 MODINFO_METADATA | MODINFOMD_EFI_ARCH);
1736 if (arch == NULL)
1737 return (0);
1738
1739 return (SYSCTL_OUT_STR(req, arch));
1740 }
1741 SYSCTL_PROC(_machdep, OID_AUTO, efi_arch,
1742 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0,
1743 efi_arch_sysctl_handler, "A", "EFI Firmware Architecture");
1744
1745 void
spinlock_enter(void)1746 spinlock_enter(void)
1747 {
1748 struct thread *td;
1749 register_t flags;
1750
1751 td = curthread;
1752 if (td->td_md.md_spinlock_count == 0) {
1753 flags = intr_disable();
1754 td->td_md.md_spinlock_count = 1;
1755 td->td_md.md_saved_flags = flags;
1756 critical_enter();
1757 } else
1758 td->td_md.md_spinlock_count++;
1759 }
1760
1761 void
spinlock_exit(void)1762 spinlock_exit(void)
1763 {
1764 struct thread *td;
1765 register_t flags;
1766
1767 td = curthread;
1768 flags = td->td_md.md_saved_flags;
1769 td->td_md.md_spinlock_count--;
1770 if (td->td_md.md_spinlock_count == 0) {
1771 critical_exit();
1772 intr_restore(flags);
1773 }
1774 }
1775
1776 /*
1777 * Construct a PCB from a trapframe. This is called from kdb_trap() where
1778 * we want to start a backtrace from the function that caused us to enter
1779 * the debugger. We have the context in the trapframe, but base the trace
1780 * on the PCB. The PCB doesn't have to be perfect, as long as it contains
1781 * enough for a backtrace.
1782 */
1783 void
makectx(struct trapframe * tf,struct pcb * pcb)1784 makectx(struct trapframe *tf, struct pcb *pcb)
1785 {
1786
1787 pcb->pcb_r12 = tf->tf_r12;
1788 pcb->pcb_r13 = tf->tf_r13;
1789 pcb->pcb_r14 = tf->tf_r14;
1790 pcb->pcb_r15 = tf->tf_r15;
1791 pcb->pcb_rbp = tf->tf_rbp;
1792 pcb->pcb_rbx = tf->tf_rbx;
1793 pcb->pcb_rip = tf->tf_rip;
1794 pcb->pcb_rsp = tf->tf_rsp;
1795 }
1796
1797 /*
1798 * The pcb_flags is only modified by current thread, or by other threads
1799 * when current thread is stopped. However, current thread may change it
1800 * from the interrupt context in cpu_switch(), or in the trap handler.
1801 * When we read-modify-write pcb_flags from C sources, compiler may generate
1802 * code that is not atomic regarding the interrupt handler. If a trap or
1803 * interrupt happens and any flag is modified from the handler, it can be
1804 * clobbered with the cached value later. Therefore, we implement setting
1805 * and clearing flags with single-instruction functions, which do not race
1806 * with possible modification of the flags from the trap or interrupt context,
1807 * because traps and interrupts are executed only on instruction boundary.
1808 */
1809 void
set_pcb_flags_raw(struct pcb * pcb,const u_int flags)1810 set_pcb_flags_raw(struct pcb *pcb, const u_int flags)
1811 {
1812
1813 __asm __volatile("orl %1,%0"
1814 : "=m" (pcb->pcb_flags) : "ir" (flags), "m" (pcb->pcb_flags)
1815 : "cc", "memory");
1816
1817 }
1818
1819 /*
1820 * The support for RDFSBASE, WRFSBASE and similar instructions for %gs
1821 * base requires that kernel saves MSR_FSBASE and MSR_{K,}GSBASE into
1822 * pcb if user space modified the bases. We must save on the context
1823 * switch or if the return to usermode happens through the doreti.
1824 *
1825 * Tracking of both events is performed by the pcb flag PCB_FULL_IRET,
1826 * which have a consequence that the base MSRs must be saved each time
1827 * the PCB_FULL_IRET flag is set. We disable interrupts to sync with
1828 * context switches.
1829 */
1830 static void
set_pcb_flags_fsgsbase(struct pcb * pcb,const u_int flags)1831 set_pcb_flags_fsgsbase(struct pcb *pcb, const u_int flags)
1832 {
1833 register_t r;
1834
1835 if (curpcb == pcb &&
1836 (flags & PCB_FULL_IRET) != 0 &&
1837 (pcb->pcb_flags & PCB_FULL_IRET) == 0) {
1838 r = intr_disable();
1839 if ((pcb->pcb_flags & PCB_FULL_IRET) == 0) {
1840 pcb->pcb_fsbase = rdfsbase();
1841 pcb->pcb_gsbase = rdmsr(MSR_KGSBASE);
1842 }
1843 set_pcb_flags_raw(pcb, flags);
1844 intr_restore(r);
1845 } else {
1846 set_pcb_flags_raw(pcb, flags);
1847 }
1848 }
1849
1850 DEFINE_IFUNC(, void, set_pcb_flags, (struct pcb *, const u_int))
1851 {
1852
1853 return ((cpu_stdext_feature & CPUID_STDEXT_FSGSBASE) != 0 ?
1854 set_pcb_flags_fsgsbase : set_pcb_flags_raw);
1855 }
1856
1857 void
clear_pcb_flags(struct pcb * pcb,const u_int flags)1858 clear_pcb_flags(struct pcb *pcb, const u_int flags)
1859 {
1860
1861 __asm __volatile("andl %1,%0"
1862 : "=m" (pcb->pcb_flags) : "ir" (~flags), "m" (pcb->pcb_flags)
1863 : "cc", "memory");
1864 }
1865
1866 extern const char wrmsr_early_safe_gp_handler[];
1867
1868 /*
1869 * What about FRED? wrmsr_early_safe_start() is used before we
1870 * switched CPU to the FRED mode. We use IDT to catch #GP from MSR
1871 * write even if BSP is switched to the FRED mode later.
1872 */
1873 void
wrmsr_early_safe_start(void)1874 wrmsr_early_safe_start(void)
1875 {
1876 struct region_descriptor efi_idt;
1877 struct gate_descriptor *gpf_descr;
1878 int i;
1879
1880 efi_idt.rd_limit = 32 * sizeof(idt0[0]);
1881 efi_idt.rd_base = (uintptr_t)idt0;
1882 lidt(&efi_idt);
1883
1884 /* Setup handler for all possible exceptions. */
1885 for (i = 0; i < 32; i++) {
1886 gpf_descr = &idt0[i];
1887 gpf_descr->gd_looffset =
1888 (uintptr_t)wrmsr_early_safe_gp_handler;
1889 gpf_descr->gd_hioffset =
1890 (uintptr_t)wrmsr_early_safe_gp_handler >> 16;
1891 gpf_descr->gd_selector = rcs();
1892 gpf_descr->gd_type = SDT_SYSTGT;
1893 gpf_descr->gd_p = 1;
1894 }
1895 }
1896
1897 void
wrmsr_early_safe_end(void)1898 wrmsr_early_safe_end(void)
1899 {
1900 }
1901
1902 int
safe_read(vm_offset_t addr,char * valp)1903 safe_read(vm_offset_t addr, char *valp)
1904 {
1905 struct uio uio;
1906 struct iovec iov;
1907
1908 iov.iov_base = valp;
1909 iov.iov_len = 1;
1910 uio.uio_offset = addr;
1911 uio.uio_iov = &iov;
1912 uio.uio_iovcnt = 1;
1913 uio.uio_resid = 1;
1914 uio.uio_segflg = UIO_SYSSPACE;
1915 uio.uio_rw = UIO_READ;
1916 uio.uio_td = NULL;
1917 return (uiomove_mem(UIO_MEM_KMEM, &uio));
1918 }
1919
1920 #ifdef KDB
1921
1922 /*
1923 * Provide inb() and outb() as functions. They are normally only available as
1924 * inline functions, thus cannot be called from the debugger.
1925 */
1926
1927 /* silence compiler warnings */
1928 u_char inb_(u_short);
1929 void outb_(u_short, u_char);
1930
1931 u_char
inb_(u_short port)1932 inb_(u_short port)
1933 {
1934 return inb(port);
1935 }
1936
1937 void
outb_(u_short port,u_char data)1938 outb_(u_short port, u_char data)
1939 {
1940 outb(port, data);
1941 }
1942
1943 #endif /* KDB */
1944
1945 #undef memset
1946 #undef memmove
1947 #undef memcpy
1948
1949 void *memset_std(void *buf, int c, size_t len);
1950 void *memset_erms(void *buf, int c, size_t len);
1951 void *memmove_std(void * _Nonnull dst, const void * _Nonnull src,
1952 size_t len);
1953 void *memmove_erms(void * _Nonnull dst, const void * _Nonnull src,
1954 size_t len);
1955 void *memcpy_std(void * _Nonnull dst, const void * _Nonnull src,
1956 size_t len);
1957 void *memcpy_erms(void * _Nonnull dst, const void * _Nonnull src,
1958 size_t len);
1959
1960 #ifdef KCSAN
1961 /*
1962 * These fail to build as ifuncs when used with KCSAN.
1963 */
1964 void *
memset(void * buf,int c,size_t len)1965 memset(void *buf, int c, size_t len)
1966 {
1967
1968 return (memset_std(buf, c, len));
1969 }
1970
1971 void *
memmove(void * _Nonnull dst,const void * _Nonnull src,size_t len)1972 memmove(void * _Nonnull dst, const void * _Nonnull src, size_t len)
1973 {
1974
1975 return (memmove_std(dst, src, len));
1976 }
1977
1978 void *
memcpy(void * _Nonnull dst,const void * _Nonnull src,size_t len)1979 memcpy(void * _Nonnull dst, const void * _Nonnull src, size_t len)
1980 {
1981
1982 return (memcpy_std(dst, src, len));
1983 }
1984 #else
1985 DEFINE_IFUNC(, void *, memset, (void *, int, size_t))
1986 {
1987
1988 return ((cpu_stdext_feature & CPUID_STDEXT_ERMS) != 0 ?
1989 memset_erms : memset_std);
1990 }
1991
1992 DEFINE_IFUNC(, void *, memmove, (void * _Nonnull, const void * _Nonnull,
1993 size_t))
1994 {
1995
1996 return ((cpu_stdext_feature & CPUID_STDEXT_ERMS) != 0 ?
1997 memmove_erms : memmove_std);
1998 }
1999
2000 DEFINE_IFUNC(, void *, memcpy, (void * _Nonnull, const void * _Nonnull,size_t))
2001 {
2002
2003 return ((cpu_stdext_feature & CPUID_STDEXT_ERMS) != 0 ?
2004 memcpy_erms : memcpy_std);
2005 }
2006 #endif
2007
2008 void pagezero_std(void *addr);
2009 void pagezero_erms(void *addr);
2010 DEFINE_IFUNC(, void , pagezero, (void *))
2011 {
2012
2013 return ((cpu_stdext_feature & CPUID_STDEXT_ERMS) != 0 ?
2014 pagezero_erms : pagezero_std);
2015 }
2016