1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 1996, by Steve Passe
5 * Copyright (c) 2003, by Peter Wemm
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. The name of the developer may NOT be used to endorse or promote products
14 * derived from this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 */
28
29 #include <sys/cdefs.h>
30 #include "opt_acpi.h"
31 #include "opt_cpu.h"
32 #include "opt_ddb.h"
33 #include "opt_kstack_pages.h"
34 #include "opt_sched.h"
35 #include "opt_smp.h"
36
37 #include <sys/param.h>
38 #include <sys/systm.h>
39 #include <sys/bus.h>
40 #include <sys/cpuset.h>
41 #include <sys/domainset.h>
42 #include <sys/kdb.h>
43 #include <sys/kernel.h>
44 #include <sys/ktr.h>
45 #include <sys/lock.h>
46 #include <sys/malloc.h>
47 #include <sys/memrange.h>
48 #include <sys/mutex.h>
49 #include <sys/pcpu.h>
50 #include <sys/proc.h>
51 #include <sys/sched.h>
52 #include <sys/smp.h>
53 #include <sys/sysctl.h>
54
55 #include <vm/vm.h>
56 #include <vm/vm_param.h>
57 #include <vm/pmap.h>
58 #include <vm/vm_kern.h>
59 #include <vm/vm_extern.h>
60 #include <vm/vm_page.h>
61 #include <vm/vm_phys.h>
62
63 #include <x86/apicreg.h>
64 #include <machine/clock.h>
65 #include <machine/cputypes.h>
66 #include <machine/cpufunc.h>
67 #include <x86/mca.h>
68 #include <machine/md_var.h>
69 #include <machine/pcb.h>
70 #include <machine/psl.h>
71 #include <machine/smp.h>
72 #include <machine/specialreg.h>
73 #include <machine/tss.h>
74 #include <x86/ucode.h>
75 #include <machine/cpu.h>
76 #include <x86/init.h>
77
78 #ifdef DEV_ACPI
79 #include <contrib/dev/acpica/include/acpi.h>
80 #include <dev/acpica/acpivar.h>
81 #endif
82
83 #define WARMBOOT_TARGET 0
84 #define WARMBOOT_OFF (KERNBASE + 0x0467)
85 #define WARMBOOT_SEG (KERNBASE + 0x0469)
86
87 #define CMOS_REG (0x70)
88 #define CMOS_DATA (0x71)
89 #define BIOS_RESET (0x0f)
90 #define BIOS_WARM (0x0a)
91
92 #define GiB(v) (v ## ULL << 30)
93
94 #define AP_BOOTPT_SZ (PAGE_SIZE * 4)
95
96 /* Temporary variables for init_secondary() */
97 static char *doublefault_stack;
98 static char *mce_stack;
99 static char *nmi_stack;
100 static char *dbg_stack;
101 void *bootpcpu;
102
103 extern u_int mptramp_la57;
104 extern u_int mptramp_nx;
105 smp_targeted_tlb_shootdown_t smp_targeted_tlb_shootdown =
106 &smp_targeted_tlb_shootdown_native;
107
108 /*
109 * Local data and functions.
110 */
111
112 static int start_ap(int apic_id, vm_paddr_t boot_address);
113 static int start_all_aps(void);
114
115 /*
116 * Initialize the IPI handlers and start up the AP's.
117 */
118 void
cpu_mp_start(void)119 cpu_mp_start(void)
120 {
121 int i;
122
123 /* Initialize the logical ID to APIC ID table. */
124 for (i = 0; i < MAXCPU; i++) {
125 cpu_apic_ids[i] = -1;
126 }
127
128 /* Install an inter-CPU IPI for cache and TLB invalidations. */
129 setidt(IPI_INVLOP, pti ? IDTVEC(invlop_pti) : IDTVEC(invlop),
130 SDT_SYSIGT, SEL_KPL, 0);
131
132 /* Install an inter-CPU IPI for all-CPU rendezvous */
133 setidt(IPI_RENDEZVOUS, pti ? IDTVEC(rendezvous_pti) :
134 IDTVEC(rendezvous), SDT_SYSIGT, SEL_KPL, 0);
135
136 /* Install generic inter-CPU IPI handler */
137 setidt(IPI_BITMAP_VECTOR, pti ? IDTVEC(ipi_intr_bitmap_handler_pti) :
138 IDTVEC(ipi_intr_bitmap_handler), SDT_SYSIGT, SEL_KPL, 0);
139
140 /* Install an inter-CPU IPI for CPU stop/restart */
141 setidt(IPI_STOP, pti ? IDTVEC(cpustop_pti) : IDTVEC(cpustop),
142 SDT_SYSIGT, SEL_KPL, 0);
143
144 /* Install an inter-CPU IPI for CPU offline */
145 setidt(IPI_OFF, pti ? IDTVEC(cpuoff_pti) : IDTVEC(cpuoff),
146 SDT_SYSIGT, SEL_KPL, 0);
147
148 /* Install an inter-CPU IPI for CPU suspend/resume */
149 setidt(IPI_SUSPEND, pti ? IDTVEC(cpususpend_pti) : IDTVEC(cpususpend),
150 SDT_SYSIGT, SEL_KPL, 0);
151
152 /* Install an IPI for calling delayed SWI */
153 setidt(IPI_SWI, pti ? IDTVEC(ipi_swi_pti) : IDTVEC(ipi_swi),
154 SDT_SYSIGT, SEL_KPL, 0);
155
156 /* Set boot_cpu_id if needed. */
157 if (boot_cpu_id == -1) {
158 boot_cpu_id = PCPU_GET(apic_id);
159 cpu_info[boot_cpu_id].cpu_bsp = 1;
160 } else
161 KASSERT(boot_cpu_id == PCPU_GET(apic_id),
162 ("BSP's APIC ID doesn't match boot_cpu_id"));
163
164 /* Probe logical/physical core configuration. */
165 topo_probe();
166
167 assign_cpu_ids();
168
169 mptramp_la57 = la57;
170 mptramp_nx = pg_nx != 0;
171 MPASS(kernel_pmap->pm_cr3 < (1UL << 32));
172 mptramp_pagetables = kernel_pmap->pm_cr3;
173
174 /* Start each Application Processor */
175 start_all_aps();
176
177 set_interrupt_apic_ids();
178
179 #if defined(DEV_ACPI) && MAXMEMDOM > 1
180 acpi_pxm_set_cpu_locality();
181 #endif
182 }
183
184 void
cpu_mp_stop(void)185 cpu_mp_stop(void)
186 {
187 cpuset_t other_cpus = all_cpus;
188
189 CPU_CLR(PCPU_GET(cpuid), &other_cpus);
190 offline_cpus(other_cpus);
191 }
192
193 /*
194 * AP CPU's call this to initialize themselves.
195 */
196 void
init_secondary(void)197 init_secondary(void)
198 {
199 struct pcpu *pc;
200 struct nmi_pcpu *np;
201 struct user_segment_descriptor *gdt;
202 struct region_descriptor ap_gdt;
203 u_int64_t cr0;
204 int cpu, gsel_tss, x;
205
206 /* Set by the startup code for us to use */
207 cpu = bootAP;
208
209 /* Update microcode before doing anything else. */
210 ucode_load_ap(cpu);
211
212 /* Initialize the PCPU area. */
213 pc = bootpcpu;
214 pcpu_init(pc, cpu, sizeof(struct pcpu));
215 dpcpu_init(dpcpu, cpu);
216 pc->pc_apic_id = cpu_apic_ids[cpu];
217 pc->pc_prvspace = pc;
218 pc->pc_curthread = 0;
219 pc->pc_tssp = &pc->pc_common_tss;
220 pc->pc_rsp0 = 0;
221 pc->pc_pti_rsp0 = STACKALIGN(((vm_offset_t)&pc->pc_pti_stack +
222 PC_PTI_STACK_SZ * sizeof(uint64_t)));
223 gdt = pc->pc_gdt;
224 pc->pc_tss = (struct system_segment_descriptor *)&gdt[GPROC0_SEL];
225 pc->pc_fs32p = &gdt[GUFS32_SEL];
226 pc->pc_gs32p = &gdt[GUGS32_SEL];
227 pc->pc_ldt = (struct system_segment_descriptor *)&gdt[GUSERLDT_SEL];
228 pc->pc_ucr3_load_mask = PMAP_UCR3_NOMASK;
229 /* See comment in pmap_bootstrap(). */
230 pc->pc_pcid_next = PMAP_PCID_KERN + 2;
231 pc->pc_pcid_gen = 1;
232 pc->pc_kpmap_store.pm_pcid = PMAP_PCID_KERN;
233 pc->pc_kpmap_store.pm_gen = 1;
234
235 pc->pc_smp_tlb_gen = 1;
236
237 /* Init tss */
238 pc->pc_common_tss = __pcpu[0].pc_common_tss;
239 pc->pc_common_tss.tss_iobase = sizeof(struct amd64tss) +
240 IOPERM_BITMAP_SIZE;
241 pc->pc_common_tss.tss_rsp0 = 0;
242
243 /*
244 * The doublefault stack.
245 * Runs on IST1 for IDT.
246 * Uses CSL 2 for FRED.
247 */
248 if (fred) {
249 wrmsr(MSR_FRED_RSP2, (uint64_t)&doublefault_stack[
250 DBLFAULT_STACK_SIZE]);
251 } else {
252 np = ((struct nmi_pcpu *)&doublefault_stack[
253 DBLFAULT_STACK_SIZE]) - 1;
254 np->np_pcpu = (register_t)pc;
255 pc->pc_common_tss.tss_ist1 = (long)np;
256 }
257
258 /*
259 * The NMI stack.
260 * Runs on IST2 for IDT.
261 * Uses CSL 1 for FRED.
262 */
263 if (fred) {
264 wrmsr(MSR_FRED_RSP1, (uint64_t)&nmi_stack[NMI_STACK_SIZE]);
265 } else {
266 np = ((struct nmi_pcpu *)&nmi_stack[NMI_STACK_SIZE]) - 1;
267 np->np_pcpu = (register_t)pc;
268 pc->pc_common_tss.tss_ist2 = (long)np;
269 }
270
271 /*
272 * The MC# stack.
273 * Runs on IST3 for IDT.
274 * Shares CSL 1 with NMI for FRED.
275 */
276 if (!fred) {
277 np = ((struct nmi_pcpu *)&mce_stack[MCE_STACK_SIZE]) - 1;
278 np->np_pcpu = (register_t)pc;
279 pc->pc_common_tss.tss_ist3 = (long)np;
280 }
281
282 if (!fred) {
283 /* The DB# stack, used for for IDT, runs on IST4. */
284 np = ((struct nmi_pcpu *)&dbg_stack[DBG_STACK_SIZE]) - 1;
285 np->np_pcpu = (register_t)pc;
286 pc->pc_common_tss.tss_ist4 = (long)np;
287 }
288
289 /* Prepare private GDT */
290 gdt_segs[GPROC0_SEL].ssd_base = (long)&pc->pc_common_tss;
291 for (x = 0; x < NGDT; x++) {
292 if (x != GPROC0_SEL && x != GPROC0_SEL + 1 &&
293 x != GUSERLDT_SEL && x != GUSERLDT_SEL + 1)
294 ssdtosd(&gdt_segs[x], &gdt[x]);
295 }
296 ssdtosyssd(&gdt_segs[GPROC0_SEL],
297 (struct system_segment_descriptor *)&gdt[GPROC0_SEL]);
298 ap_gdt.rd_limit = NGDT * sizeof(gdt[0]) - 1;
299 ap_gdt.rd_base = (u_long)gdt;
300 lgdt(&ap_gdt); /* does magic intra-segment return */
301
302 wrmsr(MSR_FSBASE, 0); /* User value */
303 wrmsr(MSR_GSBASE, (uint64_t)pc);
304 wrmsr(MSR_KGSBASE, 0); /* User value */
305 fix_cpuid();
306
307 if (fred)
308 amd64_cpu_init_fred();
309 else
310 lidt(&r_idt);
311
312 gsel_tss = GSEL(GPROC0_SEL, SEL_KPL);
313 ltr(gsel_tss);
314
315 /*
316 * Set to a known state:
317 * Set by mpboot.s: CR0_PG, CR0_PE
318 * Set by cpu_setregs: CR0_NE, CR0_MP, CR0_TS, CR0_WP, CR0_AM
319 */
320 cr0 = rcr0();
321 cr0 &= ~(CR0_CD | CR0_NW | CR0_EM);
322 load_cr0(cr0);
323
324 amd64_conf_fast_syscall();
325
326 /* signal our startup to the BSP. */
327 mp_naps++;
328
329 /* Spin until the BSP releases the AP's. */
330 while (atomic_load_acq_int(&aps_ready) == 0)
331 ia32_pause();
332
333 init_secondary_tail();
334 }
335
336 static void
amd64_mp_alloc_pcpu(void)337 amd64_mp_alloc_pcpu(void)
338 {
339 vm_page_t m;
340 int cpu;
341
342 /* Allocate pcpu areas to the correct domain. */
343 for (cpu = 1; cpu < mp_ncpus; cpu++) {
344 #ifdef NUMA
345 m = NULL;
346 if (vm_ndomains > 1) {
347 m = vm_page_alloc_noobj_domain(
348 acpi_pxm_get_cpu_locality(cpu_apic_ids[cpu]),
349 VM_ALLOC_ZERO);
350 }
351 if (m == NULL)
352 #endif
353 m = vm_page_alloc_noobj(VM_ALLOC_ZERO);
354 if (m == NULL)
355 panic("cannot alloc pcpu page for cpu %d", cpu);
356 pmap_qenter(&__pcpu[cpu], &m, 1);
357 }
358 }
359
360 /*
361 * start each AP in our list
362 */
363 static int
start_all_aps(void)364 start_all_aps(void)
365 {
366 vm_page_t m_boottramp, m_pml4, m_pdp, m_pd[4];
367 pml5_entry_t old_pml45;
368 pml4_entry_t *v_pml4;
369 pdp_entry_t *v_pdp;
370 pd_entry_t *v_pd;
371 vm_paddr_t boot_address;
372 u_int32_t mpbioswarmvec;
373 int apic_id, cpu, domain, i;
374 u_char mpbiosreason;
375
376 amd64_mp_alloc_pcpu();
377
378 MPASS(bootMP_size <= PAGE_SIZE);
379 m_boottramp = vm_page_alloc_noobj_contig(0, 1, 0,
380 (1ULL << 20), /* Trampoline should be below 1M for real mode */
381 PAGE_SIZE, 0, VM_MEMATTR_DEFAULT);
382 boot_address = VM_PAGE_TO_PHYS(m_boottramp);
383
384 /* Create a transient 1:1 mapping of low 4G */
385 if (la57) {
386 m_pml4 = pmap_page_alloc_below_4g(true);
387 v_pml4 = VM_PAGE_TO_DMAP(m_pml4);
388 } else {
389 v_pml4 = &kernel_pmap->pm_pmltop[0];
390 }
391 m_pdp = pmap_page_alloc_below_4g(true);
392 v_pdp = VM_PAGE_TO_DMAP(m_pdp);
393 m_pd[0] = pmap_page_alloc_below_4g(false);
394 v_pd = VM_PAGE_TO_DMAP(m_pd[0]);
395 for (i = 0; i < NPDEPG; i++)
396 v_pd[i] = (i << PDRSHIFT) | X86_PG_V | X86_PG_RW | X86_PG_A |
397 X86_PG_M | PG_PS;
398 m_pd[1] = pmap_page_alloc_below_4g(false);
399 v_pd = VM_PAGE_TO_DMAP(m_pd[1]);
400 for (i = 0; i < NPDEPG; i++)
401 v_pd[i] = (NBPDP + (i << PDRSHIFT)) | X86_PG_V | X86_PG_RW |
402 X86_PG_A | X86_PG_M | PG_PS;
403 m_pd[2] = pmap_page_alloc_below_4g(false);
404 v_pd = VM_PAGE_TO_DMAP(m_pd[2]);
405 for (i = 0; i < NPDEPG; i++)
406 v_pd[i] = (2UL * NBPDP + (i << PDRSHIFT)) | X86_PG_V |
407 X86_PG_RW | X86_PG_A | X86_PG_M | PG_PS;
408 m_pd[3] = pmap_page_alloc_below_4g(false);
409 v_pd = VM_PAGE_TO_DMAP(m_pd[3]);
410 for (i = 0; i < NPDEPG; i++)
411 v_pd[i] = (3UL * NBPDP + (i << PDRSHIFT)) | X86_PG_V |
412 X86_PG_RW | X86_PG_A | X86_PG_M | PG_PS;
413 v_pdp[0] = VM_PAGE_TO_PHYS(m_pd[0]) | X86_PG_V |
414 X86_PG_RW | X86_PG_A | X86_PG_M;
415 v_pdp[1] = VM_PAGE_TO_PHYS(m_pd[1]) | X86_PG_V |
416 X86_PG_RW | X86_PG_A | X86_PG_M;
417 v_pdp[2] = VM_PAGE_TO_PHYS(m_pd[2]) | X86_PG_V |
418 X86_PG_RW | X86_PG_A | X86_PG_M;
419 v_pdp[3] = VM_PAGE_TO_PHYS(m_pd[3]) | X86_PG_V |
420 X86_PG_RW | X86_PG_A | X86_PG_M;
421 old_pml45 = kernel_pmap->pm_pmltop[0];
422 if (la57) {
423 kernel_pmap->pm_pmltop[0] = VM_PAGE_TO_PHYS(m_pml4) |
424 X86_PG_V | X86_PG_RW | X86_PG_A | X86_PG_M;
425 }
426 v_pml4[0] = VM_PAGE_TO_PHYS(m_pdp) | X86_PG_V |
427 X86_PG_RW | X86_PG_A | X86_PG_M;
428 pmap_invalidate_all(kernel_pmap);
429
430 /* copy the AP 1st level boot code */
431 bcopy(mptramp_start, PHYS_TO_DMAP(boot_address), bootMP_size);
432 if (bootverbose)
433 printf("AP boot address %#lx\n", boot_address);
434
435 /* save the current value of the warm-start vector */
436 if (!efi_boot)
437 mpbioswarmvec = *((u_int32_t *) WARMBOOT_OFF);
438 outb(CMOS_REG, BIOS_RESET);
439 mpbiosreason = inb(CMOS_DATA);
440
441 /* setup a vector to our boot code */
442 if (!efi_boot) {
443 *((volatile u_short *)WARMBOOT_OFF) = WARMBOOT_TARGET;
444 *((volatile u_short *)WARMBOOT_SEG) = (boot_address >> 4);
445 }
446 outb(CMOS_REG, BIOS_RESET);
447 outb(CMOS_DATA, BIOS_WARM); /* 'warm-start' */
448
449 /* start each AP */
450 domain = 0;
451 for (cpu = 1; cpu < mp_ncpus; cpu++) {
452 apic_id = cpu_apic_ids[cpu];
453 #ifdef NUMA
454 if (vm_ndomains > 1)
455 domain = acpi_pxm_get_cpu_locality(apic_id);
456 #endif
457 /* allocate and set up an idle stack data page */
458 bootstacks[cpu] = kmem_malloc(kstack_pages * PAGE_SIZE,
459 M_WAITOK | M_ZERO);
460 doublefault_stack = kmem_malloc(DBLFAULT_STACK_SIZE,
461 M_WAITOK | M_ZERO);
462 if (!fred) {
463 mce_stack = kmem_malloc(MCE_STACK_SIZE,
464 M_WAITOK | M_ZERO);
465 }
466 nmi_stack = kmem_malloc_domainset(
467 DOMAINSET_PREF(domain), NMI_STACK_SIZE, M_WAITOK | M_ZERO);
468 if (!fred) {
469 dbg_stack = kmem_malloc_domainset(
470 DOMAINSET_PREF(domain), DBG_STACK_SIZE,
471 M_WAITOK | M_ZERO);
472 }
473 dpcpu = kmem_malloc_domainset(DOMAINSET_PREF(domain),
474 DPCPU_SIZE, M_WAITOK | M_ZERO);
475
476 bootpcpu = &__pcpu[cpu];
477 bootSTK = (char *)bootstacks[cpu] +
478 kstack_pages * PAGE_SIZE - 8;
479 bootAP = cpu;
480
481 /* attempt to start the Application Processor */
482 if (!start_ap(apic_id, boot_address)) {
483 /* restore the warmstart vector */
484 if (!efi_boot)
485 *(u_int32_t *)WARMBOOT_OFF = mpbioswarmvec;
486 panic("AP #%d (PHY# %d) failed!", cpu, apic_id);
487 }
488
489 CPU_SET(cpu, &all_cpus); /* record AP in CPU map */
490 }
491
492 /* restore the warmstart vector */
493 if (!efi_boot)
494 *(u_int32_t *)WARMBOOT_OFF = mpbioswarmvec;
495
496 outb(CMOS_REG, BIOS_RESET);
497 outb(CMOS_DATA, mpbiosreason);
498
499 /* Destroy transient 1:1 mapping */
500 kernel_pmap->pm_pmltop[0] = old_pml45;
501 invlpg(0);
502 if (la57)
503 vm_page_free(m_pml4);
504 vm_page_free(m_pd[3]);
505 vm_page_free(m_pd[2]);
506 vm_page_free(m_pd[1]);
507 vm_page_free(m_pd[0]);
508 vm_page_free(m_pdp);
509 vm_page_free(m_boottramp);
510
511 /* number of APs actually started */
512 return (mp_naps);
513 }
514
515 /*
516 * This function starts the AP (application processor) identified
517 * by the APIC ID 'physicalCpu'. It does quite a "song and dance"
518 * to accomplish this. This is necessary because of the nuances
519 * of the different hardware we might encounter. It isn't pretty,
520 * but it seems to work.
521 */
522 static int
start_ap(int apic_id,vm_paddr_t boot_address)523 start_ap(int apic_id, vm_paddr_t boot_address)
524 {
525 int vector, ms;
526 int cpus;
527
528 /* calculate the vector */
529 vector = (boot_address >> 12) & 0xff;
530
531 /* used as a watchpoint to signal AP startup */
532 cpus = mp_naps;
533
534 ipi_startup(apic_id, vector);
535
536 /* Wait up to 5 seconds for it to start. */
537 for (ms = 0; ms < 5000; ms++) {
538 if (mp_naps > cpus)
539 return 1; /* return SUCCESS */
540 DELAY(1000);
541 }
542 return 0; /* return FAILURE */
543 }
544
545 /*
546 * Flush the TLB on other CPU's
547 */
548
549 /*
550 * These variables are initialized at startup to reflect how each of
551 * the different kinds of invalidations should be performed on the
552 * current machine and environment.
553 */
554 static enum invl_op_codes invl_op_tlb;
555 static enum invl_op_codes invl_op_pgrng;
556 static enum invl_op_codes invl_op_pg;
557
558 /*
559 * Scoreboard of IPI completion notifications from target to IPI initiator.
560 *
561 * Each CPU can initiate shootdown IPI independently from other CPUs.
562 * Initiator enters critical section, then fills its local PCPU
563 * shootdown info (pc_smp_tlb_ vars), then clears scoreboard generation
564 * at location (cpu, my_cpuid) for each target cpu. After that IPI is
565 * sent to all targets which scan for zeroed scoreboard generation
566 * words. Upon finding such word the shootdown data is read from
567 * corresponding cpu's pcpu, and generation is set. Meantime initiator
568 * loops waiting for all zeroed generations in scoreboard to update.
569 */
570 static uint32_t *invl_scoreboard;
571
572 static void
invl_scoreboard_init(void * arg __unused)573 invl_scoreboard_init(void *arg __unused)
574 {
575 u_int i;
576
577 invl_scoreboard = malloc(sizeof(uint32_t) * (mp_maxid + 1) *
578 (mp_maxid + 1), M_DEVBUF, M_WAITOK);
579 for (i = 0; i < (mp_maxid + 1) * (mp_maxid + 1); i++)
580 invl_scoreboard[i] = 1;
581
582 if (pmap_pcid_enabled) {
583 if (invpcid_works) {
584 if (pti)
585 invl_op_tlb = INVL_OP_TLB_INVPCID_PTI;
586 else
587 invl_op_tlb = INVL_OP_TLB_INVPCID;
588 invl_op_pgrng = INVL_OP_PGRNG_INVPCID;
589 invl_op_pg = INVL_OP_PG_INVPCID;
590 } else {
591 invl_op_tlb = INVL_OP_TLB_PCID;
592 invl_op_pgrng = INVL_OP_PGRNG_PCID;
593 invl_op_pg = INVL_OP_PG_PCID;
594 }
595 } else {
596 invl_op_tlb = INVL_OP_TLB;
597 invl_op_pgrng = INVL_OP_PGRNG;
598 invl_op_pg = INVL_OP_PG;
599 }
600 }
601 SYSINIT(invl_ops, SI_SUB_SMP - 1, SI_ORDER_ANY, invl_scoreboard_init, NULL);
602
603 static uint32_t *
invl_scoreboard_getcpu(u_int cpu)604 invl_scoreboard_getcpu(u_int cpu)
605 {
606 return (invl_scoreboard + cpu * (mp_maxid + 1));
607 }
608
609 static uint32_t *
invl_scoreboard_slot(u_int cpu)610 invl_scoreboard_slot(u_int cpu)
611 {
612 return (invl_scoreboard_getcpu(cpu) + PCPU_GET(cpuid));
613 }
614
615 /*
616 * Used by the pmap to request cache or TLB invalidation on local and
617 * remote processors. Mask provides the set of remote CPUs that are
618 * to be signalled with the invalidation IPI. As an optimization, the
619 * curcpu_cb callback is invoked on the calling CPU in a critical
620 * section while waiting for the remote CPUs to complete the operation.
621 *
622 * The callback function is called unconditionally on the caller's
623 * underlying processor, even when this processor is not set in the
624 * mask. So, the callback function must be prepared to handle such
625 * spurious invocations.
626 *
627 * Interrupts must be enabled when calling the function with smp
628 * started, to avoid deadlock with other IPIs that are protected with
629 * smp_ipi_mtx spinlock at the initiator side.
630 *
631 * Function must be called with the thread pinned, and it unpins on
632 * completion.
633 */
634 void
smp_targeted_tlb_shootdown_native(pmap_t pmap,vm_offset_t addr1,vm_offset_t addr2,smp_invl_cb_t curcpu_cb,enum invl_op_codes op)635 smp_targeted_tlb_shootdown_native(pmap_t pmap, vm_offset_t addr1,
636 vm_offset_t addr2, smp_invl_cb_t curcpu_cb, enum invl_op_codes op)
637 {
638 cpuset_t mask;
639 uint32_t generation, *p_cpudone;
640 int cpu;
641 bool is_all;
642
643 /*
644 * It is not necessary to signal other CPUs while booting or
645 * when in the debugger.
646 */
647 if (__predict_false(kdb_active || KERNEL_PANICKED() || !smp_started))
648 goto local_cb;
649
650 KASSERT(curthread->td_pinned > 0, ("curthread not pinned"));
651
652 /*
653 * Make a stable copy of the set of CPUs on which the pmap is active.
654 * See if we have to interrupt other CPUs.
655 */
656 CPU_COPY(pmap_invalidate_cpu_mask(pmap), &mask);
657 is_all = CPU_CMP(&mask, &all_cpus) == 0;
658 CPU_CLR(curcpu, &mask);
659 if (CPU_EMPTY(&mask))
660 goto local_cb;
661
662 /*
663 * Initiator must have interrupts enabled, which prevents
664 * non-invalidation IPIs that take smp_ipi_mtx spinlock,
665 * from deadlocking with us. On the other hand, preemption
666 * must be disabled to pin initiator to the instance of the
667 * pcpu pc_smp_tlb data and scoreboard line.
668 */
669 KASSERT((read_rflags() & PSL_I) != 0,
670 ("smp_targeted_tlb_shootdown: interrupts disabled"));
671 critical_enter();
672
673 PCPU_SET(smp_tlb_addr1, addr1);
674 PCPU_SET(smp_tlb_addr2, addr2);
675 PCPU_SET(smp_tlb_pmap, pmap);
676 generation = PCPU_GET(smp_tlb_gen);
677 if (++generation == 0)
678 generation = 1;
679 PCPU_SET(smp_tlb_gen, generation);
680 PCPU_SET(smp_tlb_op, op);
681 /* Fence between filling smp_tlb fields and clearing scoreboard. */
682 atomic_thread_fence_rel();
683
684 CPU_FOREACH_ISSET(cpu, &mask) {
685 KASSERT(*invl_scoreboard_slot(cpu) != 0,
686 ("IPI scoreboard is zero, initiator %d target %d",
687 curcpu, cpu));
688 *invl_scoreboard_slot(cpu) = 0;
689 }
690
691 /*
692 * IPI acts as a fence between writing to the scoreboard above
693 * (zeroing slot) and reading from it below (wait for
694 * acknowledgment).
695 */
696 if (is_all) {
697 ipi_all_but_self(IPI_INVLOP);
698 } else {
699 ipi_selected(mask, IPI_INVLOP);
700 }
701 curcpu_cb(pmap, addr1, addr2);
702 CPU_FOREACH_ISSET(cpu, &mask) {
703 p_cpudone = invl_scoreboard_slot(cpu);
704 while (atomic_load_int(p_cpudone) != generation)
705 ia32_pause();
706 }
707
708 /*
709 * Unpin before leaving critical section. If the thread owes
710 * preemption, this allows scheduler to select thread on any
711 * CPU from its cpuset.
712 */
713 sched_unpin();
714 critical_exit();
715
716 return;
717
718 local_cb:
719 critical_enter();
720 curcpu_cb(pmap, addr1, addr2);
721 sched_unpin();
722 critical_exit();
723 }
724
725 void
smp_masked_invltlb(pmap_t pmap,smp_invl_cb_t curcpu_cb)726 smp_masked_invltlb(pmap_t pmap, smp_invl_cb_t curcpu_cb)
727 {
728 if (invlpgb_works && pmap == kernel_pmap) {
729 invlpgb(INVLPGB_GLOB, 0, 0);
730
731 /*
732 * TLBSYNC syncs only against INVLPGB executed on the
733 * same CPU. Since current thread is pinned by
734 * caller, we do not need to enter critical section to
735 * prevent migration.
736 */
737 tlbsync();
738 sched_unpin();
739 return;
740 }
741
742 smp_targeted_tlb_shootdown(pmap, 0, 0, curcpu_cb, invl_op_tlb);
743 #ifdef COUNT_XINVLTLB_HITS
744 ipi_global++;
745 #endif
746 }
747
748 void
smp_masked_invlpg(vm_offset_t addr,pmap_t pmap,smp_invl_cb_t curcpu_cb)749 smp_masked_invlpg(vm_offset_t addr, pmap_t pmap, smp_invl_cb_t curcpu_cb)
750 {
751 if (invlpgb_works && pmap == kernel_pmap) {
752 invlpgb(INVLPGB_GLOB | INVLPGB_VA | trunc_page(addr), 0, 0);
753 tlbsync();
754 sched_unpin();
755 return;
756 }
757
758 smp_targeted_tlb_shootdown(pmap, addr, 0, curcpu_cb, invl_op_pg);
759 #ifdef COUNT_XINVLTLB_HITS
760 ipi_page++;
761 #endif
762 }
763
764 void
smp_masked_invlpg_range(vm_offset_t addr1,vm_offset_t addr2,pmap_t pmap,smp_invl_cb_t curcpu_cb)765 smp_masked_invlpg_range(vm_offset_t addr1, vm_offset_t addr2, pmap_t pmap,
766 smp_invl_cb_t curcpu_cb)
767 {
768 if (invlpgb_works && pmap == kernel_pmap) {
769 vm_offset_t va;
770 uint64_t cnt, total;
771
772 addr1 = trunc_page(addr1);
773 addr2 = round_page(addr2);
774 total = atop(addr2 - addr1);
775 for (va = addr1; total > 0;) {
776 cnt = MIN(total, invlpgb_maxcnt + 1);
777 /* 4K increments because these may not be superpages. */
778 invlpgb(INVLPGB_GLOB | INVLPGB_VA | va, 0,
779 cnt - 1);
780 va += ptoa(cnt);
781 total -= cnt;
782 }
783 tlbsync();
784 sched_unpin();
785 return;
786 }
787
788 smp_targeted_tlb_shootdown(pmap, addr1, addr2, curcpu_cb,
789 invl_op_pgrng);
790 #ifdef COUNT_XINVLTLB_HITS
791 ipi_range++;
792 ipi_range_size += (addr2 - addr1) / PAGE_SIZE;
793 #endif
794 }
795
796 void
smp_cache_flush(smp_invl_cb_t curcpu_cb)797 smp_cache_flush(smp_invl_cb_t curcpu_cb)
798 {
799 smp_targeted_tlb_shootdown(kernel_pmap, 0, 0, curcpu_cb, INVL_OP_CACHE);
800 }
801
802 /*
803 * Handlers for TLB related IPIs
804 */
805 static void
invltlb_handler(pmap_t smp_tlb_pmap)806 invltlb_handler(pmap_t smp_tlb_pmap)
807 {
808 #ifdef COUNT_XINVLTLB_HITS
809 xhits_gbl[PCPU_GET(cpuid)]++;
810 #endif /* COUNT_XINVLTLB_HITS */
811 #ifdef COUNT_IPIS
812 (*ipi_invltlb_counts[PCPU_GET(cpuid)])++;
813 #endif /* COUNT_IPIS */
814
815 if (smp_tlb_pmap == kernel_pmap)
816 invltlb_glob();
817 else
818 invltlb();
819 }
820
821 static void
invltlb_invpcid_handler(pmap_t smp_tlb_pmap)822 invltlb_invpcid_handler(pmap_t smp_tlb_pmap)
823 {
824 struct invpcid_descr d;
825
826 #ifdef COUNT_XINVLTLB_HITS
827 xhits_gbl[PCPU_GET(cpuid)]++;
828 #endif /* COUNT_XINVLTLB_HITS */
829 #ifdef COUNT_IPIS
830 (*ipi_invltlb_counts[PCPU_GET(cpuid)])++;
831 #endif /* COUNT_IPIS */
832
833 d.pcid = pmap_get_pcid(smp_tlb_pmap);
834 d.pad = 0;
835 d.addr = 0;
836 invpcid(&d, smp_tlb_pmap == kernel_pmap ? INVPCID_CTXGLOB :
837 INVPCID_CTX);
838 }
839
840 static void
invltlb_invpcid_pti_handler(pmap_t smp_tlb_pmap)841 invltlb_invpcid_pti_handler(pmap_t smp_tlb_pmap)
842 {
843 struct invpcid_descr d;
844
845 #ifdef COUNT_XINVLTLB_HITS
846 xhits_gbl[PCPU_GET(cpuid)]++;
847 #endif /* COUNT_XINVLTLB_HITS */
848 #ifdef COUNT_IPIS
849 (*ipi_invltlb_counts[PCPU_GET(cpuid)])++;
850 #endif /* COUNT_IPIS */
851
852 d.pcid = pmap_get_pcid(smp_tlb_pmap);
853 d.pad = 0;
854 d.addr = 0;
855 if (smp_tlb_pmap == kernel_pmap) {
856 /*
857 * This invalidation actually needs to clear kernel
858 * mappings from the TLB in the current pmap, but
859 * since we were asked for the flush in the kernel
860 * pmap, achieve it by performing global flush.
861 */
862 invpcid(&d, INVPCID_CTXGLOB);
863 } else {
864 invpcid(&d, INVPCID_CTX);
865 if (smp_tlb_pmap == PCPU_GET(curpmap) &&
866 smp_tlb_pmap->pm_ucr3 != PMAP_NO_CR3)
867 PCPU_SET(ucr3_load_mask, ~CR3_PCID_SAVE);
868 }
869 }
870
871 static void
invltlb_pcid_handler(pmap_t smp_tlb_pmap)872 invltlb_pcid_handler(pmap_t smp_tlb_pmap)
873 {
874 #ifdef COUNT_XINVLTLB_HITS
875 xhits_gbl[PCPU_GET(cpuid)]++;
876 #endif /* COUNT_XINVLTLB_HITS */
877 #ifdef COUNT_IPIS
878 (*ipi_invltlb_counts[PCPU_GET(cpuid)])++;
879 #endif /* COUNT_IPIS */
880
881 if (smp_tlb_pmap == kernel_pmap) {
882 invltlb_glob();
883 } else {
884 /*
885 * The current pmap might not be equal to
886 * smp_tlb_pmap. The clearing of the pm_gen in
887 * pmap_invalidate_all() takes care of TLB
888 * invalidation when switching to the pmap on this
889 * CPU.
890 */
891 if (smp_tlb_pmap == PCPU_GET(curpmap)) {
892 load_cr3(smp_tlb_pmap->pm_cr3 |
893 pmap_get_pcid(smp_tlb_pmap));
894 if (smp_tlb_pmap->pm_ucr3 != PMAP_NO_CR3)
895 PCPU_SET(ucr3_load_mask, ~CR3_PCID_SAVE);
896 }
897 }
898 }
899
900 static void
invlpg_handler(vm_offset_t smp_tlb_addr1)901 invlpg_handler(vm_offset_t smp_tlb_addr1)
902 {
903 #ifdef COUNT_XINVLTLB_HITS
904 xhits_pg[PCPU_GET(cpuid)]++;
905 #endif /* COUNT_XINVLTLB_HITS */
906 #ifdef COUNT_IPIS
907 (*ipi_invlpg_counts[PCPU_GET(cpuid)])++;
908 #endif /* COUNT_IPIS */
909
910 invlpg(smp_tlb_addr1);
911 }
912
913 static void
invlpg_invpcid_handler(pmap_t smp_tlb_pmap,vm_offset_t smp_tlb_addr1)914 invlpg_invpcid_handler(pmap_t smp_tlb_pmap, vm_offset_t smp_tlb_addr1)
915 {
916 struct invpcid_descr d;
917
918 #ifdef COUNT_XINVLTLB_HITS
919 xhits_pg[PCPU_GET(cpuid)]++;
920 #endif /* COUNT_XINVLTLB_HITS */
921 #ifdef COUNT_IPIS
922 (*ipi_invlpg_counts[PCPU_GET(cpuid)])++;
923 #endif /* COUNT_IPIS */
924
925 pmap_invlpg(smp_tlb_pmap, smp_tlb_addr1);
926 if (smp_tlb_pmap == PCPU_GET(curpmap) &&
927 smp_tlb_pmap->pm_ucr3 != PMAP_NO_CR3 &&
928 PCPU_GET(ucr3_load_mask) == PMAP_UCR3_NOMASK) {
929 d.pcid = pmap_get_pcid(smp_tlb_pmap) | PMAP_PCID_USER_PT;
930 d.pad = 0;
931 d.addr = smp_tlb_addr1;
932 invpcid(&d, INVPCID_ADDR);
933 }
934 }
935
936 static void
invlpg_pcid_handler(pmap_t smp_tlb_pmap,vm_offset_t smp_tlb_addr1)937 invlpg_pcid_handler(pmap_t smp_tlb_pmap, vm_offset_t smp_tlb_addr1)
938 {
939 uint64_t kcr3, ucr3;
940 uint32_t pcid;
941
942 #ifdef COUNT_XINVLTLB_HITS
943 xhits_pg[PCPU_GET(cpuid)]++;
944 #endif /* COUNT_XINVLTLB_HITS */
945 #ifdef COUNT_IPIS
946 (*ipi_invlpg_counts[PCPU_GET(cpuid)])++;
947 #endif /* COUNT_IPIS */
948
949 invlpg(smp_tlb_addr1);
950 if (smp_tlb_pmap == PCPU_GET(curpmap) &&
951 (ucr3 = smp_tlb_pmap->pm_ucr3) != PMAP_NO_CR3 &&
952 PCPU_GET(ucr3_load_mask) == PMAP_UCR3_NOMASK) {
953 pcid = pmap_get_pcid(smp_tlb_pmap);
954 kcr3 = smp_tlb_pmap->pm_cr3 | pcid | CR3_PCID_SAVE;
955 ucr3 |= pcid | PMAP_PCID_USER_PT | CR3_PCID_SAVE;
956 pmap_pti_pcid_invlpg(ucr3, kcr3, smp_tlb_addr1);
957 }
958 }
959
960 static void
invlrng_handler(vm_offset_t smp_tlb_addr1,vm_offset_t smp_tlb_addr2)961 invlrng_handler(vm_offset_t smp_tlb_addr1, vm_offset_t smp_tlb_addr2)
962 {
963 vm_offset_t addr;
964
965 #ifdef COUNT_XINVLTLB_HITS
966 xhits_rng[PCPU_GET(cpuid)]++;
967 #endif /* COUNT_XINVLTLB_HITS */
968 #ifdef COUNT_IPIS
969 (*ipi_invlrng_counts[PCPU_GET(cpuid)])++;
970 #endif /* COUNT_IPIS */
971
972 addr = smp_tlb_addr1;
973 do {
974 invlpg(addr);
975 addr += PAGE_SIZE;
976 } while (addr < smp_tlb_addr2);
977 }
978
979 static void
invlrng_invpcid_handler(pmap_t smp_tlb_pmap,vm_offset_t smp_tlb_addr1,vm_offset_t smp_tlb_addr2)980 invlrng_invpcid_handler(pmap_t smp_tlb_pmap, vm_offset_t smp_tlb_addr1,
981 vm_offset_t smp_tlb_addr2)
982 {
983 struct invpcid_descr d;
984 vm_offset_t addr;
985
986 #ifdef COUNT_XINVLTLB_HITS
987 xhits_rng[PCPU_GET(cpuid)]++;
988 #endif /* COUNT_XINVLTLB_HITS */
989 #ifdef COUNT_IPIS
990 (*ipi_invlrng_counts[PCPU_GET(cpuid)])++;
991 #endif /* COUNT_IPIS */
992
993 addr = smp_tlb_addr1;
994 if (smp_tlb_pmap == kernel_pmap && PCPU_GET(pcid_invlpg_workaround)) {
995 struct invpcid_descr d = { 0 };
996
997 invpcid(&d, INVPCID_CTXGLOB);
998 } else {
999 do {
1000 invlpg(addr);
1001 addr += PAGE_SIZE;
1002 } while (addr < smp_tlb_addr2);
1003 }
1004 if (smp_tlb_pmap == PCPU_GET(curpmap) &&
1005 smp_tlb_pmap->pm_ucr3 != PMAP_NO_CR3 &&
1006 PCPU_GET(ucr3_load_mask) == PMAP_UCR3_NOMASK) {
1007 d.pcid = pmap_get_pcid(smp_tlb_pmap) | PMAP_PCID_USER_PT;
1008 d.pad = 0;
1009 d.addr = smp_tlb_addr1;
1010 do {
1011 invpcid(&d, INVPCID_ADDR);
1012 d.addr += PAGE_SIZE;
1013 } while (d.addr < smp_tlb_addr2);
1014 }
1015 }
1016
1017 static void
invlrng_pcid_handler(pmap_t smp_tlb_pmap,vm_offset_t smp_tlb_addr1,vm_offset_t smp_tlb_addr2)1018 invlrng_pcid_handler(pmap_t smp_tlb_pmap, vm_offset_t smp_tlb_addr1,
1019 vm_offset_t smp_tlb_addr2)
1020 {
1021 vm_offset_t addr;
1022 uint64_t kcr3, ucr3;
1023 uint32_t pcid;
1024
1025 #ifdef COUNT_XINVLTLB_HITS
1026 xhits_rng[PCPU_GET(cpuid)]++;
1027 #endif /* COUNT_XINVLTLB_HITS */
1028 #ifdef COUNT_IPIS
1029 (*ipi_invlrng_counts[PCPU_GET(cpuid)])++;
1030 #endif /* COUNT_IPIS */
1031
1032 addr = smp_tlb_addr1;
1033 do {
1034 invlpg(addr);
1035 addr += PAGE_SIZE;
1036 } while (addr < smp_tlb_addr2);
1037 if (smp_tlb_pmap == PCPU_GET(curpmap) &&
1038 (ucr3 = smp_tlb_pmap->pm_ucr3) != PMAP_NO_CR3 &&
1039 PCPU_GET(ucr3_load_mask) == PMAP_UCR3_NOMASK) {
1040 pcid = pmap_get_pcid(smp_tlb_pmap);
1041 kcr3 = smp_tlb_pmap->pm_cr3 | pcid | CR3_PCID_SAVE;
1042 ucr3 |= pcid | PMAP_PCID_USER_PT | CR3_PCID_SAVE;
1043 pmap_pti_pcid_invlrng(ucr3, kcr3, smp_tlb_addr1, smp_tlb_addr2);
1044 }
1045 }
1046
1047 static void
invlcache_handler(void)1048 invlcache_handler(void)
1049 {
1050 #ifdef COUNT_IPIS
1051 (*ipi_invlcache_counts[PCPU_GET(cpuid)])++;
1052 #endif /* COUNT_IPIS */
1053 wbinvd();
1054 }
1055
1056 static void
invlop_handler_one_req(enum invl_op_codes smp_tlb_op,pmap_t smp_tlb_pmap,vm_offset_t smp_tlb_addr1,vm_offset_t smp_tlb_addr2)1057 invlop_handler_one_req(enum invl_op_codes smp_tlb_op, pmap_t smp_tlb_pmap,
1058 vm_offset_t smp_tlb_addr1, vm_offset_t smp_tlb_addr2)
1059 {
1060 switch (smp_tlb_op) {
1061 case INVL_OP_TLB:
1062 invltlb_handler(smp_tlb_pmap);
1063 break;
1064 case INVL_OP_TLB_INVPCID:
1065 invltlb_invpcid_handler(smp_tlb_pmap);
1066 break;
1067 case INVL_OP_TLB_INVPCID_PTI:
1068 invltlb_invpcid_pti_handler(smp_tlb_pmap);
1069 break;
1070 case INVL_OP_TLB_PCID:
1071 invltlb_pcid_handler(smp_tlb_pmap);
1072 break;
1073 case INVL_OP_PGRNG:
1074 invlrng_handler(smp_tlb_addr1, smp_tlb_addr2);
1075 break;
1076 case INVL_OP_PGRNG_INVPCID:
1077 invlrng_invpcid_handler(smp_tlb_pmap, smp_tlb_addr1,
1078 smp_tlb_addr2);
1079 break;
1080 case INVL_OP_PGRNG_PCID:
1081 invlrng_pcid_handler(smp_tlb_pmap, smp_tlb_addr1,
1082 smp_tlb_addr2);
1083 break;
1084 case INVL_OP_PG:
1085 invlpg_handler(smp_tlb_addr1);
1086 break;
1087 case INVL_OP_PG_INVPCID:
1088 invlpg_invpcid_handler(smp_tlb_pmap, smp_tlb_addr1);
1089 break;
1090 case INVL_OP_PG_PCID:
1091 invlpg_pcid_handler(smp_tlb_pmap, smp_tlb_addr1);
1092 break;
1093 case INVL_OP_CACHE:
1094 invlcache_handler();
1095 break;
1096 default:
1097 __assert_unreachable();
1098 break;
1099 }
1100 }
1101
1102 void
invlop_handler(void)1103 invlop_handler(void)
1104 {
1105 struct pcpu *initiator_pc;
1106 pmap_t smp_tlb_pmap;
1107 vm_offset_t smp_tlb_addr1, smp_tlb_addr2;
1108 u_int initiator_cpu_id;
1109 enum invl_op_codes smp_tlb_op;
1110 uint32_t *scoreboard, smp_tlb_gen;
1111
1112 scoreboard = invl_scoreboard_getcpu(PCPU_GET(cpuid));
1113 for (;;) {
1114 for (initiator_cpu_id = 0; initiator_cpu_id <= mp_maxid;
1115 initiator_cpu_id++) {
1116 if (atomic_load_int(&scoreboard[initiator_cpu_id]) == 0)
1117 break;
1118 }
1119 if (initiator_cpu_id > mp_maxid)
1120 break;
1121 initiator_pc = cpuid_to_pcpu[initiator_cpu_id];
1122
1123 /*
1124 * This acquire fence and its corresponding release
1125 * fence in smp_targeted_tlb_shootdown() is between
1126 * reading zero scoreboard slot and accessing PCPU of
1127 * initiator for pc_smp_tlb values.
1128 */
1129 atomic_thread_fence_acq();
1130 smp_tlb_pmap = initiator_pc->pc_smp_tlb_pmap;
1131 smp_tlb_addr1 = initiator_pc->pc_smp_tlb_addr1;
1132 smp_tlb_addr2 = initiator_pc->pc_smp_tlb_addr2;
1133 smp_tlb_op = initiator_pc->pc_smp_tlb_op;
1134 smp_tlb_gen = initiator_pc->pc_smp_tlb_gen;
1135
1136 /*
1137 * Ensure that we do not make our scoreboard
1138 * notification visible to the initiator until the
1139 * pc_smp_tlb values are read. The corresponding
1140 * fence is implicitly provided by the barrier in the
1141 * IPI send operation before the APIC ICR register
1142 * write.
1143 *
1144 * As an optimization, the request is acknowledged
1145 * before the actual invalidation is performed. It is
1146 * safe because target CPU cannot return to userspace
1147 * before handler finishes. Only NMI can preempt the
1148 * handler, but NMI would see the kernel handler frame
1149 * and not touch not-invalidated user page table.
1150 */
1151 atomic_thread_fence_acq();
1152 atomic_store_int(&scoreboard[initiator_cpu_id], smp_tlb_gen);
1153
1154 invlop_handler_one_req(smp_tlb_op, smp_tlb_pmap, smp_tlb_addr1,
1155 smp_tlb_addr2);
1156 }
1157 }
1158