1 /*-
2 * Copyright (c) 2015 The FreeBSD Foundation
3 * Copyright (c) 2016 Ruslan Bukin <br@bsdpad.com>
4 * All rights reserved.
5 *
6 * Portions of this software were developed by Andrew Turner under
7 * sponsorship from the FreeBSD Foundation.
8 *
9 * Portions of this software were developed by SRI International and the
10 * University of Cambridge Computer Laboratory under DARPA/AFRL contract
11 * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme.
12 *
13 * Portions of this software were developed by the University of Cambridge
14 * Computer Laboratory as part of the CTSRD Project, with support from the
15 * UK Higher Education Innovation Fund (HEIF).
16 *
17 * Redistribution and use in source and binary forms, with or without
18 * modification, are permitted provided that the following conditions
19 * are met:
20 * 1. Redistributions of source code must retain the above copyright
21 * notice, this list of conditions and the following disclaimer.
22 * 2. Redistributions in binary form must reproduce the above copyright
23 * notice, this list of conditions and the following disclaimer in the
24 * documentation and/or other materials provided with the distribution.
25 *
26 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
27 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
28 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
29 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
30 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
31 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
32 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
33 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
34 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
35 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
36 * SUCH DAMAGE.
37 */
38
39 #include "opt_kstack_pages.h"
40 #include "opt_platform.h"
41
42 #include <sys/param.h>
43 #include <sys/systm.h>
44 #include <sys/bus.h>
45 #include <sys/cpu.h>
46 #include <sys/cpuset.h>
47 #include <sys/intr.h>
48 #include <sys/kernel.h>
49 #include <sys/ktr.h>
50 #include <sys/malloc.h>
51 #include <sys/module.h>
52 #include <sys/mutex.h>
53 #include <sys/proc.h>
54 #include <sys/sched.h>
55 #include <sys/smp.h>
56
57 #include <vm/vm.h>
58 #include <vm/pmap.h>
59 #include <vm/vm_extern.h>
60 #include <vm/vm_kern.h>
61 #include <vm/vm_map.h>
62
63 #include <machine/smp.h>
64 #include <machine/sbi.h>
65
66 #ifdef FDT
67 #include <dev/ofw/openfirm.h>
68 #include <dev/ofw/ofw_cpu.h>
69 #endif
70
71 #define MP_BOOTSTACK_SIZE (kstack_pages * PAGE_SIZE)
72
73 uint32_t __riscv_boot_ap[MAXCPU];
74
75 static enum {
76 CPUS_UNKNOWN,
77 #ifdef FDT
78 CPUS_FDT,
79 #endif
80 } cpu_enum_method;
81
82 static void ipi_ast(void *);
83 static void ipi_hardclock(void *);
84 static void ipi_preempt(void *);
85 static void ipi_rendezvous(void *);
86 static void ipi_stop(void *);
87
88 extern uint32_t boot_hart;
89 extern cpuset_t all_harts;
90
91 #ifdef INVARIANTS
92 static uint32_t cpu_reg[MAXCPU][2];
93 #endif
94
95 void mpentry(u_long hartid);
96 void init_secondary(uint64_t);
97
98 static struct mtx ap_boot_mtx;
99
100 /* Stacks for AP initialization, discarded once idle threads are started. */
101 void *bootstack;
102 static void *bootstacks[MAXCPU];
103
104 /* Count of started APs, used to synchronize access to bootstack. */
105 static volatile int aps_started;
106
107 /* Set to 1 once we're ready to let the APs out of the pen. */
108 static volatile int aps_ready;
109
110 /* Temporary variables for init_secondary() */
111 void *dpcpu[MAXCPU - 1];
112
113 static void
release_aps(void * dummy __unused)114 release_aps(void *dummy __unused)
115 {
116 cpuset_t mask;
117 int i;
118
119 if (mp_ncpus == 1)
120 return;
121
122 /* Setup the IPI handlers */
123 intr_ipi_setup(IPI_AST, "ast", ipi_ast, NULL);
124 intr_ipi_setup(IPI_PREEMPT, "preempt", ipi_preempt, NULL);
125 intr_ipi_setup(IPI_RENDEZVOUS, "rendezvous", ipi_rendezvous, NULL);
126 intr_ipi_setup(IPI_STOP, "stop", ipi_stop, NULL);
127 intr_ipi_setup(IPI_STOP_HARD, "stop hard", ipi_stop, NULL);
128 intr_ipi_setup(IPI_HARDCLOCK, "hardclock", ipi_hardclock, NULL);
129
130 atomic_store_rel_int(&aps_ready, 1);
131
132 /* Wake up the other CPUs */
133 mask = all_harts;
134 CPU_CLR(boot_hart, &mask);
135
136 printf("Release APs\n");
137
138 sbi_send_ipi(mask.__bits);
139
140 for (i = 0; i < 2000; i++) {
141 if (atomic_load_acq_int(&smp_started))
142 return;
143 DELAY(1000);
144 }
145
146 printf("APs not started\n");
147 }
148 SYSINIT(start_aps, SI_SUB_SMP, SI_ORDER_FIRST, release_aps, NULL);
149
150 void
init_secondary(uint64_t hart)151 init_secondary(uint64_t hart)
152 {
153 struct pcpu *pcpup;
154 u_int cpuid;
155
156 /* Renumber this cpu */
157 cpuid = hart;
158 if (cpuid < boot_hart)
159 cpuid += mp_maxid + 1;
160 cpuid -= boot_hart;
161
162 /* Setup the pcpu pointer */
163 pcpup = &__pcpu[cpuid];
164 __asm __volatile("mv tp, %0" :: "r"(pcpup));
165
166 /* Workaround: make sure wfi doesn't halt the hart */
167 csr_set(sie, SIE_SSIE);
168 csr_set(sip, SIE_SSIE);
169
170 /* Signal the BSP and spin until it has released all APs. */
171 atomic_add_int(&aps_started, 1);
172 while (!atomic_load_int(&aps_ready))
173 __asm __volatile("wfi");
174
175 /* Initialize curthread */
176 KASSERT(PCPU_GET(idlethread) != NULL, ("no idle thread"));
177 pcpup->pc_curthread = pcpup->pc_idlethread;
178 schedinit_ap();
179
180 /* Setup and enable interrupts */
181 intr_pic_init_secondary();
182
183 #ifndef EARLY_AP_STARTUP
184 /* Start per-CPU event timers. */
185 cpu_initclocks_ap();
186 #endif
187
188 /* Activate this hart in the kernel pmap. */
189 CPU_SET_ATOMIC(hart, &kernel_pmap->pm_active);
190
191 /* Activate process 0's pmap. */
192 pmap_activate_boot(vmspace_pmap(proc0.p_vmspace));
193
194 mtx_lock_spin(&ap_boot_mtx);
195
196 atomic_add_rel_32(&smp_cpus, 1);
197
198 if (smp_cpus == mp_ncpus) {
199 /* enable IPI's, tlb shootdown, freezes etc */
200 atomic_store_rel_int(&smp_started, 1);
201 }
202
203 mtx_unlock_spin(&ap_boot_mtx);
204
205 if (bootverbose)
206 printf("Secondary CPU %u fully online\n", cpuid);
207
208 /* Enter the scheduler */
209 sched_ap_entry();
210
211 panic("scheduler returned us to init_secondary");
212 /* NOTREACHED */
213 }
214
215 static void
smp_after_idle_runnable(void * arg __unused)216 smp_after_idle_runnable(void *arg __unused)
217 {
218 int cpu;
219
220 if (mp_ncpus == 1)
221 return;
222
223 KASSERT(smp_started != 0, ("%s: SMP not started yet", __func__));
224
225 /*
226 * Wait for all APs to handle an interrupt. After that, we know that
227 * the APs have entered the scheduler at least once, so the boot stacks
228 * are safe to free.
229 */
230 smp_rendezvous(smp_no_rendezvous_barrier, NULL,
231 smp_no_rendezvous_barrier, NULL);
232
233 for (cpu = 1; cpu <= mp_maxid; cpu++) {
234 if (bootstacks[cpu] != NULL)
235 kmem_free(bootstacks[cpu], MP_BOOTSTACK_SIZE);
236 }
237 }
238 SYSINIT(smp_after_idle_runnable, SI_SUB_SMP, SI_ORDER_ANY,
239 smp_after_idle_runnable, NULL);
240
241 static void
ipi_ast(void * dummy __unused)242 ipi_ast(void *dummy __unused)
243 {
244 CTR0(KTR_SMP, "IPI_AST");
245 }
246
247 static void
ipi_preempt(void * dummy __unused)248 ipi_preempt(void *dummy __unused)
249 {
250 CTR1(KTR_SMP, "%s: IPI_PREEMPT", __func__);
251 sched_preempt(curthread);
252 }
253
254 static void
ipi_rendezvous(void * dummy __unused)255 ipi_rendezvous(void *dummy __unused)
256 {
257 CTR0(KTR_SMP, "IPI_RENDEZVOUS");
258 smp_rendezvous_action();
259 }
260
261 static void
ipi_stop(void * dummy __unused)262 ipi_stop(void *dummy __unused)
263 {
264 u_int cpu;
265
266 CTR0(KTR_SMP, "IPI_STOP");
267
268 cpu = PCPU_GET(cpuid);
269 savectx(&stoppcbs[cpu]);
270
271 /* Indicate we are stopped */
272 CPU_SET_ATOMIC(cpu, &stopped_cpus);
273
274 /* Wait for restart */
275 while (!CPU_ISSET(cpu, &started_cpus))
276 cpu_spinwait();
277
278 CPU_CLR_ATOMIC(cpu, &started_cpus);
279 CPU_CLR_ATOMIC(cpu, &stopped_cpus);
280 CTR0(KTR_SMP, "IPI_STOP (restart)");
281
282 /*
283 * The kernel debugger might have set a breakpoint,
284 * so flush the instruction cache.
285 */
286 fence_i();
287 }
288
289 static void
ipi_hardclock(void * dummy __unused)290 ipi_hardclock(void *dummy __unused)
291 {
292 CTR1(KTR_SMP, "%s: IPI_HARDCLOCK", __func__);
293 hardclockintr();
294 }
295
296 struct cpu_group *
cpu_topo(void)297 cpu_topo(void)
298 {
299
300 return (smp_topo_none());
301 }
302
303 /* Determine if we running MP machine */
304 int
cpu_mp_probe(void)305 cpu_mp_probe(void)
306 {
307
308 return (mp_ncpus > 1);
309 }
310
311 #ifdef FDT
312 static bool
cpu_check_mmu(u_int id __unused,phandle_t node,u_int addr_size __unused,pcell_t * reg __unused)313 cpu_check_mmu(u_int id __unused, phandle_t node, u_int addr_size __unused,
314 pcell_t *reg __unused)
315 {
316 char type[32];
317
318 /* Check if this hart supports MMU. */
319 if (OF_getprop(node, "mmu-type", (void *)type, sizeof(type)) == -1 ||
320 strncmp(type, "riscv,none", 10) == 0)
321 return (false);
322
323 return (true);
324 }
325
326 static bool
cpu_init_fdt(u_int id,phandle_t node,u_int addr_size,pcell_t * reg)327 cpu_init_fdt(u_int id, phandle_t node, u_int addr_size, pcell_t *reg)
328 {
329 struct pcpu *pcpup;
330 vm_paddr_t start_addr;
331 uint64_t hart;
332 u_int cpuid;
333 int naps;
334 int error;
335
336 if (!cpu_check_mmu(id, node, addr_size, reg))
337 return (false);
338
339 KASSERT(id < MAXCPU, ("Too many CPUs"));
340
341 KASSERT(addr_size == 1 || addr_size == 2, ("Invalid register size"));
342 #ifdef INVARIANTS
343 cpu_reg[id][0] = reg[0];
344 if (addr_size == 2)
345 cpu_reg[id][1] = reg[1];
346 #endif
347
348 hart = reg[0];
349 if (addr_size == 2) {
350 hart <<= 32;
351 hart |= reg[1];
352 }
353
354 KASSERT(hart < MAXCPU, ("Too many harts."));
355
356 /* We are already running on this cpu */
357 if (hart == boot_hart)
358 return (true);
359
360 /*
361 * Rotate the CPU IDs to put the boot CPU as CPU 0.
362 * We keep the other CPUs ordered.
363 */
364 cpuid = hart;
365 if (cpuid < boot_hart)
366 cpuid += mp_maxid + 1;
367 cpuid -= boot_hart;
368
369 /* Check if we are able to start this cpu */
370 if (cpuid > mp_maxid)
371 return (false);
372
373 /*
374 * Depending on the SBI implementation, APs are waiting either in
375 * locore.S or to be activated explicitly, via SBI call.
376 */
377 if (sbi_probe_extension(SBI_EXT_ID_HSM) != 0) {
378 start_addr = pmap_kextract((vm_offset_t)mpentry);
379 error = sbi_hsm_hart_start(hart, start_addr, 0);
380 if (error != 0) {
381 mp_ncpus--;
382
383 /* Send a warning to the user and continue. */
384 printf("AP %u (hart %lu) failed to start, error %d\n",
385 cpuid, hart, error);
386 return (false);
387 }
388 }
389
390 pcpup = &__pcpu[cpuid];
391 pcpu_init(pcpup, cpuid, sizeof(struct pcpu));
392 pcpup->pc_hart = hart;
393
394 dpcpu[cpuid - 1] = kmem_malloc(DPCPU_SIZE, M_WAITOK | M_ZERO);
395 dpcpu_init(dpcpu[cpuid - 1], cpuid);
396
397 bootstacks[cpuid] = kmem_malloc(MP_BOOTSTACK_SIZE, M_WAITOK | M_ZERO);
398
399 naps = atomic_load_int(&aps_started);
400 bootstack = (char *)bootstacks[cpuid] + MP_BOOTSTACK_SIZE;
401
402 if (bootverbose)
403 printf("Starting CPU %u (hart %lx)\n", cpuid, hart);
404 atomic_store_32(&__riscv_boot_ap[hart], 1);
405
406 /* Wait for the AP to switch to its boot stack. */
407 while (atomic_load_int(&aps_started) < naps + 1)
408 cpu_spinwait();
409
410 CPU_SET(cpuid, &all_cpus);
411 CPU_SET(hart, &all_harts);
412
413 return (true);
414 }
415 #endif
416
417 /* Initialize and fire up non-boot processors */
418 void
cpu_mp_start(void)419 cpu_mp_start(void)
420 {
421 u_int cpu;
422
423 mtx_init(&ap_boot_mtx, "ap boot", NULL, MTX_SPIN);
424
425 CPU_SET(0, &all_cpus);
426 CPU_SET(boot_hart, &all_harts);
427
428 switch(cpu_enum_method) {
429 #ifdef FDT
430 case CPUS_FDT:
431 ofw_cpu_early_foreach(cpu_init_fdt, true);
432 break;
433 #endif
434 case CPUS_UNKNOWN:
435 break;
436 }
437
438 CPU_FOREACH(cpu) {
439 /* Already identified. */
440 if (cpu == 0)
441 continue;
442
443 identify_cpu(cpu);
444 }
445 }
446
447 /* Introduce rest of cores to the world */
448 void
cpu_mp_announce(void)449 cpu_mp_announce(void)
450 {
451 u_int cpu;
452
453 CPU_FOREACH(cpu) {
454 /* Already announced. */
455 if (cpu == 0)
456 continue;
457
458 printcpuinfo(cpu);
459 }
460 }
461
462 void
cpu_mp_setmaxid(void)463 cpu_mp_setmaxid(void)
464 {
465 int cores;
466
467 #ifdef FDT
468 cores = ofw_cpu_early_foreach(cpu_check_mmu, true);
469 if (cores > 0) {
470 cores = MIN(cores, MAXCPU);
471 if (bootverbose)
472 printf("Found %d CPUs in the device tree\n", cores);
473 mp_ncpus = cores;
474 mp_maxid = cores - 1;
475 cpu_enum_method = CPUS_FDT;
476 } else
477 #endif
478 {
479 if (bootverbose)
480 printf("No CPU data, limiting to 1 core\n");
481 mp_ncpus = 1;
482 mp_maxid = 0;
483 }
484
485 if (TUNABLE_INT_FETCH("hw.ncpu", &cores)) {
486 if (cores > 0 && cores < mp_ncpus) {
487 mp_ncpus = cores;
488 mp_maxid = cores - 1;
489 }
490 }
491 }
492
493 void
ipi_all_but_self(u_int ipi)494 ipi_all_but_self(u_int ipi)
495 {
496 cpuset_t other_cpus;
497
498 other_cpus = all_cpus;
499 CPU_CLR(PCPU_GET(cpuid), &other_cpus);
500
501 CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi);
502 intr_ipi_send(other_cpus, ipi);
503 }
504
505 void
ipi_cpu(int cpu,u_int ipi)506 ipi_cpu(int cpu, u_int ipi)
507 {
508 cpuset_t cpus;
509
510 CPU_ZERO(&cpus);
511 CPU_SET(cpu, &cpus);
512
513 CTR3(KTR_SMP, "%s: cpu: %d, ipi: %x", __func__, cpu, ipi);
514 intr_ipi_send(cpus, ipi);
515 }
516
517 void
ipi_selected(cpuset_t cpus,u_int ipi)518 ipi_selected(cpuset_t cpus, u_int ipi)
519 {
520 CTR1(KTR_SMP, "ipi_selected: ipi: %x", ipi);
521 intr_ipi_send(cpus, ipi);
522 }
523