xref: /freebsd/sys/powerpc/powerpc/mp_machdep.c (revision 4ec234c813eed05c166859bba82c882e40826eb9)
1 /*-
2  * Copyright (c) 2008 Marcel Moolenaar
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  *
9  * 1. Redistributions of source code must retain the above copyright
10  *    notice, this list of conditions and the following disclaimer.
11  * 2. Redistributions in binary form must reproduce the above copyright
12  *    notice, this list of conditions and the following disclaimer in the
13  *    documentation and/or other materials provided with the distribution.
14  *
15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
16  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
19  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
20  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25  */
26 
27 #include <sys/cdefs.h>
28 __FBSDID("$FreeBSD$");
29 
30 #include <sys/param.h>
31 #include <sys/systm.h>
32 #include <sys/kernel.h>
33 #include <sys/ktr.h>
34 #include <sys/bus.h>
35 #include <sys/cpuset.h>
36 #include <sys/lock.h>
37 #include <sys/malloc.h>
38 #include <sys/mutex.h>
39 #include <sys/pcpu.h>
40 #include <sys/proc.h>
41 #include <sys/sched.h>
42 #include <sys/smp.h>
43 
44 #include <vm/vm.h>
45 #include <vm/vm_param.h>
46 #include <vm/pmap.h>
47 #include <vm/vm_map.h>
48 #include <vm/vm_extern.h>
49 #include <vm/vm_kern.h>
50 
51 #include <machine/bus.h>
52 #include <machine/cpu.h>
53 #include <machine/intr_machdep.h>
54 #include <machine/pcb.h>
55 #include <machine/platform.h>
56 #include <machine/md_var.h>
57 #include <machine/setjmp.h>
58 #include <machine/smp.h>
59 
60 #include "pic_if.h"
61 
62 extern struct pcpu __pcpu[MAXCPU];
63 
64 volatile static int ap_awake;
65 volatile static u_int ap_letgo;
66 volatile static u_quad_t ap_timebase;
67 static u_int ipi_msg_cnt[32];
68 static struct mtx ap_boot_mtx;
69 struct pcb stoppcbs[MAXCPU];
70 int longfault(faultbuf, int);
71 
72 void
73 machdep_ap_bootstrap(void)
74 {
75 	jmp_buf *restore;
76 
77 	/* The following is needed for restoring from sleep. */
78 #ifdef __powerpc64__
79 	/* Writing to the time base register is hypervisor-privileged */
80 	if (mfmsr() & PSL_HV)
81 		mttb(0);
82 #else
83 	mttb(0);
84 #endif
85 	/* Set up important bits on the CPU (HID registers, etc.) */
86 	cpudep_ap_setup();
87 
88 	/* Set PIR */
89 	PCPU_SET(pir, mfspr(SPR_PIR));
90 	PCPU_SET(awake, 1);
91 	__asm __volatile("msync; isync");
92 
93 	restore = PCPU_GET(restore);
94 	if (restore != NULL) {
95 		longjmp(*restore, 1);
96 	}
97 
98 	while (ap_letgo == 0)
99 		;
100 
101 	/* Initialize DEC and TB, sync with the BSP values */
102 #ifdef __powerpc64__
103 	/* Writing to the time base register is hypervisor-privileged */
104 	if (mfmsr() & PSL_HV)
105 		mttb(ap_timebase);
106 #else
107 	mttb(ap_timebase);
108 #endif
109 	decr_ap_init();
110 
111 	/* Give platform code a chance to do anything necessary */
112 	platform_smp_ap_init();
113 
114 	/* Serialize console output and AP count increment */
115 	mtx_lock_spin(&ap_boot_mtx);
116 	ap_awake++;
117 	printf("SMP: AP CPU #%d launched\n", PCPU_GET(cpuid));
118 	mtx_unlock_spin(&ap_boot_mtx);
119 
120 	/* Start per-CPU event timers. */
121 	cpu_initclocks_ap();
122 
123 	/* Announce ourselves awake, and enter the scheduler */
124 	sched_throw(NULL);
125 }
126 
127 void
128 cpu_mp_setmaxid(void)
129 {
130 	struct cpuref cpuref;
131 	int error;
132 
133 	mp_ncpus = 0;
134 	error = platform_smp_first_cpu(&cpuref);
135 	while (!error) {
136 		mp_ncpus++;
137 		error = platform_smp_next_cpu(&cpuref);
138 	}
139 	/* Sanity. */
140 	if (mp_ncpus == 0)
141 		mp_ncpus = 1;
142 
143 	/*
144 	 * Set the largest cpuid we're going to use. This is necessary
145 	 * for VM initialization.
146 	 */
147 	mp_maxid = min(mp_ncpus, MAXCPU) - 1;
148 }
149 
150 int
151 cpu_mp_probe(void)
152 {
153 
154 	/*
155 	 * We're not going to enable SMP if there's only 1 processor.
156 	 */
157 	return (mp_ncpus > 1);
158 }
159 
160 void
161 cpu_mp_start(void)
162 {
163 	struct cpuref bsp, cpu;
164 	struct pcpu *pc;
165 	int error;
166 
167 	error = platform_smp_get_bsp(&bsp);
168 	KASSERT(error == 0, ("Don't know BSP"));
169 	KASSERT(bsp.cr_cpuid == 0, ("%s: cpuid != 0", __func__));
170 
171 	error = platform_smp_first_cpu(&cpu);
172 	while (!error) {
173 		if (cpu.cr_cpuid >= MAXCPU) {
174 			printf("SMP: cpu%d: skipped -- ID out of range\n",
175 			    cpu.cr_cpuid);
176 			goto next;
177 		}
178 		if (CPU_ISSET(cpu.cr_cpuid, &all_cpus)) {
179 			printf("SMP: cpu%d: skipped - duplicate ID\n",
180 			    cpu.cr_cpuid);
181 			goto next;
182 		}
183 		if (cpu.cr_cpuid != bsp.cr_cpuid) {
184 			void *dpcpu;
185 
186 			pc = &__pcpu[cpu.cr_cpuid];
187 			dpcpu = (void *)kmem_malloc(kernel_arena, DPCPU_SIZE,
188 			    M_WAITOK | M_ZERO);
189 			pcpu_init(pc, cpu.cr_cpuid, sizeof(*pc));
190 			dpcpu_init(dpcpu, cpu.cr_cpuid);
191 		} else {
192 			pc = pcpup;
193 			pc->pc_cpuid = bsp.cr_cpuid;
194 			pc->pc_bsp = 1;
195 		}
196 		pc->pc_hwref = cpu.cr_hwref;
197 		CPU_SET(pc->pc_cpuid, &all_cpus);
198 next:
199 		error = platform_smp_next_cpu(&cpu);
200 	}
201 }
202 
203 void
204 cpu_mp_announce(void)
205 {
206 	struct pcpu *pc;
207 	int i;
208 
209 	for (i = 0; i <= mp_maxid; i++) {
210 		pc = pcpu_find(i);
211 		if (pc == NULL)
212 			continue;
213 		printf("cpu%d: dev=%x", i, (int)pc->pc_hwref);
214 		if (pc->pc_bsp)
215 			printf(" (BSP)");
216 		printf("\n");
217 	}
218 }
219 
220 static void
221 cpu_mp_unleash(void *dummy)
222 {
223 	struct pcpu *pc;
224 	int cpus, timeout;
225 
226 	if (mp_ncpus <= 1)
227 		return;
228 
229 	mtx_init(&ap_boot_mtx, "ap boot", NULL, MTX_SPIN);
230 
231 	cpus = 0;
232 	smp_cpus = 0;
233 	STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) {
234 		cpus++;
235 		if (!pc->pc_bsp) {
236 			if (bootverbose)
237 				printf("Waking up CPU %d (dev=%x)\n",
238 				    pc->pc_cpuid, (int)pc->pc_hwref);
239 
240 			platform_smp_start_cpu(pc);
241 
242 			timeout = 2000;	/* wait 2sec for the AP */
243 			while (!pc->pc_awake && --timeout > 0)
244 				DELAY(1000);
245 
246 		} else {
247 			PCPU_SET(pir, mfspr(SPR_PIR));
248 			pc->pc_awake = 1;
249 		}
250 		if (pc->pc_awake) {
251 			if (bootverbose)
252 				printf("Adding CPU %d, pir=%x, awake=%x\n",
253 				    pc->pc_cpuid, pc->pc_pir, pc->pc_awake);
254 			smp_cpus++;
255 		} else
256 			CPU_SET(pc->pc_cpuid, &stopped_cpus);
257 	}
258 
259 	ap_awake = 1;
260 
261 	/* Provide our current DEC and TB values for APs */
262 	ap_timebase = mftb() + 10;
263 	__asm __volatile("msync; isync");
264 
265 	/* Let APs continue */
266 	atomic_store_rel_int(&ap_letgo, 1);
267 
268 #ifdef __powerpc64__
269 	/* Writing to the time base register is hypervisor-privileged */
270 	if (mfmsr() & PSL_HV)
271 		mttb(ap_timebase);
272 #else
273 	mttb(ap_timebase);
274 #endif
275 
276 	while (ap_awake < smp_cpus)
277 		;
278 
279 	if (smp_cpus != cpus || cpus != mp_ncpus) {
280 		printf("SMP: %d CPUs found; %d CPUs usable; %d CPUs woken\n",
281 		    mp_ncpus, cpus, smp_cpus);
282 	}
283 
284 	/* Let the APs get into the scheduler */
285 	DELAY(10000);
286 
287 	smp_active = 1;
288 	smp_started = 1;
289 }
290 
291 SYSINIT(start_aps, SI_SUB_SMP, SI_ORDER_FIRST, cpu_mp_unleash, NULL);
292 
293 int
294 powerpc_ipi_handler(void *arg)
295 {
296 	u_int cpuid;
297 	uint32_t ipimask;
298 	int msg;
299 
300 	CTR2(KTR_SMP, "%s: MSR 0x%08x", __func__, mfmsr());
301 
302 	ipimask = atomic_readandclear_32(&(pcpup->pc_ipimask));
303 	if (ipimask == 0)
304 		return (FILTER_STRAY);
305 	while ((msg = ffs(ipimask) - 1) != -1) {
306 		ipimask &= ~(1u << msg);
307 		ipi_msg_cnt[msg]++;
308 		switch (msg) {
309 		case IPI_AST:
310 			CTR1(KTR_SMP, "%s: IPI_AST", __func__);
311 			break;
312 		case IPI_PREEMPT:
313 			CTR1(KTR_SMP, "%s: IPI_PREEMPT", __func__);
314 			sched_preempt(curthread);
315 			break;
316 		case IPI_RENDEZVOUS:
317 			CTR1(KTR_SMP, "%s: IPI_RENDEZVOUS", __func__);
318 			smp_rendezvous_action();
319 			break;
320 		case IPI_STOP:
321 
322 			/*
323 			 * IPI_STOP_HARD is mapped to IPI_STOP so it is not
324 			 * necessary to add such case in the switch.
325 			 */
326 			CTR1(KTR_SMP, "%s: IPI_STOP or IPI_STOP_HARD (stop)",
327 			    __func__);
328 			cpuid = PCPU_GET(cpuid);
329 			savectx(&stoppcbs[cpuid]);
330 			savectx(PCPU_GET(curpcb));
331 			CPU_SET_ATOMIC(cpuid, &stopped_cpus);
332 			while (!CPU_ISSET(cpuid, &started_cpus))
333 				cpu_spinwait();
334 			CPU_CLR_ATOMIC(cpuid, &stopped_cpus);
335 			CPU_CLR_ATOMIC(cpuid, &started_cpus);
336 			CTR1(KTR_SMP, "%s: IPI_STOP (restart)", __func__);
337 			break;
338 		case IPI_HARDCLOCK:
339 			CTR1(KTR_SMP, "%s: IPI_HARDCLOCK", __func__);
340 			hardclockintr();
341 			break;
342 		}
343 	}
344 
345 	return (FILTER_HANDLED);
346 }
347 
348 static void
349 ipi_send(struct pcpu *pc, int ipi)
350 {
351 
352 	CTR4(KTR_SMP, "%s: pc=%p, targetcpu=%d, IPI=%d", __func__,
353 	    pc, pc->pc_cpuid, ipi);
354 
355 	atomic_set_32(&pc->pc_ipimask, (1 << ipi));
356 	PIC_IPI(root_pic, pc->pc_cpuid);
357 
358 	CTR1(KTR_SMP, "%s: sent", __func__);
359 }
360 
361 /* Send an IPI to a set of cpus. */
362 void
363 ipi_selected(cpuset_t cpus, int ipi)
364 {
365 	struct pcpu *pc;
366 
367 	STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) {
368 		if (CPU_ISSET(pc->pc_cpuid, &cpus))
369 			ipi_send(pc, ipi);
370 	}
371 }
372 
373 /* Send an IPI to a specific CPU. */
374 void
375 ipi_cpu(int cpu, u_int ipi)
376 {
377 
378 	ipi_send(cpuid_to_pcpu[cpu], ipi);
379 }
380 
381 /* Send an IPI to all CPUs EXCEPT myself. */
382 void
383 ipi_all_but_self(int ipi)
384 {
385 	struct pcpu *pc;
386 
387 	STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) {
388 		if (pc != pcpup)
389 			ipi_send(pc, ipi);
390 	}
391 }
392