xref: /linux/arch/x86/xen/smp.c (revision 4d7696f1b05f4aeb586c74868fe3da2731daca4b)
1 /*
2  * Xen SMP support
3  *
4  * This file implements the Xen versions of smp_ops.  SMP under Xen is
5  * very straightforward.  Bringing a CPU up is simply a matter of
6  * loading its initial context and setting it running.
7  *
8  * IPIs are handled through the Xen event mechanism.
9  *
10  * Because virtual CPUs can be scheduled onto any real CPU, there's no
11  * useful topology information for the kernel to make use of.  As a
12  * result, all CPUs are treated as if they're single-core and
13  * single-threaded.
14  */
15 #include <linux/sched.h>
16 #include <linux/err.h>
17 #include <linux/slab.h>
18 #include <linux/smp.h>
19 #include <linux/irq_work.h>
20 #include <linux/tick.h>
21 
22 #include <asm/paravirt.h>
23 #include <asm/desc.h>
24 #include <asm/pgtable.h>
25 #include <asm/cpu.h>
26 
27 #include <xen/interface/xen.h>
28 #include <xen/interface/vcpu.h>
29 
30 #include <asm/xen/interface.h>
31 #include <asm/xen/hypercall.h>
32 
33 #include <xen/xen.h>
34 #include <xen/page.h>
35 #include <xen/events.h>
36 
37 #include <xen/hvc-console.h>
38 #include "xen-ops.h"
39 #include "mmu.h"
40 
41 cpumask_var_t xen_cpu_initialized_map;
42 
43 struct xen_common_irq {
44 	int irq;
45 	char *name;
46 };
47 static DEFINE_PER_CPU(struct xen_common_irq, xen_resched_irq) = { .irq = -1 };
48 static DEFINE_PER_CPU(struct xen_common_irq, xen_callfunc_irq) = { .irq = -1 };
49 static DEFINE_PER_CPU(struct xen_common_irq, xen_callfuncsingle_irq) = { .irq = -1 };
50 static DEFINE_PER_CPU(struct xen_common_irq, xen_irq_work) = { .irq = -1 };
51 static DEFINE_PER_CPU(struct xen_common_irq, xen_debug_irq) = { .irq = -1 };
52 
53 static irqreturn_t xen_call_function_interrupt(int irq, void *dev_id);
54 static irqreturn_t xen_call_function_single_interrupt(int irq, void *dev_id);
55 static irqreturn_t xen_irq_work_interrupt(int irq, void *dev_id);
56 
57 /*
58  * Reschedule call back.
59  */
60 static irqreturn_t xen_reschedule_interrupt(int irq, void *dev_id)
61 {
62 	inc_irq_stat(irq_resched_count);
63 	scheduler_ipi();
64 
65 	return IRQ_HANDLED;
66 }
67 
68 static void cpu_bringup(void)
69 {
70 	int cpu;
71 
72 	cpu_init();
73 	touch_softlockup_watchdog();
74 	preempt_disable();
75 
76 	xen_enable_sysenter();
77 	xen_enable_syscall();
78 
79 	cpu = smp_processor_id();
80 	smp_store_cpu_info(cpu);
81 	cpu_data(cpu).x86_max_cores = 1;
82 	set_cpu_sibling_map(cpu);
83 
84 	xen_setup_cpu_clockevents();
85 
86 	notify_cpu_starting(cpu);
87 
88 	set_cpu_online(cpu, true);
89 
90 	this_cpu_write(cpu_state, CPU_ONLINE);
91 
92 	wmb();
93 
94 	/* We can take interrupts now: we're officially "up". */
95 	local_irq_enable();
96 
97 	wmb();			/* make sure everything is out */
98 }
99 
100 static void cpu_bringup_and_idle(void)
101 {
102 	cpu_bringup();
103 	cpu_startup_entry(CPUHP_ONLINE);
104 }
105 
106 static void xen_smp_intr_free(unsigned int cpu)
107 {
108 	if (per_cpu(xen_resched_irq, cpu).irq >= 0) {
109 		unbind_from_irqhandler(per_cpu(xen_resched_irq, cpu).irq, NULL);
110 		per_cpu(xen_resched_irq, cpu).irq = -1;
111 		kfree(per_cpu(xen_resched_irq, cpu).name);
112 		per_cpu(xen_resched_irq, cpu).name = NULL;
113 	}
114 	if (per_cpu(xen_callfunc_irq, cpu).irq >= 0) {
115 		unbind_from_irqhandler(per_cpu(xen_callfunc_irq, cpu).irq, NULL);
116 		per_cpu(xen_callfunc_irq, cpu).irq = -1;
117 		kfree(per_cpu(xen_callfunc_irq, cpu).name);
118 		per_cpu(xen_callfunc_irq, cpu).name = NULL;
119 	}
120 	if (per_cpu(xen_debug_irq, cpu).irq >= 0) {
121 		unbind_from_irqhandler(per_cpu(xen_debug_irq, cpu).irq, NULL);
122 		per_cpu(xen_debug_irq, cpu).irq = -1;
123 		kfree(per_cpu(xen_debug_irq, cpu).name);
124 		per_cpu(xen_debug_irq, cpu).name = NULL;
125 	}
126 	if (per_cpu(xen_callfuncsingle_irq, cpu).irq >= 0) {
127 		unbind_from_irqhandler(per_cpu(xen_callfuncsingle_irq, cpu).irq,
128 				       NULL);
129 		per_cpu(xen_callfuncsingle_irq, cpu).irq = -1;
130 		kfree(per_cpu(xen_callfuncsingle_irq, cpu).name);
131 		per_cpu(xen_callfuncsingle_irq, cpu).name = NULL;
132 	}
133 	if (xen_hvm_domain())
134 		return;
135 
136 	if (per_cpu(xen_irq_work, cpu).irq >= 0) {
137 		unbind_from_irqhandler(per_cpu(xen_irq_work, cpu).irq, NULL);
138 		per_cpu(xen_irq_work, cpu).irq = -1;
139 		kfree(per_cpu(xen_irq_work, cpu).name);
140 		per_cpu(xen_irq_work, cpu).name = NULL;
141 	}
142 };
143 static int xen_smp_intr_init(unsigned int cpu)
144 {
145 	int rc;
146 	char *resched_name, *callfunc_name, *debug_name;
147 
148 	resched_name = kasprintf(GFP_KERNEL, "resched%d", cpu);
149 	rc = bind_ipi_to_irqhandler(XEN_RESCHEDULE_VECTOR,
150 				    cpu,
151 				    xen_reschedule_interrupt,
152 				    IRQF_DISABLED|IRQF_PERCPU|IRQF_NOBALANCING,
153 				    resched_name,
154 				    NULL);
155 	if (rc < 0)
156 		goto fail;
157 	per_cpu(xen_resched_irq, cpu).irq = rc;
158 	per_cpu(xen_resched_irq, cpu).name = resched_name;
159 
160 	callfunc_name = kasprintf(GFP_KERNEL, "callfunc%d", cpu);
161 	rc = bind_ipi_to_irqhandler(XEN_CALL_FUNCTION_VECTOR,
162 				    cpu,
163 				    xen_call_function_interrupt,
164 				    IRQF_DISABLED|IRQF_PERCPU|IRQF_NOBALANCING,
165 				    callfunc_name,
166 				    NULL);
167 	if (rc < 0)
168 		goto fail;
169 	per_cpu(xen_callfunc_irq, cpu).irq = rc;
170 	per_cpu(xen_callfunc_irq, cpu).name = callfunc_name;
171 
172 	debug_name = kasprintf(GFP_KERNEL, "debug%d", cpu);
173 	rc = bind_virq_to_irqhandler(VIRQ_DEBUG, cpu, xen_debug_interrupt,
174 				     IRQF_DISABLED | IRQF_PERCPU | IRQF_NOBALANCING,
175 				     debug_name, NULL);
176 	if (rc < 0)
177 		goto fail;
178 	per_cpu(xen_debug_irq, cpu).irq = rc;
179 	per_cpu(xen_debug_irq, cpu).name = debug_name;
180 
181 	callfunc_name = kasprintf(GFP_KERNEL, "callfuncsingle%d", cpu);
182 	rc = bind_ipi_to_irqhandler(XEN_CALL_FUNCTION_SINGLE_VECTOR,
183 				    cpu,
184 				    xen_call_function_single_interrupt,
185 				    IRQF_DISABLED|IRQF_PERCPU|IRQF_NOBALANCING,
186 				    callfunc_name,
187 				    NULL);
188 	if (rc < 0)
189 		goto fail;
190 	per_cpu(xen_callfuncsingle_irq, cpu).irq = rc;
191 	per_cpu(xen_callfuncsingle_irq, cpu).name = callfunc_name;
192 
193 	/*
194 	 * The IRQ worker on PVHVM goes through the native path and uses the
195 	 * IPI mechanism.
196 	 */
197 	if (xen_hvm_domain())
198 		return 0;
199 
200 	callfunc_name = kasprintf(GFP_KERNEL, "irqwork%d", cpu);
201 	rc = bind_ipi_to_irqhandler(XEN_IRQ_WORK_VECTOR,
202 				    cpu,
203 				    xen_irq_work_interrupt,
204 				    IRQF_DISABLED|IRQF_PERCPU|IRQF_NOBALANCING,
205 				    callfunc_name,
206 				    NULL);
207 	if (rc < 0)
208 		goto fail;
209 	per_cpu(xen_irq_work, cpu).irq = rc;
210 	per_cpu(xen_irq_work, cpu).name = callfunc_name;
211 
212 	return 0;
213 
214  fail:
215 	xen_smp_intr_free(cpu);
216 	return rc;
217 }
218 
219 static void __init xen_fill_possible_map(void)
220 {
221 	int i, rc;
222 
223 	if (xen_initial_domain())
224 		return;
225 
226 	for (i = 0; i < nr_cpu_ids; i++) {
227 		rc = HYPERVISOR_vcpu_op(VCPUOP_is_up, i, NULL);
228 		if (rc >= 0) {
229 			num_processors++;
230 			set_cpu_possible(i, true);
231 		}
232 	}
233 }
234 
235 static void __init xen_filter_cpu_maps(void)
236 {
237 	int i, rc;
238 	unsigned int subtract = 0;
239 
240 	if (!xen_initial_domain())
241 		return;
242 
243 	num_processors = 0;
244 	disabled_cpus = 0;
245 	for (i = 0; i < nr_cpu_ids; i++) {
246 		rc = HYPERVISOR_vcpu_op(VCPUOP_is_up, i, NULL);
247 		if (rc >= 0) {
248 			num_processors++;
249 			set_cpu_possible(i, true);
250 		} else {
251 			set_cpu_possible(i, false);
252 			set_cpu_present(i, false);
253 			subtract++;
254 		}
255 	}
256 #ifdef CONFIG_HOTPLUG_CPU
257 	/* This is akin to using 'nr_cpus' on the Linux command line.
258 	 * Which is OK as when we use 'dom0_max_vcpus=X' we can only
259 	 * have up to X, while nr_cpu_ids is greater than X. This
260 	 * normally is not a problem, except when CPU hotplugging
261 	 * is involved and then there might be more than X CPUs
262 	 * in the guest - which will not work as there is no
263 	 * hypercall to expand the max number of VCPUs an already
264 	 * running guest has. So cap it up to X. */
265 	if (subtract)
266 		nr_cpu_ids = nr_cpu_ids - subtract;
267 #endif
268 
269 }
270 
271 static void __init xen_smp_prepare_boot_cpu(void)
272 {
273 	BUG_ON(smp_processor_id() != 0);
274 	native_smp_prepare_boot_cpu();
275 
276 	/* We've switched to the "real" per-cpu gdt, so make sure the
277 	   old memory can be recycled */
278 	make_lowmem_page_readwrite(xen_initial_gdt);
279 
280 	xen_filter_cpu_maps();
281 	xen_setup_vcpu_info_placement();
282 	xen_init_spinlocks();
283 }
284 
285 static void __init xen_smp_prepare_cpus(unsigned int max_cpus)
286 {
287 	unsigned cpu;
288 	unsigned int i;
289 
290 	if (skip_ioapic_setup) {
291 		char *m = (max_cpus == 0) ?
292 			"The nosmp parameter is incompatible with Xen; " \
293 			"use Xen dom0_max_vcpus=1 parameter" :
294 			"The noapic parameter is incompatible with Xen";
295 
296 		xen_raw_printk(m);
297 		panic(m);
298 	}
299 	xen_init_lock_cpu(0);
300 
301 	smp_store_boot_cpu_info();
302 	cpu_data(0).x86_max_cores = 1;
303 
304 	for_each_possible_cpu(i) {
305 		zalloc_cpumask_var(&per_cpu(cpu_sibling_map, i), GFP_KERNEL);
306 		zalloc_cpumask_var(&per_cpu(cpu_core_map, i), GFP_KERNEL);
307 		zalloc_cpumask_var(&per_cpu(cpu_llc_shared_map, i), GFP_KERNEL);
308 	}
309 	set_cpu_sibling_map(0);
310 
311 	if (xen_smp_intr_init(0))
312 		BUG();
313 
314 	if (!alloc_cpumask_var(&xen_cpu_initialized_map, GFP_KERNEL))
315 		panic("could not allocate xen_cpu_initialized_map\n");
316 
317 	cpumask_copy(xen_cpu_initialized_map, cpumask_of(0));
318 
319 	/* Restrict the possible_map according to max_cpus. */
320 	while ((num_possible_cpus() > 1) && (num_possible_cpus() > max_cpus)) {
321 		for (cpu = nr_cpu_ids - 1; !cpu_possible(cpu); cpu--)
322 			continue;
323 		set_cpu_possible(cpu, false);
324 	}
325 
326 	for_each_possible_cpu(cpu)
327 		set_cpu_present(cpu, true);
328 }
329 
330 static int
331 cpu_initialize_context(unsigned int cpu, struct task_struct *idle)
332 {
333 	struct vcpu_guest_context *ctxt;
334 	struct desc_struct *gdt;
335 	unsigned long gdt_mfn;
336 
337 	if (cpumask_test_and_set_cpu(cpu, xen_cpu_initialized_map))
338 		return 0;
339 
340 	ctxt = kzalloc(sizeof(*ctxt), GFP_KERNEL);
341 	if (ctxt == NULL)
342 		return -ENOMEM;
343 
344 	gdt = get_cpu_gdt_table(cpu);
345 
346 	ctxt->flags = VGCF_IN_KERNEL;
347 	ctxt->user_regs.ss = __KERNEL_DS;
348 #ifdef CONFIG_X86_32
349 	ctxt->user_regs.fs = __KERNEL_PERCPU;
350 	ctxt->user_regs.gs = __KERNEL_STACK_CANARY;
351 #else
352 	ctxt->gs_base_kernel = per_cpu_offset(cpu);
353 #endif
354 	ctxt->user_regs.eip = (unsigned long)cpu_bringup_and_idle;
355 
356 	memset(&ctxt->fpu_ctxt, 0, sizeof(ctxt->fpu_ctxt));
357 
358 	{
359 		ctxt->user_regs.eflags = 0x1000; /* IOPL_RING1 */
360 		ctxt->user_regs.ds = __USER_DS;
361 		ctxt->user_regs.es = __USER_DS;
362 
363 		xen_copy_trap_info(ctxt->trap_ctxt);
364 
365 		ctxt->ldt_ents = 0;
366 
367 		BUG_ON((unsigned long)gdt & ~PAGE_MASK);
368 
369 		gdt_mfn = arbitrary_virt_to_mfn(gdt);
370 		make_lowmem_page_readonly(gdt);
371 		make_lowmem_page_readonly(mfn_to_virt(gdt_mfn));
372 
373 		ctxt->gdt_frames[0] = gdt_mfn;
374 		ctxt->gdt_ents      = GDT_ENTRIES;
375 
376 		ctxt->kernel_ss = __KERNEL_DS;
377 		ctxt->kernel_sp = idle->thread.sp0;
378 
379 #ifdef CONFIG_X86_32
380 		ctxt->event_callback_cs     = __KERNEL_CS;
381 		ctxt->failsafe_callback_cs  = __KERNEL_CS;
382 #endif
383 		ctxt->event_callback_eip    =
384 					(unsigned long)xen_hypervisor_callback;
385 		ctxt->failsafe_callback_eip =
386 					(unsigned long)xen_failsafe_callback;
387 	}
388 	ctxt->user_regs.cs = __KERNEL_CS;
389 	ctxt->user_regs.esp = idle->thread.sp0 - sizeof(struct pt_regs);
390 
391 	per_cpu(xen_cr3, cpu) = __pa(swapper_pg_dir);
392 	ctxt->ctrlreg[3] = xen_pfn_to_cr3(virt_to_mfn(swapper_pg_dir));
393 
394 	if (HYPERVISOR_vcpu_op(VCPUOP_initialise, cpu, ctxt))
395 		BUG();
396 
397 	kfree(ctxt);
398 	return 0;
399 }
400 
401 static int xen_cpu_up(unsigned int cpu, struct task_struct *idle)
402 {
403 	int rc;
404 
405 	per_cpu(current_task, cpu) = idle;
406 #ifdef CONFIG_X86_32
407 	irq_ctx_init(cpu);
408 #else
409 	clear_tsk_thread_flag(idle, TIF_FORK);
410 	per_cpu(kernel_stack, cpu) =
411 		(unsigned long)task_stack_page(idle) -
412 		KERNEL_STACK_OFFSET + THREAD_SIZE;
413 #endif
414 	xen_setup_runstate_info(cpu);
415 	xen_setup_timer(cpu);
416 	xen_init_lock_cpu(cpu);
417 
418 	per_cpu(cpu_state, cpu) = CPU_UP_PREPARE;
419 
420 	/* make sure interrupts start blocked */
421 	per_cpu(xen_vcpu, cpu)->evtchn_upcall_mask = 1;
422 
423 	rc = cpu_initialize_context(cpu, idle);
424 	if (rc)
425 		return rc;
426 
427 	if (num_online_cpus() == 1)
428 		/* Just in case we booted with a single CPU. */
429 		alternatives_enable_smp();
430 
431 	rc = xen_smp_intr_init(cpu);
432 	if (rc)
433 		return rc;
434 
435 	rc = HYPERVISOR_vcpu_op(VCPUOP_up, cpu, NULL);
436 	BUG_ON(rc);
437 
438 	while(per_cpu(cpu_state, cpu) != CPU_ONLINE) {
439 		HYPERVISOR_sched_op(SCHEDOP_yield, NULL);
440 		barrier();
441 	}
442 
443 	return 0;
444 }
445 
446 static void xen_smp_cpus_done(unsigned int max_cpus)
447 {
448 }
449 
450 #ifdef CONFIG_HOTPLUG_CPU
451 static int xen_cpu_disable(void)
452 {
453 	unsigned int cpu = smp_processor_id();
454 	if (cpu == 0)
455 		return -EBUSY;
456 
457 	cpu_disable_common();
458 
459 	load_cr3(swapper_pg_dir);
460 	return 0;
461 }
462 
463 static void xen_cpu_die(unsigned int cpu)
464 {
465 	while (xen_pv_domain() && HYPERVISOR_vcpu_op(VCPUOP_is_up, cpu, NULL)) {
466 		current->state = TASK_UNINTERRUPTIBLE;
467 		schedule_timeout(HZ/10);
468 	}
469 	xen_smp_intr_free(cpu);
470 	xen_uninit_lock_cpu(cpu);
471 	xen_teardown_timer(cpu);
472 }
473 
474 static void xen_play_dead(void) /* used only with HOTPLUG_CPU */
475 {
476 	play_dead_common();
477 	HYPERVISOR_vcpu_op(VCPUOP_down, smp_processor_id(), NULL);
478 	cpu_bringup();
479 	/*
480 	 * commit 4b0c0f294 (tick: Cleanup NOHZ per cpu data on cpu down)
481 	 * clears certain data that the cpu_idle loop (which called us
482 	 * and that we return from) expects. The only way to get that
483 	 * data back is to call:
484 	 */
485 	tick_nohz_idle_enter();
486 }
487 
488 #else /* !CONFIG_HOTPLUG_CPU */
489 static int xen_cpu_disable(void)
490 {
491 	return -ENOSYS;
492 }
493 
494 static void xen_cpu_die(unsigned int cpu)
495 {
496 	BUG();
497 }
498 
499 static void xen_play_dead(void)
500 {
501 	BUG();
502 }
503 
504 #endif
505 static void stop_self(void *v)
506 {
507 	int cpu = smp_processor_id();
508 
509 	/* make sure we're not pinning something down */
510 	load_cr3(swapper_pg_dir);
511 	/* should set up a minimal gdt */
512 
513 	set_cpu_online(cpu, false);
514 
515 	HYPERVISOR_vcpu_op(VCPUOP_down, cpu, NULL);
516 	BUG();
517 }
518 
519 static void xen_stop_other_cpus(int wait)
520 {
521 	smp_call_function(stop_self, NULL, wait);
522 }
523 
524 static void xen_smp_send_reschedule(int cpu)
525 {
526 	xen_send_IPI_one(cpu, XEN_RESCHEDULE_VECTOR);
527 }
528 
529 static void __xen_send_IPI_mask(const struct cpumask *mask,
530 			      int vector)
531 {
532 	unsigned cpu;
533 
534 	for_each_cpu_and(cpu, mask, cpu_online_mask)
535 		xen_send_IPI_one(cpu, vector);
536 }
537 
538 static void xen_smp_send_call_function_ipi(const struct cpumask *mask)
539 {
540 	int cpu;
541 
542 	__xen_send_IPI_mask(mask, XEN_CALL_FUNCTION_VECTOR);
543 
544 	/* Make sure other vcpus get a chance to run if they need to. */
545 	for_each_cpu(cpu, mask) {
546 		if (xen_vcpu_stolen(cpu)) {
547 			HYPERVISOR_sched_op(SCHEDOP_yield, NULL);
548 			break;
549 		}
550 	}
551 }
552 
553 static void xen_smp_send_call_function_single_ipi(int cpu)
554 {
555 	__xen_send_IPI_mask(cpumask_of(cpu),
556 			  XEN_CALL_FUNCTION_SINGLE_VECTOR);
557 }
558 
559 static inline int xen_map_vector(int vector)
560 {
561 	int xen_vector;
562 
563 	switch (vector) {
564 	case RESCHEDULE_VECTOR:
565 		xen_vector = XEN_RESCHEDULE_VECTOR;
566 		break;
567 	case CALL_FUNCTION_VECTOR:
568 		xen_vector = XEN_CALL_FUNCTION_VECTOR;
569 		break;
570 	case CALL_FUNCTION_SINGLE_VECTOR:
571 		xen_vector = XEN_CALL_FUNCTION_SINGLE_VECTOR;
572 		break;
573 	case IRQ_WORK_VECTOR:
574 		xen_vector = XEN_IRQ_WORK_VECTOR;
575 		break;
576 #ifdef CONFIG_X86_64
577 	case NMI_VECTOR:
578 	case APIC_DM_NMI: /* Some use that instead of NMI_VECTOR */
579 		xen_vector = XEN_NMI_VECTOR;
580 		break;
581 #endif
582 	default:
583 		xen_vector = -1;
584 		printk(KERN_ERR "xen: vector 0x%x is not implemented\n",
585 			vector);
586 	}
587 
588 	return xen_vector;
589 }
590 
591 void xen_send_IPI_mask(const struct cpumask *mask,
592 			      int vector)
593 {
594 	int xen_vector = xen_map_vector(vector);
595 
596 	if (xen_vector >= 0)
597 		__xen_send_IPI_mask(mask, xen_vector);
598 }
599 
600 void xen_send_IPI_all(int vector)
601 {
602 	int xen_vector = xen_map_vector(vector);
603 
604 	if (xen_vector >= 0)
605 		__xen_send_IPI_mask(cpu_online_mask, xen_vector);
606 }
607 
608 void xen_send_IPI_self(int vector)
609 {
610 	int xen_vector = xen_map_vector(vector);
611 
612 	if (xen_vector >= 0)
613 		xen_send_IPI_one(smp_processor_id(), xen_vector);
614 }
615 
616 void xen_send_IPI_mask_allbutself(const struct cpumask *mask,
617 				int vector)
618 {
619 	unsigned cpu;
620 	unsigned int this_cpu = smp_processor_id();
621 	int xen_vector = xen_map_vector(vector);
622 
623 	if (!(num_online_cpus() > 1) || (xen_vector < 0))
624 		return;
625 
626 	for_each_cpu_and(cpu, mask, cpu_online_mask) {
627 		if (this_cpu == cpu)
628 			continue;
629 
630 		xen_send_IPI_one(cpu, xen_vector);
631 	}
632 }
633 
634 void xen_send_IPI_allbutself(int vector)
635 {
636 	xen_send_IPI_mask_allbutself(cpu_online_mask, vector);
637 }
638 
639 static irqreturn_t xen_call_function_interrupt(int irq, void *dev_id)
640 {
641 	irq_enter();
642 	generic_smp_call_function_interrupt();
643 	inc_irq_stat(irq_call_count);
644 	irq_exit();
645 
646 	return IRQ_HANDLED;
647 }
648 
649 static irqreturn_t xen_call_function_single_interrupt(int irq, void *dev_id)
650 {
651 	irq_enter();
652 	generic_smp_call_function_single_interrupt();
653 	inc_irq_stat(irq_call_count);
654 	irq_exit();
655 
656 	return IRQ_HANDLED;
657 }
658 
659 static irqreturn_t xen_irq_work_interrupt(int irq, void *dev_id)
660 {
661 	irq_enter();
662 	irq_work_run();
663 	inc_irq_stat(apic_irq_work_irqs);
664 	irq_exit();
665 
666 	return IRQ_HANDLED;
667 }
668 
669 static const struct smp_ops xen_smp_ops __initconst = {
670 	.smp_prepare_boot_cpu = xen_smp_prepare_boot_cpu,
671 	.smp_prepare_cpus = xen_smp_prepare_cpus,
672 	.smp_cpus_done = xen_smp_cpus_done,
673 
674 	.cpu_up = xen_cpu_up,
675 	.cpu_die = xen_cpu_die,
676 	.cpu_disable = xen_cpu_disable,
677 	.play_dead = xen_play_dead,
678 
679 	.stop_other_cpus = xen_stop_other_cpus,
680 	.smp_send_reschedule = xen_smp_send_reschedule,
681 
682 	.send_call_func_ipi = xen_smp_send_call_function_ipi,
683 	.send_call_func_single_ipi = xen_smp_send_call_function_single_ipi,
684 };
685 
686 void __init xen_smp_init(void)
687 {
688 	smp_ops = xen_smp_ops;
689 	xen_fill_possible_map();
690 }
691 
692 static void __init xen_hvm_smp_prepare_cpus(unsigned int max_cpus)
693 {
694 	native_smp_prepare_cpus(max_cpus);
695 	WARN_ON(xen_smp_intr_init(0));
696 
697 	xen_init_lock_cpu(0);
698 }
699 
700 static int xen_hvm_cpu_up(unsigned int cpu, struct task_struct *tidle)
701 {
702 	int rc;
703 	/*
704 	 * xen_smp_intr_init() needs to run before native_cpu_up()
705 	 * so that IPI vectors are set up on the booting CPU before
706 	 * it is marked online in native_cpu_up().
707 	*/
708 	rc = xen_smp_intr_init(cpu);
709 	WARN_ON(rc);
710 	if (!rc)
711 		rc =  native_cpu_up(cpu, tidle);
712 	return rc;
713 }
714 
715 static void xen_hvm_cpu_die(unsigned int cpu)
716 {
717 	xen_cpu_die(cpu);
718 	native_cpu_die(cpu);
719 }
720 
721 void __init xen_hvm_smp_init(void)
722 {
723 	if (!xen_have_vector_callback)
724 		return;
725 	smp_ops.smp_prepare_cpus = xen_hvm_smp_prepare_cpus;
726 	smp_ops.smp_send_reschedule = xen_smp_send_reschedule;
727 	smp_ops.cpu_up = xen_hvm_cpu_up;
728 	smp_ops.cpu_die = xen_hvm_cpu_die;
729 	smp_ops.send_call_func_ipi = xen_smp_send_call_function_ipi;
730 	smp_ops.send_call_func_single_ipi = xen_smp_send_call_function_single_ipi;
731 }
732