1 /* 2 * Xen SMP support 3 * 4 * This file implements the Xen versions of smp_ops. SMP under Xen is 5 * very straightforward. Bringing a CPU up is simply a matter of 6 * loading its initial context and setting it running. 7 * 8 * IPIs are handled through the Xen event mechanism. 9 * 10 * Because virtual CPUs can be scheduled onto any real CPU, there's no 11 * useful topology information for the kernel to make use of. As a 12 * result, all CPUs are treated as if they're single-core and 13 * single-threaded. 14 */ 15 #include <linux/sched.h> 16 #include <linux/err.h> 17 #include <linux/slab.h> 18 #include <linux/smp.h> 19 #include <linux/irq_work.h> 20 #include <linux/tick.h> 21 22 #include <asm/paravirt.h> 23 #include <asm/desc.h> 24 #include <asm/pgtable.h> 25 #include <asm/cpu.h> 26 27 #include <xen/interface/xen.h> 28 #include <xen/interface/vcpu.h> 29 #include <xen/interface/xenpmu.h> 30 31 #include <asm/xen/interface.h> 32 #include <asm/xen/hypercall.h> 33 34 #include <xen/xen.h> 35 #include <xen/page.h> 36 #include <xen/events.h> 37 38 #include <xen/hvc-console.h> 39 #include "xen-ops.h" 40 #include "mmu.h" 41 #include "smp.h" 42 #include "pmu.h" 43 44 cpumask_var_t xen_cpu_initialized_map; 45 46 struct xen_common_irq { 47 int irq; 48 char *name; 49 }; 50 static DEFINE_PER_CPU(struct xen_common_irq, xen_resched_irq) = { .irq = -1 }; 51 static DEFINE_PER_CPU(struct xen_common_irq, xen_callfunc_irq) = { .irq = -1 }; 52 static DEFINE_PER_CPU(struct xen_common_irq, xen_callfuncsingle_irq) = { .irq = -1 }; 53 static DEFINE_PER_CPU(struct xen_common_irq, xen_irq_work) = { .irq = -1 }; 54 static DEFINE_PER_CPU(struct xen_common_irq, xen_debug_irq) = { .irq = -1 }; 55 static DEFINE_PER_CPU(struct xen_common_irq, xen_pmu_irq) = { .irq = -1 }; 56 57 static irqreturn_t xen_call_function_interrupt(int irq, void *dev_id); 58 static irqreturn_t xen_call_function_single_interrupt(int irq, void *dev_id); 59 static irqreturn_t xen_irq_work_interrupt(int irq, void *dev_id); 60 61 /* 62 * Reschedule call back. 63 */ 64 static irqreturn_t xen_reschedule_interrupt(int irq, void *dev_id) 65 { 66 inc_irq_stat(irq_resched_count); 67 scheduler_ipi(); 68 69 return IRQ_HANDLED; 70 } 71 72 static void cpu_bringup(void) 73 { 74 int cpu; 75 76 cpu_init(); 77 touch_softlockup_watchdog(); 78 preempt_disable(); 79 80 /* PVH runs in ring 0 and allows us to do native syscalls. Yay! */ 81 if (!xen_feature(XENFEAT_supervisor_mode_kernel)) { 82 xen_enable_sysenter(); 83 xen_enable_syscall(); 84 } 85 cpu = smp_processor_id(); 86 smp_store_cpu_info(cpu); 87 cpu_data(cpu).x86_max_cores = 1; 88 set_cpu_sibling_map(cpu); 89 90 xen_setup_cpu_clockevents(); 91 92 notify_cpu_starting(cpu); 93 94 set_cpu_online(cpu, true); 95 96 cpu_set_state_online(cpu); /* Implies full memory barrier. */ 97 98 /* We can take interrupts now: we're officially "up". */ 99 local_irq_enable(); 100 } 101 102 /* 103 * Note: cpu parameter is only relevant for PVH. The reason for passing it 104 * is we can't do smp_processor_id until the percpu segments are loaded, for 105 * which we need the cpu number! So we pass it in rdi as first parameter. 106 */ 107 asmlinkage __visible void cpu_bringup_and_idle(int cpu) 108 { 109 #ifdef CONFIG_XEN_PVH 110 if (xen_feature(XENFEAT_auto_translated_physmap) && 111 xen_feature(XENFEAT_supervisor_mode_kernel)) 112 xen_pvh_secondary_vcpu_init(cpu); 113 #endif 114 cpu_bringup(); 115 cpu_startup_entry(CPUHP_AP_ONLINE_IDLE); 116 } 117 118 void xen_smp_intr_free(unsigned int cpu) 119 { 120 if (per_cpu(xen_resched_irq, cpu).irq >= 0) { 121 unbind_from_irqhandler(per_cpu(xen_resched_irq, cpu).irq, NULL); 122 per_cpu(xen_resched_irq, cpu).irq = -1; 123 kfree(per_cpu(xen_resched_irq, cpu).name); 124 per_cpu(xen_resched_irq, cpu).name = NULL; 125 } 126 if (per_cpu(xen_callfunc_irq, cpu).irq >= 0) { 127 unbind_from_irqhandler(per_cpu(xen_callfunc_irq, cpu).irq, NULL); 128 per_cpu(xen_callfunc_irq, cpu).irq = -1; 129 kfree(per_cpu(xen_callfunc_irq, cpu).name); 130 per_cpu(xen_callfunc_irq, cpu).name = NULL; 131 } 132 if (per_cpu(xen_debug_irq, cpu).irq >= 0) { 133 unbind_from_irqhandler(per_cpu(xen_debug_irq, cpu).irq, NULL); 134 per_cpu(xen_debug_irq, cpu).irq = -1; 135 kfree(per_cpu(xen_debug_irq, cpu).name); 136 per_cpu(xen_debug_irq, cpu).name = NULL; 137 } 138 if (per_cpu(xen_callfuncsingle_irq, cpu).irq >= 0) { 139 unbind_from_irqhandler(per_cpu(xen_callfuncsingle_irq, cpu).irq, 140 NULL); 141 per_cpu(xen_callfuncsingle_irq, cpu).irq = -1; 142 kfree(per_cpu(xen_callfuncsingle_irq, cpu).name); 143 per_cpu(xen_callfuncsingle_irq, cpu).name = NULL; 144 } 145 if (xen_hvm_domain()) 146 return; 147 148 if (per_cpu(xen_irq_work, cpu).irq >= 0) { 149 unbind_from_irqhandler(per_cpu(xen_irq_work, cpu).irq, NULL); 150 per_cpu(xen_irq_work, cpu).irq = -1; 151 kfree(per_cpu(xen_irq_work, cpu).name); 152 per_cpu(xen_irq_work, cpu).name = NULL; 153 } 154 155 if (per_cpu(xen_pmu_irq, cpu).irq >= 0) { 156 unbind_from_irqhandler(per_cpu(xen_pmu_irq, cpu).irq, NULL); 157 per_cpu(xen_pmu_irq, cpu).irq = -1; 158 kfree(per_cpu(xen_pmu_irq, cpu).name); 159 per_cpu(xen_pmu_irq, cpu).name = NULL; 160 } 161 }; 162 int xen_smp_intr_init(unsigned int cpu) 163 { 164 int rc; 165 char *resched_name, *callfunc_name, *debug_name, *pmu_name; 166 167 resched_name = kasprintf(GFP_KERNEL, "resched%d", cpu); 168 rc = bind_ipi_to_irqhandler(XEN_RESCHEDULE_VECTOR, 169 cpu, 170 xen_reschedule_interrupt, 171 IRQF_PERCPU|IRQF_NOBALANCING, 172 resched_name, 173 NULL); 174 if (rc < 0) 175 goto fail; 176 per_cpu(xen_resched_irq, cpu).irq = rc; 177 per_cpu(xen_resched_irq, cpu).name = resched_name; 178 179 callfunc_name = kasprintf(GFP_KERNEL, "callfunc%d", cpu); 180 rc = bind_ipi_to_irqhandler(XEN_CALL_FUNCTION_VECTOR, 181 cpu, 182 xen_call_function_interrupt, 183 IRQF_PERCPU|IRQF_NOBALANCING, 184 callfunc_name, 185 NULL); 186 if (rc < 0) 187 goto fail; 188 per_cpu(xen_callfunc_irq, cpu).irq = rc; 189 per_cpu(xen_callfunc_irq, cpu).name = callfunc_name; 190 191 debug_name = kasprintf(GFP_KERNEL, "debug%d", cpu); 192 rc = bind_virq_to_irqhandler(VIRQ_DEBUG, cpu, xen_debug_interrupt, 193 IRQF_PERCPU | IRQF_NOBALANCING, 194 debug_name, NULL); 195 if (rc < 0) 196 goto fail; 197 per_cpu(xen_debug_irq, cpu).irq = rc; 198 per_cpu(xen_debug_irq, cpu).name = debug_name; 199 200 callfunc_name = kasprintf(GFP_KERNEL, "callfuncsingle%d", cpu); 201 rc = bind_ipi_to_irqhandler(XEN_CALL_FUNCTION_SINGLE_VECTOR, 202 cpu, 203 xen_call_function_single_interrupt, 204 IRQF_PERCPU|IRQF_NOBALANCING, 205 callfunc_name, 206 NULL); 207 if (rc < 0) 208 goto fail; 209 per_cpu(xen_callfuncsingle_irq, cpu).irq = rc; 210 per_cpu(xen_callfuncsingle_irq, cpu).name = callfunc_name; 211 212 /* 213 * The IRQ worker on PVHVM goes through the native path and uses the 214 * IPI mechanism. 215 */ 216 if (xen_hvm_domain()) 217 return 0; 218 219 callfunc_name = kasprintf(GFP_KERNEL, "irqwork%d", cpu); 220 rc = bind_ipi_to_irqhandler(XEN_IRQ_WORK_VECTOR, 221 cpu, 222 xen_irq_work_interrupt, 223 IRQF_PERCPU|IRQF_NOBALANCING, 224 callfunc_name, 225 NULL); 226 if (rc < 0) 227 goto fail; 228 per_cpu(xen_irq_work, cpu).irq = rc; 229 per_cpu(xen_irq_work, cpu).name = callfunc_name; 230 231 if (is_xen_pmu(cpu)) { 232 pmu_name = kasprintf(GFP_KERNEL, "pmu%d", cpu); 233 rc = bind_virq_to_irqhandler(VIRQ_XENPMU, cpu, 234 xen_pmu_irq_handler, 235 IRQF_PERCPU|IRQF_NOBALANCING, 236 pmu_name, NULL); 237 if (rc < 0) 238 goto fail; 239 per_cpu(xen_pmu_irq, cpu).irq = rc; 240 per_cpu(xen_pmu_irq, cpu).name = pmu_name; 241 } 242 243 return 0; 244 245 fail: 246 xen_smp_intr_free(cpu); 247 return rc; 248 } 249 250 static void __init xen_fill_possible_map(void) 251 { 252 int i, rc; 253 254 if (xen_initial_domain()) 255 return; 256 257 for (i = 0; i < nr_cpu_ids; i++) { 258 rc = HYPERVISOR_vcpu_op(VCPUOP_is_up, i, NULL); 259 if (rc >= 0) { 260 num_processors++; 261 set_cpu_possible(i, true); 262 } 263 } 264 } 265 266 static void __init xen_filter_cpu_maps(void) 267 { 268 int i, rc; 269 unsigned int subtract = 0; 270 271 if (!xen_initial_domain()) 272 return; 273 274 num_processors = 0; 275 disabled_cpus = 0; 276 for (i = 0; i < nr_cpu_ids; i++) { 277 rc = HYPERVISOR_vcpu_op(VCPUOP_is_up, i, NULL); 278 if (rc >= 0) { 279 num_processors++; 280 set_cpu_possible(i, true); 281 } else { 282 set_cpu_possible(i, false); 283 set_cpu_present(i, false); 284 subtract++; 285 } 286 } 287 #ifdef CONFIG_HOTPLUG_CPU 288 /* This is akin to using 'nr_cpus' on the Linux command line. 289 * Which is OK as when we use 'dom0_max_vcpus=X' we can only 290 * have up to X, while nr_cpu_ids is greater than X. This 291 * normally is not a problem, except when CPU hotplugging 292 * is involved and then there might be more than X CPUs 293 * in the guest - which will not work as there is no 294 * hypercall to expand the max number of VCPUs an already 295 * running guest has. So cap it up to X. */ 296 if (subtract) 297 nr_cpu_ids = nr_cpu_ids - subtract; 298 #endif 299 300 } 301 302 static void __init xen_smp_prepare_boot_cpu(void) 303 { 304 BUG_ON(smp_processor_id() != 0); 305 native_smp_prepare_boot_cpu(); 306 307 if (xen_pv_domain()) { 308 if (!xen_feature(XENFEAT_writable_page_tables)) 309 /* We've switched to the "real" per-cpu gdt, so make 310 * sure the old memory can be recycled. */ 311 make_lowmem_page_readwrite(xen_initial_gdt); 312 313 #ifdef CONFIG_X86_32 314 /* 315 * Xen starts us with XEN_FLAT_RING1_DS, but linux code 316 * expects __USER_DS 317 */ 318 loadsegment(ds, __USER_DS); 319 loadsegment(es, __USER_DS); 320 #endif 321 322 xen_filter_cpu_maps(); 323 xen_setup_vcpu_info_placement(); 324 } 325 326 /* 327 * Setup vcpu_info for boot CPU. 328 */ 329 if (xen_hvm_domain()) 330 xen_vcpu_setup(0); 331 332 /* 333 * The alternative logic (which patches the unlock/lock) runs before 334 * the smp bootup up code is activated. Hence we need to set this up 335 * the core kernel is being patched. Otherwise we will have only 336 * modules patched but not core code. 337 */ 338 xen_init_spinlocks(); 339 } 340 341 static void __init xen_smp_prepare_cpus(unsigned int max_cpus) 342 { 343 unsigned cpu; 344 unsigned int i; 345 346 if (skip_ioapic_setup) { 347 char *m = (max_cpus == 0) ? 348 "The nosmp parameter is incompatible with Xen; " \ 349 "use Xen dom0_max_vcpus=1 parameter" : 350 "The noapic parameter is incompatible with Xen"; 351 352 xen_raw_printk(m); 353 panic(m); 354 } 355 xen_init_lock_cpu(0); 356 357 smp_store_boot_cpu_info(); 358 cpu_data(0).x86_max_cores = 1; 359 360 for_each_possible_cpu(i) { 361 zalloc_cpumask_var(&per_cpu(cpu_sibling_map, i), GFP_KERNEL); 362 zalloc_cpumask_var(&per_cpu(cpu_core_map, i), GFP_KERNEL); 363 zalloc_cpumask_var(&per_cpu(cpu_llc_shared_map, i), GFP_KERNEL); 364 } 365 set_cpu_sibling_map(0); 366 367 xen_pmu_init(0); 368 369 if (xen_smp_intr_init(0)) 370 BUG(); 371 372 if (!alloc_cpumask_var(&xen_cpu_initialized_map, GFP_KERNEL)) 373 panic("could not allocate xen_cpu_initialized_map\n"); 374 375 cpumask_copy(xen_cpu_initialized_map, cpumask_of(0)); 376 377 /* Restrict the possible_map according to max_cpus. */ 378 while ((num_possible_cpus() > 1) && (num_possible_cpus() > max_cpus)) { 379 for (cpu = nr_cpu_ids - 1; !cpu_possible(cpu); cpu--) 380 continue; 381 set_cpu_possible(cpu, false); 382 } 383 384 for_each_possible_cpu(cpu) 385 set_cpu_present(cpu, true); 386 } 387 388 static int 389 cpu_initialize_context(unsigned int cpu, struct task_struct *idle) 390 { 391 struct vcpu_guest_context *ctxt; 392 struct desc_struct *gdt; 393 unsigned long gdt_mfn; 394 395 /* used to tell cpu_init() that it can proceed with initialization */ 396 cpumask_set_cpu(cpu, cpu_callout_mask); 397 if (cpumask_test_and_set_cpu(cpu, xen_cpu_initialized_map)) 398 return 0; 399 400 ctxt = kzalloc(sizeof(*ctxt), GFP_KERNEL); 401 if (ctxt == NULL) 402 return -ENOMEM; 403 404 gdt = get_cpu_gdt_table(cpu); 405 406 #ifdef CONFIG_X86_32 407 /* Note: PVH is not yet supported on x86_32. */ 408 ctxt->user_regs.fs = __KERNEL_PERCPU; 409 ctxt->user_regs.gs = __KERNEL_STACK_CANARY; 410 #endif 411 memset(&ctxt->fpu_ctxt, 0, sizeof(ctxt->fpu_ctxt)); 412 413 if (!xen_feature(XENFEAT_auto_translated_physmap)) { 414 ctxt->user_regs.eip = (unsigned long)cpu_bringup_and_idle; 415 ctxt->flags = VGCF_IN_KERNEL; 416 ctxt->user_regs.eflags = 0x1000; /* IOPL_RING1 */ 417 ctxt->user_regs.ds = __USER_DS; 418 ctxt->user_regs.es = __USER_DS; 419 ctxt->user_regs.ss = __KERNEL_DS; 420 421 xen_copy_trap_info(ctxt->trap_ctxt); 422 423 ctxt->ldt_ents = 0; 424 425 BUG_ON((unsigned long)gdt & ~PAGE_MASK); 426 427 gdt_mfn = arbitrary_virt_to_mfn(gdt); 428 make_lowmem_page_readonly(gdt); 429 make_lowmem_page_readonly(mfn_to_virt(gdt_mfn)); 430 431 ctxt->gdt_frames[0] = gdt_mfn; 432 ctxt->gdt_ents = GDT_ENTRIES; 433 434 ctxt->kernel_ss = __KERNEL_DS; 435 ctxt->kernel_sp = idle->thread.sp0; 436 437 #ifdef CONFIG_X86_32 438 ctxt->event_callback_cs = __KERNEL_CS; 439 ctxt->failsafe_callback_cs = __KERNEL_CS; 440 #else 441 ctxt->gs_base_kernel = per_cpu_offset(cpu); 442 #endif 443 ctxt->event_callback_eip = 444 (unsigned long)xen_hypervisor_callback; 445 ctxt->failsafe_callback_eip = 446 (unsigned long)xen_failsafe_callback; 447 ctxt->user_regs.cs = __KERNEL_CS; 448 per_cpu(xen_cr3, cpu) = __pa(swapper_pg_dir); 449 } 450 #ifdef CONFIG_XEN_PVH 451 else { 452 /* 453 * The vcpu comes on kernel page tables which have the NX pte 454 * bit set. This means before DS/SS is touched, NX in 455 * EFER must be set. Hence the following assembly glue code. 456 */ 457 ctxt->user_regs.eip = (unsigned long)xen_pvh_early_cpu_init; 458 ctxt->user_regs.rdi = cpu; 459 ctxt->user_regs.rsi = true; /* entry == true */ 460 } 461 #endif 462 ctxt->user_regs.esp = idle->thread.sp0 - sizeof(struct pt_regs); 463 ctxt->ctrlreg[3] = xen_pfn_to_cr3(virt_to_gfn(swapper_pg_dir)); 464 if (HYPERVISOR_vcpu_op(VCPUOP_initialise, xen_vcpu_nr(cpu), ctxt)) 465 BUG(); 466 467 kfree(ctxt); 468 return 0; 469 } 470 471 static int xen_cpu_up(unsigned int cpu, struct task_struct *idle) 472 { 473 int rc; 474 475 common_cpu_up(cpu, idle); 476 477 xen_setup_runstate_info(cpu); 478 479 /* 480 * PV VCPUs are always successfully taken down (see 'while' loop 481 * in xen_cpu_die()), so -EBUSY is an error. 482 */ 483 rc = cpu_check_up_prepare(cpu); 484 if (rc) 485 return rc; 486 487 /* make sure interrupts start blocked */ 488 per_cpu(xen_vcpu, cpu)->evtchn_upcall_mask = 1; 489 490 rc = cpu_initialize_context(cpu, idle); 491 if (rc) 492 return rc; 493 494 xen_pmu_init(cpu); 495 496 rc = HYPERVISOR_vcpu_op(VCPUOP_up, xen_vcpu_nr(cpu), NULL); 497 BUG_ON(rc); 498 499 while (cpu_report_state(cpu) != CPU_ONLINE) 500 HYPERVISOR_sched_op(SCHEDOP_yield, NULL); 501 502 return 0; 503 } 504 505 static void xen_smp_cpus_done(unsigned int max_cpus) 506 { 507 } 508 509 #ifdef CONFIG_HOTPLUG_CPU 510 static int xen_cpu_disable(void) 511 { 512 unsigned int cpu = smp_processor_id(); 513 if (cpu == 0) 514 return -EBUSY; 515 516 cpu_disable_common(); 517 518 load_cr3(swapper_pg_dir); 519 return 0; 520 } 521 522 static void xen_cpu_die(unsigned int cpu) 523 { 524 while (xen_pv_domain() && HYPERVISOR_vcpu_op(VCPUOP_is_up, 525 xen_vcpu_nr(cpu), NULL)) { 526 __set_current_state(TASK_UNINTERRUPTIBLE); 527 schedule_timeout(HZ/10); 528 } 529 530 if (common_cpu_die(cpu) == 0) { 531 xen_smp_intr_free(cpu); 532 xen_uninit_lock_cpu(cpu); 533 xen_teardown_timer(cpu); 534 xen_pmu_finish(cpu); 535 } 536 } 537 538 static void xen_play_dead(void) /* used only with HOTPLUG_CPU */ 539 { 540 play_dead_common(); 541 HYPERVISOR_vcpu_op(VCPUOP_down, xen_vcpu_nr(smp_processor_id()), NULL); 542 cpu_bringup(); 543 /* 544 * commit 4b0c0f294 (tick: Cleanup NOHZ per cpu data on cpu down) 545 * clears certain data that the cpu_idle loop (which called us 546 * and that we return from) expects. The only way to get that 547 * data back is to call: 548 */ 549 tick_nohz_idle_enter(); 550 551 cpu_startup_entry(CPUHP_AP_ONLINE_IDLE); 552 } 553 554 #else /* !CONFIG_HOTPLUG_CPU */ 555 static int xen_cpu_disable(void) 556 { 557 return -ENOSYS; 558 } 559 560 static void xen_cpu_die(unsigned int cpu) 561 { 562 BUG(); 563 } 564 565 static void xen_play_dead(void) 566 { 567 BUG(); 568 } 569 570 #endif 571 static void stop_self(void *v) 572 { 573 int cpu = smp_processor_id(); 574 575 /* make sure we're not pinning something down */ 576 load_cr3(swapper_pg_dir); 577 /* should set up a minimal gdt */ 578 579 set_cpu_online(cpu, false); 580 581 HYPERVISOR_vcpu_op(VCPUOP_down, xen_vcpu_nr(cpu), NULL); 582 BUG(); 583 } 584 585 static void xen_stop_other_cpus(int wait) 586 { 587 smp_call_function(stop_self, NULL, wait); 588 } 589 590 static void xen_smp_send_reschedule(int cpu) 591 { 592 xen_send_IPI_one(cpu, XEN_RESCHEDULE_VECTOR); 593 } 594 595 static void __xen_send_IPI_mask(const struct cpumask *mask, 596 int vector) 597 { 598 unsigned cpu; 599 600 for_each_cpu_and(cpu, mask, cpu_online_mask) 601 xen_send_IPI_one(cpu, vector); 602 } 603 604 static void xen_smp_send_call_function_ipi(const struct cpumask *mask) 605 { 606 int cpu; 607 608 __xen_send_IPI_mask(mask, XEN_CALL_FUNCTION_VECTOR); 609 610 /* Make sure other vcpus get a chance to run if they need to. */ 611 for_each_cpu(cpu, mask) { 612 if (xen_vcpu_stolen(cpu)) { 613 HYPERVISOR_sched_op(SCHEDOP_yield, NULL); 614 break; 615 } 616 } 617 } 618 619 static void xen_smp_send_call_function_single_ipi(int cpu) 620 { 621 __xen_send_IPI_mask(cpumask_of(cpu), 622 XEN_CALL_FUNCTION_SINGLE_VECTOR); 623 } 624 625 static inline int xen_map_vector(int vector) 626 { 627 int xen_vector; 628 629 switch (vector) { 630 case RESCHEDULE_VECTOR: 631 xen_vector = XEN_RESCHEDULE_VECTOR; 632 break; 633 case CALL_FUNCTION_VECTOR: 634 xen_vector = XEN_CALL_FUNCTION_VECTOR; 635 break; 636 case CALL_FUNCTION_SINGLE_VECTOR: 637 xen_vector = XEN_CALL_FUNCTION_SINGLE_VECTOR; 638 break; 639 case IRQ_WORK_VECTOR: 640 xen_vector = XEN_IRQ_WORK_VECTOR; 641 break; 642 #ifdef CONFIG_X86_64 643 case NMI_VECTOR: 644 case APIC_DM_NMI: /* Some use that instead of NMI_VECTOR */ 645 xen_vector = XEN_NMI_VECTOR; 646 break; 647 #endif 648 default: 649 xen_vector = -1; 650 printk(KERN_ERR "xen: vector 0x%x is not implemented\n", 651 vector); 652 } 653 654 return xen_vector; 655 } 656 657 void xen_send_IPI_mask(const struct cpumask *mask, 658 int vector) 659 { 660 int xen_vector = xen_map_vector(vector); 661 662 if (xen_vector >= 0) 663 __xen_send_IPI_mask(mask, xen_vector); 664 } 665 666 void xen_send_IPI_all(int vector) 667 { 668 int xen_vector = xen_map_vector(vector); 669 670 if (xen_vector >= 0) 671 __xen_send_IPI_mask(cpu_online_mask, xen_vector); 672 } 673 674 void xen_send_IPI_self(int vector) 675 { 676 int xen_vector = xen_map_vector(vector); 677 678 if (xen_vector >= 0) 679 xen_send_IPI_one(smp_processor_id(), xen_vector); 680 } 681 682 void xen_send_IPI_mask_allbutself(const struct cpumask *mask, 683 int vector) 684 { 685 unsigned cpu; 686 unsigned int this_cpu = smp_processor_id(); 687 int xen_vector = xen_map_vector(vector); 688 689 if (!(num_online_cpus() > 1) || (xen_vector < 0)) 690 return; 691 692 for_each_cpu_and(cpu, mask, cpu_online_mask) { 693 if (this_cpu == cpu) 694 continue; 695 696 xen_send_IPI_one(cpu, xen_vector); 697 } 698 } 699 700 void xen_send_IPI_allbutself(int vector) 701 { 702 xen_send_IPI_mask_allbutself(cpu_online_mask, vector); 703 } 704 705 static irqreturn_t xen_call_function_interrupt(int irq, void *dev_id) 706 { 707 irq_enter(); 708 generic_smp_call_function_interrupt(); 709 inc_irq_stat(irq_call_count); 710 irq_exit(); 711 712 return IRQ_HANDLED; 713 } 714 715 static irqreturn_t xen_call_function_single_interrupt(int irq, void *dev_id) 716 { 717 irq_enter(); 718 generic_smp_call_function_single_interrupt(); 719 inc_irq_stat(irq_call_count); 720 irq_exit(); 721 722 return IRQ_HANDLED; 723 } 724 725 static irqreturn_t xen_irq_work_interrupt(int irq, void *dev_id) 726 { 727 irq_enter(); 728 irq_work_run(); 729 inc_irq_stat(apic_irq_work_irqs); 730 irq_exit(); 731 732 return IRQ_HANDLED; 733 } 734 735 static const struct smp_ops xen_smp_ops __initconst = { 736 .smp_prepare_boot_cpu = xen_smp_prepare_boot_cpu, 737 .smp_prepare_cpus = xen_smp_prepare_cpus, 738 .smp_cpus_done = xen_smp_cpus_done, 739 740 .cpu_up = xen_cpu_up, 741 .cpu_die = xen_cpu_die, 742 .cpu_disable = xen_cpu_disable, 743 .play_dead = xen_play_dead, 744 745 .stop_other_cpus = xen_stop_other_cpus, 746 .smp_send_reschedule = xen_smp_send_reschedule, 747 748 .send_call_func_ipi = xen_smp_send_call_function_ipi, 749 .send_call_func_single_ipi = xen_smp_send_call_function_single_ipi, 750 }; 751 752 void __init xen_smp_init(void) 753 { 754 smp_ops = xen_smp_ops; 755 xen_fill_possible_map(); 756 } 757 758 static void __init xen_hvm_smp_prepare_cpus(unsigned int max_cpus) 759 { 760 native_smp_prepare_cpus(max_cpus); 761 WARN_ON(xen_smp_intr_init(0)); 762 763 xen_init_lock_cpu(0); 764 } 765 766 void __init xen_hvm_smp_init(void) 767 { 768 smp_ops.smp_prepare_cpus = xen_hvm_smp_prepare_cpus; 769 smp_ops.smp_send_reschedule = xen_smp_send_reschedule; 770 smp_ops.cpu_die = xen_cpu_die; 771 smp_ops.send_call_func_ipi = xen_smp_send_call_function_ipi; 772 smp_ops.send_call_func_single_ipi = xen_smp_send_call_function_single_ipi; 773 smp_ops.smp_prepare_boot_cpu = xen_smp_prepare_boot_cpu; 774 } 775