1 /* 2 * SMP support for ppc. 3 * 4 * Written by Cort Dougan (cort@cs.nmt.edu) borrowing a great 5 * deal of code from the sparc and intel versions. 6 * 7 * Copyright (C) 1999 Cort Dougan <cort@cs.nmt.edu> 8 * 9 * PowerPC-64 Support added by Dave Engebretsen, Peter Bergner, and 10 * Mike Corrigan {engebret|bergner|mikec}@us.ibm.com 11 * 12 * This program is free software; you can redistribute it and/or 13 * modify it under the terms of the GNU General Public License 14 * as published by the Free Software Foundation; either version 15 * 2 of the License, or (at your option) any later version. 16 */ 17 18 #undef DEBUG 19 20 #include <linux/kernel.h> 21 #include <linux/export.h> 22 #include <linux/sched.h> 23 #include <linux/smp.h> 24 #include <linux/interrupt.h> 25 #include <linux/delay.h> 26 #include <linux/init.h> 27 #include <linux/spinlock.h> 28 #include <linux/cache.h> 29 #include <linux/err.h> 30 #include <linux/device.h> 31 #include <linux/cpu.h> 32 #include <linux/notifier.h> 33 #include <linux/topology.h> 34 #include <linux/profile.h> 35 36 #include <asm/ptrace.h> 37 #include <linux/atomic.h> 38 #include <asm/irq.h> 39 #include <asm/hw_irq.h> 40 #include <asm/kvm_ppc.h> 41 #include <asm/page.h> 42 #include <asm/pgtable.h> 43 #include <asm/prom.h> 44 #include <asm/smp.h> 45 #include <asm/time.h> 46 #include <asm/machdep.h> 47 #include <asm/cputhreads.h> 48 #include <asm/cputable.h> 49 #include <asm/mpic.h> 50 #include <asm/vdso_datapage.h> 51 #ifdef CONFIG_PPC64 52 #include <asm/paca.h> 53 #endif 54 #include <asm/vdso.h> 55 #include <asm/debug.h> 56 #include <asm/kexec.h> 57 #include <asm/asm-prototypes.h> 58 59 #ifdef DEBUG 60 #include <asm/udbg.h> 61 #define DBG(fmt...) udbg_printf(fmt) 62 #else 63 #define DBG(fmt...) 64 #endif 65 66 #ifdef CONFIG_HOTPLUG_CPU 67 /* State of each CPU during hotplug phases */ 68 static DEFINE_PER_CPU(int, cpu_state) = { 0 }; 69 #endif 70 71 struct thread_info *secondary_ti; 72 73 DEFINE_PER_CPU(cpumask_var_t, cpu_sibling_map); 74 DEFINE_PER_CPU(cpumask_var_t, cpu_core_map); 75 76 EXPORT_PER_CPU_SYMBOL(cpu_sibling_map); 77 EXPORT_PER_CPU_SYMBOL(cpu_core_map); 78 79 /* SMP operations for this machine */ 80 struct smp_ops_t *smp_ops; 81 82 /* Can't be static due to PowerMac hackery */ 83 volatile unsigned int cpu_callin_map[NR_CPUS]; 84 85 int smt_enabled_at_boot = 1; 86 87 static void (*crash_ipi_function_ptr)(struct pt_regs *) = NULL; 88 89 /* 90 * Returns 1 if the specified cpu should be brought up during boot. 91 * Used to inhibit booting threads if they've been disabled or 92 * limited on the command line 93 */ 94 int smp_generic_cpu_bootable(unsigned int nr) 95 { 96 /* Special case - we inhibit secondary thread startup 97 * during boot if the user requests it. 98 */ 99 if (system_state == SYSTEM_BOOTING && cpu_has_feature(CPU_FTR_SMT)) { 100 if (!smt_enabled_at_boot && cpu_thread_in_core(nr) != 0) 101 return 0; 102 if (smt_enabled_at_boot 103 && cpu_thread_in_core(nr) >= smt_enabled_at_boot) 104 return 0; 105 } 106 107 return 1; 108 } 109 110 111 #ifdef CONFIG_PPC64 112 int smp_generic_kick_cpu(int nr) 113 { 114 BUG_ON(nr < 0 || nr >= NR_CPUS); 115 116 /* 117 * The processor is currently spinning, waiting for the 118 * cpu_start field to become non-zero After we set cpu_start, 119 * the processor will continue on to secondary_start 120 */ 121 if (!paca[nr].cpu_start) { 122 paca[nr].cpu_start = 1; 123 smp_mb(); 124 return 0; 125 } 126 127 #ifdef CONFIG_HOTPLUG_CPU 128 /* 129 * Ok it's not there, so it might be soft-unplugged, let's 130 * try to bring it back 131 */ 132 generic_set_cpu_up(nr); 133 smp_wmb(); 134 smp_send_reschedule(nr); 135 #endif /* CONFIG_HOTPLUG_CPU */ 136 137 return 0; 138 } 139 #endif /* CONFIG_PPC64 */ 140 141 static irqreturn_t call_function_action(int irq, void *data) 142 { 143 generic_smp_call_function_interrupt(); 144 return IRQ_HANDLED; 145 } 146 147 static irqreturn_t reschedule_action(int irq, void *data) 148 { 149 scheduler_ipi(); 150 return IRQ_HANDLED; 151 } 152 153 static irqreturn_t tick_broadcast_ipi_action(int irq, void *data) 154 { 155 tick_broadcast_ipi_handler(); 156 return IRQ_HANDLED; 157 } 158 159 static irqreturn_t debug_ipi_action(int irq, void *data) 160 { 161 if (crash_ipi_function_ptr) { 162 crash_ipi_function_ptr(get_irq_regs()); 163 return IRQ_HANDLED; 164 } 165 166 #ifdef CONFIG_DEBUGGER 167 debugger_ipi(get_irq_regs()); 168 #endif /* CONFIG_DEBUGGER */ 169 170 return IRQ_HANDLED; 171 } 172 173 static irq_handler_t smp_ipi_action[] = { 174 [PPC_MSG_CALL_FUNCTION] = call_function_action, 175 [PPC_MSG_RESCHEDULE] = reschedule_action, 176 [PPC_MSG_TICK_BROADCAST] = tick_broadcast_ipi_action, 177 [PPC_MSG_DEBUGGER_BREAK] = debug_ipi_action, 178 }; 179 180 const char *smp_ipi_name[] = { 181 [PPC_MSG_CALL_FUNCTION] = "ipi call function", 182 [PPC_MSG_RESCHEDULE] = "ipi reschedule", 183 [PPC_MSG_TICK_BROADCAST] = "ipi tick-broadcast", 184 [PPC_MSG_DEBUGGER_BREAK] = "ipi debugger", 185 }; 186 187 /* optional function to request ipi, for controllers with >= 4 ipis */ 188 int smp_request_message_ipi(int virq, int msg) 189 { 190 int err; 191 192 if (msg < 0 || msg > PPC_MSG_DEBUGGER_BREAK) { 193 return -EINVAL; 194 } 195 #if !defined(CONFIG_DEBUGGER) && !defined(CONFIG_KEXEC) 196 if (msg == PPC_MSG_DEBUGGER_BREAK) { 197 return 1; 198 } 199 #endif 200 err = request_irq(virq, smp_ipi_action[msg], 201 IRQF_PERCPU | IRQF_NO_THREAD | IRQF_NO_SUSPEND, 202 smp_ipi_name[msg], NULL); 203 WARN(err < 0, "unable to request_irq %d for %s (rc %d)\n", 204 virq, smp_ipi_name[msg], err); 205 206 return err; 207 } 208 209 #ifdef CONFIG_PPC_SMP_MUXED_IPI 210 struct cpu_messages { 211 long messages; /* current messages */ 212 unsigned long data; /* data for cause ipi */ 213 }; 214 static DEFINE_PER_CPU_SHARED_ALIGNED(struct cpu_messages, ipi_message); 215 216 void smp_muxed_ipi_set_data(int cpu, unsigned long data) 217 { 218 struct cpu_messages *info = &per_cpu(ipi_message, cpu); 219 220 info->data = data; 221 } 222 223 void smp_muxed_ipi_set_message(int cpu, int msg) 224 { 225 struct cpu_messages *info = &per_cpu(ipi_message, cpu); 226 char *message = (char *)&info->messages; 227 228 /* 229 * Order previous accesses before accesses in the IPI handler. 230 */ 231 smp_mb(); 232 message[msg] = 1; 233 } 234 235 void smp_muxed_ipi_message_pass(int cpu, int msg) 236 { 237 struct cpu_messages *info = &per_cpu(ipi_message, cpu); 238 239 smp_muxed_ipi_set_message(cpu, msg); 240 /* 241 * cause_ipi functions are required to include a full barrier 242 * before doing whatever causes the IPI. 243 */ 244 smp_ops->cause_ipi(cpu, info->data); 245 } 246 247 #ifdef __BIG_ENDIAN__ 248 #define IPI_MESSAGE(A) (1uL << ((BITS_PER_LONG - 8) - 8 * (A))) 249 #else 250 #define IPI_MESSAGE(A) (1uL << (8 * (A))) 251 #endif 252 253 irqreturn_t smp_ipi_demux(void) 254 { 255 struct cpu_messages *info = this_cpu_ptr(&ipi_message); 256 unsigned long all; 257 258 mb(); /* order any irq clear */ 259 260 do { 261 all = xchg(&info->messages, 0); 262 #if defined(CONFIG_KVM_XICS) && defined(CONFIG_KVM_BOOK3S_HV_POSSIBLE) 263 /* 264 * Must check for PPC_MSG_RM_HOST_ACTION messages 265 * before PPC_MSG_CALL_FUNCTION messages because when 266 * a VM is destroyed, we call kick_all_cpus_sync() 267 * to ensure that any pending PPC_MSG_RM_HOST_ACTION 268 * messages have completed before we free any VCPUs. 269 */ 270 if (all & IPI_MESSAGE(PPC_MSG_RM_HOST_ACTION)) 271 kvmppc_xics_ipi_action(); 272 #endif 273 if (all & IPI_MESSAGE(PPC_MSG_CALL_FUNCTION)) 274 generic_smp_call_function_interrupt(); 275 if (all & IPI_MESSAGE(PPC_MSG_RESCHEDULE)) 276 scheduler_ipi(); 277 if (all & IPI_MESSAGE(PPC_MSG_TICK_BROADCAST)) 278 tick_broadcast_ipi_handler(); 279 if (all & IPI_MESSAGE(PPC_MSG_DEBUGGER_BREAK)) 280 debug_ipi_action(0, NULL); 281 } while (info->messages); 282 283 return IRQ_HANDLED; 284 } 285 #endif /* CONFIG_PPC_SMP_MUXED_IPI */ 286 287 static inline void do_message_pass(int cpu, int msg) 288 { 289 if (smp_ops->message_pass) 290 smp_ops->message_pass(cpu, msg); 291 #ifdef CONFIG_PPC_SMP_MUXED_IPI 292 else 293 smp_muxed_ipi_message_pass(cpu, msg); 294 #endif 295 } 296 297 void smp_send_reschedule(int cpu) 298 { 299 if (likely(smp_ops)) 300 do_message_pass(cpu, PPC_MSG_RESCHEDULE); 301 } 302 EXPORT_SYMBOL_GPL(smp_send_reschedule); 303 304 void arch_send_call_function_single_ipi(int cpu) 305 { 306 do_message_pass(cpu, PPC_MSG_CALL_FUNCTION); 307 } 308 309 void arch_send_call_function_ipi_mask(const struct cpumask *mask) 310 { 311 unsigned int cpu; 312 313 for_each_cpu(cpu, mask) 314 do_message_pass(cpu, PPC_MSG_CALL_FUNCTION); 315 } 316 317 #ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST 318 void tick_broadcast(const struct cpumask *mask) 319 { 320 unsigned int cpu; 321 322 for_each_cpu(cpu, mask) 323 do_message_pass(cpu, PPC_MSG_TICK_BROADCAST); 324 } 325 #endif 326 327 #if defined(CONFIG_DEBUGGER) || defined(CONFIG_KEXEC) 328 void smp_send_debugger_break(void) 329 { 330 int cpu; 331 int me = raw_smp_processor_id(); 332 333 if (unlikely(!smp_ops)) 334 return; 335 336 for_each_online_cpu(cpu) 337 if (cpu != me) 338 do_message_pass(cpu, PPC_MSG_DEBUGGER_BREAK); 339 } 340 #endif 341 342 #ifdef CONFIG_KEXEC 343 void crash_send_ipi(void (*crash_ipi_callback)(struct pt_regs *)) 344 { 345 crash_ipi_function_ptr = crash_ipi_callback; 346 if (crash_ipi_callback) { 347 mb(); 348 smp_send_debugger_break(); 349 } 350 } 351 #endif 352 353 static void stop_this_cpu(void *dummy) 354 { 355 /* Remove this CPU */ 356 set_cpu_online(smp_processor_id(), false); 357 358 local_irq_disable(); 359 while (1) 360 ; 361 } 362 363 void smp_send_stop(void) 364 { 365 smp_call_function(stop_this_cpu, NULL, 0); 366 } 367 368 struct thread_info *current_set[NR_CPUS]; 369 370 static void smp_store_cpu_info(int id) 371 { 372 per_cpu(cpu_pvr, id) = mfspr(SPRN_PVR); 373 #ifdef CONFIG_PPC_FSL_BOOK3E 374 per_cpu(next_tlbcam_idx, id) 375 = (mfspr(SPRN_TLB1CFG) & TLBnCFG_N_ENTRY) - 1; 376 #endif 377 } 378 379 void __init smp_prepare_cpus(unsigned int max_cpus) 380 { 381 unsigned int cpu; 382 383 DBG("smp_prepare_cpus\n"); 384 385 /* 386 * setup_cpu may need to be called on the boot cpu. We havent 387 * spun any cpus up but lets be paranoid. 388 */ 389 BUG_ON(boot_cpuid != smp_processor_id()); 390 391 /* Fixup boot cpu */ 392 smp_store_cpu_info(boot_cpuid); 393 cpu_callin_map[boot_cpuid] = 1; 394 395 for_each_possible_cpu(cpu) { 396 zalloc_cpumask_var_node(&per_cpu(cpu_sibling_map, cpu), 397 GFP_KERNEL, cpu_to_node(cpu)); 398 zalloc_cpumask_var_node(&per_cpu(cpu_core_map, cpu), 399 GFP_KERNEL, cpu_to_node(cpu)); 400 /* 401 * numa_node_id() works after this. 402 */ 403 if (cpu_present(cpu)) { 404 set_cpu_numa_node(cpu, numa_cpu_lookup_table[cpu]); 405 set_cpu_numa_mem(cpu, 406 local_memory_node(numa_cpu_lookup_table[cpu])); 407 } 408 } 409 410 cpumask_set_cpu(boot_cpuid, cpu_sibling_mask(boot_cpuid)); 411 cpumask_set_cpu(boot_cpuid, cpu_core_mask(boot_cpuid)); 412 413 if (smp_ops && smp_ops->probe) 414 smp_ops->probe(); 415 } 416 417 void smp_prepare_boot_cpu(void) 418 { 419 BUG_ON(smp_processor_id() != boot_cpuid); 420 #ifdef CONFIG_PPC64 421 paca[boot_cpuid].__current = current; 422 #endif 423 set_numa_node(numa_cpu_lookup_table[boot_cpuid]); 424 current_set[boot_cpuid] = task_thread_info(current); 425 } 426 427 #ifdef CONFIG_HOTPLUG_CPU 428 429 int generic_cpu_disable(void) 430 { 431 unsigned int cpu = smp_processor_id(); 432 433 if (cpu == boot_cpuid) 434 return -EBUSY; 435 436 set_cpu_online(cpu, false); 437 #ifdef CONFIG_PPC64 438 vdso_data->processorCount--; 439 #endif 440 migrate_irqs(); 441 return 0; 442 } 443 444 void generic_cpu_die(unsigned int cpu) 445 { 446 int i; 447 448 for (i = 0; i < 100; i++) { 449 smp_rmb(); 450 if (is_cpu_dead(cpu)) 451 return; 452 msleep(100); 453 } 454 printk(KERN_ERR "CPU%d didn't die...\n", cpu); 455 } 456 457 void generic_set_cpu_dead(unsigned int cpu) 458 { 459 per_cpu(cpu_state, cpu) = CPU_DEAD; 460 } 461 462 /* 463 * The cpu_state should be set to CPU_UP_PREPARE in kick_cpu(), otherwise 464 * the cpu_state is always CPU_DEAD after calling generic_set_cpu_dead(), 465 * which makes the delay in generic_cpu_die() not happen. 466 */ 467 void generic_set_cpu_up(unsigned int cpu) 468 { 469 per_cpu(cpu_state, cpu) = CPU_UP_PREPARE; 470 } 471 472 int generic_check_cpu_restart(unsigned int cpu) 473 { 474 return per_cpu(cpu_state, cpu) == CPU_UP_PREPARE; 475 } 476 477 int is_cpu_dead(unsigned int cpu) 478 { 479 return per_cpu(cpu_state, cpu) == CPU_DEAD; 480 } 481 482 static bool secondaries_inhibited(void) 483 { 484 return kvm_hv_mode_active(); 485 } 486 487 #else /* HOTPLUG_CPU */ 488 489 #define secondaries_inhibited() 0 490 491 #endif 492 493 static void cpu_idle_thread_init(unsigned int cpu, struct task_struct *idle) 494 { 495 struct thread_info *ti = task_thread_info(idle); 496 497 #ifdef CONFIG_PPC64 498 paca[cpu].__current = idle; 499 paca[cpu].kstack = (unsigned long)ti + THREAD_SIZE - STACK_FRAME_OVERHEAD; 500 #endif 501 ti->cpu = cpu; 502 secondary_ti = current_set[cpu] = ti; 503 } 504 505 int __cpu_up(unsigned int cpu, struct task_struct *tidle) 506 { 507 int rc, c; 508 509 /* 510 * Don't allow secondary threads to come online if inhibited 511 */ 512 if (threads_per_core > 1 && secondaries_inhibited() && 513 cpu_thread_in_subcore(cpu)) 514 return -EBUSY; 515 516 if (smp_ops == NULL || 517 (smp_ops->cpu_bootable && !smp_ops->cpu_bootable(cpu))) 518 return -EINVAL; 519 520 cpu_idle_thread_init(cpu, tidle); 521 522 /* Make sure callin-map entry is 0 (can be leftover a CPU 523 * hotplug 524 */ 525 cpu_callin_map[cpu] = 0; 526 527 /* The information for processor bringup must 528 * be written out to main store before we release 529 * the processor. 530 */ 531 smp_mb(); 532 533 /* wake up cpus */ 534 DBG("smp: kicking cpu %d\n", cpu); 535 rc = smp_ops->kick_cpu(cpu); 536 if (rc) { 537 pr_err("smp: failed starting cpu %d (rc %d)\n", cpu, rc); 538 return rc; 539 } 540 541 /* 542 * wait to see if the cpu made a callin (is actually up). 543 * use this value that I found through experimentation. 544 * -- Cort 545 */ 546 if (system_state < SYSTEM_RUNNING) 547 for (c = 50000; c && !cpu_callin_map[cpu]; c--) 548 udelay(100); 549 #ifdef CONFIG_HOTPLUG_CPU 550 else 551 /* 552 * CPUs can take much longer to come up in the 553 * hotplug case. Wait five seconds. 554 */ 555 for (c = 5000; c && !cpu_callin_map[cpu]; c--) 556 msleep(1); 557 #endif 558 559 if (!cpu_callin_map[cpu]) { 560 printk(KERN_ERR "Processor %u is stuck.\n", cpu); 561 return -ENOENT; 562 } 563 564 DBG("Processor %u found.\n", cpu); 565 566 if (smp_ops->give_timebase) 567 smp_ops->give_timebase(); 568 569 /* Wait until cpu puts itself in the online & active maps */ 570 while (!cpu_online(cpu)) 571 cpu_relax(); 572 573 return 0; 574 } 575 576 /* Return the value of the reg property corresponding to the given 577 * logical cpu. 578 */ 579 int cpu_to_core_id(int cpu) 580 { 581 struct device_node *np; 582 const __be32 *reg; 583 int id = -1; 584 585 np = of_get_cpu_node(cpu, NULL); 586 if (!np) 587 goto out; 588 589 reg = of_get_property(np, "reg", NULL); 590 if (!reg) 591 goto out; 592 593 id = be32_to_cpup(reg); 594 out: 595 of_node_put(np); 596 return id; 597 } 598 EXPORT_SYMBOL_GPL(cpu_to_core_id); 599 600 /* Helper routines for cpu to core mapping */ 601 int cpu_core_index_of_thread(int cpu) 602 { 603 return cpu >> threads_shift; 604 } 605 EXPORT_SYMBOL_GPL(cpu_core_index_of_thread); 606 607 int cpu_first_thread_of_core(int core) 608 { 609 return core << threads_shift; 610 } 611 EXPORT_SYMBOL_GPL(cpu_first_thread_of_core); 612 613 static void traverse_siblings_chip_id(int cpu, bool add, int chipid) 614 { 615 const struct cpumask *mask; 616 struct device_node *np; 617 int i, plen; 618 const __be32 *prop; 619 620 mask = add ? cpu_online_mask : cpu_present_mask; 621 for_each_cpu(i, mask) { 622 np = of_get_cpu_node(i, NULL); 623 if (!np) 624 continue; 625 prop = of_get_property(np, "ibm,chip-id", &plen); 626 if (prop && plen == sizeof(int) && 627 of_read_number(prop, 1) == chipid) { 628 if (add) { 629 cpumask_set_cpu(cpu, cpu_core_mask(i)); 630 cpumask_set_cpu(i, cpu_core_mask(cpu)); 631 } else { 632 cpumask_clear_cpu(cpu, cpu_core_mask(i)); 633 cpumask_clear_cpu(i, cpu_core_mask(cpu)); 634 } 635 } 636 of_node_put(np); 637 } 638 } 639 640 /* Must be called when no change can occur to cpu_present_mask, 641 * i.e. during cpu online or offline. 642 */ 643 static struct device_node *cpu_to_l2cache(int cpu) 644 { 645 struct device_node *np; 646 struct device_node *cache; 647 648 if (!cpu_present(cpu)) 649 return NULL; 650 651 np = of_get_cpu_node(cpu, NULL); 652 if (np == NULL) 653 return NULL; 654 655 cache = of_find_next_cache_node(np); 656 657 of_node_put(np); 658 659 return cache; 660 } 661 662 static void traverse_core_siblings(int cpu, bool add) 663 { 664 struct device_node *l2_cache, *np; 665 const struct cpumask *mask; 666 int i, chip, plen; 667 const __be32 *prop; 668 669 /* First see if we have ibm,chip-id properties in cpu nodes */ 670 np = of_get_cpu_node(cpu, NULL); 671 if (np) { 672 chip = -1; 673 prop = of_get_property(np, "ibm,chip-id", &plen); 674 if (prop && plen == sizeof(int)) 675 chip = of_read_number(prop, 1); 676 of_node_put(np); 677 if (chip >= 0) { 678 traverse_siblings_chip_id(cpu, add, chip); 679 return; 680 } 681 } 682 683 l2_cache = cpu_to_l2cache(cpu); 684 mask = add ? cpu_online_mask : cpu_present_mask; 685 for_each_cpu(i, mask) { 686 np = cpu_to_l2cache(i); 687 if (!np) 688 continue; 689 if (np == l2_cache) { 690 if (add) { 691 cpumask_set_cpu(cpu, cpu_core_mask(i)); 692 cpumask_set_cpu(i, cpu_core_mask(cpu)); 693 } else { 694 cpumask_clear_cpu(cpu, cpu_core_mask(i)); 695 cpumask_clear_cpu(i, cpu_core_mask(cpu)); 696 } 697 } 698 of_node_put(np); 699 } 700 of_node_put(l2_cache); 701 } 702 703 /* Activate a secondary processor. */ 704 void start_secondary(void *unused) 705 { 706 unsigned int cpu = smp_processor_id(); 707 int i, base; 708 709 atomic_inc(&init_mm.mm_count); 710 current->active_mm = &init_mm; 711 712 smp_store_cpu_info(cpu); 713 set_dec(tb_ticks_per_jiffy); 714 preempt_disable(); 715 cpu_callin_map[cpu] = 1; 716 717 if (smp_ops->setup_cpu) 718 smp_ops->setup_cpu(cpu); 719 if (smp_ops->take_timebase) 720 smp_ops->take_timebase(); 721 722 secondary_cpu_time_init(); 723 724 #ifdef CONFIG_PPC64 725 if (system_state == SYSTEM_RUNNING) 726 vdso_data->processorCount++; 727 728 vdso_getcpu_init(); 729 #endif 730 /* Update sibling maps */ 731 base = cpu_first_thread_sibling(cpu); 732 for (i = 0; i < threads_per_core; i++) { 733 if (cpu_is_offline(base + i) && (cpu != base + i)) 734 continue; 735 cpumask_set_cpu(cpu, cpu_sibling_mask(base + i)); 736 cpumask_set_cpu(base + i, cpu_sibling_mask(cpu)); 737 738 /* cpu_core_map should be a superset of 739 * cpu_sibling_map even if we don't have cache 740 * information, so update the former here, too. 741 */ 742 cpumask_set_cpu(cpu, cpu_core_mask(base + i)); 743 cpumask_set_cpu(base + i, cpu_core_mask(cpu)); 744 } 745 traverse_core_siblings(cpu, true); 746 747 set_numa_node(numa_cpu_lookup_table[cpu]); 748 set_numa_mem(local_memory_node(numa_cpu_lookup_table[cpu])); 749 750 smp_wmb(); 751 notify_cpu_starting(cpu); 752 set_cpu_online(cpu, true); 753 754 local_irq_enable(); 755 756 cpu_startup_entry(CPUHP_AP_ONLINE_IDLE); 757 758 BUG(); 759 } 760 761 int setup_profiling_timer(unsigned int multiplier) 762 { 763 return 0; 764 } 765 766 #ifdef CONFIG_SCHED_SMT 767 /* cpumask of CPUs with asymetric SMT dependancy */ 768 static int powerpc_smt_flags(void) 769 { 770 int flags = SD_SHARE_CPUCAPACITY | SD_SHARE_PKG_RESOURCES; 771 772 if (cpu_has_feature(CPU_FTR_ASYM_SMT)) { 773 printk_once(KERN_INFO "Enabling Asymmetric SMT scheduling\n"); 774 flags |= SD_ASYM_PACKING; 775 } 776 return flags; 777 } 778 #endif 779 780 static struct sched_domain_topology_level powerpc_topology[] = { 781 #ifdef CONFIG_SCHED_SMT 782 { cpu_smt_mask, powerpc_smt_flags, SD_INIT_NAME(SMT) }, 783 #endif 784 { cpu_cpu_mask, SD_INIT_NAME(DIE) }, 785 { NULL, }, 786 }; 787 788 void __init smp_cpus_done(unsigned int max_cpus) 789 { 790 cpumask_var_t old_mask; 791 792 /* We want the setup_cpu() here to be called from CPU 0, but our 793 * init thread may have been "borrowed" by another CPU in the meantime 794 * se we pin us down to CPU 0 for a short while 795 */ 796 alloc_cpumask_var(&old_mask, GFP_NOWAIT); 797 cpumask_copy(old_mask, tsk_cpus_allowed(current)); 798 set_cpus_allowed_ptr(current, cpumask_of(boot_cpuid)); 799 800 if (smp_ops && smp_ops->setup_cpu) 801 smp_ops->setup_cpu(boot_cpuid); 802 803 set_cpus_allowed_ptr(current, old_mask); 804 805 free_cpumask_var(old_mask); 806 807 if (smp_ops && smp_ops->bringup_done) 808 smp_ops->bringup_done(); 809 810 dump_numa_cpu_topology(); 811 812 set_sched_topology(powerpc_topology); 813 814 } 815 816 #ifdef CONFIG_HOTPLUG_CPU 817 int __cpu_disable(void) 818 { 819 int cpu = smp_processor_id(); 820 int base, i; 821 int err; 822 823 if (!smp_ops->cpu_disable) 824 return -ENOSYS; 825 826 err = smp_ops->cpu_disable(); 827 if (err) 828 return err; 829 830 /* Update sibling maps */ 831 base = cpu_first_thread_sibling(cpu); 832 for (i = 0; i < threads_per_core; i++) { 833 cpumask_clear_cpu(cpu, cpu_sibling_mask(base + i)); 834 cpumask_clear_cpu(base + i, cpu_sibling_mask(cpu)); 835 cpumask_clear_cpu(cpu, cpu_core_mask(base + i)); 836 cpumask_clear_cpu(base + i, cpu_core_mask(cpu)); 837 } 838 traverse_core_siblings(cpu, false); 839 840 return 0; 841 } 842 843 void __cpu_die(unsigned int cpu) 844 { 845 if (smp_ops->cpu_die) 846 smp_ops->cpu_die(cpu); 847 } 848 849 void cpu_die(void) 850 { 851 if (ppc_md.cpu_die) 852 ppc_md.cpu_die(); 853 854 /* If we return, we re-enter start_secondary */ 855 start_secondary_resume(); 856 } 857 858 #endif 859