1 /* 2 * SMP related functions 3 * 4 * Copyright IBM Corp. 1999,2012 5 * Author(s): Denis Joseph Barrow, 6 * Martin Schwidefsky <schwidefsky@de.ibm.com>, 7 * Heiko Carstens <heiko.carstens@de.ibm.com>, 8 * 9 * based on other smp stuff by 10 * (c) 1995 Alan Cox, CymruNET Ltd <alan@cymru.net> 11 * (c) 1998 Ingo Molnar 12 * 13 * The code outside of smp.c uses logical cpu numbers, only smp.c does 14 * the translation of logical to physical cpu ids. All new code that 15 * operates on physical cpu numbers needs to go into smp.c. 16 */ 17 18 #define KMSG_COMPONENT "cpu" 19 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt 20 21 #include <linux/workqueue.h> 22 #include <linux/module.h> 23 #include <linux/init.h> 24 #include <linux/mm.h> 25 #include <linux/err.h> 26 #include <linux/spinlock.h> 27 #include <linux/kernel_stat.h> 28 #include <linux/delay.h> 29 #include <linux/interrupt.h> 30 #include <linux/irqflags.h> 31 #include <linux/cpu.h> 32 #include <linux/slab.h> 33 #include <linux/crash_dump.h> 34 #include <asm/asm-offsets.h> 35 #include <asm/switch_to.h> 36 #include <asm/facility.h> 37 #include <asm/ipl.h> 38 #include <asm/setup.h> 39 #include <asm/irq.h> 40 #include <asm/tlbflush.h> 41 #include <asm/timer.h> 42 #include <asm/lowcore.h> 43 #include <asm/sclp.h> 44 #include <asm/vdso.h> 45 #include <asm/debug.h> 46 #include <asm/os_info.h> 47 #include "entry.h" 48 49 enum { 50 sigp_sense = 1, 51 sigp_external_call = 2, 52 sigp_emergency_signal = 3, 53 sigp_start = 4, 54 sigp_stop = 5, 55 sigp_restart = 6, 56 sigp_stop_and_store_status = 9, 57 sigp_initial_cpu_reset = 11, 58 sigp_cpu_reset = 12, 59 sigp_set_prefix = 13, 60 sigp_store_status_at_address = 14, 61 sigp_store_extended_status_at_address = 15, 62 sigp_set_architecture = 18, 63 sigp_conditional_emergency_signal = 19, 64 sigp_sense_running = 21, 65 }; 66 67 enum { 68 sigp_order_code_accepted = 0, 69 sigp_status_stored = 1, 70 sigp_busy = 2, 71 sigp_not_operational = 3, 72 }; 73 74 enum { 75 ec_schedule = 0, 76 ec_call_function, 77 ec_call_function_single, 78 ec_stop_cpu, 79 }; 80 81 enum { 82 CPU_STATE_STANDBY, 83 CPU_STATE_CONFIGURED, 84 }; 85 86 struct pcpu { 87 struct cpu cpu; 88 struct _lowcore *lowcore; /* lowcore page(s) for the cpu */ 89 unsigned long async_stack; /* async stack for the cpu */ 90 unsigned long panic_stack; /* panic stack for the cpu */ 91 unsigned long ec_mask; /* bit mask for ec_xxx functions */ 92 int state; /* physical cpu state */ 93 u32 status; /* last status received via sigp */ 94 u16 address; /* physical cpu address */ 95 }; 96 97 static u8 boot_cpu_type; 98 static u16 boot_cpu_address; 99 static struct pcpu pcpu_devices[NR_CPUS]; 100 101 DEFINE_MUTEX(smp_cpu_state_mutex); 102 103 /* 104 * Signal processor helper functions. 105 */ 106 static inline int __pcpu_sigp(u16 addr, u8 order, u32 parm, u32 *status) 107 { 108 register unsigned int reg1 asm ("1") = parm; 109 int cc; 110 111 asm volatile( 112 " sigp %1,%2,0(%3)\n" 113 " ipm %0\n" 114 " srl %0,28\n" 115 : "=d" (cc), "+d" (reg1) : "d" (addr), "a" (order) : "cc"); 116 if (status && cc == 1) 117 *status = reg1; 118 return cc; 119 } 120 121 static inline int __pcpu_sigp_relax(u16 addr, u8 order, u32 parm, u32 *status) 122 { 123 int cc; 124 125 while (1) { 126 cc = __pcpu_sigp(addr, order, parm, status); 127 if (cc != sigp_busy) 128 return cc; 129 cpu_relax(); 130 } 131 } 132 133 static int pcpu_sigp_retry(struct pcpu *pcpu, u8 order, u32 parm) 134 { 135 int cc, retry; 136 137 for (retry = 0; ; retry++) { 138 cc = __pcpu_sigp(pcpu->address, order, parm, &pcpu->status); 139 if (cc != sigp_busy) 140 break; 141 if (retry >= 3) 142 udelay(10); 143 } 144 return cc; 145 } 146 147 static inline int pcpu_stopped(struct pcpu *pcpu) 148 { 149 if (__pcpu_sigp(pcpu->address, sigp_sense, 150 0, &pcpu->status) != sigp_status_stored) 151 return 0; 152 /* Check for stopped and check stop state */ 153 return !!(pcpu->status & 0x50); 154 } 155 156 static inline int pcpu_running(struct pcpu *pcpu) 157 { 158 if (__pcpu_sigp(pcpu->address, sigp_sense_running, 159 0, &pcpu->status) != sigp_status_stored) 160 return 1; 161 /* Check for running status */ 162 return !(pcpu->status & 0x400); 163 } 164 165 /* 166 * Find struct pcpu by cpu address. 167 */ 168 static struct pcpu *pcpu_find_address(const struct cpumask *mask, int address) 169 { 170 int cpu; 171 172 for_each_cpu(cpu, mask) 173 if (pcpu_devices[cpu].address == address) 174 return pcpu_devices + cpu; 175 return NULL; 176 } 177 178 static void pcpu_ec_call(struct pcpu *pcpu, int ec_bit) 179 { 180 int order; 181 182 set_bit(ec_bit, &pcpu->ec_mask); 183 order = pcpu_running(pcpu) ? 184 sigp_external_call : sigp_emergency_signal; 185 pcpu_sigp_retry(pcpu, order, 0); 186 } 187 188 static int __cpuinit pcpu_alloc_lowcore(struct pcpu *pcpu, int cpu) 189 { 190 struct _lowcore *lc; 191 192 if (pcpu != &pcpu_devices[0]) { 193 pcpu->lowcore = (struct _lowcore *) 194 __get_free_pages(GFP_KERNEL | GFP_DMA, LC_ORDER); 195 pcpu->async_stack = __get_free_pages(GFP_KERNEL, ASYNC_ORDER); 196 pcpu->panic_stack = __get_free_page(GFP_KERNEL); 197 if (!pcpu->lowcore || !pcpu->panic_stack || !pcpu->async_stack) 198 goto out; 199 } 200 lc = pcpu->lowcore; 201 memcpy(lc, &S390_lowcore, 512); 202 memset((char *) lc + 512, 0, sizeof(*lc) - 512); 203 lc->async_stack = pcpu->async_stack + ASYNC_SIZE; 204 lc->panic_stack = pcpu->panic_stack + PAGE_SIZE; 205 lc->cpu_nr = cpu; 206 #ifndef CONFIG_64BIT 207 if (MACHINE_HAS_IEEE) { 208 lc->extended_save_area_addr = get_zeroed_page(GFP_KERNEL); 209 if (!lc->extended_save_area_addr) 210 goto out; 211 } 212 #else 213 if (vdso_alloc_per_cpu(lc)) 214 goto out; 215 #endif 216 lowcore_ptr[cpu] = lc; 217 pcpu_sigp_retry(pcpu, sigp_set_prefix, (u32)(unsigned long) lc); 218 return 0; 219 out: 220 if (pcpu != &pcpu_devices[0]) { 221 free_page(pcpu->panic_stack); 222 free_pages(pcpu->async_stack, ASYNC_ORDER); 223 free_pages((unsigned long) pcpu->lowcore, LC_ORDER); 224 } 225 return -ENOMEM; 226 } 227 228 #ifdef CONFIG_HOTPLUG_CPU 229 230 static void pcpu_free_lowcore(struct pcpu *pcpu) 231 { 232 pcpu_sigp_retry(pcpu, sigp_set_prefix, 0); 233 lowcore_ptr[pcpu - pcpu_devices] = NULL; 234 #ifndef CONFIG_64BIT 235 if (MACHINE_HAS_IEEE) { 236 struct _lowcore *lc = pcpu->lowcore; 237 238 free_page((unsigned long) lc->extended_save_area_addr); 239 lc->extended_save_area_addr = 0; 240 } 241 #else 242 vdso_free_per_cpu(pcpu->lowcore); 243 #endif 244 if (pcpu != &pcpu_devices[0]) { 245 free_page(pcpu->panic_stack); 246 free_pages(pcpu->async_stack, ASYNC_ORDER); 247 free_pages((unsigned long) pcpu->lowcore, LC_ORDER); 248 } 249 } 250 251 #endif /* CONFIG_HOTPLUG_CPU */ 252 253 static void pcpu_prepare_secondary(struct pcpu *pcpu, int cpu) 254 { 255 struct _lowcore *lc = pcpu->lowcore; 256 257 atomic_inc(&init_mm.context.attach_count); 258 lc->cpu_nr = cpu; 259 lc->percpu_offset = __per_cpu_offset[cpu]; 260 lc->kernel_asce = S390_lowcore.kernel_asce; 261 lc->machine_flags = S390_lowcore.machine_flags; 262 lc->ftrace_func = S390_lowcore.ftrace_func; 263 lc->user_timer = lc->system_timer = lc->steal_timer = 0; 264 __ctl_store(lc->cregs_save_area, 0, 15); 265 save_access_regs((unsigned int *) lc->access_regs_save_area); 266 memcpy(lc->stfle_fac_list, S390_lowcore.stfle_fac_list, 267 MAX_FACILITY_BIT/8); 268 } 269 270 static void pcpu_attach_task(struct pcpu *pcpu, struct task_struct *tsk) 271 { 272 struct _lowcore *lc = pcpu->lowcore; 273 struct thread_info *ti = task_thread_info(tsk); 274 275 lc->kernel_stack = (unsigned long) task_stack_page(tsk) + THREAD_SIZE; 276 lc->thread_info = (unsigned long) task_thread_info(tsk); 277 lc->current_task = (unsigned long) tsk; 278 lc->user_timer = ti->user_timer; 279 lc->system_timer = ti->system_timer; 280 lc->steal_timer = 0; 281 } 282 283 static void pcpu_start_fn(struct pcpu *pcpu, void (*func)(void *), void *data) 284 { 285 struct _lowcore *lc = pcpu->lowcore; 286 287 lc->restart_stack = lc->kernel_stack; 288 lc->restart_fn = (unsigned long) func; 289 lc->restart_data = (unsigned long) data; 290 lc->restart_source = -1UL; 291 pcpu_sigp_retry(pcpu, sigp_restart, 0); 292 } 293 294 /* 295 * Call function via PSW restart on pcpu and stop the current cpu. 296 */ 297 static void pcpu_delegate(struct pcpu *pcpu, void (*func)(void *), 298 void *data, unsigned long stack) 299 { 300 struct _lowcore *lc = lowcore_ptr[pcpu - pcpu_devices]; 301 struct { 302 unsigned long stack; 303 void *func; 304 void *data; 305 unsigned long source; 306 } restart = { stack, func, data, stap() }; 307 308 __load_psw_mask(psw_kernel_bits); 309 if (pcpu->address == restart.source) 310 func(data); /* should not return */ 311 /* Stop target cpu (if func returns this stops the current cpu). */ 312 pcpu_sigp_retry(pcpu, sigp_stop, 0); 313 /* Restart func on the target cpu and stop the current cpu. */ 314 memcpy_absolute(&lc->restart_stack, &restart, sizeof(restart)); 315 asm volatile( 316 "0: sigp 0,%0,6 # sigp restart to target cpu\n" 317 " brc 2,0b # busy, try again\n" 318 "1: sigp 0,%1,5 # sigp stop to current cpu\n" 319 " brc 2,1b # busy, try again\n" 320 : : "d" (pcpu->address), "d" (restart.source) : "0", "1", "cc"); 321 for (;;) ; 322 } 323 324 /* 325 * Call function on an online CPU. 326 */ 327 void smp_call_online_cpu(void (*func)(void *), void *data) 328 { 329 struct pcpu *pcpu; 330 331 /* Use the current cpu if it is online. */ 332 pcpu = pcpu_find_address(cpu_online_mask, stap()); 333 if (!pcpu) 334 /* Use the first online cpu. */ 335 pcpu = pcpu_devices + cpumask_first(cpu_online_mask); 336 pcpu_delegate(pcpu, func, data, (unsigned long) restart_stack); 337 } 338 339 /* 340 * Call function on the ipl CPU. 341 */ 342 void smp_call_ipl_cpu(void (*func)(void *), void *data) 343 { 344 pcpu_delegate(&pcpu_devices[0], func, data, 345 pcpu_devices->panic_stack + PAGE_SIZE); 346 } 347 348 int smp_find_processor_id(u16 address) 349 { 350 int cpu; 351 352 for_each_present_cpu(cpu) 353 if (pcpu_devices[cpu].address == address) 354 return cpu; 355 return -1; 356 } 357 358 int smp_vcpu_scheduled(int cpu) 359 { 360 return pcpu_running(pcpu_devices + cpu); 361 } 362 363 void smp_yield(void) 364 { 365 if (MACHINE_HAS_DIAG44) 366 asm volatile("diag 0,0,0x44"); 367 } 368 369 void smp_yield_cpu(int cpu) 370 { 371 if (MACHINE_HAS_DIAG9C) 372 asm volatile("diag %0,0,0x9c" 373 : : "d" (pcpu_devices[cpu].address)); 374 else if (MACHINE_HAS_DIAG44) 375 asm volatile("diag 0,0,0x44"); 376 } 377 378 /* 379 * Send cpus emergency shutdown signal. This gives the cpus the 380 * opportunity to complete outstanding interrupts. 381 */ 382 void smp_emergency_stop(cpumask_t *cpumask) 383 { 384 u64 end; 385 int cpu; 386 387 end = get_clock() + (1000000UL << 12); 388 for_each_cpu(cpu, cpumask) { 389 struct pcpu *pcpu = pcpu_devices + cpu; 390 set_bit(ec_stop_cpu, &pcpu->ec_mask); 391 while (__pcpu_sigp(pcpu->address, sigp_emergency_signal, 392 0, NULL) == sigp_busy && 393 get_clock() < end) 394 cpu_relax(); 395 } 396 while (get_clock() < end) { 397 for_each_cpu(cpu, cpumask) 398 if (pcpu_stopped(pcpu_devices + cpu)) 399 cpumask_clear_cpu(cpu, cpumask); 400 if (cpumask_empty(cpumask)) 401 break; 402 cpu_relax(); 403 } 404 } 405 406 /* 407 * Stop all cpus but the current one. 408 */ 409 void smp_send_stop(void) 410 { 411 cpumask_t cpumask; 412 int cpu; 413 414 /* Disable all interrupts/machine checks */ 415 __load_psw_mask(psw_kernel_bits | PSW_MASK_DAT); 416 trace_hardirqs_off(); 417 418 debug_set_critical(); 419 cpumask_copy(&cpumask, cpu_online_mask); 420 cpumask_clear_cpu(smp_processor_id(), &cpumask); 421 422 if (oops_in_progress) 423 smp_emergency_stop(&cpumask); 424 425 /* stop all processors */ 426 for_each_cpu(cpu, &cpumask) { 427 struct pcpu *pcpu = pcpu_devices + cpu; 428 pcpu_sigp_retry(pcpu, sigp_stop, 0); 429 while (!pcpu_stopped(pcpu)) 430 cpu_relax(); 431 } 432 } 433 434 /* 435 * Stop the current cpu. 436 */ 437 void smp_stop_cpu(void) 438 { 439 pcpu_sigp_retry(pcpu_devices + smp_processor_id(), sigp_stop, 0); 440 for (;;) ; 441 } 442 443 /* 444 * This is the main routine where commands issued by other 445 * cpus are handled. 446 */ 447 static void do_ext_call_interrupt(struct ext_code ext_code, 448 unsigned int param32, unsigned long param64) 449 { 450 unsigned long bits; 451 int cpu; 452 453 cpu = smp_processor_id(); 454 if (ext_code.code == 0x1202) 455 kstat_cpu(cpu).irqs[EXTINT_EXC]++; 456 else 457 kstat_cpu(cpu).irqs[EXTINT_EMS]++; 458 /* 459 * handle bit signal external calls 460 */ 461 bits = xchg(&pcpu_devices[cpu].ec_mask, 0); 462 463 if (test_bit(ec_stop_cpu, &bits)) 464 smp_stop_cpu(); 465 466 if (test_bit(ec_schedule, &bits)) 467 scheduler_ipi(); 468 469 if (test_bit(ec_call_function, &bits)) 470 generic_smp_call_function_interrupt(); 471 472 if (test_bit(ec_call_function_single, &bits)) 473 generic_smp_call_function_single_interrupt(); 474 475 } 476 477 void arch_send_call_function_ipi_mask(const struct cpumask *mask) 478 { 479 int cpu; 480 481 for_each_cpu(cpu, mask) 482 pcpu_ec_call(pcpu_devices + cpu, ec_call_function); 483 } 484 485 void arch_send_call_function_single_ipi(int cpu) 486 { 487 pcpu_ec_call(pcpu_devices + cpu, ec_call_function_single); 488 } 489 490 #ifndef CONFIG_64BIT 491 /* 492 * this function sends a 'purge tlb' signal to another CPU. 493 */ 494 static void smp_ptlb_callback(void *info) 495 { 496 __tlb_flush_local(); 497 } 498 499 void smp_ptlb_all(void) 500 { 501 on_each_cpu(smp_ptlb_callback, NULL, 1); 502 } 503 EXPORT_SYMBOL(smp_ptlb_all); 504 #endif /* ! CONFIG_64BIT */ 505 506 /* 507 * this function sends a 'reschedule' IPI to another CPU. 508 * it goes straight through and wastes no time serializing 509 * anything. Worst case is that we lose a reschedule ... 510 */ 511 void smp_send_reschedule(int cpu) 512 { 513 pcpu_ec_call(pcpu_devices + cpu, ec_schedule); 514 } 515 516 /* 517 * parameter area for the set/clear control bit callbacks 518 */ 519 struct ec_creg_mask_parms { 520 unsigned long orval; 521 unsigned long andval; 522 int cr; 523 }; 524 525 /* 526 * callback for setting/clearing control bits 527 */ 528 static void smp_ctl_bit_callback(void *info) 529 { 530 struct ec_creg_mask_parms *pp = info; 531 unsigned long cregs[16]; 532 533 __ctl_store(cregs, 0, 15); 534 cregs[pp->cr] = (cregs[pp->cr] & pp->andval) | pp->orval; 535 __ctl_load(cregs, 0, 15); 536 } 537 538 /* 539 * Set a bit in a control register of all cpus 540 */ 541 void smp_ctl_set_bit(int cr, int bit) 542 { 543 struct ec_creg_mask_parms parms = { 1UL << bit, -1UL, cr }; 544 545 on_each_cpu(smp_ctl_bit_callback, &parms, 1); 546 } 547 EXPORT_SYMBOL(smp_ctl_set_bit); 548 549 /* 550 * Clear a bit in a control register of all cpus 551 */ 552 void smp_ctl_clear_bit(int cr, int bit) 553 { 554 struct ec_creg_mask_parms parms = { 0, ~(1UL << bit), cr }; 555 556 on_each_cpu(smp_ctl_bit_callback, &parms, 1); 557 } 558 EXPORT_SYMBOL(smp_ctl_clear_bit); 559 560 #if defined(CONFIG_ZFCPDUMP) || defined(CONFIG_CRASH_DUMP) 561 562 struct save_area *zfcpdump_save_areas[NR_CPUS + 1]; 563 EXPORT_SYMBOL_GPL(zfcpdump_save_areas); 564 565 static void __init smp_get_save_area(int cpu, u16 address) 566 { 567 void *lc = pcpu_devices[0].lowcore; 568 struct save_area *save_area; 569 570 if (is_kdump_kernel()) 571 return; 572 if (!OLDMEM_BASE && (address == boot_cpu_address || 573 ipl_info.type != IPL_TYPE_FCP_DUMP)) 574 return; 575 if (cpu >= NR_CPUS) { 576 pr_warning("CPU %i exceeds the maximum %i and is excluded " 577 "from the dump\n", cpu, NR_CPUS - 1); 578 return; 579 } 580 save_area = kmalloc(sizeof(struct save_area), GFP_KERNEL); 581 if (!save_area) 582 panic("could not allocate memory for save area\n"); 583 zfcpdump_save_areas[cpu] = save_area; 584 #ifdef CONFIG_CRASH_DUMP 585 if (address == boot_cpu_address) { 586 /* Copy the registers of the boot cpu. */ 587 copy_oldmem_page(1, (void *) save_area, sizeof(*save_area), 588 SAVE_AREA_BASE - PAGE_SIZE, 0); 589 return; 590 } 591 #endif 592 /* Get the registers of a non-boot cpu. */ 593 __pcpu_sigp_relax(address, sigp_stop_and_store_status, 0, NULL); 594 memcpy_real(save_area, lc + SAVE_AREA_BASE, sizeof(*save_area)); 595 } 596 597 int smp_store_status(int cpu) 598 { 599 struct pcpu *pcpu; 600 601 pcpu = pcpu_devices + cpu; 602 if (__pcpu_sigp_relax(pcpu->address, sigp_stop_and_store_status, 603 0, NULL) != sigp_order_code_accepted) 604 return -EIO; 605 return 0; 606 } 607 608 #else /* CONFIG_ZFCPDUMP || CONFIG_CRASH_DUMP */ 609 610 static inline void smp_get_save_area(int cpu, u16 address) { } 611 612 #endif /* CONFIG_ZFCPDUMP || CONFIG_CRASH_DUMP */ 613 614 static struct sclp_cpu_info *smp_get_cpu_info(void) 615 { 616 static int use_sigp_detection; 617 struct sclp_cpu_info *info; 618 int address; 619 620 info = kzalloc(sizeof(*info), GFP_KERNEL); 621 if (info && (use_sigp_detection || sclp_get_cpu_info(info))) { 622 use_sigp_detection = 1; 623 for (address = 0; address <= MAX_CPU_ADDRESS; address++) { 624 if (__pcpu_sigp_relax(address, sigp_sense, 0, NULL) == 625 sigp_not_operational) 626 continue; 627 info->cpu[info->configured].address = address; 628 info->configured++; 629 } 630 info->combined = info->configured; 631 } 632 return info; 633 } 634 635 static int __devinit smp_add_present_cpu(int cpu); 636 637 static int __devinit __smp_rescan_cpus(struct sclp_cpu_info *info, 638 int sysfs_add) 639 { 640 struct pcpu *pcpu; 641 cpumask_t avail; 642 int cpu, nr, i; 643 644 nr = 0; 645 cpumask_xor(&avail, cpu_possible_mask, cpu_present_mask); 646 cpu = cpumask_first(&avail); 647 for (i = 0; (i < info->combined) && (cpu < nr_cpu_ids); i++) { 648 if (info->has_cpu_type && info->cpu[i].type != boot_cpu_type) 649 continue; 650 if (pcpu_find_address(cpu_present_mask, info->cpu[i].address)) 651 continue; 652 pcpu = pcpu_devices + cpu; 653 pcpu->address = info->cpu[i].address; 654 pcpu->state = (cpu >= info->configured) ? 655 CPU_STATE_STANDBY : CPU_STATE_CONFIGURED; 656 cpu_set_polarization(cpu, POLARIZATION_UNKNOWN); 657 set_cpu_present(cpu, true); 658 if (sysfs_add && smp_add_present_cpu(cpu) != 0) 659 set_cpu_present(cpu, false); 660 else 661 nr++; 662 cpu = cpumask_next(cpu, &avail); 663 } 664 return nr; 665 } 666 667 static void __init smp_detect_cpus(void) 668 { 669 unsigned int cpu, c_cpus, s_cpus; 670 struct sclp_cpu_info *info; 671 672 info = smp_get_cpu_info(); 673 if (!info) 674 panic("smp_detect_cpus failed to allocate memory\n"); 675 if (info->has_cpu_type) { 676 for (cpu = 0; cpu < info->combined; cpu++) { 677 if (info->cpu[cpu].address != boot_cpu_address) 678 continue; 679 /* The boot cpu dictates the cpu type. */ 680 boot_cpu_type = info->cpu[cpu].type; 681 break; 682 } 683 } 684 c_cpus = s_cpus = 0; 685 for (cpu = 0; cpu < info->combined; cpu++) { 686 if (info->has_cpu_type && info->cpu[cpu].type != boot_cpu_type) 687 continue; 688 if (cpu < info->configured) { 689 smp_get_save_area(c_cpus, info->cpu[cpu].address); 690 c_cpus++; 691 } else 692 s_cpus++; 693 } 694 pr_info("%d configured CPUs, %d standby CPUs\n", c_cpus, s_cpus); 695 get_online_cpus(); 696 __smp_rescan_cpus(info, 0); 697 put_online_cpus(); 698 kfree(info); 699 } 700 701 /* 702 * Activate a secondary processor. 703 */ 704 static void __cpuinit smp_start_secondary(void *cpuvoid) 705 { 706 S390_lowcore.last_update_clock = get_clock(); 707 S390_lowcore.restart_stack = (unsigned long) restart_stack; 708 S390_lowcore.restart_fn = (unsigned long) do_restart; 709 S390_lowcore.restart_data = 0; 710 S390_lowcore.restart_source = -1UL; 711 restore_access_regs(S390_lowcore.access_regs_save_area); 712 __ctl_load(S390_lowcore.cregs_save_area, 0, 15); 713 __load_psw_mask(psw_kernel_bits | PSW_MASK_DAT); 714 cpu_init(); 715 preempt_disable(); 716 init_cpu_timer(); 717 init_cpu_vtimer(); 718 pfault_init(); 719 notify_cpu_starting(smp_processor_id()); 720 ipi_call_lock(); 721 set_cpu_online(smp_processor_id(), true); 722 ipi_call_unlock(); 723 local_irq_enable(); 724 /* cpu_idle will call schedule for us */ 725 cpu_idle(); 726 } 727 728 /* Upping and downing of CPUs */ 729 int __cpuinit __cpu_up(unsigned int cpu, struct task_struct *tidle) 730 { 731 struct pcpu *pcpu; 732 int rc; 733 734 pcpu = pcpu_devices + cpu; 735 if (pcpu->state != CPU_STATE_CONFIGURED) 736 return -EIO; 737 if (pcpu_sigp_retry(pcpu, sigp_initial_cpu_reset, 0) != 738 sigp_order_code_accepted) 739 return -EIO; 740 741 rc = pcpu_alloc_lowcore(pcpu, cpu); 742 if (rc) 743 return rc; 744 pcpu_prepare_secondary(pcpu, cpu); 745 pcpu_attach_task(pcpu, tidle); 746 pcpu_start_fn(pcpu, smp_start_secondary, NULL); 747 while (!cpu_online(cpu)) 748 cpu_relax(); 749 return 0; 750 } 751 752 static int __init setup_possible_cpus(char *s) 753 { 754 int max, cpu; 755 756 if (kstrtoint(s, 0, &max) < 0) 757 return 0; 758 init_cpu_possible(cpumask_of(0)); 759 for (cpu = 1; cpu < max && cpu < nr_cpu_ids; cpu++) 760 set_cpu_possible(cpu, true); 761 return 0; 762 } 763 early_param("possible_cpus", setup_possible_cpus); 764 765 #ifdef CONFIG_HOTPLUG_CPU 766 767 int __cpu_disable(void) 768 { 769 unsigned long cregs[16]; 770 771 set_cpu_online(smp_processor_id(), false); 772 /* Disable pseudo page faults on this cpu. */ 773 pfault_fini(); 774 /* Disable interrupt sources via control register. */ 775 __ctl_store(cregs, 0, 15); 776 cregs[0] &= ~0x0000ee70UL; /* disable all external interrupts */ 777 cregs[6] &= ~0xff000000UL; /* disable all I/O interrupts */ 778 cregs[14] &= ~0x1f000000UL; /* disable most machine checks */ 779 __ctl_load(cregs, 0, 15); 780 return 0; 781 } 782 783 void __cpu_die(unsigned int cpu) 784 { 785 struct pcpu *pcpu; 786 787 /* Wait until target cpu is down */ 788 pcpu = pcpu_devices + cpu; 789 while (!pcpu_stopped(pcpu)) 790 cpu_relax(); 791 pcpu_free_lowcore(pcpu); 792 atomic_dec(&init_mm.context.attach_count); 793 } 794 795 void __noreturn cpu_die(void) 796 { 797 idle_task_exit(); 798 pcpu_sigp_retry(pcpu_devices + smp_processor_id(), sigp_stop, 0); 799 for (;;) ; 800 } 801 802 #endif /* CONFIG_HOTPLUG_CPU */ 803 804 void __init smp_prepare_cpus(unsigned int max_cpus) 805 { 806 /* request the 0x1201 emergency signal external interrupt */ 807 if (register_external_interrupt(0x1201, do_ext_call_interrupt) != 0) 808 panic("Couldn't request external interrupt 0x1201"); 809 /* request the 0x1202 external call external interrupt */ 810 if (register_external_interrupt(0x1202, do_ext_call_interrupt) != 0) 811 panic("Couldn't request external interrupt 0x1202"); 812 smp_detect_cpus(); 813 } 814 815 void __init smp_prepare_boot_cpu(void) 816 { 817 struct pcpu *pcpu = pcpu_devices; 818 819 boot_cpu_address = stap(); 820 pcpu->state = CPU_STATE_CONFIGURED; 821 pcpu->address = boot_cpu_address; 822 pcpu->lowcore = (struct _lowcore *)(unsigned long) store_prefix(); 823 pcpu->async_stack = S390_lowcore.async_stack - ASYNC_SIZE; 824 pcpu->panic_stack = S390_lowcore.panic_stack - PAGE_SIZE; 825 S390_lowcore.percpu_offset = __per_cpu_offset[0]; 826 cpu_set_polarization(0, POLARIZATION_UNKNOWN); 827 set_cpu_present(0, true); 828 set_cpu_online(0, true); 829 } 830 831 void __init smp_cpus_done(unsigned int max_cpus) 832 { 833 } 834 835 void __init smp_setup_processor_id(void) 836 { 837 S390_lowcore.cpu_nr = 0; 838 } 839 840 /* 841 * the frequency of the profiling timer can be changed 842 * by writing a multiplier value into /proc/profile. 843 * 844 * usually you want to run this on all CPUs ;) 845 */ 846 int setup_profiling_timer(unsigned int multiplier) 847 { 848 return 0; 849 } 850 851 #ifdef CONFIG_HOTPLUG_CPU 852 static ssize_t cpu_configure_show(struct device *dev, 853 struct device_attribute *attr, char *buf) 854 { 855 ssize_t count; 856 857 mutex_lock(&smp_cpu_state_mutex); 858 count = sprintf(buf, "%d\n", pcpu_devices[dev->id].state); 859 mutex_unlock(&smp_cpu_state_mutex); 860 return count; 861 } 862 863 static ssize_t cpu_configure_store(struct device *dev, 864 struct device_attribute *attr, 865 const char *buf, size_t count) 866 { 867 struct pcpu *pcpu; 868 int cpu, val, rc; 869 char delim; 870 871 if (sscanf(buf, "%d %c", &val, &delim) != 1) 872 return -EINVAL; 873 if (val != 0 && val != 1) 874 return -EINVAL; 875 get_online_cpus(); 876 mutex_lock(&smp_cpu_state_mutex); 877 rc = -EBUSY; 878 /* disallow configuration changes of online cpus and cpu 0 */ 879 cpu = dev->id; 880 if (cpu_online(cpu) || cpu == 0) 881 goto out; 882 pcpu = pcpu_devices + cpu; 883 rc = 0; 884 switch (val) { 885 case 0: 886 if (pcpu->state != CPU_STATE_CONFIGURED) 887 break; 888 rc = sclp_cpu_deconfigure(pcpu->address); 889 if (rc) 890 break; 891 pcpu->state = CPU_STATE_STANDBY; 892 cpu_set_polarization(cpu, POLARIZATION_UNKNOWN); 893 topology_expect_change(); 894 break; 895 case 1: 896 if (pcpu->state != CPU_STATE_STANDBY) 897 break; 898 rc = sclp_cpu_configure(pcpu->address); 899 if (rc) 900 break; 901 pcpu->state = CPU_STATE_CONFIGURED; 902 cpu_set_polarization(cpu, POLARIZATION_UNKNOWN); 903 topology_expect_change(); 904 break; 905 default: 906 break; 907 } 908 out: 909 mutex_unlock(&smp_cpu_state_mutex); 910 put_online_cpus(); 911 return rc ? rc : count; 912 } 913 static DEVICE_ATTR(configure, 0644, cpu_configure_show, cpu_configure_store); 914 #endif /* CONFIG_HOTPLUG_CPU */ 915 916 static ssize_t show_cpu_address(struct device *dev, 917 struct device_attribute *attr, char *buf) 918 { 919 return sprintf(buf, "%d\n", pcpu_devices[dev->id].address); 920 } 921 static DEVICE_ATTR(address, 0444, show_cpu_address, NULL); 922 923 static struct attribute *cpu_common_attrs[] = { 924 #ifdef CONFIG_HOTPLUG_CPU 925 &dev_attr_configure.attr, 926 #endif 927 &dev_attr_address.attr, 928 NULL, 929 }; 930 931 static struct attribute_group cpu_common_attr_group = { 932 .attrs = cpu_common_attrs, 933 }; 934 935 static ssize_t show_idle_count(struct device *dev, 936 struct device_attribute *attr, char *buf) 937 { 938 struct s390_idle_data *idle = &per_cpu(s390_idle, dev->id); 939 unsigned long long idle_count; 940 unsigned int sequence; 941 942 do { 943 sequence = ACCESS_ONCE(idle->sequence); 944 idle_count = ACCESS_ONCE(idle->idle_count); 945 if (ACCESS_ONCE(idle->idle_enter)) 946 idle_count++; 947 } while ((sequence & 1) || (idle->sequence != sequence)); 948 return sprintf(buf, "%llu\n", idle_count); 949 } 950 static DEVICE_ATTR(idle_count, 0444, show_idle_count, NULL); 951 952 static ssize_t show_idle_time(struct device *dev, 953 struct device_attribute *attr, char *buf) 954 { 955 struct s390_idle_data *idle = &per_cpu(s390_idle, dev->id); 956 unsigned long long now, idle_time, idle_enter, idle_exit; 957 unsigned int sequence; 958 959 do { 960 now = get_clock(); 961 sequence = ACCESS_ONCE(idle->sequence); 962 idle_time = ACCESS_ONCE(idle->idle_time); 963 idle_enter = ACCESS_ONCE(idle->idle_enter); 964 idle_exit = ACCESS_ONCE(idle->idle_exit); 965 } while ((sequence & 1) || (idle->sequence != sequence)); 966 idle_time += idle_enter ? ((idle_exit ? : now) - idle_enter) : 0; 967 return sprintf(buf, "%llu\n", idle_time >> 12); 968 } 969 static DEVICE_ATTR(idle_time_us, 0444, show_idle_time, NULL); 970 971 static struct attribute *cpu_online_attrs[] = { 972 &dev_attr_idle_count.attr, 973 &dev_attr_idle_time_us.attr, 974 NULL, 975 }; 976 977 static struct attribute_group cpu_online_attr_group = { 978 .attrs = cpu_online_attrs, 979 }; 980 981 static int __cpuinit smp_cpu_notify(struct notifier_block *self, 982 unsigned long action, void *hcpu) 983 { 984 unsigned int cpu = (unsigned int)(long)hcpu; 985 struct cpu *c = &pcpu_devices[cpu].cpu; 986 struct device *s = &c->dev; 987 struct s390_idle_data *idle; 988 int err = 0; 989 990 switch (action) { 991 case CPU_ONLINE: 992 case CPU_ONLINE_FROZEN: 993 idle = &per_cpu(s390_idle, cpu); 994 memset(idle, 0, sizeof(struct s390_idle_data)); 995 err = sysfs_create_group(&s->kobj, &cpu_online_attr_group); 996 break; 997 case CPU_DEAD: 998 case CPU_DEAD_FROZEN: 999 sysfs_remove_group(&s->kobj, &cpu_online_attr_group); 1000 break; 1001 } 1002 return notifier_from_errno(err); 1003 } 1004 1005 static struct notifier_block __cpuinitdata smp_cpu_nb = { 1006 .notifier_call = smp_cpu_notify, 1007 }; 1008 1009 static int __devinit smp_add_present_cpu(int cpu) 1010 { 1011 struct cpu *c = &pcpu_devices[cpu].cpu; 1012 struct device *s = &c->dev; 1013 int rc; 1014 1015 c->hotpluggable = 1; 1016 rc = register_cpu(c, cpu); 1017 if (rc) 1018 goto out; 1019 rc = sysfs_create_group(&s->kobj, &cpu_common_attr_group); 1020 if (rc) 1021 goto out_cpu; 1022 if (cpu_online(cpu)) { 1023 rc = sysfs_create_group(&s->kobj, &cpu_online_attr_group); 1024 if (rc) 1025 goto out_online; 1026 } 1027 rc = topology_cpu_init(c); 1028 if (rc) 1029 goto out_topology; 1030 return 0; 1031 1032 out_topology: 1033 if (cpu_online(cpu)) 1034 sysfs_remove_group(&s->kobj, &cpu_online_attr_group); 1035 out_online: 1036 sysfs_remove_group(&s->kobj, &cpu_common_attr_group); 1037 out_cpu: 1038 #ifdef CONFIG_HOTPLUG_CPU 1039 unregister_cpu(c); 1040 #endif 1041 out: 1042 return rc; 1043 } 1044 1045 #ifdef CONFIG_HOTPLUG_CPU 1046 1047 int __ref smp_rescan_cpus(void) 1048 { 1049 struct sclp_cpu_info *info; 1050 int nr; 1051 1052 info = smp_get_cpu_info(); 1053 if (!info) 1054 return -ENOMEM; 1055 get_online_cpus(); 1056 mutex_lock(&smp_cpu_state_mutex); 1057 nr = __smp_rescan_cpus(info, 1); 1058 mutex_unlock(&smp_cpu_state_mutex); 1059 put_online_cpus(); 1060 kfree(info); 1061 if (nr) 1062 topology_schedule_update(); 1063 return 0; 1064 } 1065 1066 static ssize_t __ref rescan_store(struct device *dev, 1067 struct device_attribute *attr, 1068 const char *buf, 1069 size_t count) 1070 { 1071 int rc; 1072 1073 rc = smp_rescan_cpus(); 1074 return rc ? rc : count; 1075 } 1076 static DEVICE_ATTR(rescan, 0200, NULL, rescan_store); 1077 #endif /* CONFIG_HOTPLUG_CPU */ 1078 1079 static int __init s390_smp_init(void) 1080 { 1081 int cpu, rc; 1082 1083 register_cpu_notifier(&smp_cpu_nb); 1084 #ifdef CONFIG_HOTPLUG_CPU 1085 rc = device_create_file(cpu_subsys.dev_root, &dev_attr_rescan); 1086 if (rc) 1087 return rc; 1088 #endif 1089 for_each_present_cpu(cpu) { 1090 rc = smp_add_present_cpu(cpu); 1091 if (rc) 1092 return rc; 1093 } 1094 return 0; 1095 } 1096 subsys_initcall(s390_smp_init); 1097