1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited 4 * 5 * Derived from MIPS: 6 * Copyright (C) 2000, 2001 Kanoj Sarcar 7 * Copyright (C) 2000, 2001 Ralf Baechle 8 * Copyright (C) 2000, 2001 Silicon Graphics, Inc. 9 * Copyright (C) 2000, 2001, 2003 Broadcom Corporation 10 */ 11 #include <linux/acpi.h> 12 #include <linux/cpu.h> 13 #include <linux/cpumask.h> 14 #include <linux/init.h> 15 #include <linux/interrupt.h> 16 #include <linux/irq_work.h> 17 #include <linux/seq_file.h> 18 #include <linux/smp.h> 19 #include <linux/threads.h> 20 #include <linux/export.h> 21 #include <linux/suspend.h> 22 #include <linux/syscore_ops.h> 23 #include <linux/time.h> 24 #include <linux/tracepoint.h> 25 #include <linux/sched/hotplug.h> 26 #include <linux/sched/task_stack.h> 27 28 #include <asm/cpu.h> 29 #include <asm/idle.h> 30 #include <asm/loongson.h> 31 #include <asm/mmu_context.h> 32 #include <asm/numa.h> 33 #include <asm/paravirt.h> 34 #include <asm/processor.h> 35 #include <asm/setup.h> 36 #include <asm/time.h> 37 38 int __cpu_number_map[NR_CPUS]; /* Map physical to logical */ 39 EXPORT_SYMBOL(__cpu_number_map); 40 41 int __cpu_logical_map[NR_CPUS]; /* Map logical to physical */ 42 EXPORT_SYMBOL(__cpu_logical_map); 43 44 /* Representing the threads (siblings) of each logical CPU */ 45 cpumask_t cpu_sibling_map[NR_CPUS] __read_mostly; 46 EXPORT_SYMBOL(cpu_sibling_map); 47 48 /* Representing the last level cache shared map of each logical CPU */ 49 cpumask_t cpu_llc_shared_map[NR_CPUS] __read_mostly; 50 EXPORT_SYMBOL(cpu_llc_shared_map); 51 52 /* Representing the core map of multi-core chips of each logical CPU */ 53 cpumask_t cpu_core_map[NR_CPUS] __read_mostly; 54 EXPORT_SYMBOL(cpu_core_map); 55 56 static DECLARE_COMPLETION(cpu_starting); 57 static DECLARE_COMPLETION(cpu_running); 58 59 /* 60 * A logcal cpu mask containing only one VPE per core to 61 * reduce the number of IPIs on large MT systems. 62 */ 63 cpumask_t cpu_foreign_map[NR_CPUS] __read_mostly; 64 EXPORT_SYMBOL(cpu_foreign_map); 65 66 /* representing cpus for which sibling maps can be computed */ 67 static cpumask_t cpu_sibling_setup_map; 68 69 /* representing cpus for which llc shared maps can be computed */ 70 static cpumask_t cpu_llc_shared_setup_map; 71 72 /* representing cpus for which core maps can be computed */ 73 static cpumask_t cpu_core_setup_map; 74 75 struct secondary_data cpuboot_data; 76 static DEFINE_PER_CPU(int, cpu_state); 77 78 static const char *ipi_types[NR_IPI] __tracepoint_string = { 79 [IPI_RESCHEDULE] = "Rescheduling interrupts", 80 [IPI_CALL_FUNCTION] = "Function call interrupts", 81 [IPI_IRQ_WORK] = "IRQ work interrupts", 82 [IPI_CLEAR_VECTOR] = "Clear vector interrupts", 83 }; 84 85 void show_ipi_list(struct seq_file *p, int prec) 86 { 87 unsigned int cpu, i; 88 89 for (i = 0; i < NR_IPI; i++) { 90 seq_printf(p, "%*s%u:", prec - 1, "IPI", i); 91 for_each_online_cpu(cpu) 92 seq_put_decimal_ull_width(p, " ", per_cpu(irq_stat, cpu).ipi_irqs[i], 10); 93 seq_printf(p, " LoongArch %d %s\n", i + 1, ipi_types[i]); 94 } 95 } 96 97 static inline void set_cpu_core_map(int cpu) 98 { 99 int i; 100 101 cpumask_set_cpu(cpu, &cpu_core_setup_map); 102 103 for_each_cpu(i, &cpu_core_setup_map) { 104 if (cpu_data[cpu].package == cpu_data[i].package) { 105 cpumask_set_cpu(i, &cpu_core_map[cpu]); 106 cpumask_set_cpu(cpu, &cpu_core_map[i]); 107 } 108 } 109 } 110 111 static inline void set_cpu_llc_shared_map(int cpu) 112 { 113 int i; 114 115 cpumask_set_cpu(cpu, &cpu_llc_shared_setup_map); 116 117 for_each_cpu(i, &cpu_llc_shared_setup_map) { 118 if (cpu_to_node(cpu) == cpu_to_node(i)) { 119 cpumask_set_cpu(i, &cpu_llc_shared_map[cpu]); 120 cpumask_set_cpu(cpu, &cpu_llc_shared_map[i]); 121 } 122 } 123 } 124 125 static inline void clear_cpu_llc_shared_map(int cpu) 126 { 127 int i; 128 129 for_each_cpu(i, &cpu_llc_shared_setup_map) { 130 if (cpu_to_node(cpu) == cpu_to_node(i)) { 131 cpumask_clear_cpu(i, &cpu_llc_shared_map[cpu]); 132 cpumask_clear_cpu(cpu, &cpu_llc_shared_map[i]); 133 } 134 } 135 136 cpumask_clear_cpu(cpu, &cpu_llc_shared_setup_map); 137 } 138 139 static inline void set_cpu_sibling_map(int cpu) 140 { 141 int i; 142 143 cpumask_set_cpu(cpu, &cpu_sibling_setup_map); 144 145 for_each_cpu(i, &cpu_sibling_setup_map) { 146 if (cpus_are_siblings(cpu, i)) { 147 cpumask_set_cpu(i, &cpu_sibling_map[cpu]); 148 cpumask_set_cpu(cpu, &cpu_sibling_map[i]); 149 } 150 } 151 } 152 153 static inline void clear_cpu_sibling_map(int cpu) 154 { 155 int i; 156 157 for_each_cpu(i, &cpu_sibling_setup_map) { 158 if (cpus_are_siblings(cpu, i)) { 159 cpumask_clear_cpu(i, &cpu_sibling_map[cpu]); 160 cpumask_clear_cpu(cpu, &cpu_sibling_map[i]); 161 } 162 } 163 164 cpumask_clear_cpu(cpu, &cpu_sibling_setup_map); 165 } 166 167 /* 168 * Calculate a new cpu_foreign_map mask whenever a 169 * new cpu appears or disappears. 170 */ 171 void calculate_cpu_foreign_map(void) 172 { 173 int i, k, core_present; 174 cpumask_t temp_foreign_map; 175 176 /* Re-calculate the mask */ 177 cpumask_clear(&temp_foreign_map); 178 for_each_online_cpu(i) { 179 core_present = 0; 180 for_each_cpu(k, &temp_foreign_map) 181 if (cpus_are_siblings(i, k)) 182 core_present = 1; 183 if (!core_present) 184 cpumask_set_cpu(i, &temp_foreign_map); 185 } 186 187 for_each_online_cpu(i) 188 cpumask_andnot(&cpu_foreign_map[i], 189 &temp_foreign_map, &cpu_sibling_map[i]); 190 } 191 192 /* Send mailbox buffer via Mail_Send */ 193 static void csr_mail_send(uint64_t data, int cpu, int mailbox) 194 { 195 uint64_t val; 196 197 /* Send high 32 bits */ 198 val = IOCSR_MBUF_SEND_BLOCKING; 199 val |= (IOCSR_MBUF_SEND_BOX_HI(mailbox) << IOCSR_MBUF_SEND_BOX_SHIFT); 200 val |= (cpu << IOCSR_MBUF_SEND_CPU_SHIFT); 201 val |= (data & IOCSR_MBUF_SEND_H32_MASK); 202 iocsr_write64(val, LOONGARCH_IOCSR_MBUF_SEND); 203 204 /* Send low 32 bits */ 205 val = IOCSR_MBUF_SEND_BLOCKING; 206 val |= (IOCSR_MBUF_SEND_BOX_LO(mailbox) << IOCSR_MBUF_SEND_BOX_SHIFT); 207 val |= (cpu << IOCSR_MBUF_SEND_CPU_SHIFT); 208 val |= (data << IOCSR_MBUF_SEND_BUF_SHIFT); 209 iocsr_write64(val, LOONGARCH_IOCSR_MBUF_SEND); 210 }; 211 212 static u32 ipi_read_clear(int cpu) 213 { 214 u32 action; 215 216 /* Load the ipi register to figure out what we're supposed to do */ 217 action = iocsr_read32(LOONGARCH_IOCSR_IPI_STATUS); 218 /* Clear the ipi register to clear the interrupt */ 219 iocsr_write32(action, LOONGARCH_IOCSR_IPI_CLEAR); 220 wbflush(); 221 222 return action; 223 } 224 225 static void ipi_write_action(int cpu, u32 action) 226 { 227 uint32_t val; 228 229 val = IOCSR_IPI_SEND_BLOCKING | action; 230 val |= (cpu << IOCSR_IPI_SEND_CPU_SHIFT); 231 iocsr_write32(val, LOONGARCH_IOCSR_IPI_SEND); 232 } 233 234 static void loongson_send_ipi_single(int cpu, unsigned int action) 235 { 236 ipi_write_action(cpu_logical_map(cpu), (u32)action); 237 } 238 239 static void loongson_send_ipi_mask(const struct cpumask *mask, unsigned int action) 240 { 241 unsigned int i; 242 243 for_each_cpu(i, mask) 244 ipi_write_action(cpu_logical_map(i), (u32)action); 245 } 246 247 /* 248 * This function sends a 'reschedule' IPI to another CPU. 249 * it goes straight through and wastes no time serializing 250 * anything. Worst case is that we lose a reschedule ... 251 */ 252 void arch_smp_send_reschedule(int cpu) 253 { 254 mp_ops.send_ipi_single(cpu, ACTION_RESCHEDULE); 255 } 256 EXPORT_SYMBOL_GPL(arch_smp_send_reschedule); 257 258 #ifdef CONFIG_IRQ_WORK 259 void arch_irq_work_raise(void) 260 { 261 mp_ops.send_ipi_single(smp_processor_id(), ACTION_IRQ_WORK); 262 } 263 #endif 264 265 static irqreturn_t loongson_ipi_interrupt(int irq, void *dev) 266 { 267 unsigned int action; 268 unsigned int cpu = smp_processor_id(); 269 270 action = ipi_read_clear(cpu_logical_map(cpu)); 271 272 if (action & SMP_RESCHEDULE) { 273 scheduler_ipi(); 274 per_cpu(irq_stat, cpu).ipi_irqs[IPI_RESCHEDULE]++; 275 } 276 277 if (action & SMP_CALL_FUNCTION) { 278 generic_smp_call_function_interrupt(); 279 per_cpu(irq_stat, cpu).ipi_irqs[IPI_CALL_FUNCTION]++; 280 } 281 282 if (action & SMP_IRQ_WORK) { 283 irq_work_run(); 284 per_cpu(irq_stat, cpu).ipi_irqs[IPI_IRQ_WORK]++; 285 } 286 287 if (action & SMP_CLEAR_VECTOR) { 288 complete_irq_moving(); 289 per_cpu(irq_stat, cpu).ipi_irqs[IPI_CLEAR_VECTOR]++; 290 } 291 292 return IRQ_HANDLED; 293 } 294 295 static void loongson_init_ipi(void) 296 { 297 int r, ipi_irq; 298 299 ipi_irq = get_percpu_irq(INT_IPI); 300 if (ipi_irq < 0) 301 panic("IPI IRQ mapping failed\n"); 302 303 irq_set_percpu_devid(ipi_irq); 304 r = request_percpu_irq(ipi_irq, loongson_ipi_interrupt, "IPI", &irq_stat); 305 if (r < 0) 306 panic("IPI IRQ request failed\n"); 307 } 308 309 struct smp_ops mp_ops = { 310 .init_ipi = loongson_init_ipi, 311 .send_ipi_single = loongson_send_ipi_single, 312 .send_ipi_mask = loongson_send_ipi_mask, 313 }; 314 315 static void __init fdt_smp_setup(void) 316 { 317 #ifdef CONFIG_OF 318 unsigned int cpu, cpuid; 319 struct device_node *node = NULL; 320 321 for_each_of_cpu_node(node) { 322 if (!of_device_is_available(node)) 323 continue; 324 325 cpuid = of_get_cpu_hwid(node, 0); 326 if (cpuid >= nr_cpu_ids) 327 continue; 328 329 if (cpuid == loongson_sysconf.boot_cpu_id) 330 cpu = 0; 331 else 332 cpu = find_first_zero_bit(cpumask_bits(cpu_present_mask), NR_CPUS); 333 334 num_processors++; 335 set_cpu_possible(cpu, true); 336 set_cpu_present(cpu, true); 337 __cpu_number_map[cpuid] = cpu; 338 __cpu_logical_map[cpu] = cpuid; 339 340 early_numa_add_cpu(cpuid, 0); 341 set_cpuid_to_node(cpuid, 0); 342 } 343 344 loongson_sysconf.nr_cpus = num_processors; 345 set_bit(0, loongson_sysconf.cores_io_master); 346 #endif 347 } 348 349 void __init loongson_smp_setup(void) 350 { 351 fdt_smp_setup(); 352 353 if (loongson_sysconf.cores_per_package == 0) 354 loongson_sysconf.cores_per_package = num_processors; 355 356 cpu_data[0].core = cpu_logical_map(0) % loongson_sysconf.cores_per_package; 357 cpu_data[0].package = cpu_logical_map(0) / loongson_sysconf.cores_per_package; 358 359 pv_ipi_init(); 360 iocsr_write32(0xffffffff, LOONGARCH_IOCSR_IPI_EN); 361 pr_info("Detected %i available CPU(s)\n", loongson_sysconf.nr_cpus); 362 } 363 364 void __init loongson_prepare_cpus(unsigned int max_cpus) 365 { 366 int i = 0; 367 int threads_per_core = 0; 368 369 parse_acpi_topology(); 370 cpu_data[0].global_id = cpu_logical_map(0); 371 372 if (!pptt_enabled) 373 threads_per_core = 1; 374 else { 375 for_each_possible_cpu(i) { 376 if (cpu_to_node(i) != 0) 377 continue; 378 if (cpus_are_siblings(0, i)) 379 threads_per_core++; 380 } 381 } 382 383 for (i = 0; i < loongson_sysconf.nr_cpus; i++) { 384 set_cpu_present(i, true); 385 csr_mail_send(0, __cpu_logical_map[i], 0); 386 } 387 388 per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE; 389 cpu_smt_set_num_threads(threads_per_core, threads_per_core); 390 } 391 392 /* 393 * Setup the PC, SP, and TP of a secondary processor and start it running! 394 */ 395 void loongson_boot_secondary(int cpu, struct task_struct *idle) 396 { 397 unsigned long entry; 398 399 pr_info("Booting CPU#%d...\n", cpu); 400 401 entry = __pa_symbol((unsigned long)&smpboot_entry); 402 cpuboot_data.task = (unsigned long)idle; 403 cpuboot_data.stack = (unsigned long)task_pt_regs(idle); 404 cpuboot_data.offset = per_cpu_offset(cpu); 405 406 csr_mail_send(entry, cpu_logical_map(cpu), 0); 407 408 loongson_send_ipi_single(cpu, ACTION_BOOT_CPU); 409 } 410 411 /* 412 * SMP init and finish on secondary CPUs 413 */ 414 void loongson_init_secondary(void) 415 { 416 unsigned int cpu = smp_processor_id(); 417 unsigned int imask = ECFGF_IP0 | ECFGF_IP1 | ECFGF_IP2 | 418 ECFGF_IPI | ECFGF_PMC | ECFGF_TIMER | ECFGF_SIP0; 419 420 change_csr_ecfg(ECFG0_IM, imask); 421 422 iocsr_write32(0xffffffff, LOONGARCH_IOCSR_IPI_EN); 423 424 #ifdef CONFIG_NUMA 425 numa_add_cpu(cpu); 426 #endif 427 per_cpu(cpu_state, cpu) = CPU_ONLINE; 428 cpu_data[cpu].core = pptt_enabled ? cpu_data[cpu].core : 429 cpu_logical_map(cpu) % loongson_sysconf.cores_per_package; 430 cpu_data[cpu].package = pptt_enabled ? cpu_data[cpu].package : 431 cpu_logical_map(cpu) / loongson_sysconf.cores_per_package; 432 cpu_data[cpu].global_id = cpu_logical_map(cpu); 433 } 434 435 void loongson_smp_finish(void) 436 { 437 local_irq_enable(); 438 iocsr_write64(0, LOONGARCH_IOCSR_MBUF0); 439 pr_info("CPU#%d finished\n", smp_processor_id()); 440 } 441 442 #ifdef CONFIG_HOTPLUG_CPU 443 444 int loongson_cpu_disable(void) 445 { 446 unsigned long flags; 447 unsigned int cpu = smp_processor_id(); 448 449 if (io_master(cpu)) 450 return -EBUSY; 451 452 #ifdef CONFIG_NUMA 453 numa_remove_cpu(cpu); 454 #endif 455 set_cpu_online(cpu, false); 456 clear_cpu_sibling_map(cpu); 457 clear_cpu_llc_shared_map(cpu); 458 calculate_cpu_foreign_map(); 459 local_irq_save(flags); 460 irq_migrate_all_off_this_cpu(); 461 clear_csr_ecfg(ECFG0_IM); 462 local_irq_restore(flags); 463 local_flush_tlb_all(); 464 465 return 0; 466 } 467 468 void loongson_cpu_die(unsigned int cpu) 469 { 470 while (per_cpu(cpu_state, cpu) != CPU_DEAD) 471 cpu_relax(); 472 473 mb(); 474 } 475 476 static void __noreturn idle_play_dead(void) 477 { 478 register uint64_t addr; 479 register void (*init_fn)(void); 480 481 idle_task_exit(); 482 local_irq_enable(); 483 set_csr_ecfg(ECFGF_IPI); 484 __this_cpu_write(cpu_state, CPU_DEAD); 485 486 __smp_mb(); 487 do { 488 __asm__ __volatile__("idle 0\n\t"); 489 addr = iocsr_read64(LOONGARCH_IOCSR_MBUF0); 490 } while (addr == 0); 491 492 local_irq_disable(); 493 init_fn = (void *)TO_CACHE(addr); 494 iocsr_write32(0xffffffff, LOONGARCH_IOCSR_IPI_CLEAR); 495 496 init_fn(); 497 BUG(); 498 } 499 500 #ifdef CONFIG_HIBERNATION 501 static void __noreturn poll_play_dead(void) 502 { 503 register uint64_t addr; 504 register void (*init_fn)(void); 505 506 idle_task_exit(); 507 __this_cpu_write(cpu_state, CPU_DEAD); 508 509 __smp_mb(); 510 do { 511 __asm__ __volatile__("nop\n\t"); 512 addr = iocsr_read64(LOONGARCH_IOCSR_MBUF0); 513 } while (addr == 0); 514 515 init_fn = (void *)TO_CACHE(addr); 516 iocsr_write32(0xffffffff, LOONGARCH_IOCSR_IPI_CLEAR); 517 518 init_fn(); 519 BUG(); 520 } 521 #endif 522 523 static void (*play_dead)(void) = idle_play_dead; 524 525 void __noreturn arch_cpu_idle_dead(void) 526 { 527 play_dead(); 528 BUG(); /* play_dead() doesn't return */ 529 } 530 531 #ifdef CONFIG_HIBERNATION 532 int hibernate_resume_nonboot_cpu_disable(void) 533 { 534 int ret; 535 536 play_dead = poll_play_dead; 537 ret = suspend_disable_secondary_cpus(); 538 play_dead = idle_play_dead; 539 540 return ret; 541 } 542 #endif 543 544 #endif 545 546 /* 547 * Power management 548 */ 549 #ifdef CONFIG_PM 550 551 static int loongson_ipi_suspend(void *data) 552 { 553 return 0; 554 } 555 556 static void loongson_ipi_resume(void *data) 557 { 558 iocsr_write32(0xffffffff, LOONGARCH_IOCSR_IPI_EN); 559 } 560 561 static const struct syscore_ops loongson_ipi_syscore_ops = { 562 .resume = loongson_ipi_resume, 563 .suspend = loongson_ipi_suspend, 564 }; 565 566 static struct syscore loongson_ipi_syscore = { 567 .ops = &loongson_ipi_syscore_ops, 568 }; 569 570 /* 571 * Enable boot cpu ipi before enabling nonboot cpus 572 * during syscore_resume. 573 */ 574 static int __init ipi_pm_init(void) 575 { 576 register_syscore(&loongson_ipi_syscore); 577 return 0; 578 } 579 580 core_initcall(ipi_pm_init); 581 #endif 582 583 /* Preload SMP state for boot cpu */ 584 void __init smp_prepare_boot_cpu(void) 585 { 586 unsigned int cpu, node, rr_node; 587 588 set_cpu_possible(0, true); 589 set_cpu_online(0, true); 590 set_my_cpu_offset(per_cpu_offset(0)); 591 numa_add_cpu(0); 592 593 rr_node = first_node(node_online_map); 594 for_each_possible_cpu(cpu) { 595 node = early_cpu_to_node(cpu); 596 597 /* 598 * The mapping between present cpus and nodes has been 599 * built during MADT and SRAT parsing. 600 * 601 * If possible cpus = present cpus here, early_cpu_to_node 602 * will return valid node. 603 * 604 * If possible cpus > present cpus here (e.g. some possible 605 * cpus will be added by cpu-hotplug later), for possible but 606 * not present cpus, early_cpu_to_node will return NUMA_NO_NODE, 607 * and we just map them to online nodes in round-robin way. 608 * Once hotplugged, new correct mapping will be built for them. 609 */ 610 if (node != NUMA_NO_NODE) 611 set_cpu_numa_node(cpu, node); 612 else { 613 set_cpu_numa_node(cpu, rr_node); 614 rr_node = next_node_in(rr_node, node_online_map); 615 } 616 } 617 618 pv_spinlock_init(); 619 } 620 621 /* called from main before smp_init() */ 622 void __init smp_prepare_cpus(unsigned int max_cpus) 623 { 624 init_new_context(current, &init_mm); 625 current_thread_info()->cpu = 0; 626 loongson_prepare_cpus(max_cpus); 627 set_cpu_sibling_map(0); 628 set_cpu_llc_shared_map(0); 629 set_cpu_core_map(0); 630 calculate_cpu_foreign_map(); 631 #ifndef CONFIG_HOTPLUG_CPU 632 init_cpu_present(cpu_possible_mask); 633 #endif 634 } 635 636 int __cpu_up(unsigned int cpu, struct task_struct *tidle) 637 { 638 loongson_boot_secondary(cpu, tidle); 639 640 /* Wait for CPU to start and be ready to sync counters */ 641 if (!wait_for_completion_timeout(&cpu_starting, 642 msecs_to_jiffies(5000))) { 643 pr_crit("CPU%u: failed to start\n", cpu); 644 return -EIO; 645 } 646 647 /* Wait for CPU to finish startup & mark itself online before return */ 648 wait_for_completion(&cpu_running); 649 650 return 0; 651 } 652 653 /* 654 * First C code run on the secondary CPUs after being started up by 655 * the master. 656 */ 657 asmlinkage void start_secondary(void) 658 { 659 unsigned int cpu; 660 661 sync_counter(); 662 cpu = raw_smp_processor_id(); 663 set_my_cpu_offset(per_cpu_offset(cpu)); 664 665 cpu_probe(); 666 set_current(current); 667 constant_clockevent_init(); 668 loongson_init_secondary(); 669 670 set_cpu_sibling_map(cpu); 671 set_cpu_llc_shared_map(cpu); 672 set_cpu_core_map(cpu); 673 674 notify_cpu_starting(cpu); 675 676 /* Notify boot CPU that we're starting */ 677 complete(&cpu_starting); 678 679 /* The CPU is running, now mark it online */ 680 set_cpu_online(cpu, true); 681 682 calculate_cpu_foreign_map(); 683 684 /* 685 * Notify boot CPU that we're up & online and it can safely return 686 * from __cpu_up() 687 */ 688 complete(&cpu_running); 689 690 /* 691 * irq will be enabled in loongson_smp_finish(), enabling it too 692 * early is dangerous. 693 */ 694 WARN_ON_ONCE(!irqs_disabled()); 695 loongson_smp_finish(); 696 697 cpu_startup_entry(CPUHP_AP_ONLINE_IDLE); 698 } 699 700 void __init smp_cpus_done(unsigned int max_cpus) 701 { 702 } 703 704 static void stop_this_cpu(void *dummy) 705 { 706 set_cpu_online(smp_processor_id(), false); 707 calculate_cpu_foreign_map(); 708 local_irq_disable(); 709 rcutree_report_cpu_dead(); 710 while (true); 711 } 712 713 void smp_send_stop(void) 714 { 715 smp_call_function(stop_this_cpu, NULL, 0); 716 } 717 718 static void flush_tlb_all_ipi(void *info) 719 { 720 local_flush_tlb_all(); 721 } 722 723 void flush_tlb_all(void) 724 { 725 on_each_cpu(flush_tlb_all_ipi, NULL, 1); 726 } 727 728 static void flush_tlb_mm_ipi(void *mm) 729 { 730 local_flush_tlb_mm((struct mm_struct *)mm); 731 } 732 733 void flush_tlb_mm(struct mm_struct *mm) 734 { 735 if (atomic_read(&mm->mm_users) == 0) 736 return; /* happens as a result of exit_mmap() */ 737 738 preempt_disable(); 739 740 if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) { 741 on_each_cpu_mask(mm_cpumask(mm), flush_tlb_mm_ipi, mm, 1); 742 } else { 743 unsigned int cpu; 744 745 for_each_online_cpu(cpu) { 746 if (cpu != smp_processor_id() && cpu_context(cpu, mm)) 747 cpu_context(cpu, mm) = 0; 748 } 749 local_flush_tlb_mm(mm); 750 } 751 752 preempt_enable(); 753 } 754 755 struct flush_tlb_data { 756 struct vm_area_struct *vma; 757 unsigned long addr1; 758 unsigned long addr2; 759 }; 760 761 static void flush_tlb_range_ipi(void *info) 762 { 763 struct flush_tlb_data *fd = info; 764 765 local_flush_tlb_range(fd->vma, fd->addr1, fd->addr2); 766 } 767 768 void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) 769 { 770 struct mm_struct *mm = vma->vm_mm; 771 772 preempt_disable(); 773 if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) { 774 struct flush_tlb_data fd = { 775 .vma = vma, 776 .addr1 = start, 777 .addr2 = end, 778 }; 779 780 on_each_cpu_mask(mm_cpumask(mm), flush_tlb_range_ipi, &fd, 1); 781 } else { 782 unsigned int cpu; 783 784 for_each_online_cpu(cpu) { 785 if (cpu != smp_processor_id() && cpu_context(cpu, mm)) 786 cpu_context(cpu, mm) = 0; 787 } 788 local_flush_tlb_range(vma, start, end); 789 } 790 preempt_enable(); 791 } 792 793 static void flush_tlb_kernel_range_ipi(void *info) 794 { 795 struct flush_tlb_data *fd = info; 796 797 local_flush_tlb_kernel_range(fd->addr1, fd->addr2); 798 } 799 800 void flush_tlb_kernel_range(unsigned long start, unsigned long end) 801 { 802 struct flush_tlb_data fd = { 803 .addr1 = start, 804 .addr2 = end, 805 }; 806 807 on_each_cpu(flush_tlb_kernel_range_ipi, &fd, 1); 808 } 809 810 static void flush_tlb_page_ipi(void *info) 811 { 812 struct flush_tlb_data *fd = info; 813 814 local_flush_tlb_page(fd->vma, fd->addr1); 815 } 816 817 void flush_tlb_page(struct vm_area_struct *vma, unsigned long page) 818 { 819 preempt_disable(); 820 if ((atomic_read(&vma->vm_mm->mm_users) != 1) || (current->mm != vma->vm_mm)) { 821 struct flush_tlb_data fd = { 822 .vma = vma, 823 .addr1 = page, 824 }; 825 826 on_each_cpu_mask(mm_cpumask(vma->vm_mm), flush_tlb_page_ipi, &fd, 1); 827 } else { 828 unsigned int cpu; 829 830 for_each_online_cpu(cpu) { 831 if (cpu != smp_processor_id() && cpu_context(cpu, vma->vm_mm)) 832 cpu_context(cpu, vma->vm_mm) = 0; 833 } 834 local_flush_tlb_page(vma, page); 835 } 836 preempt_enable(); 837 } 838 EXPORT_SYMBOL(flush_tlb_page); 839 840 static void flush_tlb_one_ipi(void *info) 841 { 842 unsigned long vaddr = (unsigned long) info; 843 844 local_flush_tlb_one(vaddr); 845 } 846 847 void flush_tlb_one(unsigned long vaddr) 848 { 849 on_each_cpu(flush_tlb_one_ipi, (void *)vaddr, 1); 850 } 851 EXPORT_SYMBOL(flush_tlb_one); 852