1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited 4 * 5 * Derived from MIPS: 6 * Copyright (C) 2000, 2001 Kanoj Sarcar 7 * Copyright (C) 2000, 2001 Ralf Baechle 8 * Copyright (C) 2000, 2001 Silicon Graphics, Inc. 9 * Copyright (C) 2000, 2001, 2003 Broadcom Corporation 10 */ 11 #include <linux/acpi.h> 12 #include <linux/cpu.h> 13 #include <linux/cpumask.h> 14 #include <linux/init.h> 15 #include <linux/interrupt.h> 16 #include <linux/irq_work.h> 17 #include <linux/profile.h> 18 #include <linux/seq_file.h> 19 #include <linux/smp.h> 20 #include <linux/threads.h> 21 #include <linux/export.h> 22 #include <linux/suspend.h> 23 #include <linux/syscore_ops.h> 24 #include <linux/time.h> 25 #include <linux/tracepoint.h> 26 #include <linux/sched/hotplug.h> 27 #include <linux/sched/task_stack.h> 28 29 #include <asm/cpu.h> 30 #include <asm/idle.h> 31 #include <asm/loongson.h> 32 #include <asm/mmu_context.h> 33 #include <asm/numa.h> 34 #include <asm/paravirt.h> 35 #include <asm/processor.h> 36 #include <asm/setup.h> 37 #include <asm/time.h> 38 39 int __cpu_number_map[NR_CPUS]; /* Map physical to logical */ 40 EXPORT_SYMBOL(__cpu_number_map); 41 42 int __cpu_logical_map[NR_CPUS]; /* Map logical to physical */ 43 EXPORT_SYMBOL(__cpu_logical_map); 44 45 /* Representing the threads (siblings) of each logical CPU */ 46 cpumask_t cpu_sibling_map[NR_CPUS] __read_mostly; 47 EXPORT_SYMBOL(cpu_sibling_map); 48 49 /* Representing the last level cache shared map of each logical CPU */ 50 cpumask_t cpu_llc_shared_map[NR_CPUS] __read_mostly; 51 EXPORT_SYMBOL(cpu_llc_shared_map); 52 53 /* Representing the core map of multi-core chips of each logical CPU */ 54 cpumask_t cpu_core_map[NR_CPUS] __read_mostly; 55 EXPORT_SYMBOL(cpu_core_map); 56 57 static DECLARE_COMPLETION(cpu_starting); 58 static DECLARE_COMPLETION(cpu_running); 59 60 /* 61 * A logcal cpu mask containing only one VPE per core to 62 * reduce the number of IPIs on large MT systems. 63 */ 64 cpumask_t cpu_foreign_map[NR_CPUS] __read_mostly; 65 EXPORT_SYMBOL(cpu_foreign_map); 66 67 /* representing cpus for which sibling maps can be computed */ 68 static cpumask_t cpu_sibling_setup_map; 69 70 /* representing cpus for which llc shared maps can be computed */ 71 static cpumask_t cpu_llc_shared_setup_map; 72 73 /* representing cpus for which core maps can be computed */ 74 static cpumask_t cpu_core_setup_map; 75 76 struct secondary_data cpuboot_data; 77 static DEFINE_PER_CPU(int, cpu_state); 78 79 static const char *ipi_types[NR_IPI] __tracepoint_string = { 80 [IPI_RESCHEDULE] = "Rescheduling interrupts", 81 [IPI_CALL_FUNCTION] = "Function call interrupts", 82 [IPI_IRQ_WORK] = "IRQ work interrupts", 83 [IPI_CLEAR_VECTOR] = "Clear vector interrupts", 84 }; 85 86 void show_ipi_list(struct seq_file *p, int prec) 87 { 88 unsigned int cpu, i; 89 90 for (i = 0; i < NR_IPI; i++) { 91 seq_printf(p, "%*s%u:%s", prec - 1, "IPI", i, prec >= 4 ? " " : ""); 92 for_each_online_cpu(cpu) 93 seq_put_decimal_ull_width(p, " ", per_cpu(irq_stat, cpu).ipi_irqs[i], 10); 94 seq_printf(p, " LoongArch %d %s\n", i + 1, ipi_types[i]); 95 } 96 } 97 98 static inline void set_cpu_core_map(int cpu) 99 { 100 int i; 101 102 cpumask_set_cpu(cpu, &cpu_core_setup_map); 103 104 for_each_cpu(i, &cpu_core_setup_map) { 105 if (cpu_data[cpu].package == cpu_data[i].package) { 106 cpumask_set_cpu(i, &cpu_core_map[cpu]); 107 cpumask_set_cpu(cpu, &cpu_core_map[i]); 108 } 109 } 110 } 111 112 static inline void set_cpu_llc_shared_map(int cpu) 113 { 114 int i; 115 116 cpumask_set_cpu(cpu, &cpu_llc_shared_setup_map); 117 118 for_each_cpu(i, &cpu_llc_shared_setup_map) { 119 if (cpu_to_node(cpu) == cpu_to_node(i)) { 120 cpumask_set_cpu(i, &cpu_llc_shared_map[cpu]); 121 cpumask_set_cpu(cpu, &cpu_llc_shared_map[i]); 122 } 123 } 124 } 125 126 static inline void clear_cpu_llc_shared_map(int cpu) 127 { 128 int i; 129 130 for_each_cpu(i, &cpu_llc_shared_setup_map) { 131 if (cpu_to_node(cpu) == cpu_to_node(i)) { 132 cpumask_clear_cpu(i, &cpu_llc_shared_map[cpu]); 133 cpumask_clear_cpu(cpu, &cpu_llc_shared_map[i]); 134 } 135 } 136 137 cpumask_clear_cpu(cpu, &cpu_llc_shared_setup_map); 138 } 139 140 static inline void set_cpu_sibling_map(int cpu) 141 { 142 int i; 143 144 cpumask_set_cpu(cpu, &cpu_sibling_setup_map); 145 146 for_each_cpu(i, &cpu_sibling_setup_map) { 147 if (cpus_are_siblings(cpu, i)) { 148 cpumask_set_cpu(i, &cpu_sibling_map[cpu]); 149 cpumask_set_cpu(cpu, &cpu_sibling_map[i]); 150 } 151 } 152 } 153 154 static inline void clear_cpu_sibling_map(int cpu) 155 { 156 int i; 157 158 for_each_cpu(i, &cpu_sibling_setup_map) { 159 if (cpus_are_siblings(cpu, i)) { 160 cpumask_clear_cpu(i, &cpu_sibling_map[cpu]); 161 cpumask_clear_cpu(cpu, &cpu_sibling_map[i]); 162 } 163 } 164 165 cpumask_clear_cpu(cpu, &cpu_sibling_setup_map); 166 } 167 168 /* 169 * Calculate a new cpu_foreign_map mask whenever a 170 * new cpu appears or disappears. 171 */ 172 void calculate_cpu_foreign_map(void) 173 { 174 int i, k, core_present; 175 cpumask_t temp_foreign_map; 176 177 /* Re-calculate the mask */ 178 cpumask_clear(&temp_foreign_map); 179 for_each_online_cpu(i) { 180 core_present = 0; 181 for_each_cpu(k, &temp_foreign_map) 182 if (cpus_are_siblings(i, k)) 183 core_present = 1; 184 if (!core_present) 185 cpumask_set_cpu(i, &temp_foreign_map); 186 } 187 188 for_each_online_cpu(i) 189 cpumask_andnot(&cpu_foreign_map[i], 190 &temp_foreign_map, &cpu_sibling_map[i]); 191 } 192 193 /* Send mailbox buffer via Mail_Send */ 194 static void csr_mail_send(uint64_t data, int cpu, int mailbox) 195 { 196 uint64_t val; 197 198 /* Send high 32 bits */ 199 val = IOCSR_MBUF_SEND_BLOCKING; 200 val |= (IOCSR_MBUF_SEND_BOX_HI(mailbox) << IOCSR_MBUF_SEND_BOX_SHIFT); 201 val |= (cpu << IOCSR_MBUF_SEND_CPU_SHIFT); 202 val |= (data & IOCSR_MBUF_SEND_H32_MASK); 203 iocsr_write64(val, LOONGARCH_IOCSR_MBUF_SEND); 204 205 /* Send low 32 bits */ 206 val = IOCSR_MBUF_SEND_BLOCKING; 207 val |= (IOCSR_MBUF_SEND_BOX_LO(mailbox) << IOCSR_MBUF_SEND_BOX_SHIFT); 208 val |= (cpu << IOCSR_MBUF_SEND_CPU_SHIFT); 209 val |= (data << IOCSR_MBUF_SEND_BUF_SHIFT); 210 iocsr_write64(val, LOONGARCH_IOCSR_MBUF_SEND); 211 }; 212 213 static u32 ipi_read_clear(int cpu) 214 { 215 u32 action; 216 217 /* Load the ipi register to figure out what we're supposed to do */ 218 action = iocsr_read32(LOONGARCH_IOCSR_IPI_STATUS); 219 /* Clear the ipi register to clear the interrupt */ 220 iocsr_write32(action, LOONGARCH_IOCSR_IPI_CLEAR); 221 wbflush(); 222 223 return action; 224 } 225 226 static void ipi_write_action(int cpu, u32 action) 227 { 228 uint32_t val; 229 230 val = IOCSR_IPI_SEND_BLOCKING | action; 231 val |= (cpu << IOCSR_IPI_SEND_CPU_SHIFT); 232 iocsr_write32(val, LOONGARCH_IOCSR_IPI_SEND); 233 } 234 235 static void loongson_send_ipi_single(int cpu, unsigned int action) 236 { 237 ipi_write_action(cpu_logical_map(cpu), (u32)action); 238 } 239 240 static void loongson_send_ipi_mask(const struct cpumask *mask, unsigned int action) 241 { 242 unsigned int i; 243 244 for_each_cpu(i, mask) 245 ipi_write_action(cpu_logical_map(i), (u32)action); 246 } 247 248 /* 249 * This function sends a 'reschedule' IPI to another CPU. 250 * it goes straight through and wastes no time serializing 251 * anything. Worst case is that we lose a reschedule ... 252 */ 253 void arch_smp_send_reschedule(int cpu) 254 { 255 mp_ops.send_ipi_single(cpu, ACTION_RESCHEDULE); 256 } 257 EXPORT_SYMBOL_GPL(arch_smp_send_reschedule); 258 259 #ifdef CONFIG_IRQ_WORK 260 void arch_irq_work_raise(void) 261 { 262 mp_ops.send_ipi_single(smp_processor_id(), ACTION_IRQ_WORK); 263 } 264 #endif 265 266 static irqreturn_t loongson_ipi_interrupt(int irq, void *dev) 267 { 268 unsigned int action; 269 unsigned int cpu = smp_processor_id(); 270 271 action = ipi_read_clear(cpu_logical_map(cpu)); 272 273 if (action & SMP_RESCHEDULE) { 274 scheduler_ipi(); 275 per_cpu(irq_stat, cpu).ipi_irqs[IPI_RESCHEDULE]++; 276 } 277 278 if (action & SMP_CALL_FUNCTION) { 279 generic_smp_call_function_interrupt(); 280 per_cpu(irq_stat, cpu).ipi_irqs[IPI_CALL_FUNCTION]++; 281 } 282 283 if (action & SMP_IRQ_WORK) { 284 irq_work_run(); 285 per_cpu(irq_stat, cpu).ipi_irqs[IPI_IRQ_WORK]++; 286 } 287 288 if (action & SMP_CLEAR_VECTOR) { 289 complete_irq_moving(); 290 per_cpu(irq_stat, cpu).ipi_irqs[IPI_CLEAR_VECTOR]++; 291 } 292 293 return IRQ_HANDLED; 294 } 295 296 static void loongson_init_ipi(void) 297 { 298 int r, ipi_irq; 299 300 ipi_irq = get_percpu_irq(INT_IPI); 301 if (ipi_irq < 0) 302 panic("IPI IRQ mapping failed\n"); 303 304 irq_set_percpu_devid(ipi_irq); 305 r = request_percpu_irq(ipi_irq, loongson_ipi_interrupt, "IPI", &irq_stat); 306 if (r < 0) 307 panic("IPI IRQ request failed\n"); 308 } 309 310 struct smp_ops mp_ops = { 311 .init_ipi = loongson_init_ipi, 312 .send_ipi_single = loongson_send_ipi_single, 313 .send_ipi_mask = loongson_send_ipi_mask, 314 }; 315 316 static void __init fdt_smp_setup(void) 317 { 318 #ifdef CONFIG_OF 319 unsigned int cpu, cpuid; 320 struct device_node *node = NULL; 321 322 for_each_of_cpu_node(node) { 323 if (!of_device_is_available(node)) 324 continue; 325 326 cpuid = of_get_cpu_hwid(node, 0); 327 if (cpuid >= nr_cpu_ids) 328 continue; 329 330 if (cpuid == loongson_sysconf.boot_cpu_id) 331 cpu = 0; 332 else 333 cpu = find_first_zero_bit(cpumask_bits(cpu_present_mask), NR_CPUS); 334 335 num_processors++; 336 set_cpu_possible(cpu, true); 337 set_cpu_present(cpu, true); 338 __cpu_number_map[cpuid] = cpu; 339 __cpu_logical_map[cpu] = cpuid; 340 341 early_numa_add_cpu(cpuid, 0); 342 set_cpuid_to_node(cpuid, 0); 343 } 344 345 loongson_sysconf.nr_cpus = num_processors; 346 set_bit(0, loongson_sysconf.cores_io_master); 347 #endif 348 } 349 350 void __init loongson_smp_setup(void) 351 { 352 fdt_smp_setup(); 353 354 if (loongson_sysconf.cores_per_package == 0) 355 loongson_sysconf.cores_per_package = num_processors; 356 357 cpu_data[0].core = cpu_logical_map(0) % loongson_sysconf.cores_per_package; 358 cpu_data[0].package = cpu_logical_map(0) / loongson_sysconf.cores_per_package; 359 360 pv_ipi_init(); 361 iocsr_write32(0xffffffff, LOONGARCH_IOCSR_IPI_EN); 362 pr_info("Detected %i available CPU(s)\n", loongson_sysconf.nr_cpus); 363 } 364 365 void __init loongson_prepare_cpus(unsigned int max_cpus) 366 { 367 int i = 0; 368 369 parse_acpi_topology(); 370 cpu_data[0].global_id = cpu_logical_map(0); 371 372 for (i = 0; i < loongson_sysconf.nr_cpus; i++) { 373 set_cpu_present(i, true); 374 csr_mail_send(0, __cpu_logical_map[i], 0); 375 } 376 377 per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE; 378 } 379 380 /* 381 * Setup the PC, SP, and TP of a secondary processor and start it running! 382 */ 383 void loongson_boot_secondary(int cpu, struct task_struct *idle) 384 { 385 unsigned long entry; 386 387 pr_info("Booting CPU#%d...\n", cpu); 388 389 entry = __pa_symbol((unsigned long)&smpboot_entry); 390 cpuboot_data.stack = (unsigned long)__KSTK_TOS(idle); 391 cpuboot_data.thread_info = (unsigned long)task_thread_info(idle); 392 393 csr_mail_send(entry, cpu_logical_map(cpu), 0); 394 395 loongson_send_ipi_single(cpu, ACTION_BOOT_CPU); 396 } 397 398 /* 399 * SMP init and finish on secondary CPUs 400 */ 401 void loongson_init_secondary(void) 402 { 403 unsigned int cpu = smp_processor_id(); 404 unsigned int imask = ECFGF_IP0 | ECFGF_IP1 | ECFGF_IP2 | 405 ECFGF_IPI | ECFGF_PMC | ECFGF_TIMER | ECFGF_SIP0; 406 407 change_csr_ecfg(ECFG0_IM, imask); 408 409 iocsr_write32(0xffffffff, LOONGARCH_IOCSR_IPI_EN); 410 411 #ifdef CONFIG_NUMA 412 numa_add_cpu(cpu); 413 #endif 414 per_cpu(cpu_state, cpu) = CPU_ONLINE; 415 cpu_data[cpu].package = 416 cpu_logical_map(cpu) / loongson_sysconf.cores_per_package; 417 cpu_data[cpu].core = pptt_enabled ? cpu_data[cpu].core : 418 cpu_logical_map(cpu) % loongson_sysconf.cores_per_package; 419 cpu_data[cpu].global_id = cpu_logical_map(cpu); 420 } 421 422 void loongson_smp_finish(void) 423 { 424 local_irq_enable(); 425 iocsr_write64(0, LOONGARCH_IOCSR_MBUF0); 426 pr_info("CPU#%d finished\n", smp_processor_id()); 427 } 428 429 #ifdef CONFIG_HOTPLUG_CPU 430 431 int loongson_cpu_disable(void) 432 { 433 unsigned long flags; 434 unsigned int cpu = smp_processor_id(); 435 436 if (io_master(cpu)) 437 return -EBUSY; 438 439 #ifdef CONFIG_NUMA 440 numa_remove_cpu(cpu); 441 #endif 442 set_cpu_online(cpu, false); 443 clear_cpu_sibling_map(cpu); 444 clear_cpu_llc_shared_map(cpu); 445 calculate_cpu_foreign_map(); 446 local_irq_save(flags); 447 irq_migrate_all_off_this_cpu(); 448 clear_csr_ecfg(ECFG0_IM); 449 local_irq_restore(flags); 450 local_flush_tlb_all(); 451 452 return 0; 453 } 454 455 void loongson_cpu_die(unsigned int cpu) 456 { 457 while (per_cpu(cpu_state, cpu) != CPU_DEAD) 458 cpu_relax(); 459 460 mb(); 461 } 462 463 static void __noreturn idle_play_dead(void) 464 { 465 register uint64_t addr; 466 register void (*init_fn)(void); 467 468 idle_task_exit(); 469 local_irq_enable(); 470 set_csr_ecfg(ECFGF_IPI); 471 __this_cpu_write(cpu_state, CPU_DEAD); 472 473 __smp_mb(); 474 do { 475 __asm__ __volatile__("idle 0\n\t"); 476 addr = iocsr_read64(LOONGARCH_IOCSR_MBUF0); 477 } while (addr == 0); 478 479 local_irq_disable(); 480 init_fn = (void *)TO_CACHE(addr); 481 iocsr_write32(0xffffffff, LOONGARCH_IOCSR_IPI_CLEAR); 482 483 init_fn(); 484 BUG(); 485 } 486 487 #ifdef CONFIG_HIBERNATION 488 static void __noreturn poll_play_dead(void) 489 { 490 register uint64_t addr; 491 register void (*init_fn)(void); 492 493 idle_task_exit(); 494 __this_cpu_write(cpu_state, CPU_DEAD); 495 496 __smp_mb(); 497 do { 498 __asm__ __volatile__("nop\n\t"); 499 addr = iocsr_read64(LOONGARCH_IOCSR_MBUF0); 500 } while (addr == 0); 501 502 init_fn = (void *)TO_CACHE(addr); 503 iocsr_write32(0xffffffff, LOONGARCH_IOCSR_IPI_CLEAR); 504 505 init_fn(); 506 BUG(); 507 } 508 #endif 509 510 static void (*play_dead)(void) = idle_play_dead; 511 512 void __noreturn arch_cpu_idle_dead(void) 513 { 514 play_dead(); 515 BUG(); /* play_dead() doesn't return */ 516 } 517 518 #ifdef CONFIG_HIBERNATION 519 int hibernate_resume_nonboot_cpu_disable(void) 520 { 521 int ret; 522 523 play_dead = poll_play_dead; 524 ret = suspend_disable_secondary_cpus(); 525 play_dead = idle_play_dead; 526 527 return ret; 528 } 529 #endif 530 531 #endif 532 533 /* 534 * Power management 535 */ 536 #ifdef CONFIG_PM 537 538 static int loongson_ipi_suspend(void *data) 539 { 540 return 0; 541 } 542 543 static void loongson_ipi_resume(void *data) 544 { 545 iocsr_write32(0xffffffff, LOONGARCH_IOCSR_IPI_EN); 546 } 547 548 static const struct syscore_ops loongson_ipi_syscore_ops = { 549 .resume = loongson_ipi_resume, 550 .suspend = loongson_ipi_suspend, 551 }; 552 553 static struct syscore loongson_ipi_syscore = { 554 .ops = &loongson_ipi_syscore_ops, 555 }; 556 557 /* 558 * Enable boot cpu ipi before enabling nonboot cpus 559 * during syscore_resume. 560 */ 561 static int __init ipi_pm_init(void) 562 { 563 register_syscore(&loongson_ipi_syscore); 564 return 0; 565 } 566 567 core_initcall(ipi_pm_init); 568 #endif 569 570 /* Preload SMP state for boot cpu */ 571 void __init smp_prepare_boot_cpu(void) 572 { 573 unsigned int cpu, node, rr_node; 574 575 set_cpu_possible(0, true); 576 set_cpu_online(0, true); 577 set_my_cpu_offset(per_cpu_offset(0)); 578 numa_add_cpu(0); 579 580 rr_node = first_node(node_online_map); 581 for_each_possible_cpu(cpu) { 582 node = early_cpu_to_node(cpu); 583 584 /* 585 * The mapping between present cpus and nodes has been 586 * built during MADT and SRAT parsing. 587 * 588 * If possible cpus = present cpus here, early_cpu_to_node 589 * will return valid node. 590 * 591 * If possible cpus > present cpus here (e.g. some possible 592 * cpus will be added by cpu-hotplug later), for possible but 593 * not present cpus, early_cpu_to_node will return NUMA_NO_NODE, 594 * and we just map them to online nodes in round-robin way. 595 * Once hotplugged, new correct mapping will be built for them. 596 */ 597 if (node != NUMA_NO_NODE) 598 set_cpu_numa_node(cpu, node); 599 else { 600 set_cpu_numa_node(cpu, rr_node); 601 rr_node = next_node_in(rr_node, node_online_map); 602 } 603 } 604 605 pv_spinlock_init(); 606 } 607 608 /* called from main before smp_init() */ 609 void __init smp_prepare_cpus(unsigned int max_cpus) 610 { 611 init_new_context(current, &init_mm); 612 current_thread_info()->cpu = 0; 613 loongson_prepare_cpus(max_cpus); 614 set_cpu_sibling_map(0); 615 set_cpu_llc_shared_map(0); 616 set_cpu_core_map(0); 617 calculate_cpu_foreign_map(); 618 #ifndef CONFIG_HOTPLUG_CPU 619 init_cpu_present(cpu_possible_mask); 620 #endif 621 } 622 623 int __cpu_up(unsigned int cpu, struct task_struct *tidle) 624 { 625 loongson_boot_secondary(cpu, tidle); 626 627 /* Wait for CPU to start and be ready to sync counters */ 628 if (!wait_for_completion_timeout(&cpu_starting, 629 msecs_to_jiffies(5000))) { 630 pr_crit("CPU%u: failed to start\n", cpu); 631 return -EIO; 632 } 633 634 /* Wait for CPU to finish startup & mark itself online before return */ 635 wait_for_completion(&cpu_running); 636 637 return 0; 638 } 639 640 /* 641 * First C code run on the secondary CPUs after being started up by 642 * the master. 643 */ 644 asmlinkage void start_secondary(void) 645 { 646 unsigned int cpu; 647 648 sync_counter(); 649 cpu = raw_smp_processor_id(); 650 set_my_cpu_offset(per_cpu_offset(cpu)); 651 652 cpu_probe(); 653 constant_clockevent_init(); 654 loongson_init_secondary(); 655 656 set_cpu_sibling_map(cpu); 657 set_cpu_llc_shared_map(cpu); 658 set_cpu_core_map(cpu); 659 660 notify_cpu_starting(cpu); 661 662 /* Notify boot CPU that we're starting */ 663 complete(&cpu_starting); 664 665 /* The CPU is running, now mark it online */ 666 set_cpu_online(cpu, true); 667 668 calculate_cpu_foreign_map(); 669 670 /* 671 * Notify boot CPU that we're up & online and it can safely return 672 * from __cpu_up() 673 */ 674 complete(&cpu_running); 675 676 /* 677 * irq will be enabled in loongson_smp_finish(), enabling it too 678 * early is dangerous. 679 */ 680 WARN_ON_ONCE(!irqs_disabled()); 681 loongson_smp_finish(); 682 683 cpu_startup_entry(CPUHP_AP_ONLINE_IDLE); 684 } 685 686 void __init smp_cpus_done(unsigned int max_cpus) 687 { 688 } 689 690 static void stop_this_cpu(void *dummy) 691 { 692 set_cpu_online(smp_processor_id(), false); 693 calculate_cpu_foreign_map(); 694 local_irq_disable(); 695 while (true); 696 } 697 698 void smp_send_stop(void) 699 { 700 smp_call_function(stop_this_cpu, NULL, 0); 701 } 702 703 #ifdef CONFIG_PROFILING 704 int setup_profiling_timer(unsigned int multiplier) 705 { 706 return 0; 707 } 708 #endif 709 710 static void flush_tlb_all_ipi(void *info) 711 { 712 local_flush_tlb_all(); 713 } 714 715 void flush_tlb_all(void) 716 { 717 on_each_cpu(flush_tlb_all_ipi, NULL, 1); 718 } 719 720 static void flush_tlb_mm_ipi(void *mm) 721 { 722 local_flush_tlb_mm((struct mm_struct *)mm); 723 } 724 725 void flush_tlb_mm(struct mm_struct *mm) 726 { 727 if (atomic_read(&mm->mm_users) == 0) 728 return; /* happens as a result of exit_mmap() */ 729 730 preempt_disable(); 731 732 if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) { 733 on_each_cpu_mask(mm_cpumask(mm), flush_tlb_mm_ipi, mm, 1); 734 } else { 735 unsigned int cpu; 736 737 for_each_online_cpu(cpu) { 738 if (cpu != smp_processor_id() && cpu_context(cpu, mm)) 739 cpu_context(cpu, mm) = 0; 740 } 741 local_flush_tlb_mm(mm); 742 } 743 744 preempt_enable(); 745 } 746 747 struct flush_tlb_data { 748 struct vm_area_struct *vma; 749 unsigned long addr1; 750 unsigned long addr2; 751 }; 752 753 static void flush_tlb_range_ipi(void *info) 754 { 755 struct flush_tlb_data *fd = info; 756 757 local_flush_tlb_range(fd->vma, fd->addr1, fd->addr2); 758 } 759 760 void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) 761 { 762 struct mm_struct *mm = vma->vm_mm; 763 764 preempt_disable(); 765 if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) { 766 struct flush_tlb_data fd = { 767 .vma = vma, 768 .addr1 = start, 769 .addr2 = end, 770 }; 771 772 on_each_cpu_mask(mm_cpumask(mm), flush_tlb_range_ipi, &fd, 1); 773 } else { 774 unsigned int cpu; 775 776 for_each_online_cpu(cpu) { 777 if (cpu != smp_processor_id() && cpu_context(cpu, mm)) 778 cpu_context(cpu, mm) = 0; 779 } 780 local_flush_tlb_range(vma, start, end); 781 } 782 preempt_enable(); 783 } 784 785 static void flush_tlb_kernel_range_ipi(void *info) 786 { 787 struct flush_tlb_data *fd = info; 788 789 local_flush_tlb_kernel_range(fd->addr1, fd->addr2); 790 } 791 792 void flush_tlb_kernel_range(unsigned long start, unsigned long end) 793 { 794 struct flush_tlb_data fd = { 795 .addr1 = start, 796 .addr2 = end, 797 }; 798 799 on_each_cpu(flush_tlb_kernel_range_ipi, &fd, 1); 800 } 801 802 static void flush_tlb_page_ipi(void *info) 803 { 804 struct flush_tlb_data *fd = info; 805 806 local_flush_tlb_page(fd->vma, fd->addr1); 807 } 808 809 void flush_tlb_page(struct vm_area_struct *vma, unsigned long page) 810 { 811 preempt_disable(); 812 if ((atomic_read(&vma->vm_mm->mm_users) != 1) || (current->mm != vma->vm_mm)) { 813 struct flush_tlb_data fd = { 814 .vma = vma, 815 .addr1 = page, 816 }; 817 818 on_each_cpu_mask(mm_cpumask(vma->vm_mm), flush_tlb_page_ipi, &fd, 1); 819 } else { 820 unsigned int cpu; 821 822 for_each_online_cpu(cpu) { 823 if (cpu != smp_processor_id() && cpu_context(cpu, vma->vm_mm)) 824 cpu_context(cpu, vma->vm_mm) = 0; 825 } 826 local_flush_tlb_page(vma, page); 827 } 828 preempt_enable(); 829 } 830 EXPORT_SYMBOL(flush_tlb_page); 831 832 static void flush_tlb_one_ipi(void *info) 833 { 834 unsigned long vaddr = (unsigned long) info; 835 836 local_flush_tlb_one(vaddr); 837 } 838 839 void flush_tlb_one(unsigned long vaddr) 840 { 841 on_each_cpu(flush_tlb_one_ipi, (void *)vaddr, 1); 842 } 843 EXPORT_SYMBOL(flush_tlb_one); 844