1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited 4 * 5 * Derived from MIPS: 6 * Copyright (C) 2000, 2001 Kanoj Sarcar 7 * Copyright (C) 2000, 2001 Ralf Baechle 8 * Copyright (C) 2000, 2001 Silicon Graphics, Inc. 9 * Copyright (C) 2000, 2001, 2003 Broadcom Corporation 10 */ 11 #include <linux/acpi.h> 12 #include <linux/cpu.h> 13 #include <linux/cpumask.h> 14 #include <linux/init.h> 15 #include <linux/interrupt.h> 16 #include <linux/irq_work.h> 17 #include <linux/profile.h> 18 #include <linux/seq_file.h> 19 #include <linux/smp.h> 20 #include <linux/threads.h> 21 #include <linux/export.h> 22 #include <linux/syscore_ops.h> 23 #include <linux/time.h> 24 #include <linux/tracepoint.h> 25 #include <linux/sched/hotplug.h> 26 #include <linux/sched/task_stack.h> 27 28 #include <asm/cpu.h> 29 #include <asm/idle.h> 30 #include <asm/loongson.h> 31 #include <asm/mmu_context.h> 32 #include <asm/numa.h> 33 #include <asm/paravirt.h> 34 #include <asm/processor.h> 35 #include <asm/setup.h> 36 #include <asm/time.h> 37 38 int __cpu_number_map[NR_CPUS]; /* Map physical to logical */ 39 EXPORT_SYMBOL(__cpu_number_map); 40 41 int __cpu_logical_map[NR_CPUS]; /* Map logical to physical */ 42 EXPORT_SYMBOL(__cpu_logical_map); 43 44 /* Representing the threads (siblings) of each logical CPU */ 45 cpumask_t cpu_sibling_map[NR_CPUS] __read_mostly; 46 EXPORT_SYMBOL(cpu_sibling_map); 47 48 /* Representing the core map of multi-core chips of each logical CPU */ 49 cpumask_t cpu_core_map[NR_CPUS] __read_mostly; 50 EXPORT_SYMBOL(cpu_core_map); 51 52 static DECLARE_COMPLETION(cpu_starting); 53 static DECLARE_COMPLETION(cpu_running); 54 55 /* 56 * A logcal cpu mask containing only one VPE per core to 57 * reduce the number of IPIs on large MT systems. 58 */ 59 cpumask_t cpu_foreign_map[NR_CPUS] __read_mostly; 60 EXPORT_SYMBOL(cpu_foreign_map); 61 62 /* representing cpus for which sibling maps can be computed */ 63 static cpumask_t cpu_sibling_setup_map; 64 65 /* representing cpus for which core maps can be computed */ 66 static cpumask_t cpu_core_setup_map; 67 68 struct secondary_data cpuboot_data; 69 static DEFINE_PER_CPU(int, cpu_state); 70 71 static const char *ipi_types[NR_IPI] __tracepoint_string = { 72 [IPI_RESCHEDULE] = "Rescheduling interrupts", 73 [IPI_CALL_FUNCTION] = "Function call interrupts", 74 [IPI_IRQ_WORK] = "IRQ work interrupts", 75 [IPI_CLEAR_VECTOR] = "Clear vector interrupts", 76 }; 77 78 void show_ipi_list(struct seq_file *p, int prec) 79 { 80 unsigned int cpu, i; 81 82 for (i = 0; i < NR_IPI; i++) { 83 seq_printf(p, "%*s%u:%s", prec - 1, "IPI", i, prec >= 4 ? " " : ""); 84 for_each_online_cpu(cpu) 85 seq_put_decimal_ull_width(p, " ", per_cpu(irq_stat, cpu).ipi_irqs[i], 10); 86 seq_printf(p, " LoongArch %d %s\n", i + 1, ipi_types[i]); 87 } 88 } 89 90 static inline void set_cpu_core_map(int cpu) 91 { 92 int i; 93 94 cpumask_set_cpu(cpu, &cpu_core_setup_map); 95 96 for_each_cpu(i, &cpu_core_setup_map) { 97 if (cpu_data[cpu].package == cpu_data[i].package) { 98 cpumask_set_cpu(i, &cpu_core_map[cpu]); 99 cpumask_set_cpu(cpu, &cpu_core_map[i]); 100 } 101 } 102 } 103 104 static inline void set_cpu_sibling_map(int cpu) 105 { 106 int i; 107 108 cpumask_set_cpu(cpu, &cpu_sibling_setup_map); 109 110 for_each_cpu(i, &cpu_sibling_setup_map) { 111 if (cpus_are_siblings(cpu, i)) { 112 cpumask_set_cpu(i, &cpu_sibling_map[cpu]); 113 cpumask_set_cpu(cpu, &cpu_sibling_map[i]); 114 } 115 } 116 } 117 118 static inline void clear_cpu_sibling_map(int cpu) 119 { 120 int i; 121 122 for_each_cpu(i, &cpu_sibling_setup_map) { 123 if (cpus_are_siblings(cpu, i)) { 124 cpumask_clear_cpu(i, &cpu_sibling_map[cpu]); 125 cpumask_clear_cpu(cpu, &cpu_sibling_map[i]); 126 } 127 } 128 129 cpumask_clear_cpu(cpu, &cpu_sibling_setup_map); 130 } 131 132 /* 133 * Calculate a new cpu_foreign_map mask whenever a 134 * new cpu appears or disappears. 135 */ 136 void calculate_cpu_foreign_map(void) 137 { 138 int i, k, core_present; 139 cpumask_t temp_foreign_map; 140 141 /* Re-calculate the mask */ 142 cpumask_clear(&temp_foreign_map); 143 for_each_online_cpu(i) { 144 core_present = 0; 145 for_each_cpu(k, &temp_foreign_map) 146 if (cpus_are_siblings(i, k)) 147 core_present = 1; 148 if (!core_present) 149 cpumask_set_cpu(i, &temp_foreign_map); 150 } 151 152 for_each_online_cpu(i) 153 cpumask_andnot(&cpu_foreign_map[i], 154 &temp_foreign_map, &cpu_sibling_map[i]); 155 } 156 157 /* Send mailbox buffer via Mail_Send */ 158 static void csr_mail_send(uint64_t data, int cpu, int mailbox) 159 { 160 uint64_t val; 161 162 /* Send high 32 bits */ 163 val = IOCSR_MBUF_SEND_BLOCKING; 164 val |= (IOCSR_MBUF_SEND_BOX_HI(mailbox) << IOCSR_MBUF_SEND_BOX_SHIFT); 165 val |= (cpu << IOCSR_MBUF_SEND_CPU_SHIFT); 166 val |= (data & IOCSR_MBUF_SEND_H32_MASK); 167 iocsr_write64(val, LOONGARCH_IOCSR_MBUF_SEND); 168 169 /* Send low 32 bits */ 170 val = IOCSR_MBUF_SEND_BLOCKING; 171 val |= (IOCSR_MBUF_SEND_BOX_LO(mailbox) << IOCSR_MBUF_SEND_BOX_SHIFT); 172 val |= (cpu << IOCSR_MBUF_SEND_CPU_SHIFT); 173 val |= (data << IOCSR_MBUF_SEND_BUF_SHIFT); 174 iocsr_write64(val, LOONGARCH_IOCSR_MBUF_SEND); 175 }; 176 177 static u32 ipi_read_clear(int cpu) 178 { 179 u32 action; 180 181 /* Load the ipi register to figure out what we're supposed to do */ 182 action = iocsr_read32(LOONGARCH_IOCSR_IPI_STATUS); 183 /* Clear the ipi register to clear the interrupt */ 184 iocsr_write32(action, LOONGARCH_IOCSR_IPI_CLEAR); 185 wbflush(); 186 187 return action; 188 } 189 190 static void ipi_write_action(int cpu, u32 action) 191 { 192 uint32_t val; 193 194 val = IOCSR_IPI_SEND_BLOCKING | action; 195 val |= (cpu << IOCSR_IPI_SEND_CPU_SHIFT); 196 iocsr_write32(val, LOONGARCH_IOCSR_IPI_SEND); 197 } 198 199 static void loongson_send_ipi_single(int cpu, unsigned int action) 200 { 201 ipi_write_action(cpu_logical_map(cpu), (u32)action); 202 } 203 204 static void loongson_send_ipi_mask(const struct cpumask *mask, unsigned int action) 205 { 206 unsigned int i; 207 208 for_each_cpu(i, mask) 209 ipi_write_action(cpu_logical_map(i), (u32)action); 210 } 211 212 /* 213 * This function sends a 'reschedule' IPI to another CPU. 214 * it goes straight through and wastes no time serializing 215 * anything. Worst case is that we lose a reschedule ... 216 */ 217 void arch_smp_send_reschedule(int cpu) 218 { 219 mp_ops.send_ipi_single(cpu, ACTION_RESCHEDULE); 220 } 221 EXPORT_SYMBOL_GPL(arch_smp_send_reschedule); 222 223 #ifdef CONFIG_IRQ_WORK 224 void arch_irq_work_raise(void) 225 { 226 mp_ops.send_ipi_single(smp_processor_id(), ACTION_IRQ_WORK); 227 } 228 #endif 229 230 static irqreturn_t loongson_ipi_interrupt(int irq, void *dev) 231 { 232 unsigned int action; 233 unsigned int cpu = smp_processor_id(); 234 235 action = ipi_read_clear(cpu_logical_map(cpu)); 236 237 if (action & SMP_RESCHEDULE) { 238 scheduler_ipi(); 239 per_cpu(irq_stat, cpu).ipi_irqs[IPI_RESCHEDULE]++; 240 } 241 242 if (action & SMP_CALL_FUNCTION) { 243 generic_smp_call_function_interrupt(); 244 per_cpu(irq_stat, cpu).ipi_irqs[IPI_CALL_FUNCTION]++; 245 } 246 247 if (action & SMP_IRQ_WORK) { 248 irq_work_run(); 249 per_cpu(irq_stat, cpu).ipi_irqs[IPI_IRQ_WORK]++; 250 } 251 252 if (action & SMP_CLEAR_VECTOR) { 253 complete_irq_moving(); 254 per_cpu(irq_stat, cpu).ipi_irqs[IPI_CLEAR_VECTOR]++; 255 } 256 257 return IRQ_HANDLED; 258 } 259 260 static void loongson_init_ipi(void) 261 { 262 int r, ipi_irq; 263 264 ipi_irq = get_percpu_irq(INT_IPI); 265 if (ipi_irq < 0) 266 panic("IPI IRQ mapping failed\n"); 267 268 irq_set_percpu_devid(ipi_irq); 269 r = request_percpu_irq(ipi_irq, loongson_ipi_interrupt, "IPI", &irq_stat); 270 if (r < 0) 271 panic("IPI IRQ request failed\n"); 272 } 273 274 struct smp_ops mp_ops = { 275 .init_ipi = loongson_init_ipi, 276 .send_ipi_single = loongson_send_ipi_single, 277 .send_ipi_mask = loongson_send_ipi_mask, 278 }; 279 280 static void __init fdt_smp_setup(void) 281 { 282 #ifdef CONFIG_OF 283 unsigned int cpu, cpuid; 284 struct device_node *node = NULL; 285 286 for_each_of_cpu_node(node) { 287 if (!of_device_is_available(node)) 288 continue; 289 290 cpuid = of_get_cpu_hwid(node, 0); 291 if (cpuid >= nr_cpu_ids) 292 continue; 293 294 if (cpuid == loongson_sysconf.boot_cpu_id) 295 cpu = 0; 296 else 297 cpu = find_first_zero_bit(cpumask_bits(cpu_present_mask), NR_CPUS); 298 299 num_processors++; 300 set_cpu_possible(cpu, true); 301 set_cpu_present(cpu, true); 302 __cpu_number_map[cpuid] = cpu; 303 __cpu_logical_map[cpu] = cpuid; 304 305 early_numa_add_cpu(cpuid, 0); 306 set_cpuid_to_node(cpuid, 0); 307 } 308 309 loongson_sysconf.nr_cpus = num_processors; 310 set_bit(0, loongson_sysconf.cores_io_master); 311 #endif 312 } 313 314 void __init loongson_smp_setup(void) 315 { 316 fdt_smp_setup(); 317 318 if (loongson_sysconf.cores_per_package == 0) 319 loongson_sysconf.cores_per_package = num_processors; 320 321 cpu_data[0].core = cpu_logical_map(0) % loongson_sysconf.cores_per_package; 322 cpu_data[0].package = cpu_logical_map(0) / loongson_sysconf.cores_per_package; 323 324 pv_ipi_init(); 325 iocsr_write32(0xffffffff, LOONGARCH_IOCSR_IPI_EN); 326 pr_info("Detected %i available CPU(s)\n", loongson_sysconf.nr_cpus); 327 } 328 329 void __init loongson_prepare_cpus(unsigned int max_cpus) 330 { 331 int i = 0; 332 333 parse_acpi_topology(); 334 cpu_data[0].global_id = cpu_logical_map(0); 335 336 for (i = 0; i < loongson_sysconf.nr_cpus; i++) { 337 set_cpu_present(i, true); 338 csr_mail_send(0, __cpu_logical_map[i], 0); 339 } 340 341 per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE; 342 } 343 344 /* 345 * Setup the PC, SP, and TP of a secondary processor and start it running! 346 */ 347 void loongson_boot_secondary(int cpu, struct task_struct *idle) 348 { 349 unsigned long entry; 350 351 pr_info("Booting CPU#%d...\n", cpu); 352 353 entry = __pa_symbol((unsigned long)&smpboot_entry); 354 cpuboot_data.stack = (unsigned long)__KSTK_TOS(idle); 355 cpuboot_data.thread_info = (unsigned long)task_thread_info(idle); 356 357 csr_mail_send(entry, cpu_logical_map(cpu), 0); 358 359 loongson_send_ipi_single(cpu, ACTION_BOOT_CPU); 360 } 361 362 /* 363 * SMP init and finish on secondary CPUs 364 */ 365 void loongson_init_secondary(void) 366 { 367 unsigned int cpu = smp_processor_id(); 368 unsigned int imask = ECFGF_IP0 | ECFGF_IP1 | ECFGF_IP2 | 369 ECFGF_IPI | ECFGF_PMC | ECFGF_TIMER | ECFGF_SIP0; 370 371 change_csr_ecfg(ECFG0_IM, imask); 372 373 iocsr_write32(0xffffffff, LOONGARCH_IOCSR_IPI_EN); 374 375 #ifdef CONFIG_NUMA 376 numa_add_cpu(cpu); 377 #endif 378 per_cpu(cpu_state, cpu) = CPU_ONLINE; 379 cpu_data[cpu].package = 380 cpu_logical_map(cpu) / loongson_sysconf.cores_per_package; 381 cpu_data[cpu].core = pptt_enabled ? cpu_data[cpu].core : 382 cpu_logical_map(cpu) % loongson_sysconf.cores_per_package; 383 cpu_data[cpu].global_id = cpu_logical_map(cpu); 384 } 385 386 void loongson_smp_finish(void) 387 { 388 local_irq_enable(); 389 iocsr_write64(0, LOONGARCH_IOCSR_MBUF0); 390 pr_info("CPU#%d finished\n", smp_processor_id()); 391 } 392 393 #ifdef CONFIG_HOTPLUG_CPU 394 395 int loongson_cpu_disable(void) 396 { 397 unsigned long flags; 398 unsigned int cpu = smp_processor_id(); 399 400 if (io_master(cpu)) 401 return -EBUSY; 402 403 #ifdef CONFIG_NUMA 404 numa_remove_cpu(cpu); 405 #endif 406 set_cpu_online(cpu, false); 407 clear_cpu_sibling_map(cpu); 408 calculate_cpu_foreign_map(); 409 local_irq_save(flags); 410 irq_migrate_all_off_this_cpu(); 411 clear_csr_ecfg(ECFG0_IM); 412 local_irq_restore(flags); 413 local_flush_tlb_all(); 414 415 return 0; 416 } 417 418 void loongson_cpu_die(unsigned int cpu) 419 { 420 while (per_cpu(cpu_state, cpu) != CPU_DEAD) 421 cpu_relax(); 422 423 mb(); 424 } 425 426 void __noreturn arch_cpu_idle_dead(void) 427 { 428 register uint64_t addr; 429 register void (*init_fn)(void); 430 431 idle_task_exit(); 432 local_irq_enable(); 433 set_csr_ecfg(ECFGF_IPI); 434 __this_cpu_write(cpu_state, CPU_DEAD); 435 436 __smp_mb(); 437 do { 438 __asm__ __volatile__("idle 0\n\t"); 439 addr = iocsr_read64(LOONGARCH_IOCSR_MBUF0); 440 } while (addr == 0); 441 442 local_irq_disable(); 443 init_fn = (void *)TO_CACHE(addr); 444 iocsr_write32(0xffffffff, LOONGARCH_IOCSR_IPI_CLEAR); 445 446 init_fn(); 447 BUG(); 448 } 449 450 #endif 451 452 /* 453 * Power management 454 */ 455 #ifdef CONFIG_PM 456 457 static int loongson_ipi_suspend(void) 458 { 459 return 0; 460 } 461 462 static void loongson_ipi_resume(void) 463 { 464 iocsr_write32(0xffffffff, LOONGARCH_IOCSR_IPI_EN); 465 } 466 467 static struct syscore_ops loongson_ipi_syscore_ops = { 468 .resume = loongson_ipi_resume, 469 .suspend = loongson_ipi_suspend, 470 }; 471 472 /* 473 * Enable boot cpu ipi before enabling nonboot cpus 474 * during syscore_resume. 475 */ 476 static int __init ipi_pm_init(void) 477 { 478 register_syscore_ops(&loongson_ipi_syscore_ops); 479 return 0; 480 } 481 482 core_initcall(ipi_pm_init); 483 #endif 484 485 /* Preload SMP state for boot cpu */ 486 void __init smp_prepare_boot_cpu(void) 487 { 488 unsigned int cpu, node, rr_node; 489 490 set_cpu_possible(0, true); 491 set_cpu_online(0, true); 492 set_my_cpu_offset(per_cpu_offset(0)); 493 numa_add_cpu(0); 494 495 rr_node = first_node(node_online_map); 496 for_each_possible_cpu(cpu) { 497 node = early_cpu_to_node(cpu); 498 499 /* 500 * The mapping between present cpus and nodes has been 501 * built during MADT and SRAT parsing. 502 * 503 * If possible cpus = present cpus here, early_cpu_to_node 504 * will return valid node. 505 * 506 * If possible cpus > present cpus here (e.g. some possible 507 * cpus will be added by cpu-hotplug later), for possible but 508 * not present cpus, early_cpu_to_node will return NUMA_NO_NODE, 509 * and we just map them to online nodes in round-robin way. 510 * Once hotplugged, new correct mapping will be built for them. 511 */ 512 if (node != NUMA_NO_NODE) 513 set_cpu_numa_node(cpu, node); 514 else { 515 set_cpu_numa_node(cpu, rr_node); 516 rr_node = next_node_in(rr_node, node_online_map); 517 } 518 } 519 520 pv_spinlock_init(); 521 } 522 523 /* called from main before smp_init() */ 524 void __init smp_prepare_cpus(unsigned int max_cpus) 525 { 526 init_new_context(current, &init_mm); 527 current_thread_info()->cpu = 0; 528 loongson_prepare_cpus(max_cpus); 529 set_cpu_sibling_map(0); 530 set_cpu_core_map(0); 531 calculate_cpu_foreign_map(); 532 #ifndef CONFIG_HOTPLUG_CPU 533 init_cpu_present(cpu_possible_mask); 534 #endif 535 } 536 537 int __cpu_up(unsigned int cpu, struct task_struct *tidle) 538 { 539 loongson_boot_secondary(cpu, tidle); 540 541 /* Wait for CPU to start and be ready to sync counters */ 542 if (!wait_for_completion_timeout(&cpu_starting, 543 msecs_to_jiffies(5000))) { 544 pr_crit("CPU%u: failed to start\n", cpu); 545 return -EIO; 546 } 547 548 /* Wait for CPU to finish startup & mark itself online before return */ 549 wait_for_completion(&cpu_running); 550 551 return 0; 552 } 553 554 /* 555 * First C code run on the secondary CPUs after being started up by 556 * the master. 557 */ 558 asmlinkage void start_secondary(void) 559 { 560 unsigned int cpu; 561 562 sync_counter(); 563 cpu = raw_smp_processor_id(); 564 set_my_cpu_offset(per_cpu_offset(cpu)); 565 566 cpu_probe(); 567 constant_clockevent_init(); 568 loongson_init_secondary(); 569 570 set_cpu_sibling_map(cpu); 571 set_cpu_core_map(cpu); 572 573 notify_cpu_starting(cpu); 574 575 /* Notify boot CPU that we're starting */ 576 complete(&cpu_starting); 577 578 /* The CPU is running, now mark it online */ 579 set_cpu_online(cpu, true); 580 581 calculate_cpu_foreign_map(); 582 583 /* 584 * Notify boot CPU that we're up & online and it can safely return 585 * from __cpu_up() 586 */ 587 complete(&cpu_running); 588 589 /* 590 * irq will be enabled in loongson_smp_finish(), enabling it too 591 * early is dangerous. 592 */ 593 WARN_ON_ONCE(!irqs_disabled()); 594 loongson_smp_finish(); 595 596 cpu_startup_entry(CPUHP_AP_ONLINE_IDLE); 597 } 598 599 void __init smp_cpus_done(unsigned int max_cpus) 600 { 601 } 602 603 static void stop_this_cpu(void *dummy) 604 { 605 set_cpu_online(smp_processor_id(), false); 606 calculate_cpu_foreign_map(); 607 local_irq_disable(); 608 while (true); 609 } 610 611 void smp_send_stop(void) 612 { 613 smp_call_function(stop_this_cpu, NULL, 0); 614 } 615 616 #ifdef CONFIG_PROFILING 617 int setup_profiling_timer(unsigned int multiplier) 618 { 619 return 0; 620 } 621 #endif 622 623 static void flush_tlb_all_ipi(void *info) 624 { 625 local_flush_tlb_all(); 626 } 627 628 void flush_tlb_all(void) 629 { 630 on_each_cpu(flush_tlb_all_ipi, NULL, 1); 631 } 632 633 static void flush_tlb_mm_ipi(void *mm) 634 { 635 local_flush_tlb_mm((struct mm_struct *)mm); 636 } 637 638 void flush_tlb_mm(struct mm_struct *mm) 639 { 640 if (atomic_read(&mm->mm_users) == 0) 641 return; /* happens as a result of exit_mmap() */ 642 643 preempt_disable(); 644 645 if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) { 646 on_each_cpu_mask(mm_cpumask(mm), flush_tlb_mm_ipi, mm, 1); 647 } else { 648 unsigned int cpu; 649 650 for_each_online_cpu(cpu) { 651 if (cpu != smp_processor_id() && cpu_context(cpu, mm)) 652 cpu_context(cpu, mm) = 0; 653 } 654 local_flush_tlb_mm(mm); 655 } 656 657 preempt_enable(); 658 } 659 660 struct flush_tlb_data { 661 struct vm_area_struct *vma; 662 unsigned long addr1; 663 unsigned long addr2; 664 }; 665 666 static void flush_tlb_range_ipi(void *info) 667 { 668 struct flush_tlb_data *fd = info; 669 670 local_flush_tlb_range(fd->vma, fd->addr1, fd->addr2); 671 } 672 673 void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) 674 { 675 struct mm_struct *mm = vma->vm_mm; 676 677 preempt_disable(); 678 if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) { 679 struct flush_tlb_data fd = { 680 .vma = vma, 681 .addr1 = start, 682 .addr2 = end, 683 }; 684 685 on_each_cpu_mask(mm_cpumask(mm), flush_tlb_range_ipi, &fd, 1); 686 } else { 687 unsigned int cpu; 688 689 for_each_online_cpu(cpu) { 690 if (cpu != smp_processor_id() && cpu_context(cpu, mm)) 691 cpu_context(cpu, mm) = 0; 692 } 693 local_flush_tlb_range(vma, start, end); 694 } 695 preempt_enable(); 696 } 697 698 static void flush_tlb_kernel_range_ipi(void *info) 699 { 700 struct flush_tlb_data *fd = info; 701 702 local_flush_tlb_kernel_range(fd->addr1, fd->addr2); 703 } 704 705 void flush_tlb_kernel_range(unsigned long start, unsigned long end) 706 { 707 struct flush_tlb_data fd = { 708 .addr1 = start, 709 .addr2 = end, 710 }; 711 712 on_each_cpu(flush_tlb_kernel_range_ipi, &fd, 1); 713 } 714 715 static void flush_tlb_page_ipi(void *info) 716 { 717 struct flush_tlb_data *fd = info; 718 719 local_flush_tlb_page(fd->vma, fd->addr1); 720 } 721 722 void flush_tlb_page(struct vm_area_struct *vma, unsigned long page) 723 { 724 preempt_disable(); 725 if ((atomic_read(&vma->vm_mm->mm_users) != 1) || (current->mm != vma->vm_mm)) { 726 struct flush_tlb_data fd = { 727 .vma = vma, 728 .addr1 = page, 729 }; 730 731 on_each_cpu_mask(mm_cpumask(vma->vm_mm), flush_tlb_page_ipi, &fd, 1); 732 } else { 733 unsigned int cpu; 734 735 for_each_online_cpu(cpu) { 736 if (cpu != smp_processor_id() && cpu_context(cpu, vma->vm_mm)) 737 cpu_context(cpu, vma->vm_mm) = 0; 738 } 739 local_flush_tlb_page(vma, page); 740 } 741 preempt_enable(); 742 } 743 EXPORT_SYMBOL(flush_tlb_page); 744 745 static void flush_tlb_one_ipi(void *info) 746 { 747 unsigned long vaddr = (unsigned long) info; 748 749 local_flush_tlb_one(vaddr); 750 } 751 752 void flush_tlb_one(unsigned long vaddr) 753 { 754 on_each_cpu(flush_tlb_one_ipi, (void *)vaddr, 1); 755 } 756 EXPORT_SYMBOL(flush_tlb_one); 757