1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited 4 * 5 * Derived from MIPS: 6 * Copyright (C) 2000, 2001 Kanoj Sarcar 7 * Copyright (C) 2000, 2001 Ralf Baechle 8 * Copyright (C) 2000, 2001 Silicon Graphics, Inc. 9 * Copyright (C) 2000, 2001, 2003 Broadcom Corporation 10 */ 11 #include <linux/acpi.h> 12 #include <linux/cpu.h> 13 #include <linux/cpumask.h> 14 #include <linux/init.h> 15 #include <linux/interrupt.h> 16 #include <linux/irq_work.h> 17 #include <linux/profile.h> 18 #include <linux/seq_file.h> 19 #include <linux/smp.h> 20 #include <linux/threads.h> 21 #include <linux/export.h> 22 #include <linux/suspend.h> 23 #include <linux/syscore_ops.h> 24 #include <linux/time.h> 25 #include <linux/tracepoint.h> 26 #include <linux/sched/hotplug.h> 27 #include <linux/sched/task_stack.h> 28 29 #include <asm/cpu.h> 30 #include <asm/idle.h> 31 #include <asm/loongson.h> 32 #include <asm/mmu_context.h> 33 #include <asm/numa.h> 34 #include <asm/paravirt.h> 35 #include <asm/processor.h> 36 #include <asm/setup.h> 37 #include <asm/time.h> 38 39 int __cpu_number_map[NR_CPUS]; /* Map physical to logical */ 40 EXPORT_SYMBOL(__cpu_number_map); 41 42 int __cpu_logical_map[NR_CPUS]; /* Map logical to physical */ 43 EXPORT_SYMBOL(__cpu_logical_map); 44 45 /* Representing the threads (siblings) of each logical CPU */ 46 cpumask_t cpu_sibling_map[NR_CPUS] __read_mostly; 47 EXPORT_SYMBOL(cpu_sibling_map); 48 49 /* Representing the core map of multi-core chips of each logical CPU */ 50 cpumask_t cpu_core_map[NR_CPUS] __read_mostly; 51 EXPORT_SYMBOL(cpu_core_map); 52 53 static DECLARE_COMPLETION(cpu_starting); 54 static DECLARE_COMPLETION(cpu_running); 55 56 /* 57 * A logcal cpu mask containing only one VPE per core to 58 * reduce the number of IPIs on large MT systems. 59 */ 60 cpumask_t cpu_foreign_map[NR_CPUS] __read_mostly; 61 EXPORT_SYMBOL(cpu_foreign_map); 62 63 /* representing cpus for which sibling maps can be computed */ 64 static cpumask_t cpu_sibling_setup_map; 65 66 /* representing cpus for which core maps can be computed */ 67 static cpumask_t cpu_core_setup_map; 68 69 struct secondary_data cpuboot_data; 70 static DEFINE_PER_CPU(int, cpu_state); 71 72 static const char *ipi_types[NR_IPI] __tracepoint_string = { 73 [IPI_RESCHEDULE] = "Rescheduling interrupts", 74 [IPI_CALL_FUNCTION] = "Function call interrupts", 75 [IPI_IRQ_WORK] = "IRQ work interrupts", 76 [IPI_CLEAR_VECTOR] = "Clear vector interrupts", 77 }; 78 79 void show_ipi_list(struct seq_file *p, int prec) 80 { 81 unsigned int cpu, i; 82 83 for (i = 0; i < NR_IPI; i++) { 84 seq_printf(p, "%*s%u:%s", prec - 1, "IPI", i, prec >= 4 ? " " : ""); 85 for_each_online_cpu(cpu) 86 seq_put_decimal_ull_width(p, " ", per_cpu(irq_stat, cpu).ipi_irqs[i], 10); 87 seq_printf(p, " LoongArch %d %s\n", i + 1, ipi_types[i]); 88 } 89 } 90 91 static inline void set_cpu_core_map(int cpu) 92 { 93 int i; 94 95 cpumask_set_cpu(cpu, &cpu_core_setup_map); 96 97 for_each_cpu(i, &cpu_core_setup_map) { 98 if (cpu_data[cpu].package == cpu_data[i].package) { 99 cpumask_set_cpu(i, &cpu_core_map[cpu]); 100 cpumask_set_cpu(cpu, &cpu_core_map[i]); 101 } 102 } 103 } 104 105 static inline void set_cpu_sibling_map(int cpu) 106 { 107 int i; 108 109 cpumask_set_cpu(cpu, &cpu_sibling_setup_map); 110 111 for_each_cpu(i, &cpu_sibling_setup_map) { 112 if (cpus_are_siblings(cpu, i)) { 113 cpumask_set_cpu(i, &cpu_sibling_map[cpu]); 114 cpumask_set_cpu(cpu, &cpu_sibling_map[i]); 115 } 116 } 117 } 118 119 static inline void clear_cpu_sibling_map(int cpu) 120 { 121 int i; 122 123 for_each_cpu(i, &cpu_sibling_setup_map) { 124 if (cpus_are_siblings(cpu, i)) { 125 cpumask_clear_cpu(i, &cpu_sibling_map[cpu]); 126 cpumask_clear_cpu(cpu, &cpu_sibling_map[i]); 127 } 128 } 129 130 cpumask_clear_cpu(cpu, &cpu_sibling_setup_map); 131 } 132 133 /* 134 * Calculate a new cpu_foreign_map mask whenever a 135 * new cpu appears or disappears. 136 */ 137 void calculate_cpu_foreign_map(void) 138 { 139 int i, k, core_present; 140 cpumask_t temp_foreign_map; 141 142 /* Re-calculate the mask */ 143 cpumask_clear(&temp_foreign_map); 144 for_each_online_cpu(i) { 145 core_present = 0; 146 for_each_cpu(k, &temp_foreign_map) 147 if (cpus_are_siblings(i, k)) 148 core_present = 1; 149 if (!core_present) 150 cpumask_set_cpu(i, &temp_foreign_map); 151 } 152 153 for_each_online_cpu(i) 154 cpumask_andnot(&cpu_foreign_map[i], 155 &temp_foreign_map, &cpu_sibling_map[i]); 156 } 157 158 /* Send mailbox buffer via Mail_Send */ 159 static void csr_mail_send(uint64_t data, int cpu, int mailbox) 160 { 161 uint64_t val; 162 163 /* Send high 32 bits */ 164 val = IOCSR_MBUF_SEND_BLOCKING; 165 val |= (IOCSR_MBUF_SEND_BOX_HI(mailbox) << IOCSR_MBUF_SEND_BOX_SHIFT); 166 val |= (cpu << IOCSR_MBUF_SEND_CPU_SHIFT); 167 val |= (data & IOCSR_MBUF_SEND_H32_MASK); 168 iocsr_write64(val, LOONGARCH_IOCSR_MBUF_SEND); 169 170 /* Send low 32 bits */ 171 val = IOCSR_MBUF_SEND_BLOCKING; 172 val |= (IOCSR_MBUF_SEND_BOX_LO(mailbox) << IOCSR_MBUF_SEND_BOX_SHIFT); 173 val |= (cpu << IOCSR_MBUF_SEND_CPU_SHIFT); 174 val |= (data << IOCSR_MBUF_SEND_BUF_SHIFT); 175 iocsr_write64(val, LOONGARCH_IOCSR_MBUF_SEND); 176 }; 177 178 static u32 ipi_read_clear(int cpu) 179 { 180 u32 action; 181 182 /* Load the ipi register to figure out what we're supposed to do */ 183 action = iocsr_read32(LOONGARCH_IOCSR_IPI_STATUS); 184 /* Clear the ipi register to clear the interrupt */ 185 iocsr_write32(action, LOONGARCH_IOCSR_IPI_CLEAR); 186 wbflush(); 187 188 return action; 189 } 190 191 static void ipi_write_action(int cpu, u32 action) 192 { 193 uint32_t val; 194 195 val = IOCSR_IPI_SEND_BLOCKING | action; 196 val |= (cpu << IOCSR_IPI_SEND_CPU_SHIFT); 197 iocsr_write32(val, LOONGARCH_IOCSR_IPI_SEND); 198 } 199 200 static void loongson_send_ipi_single(int cpu, unsigned int action) 201 { 202 ipi_write_action(cpu_logical_map(cpu), (u32)action); 203 } 204 205 static void loongson_send_ipi_mask(const struct cpumask *mask, unsigned int action) 206 { 207 unsigned int i; 208 209 for_each_cpu(i, mask) 210 ipi_write_action(cpu_logical_map(i), (u32)action); 211 } 212 213 /* 214 * This function sends a 'reschedule' IPI to another CPU. 215 * it goes straight through and wastes no time serializing 216 * anything. Worst case is that we lose a reschedule ... 217 */ 218 void arch_smp_send_reschedule(int cpu) 219 { 220 mp_ops.send_ipi_single(cpu, ACTION_RESCHEDULE); 221 } 222 EXPORT_SYMBOL_GPL(arch_smp_send_reschedule); 223 224 #ifdef CONFIG_IRQ_WORK 225 void arch_irq_work_raise(void) 226 { 227 mp_ops.send_ipi_single(smp_processor_id(), ACTION_IRQ_WORK); 228 } 229 #endif 230 231 static irqreturn_t loongson_ipi_interrupt(int irq, void *dev) 232 { 233 unsigned int action; 234 unsigned int cpu = smp_processor_id(); 235 236 action = ipi_read_clear(cpu_logical_map(cpu)); 237 238 if (action & SMP_RESCHEDULE) { 239 scheduler_ipi(); 240 per_cpu(irq_stat, cpu).ipi_irqs[IPI_RESCHEDULE]++; 241 } 242 243 if (action & SMP_CALL_FUNCTION) { 244 generic_smp_call_function_interrupt(); 245 per_cpu(irq_stat, cpu).ipi_irqs[IPI_CALL_FUNCTION]++; 246 } 247 248 if (action & SMP_IRQ_WORK) { 249 irq_work_run(); 250 per_cpu(irq_stat, cpu).ipi_irqs[IPI_IRQ_WORK]++; 251 } 252 253 if (action & SMP_CLEAR_VECTOR) { 254 complete_irq_moving(); 255 per_cpu(irq_stat, cpu).ipi_irqs[IPI_CLEAR_VECTOR]++; 256 } 257 258 return IRQ_HANDLED; 259 } 260 261 static void loongson_init_ipi(void) 262 { 263 int r, ipi_irq; 264 265 ipi_irq = get_percpu_irq(INT_IPI); 266 if (ipi_irq < 0) 267 panic("IPI IRQ mapping failed\n"); 268 269 irq_set_percpu_devid(ipi_irq); 270 r = request_percpu_irq(ipi_irq, loongson_ipi_interrupt, "IPI", &irq_stat); 271 if (r < 0) 272 panic("IPI IRQ request failed\n"); 273 } 274 275 struct smp_ops mp_ops = { 276 .init_ipi = loongson_init_ipi, 277 .send_ipi_single = loongson_send_ipi_single, 278 .send_ipi_mask = loongson_send_ipi_mask, 279 }; 280 281 static void __init fdt_smp_setup(void) 282 { 283 #ifdef CONFIG_OF 284 unsigned int cpu, cpuid; 285 struct device_node *node = NULL; 286 287 for_each_of_cpu_node(node) { 288 if (!of_device_is_available(node)) 289 continue; 290 291 cpuid = of_get_cpu_hwid(node, 0); 292 if (cpuid >= nr_cpu_ids) 293 continue; 294 295 if (cpuid == loongson_sysconf.boot_cpu_id) 296 cpu = 0; 297 else 298 cpu = find_first_zero_bit(cpumask_bits(cpu_present_mask), NR_CPUS); 299 300 num_processors++; 301 set_cpu_possible(cpu, true); 302 set_cpu_present(cpu, true); 303 __cpu_number_map[cpuid] = cpu; 304 __cpu_logical_map[cpu] = cpuid; 305 306 early_numa_add_cpu(cpuid, 0); 307 set_cpuid_to_node(cpuid, 0); 308 } 309 310 loongson_sysconf.nr_cpus = num_processors; 311 set_bit(0, loongson_sysconf.cores_io_master); 312 #endif 313 } 314 315 void __init loongson_smp_setup(void) 316 { 317 fdt_smp_setup(); 318 319 if (loongson_sysconf.cores_per_package == 0) 320 loongson_sysconf.cores_per_package = num_processors; 321 322 cpu_data[0].core = cpu_logical_map(0) % loongson_sysconf.cores_per_package; 323 cpu_data[0].package = cpu_logical_map(0) / loongson_sysconf.cores_per_package; 324 325 pv_ipi_init(); 326 iocsr_write32(0xffffffff, LOONGARCH_IOCSR_IPI_EN); 327 pr_info("Detected %i available CPU(s)\n", loongson_sysconf.nr_cpus); 328 } 329 330 void __init loongson_prepare_cpus(unsigned int max_cpus) 331 { 332 int i = 0; 333 334 parse_acpi_topology(); 335 cpu_data[0].global_id = cpu_logical_map(0); 336 337 for (i = 0; i < loongson_sysconf.nr_cpus; i++) { 338 set_cpu_present(i, true); 339 csr_mail_send(0, __cpu_logical_map[i], 0); 340 } 341 342 per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE; 343 } 344 345 /* 346 * Setup the PC, SP, and TP of a secondary processor and start it running! 347 */ 348 void loongson_boot_secondary(int cpu, struct task_struct *idle) 349 { 350 unsigned long entry; 351 352 pr_info("Booting CPU#%d...\n", cpu); 353 354 entry = __pa_symbol((unsigned long)&smpboot_entry); 355 cpuboot_data.stack = (unsigned long)__KSTK_TOS(idle); 356 cpuboot_data.thread_info = (unsigned long)task_thread_info(idle); 357 358 csr_mail_send(entry, cpu_logical_map(cpu), 0); 359 360 loongson_send_ipi_single(cpu, ACTION_BOOT_CPU); 361 } 362 363 /* 364 * SMP init and finish on secondary CPUs 365 */ 366 void loongson_init_secondary(void) 367 { 368 unsigned int cpu = smp_processor_id(); 369 unsigned int imask = ECFGF_IP0 | ECFGF_IP1 | ECFGF_IP2 | 370 ECFGF_IPI | ECFGF_PMC | ECFGF_TIMER | ECFGF_SIP0; 371 372 change_csr_ecfg(ECFG0_IM, imask); 373 374 iocsr_write32(0xffffffff, LOONGARCH_IOCSR_IPI_EN); 375 376 #ifdef CONFIG_NUMA 377 numa_add_cpu(cpu); 378 #endif 379 per_cpu(cpu_state, cpu) = CPU_ONLINE; 380 cpu_data[cpu].package = 381 cpu_logical_map(cpu) / loongson_sysconf.cores_per_package; 382 cpu_data[cpu].core = pptt_enabled ? cpu_data[cpu].core : 383 cpu_logical_map(cpu) % loongson_sysconf.cores_per_package; 384 cpu_data[cpu].global_id = cpu_logical_map(cpu); 385 } 386 387 void loongson_smp_finish(void) 388 { 389 local_irq_enable(); 390 iocsr_write64(0, LOONGARCH_IOCSR_MBUF0); 391 pr_info("CPU#%d finished\n", smp_processor_id()); 392 } 393 394 #ifdef CONFIG_HOTPLUG_CPU 395 396 int loongson_cpu_disable(void) 397 { 398 unsigned long flags; 399 unsigned int cpu = smp_processor_id(); 400 401 if (io_master(cpu)) 402 return -EBUSY; 403 404 #ifdef CONFIG_NUMA 405 numa_remove_cpu(cpu); 406 #endif 407 set_cpu_online(cpu, false); 408 clear_cpu_sibling_map(cpu); 409 calculate_cpu_foreign_map(); 410 local_irq_save(flags); 411 irq_migrate_all_off_this_cpu(); 412 clear_csr_ecfg(ECFG0_IM); 413 local_irq_restore(flags); 414 local_flush_tlb_all(); 415 416 return 0; 417 } 418 419 void loongson_cpu_die(unsigned int cpu) 420 { 421 while (per_cpu(cpu_state, cpu) != CPU_DEAD) 422 cpu_relax(); 423 424 mb(); 425 } 426 427 static void __noreturn idle_play_dead(void) 428 { 429 register uint64_t addr; 430 register void (*init_fn)(void); 431 432 idle_task_exit(); 433 local_irq_enable(); 434 set_csr_ecfg(ECFGF_IPI); 435 __this_cpu_write(cpu_state, CPU_DEAD); 436 437 __smp_mb(); 438 do { 439 __asm__ __volatile__("idle 0\n\t"); 440 addr = iocsr_read64(LOONGARCH_IOCSR_MBUF0); 441 } while (addr == 0); 442 443 local_irq_disable(); 444 init_fn = (void *)TO_CACHE(addr); 445 iocsr_write32(0xffffffff, LOONGARCH_IOCSR_IPI_CLEAR); 446 447 init_fn(); 448 BUG(); 449 } 450 451 #ifdef CONFIG_HIBERNATION 452 static void __noreturn poll_play_dead(void) 453 { 454 register uint64_t addr; 455 register void (*init_fn)(void); 456 457 idle_task_exit(); 458 __this_cpu_write(cpu_state, CPU_DEAD); 459 460 __smp_mb(); 461 do { 462 __asm__ __volatile__("nop\n\t"); 463 addr = iocsr_read64(LOONGARCH_IOCSR_MBUF0); 464 } while (addr == 0); 465 466 init_fn = (void *)TO_CACHE(addr); 467 iocsr_write32(0xffffffff, LOONGARCH_IOCSR_IPI_CLEAR); 468 469 init_fn(); 470 BUG(); 471 } 472 #endif 473 474 static void (*play_dead)(void) = idle_play_dead; 475 476 void __noreturn arch_cpu_idle_dead(void) 477 { 478 play_dead(); 479 BUG(); /* play_dead() doesn't return */ 480 } 481 482 #ifdef CONFIG_HIBERNATION 483 int hibernate_resume_nonboot_cpu_disable(void) 484 { 485 int ret; 486 487 play_dead = poll_play_dead; 488 ret = suspend_disable_secondary_cpus(); 489 play_dead = idle_play_dead; 490 491 return ret; 492 } 493 #endif 494 495 #endif 496 497 /* 498 * Power management 499 */ 500 #ifdef CONFIG_PM 501 502 static int loongson_ipi_suspend(void) 503 { 504 return 0; 505 } 506 507 static void loongson_ipi_resume(void) 508 { 509 iocsr_write32(0xffffffff, LOONGARCH_IOCSR_IPI_EN); 510 } 511 512 static struct syscore_ops loongson_ipi_syscore_ops = { 513 .resume = loongson_ipi_resume, 514 .suspend = loongson_ipi_suspend, 515 }; 516 517 /* 518 * Enable boot cpu ipi before enabling nonboot cpus 519 * during syscore_resume. 520 */ 521 static int __init ipi_pm_init(void) 522 { 523 register_syscore_ops(&loongson_ipi_syscore_ops); 524 return 0; 525 } 526 527 core_initcall(ipi_pm_init); 528 #endif 529 530 /* Preload SMP state for boot cpu */ 531 void __init smp_prepare_boot_cpu(void) 532 { 533 unsigned int cpu, node, rr_node; 534 535 set_cpu_possible(0, true); 536 set_cpu_online(0, true); 537 set_my_cpu_offset(per_cpu_offset(0)); 538 numa_add_cpu(0); 539 540 rr_node = first_node(node_online_map); 541 for_each_possible_cpu(cpu) { 542 node = early_cpu_to_node(cpu); 543 544 /* 545 * The mapping between present cpus and nodes has been 546 * built during MADT and SRAT parsing. 547 * 548 * If possible cpus = present cpus here, early_cpu_to_node 549 * will return valid node. 550 * 551 * If possible cpus > present cpus here (e.g. some possible 552 * cpus will be added by cpu-hotplug later), for possible but 553 * not present cpus, early_cpu_to_node will return NUMA_NO_NODE, 554 * and we just map them to online nodes in round-robin way. 555 * Once hotplugged, new correct mapping will be built for them. 556 */ 557 if (node != NUMA_NO_NODE) 558 set_cpu_numa_node(cpu, node); 559 else { 560 set_cpu_numa_node(cpu, rr_node); 561 rr_node = next_node_in(rr_node, node_online_map); 562 } 563 } 564 565 pv_spinlock_init(); 566 } 567 568 /* called from main before smp_init() */ 569 void __init smp_prepare_cpus(unsigned int max_cpus) 570 { 571 init_new_context(current, &init_mm); 572 current_thread_info()->cpu = 0; 573 loongson_prepare_cpus(max_cpus); 574 set_cpu_sibling_map(0); 575 set_cpu_core_map(0); 576 calculate_cpu_foreign_map(); 577 #ifndef CONFIG_HOTPLUG_CPU 578 init_cpu_present(cpu_possible_mask); 579 #endif 580 } 581 582 int __cpu_up(unsigned int cpu, struct task_struct *tidle) 583 { 584 loongson_boot_secondary(cpu, tidle); 585 586 /* Wait for CPU to start and be ready to sync counters */ 587 if (!wait_for_completion_timeout(&cpu_starting, 588 msecs_to_jiffies(5000))) { 589 pr_crit("CPU%u: failed to start\n", cpu); 590 return -EIO; 591 } 592 593 /* Wait for CPU to finish startup & mark itself online before return */ 594 wait_for_completion(&cpu_running); 595 596 return 0; 597 } 598 599 /* 600 * First C code run on the secondary CPUs after being started up by 601 * the master. 602 */ 603 asmlinkage void start_secondary(void) 604 { 605 unsigned int cpu; 606 607 sync_counter(); 608 cpu = raw_smp_processor_id(); 609 set_my_cpu_offset(per_cpu_offset(cpu)); 610 611 cpu_probe(); 612 constant_clockevent_init(); 613 loongson_init_secondary(); 614 615 set_cpu_sibling_map(cpu); 616 set_cpu_core_map(cpu); 617 618 notify_cpu_starting(cpu); 619 620 /* Notify boot CPU that we're starting */ 621 complete(&cpu_starting); 622 623 /* The CPU is running, now mark it online */ 624 set_cpu_online(cpu, true); 625 626 calculate_cpu_foreign_map(); 627 628 /* 629 * Notify boot CPU that we're up & online and it can safely return 630 * from __cpu_up() 631 */ 632 complete(&cpu_running); 633 634 /* 635 * irq will be enabled in loongson_smp_finish(), enabling it too 636 * early is dangerous. 637 */ 638 WARN_ON_ONCE(!irqs_disabled()); 639 loongson_smp_finish(); 640 641 cpu_startup_entry(CPUHP_AP_ONLINE_IDLE); 642 } 643 644 void __init smp_cpus_done(unsigned int max_cpus) 645 { 646 } 647 648 static void stop_this_cpu(void *dummy) 649 { 650 set_cpu_online(smp_processor_id(), false); 651 calculate_cpu_foreign_map(); 652 local_irq_disable(); 653 while (true); 654 } 655 656 void smp_send_stop(void) 657 { 658 smp_call_function(stop_this_cpu, NULL, 0); 659 } 660 661 #ifdef CONFIG_PROFILING 662 int setup_profiling_timer(unsigned int multiplier) 663 { 664 return 0; 665 } 666 #endif 667 668 static void flush_tlb_all_ipi(void *info) 669 { 670 local_flush_tlb_all(); 671 } 672 673 void flush_tlb_all(void) 674 { 675 on_each_cpu(flush_tlb_all_ipi, NULL, 1); 676 } 677 678 static void flush_tlb_mm_ipi(void *mm) 679 { 680 local_flush_tlb_mm((struct mm_struct *)mm); 681 } 682 683 void flush_tlb_mm(struct mm_struct *mm) 684 { 685 if (atomic_read(&mm->mm_users) == 0) 686 return; /* happens as a result of exit_mmap() */ 687 688 preempt_disable(); 689 690 if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) { 691 on_each_cpu_mask(mm_cpumask(mm), flush_tlb_mm_ipi, mm, 1); 692 } else { 693 unsigned int cpu; 694 695 for_each_online_cpu(cpu) { 696 if (cpu != smp_processor_id() && cpu_context(cpu, mm)) 697 cpu_context(cpu, mm) = 0; 698 } 699 local_flush_tlb_mm(mm); 700 } 701 702 preempt_enable(); 703 } 704 705 struct flush_tlb_data { 706 struct vm_area_struct *vma; 707 unsigned long addr1; 708 unsigned long addr2; 709 }; 710 711 static void flush_tlb_range_ipi(void *info) 712 { 713 struct flush_tlb_data *fd = info; 714 715 local_flush_tlb_range(fd->vma, fd->addr1, fd->addr2); 716 } 717 718 void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) 719 { 720 struct mm_struct *mm = vma->vm_mm; 721 722 preempt_disable(); 723 if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) { 724 struct flush_tlb_data fd = { 725 .vma = vma, 726 .addr1 = start, 727 .addr2 = end, 728 }; 729 730 on_each_cpu_mask(mm_cpumask(mm), flush_tlb_range_ipi, &fd, 1); 731 } else { 732 unsigned int cpu; 733 734 for_each_online_cpu(cpu) { 735 if (cpu != smp_processor_id() && cpu_context(cpu, mm)) 736 cpu_context(cpu, mm) = 0; 737 } 738 local_flush_tlb_range(vma, start, end); 739 } 740 preempt_enable(); 741 } 742 743 static void flush_tlb_kernel_range_ipi(void *info) 744 { 745 struct flush_tlb_data *fd = info; 746 747 local_flush_tlb_kernel_range(fd->addr1, fd->addr2); 748 } 749 750 void flush_tlb_kernel_range(unsigned long start, unsigned long end) 751 { 752 struct flush_tlb_data fd = { 753 .addr1 = start, 754 .addr2 = end, 755 }; 756 757 on_each_cpu(flush_tlb_kernel_range_ipi, &fd, 1); 758 } 759 760 static void flush_tlb_page_ipi(void *info) 761 { 762 struct flush_tlb_data *fd = info; 763 764 local_flush_tlb_page(fd->vma, fd->addr1); 765 } 766 767 void flush_tlb_page(struct vm_area_struct *vma, unsigned long page) 768 { 769 preempt_disable(); 770 if ((atomic_read(&vma->vm_mm->mm_users) != 1) || (current->mm != vma->vm_mm)) { 771 struct flush_tlb_data fd = { 772 .vma = vma, 773 .addr1 = page, 774 }; 775 776 on_each_cpu_mask(mm_cpumask(vma->vm_mm), flush_tlb_page_ipi, &fd, 1); 777 } else { 778 unsigned int cpu; 779 780 for_each_online_cpu(cpu) { 781 if (cpu != smp_processor_id() && cpu_context(cpu, vma->vm_mm)) 782 cpu_context(cpu, vma->vm_mm) = 0; 783 } 784 local_flush_tlb_page(vma, page); 785 } 786 preempt_enable(); 787 } 788 EXPORT_SYMBOL(flush_tlb_page); 789 790 static void flush_tlb_one_ipi(void *info) 791 { 792 unsigned long vaddr = (unsigned long) info; 793 794 local_flush_tlb_one(vaddr); 795 } 796 797 void flush_tlb_one(unsigned long vaddr) 798 { 799 on_each_cpu(flush_tlb_one_ipi, (void *)vaddr, 1); 800 } 801 EXPORT_SYMBOL(flush_tlb_one); 802