1 // SPDX-License-Identifier: GPL-2.0 2 3 #include <linux/mm.h> 4 #include <linux/smp.h> 5 #include <linux/sched.h> 6 #include <linux/hugetlb.h> 7 #include <linux/mmu_notifier.h> 8 #include <asm/sbi.h> 9 #include <asm/mmu_context.h> 10 #include <asm/cpufeature.h> 11 12 #define has_svinval() riscv_has_extension_unlikely(RISCV_ISA_EXT_SVINVAL) 13 14 /* 15 * Flush entire TLB if number of entries to be flushed is greater 16 * than the threshold below. 17 */ 18 unsigned long tlb_flush_all_threshold __read_mostly = 64; 19 20 static void local_flush_tlb_range_threshold_asid(unsigned long start, 21 unsigned long size, 22 unsigned long stride, 23 unsigned long asid) 24 { 25 unsigned long nr_ptes_in_range = DIV_ROUND_UP(size, stride); 26 int i; 27 28 if (nr_ptes_in_range > tlb_flush_all_threshold) { 29 local_flush_tlb_all_asid(asid); 30 return; 31 } 32 33 if (has_svinval()) { 34 local_sfence_w_inval(); 35 for (i = 0; i < nr_ptes_in_range; ++i) { 36 local_sinval_vma(start, asid); 37 start += stride; 38 } 39 local_sfence_inval_ir(); 40 return; 41 } 42 43 for (i = 0; i < nr_ptes_in_range; ++i) { 44 local_flush_tlb_page_asid(start, asid); 45 start += stride; 46 } 47 } 48 49 static inline void local_flush_tlb_range_asid(unsigned long start, 50 unsigned long size, unsigned long stride, unsigned long asid) 51 { 52 if (size <= stride) 53 local_flush_tlb_page_asid(start, asid); 54 else if (size == FLUSH_TLB_MAX_SIZE) 55 local_flush_tlb_all_asid(asid); 56 else 57 local_flush_tlb_range_threshold_asid(start, size, stride, asid); 58 } 59 60 /* Flush a range of kernel pages without broadcasting */ 61 void local_flush_tlb_kernel_range(unsigned long start, unsigned long end) 62 { 63 local_flush_tlb_range_asid(start, end - start, PAGE_SIZE, FLUSH_TLB_NO_ASID); 64 } 65 66 static void __ipi_flush_tlb_all(void *info) 67 { 68 local_flush_tlb_all(); 69 } 70 71 void flush_tlb_all(void) 72 { 73 if (num_online_cpus() < 2) 74 local_flush_tlb_all(); 75 else if (riscv_use_sbi_for_rfence()) 76 sbi_remote_sfence_vma_asid(NULL, 0, FLUSH_TLB_MAX_SIZE, FLUSH_TLB_NO_ASID); 77 else 78 on_each_cpu(__ipi_flush_tlb_all, NULL, 1); 79 } 80 81 struct flush_tlb_range_data { 82 unsigned long asid; 83 unsigned long start; 84 unsigned long size; 85 unsigned long stride; 86 }; 87 88 static void __ipi_flush_tlb_range_asid(void *info) 89 { 90 struct flush_tlb_range_data *d = info; 91 92 local_flush_tlb_range_asid(d->start, d->size, d->stride, d->asid); 93 } 94 95 static void __flush_tlb_range(struct mm_struct *mm, 96 const struct cpumask *cmask, 97 unsigned long start, unsigned long size, 98 unsigned long stride) 99 { 100 unsigned long asid = get_mm_asid(mm); 101 unsigned int cpu; 102 103 if (cpumask_empty(cmask)) 104 return; 105 106 cpu = get_cpu(); 107 108 /* Check if the TLB flush needs to be sent to other CPUs. */ 109 if (cpumask_any_but(cmask, cpu) >= nr_cpu_ids) { 110 local_flush_tlb_range_asid(start, size, stride, asid); 111 } else if (riscv_use_sbi_for_rfence()) { 112 sbi_remote_sfence_vma_asid(cmask, start, size, asid); 113 } else { 114 struct flush_tlb_range_data ftd; 115 116 ftd.asid = asid; 117 ftd.start = start; 118 ftd.size = size; 119 ftd.stride = stride; 120 on_each_cpu_mask(cmask, __ipi_flush_tlb_range_asid, &ftd, 1); 121 } 122 123 put_cpu(); 124 125 if (mm) 126 mmu_notifier_arch_invalidate_secondary_tlbs(mm, start, start + size); 127 } 128 129 void flush_tlb_mm(struct mm_struct *mm) 130 { 131 __flush_tlb_range(mm, mm_cpumask(mm), 0, FLUSH_TLB_MAX_SIZE, PAGE_SIZE); 132 } 133 134 void flush_tlb_mm_range(struct mm_struct *mm, 135 unsigned long start, unsigned long end, 136 unsigned int page_size) 137 { 138 __flush_tlb_range(mm, mm_cpumask(mm), start, end - start, page_size); 139 } 140 141 void flush_tlb_page(struct vm_area_struct *vma, unsigned long addr) 142 { 143 __flush_tlb_range(vma->vm_mm, mm_cpumask(vma->vm_mm), 144 addr, PAGE_SIZE, PAGE_SIZE); 145 } 146 147 void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, 148 unsigned long end) 149 { 150 unsigned long stride_size; 151 152 if (!is_vm_hugetlb_page(vma)) { 153 stride_size = PAGE_SIZE; 154 } else { 155 stride_size = huge_page_size(hstate_vma(vma)); 156 157 /* 158 * As stated in the privileged specification, every PTE in a 159 * NAPOT region must be invalidated, so reset the stride in that 160 * case. 161 */ 162 if (has_svnapot()) { 163 if (stride_size >= PGDIR_SIZE) 164 stride_size = PGDIR_SIZE; 165 else if (stride_size >= P4D_SIZE) 166 stride_size = P4D_SIZE; 167 else if (stride_size >= PUD_SIZE) 168 stride_size = PUD_SIZE; 169 else if (stride_size >= PMD_SIZE) 170 stride_size = PMD_SIZE; 171 else 172 stride_size = PAGE_SIZE; 173 } 174 } 175 176 __flush_tlb_range(vma->vm_mm, mm_cpumask(vma->vm_mm), 177 start, end - start, stride_size); 178 } 179 180 void flush_tlb_kernel_range(unsigned long start, unsigned long end) 181 { 182 __flush_tlb_range(NULL, cpu_online_mask, 183 start, end - start, PAGE_SIZE); 184 } 185 186 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 187 void flush_pmd_tlb_range(struct vm_area_struct *vma, unsigned long start, 188 unsigned long end) 189 { 190 __flush_tlb_range(vma->vm_mm, mm_cpumask(vma->vm_mm), 191 start, end - start, PMD_SIZE); 192 } 193 194 void flush_pud_tlb_range(struct vm_area_struct *vma, unsigned long start, 195 unsigned long end) 196 { 197 __flush_tlb_range(vma->vm_mm, mm_cpumask(vma->vm_mm), 198 start, end - start, PUD_SIZE); 199 } 200 #endif 201 202 bool arch_tlbbatch_should_defer(struct mm_struct *mm) 203 { 204 return true; 205 } 206 207 void arch_tlbbatch_add_pending(struct arch_tlbflush_unmap_batch *batch, 208 struct mm_struct *mm, unsigned long start, unsigned long end) 209 { 210 cpumask_or(&batch->cpumask, &batch->cpumask, mm_cpumask(mm)); 211 mmu_notifier_arch_invalidate_secondary_tlbs(mm, start, end); 212 } 213 214 void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch) 215 { 216 __flush_tlb_range(NULL, &batch->cpumask, 217 0, FLUSH_TLB_MAX_SIZE, PAGE_SIZE); 218 cpumask_clear(&batch->cpumask); 219 } 220