1 // SPDX-License-Identifier: GPL-2.0 2 3 #include <linux/mm.h> 4 #include <linux/smp.h> 5 #include <linux/sched.h> 6 #include <linux/hugetlb.h> 7 #include <linux/mmu_notifier.h> 8 #include <asm/sbi.h> 9 #include <asm/mmu_context.h> 10 #include <asm/cpufeature.h> 11 12 #define has_svinval() riscv_has_extension_unlikely(RISCV_ISA_EXT_SVINVAL) 13 14 static inline void local_sfence_inval_ir(void) 15 { 16 asm volatile(SFENCE_INVAL_IR() ::: "memory"); 17 } 18 19 static inline void local_sfence_w_inval(void) 20 { 21 asm volatile(SFENCE_W_INVAL() ::: "memory"); 22 } 23 24 static inline void local_sinval_vma(unsigned long vma, unsigned long asid) 25 { 26 if (asid != FLUSH_TLB_NO_ASID) 27 asm volatile(SINVAL_VMA(%0, %1) : : "r" (vma), "r" (asid) : "memory"); 28 else 29 asm volatile(SINVAL_VMA(%0, zero) : : "r" (vma) : "memory"); 30 } 31 32 /* 33 * Flush entire TLB if number of entries to be flushed is greater 34 * than the threshold below. 35 */ 36 unsigned long tlb_flush_all_threshold __read_mostly = 64; 37 38 static void local_flush_tlb_range_threshold_asid(unsigned long start, 39 unsigned long size, 40 unsigned long stride, 41 unsigned long asid) 42 { 43 unsigned long nr_ptes_in_range = DIV_ROUND_UP(size, stride); 44 int i; 45 46 if (nr_ptes_in_range > tlb_flush_all_threshold) { 47 local_flush_tlb_all_asid(asid); 48 return; 49 } 50 51 if (has_svinval()) { 52 local_sfence_w_inval(); 53 for (i = 0; i < nr_ptes_in_range; ++i) { 54 local_sinval_vma(start, asid); 55 start += stride; 56 } 57 local_sfence_inval_ir(); 58 return; 59 } 60 61 for (i = 0; i < nr_ptes_in_range; ++i) { 62 local_flush_tlb_page_asid(start, asid); 63 start += stride; 64 } 65 } 66 67 static inline void local_flush_tlb_range_asid(unsigned long start, 68 unsigned long size, unsigned long stride, unsigned long asid) 69 { 70 if (size <= stride) 71 local_flush_tlb_page_asid(start, asid); 72 else if (size == FLUSH_TLB_MAX_SIZE) 73 local_flush_tlb_all_asid(asid); 74 else 75 local_flush_tlb_range_threshold_asid(start, size, stride, asid); 76 } 77 78 /* Flush a range of kernel pages without broadcasting */ 79 void local_flush_tlb_kernel_range(unsigned long start, unsigned long end) 80 { 81 local_flush_tlb_range_asid(start, end - start, PAGE_SIZE, FLUSH_TLB_NO_ASID); 82 } 83 84 static void __ipi_flush_tlb_all(void *info) 85 { 86 local_flush_tlb_all(); 87 } 88 89 void flush_tlb_all(void) 90 { 91 if (num_online_cpus() < 2) 92 local_flush_tlb_all(); 93 else if (riscv_use_sbi_for_rfence()) 94 sbi_remote_sfence_vma_asid(NULL, 0, FLUSH_TLB_MAX_SIZE, FLUSH_TLB_NO_ASID); 95 else 96 on_each_cpu(__ipi_flush_tlb_all, NULL, 1); 97 } 98 99 struct flush_tlb_range_data { 100 unsigned long asid; 101 unsigned long start; 102 unsigned long size; 103 unsigned long stride; 104 }; 105 106 static void __ipi_flush_tlb_range_asid(void *info) 107 { 108 struct flush_tlb_range_data *d = info; 109 110 local_flush_tlb_range_asid(d->start, d->size, d->stride, d->asid); 111 } 112 113 static inline unsigned long get_mm_asid(struct mm_struct *mm) 114 { 115 return mm ? cntx2asid(atomic_long_read(&mm->context.id)) : FLUSH_TLB_NO_ASID; 116 } 117 118 static void __flush_tlb_range(struct mm_struct *mm, 119 const struct cpumask *cmask, 120 unsigned long start, unsigned long size, 121 unsigned long stride) 122 { 123 unsigned long asid = get_mm_asid(mm); 124 unsigned int cpu; 125 126 if (cpumask_empty(cmask)) 127 return; 128 129 cpu = get_cpu(); 130 131 /* Check if the TLB flush needs to be sent to other CPUs. */ 132 if (cpumask_any_but(cmask, cpu) >= nr_cpu_ids) { 133 local_flush_tlb_range_asid(start, size, stride, asid); 134 } else if (riscv_use_sbi_for_rfence()) { 135 sbi_remote_sfence_vma_asid(cmask, start, size, asid); 136 } else { 137 struct flush_tlb_range_data ftd; 138 139 ftd.asid = asid; 140 ftd.start = start; 141 ftd.size = size; 142 ftd.stride = stride; 143 on_each_cpu_mask(cmask, __ipi_flush_tlb_range_asid, &ftd, 1); 144 } 145 146 put_cpu(); 147 148 if (mm) 149 mmu_notifier_arch_invalidate_secondary_tlbs(mm, start, start + size); 150 } 151 152 void flush_tlb_mm(struct mm_struct *mm) 153 { 154 __flush_tlb_range(mm, mm_cpumask(mm), 0, FLUSH_TLB_MAX_SIZE, PAGE_SIZE); 155 } 156 157 void flush_tlb_mm_range(struct mm_struct *mm, 158 unsigned long start, unsigned long end, 159 unsigned int page_size) 160 { 161 __flush_tlb_range(mm, mm_cpumask(mm), start, end - start, page_size); 162 } 163 164 void flush_tlb_page(struct vm_area_struct *vma, unsigned long addr) 165 { 166 __flush_tlb_range(vma->vm_mm, mm_cpumask(vma->vm_mm), 167 addr, PAGE_SIZE, PAGE_SIZE); 168 } 169 170 void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, 171 unsigned long end) 172 { 173 unsigned long stride_size; 174 175 if (!is_vm_hugetlb_page(vma)) { 176 stride_size = PAGE_SIZE; 177 } else { 178 stride_size = huge_page_size(hstate_vma(vma)); 179 180 /* 181 * As stated in the privileged specification, every PTE in a 182 * NAPOT region must be invalidated, so reset the stride in that 183 * case. 184 */ 185 if (has_svnapot()) { 186 if (stride_size >= PGDIR_SIZE) 187 stride_size = PGDIR_SIZE; 188 else if (stride_size >= P4D_SIZE) 189 stride_size = P4D_SIZE; 190 else if (stride_size >= PUD_SIZE) 191 stride_size = PUD_SIZE; 192 else if (stride_size >= PMD_SIZE) 193 stride_size = PMD_SIZE; 194 else 195 stride_size = PAGE_SIZE; 196 } 197 } 198 199 __flush_tlb_range(vma->vm_mm, mm_cpumask(vma->vm_mm), 200 start, end - start, stride_size); 201 } 202 203 void flush_tlb_kernel_range(unsigned long start, unsigned long end) 204 { 205 __flush_tlb_range(NULL, cpu_online_mask, 206 start, end - start, PAGE_SIZE); 207 } 208 209 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 210 void flush_pmd_tlb_range(struct vm_area_struct *vma, unsigned long start, 211 unsigned long end) 212 { 213 __flush_tlb_range(vma->vm_mm, mm_cpumask(vma->vm_mm), 214 start, end - start, PMD_SIZE); 215 } 216 217 void flush_pud_tlb_range(struct vm_area_struct *vma, unsigned long start, 218 unsigned long end) 219 { 220 __flush_tlb_range(vma->vm_mm, mm_cpumask(vma->vm_mm), 221 start, end - start, PUD_SIZE); 222 } 223 #endif 224 225 bool arch_tlbbatch_should_defer(struct mm_struct *mm) 226 { 227 return true; 228 } 229 230 void arch_tlbbatch_add_pending(struct arch_tlbflush_unmap_batch *batch, 231 struct mm_struct *mm, unsigned long start, unsigned long end) 232 { 233 cpumask_or(&batch->cpumask, &batch->cpumask, mm_cpumask(mm)); 234 mmu_notifier_arch_invalidate_secondary_tlbs(mm, start, end); 235 } 236 237 void arch_flush_tlb_batched_pending(struct mm_struct *mm) 238 { 239 flush_tlb_mm(mm); 240 } 241 242 void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch) 243 { 244 __flush_tlb_range(NULL, &batch->cpumask, 245 0, FLUSH_TLB_MAX_SIZE, PAGE_SIZE); 246 cpumask_clear(&batch->cpumask); 247 } 248