1 // SPDX-License-Identifier: GPL-2.0 2 3 #include <linux/mm.h> 4 #include <linux/smp.h> 5 #include <linux/sched.h> 6 #include <linux/hugetlb.h> 7 #include <asm/sbi.h> 8 #include <asm/mmu_context.h> 9 10 static inline void local_flush_tlb_all_asid(unsigned long asid) 11 { 12 if (asid != FLUSH_TLB_NO_ASID) 13 __asm__ __volatile__ ("sfence.vma x0, %0" 14 : 15 : "r" (asid) 16 : "memory"); 17 else 18 local_flush_tlb_all(); 19 } 20 21 static inline void local_flush_tlb_page_asid(unsigned long addr, 22 unsigned long asid) 23 { 24 if (asid != FLUSH_TLB_NO_ASID) 25 __asm__ __volatile__ ("sfence.vma %0, %1" 26 : 27 : "r" (addr), "r" (asid) 28 : "memory"); 29 else 30 local_flush_tlb_page(addr); 31 } 32 33 /* 34 * Flush entire TLB if number of entries to be flushed is greater 35 * than the threshold below. 36 */ 37 static unsigned long tlb_flush_all_threshold __read_mostly = 64; 38 39 static void local_flush_tlb_range_threshold_asid(unsigned long start, 40 unsigned long size, 41 unsigned long stride, 42 unsigned long asid) 43 { 44 unsigned long nr_ptes_in_range = DIV_ROUND_UP(size, stride); 45 int i; 46 47 if (nr_ptes_in_range > tlb_flush_all_threshold) { 48 local_flush_tlb_all_asid(asid); 49 return; 50 } 51 52 for (i = 0; i < nr_ptes_in_range; ++i) { 53 local_flush_tlb_page_asid(start, asid); 54 start += stride; 55 } 56 } 57 58 static inline void local_flush_tlb_range_asid(unsigned long start, 59 unsigned long size, unsigned long stride, unsigned long asid) 60 { 61 if (size <= stride) 62 local_flush_tlb_page_asid(start, asid); 63 else if (size == FLUSH_TLB_MAX_SIZE) 64 local_flush_tlb_all_asid(asid); 65 else 66 local_flush_tlb_range_threshold_asid(start, size, stride, asid); 67 } 68 69 /* Flush a range of kernel pages without broadcasting */ 70 void local_flush_tlb_kernel_range(unsigned long start, unsigned long end) 71 { 72 local_flush_tlb_range_asid(start, end - start, PAGE_SIZE, FLUSH_TLB_NO_ASID); 73 } 74 75 static void __ipi_flush_tlb_all(void *info) 76 { 77 local_flush_tlb_all(); 78 } 79 80 void flush_tlb_all(void) 81 { 82 if (riscv_use_ipi_for_rfence()) 83 on_each_cpu(__ipi_flush_tlb_all, NULL, 1); 84 else 85 sbi_remote_sfence_vma_asid(NULL, 0, FLUSH_TLB_MAX_SIZE, FLUSH_TLB_NO_ASID); 86 } 87 88 struct flush_tlb_range_data { 89 unsigned long asid; 90 unsigned long start; 91 unsigned long size; 92 unsigned long stride; 93 }; 94 95 static void __ipi_flush_tlb_range_asid(void *info) 96 { 97 struct flush_tlb_range_data *d = info; 98 99 local_flush_tlb_range_asid(d->start, d->size, d->stride, d->asid); 100 } 101 102 static void __flush_tlb_range(const struct cpumask *cmask, unsigned long asid, 103 unsigned long start, unsigned long size, 104 unsigned long stride) 105 { 106 struct flush_tlb_range_data ftd; 107 bool broadcast; 108 109 if (cpumask_empty(cmask)) 110 return; 111 112 if (cmask != cpu_online_mask) { 113 unsigned int cpuid; 114 115 cpuid = get_cpu(); 116 /* check if the tlbflush needs to be sent to other CPUs */ 117 broadcast = cpumask_any_but(cmask, cpuid) < nr_cpu_ids; 118 } else { 119 broadcast = true; 120 } 121 122 if (broadcast) { 123 if (riscv_use_ipi_for_rfence()) { 124 ftd.asid = asid; 125 ftd.start = start; 126 ftd.size = size; 127 ftd.stride = stride; 128 on_each_cpu_mask(cmask, 129 __ipi_flush_tlb_range_asid, 130 &ftd, 1); 131 } else 132 sbi_remote_sfence_vma_asid(cmask, 133 start, size, asid); 134 } else { 135 local_flush_tlb_range_asid(start, size, stride, asid); 136 } 137 138 if (cmask != cpu_online_mask) 139 put_cpu(); 140 } 141 142 static inline unsigned long get_mm_asid(struct mm_struct *mm) 143 { 144 return static_branch_unlikely(&use_asid_allocator) ? 145 atomic_long_read(&mm->context.id) & asid_mask : FLUSH_TLB_NO_ASID; 146 } 147 148 void flush_tlb_mm(struct mm_struct *mm) 149 { 150 __flush_tlb_range(mm_cpumask(mm), get_mm_asid(mm), 151 0, FLUSH_TLB_MAX_SIZE, PAGE_SIZE); 152 } 153 154 void flush_tlb_mm_range(struct mm_struct *mm, 155 unsigned long start, unsigned long end, 156 unsigned int page_size) 157 { 158 __flush_tlb_range(mm_cpumask(mm), get_mm_asid(mm), 159 start, end - start, page_size); 160 } 161 162 void flush_tlb_page(struct vm_area_struct *vma, unsigned long addr) 163 { 164 __flush_tlb_range(mm_cpumask(vma->vm_mm), get_mm_asid(vma->vm_mm), 165 addr, PAGE_SIZE, PAGE_SIZE); 166 } 167 168 void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, 169 unsigned long end) 170 { 171 unsigned long stride_size; 172 173 if (!is_vm_hugetlb_page(vma)) { 174 stride_size = PAGE_SIZE; 175 } else { 176 stride_size = huge_page_size(hstate_vma(vma)); 177 178 /* 179 * As stated in the privileged specification, every PTE in a 180 * NAPOT region must be invalidated, so reset the stride in that 181 * case. 182 */ 183 if (has_svnapot()) { 184 if (stride_size >= PGDIR_SIZE) 185 stride_size = PGDIR_SIZE; 186 else if (stride_size >= P4D_SIZE) 187 stride_size = P4D_SIZE; 188 else if (stride_size >= PUD_SIZE) 189 stride_size = PUD_SIZE; 190 else if (stride_size >= PMD_SIZE) 191 stride_size = PMD_SIZE; 192 else 193 stride_size = PAGE_SIZE; 194 } 195 } 196 197 __flush_tlb_range(mm_cpumask(vma->vm_mm), get_mm_asid(vma->vm_mm), 198 start, end - start, stride_size); 199 } 200 201 void flush_tlb_kernel_range(unsigned long start, unsigned long end) 202 { 203 __flush_tlb_range(cpu_online_mask, FLUSH_TLB_NO_ASID, 204 start, end - start, PAGE_SIZE); 205 } 206 207 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 208 void flush_pmd_tlb_range(struct vm_area_struct *vma, unsigned long start, 209 unsigned long end) 210 { 211 __flush_tlb_range(mm_cpumask(vma->vm_mm), get_mm_asid(vma->vm_mm), 212 start, end - start, PMD_SIZE); 213 } 214 #endif 215 216 bool arch_tlbbatch_should_defer(struct mm_struct *mm) 217 { 218 return true; 219 } 220 221 void arch_tlbbatch_add_pending(struct arch_tlbflush_unmap_batch *batch, 222 struct mm_struct *mm, 223 unsigned long uaddr) 224 { 225 cpumask_or(&batch->cpumask, &batch->cpumask, mm_cpumask(mm)); 226 } 227 228 void arch_flush_tlb_batched_pending(struct mm_struct *mm) 229 { 230 flush_tlb_mm(mm); 231 } 232 233 void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch) 234 { 235 __flush_tlb_range(&batch->cpumask, FLUSH_TLB_NO_ASID, 0, 236 FLUSH_TLB_MAX_SIZE, PAGE_SIZE); 237 cpumask_clear(&batch->cpumask); 238 } 239