1 // SPDX-License-Identifier: GPL-2.0
2
3 #include <linux/mm.h>
4 #include <linux/smp.h>
5 #include <linux/sched.h>
6 #include <linux/hugetlb.h>
7 #include <linux/mmu_notifier.h>
8 #include <asm/sbi.h>
9 #include <asm/mmu_context.h>
10 #include <asm/cpufeature.h>
11
12 #define has_svinval() riscv_has_extension_unlikely(RISCV_ISA_EXT_SVINVAL)
13
14 /*
15 * Flush entire TLB if number of entries to be flushed is greater
16 * than the threshold below.
17 */
18 unsigned long tlb_flush_all_threshold __read_mostly = 64;
19
local_flush_tlb_range_threshold_asid(unsigned long start,unsigned long size,unsigned long stride,unsigned long asid)20 static void local_flush_tlb_range_threshold_asid(unsigned long start,
21 unsigned long size,
22 unsigned long stride,
23 unsigned long asid)
24 {
25 unsigned long nr_ptes_in_range = DIV_ROUND_UP(size, stride);
26 int i;
27
28 if (nr_ptes_in_range > tlb_flush_all_threshold) {
29 local_flush_tlb_all_asid(asid);
30 return;
31 }
32
33 if (has_svinval()) {
34 local_sfence_w_inval();
35 for (i = 0; i < nr_ptes_in_range; ++i) {
36 local_sinval_vma(start, asid);
37 start += stride;
38 }
39 local_sfence_inval_ir();
40 return;
41 }
42
43 for (i = 0; i < nr_ptes_in_range; ++i) {
44 local_flush_tlb_page_asid(start, asid);
45 start += stride;
46 }
47 }
48
local_flush_tlb_range_asid(unsigned long start,unsigned long size,unsigned long stride,unsigned long asid)49 static inline void local_flush_tlb_range_asid(unsigned long start,
50 unsigned long size, unsigned long stride, unsigned long asid)
51 {
52 if (size <= stride)
53 local_flush_tlb_page_asid(start, asid);
54 else if (size == FLUSH_TLB_MAX_SIZE)
55 local_flush_tlb_all_asid(asid);
56 else
57 local_flush_tlb_range_threshold_asid(start, size, stride, asid);
58 }
59
60 /* Flush a range of kernel pages without broadcasting */
local_flush_tlb_kernel_range(unsigned long start,unsigned long end)61 void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)
62 {
63 local_flush_tlb_range_asid(start, end - start, PAGE_SIZE, FLUSH_TLB_NO_ASID);
64 }
65
__ipi_flush_tlb_all(void * info)66 static void __ipi_flush_tlb_all(void *info)
67 {
68 local_flush_tlb_all();
69 }
70
flush_tlb_all(void)71 void flush_tlb_all(void)
72 {
73 if (num_online_cpus() < 2)
74 local_flush_tlb_all();
75 else if (riscv_use_sbi_for_rfence())
76 sbi_remote_sfence_vma_asid(NULL, 0, FLUSH_TLB_MAX_SIZE, FLUSH_TLB_NO_ASID);
77 else
78 on_each_cpu(__ipi_flush_tlb_all, NULL, 1);
79 }
80
81 struct flush_tlb_range_data {
82 unsigned long asid;
83 unsigned long start;
84 unsigned long size;
85 unsigned long stride;
86 };
87
__ipi_flush_tlb_range_asid(void * info)88 static void __ipi_flush_tlb_range_asid(void *info)
89 {
90 struct flush_tlb_range_data *d = info;
91
92 local_flush_tlb_range_asid(d->start, d->size, d->stride, d->asid);
93 }
94
__flush_tlb_range(struct mm_struct * mm,const struct cpumask * cmask,unsigned long start,unsigned long size,unsigned long stride)95 static void __flush_tlb_range(struct mm_struct *mm,
96 const struct cpumask *cmask,
97 unsigned long start, unsigned long size,
98 unsigned long stride)
99 {
100 unsigned long asid = get_mm_asid(mm);
101 unsigned int cpu;
102
103 if (cpumask_empty(cmask))
104 return;
105
106 cpu = get_cpu();
107
108 /* Check if the TLB flush needs to be sent to other CPUs. */
109 if (cpumask_any_but(cmask, cpu) >= nr_cpu_ids) {
110 local_flush_tlb_range_asid(start, size, stride, asid);
111 } else if (riscv_use_sbi_for_rfence()) {
112 sbi_remote_sfence_vma_asid(cmask, start, size, asid);
113 } else {
114 struct flush_tlb_range_data ftd;
115
116 ftd.asid = asid;
117 ftd.start = start;
118 ftd.size = size;
119 ftd.stride = stride;
120 on_each_cpu_mask(cmask, __ipi_flush_tlb_range_asid, &ftd, 1);
121 }
122
123 put_cpu();
124
125 if (mm)
126 mmu_notifier_arch_invalidate_secondary_tlbs(mm, start, start + size);
127 }
128
flush_tlb_mm(struct mm_struct * mm)129 void flush_tlb_mm(struct mm_struct *mm)
130 {
131 __flush_tlb_range(mm, mm_cpumask(mm), 0, FLUSH_TLB_MAX_SIZE, PAGE_SIZE);
132 }
133
flush_tlb_mm_range(struct mm_struct * mm,unsigned long start,unsigned long end,unsigned int page_size)134 void flush_tlb_mm_range(struct mm_struct *mm,
135 unsigned long start, unsigned long end,
136 unsigned int page_size)
137 {
138 __flush_tlb_range(mm, mm_cpumask(mm), start, end - start, page_size);
139 }
140
flush_tlb_page(struct vm_area_struct * vma,unsigned long addr)141 void flush_tlb_page(struct vm_area_struct *vma, unsigned long addr)
142 {
143 __flush_tlb_range(vma->vm_mm, mm_cpumask(vma->vm_mm),
144 addr, PAGE_SIZE, PAGE_SIZE);
145 }
146
flush_tlb_range(struct vm_area_struct * vma,unsigned long start,unsigned long end)147 void flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
148 unsigned long end)
149 {
150 unsigned long stride_size;
151
152 if (!is_vm_hugetlb_page(vma)) {
153 stride_size = PAGE_SIZE;
154 } else {
155 stride_size = huge_page_size(hstate_vma(vma));
156
157 /*
158 * As stated in the privileged specification, every PTE in a
159 * NAPOT region must be invalidated, so reset the stride in that
160 * case.
161 */
162 if (has_svnapot()) {
163 if (stride_size >= PGDIR_SIZE)
164 stride_size = PGDIR_SIZE;
165 else if (stride_size >= P4D_SIZE)
166 stride_size = P4D_SIZE;
167 else if (stride_size >= PUD_SIZE)
168 stride_size = PUD_SIZE;
169 else if (stride_size >= PMD_SIZE)
170 stride_size = PMD_SIZE;
171 else
172 stride_size = PAGE_SIZE;
173 }
174 }
175
176 __flush_tlb_range(vma->vm_mm, mm_cpumask(vma->vm_mm),
177 start, end - start, stride_size);
178 }
179
flush_tlb_kernel_range(unsigned long start,unsigned long end)180 void flush_tlb_kernel_range(unsigned long start, unsigned long end)
181 {
182 __flush_tlb_range(NULL, cpu_online_mask,
183 start, end - start, PAGE_SIZE);
184 }
185
186 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
flush_pmd_tlb_range(struct vm_area_struct * vma,unsigned long start,unsigned long end)187 void flush_pmd_tlb_range(struct vm_area_struct *vma, unsigned long start,
188 unsigned long end)
189 {
190 __flush_tlb_range(vma->vm_mm, mm_cpumask(vma->vm_mm),
191 start, end - start, PMD_SIZE);
192 }
193
flush_pud_tlb_range(struct vm_area_struct * vma,unsigned long start,unsigned long end)194 void flush_pud_tlb_range(struct vm_area_struct *vma, unsigned long start,
195 unsigned long end)
196 {
197 __flush_tlb_range(vma->vm_mm, mm_cpumask(vma->vm_mm),
198 start, end - start, PUD_SIZE);
199 }
200 #endif
201
arch_tlbbatch_should_defer(struct mm_struct * mm)202 bool arch_tlbbatch_should_defer(struct mm_struct *mm)
203 {
204 return true;
205 }
206
arch_tlbbatch_add_pending(struct arch_tlbflush_unmap_batch * batch,struct mm_struct * mm,unsigned long start,unsigned long end)207 void arch_tlbbatch_add_pending(struct arch_tlbflush_unmap_batch *batch,
208 struct mm_struct *mm, unsigned long start, unsigned long end)
209 {
210 cpumask_or(&batch->cpumask, &batch->cpumask, mm_cpumask(mm));
211 mmu_notifier_arch_invalidate_secondary_tlbs(mm, start, end);
212 }
213
arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch * batch)214 void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch)
215 {
216 __flush_tlb_range(NULL, &batch->cpumask,
217 0, FLUSH_TLB_MAX_SIZE, PAGE_SIZE);
218 cpumask_clear(&batch->cpumask);
219 }
220