1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/atomic.h>
3 #include <linux/mmu_context.h>
4 #include <linux/percpu.h>
5 #include <linux/spinlock.h>
6
7 static DEFINE_RAW_SPINLOCK(cpu_mmid_lock);
8
9 static atomic64_t mmid_version;
10 static unsigned int num_mmids;
11 static unsigned long *mmid_map;
12
13 static DEFINE_PER_CPU(u64, reserved_mmids);
14 static cpumask_t tlb_flush_pending;
15
asid_versions_eq(int cpu,u64 a,u64 b)16 static bool asid_versions_eq(int cpu, u64 a, u64 b)
17 {
18 return ((a ^ b) & asid_version_mask(cpu)) == 0;
19 }
20
get_new_mmu_context(struct mm_struct * mm)21 void get_new_mmu_context(struct mm_struct *mm)
22 {
23 unsigned int cpu;
24 u64 asid;
25
26 /*
27 * This function is specific to ASIDs, and should not be called when
28 * MMIDs are in use.
29 */
30 if (WARN_ON(IS_ENABLED(CONFIG_DEBUG_VM) && cpu_has_mmid))
31 return;
32
33 cpu = smp_processor_id();
34 asid = asid_cache(cpu);
35
36 if (!((asid += cpu_asid_inc()) & cpu_asid_mask(&cpu_data[cpu]))) {
37 if (cpu_has_vtag_icache)
38 flush_icache_all();
39 local_flush_tlb_all(); /* start new asid cycle */
40 }
41
42 set_cpu_context(cpu, mm, asid);
43 asid_cache(cpu) = asid;
44 }
45 EXPORT_SYMBOL_GPL(get_new_mmu_context);
46
check_mmu_context(struct mm_struct * mm)47 void check_mmu_context(struct mm_struct *mm)
48 {
49 unsigned int cpu = smp_processor_id();
50
51 /*
52 * This function is specific to ASIDs, and should not be called when
53 * MMIDs are in use.
54 */
55 if (WARN_ON(IS_ENABLED(CONFIG_DEBUG_VM) && cpu_has_mmid))
56 return;
57
58 /* Check if our ASID is of an older version and thus invalid */
59 if (!asid_versions_eq(cpu, cpu_context(cpu, mm), asid_cache(cpu)))
60 get_new_mmu_context(mm);
61 }
62 EXPORT_SYMBOL_GPL(check_mmu_context);
63
flush_context(void)64 static void flush_context(void)
65 {
66 u64 mmid;
67 int cpu;
68
69 /* Update the list of reserved MMIDs and the MMID bitmap */
70 bitmap_zero(mmid_map, num_mmids);
71
72 /* Reserve an MMID for kmap/wired entries */
73 __set_bit(MMID_KERNEL_WIRED, mmid_map);
74
75 for_each_possible_cpu(cpu) {
76 mmid = xchg_relaxed(&cpu_data[cpu].asid_cache, 0);
77
78 /*
79 * If this CPU has already been through a
80 * rollover, but hasn't run another task in
81 * the meantime, we must preserve its reserved
82 * MMID, as this is the only trace we have of
83 * the process it is still running.
84 */
85 if (mmid == 0)
86 mmid = per_cpu(reserved_mmids, cpu);
87
88 __set_bit(mmid & cpu_asid_mask(&cpu_data[cpu]), mmid_map);
89 per_cpu(reserved_mmids, cpu) = mmid;
90 }
91
92 /*
93 * Queue a TLB invalidation for each CPU to perform on next
94 * context-switch
95 */
96 cpumask_setall(&tlb_flush_pending);
97 }
98
check_update_reserved_mmid(u64 mmid,u64 newmmid)99 static bool check_update_reserved_mmid(u64 mmid, u64 newmmid)
100 {
101 bool hit;
102 int cpu;
103
104 /*
105 * Iterate over the set of reserved MMIDs looking for a match.
106 * If we find one, then we can update our mm to use newmmid
107 * (i.e. the same MMID in the current generation) but we can't
108 * exit the loop early, since we need to ensure that all copies
109 * of the old MMID are updated to reflect the mm. Failure to do
110 * so could result in us missing the reserved MMID in a future
111 * generation.
112 */
113 hit = false;
114 for_each_possible_cpu(cpu) {
115 if (per_cpu(reserved_mmids, cpu) == mmid) {
116 hit = true;
117 per_cpu(reserved_mmids, cpu) = newmmid;
118 }
119 }
120
121 return hit;
122 }
123
get_new_mmid(struct mm_struct * mm)124 static u64 get_new_mmid(struct mm_struct *mm)
125 {
126 static u32 cur_idx = MMID_KERNEL_WIRED + 1;
127 u64 mmid, version, mmid_mask;
128
129 mmid = cpu_context(0, mm);
130 version = atomic64_read(&mmid_version);
131 mmid_mask = cpu_asid_mask(&boot_cpu_data);
132
133 if (!asid_versions_eq(0, mmid, 0)) {
134 u64 newmmid = version | (mmid & mmid_mask);
135
136 /*
137 * If our current MMID was active during a rollover, we
138 * can continue to use it and this was just a false alarm.
139 */
140 if (check_update_reserved_mmid(mmid, newmmid)) {
141 mmid = newmmid;
142 goto set_context;
143 }
144
145 /*
146 * We had a valid MMID in a previous life, so try to re-use
147 * it if possible.
148 */
149 if (!__test_and_set_bit(mmid & mmid_mask, mmid_map)) {
150 mmid = newmmid;
151 goto set_context;
152 }
153 }
154
155 /* Allocate a free MMID */
156 mmid = find_next_zero_bit(mmid_map, num_mmids, cur_idx);
157 if (mmid != num_mmids)
158 goto reserve_mmid;
159
160 /* We're out of MMIDs, so increment the global version */
161 version = atomic64_add_return_relaxed(asid_first_version(0),
162 &mmid_version);
163
164 /* Note currently active MMIDs & mark TLBs as requiring flushes */
165 flush_context();
166
167 /* We have more MMIDs than CPUs, so this will always succeed */
168 mmid = find_first_zero_bit(mmid_map, num_mmids);
169
170 reserve_mmid:
171 __set_bit(mmid, mmid_map);
172 cur_idx = mmid;
173 mmid |= version;
174 set_context:
175 set_cpu_context(0, mm, mmid);
176 return mmid;
177 }
178
check_switch_mmu_context(struct mm_struct * mm)179 void check_switch_mmu_context(struct mm_struct *mm)
180 {
181 unsigned int cpu = smp_processor_id();
182 u64 ctx, old_active_mmid;
183 unsigned long flags;
184
185 if (!cpu_has_mmid) {
186 check_mmu_context(mm);
187 write_c0_entryhi(cpu_asid(cpu, mm));
188 goto setup_pgd;
189 }
190
191 /*
192 * MMID switch fast-path, to avoid acquiring cpu_mmid_lock when it's
193 * unnecessary.
194 *
195 * The memory ordering here is subtle. If our active_mmids is non-zero
196 * and the MMID matches the current version, then we update the CPU's
197 * asid_cache with a relaxed cmpxchg. Racing with a concurrent rollover
198 * means that either:
199 *
200 * - We get a zero back from the cmpxchg and end up waiting on
201 * cpu_mmid_lock in check_mmu_context(). Taking the lock synchronises
202 * with the rollover and so we are forced to see the updated
203 * generation.
204 *
205 * - We get a valid MMID back from the cmpxchg, which means the
206 * relaxed xchg in flush_context will treat us as reserved
207 * because atomic RmWs are totally ordered for a given location.
208 */
209 ctx = cpu_context(cpu, mm);
210 old_active_mmid = READ_ONCE(cpu_data[cpu].asid_cache);
211 if (!old_active_mmid ||
212 !asid_versions_eq(cpu, ctx, atomic64_read(&mmid_version)) ||
213 !cmpxchg_relaxed(&cpu_data[cpu].asid_cache, old_active_mmid, ctx)) {
214 raw_spin_lock_irqsave(&cpu_mmid_lock, flags);
215
216 ctx = cpu_context(cpu, mm);
217 if (!asid_versions_eq(cpu, ctx, atomic64_read(&mmid_version)))
218 ctx = get_new_mmid(mm);
219
220 WRITE_ONCE(cpu_data[cpu].asid_cache, ctx);
221 raw_spin_unlock_irqrestore(&cpu_mmid_lock, flags);
222 }
223
224 /*
225 * Invalidate the local TLB if needed. Note that we must only clear our
226 * bit in tlb_flush_pending after this is complete, so that the
227 * cpu_has_shared_ftlb_entries case below isn't misled.
228 */
229 if (cpumask_test_cpu(cpu, &tlb_flush_pending)) {
230 if (cpu_has_vtag_icache)
231 flush_icache_all();
232 local_flush_tlb_all();
233 cpumask_clear_cpu(cpu, &tlb_flush_pending);
234 }
235
236 write_c0_memorymapid(ctx & cpu_asid_mask(&boot_cpu_data));
237
238 /*
239 * If this CPU shares FTLB entries with its siblings and one or more of
240 * those siblings hasn't yet invalidated its TLB following a version
241 * increase then we need to invalidate any TLB entries for our MMID
242 * that we might otherwise pick up from a sibling.
243 *
244 * We ifdef on CONFIG_SMP because cpu_sibling_map isn't defined in
245 * CONFIG_SMP=n kernels.
246 */
247 #ifdef CONFIG_SMP
248 if (cpu_has_shared_ftlb_entries &&
249 cpumask_intersects(&tlb_flush_pending, &cpu_sibling_map[cpu])) {
250 /* Ensure we operate on the new MMID */
251 mtc0_tlbw_hazard();
252
253 /*
254 * Invalidate all TLB entries associated with the new
255 * MMID, and wait for the invalidation to complete.
256 */
257 ginvt_mmid();
258 sync_ginv();
259 }
260 #endif
261
262 setup_pgd:
263 TLBMISS_HANDLER_SETUP_PGD(mm->pgd);
264 }
265 EXPORT_SYMBOL_GPL(check_switch_mmu_context);
266
mmid_init(void)267 static int mmid_init(void)
268 {
269 if (!cpu_has_mmid)
270 return 0;
271
272 /*
273 * Expect allocation after rollover to fail if we don't have at least
274 * one more MMID than CPUs.
275 */
276 num_mmids = asid_first_version(0);
277 WARN_ON(num_mmids <= num_possible_cpus());
278
279 atomic64_set(&mmid_version, asid_first_version(0));
280 mmid_map = bitmap_zalloc(num_mmids, GFP_KERNEL);
281 if (!mmid_map)
282 panic("Failed to allocate bitmap for %u MMIDs\n", num_mmids);
283
284 /* Reserve an MMID for kmap/wired entries */
285 __set_bit(MMID_KERNEL_WIRED, mmid_map);
286
287 pr_info("MMID allocator initialised with %u entries\n", num_mmids);
288 return 0;
289 }
290 early_initcall(mmid_init);
291