1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3 * Based on arch/arm/include/asm/tlbflush.h
4 *
5 * Copyright (C) 1999-2003 Russell King
6 * Copyright (C) 2012 ARM Ltd.
7 */
8 #ifndef __ASM_TLBFLUSH_H
9 #define __ASM_TLBFLUSH_H
10
11 #ifndef __ASSEMBLER__
12
13 #include <linux/bitfield.h>
14 #include <linux/mm_types.h>
15 #include <linux/sched.h>
16 #include <linux/mmu_notifier.h>
17 #include <asm/cputype.h>
18 #include <asm/mmu.h>
19
20 /*
21 * Raw TLBI operations.
22 *
23 * Where necessary, use the __tlbi() macro to avoid asm()
24 * boilerplate. Drivers and most kernel code should use the TLB
25 * management routines in preference to the macro below.
26 *
27 * The macro can be used as __tlbi(op) or __tlbi(op, arg), depending
28 * on whether a particular TLBI operation takes an argument or
29 * not. The macros handles invoking the asm with or without the
30 * register argument as appropriate.
31 */
32 #define __TLBI_0(op, arg) asm (ARM64_ASM_PREAMBLE \
33 "tlbi " #op "\n" \
34 : : )
35
36 #define __TLBI_1(op, arg) asm (ARM64_ASM_PREAMBLE \
37 "tlbi " #op ", %x0\n" \
38 : : "rZ" (arg))
39
40 #define __TLBI_N(op, arg, n, ...) __TLBI_##n(op, arg)
41
42 #define __tlbi(op, ...) __TLBI_N(op, ##__VA_ARGS__, 1, 0)
43
44 #define __tlbi_user(op, arg) do { \
45 if (arm64_kernel_unmapped_at_el0()) \
46 __tlbi(op, (arg) | USER_ASID_FLAG); \
47 } while (0)
48
49 /* This macro creates a properly formatted VA operand for the TLBI */
50 #define __TLBI_VADDR(addr, asid) \
51 ({ \
52 unsigned long __ta = (addr) >> 12; \
53 __ta &= GENMASK_ULL(43, 0); \
54 __ta |= (unsigned long)(asid) << 48; \
55 __ta; \
56 })
57
58 /*
59 * Get translation granule of the system, which is decided by
60 * PAGE_SIZE. Used by TTL.
61 * - 4KB : 1
62 * - 16KB : 2
63 * - 64KB : 3
64 */
65 #define TLBI_TTL_TG_4K 1
66 #define TLBI_TTL_TG_16K 2
67 #define TLBI_TTL_TG_64K 3
68
get_trans_granule(void)69 static inline unsigned long get_trans_granule(void)
70 {
71 switch (PAGE_SIZE) {
72 case SZ_4K:
73 return TLBI_TTL_TG_4K;
74 case SZ_16K:
75 return TLBI_TTL_TG_16K;
76 case SZ_64K:
77 return TLBI_TTL_TG_64K;
78 default:
79 return 0;
80 }
81 }
82
83 /*
84 * Level-based TLBI operations.
85 *
86 * When ARMv8.4-TTL exists, TLBI operations take an additional hint for
87 * the level at which the invalidation must take place. If the level is
88 * wrong, no invalidation may take place. In the case where the level
89 * cannot be easily determined, the value TLBI_TTL_UNKNOWN will perform
90 * a non-hinted invalidation. Any provided level outside the hint range
91 * will also cause fall-back to non-hinted invalidation.
92 *
93 * For Stage-2 invalidation, use the level values provided to that effect
94 * in asm/stage2_pgtable.h.
95 */
96 #define TLBI_TTL_MASK GENMASK_ULL(47, 44)
97
98 #define TLBI_TTL_UNKNOWN INT_MAX
99
100 #define __tlbi_level(op, addr, level) do { \
101 u64 arg = addr; \
102 \
103 if (alternative_has_cap_unlikely(ARM64_HAS_ARMv8_4_TTL) && \
104 level >= 0 && level <= 3) { \
105 u64 ttl = level & 3; \
106 ttl |= get_trans_granule() << 2; \
107 arg &= ~TLBI_TTL_MASK; \
108 arg |= FIELD_PREP(TLBI_TTL_MASK, ttl); \
109 } \
110 \
111 __tlbi(op, arg); \
112 } while(0)
113
114 #define __tlbi_user_level(op, arg, level) do { \
115 if (arm64_kernel_unmapped_at_el0()) \
116 __tlbi_level(op, (arg | USER_ASID_FLAG), level); \
117 } while (0)
118
119 /*
120 * This macro creates a properly formatted VA operand for the TLB RANGE. The
121 * value bit assignments are:
122 *
123 * +----------+------+-------+-------+-------+----------------------+
124 * | ASID | TG | SCALE | NUM | TTL | BADDR |
125 * +-----------------+-------+-------+-------+----------------------+
126 * |63 48|47 46|45 44|43 39|38 37|36 0|
127 *
128 * The address range is determined by below formula: [BADDR, BADDR + (NUM + 1) *
129 * 2^(5*SCALE + 1) * PAGESIZE)
130 *
131 * Note that the first argument, baddr, is pre-shifted; If LPA2 is in use, BADDR
132 * holds addr[52:16]. Else BADDR holds page number. See for example ARM DDI
133 * 0487J.a section C5.5.60 "TLBI VAE1IS, TLBI VAE1ISNXS, TLB Invalidate by VA,
134 * EL1, Inner Shareable".
135 *
136 */
137 #define TLBIR_ASID_MASK GENMASK_ULL(63, 48)
138 #define TLBIR_TG_MASK GENMASK_ULL(47, 46)
139 #define TLBIR_SCALE_MASK GENMASK_ULL(45, 44)
140 #define TLBIR_NUM_MASK GENMASK_ULL(43, 39)
141 #define TLBIR_TTL_MASK GENMASK_ULL(38, 37)
142 #define TLBIR_BADDR_MASK GENMASK_ULL(36, 0)
143
144 #define __TLBI_VADDR_RANGE(baddr, asid, scale, num, ttl) \
145 ({ \
146 unsigned long __ta = 0; \
147 unsigned long __ttl = (ttl >= 1 && ttl <= 3) ? ttl : 0; \
148 __ta |= FIELD_PREP(TLBIR_BADDR_MASK, baddr); \
149 __ta |= FIELD_PREP(TLBIR_TTL_MASK, __ttl); \
150 __ta |= FIELD_PREP(TLBIR_NUM_MASK, num); \
151 __ta |= FIELD_PREP(TLBIR_SCALE_MASK, scale); \
152 __ta |= FIELD_PREP(TLBIR_TG_MASK, get_trans_granule()); \
153 __ta |= FIELD_PREP(TLBIR_ASID_MASK, asid); \
154 __ta; \
155 })
156
157 /* These macros are used by the TLBI RANGE feature. */
158 #define __TLBI_RANGE_PAGES(num, scale) \
159 ((unsigned long)((num) + 1) << (5 * (scale) + 1))
160 #define MAX_TLBI_RANGE_PAGES __TLBI_RANGE_PAGES(31, 3)
161
162 /*
163 * Generate 'num' values from -1 to 31 with -1 rejected by the
164 * __flush_tlb_range() loop below. Its return value is only
165 * significant for a maximum of MAX_TLBI_RANGE_PAGES pages. If
166 * 'pages' is more than that, you must iterate over the overall
167 * range.
168 */
169 #define __TLBI_RANGE_NUM(pages, scale) \
170 ({ \
171 int __pages = min((pages), \
172 __TLBI_RANGE_PAGES(31, (scale))); \
173 (__pages >> (5 * (scale) + 1)) - 1; \
174 })
175
176 #define __repeat_tlbi_sync(op, arg...) \
177 do { \
178 if (!alternative_has_cap_unlikely(ARM64_WORKAROUND_REPEAT_TLBI)) \
179 break; \
180 __tlbi(op, ##arg); \
181 dsb(ish); \
182 } while (0)
183
184 /*
185 * Complete broadcast TLB maintenance issued by the host which invalidates
186 * stage 1 information in the host's own translation regime.
187 */
__tlbi_sync_s1ish(void)188 static inline void __tlbi_sync_s1ish(void)
189 {
190 dsb(ish);
191 __repeat_tlbi_sync(vale1is, 0);
192 }
193
194 /*
195 * Complete broadcast TLB maintenance issued by hyp code which invalidates
196 * stage 1 translation information in any translation regime.
197 */
__tlbi_sync_s1ish_hyp(void)198 static inline void __tlbi_sync_s1ish_hyp(void)
199 {
200 dsb(ish);
201 __repeat_tlbi_sync(vale2is, 0);
202 }
203
204 /*
205 * TLB Invalidation
206 * ================
207 *
208 * This header file implements the low-level TLB invalidation routines
209 * (sometimes referred to as "flushing" in the kernel) for arm64.
210 *
211 * Every invalidation operation uses the following template:
212 *
213 * DSB ISHST // Ensure prior page-table updates have completed
214 * TLBI ... // Invalidate the TLB
215 * DSB ISH // Ensure the TLB invalidation has completed
216 * if (invalidated kernel mappings)
217 * ISB // Discard any instructions fetched from the old mapping
218 *
219 *
220 * The following functions form part of the "core" TLB invalidation API,
221 * as documented in Documentation/core-api/cachetlb.rst:
222 *
223 * flush_tlb_all()
224 * Invalidate the entire TLB (kernel + user) on all CPUs
225 *
226 * flush_tlb_mm(mm)
227 * Invalidate an entire user address space on all CPUs.
228 * The 'mm' argument identifies the ASID to invalidate.
229 *
230 * flush_tlb_range(vma, start, end)
231 * Invalidate the virtual-address range '[start, end)' on all
232 * CPUs for the user address space corresponding to 'vma->mm'.
233 * Note that this operation also invalidates any walk-cache
234 * entries associated with translations for the specified address
235 * range.
236 *
237 * flush_tlb_kernel_range(start, end)
238 * Same as flush_tlb_range(..., start, end), but applies to
239 * kernel mappings rather than a particular user address space.
240 * Whilst not explicitly documented, this function is used when
241 * unmapping pages from vmalloc/io space.
242 *
243 * flush_tlb_page(vma, addr)
244 * Invalidate a single user mapping for address 'addr' in the
245 * address space corresponding to 'vma->mm'. Note that this
246 * operation only invalidates a single, last-level page-table
247 * entry and therefore does not affect any walk-caches.
248 *
249 *
250 * Next, we have some undocumented invalidation routines that you probably
251 * don't want to call unless you know what you're doing:
252 *
253 * local_flush_tlb_all()
254 * Same as flush_tlb_all(), but only applies to the calling CPU.
255 *
256 * __flush_tlb_kernel_pgtable(addr)
257 * Invalidate a single kernel mapping for address 'addr' on all
258 * CPUs, ensuring that any walk-cache entries associated with the
259 * translation are also invalidated.
260 *
261 * __flush_tlb_range(vma, start, end, stride, last_level, tlb_level)
262 * Invalidate the virtual-address range '[start, end)' on all
263 * CPUs for the user address space corresponding to 'vma->mm'.
264 * The invalidation operations are issued at a granularity
265 * determined by 'stride' and only affect any walk-cache entries
266 * if 'last_level' is equal to false. tlb_level is the level at
267 * which the invalidation must take place. If the level is wrong,
268 * no invalidation may take place. In the case where the level
269 * cannot be easily determined, the value TLBI_TTL_UNKNOWN will
270 * perform a non-hinted invalidation.
271 *
272 * local_flush_tlb_page(vma, addr)
273 * Local variant of flush_tlb_page(). Stale TLB entries may
274 * remain in remote CPUs.
275 *
276 * local_flush_tlb_page_nonotify(vma, addr)
277 * Same as local_flush_tlb_page() except MMU notifier will not be
278 * called.
279 *
280 * local_flush_tlb_contpte(vma, addr)
281 * Invalidate the virtual-address range
282 * '[addr, addr+CONT_PTE_SIZE)' mapped with contpte on local CPU
283 * for the user address space corresponding to 'vma->mm'. Stale
284 * TLB entries may remain in remote CPUs.
285 *
286 * Finally, take a look at asm/tlb.h to see how tlb_flush() is implemented
287 * on top of these routines, since that is our interface to the mmu_gather
288 * API as used by munmap() and friends.
289 */
local_flush_tlb_all(void)290 static inline void local_flush_tlb_all(void)
291 {
292 dsb(nshst);
293 __tlbi(vmalle1);
294 dsb(nsh);
295 isb();
296 }
297
flush_tlb_all(void)298 static inline void flush_tlb_all(void)
299 {
300 dsb(ishst);
301 __tlbi(vmalle1is);
302 __tlbi_sync_s1ish();
303 isb();
304 }
305
flush_tlb_mm(struct mm_struct * mm)306 static inline void flush_tlb_mm(struct mm_struct *mm)
307 {
308 unsigned long asid;
309
310 dsb(ishst);
311 asid = __TLBI_VADDR(0, ASID(mm));
312 __tlbi(aside1is, asid);
313 __tlbi_user(aside1is, asid);
314 __tlbi_sync_s1ish();
315 mmu_notifier_arch_invalidate_secondary_tlbs(mm, 0, -1UL);
316 }
317
__local_flush_tlb_page_nonotify_nosync(struct mm_struct * mm,unsigned long uaddr)318 static inline void __local_flush_tlb_page_nonotify_nosync(struct mm_struct *mm,
319 unsigned long uaddr)
320 {
321 unsigned long addr;
322
323 dsb(nshst);
324 addr = __TLBI_VADDR(uaddr, ASID(mm));
325 __tlbi(vale1, addr);
326 __tlbi_user(vale1, addr);
327 }
328
local_flush_tlb_page_nonotify(struct vm_area_struct * vma,unsigned long uaddr)329 static inline void local_flush_tlb_page_nonotify(struct vm_area_struct *vma,
330 unsigned long uaddr)
331 {
332 __local_flush_tlb_page_nonotify_nosync(vma->vm_mm, uaddr);
333 dsb(nsh);
334 }
335
local_flush_tlb_page(struct vm_area_struct * vma,unsigned long uaddr)336 static inline void local_flush_tlb_page(struct vm_area_struct *vma,
337 unsigned long uaddr)
338 {
339 __local_flush_tlb_page_nonotify_nosync(vma->vm_mm, uaddr);
340 mmu_notifier_arch_invalidate_secondary_tlbs(vma->vm_mm, uaddr & PAGE_MASK,
341 (uaddr & PAGE_MASK) + PAGE_SIZE);
342 dsb(nsh);
343 }
344
__flush_tlb_page_nosync(struct mm_struct * mm,unsigned long uaddr)345 static inline void __flush_tlb_page_nosync(struct mm_struct *mm,
346 unsigned long uaddr)
347 {
348 unsigned long addr;
349
350 dsb(ishst);
351 addr = __TLBI_VADDR(uaddr, ASID(mm));
352 __tlbi(vale1is, addr);
353 __tlbi_user(vale1is, addr);
354 mmu_notifier_arch_invalidate_secondary_tlbs(mm, uaddr & PAGE_MASK,
355 (uaddr & PAGE_MASK) + PAGE_SIZE);
356 }
357
flush_tlb_page_nosync(struct vm_area_struct * vma,unsigned long uaddr)358 static inline void flush_tlb_page_nosync(struct vm_area_struct *vma,
359 unsigned long uaddr)
360 {
361 return __flush_tlb_page_nosync(vma->vm_mm, uaddr);
362 }
363
flush_tlb_page(struct vm_area_struct * vma,unsigned long uaddr)364 static inline void flush_tlb_page(struct vm_area_struct *vma,
365 unsigned long uaddr)
366 {
367 flush_tlb_page_nosync(vma, uaddr);
368 __tlbi_sync_s1ish();
369 }
370
arch_tlbbatch_should_defer(struct mm_struct * mm)371 static inline bool arch_tlbbatch_should_defer(struct mm_struct *mm)
372 {
373 return true;
374 }
375
376 /*
377 * To support TLB batched flush for multiple pages unmapping, we only send
378 * the TLBI for each page in arch_tlbbatch_add_pending() and wait for the
379 * completion at the end in arch_tlbbatch_flush(). Since we've already issued
380 * TLBI for each page so only a DSB is needed to synchronise its effect on the
381 * other CPUs.
382 *
383 * This will save the time waiting on DSB comparing issuing a TLBI;DSB sequence
384 * for each page.
385 */
arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch * batch)386 static inline void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch)
387 {
388 __tlbi_sync_s1ish();
389 }
390
391 /*
392 * This is meant to avoid soft lock-ups on large TLB flushing ranges and not
393 * necessarily a performance improvement.
394 */
395 #define MAX_DVM_OPS PTRS_PER_PTE
396
397 /*
398 * __flush_tlb_range_op - Perform TLBI operation upon a range
399 *
400 * @op: TLBI instruction that operates on a range (has 'r' prefix)
401 * @start: The start address of the range
402 * @pages: Range as the number of pages from 'start'
403 * @stride: Flush granularity
404 * @asid: The ASID of the task (0 for IPA instructions)
405 * @tlb_level: Translation Table level hint, if known
406 * @tlbi_user: If 'true', call an additional __tlbi_user()
407 * (typically for user ASIDs). 'flase' for IPA instructions
408 * @lpa2: If 'true', the lpa2 scheme is used as set out below
409 *
410 * When the CPU does not support TLB range operations, flush the TLB
411 * entries one by one at the granularity of 'stride'. If the TLB
412 * range ops are supported, then:
413 *
414 * 1. If FEAT_LPA2 is in use, the start address of a range operation must be
415 * 64KB aligned, so flush pages one by one until the alignment is reached
416 * using the non-range operations. This step is skipped if LPA2 is not in
417 * use.
418 *
419 * 2. The minimum range granularity is decided by 'scale', so multiple range
420 * TLBI operations may be required. Start from scale = 3, flush the largest
421 * possible number of pages ((num+1)*2^(5*scale+1)) that fit into the
422 * requested range, then decrement scale and continue until one or zero pages
423 * are left. We must start from highest scale to ensure 64KB start alignment
424 * is maintained in the LPA2 case.
425 *
426 * 3. If there is 1 page remaining, flush it through non-range operations. Range
427 * operations can only span an even number of pages. We save this for last to
428 * ensure 64KB start alignment is maintained for the LPA2 case.
429 */
430 #define __flush_tlb_range_op(op, start, pages, stride, \
431 asid, tlb_level, tlbi_user, lpa2) \
432 do { \
433 typeof(start) __flush_start = start; \
434 typeof(pages) __flush_pages = pages; \
435 int num = 0; \
436 int scale = 3; \
437 int shift = lpa2 ? 16 : PAGE_SHIFT; \
438 unsigned long addr; \
439 \
440 while (__flush_pages > 0) { \
441 if (!system_supports_tlb_range() || \
442 __flush_pages == 1 || \
443 (lpa2 && __flush_start != ALIGN(__flush_start, SZ_64K))) { \
444 addr = __TLBI_VADDR(__flush_start, asid); \
445 __tlbi_level(op, addr, tlb_level); \
446 if (tlbi_user) \
447 __tlbi_user_level(op, addr, tlb_level); \
448 __flush_start += stride; \
449 __flush_pages -= stride >> PAGE_SHIFT; \
450 continue; \
451 } \
452 \
453 num = __TLBI_RANGE_NUM(__flush_pages, scale); \
454 if (num >= 0) { \
455 addr = __TLBI_VADDR_RANGE(__flush_start >> shift, asid, \
456 scale, num, tlb_level); \
457 __tlbi(r##op, addr); \
458 if (tlbi_user) \
459 __tlbi_user(r##op, addr); \
460 __flush_start += __TLBI_RANGE_PAGES(num, scale) << PAGE_SHIFT; \
461 __flush_pages -= __TLBI_RANGE_PAGES(num, scale);\
462 } \
463 scale--; \
464 } \
465 } while (0)
466
467 #define __flush_s2_tlb_range_op(op, start, pages, stride, tlb_level) \
468 __flush_tlb_range_op(op, start, pages, stride, 0, tlb_level, false, kvm_lpa2_is_enabled());
469
__flush_tlb_range_limit_excess(unsigned long start,unsigned long end,unsigned long pages,unsigned long stride)470 static inline bool __flush_tlb_range_limit_excess(unsigned long start,
471 unsigned long end, unsigned long pages, unsigned long stride)
472 {
473 /*
474 * When the system does not support TLB range based flush
475 * operation, (MAX_DVM_OPS - 1) pages can be handled. But
476 * with TLB range based operation, MAX_TLBI_RANGE_PAGES
477 * pages can be handled.
478 */
479 if ((!system_supports_tlb_range() &&
480 (end - start) >= (MAX_DVM_OPS * stride)) ||
481 pages > MAX_TLBI_RANGE_PAGES)
482 return true;
483
484 return false;
485 }
486
__flush_tlb_range_nosync(struct mm_struct * mm,unsigned long start,unsigned long end,unsigned long stride,bool last_level,int tlb_level)487 static inline void __flush_tlb_range_nosync(struct mm_struct *mm,
488 unsigned long start, unsigned long end,
489 unsigned long stride, bool last_level,
490 int tlb_level)
491 {
492 unsigned long asid, pages;
493
494 start = round_down(start, stride);
495 end = round_up(end, stride);
496 pages = (end - start) >> PAGE_SHIFT;
497
498 if (__flush_tlb_range_limit_excess(start, end, pages, stride)) {
499 flush_tlb_mm(mm);
500 return;
501 }
502
503 dsb(ishst);
504 asid = ASID(mm);
505
506 if (last_level)
507 __flush_tlb_range_op(vale1is, start, pages, stride, asid,
508 tlb_level, true, lpa2_is_enabled());
509 else
510 __flush_tlb_range_op(vae1is, start, pages, stride, asid,
511 tlb_level, true, lpa2_is_enabled());
512
513 mmu_notifier_arch_invalidate_secondary_tlbs(mm, start, end);
514 }
515
__flush_tlb_range(struct vm_area_struct * vma,unsigned long start,unsigned long end,unsigned long stride,bool last_level,int tlb_level)516 static inline void __flush_tlb_range(struct vm_area_struct *vma,
517 unsigned long start, unsigned long end,
518 unsigned long stride, bool last_level,
519 int tlb_level)
520 {
521 __flush_tlb_range_nosync(vma->vm_mm, start, end, stride,
522 last_level, tlb_level);
523 __tlbi_sync_s1ish();
524 }
525
local_flush_tlb_contpte(struct vm_area_struct * vma,unsigned long addr)526 static inline void local_flush_tlb_contpte(struct vm_area_struct *vma,
527 unsigned long addr)
528 {
529 unsigned long asid;
530
531 addr = round_down(addr, CONT_PTE_SIZE);
532
533 dsb(nshst);
534 asid = ASID(vma->vm_mm);
535 __flush_tlb_range_op(vale1, addr, CONT_PTES, PAGE_SIZE, asid,
536 3, true, lpa2_is_enabled());
537 mmu_notifier_arch_invalidate_secondary_tlbs(vma->vm_mm, addr,
538 addr + CONT_PTE_SIZE);
539 dsb(nsh);
540 }
541
flush_tlb_range(struct vm_area_struct * vma,unsigned long start,unsigned long end)542 static inline void flush_tlb_range(struct vm_area_struct *vma,
543 unsigned long start, unsigned long end)
544 {
545 /*
546 * We cannot use leaf-only invalidation here, since we may be invalidating
547 * table entries as part of collapsing hugepages or moving page tables.
548 * Set the tlb_level to TLBI_TTL_UNKNOWN because we can not get enough
549 * information here.
550 */
551 __flush_tlb_range(vma, start, end, PAGE_SIZE, false, TLBI_TTL_UNKNOWN);
552 }
553
flush_tlb_kernel_range(unsigned long start,unsigned long end)554 static inline void flush_tlb_kernel_range(unsigned long start, unsigned long end)
555 {
556 const unsigned long stride = PAGE_SIZE;
557 unsigned long pages;
558
559 start = round_down(start, stride);
560 end = round_up(end, stride);
561 pages = (end - start) >> PAGE_SHIFT;
562
563 if (__flush_tlb_range_limit_excess(start, end, pages, stride)) {
564 flush_tlb_all();
565 return;
566 }
567
568 dsb(ishst);
569 __flush_tlb_range_op(vaale1is, start, pages, stride, 0,
570 TLBI_TTL_UNKNOWN, false, lpa2_is_enabled());
571 __tlbi_sync_s1ish();
572 isb();
573 }
574
575 /*
576 * Used to invalidate the TLB (walk caches) corresponding to intermediate page
577 * table levels (pgd/pud/pmd).
578 */
__flush_tlb_kernel_pgtable(unsigned long kaddr)579 static inline void __flush_tlb_kernel_pgtable(unsigned long kaddr)
580 {
581 unsigned long addr = __TLBI_VADDR(kaddr, 0);
582
583 dsb(ishst);
584 __tlbi(vaae1is, addr);
585 __tlbi_sync_s1ish();
586 isb();
587 }
588
arch_tlbbatch_add_pending(struct arch_tlbflush_unmap_batch * batch,struct mm_struct * mm,unsigned long start,unsigned long end)589 static inline void arch_tlbbatch_add_pending(struct arch_tlbflush_unmap_batch *batch,
590 struct mm_struct *mm, unsigned long start, unsigned long end)
591 {
592 __flush_tlb_range_nosync(mm, start, end, PAGE_SIZE, true, 3);
593 }
594
__pte_flags_need_flush(ptdesc_t oldval,ptdesc_t newval)595 static inline bool __pte_flags_need_flush(ptdesc_t oldval, ptdesc_t newval)
596 {
597 ptdesc_t diff = oldval ^ newval;
598
599 /* invalid to valid transition requires no flush */
600 if (!(oldval & PTE_VALID))
601 return false;
602
603 /* Transition in the SW bits requires no flush */
604 diff &= ~PTE_SWBITS_MASK;
605
606 return diff;
607 }
608
pte_needs_flush(pte_t oldpte,pte_t newpte)609 static inline bool pte_needs_flush(pte_t oldpte, pte_t newpte)
610 {
611 return __pte_flags_need_flush(pte_val(oldpte), pte_val(newpte));
612 }
613 #define pte_needs_flush pte_needs_flush
614
huge_pmd_needs_flush(pmd_t oldpmd,pmd_t newpmd)615 static inline bool huge_pmd_needs_flush(pmd_t oldpmd, pmd_t newpmd)
616 {
617 return __pte_flags_need_flush(pmd_val(oldpmd), pmd_val(newpmd));
618 }
619 #define huge_pmd_needs_flush huge_pmd_needs_flush
620
621 #endif
622
623 #endif
624