xref: /linux/arch/arm64/include/asm/pgtable.h (revision cb8fef4b07f71da01843457b66e579ad248a625e)
1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3  * Copyright (C) 2012 ARM Ltd.
4  */
5 #ifndef __ASM_PGTABLE_H
6 #define __ASM_PGTABLE_H
7 
8 #include <asm/bug.h>
9 #include <asm/proc-fns.h>
10 
11 #include <asm/memory.h>
12 #include <asm/mte.h>
13 #include <asm/pgtable-hwdef.h>
14 #include <asm/pgtable-prot.h>
15 #include <asm/tlbflush.h>
16 
17 /*
18  * VMALLOC range.
19  *
20  * VMALLOC_START: beginning of the kernel vmalloc space
21  * VMALLOC_END: extends to the available space below vmemmap
22  */
23 #define VMALLOC_START		(MODULES_END)
24 #if VA_BITS == VA_BITS_MIN
25 #define VMALLOC_END		(VMEMMAP_START - SZ_8M)
26 #else
27 #define VMEMMAP_UNUSED_NPAGES	((_PAGE_OFFSET(vabits_actual) - PAGE_OFFSET) >> PAGE_SHIFT)
28 #define VMALLOC_END		(VMEMMAP_START + VMEMMAP_UNUSED_NPAGES * sizeof(struct page) - SZ_8M)
29 #endif
30 
31 #define vmemmap			((struct page *)VMEMMAP_START - (memstart_addr >> PAGE_SHIFT))
32 
33 #ifndef __ASSEMBLER__
34 
35 #include <asm/cmpxchg.h>
36 #include <asm/fixmap.h>
37 #include <asm/por.h>
38 #include <linux/mmdebug.h>
39 #include <linux/mm_types.h>
40 #include <linux/sched.h>
41 #include <linux/page_table_check.h>
42 
43 static inline void emit_pte_barriers(void)
44 {
45 	/*
46 	 * These barriers are emitted under certain conditions after a pte entry
47 	 * was modified (see e.g. __set_pte_complete()). The dsb makes the store
48 	 * visible to the table walker. The isb ensures that any previous
49 	 * speculative "invalid translation" marker that is in the CPU's
50 	 * pipeline gets cleared, so that any access to that address after
51 	 * setting the pte to valid won't cause a spurious fault. If the thread
52 	 * gets preempted after storing to the pgtable but before emitting these
53 	 * barriers, __switch_to() emits a dsb which ensure the walker gets to
54 	 * see the store. There is no guarantee of an isb being issued though.
55 	 * This is safe because it will still get issued (albeit on a
56 	 * potentially different CPU) when the thread starts running again,
57 	 * before any access to the address.
58 	 */
59 	dsb(ishst);
60 	isb();
61 }
62 
63 static inline void queue_pte_barriers(void)
64 {
65 	if (is_lazy_mmu_mode_active()) {
66 		/* Avoid the atomic op if already set. */
67 		if (!test_thread_flag(TIF_LAZY_MMU_PENDING))
68 			set_thread_flag(TIF_LAZY_MMU_PENDING);
69 	} else {
70 		emit_pte_barriers();
71 	}
72 }
73 
74 static inline void arch_enter_lazy_mmu_mode(void) {}
75 
76 static inline void arch_flush_lazy_mmu_mode(void)
77 {
78 	if (test_and_clear_thread_flag(TIF_LAZY_MMU_PENDING))
79 		emit_pte_barriers();
80 }
81 
82 static inline void arch_leave_lazy_mmu_mode(void)
83 {
84 	arch_flush_lazy_mmu_mode();
85 }
86 
87 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
88 #define __HAVE_ARCH_FLUSH_PMD_TLB_RANGE
89 
90 /* Set stride and tlb_level in flush_*_tlb_range */
91 #define flush_pmd_tlb_range(vma, addr, end)	\
92 	__flush_tlb_range(vma, addr, end, PMD_SIZE, 2, TLBF_NONE)
93 #define flush_pud_tlb_range(vma, addr, end)	\
94 	__flush_tlb_range(vma, addr, end, PUD_SIZE, 1, TLBF_NONE)
95 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
96 
97 /*
98  * We use local TLB invalidation instruction when reusing page in
99  * write protection fault handler to avoid TLBI broadcast in the hot
100  * path.  This will cause spurious page faults if stale read-only TLB
101  * entries exist.
102  */
103 #define flush_tlb_fix_spurious_fault(vma, address, ptep)	\
104 	__flush_tlb_page(vma, address, TLBF_NOBROADCAST | TLBF_NONOTIFY)
105 
106 #define flush_tlb_fix_spurious_fault_pmd(vma, address, pmdp)			\
107 	__flush_tlb_range(vma, address, address + PMD_SIZE, PMD_SIZE, 2,	\
108 			  TLBF_NOBROADCAST | TLBF_NONOTIFY | TLBF_NOWALKCACHE)
109 
110 #ifdef CONFIG_ARM64_PA_BITS_52
111 static inline phys_addr_t __pte_to_phys(pte_t pte)
112 {
113 	pte_val(pte) &= ~PTE_MAYBE_SHARED;
114 	return (pte_val(pte) & PTE_ADDR_LOW) |
115 		((pte_val(pte) & PTE_ADDR_HIGH) << PTE_ADDR_HIGH_SHIFT);
116 }
117 static inline pteval_t __phys_to_pte_val(phys_addr_t phys)
118 {
119 	return (phys | (phys >> PTE_ADDR_HIGH_SHIFT)) & PHYS_TO_PTE_ADDR_MASK;
120 }
121 #else
122 static inline phys_addr_t __pte_to_phys(pte_t pte)
123 {
124 	return pte_val(pte) & PTE_ADDR_LOW;
125 }
126 
127 static inline pteval_t __phys_to_pte_val(phys_addr_t phys)
128 {
129 	return phys;
130 }
131 #endif
132 
133 #define pte_pfn(pte)		(__pte_to_phys(pte) >> PAGE_SHIFT)
134 #define pfn_pte(pfn,prot)	\
135 	__pte(__phys_to_pte_val((phys_addr_t)(pfn) << PAGE_SHIFT) | pgprot_val(prot))
136 
137 #define pte_none(pte)		(!pte_val(pte))
138 #define pte_page(pte)		(pfn_to_page(pte_pfn(pte)))
139 
140 #define pte_valid(pte)		(!!(pte_val(pte) & PTE_VALID))
141 #define pte_present_invalid(pte) \
142 	((pte_val(pte) & (PTE_VALID | PTE_PRESENT_INVALID)) == PTE_PRESENT_INVALID)
143 
144 /*
145  * The following only work if pte_present(). Undefined behaviour otherwise.
146  */
147 static __always_inline bool pte_present(pte_t pte)
148 {
149 	return pte_valid(pte) || pte_present_invalid(pte);
150 }
151 #define pte_young(pte)		(!!(pte_val(pte) & PTE_AF))
152 #define pte_special(pte)	(!!(pte_val(pte) & PTE_SPECIAL))
153 #define pte_write(pte)		(!!(pte_val(pte) & PTE_WRITE))
154 #define pte_rdonly(pte)		(!!(pte_val(pte) & PTE_RDONLY))
155 #define pte_user(pte)		(!!(pte_val(pte) & PTE_USER))
156 #define pte_user_exec(pte)	(!(pte_val(pte) & PTE_UXN))
157 #define pte_cont(pte)		(!!(pte_val(pte) & PTE_CONT))
158 #define pte_tagged(pte)		((pte_val(pte) & PTE_ATTRINDX_MASK) == \
159 				 PTE_ATTRINDX(MT_NORMAL_TAGGED))
160 
161 #define pte_cont_addr_end(addr, end)						\
162 ({	unsigned long __boundary = ((addr) + CONT_PTE_SIZE) & CONT_PTE_MASK;	\
163 	(__boundary - 1 < (end) - 1) ? __boundary : (end);			\
164 })
165 
166 #define pmd_cont_addr_end(addr, end)						\
167 ({	unsigned long __boundary = ((addr) + CONT_PMD_SIZE) & CONT_PMD_MASK;	\
168 	(__boundary - 1 < (end) - 1) ? __boundary : (end);			\
169 })
170 
171 #define pte_hw_dirty(pte)	(pte_write(pte) && !pte_rdonly(pte))
172 #define pte_sw_dirty(pte)	(!!(pte_val(pte) & PTE_DIRTY))
173 #define pte_dirty(pte)		(pte_sw_dirty(pte) || pte_hw_dirty(pte))
174 
175 /*
176  * Execute-only user mappings do not have the PTE_USER bit set. All valid
177  * kernel mappings have the PTE_UXN bit set.
178  */
179 #define pte_valid_not_user(pte) \
180 	((pte_val(pte) & (PTE_VALID | PTE_USER | PTE_UXN)) == (PTE_VALID | PTE_UXN))
181 /*
182  * Returns true if the pte is valid and has the contiguous bit set.
183  */
184 #define pte_valid_cont(pte)	(pte_valid(pte) && pte_cont(pte))
185 /*
186  * Could the pte be present in the TLB? We must check mm_tlb_flush_pending
187  * so that we don't erroneously return false for pages that have been
188  * remapped as PROT_NONE but are yet to be flushed from the TLB.
189  * Note that we can't make any assumptions based on the state of the access
190  * flag, since __ptep_clear_flush_young() elides a DSB when invalidating the
191  * TLB.
192  */
193 #define pte_accessible(mm, pte)	\
194 	(mm_tlb_flush_pending(mm) ? pte_present(pte) : pte_valid(pte))
195 
196 static inline bool por_el0_allows_pkey(u8 pkey, bool write, bool execute)
197 {
198 	u64 por;
199 
200 	if (!system_supports_poe())
201 		return true;
202 
203 	por = read_sysreg_s(SYS_POR_EL0);
204 
205 	if (write)
206 		return por_elx_allows_write(por, pkey);
207 
208 	if (execute)
209 		return por_elx_allows_exec(por, pkey);
210 
211 	return por_elx_allows_read(por, pkey);
212 }
213 
214 /*
215  * p??_access_permitted() is true for valid user mappings (PTE_USER
216  * bit set, subject to the write permission check). For execute-only
217  * mappings, like PROT_EXEC with EPAN (both PTE_USER and PTE_UXN bits
218  * not set) must return false. PROT_NONE mappings do not have the
219  * PTE_VALID bit set.
220  */
221 #define pte_access_permitted_no_overlay(pte, write) \
222 	(((pte_val(pte) & (PTE_VALID | PTE_USER)) == (PTE_VALID | PTE_USER)) && (!(write) || pte_write(pte)))
223 #define pte_access_permitted(pte, write) \
224 	(pte_access_permitted_no_overlay(pte, write) && \
225 	por_el0_allows_pkey(FIELD_GET(PTE_PO_IDX_MASK, pte_val(pte)), write, false))
226 #define pmd_access_permitted(pmd, write) \
227 	(pte_access_permitted(pmd_pte(pmd), (write)))
228 #define pud_access_permitted(pud, write) \
229 	(pte_access_permitted(pud_pte(pud), (write)))
230 
231 static inline pte_t clear_pte_bit(pte_t pte, pgprot_t prot)
232 {
233 	pte_val(pte) &= ~pgprot_val(prot);
234 	return pte;
235 }
236 
237 static inline pte_t set_pte_bit(pte_t pte, pgprot_t prot)
238 {
239 	pte_val(pte) |= pgprot_val(prot);
240 	return pte;
241 }
242 
243 static inline pmd_t clear_pmd_bit(pmd_t pmd, pgprot_t prot)
244 {
245 	pmd_val(pmd) &= ~pgprot_val(prot);
246 	return pmd;
247 }
248 
249 static inline pmd_t set_pmd_bit(pmd_t pmd, pgprot_t prot)
250 {
251 	pmd_val(pmd) |= pgprot_val(prot);
252 	return pmd;
253 }
254 
255 static inline pte_t pte_mkwrite_novma(pte_t pte)
256 {
257 	pte = set_pte_bit(pte, __pgprot(PTE_WRITE));
258 	if (pte_sw_dirty(pte))
259 		pte = clear_pte_bit(pte, __pgprot(PTE_RDONLY));
260 	return pte;
261 }
262 
263 static inline pte_t pte_mkclean(pte_t pte)
264 {
265 	pte = clear_pte_bit(pte, __pgprot(PTE_DIRTY));
266 	pte = set_pte_bit(pte, __pgprot(PTE_RDONLY));
267 
268 	return pte;
269 }
270 
271 static inline pte_t pte_mkdirty(pte_t pte)
272 {
273 	pte = set_pte_bit(pte, __pgprot(PTE_DIRTY));
274 
275 	if (pte_write(pte))
276 		pte = clear_pte_bit(pte, __pgprot(PTE_RDONLY));
277 
278 	return pte;
279 }
280 
281 static inline pte_t pte_wrprotect(pte_t pte)
282 {
283 	/*
284 	 * If hardware-dirty (PTE_WRITE/DBM bit set and PTE_RDONLY
285 	 * clear), set the PTE_DIRTY bit.
286 	 */
287 	if (pte_hw_dirty(pte))
288 		pte = set_pte_bit(pte, __pgprot(PTE_DIRTY));
289 
290 	pte = clear_pte_bit(pte, __pgprot(PTE_WRITE));
291 	pte = set_pte_bit(pte, __pgprot(PTE_RDONLY));
292 	return pte;
293 }
294 
295 static inline pte_t pte_mkold(pte_t pte)
296 {
297 	return clear_pte_bit(pte, __pgprot(PTE_AF));
298 }
299 
300 static inline pte_t pte_mkyoung(pte_t pte)
301 {
302 	return set_pte_bit(pte, __pgprot(PTE_AF));
303 }
304 
305 static inline pte_t pte_mkspecial(pte_t pte)
306 {
307 	return set_pte_bit(pte, __pgprot(PTE_SPECIAL));
308 }
309 
310 static inline pte_t pte_mkcont(pte_t pte)
311 {
312 	return set_pte_bit(pte, __pgprot(PTE_CONT));
313 }
314 
315 static inline pte_t pte_mknoncont(pte_t pte)
316 {
317 	return clear_pte_bit(pte, __pgprot(PTE_CONT));
318 }
319 
320 static inline pte_t pte_mkvalid_k(pte_t pte)
321 {
322 	pte = clear_pte_bit(pte, __pgprot(PTE_PRESENT_INVALID));
323 	pte = set_pte_bit(pte, __pgprot(PTE_PRESENT_VALID_KERNEL));
324 	return pte;
325 }
326 
327 static inline pte_t pte_mkinvalid(pte_t pte)
328 {
329 	pte = set_pte_bit(pte, __pgprot(PTE_PRESENT_INVALID));
330 	pte = clear_pte_bit(pte, __pgprot(PTE_VALID));
331 	return pte;
332 }
333 
334 static inline pmd_t pmd_mkcont(pmd_t pmd)
335 {
336 	return __pmd(pmd_val(pmd) | PMD_SECT_CONT);
337 }
338 
339 static inline pmd_t pmd_mknoncont(pmd_t pmd)
340 {
341 	return __pmd(pmd_val(pmd) & ~PMD_SECT_CONT);
342 }
343 
344 #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP
345 static inline int pte_uffd(pte_t pte)
346 {
347 	return !!(pte_val(pte) & PTE_UFFD);
348 }
349 
350 static inline pte_t pte_mkuffd(pte_t pte)
351 {
352 	return pte_wrprotect(set_pte_bit(pte, __pgprot(PTE_UFFD)));
353 }
354 
355 static inline pte_t pte_clear_uffd(pte_t pte)
356 {
357 	return clear_pte_bit(pte, __pgprot(PTE_UFFD));
358 }
359 #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */
360 
361 static inline void __set_pte_nosync(pte_t *ptep, pte_t pte)
362 {
363 	WRITE_ONCE(*ptep, pte);
364 }
365 
366 static inline void __set_pte_complete(pte_t pte)
367 {
368 	/*
369 	 * Only if the new pte is valid and kernel, otherwise TLB maintenance
370 	 * has the necessary barriers.
371 	 */
372 	if (pte_valid_not_user(pte))
373 		queue_pte_barriers();
374 }
375 
376 static inline void __set_pte(pte_t *ptep, pte_t pte)
377 {
378 	__set_pte_nosync(ptep, pte);
379 	__set_pte_complete(pte);
380 }
381 
382 static inline pte_t __ptep_get(pte_t *ptep)
383 {
384 	return READ_ONCE(*ptep);
385 }
386 
387 extern void __sync_icache_dcache(pte_t pteval);
388 bool pgattr_change_is_safe(pteval_t old, pteval_t new);
389 
390 /*
391  * PTE bits configuration in the presence of hardware Dirty Bit Management
392  * (PTE_WRITE == PTE_DBM):
393  *
394  * Dirty  Writable | PTE_RDONLY  PTE_WRITE  PTE_DIRTY (sw)
395  *   0      0      |   1           0          0
396  *   0      1      |   1           1          0
397  *   1      0      |   1           0          1
398  *   1      1      |   0           1          x
399  *
400  * When hardware DBM is not present, the software PTE_DIRTY bit is updated via
401  * the page fault mechanism. Checking the dirty status of a pte becomes:
402  *
403  *   PTE_DIRTY || (PTE_WRITE && !PTE_RDONLY)
404  */
405 
406 static inline void __check_safe_pte_update(struct mm_struct *mm, pte_t *ptep,
407 					   pte_t pte)
408 {
409 	pte_t old_pte;
410 
411 	if (!IS_ENABLED(CONFIG_DEBUG_VM))
412 		return;
413 
414 	old_pte = __ptep_get(ptep);
415 
416 	if (!pte_valid(old_pte) || !pte_valid(pte))
417 		return;
418 	if (mm != current->active_mm && atomic_read(&mm->mm_users) <= 1)
419 		return;
420 
421 	/*
422 	 * Check for potential race with hardware updates of the pte
423 	 * (__ptep_set_access_flags safely changes valid ptes without going
424 	 * through an invalid entry).
425 	 */
426 	VM_WARN_ONCE(!pte_young(pte),
427 		     "%s: racy access flag clearing: 0x%016llx -> 0x%016llx",
428 		     __func__, pte_val(old_pte), pte_val(pte));
429 	VM_WARN_ONCE(pte_write(old_pte) && !pte_dirty(pte),
430 		     "%s: racy dirty state clearing: 0x%016llx -> 0x%016llx",
431 		     __func__, pte_val(old_pte), pte_val(pte));
432 	VM_WARN_ONCE(!pgattr_change_is_safe(pte_val(old_pte), pte_val(pte)),
433 		     "%s: unsafe attribute change: 0x%016llx -> 0x%016llx",
434 		     __func__, pte_val(old_pte), pte_val(pte));
435 }
436 
437 static inline void __sync_cache_and_tags(pte_t pte, unsigned int nr_pages)
438 {
439 	if (pte_present(pte) && pte_user_exec(pte) && !pte_special(pte))
440 		__sync_icache_dcache(pte);
441 
442 	/*
443 	 * If the PTE would provide user space access to the tags associated
444 	 * with it then ensure that the MTE tags are synchronised.  Although
445 	 * pte_access_permitted_no_overlay() returns false for exec only
446 	 * mappings, they don't expose tags (instruction fetches don't check
447 	 * tags).
448 	 */
449 	if (system_supports_mte() && pte_access_permitted_no_overlay(pte, false) &&
450 	    !pte_special(pte) && pte_tagged(pte))
451 		mte_sync_tags(pte, nr_pages);
452 }
453 
454 /*
455  * Select all bits except the pfn
456  */
457 #define pte_pgprot pte_pgprot
458 static inline pgprot_t pte_pgprot(pte_t pte)
459 {
460 	unsigned long pfn = pte_pfn(pte);
461 
462 	return __pgprot(pte_val(pfn_pte(pfn, __pgprot(0))) ^ pte_val(pte));
463 }
464 
465 #define pte_advance_pfn pte_advance_pfn
466 static inline pte_t pte_advance_pfn(pte_t pte, unsigned long nr)
467 {
468 	return pfn_pte(pte_pfn(pte) + nr, pte_pgprot(pte));
469 }
470 
471 /*
472  * Hugetlb definitions.
473  */
474 #define HUGE_MAX_HSTATE		4
475 #define HPAGE_SHIFT		PMD_SHIFT
476 #define HPAGE_SIZE		(_AC(1, UL) << HPAGE_SHIFT)
477 #define HPAGE_MASK		(~(HPAGE_SIZE - 1))
478 #define HUGETLB_PAGE_ORDER	(HPAGE_SHIFT - PAGE_SHIFT)
479 
480 static inline pte_t pgd_pte(pgd_t pgd)
481 {
482 	return __pte(pgd_val(pgd));
483 }
484 
485 static inline pte_t p4d_pte(p4d_t p4d)
486 {
487 	return __pte(p4d_val(p4d));
488 }
489 
490 static inline pte_t pud_pte(pud_t pud)
491 {
492 	return __pte(pud_val(pud));
493 }
494 
495 static inline pud_t pte_pud(pte_t pte)
496 {
497 	return __pud(pte_val(pte));
498 }
499 
500 static inline pmd_t pud_pmd(pud_t pud)
501 {
502 	return __pmd(pud_val(pud));
503 }
504 
505 static inline pte_t pmd_pte(pmd_t pmd)
506 {
507 	return __pte(pmd_val(pmd));
508 }
509 
510 static inline pmd_t pte_pmd(pte_t pte)
511 {
512 	return __pmd(pte_val(pte));
513 }
514 
515 static inline pgprot_t mk_pud_sect_prot(pgprot_t prot)
516 {
517 	return __pgprot((pgprot_val(prot) & ~PUD_TYPE_MASK) | PUD_TYPE_SECT);
518 }
519 
520 static inline pgprot_t mk_pmd_sect_prot(pgprot_t prot)
521 {
522 	return __pgprot((pgprot_val(prot) & ~PMD_TYPE_MASK) | PMD_TYPE_SECT);
523 }
524 
525 static inline pte_t pte_swp_mkexclusive(pte_t pte)
526 {
527 	return set_pte_bit(pte, __pgprot(PTE_SWP_EXCLUSIVE));
528 }
529 
530 static inline bool pte_swp_exclusive(pte_t pte)
531 {
532 	return pte_val(pte) & PTE_SWP_EXCLUSIVE;
533 }
534 
535 static inline pte_t pte_swp_clear_exclusive(pte_t pte)
536 {
537 	return clear_pte_bit(pte, __pgprot(PTE_SWP_EXCLUSIVE));
538 }
539 
540 #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP
541 static inline pte_t pte_swp_mkuffd(pte_t pte)
542 {
543 	return set_pte_bit(pte, __pgprot(PTE_SWP_UFFD));
544 }
545 
546 static inline int pte_swp_uffd(pte_t pte)
547 {
548 	return !!(pte_val(pte) & PTE_SWP_UFFD);
549 }
550 
551 static inline pte_t pte_swp_clear_uffd(pte_t pte)
552 {
553 	return clear_pte_bit(pte, __pgprot(PTE_SWP_UFFD));
554 }
555 #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */
556 
557 #ifdef CONFIG_ARCH_HAS_PTE_PROTNONE
558 static inline int pte_protnone(pte_t pte)
559 {
560 	/*
561 	 * pte_present_invalid() tells us that the pte is invalid from HW
562 	 * perspective but present from SW perspective, so the fields are to be
563 	 * interpreted as per the HW layout. The second 2 checks are the unique
564 	 * encoding that we use for PROT_NONE. It is insufficient to only use
565 	 * the first check because we share the same encoding scheme with pmds
566 	 * which support pmd_mkinvalid(), so can be present-invalid without
567 	 * being PROT_NONE.
568 	 */
569 	return pte_present_invalid(pte) && !pte_user(pte) && !pte_user_exec(pte);
570 }
571 
572 static inline int pmd_protnone(pmd_t pmd)
573 {
574 	return pte_protnone(pmd_pte(pmd));
575 }
576 #endif /* CONFIG_ARCH_HAS_PTE_PROTNONE */
577 
578 #define pmd_present(pmd)	pte_present(pmd_pte(pmd))
579 #define pmd_dirty(pmd)		pte_dirty(pmd_pte(pmd))
580 #define pmd_young(pmd)		pte_young(pmd_pte(pmd))
581 #define pmd_valid(pmd)		pte_valid(pmd_pte(pmd))
582 #define pmd_user(pmd)		pte_user(pmd_pte(pmd))
583 #define pmd_user_exec(pmd)	pte_user_exec(pmd_pte(pmd))
584 #define pmd_cont(pmd)		pte_cont(pmd_pte(pmd))
585 #define pmd_wrprotect(pmd)	pte_pmd(pte_wrprotect(pmd_pte(pmd)))
586 #define pmd_mkold(pmd)		pte_pmd(pte_mkold(pmd_pte(pmd)))
587 #define pmd_mkwrite_novma(pmd)	pte_pmd(pte_mkwrite_novma(pmd_pte(pmd)))
588 #define pmd_mkclean(pmd)	pte_pmd(pte_mkclean(pmd_pte(pmd)))
589 #define pmd_mkdirty(pmd)	pte_pmd(pte_mkdirty(pmd_pte(pmd)))
590 #define pmd_mkyoung(pmd)	pte_pmd(pte_mkyoung(pmd_pte(pmd)))
591 #define pmd_mkvalid_k(pmd)	pte_pmd(pte_mkvalid_k(pmd_pte(pmd)))
592 #define pmd_mkinvalid(pmd)	pte_pmd(pte_mkinvalid(pmd_pte(pmd)))
593 #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP
594 #define pmd_uffd(pmd)	pte_uffd(pmd_pte(pmd))
595 #define pmd_mkuffd(pmd)	pte_pmd(pte_mkuffd(pmd_pte(pmd)))
596 #define pmd_clear_uffd(pmd)	pte_pmd(pte_clear_uffd(pmd_pte(pmd)))
597 #define pmd_swp_uffd(pmd)	pte_swp_uffd(pmd_pte(pmd))
598 #define pmd_swp_mkuffd(pmd)	pte_pmd(pte_swp_mkuffd(pmd_pte(pmd)))
599 #define pmd_swp_clear_uffd(pmd) \
600 				pte_pmd(pte_swp_clear_uffd(pmd_pte(pmd)))
601 #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */
602 
603 #define pmd_write(pmd)		pte_write(pmd_pte(pmd))
604 
605 static inline pmd_t pmd_mkhuge(pmd_t pmd)
606 {
607 	/*
608 	 * It's possible that the pmd is present-invalid on entry
609 	 * and in that case it needs to remain present-invalid on
610 	 * exit. So ensure the VALID bit does not get modified.
611 	 */
612 	pmdval_t mask = PMD_TYPE_MASK & ~PTE_VALID;
613 	pmdval_t val = PMD_TYPE_SECT & ~PTE_VALID;
614 
615 	return __pmd((pmd_val(pmd) & ~mask) | val);
616 }
617 
618 #ifdef CONFIG_ARCH_SUPPORTS_PMD_PFNMAP
619 #define pmd_special(pte)	(!!((pmd_val(pte) & PTE_SPECIAL)))
620 static inline pmd_t pmd_mkspecial(pmd_t pmd)
621 {
622 	return set_pmd_bit(pmd, __pgprot(PTE_SPECIAL));
623 }
624 #endif
625 
626 #define __pmd_to_phys(pmd)	__pte_to_phys(pmd_pte(pmd))
627 #define __phys_to_pmd_val(phys)	__phys_to_pte_val(phys)
628 #define pmd_pfn(pmd)		((__pmd_to_phys(pmd) & PMD_MASK) >> PAGE_SHIFT)
629 #define pfn_pmd(pfn,prot)	__pmd(__phys_to_pmd_val((phys_addr_t)(pfn) << PAGE_SHIFT) | pgprot_val(prot))
630 
631 #define pud_young(pud)		pte_young(pud_pte(pud))
632 #define pud_mkyoung(pud)	pte_pud(pte_mkyoung(pud_pte(pud)))
633 #define pud_mkwrite_novma(pud)	pte_pud(pte_mkwrite_novma(pud_pte(pud)))
634 #define pud_mkvalid_k(pud)	pte_pud(pte_mkvalid_k(pud_pte(pud)))
635 #define pud_write(pud)		pte_write(pud_pte(pud))
636 
637 static inline pud_t pud_mkhuge(pud_t pud)
638 {
639 	/*
640 	 * It's possible that the pud is present-invalid on entry
641 	 * and in that case it needs to remain present-invalid on
642 	 * exit. So ensure the VALID bit does not get modified.
643 	 */
644 	pudval_t mask = PUD_TYPE_MASK & ~PTE_VALID;
645 	pudval_t val = PUD_TYPE_SECT & ~PTE_VALID;
646 
647 	return __pud((pud_val(pud) & ~mask) | val);
648 }
649 
650 #define __pud_to_phys(pud)	__pte_to_phys(pud_pte(pud))
651 #define __phys_to_pud_val(phys)	__phys_to_pte_val(phys)
652 #define pud_pfn(pud)		((__pud_to_phys(pud) & PUD_MASK) >> PAGE_SHIFT)
653 #define pfn_pud(pfn,prot)	__pud(__phys_to_pud_val((phys_addr_t)(pfn) << PAGE_SHIFT) | pgprot_val(prot))
654 
655 #define pmd_pgprot pmd_pgprot
656 static inline pgprot_t pmd_pgprot(pmd_t pmd)
657 {
658 	unsigned long pfn = pmd_pfn(pmd);
659 
660 	return __pgprot(pmd_val(pfn_pmd(pfn, __pgprot(0))) ^ pmd_val(pmd));
661 }
662 
663 #define pud_pgprot pud_pgprot
664 static inline pgprot_t pud_pgprot(pud_t pud)
665 {
666 	unsigned long pfn = pud_pfn(pud);
667 
668 	return __pgprot(pud_val(pfn_pud(pfn, __pgprot(0))) ^ pud_val(pud));
669 }
670 
671 static inline void __set_ptes_anysz(struct mm_struct *mm, unsigned long addr,
672 				    pte_t *ptep, pte_t pte, unsigned int nr,
673 				    unsigned long pgsize)
674 {
675 	unsigned long stride = pgsize >> PAGE_SHIFT;
676 
677 	switch (pgsize) {
678 	case PAGE_SIZE:
679 		page_table_check_ptes_set(mm, addr, ptep, pte, nr);
680 		break;
681 	case PMD_SIZE:
682 		page_table_check_pmds_set(mm, addr, (pmd_t *)ptep,
683 					  pte_pmd(pte), nr);
684 		break;
685 #ifndef __PAGETABLE_PMD_FOLDED
686 	case PUD_SIZE:
687 		page_table_check_puds_set(mm, addr, (pud_t *)ptep,
688 					  pte_pud(pte), nr);
689 		break;
690 #endif
691 	default:
692 		VM_WARN_ON(1);
693 	}
694 
695 	__sync_cache_and_tags(pte, nr * stride);
696 
697 	for (;;) {
698 		__check_safe_pte_update(mm, ptep, pte);
699 		__set_pte_nosync(ptep, pte);
700 		if (--nr == 0)
701 			break;
702 		ptep++;
703 		pte = pte_advance_pfn(pte, stride);
704 	}
705 
706 	__set_pte_complete(pte);
707 }
708 
709 static inline void __set_ptes(struct mm_struct *mm, unsigned long addr,
710 			      pte_t *ptep, pte_t pte, unsigned int nr)
711 {
712 	__set_ptes_anysz(mm, addr, ptep, pte, nr, PAGE_SIZE);
713 }
714 
715 static inline void __set_pmds(struct mm_struct *mm, unsigned long addr,
716 			      pmd_t *pmdp, pmd_t pmd, unsigned int nr)
717 {
718 	__set_ptes_anysz(mm, addr, (pte_t *)pmdp, pmd_pte(pmd), nr, PMD_SIZE);
719 }
720 #define set_pmd_at(mm, addr, pmdp, pmd) __set_pmds(mm, addr, pmdp, pmd, 1)
721 
722 static inline void __set_puds(struct mm_struct *mm, unsigned long addr,
723 			      pud_t *pudp, pud_t pud, unsigned int nr)
724 {
725 	__set_ptes_anysz(mm, addr, (pte_t *)pudp, pud_pte(pud), nr, PUD_SIZE);
726 }
727 #define set_pud_at(mm, addr, pudp, pud) __set_puds(mm, addr, pudp, pud, 1)
728 
729 #define __p4d_to_phys(p4d)	__pte_to_phys(p4d_pte(p4d))
730 #define __phys_to_p4d_val(phys)	__phys_to_pte_val(phys)
731 
732 #define __pgd_to_phys(pgd)	__pte_to_phys(pgd_pte(pgd))
733 #define __phys_to_pgd_val(phys)	__phys_to_pte_val(phys)
734 
735 #define __pgprot_modify(prot,mask,bits) \
736 	__pgprot((pgprot_val(prot) & ~(mask)) | (bits))
737 
738 #define pgprot_nx(prot) \
739 	__pgprot_modify(prot, PTE_MAYBE_GP, PTE_PXN)
740 
741 #define pgprot_decrypted(prot) \
742 	__pgprot_modify(prot, PROT_NS_SHARED, PROT_NS_SHARED)
743 #define pgprot_encrypted(prot) \
744 	__pgprot_modify(prot, PROT_NS_SHARED, 0)
745 
746 /*
747  * Mark the prot value as uncacheable and unbufferable.
748  */
749 #define pgprot_noncached(prot) \
750 	__pgprot_modify(prot, PTE_ATTRINDX_MASK, PTE_ATTRINDX(MT_DEVICE_nGnRnE) | PTE_PXN | PTE_UXN)
751 #define pgprot_writecombine(prot) \
752 	__pgprot_modify(prot, PTE_ATTRINDX_MASK, PTE_ATTRINDX(MT_NORMAL_NC) | PTE_PXN | PTE_UXN)
753 #define pgprot_device(prot) \
754 	__pgprot_modify(prot, PTE_ATTRINDX_MASK, PTE_ATTRINDX(MT_DEVICE_nGnRE) | PTE_PXN | PTE_UXN)
755 #define pgprot_tagged(prot) \
756 	__pgprot_modify(prot, PTE_ATTRINDX_MASK, PTE_ATTRINDX(MT_NORMAL_TAGGED))
757 #define pgprot_mhp	pgprot_tagged
758 /*
759  * DMA allocations for non-coherent devices use what the Arm architecture calls
760  * "Normal non-cacheable" memory, which permits speculation, unaligned accesses
761  * and merging of writes.  This is different from "Device-nGnR[nE]" memory which
762  * is intended for MMIO and thus forbids speculation, preserves access size,
763  * requires strict alignment and can also force write responses to come from the
764  * endpoint.
765  */
766 #define pgprot_dmacoherent(prot) \
767 	__pgprot_modify(prot, PTE_ATTRINDX_MASK, \
768 			PTE_ATTRINDX(MT_NORMAL_NC) | PTE_PXN | PTE_UXN)
769 
770 #define __HAVE_PHYS_MEM_ACCESS_PROT
771 struct file;
772 extern pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
773 				     unsigned long size, pgprot_t vma_prot);
774 
775 #define pmd_none(pmd)		(!pmd_val(pmd))
776 
777 #define pmd_table(pmd)		((pmd_val(pmd) & PMD_TYPE_MASK) == \
778 				 PMD_TYPE_TABLE)
779 
780 #define pmd_leaf pmd_leaf
781 static inline bool pmd_leaf(pmd_t pmd)
782 {
783 	return pmd_present(pmd) && !pmd_table(pmd);
784 }
785 
786 #define pmd_bad(pmd)		(!pmd_table(pmd))
787 
788 #define pmd_leaf_size(pmd)	(pmd_cont(pmd) ? CONT_PMD_SIZE : PMD_SIZE)
789 #define pte_leaf_size(pte)	(pte_cont(pte) ? CONT_PTE_SIZE : PAGE_SIZE)
790 
791 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
792 static inline int pmd_trans_huge(pmd_t pmd)
793 {
794 	/*
795 	 * If pmd is present-invalid, pmd_table() won't detect it
796 	 * as a table, so force the valid bit for the comparison.
797 	 */
798 	return pmd_present(pmd) && !pmd_table(__pmd(pmd_val(pmd) | PTE_VALID));
799 }
800 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
801 
802 #if defined(CONFIG_ARM64_64K_PAGES) || CONFIG_PGTABLE_LEVELS < 3
803 static inline bool pud_table(pud_t pud) { return true; }
804 #else
805 #define pud_table(pud)		((pud_val(pud) & PUD_TYPE_MASK) == \
806 				 PUD_TYPE_TABLE)
807 #endif
808 
809 extern pgd_t swapper_pg_dir[];
810 extern pgd_t idmap_pg_dir[];
811 extern pgd_t tramp_pg_dir[];
812 extern pgd_t reserved_pg_dir[];
813 
814 extern void set_swapper_pgd(pgd_t *pgdp, pgd_t pgd);
815 
816 static inline bool in_swapper_pgdir(void *addr)
817 {
818 	return ((unsigned long)addr & PAGE_MASK) ==
819 	        ((unsigned long)swapper_pg_dir & PAGE_MASK);
820 }
821 
822 static inline void set_pmd(pmd_t *pmdp, pmd_t pmd)
823 {
824 #ifdef __PAGETABLE_PMD_FOLDED
825 	if (in_swapper_pgdir(pmdp)) {
826 		set_swapper_pgd((pgd_t *)pmdp, __pgd(pmd_val(pmd)));
827 		return;
828 	}
829 #endif /* __PAGETABLE_PMD_FOLDED */
830 
831 	WRITE_ONCE(*pmdp, pmd);
832 
833 	if (pmd_valid(pmd))
834 		queue_pte_barriers();
835 }
836 
837 static inline void pmd_clear(pmd_t *pmdp)
838 {
839 	set_pmd(pmdp, __pmd(0));
840 }
841 
842 static inline phys_addr_t pmd_page_paddr(pmd_t pmd)
843 {
844 	return __pmd_to_phys(pmd);
845 }
846 
847 static inline unsigned long pmd_page_vaddr(pmd_t pmd)
848 {
849 	return (unsigned long)__va(pmd_page_paddr(pmd));
850 }
851 
852 /* Find an entry in the third-level page table. */
853 #define pte_offset_phys(dir,addr)	(pmd_page_paddr(READ_ONCE(*(dir))) + pte_index(addr) * sizeof(pte_t))
854 
855 #define pte_set_fixmap(addr)		((pte_t *)set_fixmap_offset(FIX_PTE, addr))
856 #define pte_set_fixmap_offset(pmd, addr)	pte_set_fixmap(pte_offset_phys(pmd, addr))
857 #define pte_clear_fixmap()		clear_fixmap(FIX_PTE)
858 
859 #define pmd_page(pmd)			phys_to_page(__pmd_to_phys(pmd))
860 
861 /* use ONLY for statically allocated translation tables */
862 #define pte_offset_kimg(dir,addr)	((pte_t *)__phys_to_kimg(pte_offset_phys((dir), (addr))))
863 
864 #if CONFIG_PGTABLE_LEVELS > 2
865 
866 #define pud_none(pud)		(!pud_val(pud))
867 #define pud_bad(pud)		((pud_val(pud) & PUD_TYPE_MASK) != \
868 				 PUD_TYPE_TABLE)
869 #define pud_present(pud)	pte_present(pud_pte(pud))
870 #ifndef __PAGETABLE_PMD_FOLDED
871 #define pud_leaf pud_leaf
872 static inline bool pud_leaf(pud_t pud)
873 {
874 	return pud_present(pud) && !pud_table(pud);
875 }
876 #else
877 #define pud_leaf(pud)		false
878 #endif
879 #define pud_valid(pud)		pte_valid(pud_pte(pud))
880 #define pud_user(pud)		pte_user(pud_pte(pud))
881 #define pud_user_exec(pud)	pte_user_exec(pud_pte(pud))
882 
883 static inline bool pgtable_l4_enabled(void);
884 
885 static inline void set_pud(pud_t *pudp, pud_t pud)
886 {
887 	if (!pgtable_l4_enabled() && in_swapper_pgdir(pudp)) {
888 		set_swapper_pgd((pgd_t *)pudp, __pgd(pud_val(pud)));
889 		return;
890 	}
891 
892 	WRITE_ONCE(*pudp, pud);
893 
894 	if (pud_valid(pud))
895 		queue_pte_barriers();
896 }
897 
898 static inline void pud_clear(pud_t *pudp)
899 {
900 	set_pud(pudp, __pud(0));
901 }
902 
903 static inline phys_addr_t pud_page_paddr(pud_t pud)
904 {
905 	return __pud_to_phys(pud);
906 }
907 
908 static inline pmd_t *pud_pgtable(pud_t pud)
909 {
910 	return (pmd_t *)__va(pud_page_paddr(pud));
911 }
912 
913 /* Find an entry in the second-level page table. */
914 #define pmd_offset_phys(dir, addr)	(pud_page_paddr(READ_ONCE(*(dir))) + pmd_index(addr) * sizeof(pmd_t))
915 
916 #define pmd_set_fixmap(addr)		((pmd_t *)set_fixmap_offset(FIX_PMD, addr))
917 #define pmd_set_fixmap_offset(pud, addr)	pmd_set_fixmap(pmd_offset_phys(pud, addr))
918 #define pmd_clear_fixmap()		clear_fixmap(FIX_PMD)
919 
920 #define pud_page(pud)			phys_to_page(__pud_to_phys(pud))
921 
922 /* use ONLY for statically allocated translation tables */
923 #define pmd_offset_kimg(dir,addr)	((pmd_t *)__phys_to_kimg(pmd_offset_phys((dir), (addr))))
924 
925 #else
926 
927 #define pud_valid(pud)		false
928 #define pud_page_paddr(pud)	({ BUILD_BUG(); 0; })
929 #define pud_user_exec(pud)	pud_user(pud) /* Always 0 with folding */
930 
931 /* Match pmd_offset folding in <asm/generic/pgtable-nopmd.h> */
932 #define pmd_set_fixmap(addr)		NULL
933 #define pmd_set_fixmap_offset(pudp, addr)	((pmd_t *)pudp)
934 #define pmd_clear_fixmap()
935 
936 #define pmd_offset_kimg(dir,addr)	((pmd_t *)dir)
937 
938 #endif	/* CONFIG_PGTABLE_LEVELS > 2 */
939 
940 #if CONFIG_PGTABLE_LEVELS > 3
941 
942 static __always_inline bool pgtable_l4_enabled(void)
943 {
944 	if (CONFIG_PGTABLE_LEVELS > 4 || !IS_ENABLED(CONFIG_ARM64_LPA2))
945 		return true;
946 	if (!alternative_has_cap_likely(ARM64_ALWAYS_BOOT))
947 		return vabits_actual == VA_BITS;
948 	return alternative_has_cap_unlikely(ARM64_HAS_VA52);
949 }
950 
951 static inline bool mm_pud_folded(const struct mm_struct *mm)
952 {
953 	return !pgtable_l4_enabled();
954 }
955 #define mm_pud_folded  mm_pud_folded
956 
957 #define p4d_none(p4d)		(pgtable_l4_enabled() && !p4d_val(p4d))
958 #define p4d_bad(p4d)		(pgtable_l4_enabled() && \
959 				((p4d_val(p4d) & P4D_TYPE_MASK) != \
960 				 P4D_TYPE_TABLE))
961 #define p4d_present(p4d)	(!p4d_none(p4d))
962 
963 static inline void set_p4d(p4d_t *p4dp, p4d_t p4d)
964 {
965 	if (in_swapper_pgdir(p4dp)) {
966 		set_swapper_pgd((pgd_t *)p4dp, __pgd(p4d_val(p4d)));
967 		return;
968 	}
969 
970 	WRITE_ONCE(*p4dp, p4d);
971 	queue_pte_barriers();
972 }
973 
974 static inline void p4d_clear(p4d_t *p4dp)
975 {
976 	if (pgtable_l4_enabled())
977 		set_p4d(p4dp, __p4d(0));
978 }
979 
980 static inline phys_addr_t p4d_page_paddr(p4d_t p4d)
981 {
982 	return __p4d_to_phys(p4d);
983 }
984 
985 #define pud_index(addr)		(((addr) >> PUD_SHIFT) & (PTRS_PER_PUD - 1))
986 
987 static inline pud_t *p4d_to_folded_pud(p4d_t *p4dp, unsigned long addr)
988 {
989 	/* Ensure that 'p4dp' indexes a page table according to 'addr' */
990 	VM_BUG_ON(((addr >> P4D_SHIFT) ^ ((u64)p4dp >> 3)) % PTRS_PER_P4D);
991 
992 	return (pud_t *)PTR_ALIGN_DOWN(p4dp, PAGE_SIZE) + pud_index(addr);
993 }
994 
995 static inline pud_t *p4d_pgtable(p4d_t p4d)
996 {
997 	return (pud_t *)__va(p4d_page_paddr(p4d));
998 }
999 
1000 static inline phys_addr_t pud_offset_phys(p4d_t *p4dp, unsigned long addr)
1001 {
1002 	VM_WARN_ON_ONCE(!pgtable_l4_enabled());
1003 
1004 	return p4d_page_paddr(READ_ONCE(*p4dp)) + pud_index(addr) * sizeof(pud_t);
1005 }
1006 
1007 static inline
1008 pud_t *pud_offset_lockless(p4d_t *p4dp, p4d_t p4d, unsigned long addr)
1009 {
1010 	if (!pgtable_l4_enabled())
1011 		return p4d_to_folded_pud(p4dp, addr);
1012 	return (pud_t *)__va(p4d_page_paddr(p4d)) + pud_index(addr);
1013 }
1014 #define pud_offset_lockless pud_offset_lockless
1015 
1016 static inline pud_t *pud_offset(p4d_t *p4dp, unsigned long addr)
1017 {
1018 	return pud_offset_lockless(p4dp, READ_ONCE(*p4dp), addr);
1019 }
1020 #define pud_offset	pud_offset
1021 
1022 static inline pud_t *pud_set_fixmap(unsigned long addr)
1023 {
1024 	if (!pgtable_l4_enabled())
1025 		return NULL;
1026 	return (pud_t *)set_fixmap_offset(FIX_PUD, addr);
1027 }
1028 
1029 static inline pud_t *pud_set_fixmap_offset(p4d_t *p4dp, unsigned long addr)
1030 {
1031 	if (!pgtable_l4_enabled())
1032 		return p4d_to_folded_pud(p4dp, addr);
1033 	return pud_set_fixmap(pud_offset_phys(p4dp, addr));
1034 }
1035 
1036 static inline void pud_clear_fixmap(void)
1037 {
1038 	if (pgtable_l4_enabled())
1039 		clear_fixmap(FIX_PUD);
1040 }
1041 
1042 /* use ONLY for statically allocated translation tables */
1043 static inline pud_t *pud_offset_kimg(p4d_t *p4dp, u64 addr)
1044 {
1045 	if (!pgtable_l4_enabled())
1046 		return p4d_to_folded_pud(p4dp, addr);
1047 	return (pud_t *)__phys_to_kimg(pud_offset_phys(p4dp, addr));
1048 }
1049 
1050 #define p4d_page(p4d)		pfn_to_page(__phys_to_pfn(__p4d_to_phys(p4d)))
1051 
1052 #else
1053 
1054 static inline bool pgtable_l4_enabled(void) { return false; }
1055 
1056 #define p4d_page_paddr(p4d)	({ BUILD_BUG(); 0;})
1057 
1058 /* Match pud_offset folding in <asm/generic/pgtable-nopud.h> */
1059 #define pud_set_fixmap(addr)		NULL
1060 #define pud_set_fixmap_offset(pgdp, addr)	((pud_t *)pgdp)
1061 #define pud_clear_fixmap()
1062 
1063 #define pud_offset_kimg(dir,addr)	((pud_t *)dir)
1064 
1065 #endif  /* CONFIG_PGTABLE_LEVELS > 3 */
1066 
1067 #if CONFIG_PGTABLE_LEVELS > 4
1068 
1069 static __always_inline bool pgtable_l5_enabled(void)
1070 {
1071 	if (!alternative_has_cap_likely(ARM64_ALWAYS_BOOT))
1072 		return vabits_actual == VA_BITS;
1073 	return alternative_has_cap_unlikely(ARM64_HAS_VA52);
1074 }
1075 
1076 static inline bool mm_p4d_folded(const struct mm_struct *mm)
1077 {
1078 	return !pgtable_l5_enabled();
1079 }
1080 #define mm_p4d_folded  mm_p4d_folded
1081 
1082 #define pgd_none(pgd)		(pgtable_l5_enabled() && !pgd_val(pgd))
1083 #define pgd_bad(pgd)		(pgtable_l5_enabled() && \
1084 				((pgd_val(pgd) & PGD_TYPE_MASK) != \
1085 				 PGD_TYPE_TABLE))
1086 #define pgd_present(pgd)	(!pgd_none(pgd))
1087 
1088 static inline void set_pgd(pgd_t *pgdp, pgd_t pgd)
1089 {
1090 	if (in_swapper_pgdir(pgdp)) {
1091 		set_swapper_pgd(pgdp, __pgd(pgd_val(pgd)));
1092 		return;
1093 	}
1094 
1095 	WRITE_ONCE(*pgdp, pgd);
1096 	queue_pte_barriers();
1097 }
1098 
1099 static inline void pgd_clear(pgd_t *pgdp)
1100 {
1101 	if (pgtable_l5_enabled())
1102 		set_pgd(pgdp, __pgd(0));
1103 }
1104 
1105 static inline phys_addr_t pgd_page_paddr(pgd_t pgd)
1106 {
1107 	return __pgd_to_phys(pgd);
1108 }
1109 
1110 #define p4d_index(addr)		(((addr) >> P4D_SHIFT) & (PTRS_PER_P4D - 1))
1111 
1112 static inline p4d_t *pgd_to_folded_p4d(pgd_t *pgdp, unsigned long addr)
1113 {
1114 	/* Ensure that 'pgdp' indexes a page table according to 'addr' */
1115 	VM_BUG_ON(((addr >> PGDIR_SHIFT) ^ ((u64)pgdp >> 3)) % PTRS_PER_PGD);
1116 
1117 	return (p4d_t *)PTR_ALIGN_DOWN(pgdp, PAGE_SIZE) + p4d_index(addr);
1118 }
1119 
1120 static inline phys_addr_t p4d_offset_phys(pgd_t *pgdp, unsigned long addr)
1121 {
1122 	VM_WARN_ON_ONCE(!pgtable_l5_enabled());
1123 
1124 	return pgd_page_paddr(READ_ONCE(*pgdp)) + p4d_index(addr) * sizeof(p4d_t);
1125 }
1126 
1127 static inline
1128 p4d_t *p4d_offset_lockless(pgd_t *pgdp, pgd_t pgd, unsigned long addr)
1129 {
1130 	if (!pgtable_l5_enabled())
1131 		return pgd_to_folded_p4d(pgdp, addr);
1132 	return (p4d_t *)__va(pgd_page_paddr(pgd)) + p4d_index(addr);
1133 }
1134 #define p4d_offset_lockless p4d_offset_lockless
1135 
1136 static inline p4d_t *p4d_offset(pgd_t *pgdp, unsigned long addr)
1137 {
1138 	return p4d_offset_lockless(pgdp, READ_ONCE(*pgdp), addr);
1139 }
1140 
1141 static inline p4d_t *p4d_set_fixmap(unsigned long addr)
1142 {
1143 	if (!pgtable_l5_enabled())
1144 		return NULL;
1145 	return (p4d_t *)set_fixmap_offset(FIX_P4D, addr);
1146 }
1147 
1148 static inline p4d_t *p4d_set_fixmap_offset(pgd_t *pgdp, unsigned long addr)
1149 {
1150 	if (!pgtable_l5_enabled())
1151 		return pgd_to_folded_p4d(pgdp, addr);
1152 	return p4d_set_fixmap(p4d_offset_phys(pgdp, addr));
1153 }
1154 
1155 static inline void p4d_clear_fixmap(void)
1156 {
1157 	if (pgtable_l5_enabled())
1158 		clear_fixmap(FIX_P4D);
1159 }
1160 
1161 /* use ONLY for statically allocated translation tables */
1162 static inline p4d_t *p4d_offset_kimg(pgd_t *pgdp, u64 addr)
1163 {
1164 	if (!pgtable_l5_enabled())
1165 		return pgd_to_folded_p4d(pgdp, addr);
1166 	return (p4d_t *)__phys_to_kimg(p4d_offset_phys(pgdp, addr));
1167 }
1168 
1169 #define pgd_page(pgd)		pfn_to_page(__phys_to_pfn(__pgd_to_phys(pgd)))
1170 
1171 #else
1172 
1173 static inline bool pgtable_l5_enabled(void) { return false; }
1174 
1175 #define p4d_index(addr)		(((addr) >> P4D_SHIFT) & (PTRS_PER_P4D - 1))
1176 
1177 /* Match p4d_offset folding in <asm/generic/pgtable-nop4d.h> */
1178 #define p4d_set_fixmap(addr)		NULL
1179 #define p4d_set_fixmap_offset(p4dp, addr)	((p4d_t *)p4dp)
1180 #define p4d_clear_fixmap()
1181 
1182 #define p4d_offset_kimg(dir,addr)	((p4d_t *)dir)
1183 
1184 static inline
1185 p4d_t *p4d_offset_lockless_folded(pgd_t *pgdp, pgd_t pgd, unsigned long addr)
1186 {
1187 	/*
1188 	 * With runtime folding of the pud, pud_offset_lockless() passes
1189 	 * the 'pgd_t *' we return here to p4d_to_folded_pud(), which
1190 	 * will offset the pointer assuming that it points into
1191 	 * a page-table page. However, the fast GUP path passes us a
1192 	 * pgd_t allocated on the stack and so we must use the original
1193 	 * pointer in 'pgdp' to construct the p4d pointer instead of
1194 	 * using the generic p4d_offset_lockless() implementation.
1195 	 *
1196 	 * Note: reusing the original pointer means that we may
1197 	 * dereference the same (live) page-table entry multiple times.
1198 	 * This is safe because it is still only loaded once in the
1199 	 * context of each level and the CPU guarantees same-address
1200 	 * read-after-read ordering.
1201 	 */
1202 	return p4d_offset(pgdp, addr);
1203 }
1204 #define p4d_offset_lockless p4d_offset_lockless_folded
1205 
1206 #endif  /* CONFIG_PGTABLE_LEVELS > 4 */
1207 
1208 #define pgd_set_fixmap(addr)	((pgd_t *)set_fixmap_offset(FIX_PGD, addr))
1209 #define pgd_clear_fixmap()	clear_fixmap(FIX_PGD)
1210 
1211 static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
1212 {
1213 	/*
1214 	 * Normal and Normal-Tagged are two different memory types and indices
1215 	 * in MAIR_EL1. The mask below has to include PTE_ATTRINDX_MASK.
1216 	 */
1217 	const pteval_t mask = PTE_USER | PTE_PXN | PTE_UXN | PTE_RDONLY |
1218 			      PTE_PRESENT_INVALID | PTE_VALID | PTE_WRITE |
1219 			      PTE_GP | PTE_ATTRINDX_MASK | PTE_PO_IDX_MASK;
1220 
1221 	/* preserve the hardware dirty information */
1222 	if (pte_hw_dirty(pte))
1223 		pte = set_pte_bit(pte, __pgprot(PTE_DIRTY));
1224 
1225 	pte_val(pte) = (pte_val(pte) & ~mask) | (pgprot_val(newprot) & mask);
1226 	/*
1227 	 * If we end up clearing hw dirtiness for a sw-dirty PTE, set hardware
1228 	 * dirtiness again.
1229 	 */
1230 	if (pte_sw_dirty(pte))
1231 		pte = pte_mkdirty(pte);
1232 	return pte;
1233 }
1234 
1235 static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot)
1236 {
1237 	return pte_pmd(pte_modify(pmd_pte(pmd), newprot));
1238 }
1239 
1240 extern int __ptep_set_access_flags_anysz(struct vm_area_struct *vma,
1241 					 unsigned long address, pte_t *ptep,
1242 					 pte_t entry, int dirty,
1243 					 unsigned long pgsize);
1244 
1245 static inline int __ptep_set_access_flags(struct vm_area_struct *vma,
1246 					  unsigned long address, pte_t *ptep,
1247 					  pte_t entry, int dirty)
1248 {
1249 	return __ptep_set_access_flags_anysz(vma, address, ptep, entry, dirty,
1250 					     PAGE_SIZE);
1251 }
1252 
1253 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1254 #define __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS
1255 static inline int pmdp_set_access_flags(struct vm_area_struct *vma,
1256 					unsigned long address, pmd_t *pmdp,
1257 					pmd_t entry, int dirty)
1258 {
1259 	return __ptep_set_access_flags_anysz(vma, address, (pte_t *)pmdp,
1260 					     pmd_pte(entry), dirty, PMD_SIZE);
1261 }
1262 #endif
1263 
1264 #ifdef CONFIG_PAGE_TABLE_CHECK
1265 static inline bool pte_user_accessible_page(struct mm_struct *mm, unsigned long addr, pte_t pte)
1266 {
1267 	return pte_valid(pte) && (pte_user(pte) || pte_user_exec(pte));
1268 }
1269 
1270 static inline bool pmd_user_accessible_page(struct mm_struct *mm, unsigned long addr, pmd_t pmd)
1271 {
1272 	return pmd_valid(pmd) && !pmd_table(pmd) && (pmd_user(pmd) || pmd_user_exec(pmd));
1273 }
1274 
1275 static inline bool pud_user_accessible_page(struct mm_struct *mm, unsigned long addr, pud_t pud)
1276 {
1277 	return pud_valid(pud) && !pud_table(pud) && (pud_user(pud) || pud_user_exec(pud));
1278 }
1279 #endif
1280 
1281 /*
1282  * Atomic pte/pmd modifications.
1283  */
1284 
1285 static inline void __pte_clear(struct mm_struct *mm,
1286 			       unsigned long addr, pte_t *ptep)
1287 {
1288 	__set_pte(ptep, __pte(0));
1289 }
1290 
1291 static inline bool __ptep_test_and_clear_young(struct vm_area_struct *vma,
1292 		unsigned long address, pte_t *ptep)
1293 {
1294 	pte_t old_pte, pte;
1295 
1296 	pte = __ptep_get(ptep);
1297 	do {
1298 		old_pte = pte;
1299 		pte = pte_mkold(pte);
1300 		pte_val(pte) = cmpxchg_relaxed(&pte_val(*ptep),
1301 					       pte_val(old_pte), pte_val(pte));
1302 	} while (pte_val(pte) != pte_val(old_pte));
1303 
1304 	return pte_young(pte);
1305 }
1306 
1307 static inline bool __ptep_clear_flush_young(struct vm_area_struct *vma,
1308 		unsigned long address, pte_t *ptep)
1309 {
1310 	bool young = __ptep_test_and_clear_young(vma, address, ptep);
1311 
1312 	if (young) {
1313 		/*
1314 		 * We can elide the trailing DSB here since the worst that can
1315 		 * happen is that a CPU continues to use the young entry in its
1316 		 * TLB and we mistakenly reclaim the associated page. The
1317 		 * window for such an event is bounded by the next
1318 		 * context-switch, which provides a DSB to complete the TLB
1319 		 * invalidation.
1320 		 */
1321 		__flush_tlb_page(vma, address, TLBF_NOSYNC);
1322 	}
1323 
1324 	return young;
1325 }
1326 
1327 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG)
1328 #define __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG
1329 static inline bool pmdp_test_and_clear_young(struct vm_area_struct *vma,
1330 		unsigned long address, pmd_t *pmdp)
1331 {
1332 	/* Operation applies to PMD table entry only if FEAT_HAFT is enabled */
1333 	VM_WARN_ON(pmd_table(READ_ONCE(*pmdp)) && !system_supports_haft());
1334 	return __ptep_test_and_clear_young(vma, address, (pte_t *)pmdp);
1335 }
1336 #endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG */
1337 
1338 static inline pte_t __ptep_get_and_clear_anysz(struct mm_struct *mm,
1339 					       unsigned long address,
1340 					       pte_t *ptep,
1341 					       unsigned long pgsize)
1342 {
1343 	pte_t pte = __pte(xchg_relaxed(&pte_val(*ptep), 0));
1344 
1345 	switch (pgsize) {
1346 	case PAGE_SIZE:
1347 		page_table_check_pte_clear(mm, address, pte);
1348 		break;
1349 	case PMD_SIZE:
1350 		page_table_check_pmd_clear(mm, address, pte_pmd(pte));
1351 		break;
1352 #ifndef __PAGETABLE_PMD_FOLDED
1353 	case PUD_SIZE:
1354 		page_table_check_pud_clear(mm, address, pte_pud(pte));
1355 		break;
1356 #endif
1357 	default:
1358 		VM_WARN_ON(1);
1359 	}
1360 
1361 	return pte;
1362 }
1363 
1364 static inline pte_t __ptep_get_and_clear(struct mm_struct *mm,
1365 				       unsigned long address, pte_t *ptep)
1366 {
1367 	return __ptep_get_and_clear_anysz(mm, address, ptep, PAGE_SIZE);
1368 }
1369 
1370 static inline void __clear_full_ptes(struct mm_struct *mm, unsigned long addr,
1371 				pte_t *ptep, unsigned int nr, int full)
1372 {
1373 	for (;;) {
1374 		__ptep_get_and_clear(mm, addr, ptep);
1375 		if (--nr == 0)
1376 			break;
1377 		ptep++;
1378 		addr += PAGE_SIZE;
1379 	}
1380 }
1381 
1382 static inline pte_t __get_and_clear_full_ptes(struct mm_struct *mm,
1383 				unsigned long addr, pte_t *ptep,
1384 				unsigned int nr, int full)
1385 {
1386 	pte_t pte, tmp_pte;
1387 
1388 	pte = __ptep_get_and_clear(mm, addr, ptep);
1389 	while (--nr) {
1390 		ptep++;
1391 		addr += PAGE_SIZE;
1392 		tmp_pte = __ptep_get_and_clear(mm, addr, ptep);
1393 		if (pte_dirty(tmp_pte))
1394 			pte = pte_mkdirty(pte);
1395 		if (pte_young(tmp_pte))
1396 			pte = pte_mkyoung(pte);
1397 	}
1398 	return pte;
1399 }
1400 
1401 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1402 #define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR
1403 static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm,
1404 					    unsigned long address, pmd_t *pmdp)
1405 {
1406 	return pte_pmd(__ptep_get_and_clear_anysz(mm, address, (pte_t *)pmdp, PMD_SIZE));
1407 }
1408 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
1409 
1410 static inline void ___ptep_set_wrprotect(struct mm_struct *mm,
1411 					unsigned long address, pte_t *ptep,
1412 					pte_t pte)
1413 {
1414 	pte_t old_pte;
1415 
1416 	do {
1417 		old_pte = pte;
1418 		pte = pte_wrprotect(pte);
1419 		pte_val(pte) = cmpxchg_relaxed(&pte_val(*ptep),
1420 					       pte_val(old_pte), pte_val(pte));
1421 	} while (pte_val(pte) != pte_val(old_pte));
1422 }
1423 
1424 /*
1425  * __ptep_set_wrprotect - mark read-only while transferring potential hardware
1426  * dirty status (PTE_DBM && !PTE_RDONLY) to the software PTE_DIRTY bit.
1427  */
1428 static inline void __ptep_set_wrprotect(struct mm_struct *mm,
1429 					unsigned long address, pte_t *ptep)
1430 {
1431 	___ptep_set_wrprotect(mm, address, ptep, __ptep_get(ptep));
1432 }
1433 
1434 static inline void __wrprotect_ptes(struct mm_struct *mm, unsigned long address,
1435 				pte_t *ptep, unsigned int nr)
1436 {
1437 	unsigned int i;
1438 
1439 	for (i = 0; i < nr; i++, address += PAGE_SIZE, ptep++)
1440 		__ptep_set_wrprotect(mm, address, ptep);
1441 }
1442 
1443 static inline void __clear_young_dirty_pte(struct vm_area_struct *vma,
1444 					   unsigned long addr, pte_t *ptep,
1445 					   pte_t pte, cydp_t flags)
1446 {
1447 	pte_t old_pte;
1448 
1449 	do {
1450 		old_pte = pte;
1451 
1452 		if (flags & CYDP_CLEAR_YOUNG)
1453 			pte = pte_mkold(pte);
1454 		if (flags & CYDP_CLEAR_DIRTY)
1455 			pte = pte_mkclean(pte);
1456 
1457 		pte_val(pte) = cmpxchg_relaxed(&pte_val(*ptep),
1458 					       pte_val(old_pte), pte_val(pte));
1459 	} while (pte_val(pte) != pte_val(old_pte));
1460 }
1461 
1462 static inline void __clear_young_dirty_ptes(struct vm_area_struct *vma,
1463 					    unsigned long addr, pte_t *ptep,
1464 					    unsigned int nr, cydp_t flags)
1465 {
1466 	pte_t pte;
1467 
1468 	for (;;) {
1469 		pte = __ptep_get(ptep);
1470 
1471 		if (flags == (CYDP_CLEAR_YOUNG | CYDP_CLEAR_DIRTY))
1472 			__set_pte(ptep, pte_mkclean(pte_mkold(pte)));
1473 		else
1474 			__clear_young_dirty_pte(vma, addr, ptep, pte, flags);
1475 
1476 		if (--nr == 0)
1477 			break;
1478 		ptep++;
1479 		addr += PAGE_SIZE;
1480 	}
1481 }
1482 
1483 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1484 #define __HAVE_ARCH_PMDP_SET_WRPROTECT
1485 static inline void pmdp_set_wrprotect(struct mm_struct *mm,
1486 				      unsigned long address, pmd_t *pmdp)
1487 {
1488 	__ptep_set_wrprotect(mm, address, (pte_t *)pmdp);
1489 }
1490 
1491 #define pmdp_establish pmdp_establish
1492 static inline pmd_t pmdp_establish(struct vm_area_struct *vma,
1493 		unsigned long address, pmd_t *pmdp, pmd_t pmd)
1494 {
1495 	page_table_check_pmd_set(vma->vm_mm, address, pmdp, pmd);
1496 	return __pmd(xchg_relaxed(&pmd_val(*pmdp), pmd_val(pmd)));
1497 }
1498 #endif
1499 
1500 /*
1501  * Encode and decode a swap entry:
1502  *	bits 0-1:	present (must be zero)
1503  *	bits 2:		remember PG_anon_exclusive
1504  *	bit  3:		remember uffd state
1505  *	bits 6-10:	swap type
1506  *	bit  11:	PTE_PRESENT_INVALID (must be zero)
1507  *	bits 12-61:	swap offset
1508  */
1509 #define __SWP_TYPE_SHIFT	6
1510 #define __SWP_TYPE_BITS		5
1511 #define __SWP_TYPE_MASK		((1 << __SWP_TYPE_BITS) - 1)
1512 #define __SWP_OFFSET_SHIFT	12
1513 #define __SWP_OFFSET_BITS	50
1514 #define __SWP_OFFSET_MASK	((1UL << __SWP_OFFSET_BITS) - 1)
1515 
1516 #define __swp_type(x)		(((x).val >> __SWP_TYPE_SHIFT) & __SWP_TYPE_MASK)
1517 #define __swp_offset(x)		(((x).val >> __SWP_OFFSET_SHIFT) & __SWP_OFFSET_MASK)
1518 #define __swp_entry(type,offset) ((swp_entry_t) { ((type) << __SWP_TYPE_SHIFT) | ((offset) << __SWP_OFFSET_SHIFT) })
1519 
1520 #define __pte_to_swp_entry(pte)	((swp_entry_t) { pte_val(pte) })
1521 #define __swp_entry_to_pte(swp)	((pte_t) { (swp).val })
1522 
1523 #ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES
1524 #define __pmd_to_swp_entry(pmd)		((swp_entry_t) { pmd_val(pmd) })
1525 #define __swp_entry_to_pmd(swp)		__pmd((swp).val)
1526 #endif /* CONFIG_ARCH_HAS_PMD_SOFTLEAVES */
1527 
1528 /*
1529  * Ensure that there are not more swap files than can be encoded in the kernel
1530  * PTEs.
1531  */
1532 #define MAX_SWAPFILES_CHECK() BUILD_BUG_ON(MAX_SWAPFILES_SHIFT > __SWP_TYPE_BITS)
1533 
1534 #ifdef CONFIG_ARM64_MTE
1535 
1536 #define __HAVE_ARCH_PREPARE_TO_SWAP
1537 extern int arch_prepare_to_swap(struct folio *folio);
1538 
1539 #define __HAVE_ARCH_SWAP_INVALIDATE
1540 static inline void arch_swap_invalidate_page(int type, pgoff_t offset)
1541 {
1542 	if (system_supports_mte())
1543 		mte_invalidate_tags(type, offset);
1544 }
1545 
1546 static inline void arch_swap_invalidate_area(int type)
1547 {
1548 	if (system_supports_mte())
1549 		mte_invalidate_tags_area(type);
1550 }
1551 
1552 #define __HAVE_ARCH_SWAP_RESTORE
1553 extern void arch_swap_restore(swp_entry_t entry, struct folio *folio);
1554 
1555 #endif /* CONFIG_ARM64_MTE */
1556 
1557 /*
1558  * On AArch64, the cache coherency is handled via the __set_ptes() function.
1559  */
1560 static inline void update_mmu_cache_range(struct vm_fault *vmf,
1561 		struct vm_area_struct *vma, unsigned long addr, pte_t *ptep,
1562 		unsigned int nr)
1563 {
1564 	/*
1565 	 * We don't do anything here, so there's a very small chance of
1566 	 * us retaking a user fault which we just fixed up. The alternative
1567 	 * is doing a dsb(ishst), but that penalises the fastpath.
1568 	 */
1569 }
1570 
1571 #define update_mmu_cache(vma, addr, ptep) \
1572 	update_mmu_cache_range(NULL, vma, addr, ptep, 1)
1573 #define update_mmu_cache_pmd(vma, address, pmd) do { } while (0)
1574 
1575 #ifdef CONFIG_ARM64_PA_BITS_52
1576 #define phys_to_ttbr(addr)	(((addr) | ((addr) >> 46)) & TTBR_BADDR_MASK_52)
1577 #else
1578 #define phys_to_ttbr(addr)	(addr)
1579 #endif
1580 
1581 /*
1582  * On arm64 without hardware Access Flag, copying from user will fail because
1583  * the pte is old and cannot be marked young. So we always end up with zeroed
1584  * page after fork() + CoW for pfn mappings. We don't always have a
1585  * hardware-managed access flag on arm64.
1586  */
1587 #define arch_has_hw_pte_young		cpu_has_hw_af
1588 
1589 #ifdef CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG
1590 #define arch_has_hw_nonleaf_pmd_young	system_supports_haft
1591 #endif
1592 
1593 /*
1594  * Experimentally, it's cheap to set the access flag in hardware and we
1595  * benefit from prefaulting mappings as 'old' to start with.
1596  */
1597 #define arch_wants_old_prefaulted_pte	cpu_has_hw_af
1598 
1599 /*
1600  * Request exec memory is read into pagecache in at least 64K folios. This size
1601  * can be contpte-mapped when 4K base pages are in use (16 pages into 1 iTLB
1602  * entry), and HPA can coalesce it (4 pages into 1 TLB entry) when 16K base
1603  * pages are in use.
1604  */
1605 #define exec_folio_order() ilog2(SZ_64K >> PAGE_SHIFT)
1606 
1607 static inline bool pud_sect_supported(void)
1608 {
1609 	return PAGE_SIZE == SZ_4K;
1610 }
1611 
1612 
1613 #define __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION
1614 #define ptep_modify_prot_start ptep_modify_prot_start
1615 extern pte_t ptep_modify_prot_start(struct vm_area_struct *vma,
1616 				    unsigned long addr, pte_t *ptep);
1617 
1618 #define ptep_modify_prot_commit ptep_modify_prot_commit
1619 extern void ptep_modify_prot_commit(struct vm_area_struct *vma,
1620 				    unsigned long addr, pte_t *ptep,
1621 				    pte_t old_pte, pte_t new_pte);
1622 
1623 #define modify_prot_start_ptes modify_prot_start_ptes
1624 extern pte_t modify_prot_start_ptes(struct vm_area_struct *vma,
1625 				    unsigned long addr, pte_t *ptep,
1626 				    unsigned int nr);
1627 
1628 #define modify_prot_commit_ptes modify_prot_commit_ptes
1629 extern void modify_prot_commit_ptes(struct vm_area_struct *vma, unsigned long addr,
1630 				    pte_t *ptep, pte_t old_pte, pte_t pte,
1631 				    unsigned int nr);
1632 
1633 #ifdef CONFIG_ARM64_CONTPTE
1634 
1635 /*
1636  * The contpte APIs are used to transparently manage the contiguous bit in ptes
1637  * where it is possible and makes sense to do so. The PTE_CONT bit is considered
1638  * a private implementation detail of the public ptep API (see below).
1639  */
1640 extern void __contpte_try_fold(struct mm_struct *mm, unsigned long addr,
1641 				pte_t *ptep, pte_t pte);
1642 extern void __contpte_try_unfold(struct mm_struct *mm, unsigned long addr,
1643 				pte_t *ptep, pte_t pte);
1644 extern pte_t contpte_ptep_get(pte_t *ptep, pte_t orig_pte);
1645 extern pte_t contpte_ptep_get_lockless(pte_t *orig_ptep);
1646 extern void contpte_set_ptes(struct mm_struct *mm, unsigned long addr,
1647 				pte_t *ptep, pte_t pte, unsigned int nr);
1648 extern void contpte_clear_full_ptes(struct mm_struct *mm, unsigned long addr,
1649 				pte_t *ptep, unsigned int nr, int full);
1650 extern pte_t contpte_get_and_clear_full_ptes(struct mm_struct *mm,
1651 				unsigned long addr, pte_t *ptep,
1652 				unsigned int nr, int full);
1653 bool contpte_test_and_clear_young_ptes(struct vm_area_struct *vma,
1654 				unsigned long addr, pte_t *ptep, unsigned int nr);
1655 bool contpte_clear_flush_young_ptes(struct vm_area_struct *vma,
1656 				unsigned long addr, pte_t *ptep, unsigned int nr);
1657 extern void contpte_wrprotect_ptes(struct mm_struct *mm, unsigned long addr,
1658 				pte_t *ptep, unsigned int nr);
1659 extern int contpte_ptep_set_access_flags(struct vm_area_struct *vma,
1660 				unsigned long addr, pte_t *ptep,
1661 				pte_t entry, int dirty);
1662 extern void contpte_clear_young_dirty_ptes(struct vm_area_struct *vma,
1663 				unsigned long addr, pte_t *ptep,
1664 				unsigned int nr, cydp_t flags);
1665 
1666 static __always_inline void contpte_try_fold(struct mm_struct *mm,
1667 				unsigned long addr, pte_t *ptep, pte_t pte)
1668 {
1669 	/*
1670 	 * Only bother trying if both the virtual and physical addresses are
1671 	 * aligned and correspond to the last entry in a contig range. The core
1672 	 * code mostly modifies ranges from low to high, so this is the likely
1673 	 * the last modification in the contig range, so a good time to fold.
1674 	 * We can't fold special mappings, because there is no associated folio.
1675 	 */
1676 
1677 	const unsigned long contmask = CONT_PTES - 1;
1678 	bool valign = ((addr >> PAGE_SHIFT) & contmask) == contmask;
1679 
1680 	if (unlikely(valign)) {
1681 		bool palign = (pte_pfn(pte) & contmask) == contmask;
1682 
1683 		if (unlikely(palign &&
1684 		    pte_valid(pte) && !pte_cont(pte) && !pte_special(pte)))
1685 			__contpte_try_fold(mm, addr, ptep, pte);
1686 	}
1687 }
1688 
1689 static __always_inline void contpte_try_unfold(struct mm_struct *mm,
1690 				unsigned long addr, pte_t *ptep, pte_t pte)
1691 {
1692 	if (unlikely(pte_valid_cont(pte)))
1693 		__contpte_try_unfold(mm, addr, ptep, pte);
1694 }
1695 
1696 #define pte_batch_hint pte_batch_hint
1697 static inline unsigned int pte_batch_hint(pte_t *ptep, pte_t pte)
1698 {
1699 	if (!pte_valid_cont(pte))
1700 		return 1;
1701 
1702 	return CONT_PTES - (((unsigned long)ptep >> 3) & (CONT_PTES - 1));
1703 }
1704 
1705 /*
1706  * The below functions constitute the public API that arm64 presents to the
1707  * core-mm to manipulate PTE entries within their page tables (or at least this
1708  * is the subset of the API that arm64 needs to implement). These public
1709  * versions will automatically and transparently apply the contiguous bit where
1710  * it makes sense to do so. Therefore any users that are contig-aware (e.g.
1711  * hugetlb, kernel mapper) should NOT use these APIs, but instead use the
1712  * private versions, which are prefixed with double underscore. All of these
1713  * APIs except for ptep_get_lockless() are expected to be called with the PTL
1714  * held. Although the contiguous bit is considered private to the
1715  * implementation, it is deliberately allowed to leak through the getters (e.g.
1716  * ptep_get()), back to core code. This is required so that pte_leaf_size() can
1717  * provide an accurate size for perf_get_pgtable_size(). But this leakage means
1718  * its possible a pte will be passed to a setter with the contiguous bit set, so
1719  * we explicitly clear the contiguous bit in those cases to prevent accidentally
1720  * setting it in the pgtable.
1721  */
1722 
1723 #define ptep_get ptep_get
1724 static inline pte_t ptep_get(pte_t *ptep)
1725 {
1726 	pte_t pte = __ptep_get(ptep);
1727 
1728 	if (likely(!pte_valid_cont(pte)))
1729 		return pte;
1730 
1731 	return contpte_ptep_get(ptep, pte);
1732 }
1733 
1734 #define ptep_get_lockless ptep_get_lockless
1735 static inline pte_t ptep_get_lockless(pte_t *ptep)
1736 {
1737 	pte_t pte = __ptep_get(ptep);
1738 
1739 	if (likely(!pte_valid_cont(pte)))
1740 		return pte;
1741 
1742 	return contpte_ptep_get_lockless(ptep);
1743 }
1744 
1745 static inline void set_pte(pte_t *ptep, pte_t pte)
1746 {
1747 	/*
1748 	 * We don't have the mm or vaddr so cannot unfold contig entries (since
1749 	 * it requires tlb maintenance). set_pte() is not used in core code, so
1750 	 * this should never even be called. Regardless do our best to service
1751 	 * any call and emit a warning if there is any attempt to set a pte on
1752 	 * top of an existing contig range.
1753 	 */
1754 	pte_t orig_pte = __ptep_get(ptep);
1755 
1756 	WARN_ON_ONCE(pte_valid_cont(orig_pte));
1757 	__set_pte(ptep, pte_mknoncont(pte));
1758 }
1759 
1760 #define set_ptes set_ptes
1761 static __always_inline void set_ptes(struct mm_struct *mm, unsigned long addr,
1762 				pte_t *ptep, pte_t pte, unsigned int nr)
1763 {
1764 	pte = pte_mknoncont(pte);
1765 
1766 	if (likely(nr == 1)) {
1767 		contpte_try_unfold(mm, addr, ptep, __ptep_get(ptep));
1768 		__set_ptes(mm, addr, ptep, pte, 1);
1769 		contpte_try_fold(mm, addr, ptep, pte);
1770 	} else {
1771 		contpte_set_ptes(mm, addr, ptep, pte, nr);
1772 	}
1773 }
1774 
1775 static inline void pte_clear(struct mm_struct *mm,
1776 				unsigned long addr, pte_t *ptep)
1777 {
1778 	contpte_try_unfold(mm, addr, ptep, __ptep_get(ptep));
1779 	__pte_clear(mm, addr, ptep);
1780 }
1781 
1782 #define clear_full_ptes clear_full_ptes
1783 static inline void clear_full_ptes(struct mm_struct *mm, unsigned long addr,
1784 				pte_t *ptep, unsigned int nr, int full)
1785 {
1786 	if (likely(nr == 1)) {
1787 		contpte_try_unfold(mm, addr, ptep, __ptep_get(ptep));
1788 		__clear_full_ptes(mm, addr, ptep, nr, full);
1789 	} else {
1790 		contpte_clear_full_ptes(mm, addr, ptep, nr, full);
1791 	}
1792 }
1793 
1794 #define get_and_clear_full_ptes get_and_clear_full_ptes
1795 static inline pte_t get_and_clear_full_ptes(struct mm_struct *mm,
1796 				unsigned long addr, pte_t *ptep,
1797 				unsigned int nr, int full)
1798 {
1799 	pte_t pte;
1800 
1801 	if (likely(nr == 1)) {
1802 		contpte_try_unfold(mm, addr, ptep, __ptep_get(ptep));
1803 		pte = __get_and_clear_full_ptes(mm, addr, ptep, nr, full);
1804 	} else {
1805 		pte = contpte_get_and_clear_full_ptes(mm, addr, ptep, nr, full);
1806 	}
1807 
1808 	return pte;
1809 }
1810 
1811 #define __HAVE_ARCH_PTEP_GET_AND_CLEAR
1812 static inline pte_t ptep_get_and_clear(struct mm_struct *mm,
1813 				unsigned long addr, pte_t *ptep)
1814 {
1815 	contpte_try_unfold(mm, addr, ptep, __ptep_get(ptep));
1816 	return __ptep_get_and_clear(mm, addr, ptep);
1817 }
1818 
1819 static inline bool ptep_try_set(pte_t *ptep, pte_t new_pte)
1820 {
1821 	pteval_t old = 0;
1822 
1823 	if (!try_cmpxchg(&pte_val(*ptep), &old, pte_val(new_pte)))
1824 		return false;
1825 
1826 	/*
1827 	 * The store must be complete by the time this returns, but the caller
1828 	 * may be in lazy MMU mode, where __set_pte_complete() would defer the
1829 	 * barriers. Issue them directly.
1830 	 */
1831 	emit_pte_barriers();
1832 	return true;
1833 }
1834 #define ptep_try_set ptep_try_set
1835 
1836 /*
1837  * arm64 mandates break-before-make: a cleared kernel PTE must have its TLB
1838  * invalidated before a different page is installed in its place. The broadcast
1839  * TLBI is an instruction, not an IPI, so this is safe with interrupts disabled.
1840  */
1841 static inline void flush_tlb_before_set(unsigned long addr)
1842 {
1843 	flush_tlb_kernel_range(addr, addr + PAGE_SIZE);
1844 }
1845 #define flush_tlb_before_set flush_tlb_before_set
1846 
1847 #define test_and_clear_young_ptes test_and_clear_young_ptes
1848 static inline bool test_and_clear_young_ptes(struct vm_area_struct *vma,
1849 		unsigned long addr, pte_t *ptep, unsigned int nr)
1850 {
1851 	if (likely(nr == 1 && !pte_cont(__ptep_get(ptep))))
1852 		return __ptep_test_and_clear_young(vma, addr, ptep);
1853 
1854 	return contpte_test_and_clear_young_ptes(vma, addr, ptep, nr);
1855 }
1856 
1857 #define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
1858 static inline bool ptep_test_and_clear_young(struct vm_area_struct *vma,
1859 		unsigned long addr, pte_t *ptep)
1860 {
1861 	return test_and_clear_young_ptes(vma, addr, ptep, 1);
1862 }
1863 
1864 #define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
1865 static inline bool ptep_clear_flush_young(struct vm_area_struct *vma,
1866 		unsigned long addr, pte_t *ptep)
1867 {
1868 	pte_t orig_pte = __ptep_get(ptep);
1869 
1870 	if (likely(!pte_valid_cont(orig_pte)))
1871 		return __ptep_clear_flush_young(vma, addr, ptep);
1872 
1873 	return contpte_clear_flush_young_ptes(vma, addr, ptep, 1);
1874 }
1875 
1876 #define clear_flush_young_ptes clear_flush_young_ptes
1877 static inline bool clear_flush_young_ptes(struct vm_area_struct *vma,
1878 		unsigned long addr, pte_t *ptep, unsigned int nr)
1879 {
1880 	if (likely(nr == 1 && !pte_cont(__ptep_get(ptep))))
1881 		return __ptep_clear_flush_young(vma, addr, ptep);
1882 
1883 	return contpte_clear_flush_young_ptes(vma, addr, ptep, nr);
1884 }
1885 
1886 #define wrprotect_ptes wrprotect_ptes
1887 static __always_inline void wrprotect_ptes(struct mm_struct *mm,
1888 				unsigned long addr, pte_t *ptep, unsigned int nr)
1889 {
1890 	if (likely(nr == 1)) {
1891 		/*
1892 		 * Optimization: wrprotect_ptes() can only be called for present
1893 		 * ptes so we only need to check contig bit as condition for
1894 		 * unfold, and we can remove the contig bit from the pte we read
1895 		 * to avoid re-reading. This speeds up fork() which is sensitive
1896 		 * for order-0 folios. Equivalent to contpte_try_unfold().
1897 		 */
1898 		pte_t orig_pte = __ptep_get(ptep);
1899 
1900 		if (unlikely(pte_cont(orig_pte))) {
1901 			__contpte_try_unfold(mm, addr, ptep, orig_pte);
1902 			orig_pte = pte_mknoncont(orig_pte);
1903 		}
1904 		___ptep_set_wrprotect(mm, addr, ptep, orig_pte);
1905 	} else {
1906 		contpte_wrprotect_ptes(mm, addr, ptep, nr);
1907 	}
1908 }
1909 
1910 #define __HAVE_ARCH_PTEP_SET_WRPROTECT
1911 static inline void ptep_set_wrprotect(struct mm_struct *mm,
1912 				unsigned long addr, pte_t *ptep)
1913 {
1914 	wrprotect_ptes(mm, addr, ptep, 1);
1915 }
1916 
1917 #define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
1918 static inline int ptep_set_access_flags(struct vm_area_struct *vma,
1919 				unsigned long addr, pte_t *ptep,
1920 				pte_t entry, int dirty)
1921 {
1922 	pte_t orig_pte = __ptep_get(ptep);
1923 
1924 	entry = pte_mknoncont(entry);
1925 
1926 	if (likely(!pte_valid_cont(orig_pte)))
1927 		return __ptep_set_access_flags(vma, addr, ptep, entry, dirty);
1928 
1929 	return contpte_ptep_set_access_flags(vma, addr, ptep, entry, dirty);
1930 }
1931 
1932 #define clear_young_dirty_ptes clear_young_dirty_ptes
1933 static inline void clear_young_dirty_ptes(struct vm_area_struct *vma,
1934 					  unsigned long addr, pte_t *ptep,
1935 					  unsigned int nr, cydp_t flags)
1936 {
1937 	if (likely(nr == 1 && !pte_cont(__ptep_get(ptep))))
1938 		__clear_young_dirty_ptes(vma, addr, ptep, nr, flags);
1939 	else
1940 		contpte_clear_young_dirty_ptes(vma, addr, ptep, nr, flags);
1941 }
1942 
1943 #else /* CONFIG_ARM64_CONTPTE */
1944 
1945 #define ptep_get				__ptep_get
1946 #define set_pte					__set_pte
1947 #define set_ptes				__set_ptes
1948 #define pte_clear				__pte_clear
1949 #define clear_full_ptes				__clear_full_ptes
1950 #define get_and_clear_full_ptes			__get_and_clear_full_ptes
1951 #define __HAVE_ARCH_PTEP_GET_AND_CLEAR
1952 #define ptep_get_and_clear			__ptep_get_and_clear
1953 #define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
1954 #define ptep_test_and_clear_young		__ptep_test_and_clear_young
1955 #define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
1956 #define ptep_clear_flush_young			__ptep_clear_flush_young
1957 #define __HAVE_ARCH_PTEP_SET_WRPROTECT
1958 #define ptep_set_wrprotect			__ptep_set_wrprotect
1959 #define wrprotect_ptes				__wrprotect_ptes
1960 #define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
1961 #define ptep_set_access_flags			__ptep_set_access_flags
1962 #define clear_young_dirty_ptes			__clear_young_dirty_ptes
1963 
1964 #endif /* CONFIG_ARM64_CONTPTE */
1965 
1966 #endif /* !__ASSEMBLER__ */
1967 
1968 #endif /* __ASM_PGTABLE_H */
1969