1 // SPDX-License-Identifier: GPL-2.0-only
41 		unsigned long last_addr = addr + PAGE_SIZE * (nr - 1);  in contpte_try_unfold_partial()
42 		pte_t *last_ptep = ptep + nr - 1;  in contpte_try_unfold_partial()
81 	 *  - When folding N smaller non-contiguous ptes as 1 contiguous block.  in contpte_convert()
82 	 *  - When unfolding a contiguous block into N smaller non-contiguous ptes.  in contpte_convert()
86 	 *  0) Initial page-table layout:  in contpte_convert()
88 	 *   +----+----+----+----+  in contpte_convert()
89 	 *   |RO,n|RO,n|RO,n|RW,n| <--- last page being set as RO  in contpte_convert()
90 	 *   +----+----+----+----+  in contpte_convert()
94 	 *   +----+----+----+----+  in contpte_convert()
96 	 *   +----+----+----+----+  in contpte_convert()
104 	 *   +----+----+----+----+  in contpte_convert()
106 	 *   +----+----+----+----+  in contpte_convert()
110 	 *                  |____| <--- tlbi + dsb  in contpte_convert()
113 	 * only ever see an invalid (0) entry, or the new contiguous TLB entry.  in contpte_convert()
115 	 * contiguous TLB entry, which is a micro-optimisation opportunity,  in contpte_convert()
120 	 * stale cached TLB entries. This could lead to a "contiguous bit  in contpte_convert()
127 	 *  +----+----+----+----+  in contpte_convert()
128 	 *  |RO,n|RO,n|RO,n|RW,n| <--- old page tables, all non-contiguous  in contpte_convert()
129 	 *  +----+----+----+----+  in contpte_convert()
131 	 *  +----+----+----+----+  in contpte_convert()
132 	 *  |RO,c|RO,c|RO,c|RO,c| <--- new page tables, all contiguous  in contpte_convert()
133 	 *  +----+----+----+----+  in contpte_convert()
137 	 *  If both the old single (RW,n) and new contiguous (RO,c) TLB entries  in contpte_convert()
143 	 * a TLB conflict abort is raised (which we expressly forbid), or will  in contpte_convert()
147 	 * That is to say, will either raise a TLB conflict, or produce one of  in contpte_convert()
148 	 * the cached TLB entries, but never amalgamate.  in contpte_convert()
151 	 * the final tlbi+dsb (which evicts both the single stale (RW,n) TLB  in contpte_convert()
152 	 * entry as well as the new contiguous (RO,c) TLB entry), omitting the  in contpte_convert()
156 	 * case, we are still left with potentially all N TLB entries still  in contpte_convert()
157 	 * cached (the N-1 non-contiguous ptes, and the single contiguous  in contpte_convert()
158 	 * block). However, over time, natural TLB pressure will cause the  in contpte_convert()
159 	 * non-contiguous pte TLB entries to be flushed, leaving only the  in contpte_convert()
160 	 * contiguous block TLB entry. This means that omitting the tlbi+dsb is  in contpte_convert()
165 	 *  0) Initial page-table layout:  in contpte_convert()
167 	 *   +----+----+----+----+  in contpte_convert()
168 	 *   |RW,c|RW,c|RW,c|RW,c| <--- last page being set as RO  in contpte_convert()
169 	 *   +----+----+----+----+  in contpte_convert()
173 	 *   +----+----+----+----+  in contpte_convert()
175 	 *   +----+----+----+----+  in contpte_convert()
181 	 *  3) __set_ptes() to repaint as non-contiguous:  in contpte_convert()
183 	 *   +----+----+----+----+  in contpte_convert()
185 	 *   +----+----+----+----+  in contpte_convert()
189 	 *   +----+----+----+----+  in contpte_convert()
190 	 *   |RW,n|RW,n|RW,n|RO,n| <--- last page permissions set  in contpte_convert()
191 	 *   +----+----+----+----+  in contpte_convert()
195 	 *                  |____| <--- tlbi + dsb  in contpte_convert()
199 	 * guaranteed to remove the original large contiguous (RW,c) TLB entry,  in contpte_convert()
200 	 * as well as the intermediate (RW,n) TLB entry; the next access will  in contpte_convert()
201 	 * install the new (RO,n) TLB entry and the page tables are only  in contpte_convert()
229 	 * that the pte is not special - we never try to fold special mappings.  in __contpte_try_fold()
250 	folio_start = addr - (page - &folio->page) * PAGE_SIZE;  in __contpte_try_fold()
353 	 * so that it is self-consistent, without the PTL held, so we may be  in contpte_ptep_get_lockless()
383 	pfn = pte_pfn(orig_pte) - (orig_ptep - ptep);  in contpte_ptep_get_lockless()
441 	 * all ptes is not-present. Therefore we never need to unfold or  in contpte_set_ptes()
456 		nr = (next - addr) >> PAGE_SHIFT;  in contpte_set_ptes()
496 	 * flag for a _single_ pte. However, the core-mm code actually tracks  in contpte_ptep_test_and_clear_young()
529 		__flush_tlb_range_nosync(vma->vm_mm, addr, addr + CONT_PTE_SIZE,  in contpte_ptep_clear_flush_young()
543 	 * the tlb. Until the flush, the page may or may not be wrprotected.  in contpte_wrprotect_ptes()
564 	 * exactly what the core-mm asked for, it tracks access/dirty per  in contpte_clear_young_dirty_ptes()
572 	if (pte_cont(__ptep_get(ptep + nr - 1)))  in contpte_clear_young_dirty_ptes()
580 	__clear_young_dirty_ptes(vma, start, ptep, (end - start) / PAGE_SIZE, flags);  in contpte_clear_young_dirty_ptes()
609 		 * faults. Avoid per-page tlb flush in __ptep_set_access_flags()  in contpte_ptep_set_access_flags()
628 		__contpte_try_unfold(vma->vm_mm, addr, ptep, orig_pte);  in contpte_ptep_set_access_flags()