1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3 * Describes operations that can be performed on software-defined page table
4 * leaf entries. These are abstracted from the hardware page table entries
5 * themselves by the softleaf_t type, see mm_types.h.
6 */
7 #ifndef _LINUX_LEAFOPS_H
8 #define _LINUX_LEAFOPS_H
9
10 #include <linux/mm_types.h>
11 #include <linux/swapops.h>
12 #include <linux/swap.h>
13
14 #ifdef CONFIG_MMU
15
16 /* Temporary until swp_entry_t eliminated. */
17 #define LEAF_TYPE_SHIFT SWP_TYPE_SHIFT
18
19 enum softleaf_type {
20 /* Fundamental types. */
21 SOFTLEAF_NONE,
22 SOFTLEAF_SWAP,
23 /* Migration types. */
24 SOFTLEAF_MIGRATION_READ,
25 SOFTLEAF_MIGRATION_READ_EXCLUSIVE,
26 SOFTLEAF_MIGRATION_WRITE,
27 /* Device types. */
28 SOFTLEAF_DEVICE_PRIVATE_READ,
29 SOFTLEAF_DEVICE_PRIVATE_WRITE,
30 SOFTLEAF_DEVICE_EXCLUSIVE,
31 /* H/W posion types. */
32 SOFTLEAF_HWPOISON,
33 /* Marker types. */
34 SOFTLEAF_MARKER,
35 };
36
37 /**
38 * softleaf_mk_none() - Create an empty ('none') leaf entry.
39 * Returns: empty leaf entry.
40 */
softleaf_mk_none(void)41 static inline softleaf_t softleaf_mk_none(void)
42 {
43 return ((softleaf_t) { 0 });
44 }
45
46 /**
47 * softleaf_from_pte() - Obtain a leaf entry from a PTE entry.
48 * @pte: PTE entry.
49 *
50 * If @pte is present (therefore not a leaf entry) the function returns an empty
51 * leaf entry. Otherwise, it returns a leaf entry.
52 *
53 * Returns: Leaf entry.
54 */
softleaf_from_pte(pte_t pte)55 static inline softleaf_t softleaf_from_pte(pte_t pte)
56 {
57 softleaf_t arch_entry;
58
59 if (pte_present(pte) || pte_none(pte))
60 return softleaf_mk_none();
61
62 pte = pte_swp_clear_flags(pte);
63 arch_entry = __pte_to_swp_entry(pte);
64
65 /* Temporary until swp_entry_t eliminated. */
66 return swp_entry(__swp_type(arch_entry), __swp_offset(arch_entry));
67 }
68
69 /**
70 * softleaf_to_pte() - Obtain a PTE entry from a leaf entry.
71 * @entry: Leaf entry.
72 *
73 * This generates an architecture-specific PTE entry that can be utilised to
74 * encode the metadata the leaf entry encodes.
75 *
76 * Returns: Architecture-specific PTE entry encoding leaf entry.
77 */
softleaf_to_pte(softleaf_t entry)78 static inline pte_t softleaf_to_pte(softleaf_t entry)
79 {
80 /* Temporary until swp_entry_t eliminated. */
81 return swp_entry_to_pte(entry);
82 }
83
84 #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
85 /**
86 * softleaf_from_pmd() - Obtain a leaf entry from a PMD entry.
87 * @pmd: PMD entry.
88 *
89 * If @pmd is present (therefore not a leaf entry) the function returns an empty
90 * leaf entry. Otherwise, it returns a leaf entry.
91 *
92 * Returns: Leaf entry.
93 */
softleaf_from_pmd(pmd_t pmd)94 static inline softleaf_t softleaf_from_pmd(pmd_t pmd)
95 {
96 softleaf_t arch_entry;
97
98 if (pmd_present(pmd) || pmd_none(pmd))
99 return softleaf_mk_none();
100
101 if (pmd_swp_soft_dirty(pmd))
102 pmd = pmd_swp_clear_soft_dirty(pmd);
103 if (pmd_swp_uffd_wp(pmd))
104 pmd = pmd_swp_clear_uffd_wp(pmd);
105 arch_entry = __pmd_to_swp_entry(pmd);
106
107 /* Temporary until swp_entry_t eliminated. */
108 return swp_entry(__swp_type(arch_entry), __swp_offset(arch_entry));
109 }
110
111 #else
112
softleaf_from_pmd(pmd_t pmd)113 static inline softleaf_t softleaf_from_pmd(pmd_t pmd)
114 {
115 return softleaf_mk_none();
116 }
117
118 #endif
119
120 /**
121 * softleaf_is_none() - Is the leaf entry empty?
122 * @entry: Leaf entry.
123 *
124 * Empty entries are typically the result of a 'none' page table leaf entry
125 * being converted to a leaf entry.
126 *
127 * Returns: true if the entry is empty, false otherwise.
128 */
softleaf_is_none(softleaf_t entry)129 static inline bool softleaf_is_none(softleaf_t entry)
130 {
131 return entry.val == 0;
132 }
133
134 /**
135 * softleaf_type() - Identify the type of leaf entry.
136 * @enntry: Leaf entry.
137 *
138 * Returns: the leaf entry type associated with @entry.
139 */
softleaf_type(softleaf_t entry)140 static inline enum softleaf_type softleaf_type(softleaf_t entry)
141 {
142 unsigned int type_num;
143
144 if (softleaf_is_none(entry))
145 return SOFTLEAF_NONE;
146
147 type_num = entry.val >> LEAF_TYPE_SHIFT;
148
149 if (type_num < MAX_SWAPFILES)
150 return SOFTLEAF_SWAP;
151
152 switch (type_num) {
153 #ifdef CONFIG_MIGRATION
154 case SWP_MIGRATION_READ:
155 return SOFTLEAF_MIGRATION_READ;
156 case SWP_MIGRATION_READ_EXCLUSIVE:
157 return SOFTLEAF_MIGRATION_READ_EXCLUSIVE;
158 case SWP_MIGRATION_WRITE:
159 return SOFTLEAF_MIGRATION_WRITE;
160 #endif
161 #ifdef CONFIG_DEVICE_PRIVATE
162 case SWP_DEVICE_WRITE:
163 return SOFTLEAF_DEVICE_PRIVATE_WRITE;
164 case SWP_DEVICE_READ:
165 return SOFTLEAF_DEVICE_PRIVATE_READ;
166 case SWP_DEVICE_EXCLUSIVE:
167 return SOFTLEAF_DEVICE_EXCLUSIVE;
168 #endif
169 #ifdef CONFIG_MEMORY_FAILURE
170 case SWP_HWPOISON:
171 return SOFTLEAF_HWPOISON;
172 #endif
173 case SWP_PTE_MARKER:
174 return SOFTLEAF_MARKER;
175 }
176
177 /* Unknown entry type. */
178 VM_WARN_ON_ONCE(1);
179 return SOFTLEAF_NONE;
180 }
181
182 /**
183 * softleaf_is_swap() - Is this leaf entry a swap entry?
184 * @entry: Leaf entry.
185 *
186 * Returns: true if the leaf entry is a swap entry, otherwise false.
187 */
softleaf_is_swap(softleaf_t entry)188 static inline bool softleaf_is_swap(softleaf_t entry)
189 {
190 return softleaf_type(entry) == SOFTLEAF_SWAP;
191 }
192
193 /**
194 * softleaf_is_migration_write() - Is this leaf entry a writable migration entry?
195 * @entry: Leaf entry.
196 *
197 * Returns: true if the leaf entry is a writable migration entry, otherwise
198 * false.
199 */
softleaf_is_migration_write(softleaf_t entry)200 static inline bool softleaf_is_migration_write(softleaf_t entry)
201 {
202 return softleaf_type(entry) == SOFTLEAF_MIGRATION_WRITE;
203 }
204
205 /**
206 * softleaf_is_migration_read() - Is this leaf entry a readable migration entry?
207 * @entry: Leaf entry.
208 *
209 * Returns: true if the leaf entry is a readable migration entry, otherwise
210 * false.
211 */
softleaf_is_migration_read(softleaf_t entry)212 static inline bool softleaf_is_migration_read(softleaf_t entry)
213 {
214 return softleaf_type(entry) == SOFTLEAF_MIGRATION_READ;
215 }
216
217 /**
218 * softleaf_is_migration_read_exclusive() - Is this leaf entry an exclusive
219 * readable migration entry?
220 * @entry: Leaf entry.
221 *
222 * Returns: true if the leaf entry is an exclusive readable migration entry,
223 * otherwise false.
224 */
softleaf_is_migration_read_exclusive(softleaf_t entry)225 static inline bool softleaf_is_migration_read_exclusive(softleaf_t entry)
226 {
227 return softleaf_type(entry) == SOFTLEAF_MIGRATION_READ_EXCLUSIVE;
228 }
229
230 /**
231 * softleaf_is_migration() - Is this leaf entry a migration entry?
232 * @entry: Leaf entry.
233 *
234 * Returns: true if the leaf entry is a migration entry, otherwise false.
235 */
softleaf_is_migration(softleaf_t entry)236 static inline bool softleaf_is_migration(softleaf_t entry)
237 {
238 switch (softleaf_type(entry)) {
239 case SOFTLEAF_MIGRATION_READ:
240 case SOFTLEAF_MIGRATION_READ_EXCLUSIVE:
241 case SOFTLEAF_MIGRATION_WRITE:
242 return true;
243 default:
244 return false;
245 }
246 }
247
248 /**
249 * softleaf_is_device_private_write() - Is this leaf entry a device private
250 * writable entry?
251 * @entry: Leaf entry.
252 *
253 * Returns: true if the leaf entry is a device private writable entry, otherwise
254 * false.
255 */
softleaf_is_device_private_write(softleaf_t entry)256 static inline bool softleaf_is_device_private_write(softleaf_t entry)
257 {
258 return softleaf_type(entry) == SOFTLEAF_DEVICE_PRIVATE_WRITE;
259 }
260
261 /**
262 * softleaf_is_device_private() - Is this leaf entry a device private entry?
263 * @entry: Leaf entry.
264 *
265 * Returns: true if the leaf entry is a device private entry, otherwise false.
266 */
softleaf_is_device_private(softleaf_t entry)267 static inline bool softleaf_is_device_private(softleaf_t entry)
268 {
269 switch (softleaf_type(entry)) {
270 case SOFTLEAF_DEVICE_PRIVATE_WRITE:
271 case SOFTLEAF_DEVICE_PRIVATE_READ:
272 return true;
273 default:
274 return false;
275 }
276 }
277
278 /**
279 * softleaf_is_device_exclusive() - Is this leaf entry a device-exclusive entry?
280 * @entry: Leaf entry.
281 *
282 * Returns: true if the leaf entry is a device-exclusive entry, otherwise false.
283 */
softleaf_is_device_exclusive(softleaf_t entry)284 static inline bool softleaf_is_device_exclusive(softleaf_t entry)
285 {
286 return softleaf_type(entry) == SOFTLEAF_DEVICE_EXCLUSIVE;
287 }
288
289 /**
290 * softleaf_is_hwpoison() - Is this leaf entry a hardware poison entry?
291 * @entry: Leaf entry.
292 *
293 * Returns: true if the leaf entry is a hardware poison entry, otherwise false.
294 */
softleaf_is_hwpoison(softleaf_t entry)295 static inline bool softleaf_is_hwpoison(softleaf_t entry)
296 {
297 return softleaf_type(entry) == SOFTLEAF_HWPOISON;
298 }
299
300 /**
301 * softleaf_is_marker() - Is this leaf entry a marker?
302 * @entry: Leaf entry.
303 *
304 * Returns: true if the leaf entry is a marker entry, otherwise false.
305 */
softleaf_is_marker(softleaf_t entry)306 static inline bool softleaf_is_marker(softleaf_t entry)
307 {
308 return softleaf_type(entry) == SOFTLEAF_MARKER;
309 }
310
311 /**
312 * softleaf_to_marker() - Obtain marker associated with leaf entry.
313 * @entry: Leaf entry, softleaf_is_marker(@entry) must return true.
314 *
315 * Returns: Marker associated with the leaf entry.
316 */
softleaf_to_marker(softleaf_t entry)317 static inline pte_marker softleaf_to_marker(softleaf_t entry)
318 {
319 VM_WARN_ON_ONCE(!softleaf_is_marker(entry));
320
321 return swp_offset(entry) & PTE_MARKER_MASK;
322 }
323
324 /**
325 * softleaf_has_pfn() - Does this leaf entry encode a valid PFN number?
326 * @entry: Leaf entry.
327 *
328 * A pfn swap entry is a special type of swap entry that always has a pfn stored
329 * in the swap offset. They can either be used to represent unaddressable device
330 * memory, to restrict access to a page undergoing migration or to represent a
331 * pfn which has been hwpoisoned and unmapped.
332 *
333 * Returns: true if the leaf entry encodes a PFN, otherwise false.
334 */
softleaf_has_pfn(softleaf_t entry)335 static inline bool softleaf_has_pfn(softleaf_t entry)
336 {
337 /* Make sure the swp offset can always store the needed fields. */
338 BUILD_BUG_ON(SWP_TYPE_SHIFT < SWP_PFN_BITS);
339
340 if (softleaf_is_migration(entry))
341 return true;
342 if (softleaf_is_device_private(entry))
343 return true;
344 if (softleaf_is_device_exclusive(entry))
345 return true;
346 if (softleaf_is_hwpoison(entry))
347 return true;
348
349 return false;
350 }
351
352 /**
353 * softleaf_to_pfn() - Obtain PFN encoded within leaf entry.
354 * @entry: Leaf entry, softleaf_has_pfn(@entry) must return true.
355 *
356 * Returns: The PFN associated with the leaf entry.
357 */
softleaf_to_pfn(softleaf_t entry)358 static inline unsigned long softleaf_to_pfn(softleaf_t entry)
359 {
360 VM_WARN_ON_ONCE(!softleaf_has_pfn(entry));
361
362 /* Temporary until swp_entry_t eliminated. */
363 return swp_offset(entry) & SWP_PFN_MASK;
364 }
365
366 /**
367 * softleaf_to_page() - Obtains struct page for PFN encoded within leaf entry.
368 * @entry: Leaf entry, softleaf_has_pfn(@entry) must return true.
369 *
370 * Returns: Pointer to the struct page associated with the leaf entry's PFN.
371 */
softleaf_to_page(softleaf_t entry)372 static inline struct page *softleaf_to_page(softleaf_t entry)
373 {
374 struct page *page = pfn_to_page(softleaf_to_pfn(entry));
375
376 VM_WARN_ON_ONCE(!softleaf_has_pfn(entry));
377 /*
378 * Any use of migration entries may only occur while the
379 * corresponding page is locked
380 */
381 VM_WARN_ON_ONCE(softleaf_is_migration(entry) && !PageLocked(page));
382
383 return page;
384 }
385
386 /**
387 * softleaf_to_folio() - Obtains struct folio for PFN encoded within leaf entry.
388 * @entry: Leaf entry, softleaf_has_pfn(@entry) must return true.
389 *
390 * Returns: Pointer to the struct folio associated with the leaf entry's PFN.
391 */
softleaf_to_folio(softleaf_t entry)392 static inline struct folio *softleaf_to_folio(softleaf_t entry)
393 {
394 struct folio *folio = pfn_folio(softleaf_to_pfn(entry));
395
396 VM_WARN_ON_ONCE(!softleaf_has_pfn(entry));
397 /*
398 * Any use of migration entries may only occur while the
399 * corresponding folio is locked.
400 */
401 VM_WARN_ON_ONCE(softleaf_is_migration(entry) &&
402 !folio_test_locked(folio));
403
404 return folio;
405 }
406
407 /**
408 * softleaf_is_poison_marker() - Is this leaf entry a poison marker?
409 * @entry: Leaf entry.
410 *
411 * The poison marker is set via UFFDIO_POISON. Userfaultfd-specific.
412 *
413 * Returns: true if the leaf entry is a poison marker, otherwise false.
414 */
softleaf_is_poison_marker(softleaf_t entry)415 static inline bool softleaf_is_poison_marker(softleaf_t entry)
416 {
417 if (!softleaf_is_marker(entry))
418 return false;
419
420 return softleaf_to_marker(entry) & PTE_MARKER_POISONED;
421 }
422
423 /**
424 * softleaf_is_guard_marker() - Is this leaf entry a guard region marker?
425 * @entry: Leaf entry.
426 *
427 * Returns: true if the leaf entry is a guard marker, otherwise false.
428 */
softleaf_is_guard_marker(softleaf_t entry)429 static inline bool softleaf_is_guard_marker(softleaf_t entry)
430 {
431 if (!softleaf_is_marker(entry))
432 return false;
433
434 return softleaf_to_marker(entry) & PTE_MARKER_GUARD;
435 }
436
437 /**
438 * softleaf_is_uffd_wp_marker() - Is this leaf entry a userfautlfd write protect
439 * marker?
440 * @entry: Leaf entry.
441 *
442 * Userfaultfd-specific.
443 *
444 * Returns: true if the leaf entry is a UFFD WP marker, otherwise false.
445 */
softleaf_is_uffd_wp_marker(softleaf_t entry)446 static inline bool softleaf_is_uffd_wp_marker(softleaf_t entry)
447 {
448 if (!softleaf_is_marker(entry))
449 return false;
450
451 return softleaf_to_marker(entry) & PTE_MARKER_UFFD_WP;
452 }
453
454 #ifdef CONFIG_MIGRATION
455
456 /**
457 * softleaf_is_migration_young() - Does this migration entry contain an accessed
458 * bit?
459 * @entry: Leaf entry.
460 *
461 * If the architecture can support storing A/D bits in migration entries, this
462 * determines whether the accessed (or 'young') bit was set on the migrated page
463 * table entry.
464 *
465 * Returns: true if the entry contains an accessed bit, otherwise false.
466 */
softleaf_is_migration_young(softleaf_t entry)467 static inline bool softleaf_is_migration_young(softleaf_t entry)
468 {
469 VM_WARN_ON_ONCE(!softleaf_is_migration(entry));
470
471 if (migration_entry_supports_ad())
472 return swp_offset(entry) & SWP_MIG_YOUNG;
473 /* Keep the old behavior of aging page after migration */
474 return false;
475 }
476
477 /**
478 * softleaf_is_migration_dirty() - Does this migration entry contain a dirty bit?
479 * @entry: Leaf entry.
480 *
481 * If the architecture can support storing A/D bits in migration entries, this
482 * determines whether the dirty bit was set on the migrated page table entry.
483 *
484 * Returns: true if the entry contains a dirty bit, otherwise false.
485 */
softleaf_is_migration_dirty(softleaf_t entry)486 static inline bool softleaf_is_migration_dirty(softleaf_t entry)
487 {
488 VM_WARN_ON_ONCE(!softleaf_is_migration(entry));
489
490 if (migration_entry_supports_ad())
491 return swp_offset(entry) & SWP_MIG_DIRTY;
492 /* Keep the old behavior of clean page after migration */
493 return false;
494 }
495
496 #else /* CONFIG_MIGRATION */
497
softleaf_is_migration_young(softleaf_t entry)498 static inline bool softleaf_is_migration_young(softleaf_t entry)
499 {
500 return false;
501 }
502
softleaf_is_migration_dirty(softleaf_t entry)503 static inline bool softleaf_is_migration_dirty(softleaf_t entry)
504 {
505 return false;
506 }
507 #endif /* CONFIG_MIGRATION */
508
509 /**
510 * pte_is_marker() - Does the PTE entry encode a marker leaf entry?
511 * @pte: PTE entry.
512 *
513 * Returns: true if this PTE is a marker leaf entry, otherwise false.
514 */
pte_is_marker(pte_t pte)515 static inline bool pte_is_marker(pte_t pte)
516 {
517 return softleaf_is_marker(softleaf_from_pte(pte));
518 }
519
520 /**
521 * pte_is_uffd_wp_marker() - Does this PTE entry encode a userfaultfd write
522 * protect marker leaf entry?
523 * @pte: PTE entry.
524 *
525 * Returns: true if this PTE is a UFFD WP marker leaf entry, otherwise false.
526 */
pte_is_uffd_wp_marker(pte_t pte)527 static inline bool pte_is_uffd_wp_marker(pte_t pte)
528 {
529 const softleaf_t entry = softleaf_from_pte(pte);
530
531 return softleaf_is_uffd_wp_marker(entry);
532 }
533
534 /**
535 * pte_is_uffd_marker() - Does this PTE entry encode a userfault-specific marker
536 * leaf entry?
537 * @entry: Leaf entry.
538 *
539 * It's useful to be able to determine which leaf entries encode UFFD-specific
540 * markers so we can handle these correctly.
541 *
542 * Returns: true if this PTE entry is a UFFD-specific marker, otherwise false.
543 */
pte_is_uffd_marker(pte_t pte)544 static inline bool pte_is_uffd_marker(pte_t pte)
545 {
546 const softleaf_t entry = softleaf_from_pte(pte);
547
548 if (!softleaf_is_marker(entry))
549 return false;
550
551 /* UFFD WP, poisoned swap entries are UFFD-handled. */
552 if (softleaf_is_uffd_wp_marker(entry))
553 return true;
554 if (softleaf_is_poison_marker(entry))
555 return true;
556
557 return false;
558 }
559
560 #if defined(CONFIG_ZONE_DEVICE) && defined(CONFIG_ARCH_ENABLE_THP_MIGRATION)
561
562 /**
563 * pmd_is_device_private_entry() - Check if PMD contains a device private swap
564 * entry.
565 * @pmd: The PMD to check.
566 *
567 * Returns true if the PMD contains a swap entry that represents a device private
568 * page mapping. This is used for zone device private pages that have been
569 * swapped out but still need special handling during various memory management
570 * operations.
571 *
572 * Return: true if PMD contains device private entry, false otherwise
573 */
pmd_is_device_private_entry(pmd_t pmd)574 static inline bool pmd_is_device_private_entry(pmd_t pmd)
575 {
576 return softleaf_is_device_private(softleaf_from_pmd(pmd));
577 }
578
579 #else /* CONFIG_ZONE_DEVICE && CONFIG_ARCH_ENABLE_THP_MIGRATION */
580
pmd_is_device_private_entry(pmd_t pmd)581 static inline bool pmd_is_device_private_entry(pmd_t pmd)
582 {
583 return false;
584 }
585
586 #endif /* CONFIG_ZONE_DEVICE && CONFIG_ARCH_ENABLE_THP_MIGRATION */
587
588 /**
589 * pmd_is_migration_entry() - Does this PMD entry encode a migration entry?
590 * @pmd: PMD entry.
591 *
592 * Returns: true if the PMD encodes a migration entry, otherwise false.
593 */
pmd_is_migration_entry(pmd_t pmd)594 static inline bool pmd_is_migration_entry(pmd_t pmd)
595 {
596 return softleaf_is_migration(softleaf_from_pmd(pmd));
597 }
598
599 /**
600 * pmd_is_valid_softleaf() - Is this PMD entry a valid leaf entry?
601 * @pmd: PMD entry.
602 *
603 * PMD leaf entries are valid only if they are device private or migration
604 * entries. This function asserts that a PMD leaf entry is valid in this
605 * respect.
606 *
607 * Returns: true if the PMD entry is a valid leaf entry, otherwise false.
608 */
pmd_is_valid_softleaf(pmd_t pmd)609 static inline bool pmd_is_valid_softleaf(pmd_t pmd)
610 {
611 const softleaf_t entry = softleaf_from_pmd(pmd);
612
613 /* Only device private, migration entries valid for PMD. */
614 return softleaf_is_device_private(entry) ||
615 softleaf_is_migration(entry);
616 }
617
618 #endif /* CONFIG_MMU */
619 #endif /* _LINUX_LEAFOPS_H */
620