1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3 * S390 version
4 * Copyright IBM Corp. 1999, 2000
5 * Author(s): Hartmut Penner (hp@de.ibm.com)
6 * Ulrich Weigand (weigand@de.ibm.com)
7 * Martin Schwidefsky (schwidefsky@de.ibm.com)
8 *
9 * Derived from "include/asm-i386/pgtable.h"
10 */
11
12 #ifndef _ASM_S390_PGTABLE_H
13 #define _ASM_S390_PGTABLE_H
14
15 #include <linux/sched.h>
16 #include <linux/mm_types.h>
17 #include <linux/page-flags.h>
18 #include <linux/radix-tree.h>
19 #include <linux/atomic.h>
20 #include <asm/ctlreg.h>
21 #include <asm/bug.h>
22 #include <asm/page.h>
23 #include <asm/uv.h>
24
25 extern pgd_t swapper_pg_dir[];
26 extern pgd_t invalid_pg_dir[];
27 extern void paging_init(void);
28 extern struct ctlreg s390_invalid_asce;
29
30 enum {
31 PG_DIRECT_MAP_4K = 0,
32 PG_DIRECT_MAP_1M,
33 PG_DIRECT_MAP_2G,
34 PG_DIRECT_MAP_MAX
35 };
36
37 extern atomic_long_t direct_pages_count[PG_DIRECT_MAP_MAX];
38
update_page_count(int level,long count)39 static inline void update_page_count(int level, long count)
40 {
41 if (IS_ENABLED(CONFIG_PROC_FS))
42 atomic_long_add(count, &direct_pages_count[level]);
43 }
44
45 /*
46 * The S390 doesn't have any external MMU info: the kernel page
47 * tables contain all the necessary information.
48 */
49 #define update_mmu_cache(vma, address, ptep) do { } while (0)
50 #define update_mmu_cache_range(vmf, vma, addr, ptep, nr) do { } while (0)
51 #define update_mmu_cache_pmd(vma, address, ptep) do { } while (0)
52
53 /*
54 * ZERO_PAGE is a global shared page that is always zero; used
55 * for zero-mapped memory areas etc..
56 */
57
58 extern unsigned long empty_zero_page;
59 extern unsigned long zero_page_mask;
60
61 #define ZERO_PAGE(vaddr) \
62 (virt_to_page((void *)(empty_zero_page + \
63 (((unsigned long)(vaddr)) &zero_page_mask))))
64 #define __HAVE_COLOR_ZERO_PAGE
65
66 /* TODO: s390 cannot support io_remap_pfn_range... */
67
68 #define pte_ERROR(e) \
69 pr_err("%s:%d: bad pte %016lx.\n", __FILE__, __LINE__, pte_val(e))
70 #define pmd_ERROR(e) \
71 pr_err("%s:%d: bad pmd %016lx.\n", __FILE__, __LINE__, pmd_val(e))
72 #define pud_ERROR(e) \
73 pr_err("%s:%d: bad pud %016lx.\n", __FILE__, __LINE__, pud_val(e))
74 #define p4d_ERROR(e) \
75 pr_err("%s:%d: bad p4d %016lx.\n", __FILE__, __LINE__, p4d_val(e))
76 #define pgd_ERROR(e) \
77 pr_err("%s:%d: bad pgd %016lx.\n", __FILE__, __LINE__, pgd_val(e))
78
79 /*
80 * The vmalloc and module area will always be on the topmost area of the
81 * kernel mapping. 512GB are reserved for vmalloc by default.
82 * At the top of the vmalloc area a 2GB area is reserved where modules
83 * will reside. That makes sure that inter module branches always
84 * happen without trampolines and in addition the placement within a
85 * 2GB frame is branch prediction unit friendly.
86 */
87 extern unsigned long VMALLOC_START;
88 extern unsigned long VMALLOC_END;
89 #define VMALLOC_DEFAULT_SIZE ((512UL << 30) - MODULES_LEN)
90 extern struct page *vmemmap;
91 extern unsigned long vmemmap_size;
92
93 extern unsigned long MODULES_VADDR;
94 extern unsigned long MODULES_END;
95 #define MODULES_VADDR MODULES_VADDR
96 #define MODULES_END MODULES_END
97 #define MODULES_LEN (1UL << 31)
98
is_module_addr(void * addr)99 static inline int is_module_addr(void *addr)
100 {
101 BUILD_BUG_ON(MODULES_LEN > (1UL << 31));
102 if (addr < (void *)MODULES_VADDR)
103 return 0;
104 if (addr > (void *)MODULES_END)
105 return 0;
106 return 1;
107 }
108
109 #ifdef CONFIG_KMSAN
110 #define KMSAN_VMALLOC_SIZE (VMALLOC_END - VMALLOC_START)
111 #define KMSAN_VMALLOC_SHADOW_START VMALLOC_END
112 #define KMSAN_VMALLOC_SHADOW_END (KMSAN_VMALLOC_SHADOW_START + KMSAN_VMALLOC_SIZE)
113 #define KMSAN_VMALLOC_ORIGIN_START KMSAN_VMALLOC_SHADOW_END
114 #define KMSAN_VMALLOC_ORIGIN_END (KMSAN_VMALLOC_ORIGIN_START + KMSAN_VMALLOC_SIZE)
115 #define KMSAN_MODULES_SHADOW_START KMSAN_VMALLOC_ORIGIN_END
116 #define KMSAN_MODULES_SHADOW_END (KMSAN_MODULES_SHADOW_START + MODULES_LEN)
117 #define KMSAN_MODULES_ORIGIN_START KMSAN_MODULES_SHADOW_END
118 #define KMSAN_MODULES_ORIGIN_END (KMSAN_MODULES_ORIGIN_START + MODULES_LEN)
119 #endif
120
121 #ifdef CONFIG_RANDOMIZE_BASE
122 #define KASLR_LEN (1UL << 31)
123 #else
124 #define KASLR_LEN 0UL
125 #endif
126
127 void setup_protection_map(void);
128
129 /*
130 * A 64 bit pagetable entry of S390 has following format:
131 * | PFRA |0IPC| OS |
132 * 0000000000111111111122222222223333333333444444444455555555556666
133 * 0123456789012345678901234567890123456789012345678901234567890123
134 *
135 * I Page-Invalid Bit: Page is not available for address-translation
136 * P Page-Protection Bit: Store access not possible for page
137 * C Change-bit override: HW is not required to set change bit
138 *
139 * A 64 bit segmenttable entry of S390 has following format:
140 * | P-table origin | TT
141 * 0000000000111111111122222222223333333333444444444455555555556666
142 * 0123456789012345678901234567890123456789012345678901234567890123
143 *
144 * I Segment-Invalid Bit: Segment is not available for address-translation
145 * C Common-Segment Bit: Segment is not private (PoP 3-30)
146 * P Page-Protection Bit: Store access not possible for page
147 * TT Type 00
148 *
149 * A 64 bit region table entry of S390 has following format:
150 * | S-table origin | TF TTTL
151 * 0000000000111111111122222222223333333333444444444455555555556666
152 * 0123456789012345678901234567890123456789012345678901234567890123
153 *
154 * I Segment-Invalid Bit: Segment is not available for address-translation
155 * TT Type 01
156 * TF
157 * TL Table length
158 *
159 * The 64 bit regiontable origin of S390 has following format:
160 * | region table origon | DTTL
161 * 0000000000111111111122222222223333333333444444444455555555556666
162 * 0123456789012345678901234567890123456789012345678901234567890123
163 *
164 * X Space-Switch event:
165 * G Segment-Invalid Bit:
166 * P Private-Space Bit:
167 * S Storage-Alteration:
168 * R Real space
169 * TL Table-Length:
170 *
171 * A storage key has the following format:
172 * | ACC |F|R|C|0|
173 * 0 3 4 5 6 7
174 * ACC: access key
175 * F : fetch protection bit
176 * R : referenced bit
177 * C : changed bit
178 */
179
180 /* Hardware bits in the page table entry */
181 #define _PAGE_NOEXEC 0x100 /* HW no-execute bit */
182 #define _PAGE_PROTECT 0x200 /* HW read-only bit */
183 #define _PAGE_INVALID 0x400 /* HW invalid bit */
184 #define _PAGE_LARGE 0x800 /* Bit to mark a large pte */
185
186 /* Software bits in the page table entry */
187 #define _PAGE_PRESENT 0x001 /* SW pte present bit */
188 #define _PAGE_YOUNG 0x004 /* SW pte young bit */
189 #define _PAGE_DIRTY 0x008 /* SW pte dirty bit */
190 #define _PAGE_READ 0x010 /* SW pte read bit */
191 #define _PAGE_WRITE 0x020 /* SW pte write bit */
192 #define _PAGE_SPECIAL 0x040 /* SW associated with special page */
193 #define _PAGE_UNUSED 0x080 /* SW bit for pgste usage state */
194
195 #ifdef CONFIG_MEM_SOFT_DIRTY
196 #define _PAGE_SOFT_DIRTY 0x002 /* SW pte soft dirty bit */
197 #else
198 #define _PAGE_SOFT_DIRTY 0x000
199 #endif
200
201 #define _PAGE_SW_BITS 0xffUL /* All SW bits */
202
203 #define _PAGE_SWP_EXCLUSIVE _PAGE_LARGE /* SW pte exclusive swap bit */
204
205 /* Set of bits not changed in pte_modify */
206 #define _PAGE_CHG_MASK (PAGE_MASK | _PAGE_SPECIAL | _PAGE_DIRTY | \
207 _PAGE_YOUNG | _PAGE_SOFT_DIRTY)
208
209 /*
210 * Mask of bits that must not be changed with RDP. Allow only _PAGE_PROTECT
211 * HW bit and all SW bits.
212 */
213 #define _PAGE_RDP_MASK ~(_PAGE_PROTECT | _PAGE_SW_BITS)
214
215 /*
216 * handle_pte_fault uses pte_present and pte_none to find out the pte type
217 * WITHOUT holding the page table lock. The _PAGE_PRESENT bit is used to
218 * distinguish present from not-present ptes. It is changed only with the page
219 * table lock held.
220 *
221 * The following table gives the different possible bit combinations for
222 * the pte hardware and software bits in the last 12 bits of a pte
223 * (. unassigned bit, x don't care, t swap type):
224 *
225 * 842100000000
226 * 000084210000
227 * 000000008421
228 * .IR.uswrdy.p
229 * empty .10.00000000
230 * swap .11..ttttt.0
231 * prot-none, clean, old .11.xx0000.1
232 * prot-none, clean, young .11.xx0001.1
233 * prot-none, dirty, old .11.xx0010.1
234 * prot-none, dirty, young .11.xx0011.1
235 * read-only, clean, old .11.xx0100.1
236 * read-only, clean, young .01.xx0101.1
237 * read-only, dirty, old .11.xx0110.1
238 * read-only, dirty, young .01.xx0111.1
239 * read-write, clean, old .11.xx1100.1
240 * read-write, clean, young .01.xx1101.1
241 * read-write, dirty, old .10.xx1110.1
242 * read-write, dirty, young .00.xx1111.1
243 * HW-bits: R read-only, I invalid
244 * SW-bits: p present, y young, d dirty, r read, w write, s special,
245 * u unused, l large
246 *
247 * pte_none is true for the bit pattern .10.00000000, pte == 0x400
248 * pte_swap is true for the bit pattern .11..ooooo.0, (pte & 0x201) == 0x200
249 * pte_present is true for the bit pattern .xx.xxxxxx.1, (pte & 0x001) == 0x001
250 */
251
252 /* Bits in the segment/region table address-space-control-element */
253 #define _ASCE_ORIGIN ~0xfffUL/* region/segment table origin */
254 #define _ASCE_PRIVATE_SPACE 0x100 /* private space control */
255 #define _ASCE_ALT_EVENT 0x80 /* storage alteration event control */
256 #define _ASCE_SPACE_SWITCH 0x40 /* space switch event */
257 #define _ASCE_REAL_SPACE 0x20 /* real space control */
258 #define _ASCE_TYPE_MASK 0x0c /* asce table type mask */
259 #define _ASCE_TYPE_REGION1 0x0c /* region first table type */
260 #define _ASCE_TYPE_REGION2 0x08 /* region second table type */
261 #define _ASCE_TYPE_REGION3 0x04 /* region third table type */
262 #define _ASCE_TYPE_SEGMENT 0x00 /* segment table type */
263 #define _ASCE_TABLE_LENGTH 0x03 /* region table length */
264
265 /* Bits in the region table entry */
266 #define _REGION_ENTRY_ORIGIN ~0xfffUL/* region/segment table origin */
267 #define _REGION_ENTRY_PROTECT 0x200 /* region protection bit */
268 #define _REGION_ENTRY_NOEXEC 0x100 /* region no-execute bit */
269 #define _REGION_ENTRY_OFFSET 0xc0 /* region table offset */
270 #define _REGION_ENTRY_INVALID 0x20 /* invalid region table entry */
271 #define _REGION_ENTRY_TYPE_MASK 0x0c /* region table type mask */
272 #define _REGION_ENTRY_TYPE_R1 0x0c /* region first table type */
273 #define _REGION_ENTRY_TYPE_R2 0x08 /* region second table type */
274 #define _REGION_ENTRY_TYPE_R3 0x04 /* region third table type */
275 #define _REGION_ENTRY_LENGTH 0x03 /* region third length */
276
277 #define _REGION1_ENTRY (_REGION_ENTRY_TYPE_R1 | _REGION_ENTRY_LENGTH)
278 #define _REGION1_ENTRY_EMPTY (_REGION_ENTRY_TYPE_R1 | _REGION_ENTRY_INVALID)
279 #define _REGION2_ENTRY (_REGION_ENTRY_TYPE_R2 | _REGION_ENTRY_LENGTH)
280 #define _REGION2_ENTRY_EMPTY (_REGION_ENTRY_TYPE_R2 | _REGION_ENTRY_INVALID)
281 #define _REGION3_ENTRY (_REGION_ENTRY_TYPE_R3 | _REGION_ENTRY_LENGTH | \
282 _REGION3_ENTRY_PRESENT)
283 #define _REGION3_ENTRY_EMPTY (_REGION_ENTRY_TYPE_R3 | _REGION_ENTRY_INVALID)
284
285 #define _REGION3_ENTRY_HARDWARE_BITS 0xfffffffffffff6ffUL
286 #define _REGION3_ENTRY_HARDWARE_BITS_LARGE 0xffffffff8001073cUL
287 #define _REGION3_ENTRY_ORIGIN_LARGE ~0x7fffffffUL /* large page address */
288 #define _REGION3_ENTRY_DIRTY 0x2000 /* SW region dirty bit */
289 #define _REGION3_ENTRY_YOUNG 0x1000 /* SW region young bit */
290 #define _REGION3_ENTRY_COMM 0x0010 /* Common-Region, marks swap entry */
291 #define _REGION3_ENTRY_LARGE 0x0400 /* RTTE-format control, large page */
292 #define _REGION3_ENTRY_WRITE 0x8000 /* SW region write bit */
293 #define _REGION3_ENTRY_READ 0x4000 /* SW region read bit */
294
295 #ifdef CONFIG_MEM_SOFT_DIRTY
296 #define _REGION3_ENTRY_SOFT_DIRTY 0x0002 /* SW region soft dirty bit */
297 #else
298 #define _REGION3_ENTRY_SOFT_DIRTY 0x0000 /* SW region soft dirty bit */
299 #endif
300
301 #define _REGION_ENTRY_BITS 0xfffffffffffff22fUL
302
303 /*
304 * SW region present bit. For non-leaf region-third-table entries, bits 62-63
305 * indicate the TABLE LENGTH and both must be set to 1. But such entries
306 * would always be considered as present, so it is safe to use bit 63 as
307 * PRESENT bit for PUD.
308 */
309 #define _REGION3_ENTRY_PRESENT 0x0001
310
311 /* Bits in the segment table entry */
312 #define _SEGMENT_ENTRY_BITS 0xfffffffffffffe3fUL
313 #define _SEGMENT_ENTRY_HARDWARE_BITS 0xfffffffffffffe3cUL
314 #define _SEGMENT_ENTRY_HARDWARE_BITS_LARGE 0xfffffffffff1073cUL
315 #define _SEGMENT_ENTRY_ORIGIN_LARGE ~0xfffffUL /* large page address */
316 #define _SEGMENT_ENTRY_ORIGIN ~0x7ffUL/* page table origin */
317 #define _SEGMENT_ENTRY_PROTECT 0x200 /* segment protection bit */
318 #define _SEGMENT_ENTRY_NOEXEC 0x100 /* segment no-execute bit */
319 #define _SEGMENT_ENTRY_INVALID 0x20 /* invalid segment table entry */
320 #define _SEGMENT_ENTRY_TYPE_MASK 0x0c /* segment table type mask */
321
322 #define _SEGMENT_ENTRY (_SEGMENT_ENTRY_PRESENT)
323 #define _SEGMENT_ENTRY_EMPTY (_SEGMENT_ENTRY_INVALID)
324
325 #define _SEGMENT_ENTRY_DIRTY 0x2000 /* SW segment dirty bit */
326 #define _SEGMENT_ENTRY_YOUNG 0x1000 /* SW segment young bit */
327
328 #define _SEGMENT_ENTRY_COMM 0x0010 /* Common-Segment, marks swap entry */
329 #define _SEGMENT_ENTRY_LARGE 0x0400 /* STE-format control, large page */
330 #define _SEGMENT_ENTRY_WRITE 0x8000 /* SW segment write bit */
331 #define _SEGMENT_ENTRY_READ 0x4000 /* SW segment read bit */
332
333 #ifdef CONFIG_MEM_SOFT_DIRTY
334 #define _SEGMENT_ENTRY_SOFT_DIRTY 0x0002 /* SW segment soft dirty bit */
335 #else
336 #define _SEGMENT_ENTRY_SOFT_DIRTY 0x0000 /* SW segment soft dirty bit */
337 #endif
338
339 #define _SEGMENT_ENTRY_PRESENT 0x0001 /* SW segment present bit */
340
341 /* Common bits in region and segment table entries, for swap entries */
342 #define _RST_ENTRY_COMM 0x0010 /* Common-Region/Segment, marks swap entry */
343 #define _RST_ENTRY_INVALID 0x0020 /* invalid region/segment table entry */
344
345 #define _CRST_ENTRIES 2048 /* number of region/segment table entries */
346 #define _PAGE_ENTRIES 256 /* number of page table entries */
347
348 #define _CRST_TABLE_SIZE (_CRST_ENTRIES * 8)
349 #define _PAGE_TABLE_SIZE (_PAGE_ENTRIES * 8)
350
351 #define _REGION1_SHIFT 53
352 #define _REGION2_SHIFT 42
353 #define _REGION3_SHIFT 31
354 #define _SEGMENT_SHIFT 20
355
356 #define _REGION1_INDEX (0x7ffUL << _REGION1_SHIFT)
357 #define _REGION2_INDEX (0x7ffUL << _REGION2_SHIFT)
358 #define _REGION3_INDEX (0x7ffUL << _REGION3_SHIFT)
359 #define _SEGMENT_INDEX (0x7ffUL << _SEGMENT_SHIFT)
360 #define _PAGE_INDEX (0xffUL << PAGE_SHIFT)
361
362 #define _REGION1_SIZE (1UL << _REGION1_SHIFT)
363 #define _REGION2_SIZE (1UL << _REGION2_SHIFT)
364 #define _REGION3_SIZE (1UL << _REGION3_SHIFT)
365 #define _SEGMENT_SIZE (1UL << _SEGMENT_SHIFT)
366
367 #define _REGION1_MASK (~(_REGION1_SIZE - 1))
368 #define _REGION2_MASK (~(_REGION2_SIZE - 1))
369 #define _REGION3_MASK (~(_REGION3_SIZE - 1))
370 #define _SEGMENT_MASK (~(_SEGMENT_SIZE - 1))
371
372 #define PMD_SHIFT _SEGMENT_SHIFT
373 #define PUD_SHIFT _REGION3_SHIFT
374 #define P4D_SHIFT _REGION2_SHIFT
375 #define PGDIR_SHIFT _REGION1_SHIFT
376
377 #define PMD_SIZE _SEGMENT_SIZE
378 #define PUD_SIZE _REGION3_SIZE
379 #define P4D_SIZE _REGION2_SIZE
380 #define PGDIR_SIZE _REGION1_SIZE
381
382 #define PMD_MASK _SEGMENT_MASK
383 #define PUD_MASK _REGION3_MASK
384 #define P4D_MASK _REGION2_MASK
385 #define PGDIR_MASK _REGION1_MASK
386
387 #define PTRS_PER_PTE _PAGE_ENTRIES
388 #define PTRS_PER_PMD _CRST_ENTRIES
389 #define PTRS_PER_PUD _CRST_ENTRIES
390 #define PTRS_PER_P4D _CRST_ENTRIES
391 #define PTRS_PER_PGD _CRST_ENTRIES
392
393 /*
394 * Segment table and region3 table entry encoding
395 * (R = read-only, I = invalid, y = young bit):
396 * dy..R...I...wr
397 * prot-none, clean, old 00..1...1...00
398 * prot-none, clean, young 01..1...1...00
399 * prot-none, dirty, old 10..1...1...00
400 * prot-none, dirty, young 11..1...1...00
401 * read-only, clean, old 00..1...1...01
402 * read-only, clean, young 01..1...0...01
403 * read-only, dirty, old 10..1...1...01
404 * read-only, dirty, young 11..1...0...01
405 * read-write, clean, old 00..1...1...11
406 * read-write, clean, young 01..1...0...11
407 * read-write, dirty, old 10..0...1...11
408 * read-write, dirty, young 11..0...0...11
409 * The segment table origin is used to distinguish empty (origin==0) from
410 * read-write, old segment table entries (origin!=0)
411 * HW-bits: R read-only, I invalid
412 * SW-bits: y young, d dirty, r read, w write
413 */
414
415 /* Page status table bits for virtualization */
416 #define PGSTE_ACC_BITS 0xf000000000000000UL
417 #define PGSTE_FP_BIT 0x0800000000000000UL
418 #define PGSTE_PCL_BIT 0x0080000000000000UL
419 #define PGSTE_HR_BIT 0x0040000000000000UL
420 #define PGSTE_HC_BIT 0x0020000000000000UL
421 #define PGSTE_GR_BIT 0x0004000000000000UL
422 #define PGSTE_GC_BIT 0x0002000000000000UL
423 #define PGSTE_ST2_MASK 0x0000ffff00000000UL
424 #define PGSTE_UC_BIT 0x0000000000008000UL /* user dirty (migration) */
425 #define PGSTE_IN_BIT 0x0000000000004000UL /* IPTE notify bit */
426 #define PGSTE_VSIE_BIT 0x0000000000002000UL /* ref'd in a shadow table */
427
428 /* Guest Page State used for virtualization */
429 #define _PGSTE_GPS_ZERO 0x0000000080000000UL
430 #define _PGSTE_GPS_NODAT 0x0000000040000000UL
431 #define _PGSTE_GPS_USAGE_MASK 0x0000000003000000UL
432 #define _PGSTE_GPS_USAGE_STABLE 0x0000000000000000UL
433 #define _PGSTE_GPS_USAGE_UNUSED 0x0000000001000000UL
434 #define _PGSTE_GPS_USAGE_POT_VOLATILE 0x0000000002000000UL
435 #define _PGSTE_GPS_USAGE_VOLATILE _PGSTE_GPS_USAGE_MASK
436
437 /*
438 * A user page table pointer has the space-switch-event bit, the
439 * private-space-control bit and the storage-alteration-event-control
440 * bit set. A kernel page table pointer doesn't need them.
441 */
442 #define _ASCE_USER_BITS (_ASCE_SPACE_SWITCH | _ASCE_PRIVATE_SPACE | \
443 _ASCE_ALT_EVENT)
444
445 /*
446 * Page protection definitions.
447 */
448 #define __PAGE_NONE (_PAGE_PRESENT | _PAGE_INVALID | _PAGE_PROTECT)
449 #define __PAGE_RO (_PAGE_PRESENT | _PAGE_READ | \
450 _PAGE_NOEXEC | _PAGE_INVALID | _PAGE_PROTECT)
451 #define __PAGE_RX (_PAGE_PRESENT | _PAGE_READ | \
452 _PAGE_INVALID | _PAGE_PROTECT)
453 #define __PAGE_RW (_PAGE_PRESENT | _PAGE_READ | _PAGE_WRITE | \
454 _PAGE_NOEXEC | _PAGE_INVALID | _PAGE_PROTECT)
455 #define __PAGE_RWX (_PAGE_PRESENT | _PAGE_READ | _PAGE_WRITE | \
456 _PAGE_INVALID | _PAGE_PROTECT)
457 #define __PAGE_SHARED (_PAGE_PRESENT | _PAGE_READ | _PAGE_WRITE | \
458 _PAGE_YOUNG | _PAGE_DIRTY | _PAGE_NOEXEC)
459 #define __PAGE_KERNEL (_PAGE_PRESENT | _PAGE_READ | _PAGE_WRITE | \
460 _PAGE_YOUNG | _PAGE_DIRTY | _PAGE_NOEXEC)
461 #define __PAGE_KERNEL_RO (_PAGE_PRESENT | _PAGE_READ | _PAGE_YOUNG | \
462 _PAGE_PROTECT | _PAGE_NOEXEC)
463
464 extern unsigned long page_noexec_mask;
465
466 #define __pgprot_page_mask(x) __pgprot((x) & page_noexec_mask)
467
468 #define PAGE_NONE __pgprot_page_mask(__PAGE_NONE)
469 #define PAGE_RO __pgprot_page_mask(__PAGE_RO)
470 #define PAGE_RX __pgprot_page_mask(__PAGE_RX)
471 #define PAGE_RW __pgprot_page_mask(__PAGE_RW)
472 #define PAGE_RWX __pgprot_page_mask(__PAGE_RWX)
473 #define PAGE_SHARED __pgprot_page_mask(__PAGE_SHARED)
474 #define PAGE_KERNEL __pgprot_page_mask(__PAGE_KERNEL)
475 #define PAGE_KERNEL_RO __pgprot_page_mask(__PAGE_KERNEL_RO)
476
477 /*
478 * Segment entry (large page) protection definitions.
479 */
480 #define __SEGMENT_NONE (_SEGMENT_ENTRY_PRESENT | \
481 _SEGMENT_ENTRY_INVALID | \
482 _SEGMENT_ENTRY_PROTECT)
483 #define __SEGMENT_RO (_SEGMENT_ENTRY_PRESENT | \
484 _SEGMENT_ENTRY_PROTECT | \
485 _SEGMENT_ENTRY_READ | \
486 _SEGMENT_ENTRY_NOEXEC)
487 #define __SEGMENT_RX (_SEGMENT_ENTRY_PRESENT | \
488 _SEGMENT_ENTRY_PROTECT | \
489 _SEGMENT_ENTRY_READ)
490 #define __SEGMENT_RW (_SEGMENT_ENTRY_PRESENT | \
491 _SEGMENT_ENTRY_READ | \
492 _SEGMENT_ENTRY_WRITE | \
493 _SEGMENT_ENTRY_NOEXEC)
494 #define __SEGMENT_RWX (_SEGMENT_ENTRY_PRESENT | \
495 _SEGMENT_ENTRY_READ | \
496 _SEGMENT_ENTRY_WRITE)
497 #define __SEGMENT_KERNEL (_SEGMENT_ENTRY | \
498 _SEGMENT_ENTRY_LARGE | \
499 _SEGMENT_ENTRY_READ | \
500 _SEGMENT_ENTRY_WRITE | \
501 _SEGMENT_ENTRY_YOUNG | \
502 _SEGMENT_ENTRY_DIRTY | \
503 _SEGMENT_ENTRY_NOEXEC)
504 #define __SEGMENT_KERNEL_RO (_SEGMENT_ENTRY | \
505 _SEGMENT_ENTRY_LARGE | \
506 _SEGMENT_ENTRY_READ | \
507 _SEGMENT_ENTRY_YOUNG | \
508 _SEGMENT_ENTRY_PROTECT | \
509 _SEGMENT_ENTRY_NOEXEC)
510
511 extern unsigned long segment_noexec_mask;
512
513 #define __pgprot_segment_mask(x) __pgprot((x) & segment_noexec_mask)
514
515 #define SEGMENT_NONE __pgprot_segment_mask(__SEGMENT_NONE)
516 #define SEGMENT_RO __pgprot_segment_mask(__SEGMENT_RO)
517 #define SEGMENT_RX __pgprot_segment_mask(__SEGMENT_RX)
518 #define SEGMENT_RW __pgprot_segment_mask(__SEGMENT_RW)
519 #define SEGMENT_RWX __pgprot_segment_mask(__SEGMENT_RWX)
520 #define SEGMENT_KERNEL __pgprot_segment_mask(__SEGMENT_KERNEL)
521 #define SEGMENT_KERNEL_RO __pgprot_segment_mask(__SEGMENT_KERNEL_RO)
522
523 /*
524 * Region3 entry (large page) protection definitions.
525 */
526
527 #define __REGION3_KERNEL (_REGION_ENTRY_TYPE_R3 | \
528 _REGION3_ENTRY_PRESENT | \
529 _REGION3_ENTRY_LARGE | \
530 _REGION3_ENTRY_READ | \
531 _REGION3_ENTRY_WRITE | \
532 _REGION3_ENTRY_YOUNG | \
533 _REGION3_ENTRY_DIRTY | \
534 _REGION_ENTRY_NOEXEC)
535 #define __REGION3_KERNEL_RO (_REGION_ENTRY_TYPE_R3 | \
536 _REGION3_ENTRY_PRESENT | \
537 _REGION3_ENTRY_LARGE | \
538 _REGION3_ENTRY_READ | \
539 _REGION3_ENTRY_YOUNG | \
540 _REGION_ENTRY_PROTECT | \
541 _REGION_ENTRY_NOEXEC)
542
543 extern unsigned long region_noexec_mask;
544
545 #define __pgprot_region_mask(x) __pgprot((x) & region_noexec_mask)
546
547 #define REGION3_KERNEL __pgprot_region_mask(__REGION3_KERNEL)
548 #define REGION3_KERNEL_RO __pgprot_region_mask(__REGION3_KERNEL_RO)
549
mm_p4d_folded(struct mm_struct * mm)550 static inline bool mm_p4d_folded(struct mm_struct *mm)
551 {
552 return mm->context.asce_limit <= _REGION1_SIZE;
553 }
554 #define mm_p4d_folded(mm) mm_p4d_folded(mm)
555
mm_pud_folded(struct mm_struct * mm)556 static inline bool mm_pud_folded(struct mm_struct *mm)
557 {
558 return mm->context.asce_limit <= _REGION2_SIZE;
559 }
560 #define mm_pud_folded(mm) mm_pud_folded(mm)
561
mm_pmd_folded(struct mm_struct * mm)562 static inline bool mm_pmd_folded(struct mm_struct *mm)
563 {
564 return mm->context.asce_limit <= _REGION3_SIZE;
565 }
566 #define mm_pmd_folded(mm) mm_pmd_folded(mm)
567
mm_has_pgste(struct mm_struct * mm)568 static inline int mm_has_pgste(struct mm_struct *mm)
569 {
570 #ifdef CONFIG_PGSTE
571 if (unlikely(mm->context.has_pgste))
572 return 1;
573 #endif
574 return 0;
575 }
576
mm_is_protected(struct mm_struct * mm)577 static inline int mm_is_protected(struct mm_struct *mm)
578 {
579 #ifdef CONFIG_PGSTE
580 if (unlikely(atomic_read(&mm->context.protected_count)))
581 return 1;
582 #endif
583 return 0;
584 }
585
mm_alloc_pgste(struct mm_struct * mm)586 static inline int mm_alloc_pgste(struct mm_struct *mm)
587 {
588 #ifdef CONFIG_PGSTE
589 if (unlikely(mm->context.alloc_pgste))
590 return 1;
591 #endif
592 return 0;
593 }
594
clear_pte_bit(pte_t pte,pgprot_t prot)595 static inline pte_t clear_pte_bit(pte_t pte, pgprot_t prot)
596 {
597 return __pte(pte_val(pte) & ~pgprot_val(prot));
598 }
599
set_pte_bit(pte_t pte,pgprot_t prot)600 static inline pte_t set_pte_bit(pte_t pte, pgprot_t prot)
601 {
602 return __pte(pte_val(pte) | pgprot_val(prot));
603 }
604
clear_pmd_bit(pmd_t pmd,pgprot_t prot)605 static inline pmd_t clear_pmd_bit(pmd_t pmd, pgprot_t prot)
606 {
607 return __pmd(pmd_val(pmd) & ~pgprot_val(prot));
608 }
609
set_pmd_bit(pmd_t pmd,pgprot_t prot)610 static inline pmd_t set_pmd_bit(pmd_t pmd, pgprot_t prot)
611 {
612 return __pmd(pmd_val(pmd) | pgprot_val(prot));
613 }
614
clear_pud_bit(pud_t pud,pgprot_t prot)615 static inline pud_t clear_pud_bit(pud_t pud, pgprot_t prot)
616 {
617 return __pud(pud_val(pud) & ~pgprot_val(prot));
618 }
619
set_pud_bit(pud_t pud,pgprot_t prot)620 static inline pud_t set_pud_bit(pud_t pud, pgprot_t prot)
621 {
622 return __pud(pud_val(pud) | pgprot_val(prot));
623 }
624
625 /*
626 * As soon as the guest uses storage keys or enables PV, we deduplicate all
627 * mapped shared zeropages and prevent new shared zeropages from getting
628 * mapped.
629 */
630 #define mm_forbids_zeropage mm_forbids_zeropage
mm_forbids_zeropage(struct mm_struct * mm)631 static inline int mm_forbids_zeropage(struct mm_struct *mm)
632 {
633 #ifdef CONFIG_PGSTE
634 if (!mm->context.allow_cow_sharing)
635 return 1;
636 #endif
637 return 0;
638 }
639
mm_uses_skeys(struct mm_struct * mm)640 static inline int mm_uses_skeys(struct mm_struct *mm)
641 {
642 #ifdef CONFIG_PGSTE
643 if (mm->context.uses_skeys)
644 return 1;
645 #endif
646 return 0;
647 }
648
csp(unsigned int * ptr,unsigned int old,unsigned int new)649 static inline void csp(unsigned int *ptr, unsigned int old, unsigned int new)
650 {
651 union register_pair r1 = { .even = old, .odd = new, };
652 unsigned long address = (unsigned long)ptr | 1;
653
654 asm volatile(
655 " csp %[r1],%[address]"
656 : [r1] "+&d" (r1.pair), "+m" (*ptr)
657 : [address] "d" (address)
658 : "cc");
659 }
660
661 /**
662 * cspg() - Compare and Swap and Purge (CSPG)
663 * @ptr: Pointer to the value to be exchanged
664 * @old: The expected old value
665 * @new: The new value
666 *
667 * Return: True if compare and swap was successful, otherwise false.
668 */
cspg(unsigned long * ptr,unsigned long old,unsigned long new)669 static inline bool cspg(unsigned long *ptr, unsigned long old, unsigned long new)
670 {
671 union register_pair r1 = { .even = old, .odd = new, };
672 unsigned long address = (unsigned long)ptr | 1;
673
674 asm volatile(
675 " cspg %[r1],%[address]"
676 : [r1] "+&d" (r1.pair), "+m" (*ptr)
677 : [address] "d" (address)
678 : "cc");
679 return old == r1.even;
680 }
681
682 #define CRDTE_DTT_PAGE 0x00UL
683 #define CRDTE_DTT_SEGMENT 0x10UL
684 #define CRDTE_DTT_REGION3 0x14UL
685 #define CRDTE_DTT_REGION2 0x18UL
686 #define CRDTE_DTT_REGION1 0x1cUL
687
688 /**
689 * crdte() - Compare and Replace DAT Table Entry
690 * @old: The expected old value
691 * @new: The new value
692 * @table: Pointer to the value to be exchanged
693 * @dtt: Table type of the table to be exchanged
694 * @address: The address mapped by the entry to be replaced
695 * @asce: The ASCE of this entry
696 *
697 * Return: True if compare and replace was successful, otherwise false.
698 */
crdte(unsigned long old,unsigned long new,unsigned long * table,unsigned long dtt,unsigned long address,unsigned long asce)699 static inline bool crdte(unsigned long old, unsigned long new,
700 unsigned long *table, unsigned long dtt,
701 unsigned long address, unsigned long asce)
702 {
703 union register_pair r1 = { .even = old, .odd = new, };
704 union register_pair r2 = { .even = __pa(table) | dtt, .odd = address, };
705
706 asm volatile(".insn rrf,0xb98f0000,%[r1],%[r2],%[asce],0"
707 : [r1] "+&d" (r1.pair)
708 : [r2] "d" (r2.pair), [asce] "a" (asce)
709 : "memory", "cc");
710 return old == r1.even;
711 }
712
713 /*
714 * pgd/p4d/pud/pmd/pte query functions
715 */
pgd_folded(pgd_t pgd)716 static inline int pgd_folded(pgd_t pgd)
717 {
718 return (pgd_val(pgd) & _REGION_ENTRY_TYPE_MASK) < _REGION_ENTRY_TYPE_R1;
719 }
720
pgd_present(pgd_t pgd)721 static inline int pgd_present(pgd_t pgd)
722 {
723 if (pgd_folded(pgd))
724 return 1;
725 return (pgd_val(pgd) & _REGION_ENTRY_ORIGIN) != 0UL;
726 }
727
pgd_none(pgd_t pgd)728 static inline int pgd_none(pgd_t pgd)
729 {
730 if (pgd_folded(pgd))
731 return 0;
732 return (pgd_val(pgd) & _REGION_ENTRY_INVALID) != 0UL;
733 }
734
pgd_bad(pgd_t pgd)735 static inline int pgd_bad(pgd_t pgd)
736 {
737 if ((pgd_val(pgd) & _REGION_ENTRY_TYPE_MASK) < _REGION_ENTRY_TYPE_R1)
738 return 0;
739 return (pgd_val(pgd) & ~_REGION_ENTRY_BITS) != 0;
740 }
741
pgd_pfn(pgd_t pgd)742 static inline unsigned long pgd_pfn(pgd_t pgd)
743 {
744 unsigned long origin_mask;
745
746 origin_mask = _REGION_ENTRY_ORIGIN;
747 return (pgd_val(pgd) & origin_mask) >> PAGE_SHIFT;
748 }
749
p4d_folded(p4d_t p4d)750 static inline int p4d_folded(p4d_t p4d)
751 {
752 return (p4d_val(p4d) & _REGION_ENTRY_TYPE_MASK) < _REGION_ENTRY_TYPE_R2;
753 }
754
p4d_present(p4d_t p4d)755 static inline int p4d_present(p4d_t p4d)
756 {
757 if (p4d_folded(p4d))
758 return 1;
759 return (p4d_val(p4d) & _REGION_ENTRY_ORIGIN) != 0UL;
760 }
761
p4d_none(p4d_t p4d)762 static inline int p4d_none(p4d_t p4d)
763 {
764 if (p4d_folded(p4d))
765 return 0;
766 return p4d_val(p4d) == _REGION2_ENTRY_EMPTY;
767 }
768
p4d_pfn(p4d_t p4d)769 static inline unsigned long p4d_pfn(p4d_t p4d)
770 {
771 unsigned long origin_mask;
772
773 origin_mask = _REGION_ENTRY_ORIGIN;
774 return (p4d_val(p4d) & origin_mask) >> PAGE_SHIFT;
775 }
776
pud_folded(pud_t pud)777 static inline int pud_folded(pud_t pud)
778 {
779 return (pud_val(pud) & _REGION_ENTRY_TYPE_MASK) < _REGION_ENTRY_TYPE_R3;
780 }
781
pud_present(pud_t pud)782 static inline int pud_present(pud_t pud)
783 {
784 if (pud_folded(pud))
785 return 1;
786 return (pud_val(pud) & _REGION3_ENTRY_PRESENT) != 0;
787 }
788
pud_none(pud_t pud)789 static inline int pud_none(pud_t pud)
790 {
791 if (pud_folded(pud))
792 return 0;
793 return pud_val(pud) == _REGION3_ENTRY_EMPTY;
794 }
795
796 #define pud_leaf pud_leaf
pud_leaf(pud_t pud)797 static inline bool pud_leaf(pud_t pud)
798 {
799 if ((pud_val(pud) & _REGION_ENTRY_TYPE_MASK) != _REGION_ENTRY_TYPE_R3)
800 return 0;
801 return (pud_present(pud) && (pud_val(pud) & _REGION3_ENTRY_LARGE) != 0);
802 }
803
pmd_present(pmd_t pmd)804 static inline int pmd_present(pmd_t pmd)
805 {
806 return (pmd_val(pmd) & _SEGMENT_ENTRY_PRESENT) != 0;
807 }
808
809 #define pmd_leaf pmd_leaf
pmd_leaf(pmd_t pmd)810 static inline bool pmd_leaf(pmd_t pmd)
811 {
812 return (pmd_present(pmd) && (pmd_val(pmd) & _SEGMENT_ENTRY_LARGE) != 0);
813 }
814
pmd_bad(pmd_t pmd)815 static inline int pmd_bad(pmd_t pmd)
816 {
817 if ((pmd_val(pmd) & _SEGMENT_ENTRY_TYPE_MASK) > 0 || pmd_leaf(pmd))
818 return 1;
819 return (pmd_val(pmd) & ~_SEGMENT_ENTRY_BITS) != 0;
820 }
821
pud_bad(pud_t pud)822 static inline int pud_bad(pud_t pud)
823 {
824 unsigned long type = pud_val(pud) & _REGION_ENTRY_TYPE_MASK;
825
826 if (type > _REGION_ENTRY_TYPE_R3 || pud_leaf(pud))
827 return 1;
828 if (type < _REGION_ENTRY_TYPE_R3)
829 return 0;
830 return (pud_val(pud) & ~_REGION_ENTRY_BITS) != 0;
831 }
832
p4d_bad(p4d_t p4d)833 static inline int p4d_bad(p4d_t p4d)
834 {
835 unsigned long type = p4d_val(p4d) & _REGION_ENTRY_TYPE_MASK;
836
837 if (type > _REGION_ENTRY_TYPE_R2)
838 return 1;
839 if (type < _REGION_ENTRY_TYPE_R2)
840 return 0;
841 return (p4d_val(p4d) & ~_REGION_ENTRY_BITS) != 0;
842 }
843
pmd_none(pmd_t pmd)844 static inline int pmd_none(pmd_t pmd)
845 {
846 return pmd_val(pmd) == _SEGMENT_ENTRY_EMPTY;
847 }
848
849 #define pmd_write pmd_write
pmd_write(pmd_t pmd)850 static inline int pmd_write(pmd_t pmd)
851 {
852 return (pmd_val(pmd) & _SEGMENT_ENTRY_WRITE) != 0;
853 }
854
855 #define pud_write pud_write
pud_write(pud_t pud)856 static inline int pud_write(pud_t pud)
857 {
858 return (pud_val(pud) & _REGION3_ENTRY_WRITE) != 0;
859 }
860
861 #define pmd_dirty pmd_dirty
pmd_dirty(pmd_t pmd)862 static inline int pmd_dirty(pmd_t pmd)
863 {
864 return (pmd_val(pmd) & _SEGMENT_ENTRY_DIRTY) != 0;
865 }
866
867 #define pmd_young pmd_young
pmd_young(pmd_t pmd)868 static inline int pmd_young(pmd_t pmd)
869 {
870 return (pmd_val(pmd) & _SEGMENT_ENTRY_YOUNG) != 0;
871 }
872
pte_present(pte_t pte)873 static inline int pte_present(pte_t pte)
874 {
875 /* Bit pattern: (pte & 0x001) == 0x001 */
876 return (pte_val(pte) & _PAGE_PRESENT) != 0;
877 }
878
pte_none(pte_t pte)879 static inline int pte_none(pte_t pte)
880 {
881 /* Bit pattern: pte == 0x400 */
882 return pte_val(pte) == _PAGE_INVALID;
883 }
884
pte_swap(pte_t pte)885 static inline int pte_swap(pte_t pte)
886 {
887 /* Bit pattern: (pte & 0x201) == 0x200 */
888 return (pte_val(pte) & (_PAGE_PROTECT | _PAGE_PRESENT))
889 == _PAGE_PROTECT;
890 }
891
pte_special(pte_t pte)892 static inline int pte_special(pte_t pte)
893 {
894 return (pte_val(pte) & _PAGE_SPECIAL);
895 }
896
897 #define __HAVE_ARCH_PTE_SAME
pte_same(pte_t a,pte_t b)898 static inline int pte_same(pte_t a, pte_t b)
899 {
900 return pte_val(a) == pte_val(b);
901 }
902
903 #ifdef CONFIG_NUMA_BALANCING
pte_protnone(pte_t pte)904 static inline int pte_protnone(pte_t pte)
905 {
906 return pte_present(pte) && !(pte_val(pte) & _PAGE_READ);
907 }
908
pmd_protnone(pmd_t pmd)909 static inline int pmd_protnone(pmd_t pmd)
910 {
911 /* pmd_leaf(pmd) implies pmd_present(pmd) */
912 return pmd_leaf(pmd) && !(pmd_val(pmd) & _SEGMENT_ENTRY_READ);
913 }
914 #endif
915
pte_swp_exclusive(pte_t pte)916 static inline int pte_swp_exclusive(pte_t pte)
917 {
918 return pte_val(pte) & _PAGE_SWP_EXCLUSIVE;
919 }
920
pte_swp_mkexclusive(pte_t pte)921 static inline pte_t pte_swp_mkexclusive(pte_t pte)
922 {
923 return set_pte_bit(pte, __pgprot(_PAGE_SWP_EXCLUSIVE));
924 }
925
pte_swp_clear_exclusive(pte_t pte)926 static inline pte_t pte_swp_clear_exclusive(pte_t pte)
927 {
928 return clear_pte_bit(pte, __pgprot(_PAGE_SWP_EXCLUSIVE));
929 }
930
pte_soft_dirty(pte_t pte)931 static inline int pte_soft_dirty(pte_t pte)
932 {
933 return pte_val(pte) & _PAGE_SOFT_DIRTY;
934 }
935 #define pte_swp_soft_dirty pte_soft_dirty
936
pte_mksoft_dirty(pte_t pte)937 static inline pte_t pte_mksoft_dirty(pte_t pte)
938 {
939 return set_pte_bit(pte, __pgprot(_PAGE_SOFT_DIRTY));
940 }
941 #define pte_swp_mksoft_dirty pte_mksoft_dirty
942
pte_clear_soft_dirty(pte_t pte)943 static inline pte_t pte_clear_soft_dirty(pte_t pte)
944 {
945 return clear_pte_bit(pte, __pgprot(_PAGE_SOFT_DIRTY));
946 }
947 #define pte_swp_clear_soft_dirty pte_clear_soft_dirty
948
pmd_soft_dirty(pmd_t pmd)949 static inline int pmd_soft_dirty(pmd_t pmd)
950 {
951 return pmd_val(pmd) & _SEGMENT_ENTRY_SOFT_DIRTY;
952 }
953
pmd_mksoft_dirty(pmd_t pmd)954 static inline pmd_t pmd_mksoft_dirty(pmd_t pmd)
955 {
956 return set_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_SOFT_DIRTY));
957 }
958
pmd_clear_soft_dirty(pmd_t pmd)959 static inline pmd_t pmd_clear_soft_dirty(pmd_t pmd)
960 {
961 return clear_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_SOFT_DIRTY));
962 }
963
964 /*
965 * query functions pte_write/pte_dirty/pte_young only work if
966 * pte_present() is true. Undefined behaviour if not..
967 */
pte_write(pte_t pte)968 static inline int pte_write(pte_t pte)
969 {
970 return (pte_val(pte) & _PAGE_WRITE) != 0;
971 }
972
pte_dirty(pte_t pte)973 static inline int pte_dirty(pte_t pte)
974 {
975 return (pte_val(pte) & _PAGE_DIRTY) != 0;
976 }
977
pte_young(pte_t pte)978 static inline int pte_young(pte_t pte)
979 {
980 return (pte_val(pte) & _PAGE_YOUNG) != 0;
981 }
982
983 #define __HAVE_ARCH_PTE_UNUSED
pte_unused(pte_t pte)984 static inline int pte_unused(pte_t pte)
985 {
986 return pte_val(pte) & _PAGE_UNUSED;
987 }
988
989 /*
990 * Extract the pgprot value from the given pte while at the same time making it
991 * usable for kernel address space mappings where fault driven dirty and
992 * young/old accounting is not supported, i.e _PAGE_PROTECT and _PAGE_INVALID
993 * must not be set.
994 */
995 #define pte_pgprot pte_pgprot
pte_pgprot(pte_t pte)996 static inline pgprot_t pte_pgprot(pte_t pte)
997 {
998 unsigned long pte_flags = pte_val(pte) & _PAGE_CHG_MASK;
999
1000 if (pte_write(pte))
1001 pte_flags |= pgprot_val(PAGE_KERNEL);
1002 else
1003 pte_flags |= pgprot_val(PAGE_KERNEL_RO);
1004 pte_flags |= pte_val(pte) & mio_wb_bit_mask;
1005
1006 return __pgprot(pte_flags);
1007 }
1008
1009 /*
1010 * pgd/pmd/pte modification functions
1011 */
1012
set_pgd(pgd_t * pgdp,pgd_t pgd)1013 static inline void set_pgd(pgd_t *pgdp, pgd_t pgd)
1014 {
1015 WRITE_ONCE(*pgdp, pgd);
1016 }
1017
set_p4d(p4d_t * p4dp,p4d_t p4d)1018 static inline void set_p4d(p4d_t *p4dp, p4d_t p4d)
1019 {
1020 WRITE_ONCE(*p4dp, p4d);
1021 }
1022
set_pud(pud_t * pudp,pud_t pud)1023 static inline void set_pud(pud_t *pudp, pud_t pud)
1024 {
1025 WRITE_ONCE(*pudp, pud);
1026 }
1027
set_pmd(pmd_t * pmdp,pmd_t pmd)1028 static inline void set_pmd(pmd_t *pmdp, pmd_t pmd)
1029 {
1030 WRITE_ONCE(*pmdp, pmd);
1031 }
1032
set_pte(pte_t * ptep,pte_t pte)1033 static inline void set_pte(pte_t *ptep, pte_t pte)
1034 {
1035 WRITE_ONCE(*ptep, pte);
1036 }
1037
pgd_clear(pgd_t * pgd)1038 static inline void pgd_clear(pgd_t *pgd)
1039 {
1040 if ((pgd_val(*pgd) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R1)
1041 set_pgd(pgd, __pgd(_REGION1_ENTRY_EMPTY));
1042 }
1043
p4d_clear(p4d_t * p4d)1044 static inline void p4d_clear(p4d_t *p4d)
1045 {
1046 if ((p4d_val(*p4d) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R2)
1047 set_p4d(p4d, __p4d(_REGION2_ENTRY_EMPTY));
1048 }
1049
pud_clear(pud_t * pud)1050 static inline void pud_clear(pud_t *pud)
1051 {
1052 if ((pud_val(*pud) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3)
1053 set_pud(pud, __pud(_REGION3_ENTRY_EMPTY));
1054 }
1055
pmd_clear(pmd_t * pmdp)1056 static inline void pmd_clear(pmd_t *pmdp)
1057 {
1058 set_pmd(pmdp, __pmd(_SEGMENT_ENTRY_EMPTY));
1059 }
1060
pte_clear(struct mm_struct * mm,unsigned long addr,pte_t * ptep)1061 static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
1062 {
1063 set_pte(ptep, __pte(_PAGE_INVALID));
1064 }
1065
1066 /*
1067 * The following pte modification functions only work if
1068 * pte_present() is true. Undefined behaviour if not..
1069 */
pte_modify(pte_t pte,pgprot_t newprot)1070 static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
1071 {
1072 pte = clear_pte_bit(pte, __pgprot(~_PAGE_CHG_MASK));
1073 pte = set_pte_bit(pte, newprot);
1074 /*
1075 * newprot for PAGE_NONE, PAGE_RO, PAGE_RX, PAGE_RW and PAGE_RWX
1076 * has the invalid bit set, clear it again for readable, young pages
1077 */
1078 if ((pte_val(pte) & _PAGE_YOUNG) && (pte_val(pte) & _PAGE_READ))
1079 pte = clear_pte_bit(pte, __pgprot(_PAGE_INVALID));
1080 /*
1081 * newprot for PAGE_RO, PAGE_RX, PAGE_RW and PAGE_RWX has the page
1082 * protection bit set, clear it again for writable, dirty pages
1083 */
1084 if ((pte_val(pte) & _PAGE_DIRTY) && (pte_val(pte) & _PAGE_WRITE))
1085 pte = clear_pte_bit(pte, __pgprot(_PAGE_PROTECT));
1086 return pte;
1087 }
1088
pte_wrprotect(pte_t pte)1089 static inline pte_t pte_wrprotect(pte_t pte)
1090 {
1091 pte = clear_pte_bit(pte, __pgprot(_PAGE_WRITE));
1092 return set_pte_bit(pte, __pgprot(_PAGE_PROTECT));
1093 }
1094
pte_mkwrite_novma(pte_t pte)1095 static inline pte_t pte_mkwrite_novma(pte_t pte)
1096 {
1097 pte = set_pte_bit(pte, __pgprot(_PAGE_WRITE));
1098 if (pte_val(pte) & _PAGE_DIRTY)
1099 pte = clear_pte_bit(pte, __pgprot(_PAGE_PROTECT));
1100 return pte;
1101 }
1102
pte_mkclean(pte_t pte)1103 static inline pte_t pte_mkclean(pte_t pte)
1104 {
1105 pte = clear_pte_bit(pte, __pgprot(_PAGE_DIRTY));
1106 return set_pte_bit(pte, __pgprot(_PAGE_PROTECT));
1107 }
1108
pte_mkdirty(pte_t pte)1109 static inline pte_t pte_mkdirty(pte_t pte)
1110 {
1111 pte = set_pte_bit(pte, __pgprot(_PAGE_DIRTY | _PAGE_SOFT_DIRTY));
1112 if (pte_val(pte) & _PAGE_WRITE)
1113 pte = clear_pte_bit(pte, __pgprot(_PAGE_PROTECT));
1114 return pte;
1115 }
1116
pte_mkold(pte_t pte)1117 static inline pte_t pte_mkold(pte_t pte)
1118 {
1119 pte = clear_pte_bit(pte, __pgprot(_PAGE_YOUNG));
1120 return set_pte_bit(pte, __pgprot(_PAGE_INVALID));
1121 }
1122
pte_mkyoung(pte_t pte)1123 static inline pte_t pte_mkyoung(pte_t pte)
1124 {
1125 pte = set_pte_bit(pte, __pgprot(_PAGE_YOUNG));
1126 if (pte_val(pte) & _PAGE_READ)
1127 pte = clear_pte_bit(pte, __pgprot(_PAGE_INVALID));
1128 return pte;
1129 }
1130
pte_mkspecial(pte_t pte)1131 static inline pte_t pte_mkspecial(pte_t pte)
1132 {
1133 return set_pte_bit(pte, __pgprot(_PAGE_SPECIAL));
1134 }
1135
1136 #ifdef CONFIG_HUGETLB_PAGE
pte_mkhuge(pte_t pte)1137 static inline pte_t pte_mkhuge(pte_t pte)
1138 {
1139 return set_pte_bit(pte, __pgprot(_PAGE_LARGE));
1140 }
1141 #endif
1142
1143 #define IPTE_GLOBAL 0
1144 #define IPTE_LOCAL 1
1145
1146 #define IPTE_NODAT 0x400
1147 #define IPTE_GUEST_ASCE 0x800
1148
__ptep_rdp(unsigned long addr,pte_t * ptep,unsigned long opt,unsigned long asce,int local)1149 static __always_inline void __ptep_rdp(unsigned long addr, pte_t *ptep,
1150 unsigned long opt, unsigned long asce,
1151 int local)
1152 {
1153 unsigned long pto;
1154
1155 pto = __pa(ptep) & ~(PTRS_PER_PTE * sizeof(pte_t) - 1);
1156 asm volatile(".insn rrf,0xb98b0000,%[r1],%[r2],%[asce],%[m4]"
1157 : "+m" (*ptep)
1158 : [r1] "a" (pto), [r2] "a" ((addr & PAGE_MASK) | opt),
1159 [asce] "a" (asce), [m4] "i" (local));
1160 }
1161
__ptep_ipte(unsigned long address,pte_t * ptep,unsigned long opt,unsigned long asce,int local)1162 static __always_inline void __ptep_ipte(unsigned long address, pte_t *ptep,
1163 unsigned long opt, unsigned long asce,
1164 int local)
1165 {
1166 unsigned long pto = __pa(ptep);
1167
1168 if (__builtin_constant_p(opt) && opt == 0) {
1169 /* Invalidation + TLB flush for the pte */
1170 asm volatile(
1171 " ipte %[r1],%[r2],0,%[m4]"
1172 : "+m" (*ptep) : [r1] "a" (pto), [r2] "a" (address),
1173 [m4] "i" (local));
1174 return;
1175 }
1176
1177 /* Invalidate ptes with options + TLB flush of the ptes */
1178 opt = opt | (asce & _ASCE_ORIGIN);
1179 asm volatile(
1180 " ipte %[r1],%[r2],%[r3],%[m4]"
1181 : [r2] "+a" (address), [r3] "+a" (opt)
1182 : [r1] "a" (pto), [m4] "i" (local) : "memory");
1183 }
1184
__ptep_ipte_range(unsigned long address,int nr,pte_t * ptep,int local)1185 static __always_inline void __ptep_ipte_range(unsigned long address, int nr,
1186 pte_t *ptep, int local)
1187 {
1188 unsigned long pto = __pa(ptep);
1189
1190 /* Invalidate a range of ptes + TLB flush of the ptes */
1191 do {
1192 asm volatile(
1193 " ipte %[r1],%[r2],%[r3],%[m4]"
1194 : [r2] "+a" (address), [r3] "+a" (nr)
1195 : [r1] "a" (pto), [m4] "i" (local) : "memory");
1196 } while (nr != 255);
1197 }
1198
1199 /*
1200 * This is hard to understand. ptep_get_and_clear and ptep_clear_flush
1201 * both clear the TLB for the unmapped pte. The reason is that
1202 * ptep_get_and_clear is used in common code (e.g. change_pte_range)
1203 * to modify an active pte. The sequence is
1204 * 1) ptep_get_and_clear
1205 * 2) set_pte_at
1206 * 3) flush_tlb_range
1207 * On s390 the tlb needs to get flushed with the modification of the pte
1208 * if the pte is active. The only way how this can be implemented is to
1209 * have ptep_get_and_clear do the tlb flush. In exchange flush_tlb_range
1210 * is a nop.
1211 */
1212 pte_t ptep_xchg_direct(struct mm_struct *, unsigned long, pte_t *, pte_t);
1213 pte_t ptep_xchg_lazy(struct mm_struct *, unsigned long, pte_t *, pte_t);
1214
1215 #define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
ptep_test_and_clear_young(struct vm_area_struct * vma,unsigned long addr,pte_t * ptep)1216 static inline int ptep_test_and_clear_young(struct vm_area_struct *vma,
1217 unsigned long addr, pte_t *ptep)
1218 {
1219 pte_t pte = *ptep;
1220
1221 pte = ptep_xchg_direct(vma->vm_mm, addr, ptep, pte_mkold(pte));
1222 return pte_young(pte);
1223 }
1224
1225 #define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
ptep_clear_flush_young(struct vm_area_struct * vma,unsigned long address,pte_t * ptep)1226 static inline int ptep_clear_flush_young(struct vm_area_struct *vma,
1227 unsigned long address, pte_t *ptep)
1228 {
1229 return ptep_test_and_clear_young(vma, address, ptep);
1230 }
1231
1232 #define __HAVE_ARCH_PTEP_GET_AND_CLEAR
ptep_get_and_clear(struct mm_struct * mm,unsigned long addr,pte_t * ptep)1233 static inline pte_t ptep_get_and_clear(struct mm_struct *mm,
1234 unsigned long addr, pte_t *ptep)
1235 {
1236 pte_t res;
1237
1238 res = ptep_xchg_lazy(mm, addr, ptep, __pte(_PAGE_INVALID));
1239 /* At this point the reference through the mapping is still present */
1240 if (mm_is_protected(mm) && pte_present(res))
1241 uv_convert_from_secure_pte(res);
1242 return res;
1243 }
1244
1245 #define __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION
1246 pte_t ptep_modify_prot_start(struct vm_area_struct *, unsigned long, pte_t *);
1247 void ptep_modify_prot_commit(struct vm_area_struct *, unsigned long,
1248 pte_t *, pte_t, pte_t);
1249
1250 #define __HAVE_ARCH_PTEP_CLEAR_FLUSH
ptep_clear_flush(struct vm_area_struct * vma,unsigned long addr,pte_t * ptep)1251 static inline pte_t ptep_clear_flush(struct vm_area_struct *vma,
1252 unsigned long addr, pte_t *ptep)
1253 {
1254 pte_t res;
1255
1256 res = ptep_xchg_direct(vma->vm_mm, addr, ptep, __pte(_PAGE_INVALID));
1257 /* At this point the reference through the mapping is still present */
1258 if (mm_is_protected(vma->vm_mm) && pte_present(res))
1259 uv_convert_from_secure_pte(res);
1260 return res;
1261 }
1262
1263 /*
1264 * The batched pte unmap code uses ptep_get_and_clear_full to clear the
1265 * ptes. Here an optimization is possible. tlb_gather_mmu flushes all
1266 * tlbs of an mm if it can guarantee that the ptes of the mm_struct
1267 * cannot be accessed while the batched unmap is running. In this case
1268 * full==1 and a simple pte_clear is enough. See tlb.h.
1269 */
1270 #define __HAVE_ARCH_PTEP_GET_AND_CLEAR_FULL
ptep_get_and_clear_full(struct mm_struct * mm,unsigned long addr,pte_t * ptep,int full)1271 static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm,
1272 unsigned long addr,
1273 pte_t *ptep, int full)
1274 {
1275 pte_t res;
1276
1277 if (full) {
1278 res = *ptep;
1279 set_pte(ptep, __pte(_PAGE_INVALID));
1280 } else {
1281 res = ptep_xchg_lazy(mm, addr, ptep, __pte(_PAGE_INVALID));
1282 }
1283 /* Nothing to do */
1284 if (!mm_is_protected(mm) || !pte_present(res))
1285 return res;
1286 /*
1287 * At this point the reference through the mapping is still present.
1288 * The notifier should have destroyed all protected vCPUs at this
1289 * point, so the destroy should be successful.
1290 */
1291 if (full && !uv_destroy_pte(res))
1292 return res;
1293 /*
1294 * If something went wrong and the page could not be destroyed, or
1295 * if this is not a mm teardown, the slower export is used as
1296 * fallback instead.
1297 */
1298 uv_convert_from_secure_pte(res);
1299 return res;
1300 }
1301
1302 #define __HAVE_ARCH_PTEP_SET_WRPROTECT
ptep_set_wrprotect(struct mm_struct * mm,unsigned long addr,pte_t * ptep)1303 static inline void ptep_set_wrprotect(struct mm_struct *mm,
1304 unsigned long addr, pte_t *ptep)
1305 {
1306 pte_t pte = *ptep;
1307
1308 if (pte_write(pte))
1309 ptep_xchg_lazy(mm, addr, ptep, pte_wrprotect(pte));
1310 }
1311
1312 /*
1313 * Check if PTEs only differ in _PAGE_PROTECT HW bit, but also allow SW PTE
1314 * bits in the comparison. Those might change e.g. because of dirty and young
1315 * tracking.
1316 */
pte_allow_rdp(pte_t old,pte_t new)1317 static inline int pte_allow_rdp(pte_t old, pte_t new)
1318 {
1319 /*
1320 * Only allow changes from RO to RW
1321 */
1322 if (!(pte_val(old) & _PAGE_PROTECT) || pte_val(new) & _PAGE_PROTECT)
1323 return 0;
1324
1325 return (pte_val(old) & _PAGE_RDP_MASK) == (pte_val(new) & _PAGE_RDP_MASK);
1326 }
1327
flush_tlb_fix_spurious_fault(struct vm_area_struct * vma,unsigned long address,pte_t * ptep)1328 static inline void flush_tlb_fix_spurious_fault(struct vm_area_struct *vma,
1329 unsigned long address,
1330 pte_t *ptep)
1331 {
1332 /*
1333 * RDP might not have propagated the PTE protection reset to all CPUs,
1334 * so there could be spurious TLB protection faults.
1335 * NOTE: This will also be called when a racing pagetable update on
1336 * another thread already installed the correct PTE. Both cases cannot
1337 * really be distinguished.
1338 * Therefore, only do the local TLB flush when RDP can be used, and the
1339 * PTE does not have _PAGE_PROTECT set, to avoid unnecessary overhead.
1340 * A local RDP can be used to do the flush.
1341 */
1342 if (MACHINE_HAS_RDP && !(pte_val(*ptep) & _PAGE_PROTECT))
1343 __ptep_rdp(address, ptep, 0, 0, 1);
1344 }
1345 #define flush_tlb_fix_spurious_fault flush_tlb_fix_spurious_fault
1346
1347 void ptep_reset_dat_prot(struct mm_struct *mm, unsigned long addr, pte_t *ptep,
1348 pte_t new);
1349
1350 #define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
ptep_set_access_flags(struct vm_area_struct * vma,unsigned long addr,pte_t * ptep,pte_t entry,int dirty)1351 static inline int ptep_set_access_flags(struct vm_area_struct *vma,
1352 unsigned long addr, pte_t *ptep,
1353 pte_t entry, int dirty)
1354 {
1355 if (pte_same(*ptep, entry))
1356 return 0;
1357 if (MACHINE_HAS_RDP && !mm_has_pgste(vma->vm_mm) && pte_allow_rdp(*ptep, entry))
1358 ptep_reset_dat_prot(vma->vm_mm, addr, ptep, entry);
1359 else
1360 ptep_xchg_direct(vma->vm_mm, addr, ptep, entry);
1361 return 1;
1362 }
1363
1364 /*
1365 * Additional functions to handle KVM guest page tables
1366 */
1367 void ptep_set_pte_at(struct mm_struct *mm, unsigned long addr,
1368 pte_t *ptep, pte_t entry);
1369 void ptep_set_notify(struct mm_struct *mm, unsigned long addr, pte_t *ptep);
1370 void ptep_notify(struct mm_struct *mm, unsigned long addr,
1371 pte_t *ptep, unsigned long bits);
1372 int ptep_force_prot(struct mm_struct *mm, unsigned long gaddr,
1373 pte_t *ptep, int prot, unsigned long bit);
1374 void ptep_zap_unused(struct mm_struct *mm, unsigned long addr,
1375 pte_t *ptep , int reset);
1376 void ptep_zap_key(struct mm_struct *mm, unsigned long addr, pte_t *ptep);
1377 int ptep_shadow_pte(struct mm_struct *mm, unsigned long saddr,
1378 pte_t *sptep, pte_t *tptep, pte_t pte);
1379 void ptep_unshadow_pte(struct mm_struct *mm, unsigned long saddr, pte_t *ptep);
1380
1381 bool ptep_test_and_clear_uc(struct mm_struct *mm, unsigned long address,
1382 pte_t *ptep);
1383 int set_guest_storage_key(struct mm_struct *mm, unsigned long addr,
1384 unsigned char key, bool nq);
1385 int cond_set_guest_storage_key(struct mm_struct *mm, unsigned long addr,
1386 unsigned char key, unsigned char *oldkey,
1387 bool nq, bool mr, bool mc);
1388 int reset_guest_reference_bit(struct mm_struct *mm, unsigned long addr);
1389 int get_guest_storage_key(struct mm_struct *mm, unsigned long addr,
1390 unsigned char *key);
1391
1392 int set_pgste_bits(struct mm_struct *mm, unsigned long addr,
1393 unsigned long bits, unsigned long value);
1394 int get_pgste(struct mm_struct *mm, unsigned long hva, unsigned long *pgstep);
1395 int pgste_perform_essa(struct mm_struct *mm, unsigned long hva, int orc,
1396 unsigned long *oldpte, unsigned long *oldpgste);
1397 void gmap_pmdp_csp(struct mm_struct *mm, unsigned long vmaddr);
1398 void gmap_pmdp_invalidate(struct mm_struct *mm, unsigned long vmaddr);
1399 void gmap_pmdp_idte_local(struct mm_struct *mm, unsigned long vmaddr);
1400 void gmap_pmdp_idte_global(struct mm_struct *mm, unsigned long vmaddr);
1401
1402 #define pgprot_writecombine pgprot_writecombine
1403 pgprot_t pgprot_writecombine(pgprot_t prot);
1404
1405 #define pgprot_writethrough pgprot_writethrough
1406 pgprot_t pgprot_writethrough(pgprot_t prot);
1407
1408 #define PFN_PTE_SHIFT PAGE_SHIFT
1409
1410 /*
1411 * Set multiple PTEs to consecutive pages with a single call. All PTEs
1412 * are within the same folio, PMD and VMA.
1413 */
set_ptes(struct mm_struct * mm,unsigned long addr,pte_t * ptep,pte_t entry,unsigned int nr)1414 static inline void set_ptes(struct mm_struct *mm, unsigned long addr,
1415 pte_t *ptep, pte_t entry, unsigned int nr)
1416 {
1417 if (pte_present(entry))
1418 entry = clear_pte_bit(entry, __pgprot(_PAGE_UNUSED));
1419 if (mm_has_pgste(mm)) {
1420 for (;;) {
1421 ptep_set_pte_at(mm, addr, ptep, entry);
1422 if (--nr == 0)
1423 break;
1424 ptep++;
1425 entry = __pte(pte_val(entry) + PAGE_SIZE);
1426 addr += PAGE_SIZE;
1427 }
1428 } else {
1429 for (;;) {
1430 set_pte(ptep, entry);
1431 if (--nr == 0)
1432 break;
1433 ptep++;
1434 entry = __pte(pte_val(entry) + PAGE_SIZE);
1435 }
1436 }
1437 }
1438 #define set_ptes set_ptes
1439
1440 /*
1441 * Conversion functions: convert a page and protection to a page entry,
1442 * and a page entry and page directory to the page they refer to.
1443 */
mk_pte_phys(unsigned long physpage,pgprot_t pgprot)1444 static inline pte_t mk_pte_phys(unsigned long physpage, pgprot_t pgprot)
1445 {
1446 pte_t __pte;
1447
1448 __pte = __pte(physpage | pgprot_val(pgprot));
1449 return pte_mkyoung(__pte);
1450 }
1451
mk_pte(struct page * page,pgprot_t pgprot)1452 static inline pte_t mk_pte(struct page *page, pgprot_t pgprot)
1453 {
1454 unsigned long physpage = page_to_phys(page);
1455 pte_t __pte = mk_pte_phys(physpage, pgprot);
1456
1457 if (pte_write(__pte) && PageDirty(page))
1458 __pte = pte_mkdirty(__pte);
1459 return __pte;
1460 }
1461
1462 #define pgd_index(address) (((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD-1))
1463 #define p4d_index(address) (((address) >> P4D_SHIFT) & (PTRS_PER_P4D-1))
1464 #define pud_index(address) (((address) >> PUD_SHIFT) & (PTRS_PER_PUD-1))
1465 #define pmd_index(address) (((address) >> PMD_SHIFT) & (PTRS_PER_PMD-1))
1466
1467 #define p4d_deref(pud) ((unsigned long)__va(p4d_val(pud) & _REGION_ENTRY_ORIGIN))
1468 #define pgd_deref(pgd) ((unsigned long)__va(pgd_val(pgd) & _REGION_ENTRY_ORIGIN))
1469
pmd_deref(pmd_t pmd)1470 static inline unsigned long pmd_deref(pmd_t pmd)
1471 {
1472 unsigned long origin_mask;
1473
1474 origin_mask = _SEGMENT_ENTRY_ORIGIN;
1475 if (pmd_leaf(pmd))
1476 origin_mask = _SEGMENT_ENTRY_ORIGIN_LARGE;
1477 return (unsigned long)__va(pmd_val(pmd) & origin_mask);
1478 }
1479
pmd_pfn(pmd_t pmd)1480 static inline unsigned long pmd_pfn(pmd_t pmd)
1481 {
1482 return __pa(pmd_deref(pmd)) >> PAGE_SHIFT;
1483 }
1484
pud_deref(pud_t pud)1485 static inline unsigned long pud_deref(pud_t pud)
1486 {
1487 unsigned long origin_mask;
1488
1489 origin_mask = _REGION_ENTRY_ORIGIN;
1490 if (pud_leaf(pud))
1491 origin_mask = _REGION3_ENTRY_ORIGIN_LARGE;
1492 return (unsigned long)__va(pud_val(pud) & origin_mask);
1493 }
1494
1495 #define pud_pfn pud_pfn
pud_pfn(pud_t pud)1496 static inline unsigned long pud_pfn(pud_t pud)
1497 {
1498 return __pa(pud_deref(pud)) >> PAGE_SHIFT;
1499 }
1500
1501 /*
1502 * The pgd_offset function *always* adds the index for the top-level
1503 * region/segment table. This is done to get a sequence like the
1504 * following to work:
1505 * pgdp = pgd_offset(current->mm, addr);
1506 * pgd = READ_ONCE(*pgdp);
1507 * p4dp = p4d_offset(&pgd, addr);
1508 * ...
1509 * The subsequent p4d_offset, pud_offset and pmd_offset functions
1510 * only add an index if they dereferenced the pointer.
1511 */
pgd_offset_raw(pgd_t * pgd,unsigned long address)1512 static inline pgd_t *pgd_offset_raw(pgd_t *pgd, unsigned long address)
1513 {
1514 unsigned long rste;
1515 unsigned int shift;
1516
1517 /* Get the first entry of the top level table */
1518 rste = pgd_val(*pgd);
1519 /* Pick up the shift from the table type of the first entry */
1520 shift = ((rste & _REGION_ENTRY_TYPE_MASK) >> 2) * 11 + 20;
1521 return pgd + ((address >> shift) & (PTRS_PER_PGD - 1));
1522 }
1523
1524 #define pgd_offset(mm, address) pgd_offset_raw(READ_ONCE((mm)->pgd), address)
1525
p4d_offset_lockless(pgd_t * pgdp,pgd_t pgd,unsigned long address)1526 static inline p4d_t *p4d_offset_lockless(pgd_t *pgdp, pgd_t pgd, unsigned long address)
1527 {
1528 if ((pgd_val(pgd) & _REGION_ENTRY_TYPE_MASK) >= _REGION_ENTRY_TYPE_R1)
1529 return (p4d_t *) pgd_deref(pgd) + p4d_index(address);
1530 return (p4d_t *) pgdp;
1531 }
1532 #define p4d_offset_lockless p4d_offset_lockless
1533
p4d_offset(pgd_t * pgdp,unsigned long address)1534 static inline p4d_t *p4d_offset(pgd_t *pgdp, unsigned long address)
1535 {
1536 return p4d_offset_lockless(pgdp, *pgdp, address);
1537 }
1538
pud_offset_lockless(p4d_t * p4dp,p4d_t p4d,unsigned long address)1539 static inline pud_t *pud_offset_lockless(p4d_t *p4dp, p4d_t p4d, unsigned long address)
1540 {
1541 if ((p4d_val(p4d) & _REGION_ENTRY_TYPE_MASK) >= _REGION_ENTRY_TYPE_R2)
1542 return (pud_t *) p4d_deref(p4d) + pud_index(address);
1543 return (pud_t *) p4dp;
1544 }
1545 #define pud_offset_lockless pud_offset_lockless
1546
pud_offset(p4d_t * p4dp,unsigned long address)1547 static inline pud_t *pud_offset(p4d_t *p4dp, unsigned long address)
1548 {
1549 return pud_offset_lockless(p4dp, *p4dp, address);
1550 }
1551 #define pud_offset pud_offset
1552
pmd_offset_lockless(pud_t * pudp,pud_t pud,unsigned long address)1553 static inline pmd_t *pmd_offset_lockless(pud_t *pudp, pud_t pud, unsigned long address)
1554 {
1555 if ((pud_val(pud) & _REGION_ENTRY_TYPE_MASK) >= _REGION_ENTRY_TYPE_R3)
1556 return (pmd_t *) pud_deref(pud) + pmd_index(address);
1557 return (pmd_t *) pudp;
1558 }
1559 #define pmd_offset_lockless pmd_offset_lockless
1560
pmd_offset(pud_t * pudp,unsigned long address)1561 static inline pmd_t *pmd_offset(pud_t *pudp, unsigned long address)
1562 {
1563 return pmd_offset_lockless(pudp, *pudp, address);
1564 }
1565 #define pmd_offset pmd_offset
1566
pmd_page_vaddr(pmd_t pmd)1567 static inline unsigned long pmd_page_vaddr(pmd_t pmd)
1568 {
1569 return (unsigned long) pmd_deref(pmd);
1570 }
1571
gup_fast_permitted(unsigned long start,unsigned long end)1572 static inline bool gup_fast_permitted(unsigned long start, unsigned long end)
1573 {
1574 return end <= current->mm->context.asce_limit;
1575 }
1576 #define gup_fast_permitted gup_fast_permitted
1577
1578 #define pfn_pte(pfn, pgprot) mk_pte_phys(((pfn) << PAGE_SHIFT), (pgprot))
1579 #define pte_pfn(x) (pte_val(x) >> PAGE_SHIFT)
1580 #define pte_page(x) pfn_to_page(pte_pfn(x))
1581
1582 #define pmd_page(pmd) pfn_to_page(pmd_pfn(pmd))
1583 #define pud_page(pud) pfn_to_page(pud_pfn(pud))
1584 #define p4d_page(p4d) pfn_to_page(p4d_pfn(p4d))
1585 #define pgd_page(pgd) pfn_to_page(pgd_pfn(pgd))
1586
pmd_wrprotect(pmd_t pmd)1587 static inline pmd_t pmd_wrprotect(pmd_t pmd)
1588 {
1589 pmd = clear_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_WRITE));
1590 return set_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_PROTECT));
1591 }
1592
pmd_mkwrite_novma(pmd_t pmd)1593 static inline pmd_t pmd_mkwrite_novma(pmd_t pmd)
1594 {
1595 pmd = set_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_WRITE));
1596 if (pmd_val(pmd) & _SEGMENT_ENTRY_DIRTY)
1597 pmd = clear_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_PROTECT));
1598 return pmd;
1599 }
1600
pmd_mkclean(pmd_t pmd)1601 static inline pmd_t pmd_mkclean(pmd_t pmd)
1602 {
1603 pmd = clear_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_DIRTY));
1604 return set_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_PROTECT));
1605 }
1606
pmd_mkdirty(pmd_t pmd)1607 static inline pmd_t pmd_mkdirty(pmd_t pmd)
1608 {
1609 pmd = set_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_DIRTY | _SEGMENT_ENTRY_SOFT_DIRTY));
1610 if (pmd_val(pmd) & _SEGMENT_ENTRY_WRITE)
1611 pmd = clear_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_PROTECT));
1612 return pmd;
1613 }
1614
pud_wrprotect(pud_t pud)1615 static inline pud_t pud_wrprotect(pud_t pud)
1616 {
1617 pud = clear_pud_bit(pud, __pgprot(_REGION3_ENTRY_WRITE));
1618 return set_pud_bit(pud, __pgprot(_REGION_ENTRY_PROTECT));
1619 }
1620
pud_mkwrite(pud_t pud)1621 static inline pud_t pud_mkwrite(pud_t pud)
1622 {
1623 pud = set_pud_bit(pud, __pgprot(_REGION3_ENTRY_WRITE));
1624 if (pud_val(pud) & _REGION3_ENTRY_DIRTY)
1625 pud = clear_pud_bit(pud, __pgprot(_REGION_ENTRY_PROTECT));
1626 return pud;
1627 }
1628
pud_mkclean(pud_t pud)1629 static inline pud_t pud_mkclean(pud_t pud)
1630 {
1631 pud = clear_pud_bit(pud, __pgprot(_REGION3_ENTRY_DIRTY));
1632 return set_pud_bit(pud, __pgprot(_REGION_ENTRY_PROTECT));
1633 }
1634
pud_mkdirty(pud_t pud)1635 static inline pud_t pud_mkdirty(pud_t pud)
1636 {
1637 pud = set_pud_bit(pud, __pgprot(_REGION3_ENTRY_DIRTY | _REGION3_ENTRY_SOFT_DIRTY));
1638 if (pud_val(pud) & _REGION3_ENTRY_WRITE)
1639 pud = clear_pud_bit(pud, __pgprot(_REGION_ENTRY_PROTECT));
1640 return pud;
1641 }
1642
1643 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLB_PAGE)
massage_pgprot_pmd(pgprot_t pgprot)1644 static inline unsigned long massage_pgprot_pmd(pgprot_t pgprot)
1645 {
1646 /*
1647 * pgprot is PAGE_NONE, PAGE_RO, PAGE_RX, PAGE_RW or PAGE_RWX
1648 * (see __Pxxx / __Sxxx). Convert to segment table entry format.
1649 */
1650 if (pgprot_val(pgprot) == pgprot_val(PAGE_NONE))
1651 return pgprot_val(SEGMENT_NONE);
1652 if (pgprot_val(pgprot) == pgprot_val(PAGE_RO))
1653 return pgprot_val(SEGMENT_RO);
1654 if (pgprot_val(pgprot) == pgprot_val(PAGE_RX))
1655 return pgprot_val(SEGMENT_RX);
1656 if (pgprot_val(pgprot) == pgprot_val(PAGE_RW))
1657 return pgprot_val(SEGMENT_RW);
1658 return pgprot_val(SEGMENT_RWX);
1659 }
1660
pmd_mkyoung(pmd_t pmd)1661 static inline pmd_t pmd_mkyoung(pmd_t pmd)
1662 {
1663 pmd = set_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_YOUNG));
1664 if (pmd_val(pmd) & _SEGMENT_ENTRY_READ)
1665 pmd = clear_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_INVALID));
1666 return pmd;
1667 }
1668
pmd_mkold(pmd_t pmd)1669 static inline pmd_t pmd_mkold(pmd_t pmd)
1670 {
1671 pmd = clear_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_YOUNG));
1672 return set_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_INVALID));
1673 }
1674
pmd_modify(pmd_t pmd,pgprot_t newprot)1675 static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot)
1676 {
1677 unsigned long mask;
1678
1679 mask = _SEGMENT_ENTRY_ORIGIN_LARGE;
1680 mask |= _SEGMENT_ENTRY_DIRTY;
1681 mask |= _SEGMENT_ENTRY_YOUNG;
1682 mask |= _SEGMENT_ENTRY_LARGE;
1683 mask |= _SEGMENT_ENTRY_SOFT_DIRTY;
1684 pmd = __pmd(pmd_val(pmd) & mask);
1685 pmd = set_pmd_bit(pmd, __pgprot(massage_pgprot_pmd(newprot)));
1686 if (!(pmd_val(pmd) & _SEGMENT_ENTRY_DIRTY))
1687 pmd = set_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_PROTECT));
1688 if (!(pmd_val(pmd) & _SEGMENT_ENTRY_YOUNG))
1689 pmd = set_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_INVALID));
1690 return pmd;
1691 }
1692
mk_pmd_phys(unsigned long physpage,pgprot_t pgprot)1693 static inline pmd_t mk_pmd_phys(unsigned long physpage, pgprot_t pgprot)
1694 {
1695 return __pmd(physpage + massage_pgprot_pmd(pgprot));
1696 }
1697
1698 #endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLB_PAGE */
1699
__pmdp_csp(pmd_t * pmdp)1700 static inline void __pmdp_csp(pmd_t *pmdp)
1701 {
1702 csp((unsigned int *)pmdp + 1, pmd_val(*pmdp),
1703 pmd_val(*pmdp) | _SEGMENT_ENTRY_INVALID);
1704 }
1705
1706 #define IDTE_GLOBAL 0
1707 #define IDTE_LOCAL 1
1708
1709 #define IDTE_PTOA 0x0800
1710 #define IDTE_NODAT 0x1000
1711 #define IDTE_GUEST_ASCE 0x2000
1712
__pmdp_idte(unsigned long addr,pmd_t * pmdp,unsigned long opt,unsigned long asce,int local)1713 static __always_inline void __pmdp_idte(unsigned long addr, pmd_t *pmdp,
1714 unsigned long opt, unsigned long asce,
1715 int local)
1716 {
1717 unsigned long sto;
1718
1719 sto = __pa(pmdp) - pmd_index(addr) * sizeof(pmd_t);
1720 if (__builtin_constant_p(opt) && opt == 0) {
1721 /* flush without guest asce */
1722 asm volatile(
1723 " idte %[r1],0,%[r2],%[m4]"
1724 : "+m" (*pmdp)
1725 : [r1] "a" (sto), [r2] "a" ((addr & HPAGE_MASK)),
1726 [m4] "i" (local)
1727 : "cc" );
1728 } else {
1729 /* flush with guest asce */
1730 asm volatile(
1731 " idte %[r1],%[r3],%[r2],%[m4]"
1732 : "+m" (*pmdp)
1733 : [r1] "a" (sto), [r2] "a" ((addr & HPAGE_MASK) | opt),
1734 [r3] "a" (asce), [m4] "i" (local)
1735 : "cc" );
1736 }
1737 }
1738
__pudp_idte(unsigned long addr,pud_t * pudp,unsigned long opt,unsigned long asce,int local)1739 static __always_inline void __pudp_idte(unsigned long addr, pud_t *pudp,
1740 unsigned long opt, unsigned long asce,
1741 int local)
1742 {
1743 unsigned long r3o;
1744
1745 r3o = __pa(pudp) - pud_index(addr) * sizeof(pud_t);
1746 r3o |= _ASCE_TYPE_REGION3;
1747 if (__builtin_constant_p(opt) && opt == 0) {
1748 /* flush without guest asce */
1749 asm volatile(
1750 " idte %[r1],0,%[r2],%[m4]"
1751 : "+m" (*pudp)
1752 : [r1] "a" (r3o), [r2] "a" ((addr & PUD_MASK)),
1753 [m4] "i" (local)
1754 : "cc");
1755 } else {
1756 /* flush with guest asce */
1757 asm volatile(
1758 " idte %[r1],%[r3],%[r2],%[m4]"
1759 : "+m" (*pudp)
1760 : [r1] "a" (r3o), [r2] "a" ((addr & PUD_MASK) | opt),
1761 [r3] "a" (asce), [m4] "i" (local)
1762 : "cc" );
1763 }
1764 }
1765
1766 pmd_t pmdp_xchg_direct(struct mm_struct *, unsigned long, pmd_t *, pmd_t);
1767 pmd_t pmdp_xchg_lazy(struct mm_struct *, unsigned long, pmd_t *, pmd_t);
1768 pud_t pudp_xchg_direct(struct mm_struct *, unsigned long, pud_t *, pud_t);
1769
1770 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1771
1772 #define __HAVE_ARCH_PGTABLE_DEPOSIT
1773 void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
1774 pgtable_t pgtable);
1775
1776 #define __HAVE_ARCH_PGTABLE_WITHDRAW
1777 pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp);
1778
1779 #define __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS
pmdp_set_access_flags(struct vm_area_struct * vma,unsigned long addr,pmd_t * pmdp,pmd_t entry,int dirty)1780 static inline int pmdp_set_access_flags(struct vm_area_struct *vma,
1781 unsigned long addr, pmd_t *pmdp,
1782 pmd_t entry, int dirty)
1783 {
1784 VM_BUG_ON(addr & ~HPAGE_MASK);
1785
1786 entry = pmd_mkyoung(entry);
1787 if (dirty)
1788 entry = pmd_mkdirty(entry);
1789 if (pmd_val(*pmdp) == pmd_val(entry))
1790 return 0;
1791 pmdp_xchg_direct(vma->vm_mm, addr, pmdp, entry);
1792 return 1;
1793 }
1794
1795 #define __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG
pmdp_test_and_clear_young(struct vm_area_struct * vma,unsigned long addr,pmd_t * pmdp)1796 static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma,
1797 unsigned long addr, pmd_t *pmdp)
1798 {
1799 pmd_t pmd = *pmdp;
1800
1801 pmd = pmdp_xchg_direct(vma->vm_mm, addr, pmdp, pmd_mkold(pmd));
1802 return pmd_young(pmd);
1803 }
1804
1805 #define __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH
pmdp_clear_flush_young(struct vm_area_struct * vma,unsigned long addr,pmd_t * pmdp)1806 static inline int pmdp_clear_flush_young(struct vm_area_struct *vma,
1807 unsigned long addr, pmd_t *pmdp)
1808 {
1809 VM_BUG_ON(addr & ~HPAGE_MASK);
1810 return pmdp_test_and_clear_young(vma, addr, pmdp);
1811 }
1812
set_pmd_at(struct mm_struct * mm,unsigned long addr,pmd_t * pmdp,pmd_t entry)1813 static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr,
1814 pmd_t *pmdp, pmd_t entry)
1815 {
1816 set_pmd(pmdp, entry);
1817 }
1818
pmd_mkhuge(pmd_t pmd)1819 static inline pmd_t pmd_mkhuge(pmd_t pmd)
1820 {
1821 pmd = set_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_LARGE));
1822 pmd = set_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_YOUNG));
1823 return set_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_PROTECT));
1824 }
1825
1826 #define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR
pmdp_huge_get_and_clear(struct mm_struct * mm,unsigned long addr,pmd_t * pmdp)1827 static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm,
1828 unsigned long addr, pmd_t *pmdp)
1829 {
1830 return pmdp_xchg_direct(mm, addr, pmdp, __pmd(_SEGMENT_ENTRY_EMPTY));
1831 }
1832
1833 #define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR_FULL
pmdp_huge_get_and_clear_full(struct vm_area_struct * vma,unsigned long addr,pmd_t * pmdp,int full)1834 static inline pmd_t pmdp_huge_get_and_clear_full(struct vm_area_struct *vma,
1835 unsigned long addr,
1836 pmd_t *pmdp, int full)
1837 {
1838 if (full) {
1839 pmd_t pmd = *pmdp;
1840 set_pmd(pmdp, __pmd(_SEGMENT_ENTRY_EMPTY));
1841 return pmd;
1842 }
1843 return pmdp_xchg_lazy(vma->vm_mm, addr, pmdp, __pmd(_SEGMENT_ENTRY_EMPTY));
1844 }
1845
1846 #define __HAVE_ARCH_PMDP_HUGE_CLEAR_FLUSH
pmdp_huge_clear_flush(struct vm_area_struct * vma,unsigned long addr,pmd_t * pmdp)1847 static inline pmd_t pmdp_huge_clear_flush(struct vm_area_struct *vma,
1848 unsigned long addr, pmd_t *pmdp)
1849 {
1850 return pmdp_huge_get_and_clear(vma->vm_mm, addr, pmdp);
1851 }
1852
1853 #define __HAVE_ARCH_PMDP_INVALIDATE
pmdp_invalidate(struct vm_area_struct * vma,unsigned long addr,pmd_t * pmdp)1854 static inline pmd_t pmdp_invalidate(struct vm_area_struct *vma,
1855 unsigned long addr, pmd_t *pmdp)
1856 {
1857 pmd_t pmd;
1858
1859 VM_WARN_ON_ONCE(!pmd_present(*pmdp));
1860 pmd = __pmd(pmd_val(*pmdp) | _SEGMENT_ENTRY_INVALID);
1861 return pmdp_xchg_direct(vma->vm_mm, addr, pmdp, pmd);
1862 }
1863
1864 #define __HAVE_ARCH_PMDP_SET_WRPROTECT
pmdp_set_wrprotect(struct mm_struct * mm,unsigned long addr,pmd_t * pmdp)1865 static inline void pmdp_set_wrprotect(struct mm_struct *mm,
1866 unsigned long addr, pmd_t *pmdp)
1867 {
1868 pmd_t pmd = *pmdp;
1869
1870 if (pmd_write(pmd))
1871 pmd = pmdp_xchg_lazy(mm, addr, pmdp, pmd_wrprotect(pmd));
1872 }
1873
pmdp_collapse_flush(struct vm_area_struct * vma,unsigned long address,pmd_t * pmdp)1874 static inline pmd_t pmdp_collapse_flush(struct vm_area_struct *vma,
1875 unsigned long address,
1876 pmd_t *pmdp)
1877 {
1878 return pmdp_huge_get_and_clear(vma->vm_mm, address, pmdp);
1879 }
1880 #define pmdp_collapse_flush pmdp_collapse_flush
1881
1882 #define pfn_pmd(pfn, pgprot) mk_pmd_phys(((pfn) << PAGE_SHIFT), (pgprot))
1883 #define mk_pmd(page, pgprot) pfn_pmd(page_to_pfn(page), (pgprot))
1884
pmd_trans_huge(pmd_t pmd)1885 static inline int pmd_trans_huge(pmd_t pmd)
1886 {
1887 return pmd_leaf(pmd);
1888 }
1889
1890 #define has_transparent_hugepage has_transparent_hugepage
has_transparent_hugepage(void)1891 static inline int has_transparent_hugepage(void)
1892 {
1893 return MACHINE_HAS_EDAT1 ? 1 : 0;
1894 }
1895 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
1896
1897 /*
1898 * 64 bit swap entry format:
1899 * A page-table entry has some bits we have to treat in a special way.
1900 * Bits 54 and 63 are used to indicate the page type. Bit 53 marks the pte
1901 * as invalid.
1902 * A swap pte is indicated by bit pattern (pte & 0x201) == 0x200
1903 * | offset |E11XX|type |S0|
1904 * |0000000000111111111122222222223333333333444444444455|55555|55566|66|
1905 * |0123456789012345678901234567890123456789012345678901|23456|78901|23|
1906 *
1907 * Bits 0-51 store the offset.
1908 * Bit 52 (E) is used to remember PG_anon_exclusive.
1909 * Bits 57-61 store the type.
1910 * Bit 62 (S) is used for softdirty tracking.
1911 * Bits 55 and 56 (X) are unused.
1912 */
1913
1914 #define __SWP_OFFSET_MASK ((1UL << 52) - 1)
1915 #define __SWP_OFFSET_SHIFT 12
1916 #define __SWP_TYPE_MASK ((1UL << 5) - 1)
1917 #define __SWP_TYPE_SHIFT 2
1918
mk_swap_pte(unsigned long type,unsigned long offset)1919 static inline pte_t mk_swap_pte(unsigned long type, unsigned long offset)
1920 {
1921 unsigned long pteval;
1922
1923 pteval = _PAGE_INVALID | _PAGE_PROTECT;
1924 pteval |= (offset & __SWP_OFFSET_MASK) << __SWP_OFFSET_SHIFT;
1925 pteval |= (type & __SWP_TYPE_MASK) << __SWP_TYPE_SHIFT;
1926 return __pte(pteval);
1927 }
1928
__swp_type(swp_entry_t entry)1929 static inline unsigned long __swp_type(swp_entry_t entry)
1930 {
1931 return (entry.val >> __SWP_TYPE_SHIFT) & __SWP_TYPE_MASK;
1932 }
1933
__swp_offset(swp_entry_t entry)1934 static inline unsigned long __swp_offset(swp_entry_t entry)
1935 {
1936 return (entry.val >> __SWP_OFFSET_SHIFT) & __SWP_OFFSET_MASK;
1937 }
1938
__swp_entry(unsigned long type,unsigned long offset)1939 static inline swp_entry_t __swp_entry(unsigned long type, unsigned long offset)
1940 {
1941 return (swp_entry_t) { pte_val(mk_swap_pte(type, offset)) };
1942 }
1943
1944 #define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val(pte) })
1945 #define __swp_entry_to_pte(x) ((pte_t) { (x).val })
1946
1947 /*
1948 * 64 bit swap entry format for REGION3 and SEGMENT table entries (RSTE)
1949 * Bits 59 and 63 are used to indicate the swap entry. Bit 58 marks the rste
1950 * as invalid.
1951 * A swap entry is indicated by bit pattern (rste & 0x011) == 0x010
1952 * | offset |Xtype |11TT|S0|
1953 * |0000000000111111111122222222223333333333444444444455|555555|5566|66|
1954 * |0123456789012345678901234567890123456789012345678901|234567|8901|23|
1955 *
1956 * Bits 0-51 store the offset.
1957 * Bits 53-57 store the type.
1958 * Bit 62 (S) is used for softdirty tracking.
1959 * Bits 60-61 (TT) indicate the table type: 0x01 for REGION3 and 0x00 for SEGMENT.
1960 * Bit 52 (X) is unused.
1961 */
1962
1963 #define __SWP_OFFSET_MASK_RSTE ((1UL << 52) - 1)
1964 #define __SWP_OFFSET_SHIFT_RSTE 12
1965 #define __SWP_TYPE_MASK_RSTE ((1UL << 5) - 1)
1966 #define __SWP_TYPE_SHIFT_RSTE 6
1967
1968 /*
1969 * TT bits set to 0x00 == SEGMENT. For REGION3 entries, caller must add R3
1970 * bits 0x01. See also __set_huge_pte_at().
1971 */
mk_swap_rste(unsigned long type,unsigned long offset)1972 static inline unsigned long mk_swap_rste(unsigned long type, unsigned long offset)
1973 {
1974 unsigned long rste;
1975
1976 rste = _RST_ENTRY_INVALID | _RST_ENTRY_COMM;
1977 rste |= (offset & __SWP_OFFSET_MASK_RSTE) << __SWP_OFFSET_SHIFT_RSTE;
1978 rste |= (type & __SWP_TYPE_MASK_RSTE) << __SWP_TYPE_SHIFT_RSTE;
1979 return rste;
1980 }
1981
__swp_type_rste(swp_entry_t entry)1982 static inline unsigned long __swp_type_rste(swp_entry_t entry)
1983 {
1984 return (entry.val >> __SWP_TYPE_SHIFT_RSTE) & __SWP_TYPE_MASK_RSTE;
1985 }
1986
__swp_offset_rste(swp_entry_t entry)1987 static inline unsigned long __swp_offset_rste(swp_entry_t entry)
1988 {
1989 return (entry.val >> __SWP_OFFSET_SHIFT_RSTE) & __SWP_OFFSET_MASK_RSTE;
1990 }
1991
1992 #define __rste_to_swp_entry(rste) ((swp_entry_t) { rste })
1993
1994 extern int vmem_add_mapping(unsigned long start, unsigned long size);
1995 extern void vmem_remove_mapping(unsigned long start, unsigned long size);
1996 extern int __vmem_map_4k_page(unsigned long addr, unsigned long phys, pgprot_t prot, bool alloc);
1997 extern int vmem_map_4k_page(unsigned long addr, unsigned long phys, pgprot_t prot);
1998 extern void vmem_unmap_4k_page(unsigned long addr);
1999 extern pte_t *vmem_get_alloc_pte(unsigned long addr, bool alloc);
2000 extern int s390_enable_sie(void);
2001 extern int s390_enable_skey(void);
2002 extern void s390_reset_cmma(struct mm_struct *mm);
2003
2004 /* s390 has a private copy of get unmapped area to deal with cache synonyms */
2005 #define HAVE_ARCH_UNMAPPED_AREA
2006 #define HAVE_ARCH_UNMAPPED_AREA_TOPDOWN
2007
2008 #define pmd_pgtable(pmd) \
2009 ((pgtable_t)__va(pmd_val(pmd) & -sizeof(pte_t)*PTRS_PER_PTE))
2010
gmap_pgste_get_pgt_addr(unsigned long * pgt)2011 static inline unsigned long gmap_pgste_get_pgt_addr(unsigned long *pgt)
2012 {
2013 unsigned long *pgstes, res;
2014
2015 pgstes = pgt + _PAGE_ENTRIES;
2016
2017 res = (pgstes[0] & PGSTE_ST2_MASK) << 16;
2018 res |= pgstes[1] & PGSTE_ST2_MASK;
2019 res |= (pgstes[2] & PGSTE_ST2_MASK) >> 16;
2020 res |= (pgstes[3] & PGSTE_ST2_MASK) >> 32;
2021
2022 return res;
2023 }
2024
2025 #endif /* _S390_PAGE_H */
2026