xref: /linux/arch/s390/include/asm/pgtable.h (revision 954a209f431c06b62718a49b403bd4c549f0d6fb)
1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3  *  S390 version
4  *    Copyright IBM Corp. 1999, 2000
5  *    Author(s): Hartmut Penner (hp@de.ibm.com)
6  *               Ulrich Weigand (weigand@de.ibm.com)
7  *               Martin Schwidefsky (schwidefsky@de.ibm.com)
8  *
9  *  Derived from "include/asm-i386/pgtable.h"
10  */
11 
12 #ifndef _ASM_S390_PGTABLE_H
13 #define _ASM_S390_PGTABLE_H
14 
15 #include <linux/sched.h>
16 #include <linux/mm_types.h>
17 #include <linux/page-flags.h>
18 #include <linux/radix-tree.h>
19 #include <linux/atomic.h>
20 #include <asm/ctlreg.h>
21 #include <asm/bug.h>
22 #include <asm/page.h>
23 #include <asm/uv.h>
24 
25 extern pgd_t swapper_pg_dir[];
26 extern pgd_t invalid_pg_dir[];
27 extern void paging_init(void);
28 extern struct ctlreg s390_invalid_asce;
29 
30 enum {
31 	PG_DIRECT_MAP_4K = 0,
32 	PG_DIRECT_MAP_1M,
33 	PG_DIRECT_MAP_2G,
34 	PG_DIRECT_MAP_MAX
35 };
36 
37 extern atomic_long_t direct_pages_count[PG_DIRECT_MAP_MAX];
38 
update_page_count(int level,long count)39 static inline void update_page_count(int level, long count)
40 {
41 	if (IS_ENABLED(CONFIG_PROC_FS))
42 		atomic_long_add(count, &direct_pages_count[level]);
43 }
44 
45 /*
46  * The S390 doesn't have any external MMU info: the kernel page
47  * tables contain all the necessary information.
48  */
49 #define update_mmu_cache(vma, address, ptep)     do { } while (0)
50 #define update_mmu_cache_range(vmf, vma, addr, ptep, nr) do { } while (0)
51 #define update_mmu_cache_pmd(vma, address, ptep) do { } while (0)
52 
53 /*
54  * ZERO_PAGE is a global shared page that is always zero; used
55  * for zero-mapped memory areas etc..
56  */
57 
58 extern unsigned long empty_zero_page;
59 extern unsigned long zero_page_mask;
60 
61 #define ZERO_PAGE(vaddr) \
62 	(virt_to_page((void *)(empty_zero_page + \
63 	 (((unsigned long)(vaddr)) &zero_page_mask))))
64 #define __HAVE_COLOR_ZERO_PAGE
65 
66 /* TODO: s390 cannot support io_remap_pfn_range... */
67 
68 #define pte_ERROR(e) \
69 	pr_err("%s:%d: bad pte %016lx.\n", __FILE__, __LINE__, pte_val(e))
70 #define pmd_ERROR(e) \
71 	pr_err("%s:%d: bad pmd %016lx.\n", __FILE__, __LINE__, pmd_val(e))
72 #define pud_ERROR(e) \
73 	pr_err("%s:%d: bad pud %016lx.\n", __FILE__, __LINE__, pud_val(e))
74 #define p4d_ERROR(e) \
75 	pr_err("%s:%d: bad p4d %016lx.\n", __FILE__, __LINE__, p4d_val(e))
76 #define pgd_ERROR(e) \
77 	pr_err("%s:%d: bad pgd %016lx.\n", __FILE__, __LINE__, pgd_val(e))
78 
79 /*
80  * The vmalloc and module area will always be on the topmost area of the
81  * kernel mapping. 512GB are reserved for vmalloc by default.
82  * At the top of the vmalloc area a 2GB area is reserved where modules
83  * will reside. That makes sure that inter module branches always
84  * happen without trampolines and in addition the placement within a
85  * 2GB frame is branch prediction unit friendly.
86  */
87 extern unsigned long VMALLOC_START;
88 extern unsigned long VMALLOC_END;
89 #define VMALLOC_DEFAULT_SIZE	((512UL << 30) - MODULES_LEN)
90 extern struct page *vmemmap;
91 extern unsigned long vmemmap_size;
92 
93 extern unsigned long MODULES_VADDR;
94 extern unsigned long MODULES_END;
95 #define MODULES_VADDR	MODULES_VADDR
96 #define MODULES_END	MODULES_END
97 #define MODULES_LEN	(1UL << 31)
98 
is_module_addr(void * addr)99 static inline int is_module_addr(void *addr)
100 {
101 	BUILD_BUG_ON(MODULES_LEN > (1UL << 31));
102 	if (addr < (void *)MODULES_VADDR)
103 		return 0;
104 	if (addr > (void *)MODULES_END)
105 		return 0;
106 	return 1;
107 }
108 
109 #ifdef CONFIG_KMSAN
110 #define KMSAN_VMALLOC_SIZE (VMALLOC_END - VMALLOC_START)
111 #define KMSAN_VMALLOC_SHADOW_START VMALLOC_END
112 #define KMSAN_VMALLOC_SHADOW_END (KMSAN_VMALLOC_SHADOW_START + KMSAN_VMALLOC_SIZE)
113 #define KMSAN_VMALLOC_ORIGIN_START KMSAN_VMALLOC_SHADOW_END
114 #define KMSAN_VMALLOC_ORIGIN_END (KMSAN_VMALLOC_ORIGIN_START + KMSAN_VMALLOC_SIZE)
115 #define KMSAN_MODULES_SHADOW_START KMSAN_VMALLOC_ORIGIN_END
116 #define KMSAN_MODULES_SHADOW_END (KMSAN_MODULES_SHADOW_START + MODULES_LEN)
117 #define KMSAN_MODULES_ORIGIN_START KMSAN_MODULES_SHADOW_END
118 #define KMSAN_MODULES_ORIGIN_END (KMSAN_MODULES_ORIGIN_START + MODULES_LEN)
119 #endif
120 
121 #ifdef CONFIG_RANDOMIZE_BASE
122 #define KASLR_LEN	(1UL << 31)
123 #else
124 #define KASLR_LEN	0UL
125 #endif
126 
127 void setup_protection_map(void);
128 
129 /*
130  * A 64 bit pagetable entry of S390 has following format:
131  * |			 PFRA			      |0IPC|  OS  |
132  * 0000000000111111111122222222223333333333444444444455555555556666
133  * 0123456789012345678901234567890123456789012345678901234567890123
134  *
135  * I Page-Invalid Bit:    Page is not available for address-translation
136  * P Page-Protection Bit: Store access not possible for page
137  * C Change-bit override: HW is not required to set change bit
138  *
139  * A 64 bit segmenttable entry of S390 has following format:
140  * |        P-table origin                              |      TT
141  * 0000000000111111111122222222223333333333444444444455555555556666
142  * 0123456789012345678901234567890123456789012345678901234567890123
143  *
144  * I Segment-Invalid Bit:    Segment is not available for address-translation
145  * C Common-Segment Bit:     Segment is not private (PoP 3-30)
146  * P Page-Protection Bit: Store access not possible for page
147  * TT Type 00
148  *
149  * A 64 bit region table entry of S390 has following format:
150  * |        S-table origin                             |   TF  TTTL
151  * 0000000000111111111122222222223333333333444444444455555555556666
152  * 0123456789012345678901234567890123456789012345678901234567890123
153  *
154  * I Segment-Invalid Bit:    Segment is not available for address-translation
155  * TT Type 01
156  * TF
157  * TL Table length
158  *
159  * The 64 bit regiontable origin of S390 has following format:
160  * |      region table origon                          |       DTTL
161  * 0000000000111111111122222222223333333333444444444455555555556666
162  * 0123456789012345678901234567890123456789012345678901234567890123
163  *
164  * X Space-Switch event:
165  * G Segment-Invalid Bit:
166  * P Private-Space Bit:
167  * S Storage-Alteration:
168  * R Real space
169  * TL Table-Length:
170  *
171  * A storage key has the following format:
172  * | ACC |F|R|C|0|
173  *  0   3 4 5 6 7
174  * ACC: access key
175  * F  : fetch protection bit
176  * R  : referenced bit
177  * C  : changed bit
178  */
179 
180 /* Hardware bits in the page table entry */
181 #define _PAGE_NOEXEC	0x100		/* HW no-execute bit  */
182 #define _PAGE_PROTECT	0x200		/* HW read-only bit  */
183 #define _PAGE_INVALID	0x400		/* HW invalid bit    */
184 #define _PAGE_LARGE	0x800		/* Bit to mark a large pte */
185 
186 /* Software bits in the page table entry */
187 #define _PAGE_PRESENT	0x001		/* SW pte present bit */
188 #define _PAGE_YOUNG	0x004		/* SW pte young bit */
189 #define _PAGE_DIRTY	0x008		/* SW pte dirty bit */
190 #define _PAGE_READ	0x010		/* SW pte read bit */
191 #define _PAGE_WRITE	0x020		/* SW pte write bit */
192 #define _PAGE_SPECIAL	0x040		/* SW associated with special page */
193 #define _PAGE_UNUSED	0x080		/* SW bit for pgste usage state */
194 
195 #ifdef CONFIG_MEM_SOFT_DIRTY
196 #define _PAGE_SOFT_DIRTY 0x002		/* SW pte soft dirty bit */
197 #else
198 #define _PAGE_SOFT_DIRTY 0x000
199 #endif
200 
201 #define _PAGE_SW_BITS	0xffUL		/* All SW bits */
202 
203 #define _PAGE_SWP_EXCLUSIVE _PAGE_LARGE	/* SW pte exclusive swap bit */
204 
205 /* Set of bits not changed in pte_modify */
206 #define _PAGE_CHG_MASK		(PAGE_MASK | _PAGE_SPECIAL | _PAGE_DIRTY | \
207 				 _PAGE_YOUNG | _PAGE_SOFT_DIRTY)
208 
209 /*
210  * Mask of bits that must not be changed with RDP. Allow only _PAGE_PROTECT
211  * HW bit and all SW bits.
212  */
213 #define _PAGE_RDP_MASK		~(_PAGE_PROTECT | _PAGE_SW_BITS)
214 
215 /*
216  * handle_pte_fault uses pte_present and pte_none to find out the pte type
217  * WITHOUT holding the page table lock. The _PAGE_PRESENT bit is used to
218  * distinguish present from not-present ptes. It is changed only with the page
219  * table lock held.
220  *
221  * The following table gives the different possible bit combinations for
222  * the pte hardware and software bits in the last 12 bits of a pte
223  * (. unassigned bit, x don't care, t swap type):
224  *
225  *				842100000000
226  *				000084210000
227  *				000000008421
228  *				.IR.uswrdy.p
229  * empty			.10.00000000
230  * swap				.11..ttttt.0
231  * prot-none, clean, old	.11.xx0000.1
232  * prot-none, clean, young	.11.xx0001.1
233  * prot-none, dirty, old	.11.xx0010.1
234  * prot-none, dirty, young	.11.xx0011.1
235  * read-only, clean, old	.11.xx0100.1
236  * read-only, clean, young	.01.xx0101.1
237  * read-only, dirty, old	.11.xx0110.1
238  * read-only, dirty, young	.01.xx0111.1
239  * read-write, clean, old	.11.xx1100.1
240  * read-write, clean, young	.01.xx1101.1
241  * read-write, dirty, old	.10.xx1110.1
242  * read-write, dirty, young	.00.xx1111.1
243  * HW-bits: R read-only, I invalid
244  * SW-bits: p present, y young, d dirty, r read, w write, s special,
245  *	    u unused, l large
246  *
247  * pte_none    is true for the bit pattern .10.00000000, pte == 0x400
248  * pte_swap    is true for the bit pattern .11..ooooo.0, (pte & 0x201) == 0x200
249  * pte_present is true for the bit pattern .xx.xxxxxx.1, (pte & 0x001) == 0x001
250  */
251 
252 /* Bits in the segment/region table address-space-control-element */
253 #define _ASCE_ORIGIN		~0xfffUL/* region/segment table origin	    */
254 #define _ASCE_PRIVATE_SPACE	0x100	/* private space control	    */
255 #define _ASCE_ALT_EVENT		0x80	/* storage alteration event control */
256 #define _ASCE_SPACE_SWITCH	0x40	/* space switch event		    */
257 #define _ASCE_REAL_SPACE	0x20	/* real space control		    */
258 #define _ASCE_TYPE_MASK		0x0c	/* asce table type mask		    */
259 #define _ASCE_TYPE_REGION1	0x0c	/* region first table type	    */
260 #define _ASCE_TYPE_REGION2	0x08	/* region second table type	    */
261 #define _ASCE_TYPE_REGION3	0x04	/* region third table type	    */
262 #define _ASCE_TYPE_SEGMENT	0x00	/* segment table type		    */
263 #define _ASCE_TABLE_LENGTH	0x03	/* region table length		    */
264 
265 /* Bits in the region table entry */
266 #define _REGION_ENTRY_ORIGIN	~0xfffUL/* region/segment table origin	    */
267 #define _REGION_ENTRY_PROTECT	0x200	/* region protection bit	    */
268 #define _REGION_ENTRY_NOEXEC	0x100	/* region no-execute bit	    */
269 #define _REGION_ENTRY_OFFSET	0xc0	/* region table offset		    */
270 #define _REGION_ENTRY_INVALID	0x20	/* invalid region table entry	    */
271 #define _REGION_ENTRY_TYPE_MASK	0x0c	/* region table type mask	    */
272 #define _REGION_ENTRY_TYPE_R1	0x0c	/* region first table type	    */
273 #define _REGION_ENTRY_TYPE_R2	0x08	/* region second table type	    */
274 #define _REGION_ENTRY_TYPE_R3	0x04	/* region third table type	    */
275 #define _REGION_ENTRY_LENGTH	0x03	/* region third length		    */
276 
277 #define _REGION1_ENTRY		(_REGION_ENTRY_TYPE_R1 | _REGION_ENTRY_LENGTH)
278 #define _REGION1_ENTRY_EMPTY	(_REGION_ENTRY_TYPE_R1 | _REGION_ENTRY_INVALID)
279 #define _REGION2_ENTRY		(_REGION_ENTRY_TYPE_R2 | _REGION_ENTRY_LENGTH)
280 #define _REGION2_ENTRY_EMPTY	(_REGION_ENTRY_TYPE_R2 | _REGION_ENTRY_INVALID)
281 #define _REGION3_ENTRY		(_REGION_ENTRY_TYPE_R3 | _REGION_ENTRY_LENGTH | \
282 				 _REGION3_ENTRY_PRESENT)
283 #define _REGION3_ENTRY_EMPTY	(_REGION_ENTRY_TYPE_R3 | _REGION_ENTRY_INVALID)
284 
285 #define _REGION3_ENTRY_HARDWARE_BITS		0xfffffffffffff6ffUL
286 #define _REGION3_ENTRY_HARDWARE_BITS_LARGE	0xffffffff8001073cUL
287 #define _REGION3_ENTRY_ORIGIN_LARGE ~0x7fffffffUL /* large page address	     */
288 #define _REGION3_ENTRY_DIRTY	0x2000	/* SW region dirty bit */
289 #define _REGION3_ENTRY_YOUNG	0x1000	/* SW region young bit */
290 #define _REGION3_ENTRY_COMM	0x0010	/* Common-Region, marks swap entry */
291 #define _REGION3_ENTRY_LARGE	0x0400	/* RTTE-format control, large page  */
292 #define _REGION3_ENTRY_WRITE	0x8000	/* SW region write bit */
293 #define _REGION3_ENTRY_READ	0x4000	/* SW region read bit */
294 
295 #ifdef CONFIG_MEM_SOFT_DIRTY
296 #define _REGION3_ENTRY_SOFT_DIRTY 0x0002 /* SW region soft dirty bit */
297 #else
298 #define _REGION3_ENTRY_SOFT_DIRTY 0x0000 /* SW region soft dirty bit */
299 #endif
300 
301 #define _REGION_ENTRY_BITS	 0xfffffffffffff22fUL
302 
303 /*
304  * SW region present bit. For non-leaf region-third-table entries, bits 62-63
305  * indicate the TABLE LENGTH and both must be set to 1. But such entries
306  * would always be considered as present, so it is safe to use bit 63 as
307  * PRESENT bit for PUD.
308  */
309 #define _REGION3_ENTRY_PRESENT	0x0001
310 
311 /* Bits in the segment table entry */
312 #define _SEGMENT_ENTRY_BITS			0xfffffffffffffe3fUL
313 #define _SEGMENT_ENTRY_HARDWARE_BITS		0xfffffffffffffe3cUL
314 #define _SEGMENT_ENTRY_HARDWARE_BITS_LARGE	0xfffffffffff1073cUL
315 #define _SEGMENT_ENTRY_ORIGIN_LARGE ~0xfffffUL /* large page address	    */
316 #define _SEGMENT_ENTRY_ORIGIN	~0x7ffUL/* page table origin		    */
317 #define _SEGMENT_ENTRY_PROTECT	0x200	/* segment protection bit	    */
318 #define _SEGMENT_ENTRY_NOEXEC	0x100	/* segment no-execute bit	    */
319 #define _SEGMENT_ENTRY_INVALID	0x20	/* invalid segment table entry	    */
320 #define _SEGMENT_ENTRY_TYPE_MASK 0x0c	/* segment table type mask	    */
321 
322 #define _SEGMENT_ENTRY		(_SEGMENT_ENTRY_PRESENT)
323 #define _SEGMENT_ENTRY_EMPTY	(_SEGMENT_ENTRY_INVALID)
324 
325 #define _SEGMENT_ENTRY_DIRTY	0x2000	/* SW segment dirty bit */
326 #define _SEGMENT_ENTRY_YOUNG	0x1000	/* SW segment young bit */
327 
328 #define _SEGMENT_ENTRY_COMM	0x0010	/* Common-Segment, marks swap entry */
329 #define _SEGMENT_ENTRY_LARGE	0x0400	/* STE-format control, large page */
330 #define _SEGMENT_ENTRY_WRITE	0x8000	/* SW segment write bit */
331 #define _SEGMENT_ENTRY_READ	0x4000	/* SW segment read bit */
332 
333 #ifdef CONFIG_MEM_SOFT_DIRTY
334 #define _SEGMENT_ENTRY_SOFT_DIRTY 0x0002 /* SW segment soft dirty bit */
335 #else
336 #define _SEGMENT_ENTRY_SOFT_DIRTY 0x0000 /* SW segment soft dirty bit */
337 #endif
338 
339 #define _SEGMENT_ENTRY_PRESENT	0x0001	/* SW segment present bit */
340 
341 /* Common bits in region and segment table entries, for swap entries */
342 #define _RST_ENTRY_COMM		0x0010	/* Common-Region/Segment, marks swap entry */
343 #define _RST_ENTRY_INVALID	0x0020	/* invalid region/segment table entry */
344 
345 #define _CRST_ENTRIES	2048	/* number of region/segment table entries */
346 #define _PAGE_ENTRIES	256	/* number of page table entries	*/
347 
348 #define _CRST_TABLE_SIZE (_CRST_ENTRIES * 8)
349 #define _PAGE_TABLE_SIZE (_PAGE_ENTRIES * 8)
350 
351 #define _REGION1_SHIFT	53
352 #define _REGION2_SHIFT	42
353 #define _REGION3_SHIFT	31
354 #define _SEGMENT_SHIFT	20
355 
356 #define _REGION1_INDEX	(0x7ffUL << _REGION1_SHIFT)
357 #define _REGION2_INDEX	(0x7ffUL << _REGION2_SHIFT)
358 #define _REGION3_INDEX	(0x7ffUL << _REGION3_SHIFT)
359 #define _SEGMENT_INDEX	(0x7ffUL << _SEGMENT_SHIFT)
360 #define _PAGE_INDEX	(0xffUL  << PAGE_SHIFT)
361 
362 #define _REGION1_SIZE	(1UL << _REGION1_SHIFT)
363 #define _REGION2_SIZE	(1UL << _REGION2_SHIFT)
364 #define _REGION3_SIZE	(1UL << _REGION3_SHIFT)
365 #define _SEGMENT_SIZE	(1UL << _SEGMENT_SHIFT)
366 
367 #define _REGION1_MASK	(~(_REGION1_SIZE - 1))
368 #define _REGION2_MASK	(~(_REGION2_SIZE - 1))
369 #define _REGION3_MASK	(~(_REGION3_SIZE - 1))
370 #define _SEGMENT_MASK	(~(_SEGMENT_SIZE - 1))
371 
372 #define PMD_SHIFT	_SEGMENT_SHIFT
373 #define PUD_SHIFT	_REGION3_SHIFT
374 #define P4D_SHIFT	_REGION2_SHIFT
375 #define PGDIR_SHIFT	_REGION1_SHIFT
376 
377 #define PMD_SIZE	_SEGMENT_SIZE
378 #define PUD_SIZE	_REGION3_SIZE
379 #define P4D_SIZE	_REGION2_SIZE
380 #define PGDIR_SIZE	_REGION1_SIZE
381 
382 #define PMD_MASK	_SEGMENT_MASK
383 #define PUD_MASK	_REGION3_MASK
384 #define P4D_MASK	_REGION2_MASK
385 #define PGDIR_MASK	_REGION1_MASK
386 
387 #define PTRS_PER_PTE	_PAGE_ENTRIES
388 #define PTRS_PER_PMD	_CRST_ENTRIES
389 #define PTRS_PER_PUD	_CRST_ENTRIES
390 #define PTRS_PER_P4D	_CRST_ENTRIES
391 #define PTRS_PER_PGD	_CRST_ENTRIES
392 
393 /*
394  * Segment table and region3 table entry encoding
395  * (R = read-only, I = invalid, y = young bit):
396  *				dy..R...I...wr
397  * prot-none, clean, old	00..1...1...00
398  * prot-none, clean, young	01..1...1...00
399  * prot-none, dirty, old	10..1...1...00
400  * prot-none, dirty, young	11..1...1...00
401  * read-only, clean, old	00..1...1...01
402  * read-only, clean, young	01..1...0...01
403  * read-only, dirty, old	10..1...1...01
404  * read-only, dirty, young	11..1...0...01
405  * read-write, clean, old	00..1...1...11
406  * read-write, clean, young	01..1...0...11
407  * read-write, dirty, old	10..0...1...11
408  * read-write, dirty, young	11..0...0...11
409  * The segment table origin is used to distinguish empty (origin==0) from
410  * read-write, old segment table entries (origin!=0)
411  * HW-bits: R read-only, I invalid
412  * SW-bits: y young, d dirty, r read, w write
413  */
414 
415 /* Page status table bits for virtualization */
416 #define PGSTE_ACC_BITS	0xf000000000000000UL
417 #define PGSTE_FP_BIT	0x0800000000000000UL
418 #define PGSTE_PCL_BIT	0x0080000000000000UL
419 #define PGSTE_HR_BIT	0x0040000000000000UL
420 #define PGSTE_HC_BIT	0x0020000000000000UL
421 #define PGSTE_GR_BIT	0x0004000000000000UL
422 #define PGSTE_GC_BIT	0x0002000000000000UL
423 #define PGSTE_ST2_MASK	0x0000ffff00000000UL
424 #define PGSTE_UC_BIT	0x0000000000008000UL	/* user dirty (migration) */
425 #define PGSTE_IN_BIT	0x0000000000004000UL	/* IPTE notify bit */
426 #define PGSTE_VSIE_BIT	0x0000000000002000UL	/* ref'd in a shadow table */
427 
428 /* Guest Page State used for virtualization */
429 #define _PGSTE_GPS_ZERO			0x0000000080000000UL
430 #define _PGSTE_GPS_NODAT		0x0000000040000000UL
431 #define _PGSTE_GPS_USAGE_MASK		0x0000000003000000UL
432 #define _PGSTE_GPS_USAGE_STABLE		0x0000000000000000UL
433 #define _PGSTE_GPS_USAGE_UNUSED		0x0000000001000000UL
434 #define _PGSTE_GPS_USAGE_POT_VOLATILE	0x0000000002000000UL
435 #define _PGSTE_GPS_USAGE_VOLATILE	_PGSTE_GPS_USAGE_MASK
436 
437 /*
438  * A user page table pointer has the space-switch-event bit, the
439  * private-space-control bit and the storage-alteration-event-control
440  * bit set. A kernel page table pointer doesn't need them.
441  */
442 #define _ASCE_USER_BITS		(_ASCE_SPACE_SWITCH | _ASCE_PRIVATE_SPACE | \
443 				 _ASCE_ALT_EVENT)
444 
445 /*
446  * Page protection definitions.
447  */
448 #define __PAGE_NONE		(_PAGE_PRESENT | _PAGE_INVALID | _PAGE_PROTECT)
449 #define __PAGE_RO		(_PAGE_PRESENT | _PAGE_READ | \
450 				 _PAGE_NOEXEC  | _PAGE_INVALID | _PAGE_PROTECT)
451 #define __PAGE_RX		(_PAGE_PRESENT | _PAGE_READ | \
452 				 _PAGE_INVALID | _PAGE_PROTECT)
453 #define __PAGE_RW		(_PAGE_PRESENT | _PAGE_READ | _PAGE_WRITE | \
454 				 _PAGE_NOEXEC  | _PAGE_INVALID | _PAGE_PROTECT)
455 #define __PAGE_RWX		(_PAGE_PRESENT | _PAGE_READ | _PAGE_WRITE | \
456 				 _PAGE_INVALID | _PAGE_PROTECT)
457 #define __PAGE_SHARED		(_PAGE_PRESENT | _PAGE_READ | _PAGE_WRITE | \
458 				 _PAGE_YOUNG | _PAGE_DIRTY | _PAGE_NOEXEC)
459 #define __PAGE_KERNEL		(_PAGE_PRESENT | _PAGE_READ | _PAGE_WRITE | \
460 				 _PAGE_YOUNG | _PAGE_DIRTY | _PAGE_NOEXEC)
461 #define __PAGE_KERNEL_RO	(_PAGE_PRESENT | _PAGE_READ | _PAGE_YOUNG | \
462 				 _PAGE_PROTECT | _PAGE_NOEXEC)
463 
464 extern unsigned long page_noexec_mask;
465 
466 #define __pgprot_page_mask(x)	__pgprot((x) & page_noexec_mask)
467 
468 #define PAGE_NONE		__pgprot_page_mask(__PAGE_NONE)
469 #define PAGE_RO			__pgprot_page_mask(__PAGE_RO)
470 #define PAGE_RX			__pgprot_page_mask(__PAGE_RX)
471 #define PAGE_RW			__pgprot_page_mask(__PAGE_RW)
472 #define PAGE_RWX		__pgprot_page_mask(__PAGE_RWX)
473 #define PAGE_SHARED		__pgprot_page_mask(__PAGE_SHARED)
474 #define PAGE_KERNEL		__pgprot_page_mask(__PAGE_KERNEL)
475 #define PAGE_KERNEL_RO		__pgprot_page_mask(__PAGE_KERNEL_RO)
476 
477 /*
478  * Segment entry (large page) protection definitions.
479  */
480 #define __SEGMENT_NONE		(_SEGMENT_ENTRY_PRESENT | \
481 				 _SEGMENT_ENTRY_INVALID | \
482 				 _SEGMENT_ENTRY_PROTECT)
483 #define __SEGMENT_RO		(_SEGMENT_ENTRY_PRESENT | \
484 				 _SEGMENT_ENTRY_PROTECT | \
485 				 _SEGMENT_ENTRY_READ | \
486 				 _SEGMENT_ENTRY_NOEXEC)
487 #define __SEGMENT_RX		(_SEGMENT_ENTRY_PRESENT | \
488 				 _SEGMENT_ENTRY_PROTECT | \
489 				 _SEGMENT_ENTRY_READ)
490 #define __SEGMENT_RW		(_SEGMENT_ENTRY_PRESENT | \
491 				 _SEGMENT_ENTRY_READ | \
492 				 _SEGMENT_ENTRY_WRITE | \
493 				 _SEGMENT_ENTRY_NOEXEC)
494 #define __SEGMENT_RWX		(_SEGMENT_ENTRY_PRESENT | \
495 				 _SEGMENT_ENTRY_READ | \
496 				 _SEGMENT_ENTRY_WRITE)
497 #define __SEGMENT_KERNEL	(_SEGMENT_ENTRY |	\
498 				 _SEGMENT_ENTRY_LARGE |	\
499 				 _SEGMENT_ENTRY_READ |	\
500 				 _SEGMENT_ENTRY_WRITE | \
501 				 _SEGMENT_ENTRY_YOUNG | \
502 				 _SEGMENT_ENTRY_DIRTY | \
503 				 _SEGMENT_ENTRY_NOEXEC)
504 #define __SEGMENT_KERNEL_RO	(_SEGMENT_ENTRY |	\
505 				 _SEGMENT_ENTRY_LARGE |	\
506 				 _SEGMENT_ENTRY_READ |	\
507 				 _SEGMENT_ENTRY_YOUNG |	\
508 				 _SEGMENT_ENTRY_PROTECT | \
509 				 _SEGMENT_ENTRY_NOEXEC)
510 
511 extern unsigned long segment_noexec_mask;
512 
513 #define __pgprot_segment_mask(x) __pgprot((x) & segment_noexec_mask)
514 
515 #define SEGMENT_NONE		__pgprot_segment_mask(__SEGMENT_NONE)
516 #define SEGMENT_RO		__pgprot_segment_mask(__SEGMENT_RO)
517 #define SEGMENT_RX		__pgprot_segment_mask(__SEGMENT_RX)
518 #define SEGMENT_RW		__pgprot_segment_mask(__SEGMENT_RW)
519 #define SEGMENT_RWX		__pgprot_segment_mask(__SEGMENT_RWX)
520 #define SEGMENT_KERNEL		__pgprot_segment_mask(__SEGMENT_KERNEL)
521 #define SEGMENT_KERNEL_RO	__pgprot_segment_mask(__SEGMENT_KERNEL_RO)
522 
523 /*
524  * Region3 entry (large page) protection definitions.
525  */
526 
527 #define __REGION3_KERNEL	(_REGION_ENTRY_TYPE_R3 | \
528 				 _REGION3_ENTRY_PRESENT | \
529 				 _REGION3_ENTRY_LARGE | \
530 				 _REGION3_ENTRY_READ | \
531 				 _REGION3_ENTRY_WRITE | \
532 				 _REGION3_ENTRY_YOUNG | \
533 				 _REGION3_ENTRY_DIRTY | \
534 				 _REGION_ENTRY_NOEXEC)
535 #define __REGION3_KERNEL_RO	(_REGION_ENTRY_TYPE_R3 | \
536 				 _REGION3_ENTRY_PRESENT | \
537 				 _REGION3_ENTRY_LARGE | \
538 				 _REGION3_ENTRY_READ | \
539 				 _REGION3_ENTRY_YOUNG | \
540 				 _REGION_ENTRY_PROTECT | \
541 				 _REGION_ENTRY_NOEXEC)
542 
543 extern unsigned long region_noexec_mask;
544 
545 #define __pgprot_region_mask(x)	__pgprot((x) & region_noexec_mask)
546 
547 #define REGION3_KERNEL		__pgprot_region_mask(__REGION3_KERNEL)
548 #define REGION3_KERNEL_RO	__pgprot_region_mask(__REGION3_KERNEL_RO)
549 
mm_p4d_folded(struct mm_struct * mm)550 static inline bool mm_p4d_folded(struct mm_struct *mm)
551 {
552 	return mm->context.asce_limit <= _REGION1_SIZE;
553 }
554 #define mm_p4d_folded(mm) mm_p4d_folded(mm)
555 
mm_pud_folded(struct mm_struct * mm)556 static inline bool mm_pud_folded(struct mm_struct *mm)
557 {
558 	return mm->context.asce_limit <= _REGION2_SIZE;
559 }
560 #define mm_pud_folded(mm) mm_pud_folded(mm)
561 
mm_pmd_folded(struct mm_struct * mm)562 static inline bool mm_pmd_folded(struct mm_struct *mm)
563 {
564 	return mm->context.asce_limit <= _REGION3_SIZE;
565 }
566 #define mm_pmd_folded(mm) mm_pmd_folded(mm)
567 
mm_has_pgste(struct mm_struct * mm)568 static inline int mm_has_pgste(struct mm_struct *mm)
569 {
570 #ifdef CONFIG_PGSTE
571 	if (unlikely(mm->context.has_pgste))
572 		return 1;
573 #endif
574 	return 0;
575 }
576 
mm_is_protected(struct mm_struct * mm)577 static inline int mm_is_protected(struct mm_struct *mm)
578 {
579 #ifdef CONFIG_PGSTE
580 	if (unlikely(atomic_read(&mm->context.protected_count)))
581 		return 1;
582 #endif
583 	return 0;
584 }
585 
mm_alloc_pgste(struct mm_struct * mm)586 static inline int mm_alloc_pgste(struct mm_struct *mm)
587 {
588 #ifdef CONFIG_PGSTE
589 	if (unlikely(mm->context.alloc_pgste))
590 		return 1;
591 #endif
592 	return 0;
593 }
594 
clear_pte_bit(pte_t pte,pgprot_t prot)595 static inline pte_t clear_pte_bit(pte_t pte, pgprot_t prot)
596 {
597 	return __pte(pte_val(pte) & ~pgprot_val(prot));
598 }
599 
set_pte_bit(pte_t pte,pgprot_t prot)600 static inline pte_t set_pte_bit(pte_t pte, pgprot_t prot)
601 {
602 	return __pte(pte_val(pte) | pgprot_val(prot));
603 }
604 
clear_pmd_bit(pmd_t pmd,pgprot_t prot)605 static inline pmd_t clear_pmd_bit(pmd_t pmd, pgprot_t prot)
606 {
607 	return __pmd(pmd_val(pmd) & ~pgprot_val(prot));
608 }
609 
set_pmd_bit(pmd_t pmd,pgprot_t prot)610 static inline pmd_t set_pmd_bit(pmd_t pmd, pgprot_t prot)
611 {
612 	return __pmd(pmd_val(pmd) | pgprot_val(prot));
613 }
614 
clear_pud_bit(pud_t pud,pgprot_t prot)615 static inline pud_t clear_pud_bit(pud_t pud, pgprot_t prot)
616 {
617 	return __pud(pud_val(pud) & ~pgprot_val(prot));
618 }
619 
set_pud_bit(pud_t pud,pgprot_t prot)620 static inline pud_t set_pud_bit(pud_t pud, pgprot_t prot)
621 {
622 	return __pud(pud_val(pud) | pgprot_val(prot));
623 }
624 
625 /*
626  * As soon as the guest uses storage keys or enables PV, we deduplicate all
627  * mapped shared zeropages and prevent new shared zeropages from getting
628  * mapped.
629  */
630 #define mm_forbids_zeropage mm_forbids_zeropage
mm_forbids_zeropage(struct mm_struct * mm)631 static inline int mm_forbids_zeropage(struct mm_struct *mm)
632 {
633 #ifdef CONFIG_PGSTE
634 	if (!mm->context.allow_cow_sharing)
635 		return 1;
636 #endif
637 	return 0;
638 }
639 
mm_uses_skeys(struct mm_struct * mm)640 static inline int mm_uses_skeys(struct mm_struct *mm)
641 {
642 #ifdef CONFIG_PGSTE
643 	if (mm->context.uses_skeys)
644 		return 1;
645 #endif
646 	return 0;
647 }
648 
csp(unsigned int * ptr,unsigned int old,unsigned int new)649 static inline void csp(unsigned int *ptr, unsigned int old, unsigned int new)
650 {
651 	union register_pair r1 = { .even = old, .odd = new, };
652 	unsigned long address = (unsigned long)ptr | 1;
653 
654 	asm volatile(
655 		"	csp	%[r1],%[address]"
656 		: [r1] "+&d" (r1.pair), "+m" (*ptr)
657 		: [address] "d" (address)
658 		: "cc");
659 }
660 
661 /**
662  * cspg() - Compare and Swap and Purge (CSPG)
663  * @ptr: Pointer to the value to be exchanged
664  * @old: The expected old value
665  * @new: The new value
666  *
667  * Return: True if compare and swap was successful, otherwise false.
668  */
cspg(unsigned long * ptr,unsigned long old,unsigned long new)669 static inline bool cspg(unsigned long *ptr, unsigned long old, unsigned long new)
670 {
671 	union register_pair r1 = { .even = old, .odd = new, };
672 	unsigned long address = (unsigned long)ptr | 1;
673 
674 	asm volatile(
675 		"	cspg	%[r1],%[address]"
676 		: [r1] "+&d" (r1.pair), "+m" (*ptr)
677 		: [address] "d" (address)
678 		: "cc");
679 	return old == r1.even;
680 }
681 
682 #define CRDTE_DTT_PAGE		0x00UL
683 #define CRDTE_DTT_SEGMENT	0x10UL
684 #define CRDTE_DTT_REGION3	0x14UL
685 #define CRDTE_DTT_REGION2	0x18UL
686 #define CRDTE_DTT_REGION1	0x1cUL
687 
688 /**
689  * crdte() - Compare and Replace DAT Table Entry
690  * @old:     The expected old value
691  * @new:     The new value
692  * @table:   Pointer to the value to be exchanged
693  * @dtt:     Table type of the table to be exchanged
694  * @address: The address mapped by the entry to be replaced
695  * @asce:    The ASCE of this entry
696  *
697  * Return: True if compare and replace was successful, otherwise false.
698  */
crdte(unsigned long old,unsigned long new,unsigned long * table,unsigned long dtt,unsigned long address,unsigned long asce)699 static inline bool crdte(unsigned long old, unsigned long new,
700 			 unsigned long *table, unsigned long dtt,
701 			 unsigned long address, unsigned long asce)
702 {
703 	union register_pair r1 = { .even = old, .odd = new, };
704 	union register_pair r2 = { .even = __pa(table) | dtt, .odd = address, };
705 
706 	asm volatile(".insn rrf,0xb98f0000,%[r1],%[r2],%[asce],0"
707 		     : [r1] "+&d" (r1.pair)
708 		     : [r2] "d" (r2.pair), [asce] "a" (asce)
709 		     : "memory", "cc");
710 	return old == r1.even;
711 }
712 
713 /*
714  * pgd/p4d/pud/pmd/pte query functions
715  */
pgd_folded(pgd_t pgd)716 static inline int pgd_folded(pgd_t pgd)
717 {
718 	return (pgd_val(pgd) & _REGION_ENTRY_TYPE_MASK) < _REGION_ENTRY_TYPE_R1;
719 }
720 
pgd_present(pgd_t pgd)721 static inline int pgd_present(pgd_t pgd)
722 {
723 	if (pgd_folded(pgd))
724 		return 1;
725 	return (pgd_val(pgd) & _REGION_ENTRY_ORIGIN) != 0UL;
726 }
727 
pgd_none(pgd_t pgd)728 static inline int pgd_none(pgd_t pgd)
729 {
730 	if (pgd_folded(pgd))
731 		return 0;
732 	return (pgd_val(pgd) & _REGION_ENTRY_INVALID) != 0UL;
733 }
734 
pgd_bad(pgd_t pgd)735 static inline int pgd_bad(pgd_t pgd)
736 {
737 	if ((pgd_val(pgd) & _REGION_ENTRY_TYPE_MASK) < _REGION_ENTRY_TYPE_R1)
738 		return 0;
739 	return (pgd_val(pgd) & ~_REGION_ENTRY_BITS) != 0;
740 }
741 
pgd_pfn(pgd_t pgd)742 static inline unsigned long pgd_pfn(pgd_t pgd)
743 {
744 	unsigned long origin_mask;
745 
746 	origin_mask = _REGION_ENTRY_ORIGIN;
747 	return (pgd_val(pgd) & origin_mask) >> PAGE_SHIFT;
748 }
749 
p4d_folded(p4d_t p4d)750 static inline int p4d_folded(p4d_t p4d)
751 {
752 	return (p4d_val(p4d) & _REGION_ENTRY_TYPE_MASK) < _REGION_ENTRY_TYPE_R2;
753 }
754 
p4d_present(p4d_t p4d)755 static inline int p4d_present(p4d_t p4d)
756 {
757 	if (p4d_folded(p4d))
758 		return 1;
759 	return (p4d_val(p4d) & _REGION_ENTRY_ORIGIN) != 0UL;
760 }
761 
p4d_none(p4d_t p4d)762 static inline int p4d_none(p4d_t p4d)
763 {
764 	if (p4d_folded(p4d))
765 		return 0;
766 	return p4d_val(p4d) == _REGION2_ENTRY_EMPTY;
767 }
768 
p4d_pfn(p4d_t p4d)769 static inline unsigned long p4d_pfn(p4d_t p4d)
770 {
771 	unsigned long origin_mask;
772 
773 	origin_mask = _REGION_ENTRY_ORIGIN;
774 	return (p4d_val(p4d) & origin_mask) >> PAGE_SHIFT;
775 }
776 
pud_folded(pud_t pud)777 static inline int pud_folded(pud_t pud)
778 {
779 	return (pud_val(pud) & _REGION_ENTRY_TYPE_MASK) < _REGION_ENTRY_TYPE_R3;
780 }
781 
pud_present(pud_t pud)782 static inline int pud_present(pud_t pud)
783 {
784 	if (pud_folded(pud))
785 		return 1;
786 	return (pud_val(pud) & _REGION3_ENTRY_PRESENT) != 0;
787 }
788 
pud_none(pud_t pud)789 static inline int pud_none(pud_t pud)
790 {
791 	if (pud_folded(pud))
792 		return 0;
793 	return pud_val(pud) == _REGION3_ENTRY_EMPTY;
794 }
795 
796 #define pud_leaf pud_leaf
pud_leaf(pud_t pud)797 static inline bool pud_leaf(pud_t pud)
798 {
799 	if ((pud_val(pud) & _REGION_ENTRY_TYPE_MASK) != _REGION_ENTRY_TYPE_R3)
800 		return 0;
801 	return (pud_present(pud) && (pud_val(pud) & _REGION3_ENTRY_LARGE) != 0);
802 }
803 
pmd_present(pmd_t pmd)804 static inline int pmd_present(pmd_t pmd)
805 {
806 	return (pmd_val(pmd) & _SEGMENT_ENTRY_PRESENT) != 0;
807 }
808 
809 #define pmd_leaf pmd_leaf
pmd_leaf(pmd_t pmd)810 static inline bool pmd_leaf(pmd_t pmd)
811 {
812 	return (pmd_present(pmd) && (pmd_val(pmd) & _SEGMENT_ENTRY_LARGE) != 0);
813 }
814 
pmd_bad(pmd_t pmd)815 static inline int pmd_bad(pmd_t pmd)
816 {
817 	if ((pmd_val(pmd) & _SEGMENT_ENTRY_TYPE_MASK) > 0 || pmd_leaf(pmd))
818 		return 1;
819 	return (pmd_val(pmd) & ~_SEGMENT_ENTRY_BITS) != 0;
820 }
821 
pud_bad(pud_t pud)822 static inline int pud_bad(pud_t pud)
823 {
824 	unsigned long type = pud_val(pud) & _REGION_ENTRY_TYPE_MASK;
825 
826 	if (type > _REGION_ENTRY_TYPE_R3 || pud_leaf(pud))
827 		return 1;
828 	if (type < _REGION_ENTRY_TYPE_R3)
829 		return 0;
830 	return (pud_val(pud) & ~_REGION_ENTRY_BITS) != 0;
831 }
832 
p4d_bad(p4d_t p4d)833 static inline int p4d_bad(p4d_t p4d)
834 {
835 	unsigned long type = p4d_val(p4d) & _REGION_ENTRY_TYPE_MASK;
836 
837 	if (type > _REGION_ENTRY_TYPE_R2)
838 		return 1;
839 	if (type < _REGION_ENTRY_TYPE_R2)
840 		return 0;
841 	return (p4d_val(p4d) & ~_REGION_ENTRY_BITS) != 0;
842 }
843 
pmd_none(pmd_t pmd)844 static inline int pmd_none(pmd_t pmd)
845 {
846 	return pmd_val(pmd) == _SEGMENT_ENTRY_EMPTY;
847 }
848 
849 #define pmd_write pmd_write
pmd_write(pmd_t pmd)850 static inline int pmd_write(pmd_t pmd)
851 {
852 	return (pmd_val(pmd) & _SEGMENT_ENTRY_WRITE) != 0;
853 }
854 
855 #define pud_write pud_write
pud_write(pud_t pud)856 static inline int pud_write(pud_t pud)
857 {
858 	return (pud_val(pud) & _REGION3_ENTRY_WRITE) != 0;
859 }
860 
861 #define pmd_dirty pmd_dirty
pmd_dirty(pmd_t pmd)862 static inline int pmd_dirty(pmd_t pmd)
863 {
864 	return (pmd_val(pmd) & _SEGMENT_ENTRY_DIRTY) != 0;
865 }
866 
867 #define pmd_young pmd_young
pmd_young(pmd_t pmd)868 static inline int pmd_young(pmd_t pmd)
869 {
870 	return (pmd_val(pmd) & _SEGMENT_ENTRY_YOUNG) != 0;
871 }
872 
pte_present(pte_t pte)873 static inline int pte_present(pte_t pte)
874 {
875 	/* Bit pattern: (pte & 0x001) == 0x001 */
876 	return (pte_val(pte) & _PAGE_PRESENT) != 0;
877 }
878 
pte_none(pte_t pte)879 static inline int pte_none(pte_t pte)
880 {
881 	/* Bit pattern: pte == 0x400 */
882 	return pte_val(pte) == _PAGE_INVALID;
883 }
884 
pte_swap(pte_t pte)885 static inline int pte_swap(pte_t pte)
886 {
887 	/* Bit pattern: (pte & 0x201) == 0x200 */
888 	return (pte_val(pte) & (_PAGE_PROTECT | _PAGE_PRESENT))
889 		== _PAGE_PROTECT;
890 }
891 
pte_special(pte_t pte)892 static inline int pte_special(pte_t pte)
893 {
894 	return (pte_val(pte) & _PAGE_SPECIAL);
895 }
896 
897 #define __HAVE_ARCH_PTE_SAME
pte_same(pte_t a,pte_t b)898 static inline int pte_same(pte_t a, pte_t b)
899 {
900 	return pte_val(a) == pte_val(b);
901 }
902 
903 #ifdef CONFIG_NUMA_BALANCING
pte_protnone(pte_t pte)904 static inline int pte_protnone(pte_t pte)
905 {
906 	return pte_present(pte) && !(pte_val(pte) & _PAGE_READ);
907 }
908 
pmd_protnone(pmd_t pmd)909 static inline int pmd_protnone(pmd_t pmd)
910 {
911 	/* pmd_leaf(pmd) implies pmd_present(pmd) */
912 	return pmd_leaf(pmd) && !(pmd_val(pmd) & _SEGMENT_ENTRY_READ);
913 }
914 #endif
915 
pte_swp_exclusive(pte_t pte)916 static inline int pte_swp_exclusive(pte_t pte)
917 {
918 	return pte_val(pte) & _PAGE_SWP_EXCLUSIVE;
919 }
920 
pte_swp_mkexclusive(pte_t pte)921 static inline pte_t pte_swp_mkexclusive(pte_t pte)
922 {
923 	return set_pte_bit(pte, __pgprot(_PAGE_SWP_EXCLUSIVE));
924 }
925 
pte_swp_clear_exclusive(pte_t pte)926 static inline pte_t pte_swp_clear_exclusive(pte_t pte)
927 {
928 	return clear_pte_bit(pte, __pgprot(_PAGE_SWP_EXCLUSIVE));
929 }
930 
pte_soft_dirty(pte_t pte)931 static inline int pte_soft_dirty(pte_t pte)
932 {
933 	return pte_val(pte) & _PAGE_SOFT_DIRTY;
934 }
935 #define pte_swp_soft_dirty pte_soft_dirty
936 
pte_mksoft_dirty(pte_t pte)937 static inline pte_t pte_mksoft_dirty(pte_t pte)
938 {
939 	return set_pte_bit(pte, __pgprot(_PAGE_SOFT_DIRTY));
940 }
941 #define pte_swp_mksoft_dirty pte_mksoft_dirty
942 
pte_clear_soft_dirty(pte_t pte)943 static inline pte_t pte_clear_soft_dirty(pte_t pte)
944 {
945 	return clear_pte_bit(pte, __pgprot(_PAGE_SOFT_DIRTY));
946 }
947 #define pte_swp_clear_soft_dirty pte_clear_soft_dirty
948 
pmd_soft_dirty(pmd_t pmd)949 static inline int pmd_soft_dirty(pmd_t pmd)
950 {
951 	return pmd_val(pmd) & _SEGMENT_ENTRY_SOFT_DIRTY;
952 }
953 
pmd_mksoft_dirty(pmd_t pmd)954 static inline pmd_t pmd_mksoft_dirty(pmd_t pmd)
955 {
956 	return set_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_SOFT_DIRTY));
957 }
958 
pmd_clear_soft_dirty(pmd_t pmd)959 static inline pmd_t pmd_clear_soft_dirty(pmd_t pmd)
960 {
961 	return clear_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_SOFT_DIRTY));
962 }
963 
964 /*
965  * query functions pte_write/pte_dirty/pte_young only work if
966  * pte_present() is true. Undefined behaviour if not..
967  */
pte_write(pte_t pte)968 static inline int pte_write(pte_t pte)
969 {
970 	return (pte_val(pte) & _PAGE_WRITE) != 0;
971 }
972 
pte_dirty(pte_t pte)973 static inline int pte_dirty(pte_t pte)
974 {
975 	return (pte_val(pte) & _PAGE_DIRTY) != 0;
976 }
977 
pte_young(pte_t pte)978 static inline int pte_young(pte_t pte)
979 {
980 	return (pte_val(pte) & _PAGE_YOUNG) != 0;
981 }
982 
983 #define __HAVE_ARCH_PTE_UNUSED
pte_unused(pte_t pte)984 static inline int pte_unused(pte_t pte)
985 {
986 	return pte_val(pte) & _PAGE_UNUSED;
987 }
988 
989 /*
990  * Extract the pgprot value from the given pte while at the same time making it
991  * usable for kernel address space mappings where fault driven dirty and
992  * young/old accounting is not supported, i.e _PAGE_PROTECT and _PAGE_INVALID
993  * must not be set.
994  */
995 #define pte_pgprot pte_pgprot
pte_pgprot(pte_t pte)996 static inline pgprot_t pte_pgprot(pte_t pte)
997 {
998 	unsigned long pte_flags = pte_val(pte) & _PAGE_CHG_MASK;
999 
1000 	if (pte_write(pte))
1001 		pte_flags |= pgprot_val(PAGE_KERNEL);
1002 	else
1003 		pte_flags |= pgprot_val(PAGE_KERNEL_RO);
1004 	pte_flags |= pte_val(pte) & mio_wb_bit_mask;
1005 
1006 	return __pgprot(pte_flags);
1007 }
1008 
1009 /*
1010  * pgd/pmd/pte modification functions
1011  */
1012 
set_pgd(pgd_t * pgdp,pgd_t pgd)1013 static inline void set_pgd(pgd_t *pgdp, pgd_t pgd)
1014 {
1015 	WRITE_ONCE(*pgdp, pgd);
1016 }
1017 
set_p4d(p4d_t * p4dp,p4d_t p4d)1018 static inline void set_p4d(p4d_t *p4dp, p4d_t p4d)
1019 {
1020 	WRITE_ONCE(*p4dp, p4d);
1021 }
1022 
set_pud(pud_t * pudp,pud_t pud)1023 static inline void set_pud(pud_t *pudp, pud_t pud)
1024 {
1025 	WRITE_ONCE(*pudp, pud);
1026 }
1027 
set_pmd(pmd_t * pmdp,pmd_t pmd)1028 static inline void set_pmd(pmd_t *pmdp, pmd_t pmd)
1029 {
1030 	WRITE_ONCE(*pmdp, pmd);
1031 }
1032 
set_pte(pte_t * ptep,pte_t pte)1033 static inline void set_pte(pte_t *ptep, pte_t pte)
1034 {
1035 	WRITE_ONCE(*ptep, pte);
1036 }
1037 
pgd_clear(pgd_t * pgd)1038 static inline void pgd_clear(pgd_t *pgd)
1039 {
1040 	if ((pgd_val(*pgd) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R1)
1041 		set_pgd(pgd, __pgd(_REGION1_ENTRY_EMPTY));
1042 }
1043 
p4d_clear(p4d_t * p4d)1044 static inline void p4d_clear(p4d_t *p4d)
1045 {
1046 	if ((p4d_val(*p4d) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R2)
1047 		set_p4d(p4d, __p4d(_REGION2_ENTRY_EMPTY));
1048 }
1049 
pud_clear(pud_t * pud)1050 static inline void pud_clear(pud_t *pud)
1051 {
1052 	if ((pud_val(*pud) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3)
1053 		set_pud(pud, __pud(_REGION3_ENTRY_EMPTY));
1054 }
1055 
pmd_clear(pmd_t * pmdp)1056 static inline void pmd_clear(pmd_t *pmdp)
1057 {
1058 	set_pmd(pmdp, __pmd(_SEGMENT_ENTRY_EMPTY));
1059 }
1060 
pte_clear(struct mm_struct * mm,unsigned long addr,pte_t * ptep)1061 static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
1062 {
1063 	set_pte(ptep, __pte(_PAGE_INVALID));
1064 }
1065 
1066 /*
1067  * The following pte modification functions only work if
1068  * pte_present() is true. Undefined behaviour if not..
1069  */
pte_modify(pte_t pte,pgprot_t newprot)1070 static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
1071 {
1072 	pte = clear_pte_bit(pte, __pgprot(~_PAGE_CHG_MASK));
1073 	pte = set_pte_bit(pte, newprot);
1074 	/*
1075 	 * newprot for PAGE_NONE, PAGE_RO, PAGE_RX, PAGE_RW and PAGE_RWX
1076 	 * has the invalid bit set, clear it again for readable, young pages
1077 	 */
1078 	if ((pte_val(pte) & _PAGE_YOUNG) && (pte_val(pte) & _PAGE_READ))
1079 		pte = clear_pte_bit(pte, __pgprot(_PAGE_INVALID));
1080 	/*
1081 	 * newprot for PAGE_RO, PAGE_RX, PAGE_RW and PAGE_RWX has the page
1082 	 * protection bit set, clear it again for writable, dirty pages
1083 	 */
1084 	if ((pte_val(pte) & _PAGE_DIRTY) && (pte_val(pte) & _PAGE_WRITE))
1085 		pte = clear_pte_bit(pte, __pgprot(_PAGE_PROTECT));
1086 	return pte;
1087 }
1088 
pte_wrprotect(pte_t pte)1089 static inline pte_t pte_wrprotect(pte_t pte)
1090 {
1091 	pte = clear_pte_bit(pte, __pgprot(_PAGE_WRITE));
1092 	return set_pte_bit(pte, __pgprot(_PAGE_PROTECT));
1093 }
1094 
pte_mkwrite_novma(pte_t pte)1095 static inline pte_t pte_mkwrite_novma(pte_t pte)
1096 {
1097 	pte = set_pte_bit(pte, __pgprot(_PAGE_WRITE));
1098 	if (pte_val(pte) & _PAGE_DIRTY)
1099 		pte = clear_pte_bit(pte, __pgprot(_PAGE_PROTECT));
1100 	return pte;
1101 }
1102 
pte_mkclean(pte_t pte)1103 static inline pte_t pte_mkclean(pte_t pte)
1104 {
1105 	pte = clear_pte_bit(pte, __pgprot(_PAGE_DIRTY));
1106 	return set_pte_bit(pte, __pgprot(_PAGE_PROTECT));
1107 }
1108 
pte_mkdirty(pte_t pte)1109 static inline pte_t pte_mkdirty(pte_t pte)
1110 {
1111 	pte = set_pte_bit(pte, __pgprot(_PAGE_DIRTY | _PAGE_SOFT_DIRTY));
1112 	if (pte_val(pte) & _PAGE_WRITE)
1113 		pte = clear_pte_bit(pte, __pgprot(_PAGE_PROTECT));
1114 	return pte;
1115 }
1116 
pte_mkold(pte_t pte)1117 static inline pte_t pte_mkold(pte_t pte)
1118 {
1119 	pte = clear_pte_bit(pte, __pgprot(_PAGE_YOUNG));
1120 	return set_pte_bit(pte, __pgprot(_PAGE_INVALID));
1121 }
1122 
pte_mkyoung(pte_t pte)1123 static inline pte_t pte_mkyoung(pte_t pte)
1124 {
1125 	pte = set_pte_bit(pte, __pgprot(_PAGE_YOUNG));
1126 	if (pte_val(pte) & _PAGE_READ)
1127 		pte = clear_pte_bit(pte, __pgprot(_PAGE_INVALID));
1128 	return pte;
1129 }
1130 
pte_mkspecial(pte_t pte)1131 static inline pte_t pte_mkspecial(pte_t pte)
1132 {
1133 	return set_pte_bit(pte, __pgprot(_PAGE_SPECIAL));
1134 }
1135 
1136 #ifdef CONFIG_HUGETLB_PAGE
pte_mkhuge(pte_t pte)1137 static inline pte_t pte_mkhuge(pte_t pte)
1138 {
1139 	return set_pte_bit(pte, __pgprot(_PAGE_LARGE));
1140 }
1141 #endif
1142 
1143 #define IPTE_GLOBAL	0
1144 #define	IPTE_LOCAL	1
1145 
1146 #define IPTE_NODAT	0x400
1147 #define IPTE_GUEST_ASCE	0x800
1148 
__ptep_rdp(unsigned long addr,pte_t * ptep,unsigned long opt,unsigned long asce,int local)1149 static __always_inline void __ptep_rdp(unsigned long addr, pte_t *ptep,
1150 				       unsigned long opt, unsigned long asce,
1151 				       int local)
1152 {
1153 	unsigned long pto;
1154 
1155 	pto = __pa(ptep) & ~(PTRS_PER_PTE * sizeof(pte_t) - 1);
1156 	asm volatile(".insn rrf,0xb98b0000,%[r1],%[r2],%[asce],%[m4]"
1157 		     : "+m" (*ptep)
1158 		     : [r1] "a" (pto), [r2] "a" ((addr & PAGE_MASK) | opt),
1159 		       [asce] "a" (asce), [m4] "i" (local));
1160 }
1161 
__ptep_ipte(unsigned long address,pte_t * ptep,unsigned long opt,unsigned long asce,int local)1162 static __always_inline void __ptep_ipte(unsigned long address, pte_t *ptep,
1163 					unsigned long opt, unsigned long asce,
1164 					int local)
1165 {
1166 	unsigned long pto = __pa(ptep);
1167 
1168 	if (__builtin_constant_p(opt) && opt == 0) {
1169 		/* Invalidation + TLB flush for the pte */
1170 		asm volatile(
1171 			"	ipte	%[r1],%[r2],0,%[m4]"
1172 			: "+m" (*ptep) : [r1] "a" (pto), [r2] "a" (address),
1173 			  [m4] "i" (local));
1174 		return;
1175 	}
1176 
1177 	/* Invalidate ptes with options + TLB flush of the ptes */
1178 	opt = opt | (asce & _ASCE_ORIGIN);
1179 	asm volatile(
1180 		"	ipte	%[r1],%[r2],%[r3],%[m4]"
1181 		: [r2] "+a" (address), [r3] "+a" (opt)
1182 		: [r1] "a" (pto), [m4] "i" (local) : "memory");
1183 }
1184 
__ptep_ipte_range(unsigned long address,int nr,pte_t * ptep,int local)1185 static __always_inline void __ptep_ipte_range(unsigned long address, int nr,
1186 					      pte_t *ptep, int local)
1187 {
1188 	unsigned long pto = __pa(ptep);
1189 
1190 	/* Invalidate a range of ptes + TLB flush of the ptes */
1191 	do {
1192 		asm volatile(
1193 			"	ipte %[r1],%[r2],%[r3],%[m4]"
1194 			: [r2] "+a" (address), [r3] "+a" (nr)
1195 			: [r1] "a" (pto), [m4] "i" (local) : "memory");
1196 	} while (nr != 255);
1197 }
1198 
1199 /*
1200  * This is hard to understand. ptep_get_and_clear and ptep_clear_flush
1201  * both clear the TLB for the unmapped pte. The reason is that
1202  * ptep_get_and_clear is used in common code (e.g. change_pte_range)
1203  * to modify an active pte. The sequence is
1204  *   1) ptep_get_and_clear
1205  *   2) set_pte_at
1206  *   3) flush_tlb_range
1207  * On s390 the tlb needs to get flushed with the modification of the pte
1208  * if the pte is active. The only way how this can be implemented is to
1209  * have ptep_get_and_clear do the tlb flush. In exchange flush_tlb_range
1210  * is a nop.
1211  */
1212 pte_t ptep_xchg_direct(struct mm_struct *, unsigned long, pte_t *, pte_t);
1213 pte_t ptep_xchg_lazy(struct mm_struct *, unsigned long, pte_t *, pte_t);
1214 
1215 #define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
ptep_test_and_clear_young(struct vm_area_struct * vma,unsigned long addr,pte_t * ptep)1216 static inline int ptep_test_and_clear_young(struct vm_area_struct *vma,
1217 					    unsigned long addr, pte_t *ptep)
1218 {
1219 	pte_t pte = *ptep;
1220 
1221 	pte = ptep_xchg_direct(vma->vm_mm, addr, ptep, pte_mkold(pte));
1222 	return pte_young(pte);
1223 }
1224 
1225 #define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
ptep_clear_flush_young(struct vm_area_struct * vma,unsigned long address,pte_t * ptep)1226 static inline int ptep_clear_flush_young(struct vm_area_struct *vma,
1227 					 unsigned long address, pte_t *ptep)
1228 {
1229 	return ptep_test_and_clear_young(vma, address, ptep);
1230 }
1231 
1232 #define __HAVE_ARCH_PTEP_GET_AND_CLEAR
ptep_get_and_clear(struct mm_struct * mm,unsigned long addr,pte_t * ptep)1233 static inline pte_t ptep_get_and_clear(struct mm_struct *mm,
1234 				       unsigned long addr, pte_t *ptep)
1235 {
1236 	pte_t res;
1237 
1238 	res = ptep_xchg_lazy(mm, addr, ptep, __pte(_PAGE_INVALID));
1239 	/* At this point the reference through the mapping is still present */
1240 	if (mm_is_protected(mm) && pte_present(res))
1241 		uv_convert_from_secure_pte(res);
1242 	return res;
1243 }
1244 
1245 #define __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION
1246 pte_t ptep_modify_prot_start(struct vm_area_struct *, unsigned long, pte_t *);
1247 void ptep_modify_prot_commit(struct vm_area_struct *, unsigned long,
1248 			     pte_t *, pte_t, pte_t);
1249 
1250 #define __HAVE_ARCH_PTEP_CLEAR_FLUSH
ptep_clear_flush(struct vm_area_struct * vma,unsigned long addr,pte_t * ptep)1251 static inline pte_t ptep_clear_flush(struct vm_area_struct *vma,
1252 				     unsigned long addr, pte_t *ptep)
1253 {
1254 	pte_t res;
1255 
1256 	res = ptep_xchg_direct(vma->vm_mm, addr, ptep, __pte(_PAGE_INVALID));
1257 	/* At this point the reference through the mapping is still present */
1258 	if (mm_is_protected(vma->vm_mm) && pte_present(res))
1259 		uv_convert_from_secure_pte(res);
1260 	return res;
1261 }
1262 
1263 /*
1264  * The batched pte unmap code uses ptep_get_and_clear_full to clear the
1265  * ptes. Here an optimization is possible. tlb_gather_mmu flushes all
1266  * tlbs of an mm if it can guarantee that the ptes of the mm_struct
1267  * cannot be accessed while the batched unmap is running. In this case
1268  * full==1 and a simple pte_clear is enough. See tlb.h.
1269  */
1270 #define __HAVE_ARCH_PTEP_GET_AND_CLEAR_FULL
ptep_get_and_clear_full(struct mm_struct * mm,unsigned long addr,pte_t * ptep,int full)1271 static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm,
1272 					    unsigned long addr,
1273 					    pte_t *ptep, int full)
1274 {
1275 	pte_t res;
1276 
1277 	if (full) {
1278 		res = *ptep;
1279 		set_pte(ptep, __pte(_PAGE_INVALID));
1280 	} else {
1281 		res = ptep_xchg_lazy(mm, addr, ptep, __pte(_PAGE_INVALID));
1282 	}
1283 	/* Nothing to do */
1284 	if (!mm_is_protected(mm) || !pte_present(res))
1285 		return res;
1286 	/*
1287 	 * At this point the reference through the mapping is still present.
1288 	 * The notifier should have destroyed all protected vCPUs at this
1289 	 * point, so the destroy should be successful.
1290 	 */
1291 	if (full && !uv_destroy_pte(res))
1292 		return res;
1293 	/*
1294 	 * If something went wrong and the page could not be destroyed, or
1295 	 * if this is not a mm teardown, the slower export is used as
1296 	 * fallback instead.
1297 	 */
1298 	uv_convert_from_secure_pte(res);
1299 	return res;
1300 }
1301 
1302 #define __HAVE_ARCH_PTEP_SET_WRPROTECT
ptep_set_wrprotect(struct mm_struct * mm,unsigned long addr,pte_t * ptep)1303 static inline void ptep_set_wrprotect(struct mm_struct *mm,
1304 				      unsigned long addr, pte_t *ptep)
1305 {
1306 	pte_t pte = *ptep;
1307 
1308 	if (pte_write(pte))
1309 		ptep_xchg_lazy(mm, addr, ptep, pte_wrprotect(pte));
1310 }
1311 
1312 /*
1313  * Check if PTEs only differ in _PAGE_PROTECT HW bit, but also allow SW PTE
1314  * bits in the comparison. Those might change e.g. because of dirty and young
1315  * tracking.
1316  */
pte_allow_rdp(pte_t old,pte_t new)1317 static inline int pte_allow_rdp(pte_t old, pte_t new)
1318 {
1319 	/*
1320 	 * Only allow changes from RO to RW
1321 	 */
1322 	if (!(pte_val(old) & _PAGE_PROTECT) || pte_val(new) & _PAGE_PROTECT)
1323 		return 0;
1324 
1325 	return (pte_val(old) & _PAGE_RDP_MASK) == (pte_val(new) & _PAGE_RDP_MASK);
1326 }
1327 
flush_tlb_fix_spurious_fault(struct vm_area_struct * vma,unsigned long address,pte_t * ptep)1328 static inline void flush_tlb_fix_spurious_fault(struct vm_area_struct *vma,
1329 						unsigned long address,
1330 						pte_t *ptep)
1331 {
1332 	/*
1333 	 * RDP might not have propagated the PTE protection reset to all CPUs,
1334 	 * so there could be spurious TLB protection faults.
1335 	 * NOTE: This will also be called when a racing pagetable update on
1336 	 * another thread already installed the correct PTE. Both cases cannot
1337 	 * really be distinguished.
1338 	 * Therefore, only do the local TLB flush when RDP can be used, and the
1339 	 * PTE does not have _PAGE_PROTECT set, to avoid unnecessary overhead.
1340 	 * A local RDP can be used to do the flush.
1341 	 */
1342 	if (MACHINE_HAS_RDP && !(pte_val(*ptep) & _PAGE_PROTECT))
1343 		__ptep_rdp(address, ptep, 0, 0, 1);
1344 }
1345 #define flush_tlb_fix_spurious_fault flush_tlb_fix_spurious_fault
1346 
1347 void ptep_reset_dat_prot(struct mm_struct *mm, unsigned long addr, pte_t *ptep,
1348 			 pte_t new);
1349 
1350 #define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
ptep_set_access_flags(struct vm_area_struct * vma,unsigned long addr,pte_t * ptep,pte_t entry,int dirty)1351 static inline int ptep_set_access_flags(struct vm_area_struct *vma,
1352 					unsigned long addr, pte_t *ptep,
1353 					pte_t entry, int dirty)
1354 {
1355 	if (pte_same(*ptep, entry))
1356 		return 0;
1357 	if (MACHINE_HAS_RDP && !mm_has_pgste(vma->vm_mm) && pte_allow_rdp(*ptep, entry))
1358 		ptep_reset_dat_prot(vma->vm_mm, addr, ptep, entry);
1359 	else
1360 		ptep_xchg_direct(vma->vm_mm, addr, ptep, entry);
1361 	return 1;
1362 }
1363 
1364 /*
1365  * Additional functions to handle KVM guest page tables
1366  */
1367 void ptep_set_pte_at(struct mm_struct *mm, unsigned long addr,
1368 		     pte_t *ptep, pte_t entry);
1369 void ptep_set_notify(struct mm_struct *mm, unsigned long addr, pte_t *ptep);
1370 void ptep_notify(struct mm_struct *mm, unsigned long addr,
1371 		 pte_t *ptep, unsigned long bits);
1372 int ptep_force_prot(struct mm_struct *mm, unsigned long gaddr,
1373 		    pte_t *ptep, int prot, unsigned long bit);
1374 void ptep_zap_unused(struct mm_struct *mm, unsigned long addr,
1375 		     pte_t *ptep , int reset);
1376 void ptep_zap_key(struct mm_struct *mm, unsigned long addr, pte_t *ptep);
1377 int ptep_shadow_pte(struct mm_struct *mm, unsigned long saddr,
1378 		    pte_t *sptep, pte_t *tptep, pte_t pte);
1379 void ptep_unshadow_pte(struct mm_struct *mm, unsigned long saddr, pte_t *ptep);
1380 
1381 bool ptep_test_and_clear_uc(struct mm_struct *mm, unsigned long address,
1382 			    pte_t *ptep);
1383 int set_guest_storage_key(struct mm_struct *mm, unsigned long addr,
1384 			  unsigned char key, bool nq);
1385 int cond_set_guest_storage_key(struct mm_struct *mm, unsigned long addr,
1386 			       unsigned char key, unsigned char *oldkey,
1387 			       bool nq, bool mr, bool mc);
1388 int reset_guest_reference_bit(struct mm_struct *mm, unsigned long addr);
1389 int get_guest_storage_key(struct mm_struct *mm, unsigned long addr,
1390 			  unsigned char *key);
1391 
1392 int set_pgste_bits(struct mm_struct *mm, unsigned long addr,
1393 				unsigned long bits, unsigned long value);
1394 int get_pgste(struct mm_struct *mm, unsigned long hva, unsigned long *pgstep);
1395 int pgste_perform_essa(struct mm_struct *mm, unsigned long hva, int orc,
1396 			unsigned long *oldpte, unsigned long *oldpgste);
1397 void gmap_pmdp_csp(struct mm_struct *mm, unsigned long vmaddr);
1398 void gmap_pmdp_invalidate(struct mm_struct *mm, unsigned long vmaddr);
1399 void gmap_pmdp_idte_local(struct mm_struct *mm, unsigned long vmaddr);
1400 void gmap_pmdp_idte_global(struct mm_struct *mm, unsigned long vmaddr);
1401 
1402 #define pgprot_writecombine	pgprot_writecombine
1403 pgprot_t pgprot_writecombine(pgprot_t prot);
1404 
1405 #define pgprot_writethrough	pgprot_writethrough
1406 pgprot_t pgprot_writethrough(pgprot_t prot);
1407 
1408 #define PFN_PTE_SHIFT		PAGE_SHIFT
1409 
1410 /*
1411  * Set multiple PTEs to consecutive pages with a single call.  All PTEs
1412  * are within the same folio, PMD and VMA.
1413  */
set_ptes(struct mm_struct * mm,unsigned long addr,pte_t * ptep,pte_t entry,unsigned int nr)1414 static inline void set_ptes(struct mm_struct *mm, unsigned long addr,
1415 			      pte_t *ptep, pte_t entry, unsigned int nr)
1416 {
1417 	if (pte_present(entry))
1418 		entry = clear_pte_bit(entry, __pgprot(_PAGE_UNUSED));
1419 	if (mm_has_pgste(mm)) {
1420 		for (;;) {
1421 			ptep_set_pte_at(mm, addr, ptep, entry);
1422 			if (--nr == 0)
1423 				break;
1424 			ptep++;
1425 			entry = __pte(pte_val(entry) + PAGE_SIZE);
1426 			addr += PAGE_SIZE;
1427 		}
1428 	} else {
1429 		for (;;) {
1430 			set_pte(ptep, entry);
1431 			if (--nr == 0)
1432 				break;
1433 			ptep++;
1434 			entry = __pte(pte_val(entry) + PAGE_SIZE);
1435 		}
1436 	}
1437 }
1438 #define set_ptes set_ptes
1439 
1440 /*
1441  * Conversion functions: convert a page and protection to a page entry,
1442  * and a page entry and page directory to the page they refer to.
1443  */
mk_pte_phys(unsigned long physpage,pgprot_t pgprot)1444 static inline pte_t mk_pte_phys(unsigned long physpage, pgprot_t pgprot)
1445 {
1446 	pte_t __pte;
1447 
1448 	__pte = __pte(physpage | pgprot_val(pgprot));
1449 	return pte_mkyoung(__pte);
1450 }
1451 
mk_pte(struct page * page,pgprot_t pgprot)1452 static inline pte_t mk_pte(struct page *page, pgprot_t pgprot)
1453 {
1454 	unsigned long physpage = page_to_phys(page);
1455 	pte_t __pte = mk_pte_phys(physpage, pgprot);
1456 
1457 	if (pte_write(__pte) && PageDirty(page))
1458 		__pte = pte_mkdirty(__pte);
1459 	return __pte;
1460 }
1461 
1462 #define pgd_index(address) (((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD-1))
1463 #define p4d_index(address) (((address) >> P4D_SHIFT) & (PTRS_PER_P4D-1))
1464 #define pud_index(address) (((address) >> PUD_SHIFT) & (PTRS_PER_PUD-1))
1465 #define pmd_index(address) (((address) >> PMD_SHIFT) & (PTRS_PER_PMD-1))
1466 
1467 #define p4d_deref(pud) ((unsigned long)__va(p4d_val(pud) & _REGION_ENTRY_ORIGIN))
1468 #define pgd_deref(pgd) ((unsigned long)__va(pgd_val(pgd) & _REGION_ENTRY_ORIGIN))
1469 
pmd_deref(pmd_t pmd)1470 static inline unsigned long pmd_deref(pmd_t pmd)
1471 {
1472 	unsigned long origin_mask;
1473 
1474 	origin_mask = _SEGMENT_ENTRY_ORIGIN;
1475 	if (pmd_leaf(pmd))
1476 		origin_mask = _SEGMENT_ENTRY_ORIGIN_LARGE;
1477 	return (unsigned long)__va(pmd_val(pmd) & origin_mask);
1478 }
1479 
pmd_pfn(pmd_t pmd)1480 static inline unsigned long pmd_pfn(pmd_t pmd)
1481 {
1482 	return __pa(pmd_deref(pmd)) >> PAGE_SHIFT;
1483 }
1484 
pud_deref(pud_t pud)1485 static inline unsigned long pud_deref(pud_t pud)
1486 {
1487 	unsigned long origin_mask;
1488 
1489 	origin_mask = _REGION_ENTRY_ORIGIN;
1490 	if (pud_leaf(pud))
1491 		origin_mask = _REGION3_ENTRY_ORIGIN_LARGE;
1492 	return (unsigned long)__va(pud_val(pud) & origin_mask);
1493 }
1494 
1495 #define pud_pfn pud_pfn
pud_pfn(pud_t pud)1496 static inline unsigned long pud_pfn(pud_t pud)
1497 {
1498 	return __pa(pud_deref(pud)) >> PAGE_SHIFT;
1499 }
1500 
1501 /*
1502  * The pgd_offset function *always* adds the index for the top-level
1503  * region/segment table. This is done to get a sequence like the
1504  * following to work:
1505  *	pgdp = pgd_offset(current->mm, addr);
1506  *	pgd = READ_ONCE(*pgdp);
1507  *	p4dp = p4d_offset(&pgd, addr);
1508  *	...
1509  * The subsequent p4d_offset, pud_offset and pmd_offset functions
1510  * only add an index if they dereferenced the pointer.
1511  */
pgd_offset_raw(pgd_t * pgd,unsigned long address)1512 static inline pgd_t *pgd_offset_raw(pgd_t *pgd, unsigned long address)
1513 {
1514 	unsigned long rste;
1515 	unsigned int shift;
1516 
1517 	/* Get the first entry of the top level table */
1518 	rste = pgd_val(*pgd);
1519 	/* Pick up the shift from the table type of the first entry */
1520 	shift = ((rste & _REGION_ENTRY_TYPE_MASK) >> 2) * 11 + 20;
1521 	return pgd + ((address >> shift) & (PTRS_PER_PGD - 1));
1522 }
1523 
1524 #define pgd_offset(mm, address) pgd_offset_raw(READ_ONCE((mm)->pgd), address)
1525 
p4d_offset_lockless(pgd_t * pgdp,pgd_t pgd,unsigned long address)1526 static inline p4d_t *p4d_offset_lockless(pgd_t *pgdp, pgd_t pgd, unsigned long address)
1527 {
1528 	if ((pgd_val(pgd) & _REGION_ENTRY_TYPE_MASK) >= _REGION_ENTRY_TYPE_R1)
1529 		return (p4d_t *) pgd_deref(pgd) + p4d_index(address);
1530 	return (p4d_t *) pgdp;
1531 }
1532 #define p4d_offset_lockless p4d_offset_lockless
1533 
p4d_offset(pgd_t * pgdp,unsigned long address)1534 static inline p4d_t *p4d_offset(pgd_t *pgdp, unsigned long address)
1535 {
1536 	return p4d_offset_lockless(pgdp, *pgdp, address);
1537 }
1538 
pud_offset_lockless(p4d_t * p4dp,p4d_t p4d,unsigned long address)1539 static inline pud_t *pud_offset_lockless(p4d_t *p4dp, p4d_t p4d, unsigned long address)
1540 {
1541 	if ((p4d_val(p4d) & _REGION_ENTRY_TYPE_MASK) >= _REGION_ENTRY_TYPE_R2)
1542 		return (pud_t *) p4d_deref(p4d) + pud_index(address);
1543 	return (pud_t *) p4dp;
1544 }
1545 #define pud_offset_lockless pud_offset_lockless
1546 
pud_offset(p4d_t * p4dp,unsigned long address)1547 static inline pud_t *pud_offset(p4d_t *p4dp, unsigned long address)
1548 {
1549 	return pud_offset_lockless(p4dp, *p4dp, address);
1550 }
1551 #define pud_offset pud_offset
1552 
pmd_offset_lockless(pud_t * pudp,pud_t pud,unsigned long address)1553 static inline pmd_t *pmd_offset_lockless(pud_t *pudp, pud_t pud, unsigned long address)
1554 {
1555 	if ((pud_val(pud) & _REGION_ENTRY_TYPE_MASK) >= _REGION_ENTRY_TYPE_R3)
1556 		return (pmd_t *) pud_deref(pud) + pmd_index(address);
1557 	return (pmd_t *) pudp;
1558 }
1559 #define pmd_offset_lockless pmd_offset_lockless
1560 
pmd_offset(pud_t * pudp,unsigned long address)1561 static inline pmd_t *pmd_offset(pud_t *pudp, unsigned long address)
1562 {
1563 	return pmd_offset_lockless(pudp, *pudp, address);
1564 }
1565 #define pmd_offset pmd_offset
1566 
pmd_page_vaddr(pmd_t pmd)1567 static inline unsigned long pmd_page_vaddr(pmd_t pmd)
1568 {
1569 	return (unsigned long) pmd_deref(pmd);
1570 }
1571 
gup_fast_permitted(unsigned long start,unsigned long end)1572 static inline bool gup_fast_permitted(unsigned long start, unsigned long end)
1573 {
1574 	return end <= current->mm->context.asce_limit;
1575 }
1576 #define gup_fast_permitted gup_fast_permitted
1577 
1578 #define pfn_pte(pfn, pgprot)	mk_pte_phys(((pfn) << PAGE_SHIFT), (pgprot))
1579 #define pte_pfn(x) (pte_val(x) >> PAGE_SHIFT)
1580 #define pte_page(x) pfn_to_page(pte_pfn(x))
1581 
1582 #define pmd_page(pmd) pfn_to_page(pmd_pfn(pmd))
1583 #define pud_page(pud) pfn_to_page(pud_pfn(pud))
1584 #define p4d_page(p4d) pfn_to_page(p4d_pfn(p4d))
1585 #define pgd_page(pgd) pfn_to_page(pgd_pfn(pgd))
1586 
pmd_wrprotect(pmd_t pmd)1587 static inline pmd_t pmd_wrprotect(pmd_t pmd)
1588 {
1589 	pmd = clear_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_WRITE));
1590 	return set_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_PROTECT));
1591 }
1592 
pmd_mkwrite_novma(pmd_t pmd)1593 static inline pmd_t pmd_mkwrite_novma(pmd_t pmd)
1594 {
1595 	pmd = set_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_WRITE));
1596 	if (pmd_val(pmd) & _SEGMENT_ENTRY_DIRTY)
1597 		pmd = clear_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_PROTECT));
1598 	return pmd;
1599 }
1600 
pmd_mkclean(pmd_t pmd)1601 static inline pmd_t pmd_mkclean(pmd_t pmd)
1602 {
1603 	pmd = clear_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_DIRTY));
1604 	return set_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_PROTECT));
1605 }
1606 
pmd_mkdirty(pmd_t pmd)1607 static inline pmd_t pmd_mkdirty(pmd_t pmd)
1608 {
1609 	pmd = set_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_DIRTY | _SEGMENT_ENTRY_SOFT_DIRTY));
1610 	if (pmd_val(pmd) & _SEGMENT_ENTRY_WRITE)
1611 		pmd = clear_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_PROTECT));
1612 	return pmd;
1613 }
1614 
pud_wrprotect(pud_t pud)1615 static inline pud_t pud_wrprotect(pud_t pud)
1616 {
1617 	pud = clear_pud_bit(pud, __pgprot(_REGION3_ENTRY_WRITE));
1618 	return set_pud_bit(pud, __pgprot(_REGION_ENTRY_PROTECT));
1619 }
1620 
pud_mkwrite(pud_t pud)1621 static inline pud_t pud_mkwrite(pud_t pud)
1622 {
1623 	pud = set_pud_bit(pud, __pgprot(_REGION3_ENTRY_WRITE));
1624 	if (pud_val(pud) & _REGION3_ENTRY_DIRTY)
1625 		pud = clear_pud_bit(pud, __pgprot(_REGION_ENTRY_PROTECT));
1626 	return pud;
1627 }
1628 
pud_mkclean(pud_t pud)1629 static inline pud_t pud_mkclean(pud_t pud)
1630 {
1631 	pud = clear_pud_bit(pud, __pgprot(_REGION3_ENTRY_DIRTY));
1632 	return set_pud_bit(pud, __pgprot(_REGION_ENTRY_PROTECT));
1633 }
1634 
pud_mkdirty(pud_t pud)1635 static inline pud_t pud_mkdirty(pud_t pud)
1636 {
1637 	pud = set_pud_bit(pud, __pgprot(_REGION3_ENTRY_DIRTY | _REGION3_ENTRY_SOFT_DIRTY));
1638 	if (pud_val(pud) & _REGION3_ENTRY_WRITE)
1639 		pud = clear_pud_bit(pud, __pgprot(_REGION_ENTRY_PROTECT));
1640 	return pud;
1641 }
1642 
1643 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLB_PAGE)
massage_pgprot_pmd(pgprot_t pgprot)1644 static inline unsigned long massage_pgprot_pmd(pgprot_t pgprot)
1645 {
1646 	/*
1647 	 * pgprot is PAGE_NONE, PAGE_RO, PAGE_RX, PAGE_RW or PAGE_RWX
1648 	 * (see __Pxxx / __Sxxx). Convert to segment table entry format.
1649 	 */
1650 	if (pgprot_val(pgprot) == pgprot_val(PAGE_NONE))
1651 		return pgprot_val(SEGMENT_NONE);
1652 	if (pgprot_val(pgprot) == pgprot_val(PAGE_RO))
1653 		return pgprot_val(SEGMENT_RO);
1654 	if (pgprot_val(pgprot) == pgprot_val(PAGE_RX))
1655 		return pgprot_val(SEGMENT_RX);
1656 	if (pgprot_val(pgprot) == pgprot_val(PAGE_RW))
1657 		return pgprot_val(SEGMENT_RW);
1658 	return pgprot_val(SEGMENT_RWX);
1659 }
1660 
pmd_mkyoung(pmd_t pmd)1661 static inline pmd_t pmd_mkyoung(pmd_t pmd)
1662 {
1663 	pmd = set_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_YOUNG));
1664 	if (pmd_val(pmd) & _SEGMENT_ENTRY_READ)
1665 		pmd = clear_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_INVALID));
1666 	return pmd;
1667 }
1668 
pmd_mkold(pmd_t pmd)1669 static inline pmd_t pmd_mkold(pmd_t pmd)
1670 {
1671 	pmd = clear_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_YOUNG));
1672 	return set_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_INVALID));
1673 }
1674 
pmd_modify(pmd_t pmd,pgprot_t newprot)1675 static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot)
1676 {
1677 	unsigned long mask;
1678 
1679 	mask  = _SEGMENT_ENTRY_ORIGIN_LARGE;
1680 	mask |= _SEGMENT_ENTRY_DIRTY;
1681 	mask |= _SEGMENT_ENTRY_YOUNG;
1682 	mask |=	_SEGMENT_ENTRY_LARGE;
1683 	mask |= _SEGMENT_ENTRY_SOFT_DIRTY;
1684 	pmd = __pmd(pmd_val(pmd) & mask);
1685 	pmd = set_pmd_bit(pmd, __pgprot(massage_pgprot_pmd(newprot)));
1686 	if (!(pmd_val(pmd) & _SEGMENT_ENTRY_DIRTY))
1687 		pmd = set_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_PROTECT));
1688 	if (!(pmd_val(pmd) & _SEGMENT_ENTRY_YOUNG))
1689 		pmd = set_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_INVALID));
1690 	return pmd;
1691 }
1692 
mk_pmd_phys(unsigned long physpage,pgprot_t pgprot)1693 static inline pmd_t mk_pmd_phys(unsigned long physpage, pgprot_t pgprot)
1694 {
1695 	return __pmd(physpage + massage_pgprot_pmd(pgprot));
1696 }
1697 
1698 #endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLB_PAGE */
1699 
__pmdp_csp(pmd_t * pmdp)1700 static inline void __pmdp_csp(pmd_t *pmdp)
1701 {
1702 	csp((unsigned int *)pmdp + 1, pmd_val(*pmdp),
1703 	    pmd_val(*pmdp) | _SEGMENT_ENTRY_INVALID);
1704 }
1705 
1706 #define IDTE_GLOBAL	0
1707 #define IDTE_LOCAL	1
1708 
1709 #define IDTE_PTOA	0x0800
1710 #define IDTE_NODAT	0x1000
1711 #define IDTE_GUEST_ASCE	0x2000
1712 
__pmdp_idte(unsigned long addr,pmd_t * pmdp,unsigned long opt,unsigned long asce,int local)1713 static __always_inline void __pmdp_idte(unsigned long addr, pmd_t *pmdp,
1714 					unsigned long opt, unsigned long asce,
1715 					int local)
1716 {
1717 	unsigned long sto;
1718 
1719 	sto = __pa(pmdp) - pmd_index(addr) * sizeof(pmd_t);
1720 	if (__builtin_constant_p(opt) && opt == 0) {
1721 		/* flush without guest asce */
1722 		asm volatile(
1723 			"	idte	%[r1],0,%[r2],%[m4]"
1724 			: "+m" (*pmdp)
1725 			: [r1] "a" (sto), [r2] "a" ((addr & HPAGE_MASK)),
1726 			  [m4] "i" (local)
1727 			: "cc" );
1728 	} else {
1729 		/* flush with guest asce */
1730 		asm volatile(
1731 			"	idte	%[r1],%[r3],%[r2],%[m4]"
1732 			: "+m" (*pmdp)
1733 			: [r1] "a" (sto), [r2] "a" ((addr & HPAGE_MASK) | opt),
1734 			  [r3] "a" (asce), [m4] "i" (local)
1735 			: "cc" );
1736 	}
1737 }
1738 
__pudp_idte(unsigned long addr,pud_t * pudp,unsigned long opt,unsigned long asce,int local)1739 static __always_inline void __pudp_idte(unsigned long addr, pud_t *pudp,
1740 					unsigned long opt, unsigned long asce,
1741 					int local)
1742 {
1743 	unsigned long r3o;
1744 
1745 	r3o = __pa(pudp) - pud_index(addr) * sizeof(pud_t);
1746 	r3o |= _ASCE_TYPE_REGION3;
1747 	if (__builtin_constant_p(opt) && opt == 0) {
1748 		/* flush without guest asce */
1749 		asm volatile(
1750 			"	idte	%[r1],0,%[r2],%[m4]"
1751 			: "+m" (*pudp)
1752 			: [r1] "a" (r3o), [r2] "a" ((addr & PUD_MASK)),
1753 			  [m4] "i" (local)
1754 			: "cc");
1755 	} else {
1756 		/* flush with guest asce */
1757 		asm volatile(
1758 			"	idte	%[r1],%[r3],%[r2],%[m4]"
1759 			: "+m" (*pudp)
1760 			: [r1] "a" (r3o), [r2] "a" ((addr & PUD_MASK) | opt),
1761 			  [r3] "a" (asce), [m4] "i" (local)
1762 			: "cc" );
1763 	}
1764 }
1765 
1766 pmd_t pmdp_xchg_direct(struct mm_struct *, unsigned long, pmd_t *, pmd_t);
1767 pmd_t pmdp_xchg_lazy(struct mm_struct *, unsigned long, pmd_t *, pmd_t);
1768 pud_t pudp_xchg_direct(struct mm_struct *, unsigned long, pud_t *, pud_t);
1769 
1770 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1771 
1772 #define __HAVE_ARCH_PGTABLE_DEPOSIT
1773 void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
1774 				pgtable_t pgtable);
1775 
1776 #define __HAVE_ARCH_PGTABLE_WITHDRAW
1777 pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp);
1778 
1779 #define  __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS
pmdp_set_access_flags(struct vm_area_struct * vma,unsigned long addr,pmd_t * pmdp,pmd_t entry,int dirty)1780 static inline int pmdp_set_access_flags(struct vm_area_struct *vma,
1781 					unsigned long addr, pmd_t *pmdp,
1782 					pmd_t entry, int dirty)
1783 {
1784 	VM_BUG_ON(addr & ~HPAGE_MASK);
1785 
1786 	entry = pmd_mkyoung(entry);
1787 	if (dirty)
1788 		entry = pmd_mkdirty(entry);
1789 	if (pmd_val(*pmdp) == pmd_val(entry))
1790 		return 0;
1791 	pmdp_xchg_direct(vma->vm_mm, addr, pmdp, entry);
1792 	return 1;
1793 }
1794 
1795 #define __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG
pmdp_test_and_clear_young(struct vm_area_struct * vma,unsigned long addr,pmd_t * pmdp)1796 static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma,
1797 					    unsigned long addr, pmd_t *pmdp)
1798 {
1799 	pmd_t pmd = *pmdp;
1800 
1801 	pmd = pmdp_xchg_direct(vma->vm_mm, addr, pmdp, pmd_mkold(pmd));
1802 	return pmd_young(pmd);
1803 }
1804 
1805 #define __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH
pmdp_clear_flush_young(struct vm_area_struct * vma,unsigned long addr,pmd_t * pmdp)1806 static inline int pmdp_clear_flush_young(struct vm_area_struct *vma,
1807 					 unsigned long addr, pmd_t *pmdp)
1808 {
1809 	VM_BUG_ON(addr & ~HPAGE_MASK);
1810 	return pmdp_test_and_clear_young(vma, addr, pmdp);
1811 }
1812 
set_pmd_at(struct mm_struct * mm,unsigned long addr,pmd_t * pmdp,pmd_t entry)1813 static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr,
1814 			      pmd_t *pmdp, pmd_t entry)
1815 {
1816 	set_pmd(pmdp, entry);
1817 }
1818 
pmd_mkhuge(pmd_t pmd)1819 static inline pmd_t pmd_mkhuge(pmd_t pmd)
1820 {
1821 	pmd = set_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_LARGE));
1822 	pmd = set_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_YOUNG));
1823 	return set_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_PROTECT));
1824 }
1825 
1826 #define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR
pmdp_huge_get_and_clear(struct mm_struct * mm,unsigned long addr,pmd_t * pmdp)1827 static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm,
1828 					    unsigned long addr, pmd_t *pmdp)
1829 {
1830 	return pmdp_xchg_direct(mm, addr, pmdp, __pmd(_SEGMENT_ENTRY_EMPTY));
1831 }
1832 
1833 #define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR_FULL
pmdp_huge_get_and_clear_full(struct vm_area_struct * vma,unsigned long addr,pmd_t * pmdp,int full)1834 static inline pmd_t pmdp_huge_get_and_clear_full(struct vm_area_struct *vma,
1835 						 unsigned long addr,
1836 						 pmd_t *pmdp, int full)
1837 {
1838 	if (full) {
1839 		pmd_t pmd = *pmdp;
1840 		set_pmd(pmdp, __pmd(_SEGMENT_ENTRY_EMPTY));
1841 		return pmd;
1842 	}
1843 	return pmdp_xchg_lazy(vma->vm_mm, addr, pmdp, __pmd(_SEGMENT_ENTRY_EMPTY));
1844 }
1845 
1846 #define __HAVE_ARCH_PMDP_HUGE_CLEAR_FLUSH
pmdp_huge_clear_flush(struct vm_area_struct * vma,unsigned long addr,pmd_t * pmdp)1847 static inline pmd_t pmdp_huge_clear_flush(struct vm_area_struct *vma,
1848 					  unsigned long addr, pmd_t *pmdp)
1849 {
1850 	return pmdp_huge_get_and_clear(vma->vm_mm, addr, pmdp);
1851 }
1852 
1853 #define __HAVE_ARCH_PMDP_INVALIDATE
pmdp_invalidate(struct vm_area_struct * vma,unsigned long addr,pmd_t * pmdp)1854 static inline pmd_t pmdp_invalidate(struct vm_area_struct *vma,
1855 				   unsigned long addr, pmd_t *pmdp)
1856 {
1857 	pmd_t pmd;
1858 
1859 	VM_WARN_ON_ONCE(!pmd_present(*pmdp));
1860 	pmd = __pmd(pmd_val(*pmdp) | _SEGMENT_ENTRY_INVALID);
1861 	return pmdp_xchg_direct(vma->vm_mm, addr, pmdp, pmd);
1862 }
1863 
1864 #define __HAVE_ARCH_PMDP_SET_WRPROTECT
pmdp_set_wrprotect(struct mm_struct * mm,unsigned long addr,pmd_t * pmdp)1865 static inline void pmdp_set_wrprotect(struct mm_struct *mm,
1866 				      unsigned long addr, pmd_t *pmdp)
1867 {
1868 	pmd_t pmd = *pmdp;
1869 
1870 	if (pmd_write(pmd))
1871 		pmd = pmdp_xchg_lazy(mm, addr, pmdp, pmd_wrprotect(pmd));
1872 }
1873 
pmdp_collapse_flush(struct vm_area_struct * vma,unsigned long address,pmd_t * pmdp)1874 static inline pmd_t pmdp_collapse_flush(struct vm_area_struct *vma,
1875 					unsigned long address,
1876 					pmd_t *pmdp)
1877 {
1878 	return pmdp_huge_get_and_clear(vma->vm_mm, address, pmdp);
1879 }
1880 #define pmdp_collapse_flush pmdp_collapse_flush
1881 
1882 #define pfn_pmd(pfn, pgprot)	mk_pmd_phys(((pfn) << PAGE_SHIFT), (pgprot))
1883 #define mk_pmd(page, pgprot)	pfn_pmd(page_to_pfn(page), (pgprot))
1884 
pmd_trans_huge(pmd_t pmd)1885 static inline int pmd_trans_huge(pmd_t pmd)
1886 {
1887 	return pmd_leaf(pmd);
1888 }
1889 
1890 #define has_transparent_hugepage has_transparent_hugepage
has_transparent_hugepage(void)1891 static inline int has_transparent_hugepage(void)
1892 {
1893 	return MACHINE_HAS_EDAT1 ? 1 : 0;
1894 }
1895 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
1896 
1897 /*
1898  * 64 bit swap entry format:
1899  * A page-table entry has some bits we have to treat in a special way.
1900  * Bits 54 and 63 are used to indicate the page type. Bit 53 marks the pte
1901  * as invalid.
1902  * A swap pte is indicated by bit pattern (pte & 0x201) == 0x200
1903  * |			  offset			|E11XX|type |S0|
1904  * |0000000000111111111122222222223333333333444444444455|55555|55566|66|
1905  * |0123456789012345678901234567890123456789012345678901|23456|78901|23|
1906  *
1907  * Bits 0-51 store the offset.
1908  * Bit 52 (E) is used to remember PG_anon_exclusive.
1909  * Bits 57-61 store the type.
1910  * Bit 62 (S) is used for softdirty tracking.
1911  * Bits 55 and 56 (X) are unused.
1912  */
1913 
1914 #define __SWP_OFFSET_MASK	((1UL << 52) - 1)
1915 #define __SWP_OFFSET_SHIFT	12
1916 #define __SWP_TYPE_MASK		((1UL << 5) - 1)
1917 #define __SWP_TYPE_SHIFT	2
1918 
mk_swap_pte(unsigned long type,unsigned long offset)1919 static inline pte_t mk_swap_pte(unsigned long type, unsigned long offset)
1920 {
1921 	unsigned long pteval;
1922 
1923 	pteval = _PAGE_INVALID | _PAGE_PROTECT;
1924 	pteval |= (offset & __SWP_OFFSET_MASK) << __SWP_OFFSET_SHIFT;
1925 	pteval |= (type & __SWP_TYPE_MASK) << __SWP_TYPE_SHIFT;
1926 	return __pte(pteval);
1927 }
1928 
__swp_type(swp_entry_t entry)1929 static inline unsigned long __swp_type(swp_entry_t entry)
1930 {
1931 	return (entry.val >> __SWP_TYPE_SHIFT) & __SWP_TYPE_MASK;
1932 }
1933 
__swp_offset(swp_entry_t entry)1934 static inline unsigned long __swp_offset(swp_entry_t entry)
1935 {
1936 	return (entry.val >> __SWP_OFFSET_SHIFT) & __SWP_OFFSET_MASK;
1937 }
1938 
__swp_entry(unsigned long type,unsigned long offset)1939 static inline swp_entry_t __swp_entry(unsigned long type, unsigned long offset)
1940 {
1941 	return (swp_entry_t) { pte_val(mk_swap_pte(type, offset)) };
1942 }
1943 
1944 #define __pte_to_swp_entry(pte)	((swp_entry_t) { pte_val(pte) })
1945 #define __swp_entry_to_pte(x)	((pte_t) { (x).val })
1946 
1947 /*
1948  * 64 bit swap entry format for REGION3 and SEGMENT table entries (RSTE)
1949  * Bits 59 and 63 are used to indicate the swap entry. Bit 58 marks the rste
1950  * as invalid.
1951  * A swap entry is indicated by bit pattern (rste & 0x011) == 0x010
1952  * |			  offset			|Xtype |11TT|S0|
1953  * |0000000000111111111122222222223333333333444444444455|555555|5566|66|
1954  * |0123456789012345678901234567890123456789012345678901|234567|8901|23|
1955  *
1956  * Bits 0-51 store the offset.
1957  * Bits 53-57 store the type.
1958  * Bit 62 (S) is used for softdirty tracking.
1959  * Bits 60-61 (TT) indicate the table type: 0x01 for REGION3 and 0x00 for SEGMENT.
1960  * Bit 52 (X) is unused.
1961  */
1962 
1963 #define __SWP_OFFSET_MASK_RSTE	((1UL << 52) - 1)
1964 #define __SWP_OFFSET_SHIFT_RSTE	12
1965 #define __SWP_TYPE_MASK_RSTE		((1UL << 5) - 1)
1966 #define __SWP_TYPE_SHIFT_RSTE	6
1967 
1968 /*
1969  * TT bits set to 0x00 == SEGMENT. For REGION3 entries, caller must add R3
1970  * bits 0x01. See also __set_huge_pte_at().
1971  */
mk_swap_rste(unsigned long type,unsigned long offset)1972 static inline unsigned long mk_swap_rste(unsigned long type, unsigned long offset)
1973 {
1974 	unsigned long rste;
1975 
1976 	rste = _RST_ENTRY_INVALID | _RST_ENTRY_COMM;
1977 	rste |= (offset & __SWP_OFFSET_MASK_RSTE) << __SWP_OFFSET_SHIFT_RSTE;
1978 	rste |= (type & __SWP_TYPE_MASK_RSTE) << __SWP_TYPE_SHIFT_RSTE;
1979 	return rste;
1980 }
1981 
__swp_type_rste(swp_entry_t entry)1982 static inline unsigned long __swp_type_rste(swp_entry_t entry)
1983 {
1984 	return (entry.val >> __SWP_TYPE_SHIFT_RSTE) & __SWP_TYPE_MASK_RSTE;
1985 }
1986 
__swp_offset_rste(swp_entry_t entry)1987 static inline unsigned long __swp_offset_rste(swp_entry_t entry)
1988 {
1989 	return (entry.val >> __SWP_OFFSET_SHIFT_RSTE) & __SWP_OFFSET_MASK_RSTE;
1990 }
1991 
1992 #define __rste_to_swp_entry(rste)	((swp_entry_t) { rste })
1993 
1994 extern int vmem_add_mapping(unsigned long start, unsigned long size);
1995 extern void vmem_remove_mapping(unsigned long start, unsigned long size);
1996 extern int __vmem_map_4k_page(unsigned long addr, unsigned long phys, pgprot_t prot, bool alloc);
1997 extern int vmem_map_4k_page(unsigned long addr, unsigned long phys, pgprot_t prot);
1998 extern void vmem_unmap_4k_page(unsigned long addr);
1999 extern pte_t *vmem_get_alloc_pte(unsigned long addr, bool alloc);
2000 extern int s390_enable_sie(void);
2001 extern int s390_enable_skey(void);
2002 extern void s390_reset_cmma(struct mm_struct *mm);
2003 
2004 /* s390 has a private copy of get unmapped area to deal with cache synonyms */
2005 #define HAVE_ARCH_UNMAPPED_AREA
2006 #define HAVE_ARCH_UNMAPPED_AREA_TOPDOWN
2007 
2008 #define pmd_pgtable(pmd) \
2009 	((pgtable_t)__va(pmd_val(pmd) & -sizeof(pte_t)*PTRS_PER_PTE))
2010 
gmap_pgste_get_pgt_addr(unsigned long * pgt)2011 static inline unsigned long gmap_pgste_get_pgt_addr(unsigned long *pgt)
2012 {
2013 	unsigned long *pgstes, res;
2014 
2015 	pgstes = pgt + _PAGE_ENTRIES;
2016 
2017 	res = (pgstes[0] & PGSTE_ST2_MASK) << 16;
2018 	res |= pgstes[1] & PGSTE_ST2_MASK;
2019 	res |= (pgstes[2] & PGSTE_ST2_MASK) >> 16;
2020 	res |= (pgstes[3] & PGSTE_ST2_MASK) >> 32;
2021 
2022 	return res;
2023 }
2024 
2025 #endif /* _S390_PAGE_H */
2026