1 // SPDX-License-Identifier: GPL-2.0
5 #include <asm/page-states.h>
40 static void pgtable_populate(unsigned long addr, unsigned long end, enum populate_mode mode);
53 static inline void kasan_populate(unsigned long start, unsigned long end, enum populate_mode mode)  in kasan_populate()  argument
57 	pgtable_populate(start, end, mode);  in kasan_populate()
110 					   unsigned long end, enum populate_mode mode)  in kasan_pgd_populate_zero_shadow()  argument
112 	if (mode == POPULATE_KASAN_ZERO_SHADOW &&  in kasan_pgd_populate_zero_shadow()
113 	    IS_ALIGNED(addr, PGDIR_SIZE) && end - addr >= PGDIR_SIZE) {  in kasan_pgd_populate_zero_shadow()
121 					   unsigned long end, enum populate_mode mode)  in kasan_p4d_populate_zero_shadow()  argument
123 	if (mode == POPULATE_KASAN_ZERO_SHADOW &&  in kasan_p4d_populate_zero_shadow()
124 	    IS_ALIGNED(addr, P4D_SIZE) && end - addr >= P4D_SIZE) {  in kasan_p4d_populate_zero_shadow()
132 					   unsigned long end, enum populate_mode mode)  in kasan_pud_populate_zero_shadow()  argument
134 	if (mode == POPULATE_KASAN_ZERO_SHADOW &&  in kasan_pud_populate_zero_shadow()
135 	    IS_ALIGNED(addr, PUD_SIZE) && end - addr >= PUD_SIZE) {  in kasan_pud_populate_zero_shadow()
143 					   unsigned long end, enum populate_mode mode)  in kasan_pmd_populate_zero_shadow()  argument
145 	if (mode == POPULATE_KASAN_ZERO_SHADOW &&  in kasan_pmd_populate_zero_shadow()
146 	    IS_ALIGNED(addr, PMD_SIZE) && end - addr >= PMD_SIZE) {  in kasan_pmd_populate_zero_shadow()
153 static bool kasan_pte_populate_zero_shadow(pte_t *pte, enum populate_mode mode)  in kasan_pte_populate_zero_shadow()  argument
155 	if (mode == POPULATE_KASAN_ZERO_SHADOW) {  in kasan_pte_populate_zero_shadow()
168 						  unsigned long end, enum populate_mode mode)  in kasan_pgd_populate_zero_shadow()  argument
174 						  unsigned long end, enum populate_mode mode)  in kasan_p4d_populate_zero_shadow()  argument
180 						  unsigned long end, enum populate_mode mode)  in kasan_pud_populate_zero_shadow()  argument
186 						  unsigned long end, enum populate_mode mode)  in kasan_pmd_populate_zero_shadow()  argument
191 static bool kasan_pte_populate_zero_shadow(pte_t *pte, enum populate_mode mode)  in kasan_pte_populate_zero_shadow()  argument
239 static unsigned long _pa(unsigned long addr, unsigned long size, enum populate_mode mode)  in _pa()  argument
241 	switch (mode) {  in _pa()
243 		return -1;  in _pa()
261 		return -1;  in _pa()
265 static bool large_allowed(enum populate_mode mode)  in large_allowed()  argument
267 	return (mode == POPULATE_DIRECT) || (mode == POPULATE_IDENTITY) || (mode == POPULATE_KERNEL);  in large_allowed()
271 			  enum populate_mode mode)  in can_large_pud()  argument
273 	unsigned long size = end - addr;  in can_large_pud()
275 	return machine.has_edat2 && large_allowed(mode) &&  in can_large_pud()
277 	       IS_ALIGNED(_pa(addr, size, mode), PUD_SIZE);  in can_large_pud()
281 			  enum populate_mode mode)  in can_large_pmd()  argument
283 	unsigned long size = end - addr;  in can_large_pmd()
285 	return machine.has_edat1 && large_allowed(mode) &&  in can_large_pmd()
287 	       IS_ALIGNED(_pa(addr, size, mode), PMD_SIZE);  in can_large_pmd()
291 				 enum populate_mode mode)  in pgtable_pte_populate()  argument
299 			if (kasan_pte_populate_zero_shadow(pte, mode))  in pgtable_pte_populate()
301 			entry = __pte(_pa(addr, PAGE_SIZE, mode));  in pgtable_pte_populate()
309 	if (mode == POPULATE_IDENTITY)  in pgtable_pte_populate()
314 				 enum populate_mode mode)  in pgtable_pmd_populate()  argument
316 	unsigned long next, pages = 0;  in pgtable_pmd_populate()  local
321 	for (; addr < end; addr = next, pmd++) {  in pgtable_pmd_populate()
322 		next = pmd_addr_end(addr, end);  in pgtable_pmd_populate()
324 			if (kasan_pmd_populate_zero_shadow(pmd, addr, next, mode))  in pgtable_pmd_populate()
326 			if (can_large_pmd(pmd, addr, next, mode)) {  in pgtable_pmd_populate()
327 				entry = __pmd(_pa(addr, _SEGMENT_SIZE, mode));  in pgtable_pmd_populate()
340 		pgtable_pte_populate(pmd, addr, next, mode);  in pgtable_pmd_populate()
342 	if (mode == POPULATE_IDENTITY)  in pgtable_pmd_populate()
347 				 enum populate_mode mode)  in pgtable_pud_populate()  argument
349 	unsigned long next, pages = 0;  in pgtable_pud_populate()  local
354 	for (; addr < end; addr = next, pud++) {  in pgtable_pud_populate()
355 		next = pud_addr_end(addr, end);  in pgtable_pud_populate()
357 			if (kasan_pud_populate_zero_shadow(pud, addr, next, mode))  in pgtable_pud_populate()
359 			if (can_large_pud(pud, addr, next, mode)) {  in pgtable_pud_populate()
360 				entry = __pud(_pa(addr, _REGION3_SIZE, mode));  in pgtable_pud_populate()
373 		pgtable_pmd_populate(pud, addr, next, mode);  in pgtable_pud_populate()
375 	if (mode == POPULATE_IDENTITY)  in pgtable_pud_populate()
380 				 enum populate_mode mode)  in pgtable_p4d_populate()  argument
382 	unsigned long next;  in pgtable_p4d_populate()  local
387 	for (; addr < end; addr = next, p4d++) {  in pgtable_p4d_populate()
388 		next = p4d_addr_end(addr, end);  in pgtable_p4d_populate()
390 			if (kasan_p4d_populate_zero_shadow(p4d, addr, next, mode))  in pgtable_p4d_populate()
395 		pgtable_pud_populate(p4d, addr, next, mode);  in pgtable_p4d_populate()
399 static void pgtable_populate(unsigned long addr, unsigned long end, enum populate_mode mode)  in pgtable_populate()  argument
401 	unsigned long next;  in pgtable_populate()  local
406 	for (; addr < end; addr = next, pgd++) {  in pgtable_populate()
407 		next = pgd_addr_end(addr, end);  in pgtable_populate()
409 			if (kasan_pgd_populate_zero_shadow(pgd, addr, next, mode))  in pgtable_populate()
415 		if (mode == POPULATE_KASAN_SHALLOW)  in pgtable_populate()
418 		pgtable_p4d_populate(pgd, addr, next, mode);  in pgtable_populate()
432 	 * Mark whole memory as no-dat. This must be done before any  in setup_vmem()
437 		__arch_set_page_nodat((void *)start, (end - start) >> PAGE_SHIFT);  in setup_vmem()
440 	 * init_mm->pgd contains virtual address of swapper_pg_dir.  in setup_vmem()
498 	get_lowcore()->kernel_asce.val = swapper_pg_dir | asce_bits;  in setup_vmem()
499 	get_lowcore()->user_asce = s390_invalid_asce;  in setup_vmem()
501 	local_ctl_load(1, &get_lowcore()->kernel_asce);  in setup_vmem()
502 	local_ctl_load(7, &get_lowcore()->user_asce);  in setup_vmem()
503 	local_ctl_load(13, &get_lowcore()->kernel_asce);  in setup_vmem()
505 	init_mm.context.asce = get_lowcore()->kernel_asce.val;  in setup_vmem()