73 static void pgtable_populate(unsigned long addr, unsigned long end, enum populate_mode mode);
137 static bool kasan_pgd_populate_zero_shadow(pgd_t *pgd, unsigned long addr,  in kasan_pgd_populate_zero_shadow()  argument
141 	    IS_ALIGNED(addr, PGDIR_SIZE) && end - addr >= PGDIR_SIZE) {  in kasan_pgd_populate_zero_shadow()
148 static bool kasan_p4d_populate_zero_shadow(p4d_t *p4d, unsigned long addr,  in kasan_p4d_populate_zero_shadow()  argument
152 	    IS_ALIGNED(addr, P4D_SIZE) && end - addr >= P4D_SIZE) {  in kasan_p4d_populate_zero_shadow()
159 static bool kasan_pud_populate_zero_shadow(pud_t *pud, unsigned long addr,  in kasan_pud_populate_zero_shadow()  argument
163 	    IS_ALIGNED(addr, PUD_SIZE) && end - addr >= PUD_SIZE) {  in kasan_pud_populate_zero_shadow()
170 static bool kasan_pmd_populate_zero_shadow(pmd_t *pmd, unsigned long addr,  in kasan_pmd_populate_zero_shadow()  argument
174 	    IS_ALIGNED(addr, PMD_SIZE) && end - addr >= PMD_SIZE) {  in kasan_pmd_populate_zero_shadow()
195 static inline bool kasan_pgd_populate_zero_shadow(pgd_t *pgd, unsigned long addr,  in kasan_pgd_populate_zero_shadow()  argument
201 static inline bool kasan_p4d_populate_zero_shadow(p4d_t *p4d, unsigned long addr,  in kasan_p4d_populate_zero_shadow()  argument
207 static inline bool kasan_pud_populate_zero_shadow(pud_t *pud, unsigned long addr,  in kasan_pud_populate_zero_shadow()  argument
213 static inline bool kasan_pmd_populate_zero_shadow(pmd_t *pmd, unsigned long addr,  in kasan_pmd_populate_zero_shadow()  argument
255 static unsigned long resolve_pa_may_alloc(unsigned long addr, unsigned long size,  in resolve_pa_may_alloc()  argument
262 		return addr;  in resolve_pa_may_alloc()
264 		return __lowcore_pa(addr);  in resolve_pa_may_alloc()
266 		return __abs_lowcore_pa(addr);  in resolve_pa_may_alloc()
268 		return __kernel_pa(addr);  in resolve_pa_may_alloc()
270 		return __identity_pa(addr);  in resolve_pa_may_alloc()
274 		addr = physmem_alloc(RR_VMEM, size, size, size == PAGE_SIZE);  in resolve_pa_may_alloc()
275 		if (addr) {  in resolve_pa_may_alloc()
276 			memset((void *)addr, 0, size);  in resolve_pa_may_alloc()
277 			return addr;  in resolve_pa_may_alloc()
301 static unsigned long try_get_large_pud_pa(pud_t *pu_dir, unsigned long addr, unsigned long end,  in try_get_large_pud_pa()  argument
304 	unsigned long pa, size = end - addr;  in try_get_large_pud_pa()
307 	    !IS_ALIGNED(addr, PUD_SIZE) || (size < PUD_SIZE))  in try_get_large_pud_pa()
310 	pa = resolve_pa_may_alloc(addr, size, mode);  in try_get_large_pud_pa()
317 static unsigned long try_get_large_pmd_pa(pmd_t *pm_dir, unsigned long addr, unsigned long end,  in try_get_large_pmd_pa()  argument
320 	unsigned long pa, size = end - addr;  in try_get_large_pmd_pa()
323 	    !IS_ALIGNED(addr, PMD_SIZE) || (size < PMD_SIZE))  in try_get_large_pmd_pa()
326 	pa = resolve_pa_may_alloc(addr, size, mode);  in try_get_large_pmd_pa()
333 static void pgtable_pte_populate(pmd_t *pmd, unsigned long addr, unsigned long end,  in pgtable_pte_populate()  argument
339 	pte = pte_offset_kernel(pmd, addr);  in pgtable_pte_populate()
340 	for (; addr < end; addr += PAGE_SIZE, pte++) {  in pgtable_pte_populate()
344 			entry = __pte(resolve_pa_may_alloc(addr, PAGE_SIZE, mode));  in pgtable_pte_populate()
354 static void pgtable_pmd_populate(pud_t *pud, unsigned long addr, unsigned long end,  in pgtable_pmd_populate()  argument
361 	pmd = pmd_offset(pud, addr);  in pgtable_pmd_populate()
362 	for (; addr < end; addr = next, pmd++) {  in pgtable_pmd_populate()
363 		next = pmd_addr_end(addr, end);  in pgtable_pmd_populate()
365 			if (kasan_pmd_populate_zero_shadow(pmd, addr, next, mode))  in pgtable_pmd_populate()
367 			pa = try_get_large_pmd_pa(pmd, addr, next, mode);  in pgtable_pmd_populate()
380 		pgtable_pte_populate(pmd, addr, next, mode);  in pgtable_pmd_populate()
386 static void pgtable_pud_populate(p4d_t *p4d, unsigned long addr, unsigned long end,  in pgtable_pud_populate()  argument
393 	pud = pud_offset(p4d, addr);  in pgtable_pud_populate()
394 	for (; addr < end; addr = next, pud++) {  in pgtable_pud_populate()
395 		next = pud_addr_end(addr, end);  in pgtable_pud_populate()
397 			if (kasan_pud_populate_zero_shadow(pud, addr, next, mode))  in pgtable_pud_populate()
399 			pa = try_get_large_pud_pa(pud, addr, next, mode);  in pgtable_pud_populate()
412 		pgtable_pmd_populate(pud, addr, next, mode);  in pgtable_pud_populate()
418 static void pgtable_p4d_populate(pgd_t *pgd, unsigned long addr, unsigned long end,  in pgtable_p4d_populate()  argument
425 	p4d = p4d_offset(pgd, addr);  in pgtable_p4d_populate()
426 	for (; addr < end; addr = next, p4d++) {  in pgtable_p4d_populate()
427 		next = p4d_addr_end(addr, end);  in pgtable_p4d_populate()
429 			if (kasan_p4d_populate_zero_shadow(p4d, addr, next, mode))  in pgtable_p4d_populate()
434 		pgtable_pud_populate(p4d, addr, next, mode);  in pgtable_p4d_populate()
438 static void pgtable_populate(unsigned long addr, unsigned long end, enum populate_mode mode)  in pgtable_populate()  argument
446 			   get_populate_mode_name(mode), addr, end,  in pgtable_populate()
447 			   resolve_pa_may_alloc(addr, 0, mode),  in pgtable_populate()
451 	pgd = pgd_offset(&init_mm, addr);  in pgtable_populate()
452 	for (; addr < end; addr = next, pgd++) {  in pgtable_populate()
453 		next = pgd_addr_end(addr, end);  in pgtable_populate()
455 			if (kasan_pgd_populate_zero_shadow(pgd, addr, next, mode))  in pgtable_populate()
464 		pgtable_p4d_populate(pgd, addr, next, mode);  in pgtable_populate()