40 static void pgtable_populate(unsigned long addr, unsigned long end, enum populate_mode mode);
109 static bool kasan_pgd_populate_zero_shadow(pgd_t *pgd, unsigned long addr,  in kasan_pgd_populate_zero_shadow()  argument
113 	    IS_ALIGNED(addr, PGDIR_SIZE) && end - addr >= PGDIR_SIZE) {  in kasan_pgd_populate_zero_shadow()
120 static bool kasan_p4d_populate_zero_shadow(p4d_t *p4d, unsigned long addr,  in kasan_p4d_populate_zero_shadow()  argument
124 	    IS_ALIGNED(addr, P4D_SIZE) && end - addr >= P4D_SIZE) {  in kasan_p4d_populate_zero_shadow()
131 static bool kasan_pud_populate_zero_shadow(pud_t *pud, unsigned long addr,  in kasan_pud_populate_zero_shadow()  argument
135 	    IS_ALIGNED(addr, PUD_SIZE) && end - addr >= PUD_SIZE) {  in kasan_pud_populate_zero_shadow()
142 static bool kasan_pmd_populate_zero_shadow(pmd_t *pmd, unsigned long addr,  in kasan_pmd_populate_zero_shadow()  argument
146 	    IS_ALIGNED(addr, PMD_SIZE) && end - addr >= PMD_SIZE) {  in kasan_pmd_populate_zero_shadow()
167 static inline bool kasan_pgd_populate_zero_shadow(pgd_t *pgd, unsigned long addr,  in kasan_pgd_populate_zero_shadow()  argument
173 static inline bool kasan_p4d_populate_zero_shadow(p4d_t *p4d, unsigned long addr,  in kasan_p4d_populate_zero_shadow()  argument
179 static inline bool kasan_pud_populate_zero_shadow(pud_t *pud, unsigned long addr,  in kasan_pud_populate_zero_shadow()  argument
185 static inline bool kasan_pmd_populate_zero_shadow(pmd_t *pmd, unsigned long addr,  in kasan_pmd_populate_zero_shadow()  argument
239 static unsigned long _pa(unsigned long addr, unsigned long size, enum populate_mode mode)  in _pa()  argument
245 		return addr;  in _pa()
247 		return __lowcore_pa(addr);  in _pa()
249 		return __abs_lowcore_pa(addr);  in _pa()
251 		return __kernel_pa(addr);  in _pa()
253 		return __identity_pa(addr);  in _pa()
256 		addr = physmem_alloc_top_down(RR_VMEM, size, size);  in _pa()
257 		memset((void *)addr, 0, size);  in _pa()
258 		return addr;  in _pa()
270 static bool can_large_pud(pud_t *pu_dir, unsigned long addr, unsigned long end,  in can_large_pud()  argument
273 	unsigned long size = end - addr;  in can_large_pud()
276 	       IS_ALIGNED(addr, PUD_SIZE) && (size >= PUD_SIZE) &&  in can_large_pud()
277 	       IS_ALIGNED(_pa(addr, size, mode), PUD_SIZE);  in can_large_pud()
280 static bool can_large_pmd(pmd_t *pm_dir, unsigned long addr, unsigned long end,  in can_large_pmd()  argument
283 	unsigned long size = end - addr;  in can_large_pmd()
286 	       IS_ALIGNED(addr, PMD_SIZE) && (size >= PMD_SIZE) &&  in can_large_pmd()
287 	       IS_ALIGNED(_pa(addr, size, mode), PMD_SIZE);  in can_large_pmd()
290 static void pgtable_pte_populate(pmd_t *pmd, unsigned long addr, unsigned long end,  in pgtable_pte_populate()  argument
296 	pte = pte_offset_kernel(pmd, addr);  in pgtable_pte_populate()
297 	for (; addr < end; addr += PAGE_SIZE, pte++) {  in pgtable_pte_populate()
301 			entry = __pte(_pa(addr, PAGE_SIZE, mode));  in pgtable_pte_populate()
313 static void pgtable_pmd_populate(pud_t *pud, unsigned long addr, unsigned long end,  in pgtable_pmd_populate()  argument
320 	pmd = pmd_offset(pud, addr);  in pgtable_pmd_populate()
321 	for (; addr < end; addr = next, pmd++) {  in pgtable_pmd_populate()
322 		next = pmd_addr_end(addr, end);  in pgtable_pmd_populate()
324 			if (kasan_pmd_populate_zero_shadow(pmd, addr, next, mode))  in pgtable_pmd_populate()
326 			if (can_large_pmd(pmd, addr, next, mode)) {  in pgtable_pmd_populate()
327 				entry = __pmd(_pa(addr, _SEGMENT_SIZE, mode));  in pgtable_pmd_populate()
340 		pgtable_pte_populate(pmd, addr, next, mode);  in pgtable_pmd_populate()
346 static void pgtable_pud_populate(p4d_t *p4d, unsigned long addr, unsigned long end,  in pgtable_pud_populate()  argument
353 	pud = pud_offset(p4d, addr);  in pgtable_pud_populate()
354 	for (; addr < end; addr = next, pud++) {  in pgtable_pud_populate()
355 		next = pud_addr_end(addr, end);  in pgtable_pud_populate()
357 			if (kasan_pud_populate_zero_shadow(pud, addr, next, mode))  in pgtable_pud_populate()
359 			if (can_large_pud(pud, addr, next, mode)) {  in pgtable_pud_populate()
360 				entry = __pud(_pa(addr, _REGION3_SIZE, mode));  in pgtable_pud_populate()
373 		pgtable_pmd_populate(pud, addr, next, mode);  in pgtable_pud_populate()
379 static void pgtable_p4d_populate(pgd_t *pgd, unsigned long addr, unsigned long end,  in pgtable_p4d_populate()  argument
386 	p4d = p4d_offset(pgd, addr);  in pgtable_p4d_populate()
387 	for (; addr < end; addr = next, p4d++) {  in pgtable_p4d_populate()
388 		next = p4d_addr_end(addr, end);  in pgtable_p4d_populate()
390 			if (kasan_p4d_populate_zero_shadow(p4d, addr, next, mode))  in pgtable_p4d_populate()
395 		pgtable_pud_populate(p4d, addr, next, mode);  in pgtable_p4d_populate()
399 static void pgtable_populate(unsigned long addr, unsigned long end, enum populate_mode mode)  in pgtable_populate()  argument
405 	pgd = pgd_offset(&init_mm, addr);  in pgtable_populate()
406 	for (; addr < end; addr = next, pgd++) {  in pgtable_populate()
407 		next = pgd_addr_end(addr, end);  in pgtable_populate()
409 			if (kasan_pgd_populate_zero_shadow(pgd, addr, next, mode))  in pgtable_populate()
418 		pgtable_p4d_populate(pgd, addr, next, mode);  in pgtable_populate()