15615f69bSLinus Walleij // SPDX-License-Identifier: GPL-2.0-only 25615f69bSLinus Walleij /* 35615f69bSLinus Walleij * This file contains kasan initialization code for ARM. 45615f69bSLinus Walleij * 55615f69bSLinus Walleij * Copyright (c) 2018 Samsung Electronics Co., Ltd. 65615f69bSLinus Walleij * Author: Andrey Ryabinin <ryabinin.a.a@gmail.com> 75615f69bSLinus Walleij * Author: Linus Walleij <linus.walleij@linaro.org> 85615f69bSLinus Walleij */ 95615f69bSLinus Walleij 105615f69bSLinus Walleij #define pr_fmt(fmt) "kasan: " fmt 115615f69bSLinus Walleij #include <linux/kasan.h> 125615f69bSLinus Walleij #include <linux/kernel.h> 135615f69bSLinus Walleij #include <linux/memblock.h> 145615f69bSLinus Walleij #include <linux/sched/task.h> 155615f69bSLinus Walleij #include <linux/start_kernel.h> 165615f69bSLinus Walleij #include <linux/pgtable.h> 175615f69bSLinus Walleij #include <asm/cputype.h> 185615f69bSLinus Walleij #include <asm/highmem.h> 195615f69bSLinus Walleij #include <asm/mach/map.h> 205615f69bSLinus Walleij #include <asm/memory.h> 215615f69bSLinus Walleij #include <asm/page.h> 225615f69bSLinus Walleij #include <asm/pgalloc.h> 235615f69bSLinus Walleij #include <asm/procinfo.h> 245615f69bSLinus Walleij #include <asm/proc-fns.h> 255615f69bSLinus Walleij 265615f69bSLinus Walleij #include "mm.h" 275615f69bSLinus Walleij 285615f69bSLinus Walleij static pgd_t tmp_pgd_table[PTRS_PER_PGD] __initdata __aligned(PGD_SIZE); 295615f69bSLinus Walleij 305615f69bSLinus Walleij pmd_t tmp_pmd_table[PTRS_PER_PMD] __page_aligned_bss; 315615f69bSLinus Walleij 325615f69bSLinus Walleij static __init void *kasan_alloc_block(size_t size) 335615f69bSLinus Walleij { 345615f69bSLinus Walleij return memblock_alloc_try_nid(size, size, __pa(MAX_DMA_ADDRESS), 355615f69bSLinus Walleij MEMBLOCK_ALLOC_KASAN, NUMA_NO_NODE); 365615f69bSLinus Walleij } 375615f69bSLinus Walleij 385615f69bSLinus Walleij static void __init kasan_pte_populate(pmd_t *pmdp, unsigned long addr, 395615f69bSLinus Walleij unsigned long end, bool early) 405615f69bSLinus Walleij { 415615f69bSLinus Walleij unsigned long next; 425615f69bSLinus Walleij pte_t *ptep = pte_offset_kernel(pmdp, addr); 435615f69bSLinus Walleij 445615f69bSLinus Walleij do { 455615f69bSLinus Walleij pte_t entry; 465615f69bSLinus Walleij void *p; 475615f69bSLinus Walleij 485615f69bSLinus Walleij next = addr + PAGE_SIZE; 495615f69bSLinus Walleij 505615f69bSLinus Walleij if (!early) { 515615f69bSLinus Walleij if (!pte_none(READ_ONCE(*ptep))) 525615f69bSLinus Walleij continue; 535615f69bSLinus Walleij 545615f69bSLinus Walleij p = kasan_alloc_block(PAGE_SIZE); 555615f69bSLinus Walleij if (!p) { 565615f69bSLinus Walleij panic("%s failed to allocate shadow page for address 0x%lx\n", 575615f69bSLinus Walleij __func__, addr); 585615f69bSLinus Walleij return; 595615f69bSLinus Walleij } 605615f69bSLinus Walleij memset(p, KASAN_SHADOW_INIT, PAGE_SIZE); 615615f69bSLinus Walleij entry = pfn_pte(virt_to_pfn(p), 625615f69bSLinus Walleij __pgprot(pgprot_val(PAGE_KERNEL))); 635615f69bSLinus Walleij } else if (pte_none(READ_ONCE(*ptep))) { 645615f69bSLinus Walleij /* 655615f69bSLinus Walleij * The early shadow memory is mapping all KASan 665615f69bSLinus Walleij * operations to one and the same page in memory, 675615f69bSLinus Walleij * "kasan_early_shadow_page" so that the instrumentation 685615f69bSLinus Walleij * will work on a scratch area until we can set up the 695615f69bSLinus Walleij * proper KASan shadow memory. 705615f69bSLinus Walleij */ 715615f69bSLinus Walleij entry = pfn_pte(virt_to_pfn(kasan_early_shadow_page), 725615f69bSLinus Walleij __pgprot(_L_PTE_DEFAULT | L_PTE_DIRTY | L_PTE_XN)); 735615f69bSLinus Walleij } else { 745615f69bSLinus Walleij /* 755615f69bSLinus Walleij * Early shadow mappings are PMD_SIZE aligned, so if the 765615f69bSLinus Walleij * first entry is already set, they must all be set. 775615f69bSLinus Walleij */ 785615f69bSLinus Walleij return; 795615f69bSLinus Walleij } 805615f69bSLinus Walleij 815615f69bSLinus Walleij set_pte_at(&init_mm, addr, ptep, entry); 825615f69bSLinus Walleij } while (ptep++, addr = next, addr != end); 835615f69bSLinus Walleij } 845615f69bSLinus Walleij 855615f69bSLinus Walleij /* 865615f69bSLinus Walleij * The pmd (page middle directory) is only used on LPAE 875615f69bSLinus Walleij */ 885615f69bSLinus Walleij static void __init kasan_pmd_populate(pud_t *pudp, unsigned long addr, 895615f69bSLinus Walleij unsigned long end, bool early) 905615f69bSLinus Walleij { 915615f69bSLinus Walleij unsigned long next; 925615f69bSLinus Walleij pmd_t *pmdp = pmd_offset(pudp, addr); 935615f69bSLinus Walleij 945615f69bSLinus Walleij do { 955615f69bSLinus Walleij if (pmd_none(*pmdp)) { 965615f69bSLinus Walleij /* 975615f69bSLinus Walleij * We attempt to allocate a shadow block for the PMDs 985615f69bSLinus Walleij * used by the PTEs for this address if it isn't already 995615f69bSLinus Walleij * allocated. 1005615f69bSLinus Walleij */ 1015615f69bSLinus Walleij void *p = early ? kasan_early_shadow_pte : 1025615f69bSLinus Walleij kasan_alloc_block(PAGE_SIZE); 1035615f69bSLinus Walleij 1045615f69bSLinus Walleij if (!p) { 1055615f69bSLinus Walleij panic("%s failed to allocate shadow block for address 0x%lx\n", 1065615f69bSLinus Walleij __func__, addr); 1075615f69bSLinus Walleij return; 1085615f69bSLinus Walleij } 1095615f69bSLinus Walleij pmd_populate_kernel(&init_mm, pmdp, p); 1105615f69bSLinus Walleij flush_pmd_entry(pmdp); 1115615f69bSLinus Walleij } 1125615f69bSLinus Walleij 1135615f69bSLinus Walleij next = pmd_addr_end(addr, end); 1145615f69bSLinus Walleij kasan_pte_populate(pmdp, addr, next, early); 1155615f69bSLinus Walleij } while (pmdp++, addr = next, addr != end); 1165615f69bSLinus Walleij } 1175615f69bSLinus Walleij 1185615f69bSLinus Walleij static void __init kasan_pgd_populate(unsigned long addr, unsigned long end, 1195615f69bSLinus Walleij bool early) 1205615f69bSLinus Walleij { 1215615f69bSLinus Walleij unsigned long next; 1225615f69bSLinus Walleij pgd_t *pgdp; 1235615f69bSLinus Walleij p4d_t *p4dp; 1245615f69bSLinus Walleij pud_t *pudp; 1255615f69bSLinus Walleij 1265615f69bSLinus Walleij pgdp = pgd_offset_k(addr); 1275615f69bSLinus Walleij 1285615f69bSLinus Walleij do { 1295615f69bSLinus Walleij /* 1305615f69bSLinus Walleij * Allocate and populate the shadow block of p4d folded into 1315615f69bSLinus Walleij * pud folded into pmd if it doesn't already exist 1325615f69bSLinus Walleij */ 1335615f69bSLinus Walleij if (!early && pgd_none(*pgdp)) { 1345615f69bSLinus Walleij void *p = kasan_alloc_block(PAGE_SIZE); 1355615f69bSLinus Walleij 1365615f69bSLinus Walleij if (!p) { 1375615f69bSLinus Walleij panic("%s failed to allocate shadow block for address 0x%lx\n", 1385615f69bSLinus Walleij __func__, addr); 1395615f69bSLinus Walleij return; 1405615f69bSLinus Walleij } 1415615f69bSLinus Walleij pgd_populate(&init_mm, pgdp, p); 1425615f69bSLinus Walleij } 1435615f69bSLinus Walleij 1445615f69bSLinus Walleij next = pgd_addr_end(addr, end); 1455615f69bSLinus Walleij /* 1465615f69bSLinus Walleij * We just immediately jump over the p4d and pud page 1475615f69bSLinus Walleij * directories since we believe ARM32 will never gain four 1485615f69bSLinus Walleij * nor five level page tables. 1495615f69bSLinus Walleij */ 1505615f69bSLinus Walleij p4dp = p4d_offset(pgdp, addr); 1515615f69bSLinus Walleij pudp = pud_offset(p4dp, addr); 1525615f69bSLinus Walleij 1535615f69bSLinus Walleij kasan_pmd_populate(pudp, addr, next, early); 1545615f69bSLinus Walleij } while (pgdp++, addr = next, addr != end); 1555615f69bSLinus Walleij } 1565615f69bSLinus Walleij 1575615f69bSLinus Walleij extern struct proc_info_list *lookup_processor_type(unsigned int); 1585615f69bSLinus Walleij 1595615f69bSLinus Walleij void __init kasan_early_init(void) 1605615f69bSLinus Walleij { 1615615f69bSLinus Walleij struct proc_info_list *list; 1625615f69bSLinus Walleij 1635615f69bSLinus Walleij /* 1645615f69bSLinus Walleij * locate processor in the list of supported processor 1655615f69bSLinus Walleij * types. The linker builds this table for us from the 1665615f69bSLinus Walleij * entries in arch/arm/mm/proc-*.S 1675615f69bSLinus Walleij */ 1685615f69bSLinus Walleij list = lookup_processor_type(read_cpuid_id()); 1695615f69bSLinus Walleij if (list) { 1705615f69bSLinus Walleij #ifdef MULTI_CPU 1715615f69bSLinus Walleij processor = *list->proc; 1725615f69bSLinus Walleij #endif 1735615f69bSLinus Walleij } 1745615f69bSLinus Walleij 1755615f69bSLinus Walleij BUILD_BUG_ON((KASAN_SHADOW_END - (1UL << 29)) != KASAN_SHADOW_OFFSET); 1765615f69bSLinus Walleij /* 1775615f69bSLinus Walleij * We walk the page table and set all of the shadow memory to point 1785615f69bSLinus Walleij * to the scratch page. 1795615f69bSLinus Walleij */ 1805615f69bSLinus Walleij kasan_pgd_populate(KASAN_SHADOW_START, KASAN_SHADOW_END, true); 1815615f69bSLinus Walleij } 1825615f69bSLinus Walleij 1835615f69bSLinus Walleij static void __init clear_pgds(unsigned long start, 1845615f69bSLinus Walleij unsigned long end) 1855615f69bSLinus Walleij { 1865615f69bSLinus Walleij for (; start && start < end; start += PMD_SIZE) 1875615f69bSLinus Walleij pmd_clear(pmd_off_k(start)); 1885615f69bSLinus Walleij } 1895615f69bSLinus Walleij 1905615f69bSLinus Walleij static int __init create_mapping(void *start, void *end) 1915615f69bSLinus Walleij { 1925615f69bSLinus Walleij void *shadow_start, *shadow_end; 1935615f69bSLinus Walleij 1945615f69bSLinus Walleij shadow_start = kasan_mem_to_shadow(start); 1955615f69bSLinus Walleij shadow_end = kasan_mem_to_shadow(end); 1965615f69bSLinus Walleij 1975615f69bSLinus Walleij pr_info("Mapping kernel virtual memory block: %px-%px at shadow: %px-%px\n", 1985615f69bSLinus Walleij start, end, shadow_start, shadow_end); 1995615f69bSLinus Walleij 2005615f69bSLinus Walleij kasan_pgd_populate((unsigned long)shadow_start & PAGE_MASK, 2015615f69bSLinus Walleij PAGE_ALIGN((unsigned long)shadow_end), false); 2025615f69bSLinus Walleij return 0; 2035615f69bSLinus Walleij } 2045615f69bSLinus Walleij 2055615f69bSLinus Walleij void __init kasan_init(void) 2065615f69bSLinus Walleij { 2075615f69bSLinus Walleij phys_addr_t pa_start, pa_end; 2085615f69bSLinus Walleij u64 i; 2095615f69bSLinus Walleij 2105615f69bSLinus Walleij /* 2115615f69bSLinus Walleij * We are going to perform proper setup of shadow memory. 2125615f69bSLinus Walleij * 2135615f69bSLinus Walleij * At first we should unmap early shadow (clear_pgds() call bellow). 2145615f69bSLinus Walleij * However, instrumented code can't execute without shadow memory. 2155615f69bSLinus Walleij * 2165615f69bSLinus Walleij * To keep the early shadow memory MMU tables around while setting up 2175615f69bSLinus Walleij * the proper shadow memory, we copy swapper_pg_dir (the initial page 2185615f69bSLinus Walleij * table) to tmp_pgd_table and use that to keep the early shadow memory 2195615f69bSLinus Walleij * mapped until the full shadow setup is finished. Then we swap back 2205615f69bSLinus Walleij * to the proper swapper_pg_dir. 2215615f69bSLinus Walleij */ 2225615f69bSLinus Walleij 2235615f69bSLinus Walleij memcpy(tmp_pgd_table, swapper_pg_dir, sizeof(tmp_pgd_table)); 2245615f69bSLinus Walleij #ifdef CONFIG_ARM_LPAE 2255615f69bSLinus Walleij /* We need to be in the same PGD or this won't work */ 2265615f69bSLinus Walleij BUILD_BUG_ON(pgd_index(KASAN_SHADOW_START) != 2275615f69bSLinus Walleij pgd_index(KASAN_SHADOW_END)); 2285615f69bSLinus Walleij memcpy(tmp_pmd_table, 229*c2e6df3eSArnd Bergmann (void*)pgd_page_vaddr(*pgd_offset_k(KASAN_SHADOW_START)), 2305615f69bSLinus Walleij sizeof(tmp_pmd_table)); 2315615f69bSLinus Walleij set_pgd(&tmp_pgd_table[pgd_index(KASAN_SHADOW_START)], 2325615f69bSLinus Walleij __pgd(__pa(tmp_pmd_table) | PMD_TYPE_TABLE | L_PGD_SWAPPER)); 2335615f69bSLinus Walleij #endif 2345615f69bSLinus Walleij cpu_switch_mm(tmp_pgd_table, &init_mm); 2355615f69bSLinus Walleij local_flush_tlb_all(); 2365615f69bSLinus Walleij 2375615f69bSLinus Walleij clear_pgds(KASAN_SHADOW_START, KASAN_SHADOW_END); 2385615f69bSLinus Walleij 2395615f69bSLinus Walleij kasan_populate_early_shadow(kasan_mem_to_shadow((void *)VMALLOC_START), 2405615f69bSLinus Walleij kasan_mem_to_shadow((void *)-1UL) + 1); 2415615f69bSLinus Walleij 2425615f69bSLinus Walleij for_each_mem_range(i, &pa_start, &pa_end) { 2435615f69bSLinus Walleij void *start = __va(pa_start); 2445615f69bSLinus Walleij void *end = __va(pa_end); 2455615f69bSLinus Walleij 2465615f69bSLinus Walleij /* Do not attempt to shadow highmem */ 2475615f69bSLinus Walleij if (pa_start >= arm_lowmem_limit) { 2485615f69bSLinus Walleij pr_info("Skip highmem block at %pa-%pa\n", &pa_start, &pa_end); 2495615f69bSLinus Walleij continue; 2505615f69bSLinus Walleij } 2515615f69bSLinus Walleij if (pa_end > arm_lowmem_limit) { 2525615f69bSLinus Walleij pr_info("Truncating shadow for memory block at %pa-%pa to lowmem region at %pa\n", 2535615f69bSLinus Walleij &pa_start, &pa_end, &arm_lowmem_limit); 2545615f69bSLinus Walleij end = __va(arm_lowmem_limit); 2555615f69bSLinus Walleij } 2565615f69bSLinus Walleij if (start >= end) { 2575615f69bSLinus Walleij pr_info("Skipping invalid memory block %pa-%pa (virtual %p-%p)\n", 2585615f69bSLinus Walleij &pa_start, &pa_end, start, end); 2595615f69bSLinus Walleij continue; 2605615f69bSLinus Walleij } 2615615f69bSLinus Walleij 2625615f69bSLinus Walleij create_mapping(start, end); 2635615f69bSLinus Walleij } 2645615f69bSLinus Walleij 2655615f69bSLinus Walleij /* 2665615f69bSLinus Walleij * 1. The module global variables are in MODULES_VADDR ~ MODULES_END, 2675615f69bSLinus Walleij * so we need to map this area. 2685615f69bSLinus Walleij * 2. PKMAP_BASE ~ PKMAP_BASE+PMD_SIZE's shadow and MODULES_VADDR 2695615f69bSLinus Walleij * ~ MODULES_END's shadow is in the same PMD_SIZE, so we can't 2705615f69bSLinus Walleij * use kasan_populate_zero_shadow. 2715615f69bSLinus Walleij */ 2725615f69bSLinus Walleij create_mapping((void *)MODULES_VADDR, (void *)(PKMAP_BASE + PMD_SIZE)); 2735615f69bSLinus Walleij 2745615f69bSLinus Walleij /* 2755615f69bSLinus Walleij * KAsan may reuse the contents of kasan_early_shadow_pte directly, so 2765615f69bSLinus Walleij * we should make sure that it maps the zero page read-only. 2775615f69bSLinus Walleij */ 2785615f69bSLinus Walleij for (i = 0; i < PTRS_PER_PTE; i++) 2795615f69bSLinus Walleij set_pte_at(&init_mm, KASAN_SHADOW_START + i*PAGE_SIZE, 2805615f69bSLinus Walleij &kasan_early_shadow_pte[i], 2815615f69bSLinus Walleij pfn_pte(virt_to_pfn(kasan_early_shadow_page), 2825615f69bSLinus Walleij __pgprot(pgprot_val(PAGE_KERNEL) 2835615f69bSLinus Walleij | L_PTE_RDONLY))); 2845615f69bSLinus Walleij 2855615f69bSLinus Walleij cpu_switch_mm(swapper_pg_dir, &init_mm); 2865615f69bSLinus Walleij local_flush_tlb_all(); 2875615f69bSLinus Walleij 2885615f69bSLinus Walleij memset(kasan_early_shadow_page, 0, PAGE_SIZE); 2895615f69bSLinus Walleij pr_info("Kernel address sanitizer initialized\n"); 2905615f69bSLinus Walleij init_task.kasan_depth = 0; 2915615f69bSLinus Walleij } 292