1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0
2da1694adSHeiko Carstens #include <linux/set_memory.h>
39d719d39SHeiko Carstens #include <linux/ptdump.h>
4e76e82d7SHeiko Carstens #include <linux/seq_file.h>
5e76e82d7SHeiko Carstens #include <linux/debugfs.h>
6*7e12284cSHeiko Carstens #include <linux/sort.h>
7e76e82d7SHeiko Carstens #include <linux/mm.h>
8d6de72cfSSven Schnelle #include <linux/kfence.h>
9e006222bSVasily Gorbik #include <linux/kasan.h>
100dac8f6bSVasily Gorbik #include <asm/kasan.h>
1150787755SAlexander Gordeev #include <asm/abs_lowcore.h>
123b051e89SSven Schnelle #include <asm/nospec-branch.h>
13e76e82d7SHeiko Carstens #include <asm/sections.h>
14c0ceb944SAlexander Gordeev #include <asm/maccess.h>
15e76e82d7SHeiko Carstens
16e76e82d7SHeiko Carstens static unsigned long max_addr;
17e76e82d7SHeiko Carstens
18e76e82d7SHeiko Carstens struct addr_marker {
19*7e12284cSHeiko Carstens int is_start;
20e76e82d7SHeiko Carstens unsigned long start_address;
21e76e82d7SHeiko Carstens unsigned long size;
22e76e82d7SHeiko Carstens const char *name;
23e76e82d7SHeiko Carstens };
24e76e82d7SHeiko Carstens
257e4d4cfeSHeiko Carstens static struct addr_marker *markers;
267e4d4cfeSHeiko Carstens static unsigned int markers_cnt;
2737395344SHeiko Carstens
28edcfc9c7SHeiko Carstens struct pg_state {
29edcfc9c7SHeiko Carstens struct ptdump_state ptdump;
30e76e82d7SHeiko Carstens struct seq_file *seq;
31e76e82d7SHeiko Carstens int level;
32d6de72cfSSven Schnelle unsigned int current_prot;
33d6de72cfSSven Schnelle bool check_wx;
34d6de72cfSSven Schnelle unsigned long wx_pages;
35d6de72cfSSven Schnelle unsigned long start_address;
3637395344SHeiko Carstens const struct addr_marker *marker;
3737395344SHeiko Carstens };
38e76e82d7SHeiko Carstens
39e670e64aSVasily Gorbik #define pt_dump_seq_printf(m, fmt, args...) \
40e76e82d7SHeiko Carstens ({ \
41e670e64aSVasily Gorbik struct seq_file *__m = (m); \
4219af2887SIlya Leoshkevich \
4319af2887SIlya Leoshkevich if (__m) \
4419af2887SIlya Leoshkevich seq_printf(__m, fmt, ##args); \
4519af2887SIlya Leoshkevich })
4619af2887SIlya Leoshkevich
4719af2887SIlya Leoshkevich #define pt_dump_seq_puts(m, fmt) \
4819af2887SIlya Leoshkevich ({ \
4919af2887SIlya Leoshkevich struct seq_file *__m = (m); \
5019af2887SIlya Leoshkevich \
5119af2887SIlya Leoshkevich if (__m) \
52c972cc60SHeiko Carstens seq_printf(__m, fmt); \
53e670e64aSVasily Gorbik })
5450787755SAlexander Gordeev
print_prot(struct seq_file * m,unsigned int pr,int level)5550787755SAlexander Gordeev static void print_prot(struct seq_file *m, unsigned int pr, int level)
56c0ceb944SAlexander Gordeev {
57c0ceb944SAlexander Gordeev static const char * const level_name[] =
5805178996SVasily Gorbik { "ASCE", "PGD", "PUD", "PMD", "PTE" };
5905178996SVasily Gorbik
6005178996SVasily Gorbik pt_dump_seq_printf(m, "%s ", level_name[level]);
6105178996SVasily Gorbik if (pr & _PAGE_INVALID) {
62e76e82d7SHeiko Carstens pt_dump_seq_printf(m, "I\n");
63e76e82d7SHeiko Carstens return;
64e76e82d7SHeiko Carstens }
65*7e12284cSHeiko Carstens pt_dump_seq_puts(m, (pr & _PAGE_PROTECT) ? "RO " : "RW ");
66*7e12284cSHeiko Carstens pt_dump_seq_puts(m, (pr & _PAGE_NOEXEC) ? "NX\n" : "X\n");
67*7e12284cSHeiko Carstens }
68*7e12284cSHeiko Carstens
note_prot_wx(struct pg_state * st,unsigned long addr)69*7e12284cSHeiko Carstens static void note_prot_wx(struct pg_state *st, unsigned long addr)
70*7e12284cSHeiko Carstens {
71*7e12284cSHeiko Carstens if (!st->check_wx)
72*7e12284cSHeiko Carstens return;
73*7e12284cSHeiko Carstens if (st->current_prot & _PAGE_INVALID)
74d6de72cfSSven Schnelle return;
75*7e12284cSHeiko Carstens if (st->current_prot & _PAGE_PROTECT)
76*7e12284cSHeiko Carstens return;
77d6de72cfSSven Schnelle if (st->current_prot & _PAGE_NOEXEC)
78*7e12284cSHeiko Carstens return;
79*7e12284cSHeiko Carstens /*
80*7e12284cSHeiko Carstens * The first lowcore page is W+X if spectre mitigations are using
81*7e12284cSHeiko Carstens * trampolines or the BEAR enhancements facility is not installed,
8219af2887SIlya Leoshkevich * in which case we have two lpswe instructions in lowcore that need
83*7e12284cSHeiko Carstens * to be executable.
84*7e12284cSHeiko Carstens */
85*7e12284cSHeiko Carstens if (addr == PAGE_SIZE && (nospec_uses_trampoline() || !static_key_enabled(&cpu_has_bear)))
86*7e12284cSHeiko Carstens return;
87*7e12284cSHeiko Carstens WARN_ONCE(IS_ENABLED(CONFIG_DEBUG_WX),
88*7e12284cSHeiko Carstens "s390/mm: Found insecure W+X mapping at address %pS\n",
89*7e12284cSHeiko Carstens (void *)st->start_address);
90*7e12284cSHeiko Carstens st->wx_pages += (addr - st->start_address) / PAGE_SIZE;
9119af2887SIlya Leoshkevich }
92*7e12284cSHeiko Carstens
note_page_update_state(struct pg_state * st,unsigned long addr,unsigned int prot,int level)93*7e12284cSHeiko Carstens static void note_page_update_state(struct pg_state *st, unsigned long addr, unsigned int prot, int level)
94*7e12284cSHeiko Carstens {
95*7e12284cSHeiko Carstens struct seq_file *m = st->seq;
96*7e12284cSHeiko Carstens
97*7e12284cSHeiko Carstens while (addr >= st->marker[1].start_address) {
9805178996SVasily Gorbik st->marker++;
99*7e12284cSHeiko Carstens pt_dump_seq_printf(m, "---[ %s %s ]---\n", st->marker->name,
100*7e12284cSHeiko Carstens st->marker->is_start ? "Start" : "End");
10105178996SVasily Gorbik }
102*7e12284cSHeiko Carstens st->start_address = addr;
103e76e82d7SHeiko Carstens st->current_prot = prot;
104e76e82d7SHeiko Carstens st->level = level;
105e76e82d7SHeiko Carstens }
1069d719d39SHeiko Carstens
note_page(struct ptdump_state * pt_st,unsigned long addr,int level,u64 val)1079d719d39SHeiko Carstens static void note_page(struct ptdump_state *pt_st, unsigned long addr, int level, u64 val)
108e76e82d7SHeiko Carstens {
109e76e82d7SHeiko Carstens int width = sizeof(unsigned long) * 2;
11008c8e685SHeiko Carstens static const char units[] = "KMGTPE";
11108c8e685SHeiko Carstens const char *unit = units;
112e76e82d7SHeiko Carstens unsigned long delta;
113e76e82d7SHeiko Carstens struct pg_state *st;
114e76e82d7SHeiko Carstens struct seq_file *m;
115e76e82d7SHeiko Carstens unsigned int prot;
1166bf9a639SHeiko Carstens
1176bf9a639SHeiko Carstens st = container_of(pt_st, struct pg_state, ptdump);
1186bf9a639SHeiko Carstens m = st->seq;
1196bf9a639SHeiko Carstens prot = val & (_PAGE_PROTECT | _PAGE_NOEXEC);
1206bf9a639SHeiko Carstens if (level == 4 && (val & _PAGE_INVALID))
1216bf9a639SHeiko Carstens prot = _PAGE_INVALID;
1226bf9a639SHeiko Carstens /* For pmd_none() & friends val gets passed as zero. */
1236bf9a639SHeiko Carstens if (level != 4 && !val)
1246bf9a639SHeiko Carstens prot = _PAGE_INVALID;
1256bf9a639SHeiko Carstens /* Final flush from generic code. */
1266bf9a639SHeiko Carstens if (level == -1)
1276bf9a639SHeiko Carstens addr = max_addr;
1286bf9a639SHeiko Carstens if (st->level == -1) {
1296bf9a639SHeiko Carstens pt_dump_seq_puts(m, "---[ Kernel Virtual Address Space ]---\n");
1306bf9a639SHeiko Carstens note_page_update_state(st, addr, prot, level);
1316bf9a639SHeiko Carstens } else if (prot != st->current_prot || level != st->level ||
132e76e82d7SHeiko Carstens addr >= st->marker[1].start_address) {
133e76e82d7SHeiko Carstens note_prot_wx(st, addr);
134e76e82d7SHeiko Carstens pt_dump_seq_printf(m, "0x%0*lx-0x%0*lx ",
135e76e82d7SHeiko Carstens width, st->start_address,
136e76e82d7SHeiko Carstens width, addr);
1376bf9a639SHeiko Carstens delta = (addr - st->start_address) >> 10;
1381819ed1fSHeiko Carstens while (!(delta & 0x3ff) && unit[1]) {
1396bf9a639SHeiko Carstens delta >>= 10;
1401819ed1fSHeiko Carstens unit++;
1411819ed1fSHeiko Carstens }
1426bf9a639SHeiko Carstens pt_dump_seq_printf(m, "%9lu%c ", delta, *unit);
1436bf9a639SHeiko Carstens print_prot(m, st->current_prot, st->level);
144e76e82d7SHeiko Carstens note_page_update_state(st, addr, prot, level);
145e76e82d7SHeiko Carstens }
14608c8e685SHeiko Carstens }
14708c8e685SHeiko Carstens
ptdump_check_wx(void)14808c8e685SHeiko Carstens bool ptdump_check_wx(void)
14908c8e685SHeiko Carstens {
15008c8e685SHeiko Carstens struct pg_state st = {
15108c8e685SHeiko Carstens .ptdump = {
15208c8e685SHeiko Carstens .note_page = note_page,
15308c8e685SHeiko Carstens .range = (struct ptdump_range[]) {
15408c8e685SHeiko Carstens {.start = 0, .end = max_addr},
15508c8e685SHeiko Carstens {.start = 0, .end = 0},
1563b051e89SSven Schnelle }
1573b051e89SSven Schnelle },
1583b051e89SSven Schnelle .seq = NULL,
1593b051e89SSven Schnelle .level = -1,
1603b051e89SSven Schnelle .current_prot = 0,
1613b051e89SSven Schnelle .check_wx = true,
1623b051e89SSven Schnelle .wx_pages = 0,
16308c8e685SHeiko Carstens .start_address = 0,
164592e15f6SChristophe Leroy .marker = (struct addr_marker[]) {
165592e15f6SChristophe Leroy { .start_address = 0, .name = NULL},
16608c8e685SHeiko Carstens { .start_address = -1, .name = NULL},
16708c8e685SHeiko Carstens },
16808c8e685SHeiko Carstens };
16908c8e685SHeiko Carstens
1707e4d4cfeSHeiko Carstens if (!MACHINE_HAS_NX)
1717e4d4cfeSHeiko Carstens return true;
1727e4d4cfeSHeiko Carstens ptdump_walk_pgd(&st.ptdump, &init_mm, NULL);
1737e4d4cfeSHeiko Carstens if (st.wx_pages) {
1747e4d4cfeSHeiko Carstens pr_warn("Checked W+X mappings: FAILED, %lu W+X pages found\n", st.wx_pages);
1757e4d4cfeSHeiko Carstens
1767e4d4cfeSHeiko Carstens return false;
1777e4d4cfeSHeiko Carstens } else {
1787e4d4cfeSHeiko Carstens pr_info("Checked W+X mappings: passed, no %sW+X pages found\n",
1797e4d4cfeSHeiko Carstens (nospec_uses_trampoline() || !static_key_enabled(&cpu_has_bear)) ?
1807e4d4cfeSHeiko Carstens "unexpected " : "");
1817e4d4cfeSHeiko Carstens
1827e4d4cfeSHeiko Carstens return true;
1839d719d39SHeiko Carstens }
184e76e82d7SHeiko Carstens }
185e76e82d7SHeiko Carstens
1869d719d39SHeiko Carstens #ifdef CONFIG_PTDUMP_DEBUGFS
ptdump_show(struct seq_file * m,void * v)187e76e82d7SHeiko Carstens static int ptdump_show(struct seq_file *m, void *v)
188e76e82d7SHeiko Carstens {
1899d719d39SHeiko Carstens struct pg_state st = {
1909d719d39SHeiko Carstens .ptdump = {
1919d719d39SHeiko Carstens .note_page = note_page,
192e76e82d7SHeiko Carstens .range = (struct ptdump_range[]) {
1939d719d39SHeiko Carstens {.start = 0, .end = max_addr},
1949d719d39SHeiko Carstens {.start = 0, .end = 0},
1959d719d39SHeiko Carstens }
1969d719d39SHeiko Carstens },
1979d719d39SHeiko Carstens .seq = m,
1989d719d39SHeiko Carstens .level = -1,
1999d719d39SHeiko Carstens .current_prot = 0,
2009d719d39SHeiko Carstens .check_wx = false,
2019d719d39SHeiko Carstens .wx_pages = 0,
2029d719d39SHeiko Carstens .start_address = 0,
2039d719d39SHeiko Carstens .marker = markers,
2049d719d39SHeiko Carstens };
2056bf9a639SHeiko Carstens
2067e4d4cfeSHeiko Carstens get_online_mems();
2079d719d39SHeiko Carstens mutex_lock(&cpa_mutex);
2089d719d39SHeiko Carstens ptdump_walk_pgd(&st.ptdump, &init_mm, NULL);
20908c8e685SHeiko Carstens mutex_unlock(&cpa_mutex);
2106bf9a639SHeiko Carstens put_online_mems();
211e76e82d7SHeiko Carstens return 0;
2129d719d39SHeiko Carstens }
2139d719d39SHeiko Carstens DEFINE_SHOW_ATTRIBUTE(ptdump);
214e76e82d7SHeiko Carstens #endif /* CONFIG_PTDUMP_DEBUGFS */
215e76e82d7SHeiko Carstens
ptdump_cmp(const void * a,const void * b)216e76e82d7SHeiko Carstens static int ptdump_cmp(const void *a, const void *b)
217e76e82d7SHeiko Carstens {
2186bf9a639SHeiko Carstens const struct addr_marker *ama = a;
219e76e82d7SHeiko Carstens const struct addr_marker *amb = b;
2207e4d4cfeSHeiko Carstens
221e76e82d7SHeiko Carstens if (ama->start_address > amb->start_address)
222e76e82d7SHeiko Carstens return 1;
223e76e82d7SHeiko Carstens if (ama->start_address < amb->start_address)
2246cdc82dbSChristophe Leroy return -1;
22508c8e685SHeiko Carstens /*
22608c8e685SHeiko Carstens * If the start addresses of two markers are identical sort markers in an
22708c8e685SHeiko Carstens * order that considers areas contained within other areas correctly.
22808c8e685SHeiko Carstens */
22908c8e685SHeiko Carstens if (ama->is_start && amb->is_start) {
23008c8e685SHeiko Carstens if (ama->size > amb->size)
23108c8e685SHeiko Carstens return -1;
23208c8e685SHeiko Carstens if (ama->size < amb->size)
23308c8e685SHeiko Carstens return 1;
23408c8e685SHeiko Carstens return 0;
23508c8e685SHeiko Carstens }
23608c8e685SHeiko Carstens if (!ama->is_start && !amb->is_start) {
23708c8e685SHeiko Carstens if (ama->size > amb->size)
23808c8e685SHeiko Carstens return 1;
23908c8e685SHeiko Carstens if (ama->size < amb->size)
24008c8e685SHeiko Carstens return -1;
24108c8e685SHeiko Carstens return 0;
24208c8e685SHeiko Carstens }
24308c8e685SHeiko Carstens if (ama->is_start)
24408c8e685SHeiko Carstens return 1;
24508c8e685SHeiko Carstens if (amb->is_start)
24608c8e685SHeiko Carstens return -1;
2476cdc82dbSChristophe Leroy return 0;
24808c8e685SHeiko Carstens }
2496cdc82dbSChristophe Leroy
add_marker(unsigned long start,unsigned long end,const char * name)25008c8e685SHeiko Carstens static int add_marker(unsigned long start, unsigned long end, const char *name)
2516cdc82dbSChristophe Leroy {
2526cdc82dbSChristophe Leroy size_t oldsize, newsize;
2536cdc82dbSChristophe Leroy
2543b051e89SSven Schnelle oldsize = markers_cnt * sizeof(*markers);
2553b051e89SSven Schnelle newsize = oldsize + 2 * sizeof(*markers);
2563b051e89SSven Schnelle if (!oldsize)
2576cdc82dbSChristophe Leroy markers = kvmalloc(newsize, GFP_KERNEL);
2586cdc82dbSChristophe Leroy else
2596cdc82dbSChristophe Leroy markers = kvrealloc(markers, newsize, GFP_KERNEL);
26008c8e685SHeiko Carstens if (!markers)
26108c8e685SHeiko Carstens goto error;
26208c8e685SHeiko Carstens markers[markers_cnt].is_start = 1;
263e76e82d7SHeiko Carstens markers[markers_cnt].start_address = start;
264e76e82d7SHeiko Carstens markers[markers_cnt].size = end - start;
2659d719d39SHeiko Carstens markers[markers_cnt].name = name;
2669d719d39SHeiko Carstens markers_cnt++;
2679d719d39SHeiko Carstens markers[markers_cnt].is_start = 0;
2689d719d39SHeiko Carstens markers[markers_cnt].start_address = end;
2699d719d39SHeiko Carstens markers[markers_cnt].size = end - start;
2709d719d39SHeiko Carstens markers[markers_cnt].name = name;
2719d719d39SHeiko Carstens markers_cnt++;
2729d719d39SHeiko Carstens return 0;
2739d719d39SHeiko Carstens error:
2749d719d39SHeiko Carstens markers_cnt = 0;
2759d719d39SHeiko Carstens return -ENOMEM;
27608c8e685SHeiko Carstens }
27708c8e685SHeiko Carstens
pt_dump_init(void)2789d719d39SHeiko Carstens static int pt_dump_init(void)
2799d719d39SHeiko Carstens {
2809d719d39SHeiko Carstens #ifdef CONFIG_KFENCE
2819d719d39SHeiko Carstens unsigned long kfence_start = (unsigned long)__kfence_pool;
28236c2733cSHeiko Carstens #endif
283da1694adSHeiko Carstens unsigned long lowcore = (unsigned long)get_lowcore();
2849d719d39SHeiko Carstens int rc;
285da1694adSHeiko Carstens
28636c2733cSHeiko Carstens /*
287e76e82d7SHeiko Carstens * Figure out the maximum virtual address being accessible with the
288e76e82d7SHeiko Carstens * kernel ASCE. We need this to keep the page table walker functions
2899d719d39SHeiko Carstens * from accessing non-existent entries.
29008c8e685SHeiko Carstens */
291e76e82d7SHeiko Carstens max_addr = (get_lowcore()->kernel_asce.val & _REGION_ENTRY_TYPE_MASK) >> 2;
292*7e12284cSHeiko Carstens max_addr = 1UL << (max_addr * 11 + 31);
293ee4b2ce6SVasily Gorbik /* start + end markers - must be added first */
294*7e12284cSHeiko Carstens rc = add_marker(0, -1UL, NULL);
295*7e12284cSHeiko Carstens rc |= add_marker((unsigned long)_stext, (unsigned long)_end, "Kernel Image");
296ee4b2ce6SVasily Gorbik rc |= add_marker(lowcore, lowcore + sizeof(struct lowcore), "Lowcore");
297*7e12284cSHeiko Carstens rc |= add_marker(__identity_base, __identity_base + ident_map_size, "Identity Mapping");
298*7e12284cSHeiko Carstens rc |= add_marker((unsigned long)__samode31, (unsigned long)__eamode31, "Amode31 Area");
299*7e12284cSHeiko Carstens rc |= add_marker(MODULES_VADDR, MODULES_END, "Modules Area");
300*7e12284cSHeiko Carstens rc |= add_marker(__abs_lowcore, __abs_lowcore + ABS_LOWCORE_MAP_SIZE, "Lowcore Area");
301*7e12284cSHeiko Carstens rc |= add_marker(__memcpy_real_area, __memcpy_real_area + MEMCPY_REAL_SIZE, "Real Memory Copy Area");
302*7e12284cSHeiko Carstens rc |= add_marker((unsigned long)vmemmap, (unsigned long)vmemmap + vmemmap_size, "vmemmap Area");
303*7e12284cSHeiko Carstens rc |= add_marker(VMALLOC_START, VMALLOC_END, "vmalloc Area");
304*7e12284cSHeiko Carstens #ifdef CONFIG_KFENCE
305*7e12284cSHeiko Carstens rc |= add_marker(kfence_start, kfence_start + KFENCE_POOL_SIZE, "KFence Pool");
306*7e12284cSHeiko Carstens #endif
307*7e12284cSHeiko Carstens #ifdef CONFIG_KMSAN
308*7e12284cSHeiko Carstens rc |= add_marker(KMSAN_VMALLOC_SHADOW_START, KMSAN_VMALLOC_SHADOW_END, "Kmsan vmalloc Shadow");
309*7e12284cSHeiko Carstens rc |= add_marker(KMSAN_VMALLOC_ORIGIN_START, KMSAN_VMALLOC_ORIGIN_END, "Kmsan vmalloc Origins");
310*7e12284cSHeiko Carstens rc |= add_marker(KMSAN_MODULES_SHADOW_START, KMSAN_MODULES_SHADOW_END, "Kmsan Modules Shadow");
311ee4b2ce6SVasily Gorbik rc |= add_marker(KMSAN_MODULES_ORIGIN_START, KMSAN_MODULES_ORIGIN_END, "Kmsan Modules Origins");
312ee4b2ce6SVasily Gorbik #endif
313e76e82d7SHeiko Carstens #ifdef CONFIG_KASAN
314e76e82d7SHeiko Carstens rc |= add_marker(KASAN_SHADOW_START, KASAN_SHADOW_END, "Kasan Shadow");
315d6de72cfSSven Schnelle #endif
316d6de72cfSSven Schnelle if (rc)
317d6de72cfSSven Schnelle goto error;
3187e4d4cfeSHeiko Carstens sort(&markers[1], markers_cnt - 1, sizeof(*markers), ptdump_cmp, NULL);
3197e4d4cfeSHeiko Carstens #ifdef CONFIG_PTDUMP_DEBUGFS
320e76e82d7SHeiko Carstens debugfs_create_file("kernel_page_tables", 0400, NULL, NULL, &ptdump_fops);
321e76e82d7SHeiko Carstens #endif /* CONFIG_PTDUMP_DEBUGFS */
322e76e82d7SHeiko Carstens return 0;
323e76e82d7SHeiko Carstens error:
324e76e82d7SHeiko Carstens kvfree(markers);
325208da1d5SSven Schnelle return -ENOMEM;
326e76e82d7SHeiko Carstens }
3277e4d4cfeSHeiko Carstens device_initcall(pt_dump_init);
3287e4d4cfeSHeiko Carstens