1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/set_memory.h> 3 #include <linux/ptdump.h> 4 #include <linux/seq_file.h> 5 #include <linux/debugfs.h> 6 #include <linux/sort.h> 7 #include <linux/mm.h> 8 #include <linux/kfence.h> 9 #include <linux/kasan.h> 10 #include <asm/kasan.h> 11 #include <asm/abs_lowcore.h> 12 #include <asm/nospec-branch.h> 13 #include <asm/sections.h> 14 #include <asm/maccess.h> 15 16 static unsigned long max_addr; 17 18 struct addr_marker { 19 int is_start; 20 unsigned long start_address; 21 const char *name; 22 }; 23 24 enum address_markers_idx { 25 KVA_NR = 0, 26 LOWCORE_START_NR, 27 LOWCORE_END_NR, 28 AMODE31_START_NR, 29 AMODE31_END_NR, 30 KERNEL_START_NR, 31 KERNEL_END_NR, 32 #ifdef CONFIG_KFENCE 33 KFENCE_START_NR, 34 KFENCE_END_NR, 35 #endif 36 IDENTITY_START_NR, 37 IDENTITY_END_NR, 38 VMEMMAP_NR, 39 VMEMMAP_END_NR, 40 VMALLOC_NR, 41 VMALLOC_END_NR, 42 #ifdef CONFIG_KMSAN 43 KMSAN_VMALLOC_SHADOW_START_NR, 44 KMSAN_VMALLOC_SHADOW_END_NR, 45 KMSAN_VMALLOC_ORIGIN_START_NR, 46 KMSAN_VMALLOC_ORIGIN_END_NR, 47 KMSAN_MODULES_SHADOW_START_NR, 48 KMSAN_MODULES_SHADOW_END_NR, 49 KMSAN_MODULES_ORIGIN_START_NR, 50 KMSAN_MODULES_ORIGIN_END_NR, 51 #endif 52 MODULES_NR, 53 MODULES_END_NR, 54 ABS_LOWCORE_NR, 55 ABS_LOWCORE_END_NR, 56 MEMCPY_REAL_NR, 57 MEMCPY_REAL_END_NR, 58 #ifdef CONFIG_KASAN 59 KASAN_SHADOW_START_NR, 60 KASAN_SHADOW_END_NR, 61 #endif 62 }; 63 64 static struct addr_marker address_markers[] = { 65 [KVA_NR] = {0, 0, "Kernel Virtual Address Space"}, 66 [LOWCORE_START_NR] = {1, 0, "Lowcore Start"}, 67 [LOWCORE_END_NR] = {0, 0, "Lowcore End"}, 68 [IDENTITY_START_NR] = {1, 0, "Identity Mapping Start"}, 69 [IDENTITY_END_NR] = {0, 0, "Identity Mapping End"}, 70 [AMODE31_START_NR] = {1, 0, "Amode31 Area Start"}, 71 [AMODE31_END_NR] = {0, 0, "Amode31 Area End"}, 72 [KERNEL_START_NR] = {1, (unsigned long)_stext, "Kernel Image Start"}, 73 [KERNEL_END_NR] = {0, (unsigned long)_end, "Kernel Image End"}, 74 #ifdef CONFIG_KFENCE 75 [KFENCE_START_NR] = {1, 0, "KFence Pool Start"}, 76 [KFENCE_END_NR] = {0, 0, "KFence Pool End"}, 77 #endif 78 [VMEMMAP_NR] = {1, 0, "vmemmap Area Start"}, 79 [VMEMMAP_END_NR] = {0, 0, "vmemmap Area End"}, 80 [VMALLOC_NR] = {1, 0, "vmalloc Area Start"}, 81 [VMALLOC_END_NR] = {0, 0, "vmalloc Area End"}, 82 #ifdef CONFIG_KMSAN 83 [KMSAN_VMALLOC_SHADOW_START_NR] = {1, 0, "Kmsan vmalloc Shadow Start"}, 84 [KMSAN_VMALLOC_SHADOW_END_NR] = {0, 0, "Kmsan vmalloc Shadow End"}, 85 [KMSAN_VMALLOC_ORIGIN_START_NR] = {1, 0, "Kmsan vmalloc Origins Start"}, 86 [KMSAN_VMALLOC_ORIGIN_END_NR] = {0, 0, "Kmsan vmalloc Origins End"}, 87 [KMSAN_MODULES_SHADOW_START_NR] = {1, 0, "Kmsan Modules Shadow Start"}, 88 [KMSAN_MODULES_SHADOW_END_NR] = {0, 0, "Kmsan Modules Shadow End"}, 89 [KMSAN_MODULES_ORIGIN_START_NR] = {1, 0, "Kmsan Modules Origins Start"}, 90 [KMSAN_MODULES_ORIGIN_END_NR] = {0, 0, "Kmsan Modules Origins End"}, 91 #endif 92 [MODULES_NR] = {1, 0, "Modules Area Start"}, 93 [MODULES_END_NR] = {0, 0, "Modules Area End"}, 94 [ABS_LOWCORE_NR] = {1, 0, "Lowcore Area Start"}, 95 [ABS_LOWCORE_END_NR] = {0, 0, "Lowcore Area End"}, 96 [MEMCPY_REAL_NR] = {1, 0, "Real Memory Copy Area Start"}, 97 [MEMCPY_REAL_END_NR] = {0, 0, "Real Memory Copy Area End"}, 98 #ifdef CONFIG_KASAN 99 [KASAN_SHADOW_START_NR] = {1, KASAN_SHADOW_START, "Kasan Shadow Start"}, 100 [KASAN_SHADOW_END_NR] = {0, KASAN_SHADOW_END, "Kasan Shadow End"}, 101 #endif 102 {1, -1UL, NULL} 103 }; 104 105 struct pg_state { 106 struct ptdump_state ptdump; 107 struct seq_file *seq; 108 int level; 109 unsigned int current_prot; 110 bool check_wx; 111 unsigned long wx_pages; 112 unsigned long start_address; 113 const struct addr_marker *marker; 114 }; 115 116 #define pt_dump_seq_printf(m, fmt, args...) \ 117 ({ \ 118 struct seq_file *__m = (m); \ 119 \ 120 if (__m) \ 121 seq_printf(__m, fmt, ##args); \ 122 }) 123 124 #define pt_dump_seq_puts(m, fmt) \ 125 ({ \ 126 struct seq_file *__m = (m); \ 127 \ 128 if (__m) \ 129 seq_printf(__m, fmt); \ 130 }) 131 132 static void print_prot(struct seq_file *m, unsigned int pr, int level) 133 { 134 static const char * const level_name[] = 135 { "ASCE", "PGD", "PUD", "PMD", "PTE" }; 136 137 pt_dump_seq_printf(m, "%s ", level_name[level]); 138 if (pr & _PAGE_INVALID) { 139 pt_dump_seq_printf(m, "I\n"); 140 return; 141 } 142 pt_dump_seq_puts(m, (pr & _PAGE_PROTECT) ? "RO " : "RW "); 143 pt_dump_seq_puts(m, (pr & _PAGE_NOEXEC) ? "NX\n" : "X\n"); 144 } 145 146 static void note_prot_wx(struct pg_state *st, unsigned long addr) 147 { 148 if (!st->check_wx) 149 return; 150 if (st->current_prot & _PAGE_INVALID) 151 return; 152 if (st->current_prot & _PAGE_PROTECT) 153 return; 154 if (st->current_prot & _PAGE_NOEXEC) 155 return; 156 /* 157 * The first lowcore page is W+X if spectre mitigations are using 158 * trampolines or the BEAR enhancements facility is not installed, 159 * in which case we have two lpswe instructions in lowcore that need 160 * to be executable. 161 */ 162 if (addr == PAGE_SIZE && (nospec_uses_trampoline() || !static_key_enabled(&cpu_has_bear))) 163 return; 164 WARN_ONCE(IS_ENABLED(CONFIG_DEBUG_WX), 165 "s390/mm: Found insecure W+X mapping at address %pS\n", 166 (void *)st->start_address); 167 st->wx_pages += (addr - st->start_address) / PAGE_SIZE; 168 } 169 170 static void note_page_update_state(struct pg_state *st, unsigned long addr, unsigned int prot, int level) 171 { 172 struct seq_file *m = st->seq; 173 174 while (addr >= st->marker[1].start_address) { 175 st->marker++; 176 pt_dump_seq_printf(m, "---[ %s ]---\n", st->marker->name); 177 } 178 st->start_address = addr; 179 st->current_prot = prot; 180 st->level = level; 181 } 182 183 static void note_page(struct ptdump_state *pt_st, unsigned long addr, int level, u64 val) 184 { 185 int width = sizeof(unsigned long) * 2; 186 static const char units[] = "KMGTPE"; 187 const char *unit = units; 188 unsigned long delta; 189 struct pg_state *st; 190 struct seq_file *m; 191 unsigned int prot; 192 193 st = container_of(pt_st, struct pg_state, ptdump); 194 m = st->seq; 195 prot = val & (_PAGE_PROTECT | _PAGE_NOEXEC); 196 if (level == 4 && (val & _PAGE_INVALID)) 197 prot = _PAGE_INVALID; 198 /* For pmd_none() & friends val gets passed as zero. */ 199 if (level != 4 && !val) 200 prot = _PAGE_INVALID; 201 /* Final flush from generic code. */ 202 if (level == -1) 203 addr = max_addr; 204 if (st->level == -1) { 205 pt_dump_seq_printf(m, "---[ %s ]---\n", st->marker->name); 206 note_page_update_state(st, addr, prot, level); 207 } else if (prot != st->current_prot || level != st->level || 208 addr >= st->marker[1].start_address) { 209 note_prot_wx(st, addr); 210 pt_dump_seq_printf(m, "0x%0*lx-0x%0*lx ", 211 width, st->start_address, 212 width, addr); 213 delta = (addr - st->start_address) >> 10; 214 while (!(delta & 0x3ff) && unit[1]) { 215 delta >>= 10; 216 unit++; 217 } 218 pt_dump_seq_printf(m, "%9lu%c ", delta, *unit); 219 print_prot(m, st->current_prot, st->level); 220 note_page_update_state(st, addr, prot, level); 221 } 222 } 223 224 bool ptdump_check_wx(void) 225 { 226 struct pg_state st = { 227 .ptdump = { 228 .note_page = note_page, 229 .range = (struct ptdump_range[]) { 230 {.start = 0, .end = max_addr}, 231 {.start = 0, .end = 0}, 232 } 233 }, 234 .seq = NULL, 235 .level = -1, 236 .current_prot = 0, 237 .check_wx = true, 238 .wx_pages = 0, 239 .start_address = 0, 240 .marker = (struct addr_marker[]) { 241 { .start_address = 0, .name = NULL}, 242 { .start_address = -1, .name = NULL}, 243 }, 244 }; 245 246 if (!MACHINE_HAS_NX) 247 return true; 248 ptdump_walk_pgd(&st.ptdump, &init_mm, NULL); 249 if (st.wx_pages) { 250 pr_warn("Checked W+X mappings: FAILED, %lu W+X pages found\n", st.wx_pages); 251 252 return false; 253 } else { 254 pr_info("Checked W+X mappings: passed, no %sW+X pages found\n", 255 (nospec_uses_trampoline() || !static_key_enabled(&cpu_has_bear)) ? 256 "unexpected " : ""); 257 258 return true; 259 } 260 } 261 262 #ifdef CONFIG_PTDUMP_DEBUGFS 263 static int ptdump_show(struct seq_file *m, void *v) 264 { 265 struct pg_state st = { 266 .ptdump = { 267 .note_page = note_page, 268 .range = (struct ptdump_range[]) { 269 {.start = 0, .end = max_addr}, 270 {.start = 0, .end = 0}, 271 } 272 }, 273 .seq = m, 274 .level = -1, 275 .current_prot = 0, 276 .check_wx = false, 277 .wx_pages = 0, 278 .start_address = 0, 279 .marker = address_markers, 280 }; 281 282 get_online_mems(); 283 mutex_lock(&cpa_mutex); 284 ptdump_walk_pgd(&st.ptdump, &init_mm, NULL); 285 mutex_unlock(&cpa_mutex); 286 put_online_mems(); 287 return 0; 288 } 289 DEFINE_SHOW_ATTRIBUTE(ptdump); 290 #endif /* CONFIG_PTDUMP_DEBUGFS */ 291 292 static int ptdump_cmp(const void *a, const void *b) 293 { 294 const struct addr_marker *ama = a; 295 const struct addr_marker *amb = b; 296 297 if (ama->start_address > amb->start_address) 298 return 1; 299 if (ama->start_address < amb->start_address) 300 return -1; 301 /* 302 * If the start addresses of two markers are identical consider the 303 * marker which defines the start of an area higher than the one which 304 * defines the end of an area. This keeps pairs of markers sorted. 305 */ 306 if (ama->is_start) 307 return 1; 308 if (amb->is_start) 309 return -1; 310 return 0; 311 } 312 313 static int pt_dump_init(void) 314 { 315 #ifdef CONFIG_KFENCE 316 unsigned long kfence_start = (unsigned long)__kfence_pool; 317 #endif 318 unsigned long lowcore = (unsigned long)get_lowcore(); 319 320 /* 321 * Figure out the maximum virtual address being accessible with the 322 * kernel ASCE. We need this to keep the page table walker functions 323 * from accessing non-existent entries. 324 */ 325 max_addr = (get_lowcore()->kernel_asce.val & _REGION_ENTRY_TYPE_MASK) >> 2; 326 max_addr = 1UL << (max_addr * 11 + 31); 327 address_markers[LOWCORE_START_NR].start_address = lowcore; 328 address_markers[LOWCORE_END_NR].start_address = lowcore + sizeof(struct lowcore); 329 address_markers[IDENTITY_START_NR].start_address = __identity_base; 330 address_markers[IDENTITY_END_NR].start_address = __identity_base + ident_map_size; 331 address_markers[AMODE31_START_NR].start_address = (unsigned long)__samode31; 332 address_markers[AMODE31_END_NR].start_address = (unsigned long)__eamode31; 333 address_markers[MODULES_NR].start_address = MODULES_VADDR; 334 address_markers[MODULES_END_NR].start_address = MODULES_END; 335 address_markers[ABS_LOWCORE_NR].start_address = __abs_lowcore; 336 address_markers[ABS_LOWCORE_END_NR].start_address = __abs_lowcore + ABS_LOWCORE_MAP_SIZE; 337 address_markers[MEMCPY_REAL_NR].start_address = __memcpy_real_area; 338 address_markers[MEMCPY_REAL_END_NR].start_address = __memcpy_real_area + MEMCPY_REAL_SIZE; 339 address_markers[VMEMMAP_NR].start_address = (unsigned long) vmemmap; 340 address_markers[VMEMMAP_END_NR].start_address = (unsigned long)vmemmap + vmemmap_size; 341 address_markers[VMALLOC_NR].start_address = VMALLOC_START; 342 address_markers[VMALLOC_END_NR].start_address = VMALLOC_END; 343 #ifdef CONFIG_KFENCE 344 address_markers[KFENCE_START_NR].start_address = kfence_start; 345 address_markers[KFENCE_END_NR].start_address = kfence_start + KFENCE_POOL_SIZE; 346 #endif 347 #ifdef CONFIG_KMSAN 348 address_markers[KMSAN_VMALLOC_SHADOW_START_NR].start_address = KMSAN_VMALLOC_SHADOW_START; 349 address_markers[KMSAN_VMALLOC_SHADOW_END_NR].start_address = KMSAN_VMALLOC_SHADOW_END; 350 address_markers[KMSAN_VMALLOC_ORIGIN_START_NR].start_address = KMSAN_VMALLOC_ORIGIN_START; 351 address_markers[KMSAN_VMALLOC_ORIGIN_END_NR].start_address = KMSAN_VMALLOC_ORIGIN_END; 352 address_markers[KMSAN_MODULES_SHADOW_START_NR].start_address = KMSAN_MODULES_SHADOW_START; 353 address_markers[KMSAN_MODULES_SHADOW_END_NR].start_address = KMSAN_MODULES_SHADOW_END; 354 address_markers[KMSAN_MODULES_ORIGIN_START_NR].start_address = KMSAN_MODULES_ORIGIN_START; 355 address_markers[KMSAN_MODULES_ORIGIN_END_NR].start_address = KMSAN_MODULES_ORIGIN_END; 356 #endif 357 sort(address_markers, ARRAY_SIZE(address_markers) - 1, 358 sizeof(address_markers[0]), ptdump_cmp, NULL); 359 #ifdef CONFIG_PTDUMP_DEBUGFS 360 debugfs_create_file("kernel_page_tables", 0400, NULL, NULL, &ptdump_fops); 361 #endif /* CONFIG_PTDUMP_DEBUGFS */ 362 return 0; 363 } 364 device_initcall(pt_dump_init); 365