1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * This kernel test validates architecture page table helpers and 4 * accessors and helps in verifying their continued compliance with 5 * expected generic MM semantics. 6 * 7 * Copyright (C) 2019 ARM Ltd. 8 * 9 * Author: Anshuman Khandual <anshuman.khandual@arm.com> 10 */ 11 #define pr_fmt(fmt) "debug_vm_pgtable: [%-25s]: " fmt, __func__ 12 13 #include <linux/gfp.h> 14 #include <linux/highmem.h> 15 #include <linux/hugetlb.h> 16 #include <linux/kernel.h> 17 #include <linux/kconfig.h> 18 #include <linux/memblock.h> 19 #include <linux/mm.h> 20 #include <linux/mman.h> 21 #include <linux/mm_types.h> 22 #include <linux/module.h> 23 #include <linux/pfn_t.h> 24 #include <linux/printk.h> 25 #include <linux/pgtable.h> 26 #include <linux/random.h> 27 #include <linux/spinlock.h> 28 #include <linux/swap.h> 29 #include <linux/swapops.h> 30 #include <linux/start_kernel.h> 31 #include <linux/sched/mm.h> 32 #include <linux/io.h> 33 #include <linux/vmalloc.h> 34 35 #include <asm/cacheflush.h> 36 #include <asm/pgalloc.h> 37 #include <asm/tlbflush.h> 38 39 /* 40 * Please refer Documentation/mm/arch_pgtable_helpers.rst for the semantics 41 * expectations that are being validated here. All future changes in here 42 * or the documentation need to be in sync. 43 */ 44 #define RANDOM_NZVALUE GENMASK(7, 0) 45 46 struct pgtable_debug_args { 47 struct mm_struct *mm; 48 struct vm_area_struct *vma; 49 50 pgd_t *pgdp; 51 p4d_t *p4dp; 52 pud_t *pudp; 53 pmd_t *pmdp; 54 pte_t *ptep; 55 56 p4d_t *start_p4dp; 57 pud_t *start_pudp; 58 pmd_t *start_pmdp; 59 pgtable_t start_ptep; 60 61 unsigned long vaddr; 62 pgprot_t page_prot; 63 pgprot_t page_prot_none; 64 65 bool is_contiguous_page; 66 unsigned long pud_pfn; 67 unsigned long pmd_pfn; 68 unsigned long pte_pfn; 69 70 unsigned long fixed_alignment; 71 unsigned long fixed_pgd_pfn; 72 unsigned long fixed_p4d_pfn; 73 unsigned long fixed_pud_pfn; 74 unsigned long fixed_pmd_pfn; 75 unsigned long fixed_pte_pfn; 76 }; 77 78 static void __init pte_basic_tests(struct pgtable_debug_args *args, int idx) 79 { 80 pgprot_t prot = vm_get_page_prot(idx); 81 pte_t pte = pfn_pte(args->fixed_pte_pfn, prot); 82 unsigned long val = idx, *ptr = &val; 83 84 pr_debug("Validating PTE basic (%pGv)\n", ptr); 85 86 /* 87 * This test needs to be executed after the given page table entry 88 * is created with pfn_pte() to make sure that vm_get_page_prot(idx) 89 * does not have the dirty bit enabled from the beginning. This is 90 * important for platforms like arm64 where (!PTE_RDONLY) indicate 91 * dirty bit being set. 92 */ 93 WARN_ON(pte_dirty(pte_wrprotect(pte))); 94 95 WARN_ON(!pte_same(pte, pte)); 96 WARN_ON(!pte_young(pte_mkyoung(pte_mkold(pte)))); 97 WARN_ON(!pte_dirty(pte_mkdirty(pte_mkclean(pte)))); 98 WARN_ON(!pte_write(pte_mkwrite(pte_wrprotect(pte), args->vma))); 99 WARN_ON(pte_young(pte_mkold(pte_mkyoung(pte)))); 100 WARN_ON(pte_dirty(pte_mkclean(pte_mkdirty(pte)))); 101 WARN_ON(pte_write(pte_wrprotect(pte_mkwrite(pte, args->vma)))); 102 WARN_ON(pte_dirty(pte_wrprotect(pte_mkclean(pte)))); 103 WARN_ON(!pte_dirty(pte_wrprotect(pte_mkdirty(pte)))); 104 } 105 106 static void __init pte_advanced_tests(struct pgtable_debug_args *args) 107 { 108 struct page *page; 109 pte_t pte; 110 111 /* 112 * Architectures optimize set_pte_at by avoiding TLB flush. 113 * This requires set_pte_at to be not used to update an 114 * existing pte entry. Clear pte before we do set_pte_at 115 * 116 * flush_dcache_page() is called after set_pte_at() to clear 117 * PG_arch_1 for the page on ARM64. The page flag isn't cleared 118 * when it's released and page allocation check will fail when 119 * the page is allocated again. For architectures other than ARM64, 120 * the unexpected overhead of cache flushing is acceptable. 121 */ 122 page = (args->pte_pfn != ULONG_MAX) ? pfn_to_page(args->pte_pfn) : NULL; 123 if (!page) 124 return; 125 126 pr_debug("Validating PTE advanced\n"); 127 if (WARN_ON(!args->ptep)) 128 return; 129 130 pte = pfn_pte(args->pte_pfn, args->page_prot); 131 set_pte_at(args->mm, args->vaddr, args->ptep, pte); 132 flush_dcache_page(page); 133 ptep_set_wrprotect(args->mm, args->vaddr, args->ptep); 134 pte = ptep_get(args->ptep); 135 WARN_ON(pte_write(pte)); 136 ptep_get_and_clear(args->mm, args->vaddr, args->ptep); 137 pte = ptep_get(args->ptep); 138 WARN_ON(!pte_none(pte)); 139 140 pte = pfn_pte(args->pte_pfn, args->page_prot); 141 pte = pte_wrprotect(pte); 142 pte = pte_mkclean(pte); 143 set_pte_at(args->mm, args->vaddr, args->ptep, pte); 144 flush_dcache_page(page); 145 pte = pte_mkwrite(pte, args->vma); 146 pte = pte_mkdirty(pte); 147 ptep_set_access_flags(args->vma, args->vaddr, args->ptep, pte, 1); 148 pte = ptep_get(args->ptep); 149 WARN_ON(!(pte_write(pte) && pte_dirty(pte))); 150 ptep_get_and_clear_full(args->mm, args->vaddr, args->ptep, 1); 151 pte = ptep_get(args->ptep); 152 WARN_ON(!pte_none(pte)); 153 154 pte = pfn_pte(args->pte_pfn, args->page_prot); 155 pte = pte_mkyoung(pte); 156 set_pte_at(args->mm, args->vaddr, args->ptep, pte); 157 flush_dcache_page(page); 158 ptep_test_and_clear_young(args->vma, args->vaddr, args->ptep); 159 pte = ptep_get(args->ptep); 160 WARN_ON(pte_young(pte)); 161 162 ptep_get_and_clear_full(args->mm, args->vaddr, args->ptep, 1); 163 } 164 165 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 166 static void __init pmd_basic_tests(struct pgtable_debug_args *args, int idx) 167 { 168 pgprot_t prot = vm_get_page_prot(idx); 169 unsigned long val = idx, *ptr = &val; 170 pmd_t pmd; 171 172 if (!has_transparent_hugepage()) 173 return; 174 175 pr_debug("Validating PMD basic (%pGv)\n", ptr); 176 pmd = pfn_pmd(args->fixed_pmd_pfn, prot); 177 178 /* 179 * This test needs to be executed after the given page table entry 180 * is created with pfn_pmd() to make sure that vm_get_page_prot(idx) 181 * does not have the dirty bit enabled from the beginning. This is 182 * important for platforms like arm64 where (!PTE_RDONLY) indicate 183 * dirty bit being set. 184 */ 185 WARN_ON(pmd_dirty(pmd_wrprotect(pmd))); 186 187 188 WARN_ON(!pmd_same(pmd, pmd)); 189 WARN_ON(!pmd_young(pmd_mkyoung(pmd_mkold(pmd)))); 190 WARN_ON(!pmd_dirty(pmd_mkdirty(pmd_mkclean(pmd)))); 191 WARN_ON(!pmd_write(pmd_mkwrite(pmd_wrprotect(pmd), args->vma))); 192 WARN_ON(pmd_young(pmd_mkold(pmd_mkyoung(pmd)))); 193 WARN_ON(pmd_dirty(pmd_mkclean(pmd_mkdirty(pmd)))); 194 WARN_ON(pmd_write(pmd_wrprotect(pmd_mkwrite(pmd, args->vma)))); 195 WARN_ON(pmd_dirty(pmd_wrprotect(pmd_mkclean(pmd)))); 196 WARN_ON(!pmd_dirty(pmd_wrprotect(pmd_mkdirty(pmd)))); 197 /* 198 * A huge page does not point to next level page table 199 * entry. Hence this must qualify as pmd_bad(). 200 */ 201 WARN_ON(!pmd_bad(pmd_mkhuge(pmd))); 202 } 203 204 static void __init pmd_advanced_tests(struct pgtable_debug_args *args) 205 { 206 struct page *page; 207 pmd_t pmd; 208 unsigned long vaddr = args->vaddr; 209 210 if (!has_transparent_hugepage()) 211 return; 212 213 page = (args->pmd_pfn != ULONG_MAX) ? pfn_to_page(args->pmd_pfn) : NULL; 214 if (!page) 215 return; 216 217 /* 218 * flush_dcache_page() is called after set_pmd_at() to clear 219 * PG_arch_1 for the page on ARM64. The page flag isn't cleared 220 * when it's released and page allocation check will fail when 221 * the page is allocated again. For architectures other than ARM64, 222 * the unexpected overhead of cache flushing is acceptable. 223 */ 224 pr_debug("Validating PMD advanced\n"); 225 /* Align the address wrt HPAGE_PMD_SIZE */ 226 vaddr &= HPAGE_PMD_MASK; 227 228 pgtable_trans_huge_deposit(args->mm, args->pmdp, args->start_ptep); 229 230 pmd = pfn_pmd(args->pmd_pfn, args->page_prot); 231 set_pmd_at(args->mm, vaddr, args->pmdp, pmd); 232 flush_dcache_page(page); 233 pmdp_set_wrprotect(args->mm, vaddr, args->pmdp); 234 pmd = pmdp_get(args->pmdp); 235 WARN_ON(pmd_write(pmd)); 236 pmdp_huge_get_and_clear(args->mm, vaddr, args->pmdp); 237 pmd = pmdp_get(args->pmdp); 238 WARN_ON(!pmd_none(pmd)); 239 240 pmd = pfn_pmd(args->pmd_pfn, args->page_prot); 241 pmd = pmd_wrprotect(pmd); 242 pmd = pmd_mkclean(pmd); 243 set_pmd_at(args->mm, vaddr, args->pmdp, pmd); 244 flush_dcache_page(page); 245 pmd = pmd_mkwrite(pmd, args->vma); 246 pmd = pmd_mkdirty(pmd); 247 pmdp_set_access_flags(args->vma, vaddr, args->pmdp, pmd, 1); 248 pmd = pmdp_get(args->pmdp); 249 WARN_ON(!(pmd_write(pmd) && pmd_dirty(pmd))); 250 pmdp_huge_get_and_clear_full(args->vma, vaddr, args->pmdp, 1); 251 pmd = pmdp_get(args->pmdp); 252 WARN_ON(!pmd_none(pmd)); 253 254 pmd = pmd_mkhuge(pfn_pmd(args->pmd_pfn, args->page_prot)); 255 pmd = pmd_mkyoung(pmd); 256 set_pmd_at(args->mm, vaddr, args->pmdp, pmd); 257 flush_dcache_page(page); 258 pmdp_test_and_clear_young(args->vma, vaddr, args->pmdp); 259 pmd = pmdp_get(args->pmdp); 260 WARN_ON(pmd_young(pmd)); 261 262 /* Clear the pte entries */ 263 pmdp_huge_get_and_clear(args->mm, vaddr, args->pmdp); 264 pgtable_trans_huge_withdraw(args->mm, args->pmdp); 265 } 266 267 static void __init pmd_leaf_tests(struct pgtable_debug_args *args) 268 { 269 pmd_t pmd; 270 271 if (!has_transparent_hugepage()) 272 return; 273 274 pr_debug("Validating PMD leaf\n"); 275 pmd = pfn_pmd(args->fixed_pmd_pfn, args->page_prot); 276 277 /* 278 * PMD based THP is a leaf entry. 279 */ 280 pmd = pmd_mkhuge(pmd); 281 WARN_ON(!pmd_leaf(pmd)); 282 } 283 284 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD 285 static void __init pud_basic_tests(struct pgtable_debug_args *args, int idx) 286 { 287 pgprot_t prot = vm_get_page_prot(idx); 288 unsigned long val = idx, *ptr = &val; 289 pud_t pud; 290 291 if (!has_transparent_pud_hugepage()) 292 return; 293 294 pr_debug("Validating PUD basic (%pGv)\n", ptr); 295 pud = pfn_pud(args->fixed_pud_pfn, prot); 296 297 /* 298 * This test needs to be executed after the given page table entry 299 * is created with pfn_pud() to make sure that vm_get_page_prot(idx) 300 * does not have the dirty bit enabled from the beginning. This is 301 * important for platforms like arm64 where (!PTE_RDONLY) indicate 302 * dirty bit being set. 303 */ 304 WARN_ON(pud_dirty(pud_wrprotect(pud))); 305 306 WARN_ON(!pud_same(pud, pud)); 307 WARN_ON(!pud_young(pud_mkyoung(pud_mkold(pud)))); 308 WARN_ON(!pud_dirty(pud_mkdirty(pud_mkclean(pud)))); 309 WARN_ON(pud_dirty(pud_mkclean(pud_mkdirty(pud)))); 310 WARN_ON(!pud_write(pud_mkwrite(pud_wrprotect(pud)))); 311 WARN_ON(pud_write(pud_wrprotect(pud_mkwrite(pud)))); 312 WARN_ON(pud_young(pud_mkold(pud_mkyoung(pud)))); 313 WARN_ON(pud_dirty(pud_wrprotect(pud_mkclean(pud)))); 314 WARN_ON(!pud_dirty(pud_wrprotect(pud_mkdirty(pud)))); 315 316 if (mm_pmd_folded(args->mm)) 317 return; 318 319 /* 320 * A huge page does not point to next level page table 321 * entry. Hence this must qualify as pud_bad(). 322 */ 323 WARN_ON(!pud_bad(pud_mkhuge(pud))); 324 } 325 326 static void __init pud_advanced_tests(struct pgtable_debug_args *args) 327 { 328 struct page *page; 329 unsigned long vaddr = args->vaddr; 330 pud_t pud; 331 332 if (!has_transparent_pud_hugepage()) 333 return; 334 335 page = (args->pud_pfn != ULONG_MAX) ? pfn_to_page(args->pud_pfn) : NULL; 336 if (!page) 337 return; 338 339 /* 340 * flush_dcache_page() is called after set_pud_at() to clear 341 * PG_arch_1 for the page on ARM64. The page flag isn't cleared 342 * when it's released and page allocation check will fail when 343 * the page is allocated again. For architectures other than ARM64, 344 * the unexpected overhead of cache flushing is acceptable. 345 */ 346 pr_debug("Validating PUD advanced\n"); 347 /* Align the address wrt HPAGE_PUD_SIZE */ 348 vaddr &= HPAGE_PUD_MASK; 349 350 pud = pfn_pud(args->pud_pfn, args->page_prot); 351 set_pud_at(args->mm, vaddr, args->pudp, pud); 352 flush_dcache_page(page); 353 pudp_set_wrprotect(args->mm, vaddr, args->pudp); 354 pud = pudp_get(args->pudp); 355 WARN_ON(pud_write(pud)); 356 357 #ifndef __PAGETABLE_PMD_FOLDED 358 pudp_huge_get_and_clear(args->mm, vaddr, args->pudp); 359 pud = pudp_get(args->pudp); 360 WARN_ON(!pud_none(pud)); 361 #endif /* __PAGETABLE_PMD_FOLDED */ 362 pud = pfn_pud(args->pud_pfn, args->page_prot); 363 pud = pud_wrprotect(pud); 364 pud = pud_mkclean(pud); 365 set_pud_at(args->mm, vaddr, args->pudp, pud); 366 flush_dcache_page(page); 367 pud = pud_mkwrite(pud); 368 pud = pud_mkdirty(pud); 369 pudp_set_access_flags(args->vma, vaddr, args->pudp, pud, 1); 370 pud = pudp_get(args->pudp); 371 WARN_ON(!(pud_write(pud) && pud_dirty(pud))); 372 373 #ifndef __PAGETABLE_PMD_FOLDED 374 pudp_huge_get_and_clear_full(args->vma, vaddr, args->pudp, 1); 375 pud = pudp_get(args->pudp); 376 WARN_ON(!pud_none(pud)); 377 #endif /* __PAGETABLE_PMD_FOLDED */ 378 379 pud = pfn_pud(args->pud_pfn, args->page_prot); 380 pud = pud_mkyoung(pud); 381 set_pud_at(args->mm, vaddr, args->pudp, pud); 382 flush_dcache_page(page); 383 pudp_test_and_clear_young(args->vma, vaddr, args->pudp); 384 pud = pudp_get(args->pudp); 385 WARN_ON(pud_young(pud)); 386 387 pudp_huge_get_and_clear(args->mm, vaddr, args->pudp); 388 } 389 390 static void __init pud_leaf_tests(struct pgtable_debug_args *args) 391 { 392 pud_t pud; 393 394 if (!has_transparent_pud_hugepage()) 395 return; 396 397 pr_debug("Validating PUD leaf\n"); 398 pud = pfn_pud(args->fixed_pud_pfn, args->page_prot); 399 /* 400 * PUD based THP is a leaf entry. 401 */ 402 pud = pud_mkhuge(pud); 403 WARN_ON(!pud_leaf(pud)); 404 } 405 #else /* !CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */ 406 static void __init pud_basic_tests(struct pgtable_debug_args *args, int idx) { } 407 static void __init pud_advanced_tests(struct pgtable_debug_args *args) { } 408 static void __init pud_leaf_tests(struct pgtable_debug_args *args) { } 409 #endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */ 410 #else /* !CONFIG_TRANSPARENT_HUGEPAGE */ 411 static void __init pmd_basic_tests(struct pgtable_debug_args *args, int idx) { } 412 static void __init pud_basic_tests(struct pgtable_debug_args *args, int idx) { } 413 static void __init pmd_advanced_tests(struct pgtable_debug_args *args) { } 414 static void __init pud_advanced_tests(struct pgtable_debug_args *args) { } 415 static void __init pmd_leaf_tests(struct pgtable_debug_args *args) { } 416 static void __init pud_leaf_tests(struct pgtable_debug_args *args) { } 417 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 418 419 #ifdef CONFIG_HAVE_ARCH_HUGE_VMAP 420 static void __init pmd_huge_tests(struct pgtable_debug_args *args) 421 { 422 pmd_t pmd; 423 424 if (!arch_vmap_pmd_supported(args->page_prot) || 425 args->fixed_alignment < PMD_SIZE) 426 return; 427 428 pr_debug("Validating PMD huge\n"); 429 /* 430 * X86 defined pmd_set_huge() verifies that the given 431 * PMD is not a populated non-leaf entry. 432 */ 433 WRITE_ONCE(*args->pmdp, __pmd(0)); 434 WARN_ON(!pmd_set_huge(args->pmdp, __pfn_to_phys(args->fixed_pmd_pfn), args->page_prot)); 435 WARN_ON(!pmd_clear_huge(args->pmdp)); 436 pmd = pmdp_get(args->pmdp); 437 WARN_ON(!pmd_none(pmd)); 438 } 439 440 static void __init pud_huge_tests(struct pgtable_debug_args *args) 441 { 442 pud_t pud; 443 444 if (!arch_vmap_pud_supported(args->page_prot) || 445 args->fixed_alignment < PUD_SIZE) 446 return; 447 448 pr_debug("Validating PUD huge\n"); 449 /* 450 * X86 defined pud_set_huge() verifies that the given 451 * PUD is not a populated non-leaf entry. 452 */ 453 WRITE_ONCE(*args->pudp, __pud(0)); 454 WARN_ON(!pud_set_huge(args->pudp, __pfn_to_phys(args->fixed_pud_pfn), args->page_prot)); 455 WARN_ON(!pud_clear_huge(args->pudp)); 456 pud = pudp_get(args->pudp); 457 WARN_ON(!pud_none(pud)); 458 } 459 #else /* !CONFIG_HAVE_ARCH_HUGE_VMAP */ 460 static void __init pmd_huge_tests(struct pgtable_debug_args *args) { } 461 static void __init pud_huge_tests(struct pgtable_debug_args *args) { } 462 #endif /* CONFIG_HAVE_ARCH_HUGE_VMAP */ 463 464 static void __init p4d_basic_tests(struct pgtable_debug_args *args) 465 { 466 p4d_t p4d; 467 468 pr_debug("Validating P4D basic\n"); 469 memset(&p4d, RANDOM_NZVALUE, sizeof(p4d_t)); 470 WARN_ON(!p4d_same(p4d, p4d)); 471 } 472 473 static void __init pgd_basic_tests(struct pgtable_debug_args *args) 474 { 475 pgd_t pgd; 476 477 pr_debug("Validating PGD basic\n"); 478 memset(&pgd, RANDOM_NZVALUE, sizeof(pgd_t)); 479 WARN_ON(!pgd_same(pgd, pgd)); 480 } 481 482 #ifndef __PAGETABLE_PUD_FOLDED 483 static void __init pud_clear_tests(struct pgtable_debug_args *args) 484 { 485 pud_t pud = pudp_get(args->pudp); 486 487 if (mm_pmd_folded(args->mm)) 488 return; 489 490 pr_debug("Validating PUD clear\n"); 491 WARN_ON(pud_none(pud)); 492 pud_clear(args->pudp); 493 pud = pudp_get(args->pudp); 494 WARN_ON(!pud_none(pud)); 495 } 496 497 static void __init pud_populate_tests(struct pgtable_debug_args *args) 498 { 499 pud_t pud; 500 501 if (mm_pmd_folded(args->mm)) 502 return; 503 504 pr_debug("Validating PUD populate\n"); 505 /* 506 * This entry points to next level page table page. 507 * Hence this must not qualify as pud_bad(). 508 */ 509 pud_populate(args->mm, args->pudp, args->start_pmdp); 510 pud = pudp_get(args->pudp); 511 WARN_ON(pud_bad(pud)); 512 } 513 #else /* !__PAGETABLE_PUD_FOLDED */ 514 static void __init pud_clear_tests(struct pgtable_debug_args *args) { } 515 static void __init pud_populate_tests(struct pgtable_debug_args *args) { } 516 #endif /* PAGETABLE_PUD_FOLDED */ 517 518 #ifndef __PAGETABLE_P4D_FOLDED 519 static void __init p4d_clear_tests(struct pgtable_debug_args *args) 520 { 521 p4d_t p4d = p4dp_get(args->p4dp); 522 523 if (mm_pud_folded(args->mm)) 524 return; 525 526 pr_debug("Validating P4D clear\n"); 527 WARN_ON(p4d_none(p4d)); 528 p4d_clear(args->p4dp); 529 p4d = p4dp_get(args->p4dp); 530 WARN_ON(!p4d_none(p4d)); 531 } 532 533 static void __init p4d_populate_tests(struct pgtable_debug_args *args) 534 { 535 p4d_t p4d; 536 537 if (mm_pud_folded(args->mm)) 538 return; 539 540 pr_debug("Validating P4D populate\n"); 541 /* 542 * This entry points to next level page table page. 543 * Hence this must not qualify as p4d_bad(). 544 */ 545 pud_clear(args->pudp); 546 p4d_clear(args->p4dp); 547 p4d_populate(args->mm, args->p4dp, args->start_pudp); 548 p4d = p4dp_get(args->p4dp); 549 WARN_ON(p4d_bad(p4d)); 550 } 551 552 static void __init pgd_clear_tests(struct pgtable_debug_args *args) 553 { 554 pgd_t pgd = pgdp_get(args->pgdp); 555 556 if (mm_p4d_folded(args->mm)) 557 return; 558 559 pr_debug("Validating PGD clear\n"); 560 WARN_ON(pgd_none(pgd)); 561 pgd_clear(args->pgdp); 562 pgd = pgdp_get(args->pgdp); 563 WARN_ON(!pgd_none(pgd)); 564 } 565 566 static void __init pgd_populate_tests(struct pgtable_debug_args *args) 567 { 568 pgd_t pgd; 569 570 if (mm_p4d_folded(args->mm)) 571 return; 572 573 pr_debug("Validating PGD populate\n"); 574 /* 575 * This entry points to next level page table page. 576 * Hence this must not qualify as pgd_bad(). 577 */ 578 p4d_clear(args->p4dp); 579 pgd_clear(args->pgdp); 580 pgd_populate(args->mm, args->pgdp, args->start_p4dp); 581 pgd = pgdp_get(args->pgdp); 582 WARN_ON(pgd_bad(pgd)); 583 } 584 #else /* !__PAGETABLE_P4D_FOLDED */ 585 static void __init p4d_clear_tests(struct pgtable_debug_args *args) { } 586 static void __init pgd_clear_tests(struct pgtable_debug_args *args) { } 587 static void __init p4d_populate_tests(struct pgtable_debug_args *args) { } 588 static void __init pgd_populate_tests(struct pgtable_debug_args *args) { } 589 #endif /* PAGETABLE_P4D_FOLDED */ 590 591 static void __init pte_clear_tests(struct pgtable_debug_args *args) 592 { 593 struct page *page; 594 pte_t pte = pfn_pte(args->pte_pfn, args->page_prot); 595 596 page = (args->pte_pfn != ULONG_MAX) ? pfn_to_page(args->pte_pfn) : NULL; 597 if (!page) 598 return; 599 600 /* 601 * flush_dcache_page() is called after set_pte_at() to clear 602 * PG_arch_1 for the page on ARM64. The page flag isn't cleared 603 * when it's released and page allocation check will fail when 604 * the page is allocated again. For architectures other than ARM64, 605 * the unexpected overhead of cache flushing is acceptable. 606 */ 607 pr_debug("Validating PTE clear\n"); 608 if (WARN_ON(!args->ptep)) 609 return; 610 611 set_pte_at(args->mm, args->vaddr, args->ptep, pte); 612 WARN_ON(pte_none(pte)); 613 flush_dcache_page(page); 614 barrier(); 615 ptep_clear(args->mm, args->vaddr, args->ptep); 616 pte = ptep_get(args->ptep); 617 WARN_ON(!pte_none(pte)); 618 } 619 620 static void __init pmd_clear_tests(struct pgtable_debug_args *args) 621 { 622 pmd_t pmd = pmdp_get(args->pmdp); 623 624 pr_debug("Validating PMD clear\n"); 625 WARN_ON(pmd_none(pmd)); 626 pmd_clear(args->pmdp); 627 pmd = pmdp_get(args->pmdp); 628 WARN_ON(!pmd_none(pmd)); 629 } 630 631 static void __init pmd_populate_tests(struct pgtable_debug_args *args) 632 { 633 pmd_t pmd; 634 635 pr_debug("Validating PMD populate\n"); 636 /* 637 * This entry points to next level page table page. 638 * Hence this must not qualify as pmd_bad(). 639 */ 640 pmd_populate(args->mm, args->pmdp, args->start_ptep); 641 pmd = pmdp_get(args->pmdp); 642 WARN_ON(pmd_bad(pmd)); 643 } 644 645 static void __init pte_special_tests(struct pgtable_debug_args *args) 646 { 647 pte_t pte = pfn_pte(args->fixed_pte_pfn, args->page_prot); 648 649 if (!IS_ENABLED(CONFIG_ARCH_HAS_PTE_SPECIAL)) 650 return; 651 652 pr_debug("Validating PTE special\n"); 653 WARN_ON(!pte_special(pte_mkspecial(pte))); 654 } 655 656 static void __init pte_protnone_tests(struct pgtable_debug_args *args) 657 { 658 pte_t pte = pfn_pte(args->fixed_pte_pfn, args->page_prot_none); 659 660 if (!IS_ENABLED(CONFIG_NUMA_BALANCING)) 661 return; 662 663 pr_debug("Validating PTE protnone\n"); 664 WARN_ON(!pte_protnone(pte)); 665 WARN_ON(!pte_present(pte)); 666 } 667 668 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 669 static void __init pmd_protnone_tests(struct pgtable_debug_args *args) 670 { 671 pmd_t pmd; 672 673 if (!IS_ENABLED(CONFIG_NUMA_BALANCING)) 674 return; 675 676 if (!has_transparent_hugepage()) 677 return; 678 679 pr_debug("Validating PMD protnone\n"); 680 pmd = pmd_mkhuge(pfn_pmd(args->fixed_pmd_pfn, args->page_prot_none)); 681 WARN_ON(!pmd_protnone(pmd)); 682 WARN_ON(!pmd_present(pmd)); 683 } 684 #else /* !CONFIG_TRANSPARENT_HUGEPAGE */ 685 static void __init pmd_protnone_tests(struct pgtable_debug_args *args) { } 686 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 687 688 static void __init pte_soft_dirty_tests(struct pgtable_debug_args *args) 689 { 690 pte_t pte = pfn_pte(args->fixed_pte_pfn, args->page_prot); 691 692 if (!IS_ENABLED(CONFIG_MEM_SOFT_DIRTY)) 693 return; 694 695 pr_debug("Validating PTE soft dirty\n"); 696 WARN_ON(!pte_soft_dirty(pte_mksoft_dirty(pte))); 697 WARN_ON(pte_soft_dirty(pte_clear_soft_dirty(pte))); 698 } 699 700 static void __init pte_swap_soft_dirty_tests(struct pgtable_debug_args *args) 701 { 702 pte_t pte = pfn_pte(args->fixed_pte_pfn, args->page_prot); 703 704 if (!IS_ENABLED(CONFIG_MEM_SOFT_DIRTY)) 705 return; 706 707 pr_debug("Validating PTE swap soft dirty\n"); 708 WARN_ON(!pte_swp_soft_dirty(pte_swp_mksoft_dirty(pte))); 709 WARN_ON(pte_swp_soft_dirty(pte_swp_clear_soft_dirty(pte))); 710 } 711 712 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 713 static void __init pmd_soft_dirty_tests(struct pgtable_debug_args *args) 714 { 715 pmd_t pmd; 716 717 if (!IS_ENABLED(CONFIG_MEM_SOFT_DIRTY)) 718 return; 719 720 if (!has_transparent_hugepage()) 721 return; 722 723 pr_debug("Validating PMD soft dirty\n"); 724 pmd = pfn_pmd(args->fixed_pmd_pfn, args->page_prot); 725 WARN_ON(!pmd_soft_dirty(pmd_mksoft_dirty(pmd))); 726 WARN_ON(pmd_soft_dirty(pmd_clear_soft_dirty(pmd))); 727 } 728 729 static void __init pmd_swap_soft_dirty_tests(struct pgtable_debug_args *args) 730 { 731 pmd_t pmd; 732 733 if (!IS_ENABLED(CONFIG_MEM_SOFT_DIRTY) || 734 !IS_ENABLED(CONFIG_ARCH_ENABLE_THP_MIGRATION)) 735 return; 736 737 if (!has_transparent_hugepage()) 738 return; 739 740 pr_debug("Validating PMD swap soft dirty\n"); 741 pmd = pfn_pmd(args->fixed_pmd_pfn, args->page_prot); 742 WARN_ON(!pmd_swp_soft_dirty(pmd_swp_mksoft_dirty(pmd))); 743 WARN_ON(pmd_swp_soft_dirty(pmd_swp_clear_soft_dirty(pmd))); 744 } 745 #else /* !CONFIG_TRANSPARENT_HUGEPAGE */ 746 static void __init pmd_soft_dirty_tests(struct pgtable_debug_args *args) { } 747 static void __init pmd_swap_soft_dirty_tests(struct pgtable_debug_args *args) { } 748 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 749 750 static void __init pte_swap_exclusive_tests(struct pgtable_debug_args *args) 751 { 752 unsigned long max_swap_offset; 753 swp_entry_t entry, entry2; 754 pte_t pte; 755 756 pr_debug("Validating PTE swap exclusive\n"); 757 758 /* See generic_max_swapfile_size(): probe the maximum offset */ 759 max_swap_offset = swp_offset(pte_to_swp_entry(swp_entry_to_pte(swp_entry(0, ~0UL)))); 760 761 /* Create a swp entry with all possible bits set */ 762 entry = swp_entry((1 << MAX_SWAPFILES_SHIFT) - 1, max_swap_offset); 763 764 pte = swp_entry_to_pte(entry); 765 WARN_ON(pte_swp_exclusive(pte)); 766 WARN_ON(!is_swap_pte(pte)); 767 entry2 = pte_to_swp_entry(pte); 768 WARN_ON(memcmp(&entry, &entry2, sizeof(entry))); 769 770 pte = pte_swp_mkexclusive(pte); 771 WARN_ON(!pte_swp_exclusive(pte)); 772 WARN_ON(!is_swap_pte(pte)); 773 WARN_ON(pte_swp_soft_dirty(pte)); 774 entry2 = pte_to_swp_entry(pte); 775 WARN_ON(memcmp(&entry, &entry2, sizeof(entry))); 776 777 pte = pte_swp_clear_exclusive(pte); 778 WARN_ON(pte_swp_exclusive(pte)); 779 WARN_ON(!is_swap_pte(pte)); 780 entry2 = pte_to_swp_entry(pte); 781 WARN_ON(memcmp(&entry, &entry2, sizeof(entry))); 782 } 783 784 static void __init pte_swap_tests(struct pgtable_debug_args *args) 785 { 786 swp_entry_t swp; 787 pte_t pte; 788 789 pr_debug("Validating PTE swap\n"); 790 pte = pfn_pte(args->fixed_pte_pfn, args->page_prot); 791 swp = __pte_to_swp_entry(pte); 792 pte = __swp_entry_to_pte(swp); 793 WARN_ON(args->fixed_pte_pfn != pte_pfn(pte)); 794 } 795 796 #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION 797 static void __init pmd_swap_tests(struct pgtable_debug_args *args) 798 { 799 swp_entry_t swp; 800 pmd_t pmd; 801 802 if (!has_transparent_hugepage()) 803 return; 804 805 pr_debug("Validating PMD swap\n"); 806 pmd = pfn_pmd(args->fixed_pmd_pfn, args->page_prot); 807 swp = __pmd_to_swp_entry(pmd); 808 pmd = __swp_entry_to_pmd(swp); 809 WARN_ON(args->fixed_pmd_pfn != pmd_pfn(pmd)); 810 } 811 #else /* !CONFIG_ARCH_ENABLE_THP_MIGRATION */ 812 static void __init pmd_swap_tests(struct pgtable_debug_args *args) { } 813 #endif /* CONFIG_ARCH_ENABLE_THP_MIGRATION */ 814 815 static void __init swap_migration_tests(struct pgtable_debug_args *args) 816 { 817 struct page *page; 818 swp_entry_t swp; 819 820 if (!IS_ENABLED(CONFIG_MIGRATION)) 821 return; 822 823 /* 824 * swap_migration_tests() requires a dedicated page as it needs to 825 * be locked before creating a migration entry from it. Locking the 826 * page that actually maps kernel text ('start_kernel') can be real 827 * problematic. Lets use the allocated page explicitly for this 828 * purpose. 829 */ 830 page = (args->pte_pfn != ULONG_MAX) ? pfn_to_page(args->pte_pfn) : NULL; 831 if (!page) 832 return; 833 834 pr_debug("Validating swap migration\n"); 835 836 /* 837 * make_[readable|writable]_migration_entry() expects given page to 838 * be locked, otherwise it stumbles upon a BUG_ON(). 839 */ 840 __SetPageLocked(page); 841 swp = make_writable_migration_entry(page_to_pfn(page)); 842 WARN_ON(!is_migration_entry(swp)); 843 WARN_ON(!is_writable_migration_entry(swp)); 844 845 swp = make_readable_migration_entry(swp_offset(swp)); 846 WARN_ON(!is_migration_entry(swp)); 847 WARN_ON(is_writable_migration_entry(swp)); 848 849 swp = make_readable_migration_entry(page_to_pfn(page)); 850 WARN_ON(!is_migration_entry(swp)); 851 WARN_ON(is_writable_migration_entry(swp)); 852 __ClearPageLocked(page); 853 } 854 855 #ifdef CONFIG_HUGETLB_PAGE 856 static void __init hugetlb_basic_tests(struct pgtable_debug_args *args) 857 { 858 pte_t pte; 859 860 pr_debug("Validating HugeTLB basic\n"); 861 pte = pfn_pte(args->fixed_pmd_pfn, args->page_prot); 862 pte = arch_make_huge_pte(pte, PMD_SHIFT, VM_ACCESS_FLAGS); 863 864 #ifdef CONFIG_ARCH_WANT_GENERAL_HUGETLB 865 WARN_ON(!pte_huge(pte)); 866 #endif 867 WARN_ON(!huge_pte_dirty(huge_pte_mkdirty(pte))); 868 WARN_ON(!huge_pte_write(huge_pte_mkwrite(huge_pte_wrprotect(pte)))); 869 WARN_ON(huge_pte_write(huge_pte_wrprotect(huge_pte_mkwrite(pte)))); 870 } 871 #else /* !CONFIG_HUGETLB_PAGE */ 872 static void __init hugetlb_basic_tests(struct pgtable_debug_args *args) { } 873 #endif /* CONFIG_HUGETLB_PAGE */ 874 875 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 876 static void __init pmd_thp_tests(struct pgtable_debug_args *args) 877 { 878 pmd_t pmd; 879 880 if (!has_transparent_hugepage()) 881 return; 882 883 pr_debug("Validating PMD based THP\n"); 884 /* 885 * pmd_trans_huge() and pmd_present() must return positive after 886 * MMU invalidation with pmd_mkinvalid(). This behavior is an 887 * optimization for transparent huge page. pmd_trans_huge() must 888 * be true if pmd_page() returns a valid THP to avoid taking the 889 * pmd_lock when others walk over non transhuge pmds (i.e. there 890 * are no THP allocated). Especially when splitting a THP and 891 * removing the present bit from the pmd, pmd_trans_huge() still 892 * needs to return true. pmd_present() should be true whenever 893 * pmd_trans_huge() returns true. 894 */ 895 pmd = pfn_pmd(args->fixed_pmd_pfn, args->page_prot); 896 WARN_ON(!pmd_trans_huge(pmd_mkhuge(pmd))); 897 898 #ifndef __HAVE_ARCH_PMDP_INVALIDATE 899 WARN_ON(!pmd_trans_huge(pmd_mkinvalid(pmd_mkhuge(pmd)))); 900 WARN_ON(!pmd_present(pmd_mkinvalid(pmd_mkhuge(pmd)))); 901 WARN_ON(!pmd_leaf(pmd_mkinvalid(pmd_mkhuge(pmd)))); 902 #endif /* __HAVE_ARCH_PMDP_INVALIDATE */ 903 } 904 905 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD 906 static void __init pud_thp_tests(struct pgtable_debug_args *args) 907 { 908 pud_t pud; 909 910 if (!has_transparent_pud_hugepage()) 911 return; 912 913 pr_debug("Validating PUD based THP\n"); 914 pud = pfn_pud(args->fixed_pud_pfn, args->page_prot); 915 WARN_ON(!pud_trans_huge(pud_mkhuge(pud))); 916 917 /* 918 * pud_mkinvalid() has been dropped for now. Enable back 919 * these tests when it comes back with a modified pud_present(). 920 * 921 * WARN_ON(!pud_trans_huge(pud_mkinvalid(pud_mkhuge(pud)))); 922 * WARN_ON(!pud_present(pud_mkinvalid(pud_mkhuge(pud)))); 923 */ 924 } 925 #else /* !CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */ 926 static void __init pud_thp_tests(struct pgtable_debug_args *args) { } 927 #endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */ 928 #else /* !CONFIG_TRANSPARENT_HUGEPAGE */ 929 static void __init pmd_thp_tests(struct pgtable_debug_args *args) { } 930 static void __init pud_thp_tests(struct pgtable_debug_args *args) { } 931 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 932 933 static unsigned long __init get_random_vaddr(void) 934 { 935 unsigned long random_vaddr, random_pages, total_user_pages; 936 937 total_user_pages = (TASK_SIZE - FIRST_USER_ADDRESS) / PAGE_SIZE; 938 939 random_pages = get_random_long() % total_user_pages; 940 random_vaddr = FIRST_USER_ADDRESS + random_pages * PAGE_SIZE; 941 942 return random_vaddr; 943 } 944 945 static void __init destroy_args(struct pgtable_debug_args *args) 946 { 947 struct page *page = NULL; 948 949 /* Free (huge) page */ 950 if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) && 951 has_transparent_pud_hugepage() && 952 args->pud_pfn != ULONG_MAX) { 953 if (args->is_contiguous_page) { 954 free_contig_range(args->pud_pfn, 955 (1 << (HPAGE_PUD_SHIFT - PAGE_SHIFT))); 956 } else { 957 page = pfn_to_page(args->pud_pfn); 958 __free_pages(page, HPAGE_PUD_SHIFT - PAGE_SHIFT); 959 } 960 961 args->pud_pfn = ULONG_MAX; 962 args->pmd_pfn = ULONG_MAX; 963 args->pte_pfn = ULONG_MAX; 964 } 965 966 if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) && 967 has_transparent_hugepage() && 968 args->pmd_pfn != ULONG_MAX) { 969 if (args->is_contiguous_page) { 970 free_contig_range(args->pmd_pfn, (1 << HPAGE_PMD_ORDER)); 971 } else { 972 page = pfn_to_page(args->pmd_pfn); 973 __free_pages(page, HPAGE_PMD_ORDER); 974 } 975 976 args->pmd_pfn = ULONG_MAX; 977 args->pte_pfn = ULONG_MAX; 978 } 979 980 if (args->pte_pfn != ULONG_MAX) { 981 page = pfn_to_page(args->pte_pfn); 982 __free_page(page); 983 984 args->pte_pfn = ULONG_MAX; 985 } 986 987 /* Free page table entries */ 988 if (args->start_ptep) { 989 pte_free(args->mm, args->start_ptep); 990 mm_dec_nr_ptes(args->mm); 991 } 992 993 if (args->start_pmdp) { 994 pmd_free(args->mm, args->start_pmdp); 995 mm_dec_nr_pmds(args->mm); 996 } 997 998 if (args->start_pudp) { 999 pud_free(args->mm, args->start_pudp); 1000 mm_dec_nr_puds(args->mm); 1001 } 1002 1003 if (args->start_p4dp) 1004 p4d_free(args->mm, args->start_p4dp); 1005 1006 /* Free vma and mm struct */ 1007 if (args->vma) 1008 vm_area_free(args->vma); 1009 1010 if (args->mm) 1011 mmdrop(args->mm); 1012 } 1013 1014 static struct page * __init 1015 debug_vm_pgtable_alloc_huge_page(struct pgtable_debug_args *args, int order) 1016 { 1017 struct page *page = NULL; 1018 1019 #ifdef CONFIG_CONTIG_ALLOC 1020 if (order > MAX_PAGE_ORDER) { 1021 page = alloc_contig_pages((1 << order), GFP_KERNEL, 1022 first_online_node, NULL); 1023 if (page) { 1024 args->is_contiguous_page = true; 1025 return page; 1026 } 1027 } 1028 #endif 1029 1030 if (order <= MAX_PAGE_ORDER) 1031 page = alloc_pages(GFP_KERNEL, order); 1032 1033 return page; 1034 } 1035 1036 /* 1037 * Check if a physical memory range described by <pstart, pend> contains 1038 * an area that is of size psize, and aligned to psize. 1039 * 1040 * Don't use address 0, an all-zeroes physical address might mask bugs, and 1041 * it's not used on x86. 1042 */ 1043 static void __init phys_align_check(phys_addr_t pstart, 1044 phys_addr_t pend, unsigned long psize, 1045 phys_addr_t *physp, unsigned long *alignp) 1046 { 1047 phys_addr_t aligned_start, aligned_end; 1048 1049 if (pstart == 0) 1050 pstart = PAGE_SIZE; 1051 1052 aligned_start = ALIGN(pstart, psize); 1053 aligned_end = aligned_start + psize; 1054 1055 if (aligned_end > aligned_start && aligned_end <= pend) { 1056 *alignp = psize; 1057 *physp = aligned_start; 1058 } 1059 } 1060 1061 static void __init init_fixed_pfns(struct pgtable_debug_args *args) 1062 { 1063 u64 idx; 1064 phys_addr_t phys, pstart, pend; 1065 1066 /* 1067 * Initialize the fixed pfns. To do this, try to find a 1068 * valid physical range, preferably aligned to PUD_SIZE, 1069 * but settling for aligned to PMD_SIZE as a fallback. If 1070 * neither of those is found, use the physical address of 1071 * the start_kernel symbol. 1072 * 1073 * The memory doesn't need to be allocated, it just needs to exist 1074 * as usable memory. It won't be touched. 1075 * 1076 * The alignment is recorded, and can be checked to see if we 1077 * can run the tests that require an actual valid physical 1078 * address range on some architectures ({pmd,pud}_huge_test 1079 * on x86). 1080 */ 1081 1082 phys = __pa_symbol(&start_kernel); 1083 args->fixed_alignment = PAGE_SIZE; 1084 1085 for_each_mem_range(idx, &pstart, &pend) { 1086 /* First check for a PUD-aligned area */ 1087 phys_align_check(pstart, pend, PUD_SIZE, &phys, 1088 &args->fixed_alignment); 1089 1090 /* If a PUD-aligned area is found, we're done */ 1091 if (args->fixed_alignment == PUD_SIZE) 1092 break; 1093 1094 /* 1095 * If no PMD-aligned area found yet, check for one, 1096 * but continue the loop to look for a PUD-aligned area. 1097 */ 1098 if (args->fixed_alignment < PMD_SIZE) 1099 phys_align_check(pstart, pend, PMD_SIZE, &phys, 1100 &args->fixed_alignment); 1101 } 1102 1103 args->fixed_pgd_pfn = __phys_to_pfn(phys & PGDIR_MASK); 1104 args->fixed_p4d_pfn = __phys_to_pfn(phys & P4D_MASK); 1105 args->fixed_pud_pfn = __phys_to_pfn(phys & PUD_MASK); 1106 args->fixed_pmd_pfn = __phys_to_pfn(phys & PMD_MASK); 1107 args->fixed_pte_pfn = __phys_to_pfn(phys & PAGE_MASK); 1108 WARN_ON(!pfn_valid(args->fixed_pte_pfn)); 1109 } 1110 1111 1112 static int __init init_args(struct pgtable_debug_args *args) 1113 { 1114 struct page *page = NULL; 1115 int ret = 0; 1116 1117 /* 1118 * Initialize the debugging data. 1119 * 1120 * vm_get_page_prot(VM_NONE) or vm_get_page_prot(VM_SHARED|VM_NONE) 1121 * will help create page table entries with PROT_NONE permission as 1122 * required for pxx_protnone_tests(). 1123 */ 1124 memset(args, 0, sizeof(*args)); 1125 args->vaddr = get_random_vaddr(); 1126 args->page_prot = vm_get_page_prot(VM_ACCESS_FLAGS); 1127 args->page_prot_none = vm_get_page_prot(VM_NONE); 1128 args->is_contiguous_page = false; 1129 args->pud_pfn = ULONG_MAX; 1130 args->pmd_pfn = ULONG_MAX; 1131 args->pte_pfn = ULONG_MAX; 1132 args->fixed_pgd_pfn = ULONG_MAX; 1133 args->fixed_p4d_pfn = ULONG_MAX; 1134 args->fixed_pud_pfn = ULONG_MAX; 1135 args->fixed_pmd_pfn = ULONG_MAX; 1136 args->fixed_pte_pfn = ULONG_MAX; 1137 1138 /* Allocate mm and vma */ 1139 args->mm = mm_alloc(); 1140 if (!args->mm) { 1141 pr_err("Failed to allocate mm struct\n"); 1142 ret = -ENOMEM; 1143 goto error; 1144 } 1145 1146 args->vma = vm_area_alloc(args->mm); 1147 if (!args->vma) { 1148 pr_err("Failed to allocate vma\n"); 1149 ret = -ENOMEM; 1150 goto error; 1151 } 1152 1153 /* 1154 * Allocate page table entries. They will be modified in the tests. 1155 * Lets save the page table entries so that they can be released 1156 * when the tests are completed. 1157 */ 1158 args->pgdp = pgd_offset(args->mm, args->vaddr); 1159 args->p4dp = p4d_alloc(args->mm, args->pgdp, args->vaddr); 1160 if (!args->p4dp) { 1161 pr_err("Failed to allocate p4d entries\n"); 1162 ret = -ENOMEM; 1163 goto error; 1164 } 1165 args->start_p4dp = p4d_offset(args->pgdp, 0UL); 1166 WARN_ON(!args->start_p4dp); 1167 1168 args->pudp = pud_alloc(args->mm, args->p4dp, args->vaddr); 1169 if (!args->pudp) { 1170 pr_err("Failed to allocate pud entries\n"); 1171 ret = -ENOMEM; 1172 goto error; 1173 } 1174 args->start_pudp = pud_offset(args->p4dp, 0UL); 1175 WARN_ON(!args->start_pudp); 1176 1177 args->pmdp = pmd_alloc(args->mm, args->pudp, args->vaddr); 1178 if (!args->pmdp) { 1179 pr_err("Failed to allocate pmd entries\n"); 1180 ret = -ENOMEM; 1181 goto error; 1182 } 1183 args->start_pmdp = pmd_offset(args->pudp, 0UL); 1184 WARN_ON(!args->start_pmdp); 1185 1186 if (pte_alloc(args->mm, args->pmdp)) { 1187 pr_err("Failed to allocate pte entries\n"); 1188 ret = -ENOMEM; 1189 goto error; 1190 } 1191 args->start_ptep = pmd_pgtable(pmdp_get(args->pmdp)); 1192 WARN_ON(!args->start_ptep); 1193 1194 init_fixed_pfns(args); 1195 1196 /* 1197 * Allocate (huge) pages because some of the tests need to access 1198 * the data in the pages. The corresponding tests will be skipped 1199 * if we fail to allocate (huge) pages. 1200 */ 1201 if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) && 1202 has_transparent_pud_hugepage()) { 1203 page = debug_vm_pgtable_alloc_huge_page(args, 1204 HPAGE_PUD_SHIFT - PAGE_SHIFT); 1205 if (page) { 1206 args->pud_pfn = page_to_pfn(page); 1207 args->pmd_pfn = args->pud_pfn; 1208 args->pte_pfn = args->pud_pfn; 1209 return 0; 1210 } 1211 } 1212 1213 if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) && 1214 has_transparent_hugepage()) { 1215 page = debug_vm_pgtable_alloc_huge_page(args, HPAGE_PMD_ORDER); 1216 if (page) { 1217 args->pmd_pfn = page_to_pfn(page); 1218 args->pte_pfn = args->pmd_pfn; 1219 return 0; 1220 } 1221 } 1222 1223 page = alloc_page(GFP_KERNEL); 1224 if (page) 1225 args->pte_pfn = page_to_pfn(page); 1226 1227 return 0; 1228 1229 error: 1230 destroy_args(args); 1231 return ret; 1232 } 1233 1234 static int __init debug_vm_pgtable(void) 1235 { 1236 struct pgtable_debug_args args; 1237 spinlock_t *ptl = NULL; 1238 int idx, ret; 1239 1240 pr_info("Validating architecture page table helpers\n"); 1241 ret = init_args(&args); 1242 if (ret) 1243 return ret; 1244 1245 /* 1246 * Iterate over each possible vm_flags to make sure that all 1247 * the basic page table transformation validations just hold 1248 * true irrespective of the starting protection value for a 1249 * given page table entry. 1250 * 1251 * Protection based vm_flags combinations are always linear 1252 * and increasing i.e starting from VM_NONE and going up to 1253 * (VM_SHARED | READ | WRITE | EXEC). 1254 */ 1255 #define VM_FLAGS_START (VM_NONE) 1256 #define VM_FLAGS_END (VM_SHARED | VM_EXEC | VM_WRITE | VM_READ) 1257 1258 for (idx = VM_FLAGS_START; idx <= VM_FLAGS_END; idx++) { 1259 pte_basic_tests(&args, idx); 1260 pmd_basic_tests(&args, idx); 1261 pud_basic_tests(&args, idx); 1262 } 1263 1264 /* 1265 * Both P4D and PGD level tests are very basic which do not 1266 * involve creating page table entries from the protection 1267 * value and the given pfn. Hence just keep them out from 1268 * the above iteration for now to save some test execution 1269 * time. 1270 */ 1271 p4d_basic_tests(&args); 1272 pgd_basic_tests(&args); 1273 1274 pmd_leaf_tests(&args); 1275 pud_leaf_tests(&args); 1276 1277 pte_special_tests(&args); 1278 pte_protnone_tests(&args); 1279 pmd_protnone_tests(&args); 1280 1281 pte_soft_dirty_tests(&args); 1282 pmd_soft_dirty_tests(&args); 1283 pte_swap_soft_dirty_tests(&args); 1284 pmd_swap_soft_dirty_tests(&args); 1285 1286 pte_swap_exclusive_tests(&args); 1287 1288 pte_swap_tests(&args); 1289 pmd_swap_tests(&args); 1290 1291 swap_migration_tests(&args); 1292 1293 pmd_thp_tests(&args); 1294 pud_thp_tests(&args); 1295 1296 hugetlb_basic_tests(&args); 1297 1298 /* 1299 * Page table modifying tests. They need to hold 1300 * proper page table lock. 1301 */ 1302 1303 args.ptep = pte_offset_map_lock(args.mm, args.pmdp, args.vaddr, &ptl); 1304 pte_clear_tests(&args); 1305 pte_advanced_tests(&args); 1306 if (args.ptep) 1307 pte_unmap_unlock(args.ptep, ptl); 1308 1309 ptl = pmd_lock(args.mm, args.pmdp); 1310 pmd_clear_tests(&args); 1311 pmd_advanced_tests(&args); 1312 pmd_huge_tests(&args); 1313 pmd_populate_tests(&args); 1314 spin_unlock(ptl); 1315 1316 ptl = pud_lock(args.mm, args.pudp); 1317 pud_clear_tests(&args); 1318 pud_advanced_tests(&args); 1319 pud_huge_tests(&args); 1320 pud_populate_tests(&args); 1321 spin_unlock(ptl); 1322 1323 spin_lock(&(args.mm->page_table_lock)); 1324 p4d_clear_tests(&args); 1325 pgd_clear_tests(&args); 1326 p4d_populate_tests(&args); 1327 pgd_populate_tests(&args); 1328 spin_unlock(&(args.mm->page_table_lock)); 1329 1330 destroy_args(&args); 1331 return 0; 1332 } 1333 late_initcall(debug_vm_pgtable); 1334