1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2018 Matthew Macy 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 #include "opt_platform.h" 29 30 #include <sys/param.h> 31 #include <sys/kernel.h> 32 #include <sys/systm.h> 33 #include <sys/conf.h> 34 #include <sys/bitstring.h> 35 #include <sys/queue.h> 36 #include <sys/cpuset.h> 37 #include <sys/endian.h> 38 #include <sys/kerneldump.h> 39 #include <sys/ktr.h> 40 #include <sys/lock.h> 41 #include <sys/syslog.h> 42 #include <sys/msgbuf.h> 43 #include <sys/malloc.h> 44 #include <sys/mman.h> 45 #include <sys/mutex.h> 46 #include <sys/proc.h> 47 #include <sys/rwlock.h> 48 #include <sys/sched.h> 49 #include <sys/sysctl.h> 50 #include <sys/systm.h> 51 #include <sys/vmem.h> 52 #include <sys/vmmeter.h> 53 #include <sys/smp.h> 54 55 #include <sys/kdb.h> 56 57 #include <dev/ofw/openfirm.h> 58 59 #include <vm/vm.h> 60 #include <vm/pmap.h> 61 #include <vm/vm_param.h> 62 #include <vm/vm_kern.h> 63 #include <vm/vm_page.h> 64 #include <vm/vm_map.h> 65 #include <vm/vm_object.h> 66 #include <vm/vm_extern.h> 67 #include <vm/vm_pageout.h> 68 #include <vm/vm_phys.h> 69 #include <vm/vm_radix.h> 70 #include <vm/vm_reserv.h> 71 #include <vm/vm_dumpset.h> 72 #include <vm/uma.h> 73 74 #include <machine/_inttypes.h> 75 #include <machine/cpu.h> 76 #include <machine/platform.h> 77 #include <machine/frame.h> 78 #include <machine/md_var.h> 79 #include <machine/psl.h> 80 #include <machine/bat.h> 81 #include <machine/hid.h> 82 #include <machine/pte.h> 83 #include <machine/sr.h> 84 #include <machine/trap.h> 85 #include <machine/mmuvar.h> 86 87 /* For pseries bit. */ 88 #include <powerpc/pseries/phyp-hvcall.h> 89 90 #ifdef INVARIANTS 91 #include <vm/uma_dbg.h> 92 #endif 93 94 #define PPC_BITLSHIFT(bit) (sizeof(long)*NBBY - 1 - (bit)) 95 #define PPC_BIT(bit) (1UL << PPC_BITLSHIFT(bit)) 96 #define PPC_BITLSHIFT_VAL(val, bit) ((val) << PPC_BITLSHIFT(bit)) 97 98 #include "opt_ddb.h" 99 100 #ifdef DDB 101 static void pmap_pte_walk(pml1_entry_t *l1, vm_offset_t va); 102 #endif 103 104 #define PG_W RPTE_WIRED 105 #define PG_V RPTE_VALID 106 #define PG_MANAGED RPTE_MANAGED 107 #define PG_PROMOTED RPTE_PROMOTED 108 #define PG_M RPTE_C 109 #define PG_A RPTE_R 110 #define PG_X RPTE_EAA_X 111 #define PG_RW RPTE_EAA_W 112 #define PG_PTE_CACHE RPTE_ATTR_MASK 113 114 #define RPTE_SHIFT 9 115 #define NLS_MASK ((1UL<<5)-1) 116 #define RPTE_ENTRIES (1UL<<RPTE_SHIFT) 117 #define RPTE_MASK (RPTE_ENTRIES-1) 118 119 #define NLB_SHIFT 0 120 #define NLB_MASK (((1UL<<52)-1) << 8) 121 122 extern int nkpt; 123 extern caddr_t crashdumpmap; 124 125 #define RIC_FLUSH_TLB 0 126 #define RIC_FLUSH_PWC 1 127 #define RIC_FLUSH_ALL 2 128 129 #define POWER9_TLB_SETS_RADIX 128 /* # sets in POWER9 TLB Radix mode */ 130 131 #define PPC_INST_TLBIE 0x7c000264 132 #define PPC_INST_TLBIEL 0x7c000224 133 #define PPC_INST_SLBIA 0x7c0003e4 134 135 #define ___PPC_RA(a) (((a) & 0x1f) << 16) 136 #define ___PPC_RB(b) (((b) & 0x1f) << 11) 137 #define ___PPC_RS(s) (((s) & 0x1f) << 21) 138 #define ___PPC_RT(t) ___PPC_RS(t) 139 #define ___PPC_R(r) (((r) & 0x1) << 16) 140 #define ___PPC_PRS(prs) (((prs) & 0x1) << 17) 141 #define ___PPC_RIC(ric) (((ric) & 0x3) << 18) 142 143 #define PPC_SLBIA(IH) __XSTRING(.long PPC_INST_SLBIA | \ 144 ((IH & 0x7) << 21)) 145 #define PPC_TLBIE_5(rb,rs,ric,prs,r) \ 146 __XSTRING(.long PPC_INST_TLBIE | \ 147 ___PPC_RB(rb) | ___PPC_RS(rs) | \ 148 ___PPC_RIC(ric) | ___PPC_PRS(prs) | \ 149 ___PPC_R(r)) 150 151 #define PPC_TLBIEL(rb,rs,ric,prs,r) \ 152 __XSTRING(.long PPC_INST_TLBIEL | \ 153 ___PPC_RB(rb) | ___PPC_RS(rs) | \ 154 ___PPC_RIC(ric) | ___PPC_PRS(prs) | \ 155 ___PPC_R(r)) 156 157 #define PPC_INVALIDATE_ERAT PPC_SLBIA(7) 158 159 static __inline void 160 ttusync(void) 161 { 162 __asm __volatile("eieio; tlbsync; ptesync" ::: "memory"); 163 } 164 165 #define TLBIEL_INVAL_SEL_MASK 0xc00 /* invalidation selector */ 166 #define TLBIEL_INVAL_PAGE 0x000 /* invalidate a single page */ 167 #define TLBIEL_INVAL_SET_PID 0x400 /* invalidate a set for the current PID */ 168 #define TLBIEL_INVAL_SET_LPID 0x800 /* invalidate a set for current LPID */ 169 #define TLBIEL_INVAL_SET 0xc00 /* invalidate a set for all LPIDs */ 170 171 #define TLBIE_ACTUAL_PAGE_MASK 0xe0 172 #define TLBIE_ACTUAL_PAGE_4K 0x00 173 #define TLBIE_ACTUAL_PAGE_64K 0xa0 174 #define TLBIE_ACTUAL_PAGE_2M 0x20 175 #define TLBIE_ACTUAL_PAGE_1G 0x40 176 177 #define TLBIE_PRS_PARTITION_SCOPE 0x0 178 #define TLBIE_PRS_PROCESS_SCOPE 0x1 179 180 #define TLBIE_RIC_INVALIDATE_TLB 0x0 /* Invalidate just TLB */ 181 #define TLBIE_RIC_INVALIDATE_PWC 0x1 /* Invalidate just PWC */ 182 #define TLBIE_RIC_INVALIDATE_ALL 0x2 /* Invalidate TLB, PWC, 183 * cached {proc, part}tab entries 184 */ 185 #define TLBIE_RIC_INVALIDATE_SEQ 0x3 /* HPT - only: 186 * Invalidate a range of translations 187 */ 188 189 static __always_inline void 190 radix_tlbie(uint8_t ric, uint8_t prs, uint16_t is, uint32_t pid, uint32_t lpid, 191 vm_offset_t va, uint16_t ap) 192 { 193 uint64_t rb, rs; 194 195 MPASS((va & PAGE_MASK) == 0); 196 197 rs = ((uint64_t)pid << 32) | lpid; 198 rb = va | is | ap; 199 __asm __volatile(PPC_TLBIE_5(%0, %1, %2, %3, 1) : : 200 "r" (rb), "r" (rs), "i" (ric), "i" (prs) : "memory"); 201 } 202 203 static __inline void 204 radix_tlbie_fixup(uint32_t pid, vm_offset_t va, int ap) 205 { 206 207 __asm __volatile("ptesync" ::: "memory"); 208 radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE, 209 TLBIEL_INVAL_PAGE, 0, 0, va, ap); 210 __asm __volatile("ptesync" ::: "memory"); 211 radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE, 212 TLBIEL_INVAL_PAGE, pid, 0, va, ap); 213 } 214 215 static __inline void 216 radix_tlbie_invlpg_user_4k(uint32_t pid, vm_offset_t va) 217 { 218 219 radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE, 220 TLBIEL_INVAL_PAGE, pid, 0, va, TLBIE_ACTUAL_PAGE_4K); 221 radix_tlbie_fixup(pid, va, TLBIE_ACTUAL_PAGE_4K); 222 } 223 224 static __inline void 225 radix_tlbie_invlpg_user_2m(uint32_t pid, vm_offset_t va) 226 { 227 228 radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE, 229 TLBIEL_INVAL_PAGE, pid, 0, va, TLBIE_ACTUAL_PAGE_2M); 230 radix_tlbie_fixup(pid, va, TLBIE_ACTUAL_PAGE_2M); 231 } 232 233 static __inline void 234 radix_tlbie_invlpwc_user(uint32_t pid) 235 { 236 237 radix_tlbie(TLBIE_RIC_INVALIDATE_PWC, TLBIE_PRS_PROCESS_SCOPE, 238 TLBIEL_INVAL_SET_PID, pid, 0, 0, 0); 239 } 240 241 static __inline void 242 radix_tlbie_flush_user(uint32_t pid) 243 { 244 245 radix_tlbie(TLBIE_RIC_INVALIDATE_ALL, TLBIE_PRS_PROCESS_SCOPE, 246 TLBIEL_INVAL_SET_PID, pid, 0, 0, 0); 247 } 248 249 static __inline void 250 radix_tlbie_invlpg_kernel_4k(vm_offset_t va) 251 { 252 253 radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE, 254 TLBIEL_INVAL_PAGE, 0, 0, va, TLBIE_ACTUAL_PAGE_4K); 255 radix_tlbie_fixup(0, va, TLBIE_ACTUAL_PAGE_4K); 256 } 257 258 static __inline void 259 radix_tlbie_invlpg_kernel_2m(vm_offset_t va) 260 { 261 262 radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE, 263 TLBIEL_INVAL_PAGE, 0, 0, va, TLBIE_ACTUAL_PAGE_2M); 264 radix_tlbie_fixup(0, va, TLBIE_ACTUAL_PAGE_2M); 265 } 266 267 /* 1GB pages aren't currently supported. */ 268 static __inline __unused void 269 radix_tlbie_invlpg_kernel_1g(vm_offset_t va) 270 { 271 272 radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE, 273 TLBIEL_INVAL_PAGE, 0, 0, va, TLBIE_ACTUAL_PAGE_1G); 274 radix_tlbie_fixup(0, va, TLBIE_ACTUAL_PAGE_1G); 275 } 276 277 static __inline void 278 radix_tlbie_invlpwc_kernel(void) 279 { 280 281 radix_tlbie(TLBIE_RIC_INVALIDATE_PWC, TLBIE_PRS_PROCESS_SCOPE, 282 TLBIEL_INVAL_SET_LPID, 0, 0, 0, 0); 283 } 284 285 static __inline void 286 radix_tlbie_flush_kernel(void) 287 { 288 289 radix_tlbie(TLBIE_RIC_INVALIDATE_ALL, TLBIE_PRS_PROCESS_SCOPE, 290 TLBIEL_INVAL_SET_LPID, 0, 0, 0, 0); 291 } 292 293 static __inline vm_pindex_t 294 pmap_l3e_pindex(vm_offset_t va) 295 { 296 return ((va & PG_FRAME) >> L3_PAGE_SIZE_SHIFT); 297 } 298 299 static __inline vm_pindex_t 300 pmap_pml3e_index(vm_offset_t va) 301 { 302 303 return ((va >> L3_PAGE_SIZE_SHIFT) & RPTE_MASK); 304 } 305 306 static __inline vm_pindex_t 307 pmap_pml2e_index(vm_offset_t va) 308 { 309 return ((va >> L2_PAGE_SIZE_SHIFT) & RPTE_MASK); 310 } 311 312 static __inline vm_pindex_t 313 pmap_pml1e_index(vm_offset_t va) 314 { 315 return ((va & PG_FRAME) >> L1_PAGE_SIZE_SHIFT); 316 } 317 318 /* Return various clipped indexes for a given VA */ 319 static __inline vm_pindex_t 320 pmap_pte_index(vm_offset_t va) 321 { 322 323 return ((va >> PAGE_SHIFT) & RPTE_MASK); 324 } 325 326 /* Return a pointer to the PT slot that corresponds to a VA */ 327 static __inline pt_entry_t * 328 pmap_l3e_to_pte(pt_entry_t *l3e, vm_offset_t va) 329 { 330 pt_entry_t *pte; 331 vm_paddr_t ptepa; 332 333 ptepa = (be64toh(*l3e) & NLB_MASK); 334 pte = (pt_entry_t *)PHYS_TO_DMAP(ptepa); 335 return (&pte[pmap_pte_index(va)]); 336 } 337 338 /* Return a pointer to the PD slot that corresponds to a VA */ 339 static __inline pt_entry_t * 340 pmap_l2e_to_l3e(pt_entry_t *l2e, vm_offset_t va) 341 { 342 pt_entry_t *l3e; 343 vm_paddr_t l3pa; 344 345 l3pa = (be64toh(*l2e) & NLB_MASK); 346 l3e = (pml3_entry_t *)PHYS_TO_DMAP(l3pa); 347 return (&l3e[pmap_pml3e_index(va)]); 348 } 349 350 /* Return a pointer to the PD slot that corresponds to a VA */ 351 static __inline pt_entry_t * 352 pmap_l1e_to_l2e(pt_entry_t *l1e, vm_offset_t va) 353 { 354 pt_entry_t *l2e; 355 vm_paddr_t l2pa; 356 357 l2pa = (be64toh(*l1e) & NLB_MASK); 358 359 l2e = (pml2_entry_t *)PHYS_TO_DMAP(l2pa); 360 return (&l2e[pmap_pml2e_index(va)]); 361 } 362 363 static __inline pml1_entry_t * 364 pmap_pml1e(pmap_t pmap, vm_offset_t va) 365 { 366 367 return (&pmap->pm_pml1[pmap_pml1e_index(va)]); 368 } 369 370 static pt_entry_t * 371 pmap_pml2e(pmap_t pmap, vm_offset_t va) 372 { 373 pt_entry_t *l1e; 374 375 l1e = pmap_pml1e(pmap, va); 376 if (l1e == NULL || (be64toh(*l1e) & RPTE_VALID) == 0) 377 return (NULL); 378 return (pmap_l1e_to_l2e(l1e, va)); 379 } 380 381 static __inline pt_entry_t * 382 pmap_pml3e(pmap_t pmap, vm_offset_t va) 383 { 384 pt_entry_t *l2e; 385 386 l2e = pmap_pml2e(pmap, va); 387 if (l2e == NULL || (be64toh(*l2e) & RPTE_VALID) == 0) 388 return (NULL); 389 return (pmap_l2e_to_l3e(l2e, va)); 390 } 391 392 static __inline pt_entry_t * 393 pmap_pte(pmap_t pmap, vm_offset_t va) 394 { 395 pt_entry_t *l3e; 396 397 l3e = pmap_pml3e(pmap, va); 398 if (l3e == NULL || (be64toh(*l3e) & RPTE_VALID) == 0) 399 return (NULL); 400 return (pmap_l3e_to_pte(l3e, va)); 401 } 402 403 int nkpt = 64; 404 SYSCTL_INT(_machdep, OID_AUTO, nkpt, CTLFLAG_RD, &nkpt, 0, 405 "Number of kernel page table pages allocated on bootup"); 406 407 vm_paddr_t dmaplimit; 408 409 SYSCTL_DECL(_vm_pmap); 410 411 #ifdef INVARIANTS 412 #define VERBOSE_PMAP 0 413 #define VERBOSE_PROTECT 0 414 static int pmap_logging; 415 SYSCTL_INT(_vm_pmap, OID_AUTO, pmap_logging, CTLFLAG_RWTUN, 416 &pmap_logging, 0, "verbose debug logging"); 417 #endif 418 419 static u_int64_t KPTphys; /* phys addr of kernel level 1 */ 420 421 //static vm_paddr_t KERNend; /* phys addr of end of bootstrap data */ 422 423 static vm_offset_t qframe = 0; 424 static struct mtx qframe_mtx; 425 426 void mmu_radix_activate(struct thread *); 427 void mmu_radix_advise(pmap_t, vm_offset_t, vm_offset_t, int); 428 void mmu_radix_align_superpage(vm_object_t, vm_ooffset_t, vm_offset_t *, 429 vm_size_t); 430 void mmu_radix_clear_modify(vm_page_t); 431 void mmu_radix_copy(pmap_t, pmap_t, vm_offset_t, vm_size_t, vm_offset_t); 432 int mmu_radix_decode_kernel_ptr(vm_offset_t, int *, vm_offset_t *); 433 int mmu_radix_enter(pmap_t, vm_offset_t, vm_page_t, vm_prot_t, u_int, int8_t); 434 void mmu_radix_enter_object(pmap_t, vm_offset_t, vm_offset_t, vm_page_t, 435 vm_prot_t); 436 void mmu_radix_enter_quick(pmap_t, vm_offset_t, vm_page_t, vm_prot_t); 437 vm_paddr_t mmu_radix_extract(pmap_t pmap, vm_offset_t va); 438 vm_page_t mmu_radix_extract_and_hold(pmap_t, vm_offset_t, vm_prot_t); 439 void mmu_radix_kenter(vm_offset_t, vm_paddr_t); 440 vm_paddr_t mmu_radix_kextract(vm_offset_t); 441 void mmu_radix_kremove(vm_offset_t); 442 bool mmu_radix_is_modified(vm_page_t); 443 bool mmu_radix_is_prefaultable(pmap_t, vm_offset_t); 444 bool mmu_radix_is_referenced(vm_page_t); 445 void mmu_radix_object_init_pt(pmap_t, vm_offset_t, vm_object_t, 446 vm_pindex_t, vm_size_t); 447 bool mmu_radix_page_exists_quick(pmap_t, vm_page_t); 448 void mmu_radix_page_init(vm_page_t); 449 bool mmu_radix_page_is_mapped(vm_page_t m); 450 void mmu_radix_page_set_memattr(vm_page_t, vm_memattr_t); 451 int mmu_radix_page_wired_mappings(vm_page_t); 452 int mmu_radix_pinit(pmap_t); 453 void mmu_radix_protect(pmap_t, vm_offset_t, vm_offset_t, vm_prot_t); 454 bool mmu_radix_ps_enabled(pmap_t); 455 void mmu_radix_qenter(vm_offset_t, vm_page_t *, int); 456 void mmu_radix_qremove(vm_offset_t, int); 457 vm_offset_t mmu_radix_quick_enter_page(vm_page_t); 458 void mmu_radix_quick_remove_page(vm_offset_t); 459 int mmu_radix_ts_referenced(vm_page_t); 460 void mmu_radix_release(pmap_t); 461 void mmu_radix_remove(pmap_t, vm_offset_t, vm_offset_t); 462 void mmu_radix_remove_all(vm_page_t); 463 void mmu_radix_remove_pages(pmap_t); 464 void mmu_radix_remove_write(vm_page_t); 465 void mmu_radix_sync_icache(pmap_t pm, vm_offset_t va, vm_size_t sz); 466 void mmu_radix_unwire(pmap_t, vm_offset_t, vm_offset_t); 467 void mmu_radix_zero_page(vm_page_t); 468 void mmu_radix_zero_page_area(vm_page_t, int, int); 469 int mmu_radix_change_attr(vm_offset_t, vm_size_t, vm_memattr_t); 470 void mmu_radix_page_array_startup(long pages); 471 472 #include "mmu_oea64.h" 473 474 /* 475 * Kernel MMU interface 476 */ 477 478 static void mmu_radix_bootstrap(vm_offset_t, vm_offset_t); 479 480 static void mmu_radix_copy_page(vm_page_t, vm_page_t); 481 static void mmu_radix_copy_pages(vm_page_t *ma, vm_offset_t a_offset, 482 vm_page_t *mb, vm_offset_t b_offset, int xfersize); 483 static int mmu_radix_growkernel(vm_offset_t); 484 static void mmu_radix_init(void); 485 static int mmu_radix_mincore(pmap_t, vm_offset_t, vm_paddr_t *); 486 static vm_offset_t mmu_radix_map(vm_offset_t *, vm_paddr_t, vm_paddr_t, int); 487 static void mmu_radix_pinit0(pmap_t); 488 489 static void *mmu_radix_mapdev(vm_paddr_t, vm_size_t); 490 static void *mmu_radix_mapdev_attr(vm_paddr_t, vm_size_t, vm_memattr_t); 491 static void mmu_radix_unmapdev(void *, vm_size_t); 492 static void mmu_radix_kenter_attr(vm_offset_t, vm_paddr_t, vm_memattr_t ma); 493 static int mmu_radix_dev_direct_mapped(vm_paddr_t, vm_size_t); 494 static void mmu_radix_dumpsys_map(vm_paddr_t pa, size_t sz, void **va); 495 static void mmu_radix_scan_init(void); 496 static void mmu_radix_cpu_bootstrap(int ap); 497 static void mmu_radix_tlbie_all(void); 498 499 static struct pmap_funcs mmu_radix_methods = { 500 .bootstrap = mmu_radix_bootstrap, 501 .copy_page = mmu_radix_copy_page, 502 .copy_pages = mmu_radix_copy_pages, 503 .cpu_bootstrap = mmu_radix_cpu_bootstrap, 504 .growkernel_nopanic = mmu_radix_growkernel, 505 .init = mmu_radix_init, 506 .map = mmu_radix_map, 507 .mincore = mmu_radix_mincore, 508 .pinit = mmu_radix_pinit, 509 .pinit0 = mmu_radix_pinit0, 510 511 .mapdev = mmu_radix_mapdev, 512 .mapdev_attr = mmu_radix_mapdev_attr, 513 .unmapdev = mmu_radix_unmapdev, 514 .kenter_attr = mmu_radix_kenter_attr, 515 .dev_direct_mapped = mmu_radix_dev_direct_mapped, 516 .dumpsys_pa_init = mmu_radix_scan_init, 517 .dumpsys_map_chunk = mmu_radix_dumpsys_map, 518 .page_is_mapped = mmu_radix_page_is_mapped, 519 .ps_enabled = mmu_radix_ps_enabled, 520 .align_superpage = mmu_radix_align_superpage, 521 .object_init_pt = mmu_radix_object_init_pt, 522 .protect = mmu_radix_protect, 523 /* pmap dispatcher interface */ 524 .clear_modify = mmu_radix_clear_modify, 525 .copy = mmu_radix_copy, 526 .enter = mmu_radix_enter, 527 .enter_object = mmu_radix_enter_object, 528 .enter_quick = mmu_radix_enter_quick, 529 .extract = mmu_radix_extract, 530 .extract_and_hold = mmu_radix_extract_and_hold, 531 .is_modified = mmu_radix_is_modified, 532 .is_prefaultable = mmu_radix_is_prefaultable, 533 .is_referenced = mmu_radix_is_referenced, 534 .ts_referenced = mmu_radix_ts_referenced, 535 .page_exists_quick = mmu_radix_page_exists_quick, 536 .page_init = mmu_radix_page_init, 537 .page_wired_mappings = mmu_radix_page_wired_mappings, 538 .qenter = mmu_radix_qenter, 539 .qremove = mmu_radix_qremove, 540 .release = mmu_radix_release, 541 .remove = mmu_radix_remove, 542 .remove_all = mmu_radix_remove_all, 543 .remove_write = mmu_radix_remove_write, 544 .sync_icache = mmu_radix_sync_icache, 545 .unwire = mmu_radix_unwire, 546 .zero_page = mmu_radix_zero_page, 547 .zero_page_area = mmu_radix_zero_page_area, 548 .activate = mmu_radix_activate, 549 .quick_enter_page = mmu_radix_quick_enter_page, 550 .quick_remove_page = mmu_radix_quick_remove_page, 551 .page_set_memattr = mmu_radix_page_set_memattr, 552 .page_array_startup = mmu_radix_page_array_startup, 553 554 /* Internal interfaces */ 555 .kenter = mmu_radix_kenter, 556 .kextract = mmu_radix_kextract, 557 .kremove = mmu_radix_kremove, 558 .change_attr = mmu_radix_change_attr, 559 .decode_kernel_ptr = mmu_radix_decode_kernel_ptr, 560 561 .tlbie_all = mmu_radix_tlbie_all, 562 }; 563 564 MMU_DEF(mmu_radix, MMU_TYPE_RADIX, mmu_radix_methods); 565 566 static bool pmap_demote_l3e_locked(pmap_t pmap, pml3_entry_t *l3e, vm_offset_t va, 567 struct rwlock **lockp); 568 static bool pmap_demote_l3e(pmap_t pmap, pml3_entry_t *pde, vm_offset_t va); 569 static int pmap_unuse_pt(pmap_t, vm_offset_t, pml3_entry_t, struct spglist *); 570 static int pmap_remove_l3e(pmap_t pmap, pml3_entry_t *pdq, vm_offset_t sva, 571 struct spglist *free, struct rwlock **lockp); 572 static int pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t sva, 573 pml3_entry_t ptepde, struct spglist *free, struct rwlock **lockp); 574 static vm_page_t pmap_remove_pt_page(pmap_t pmap, vm_offset_t va); 575 static bool pmap_remove_page(pmap_t pmap, vm_offset_t va, pml3_entry_t *pde, 576 struct spglist *free); 577 static bool pmap_remove_ptes(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, 578 pml3_entry_t *l3e, struct spglist *free, struct rwlock **lockp); 579 580 static bool pmap_pv_insert_l3e(pmap_t pmap, vm_offset_t va, pml3_entry_t l3e, 581 u_int flags, struct rwlock **lockp); 582 #if VM_NRESERVLEVEL > 0 583 static void pmap_pv_promote_l3e(pmap_t pmap, vm_offset_t va, vm_paddr_t pa, 584 struct rwlock **lockp); 585 #endif 586 static void pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va); 587 static int pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte); 588 static vm_page_t mmu_radix_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, 589 vm_prot_t prot, vm_page_t mpte, struct rwlock **lockp, bool *invalidate); 590 591 static bool pmap_enter_2mpage(pmap_t pmap, vm_offset_t va, vm_page_t m, 592 vm_prot_t prot, struct rwlock **lockp); 593 static int pmap_enter_l3e(pmap_t pmap, vm_offset_t va, pml3_entry_t newpde, 594 u_int flags, vm_page_t m, struct rwlock **lockp); 595 596 static vm_page_t reclaim_pv_chunk(pmap_t locked_pmap, struct rwlock **lockp); 597 static void free_pv_chunk(struct pv_chunk *pc); 598 static vm_page_t _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, struct rwlock **lockp); 599 static vm_page_t pmap_allocl3e(pmap_t pmap, vm_offset_t va, 600 struct rwlock **lockp); 601 static vm_page_t pmap_allocpte(pmap_t pmap, vm_offset_t va, 602 struct rwlock **lockp); 603 static void _pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, 604 struct spglist *free); 605 static bool pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free); 606 607 static void pmap_invalidate_page(pmap_t pmap, vm_offset_t start); 608 static void pmap_invalidate_all(pmap_t pmap); 609 static int pmap_change_attr_locked(vm_offset_t va, vm_size_t size, int mode, bool flush); 610 static void pmap_fill_ptp(pt_entry_t *firstpte, pt_entry_t newpte); 611 612 /* 613 * Internal flags for pmap_enter()'s helper functions. 614 */ 615 #define PMAP_ENTER_NORECLAIM 0x1000000 /* Don't reclaim PV entries. */ 616 #define PMAP_ENTER_NOREPLACE 0x2000000 /* Don't replace mappings. */ 617 618 #define UNIMPLEMENTED() panic("%s not implemented", __func__) 619 #define UNTESTED() panic("%s not yet tested", __func__) 620 621 /* Number of supported PID bits */ 622 static unsigned int isa3_pid_bits; 623 624 /* PID to start allocating from */ 625 static unsigned int isa3_base_pid; 626 627 #define PROCTAB_SIZE_SHIFT (isa3_pid_bits + 4) 628 #define PROCTAB_ENTRIES (1ul << isa3_pid_bits) 629 630 /* 631 * Map of physical memory regions. 632 */ 633 static struct mem_region *regions, *pregions; 634 static struct numa_mem_region *numa_pregions; 635 static u_int phys_avail_count; 636 static int regions_sz, pregions_sz, numa_pregions_sz; 637 static struct pate *isa3_parttab; 638 static struct prte *isa3_proctab; 639 static vmem_t *asid_arena; 640 641 extern void bs_remap_earlyboot(void); 642 643 #define RADIX_PGD_SIZE_SHIFT 16 644 #define RADIX_PGD_SIZE (1UL << RADIX_PGD_SIZE_SHIFT) 645 646 #define RADIX_PGD_INDEX_SHIFT (RADIX_PGD_SIZE_SHIFT-3) 647 #define NL2EPG (PAGE_SIZE/sizeof(pml2_entry_t)) 648 #define NL3EPG (PAGE_SIZE/sizeof(pml3_entry_t)) 649 650 #define NUPML1E (RADIX_PGD_SIZE/sizeof(uint64_t)) /* number of userland PML1 pages */ 651 #define NUPDPE (NUPML1E * NL2EPG)/* number of userland PDP pages */ 652 #define NUPDE (NUPDPE * NL3EPG) /* number of userland PD entries */ 653 654 /* POWER9 only permits a 64k partition table size. */ 655 #define PARTTAB_SIZE_SHIFT 16 656 #define PARTTAB_SIZE (1UL << PARTTAB_SIZE_SHIFT) 657 658 #define PARTTAB_HR (1UL << 63) /* host uses radix */ 659 #define PARTTAB_GR (1UL << 63) /* guest uses radix must match host */ 660 661 /* TLB flush actions. Used as argument to tlbiel_flush() */ 662 enum { 663 TLB_INVAL_SCOPE_LPID = 2, /* invalidate TLBs for current LPID */ 664 TLB_INVAL_SCOPE_GLOBAL = 3, /* invalidate all TLBs */ 665 }; 666 667 #define NPV_LIST_LOCKS MAXCPU 668 static int pmap_initialized; 669 static vm_paddr_t proctab0pa; 670 static vm_paddr_t parttab_phys; 671 CTASSERT(sizeof(struct pv_chunk) == PAGE_SIZE); 672 673 /* 674 * Data for the pv entry allocation mechanism. 675 * Updates to pv_invl_gen are protected by the pv_list_locks[] 676 * elements, but reads are not. 677 */ 678 static TAILQ_HEAD(pch, pv_chunk) pv_chunks = TAILQ_HEAD_INITIALIZER(pv_chunks); 679 static struct mtx __exclusive_cache_line pv_chunks_mutex; 680 static struct rwlock __exclusive_cache_line pv_list_locks[NPV_LIST_LOCKS]; 681 static struct md_page *pv_table; 682 static struct md_page pv_dummy; 683 684 #ifdef PV_STATS 685 #define PV_STAT(x) do { x ; } while (0) 686 #else 687 #define PV_STAT(x) do { } while (0) 688 #endif 689 690 #define pa_radix_index(pa) ((pa) >> L3_PAGE_SIZE_SHIFT) 691 #define pa_to_pvh(pa) (&pv_table[pa_radix_index(pa)]) 692 693 #define PHYS_TO_PV_LIST_LOCK(pa) \ 694 (&pv_list_locks[pa_radix_index(pa) % NPV_LIST_LOCKS]) 695 696 #define CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa) do { \ 697 struct rwlock **_lockp = (lockp); \ 698 struct rwlock *_new_lock; \ 699 \ 700 _new_lock = PHYS_TO_PV_LIST_LOCK(pa); \ 701 if (_new_lock != *_lockp) { \ 702 if (*_lockp != NULL) \ 703 rw_wunlock(*_lockp); \ 704 *_lockp = _new_lock; \ 705 rw_wlock(*_lockp); \ 706 } \ 707 } while (0) 708 709 #define CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m) \ 710 CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, VM_PAGE_TO_PHYS(m)) 711 712 #define RELEASE_PV_LIST_LOCK(lockp) do { \ 713 struct rwlock **_lockp = (lockp); \ 714 \ 715 if (*_lockp != NULL) { \ 716 rw_wunlock(*_lockp); \ 717 *_lockp = NULL; \ 718 } \ 719 } while (0) 720 721 #define VM_PAGE_TO_PV_LIST_LOCK(m) \ 722 PHYS_TO_PV_LIST_LOCK(VM_PAGE_TO_PHYS(m)) 723 724 /* 725 * We support 52 bits, hence: 726 * bits 52 - 31 = 21, 0b10101 727 * RTS encoding details 728 * bits 0 - 3 of rts -> bits 6 - 8 unsigned long 729 * bits 4 - 5 of rts -> bits 62 - 63 of unsigned long 730 */ 731 #define RTS_SIZE ((0x2UL << 61) | (0x5UL << 5)) 732 733 static int powernv_enabled = 1; 734 735 static __always_inline void 736 tlbiel_radix_set_isa300(uint32_t set, uint32_t is, 737 uint32_t pid, uint32_t ric, uint32_t prs) 738 { 739 uint64_t rb; 740 uint64_t rs; 741 742 rb = PPC_BITLSHIFT_VAL(set, 51) | PPC_BITLSHIFT_VAL(is, 53); 743 rs = PPC_BITLSHIFT_VAL((uint64_t)pid, 31); 744 745 __asm __volatile(PPC_TLBIEL(%0, %1, %2, %3, 1) 746 : : "r"(rb), "r"(rs), "i"(ric), "i"(prs) 747 : "memory"); 748 } 749 750 static void 751 tlbiel_flush_isa3(uint32_t num_sets, uint32_t is) 752 { 753 uint32_t set; 754 755 __asm __volatile("ptesync": : :"memory"); 756 757 /* 758 * Flush the first set of the TLB, and the entire Page Walk Cache 759 * and partition table entries. Then flush the remaining sets of the 760 * TLB. 761 */ 762 if (is == TLB_INVAL_SCOPE_GLOBAL) { 763 tlbiel_radix_set_isa300(0, is, 0, RIC_FLUSH_ALL, 0); 764 for (set = 1; set < num_sets; set++) 765 tlbiel_radix_set_isa300(set, is, 0, RIC_FLUSH_TLB, 0); 766 } 767 768 /* Do the same for process scoped entries. */ 769 tlbiel_radix_set_isa300(0, is, 0, RIC_FLUSH_ALL, 1); 770 for (set = 1; set < num_sets; set++) 771 tlbiel_radix_set_isa300(set, is, 0, RIC_FLUSH_TLB, 1); 772 773 __asm __volatile("ptesync": : :"memory"); 774 } 775 776 static void 777 mmu_radix_tlbiel_flush(int scope) 778 { 779 MPASS(scope == TLB_INVAL_SCOPE_LPID || 780 scope == TLB_INVAL_SCOPE_GLOBAL); 781 782 tlbiel_flush_isa3(POWER9_TLB_SETS_RADIX, scope); 783 __asm __volatile(PPC_INVALIDATE_ERAT "; isync" : : :"memory"); 784 } 785 786 static void 787 mmu_radix_tlbie_all(void) 788 { 789 if (powernv_enabled) 790 mmu_radix_tlbiel_flush(TLB_INVAL_SCOPE_GLOBAL); 791 else 792 mmu_radix_tlbiel_flush(TLB_INVAL_SCOPE_LPID); 793 } 794 795 static void 796 mmu_radix_init_amor(void) 797 { 798 /* 799 * In HV mode, we init AMOR (Authority Mask Override Register) so that 800 * the hypervisor and guest can setup IAMR (Instruction Authority Mask 801 * Register), enable key 0 and set it to 1. 802 * 803 * AMOR = 0b1100 .... 0000 (Mask for key 0 is 11) 804 */ 805 mtspr(SPR_AMOR, (3ul << 62)); 806 } 807 808 static void 809 mmu_radix_init_iamr(void) 810 { 811 /* 812 * Radix always uses key0 of the IAMR to determine if an access is 813 * allowed. We set bit 0 (IBM bit 1) of key0, to prevent instruction 814 * fetch. 815 */ 816 mtspr(SPR_IAMR, (1ul << 62)); 817 } 818 819 static void 820 mmu_radix_pid_set(pmap_t pmap) 821 { 822 823 mtspr(SPR_PID, pmap->pm_pid); 824 isync(); 825 } 826 827 /* Quick sort callout for comparing physical addresses. */ 828 static int 829 pa_cmp(const void *a, const void *b) 830 { 831 const vm_paddr_t *pa = a, *pb = b; 832 833 if (*pa < *pb) 834 return (-1); 835 else if (*pa > *pb) 836 return (1); 837 else 838 return (0); 839 } 840 841 #define pte_load_store(ptep, pte) atomic_swap_long(ptep, pte) 842 #define pte_load_clear(ptep) atomic_swap_long(ptep, 0) 843 #define pte_store(ptep, pte) do { \ 844 MPASS((pte) & (RPTE_EAA_R | RPTE_EAA_W | RPTE_EAA_X)); \ 845 *(u_long *)(ptep) = htobe64((u_long)((pte) | PG_V | RPTE_LEAF)); \ 846 } while (0) 847 /* 848 * NB: should only be used for adding directories - not for direct mappings 849 */ 850 #define pde_store(ptep, pa) do { \ 851 *(u_long *)(ptep) = htobe64((u_long)(pa|RPTE_VALID|RPTE_SHIFT)); \ 852 } while (0) 853 854 #define pte_clear(ptep) do { \ 855 *(u_long *)(ptep) = (u_long)(0); \ 856 } while (0) 857 858 #define PMAP_PDE_SUPERPAGE (1 << 8) /* supports 2MB superpages */ 859 860 /* 861 * Promotion to a 2MB (PDE) page mapping requires that the corresponding 4KB 862 * (PTE) page mappings have identical settings for the following fields: 863 */ 864 #define PG_PTE_PROMOTE (PG_X | PG_MANAGED | PG_W | PG_PTE_CACHE | \ 865 PG_M | PG_A | RPTE_EAA_MASK | PG_V) 866 867 static __inline void 868 pmap_resident_count_inc(pmap_t pmap, int count) 869 { 870 871 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 872 pmap->pm_stats.resident_count += count; 873 } 874 875 static __inline void 876 pmap_resident_count_dec(pmap_t pmap, int count) 877 { 878 879 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 880 KASSERT(pmap->pm_stats.resident_count >= count, 881 ("pmap %p resident count underflow %ld %d", pmap, 882 pmap->pm_stats.resident_count, count)); 883 pmap->pm_stats.resident_count -= count; 884 } 885 886 static void 887 pagezero(vm_offset_t va) 888 { 889 va = trunc_page(va); 890 891 bzero((void *)va, PAGE_SIZE); 892 } 893 894 static uint64_t 895 allocpages(int n) 896 { 897 u_int64_t ret; 898 899 ret = moea64_bootstrap_alloc(n * PAGE_SIZE, PAGE_SIZE); 900 for (int i = 0; i < n; i++) 901 pagezero(PHYS_TO_DMAP(ret + i * PAGE_SIZE)); 902 return (ret); 903 } 904 905 static pt_entry_t * 906 kvtopte(vm_offset_t va) 907 { 908 pt_entry_t *l3e; 909 910 l3e = pmap_pml3e(kernel_pmap, va); 911 if (l3e == NULL || (be64toh(*l3e) & RPTE_VALID) == 0) 912 return (NULL); 913 return (pmap_l3e_to_pte(l3e, va)); 914 } 915 916 void 917 mmu_radix_kenter(vm_offset_t va, vm_paddr_t pa) 918 { 919 pt_entry_t *pte; 920 921 pte = kvtopte(va); 922 MPASS(pte != NULL); 923 *pte = htobe64(pa | RPTE_VALID | RPTE_LEAF | RPTE_EAA_R | \ 924 RPTE_EAA_W | RPTE_EAA_P | PG_M | PG_A); 925 } 926 927 bool 928 mmu_radix_ps_enabled(pmap_t pmap) 929 { 930 return (superpages_enabled && (pmap->pm_flags & PMAP_PDE_SUPERPAGE) != 0); 931 } 932 933 static pt_entry_t * 934 pmap_nofault_pte(pmap_t pmap, vm_offset_t va, int *is_l3e) 935 { 936 pml3_entry_t *l3e; 937 pt_entry_t *pte; 938 939 va &= PG_PS_FRAME; 940 l3e = pmap_pml3e(pmap, va); 941 if (l3e == NULL || (be64toh(*l3e) & PG_V) == 0) 942 return (NULL); 943 944 if (be64toh(*l3e) & RPTE_LEAF) { 945 *is_l3e = 1; 946 return (l3e); 947 } 948 *is_l3e = 0; 949 va &= PG_FRAME; 950 pte = pmap_l3e_to_pte(l3e, va); 951 if (pte == NULL || (be64toh(*pte) & PG_V) == 0) 952 return (NULL); 953 return (pte); 954 } 955 956 int 957 pmap_nofault(pmap_t pmap, vm_offset_t va, vm_prot_t flags) 958 { 959 pt_entry_t *pte; 960 pt_entry_t startpte, origpte, newpte; 961 vm_page_t m; 962 int is_l3e; 963 964 startpte = 0; 965 retry: 966 if ((pte = pmap_nofault_pte(pmap, va, &is_l3e)) == NULL) 967 return (KERN_INVALID_ADDRESS); 968 origpte = newpte = be64toh(*pte); 969 if (startpte == 0) { 970 startpte = origpte; 971 if (((flags & VM_PROT_WRITE) && (startpte & PG_M)) || 972 ((flags & VM_PROT_READ) && (startpte & PG_A))) { 973 pmap_invalidate_all(pmap); 974 #ifdef INVARIANTS 975 if (VERBOSE_PMAP || pmap_logging) 976 printf("%s(%p, %#lx, %#x) (%#lx) -- invalidate all\n", 977 __func__, pmap, va, flags, origpte); 978 #endif 979 return (KERN_FAILURE); 980 } 981 } 982 #ifdef INVARIANTS 983 if (VERBOSE_PMAP || pmap_logging) 984 printf("%s(%p, %#lx, %#x) (%#lx)\n", __func__, pmap, va, 985 flags, origpte); 986 #endif 987 PMAP_LOCK(pmap); 988 if ((pte = pmap_nofault_pte(pmap, va, &is_l3e)) == NULL || 989 be64toh(*pte) != origpte) { 990 PMAP_UNLOCK(pmap); 991 return (KERN_FAILURE); 992 } 993 m = PHYS_TO_VM_PAGE(newpte & PG_FRAME); 994 MPASS(m != NULL); 995 switch (flags) { 996 case VM_PROT_READ: 997 if ((newpte & (RPTE_EAA_R|RPTE_EAA_X)) == 0) 998 goto protfail; 999 newpte |= PG_A; 1000 vm_page_aflag_set(m, PGA_REFERENCED); 1001 break; 1002 case VM_PROT_WRITE: 1003 if ((newpte & RPTE_EAA_W) == 0) 1004 goto protfail; 1005 if (is_l3e) 1006 goto protfail; 1007 newpte |= PG_M; 1008 vm_page_dirty(m); 1009 break; 1010 case VM_PROT_EXECUTE: 1011 if ((newpte & RPTE_EAA_X) == 0) 1012 goto protfail; 1013 newpte |= PG_A; 1014 vm_page_aflag_set(m, PGA_REFERENCED); 1015 break; 1016 } 1017 1018 if (!atomic_cmpset_long(pte, htobe64(origpte), htobe64(newpte))) 1019 goto retry; 1020 ptesync(); 1021 PMAP_UNLOCK(pmap); 1022 if (startpte == newpte) 1023 return (KERN_FAILURE); 1024 return (0); 1025 protfail: 1026 PMAP_UNLOCK(pmap); 1027 return (KERN_PROTECTION_FAILURE); 1028 } 1029 1030 /* 1031 * Returns true if the given page is mapped individually or as part of 1032 * a 2mpage. Otherwise, returns false. 1033 */ 1034 bool 1035 mmu_radix_page_is_mapped(vm_page_t m) 1036 { 1037 struct rwlock *lock; 1038 bool rv; 1039 1040 if ((m->oflags & VPO_UNMANAGED) != 0) 1041 return (false); 1042 lock = VM_PAGE_TO_PV_LIST_LOCK(m); 1043 rw_rlock(lock); 1044 rv = !TAILQ_EMPTY(&m->md.pv_list) || 1045 ((m->flags & PG_FICTITIOUS) == 0 && 1046 !TAILQ_EMPTY(&pa_to_pvh(VM_PAGE_TO_PHYS(m))->pv_list)); 1047 rw_runlock(lock); 1048 return (rv); 1049 } 1050 1051 /* 1052 * Determine the appropriate bits to set in a PTE or PDE for a specified 1053 * caching mode. 1054 */ 1055 static int 1056 pmap_cache_bits(vm_memattr_t ma) 1057 { 1058 if (ma != VM_MEMATTR_DEFAULT) { 1059 switch (ma) { 1060 case VM_MEMATTR_UNCACHEABLE: 1061 return (RPTE_ATTR_GUARDEDIO); 1062 case VM_MEMATTR_CACHEABLE: 1063 return (RPTE_ATTR_MEM); 1064 case VM_MEMATTR_WRITE_BACK: 1065 case VM_MEMATTR_PREFETCHABLE: 1066 case VM_MEMATTR_WRITE_COMBINING: 1067 return (RPTE_ATTR_UNGUARDEDIO); 1068 } 1069 } 1070 return (0); 1071 } 1072 1073 static void 1074 pmap_invalidate_page(pmap_t pmap, vm_offset_t start) 1075 { 1076 ptesync(); 1077 if (pmap == kernel_pmap) 1078 radix_tlbie_invlpg_kernel_4k(start); 1079 else 1080 radix_tlbie_invlpg_user_4k(pmap->pm_pid, start); 1081 ttusync(); 1082 } 1083 1084 static void 1085 pmap_invalidate_page_2m(pmap_t pmap, vm_offset_t start) 1086 { 1087 ptesync(); 1088 if (pmap == kernel_pmap) 1089 radix_tlbie_invlpg_kernel_2m(start); 1090 else 1091 radix_tlbie_invlpg_user_2m(pmap->pm_pid, start); 1092 ttusync(); 1093 } 1094 1095 static void 1096 pmap_invalidate_pwc(pmap_t pmap) 1097 { 1098 ptesync(); 1099 if (pmap == kernel_pmap) 1100 radix_tlbie_invlpwc_kernel(); 1101 else 1102 radix_tlbie_invlpwc_user(pmap->pm_pid); 1103 ttusync(); 1104 } 1105 1106 static void 1107 pmap_invalidate_range(pmap_t pmap, vm_offset_t start, vm_offset_t end) 1108 { 1109 if (((start - end) >> PAGE_SHIFT) > 8) { 1110 pmap_invalidate_all(pmap); 1111 return; 1112 } 1113 ptesync(); 1114 if (pmap == kernel_pmap) { 1115 while (start < end) { 1116 radix_tlbie_invlpg_kernel_4k(start); 1117 start += PAGE_SIZE; 1118 } 1119 } else { 1120 while (start < end) { 1121 radix_tlbie_invlpg_user_4k(pmap->pm_pid, start); 1122 start += PAGE_SIZE; 1123 } 1124 } 1125 ttusync(); 1126 } 1127 1128 static void 1129 pmap_invalidate_all(pmap_t pmap) 1130 { 1131 ptesync(); 1132 if (pmap == kernel_pmap) 1133 radix_tlbie_flush_kernel(); 1134 else 1135 radix_tlbie_flush_user(pmap->pm_pid); 1136 ttusync(); 1137 } 1138 1139 static void 1140 pmap_invalidate_l3e_page(pmap_t pmap, vm_offset_t va, pml3_entry_t l3e) 1141 { 1142 1143 /* 1144 * When the PDE has PG_PROMOTED set, the 2MB page mapping was created 1145 * by a promotion that did not invalidate the 512 4KB page mappings 1146 * that might exist in the TLB. Consequently, at this point, the TLB 1147 * may hold both 4KB and 2MB page mappings for the address range [va, 1148 * va + L3_PAGE_SIZE). Therefore, the entire range must be invalidated here. 1149 * In contrast, when PG_PROMOTED is clear, the TLB will not hold any 1150 * 4KB page mappings for the address range [va, va + L3_PAGE_SIZE), and so a 1151 * single INVLPG suffices to invalidate the 2MB page mapping from the 1152 * TLB. 1153 */ 1154 ptesync(); 1155 if ((l3e & PG_PROMOTED) != 0) 1156 pmap_invalidate_range(pmap, va, va + L3_PAGE_SIZE - 1); 1157 else 1158 pmap_invalidate_page_2m(pmap, va); 1159 1160 pmap_invalidate_pwc(pmap); 1161 } 1162 1163 static __inline struct pv_chunk * 1164 pv_to_chunk(pv_entry_t pv) 1165 { 1166 1167 return ((struct pv_chunk *)((uintptr_t)pv & ~(uintptr_t)PAGE_MASK)); 1168 } 1169 1170 #define PV_PMAP(pv) (pv_to_chunk(pv)->pc_pmap) 1171 1172 #define PC_FREE0 0xfffffffffffffffful 1173 #define PC_FREE1 ((1ul << (_NPCPV % 64)) - 1) 1174 1175 static const uint64_t pc_freemask[_NPCM] = { PC_FREE0, PC_FREE1 }; 1176 1177 /* 1178 * Ensure that the number of spare PV entries in the specified pmap meets or 1179 * exceeds the given count, "needed". 1180 * 1181 * The given PV list lock may be released. 1182 */ 1183 static void 1184 reserve_pv_entries(pmap_t pmap, int needed, struct rwlock **lockp) 1185 { 1186 struct pch new_tail; 1187 struct pv_chunk *pc; 1188 vm_page_t m; 1189 int avail, free; 1190 bool reclaimed; 1191 1192 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1193 KASSERT(lockp != NULL, ("reserve_pv_entries: lockp is NULL")); 1194 1195 /* 1196 * Newly allocated PV chunks must be stored in a private list until 1197 * the required number of PV chunks have been allocated. Otherwise, 1198 * reclaim_pv_chunk() could recycle one of these chunks. In 1199 * contrast, these chunks must be added to the pmap upon allocation. 1200 */ 1201 TAILQ_INIT(&new_tail); 1202 retry: 1203 avail = 0; 1204 TAILQ_FOREACH(pc, &pmap->pm_pvchunk, pc_list) { 1205 // if ((cpu_feature2 & CPUID2_POPCNT) == 0) 1206 bit_count((bitstr_t *)pc->pc_map, 0, 1207 sizeof(pc->pc_map) * NBBY, &free); 1208 #if 0 1209 free = popcnt_pc_map_pq(pc->pc_map); 1210 #endif 1211 if (free == 0) 1212 break; 1213 avail += free; 1214 if (avail >= needed) 1215 break; 1216 } 1217 for (reclaimed = false; avail < needed; avail += _NPCPV) { 1218 m = vm_page_alloc_noobj(VM_ALLOC_WIRED); 1219 if (m == NULL) { 1220 m = reclaim_pv_chunk(pmap, lockp); 1221 if (m == NULL) 1222 goto retry; 1223 reclaimed = true; 1224 } 1225 PV_STAT(atomic_add_int(&pc_chunk_count, 1)); 1226 PV_STAT(atomic_add_int(&pc_chunk_allocs, 1)); 1227 dump_add_page(m->phys_addr); 1228 pc = (void *)PHYS_TO_DMAP(m->phys_addr); 1229 pc->pc_pmap = pmap; 1230 pc->pc_map[0] = PC_FREE0; 1231 pc->pc_map[1] = PC_FREE1; 1232 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list); 1233 TAILQ_INSERT_TAIL(&new_tail, pc, pc_lru); 1234 PV_STAT(atomic_add_int(&pv_entry_spare, _NPCPV)); 1235 1236 /* 1237 * The reclaim might have freed a chunk from the current pmap. 1238 * If that chunk contained available entries, we need to 1239 * re-count the number of available entries. 1240 */ 1241 if (reclaimed) 1242 goto retry; 1243 } 1244 if (!TAILQ_EMPTY(&new_tail)) { 1245 mtx_lock(&pv_chunks_mutex); 1246 TAILQ_CONCAT(&pv_chunks, &new_tail, pc_lru); 1247 mtx_unlock(&pv_chunks_mutex); 1248 } 1249 } 1250 1251 /* 1252 * First find and then remove the pv entry for the specified pmap and virtual 1253 * address from the specified pv list. Returns the pv entry if found and NULL 1254 * otherwise. This operation can be performed on pv lists for either 4KB or 1255 * 2MB page mappings. 1256 */ 1257 static __inline pv_entry_t 1258 pmap_pvh_remove(struct md_page *pvh, pmap_t pmap, vm_offset_t va) 1259 { 1260 pv_entry_t pv; 1261 1262 TAILQ_FOREACH(pv, &pvh->pv_list, pv_link) { 1263 #ifdef INVARIANTS 1264 if (PV_PMAP(pv) == NULL) { 1265 printf("corrupted pv_chunk/pv %p\n", pv); 1266 printf("pv_chunk: %64D\n", pv_to_chunk(pv), ":"); 1267 } 1268 MPASS(PV_PMAP(pv) != NULL); 1269 MPASS(pv->pv_va != 0); 1270 #endif 1271 if (pmap == PV_PMAP(pv) && va == pv->pv_va) { 1272 TAILQ_REMOVE(&pvh->pv_list, pv, pv_link); 1273 pvh->pv_gen++; 1274 break; 1275 } 1276 } 1277 return (pv); 1278 } 1279 1280 /* 1281 * After demotion from a 2MB page mapping to 512 4KB page mappings, 1282 * destroy the pv entry for the 2MB page mapping and reinstantiate the pv 1283 * entries for each of the 4KB page mappings. 1284 */ 1285 static void 1286 pmap_pv_demote_l3e(pmap_t pmap, vm_offset_t va, vm_paddr_t pa, 1287 struct rwlock **lockp) 1288 { 1289 struct md_page *pvh; 1290 struct pv_chunk *pc; 1291 pv_entry_t pv; 1292 vm_offset_t va_last; 1293 vm_page_t m; 1294 int bit, field; 1295 1296 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1297 KASSERT((pa & L3_PAGE_MASK) == 0, 1298 ("pmap_pv_demote_pde: pa is not 2mpage aligned")); 1299 CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa); 1300 1301 /* 1302 * Transfer the 2mpage's pv entry for this mapping to the first 1303 * page's pv list. Once this transfer begins, the pv list lock 1304 * must not be released until the last pv entry is reinstantiated. 1305 */ 1306 pvh = pa_to_pvh(pa); 1307 va = trunc_2mpage(va); 1308 pv = pmap_pvh_remove(pvh, pmap, va); 1309 KASSERT(pv != NULL, ("pmap_pv_demote_pde: pv not found")); 1310 m = PHYS_TO_VM_PAGE(pa); 1311 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_link); 1312 1313 m->md.pv_gen++; 1314 /* Instantiate the remaining NPTEPG - 1 pv entries. */ 1315 PV_STAT(atomic_add_long(&pv_entry_allocs, NPTEPG - 1)); 1316 va_last = va + L3_PAGE_SIZE - PAGE_SIZE; 1317 for (;;) { 1318 pc = TAILQ_FIRST(&pmap->pm_pvchunk); 1319 KASSERT(pc->pc_map[0] != 0 || pc->pc_map[1] != 0 1320 , ("pmap_pv_demote_pde: missing spare")); 1321 for (field = 0; field < _NPCM; field++) { 1322 while (pc->pc_map[field]) { 1323 bit = cnttzd(pc->pc_map[field]); 1324 pc->pc_map[field] &= ~(1ul << bit); 1325 pv = &pc->pc_pventry[field * 64 + bit]; 1326 va += PAGE_SIZE; 1327 pv->pv_va = va; 1328 m++; 1329 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 1330 ("pmap_pv_demote_pde: page %p is not managed", m)); 1331 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_link); 1332 1333 m->md.pv_gen++; 1334 if (va == va_last) 1335 goto out; 1336 } 1337 } 1338 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 1339 TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list); 1340 } 1341 out: 1342 if (pc->pc_map[0] == 0 && pc->pc_map[1] == 0) { 1343 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 1344 TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list); 1345 } 1346 PV_STAT(atomic_add_long(&pv_entry_count, NPTEPG - 1)); 1347 PV_STAT(atomic_subtract_int(&pv_entry_spare, NPTEPG - 1)); 1348 } 1349 1350 static void 1351 reclaim_pv_chunk_leave_pmap(pmap_t pmap, pmap_t locked_pmap) 1352 { 1353 1354 if (pmap == NULL) 1355 return; 1356 pmap_invalidate_all(pmap); 1357 if (pmap != locked_pmap) 1358 PMAP_UNLOCK(pmap); 1359 } 1360 1361 /* 1362 * We are in a serious low memory condition. Resort to 1363 * drastic measures to free some pages so we can allocate 1364 * another pv entry chunk. 1365 * 1366 * Returns NULL if PV entries were reclaimed from the specified pmap. 1367 * 1368 * We do not, however, unmap 2mpages because subsequent accesses will 1369 * allocate per-page pv entries until repromotion occurs, thereby 1370 * exacerbating the shortage of free pv entries. 1371 */ 1372 static int active_reclaims = 0; 1373 static vm_page_t 1374 reclaim_pv_chunk(pmap_t locked_pmap, struct rwlock **lockp) 1375 { 1376 struct pv_chunk *pc, *pc_marker, *pc_marker_end; 1377 struct pv_chunk_header pc_marker_b, pc_marker_end_b; 1378 struct md_page *pvh; 1379 pml3_entry_t *l3e; 1380 pmap_t next_pmap, pmap; 1381 pt_entry_t *pte, tpte; 1382 pv_entry_t pv; 1383 vm_offset_t va; 1384 vm_page_t m, m_pc; 1385 struct spglist free; 1386 uint64_t inuse; 1387 int bit, field, freed; 1388 1389 PMAP_LOCK_ASSERT(locked_pmap, MA_OWNED); 1390 KASSERT(lockp != NULL, ("reclaim_pv_chunk: lockp is NULL")); 1391 pmap = NULL; 1392 m_pc = NULL; 1393 SLIST_INIT(&free); 1394 bzero(&pc_marker_b, sizeof(pc_marker_b)); 1395 bzero(&pc_marker_end_b, sizeof(pc_marker_end_b)); 1396 pc_marker = (struct pv_chunk *)&pc_marker_b; 1397 pc_marker_end = (struct pv_chunk *)&pc_marker_end_b; 1398 1399 mtx_lock(&pv_chunks_mutex); 1400 active_reclaims++; 1401 TAILQ_INSERT_HEAD(&pv_chunks, pc_marker, pc_lru); 1402 TAILQ_INSERT_TAIL(&pv_chunks, pc_marker_end, pc_lru); 1403 while ((pc = TAILQ_NEXT(pc_marker, pc_lru)) != pc_marker_end && 1404 SLIST_EMPTY(&free)) { 1405 next_pmap = pc->pc_pmap; 1406 if (next_pmap == NULL) { 1407 /* 1408 * The next chunk is a marker. However, it is 1409 * not our marker, so active_reclaims must be 1410 * > 1. Consequently, the next_chunk code 1411 * will not rotate the pv_chunks list. 1412 */ 1413 goto next_chunk; 1414 } 1415 mtx_unlock(&pv_chunks_mutex); 1416 1417 /* 1418 * A pv_chunk can only be removed from the pc_lru list 1419 * when both pc_chunks_mutex is owned and the 1420 * corresponding pmap is locked. 1421 */ 1422 if (pmap != next_pmap) { 1423 reclaim_pv_chunk_leave_pmap(pmap, locked_pmap); 1424 pmap = next_pmap; 1425 /* Avoid deadlock and lock recursion. */ 1426 if (pmap > locked_pmap) { 1427 RELEASE_PV_LIST_LOCK(lockp); 1428 PMAP_LOCK(pmap); 1429 mtx_lock(&pv_chunks_mutex); 1430 continue; 1431 } else if (pmap != locked_pmap) { 1432 if (PMAP_TRYLOCK(pmap)) { 1433 mtx_lock(&pv_chunks_mutex); 1434 continue; 1435 } else { 1436 pmap = NULL; /* pmap is not locked */ 1437 mtx_lock(&pv_chunks_mutex); 1438 pc = TAILQ_NEXT(pc_marker, pc_lru); 1439 if (pc == NULL || 1440 pc->pc_pmap != next_pmap) 1441 continue; 1442 goto next_chunk; 1443 } 1444 } 1445 } 1446 1447 /* 1448 * Destroy every non-wired, 4 KB page mapping in the chunk. 1449 */ 1450 freed = 0; 1451 for (field = 0; field < _NPCM; field++) { 1452 for (inuse = ~pc->pc_map[field] & pc_freemask[field]; 1453 inuse != 0; inuse &= ~(1UL << bit)) { 1454 bit = cnttzd(inuse); 1455 pv = &pc->pc_pventry[field * 64 + bit]; 1456 va = pv->pv_va; 1457 l3e = pmap_pml3e(pmap, va); 1458 if ((be64toh(*l3e) & RPTE_LEAF) != 0) 1459 continue; 1460 pte = pmap_l3e_to_pte(l3e, va); 1461 if ((be64toh(*pte) & PG_W) != 0) 1462 continue; 1463 tpte = be64toh(pte_load_clear(pte)); 1464 m = PHYS_TO_VM_PAGE(tpte & PG_FRAME); 1465 if ((tpte & (PG_M | PG_RW)) == (PG_M | PG_RW)) 1466 vm_page_dirty(m); 1467 if ((tpte & PG_A) != 0) 1468 vm_page_aflag_set(m, PGA_REFERENCED); 1469 CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m); 1470 TAILQ_REMOVE(&m->md.pv_list, pv, pv_link); 1471 1472 m->md.pv_gen++; 1473 if (TAILQ_EMPTY(&m->md.pv_list) && 1474 (m->flags & PG_FICTITIOUS) == 0) { 1475 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); 1476 if (TAILQ_EMPTY(&pvh->pv_list)) { 1477 vm_page_aflag_clear(m, 1478 PGA_WRITEABLE); 1479 } 1480 } 1481 pc->pc_map[field] |= 1UL << bit; 1482 pmap_unuse_pt(pmap, va, be64toh(*l3e), &free); 1483 freed++; 1484 } 1485 } 1486 if (freed == 0) { 1487 mtx_lock(&pv_chunks_mutex); 1488 goto next_chunk; 1489 } 1490 /* Every freed mapping is for a 4 KB page. */ 1491 pmap_resident_count_dec(pmap, freed); 1492 PV_STAT(atomic_add_long(&pv_entry_frees, freed)); 1493 PV_STAT(atomic_add_int(&pv_entry_spare, freed)); 1494 PV_STAT(atomic_subtract_long(&pv_entry_count, freed)); 1495 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 1496 if (pc->pc_map[0] == PC_FREE0 && pc->pc_map[1] == PC_FREE1) { 1497 PV_STAT(atomic_subtract_int(&pv_entry_spare, _NPCPV)); 1498 PV_STAT(atomic_subtract_int(&pc_chunk_count, 1)); 1499 PV_STAT(atomic_add_int(&pc_chunk_frees, 1)); 1500 /* Entire chunk is free; return it. */ 1501 m_pc = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pc)); 1502 dump_drop_page(m_pc->phys_addr); 1503 mtx_lock(&pv_chunks_mutex); 1504 TAILQ_REMOVE(&pv_chunks, pc, pc_lru); 1505 break; 1506 } 1507 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list); 1508 mtx_lock(&pv_chunks_mutex); 1509 /* One freed pv entry in locked_pmap is sufficient. */ 1510 if (pmap == locked_pmap) 1511 break; 1512 next_chunk: 1513 TAILQ_REMOVE(&pv_chunks, pc_marker, pc_lru); 1514 TAILQ_INSERT_AFTER(&pv_chunks, pc, pc_marker, pc_lru); 1515 if (active_reclaims == 1 && pmap != NULL) { 1516 /* 1517 * Rotate the pv chunks list so that we do not 1518 * scan the same pv chunks that could not be 1519 * freed (because they contained a wired 1520 * and/or superpage mapping) on every 1521 * invocation of reclaim_pv_chunk(). 1522 */ 1523 while ((pc = TAILQ_FIRST(&pv_chunks)) != pc_marker) { 1524 MPASS(pc->pc_pmap != NULL); 1525 TAILQ_REMOVE(&pv_chunks, pc, pc_lru); 1526 TAILQ_INSERT_TAIL(&pv_chunks, pc, pc_lru); 1527 } 1528 } 1529 } 1530 TAILQ_REMOVE(&pv_chunks, pc_marker, pc_lru); 1531 TAILQ_REMOVE(&pv_chunks, pc_marker_end, pc_lru); 1532 active_reclaims--; 1533 mtx_unlock(&pv_chunks_mutex); 1534 reclaim_pv_chunk_leave_pmap(pmap, locked_pmap); 1535 if (m_pc == NULL && !SLIST_EMPTY(&free)) { 1536 m_pc = SLIST_FIRST(&free); 1537 SLIST_REMOVE_HEAD(&free, plinks.s.ss); 1538 /* Recycle a freed page table page. */ 1539 m_pc->ref_count = 1; 1540 } 1541 vm_page_free_pages_toq(&free, true); 1542 return (m_pc); 1543 } 1544 1545 /* 1546 * free the pv_entry back to the free list 1547 */ 1548 static void 1549 free_pv_entry(pmap_t pmap, pv_entry_t pv) 1550 { 1551 struct pv_chunk *pc; 1552 int idx, field, bit; 1553 1554 #ifdef VERBOSE_PV 1555 if (pmap != kernel_pmap) 1556 printf("%s(%p, %p)\n", __func__, pmap, pv); 1557 #endif 1558 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1559 PV_STAT(atomic_add_long(&pv_entry_frees, 1)); 1560 PV_STAT(atomic_add_int(&pv_entry_spare, 1)); 1561 PV_STAT(atomic_subtract_long(&pv_entry_count, 1)); 1562 pc = pv_to_chunk(pv); 1563 idx = pv - &pc->pc_pventry[0]; 1564 field = idx / 64; 1565 bit = idx % 64; 1566 pc->pc_map[field] |= 1ul << bit; 1567 if (pc->pc_map[0] != PC_FREE0 || pc->pc_map[1] != PC_FREE1) { 1568 /* 98% of the time, pc is already at the head of the list. */ 1569 if (__predict_false(pc != TAILQ_FIRST(&pmap->pm_pvchunk))) { 1570 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 1571 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list); 1572 } 1573 return; 1574 } 1575 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 1576 free_pv_chunk(pc); 1577 } 1578 1579 static void 1580 free_pv_chunk(struct pv_chunk *pc) 1581 { 1582 vm_page_t m; 1583 1584 mtx_lock(&pv_chunks_mutex); 1585 TAILQ_REMOVE(&pv_chunks, pc, pc_lru); 1586 mtx_unlock(&pv_chunks_mutex); 1587 PV_STAT(atomic_subtract_int(&pv_entry_spare, _NPCPV)); 1588 PV_STAT(atomic_subtract_int(&pc_chunk_count, 1)); 1589 PV_STAT(atomic_add_int(&pc_chunk_frees, 1)); 1590 /* entire chunk is free, return it */ 1591 m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pc)); 1592 dump_drop_page(m->phys_addr); 1593 vm_page_unwire_noq(m); 1594 vm_page_free(m); 1595 } 1596 1597 /* 1598 * Returns a new PV entry, allocating a new PV chunk from the system when 1599 * needed. If this PV chunk allocation fails and a PV list lock pointer was 1600 * given, a PV chunk is reclaimed from an arbitrary pmap. Otherwise, NULL is 1601 * returned. 1602 * 1603 * The given PV list lock may be released. 1604 */ 1605 static pv_entry_t 1606 get_pv_entry(pmap_t pmap, struct rwlock **lockp) 1607 { 1608 int bit, field; 1609 pv_entry_t pv; 1610 struct pv_chunk *pc; 1611 vm_page_t m; 1612 1613 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1614 PV_STAT(atomic_add_long(&pv_entry_allocs, 1)); 1615 retry: 1616 pc = TAILQ_FIRST(&pmap->pm_pvchunk); 1617 if (pc != NULL) { 1618 for (field = 0; field < _NPCM; field++) { 1619 if (pc->pc_map[field]) { 1620 bit = cnttzd(pc->pc_map[field]); 1621 break; 1622 } 1623 } 1624 if (field < _NPCM) { 1625 pv = &pc->pc_pventry[field * 64 + bit]; 1626 pc->pc_map[field] &= ~(1ul << bit); 1627 /* If this was the last item, move it to tail */ 1628 if (pc->pc_map[0] == 0 && pc->pc_map[1] == 0) { 1629 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 1630 TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, 1631 pc_list); 1632 } 1633 PV_STAT(atomic_add_long(&pv_entry_count, 1)); 1634 PV_STAT(atomic_subtract_int(&pv_entry_spare, 1)); 1635 MPASS(PV_PMAP(pv) != NULL); 1636 return (pv); 1637 } 1638 } 1639 /* No free items, allocate another chunk */ 1640 m = vm_page_alloc_noobj(VM_ALLOC_WIRED); 1641 if (m == NULL) { 1642 if (lockp == NULL) { 1643 PV_STAT(pc_chunk_tryfail++); 1644 return (NULL); 1645 } 1646 m = reclaim_pv_chunk(pmap, lockp); 1647 if (m == NULL) 1648 goto retry; 1649 } 1650 PV_STAT(atomic_add_int(&pc_chunk_count, 1)); 1651 PV_STAT(atomic_add_int(&pc_chunk_allocs, 1)); 1652 dump_add_page(m->phys_addr); 1653 pc = (void *)PHYS_TO_DMAP(m->phys_addr); 1654 pc->pc_pmap = pmap; 1655 pc->pc_map[0] = PC_FREE0 & ~1ul; /* preallocated bit 0 */ 1656 pc->pc_map[1] = PC_FREE1; 1657 mtx_lock(&pv_chunks_mutex); 1658 TAILQ_INSERT_TAIL(&pv_chunks, pc, pc_lru); 1659 mtx_unlock(&pv_chunks_mutex); 1660 pv = &pc->pc_pventry[0]; 1661 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list); 1662 PV_STAT(atomic_add_long(&pv_entry_count, 1)); 1663 PV_STAT(atomic_add_int(&pv_entry_spare, _NPCPV - 1)); 1664 MPASS(PV_PMAP(pv) != NULL); 1665 return (pv); 1666 } 1667 1668 #if VM_NRESERVLEVEL > 0 1669 /* 1670 * After promotion from 512 4KB page mappings to a single 2MB page mapping, 1671 * replace the many pv entries for the 4KB page mappings by a single pv entry 1672 * for the 2MB page mapping. 1673 */ 1674 static void 1675 pmap_pv_promote_l3e(pmap_t pmap, vm_offset_t va, vm_paddr_t pa, 1676 struct rwlock **lockp) 1677 { 1678 struct md_page *pvh; 1679 pv_entry_t pv; 1680 vm_offset_t va_last; 1681 vm_page_t m; 1682 1683 KASSERT((pa & L3_PAGE_MASK) == 0, 1684 ("pmap_pv_promote_pde: pa is not 2mpage aligned")); 1685 CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa); 1686 1687 /* 1688 * Transfer the first page's pv entry for this mapping to the 2mpage's 1689 * pv list. Aside from avoiding the cost of a call to get_pv_entry(), 1690 * a transfer avoids the possibility that get_pv_entry() calls 1691 * reclaim_pv_chunk() and that reclaim_pv_chunk() removes one of the 1692 * mappings that is being promoted. 1693 */ 1694 m = PHYS_TO_VM_PAGE(pa); 1695 va = trunc_2mpage(va); 1696 pv = pmap_pvh_remove(&m->md, pmap, va); 1697 KASSERT(pv != NULL, ("pmap_pv_promote_pde: pv not found")); 1698 pvh = pa_to_pvh(pa); 1699 TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_link); 1700 pvh->pv_gen++; 1701 /* Free the remaining NPTEPG - 1 pv entries. */ 1702 va_last = va + L3_PAGE_SIZE - PAGE_SIZE; 1703 do { 1704 m++; 1705 va += PAGE_SIZE; 1706 pmap_pvh_free(&m->md, pmap, va); 1707 } while (va < va_last); 1708 } 1709 #endif /* VM_NRESERVLEVEL > 0 */ 1710 1711 /* 1712 * First find and then destroy the pv entry for the specified pmap and virtual 1713 * address. This operation can be performed on pv lists for either 4KB or 2MB 1714 * page mappings. 1715 */ 1716 static void 1717 pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va) 1718 { 1719 pv_entry_t pv; 1720 1721 pv = pmap_pvh_remove(pvh, pmap, va); 1722 KASSERT(pv != NULL, ("pmap_pvh_free: pv not found")); 1723 free_pv_entry(pmap, pv); 1724 } 1725 1726 /* 1727 * Conditionally create the PV entry for a 4KB page mapping if the required 1728 * memory can be allocated without resorting to reclamation. 1729 */ 1730 static bool 1731 pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, vm_page_t m, 1732 struct rwlock **lockp) 1733 { 1734 pv_entry_t pv; 1735 1736 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1737 /* Pass NULL instead of the lock pointer to disable reclamation. */ 1738 if ((pv = get_pv_entry(pmap, NULL)) != NULL) { 1739 pv->pv_va = va; 1740 CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m); 1741 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_link); 1742 m->md.pv_gen++; 1743 return (true); 1744 } else 1745 return (false); 1746 } 1747 1748 vm_paddr_t phys_avail_debug[2 * VM_PHYSSEG_MAX]; 1749 #ifdef INVARIANTS 1750 static void 1751 validate_addr(vm_paddr_t addr, vm_size_t size) 1752 { 1753 vm_paddr_t end = addr + size; 1754 bool found = false; 1755 1756 for (int i = 0; i < 2 * phys_avail_count; i += 2) { 1757 if (addr >= phys_avail_debug[i] && 1758 end <= phys_avail_debug[i + 1]) { 1759 found = true; 1760 break; 1761 } 1762 } 1763 KASSERT(found, ("%#lx-%#lx outside of initial phys_avail array", 1764 addr, end)); 1765 } 1766 #else 1767 static void validate_addr(vm_paddr_t addr, vm_size_t size) {} 1768 #endif 1769 #define DMAP_PAGE_BITS (RPTE_VALID | RPTE_LEAF | RPTE_EAA_MASK | PG_M | PG_A) 1770 1771 static vm_paddr_t 1772 alloc_pt_page(void) 1773 { 1774 vm_paddr_t page; 1775 1776 page = allocpages(1); 1777 pagezero(PHYS_TO_DMAP(page)); 1778 return (page); 1779 } 1780 1781 static void 1782 mmu_radix_dmap_range(vm_paddr_t start, vm_paddr_t end) 1783 { 1784 pt_entry_t *pte, pteval; 1785 vm_paddr_t page; 1786 1787 if (bootverbose) 1788 printf("%s %lx -> %lx\n", __func__, start, end); 1789 while (start < end) { 1790 pteval = start | DMAP_PAGE_BITS; 1791 pte = pmap_pml1e(kernel_pmap, PHYS_TO_DMAP(start)); 1792 if ((be64toh(*pte) & RPTE_VALID) == 0) { 1793 page = alloc_pt_page(); 1794 pde_store(pte, page); 1795 } 1796 pte = pmap_l1e_to_l2e(pte, PHYS_TO_DMAP(start)); 1797 if ((start & L2_PAGE_MASK) == 0 && 1798 end - start >= L2_PAGE_SIZE) { 1799 start += L2_PAGE_SIZE; 1800 goto done; 1801 } else if ((be64toh(*pte) & RPTE_VALID) == 0) { 1802 page = alloc_pt_page(); 1803 pde_store(pte, page); 1804 } 1805 1806 pte = pmap_l2e_to_l3e(pte, PHYS_TO_DMAP(start)); 1807 if ((start & L3_PAGE_MASK) == 0 && 1808 end - start >= L3_PAGE_SIZE) { 1809 start += L3_PAGE_SIZE; 1810 goto done; 1811 } else if ((be64toh(*pte) & RPTE_VALID) == 0) { 1812 page = alloc_pt_page(); 1813 pde_store(pte, page); 1814 } 1815 pte = pmap_l3e_to_pte(pte, PHYS_TO_DMAP(start)); 1816 start += PAGE_SIZE; 1817 done: 1818 pte_store(pte, pteval); 1819 } 1820 } 1821 1822 static void 1823 mmu_radix_dmap_populate(vm_size_t hwphyssz) 1824 { 1825 vm_paddr_t start, end; 1826 1827 for (int i = 0; i < pregions_sz; i++) { 1828 start = pregions[i].mr_start; 1829 end = start + pregions[i].mr_size; 1830 if (hwphyssz && start >= hwphyssz) 1831 break; 1832 if (hwphyssz && hwphyssz < end) 1833 end = hwphyssz; 1834 mmu_radix_dmap_range(start, end); 1835 } 1836 } 1837 1838 static void 1839 mmu_radix_setup_pagetables(vm_size_t hwphyssz) 1840 { 1841 vm_paddr_t ptpages, pages; 1842 pt_entry_t *pte; 1843 vm_paddr_t l1phys; 1844 1845 bzero(kernel_pmap, sizeof(struct pmap)); 1846 PMAP_LOCK_INIT(kernel_pmap); 1847 vm_radix_init(&kernel_pmap->pm_radix); 1848 1849 ptpages = allocpages(3); 1850 l1phys = moea64_bootstrap_alloc(RADIX_PGD_SIZE, RADIX_PGD_SIZE); 1851 validate_addr(l1phys, RADIX_PGD_SIZE); 1852 if (bootverbose) 1853 printf("l1phys=%lx\n", l1phys); 1854 MPASS((l1phys & (RADIX_PGD_SIZE-1)) == 0); 1855 for (int i = 0; i < RADIX_PGD_SIZE/PAGE_SIZE; i++) 1856 pagezero(PHYS_TO_DMAP(l1phys + i * PAGE_SIZE)); 1857 kernel_pmap->pm_pml1 = (pml1_entry_t *)PHYS_TO_DMAP(l1phys); 1858 1859 mmu_radix_dmap_populate(hwphyssz); 1860 1861 /* 1862 * Create page tables for first 128MB of KVA 1863 */ 1864 pages = ptpages; 1865 pte = pmap_pml1e(kernel_pmap, VM_MIN_KERNEL_ADDRESS); 1866 *pte = htobe64(pages | RPTE_VALID | RPTE_SHIFT); 1867 pages += PAGE_SIZE; 1868 pte = pmap_l1e_to_l2e(pte, VM_MIN_KERNEL_ADDRESS); 1869 *pte = htobe64(pages | RPTE_VALID | RPTE_SHIFT); 1870 pages += PAGE_SIZE; 1871 pte = pmap_l2e_to_l3e(pte, VM_MIN_KERNEL_ADDRESS); 1872 /* 1873 * the kernel page table pages need to be preserved in 1874 * phys_avail and not overlap with previous allocations 1875 */ 1876 pages = allocpages(nkpt); 1877 if (bootverbose) { 1878 printf("phys_avail after dmap populate and nkpt allocation\n"); 1879 for (int j = 0; j < 2 * phys_avail_count; j+=2) 1880 printf("phys_avail[%d]=%08lx - phys_avail[%d]=%08lx\n", 1881 j, phys_avail[j], j + 1, phys_avail[j + 1]); 1882 } 1883 KPTphys = pages; 1884 for (int i = 0; i < nkpt; i++, pte++, pages += PAGE_SIZE) 1885 *pte = htobe64(pages | RPTE_VALID | RPTE_SHIFT); 1886 kernel_vm_end = VM_MIN_KERNEL_ADDRESS + nkpt * L3_PAGE_SIZE; 1887 if (bootverbose) 1888 printf("kernel_pmap pml1 %p\n", kernel_pmap->pm_pml1); 1889 /* 1890 * Add a physical memory segment (vm_phys_seg) corresponding to the 1891 * preallocated kernel page table pages so that vm_page structures 1892 * representing these pages will be created. The vm_page structures 1893 * are required for promotion of the corresponding kernel virtual 1894 * addresses to superpage mappings. 1895 */ 1896 vm_phys_add_seg(KPTphys, KPTphys + ptoa(nkpt)); 1897 } 1898 1899 static void 1900 mmu_radix_early_bootstrap(vm_offset_t start, vm_offset_t end) 1901 { 1902 vm_paddr_t kpstart, kpend; 1903 vm_size_t physsz, hwphyssz; 1904 //uint64_t l2virt; 1905 int rm_pavail, proctab_size; 1906 int i, j; 1907 1908 kpstart = start & ~DMAP_BASE_ADDRESS; 1909 kpend = end & ~DMAP_BASE_ADDRESS; 1910 1911 /* Get physical memory regions from firmware */ 1912 mem_regions(&pregions, &pregions_sz, ®ions, ®ions_sz); 1913 CTR0(KTR_PMAP, "mmu_radix_early_bootstrap: physical memory"); 1914 1915 if (2 * VM_PHYSSEG_MAX < regions_sz) 1916 panic("mmu_radix_early_bootstrap: phys_avail too small"); 1917 1918 if (bootverbose) 1919 for (int i = 0; i < regions_sz; i++) 1920 printf("regions[%d].mr_start=%lx regions[%d].mr_size=%lx\n", 1921 i, regions[i].mr_start, i, regions[i].mr_size); 1922 /* 1923 * XXX workaround a simulator bug 1924 */ 1925 for (int i = 0; i < regions_sz; i++) 1926 if (regions[i].mr_start & PAGE_MASK) { 1927 regions[i].mr_start += PAGE_MASK; 1928 regions[i].mr_start &= ~PAGE_MASK; 1929 regions[i].mr_size &= ~PAGE_MASK; 1930 } 1931 if (bootverbose) 1932 for (int i = 0; i < pregions_sz; i++) 1933 printf("pregions[%d].mr_start=%lx pregions[%d].mr_size=%lx\n", 1934 i, pregions[i].mr_start, i, pregions[i].mr_size); 1935 1936 phys_avail_count = 0; 1937 physsz = 0; 1938 hwphyssz = 0; 1939 TUNABLE_ULONG_FETCH("hw.physmem", (u_long *) &hwphyssz); 1940 for (i = 0, j = 0; i < regions_sz; i++) { 1941 if (bootverbose) 1942 printf("regions[%d].mr_start=%016lx regions[%d].mr_size=%016lx\n", 1943 i, regions[i].mr_start, i, regions[i].mr_size); 1944 1945 if (regions[i].mr_size < PAGE_SIZE) 1946 continue; 1947 1948 if (hwphyssz != 0 && 1949 (physsz + regions[i].mr_size) >= hwphyssz) { 1950 if (physsz < hwphyssz) { 1951 phys_avail[j] = regions[i].mr_start; 1952 phys_avail[j + 1] = regions[i].mr_start + 1953 (hwphyssz - physsz); 1954 physsz = hwphyssz; 1955 phys_avail_count++; 1956 dump_avail[j] = phys_avail[j]; 1957 dump_avail[j + 1] = phys_avail[j + 1]; 1958 } 1959 break; 1960 } 1961 phys_avail[j] = regions[i].mr_start; 1962 phys_avail[j + 1] = regions[i].mr_start + regions[i].mr_size; 1963 dump_avail[j] = phys_avail[j]; 1964 dump_avail[j + 1] = phys_avail[j + 1]; 1965 1966 phys_avail_count++; 1967 physsz += regions[i].mr_size; 1968 j += 2; 1969 } 1970 1971 /* Check for overlap with the kernel and exception vectors */ 1972 rm_pavail = 0; 1973 for (j = 0; j < 2 * phys_avail_count; j+=2) { 1974 if (phys_avail[j] < EXC_LAST) 1975 phys_avail[j] += EXC_LAST; 1976 1977 if (phys_avail[j] >= kpstart && 1978 phys_avail[j + 1] <= kpend) { 1979 phys_avail[j] = phys_avail[j + 1] = ~0; 1980 rm_pavail++; 1981 continue; 1982 } 1983 1984 if (kpstart >= phys_avail[j] && 1985 kpstart < phys_avail[j + 1]) { 1986 if (kpend < phys_avail[j + 1]) { 1987 phys_avail[2 * phys_avail_count] = 1988 (kpend & ~PAGE_MASK) + PAGE_SIZE; 1989 phys_avail[2 * phys_avail_count + 1] = 1990 phys_avail[j + 1]; 1991 phys_avail_count++; 1992 } 1993 1994 phys_avail[j + 1] = kpstart & ~PAGE_MASK; 1995 } 1996 1997 if (kpend >= phys_avail[j] && 1998 kpend < phys_avail[j + 1]) { 1999 if (kpstart > phys_avail[j]) { 2000 phys_avail[2 * phys_avail_count] = phys_avail[j]; 2001 phys_avail[2 * phys_avail_count + 1] = 2002 kpstart & ~PAGE_MASK; 2003 phys_avail_count++; 2004 } 2005 2006 phys_avail[j] = (kpend & ~PAGE_MASK) + 2007 PAGE_SIZE; 2008 } 2009 } 2010 qsort(phys_avail, 2 * phys_avail_count, sizeof(phys_avail[0]), pa_cmp); 2011 for (i = 0; i < 2 * phys_avail_count; i++) 2012 phys_avail_debug[i] = phys_avail[i]; 2013 2014 /* Remove physical available regions marked for removal (~0) */ 2015 if (rm_pavail) { 2016 phys_avail_count -= rm_pavail; 2017 for (i = 2 * phys_avail_count; 2018 i < 2*(phys_avail_count + rm_pavail); i+=2) 2019 phys_avail[i] = phys_avail[i + 1] = 0; 2020 } 2021 if (bootverbose) { 2022 printf("phys_avail ranges after filtering:\n"); 2023 for (j = 0; j < 2 * phys_avail_count; j+=2) 2024 printf("phys_avail[%d]=%08lx - phys_avail[%d]=%08lx\n", 2025 j, phys_avail[j], j + 1, phys_avail[j + 1]); 2026 } 2027 physmem = btoc(physsz); 2028 2029 /* XXX assume we're running non-virtualized and 2030 * we don't support BHYVE 2031 */ 2032 if (isa3_pid_bits == 0) 2033 isa3_pid_bits = 20; 2034 if (powernv_enabled) { 2035 parttab_phys = 2036 moea64_bootstrap_alloc(PARTTAB_SIZE, PARTTAB_SIZE); 2037 validate_addr(parttab_phys, PARTTAB_SIZE); 2038 for (int i = 0; i < PARTTAB_SIZE/PAGE_SIZE; i++) 2039 pagezero(PHYS_TO_DMAP(parttab_phys + i * PAGE_SIZE)); 2040 2041 } 2042 proctab_size = 1UL << PROCTAB_SIZE_SHIFT; 2043 proctab0pa = moea64_bootstrap_alloc(proctab_size, proctab_size); 2044 validate_addr(proctab0pa, proctab_size); 2045 for (int i = 0; i < proctab_size/PAGE_SIZE; i++) 2046 pagezero(PHYS_TO_DMAP(proctab0pa + i * PAGE_SIZE)); 2047 2048 mmu_radix_setup_pagetables(hwphyssz); 2049 } 2050 2051 static void 2052 mmu_radix_late_bootstrap(vm_offset_t start, vm_offset_t end) 2053 { 2054 int i; 2055 vm_paddr_t pa; 2056 void *dpcpu; 2057 vm_offset_t va; 2058 2059 /* 2060 * Set up the Open Firmware pmap and add its mappings if not in real 2061 * mode. 2062 */ 2063 if (bootverbose) 2064 printf("%s enter\n", __func__); 2065 2066 /* 2067 * Calculate the last available physical address, and reserve the 2068 * vm_page_array (upper bound). 2069 */ 2070 Maxmem = 0; 2071 for (i = 0; phys_avail[i + 1] != 0; i += 2) 2072 Maxmem = MAX(Maxmem, powerpc_btop(phys_avail[i + 1])); 2073 2074 /* 2075 * Remap any early IO mappings (console framebuffer, etc.) 2076 */ 2077 bs_remap_earlyboot(); 2078 2079 /* 2080 * Allocate a kernel stack with a guard page for thread0 and map it 2081 * into the kernel page map. 2082 */ 2083 pa = allocpages(kstack_pages); 2084 va = virtual_avail + KSTACK_GUARD_PAGES * PAGE_SIZE; 2085 virtual_avail = va + kstack_pages * PAGE_SIZE; 2086 CTR2(KTR_PMAP, "moea64_bootstrap: kstack0 at %#x (%#x)", pa, va); 2087 thread0.td_kstack = va; 2088 for (i = 0; i < kstack_pages; i++) { 2089 mmu_radix_kenter(va, pa); 2090 pa += PAGE_SIZE; 2091 va += PAGE_SIZE; 2092 } 2093 thread0.td_kstack_pages = kstack_pages; 2094 2095 /* 2096 * Allocate virtual address space for the message buffer. 2097 */ 2098 pa = msgbuf_phys = allocpages((msgbufsize + PAGE_MASK) >> PAGE_SHIFT); 2099 msgbufp = (struct msgbuf *)PHYS_TO_DMAP(pa); 2100 2101 /* 2102 * Allocate virtual address space for the dynamic percpu area. 2103 */ 2104 pa = allocpages(DPCPU_SIZE >> PAGE_SHIFT); 2105 dpcpu = (void *)PHYS_TO_DMAP(pa); 2106 dpcpu_init(dpcpu, curcpu); 2107 2108 crashdumpmap = (caddr_t)virtual_avail; 2109 virtual_avail += MAXDUMPPGS * PAGE_SIZE; 2110 2111 /* 2112 * Reserve some special page table entries/VA space for temporary 2113 * mapping of pages. 2114 */ 2115 } 2116 2117 static void 2118 mmu_parttab_init(void) 2119 { 2120 uint64_t ptcr; 2121 2122 isa3_parttab = (struct pate *)PHYS_TO_DMAP(parttab_phys); 2123 2124 if (bootverbose) 2125 printf("%s parttab: %p\n", __func__, isa3_parttab); 2126 ptcr = parttab_phys | (PARTTAB_SIZE_SHIFT-12); 2127 if (bootverbose) 2128 printf("setting ptcr %lx\n", ptcr); 2129 mtspr(SPR_PTCR, ptcr); 2130 } 2131 2132 static void 2133 mmu_parttab_update(uint64_t lpid, uint64_t pagetab, uint64_t proctab) 2134 { 2135 uint64_t prev; 2136 2137 if (bootverbose) 2138 printf("%s isa3_parttab %p lpid %lx pagetab %lx proctab %lx\n", __func__, isa3_parttab, 2139 lpid, pagetab, proctab); 2140 prev = be64toh(isa3_parttab[lpid].pagetab); 2141 isa3_parttab[lpid].pagetab = htobe64(pagetab); 2142 isa3_parttab[lpid].proctab = htobe64(proctab); 2143 2144 if (prev & PARTTAB_HR) { 2145 __asm __volatile(PPC_TLBIE_5(%0,%1,2,0,1) : : 2146 "r" (TLBIEL_INVAL_SET_LPID), "r" (lpid)); 2147 __asm __volatile(PPC_TLBIE_5(%0,%1,2,1,1) : : 2148 "r" (TLBIEL_INVAL_SET_LPID), "r" (lpid)); 2149 } else { 2150 __asm __volatile(PPC_TLBIE_5(%0,%1,2,0,0) : : 2151 "r" (TLBIEL_INVAL_SET_LPID), "r" (lpid)); 2152 } 2153 ttusync(); 2154 } 2155 2156 static void 2157 mmu_radix_parttab_init(void) 2158 { 2159 uint64_t pagetab; 2160 2161 mmu_parttab_init(); 2162 pagetab = RTS_SIZE | DMAP_TO_PHYS((vm_offset_t)kernel_pmap->pm_pml1) | \ 2163 RADIX_PGD_INDEX_SHIFT | PARTTAB_HR; 2164 mmu_parttab_update(0, pagetab, 0); 2165 } 2166 2167 static void 2168 mmu_radix_proctab_register(vm_paddr_t proctabpa, uint64_t table_size) 2169 { 2170 uint64_t pagetab, proctab; 2171 2172 pagetab = be64toh(isa3_parttab[0].pagetab); 2173 proctab = proctabpa | table_size | PARTTAB_GR; 2174 mmu_parttab_update(0, pagetab, proctab); 2175 } 2176 2177 static void 2178 mmu_radix_proctab_init(void) 2179 { 2180 2181 isa3_base_pid = 1; 2182 2183 isa3_proctab = (void*)PHYS_TO_DMAP(proctab0pa); 2184 isa3_proctab->proctab0 = 2185 htobe64(RTS_SIZE | DMAP_TO_PHYS((vm_offset_t)kernel_pmap->pm_pml1) | 2186 RADIX_PGD_INDEX_SHIFT); 2187 2188 if (powernv_enabled) { 2189 mmu_radix_proctab_register(proctab0pa, PROCTAB_SIZE_SHIFT - 12); 2190 __asm __volatile("ptesync" : : : "memory"); 2191 __asm __volatile(PPC_TLBIE_5(%0,%1,2,1,1) : : 2192 "r" (TLBIEL_INVAL_SET_LPID), "r" (0)); 2193 __asm __volatile("eieio; tlbsync; ptesync" : : : "memory"); 2194 #ifdef PSERIES 2195 } else { 2196 int64_t rc; 2197 2198 rc = phyp_hcall(H_REGISTER_PROC_TBL, 2199 PROC_TABLE_NEW | PROC_TABLE_RADIX | PROC_TABLE_GTSE, 2200 proctab0pa, 0, PROCTAB_SIZE_SHIFT - 12); 2201 if (rc != H_SUCCESS) 2202 panic("mmu_radix_proctab_init: " 2203 "failed to register process table: rc=%jd", 2204 (intmax_t)rc); 2205 #endif 2206 } 2207 2208 if (bootverbose) 2209 printf("process table %p and kernel radix PDE: %p\n", 2210 isa3_proctab, kernel_pmap->pm_pml1); 2211 mtmsr(mfmsr() | PSL_DR ); 2212 mtmsr(mfmsr() & ~PSL_DR); 2213 kernel_pmap->pm_pid = isa3_base_pid; 2214 isa3_base_pid++; 2215 } 2216 2217 void 2218 mmu_radix_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, 2219 int advice) 2220 { 2221 struct rwlock *lock; 2222 pml1_entry_t *l1e; 2223 pml2_entry_t *l2e; 2224 pml3_entry_t oldl3e, *l3e; 2225 pt_entry_t *pte; 2226 vm_offset_t va, va_next; 2227 vm_page_t m; 2228 bool anychanged; 2229 2230 if (advice != MADV_DONTNEED && advice != MADV_FREE) 2231 return; 2232 anychanged = false; 2233 PMAP_LOCK(pmap); 2234 for (; sva < eva; sva = va_next) { 2235 l1e = pmap_pml1e(pmap, sva); 2236 if ((be64toh(*l1e) & PG_V) == 0) { 2237 va_next = (sva + L1_PAGE_SIZE) & ~L1_PAGE_MASK; 2238 if (va_next < sva) 2239 va_next = eva; 2240 continue; 2241 } 2242 l2e = pmap_l1e_to_l2e(l1e, sva); 2243 if ((be64toh(*l2e) & PG_V) == 0) { 2244 va_next = (sva + L2_PAGE_SIZE) & ~L2_PAGE_MASK; 2245 if (va_next < sva) 2246 va_next = eva; 2247 continue; 2248 } 2249 va_next = (sva + L3_PAGE_SIZE) & ~L3_PAGE_MASK; 2250 if (va_next < sva) 2251 va_next = eva; 2252 l3e = pmap_l2e_to_l3e(l2e, sva); 2253 oldl3e = be64toh(*l3e); 2254 if ((oldl3e & PG_V) == 0) 2255 continue; 2256 else if ((oldl3e & RPTE_LEAF) != 0) { 2257 if ((oldl3e & PG_MANAGED) == 0) 2258 continue; 2259 lock = NULL; 2260 if (!pmap_demote_l3e_locked(pmap, l3e, sva, &lock)) { 2261 if (lock != NULL) 2262 rw_wunlock(lock); 2263 2264 /* 2265 * The large page mapping was destroyed. 2266 */ 2267 continue; 2268 } 2269 2270 /* 2271 * Unless the page mappings are wired, remove the 2272 * mapping to a single page so that a subsequent 2273 * access may repromote. Choosing the last page 2274 * within the address range [sva, min(va_next, eva)) 2275 * generally results in more repromotions. Since the 2276 * underlying page table page is fully populated, this 2277 * removal never frees a page table page. 2278 */ 2279 if ((oldl3e & PG_W) == 0) { 2280 va = eva; 2281 if (va > va_next) 2282 va = va_next; 2283 va -= PAGE_SIZE; 2284 KASSERT(va >= sva, 2285 ("mmu_radix_advise: no address gap")); 2286 pte = pmap_l3e_to_pte(l3e, va); 2287 KASSERT((be64toh(*pte) & PG_V) != 0, 2288 ("pmap_advise: invalid PTE")); 2289 pmap_remove_pte(pmap, pte, va, be64toh(*l3e), NULL, 2290 &lock); 2291 anychanged = true; 2292 } 2293 if (lock != NULL) 2294 rw_wunlock(lock); 2295 } 2296 if (va_next > eva) 2297 va_next = eva; 2298 va = va_next; 2299 for (pte = pmap_l3e_to_pte(l3e, sva); sva != va_next; 2300 pte++, sva += PAGE_SIZE) { 2301 MPASS(pte == pmap_pte(pmap, sva)); 2302 2303 if ((be64toh(*pte) & (PG_MANAGED | PG_V)) != (PG_MANAGED | PG_V)) 2304 goto maybe_invlrng; 2305 else if ((be64toh(*pte) & (PG_M | PG_RW)) == (PG_M | PG_RW)) { 2306 if (advice == MADV_DONTNEED) { 2307 /* 2308 * Future calls to pmap_is_modified() 2309 * can be avoided by making the page 2310 * dirty now. 2311 */ 2312 m = PHYS_TO_VM_PAGE(be64toh(*pte) & PG_FRAME); 2313 vm_page_dirty(m); 2314 } 2315 atomic_clear_long(pte, htobe64(PG_M | PG_A)); 2316 } else if ((be64toh(*pte) & PG_A) != 0) 2317 atomic_clear_long(pte, htobe64(PG_A)); 2318 else 2319 goto maybe_invlrng; 2320 anychanged = true; 2321 continue; 2322 maybe_invlrng: 2323 if (va != va_next) { 2324 anychanged = true; 2325 va = va_next; 2326 } 2327 } 2328 if (va != va_next) 2329 anychanged = true; 2330 } 2331 if (anychanged) 2332 pmap_invalidate_all(pmap); 2333 PMAP_UNLOCK(pmap); 2334 } 2335 2336 /* 2337 * Routines used in machine-dependent code 2338 */ 2339 static void 2340 mmu_radix_bootstrap(vm_offset_t start, vm_offset_t end) 2341 { 2342 uint64_t lpcr; 2343 2344 if (bootverbose) 2345 printf("%s\n", __func__); 2346 hw_direct_map = 1; 2347 powernv_enabled = (mfmsr() & PSL_HV) ? 1 : 0; 2348 mmu_radix_early_bootstrap(start, end); 2349 if (bootverbose) 2350 printf("early bootstrap complete\n"); 2351 if (powernv_enabled) { 2352 lpcr = mfspr(SPR_LPCR); 2353 mtspr(SPR_LPCR, lpcr | LPCR_UPRT | LPCR_HR); 2354 mmu_radix_parttab_init(); 2355 mmu_radix_init_amor(); 2356 if (bootverbose) 2357 printf("powernv init complete\n"); 2358 } 2359 mmu_radix_init_iamr(); 2360 mmu_radix_proctab_init(); 2361 mmu_radix_pid_set(kernel_pmap); 2362 if (powernv_enabled) 2363 mmu_radix_tlbiel_flush(TLB_INVAL_SCOPE_GLOBAL); 2364 else 2365 mmu_radix_tlbiel_flush(TLB_INVAL_SCOPE_LPID); 2366 2367 mmu_radix_late_bootstrap(start, end); 2368 numa_mem_regions(&numa_pregions, &numa_pregions_sz); 2369 if (bootverbose) 2370 printf("%s done\n", __func__); 2371 pmap_bootstrapped = 1; 2372 dmaplimit = roundup2(powerpc_ptob(Maxmem), L2_PAGE_SIZE); 2373 PCPU_SET(flags, PCPU_GET(flags) | PC_FLAG_NOSRS); 2374 } 2375 2376 static void 2377 mmu_radix_cpu_bootstrap(int ap) 2378 { 2379 uint64_t lpcr; 2380 uint64_t ptcr; 2381 2382 if (powernv_enabled) { 2383 lpcr = mfspr(SPR_LPCR); 2384 mtspr(SPR_LPCR, lpcr | LPCR_UPRT | LPCR_HR); 2385 2386 ptcr = parttab_phys | (PARTTAB_SIZE_SHIFT-12); 2387 mtspr(SPR_PTCR, ptcr); 2388 mmu_radix_init_amor(); 2389 } 2390 mmu_radix_init_iamr(); 2391 mmu_radix_pid_set(kernel_pmap); 2392 if (powernv_enabled) 2393 mmu_radix_tlbiel_flush(TLB_INVAL_SCOPE_GLOBAL); 2394 else 2395 mmu_radix_tlbiel_flush(TLB_INVAL_SCOPE_LPID); 2396 } 2397 2398 static SYSCTL_NODE(_vm_pmap, OID_AUTO, l3e, CTLFLAG_RD, 0, 2399 "2MB page mapping counters"); 2400 2401 static COUNTER_U64_DEFINE_EARLY(pmap_l3e_demotions); 2402 SYSCTL_COUNTER_U64(_vm_pmap_l3e, OID_AUTO, demotions, CTLFLAG_RD, 2403 &pmap_l3e_demotions, "2MB page demotions"); 2404 2405 static COUNTER_U64_DEFINE_EARLY(pmap_l3e_mappings); 2406 SYSCTL_COUNTER_U64(_vm_pmap_l3e, OID_AUTO, mappings, CTLFLAG_RD, 2407 &pmap_l3e_mappings, "2MB page mappings"); 2408 2409 static COUNTER_U64_DEFINE_EARLY(pmap_l3e_p_failures); 2410 SYSCTL_COUNTER_U64(_vm_pmap_l3e, OID_AUTO, p_failures, CTLFLAG_RD, 2411 &pmap_l3e_p_failures, "2MB page promotion failures"); 2412 2413 static COUNTER_U64_DEFINE_EARLY(pmap_l3e_promotions); 2414 SYSCTL_COUNTER_U64(_vm_pmap_l3e, OID_AUTO, promotions, CTLFLAG_RD, 2415 &pmap_l3e_promotions, "2MB page promotions"); 2416 2417 static SYSCTL_NODE(_vm_pmap, OID_AUTO, l2e, CTLFLAG_RD, 0, 2418 "1GB page mapping counters"); 2419 2420 static COUNTER_U64_DEFINE_EARLY(pmap_l2e_demotions); 2421 SYSCTL_COUNTER_U64(_vm_pmap_l2e, OID_AUTO, demotions, CTLFLAG_RD, 2422 &pmap_l2e_demotions, "1GB page demotions"); 2423 2424 void 2425 mmu_radix_clear_modify(vm_page_t m) 2426 { 2427 struct md_page *pvh; 2428 pmap_t pmap; 2429 pv_entry_t next_pv, pv; 2430 pml3_entry_t oldl3e, *l3e; 2431 pt_entry_t oldpte, *pte; 2432 struct rwlock *lock; 2433 vm_offset_t va; 2434 int md_gen, pvh_gen; 2435 2436 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 2437 ("pmap_clear_modify: page %p is not managed", m)); 2438 vm_page_assert_busied(m); 2439 CTR2(KTR_PMAP, "%s(%p)", __func__, m); 2440 2441 /* 2442 * If the page is not PGA_WRITEABLE, then no PTEs can have PG_M set. 2443 * If the object containing the page is locked and the page is not 2444 * exclusive busied, then PGA_WRITEABLE cannot be concurrently set. 2445 */ 2446 if ((m->a.flags & PGA_WRITEABLE) == 0) 2447 return; 2448 pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy : 2449 pa_to_pvh(VM_PAGE_TO_PHYS(m)); 2450 lock = VM_PAGE_TO_PV_LIST_LOCK(m); 2451 rw_wlock(lock); 2452 restart: 2453 TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_link, next_pv) { 2454 pmap = PV_PMAP(pv); 2455 if (!PMAP_TRYLOCK(pmap)) { 2456 pvh_gen = pvh->pv_gen; 2457 rw_wunlock(lock); 2458 PMAP_LOCK(pmap); 2459 rw_wlock(lock); 2460 if (pvh_gen != pvh->pv_gen) { 2461 PMAP_UNLOCK(pmap); 2462 goto restart; 2463 } 2464 } 2465 va = pv->pv_va; 2466 l3e = pmap_pml3e(pmap, va); 2467 oldl3e = be64toh(*l3e); 2468 if ((oldl3e & PG_RW) != 0 && 2469 pmap_demote_l3e_locked(pmap, l3e, va, &lock) && 2470 (oldl3e & PG_W) == 0) { 2471 /* 2472 * Write protect the mapping to a 2473 * single page so that a subsequent 2474 * write access may repromote. 2475 */ 2476 va += VM_PAGE_TO_PHYS(m) - (oldl3e & 2477 PG_PS_FRAME); 2478 pte = pmap_l3e_to_pte(l3e, va); 2479 oldpte = be64toh(*pte); 2480 while (!atomic_cmpset_long(pte, 2481 htobe64(oldpte), 2482 htobe64((oldpte | RPTE_EAA_R) & ~(PG_M | PG_RW)))) 2483 oldpte = be64toh(*pte); 2484 vm_page_dirty(m); 2485 pmap_invalidate_page(pmap, va); 2486 } 2487 PMAP_UNLOCK(pmap); 2488 } 2489 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) { 2490 pmap = PV_PMAP(pv); 2491 if (!PMAP_TRYLOCK(pmap)) { 2492 md_gen = m->md.pv_gen; 2493 pvh_gen = pvh->pv_gen; 2494 rw_wunlock(lock); 2495 PMAP_LOCK(pmap); 2496 rw_wlock(lock); 2497 if (pvh_gen != pvh->pv_gen || md_gen != m->md.pv_gen) { 2498 PMAP_UNLOCK(pmap); 2499 goto restart; 2500 } 2501 } 2502 l3e = pmap_pml3e(pmap, pv->pv_va); 2503 KASSERT((be64toh(*l3e) & RPTE_LEAF) == 0, ("pmap_clear_modify: found" 2504 " a 2mpage in page %p's pv list", m)); 2505 pte = pmap_l3e_to_pte(l3e, pv->pv_va); 2506 if ((be64toh(*pte) & (PG_M | PG_RW)) == (PG_M | PG_RW)) { 2507 atomic_clear_long(pte, htobe64(PG_M)); 2508 pmap_invalidate_page(pmap, pv->pv_va); 2509 } 2510 PMAP_UNLOCK(pmap); 2511 } 2512 rw_wunlock(lock); 2513 } 2514 2515 void 2516 mmu_radix_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, 2517 vm_size_t len, vm_offset_t src_addr) 2518 { 2519 struct rwlock *lock; 2520 struct spglist free; 2521 vm_offset_t addr; 2522 vm_offset_t end_addr = src_addr + len; 2523 vm_offset_t va_next; 2524 vm_page_t dst_pdpg, dstmpte, srcmpte; 2525 bool invalidate_all; 2526 2527 CTR6(KTR_PMAP, 2528 "%s(dst_pmap=%p, src_pmap=%p, dst_addr=%lx, len=%lu, src_addr=%lx)\n", 2529 __func__, dst_pmap, src_pmap, dst_addr, len, src_addr); 2530 2531 if (dst_addr != src_addr) 2532 return; 2533 lock = NULL; 2534 invalidate_all = false; 2535 if (dst_pmap < src_pmap) { 2536 PMAP_LOCK(dst_pmap); 2537 PMAP_LOCK(src_pmap); 2538 } else { 2539 PMAP_LOCK(src_pmap); 2540 PMAP_LOCK(dst_pmap); 2541 } 2542 2543 for (addr = src_addr; addr < end_addr; addr = va_next) { 2544 pml1_entry_t *l1e; 2545 pml2_entry_t *l2e; 2546 pml3_entry_t srcptepaddr, *l3e; 2547 pt_entry_t *src_pte, *dst_pte; 2548 2549 l1e = pmap_pml1e(src_pmap, addr); 2550 if ((be64toh(*l1e) & PG_V) == 0) { 2551 va_next = (addr + L1_PAGE_SIZE) & ~L1_PAGE_MASK; 2552 if (va_next < addr) 2553 va_next = end_addr; 2554 continue; 2555 } 2556 2557 l2e = pmap_l1e_to_l2e(l1e, addr); 2558 if ((be64toh(*l2e) & PG_V) == 0) { 2559 va_next = (addr + L2_PAGE_SIZE) & ~L2_PAGE_MASK; 2560 if (va_next < addr) 2561 va_next = end_addr; 2562 continue; 2563 } 2564 2565 va_next = (addr + L3_PAGE_SIZE) & ~L3_PAGE_MASK; 2566 if (va_next < addr) 2567 va_next = end_addr; 2568 2569 l3e = pmap_l2e_to_l3e(l2e, addr); 2570 srcptepaddr = be64toh(*l3e); 2571 if (srcptepaddr == 0) 2572 continue; 2573 2574 if (srcptepaddr & RPTE_LEAF) { 2575 if ((addr & L3_PAGE_MASK) != 0 || 2576 addr + L3_PAGE_SIZE > end_addr) 2577 continue; 2578 dst_pdpg = pmap_allocl3e(dst_pmap, addr, NULL); 2579 if (dst_pdpg == NULL) 2580 break; 2581 l3e = (pml3_entry_t *) 2582 PHYS_TO_DMAP(VM_PAGE_TO_PHYS(dst_pdpg)); 2583 l3e = &l3e[pmap_pml3e_index(addr)]; 2584 if (be64toh(*l3e) == 0 && ((srcptepaddr & PG_MANAGED) == 0 || 2585 pmap_pv_insert_l3e(dst_pmap, addr, srcptepaddr, 2586 PMAP_ENTER_NORECLAIM, &lock))) { 2587 *l3e = htobe64(srcptepaddr & ~PG_W); 2588 pmap_resident_count_inc(dst_pmap, 2589 L3_PAGE_SIZE / PAGE_SIZE); 2590 counter_u64_add(pmap_l3e_mappings, 1); 2591 } else 2592 dst_pdpg->ref_count--; 2593 continue; 2594 } 2595 2596 srcptepaddr &= PG_FRAME; 2597 srcmpte = PHYS_TO_VM_PAGE(srcptepaddr); 2598 KASSERT(srcmpte->ref_count > 0, 2599 ("pmap_copy: source page table page is unused")); 2600 2601 if (va_next > end_addr) 2602 va_next = end_addr; 2603 2604 src_pte = (pt_entry_t *)PHYS_TO_DMAP(srcptepaddr); 2605 src_pte = &src_pte[pmap_pte_index(addr)]; 2606 dstmpte = NULL; 2607 while (addr < va_next) { 2608 pt_entry_t ptetemp; 2609 ptetemp = be64toh(*src_pte); 2610 /* 2611 * we only virtual copy managed pages 2612 */ 2613 if ((ptetemp & PG_MANAGED) != 0) { 2614 if (dstmpte != NULL && 2615 dstmpte->pindex == pmap_l3e_pindex(addr)) 2616 dstmpte->ref_count++; 2617 else if ((dstmpte = pmap_allocpte(dst_pmap, 2618 addr, NULL)) == NULL) 2619 goto out; 2620 dst_pte = (pt_entry_t *) 2621 PHYS_TO_DMAP(VM_PAGE_TO_PHYS(dstmpte)); 2622 dst_pte = &dst_pte[pmap_pte_index(addr)]; 2623 if (be64toh(*dst_pte) == 0 && 2624 pmap_try_insert_pv_entry(dst_pmap, addr, 2625 PHYS_TO_VM_PAGE(ptetemp & PG_FRAME), 2626 &lock)) { 2627 /* 2628 * Clear the wired, modified, and 2629 * accessed (referenced) bits 2630 * during the copy. 2631 */ 2632 *dst_pte = htobe64(ptetemp & ~(PG_W | PG_M | 2633 PG_A)); 2634 pmap_resident_count_inc(dst_pmap, 1); 2635 } else { 2636 SLIST_INIT(&free); 2637 if (pmap_unwire_ptp(dst_pmap, addr, 2638 dstmpte, &free)) { 2639 /* 2640 * Although "addr" is not 2641 * mapped, paging-structure 2642 * caches could nonetheless 2643 * have entries that refer to 2644 * the freed page table pages. 2645 * Invalidate those entries. 2646 */ 2647 invalidate_all = true; 2648 vm_page_free_pages_toq(&free, 2649 true); 2650 } 2651 goto out; 2652 } 2653 if (dstmpte->ref_count >= srcmpte->ref_count) 2654 break; 2655 } 2656 addr += PAGE_SIZE; 2657 if (__predict_false((addr & L3_PAGE_MASK) == 0)) 2658 src_pte = pmap_pte(src_pmap, addr); 2659 else 2660 src_pte++; 2661 } 2662 } 2663 out: 2664 if (invalidate_all) 2665 pmap_invalidate_all(dst_pmap); 2666 if (lock != NULL) 2667 rw_wunlock(lock); 2668 PMAP_UNLOCK(src_pmap); 2669 PMAP_UNLOCK(dst_pmap); 2670 } 2671 2672 static void 2673 mmu_radix_copy_page(vm_page_t msrc, vm_page_t mdst) 2674 { 2675 vm_offset_t src = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(msrc)); 2676 vm_offset_t dst = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mdst)); 2677 2678 CTR3(KTR_PMAP, "%s(%p, %p)", __func__, src, dst); 2679 /* 2680 * XXX slow 2681 */ 2682 bcopy((void *)src, (void *)dst, PAGE_SIZE); 2683 } 2684 2685 static void 2686 mmu_radix_copy_pages(vm_page_t ma[], vm_offset_t a_offset, vm_page_t mb[], 2687 vm_offset_t b_offset, int xfersize) 2688 { 2689 void *a_cp, *b_cp; 2690 vm_offset_t a_pg_offset, b_pg_offset; 2691 int cnt; 2692 2693 CTR6(KTR_PMAP, "%s(%p, %#x, %p, %#x, %#x)", __func__, ma, 2694 a_offset, mb, b_offset, xfersize); 2695 2696 while (xfersize > 0) { 2697 a_pg_offset = a_offset & PAGE_MASK; 2698 cnt = min(xfersize, PAGE_SIZE - a_pg_offset); 2699 a_cp = (char *)(uintptr_t)PHYS_TO_DMAP( 2700 VM_PAGE_TO_PHYS(ma[a_offset >> PAGE_SHIFT])) + 2701 a_pg_offset; 2702 b_pg_offset = b_offset & PAGE_MASK; 2703 cnt = min(cnt, PAGE_SIZE - b_pg_offset); 2704 b_cp = (char *)(uintptr_t)PHYS_TO_DMAP( 2705 VM_PAGE_TO_PHYS(mb[b_offset >> PAGE_SHIFT])) + 2706 b_pg_offset; 2707 bcopy(a_cp, b_cp, cnt); 2708 a_offset += cnt; 2709 b_offset += cnt; 2710 xfersize -= cnt; 2711 } 2712 } 2713 2714 #if VM_NRESERVLEVEL > 0 2715 /* 2716 * Tries to promote the 512, contiguous 4KB page mappings that are within a 2717 * single page table page (PTP) to a single 2MB page mapping. For promotion 2718 * to occur, two conditions must be met: (1) the 4KB page mappings must map 2719 * aligned, contiguous physical memory and (2) the 4KB page mappings must have 2720 * identical characteristics. 2721 */ 2722 static int 2723 pmap_promote_l3e(pmap_t pmap, pml3_entry_t *pde, vm_offset_t va, 2724 struct rwlock **lockp) 2725 { 2726 pml3_entry_t newpde; 2727 pt_entry_t *firstpte, oldpte, pa, *pte; 2728 vm_page_t mpte; 2729 2730 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 2731 2732 /* 2733 * Examine the first PTE in the specified PTP. Abort if this PTE is 2734 * either invalid, unused, or does not map the first 4KB physical page 2735 * within a 2MB page. 2736 */ 2737 firstpte = (pt_entry_t *)PHYS_TO_DMAP(be64toh(*pde) & PG_FRAME); 2738 setpde: 2739 newpde = be64toh(*firstpte); 2740 if ((newpde & ((PG_FRAME & L3_PAGE_MASK) | PG_A | PG_V)) != (PG_A | PG_V)) { 2741 CTR2(KTR_PMAP, "pmap_promote_l3e: failure for va %#lx" 2742 " in pmap %p", va, pmap); 2743 goto fail; 2744 } 2745 if ((newpde & (PG_M | PG_RW)) == PG_RW) { 2746 /* 2747 * When PG_M is already clear, PG_RW can be cleared without 2748 * a TLB invalidation. 2749 */ 2750 if (!atomic_cmpset_long(firstpte, htobe64(newpde), htobe64((newpde | RPTE_EAA_R) & ~RPTE_EAA_W))) 2751 goto setpde; 2752 newpde &= ~RPTE_EAA_W; 2753 } 2754 2755 /* 2756 * Examine each of the other PTEs in the specified PTP. Abort if this 2757 * PTE maps an unexpected 4KB physical page or does not have identical 2758 * characteristics to the first PTE. 2759 */ 2760 pa = (newpde & (PG_PS_FRAME | PG_A | PG_V)) + L3_PAGE_SIZE - PAGE_SIZE; 2761 for (pte = firstpte + NPTEPG - 1; pte > firstpte; pte--) { 2762 setpte: 2763 oldpte = be64toh(*pte); 2764 if ((oldpte & (PG_FRAME | PG_A | PG_V)) != pa) { 2765 CTR2(KTR_PMAP, "pmap_promote_l3e: failure for va %#lx" 2766 " in pmap %p", va, pmap); 2767 goto fail; 2768 } 2769 if ((oldpte & (PG_M | PG_RW)) == PG_RW) { 2770 /* 2771 * When PG_M is already clear, PG_RW can be cleared 2772 * without a TLB invalidation. 2773 */ 2774 if (!atomic_cmpset_long(pte, htobe64(oldpte), htobe64((oldpte | RPTE_EAA_R) & ~RPTE_EAA_W))) 2775 goto setpte; 2776 oldpte &= ~RPTE_EAA_W; 2777 CTR2(KTR_PMAP, "pmap_promote_l3e: protect for va %#lx" 2778 " in pmap %p", (oldpte & PG_FRAME & L3_PAGE_MASK) | 2779 (va & ~L3_PAGE_MASK), pmap); 2780 } 2781 if ((oldpte & PG_PTE_PROMOTE) != (newpde & PG_PTE_PROMOTE)) { 2782 CTR2(KTR_PMAP, "pmap_promote_l3e: failure for va %#lx" 2783 " in pmap %p", va, pmap); 2784 goto fail; 2785 } 2786 pa -= PAGE_SIZE; 2787 } 2788 2789 /* 2790 * Save the page table page in its current state until the PDE 2791 * mapping the superpage is demoted by pmap_demote_pde() or 2792 * destroyed by pmap_remove_pde(). 2793 */ 2794 mpte = PHYS_TO_VM_PAGE(be64toh(*pde) & PG_FRAME); 2795 KASSERT(mpte >= vm_page_array && 2796 mpte < &vm_page_array[vm_page_array_size], 2797 ("pmap_promote_l3e: page table page is out of range")); 2798 KASSERT(mpte->pindex == pmap_l3e_pindex(va), 2799 ("pmap_promote_l3e: page table page's pindex is wrong")); 2800 if (pmap_insert_pt_page(pmap, mpte)) { 2801 CTR2(KTR_PMAP, 2802 "pmap_promote_l3e: failure for va %#lx in pmap %p", va, 2803 pmap); 2804 goto fail; 2805 } 2806 2807 /* 2808 * Promote the pv entries. 2809 */ 2810 if ((newpde & PG_MANAGED) != 0) 2811 pmap_pv_promote_l3e(pmap, va, newpde & PG_PS_FRAME, lockp); 2812 2813 pte_store(pde, PG_PROMOTED | newpde); 2814 ptesync(); 2815 counter_u64_add(pmap_l3e_promotions, 1); 2816 CTR2(KTR_PMAP, "pmap_promote_l3e: success for va %#lx" 2817 " in pmap %p", va, pmap); 2818 return (0); 2819 fail: 2820 counter_u64_add(pmap_l3e_p_failures, 1); 2821 return (KERN_FAILURE); 2822 } 2823 #endif /* VM_NRESERVLEVEL > 0 */ 2824 2825 int 2826 mmu_radix_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, 2827 vm_prot_t prot, u_int flags, int8_t psind) 2828 { 2829 struct rwlock *lock; 2830 pml3_entry_t *l3e; 2831 pt_entry_t *pte; 2832 pt_entry_t newpte, origpte; 2833 pv_entry_t pv; 2834 vm_paddr_t opa, pa; 2835 vm_page_t mpte, om; 2836 int rv, retrycount; 2837 bool nosleep, invalidate_all, invalidate_page; 2838 2839 va = trunc_page(va); 2840 retrycount = 0; 2841 invalidate_page = invalidate_all = false; 2842 CTR6(KTR_PMAP, "pmap_enter(%p, %#lx, %p, %#x, %#x, %d)", pmap, va, 2843 m, prot, flags, psind); 2844 KASSERT(va <= VM_MAX_KERNEL_ADDRESS, ("pmap_enter: toobig")); 2845 KASSERT((m->oflags & VPO_UNMANAGED) != 0 || !VA_IS_CLEANMAP(va), 2846 ("pmap_enter: managed mapping within the clean submap")); 2847 if ((m->oflags & VPO_UNMANAGED) == 0) 2848 VM_PAGE_OBJECT_BUSY_ASSERT(m); 2849 2850 KASSERT((flags & PMAP_ENTER_RESERVED) == 0, 2851 ("pmap_enter: flags %u has reserved bits set", flags)); 2852 pa = VM_PAGE_TO_PHYS(m); 2853 newpte = (pt_entry_t)(pa | PG_A | PG_V | RPTE_LEAF); 2854 if ((flags & VM_PROT_WRITE) != 0) 2855 newpte |= PG_M; 2856 if ((flags & VM_PROT_READ) != 0) 2857 newpte |= PG_A; 2858 if (prot & VM_PROT_READ) 2859 newpte |= RPTE_EAA_R; 2860 if ((prot & VM_PROT_WRITE) != 0) 2861 newpte |= RPTE_EAA_W; 2862 KASSERT((newpte & (PG_M | PG_RW)) != PG_M, 2863 ("pmap_enter: flags includes VM_PROT_WRITE but prot doesn't")); 2864 2865 if (prot & VM_PROT_EXECUTE) 2866 newpte |= PG_X; 2867 if ((flags & PMAP_ENTER_WIRED) != 0) 2868 newpte |= PG_W; 2869 if (va >= DMAP_MIN_ADDRESS) 2870 newpte |= RPTE_EAA_P; 2871 newpte |= pmap_cache_bits(m->md.mdpg_cache_attrs); 2872 /* 2873 * Set modified bit gratuitously for writeable mappings if 2874 * the page is unmanaged. We do not want to take a fault 2875 * to do the dirty bit accounting for these mappings. 2876 */ 2877 if ((m->oflags & VPO_UNMANAGED) != 0) { 2878 if ((newpte & PG_RW) != 0) 2879 newpte |= PG_M; 2880 } else 2881 newpte |= PG_MANAGED; 2882 2883 lock = NULL; 2884 PMAP_LOCK(pmap); 2885 if (psind == 1) { 2886 /* Assert the required virtual and physical alignment. */ 2887 KASSERT((va & L3_PAGE_MASK) == 0, ("pmap_enter: va unaligned")); 2888 KASSERT(m->psind > 0, ("pmap_enter: m->psind < psind")); 2889 rv = pmap_enter_l3e(pmap, va, newpte | RPTE_LEAF, flags, m, &lock); 2890 goto out; 2891 } 2892 mpte = NULL; 2893 2894 /* 2895 * In the case that a page table page is not 2896 * resident, we are creating it here. 2897 */ 2898 retry: 2899 l3e = pmap_pml3e(pmap, va); 2900 if (l3e != NULL && (be64toh(*l3e) & PG_V) != 0 && ((be64toh(*l3e) & RPTE_LEAF) == 0 || 2901 pmap_demote_l3e_locked(pmap, l3e, va, &lock))) { 2902 pte = pmap_l3e_to_pte(l3e, va); 2903 if (va < VM_MAXUSER_ADDRESS && mpte == NULL) { 2904 mpte = PHYS_TO_VM_PAGE(be64toh(*l3e) & PG_FRAME); 2905 mpte->ref_count++; 2906 } 2907 } else if (va < VM_MAXUSER_ADDRESS) { 2908 /* 2909 * Here if the pte page isn't mapped, or if it has been 2910 * deallocated. 2911 */ 2912 nosleep = (flags & PMAP_ENTER_NOSLEEP) != 0; 2913 mpte = _pmap_allocpte(pmap, pmap_l3e_pindex(va), 2914 nosleep ? NULL : &lock); 2915 if (mpte == NULL && nosleep) { 2916 rv = KERN_RESOURCE_SHORTAGE; 2917 goto out; 2918 } 2919 if (__predict_false(retrycount++ == 6)) 2920 panic("too many retries"); 2921 invalidate_all = true; 2922 goto retry; 2923 } else 2924 panic("pmap_enter: invalid page directory va=%#lx", va); 2925 2926 origpte = be64toh(*pte); 2927 pv = NULL; 2928 2929 /* 2930 * Is the specified virtual address already mapped? 2931 */ 2932 if ((origpte & PG_V) != 0) { 2933 #ifdef INVARIANTS 2934 if (VERBOSE_PMAP || pmap_logging) { 2935 printf("cow fault pmap_enter(%p, %#lx, %p, %#x, %x, %d) --" 2936 " asid=%lu curpid=%d name=%s origpte0x%lx\n", 2937 pmap, va, m, prot, flags, psind, pmap->pm_pid, 2938 curproc->p_pid, curproc->p_comm, origpte); 2939 #ifdef DDB 2940 pmap_pte_walk(pmap->pm_pml1, va); 2941 #endif 2942 } 2943 #endif 2944 /* 2945 * Wiring change, just update stats. We don't worry about 2946 * wiring PT pages as they remain resident as long as there 2947 * are valid mappings in them. Hence, if a user page is wired, 2948 * the PT page will be also. 2949 */ 2950 if ((newpte & PG_W) != 0 && (origpte & PG_W) == 0) 2951 pmap->pm_stats.wired_count++; 2952 else if ((newpte & PG_W) == 0 && (origpte & PG_W) != 0) 2953 pmap->pm_stats.wired_count--; 2954 2955 /* 2956 * Remove the extra PT page reference. 2957 */ 2958 if (mpte != NULL) { 2959 mpte->ref_count--; 2960 KASSERT(mpte->ref_count > 0, 2961 ("pmap_enter: missing reference to page table page," 2962 " va: 0x%lx", va)); 2963 } 2964 2965 /* 2966 * Has the physical page changed? 2967 */ 2968 opa = origpte & PG_FRAME; 2969 if (opa == pa) { 2970 /* 2971 * No, might be a protection or wiring change. 2972 */ 2973 if ((origpte & PG_MANAGED) != 0 && 2974 (newpte & PG_RW) != 0) 2975 vm_page_aflag_set(m, PGA_WRITEABLE); 2976 if (((origpte ^ newpte) & ~(PG_M | PG_A)) == 0) { 2977 if ((newpte & (PG_A|PG_M)) != (origpte & (PG_A|PG_M))) { 2978 if (!atomic_cmpset_long(pte, htobe64(origpte), htobe64(newpte))) 2979 goto retry; 2980 if ((newpte & PG_M) != (origpte & PG_M)) 2981 vm_page_dirty(m); 2982 if ((newpte & PG_A) != (origpte & PG_A)) 2983 vm_page_aflag_set(m, PGA_REFERENCED); 2984 ptesync(); 2985 } else 2986 invalidate_all = true; 2987 if (((origpte ^ newpte) & ~(PG_M | PG_A)) == 0) 2988 goto unchanged; 2989 } 2990 goto validate; 2991 } 2992 2993 /* 2994 * The physical page has changed. Temporarily invalidate 2995 * the mapping. This ensures that all threads sharing the 2996 * pmap keep a consistent view of the mapping, which is 2997 * necessary for the correct handling of COW faults. It 2998 * also permits reuse of the old mapping's PV entry, 2999 * avoiding an allocation. 3000 * 3001 * For consistency, handle unmanaged mappings the same way. 3002 */ 3003 origpte = be64toh(pte_load_clear(pte)); 3004 KASSERT((origpte & PG_FRAME) == opa, 3005 ("pmap_enter: unexpected pa update for %#lx", va)); 3006 if ((origpte & PG_MANAGED) != 0) { 3007 om = PHYS_TO_VM_PAGE(opa); 3008 3009 /* 3010 * The pmap lock is sufficient to synchronize with 3011 * concurrent calls to pmap_page_test_mappings() and 3012 * pmap_ts_referenced(). 3013 */ 3014 if ((origpte & (PG_M | PG_RW)) == (PG_M | PG_RW)) 3015 vm_page_dirty(om); 3016 if ((origpte & PG_A) != 0) 3017 vm_page_aflag_set(om, PGA_REFERENCED); 3018 CHANGE_PV_LIST_LOCK_TO_PHYS(&lock, opa); 3019 pv = pmap_pvh_remove(&om->md, pmap, va); 3020 if ((newpte & PG_MANAGED) == 0) 3021 free_pv_entry(pmap, pv); 3022 #ifdef INVARIANTS 3023 else if (origpte & PG_MANAGED) { 3024 if (pv == NULL) { 3025 #ifdef DDB 3026 pmap_page_print_mappings(om); 3027 #endif 3028 MPASS(pv != NULL); 3029 } 3030 } 3031 #endif 3032 if ((om->a.flags & PGA_WRITEABLE) != 0 && 3033 TAILQ_EMPTY(&om->md.pv_list) && 3034 ((om->flags & PG_FICTITIOUS) != 0 || 3035 TAILQ_EMPTY(&pa_to_pvh(opa)->pv_list))) 3036 vm_page_aflag_clear(om, PGA_WRITEABLE); 3037 } 3038 if ((origpte & PG_A) != 0) 3039 invalidate_page = true; 3040 origpte = 0; 3041 } else { 3042 if (pmap != kernel_pmap) { 3043 #ifdef INVARIANTS 3044 if (VERBOSE_PMAP || pmap_logging) 3045 printf("pmap_enter(%p, %#lx, %p, %#x, %x, %d) -- asid=%lu curpid=%d name=%s\n", 3046 pmap, va, m, prot, flags, psind, 3047 pmap->pm_pid, curproc->p_pid, 3048 curproc->p_comm); 3049 #endif 3050 } 3051 3052 /* 3053 * Increment the counters. 3054 */ 3055 if ((newpte & PG_W) != 0) 3056 pmap->pm_stats.wired_count++; 3057 pmap_resident_count_inc(pmap, 1); 3058 } 3059 3060 /* 3061 * Enter on the PV list if part of our managed memory. 3062 */ 3063 if ((newpte & PG_MANAGED) != 0) { 3064 if (pv == NULL) { 3065 pv = get_pv_entry(pmap, &lock); 3066 pv->pv_va = va; 3067 } 3068 #ifdef VERBOSE_PV 3069 else 3070 printf("reassigning pv: %p to pmap: %p\n", 3071 pv, pmap); 3072 #endif 3073 CHANGE_PV_LIST_LOCK_TO_PHYS(&lock, pa); 3074 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_link); 3075 m->md.pv_gen++; 3076 if ((newpte & PG_RW) != 0) 3077 vm_page_aflag_set(m, PGA_WRITEABLE); 3078 } 3079 3080 /* 3081 * Update the PTE. 3082 */ 3083 if ((origpte & PG_V) != 0) { 3084 validate: 3085 origpte = be64toh(pte_load_store(pte, htobe64(newpte))); 3086 KASSERT((origpte & PG_FRAME) == pa, 3087 ("pmap_enter: unexpected pa update for %#lx", va)); 3088 if ((newpte & PG_M) == 0 && (origpte & (PG_M | PG_RW)) == 3089 (PG_M | PG_RW)) { 3090 if ((origpte & PG_MANAGED) != 0) 3091 vm_page_dirty(m); 3092 invalidate_page = true; 3093 3094 /* 3095 * Although the PTE may still have PG_RW set, TLB 3096 * invalidation may nonetheless be required because 3097 * the PTE no longer has PG_M set. 3098 */ 3099 } else if ((origpte & PG_X) != 0 || (newpte & PG_X) == 0) { 3100 /* 3101 * Removing capabilities requires invalidation on POWER 3102 */ 3103 invalidate_page = true; 3104 goto unchanged; 3105 } 3106 if ((origpte & PG_A) != 0) 3107 invalidate_page = true; 3108 } else { 3109 pte_store(pte, newpte); 3110 ptesync(); 3111 } 3112 unchanged: 3113 3114 #if VM_NRESERVLEVEL > 0 3115 /* 3116 * If both the page table page and the reservation are fully 3117 * populated, then attempt promotion. 3118 */ 3119 if ((mpte == NULL || mpte->ref_count == NPTEPG) && 3120 mmu_radix_ps_enabled(pmap) && 3121 (m->flags & PG_FICTITIOUS) == 0 && 3122 vm_reserv_level_iffullpop(m) == 0 && 3123 pmap_promote_l3e(pmap, l3e, va, &lock) == 0) 3124 invalidate_all = true; 3125 #endif 3126 if (invalidate_all) 3127 pmap_invalidate_all(pmap); 3128 else if (invalidate_page) 3129 pmap_invalidate_page(pmap, va); 3130 3131 rv = KERN_SUCCESS; 3132 out: 3133 if (lock != NULL) 3134 rw_wunlock(lock); 3135 PMAP_UNLOCK(pmap); 3136 3137 return (rv); 3138 } 3139 3140 /* 3141 * Release a page table page reference after a failed attempt to create a 3142 * mapping. 3143 */ 3144 static void 3145 pmap_abort_ptp(pmap_t pmap, vm_offset_t va, vm_page_t pdpg) 3146 { 3147 struct spglist free; 3148 3149 SLIST_INIT(&free); 3150 if (pmap_unwire_ptp(pmap, va, pdpg, &free)) { 3151 /* 3152 * Although "va" is not mapped, paging- 3153 * structure caches could nonetheless have 3154 * entries that refer to the freed page table 3155 * pages. Invalidate those entries. 3156 */ 3157 pmap_invalidate_page(pmap, va); 3158 vm_page_free_pages_toq(&free, true); 3159 } 3160 } 3161 3162 /* 3163 * Tries to create a read- and/or execute-only 2MB page mapping. Returns true 3164 * if successful. Returns false if (1) a page table page cannot be allocated 3165 * without sleeping, (2) a mapping already exists at the specified virtual 3166 * address, or (3) a PV entry cannot be allocated without reclaiming another 3167 * PV entry. 3168 */ 3169 static bool 3170 pmap_enter_2mpage(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, 3171 struct rwlock **lockp) 3172 { 3173 pml3_entry_t newpde; 3174 3175 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 3176 newpde = VM_PAGE_TO_PHYS(m) | pmap_cache_bits(m->md.mdpg_cache_attrs) | 3177 RPTE_LEAF | PG_V; 3178 if ((m->oflags & VPO_UNMANAGED) == 0) 3179 newpde |= PG_MANAGED; 3180 if (prot & VM_PROT_EXECUTE) 3181 newpde |= PG_X; 3182 if (prot & VM_PROT_READ) 3183 newpde |= RPTE_EAA_R; 3184 if (va >= DMAP_MIN_ADDRESS) 3185 newpde |= RPTE_EAA_P; 3186 return (pmap_enter_l3e(pmap, va, newpde, PMAP_ENTER_NOSLEEP | 3187 PMAP_ENTER_NOREPLACE | PMAP_ENTER_NORECLAIM, NULL, lockp) == 3188 KERN_SUCCESS); 3189 } 3190 3191 /* 3192 * Tries to create the specified 2MB page mapping. Returns KERN_SUCCESS if 3193 * the mapping was created, and either KERN_FAILURE or KERN_RESOURCE_SHORTAGE 3194 * otherwise. Returns KERN_FAILURE if PMAP_ENTER_NOREPLACE was specified and 3195 * a mapping already exists at the specified virtual address. Returns 3196 * KERN_RESOURCE_SHORTAGE if PMAP_ENTER_NOSLEEP was specified and a page table 3197 * page allocation failed. Returns KERN_RESOURCE_SHORTAGE if 3198 * PMAP_ENTER_NORECLAIM was specified and a PV entry allocation failed. 3199 * 3200 * The parameter "m" is only used when creating a managed, writeable mapping. 3201 */ 3202 static int 3203 pmap_enter_l3e(pmap_t pmap, vm_offset_t va, pml3_entry_t newpde, u_int flags, 3204 vm_page_t m, struct rwlock **lockp) 3205 { 3206 struct spglist free; 3207 pml3_entry_t oldl3e, *l3e; 3208 vm_page_t mt, pdpg; 3209 vm_page_t uwptpg; 3210 3211 KASSERT((newpde & (PG_M | PG_RW)) != PG_RW, 3212 ("pmap_enter_pde: newpde is missing PG_M")); 3213 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 3214 3215 if ((pdpg = pmap_allocl3e(pmap, va, (flags & PMAP_ENTER_NOSLEEP) != 0 ? 3216 NULL : lockp)) == NULL) { 3217 CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx" 3218 " in pmap %p", va, pmap); 3219 return (KERN_RESOURCE_SHORTAGE); 3220 } 3221 l3e = (pml3_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pdpg)); 3222 l3e = &l3e[pmap_pml3e_index(va)]; 3223 oldl3e = be64toh(*l3e); 3224 if ((oldl3e & PG_V) != 0) { 3225 KASSERT(pdpg->ref_count > 1, 3226 ("pmap_enter_pde: pdpg's wire count is too low")); 3227 if ((flags & PMAP_ENTER_NOREPLACE) != 0) { 3228 pdpg->ref_count--; 3229 CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx" 3230 " in pmap %p", va, pmap); 3231 return (KERN_FAILURE); 3232 } 3233 /* Break the existing mapping(s). */ 3234 SLIST_INIT(&free); 3235 if ((oldl3e & RPTE_LEAF) != 0) { 3236 /* 3237 * The reference to the PD page that was acquired by 3238 * pmap_allocl3e() ensures that it won't be freed. 3239 * However, if the PDE resulted from a promotion, then 3240 * a reserved PT page could be freed. 3241 */ 3242 (void)pmap_remove_l3e(pmap, l3e, va, &free, lockp); 3243 pmap_invalidate_l3e_page(pmap, va, oldl3e); 3244 } else { 3245 if (pmap_remove_ptes(pmap, va, va + L3_PAGE_SIZE, l3e, 3246 &free, lockp)) 3247 pmap_invalidate_all(pmap); 3248 } 3249 vm_page_free_pages_toq(&free, true); 3250 if (va >= VM_MAXUSER_ADDRESS) { 3251 mt = PHYS_TO_VM_PAGE(be64toh(*l3e) & PG_FRAME); 3252 if (pmap_insert_pt_page(pmap, mt)) { 3253 /* 3254 * XXX Currently, this can't happen because 3255 * we do not perform pmap_enter(psind == 1) 3256 * on the kernel pmap. 3257 */ 3258 panic("pmap_enter_pde: trie insert failed"); 3259 } 3260 } else 3261 KASSERT(be64toh(*l3e) == 0, ("pmap_enter_pde: non-zero pde %p", 3262 l3e)); 3263 } 3264 3265 /* 3266 * Allocate leaf ptpage for wired userspace pages. 3267 */ 3268 uwptpg = NULL; 3269 if ((newpde & PG_W) != 0 && pmap != kernel_pmap) { 3270 uwptpg = vm_page_alloc_noobj(VM_ALLOC_WIRED); 3271 if (uwptpg == NULL) { 3272 pmap_abort_ptp(pmap, va, pdpg); 3273 return (KERN_RESOURCE_SHORTAGE); 3274 } 3275 uwptpg->pindex = pmap_l3e_pindex(va); 3276 if (pmap_insert_pt_page(pmap, uwptpg)) { 3277 vm_page_unwire_noq(uwptpg); 3278 vm_page_free(uwptpg); 3279 pmap_abort_ptp(pmap, va, pdpg); 3280 return (KERN_RESOURCE_SHORTAGE); 3281 } 3282 pmap_resident_count_inc(pmap, 1); 3283 uwptpg->ref_count = NPTEPG; 3284 pmap_fill_ptp((pt_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(uwptpg)), 3285 newpde); 3286 } 3287 if ((newpde & PG_MANAGED) != 0) { 3288 /* 3289 * Abort this mapping if its PV entry could not be created. 3290 */ 3291 if (!pmap_pv_insert_l3e(pmap, va, newpde, flags, lockp)) { 3292 pmap_abort_ptp(pmap, va, pdpg); 3293 if (uwptpg != NULL) { 3294 mt = pmap_remove_pt_page(pmap, va); 3295 KASSERT(mt == uwptpg, 3296 ("removed pt page %p, expected %p", mt, 3297 uwptpg)); 3298 pmap_resident_count_dec(pmap, 1); 3299 uwptpg->ref_count = 1; 3300 vm_page_unwire_noq(uwptpg); 3301 vm_page_free(uwptpg); 3302 } 3303 CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx" 3304 " in pmap %p", va, pmap); 3305 return (KERN_RESOURCE_SHORTAGE); 3306 } 3307 if ((newpde & PG_RW) != 0) { 3308 for (mt = m; mt < &m[L3_PAGE_SIZE / PAGE_SIZE]; mt++) 3309 vm_page_aflag_set(mt, PGA_WRITEABLE); 3310 } 3311 } 3312 3313 /* 3314 * Increment counters. 3315 */ 3316 if ((newpde & PG_W) != 0) 3317 pmap->pm_stats.wired_count += L3_PAGE_SIZE / PAGE_SIZE; 3318 pmap_resident_count_inc(pmap, L3_PAGE_SIZE / PAGE_SIZE); 3319 3320 /* 3321 * Map the superpage. (This is not a promoted mapping; there will not 3322 * be any lingering 4KB page mappings in the TLB.) 3323 */ 3324 pte_store(l3e, newpde); 3325 ptesync(); 3326 3327 counter_u64_add(pmap_l3e_mappings, 1); 3328 CTR2(KTR_PMAP, "pmap_enter_pde: success for va %#lx" 3329 " in pmap %p", va, pmap); 3330 return (KERN_SUCCESS); 3331 } 3332 3333 void 3334 mmu_radix_enter_object(pmap_t pmap, vm_offset_t start, 3335 vm_offset_t end, vm_page_t m_start, vm_prot_t prot) 3336 { 3337 struct pctrie_iter pages; 3338 struct rwlock *lock; 3339 vm_offset_t va; 3340 vm_page_t m, mpte; 3341 bool invalidate; 3342 3343 VM_OBJECT_ASSERT_LOCKED(m_start->object); 3344 3345 CTR6(KTR_PMAP, "%s(%p, %#x, %#x, %p, %#x)", __func__, pmap, start, 3346 end, m_start, prot); 3347 invalidate = false; 3348 mpte = NULL; 3349 vm_page_iter_limit_init(&pages, m_start->object, 3350 m_start->pindex + atop(end - start)); 3351 m = vm_radix_iter_lookup(&pages, m_start->pindex); 3352 lock = NULL; 3353 PMAP_LOCK(pmap); 3354 while (m != NULL) { 3355 va = start + ptoa(m->pindex - m_start->pindex); 3356 if ((va & L3_PAGE_MASK) == 0 && va + L3_PAGE_SIZE <= end && 3357 m->psind == 1 && mmu_radix_ps_enabled(pmap) && 3358 pmap_enter_2mpage(pmap, va, m, prot, &lock)) { 3359 m = vm_radix_iter_jump(&pages, L3_PAGE_SIZE / PAGE_SIZE); 3360 } else { 3361 mpte = mmu_radix_enter_quick_locked(pmap, va, m, prot, 3362 mpte, &lock, &invalidate); 3363 m = vm_radix_iter_step(&pages); 3364 } 3365 } 3366 ptesync(); 3367 if (lock != NULL) 3368 rw_wunlock(lock); 3369 if (invalidate) 3370 pmap_invalidate_all(pmap); 3371 PMAP_UNLOCK(pmap); 3372 } 3373 3374 static vm_page_t 3375 mmu_radix_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, 3376 vm_prot_t prot, vm_page_t mpte, struct rwlock **lockp, bool *invalidate) 3377 { 3378 struct spglist free; 3379 pt_entry_t *pte; 3380 vm_paddr_t pa; 3381 3382 KASSERT(!VA_IS_CLEANMAP(va) || 3383 (m->oflags & VPO_UNMANAGED) != 0, 3384 ("mmu_radix_enter_quick_locked: managed mapping within the clean submap")); 3385 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 3386 3387 /* 3388 * In the case that a page table page is not 3389 * resident, we are creating it here. 3390 */ 3391 if (va < VM_MAXUSER_ADDRESS) { 3392 vm_pindex_t ptepindex; 3393 pml3_entry_t *ptepa; 3394 3395 /* 3396 * Calculate pagetable page index 3397 */ 3398 ptepindex = pmap_l3e_pindex(va); 3399 if (mpte && (mpte->pindex == ptepindex)) { 3400 mpte->ref_count++; 3401 } else { 3402 /* 3403 * Get the page directory entry 3404 */ 3405 ptepa = pmap_pml3e(pmap, va); 3406 3407 /* 3408 * If the page table page is mapped, we just increment 3409 * the hold count, and activate it. Otherwise, we 3410 * attempt to allocate a page table page. If this 3411 * attempt fails, we don't retry. Instead, we give up. 3412 */ 3413 if (ptepa && (be64toh(*ptepa) & PG_V) != 0) { 3414 if (be64toh(*ptepa) & RPTE_LEAF) 3415 return (NULL); 3416 mpte = PHYS_TO_VM_PAGE(be64toh(*ptepa) & PG_FRAME); 3417 mpte->ref_count++; 3418 } else { 3419 /* 3420 * Pass NULL instead of the PV list lock 3421 * pointer, because we don't intend to sleep. 3422 */ 3423 mpte = _pmap_allocpte(pmap, ptepindex, NULL); 3424 if (mpte == NULL) 3425 return (mpte); 3426 } 3427 } 3428 pte = (pt_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mpte)); 3429 pte = &pte[pmap_pte_index(va)]; 3430 } else { 3431 mpte = NULL; 3432 pte = pmap_pte(pmap, va); 3433 } 3434 if (be64toh(*pte)) { 3435 if (mpte != NULL) { 3436 mpte->ref_count--; 3437 mpte = NULL; 3438 } 3439 return (mpte); 3440 } 3441 3442 /* 3443 * Enter on the PV list if part of our managed memory. 3444 */ 3445 if ((m->oflags & VPO_UNMANAGED) == 0 && 3446 !pmap_try_insert_pv_entry(pmap, va, m, lockp)) { 3447 if (mpte != NULL) { 3448 SLIST_INIT(&free); 3449 if (pmap_unwire_ptp(pmap, va, mpte, &free)) { 3450 /* 3451 * Although "va" is not mapped, paging- 3452 * structure caches could nonetheless have 3453 * entries that refer to the freed page table 3454 * pages. Invalidate those entries. 3455 */ 3456 *invalidate = true; 3457 vm_page_free_pages_toq(&free, true); 3458 } 3459 mpte = NULL; 3460 } 3461 return (mpte); 3462 } 3463 3464 /* 3465 * Increment counters 3466 */ 3467 pmap_resident_count_inc(pmap, 1); 3468 3469 pa = VM_PAGE_TO_PHYS(m) | pmap_cache_bits(m->md.mdpg_cache_attrs); 3470 if (prot & VM_PROT_EXECUTE) 3471 pa |= PG_X; 3472 else 3473 pa |= RPTE_EAA_R; 3474 if ((m->oflags & VPO_UNMANAGED) == 0) 3475 pa |= PG_MANAGED; 3476 3477 pte_store(pte, pa); 3478 return (mpte); 3479 } 3480 3481 void 3482 mmu_radix_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, 3483 vm_prot_t prot) 3484 { 3485 struct rwlock *lock; 3486 bool invalidate; 3487 3488 lock = NULL; 3489 invalidate = false; 3490 PMAP_LOCK(pmap); 3491 mmu_radix_enter_quick_locked(pmap, va, m, prot, NULL, &lock, 3492 &invalidate); 3493 ptesync(); 3494 if (lock != NULL) 3495 rw_wunlock(lock); 3496 if (invalidate) 3497 pmap_invalidate_all(pmap); 3498 PMAP_UNLOCK(pmap); 3499 } 3500 3501 vm_paddr_t 3502 mmu_radix_extract(pmap_t pmap, vm_offset_t va) 3503 { 3504 pml3_entry_t *l3e; 3505 pt_entry_t *pte; 3506 vm_paddr_t pa; 3507 3508 l3e = pmap_pml3e(pmap, va); 3509 if (__predict_false(l3e == NULL)) 3510 return (0); 3511 if (be64toh(*l3e) & RPTE_LEAF) { 3512 pa = (be64toh(*l3e) & PG_PS_FRAME) | (va & L3_PAGE_MASK); 3513 pa |= (va & L3_PAGE_MASK); 3514 } else { 3515 /* 3516 * Beware of a concurrent promotion that changes the 3517 * PDE at this point! For example, vtopte() must not 3518 * be used to access the PTE because it would use the 3519 * new PDE. It is, however, safe to use the old PDE 3520 * because the page table page is preserved by the 3521 * promotion. 3522 */ 3523 pte = pmap_l3e_to_pte(l3e, va); 3524 if (__predict_false(pte == NULL)) 3525 return (0); 3526 pa = be64toh(*pte); 3527 pa = (pa & PG_FRAME) | (va & PAGE_MASK); 3528 pa |= (va & PAGE_MASK); 3529 } 3530 return (pa); 3531 } 3532 3533 vm_page_t 3534 mmu_radix_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot) 3535 { 3536 pml3_entry_t l3e, *l3ep; 3537 pt_entry_t pte; 3538 vm_page_t m; 3539 3540 m = NULL; 3541 CTR4(KTR_PMAP, "%s(%p, %#x, %#x)", __func__, pmap, va, prot); 3542 PMAP_LOCK(pmap); 3543 l3ep = pmap_pml3e(pmap, va); 3544 if (l3ep != NULL && (l3e = be64toh(*l3ep))) { 3545 if (l3e & RPTE_LEAF) { 3546 if ((l3e & PG_RW) || (prot & VM_PROT_WRITE) == 0) 3547 m = PHYS_TO_VM_PAGE((l3e & PG_PS_FRAME) | 3548 (va & L3_PAGE_MASK)); 3549 } else { 3550 /* Native endian PTE, do not pass to pmap functions */ 3551 pte = be64toh(*pmap_l3e_to_pte(l3ep, va)); 3552 if ((pte & PG_V) && 3553 ((pte & PG_RW) || (prot & VM_PROT_WRITE) == 0)) 3554 m = PHYS_TO_VM_PAGE(pte & PG_FRAME); 3555 } 3556 if (m != NULL && !vm_page_wire_mapped(m)) 3557 m = NULL; 3558 } 3559 PMAP_UNLOCK(pmap); 3560 return (m); 3561 } 3562 3563 static int 3564 mmu_radix_growkernel(vm_offset_t addr) 3565 { 3566 vm_paddr_t paddr; 3567 vm_page_t nkpg; 3568 pml3_entry_t *l3e; 3569 pml2_entry_t *l2e; 3570 3571 CTR2(KTR_PMAP, "%s(%#x)", __func__, addr); 3572 if (VM_MIN_KERNEL_ADDRESS < addr && 3573 addr < (VM_MIN_KERNEL_ADDRESS + nkpt * L3_PAGE_SIZE)) 3574 return (KERN_SUCCESS); 3575 3576 addr = roundup2(addr, L3_PAGE_SIZE); 3577 if (addr - 1 >= vm_map_max(kernel_map)) 3578 addr = vm_map_max(kernel_map); 3579 while (kernel_vm_end < addr) { 3580 l2e = pmap_pml2e(kernel_pmap, kernel_vm_end); 3581 if ((be64toh(*l2e) & PG_V) == 0) { 3582 /* We need a new PDP entry */ 3583 nkpg = vm_page_alloc_noobj(VM_ALLOC_INTERRUPT | 3584 VM_ALLOC_NOFREE | VM_ALLOC_WIRED | VM_ALLOC_ZERO); 3585 if (nkpg == NULL) 3586 return (KERN_RESOURCE_SHORTAGE); 3587 nkpg->pindex = kernel_vm_end >> L2_PAGE_SIZE_SHIFT; 3588 paddr = VM_PAGE_TO_PHYS(nkpg); 3589 pde_store(l2e, paddr); 3590 continue; /* try again */ 3591 } 3592 l3e = pmap_l2e_to_l3e(l2e, kernel_vm_end); 3593 if ((be64toh(*l3e) & PG_V) != 0) { 3594 kernel_vm_end = (kernel_vm_end + L3_PAGE_SIZE) & ~L3_PAGE_MASK; 3595 if (kernel_vm_end - 1 >= vm_map_max(kernel_map)) { 3596 kernel_vm_end = vm_map_max(kernel_map); 3597 break; 3598 } 3599 continue; 3600 } 3601 3602 nkpg = vm_page_alloc_noobj(VM_ALLOC_INTERRUPT | 3603 VM_ALLOC_NOFREE | VM_ALLOC_WIRED | VM_ALLOC_ZERO); 3604 if (nkpg == NULL) 3605 return (KERN_RESOURCE_SHORTAGE); 3606 nkpg->pindex = pmap_l3e_pindex(kernel_vm_end); 3607 paddr = VM_PAGE_TO_PHYS(nkpg); 3608 pde_store(l3e, paddr); 3609 3610 kernel_vm_end = (kernel_vm_end + L3_PAGE_SIZE) & ~L3_PAGE_MASK; 3611 if (kernel_vm_end - 1 >= vm_map_max(kernel_map)) { 3612 kernel_vm_end = vm_map_max(kernel_map); 3613 break; 3614 } 3615 } 3616 ptesync(); 3617 return (KERN_SUCCESS); 3618 } 3619 3620 static MALLOC_DEFINE(M_RADIX_PGD, "radix_pgd", "radix page table root directory"); 3621 static uma_zone_t zone_radix_pgd; 3622 3623 static int 3624 radix_pgd_import(void *arg __unused, void **store, int count, int domain __unused, 3625 int flags) 3626 { 3627 int req; 3628 3629 req = VM_ALLOC_WIRED | malloc2vm_flags(flags); 3630 for (int i = 0; i < count; i++) { 3631 vm_page_t m = vm_page_alloc_noobj_contig(req, 3632 RADIX_PGD_SIZE / PAGE_SIZE, 3633 0, (vm_paddr_t)-1, RADIX_PGD_SIZE, L1_PAGE_SIZE, 3634 VM_MEMATTR_DEFAULT); 3635 store[i] = (void *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)); 3636 } 3637 return (count); 3638 } 3639 3640 static void 3641 radix_pgd_release(void *arg __unused, void **store, int count) 3642 { 3643 vm_page_t m; 3644 struct spglist free; 3645 int page_count; 3646 3647 SLIST_INIT(&free); 3648 page_count = RADIX_PGD_SIZE/PAGE_SIZE; 3649 3650 for (int i = 0; i < count; i++) { 3651 /* 3652 * XXX selectively remove dmap and KVA entries so we don't 3653 * need to bzero 3654 */ 3655 m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)store[i])); 3656 for (int j = page_count-1; j >= 0; j--) { 3657 vm_page_unwire_noq(&m[j]); 3658 SLIST_INSERT_HEAD(&free, &m[j], plinks.s.ss); 3659 } 3660 vm_page_free_pages_toq(&free, false); 3661 } 3662 } 3663 3664 static void 3665 mmu_radix_init(void) 3666 { 3667 vm_page_t mpte; 3668 vm_size_t s; 3669 int error, i, pv_npg; 3670 3671 /* XXX is this really needed for POWER? */ 3672 /* L1TF, reserve page @0 unconditionally */ 3673 vm_page_blacklist_add(0, bootverbose); 3674 3675 zone_radix_pgd = uma_zcache_create("radix_pgd_cache", 3676 RADIX_PGD_SIZE, NULL, NULL, 3677 #ifdef INVARIANTS 3678 trash_init, trash_fini, 3679 #else 3680 NULL, NULL, 3681 #endif 3682 radix_pgd_import, radix_pgd_release, 3683 NULL, UMA_ZONE_NOBUCKET); 3684 3685 /* 3686 * Initialize the vm page array entries for the kernel pmap's 3687 * page table pages. 3688 */ 3689 PMAP_LOCK(kernel_pmap); 3690 for (i = 0; i < nkpt; i++) { 3691 mpte = PHYS_TO_VM_PAGE(KPTphys + (i << PAGE_SHIFT)); 3692 KASSERT(mpte >= vm_page_array && 3693 mpte < &vm_page_array[vm_page_array_size], 3694 ("pmap_init: page table page is out of range size: %lu", 3695 vm_page_array_size)); 3696 mpte->pindex = pmap_l3e_pindex(VM_MIN_KERNEL_ADDRESS) + i; 3697 mpte->phys_addr = KPTphys + (i << PAGE_SHIFT); 3698 MPASS(PHYS_TO_VM_PAGE(mpte->phys_addr) == mpte); 3699 //pmap_insert_pt_page(kernel_pmap, mpte); 3700 mpte->ref_count = 1; 3701 } 3702 PMAP_UNLOCK(kernel_pmap); 3703 vm_wire_add(nkpt); 3704 3705 CTR1(KTR_PMAP, "%s()", __func__); 3706 TAILQ_INIT(&pv_dummy.pv_list); 3707 3708 /* 3709 * Are large page mappings enabled? 3710 */ 3711 TUNABLE_INT_FETCH("vm.pmap.superpages_enabled", &superpages_enabled); 3712 if (superpages_enabled) { 3713 KASSERT(MAXPAGESIZES > 1 && pagesizes[1] == 0, 3714 ("pmap_init: can't assign to pagesizes[1]")); 3715 pagesizes[1] = L3_PAGE_SIZE; 3716 } 3717 3718 /* 3719 * Initialize the pv chunk list mutex. 3720 */ 3721 mtx_init(&pv_chunks_mutex, "pmap pv chunk list", NULL, MTX_DEF); 3722 3723 /* 3724 * Initialize the pool of pv list locks. 3725 */ 3726 for (i = 0; i < NPV_LIST_LOCKS; i++) 3727 rw_init(&pv_list_locks[i], "pmap pv list"); 3728 3729 /* 3730 * Calculate the size of the pv head table for superpages. 3731 */ 3732 pv_npg = howmany(vm_phys_segs[vm_phys_nsegs - 1].end, L3_PAGE_SIZE); 3733 3734 /* 3735 * Allocate memory for the pv head table for superpages. 3736 */ 3737 s = (vm_size_t)(pv_npg * sizeof(struct md_page)); 3738 s = round_page(s); 3739 pv_table = kmem_malloc(s, M_WAITOK | M_ZERO); 3740 for (i = 0; i < pv_npg; i++) 3741 TAILQ_INIT(&pv_table[i].pv_list); 3742 TAILQ_INIT(&pv_dummy.pv_list); 3743 3744 pmap_initialized = 1; 3745 mtx_init(&qframe_mtx, "qfrmlk", NULL, MTX_SPIN); 3746 error = vmem_alloc(kernel_arena, PAGE_SIZE, M_BESTFIT | M_WAITOK, 3747 (vmem_addr_t *)&qframe); 3748 3749 if (error != 0) 3750 panic("qframe allocation failed"); 3751 asid_arena = vmem_create("ASID", isa3_base_pid + 1, (1<<isa3_pid_bits), 3752 1, 1, M_WAITOK); 3753 } 3754 3755 static bool 3756 pmap_page_test_mappings(vm_page_t m, bool accessed, bool modified) 3757 { 3758 struct rwlock *lock; 3759 pv_entry_t pv; 3760 struct md_page *pvh; 3761 pt_entry_t *pte, mask; 3762 pmap_t pmap; 3763 int md_gen, pvh_gen; 3764 bool rv; 3765 3766 rv = false; 3767 lock = VM_PAGE_TO_PV_LIST_LOCK(m); 3768 rw_rlock(lock); 3769 restart: 3770 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) { 3771 pmap = PV_PMAP(pv); 3772 if (!PMAP_TRYLOCK(pmap)) { 3773 md_gen = m->md.pv_gen; 3774 rw_runlock(lock); 3775 PMAP_LOCK(pmap); 3776 rw_rlock(lock); 3777 if (md_gen != m->md.pv_gen) { 3778 PMAP_UNLOCK(pmap); 3779 goto restart; 3780 } 3781 } 3782 pte = pmap_pte(pmap, pv->pv_va); 3783 mask = 0; 3784 if (modified) 3785 mask |= PG_RW | PG_M; 3786 if (accessed) 3787 mask |= PG_V | PG_A; 3788 rv = (be64toh(*pte) & mask) == mask; 3789 PMAP_UNLOCK(pmap); 3790 if (rv) 3791 goto out; 3792 } 3793 if ((m->flags & PG_FICTITIOUS) == 0) { 3794 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); 3795 TAILQ_FOREACH(pv, &pvh->pv_list, pv_link) { 3796 pmap = PV_PMAP(pv); 3797 if (!PMAP_TRYLOCK(pmap)) { 3798 md_gen = m->md.pv_gen; 3799 pvh_gen = pvh->pv_gen; 3800 rw_runlock(lock); 3801 PMAP_LOCK(pmap); 3802 rw_rlock(lock); 3803 if (md_gen != m->md.pv_gen || 3804 pvh_gen != pvh->pv_gen) { 3805 PMAP_UNLOCK(pmap); 3806 goto restart; 3807 } 3808 } 3809 pte = pmap_pml3e(pmap, pv->pv_va); 3810 mask = 0; 3811 if (modified) 3812 mask |= PG_RW | PG_M; 3813 if (accessed) 3814 mask |= PG_V | PG_A; 3815 rv = (be64toh(*pte) & mask) == mask; 3816 PMAP_UNLOCK(pmap); 3817 if (rv) 3818 goto out; 3819 } 3820 } 3821 out: 3822 rw_runlock(lock); 3823 return (rv); 3824 } 3825 3826 /* 3827 * pmap_is_modified: 3828 * 3829 * Return whether or not the specified physical page was modified 3830 * in any physical maps. 3831 */ 3832 bool 3833 mmu_radix_is_modified(vm_page_t m) 3834 { 3835 3836 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 3837 ("pmap_is_modified: page %p is not managed", m)); 3838 3839 CTR2(KTR_PMAP, "%s(%p)", __func__, m); 3840 /* 3841 * If the page is not busied then this check is racy. 3842 */ 3843 if (!pmap_page_is_write_mapped(m)) 3844 return (false); 3845 return (pmap_page_test_mappings(m, false, true)); 3846 } 3847 3848 bool 3849 mmu_radix_is_prefaultable(pmap_t pmap, vm_offset_t addr) 3850 { 3851 pml3_entry_t *l3e; 3852 pt_entry_t *pte; 3853 bool rv; 3854 3855 CTR3(KTR_PMAP, "%s(%p, %#x)", __func__, pmap, addr); 3856 rv = false; 3857 PMAP_LOCK(pmap); 3858 l3e = pmap_pml3e(pmap, addr); 3859 if (l3e != NULL && (be64toh(*l3e) & (RPTE_LEAF | PG_V)) == PG_V) { 3860 pte = pmap_l3e_to_pte(l3e, addr); 3861 rv = (be64toh(*pte) & PG_V) == 0; 3862 } 3863 PMAP_UNLOCK(pmap); 3864 return (rv); 3865 } 3866 3867 bool 3868 mmu_radix_is_referenced(vm_page_t m) 3869 { 3870 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 3871 ("pmap_is_referenced: page %p is not managed", m)); 3872 CTR2(KTR_PMAP, "%s(%p)", __func__, m); 3873 return (pmap_page_test_mappings(m, true, false)); 3874 } 3875 3876 /* 3877 * pmap_ts_referenced: 3878 * 3879 * Return a count of reference bits for a page, clearing those bits. 3880 * It is not necessary for every reference bit to be cleared, but it 3881 * is necessary that 0 only be returned when there are truly no 3882 * reference bits set. 3883 * 3884 * As an optimization, update the page's dirty field if a modified bit is 3885 * found while counting reference bits. This opportunistic update can be 3886 * performed at low cost and can eliminate the need for some future calls 3887 * to pmap_is_modified(). However, since this function stops after 3888 * finding PMAP_TS_REFERENCED_MAX reference bits, it may not detect some 3889 * dirty pages. Those dirty pages will only be detected by a future call 3890 * to pmap_is_modified(). 3891 * 3892 * A DI block is not needed within this function, because 3893 * invalidations are performed before the PV list lock is 3894 * released. 3895 */ 3896 int 3897 mmu_radix_ts_referenced(vm_page_t m) 3898 { 3899 struct md_page *pvh; 3900 pv_entry_t pv, pvf; 3901 pmap_t pmap; 3902 struct rwlock *lock; 3903 pml3_entry_t oldl3e, *l3e; 3904 pt_entry_t *pte; 3905 vm_paddr_t pa; 3906 int cleared, md_gen, not_cleared, pvh_gen; 3907 struct spglist free; 3908 3909 CTR2(KTR_PMAP, "%s(%p)", __func__, m); 3910 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 3911 ("pmap_ts_referenced: page %p is not managed", m)); 3912 SLIST_INIT(&free); 3913 cleared = 0; 3914 pa = VM_PAGE_TO_PHYS(m); 3915 lock = PHYS_TO_PV_LIST_LOCK(pa); 3916 pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy : pa_to_pvh(pa); 3917 rw_wlock(lock); 3918 retry: 3919 not_cleared = 0; 3920 if ((pvf = TAILQ_FIRST(&pvh->pv_list)) == NULL) 3921 goto small_mappings; 3922 pv = pvf; 3923 do { 3924 if (pvf == NULL) 3925 pvf = pv; 3926 pmap = PV_PMAP(pv); 3927 if (!PMAP_TRYLOCK(pmap)) { 3928 pvh_gen = pvh->pv_gen; 3929 rw_wunlock(lock); 3930 PMAP_LOCK(pmap); 3931 rw_wlock(lock); 3932 if (pvh_gen != pvh->pv_gen) { 3933 PMAP_UNLOCK(pmap); 3934 goto retry; 3935 } 3936 } 3937 l3e = pmap_pml3e(pmap, pv->pv_va); 3938 oldl3e = be64toh(*l3e); 3939 if ((oldl3e & (PG_M | PG_RW)) == (PG_M | PG_RW)) { 3940 /* 3941 * Although "oldpde" is mapping a 2MB page, because 3942 * this function is called at a 4KB page granularity, 3943 * we only update the 4KB page under test. 3944 */ 3945 vm_page_dirty(m); 3946 } 3947 if ((oldl3e & PG_A) != 0) { 3948 /* 3949 * Since this reference bit is shared by 512 4KB 3950 * pages, it should not be cleared every time it is 3951 * tested. Apply a simple "hash" function on the 3952 * physical page number, the virtual superpage number, 3953 * and the pmap address to select one 4KB page out of 3954 * the 512 on which testing the reference bit will 3955 * result in clearing that reference bit. This 3956 * function is designed to avoid the selection of the 3957 * same 4KB page for every 2MB page mapping. 3958 * 3959 * On demotion, a mapping that hasn't been referenced 3960 * is simply destroyed. To avoid the possibility of a 3961 * subsequent page fault on a demoted wired mapping, 3962 * always leave its reference bit set. Moreover, 3963 * since the superpage is wired, the current state of 3964 * its reference bit won't affect page replacement. 3965 */ 3966 if ((((pa >> PAGE_SHIFT) ^ (pv->pv_va >> L3_PAGE_SIZE_SHIFT) ^ 3967 (uintptr_t)pmap) & (NPTEPG - 1)) == 0 && 3968 (oldl3e & PG_W) == 0) { 3969 atomic_clear_long(l3e, htobe64(PG_A)); 3970 pmap_invalidate_page(pmap, pv->pv_va); 3971 cleared++; 3972 KASSERT(lock == VM_PAGE_TO_PV_LIST_LOCK(m), 3973 ("inconsistent pv lock %p %p for page %p", 3974 lock, VM_PAGE_TO_PV_LIST_LOCK(m), m)); 3975 } else 3976 not_cleared++; 3977 } 3978 PMAP_UNLOCK(pmap); 3979 /* Rotate the PV list if it has more than one entry. */ 3980 if (pv != NULL && TAILQ_NEXT(pv, pv_link) != NULL) { 3981 TAILQ_REMOVE(&pvh->pv_list, pv, pv_link); 3982 TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_link); 3983 pvh->pv_gen++; 3984 } 3985 if (cleared + not_cleared >= PMAP_TS_REFERENCED_MAX) 3986 goto out; 3987 } while ((pv = TAILQ_FIRST(&pvh->pv_list)) != pvf); 3988 small_mappings: 3989 if ((pvf = TAILQ_FIRST(&m->md.pv_list)) == NULL) 3990 goto out; 3991 pv = pvf; 3992 do { 3993 if (pvf == NULL) 3994 pvf = pv; 3995 pmap = PV_PMAP(pv); 3996 if (!PMAP_TRYLOCK(pmap)) { 3997 pvh_gen = pvh->pv_gen; 3998 md_gen = m->md.pv_gen; 3999 rw_wunlock(lock); 4000 PMAP_LOCK(pmap); 4001 rw_wlock(lock); 4002 if (pvh_gen != pvh->pv_gen || md_gen != m->md.pv_gen) { 4003 PMAP_UNLOCK(pmap); 4004 goto retry; 4005 } 4006 } 4007 l3e = pmap_pml3e(pmap, pv->pv_va); 4008 KASSERT((be64toh(*l3e) & RPTE_LEAF) == 0, 4009 ("pmap_ts_referenced: found a 2mpage in page %p's pv list", 4010 m)); 4011 pte = pmap_l3e_to_pte(l3e, pv->pv_va); 4012 if ((be64toh(*pte) & (PG_M | PG_RW)) == (PG_M | PG_RW)) 4013 vm_page_dirty(m); 4014 if ((be64toh(*pte) & PG_A) != 0) { 4015 atomic_clear_long(pte, htobe64(PG_A)); 4016 pmap_invalidate_page(pmap, pv->pv_va); 4017 cleared++; 4018 } 4019 PMAP_UNLOCK(pmap); 4020 /* Rotate the PV list if it has more than one entry. */ 4021 if (pv != NULL && TAILQ_NEXT(pv, pv_link) != NULL) { 4022 TAILQ_REMOVE(&m->md.pv_list, pv, pv_link); 4023 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_link); 4024 m->md.pv_gen++; 4025 } 4026 } while ((pv = TAILQ_FIRST(&m->md.pv_list)) != pvf && cleared + 4027 not_cleared < PMAP_TS_REFERENCED_MAX); 4028 out: 4029 rw_wunlock(lock); 4030 vm_page_free_pages_toq(&free, true); 4031 return (cleared + not_cleared); 4032 } 4033 4034 static vm_offset_t 4035 mmu_radix_map(vm_offset_t *virt __unused, vm_paddr_t start, 4036 vm_paddr_t end, int prot __unused) 4037 { 4038 4039 CTR5(KTR_PMAP, "%s(%p, %#x, %#x, %#x)", __func__, virt, start, end, 4040 prot); 4041 return (PHYS_TO_DMAP(start)); 4042 } 4043 4044 void 4045 mmu_radix_object_init_pt(pmap_t pmap, vm_offset_t addr, 4046 vm_object_t object, vm_pindex_t pindex, vm_size_t size) 4047 { 4048 struct pctrie_iter pages; 4049 pml3_entry_t *l3e; 4050 vm_paddr_t pa, ptepa; 4051 vm_page_t p, pdpg; 4052 vm_memattr_t ma; 4053 4054 CTR6(KTR_PMAP, "%s(%p, %#x, %p, %u, %#x)", __func__, pmap, addr, 4055 object, pindex, size); 4056 VM_OBJECT_ASSERT_WLOCKED(object); 4057 KASSERT(object->type == OBJT_DEVICE || object->type == OBJT_SG, 4058 ("pmap_object_init_pt: non-device object")); 4059 /* NB: size can be logically ored with addr here */ 4060 if ((addr & L3_PAGE_MASK) == 0 && (size & L3_PAGE_MASK) == 0) { 4061 if (!mmu_radix_ps_enabled(pmap)) 4062 return; 4063 if (!vm_object_populate(object, pindex, pindex + atop(size))) 4064 return; 4065 vm_page_iter_init(&pages, object); 4066 p = vm_radix_iter_lookup(&pages, pindex); 4067 4068 KASSERT(p->valid == VM_PAGE_BITS_ALL, 4069 ("pmap_object_init_pt: invalid page %p", p)); 4070 ma = p->md.mdpg_cache_attrs; 4071 4072 /* 4073 * Abort the mapping if the first page is not physically 4074 * aligned to a 2MB page boundary. 4075 */ 4076 ptepa = VM_PAGE_TO_PHYS(p); 4077 if (ptepa & L3_PAGE_MASK) 4078 return; 4079 4080 /* 4081 * Skip the first page. Abort the mapping if the rest of 4082 * the pages are not physically contiguous or have differing 4083 * memory attributes. 4084 */ 4085 for (pa = ptepa + PAGE_SIZE; pa < ptepa + size; 4086 pa += PAGE_SIZE) { 4087 p = vm_radix_iter_next(&pages); 4088 KASSERT(p->valid == VM_PAGE_BITS_ALL, 4089 ("pmap_object_init_pt: invalid page %p", p)); 4090 if (pa != VM_PAGE_TO_PHYS(p) || 4091 ma != p->md.mdpg_cache_attrs) 4092 return; 4093 } 4094 4095 PMAP_LOCK(pmap); 4096 for (pa = ptepa | pmap_cache_bits(ma); 4097 pa < ptepa + size; pa += L3_PAGE_SIZE) { 4098 pdpg = pmap_allocl3e(pmap, addr, NULL); 4099 if (pdpg == NULL) { 4100 /* 4101 * The creation of mappings below is only an 4102 * optimization. If a page directory page 4103 * cannot be allocated without blocking, 4104 * continue on to the next mapping rather than 4105 * blocking. 4106 */ 4107 addr += L3_PAGE_SIZE; 4108 continue; 4109 } 4110 l3e = (pml3_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pdpg)); 4111 l3e = &l3e[pmap_pml3e_index(addr)]; 4112 if ((be64toh(*l3e) & PG_V) == 0) { 4113 pa |= PG_M | PG_A | PG_RW; 4114 pte_store(l3e, pa); 4115 pmap_resident_count_inc(pmap, L3_PAGE_SIZE / PAGE_SIZE); 4116 counter_u64_add(pmap_l3e_mappings, 1); 4117 } else { 4118 /* Continue on if the PDE is already valid. */ 4119 pdpg->ref_count--; 4120 KASSERT(pdpg->ref_count > 0, 4121 ("pmap_object_init_pt: missing reference " 4122 "to page directory page, va: 0x%lx", addr)); 4123 } 4124 addr += L3_PAGE_SIZE; 4125 } 4126 ptesync(); 4127 PMAP_UNLOCK(pmap); 4128 } 4129 } 4130 4131 bool 4132 mmu_radix_page_exists_quick(pmap_t pmap, vm_page_t m) 4133 { 4134 struct md_page *pvh; 4135 struct rwlock *lock; 4136 pv_entry_t pv; 4137 int loops = 0; 4138 bool rv; 4139 4140 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 4141 ("pmap_page_exists_quick: page %p is not managed", m)); 4142 CTR3(KTR_PMAP, "%s(%p, %p)", __func__, pmap, m); 4143 rv = false; 4144 lock = VM_PAGE_TO_PV_LIST_LOCK(m); 4145 rw_rlock(lock); 4146 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) { 4147 if (PV_PMAP(pv) == pmap) { 4148 rv = true; 4149 break; 4150 } 4151 loops++; 4152 if (loops >= 16) 4153 break; 4154 } 4155 if (!rv && loops < 16 && (m->flags & PG_FICTITIOUS) == 0) { 4156 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); 4157 TAILQ_FOREACH(pv, &pvh->pv_list, pv_link) { 4158 if (PV_PMAP(pv) == pmap) { 4159 rv = true; 4160 break; 4161 } 4162 loops++; 4163 if (loops >= 16) 4164 break; 4165 } 4166 } 4167 rw_runlock(lock); 4168 return (rv); 4169 } 4170 4171 void 4172 mmu_radix_page_init(vm_page_t m) 4173 { 4174 4175 CTR2(KTR_PMAP, "%s(%p)", __func__, m); 4176 TAILQ_INIT(&m->md.pv_list); 4177 m->md.mdpg_cache_attrs = VM_MEMATTR_DEFAULT; 4178 } 4179 4180 int 4181 mmu_radix_page_wired_mappings(vm_page_t m) 4182 { 4183 struct rwlock *lock; 4184 struct md_page *pvh; 4185 pmap_t pmap; 4186 pt_entry_t *pte; 4187 pv_entry_t pv; 4188 int count, md_gen, pvh_gen; 4189 4190 if ((m->oflags & VPO_UNMANAGED) != 0) 4191 return (0); 4192 CTR2(KTR_PMAP, "%s(%p)", __func__, m); 4193 lock = VM_PAGE_TO_PV_LIST_LOCK(m); 4194 rw_rlock(lock); 4195 restart: 4196 count = 0; 4197 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) { 4198 pmap = PV_PMAP(pv); 4199 if (!PMAP_TRYLOCK(pmap)) { 4200 md_gen = m->md.pv_gen; 4201 rw_runlock(lock); 4202 PMAP_LOCK(pmap); 4203 rw_rlock(lock); 4204 if (md_gen != m->md.pv_gen) { 4205 PMAP_UNLOCK(pmap); 4206 goto restart; 4207 } 4208 } 4209 pte = pmap_pte(pmap, pv->pv_va); 4210 if ((be64toh(*pte) & PG_W) != 0) 4211 count++; 4212 PMAP_UNLOCK(pmap); 4213 } 4214 if ((m->flags & PG_FICTITIOUS) == 0) { 4215 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); 4216 TAILQ_FOREACH(pv, &pvh->pv_list, pv_link) { 4217 pmap = PV_PMAP(pv); 4218 if (!PMAP_TRYLOCK(pmap)) { 4219 md_gen = m->md.pv_gen; 4220 pvh_gen = pvh->pv_gen; 4221 rw_runlock(lock); 4222 PMAP_LOCK(pmap); 4223 rw_rlock(lock); 4224 if (md_gen != m->md.pv_gen || 4225 pvh_gen != pvh->pv_gen) { 4226 PMAP_UNLOCK(pmap); 4227 goto restart; 4228 } 4229 } 4230 pte = pmap_pml3e(pmap, pv->pv_va); 4231 if ((be64toh(*pte) & PG_W) != 0) 4232 count++; 4233 PMAP_UNLOCK(pmap); 4234 } 4235 } 4236 rw_runlock(lock); 4237 return (count); 4238 } 4239 4240 static void 4241 mmu_radix_update_proctab(int pid, pml1_entry_t l1pa) 4242 { 4243 isa3_proctab[pid].proctab0 = htobe64(RTS_SIZE | l1pa | RADIX_PGD_INDEX_SHIFT); 4244 } 4245 4246 int 4247 mmu_radix_pinit(pmap_t pmap) 4248 { 4249 vmem_addr_t pid; 4250 vm_paddr_t l1pa; 4251 4252 CTR2(KTR_PMAP, "%s(%p)", __func__, pmap); 4253 4254 /* 4255 * allocate the page directory page 4256 */ 4257 pmap->pm_pml1 = uma_zalloc(zone_radix_pgd, M_WAITOK); 4258 4259 for (int j = 0; j < RADIX_PGD_SIZE_SHIFT; j++) 4260 pagezero((vm_offset_t)pmap->pm_pml1 + j * PAGE_SIZE); 4261 vm_radix_init(&pmap->pm_radix); 4262 TAILQ_INIT(&pmap->pm_pvchunk); 4263 bzero(&pmap->pm_stats, sizeof pmap->pm_stats); 4264 pmap->pm_flags = PMAP_PDE_SUPERPAGE; 4265 vmem_alloc(asid_arena, 1, M_FIRSTFIT|M_WAITOK, &pid); 4266 4267 pmap->pm_pid = pid; 4268 l1pa = DMAP_TO_PHYS((vm_offset_t)pmap->pm_pml1); 4269 mmu_radix_update_proctab(pid, l1pa); 4270 __asm __volatile("ptesync;isync" : : : "memory"); 4271 4272 return (1); 4273 } 4274 4275 /* 4276 * This routine is called if the desired page table page does not exist. 4277 * 4278 * If page table page allocation fails, this routine may sleep before 4279 * returning NULL. It sleeps only if a lock pointer was given. 4280 * 4281 * Note: If a page allocation fails at page table level two or three, 4282 * one or two pages may be held during the wait, only to be released 4283 * afterwards. This conservative approach is easily argued to avoid 4284 * race conditions. 4285 */ 4286 static vm_page_t 4287 _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, struct rwlock **lockp) 4288 { 4289 vm_page_t m, pdppg, pdpg; 4290 4291 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 4292 4293 /* 4294 * Allocate a page table page. 4295 */ 4296 if ((m = vm_page_alloc_noobj(VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) { 4297 if (lockp != NULL) { 4298 RELEASE_PV_LIST_LOCK(lockp); 4299 PMAP_UNLOCK(pmap); 4300 vm_wait(NULL); 4301 PMAP_LOCK(pmap); 4302 } 4303 /* 4304 * Indicate the need to retry. While waiting, the page table 4305 * page may have been allocated. 4306 */ 4307 return (NULL); 4308 } 4309 m->pindex = ptepindex; 4310 4311 /* 4312 * Map the pagetable page into the process address space, if 4313 * it isn't already there. 4314 */ 4315 4316 if (ptepindex >= (NUPDE + NUPDPE)) { 4317 pml1_entry_t *l1e; 4318 vm_pindex_t pml1index; 4319 4320 /* Wire up a new PDPE page */ 4321 pml1index = ptepindex - (NUPDE + NUPDPE); 4322 l1e = &pmap->pm_pml1[pml1index]; 4323 KASSERT((be64toh(*l1e) & PG_V) == 0, 4324 ("%s: L1 entry %#lx is valid", __func__, *l1e)); 4325 pde_store(l1e, VM_PAGE_TO_PHYS(m)); 4326 } else if (ptepindex >= NUPDE) { 4327 vm_pindex_t pml1index; 4328 vm_pindex_t pdpindex; 4329 pml1_entry_t *l1e; 4330 pml2_entry_t *l2e; 4331 4332 /* Wire up a new l2e page */ 4333 pdpindex = ptepindex - NUPDE; 4334 pml1index = pdpindex >> RPTE_SHIFT; 4335 4336 l1e = &pmap->pm_pml1[pml1index]; 4337 if ((be64toh(*l1e) & PG_V) == 0) { 4338 /* Have to allocate a new pdp, recurse */ 4339 if (_pmap_allocpte(pmap, NUPDE + NUPDPE + pml1index, 4340 lockp) == NULL) { 4341 vm_page_unwire_noq(m); 4342 vm_page_free_zero(m); 4343 return (NULL); 4344 } 4345 } else { 4346 /* Add reference to l2e page */ 4347 pdppg = PHYS_TO_VM_PAGE(be64toh(*l1e) & PG_FRAME); 4348 pdppg->ref_count++; 4349 } 4350 l2e = (pml2_entry_t *)PHYS_TO_DMAP(be64toh(*l1e) & PG_FRAME); 4351 4352 /* Now find the pdp page */ 4353 l2e = &l2e[pdpindex & RPTE_MASK]; 4354 KASSERT((be64toh(*l2e) & PG_V) == 0, 4355 ("%s: L2 entry %#lx is valid", __func__, *l2e)); 4356 pde_store(l2e, VM_PAGE_TO_PHYS(m)); 4357 } else { 4358 vm_pindex_t pml1index; 4359 vm_pindex_t pdpindex; 4360 pml1_entry_t *l1e; 4361 pml2_entry_t *l2e; 4362 pml3_entry_t *l3e; 4363 4364 /* Wire up a new PTE page */ 4365 pdpindex = ptepindex >> RPTE_SHIFT; 4366 pml1index = pdpindex >> RPTE_SHIFT; 4367 4368 /* First, find the pdp and check that its valid. */ 4369 l1e = &pmap->pm_pml1[pml1index]; 4370 if ((be64toh(*l1e) & PG_V) == 0) { 4371 /* Have to allocate a new pd, recurse */ 4372 if (_pmap_allocpte(pmap, NUPDE + pdpindex, 4373 lockp) == NULL) { 4374 vm_page_unwire_noq(m); 4375 vm_page_free_zero(m); 4376 return (NULL); 4377 } 4378 l2e = (pml2_entry_t *)PHYS_TO_DMAP(be64toh(*l1e) & PG_FRAME); 4379 l2e = &l2e[pdpindex & RPTE_MASK]; 4380 } else { 4381 l2e = (pml2_entry_t *)PHYS_TO_DMAP(be64toh(*l1e) & PG_FRAME); 4382 l2e = &l2e[pdpindex & RPTE_MASK]; 4383 if ((be64toh(*l2e) & PG_V) == 0) { 4384 /* Have to allocate a new pd, recurse */ 4385 if (_pmap_allocpte(pmap, NUPDE + pdpindex, 4386 lockp) == NULL) { 4387 vm_page_unwire_noq(m); 4388 vm_page_free_zero(m); 4389 return (NULL); 4390 } 4391 } else { 4392 /* Add reference to the pd page */ 4393 pdpg = PHYS_TO_VM_PAGE(be64toh(*l2e) & PG_FRAME); 4394 pdpg->ref_count++; 4395 } 4396 } 4397 l3e = (pml3_entry_t *)PHYS_TO_DMAP(be64toh(*l2e) & PG_FRAME); 4398 4399 /* Now we know where the page directory page is */ 4400 l3e = &l3e[ptepindex & RPTE_MASK]; 4401 KASSERT((be64toh(*l3e) & PG_V) == 0, 4402 ("%s: L3 entry %#lx is valid", __func__, *l3e)); 4403 pde_store(l3e, VM_PAGE_TO_PHYS(m)); 4404 } 4405 4406 pmap_resident_count_inc(pmap, 1); 4407 return (m); 4408 } 4409 static vm_page_t 4410 pmap_allocl3e(pmap_t pmap, vm_offset_t va, struct rwlock **lockp) 4411 { 4412 vm_pindex_t pdpindex, ptepindex; 4413 pml2_entry_t *pdpe; 4414 vm_page_t pdpg; 4415 4416 retry: 4417 pdpe = pmap_pml2e(pmap, va); 4418 if (pdpe != NULL && (be64toh(*pdpe) & PG_V) != 0) { 4419 /* Add a reference to the pd page. */ 4420 pdpg = PHYS_TO_VM_PAGE(be64toh(*pdpe) & PG_FRAME); 4421 pdpg->ref_count++; 4422 } else { 4423 /* Allocate a pd page. */ 4424 ptepindex = pmap_l3e_pindex(va); 4425 pdpindex = ptepindex >> RPTE_SHIFT; 4426 pdpg = _pmap_allocpte(pmap, NUPDE + pdpindex, lockp); 4427 if (pdpg == NULL && lockp != NULL) 4428 goto retry; 4429 } 4430 return (pdpg); 4431 } 4432 4433 static vm_page_t 4434 pmap_allocpte(pmap_t pmap, vm_offset_t va, struct rwlock **lockp) 4435 { 4436 vm_pindex_t ptepindex; 4437 pml3_entry_t *pd; 4438 vm_page_t m; 4439 4440 /* 4441 * Calculate pagetable page index 4442 */ 4443 ptepindex = pmap_l3e_pindex(va); 4444 retry: 4445 /* 4446 * Get the page directory entry 4447 */ 4448 pd = pmap_pml3e(pmap, va); 4449 4450 /* 4451 * This supports switching from a 2MB page to a 4452 * normal 4K page. 4453 */ 4454 if (pd != NULL && (be64toh(*pd) & (RPTE_LEAF | PG_V)) == (RPTE_LEAF | PG_V)) { 4455 if (!pmap_demote_l3e_locked(pmap, pd, va, lockp)) { 4456 /* 4457 * Invalidation of the 2MB page mapping may have caused 4458 * the deallocation of the underlying PD page. 4459 */ 4460 pd = NULL; 4461 } 4462 } 4463 4464 /* 4465 * If the page table page is mapped, we just increment the 4466 * hold count, and activate it. 4467 */ 4468 if (pd != NULL && (be64toh(*pd) & PG_V) != 0) { 4469 m = PHYS_TO_VM_PAGE(be64toh(*pd) & PG_FRAME); 4470 m->ref_count++; 4471 } else { 4472 /* 4473 * Here if the pte page isn't mapped, or if it has been 4474 * deallocated. 4475 */ 4476 m = _pmap_allocpte(pmap, ptepindex, lockp); 4477 if (m == NULL && lockp != NULL) 4478 goto retry; 4479 } 4480 return (m); 4481 } 4482 4483 static void 4484 mmu_radix_pinit0(pmap_t pmap) 4485 { 4486 4487 CTR2(KTR_PMAP, "%s(%p)", __func__, pmap); 4488 PMAP_LOCK_INIT(pmap); 4489 pmap->pm_pml1 = kernel_pmap->pm_pml1; 4490 pmap->pm_pid = kernel_pmap->pm_pid; 4491 4492 vm_radix_init(&pmap->pm_radix); 4493 TAILQ_INIT(&pmap->pm_pvchunk); 4494 bzero(&pmap->pm_stats, sizeof pmap->pm_stats); 4495 kernel_pmap->pm_flags = 4496 pmap->pm_flags = PMAP_PDE_SUPERPAGE; 4497 } 4498 /* 4499 * pmap_protect_l3e: do the things to protect a 2mpage in a process 4500 */ 4501 static bool 4502 pmap_protect_l3e(pmap_t pmap, pt_entry_t *l3e, vm_offset_t sva, vm_prot_t prot) 4503 { 4504 pt_entry_t newpde, oldpde; 4505 vm_offset_t eva, va; 4506 vm_page_t m; 4507 bool anychanged; 4508 4509 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 4510 KASSERT((sva & L3_PAGE_MASK) == 0, 4511 ("pmap_protect_l3e: sva is not 2mpage aligned")); 4512 anychanged = false; 4513 retry: 4514 oldpde = newpde = be64toh(*l3e); 4515 if ((oldpde & (PG_MANAGED | PG_M | PG_RW)) == 4516 (PG_MANAGED | PG_M | PG_RW)) { 4517 eva = sva + L3_PAGE_SIZE; 4518 for (va = sva, m = PHYS_TO_VM_PAGE(oldpde & PG_PS_FRAME); 4519 va < eva; va += PAGE_SIZE, m++) 4520 vm_page_dirty(m); 4521 } 4522 if ((prot & VM_PROT_WRITE) == 0) { 4523 newpde &= ~(PG_RW | PG_M); 4524 newpde |= RPTE_EAA_R; 4525 } 4526 if (prot & VM_PROT_EXECUTE) 4527 newpde |= PG_X; 4528 if (newpde != oldpde) { 4529 /* 4530 * As an optimization to future operations on this PDE, clear 4531 * PG_PROMOTED. The impending invalidation will remove any 4532 * lingering 4KB page mappings from the TLB. 4533 */ 4534 if (!atomic_cmpset_long(l3e, htobe64(oldpde), htobe64(newpde & ~PG_PROMOTED))) 4535 goto retry; 4536 anychanged = true; 4537 } 4538 return (anychanged); 4539 } 4540 4541 void 4542 mmu_radix_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, 4543 vm_prot_t prot) 4544 { 4545 vm_offset_t va_next; 4546 pml1_entry_t *l1e; 4547 pml2_entry_t *l2e; 4548 pml3_entry_t ptpaddr, *l3e; 4549 pt_entry_t *pte; 4550 bool anychanged; 4551 4552 CTR5(KTR_PMAP, "%s(%p, %#x, %#x, %#x)", __func__, pmap, sva, eva, 4553 prot); 4554 4555 KASSERT((prot & ~VM_PROT_ALL) == 0, ("invalid prot %x", prot)); 4556 if (prot == VM_PROT_NONE) { 4557 mmu_radix_remove(pmap, sva, eva); 4558 return; 4559 } 4560 4561 if ((prot & (VM_PROT_WRITE|VM_PROT_EXECUTE)) == 4562 (VM_PROT_WRITE|VM_PROT_EXECUTE)) 4563 return; 4564 4565 #ifdef INVARIANTS 4566 if (VERBOSE_PROTECT || pmap_logging) 4567 printf("pmap_protect(%p, %#lx, %#lx, %x) - asid: %lu\n", 4568 pmap, sva, eva, prot, pmap->pm_pid); 4569 #endif 4570 anychanged = false; 4571 4572 PMAP_LOCK(pmap); 4573 for (; sva < eva; sva = va_next) { 4574 l1e = pmap_pml1e(pmap, sva); 4575 if ((be64toh(*l1e) & PG_V) == 0) { 4576 va_next = (sva + L1_PAGE_SIZE) & ~L1_PAGE_MASK; 4577 if (va_next < sva) 4578 va_next = eva; 4579 continue; 4580 } 4581 4582 l2e = pmap_l1e_to_l2e(l1e, sva); 4583 if ((be64toh(*l2e) & PG_V) == 0) { 4584 va_next = (sva + L2_PAGE_SIZE) & ~L2_PAGE_MASK; 4585 if (va_next < sva) 4586 va_next = eva; 4587 continue; 4588 } 4589 4590 va_next = (sva + L3_PAGE_SIZE) & ~L3_PAGE_MASK; 4591 if (va_next < sva) 4592 va_next = eva; 4593 4594 l3e = pmap_l2e_to_l3e(l2e, sva); 4595 ptpaddr = be64toh(*l3e); 4596 4597 /* 4598 * Weed out invalid mappings. 4599 */ 4600 if (ptpaddr == 0) 4601 continue; 4602 4603 /* 4604 * Check for large page. 4605 */ 4606 if ((ptpaddr & RPTE_LEAF) != 0) { 4607 /* 4608 * Are we protecting the entire large page? If not, 4609 * demote the mapping and fall through. 4610 */ 4611 if (sva + L3_PAGE_SIZE == va_next && eva >= va_next) { 4612 if (pmap_protect_l3e(pmap, l3e, sva, prot)) 4613 anychanged = true; 4614 continue; 4615 } else if (!pmap_demote_l3e(pmap, l3e, sva)) { 4616 /* 4617 * The large page mapping was destroyed. 4618 */ 4619 continue; 4620 } 4621 } 4622 4623 if (va_next > eva) 4624 va_next = eva; 4625 4626 for (pte = pmap_l3e_to_pte(l3e, sva); sva != va_next; pte++, 4627 sva += PAGE_SIZE) { 4628 pt_entry_t obits, pbits; 4629 vm_page_t m; 4630 4631 retry: 4632 MPASS(pte == pmap_pte(pmap, sva)); 4633 obits = pbits = be64toh(*pte); 4634 if ((pbits & PG_V) == 0) 4635 continue; 4636 4637 if ((prot & VM_PROT_WRITE) == 0) { 4638 if ((pbits & (PG_MANAGED | PG_M | PG_RW)) == 4639 (PG_MANAGED | PG_M | PG_RW)) { 4640 m = PHYS_TO_VM_PAGE(pbits & PG_FRAME); 4641 vm_page_dirty(m); 4642 } 4643 pbits &= ~(PG_RW | PG_M); 4644 pbits |= RPTE_EAA_R; 4645 } 4646 if (prot & VM_PROT_EXECUTE) 4647 pbits |= PG_X; 4648 4649 if (pbits != obits) { 4650 if (!atomic_cmpset_long(pte, htobe64(obits), htobe64(pbits))) 4651 goto retry; 4652 if (obits & (PG_A|PG_M)) { 4653 anychanged = true; 4654 #ifdef INVARIANTS 4655 if (VERBOSE_PROTECT || pmap_logging) 4656 printf("%#lx %#lx -> %#lx\n", 4657 sva, obits, pbits); 4658 #endif 4659 } 4660 } 4661 } 4662 } 4663 if (anychanged) 4664 pmap_invalidate_all(pmap); 4665 PMAP_UNLOCK(pmap); 4666 } 4667 4668 void 4669 mmu_radix_qenter(vm_offset_t sva, vm_page_t *ma, int count) 4670 { 4671 4672 CTR4(KTR_PMAP, "%s(%#x, %p, %d)", __func__, sva, ma, count); 4673 pt_entry_t oldpte, pa, *pte; 4674 vm_page_t m; 4675 uint64_t cache_bits, attr_bits; 4676 vm_offset_t va; 4677 4678 oldpte = 0; 4679 attr_bits = RPTE_EAA_R | RPTE_EAA_W | RPTE_EAA_P | PG_M | PG_A; 4680 va = sva; 4681 pte = kvtopte(va); 4682 while (va < sva + PAGE_SIZE * count) { 4683 if (__predict_false((va & L3_PAGE_MASK) == 0)) 4684 pte = kvtopte(va); 4685 MPASS(pte == pmap_pte(kernel_pmap, va)); 4686 4687 /* 4688 * XXX there has to be a more efficient way than traversing 4689 * the page table every time - but go for correctness for 4690 * today 4691 */ 4692 4693 m = *ma++; 4694 cache_bits = pmap_cache_bits(m->md.mdpg_cache_attrs); 4695 pa = VM_PAGE_TO_PHYS(m) | cache_bits | attr_bits; 4696 if (be64toh(*pte) != pa) { 4697 oldpte |= be64toh(*pte); 4698 pte_store(pte, pa); 4699 } 4700 va += PAGE_SIZE; 4701 pte++; 4702 } 4703 if (__predict_false((oldpte & RPTE_VALID) != 0)) 4704 pmap_invalidate_range(kernel_pmap, sva, sva + count * 4705 PAGE_SIZE); 4706 else 4707 ptesync(); 4708 } 4709 4710 void 4711 mmu_radix_qremove(vm_offset_t sva, int count) 4712 { 4713 vm_offset_t va; 4714 pt_entry_t *pte; 4715 4716 CTR3(KTR_PMAP, "%s(%#x, %d)", __func__, sva, count); 4717 KASSERT(sva >= VM_MIN_KERNEL_ADDRESS, ("usermode or dmap va %lx", sva)); 4718 4719 va = sva; 4720 pte = kvtopte(va); 4721 while (va < sva + PAGE_SIZE * count) { 4722 if (__predict_false((va & L3_PAGE_MASK) == 0)) 4723 pte = kvtopte(va); 4724 pte_clear(pte); 4725 pte++; 4726 va += PAGE_SIZE; 4727 } 4728 pmap_invalidate_range(kernel_pmap, sva, va); 4729 } 4730 4731 /*************************************************** 4732 * Page table page management routines..... 4733 ***************************************************/ 4734 /* 4735 * Schedule the specified unused page table page to be freed. Specifically, 4736 * add the page to the specified list of pages that will be released to the 4737 * physical memory manager after the TLB has been updated. 4738 */ 4739 static __inline void 4740 pmap_add_delayed_free_list(vm_page_t m, struct spglist *free, bool set_PG_ZERO) 4741 { 4742 4743 if (set_PG_ZERO) 4744 m->flags |= PG_ZERO; 4745 else 4746 m->flags &= ~PG_ZERO; 4747 SLIST_INSERT_HEAD(free, m, plinks.s.ss); 4748 } 4749 4750 /* 4751 * Inserts the specified page table page into the specified pmap's collection 4752 * of idle page table pages. Each of a pmap's page table pages is responsible 4753 * for mapping a distinct range of virtual addresses. The pmap's collection is 4754 * ordered by this virtual address range. 4755 */ 4756 static __inline int 4757 pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte) 4758 { 4759 4760 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 4761 return (vm_radix_insert(&pmap->pm_radix, mpte)); 4762 } 4763 4764 /* 4765 * Removes the page table page mapping the specified virtual address from the 4766 * specified pmap's collection of idle page table pages, and returns it. 4767 * Otherwise, returns NULL if there is no page table page corresponding to the 4768 * specified virtual address. 4769 */ 4770 static __inline vm_page_t 4771 pmap_remove_pt_page(pmap_t pmap, vm_offset_t va) 4772 { 4773 4774 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 4775 return (vm_radix_remove(&pmap->pm_radix, pmap_l3e_pindex(va))); 4776 } 4777 4778 /* 4779 * Decrements a page table page's wire count, which is used to record the 4780 * number of valid page table entries within the page. If the wire count 4781 * drops to zero, then the page table page is unmapped. Returns true if the 4782 * page table page was unmapped and false otherwise. 4783 */ 4784 static inline bool 4785 pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free) 4786 { 4787 4788 --m->ref_count; 4789 if (m->ref_count == 0) { 4790 _pmap_unwire_ptp(pmap, va, m, free); 4791 return (true); 4792 } else 4793 return (false); 4794 } 4795 4796 static void 4797 _pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free) 4798 { 4799 4800 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 4801 /* 4802 * unmap the page table page 4803 */ 4804 if (m->pindex >= NUPDE + NUPDPE) { 4805 /* PDP page */ 4806 pml1_entry_t *pml1; 4807 pml1 = pmap_pml1e(pmap, va); 4808 *pml1 = 0; 4809 } else if (m->pindex >= NUPDE) { 4810 /* PD page */ 4811 pml2_entry_t *l2e; 4812 l2e = pmap_pml2e(pmap, va); 4813 *l2e = 0; 4814 } else { 4815 /* PTE page */ 4816 pml3_entry_t *l3e; 4817 l3e = pmap_pml3e(pmap, va); 4818 *l3e = 0; 4819 } 4820 pmap_resident_count_dec(pmap, 1); 4821 if (m->pindex < NUPDE) { 4822 /* We just released a PT, unhold the matching PD */ 4823 vm_page_t pdpg; 4824 4825 pdpg = PHYS_TO_VM_PAGE(be64toh(*pmap_pml2e(pmap, va)) & PG_FRAME); 4826 pmap_unwire_ptp(pmap, va, pdpg, free); 4827 } 4828 else if (m->pindex >= NUPDE && m->pindex < (NUPDE + NUPDPE)) { 4829 /* We just released a PD, unhold the matching PDP */ 4830 vm_page_t pdppg; 4831 4832 pdppg = PHYS_TO_VM_PAGE(be64toh(*pmap_pml1e(pmap, va)) & PG_FRAME); 4833 pmap_unwire_ptp(pmap, va, pdppg, free); 4834 } 4835 4836 /* 4837 * Put page on a list so that it is released after 4838 * *ALL* TLB shootdown is done 4839 */ 4840 pmap_add_delayed_free_list(m, free, true); 4841 } 4842 4843 /* 4844 * After removing a page table entry, this routine is used to 4845 * conditionally free the page, and manage the hold/wire counts. 4846 */ 4847 static int 4848 pmap_unuse_pt(pmap_t pmap, vm_offset_t va, pml3_entry_t ptepde, 4849 struct spglist *free) 4850 { 4851 vm_page_t mpte; 4852 4853 if (va >= VM_MAXUSER_ADDRESS) 4854 return (0); 4855 KASSERT(ptepde != 0, ("pmap_unuse_pt: ptepde != 0")); 4856 mpte = PHYS_TO_VM_PAGE(ptepde & PG_FRAME); 4857 return (pmap_unwire_ptp(pmap, va, mpte, free)); 4858 } 4859 4860 void 4861 mmu_radix_release(pmap_t pmap) 4862 { 4863 4864 CTR2(KTR_PMAP, "%s(%p)", __func__, pmap); 4865 KASSERT(pmap->pm_stats.resident_count == 0, 4866 ("pmap_release: pmap resident count %ld != 0", 4867 pmap->pm_stats.resident_count)); 4868 KASSERT(vm_radix_is_empty(&pmap->pm_radix), 4869 ("pmap_release: pmap has reserved page table page(s)")); 4870 4871 pmap_invalidate_all(pmap); 4872 isa3_proctab[pmap->pm_pid].proctab0 = 0; 4873 uma_zfree(zone_radix_pgd, pmap->pm_pml1); 4874 vmem_free(asid_arena, pmap->pm_pid, 1); 4875 } 4876 4877 /* 4878 * Create the PV entry for a 2MB page mapping. Always returns true unless the 4879 * flag PMAP_ENTER_NORECLAIM is specified. If that flag is specified, returns 4880 * false if the PV entry cannot be allocated without resorting to reclamation. 4881 */ 4882 static bool 4883 pmap_pv_insert_l3e(pmap_t pmap, vm_offset_t va, pml3_entry_t pde, u_int flags, 4884 struct rwlock **lockp) 4885 { 4886 struct md_page *pvh; 4887 pv_entry_t pv; 4888 vm_paddr_t pa; 4889 4890 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 4891 /* Pass NULL instead of the lock pointer to disable reclamation. */ 4892 if ((pv = get_pv_entry(pmap, (flags & PMAP_ENTER_NORECLAIM) != 0 ? 4893 NULL : lockp)) == NULL) 4894 return (false); 4895 pv->pv_va = va; 4896 pa = pde & PG_PS_FRAME; 4897 CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa); 4898 pvh = pa_to_pvh(pa); 4899 TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_link); 4900 pvh->pv_gen++; 4901 return (true); 4902 } 4903 4904 /* 4905 * Fills a page table page with mappings to consecutive physical pages. 4906 */ 4907 static void 4908 pmap_fill_ptp(pt_entry_t *firstpte, pt_entry_t newpte) 4909 { 4910 pt_entry_t *pte; 4911 4912 for (pte = firstpte; pte < firstpte + NPTEPG; pte++) { 4913 *pte = htobe64(newpte); 4914 newpte += PAGE_SIZE; 4915 } 4916 } 4917 4918 static bool 4919 pmap_demote_l3e(pmap_t pmap, pml3_entry_t *pde, vm_offset_t va) 4920 { 4921 struct rwlock *lock; 4922 bool rv; 4923 4924 lock = NULL; 4925 rv = pmap_demote_l3e_locked(pmap, pde, va, &lock); 4926 if (lock != NULL) 4927 rw_wunlock(lock); 4928 return (rv); 4929 } 4930 4931 static bool 4932 pmap_demote_l3e_locked(pmap_t pmap, pml3_entry_t *l3e, vm_offset_t va, 4933 struct rwlock **lockp) 4934 { 4935 pml3_entry_t oldpde; 4936 pt_entry_t *firstpte; 4937 vm_paddr_t mptepa; 4938 vm_page_t mpte; 4939 struct spglist free; 4940 vm_offset_t sva; 4941 4942 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 4943 oldpde = be64toh(*l3e); 4944 KASSERT((oldpde & (RPTE_LEAF | PG_V)) == (RPTE_LEAF | PG_V), 4945 ("pmap_demote_l3e: oldpde is missing RPTE_LEAF and/or PG_V %lx", 4946 oldpde)); 4947 if ((oldpde & PG_A) == 0 || (mpte = pmap_remove_pt_page(pmap, va)) == 4948 NULL) { 4949 KASSERT((oldpde & PG_W) == 0, 4950 ("pmap_demote_l3e: page table page for a wired mapping" 4951 " is missing")); 4952 4953 /* 4954 * Invalidate the 2MB page mapping and return "failure" if the 4955 * mapping was never accessed or the allocation of the new 4956 * page table page fails. If the 2MB page mapping belongs to 4957 * the direct map region of the kernel's address space, then 4958 * the page allocation request specifies the highest possible 4959 * priority (VM_ALLOC_INTERRUPT). Otherwise, the priority is 4960 * normal. Page table pages are preallocated for every other 4961 * part of the kernel address space, so the direct map region 4962 * is the only part of the kernel address space that must be 4963 * handled here. 4964 */ 4965 if ((oldpde & PG_A) == 0 || (mpte = vm_page_alloc_noobj( 4966 (va >= DMAP_MIN_ADDRESS && va < DMAP_MAX_ADDRESS ? 4967 VM_ALLOC_INTERRUPT : 0) | VM_ALLOC_WIRED)) == NULL) { 4968 SLIST_INIT(&free); 4969 sva = trunc_2mpage(va); 4970 pmap_remove_l3e(pmap, l3e, sva, &free, lockp); 4971 pmap_invalidate_l3e_page(pmap, sva, oldpde); 4972 vm_page_free_pages_toq(&free, true); 4973 CTR2(KTR_PMAP, "pmap_demote_l3e: failure for va %#lx" 4974 " in pmap %p", va, pmap); 4975 return (false); 4976 } 4977 mpte->pindex = pmap_l3e_pindex(va); 4978 if (va < VM_MAXUSER_ADDRESS) 4979 pmap_resident_count_inc(pmap, 1); 4980 } 4981 mptepa = VM_PAGE_TO_PHYS(mpte); 4982 firstpte = (pt_entry_t *)PHYS_TO_DMAP(mptepa); 4983 KASSERT((oldpde & PG_A) != 0, 4984 ("pmap_demote_l3e: oldpde is missing PG_A")); 4985 KASSERT((oldpde & (PG_M | PG_RW)) != PG_RW, 4986 ("pmap_demote_l3e: oldpde is missing PG_M")); 4987 4988 /* 4989 * If the page table page is new, initialize it. 4990 */ 4991 if (mpte->ref_count == 1) { 4992 mpte->ref_count = NPTEPG; 4993 pmap_fill_ptp(firstpte, oldpde); 4994 } 4995 4996 KASSERT((be64toh(*firstpte) & PG_FRAME) == (oldpde & PG_FRAME), 4997 ("pmap_demote_l3e: firstpte and newpte map different physical" 4998 " addresses")); 4999 5000 /* 5001 * If the mapping has changed attributes, update the page table 5002 * entries. 5003 */ 5004 if ((be64toh(*firstpte) & PG_PTE_PROMOTE) != (oldpde & PG_PTE_PROMOTE)) 5005 pmap_fill_ptp(firstpte, oldpde); 5006 5007 /* 5008 * The spare PV entries must be reserved prior to demoting the 5009 * mapping, that is, prior to changing the PDE. Otherwise, the state 5010 * of the PDE and the PV lists will be inconsistent, which can result 5011 * in reclaim_pv_chunk() attempting to remove a PV entry from the 5012 * wrong PV list and pmap_pv_demote_l3e() failing to find the expected 5013 * PV entry for the 2MB page mapping that is being demoted. 5014 */ 5015 if ((oldpde & PG_MANAGED) != 0) 5016 reserve_pv_entries(pmap, NPTEPG - 1, lockp); 5017 5018 /* 5019 * Demote the mapping. This pmap is locked. The old PDE has 5020 * PG_A set. If the old PDE has PG_RW set, it also has PG_M 5021 * set. Thus, there is no danger of a race with another 5022 * processor changing the setting of PG_A and/or PG_M between 5023 * the read above and the store below. 5024 */ 5025 pde_store(l3e, mptepa); 5026 pmap_invalidate_l3e_page(pmap, trunc_2mpage(va), oldpde); 5027 /* 5028 * Demote the PV entry. 5029 */ 5030 if ((oldpde & PG_MANAGED) != 0) 5031 pmap_pv_demote_l3e(pmap, va, oldpde & PG_PS_FRAME, lockp); 5032 5033 counter_u64_add(pmap_l3e_demotions, 1); 5034 CTR2(KTR_PMAP, "pmap_demote_l3e: success for va %#lx" 5035 " in pmap %p", va, pmap); 5036 return (true); 5037 } 5038 5039 /* 5040 * pmap_remove_kernel_pde: Remove a kernel superpage mapping. 5041 */ 5042 static void 5043 pmap_remove_kernel_l3e(pmap_t pmap, pml3_entry_t *l3e, vm_offset_t va) 5044 { 5045 vm_paddr_t mptepa; 5046 vm_page_t mpte; 5047 5048 KASSERT(pmap == kernel_pmap, ("pmap %p is not kernel_pmap", pmap)); 5049 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 5050 mpte = pmap_remove_pt_page(pmap, va); 5051 if (mpte == NULL) 5052 panic("pmap_remove_kernel_pde: Missing pt page."); 5053 5054 mptepa = VM_PAGE_TO_PHYS(mpte); 5055 5056 /* 5057 * Initialize the page table page. 5058 */ 5059 pagezero(PHYS_TO_DMAP(mptepa)); 5060 5061 /* 5062 * Demote the mapping. 5063 */ 5064 pde_store(l3e, mptepa); 5065 ptesync(); 5066 } 5067 5068 /* 5069 * pmap_remove_l3e: do the things to unmap a superpage in a process 5070 */ 5071 static int 5072 pmap_remove_l3e(pmap_t pmap, pml3_entry_t *pdq, vm_offset_t sva, 5073 struct spglist *free, struct rwlock **lockp) 5074 { 5075 struct md_page *pvh; 5076 pml3_entry_t oldpde; 5077 vm_offset_t eva, va; 5078 vm_page_t m, mpte; 5079 5080 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 5081 KASSERT((sva & L3_PAGE_MASK) == 0, 5082 ("pmap_remove_l3e: sva is not 2mpage aligned")); 5083 oldpde = be64toh(pte_load_clear(pdq)); 5084 if (oldpde & PG_W) 5085 pmap->pm_stats.wired_count -= (L3_PAGE_SIZE / PAGE_SIZE); 5086 pmap_resident_count_dec(pmap, L3_PAGE_SIZE / PAGE_SIZE); 5087 if (oldpde & PG_MANAGED) { 5088 CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, oldpde & PG_PS_FRAME); 5089 pvh = pa_to_pvh(oldpde & PG_PS_FRAME); 5090 pmap_pvh_free(pvh, pmap, sva); 5091 eva = sva + L3_PAGE_SIZE; 5092 for (va = sva, m = PHYS_TO_VM_PAGE(oldpde & PG_PS_FRAME); 5093 va < eva; va += PAGE_SIZE, m++) { 5094 if ((oldpde & (PG_M | PG_RW)) == (PG_M | PG_RW)) 5095 vm_page_dirty(m); 5096 if (oldpde & PG_A) 5097 vm_page_aflag_set(m, PGA_REFERENCED); 5098 if (TAILQ_EMPTY(&m->md.pv_list) && 5099 TAILQ_EMPTY(&pvh->pv_list)) 5100 vm_page_aflag_clear(m, PGA_WRITEABLE); 5101 } 5102 } 5103 if (pmap == kernel_pmap) { 5104 pmap_remove_kernel_l3e(pmap, pdq, sva); 5105 } else { 5106 mpte = pmap_remove_pt_page(pmap, sva); 5107 if (mpte != NULL) { 5108 pmap_resident_count_dec(pmap, 1); 5109 KASSERT(mpte->ref_count == NPTEPG, 5110 ("pmap_remove_l3e: pte page wire count error")); 5111 mpte->ref_count = 0; 5112 pmap_add_delayed_free_list(mpte, free, false); 5113 } 5114 } 5115 return (pmap_unuse_pt(pmap, sva, be64toh(*pmap_pml2e(pmap, sva)), free)); 5116 } 5117 5118 /* 5119 * pmap_remove_pte: do the things to unmap a page in a process 5120 */ 5121 static int 5122 pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t va, 5123 pml3_entry_t ptepde, struct spglist *free, struct rwlock **lockp) 5124 { 5125 struct md_page *pvh; 5126 pt_entry_t oldpte; 5127 vm_page_t m; 5128 5129 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 5130 oldpte = be64toh(pte_load_clear(ptq)); 5131 if (oldpte & RPTE_WIRED) 5132 pmap->pm_stats.wired_count -= 1; 5133 pmap_resident_count_dec(pmap, 1); 5134 if (oldpte & RPTE_MANAGED) { 5135 m = PHYS_TO_VM_PAGE(oldpte & PG_FRAME); 5136 if ((oldpte & (PG_M | PG_RW)) == (PG_M | PG_RW)) 5137 vm_page_dirty(m); 5138 if (oldpte & PG_A) 5139 vm_page_aflag_set(m, PGA_REFERENCED); 5140 CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m); 5141 pmap_pvh_free(&m->md, pmap, va); 5142 if (TAILQ_EMPTY(&m->md.pv_list) && 5143 (m->flags & PG_FICTITIOUS) == 0) { 5144 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); 5145 if (TAILQ_EMPTY(&pvh->pv_list)) 5146 vm_page_aflag_clear(m, PGA_WRITEABLE); 5147 } 5148 } 5149 return (pmap_unuse_pt(pmap, va, ptepde, free)); 5150 } 5151 5152 /* 5153 * Remove a single page from a process address space 5154 */ 5155 static bool 5156 pmap_remove_page(pmap_t pmap, vm_offset_t va, pml3_entry_t *l3e, 5157 struct spglist *free) 5158 { 5159 struct rwlock *lock; 5160 pt_entry_t *pte; 5161 bool invalidate_all; 5162 5163 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 5164 if ((be64toh(*l3e) & RPTE_VALID) == 0) { 5165 return (false); 5166 } 5167 pte = pmap_l3e_to_pte(l3e, va); 5168 if ((be64toh(*pte) & RPTE_VALID) == 0) { 5169 return (false); 5170 } 5171 lock = NULL; 5172 5173 invalidate_all = pmap_remove_pte(pmap, pte, va, be64toh(*l3e), free, &lock); 5174 if (lock != NULL) 5175 rw_wunlock(lock); 5176 if (!invalidate_all) 5177 pmap_invalidate_page(pmap, va); 5178 return (invalidate_all); 5179 } 5180 5181 /* 5182 * Removes the specified range of addresses from the page table page. 5183 */ 5184 static bool 5185 pmap_remove_ptes(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, 5186 pml3_entry_t *l3e, struct spglist *free, struct rwlock **lockp) 5187 { 5188 pt_entry_t *pte; 5189 vm_offset_t va; 5190 bool anyvalid; 5191 5192 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 5193 anyvalid = false; 5194 va = eva; 5195 for (pte = pmap_l3e_to_pte(l3e, sva); sva != eva; pte++, 5196 sva += PAGE_SIZE) { 5197 MPASS(pte == pmap_pte(pmap, sva)); 5198 if (*pte == 0) { 5199 if (va != eva) { 5200 anyvalid = true; 5201 va = eva; 5202 } 5203 continue; 5204 } 5205 if (va == eva) 5206 va = sva; 5207 if (pmap_remove_pte(pmap, pte, sva, be64toh(*l3e), free, lockp)) { 5208 anyvalid = true; 5209 sva += PAGE_SIZE; 5210 break; 5211 } 5212 } 5213 if (anyvalid) 5214 pmap_invalidate_all(pmap); 5215 else if (va != eva) 5216 pmap_invalidate_range(pmap, va, sva); 5217 return (anyvalid); 5218 } 5219 5220 void 5221 mmu_radix_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) 5222 { 5223 struct rwlock *lock; 5224 vm_offset_t va_next; 5225 pml1_entry_t *l1e; 5226 pml2_entry_t *l2e; 5227 pml3_entry_t ptpaddr, *l3e; 5228 struct spglist free; 5229 bool anyvalid; 5230 5231 CTR4(KTR_PMAP, "%s(%p, %#x, %#x)", __func__, pmap, sva, eva); 5232 5233 /* 5234 * Perform an unsynchronized read. This is, however, safe. 5235 */ 5236 if (pmap->pm_stats.resident_count == 0) 5237 return; 5238 5239 anyvalid = false; 5240 SLIST_INIT(&free); 5241 5242 /* XXX something fishy here */ 5243 sva = (sva + PAGE_MASK) & ~PAGE_MASK; 5244 eva = (eva + PAGE_MASK) & ~PAGE_MASK; 5245 5246 PMAP_LOCK(pmap); 5247 5248 /* 5249 * special handling of removing one page. a very 5250 * common operation and easy to short circuit some 5251 * code. 5252 */ 5253 if (sva + PAGE_SIZE == eva) { 5254 l3e = pmap_pml3e(pmap, sva); 5255 if (l3e && (be64toh(*l3e) & RPTE_LEAF) == 0) { 5256 anyvalid = pmap_remove_page(pmap, sva, l3e, &free); 5257 goto out; 5258 } 5259 } 5260 5261 lock = NULL; 5262 for (; sva < eva; sva = va_next) { 5263 if (pmap->pm_stats.resident_count == 0) 5264 break; 5265 l1e = pmap_pml1e(pmap, sva); 5266 if (l1e == NULL || (be64toh(*l1e) & PG_V) == 0) { 5267 va_next = (sva + L1_PAGE_SIZE) & ~L1_PAGE_MASK; 5268 if (va_next < sva) 5269 va_next = eva; 5270 continue; 5271 } 5272 5273 l2e = pmap_l1e_to_l2e(l1e, sva); 5274 if (l2e == NULL || (be64toh(*l2e) & PG_V) == 0) { 5275 va_next = (sva + L2_PAGE_SIZE) & ~L2_PAGE_MASK; 5276 if (va_next < sva) 5277 va_next = eva; 5278 continue; 5279 } 5280 5281 /* 5282 * Calculate index for next page table. 5283 */ 5284 va_next = (sva + L3_PAGE_SIZE) & ~L3_PAGE_MASK; 5285 if (va_next < sva) 5286 va_next = eva; 5287 5288 l3e = pmap_l2e_to_l3e(l2e, sva); 5289 ptpaddr = be64toh(*l3e); 5290 5291 /* 5292 * Weed out invalid mappings. 5293 */ 5294 if (ptpaddr == 0) 5295 continue; 5296 5297 /* 5298 * Check for large page. 5299 */ 5300 if ((ptpaddr & RPTE_LEAF) != 0) { 5301 /* 5302 * Are we removing the entire large page? If not, 5303 * demote the mapping and fall through. 5304 */ 5305 if (sva + L3_PAGE_SIZE == va_next && eva >= va_next) { 5306 pmap_remove_l3e(pmap, l3e, sva, &free, &lock); 5307 anyvalid = true; 5308 continue; 5309 } else if (!pmap_demote_l3e_locked(pmap, l3e, sva, 5310 &lock)) { 5311 /* The large page mapping was destroyed. */ 5312 continue; 5313 } else 5314 ptpaddr = be64toh(*l3e); 5315 } 5316 5317 /* 5318 * Limit our scan to either the end of the va represented 5319 * by the current page table page, or to the end of the 5320 * range being removed. 5321 */ 5322 if (va_next > eva) 5323 va_next = eva; 5324 5325 if (pmap_remove_ptes(pmap, sva, va_next, l3e, &free, &lock)) 5326 anyvalid = true; 5327 } 5328 if (lock != NULL) 5329 rw_wunlock(lock); 5330 out: 5331 if (anyvalid) 5332 pmap_invalidate_all(pmap); 5333 PMAP_UNLOCK(pmap); 5334 vm_page_free_pages_toq(&free, true); 5335 } 5336 5337 void 5338 mmu_radix_remove_all(vm_page_t m) 5339 { 5340 struct md_page *pvh; 5341 pv_entry_t pv; 5342 pmap_t pmap; 5343 struct rwlock *lock; 5344 pt_entry_t *pte, tpte; 5345 pml3_entry_t *l3e; 5346 vm_offset_t va; 5347 struct spglist free; 5348 int pvh_gen, md_gen; 5349 5350 CTR2(KTR_PMAP, "%s(%p)", __func__, m); 5351 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 5352 ("pmap_remove_all: page %p is not managed", m)); 5353 SLIST_INIT(&free); 5354 lock = VM_PAGE_TO_PV_LIST_LOCK(m); 5355 pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy : 5356 pa_to_pvh(VM_PAGE_TO_PHYS(m)); 5357 retry: 5358 rw_wlock(lock); 5359 while ((pv = TAILQ_FIRST(&pvh->pv_list)) != NULL) { 5360 pmap = PV_PMAP(pv); 5361 if (!PMAP_TRYLOCK(pmap)) { 5362 pvh_gen = pvh->pv_gen; 5363 rw_wunlock(lock); 5364 PMAP_LOCK(pmap); 5365 rw_wlock(lock); 5366 if (pvh_gen != pvh->pv_gen) { 5367 rw_wunlock(lock); 5368 PMAP_UNLOCK(pmap); 5369 goto retry; 5370 } 5371 } 5372 va = pv->pv_va; 5373 l3e = pmap_pml3e(pmap, va); 5374 (void)pmap_demote_l3e_locked(pmap, l3e, va, &lock); 5375 PMAP_UNLOCK(pmap); 5376 } 5377 while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) { 5378 pmap = PV_PMAP(pv); 5379 if (!PMAP_TRYLOCK(pmap)) { 5380 pvh_gen = pvh->pv_gen; 5381 md_gen = m->md.pv_gen; 5382 rw_wunlock(lock); 5383 PMAP_LOCK(pmap); 5384 rw_wlock(lock); 5385 if (pvh_gen != pvh->pv_gen || md_gen != m->md.pv_gen) { 5386 rw_wunlock(lock); 5387 PMAP_UNLOCK(pmap); 5388 goto retry; 5389 } 5390 } 5391 pmap_resident_count_dec(pmap, 1); 5392 l3e = pmap_pml3e(pmap, pv->pv_va); 5393 KASSERT((be64toh(*l3e) & RPTE_LEAF) == 0, ("pmap_remove_all: found" 5394 " a 2mpage in page %p's pv list", m)); 5395 pte = pmap_l3e_to_pte(l3e, pv->pv_va); 5396 tpte = be64toh(pte_load_clear(pte)); 5397 if (tpte & PG_W) 5398 pmap->pm_stats.wired_count--; 5399 if (tpte & PG_A) 5400 vm_page_aflag_set(m, PGA_REFERENCED); 5401 5402 /* 5403 * Update the vm_page_t clean and reference bits. 5404 */ 5405 if ((tpte & (PG_M | PG_RW)) == (PG_M | PG_RW)) 5406 vm_page_dirty(m); 5407 pmap_unuse_pt(pmap, pv->pv_va, be64toh(*l3e), &free); 5408 pmap_invalidate_page(pmap, pv->pv_va); 5409 TAILQ_REMOVE(&m->md.pv_list, pv, pv_link); 5410 m->md.pv_gen++; 5411 free_pv_entry(pmap, pv); 5412 PMAP_UNLOCK(pmap); 5413 } 5414 vm_page_aflag_clear(m, PGA_WRITEABLE); 5415 rw_wunlock(lock); 5416 vm_page_free_pages_toq(&free, true); 5417 } 5418 5419 /* 5420 * Destroy all managed, non-wired mappings in the given user-space 5421 * pmap. This pmap cannot be active on any processor besides the 5422 * caller. 5423 * 5424 * This function cannot be applied to the kernel pmap. Moreover, it 5425 * is not intended for general use. It is only to be used during 5426 * process termination. Consequently, it can be implemented in ways 5427 * that make it faster than pmap_remove(). First, it can more quickly 5428 * destroy mappings by iterating over the pmap's collection of PV 5429 * entries, rather than searching the page table. Second, it doesn't 5430 * have to test and clear the page table entries atomically, because 5431 * no processor is currently accessing the user address space. In 5432 * particular, a page table entry's dirty bit won't change state once 5433 * this function starts. 5434 * 5435 * Although this function destroys all of the pmap's managed, 5436 * non-wired mappings, it can delay and batch the invalidation of TLB 5437 * entries without calling pmap_delayed_invl_started() and 5438 * pmap_delayed_invl_finished(). Because the pmap is not active on 5439 * any other processor, none of these TLB entries will ever be used 5440 * before their eventual invalidation. Consequently, there is no need 5441 * for either pmap_remove_all() or pmap_remove_write() to wait for 5442 * that eventual TLB invalidation. 5443 */ 5444 5445 void 5446 mmu_radix_remove_pages(pmap_t pmap) 5447 { 5448 5449 CTR2(KTR_PMAP, "%s(%p)", __func__, pmap); 5450 pml3_entry_t ptel3e; 5451 pt_entry_t *pte, tpte; 5452 struct spglist free; 5453 vm_page_t m, mpte, mt; 5454 pv_entry_t pv; 5455 struct md_page *pvh; 5456 struct pv_chunk *pc, *npc; 5457 struct rwlock *lock; 5458 int64_t bit; 5459 uint64_t inuse, bitmask; 5460 int allfree, field, idx; 5461 #ifdef PV_STATS 5462 int freed; 5463 #endif 5464 bool superpage; 5465 vm_paddr_t pa; 5466 5467 /* 5468 * Assert that the given pmap is only active on the current 5469 * CPU. Unfortunately, we cannot block another CPU from 5470 * activating the pmap while this function is executing. 5471 */ 5472 KASSERT(pmap->pm_pid == mfspr(SPR_PID), 5473 ("non-current asid %lu - expected %lu", pmap->pm_pid, 5474 mfspr(SPR_PID))); 5475 5476 lock = NULL; 5477 5478 SLIST_INIT(&free); 5479 PMAP_LOCK(pmap); 5480 TAILQ_FOREACH_SAFE(pc, &pmap->pm_pvchunk, pc_list, npc) { 5481 allfree = 1; 5482 #ifdef PV_STATS 5483 freed = 0; 5484 #endif 5485 for (field = 0; field < _NPCM; field++) { 5486 inuse = ~pc->pc_map[field] & pc_freemask[field]; 5487 while (inuse != 0) { 5488 bit = cnttzd(inuse); 5489 bitmask = 1UL << bit; 5490 idx = field * 64 + bit; 5491 pv = &pc->pc_pventry[idx]; 5492 inuse &= ~bitmask; 5493 5494 pte = pmap_pml2e(pmap, pv->pv_va); 5495 ptel3e = be64toh(*pte); 5496 pte = pmap_l2e_to_l3e(pte, pv->pv_va); 5497 tpte = be64toh(*pte); 5498 if ((tpte & (RPTE_LEAF | PG_V)) == PG_V) { 5499 superpage = false; 5500 ptel3e = tpte; 5501 pte = (pt_entry_t *)PHYS_TO_DMAP(tpte & 5502 PG_FRAME); 5503 pte = &pte[pmap_pte_index(pv->pv_va)]; 5504 tpte = be64toh(*pte); 5505 } else { 5506 /* 5507 * Keep track whether 'tpte' is a 5508 * superpage explicitly instead of 5509 * relying on RPTE_LEAF being set. 5510 * 5511 * This is because RPTE_LEAF is numerically 5512 * identical to PG_PTE_PAT and thus a 5513 * regular page could be mistaken for 5514 * a superpage. 5515 */ 5516 superpage = true; 5517 } 5518 5519 if ((tpte & PG_V) == 0) { 5520 panic("bad pte va %lx pte %lx", 5521 pv->pv_va, tpte); 5522 } 5523 5524 /* 5525 * We cannot remove wired pages from a process' mapping at this time 5526 */ 5527 if (tpte & PG_W) { 5528 allfree = 0; 5529 continue; 5530 } 5531 5532 if (superpage) 5533 pa = tpte & PG_PS_FRAME; 5534 else 5535 pa = tpte & PG_FRAME; 5536 5537 m = PHYS_TO_VM_PAGE(pa); 5538 KASSERT(m->phys_addr == pa, 5539 ("vm_page_t %p phys_addr mismatch %016jx %016jx", 5540 m, (uintmax_t)m->phys_addr, 5541 (uintmax_t)tpte)); 5542 5543 KASSERT((m->flags & PG_FICTITIOUS) != 0 || 5544 m < &vm_page_array[vm_page_array_size], 5545 ("pmap_remove_pages: bad tpte %#jx", 5546 (uintmax_t)tpte)); 5547 5548 pte_clear(pte); 5549 5550 /* 5551 * Update the vm_page_t clean/reference bits. 5552 */ 5553 if ((tpte & (PG_M | PG_RW)) == (PG_M | PG_RW)) { 5554 if (superpage) { 5555 for (mt = m; mt < &m[L3_PAGE_SIZE / PAGE_SIZE]; mt++) 5556 vm_page_dirty(mt); 5557 } else 5558 vm_page_dirty(m); 5559 } 5560 5561 CHANGE_PV_LIST_LOCK_TO_VM_PAGE(&lock, m); 5562 5563 /* Mark free */ 5564 pc->pc_map[field] |= bitmask; 5565 if (superpage) { 5566 pmap_resident_count_dec(pmap, L3_PAGE_SIZE / PAGE_SIZE); 5567 pvh = pa_to_pvh(tpte & PG_PS_FRAME); 5568 TAILQ_REMOVE(&pvh->pv_list, pv, pv_link); 5569 pvh->pv_gen++; 5570 if (TAILQ_EMPTY(&pvh->pv_list)) { 5571 for (mt = m; mt < &m[L3_PAGE_SIZE / PAGE_SIZE]; mt++) 5572 if ((mt->a.flags & PGA_WRITEABLE) != 0 && 5573 TAILQ_EMPTY(&mt->md.pv_list)) 5574 vm_page_aflag_clear(mt, PGA_WRITEABLE); 5575 } 5576 mpte = pmap_remove_pt_page(pmap, pv->pv_va); 5577 if (mpte != NULL) { 5578 pmap_resident_count_dec(pmap, 1); 5579 KASSERT(mpte->ref_count == NPTEPG, 5580 ("pmap_remove_pages: pte page wire count error")); 5581 mpte->ref_count = 0; 5582 pmap_add_delayed_free_list(mpte, &free, false); 5583 } 5584 } else { 5585 pmap_resident_count_dec(pmap, 1); 5586 #ifdef VERBOSE_PV 5587 printf("freeing pv (%p, %p)\n", 5588 pmap, pv); 5589 #endif 5590 TAILQ_REMOVE(&m->md.pv_list, pv, pv_link); 5591 m->md.pv_gen++; 5592 if ((m->a.flags & PGA_WRITEABLE) != 0 && 5593 TAILQ_EMPTY(&m->md.pv_list) && 5594 (m->flags & PG_FICTITIOUS) == 0) { 5595 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); 5596 if (TAILQ_EMPTY(&pvh->pv_list)) 5597 vm_page_aflag_clear(m, PGA_WRITEABLE); 5598 } 5599 } 5600 pmap_unuse_pt(pmap, pv->pv_va, ptel3e, &free); 5601 #ifdef PV_STATS 5602 freed++; 5603 #endif 5604 } 5605 } 5606 PV_STAT(atomic_add_long(&pv_entry_frees, freed)); 5607 PV_STAT(atomic_add_int(&pv_entry_spare, freed)); 5608 PV_STAT(atomic_subtract_long(&pv_entry_count, freed)); 5609 if (allfree) { 5610 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 5611 free_pv_chunk(pc); 5612 } 5613 } 5614 if (lock != NULL) 5615 rw_wunlock(lock); 5616 pmap_invalidate_all(pmap); 5617 PMAP_UNLOCK(pmap); 5618 vm_page_free_pages_toq(&free, true); 5619 } 5620 5621 void 5622 mmu_radix_remove_write(vm_page_t m) 5623 { 5624 struct md_page *pvh; 5625 pmap_t pmap; 5626 struct rwlock *lock; 5627 pv_entry_t next_pv, pv; 5628 pml3_entry_t *l3e; 5629 pt_entry_t oldpte, *pte; 5630 int pvh_gen, md_gen; 5631 5632 CTR2(KTR_PMAP, "%s(%p)", __func__, m); 5633 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 5634 ("pmap_remove_write: page %p is not managed", m)); 5635 vm_page_assert_busied(m); 5636 5637 if (!pmap_page_is_write_mapped(m)) 5638 return; 5639 lock = VM_PAGE_TO_PV_LIST_LOCK(m); 5640 pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy : 5641 pa_to_pvh(VM_PAGE_TO_PHYS(m)); 5642 retry_pv_loop: 5643 rw_wlock(lock); 5644 TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_link, next_pv) { 5645 pmap = PV_PMAP(pv); 5646 if (!PMAP_TRYLOCK(pmap)) { 5647 pvh_gen = pvh->pv_gen; 5648 rw_wunlock(lock); 5649 PMAP_LOCK(pmap); 5650 rw_wlock(lock); 5651 if (pvh_gen != pvh->pv_gen) { 5652 PMAP_UNLOCK(pmap); 5653 rw_wunlock(lock); 5654 goto retry_pv_loop; 5655 } 5656 } 5657 l3e = pmap_pml3e(pmap, pv->pv_va); 5658 if ((be64toh(*l3e) & PG_RW) != 0) 5659 (void)pmap_demote_l3e_locked(pmap, l3e, pv->pv_va, &lock); 5660 KASSERT(lock == VM_PAGE_TO_PV_LIST_LOCK(m), 5661 ("inconsistent pv lock %p %p for page %p", 5662 lock, VM_PAGE_TO_PV_LIST_LOCK(m), m)); 5663 PMAP_UNLOCK(pmap); 5664 } 5665 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) { 5666 pmap = PV_PMAP(pv); 5667 if (!PMAP_TRYLOCK(pmap)) { 5668 pvh_gen = pvh->pv_gen; 5669 md_gen = m->md.pv_gen; 5670 rw_wunlock(lock); 5671 PMAP_LOCK(pmap); 5672 rw_wlock(lock); 5673 if (pvh_gen != pvh->pv_gen || 5674 md_gen != m->md.pv_gen) { 5675 PMAP_UNLOCK(pmap); 5676 rw_wunlock(lock); 5677 goto retry_pv_loop; 5678 } 5679 } 5680 l3e = pmap_pml3e(pmap, pv->pv_va); 5681 KASSERT((be64toh(*l3e) & RPTE_LEAF) == 0, 5682 ("pmap_remove_write: found a 2mpage in page %p's pv list", 5683 m)); 5684 pte = pmap_l3e_to_pte(l3e, pv->pv_va); 5685 retry: 5686 oldpte = be64toh(*pte); 5687 if (oldpte & PG_RW) { 5688 if (!atomic_cmpset_long(pte, htobe64(oldpte), 5689 htobe64((oldpte | RPTE_EAA_R) & ~(PG_RW | PG_M)))) 5690 goto retry; 5691 if ((oldpte & PG_M) != 0) 5692 vm_page_dirty(m); 5693 pmap_invalidate_page(pmap, pv->pv_va); 5694 } 5695 PMAP_UNLOCK(pmap); 5696 } 5697 rw_wunlock(lock); 5698 vm_page_aflag_clear(m, PGA_WRITEABLE); 5699 } 5700 5701 /* 5702 * Clear the wired attribute from the mappings for the specified range of 5703 * addresses in the given pmap. Every valid mapping within that range 5704 * must have the wired attribute set. In contrast, invalid mappings 5705 * cannot have the wired attribute set, so they are ignored. 5706 * 5707 * The wired attribute of the page table entry is not a hardware 5708 * feature, so there is no need to invalidate any TLB entries. 5709 * Since pmap_demote_l3e() for the wired entry must never fail, 5710 * pmap_delayed_invl_started()/finished() calls around the 5711 * function are not needed. 5712 */ 5713 void 5714 mmu_radix_unwire(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) 5715 { 5716 vm_offset_t va_next; 5717 pml1_entry_t *l1e; 5718 pml2_entry_t *l2e; 5719 pml3_entry_t *l3e; 5720 pt_entry_t *pte; 5721 5722 CTR4(KTR_PMAP, "%s(%p, %#x, %#x)", __func__, pmap, sva, eva); 5723 PMAP_LOCK(pmap); 5724 for (; sva < eva; sva = va_next) { 5725 l1e = pmap_pml1e(pmap, sva); 5726 if ((be64toh(*l1e) & PG_V) == 0) { 5727 va_next = (sva + L1_PAGE_SIZE) & ~L1_PAGE_MASK; 5728 if (va_next < sva) 5729 va_next = eva; 5730 continue; 5731 } 5732 l2e = pmap_l1e_to_l2e(l1e, sva); 5733 if ((be64toh(*l2e) & PG_V) == 0) { 5734 va_next = (sva + L2_PAGE_SIZE) & ~L2_PAGE_MASK; 5735 if (va_next < sva) 5736 va_next = eva; 5737 continue; 5738 } 5739 va_next = (sva + L3_PAGE_SIZE) & ~L3_PAGE_MASK; 5740 if (va_next < sva) 5741 va_next = eva; 5742 l3e = pmap_l2e_to_l3e(l2e, sva); 5743 if ((be64toh(*l3e) & PG_V) == 0) 5744 continue; 5745 if ((be64toh(*l3e) & RPTE_LEAF) != 0) { 5746 if ((be64toh(*l3e) & PG_W) == 0) 5747 panic("pmap_unwire: pde %#jx is missing PG_W", 5748 (uintmax_t)(be64toh(*l3e))); 5749 5750 /* 5751 * Are we unwiring the entire large page? If not, 5752 * demote the mapping and fall through. 5753 */ 5754 if (sva + L3_PAGE_SIZE == va_next && eva >= va_next) { 5755 atomic_clear_long(l3e, htobe64(PG_W)); 5756 pmap->pm_stats.wired_count -= L3_PAGE_SIZE / 5757 PAGE_SIZE; 5758 continue; 5759 } else if (!pmap_demote_l3e(pmap, l3e, sva)) 5760 panic("pmap_unwire: demotion failed"); 5761 } 5762 if (va_next > eva) 5763 va_next = eva; 5764 for (pte = pmap_l3e_to_pte(l3e, sva); sva != va_next; pte++, 5765 sva += PAGE_SIZE) { 5766 MPASS(pte == pmap_pte(pmap, sva)); 5767 if ((be64toh(*pte) & PG_V) == 0) 5768 continue; 5769 if ((be64toh(*pte) & PG_W) == 0) 5770 panic("pmap_unwire: pte %#jx is missing PG_W", 5771 (uintmax_t)(be64toh(*pte))); 5772 5773 /* 5774 * PG_W must be cleared atomically. Although the pmap 5775 * lock synchronizes access to PG_W, another processor 5776 * could be setting PG_M and/or PG_A concurrently. 5777 */ 5778 atomic_clear_long(pte, htobe64(PG_W)); 5779 pmap->pm_stats.wired_count--; 5780 } 5781 } 5782 PMAP_UNLOCK(pmap); 5783 } 5784 5785 void 5786 mmu_radix_zero_page(vm_page_t m) 5787 { 5788 vm_offset_t addr; 5789 5790 CTR2(KTR_PMAP, "%s(%p)", __func__, m); 5791 addr = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)); 5792 pagezero(addr); 5793 } 5794 5795 void 5796 mmu_radix_zero_page_area(vm_page_t m, int off, int size) 5797 { 5798 caddr_t addr; 5799 5800 CTR4(KTR_PMAP, "%s(%p, %d, %d)", __func__, m, off, size); 5801 MPASS(off + size <= PAGE_SIZE); 5802 addr = (caddr_t)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)); 5803 memset(addr + off, 0, size); 5804 } 5805 5806 static int 5807 mmu_radix_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *locked_pa) 5808 { 5809 pml3_entry_t *l3ep; 5810 pt_entry_t pte; 5811 vm_paddr_t pa; 5812 int val; 5813 5814 CTR3(KTR_PMAP, "%s(%p, %#x)", __func__, pmap, addr); 5815 PMAP_LOCK(pmap); 5816 5817 l3ep = pmap_pml3e(pmap, addr); 5818 if (l3ep != NULL && (be64toh(*l3ep) & PG_V)) { 5819 if (be64toh(*l3ep) & RPTE_LEAF) { 5820 pte = be64toh(*l3ep); 5821 /* Compute the physical address of the 4KB page. */ 5822 pa = ((be64toh(*l3ep) & PG_PS_FRAME) | (addr & L3_PAGE_MASK)) & 5823 PG_FRAME; 5824 val = MINCORE_PSIND(1); 5825 } else { 5826 /* Native endian PTE, do not pass to functions */ 5827 pte = be64toh(*pmap_l3e_to_pte(l3ep, addr)); 5828 pa = pte & PG_FRAME; 5829 val = 0; 5830 } 5831 } else { 5832 pte = 0; 5833 pa = 0; 5834 val = 0; 5835 } 5836 if ((pte & PG_V) != 0) { 5837 val |= MINCORE_INCORE; 5838 if ((pte & (PG_M | PG_RW)) == (PG_M | PG_RW)) 5839 val |= MINCORE_MODIFIED | MINCORE_MODIFIED_OTHER; 5840 if ((pte & PG_A) != 0) 5841 val |= MINCORE_REFERENCED | MINCORE_REFERENCED_OTHER; 5842 } 5843 if ((val & (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER)) != 5844 (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER) && 5845 (pte & (PG_MANAGED | PG_V)) == (PG_MANAGED | PG_V)) { 5846 *locked_pa = pa; 5847 } 5848 PMAP_UNLOCK(pmap); 5849 return (val); 5850 } 5851 5852 void 5853 mmu_radix_activate(struct thread *td) 5854 { 5855 pmap_t pmap; 5856 uint32_t curpid; 5857 5858 CTR2(KTR_PMAP, "%s(%p)", __func__, td); 5859 critical_enter(); 5860 pmap = vmspace_pmap(td->td_proc->p_vmspace); 5861 curpid = mfspr(SPR_PID); 5862 if (pmap->pm_pid > isa3_base_pid && 5863 curpid != pmap->pm_pid) { 5864 mmu_radix_pid_set(pmap); 5865 } 5866 critical_exit(); 5867 } 5868 5869 /* 5870 * Increase the starting virtual address of the given mapping if a 5871 * different alignment might result in more superpage mappings. 5872 */ 5873 void 5874 mmu_radix_align_superpage(vm_object_t object, vm_ooffset_t offset, 5875 vm_offset_t *addr, vm_size_t size) 5876 { 5877 5878 CTR5(KTR_PMAP, "%s(%p, %#x, %p, %#x)", __func__, object, offset, addr, 5879 size); 5880 vm_offset_t superpage_offset; 5881 5882 if (size < L3_PAGE_SIZE) 5883 return; 5884 if (object != NULL && (object->flags & OBJ_COLORED) != 0) 5885 offset += ptoa(object->pg_color); 5886 superpage_offset = offset & L3_PAGE_MASK; 5887 if (size - ((L3_PAGE_SIZE - superpage_offset) & L3_PAGE_MASK) < L3_PAGE_SIZE || 5888 (*addr & L3_PAGE_MASK) == superpage_offset) 5889 return; 5890 if ((*addr & L3_PAGE_MASK) < superpage_offset) 5891 *addr = (*addr & ~L3_PAGE_MASK) + superpage_offset; 5892 else 5893 *addr = ((*addr + L3_PAGE_MASK) & ~L3_PAGE_MASK) + superpage_offset; 5894 } 5895 5896 static void * 5897 mmu_radix_mapdev_attr(vm_paddr_t pa, vm_size_t size, vm_memattr_t attr) 5898 { 5899 vm_offset_t va, tmpva, ppa, offset; 5900 5901 ppa = trunc_page(pa); 5902 offset = pa & PAGE_MASK; 5903 size = roundup2(offset + size, PAGE_SIZE); 5904 if (pa < powerpc_ptob(Maxmem)) 5905 panic("bad pa: %#lx less than Maxmem %#lx\n", 5906 pa, powerpc_ptob(Maxmem)); 5907 va = kva_alloc(size); 5908 if (bootverbose) 5909 printf("%s(%#lx, %lu, %d)\n", __func__, pa, size, attr); 5910 KASSERT(size > 0, ("%s(%#lx, %lu, %d)", __func__, pa, size, attr)); 5911 5912 if (!va) 5913 panic("%s: Couldn't alloc kernel virtual memory", __func__); 5914 5915 for (tmpva = va; size > 0;) { 5916 mmu_radix_kenter_attr(tmpva, ppa, attr); 5917 size -= PAGE_SIZE; 5918 tmpva += PAGE_SIZE; 5919 ppa += PAGE_SIZE; 5920 } 5921 ptesync(); 5922 5923 return ((void *)(va + offset)); 5924 } 5925 5926 static void * 5927 mmu_radix_mapdev(vm_paddr_t pa, vm_size_t size) 5928 { 5929 5930 CTR3(KTR_PMAP, "%s(%#x, %#x)", __func__, pa, size); 5931 5932 return (mmu_radix_mapdev_attr(pa, size, VM_MEMATTR_DEFAULT)); 5933 } 5934 5935 void 5936 mmu_radix_page_set_memattr(vm_page_t m, vm_memattr_t ma) 5937 { 5938 5939 CTR3(KTR_PMAP, "%s(%p, %#x)", __func__, m, ma); 5940 5941 if (m->md.mdpg_cache_attrs == ma) 5942 return; 5943 5944 m->md.mdpg_cache_attrs = ma; 5945 5946 /* 5947 * If "m" is a normal page, update its direct mapping. This update 5948 * can be relied upon to perform any cache operations that are 5949 * required for data coherence. 5950 */ 5951 if ((m->flags & PG_FICTITIOUS) == 0 && 5952 mmu_radix_change_attr(PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)), 5953 PAGE_SIZE, m->md.mdpg_cache_attrs)) 5954 panic("memory attribute change on the direct map failed"); 5955 } 5956 5957 static void 5958 mmu_radix_unmapdev(void *p, vm_size_t size) 5959 { 5960 vm_offset_t offset, va; 5961 5962 CTR3(KTR_PMAP, "%s(%p, %#x)", __func__, p, size); 5963 5964 /* If we gave a direct map region in pmap_mapdev, do nothing */ 5965 va = (vm_offset_t)p; 5966 if (va >= DMAP_MIN_ADDRESS && va < DMAP_MAX_ADDRESS) 5967 return; 5968 5969 offset = va & PAGE_MASK; 5970 size = round_page(offset + size); 5971 va = trunc_page(va); 5972 5973 if (pmap_initialized) { 5974 mmu_radix_qremove(va, atop(size)); 5975 kva_free(va, size); 5976 } 5977 } 5978 5979 void 5980 mmu_radix_sync_icache(pmap_t pm, vm_offset_t va, vm_size_t sz) 5981 { 5982 vm_paddr_t pa = 0; 5983 int sync_sz; 5984 5985 if (__predict_false(pm == NULL)) 5986 pm = &curthread->td_proc->p_vmspace->vm_pmap; 5987 5988 while (sz > 0) { 5989 pa = pmap_extract(pm, va); 5990 sync_sz = PAGE_SIZE - (va & PAGE_MASK); 5991 sync_sz = min(sync_sz, sz); 5992 if (pa != 0) { 5993 pa += (va & PAGE_MASK); 5994 __syncicache((void *)PHYS_TO_DMAP(pa), sync_sz); 5995 } 5996 va += sync_sz; 5997 sz -= sync_sz; 5998 } 5999 } 6000 6001 static __inline void 6002 pmap_pte_attr(pt_entry_t *pte, uint64_t cache_bits, uint64_t mask) 6003 { 6004 uint64_t opte, npte; 6005 6006 /* 6007 * The cache mode bits are all in the low 32-bits of the 6008 * PTE, so we can just spin on updating the low 32-bits. 6009 */ 6010 do { 6011 opte = be64toh(*pte); 6012 npte = opte & ~mask; 6013 npte |= cache_bits; 6014 } while (npte != opte && !atomic_cmpset_long(pte, htobe64(opte), htobe64(npte))); 6015 } 6016 6017 /* 6018 * Tries to demote a 1GB page mapping. 6019 */ 6020 static bool 6021 pmap_demote_l2e(pmap_t pmap, pml2_entry_t *l2e, vm_offset_t va) 6022 { 6023 pml2_entry_t oldpdpe; 6024 pml3_entry_t *firstpde, newpde, *pde; 6025 vm_paddr_t pdpgpa; 6026 vm_page_t pdpg; 6027 6028 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 6029 oldpdpe = be64toh(*l2e); 6030 KASSERT((oldpdpe & (RPTE_LEAF | PG_V)) == (RPTE_LEAF | PG_V), 6031 ("pmap_demote_pdpe: oldpdpe is missing PG_PS and/or PG_V")); 6032 pdpg = vm_page_alloc_noobj(VM_ALLOC_INTERRUPT | VM_ALLOC_WIRED); 6033 if (pdpg == NULL) { 6034 CTR2(KTR_PMAP, "pmap_demote_pdpe: failure for va %#lx" 6035 " in pmap %p", va, pmap); 6036 return (false); 6037 } 6038 pdpg->pindex = va >> L2_PAGE_SIZE_SHIFT; 6039 pdpgpa = VM_PAGE_TO_PHYS(pdpg); 6040 firstpde = (pml3_entry_t *)PHYS_TO_DMAP(pdpgpa); 6041 KASSERT((oldpdpe & PG_A) != 0, 6042 ("pmap_demote_pdpe: oldpdpe is missing PG_A")); 6043 KASSERT((oldpdpe & (PG_M | PG_RW)) != PG_RW, 6044 ("pmap_demote_pdpe: oldpdpe is missing PG_M")); 6045 newpde = oldpdpe; 6046 6047 /* 6048 * Initialize the page directory page. 6049 */ 6050 for (pde = firstpde; pde < firstpde + NPDEPG; pde++) { 6051 *pde = htobe64(newpde); 6052 newpde += L3_PAGE_SIZE; 6053 } 6054 6055 /* 6056 * Demote the mapping. 6057 */ 6058 pde_store(l2e, pdpgpa); 6059 6060 /* 6061 * Flush PWC --- XXX revisit 6062 */ 6063 pmap_invalidate_all(pmap); 6064 6065 counter_u64_add(pmap_l2e_demotions, 1); 6066 CTR2(KTR_PMAP, "pmap_demote_pdpe: success for va %#lx" 6067 " in pmap %p", va, pmap); 6068 return (true); 6069 } 6070 6071 vm_paddr_t 6072 mmu_radix_kextract(vm_offset_t va) 6073 { 6074 pml3_entry_t l3e; 6075 vm_paddr_t pa; 6076 6077 CTR2(KTR_PMAP, "%s(%#x)", __func__, va); 6078 if (va >= DMAP_MIN_ADDRESS && va < DMAP_MAX_ADDRESS) { 6079 pa = DMAP_TO_PHYS(va); 6080 } else { 6081 /* Big-endian PTE on stack */ 6082 l3e = *pmap_pml3e(kernel_pmap, va); 6083 if (be64toh(l3e) & RPTE_LEAF) { 6084 pa = (be64toh(l3e) & PG_PS_FRAME) | (va & L3_PAGE_MASK); 6085 pa |= (va & L3_PAGE_MASK); 6086 } else { 6087 /* 6088 * Beware of a concurrent promotion that changes the 6089 * PDE at this point! For example, vtopte() must not 6090 * be used to access the PTE because it would use the 6091 * new PDE. It is, however, safe to use the old PDE 6092 * because the page table page is preserved by the 6093 * promotion. 6094 */ 6095 pa = be64toh(*pmap_l3e_to_pte(&l3e, va)); 6096 pa = (pa & PG_FRAME) | (va & PAGE_MASK); 6097 pa |= (va & PAGE_MASK); 6098 } 6099 } 6100 return (pa); 6101 } 6102 6103 static pt_entry_t 6104 mmu_radix_calc_wimg(vm_paddr_t pa, vm_memattr_t ma) 6105 { 6106 6107 if (ma != VM_MEMATTR_DEFAULT) { 6108 return pmap_cache_bits(ma); 6109 } 6110 6111 /* 6112 * Assume the page is cache inhibited and access is guarded unless 6113 * it's in our available memory array. 6114 */ 6115 for (int i = 0; i < pregions_sz; i++) { 6116 if ((pa >= pregions[i].mr_start) && 6117 (pa < (pregions[i].mr_start + pregions[i].mr_size))) 6118 return (RPTE_ATTR_MEM); 6119 } 6120 return (RPTE_ATTR_GUARDEDIO); 6121 } 6122 6123 static void 6124 mmu_radix_kenter_attr(vm_offset_t va, vm_paddr_t pa, vm_memattr_t ma) 6125 { 6126 pt_entry_t *pte, pteval; 6127 uint64_t cache_bits; 6128 6129 pte = kvtopte(va); 6130 MPASS(pte != NULL); 6131 pteval = pa | RPTE_EAA_R | RPTE_EAA_W | RPTE_EAA_P | PG_M | PG_A; 6132 cache_bits = mmu_radix_calc_wimg(pa, ma); 6133 pte_store(pte, pteval | cache_bits); 6134 } 6135 6136 void 6137 mmu_radix_kremove(vm_offset_t va) 6138 { 6139 pt_entry_t *pte; 6140 6141 CTR2(KTR_PMAP, "%s(%#x)", __func__, va); 6142 6143 pte = kvtopte(va); 6144 pte_clear(pte); 6145 } 6146 6147 int 6148 mmu_radix_decode_kernel_ptr(vm_offset_t addr, 6149 int *is_user, vm_offset_t *decoded) 6150 { 6151 6152 CTR2(KTR_PMAP, "%s(%#jx)", __func__, (uintmax_t)addr); 6153 *decoded = addr; 6154 *is_user = (addr < VM_MAXUSER_ADDRESS); 6155 return (0); 6156 } 6157 6158 static int 6159 mmu_radix_dev_direct_mapped(vm_paddr_t pa, vm_size_t size) 6160 { 6161 6162 CTR3(KTR_PMAP, "%s(%#x, %#x)", __func__, pa, size); 6163 return (mem_valid(pa, size)); 6164 } 6165 6166 static void 6167 mmu_radix_scan_init(void) 6168 { 6169 6170 CTR1(KTR_PMAP, "%s()", __func__); 6171 UNIMPLEMENTED(); 6172 } 6173 6174 static void 6175 mmu_radix_dumpsys_map(vm_paddr_t pa, size_t sz, 6176 void **va) 6177 { 6178 CTR4(KTR_PMAP, "%s(%#jx, %#zx, %p)", __func__, (uintmax_t)pa, sz, va); 6179 UNIMPLEMENTED(); 6180 } 6181 6182 vm_offset_t 6183 mmu_radix_quick_enter_page(vm_page_t m) 6184 { 6185 vm_paddr_t paddr; 6186 6187 CTR2(KTR_PMAP, "%s(%p)", __func__, m); 6188 paddr = VM_PAGE_TO_PHYS(m); 6189 return (PHYS_TO_DMAP(paddr)); 6190 } 6191 6192 void 6193 mmu_radix_quick_remove_page(vm_offset_t addr __unused) 6194 { 6195 /* no work to do here */ 6196 CTR2(KTR_PMAP, "%s(%#x)", __func__, addr); 6197 } 6198 6199 static void 6200 pmap_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva) 6201 { 6202 cpu_flush_dcache((void *)sva, eva - sva); 6203 } 6204 6205 int 6206 mmu_radix_change_attr(vm_offset_t va, vm_size_t size, 6207 vm_memattr_t mode) 6208 { 6209 int error; 6210 6211 CTR4(KTR_PMAP, "%s(%#x, %#zx, %d)", __func__, va, size, mode); 6212 PMAP_LOCK(kernel_pmap); 6213 error = pmap_change_attr_locked(va, size, mode, true); 6214 PMAP_UNLOCK(kernel_pmap); 6215 return (error); 6216 } 6217 6218 static int 6219 pmap_change_attr_locked(vm_offset_t va, vm_size_t size, int mode, bool flush) 6220 { 6221 vm_offset_t base, offset, tmpva; 6222 vm_paddr_t pa_start, pa_end, pa_end1; 6223 pml2_entry_t *l2e; 6224 pml3_entry_t *l3e; 6225 pt_entry_t *pte; 6226 int cache_bits, error; 6227 bool changed; 6228 6229 PMAP_LOCK_ASSERT(kernel_pmap, MA_OWNED); 6230 base = trunc_page(va); 6231 offset = va & PAGE_MASK; 6232 size = round_page(offset + size); 6233 6234 /* 6235 * Only supported on kernel virtual addresses, including the direct 6236 * map but excluding the recursive map. 6237 */ 6238 if (base < DMAP_MIN_ADDRESS) 6239 return (EINVAL); 6240 6241 cache_bits = pmap_cache_bits(mode); 6242 changed = false; 6243 6244 /* 6245 * Pages that aren't mapped aren't supported. Also break down 2MB pages 6246 * into 4KB pages if required. 6247 */ 6248 for (tmpva = base; tmpva < base + size; ) { 6249 l2e = pmap_pml2e(kernel_pmap, tmpva); 6250 if (l2e == NULL || *l2e == 0) 6251 return (EINVAL); 6252 if (be64toh(*l2e) & RPTE_LEAF) { 6253 /* 6254 * If the current 1GB page already has the required 6255 * memory type, then we need not demote this page. Just 6256 * increment tmpva to the next 1GB page frame. 6257 */ 6258 if ((be64toh(*l2e) & RPTE_ATTR_MASK) == cache_bits) { 6259 tmpva = trunc_1gpage(tmpva) + L2_PAGE_SIZE; 6260 continue; 6261 } 6262 6263 /* 6264 * If the current offset aligns with a 1GB page frame 6265 * and there is at least 1GB left within the range, then 6266 * we need not break down this page into 2MB pages. 6267 */ 6268 if ((tmpva & L2_PAGE_MASK) == 0 && 6269 tmpva + L2_PAGE_MASK < base + size) { 6270 tmpva += L2_PAGE_MASK; 6271 continue; 6272 } 6273 if (!pmap_demote_l2e(kernel_pmap, l2e, tmpva)) 6274 return (ENOMEM); 6275 } 6276 l3e = pmap_l2e_to_l3e(l2e, tmpva); 6277 KASSERT(l3e != NULL, ("no l3e entry for %#lx in %p\n", 6278 tmpva, l2e)); 6279 if (*l3e == 0) 6280 return (EINVAL); 6281 if (be64toh(*l3e) & RPTE_LEAF) { 6282 /* 6283 * If the current 2MB page already has the required 6284 * memory type, then we need not demote this page. Just 6285 * increment tmpva to the next 2MB page frame. 6286 */ 6287 if ((be64toh(*l3e) & RPTE_ATTR_MASK) == cache_bits) { 6288 tmpva = trunc_2mpage(tmpva) + L3_PAGE_SIZE; 6289 continue; 6290 } 6291 6292 /* 6293 * If the current offset aligns with a 2MB page frame 6294 * and there is at least 2MB left within the range, then 6295 * we need not break down this page into 4KB pages. 6296 */ 6297 if ((tmpva & L3_PAGE_MASK) == 0 && 6298 tmpva + L3_PAGE_MASK < base + size) { 6299 tmpva += L3_PAGE_SIZE; 6300 continue; 6301 } 6302 if (!pmap_demote_l3e(kernel_pmap, l3e, tmpva)) 6303 return (ENOMEM); 6304 } 6305 pte = pmap_l3e_to_pte(l3e, tmpva); 6306 if (*pte == 0) 6307 return (EINVAL); 6308 tmpva += PAGE_SIZE; 6309 } 6310 error = 0; 6311 6312 /* 6313 * Ok, all the pages exist, so run through them updating their 6314 * cache mode if required. 6315 */ 6316 pa_start = pa_end = 0; 6317 for (tmpva = base; tmpva < base + size; ) { 6318 l2e = pmap_pml2e(kernel_pmap, tmpva); 6319 if (be64toh(*l2e) & RPTE_LEAF) { 6320 if ((be64toh(*l2e) & RPTE_ATTR_MASK) != cache_bits) { 6321 pmap_pte_attr(l2e, cache_bits, 6322 RPTE_ATTR_MASK); 6323 changed = true; 6324 } 6325 if (tmpva >= VM_MIN_KERNEL_ADDRESS && 6326 (*l2e & PG_PS_FRAME) < dmaplimit) { 6327 if (pa_start == pa_end) { 6328 /* Start physical address run. */ 6329 pa_start = be64toh(*l2e) & PG_PS_FRAME; 6330 pa_end = pa_start + L2_PAGE_SIZE; 6331 } else if (pa_end == (be64toh(*l2e) & PG_PS_FRAME)) 6332 pa_end += L2_PAGE_SIZE; 6333 else { 6334 /* Run ended, update direct map. */ 6335 error = pmap_change_attr_locked( 6336 PHYS_TO_DMAP(pa_start), 6337 pa_end - pa_start, mode, flush); 6338 if (error != 0) 6339 break; 6340 /* Start physical address run. */ 6341 pa_start = be64toh(*l2e) & PG_PS_FRAME; 6342 pa_end = pa_start + L2_PAGE_SIZE; 6343 } 6344 } 6345 tmpva = trunc_1gpage(tmpva) + L2_PAGE_SIZE; 6346 continue; 6347 } 6348 l3e = pmap_l2e_to_l3e(l2e, tmpva); 6349 if (be64toh(*l3e) & RPTE_LEAF) { 6350 if ((be64toh(*l3e) & RPTE_ATTR_MASK) != cache_bits) { 6351 pmap_pte_attr(l3e, cache_bits, 6352 RPTE_ATTR_MASK); 6353 changed = true; 6354 } 6355 if (tmpva >= VM_MIN_KERNEL_ADDRESS && 6356 (be64toh(*l3e) & PG_PS_FRAME) < dmaplimit) { 6357 if (pa_start == pa_end) { 6358 /* Start physical address run. */ 6359 pa_start = be64toh(*l3e) & PG_PS_FRAME; 6360 pa_end = pa_start + L3_PAGE_SIZE; 6361 } else if (pa_end == (be64toh(*l3e) & PG_PS_FRAME)) 6362 pa_end += L3_PAGE_SIZE; 6363 else { 6364 /* Run ended, update direct map. */ 6365 error = pmap_change_attr_locked( 6366 PHYS_TO_DMAP(pa_start), 6367 pa_end - pa_start, mode, flush); 6368 if (error != 0) 6369 break; 6370 /* Start physical address run. */ 6371 pa_start = be64toh(*l3e) & PG_PS_FRAME; 6372 pa_end = pa_start + L3_PAGE_SIZE; 6373 } 6374 } 6375 tmpva = trunc_2mpage(tmpva) + L3_PAGE_SIZE; 6376 } else { 6377 pte = pmap_l3e_to_pte(l3e, tmpva); 6378 if ((be64toh(*pte) & RPTE_ATTR_MASK) != cache_bits) { 6379 pmap_pte_attr(pte, cache_bits, 6380 RPTE_ATTR_MASK); 6381 changed = true; 6382 } 6383 if (tmpva >= VM_MIN_KERNEL_ADDRESS && 6384 (be64toh(*pte) & PG_FRAME) < dmaplimit) { 6385 if (pa_start == pa_end) { 6386 /* Start physical address run. */ 6387 pa_start = be64toh(*pte) & PG_FRAME; 6388 pa_end = pa_start + PAGE_SIZE; 6389 } else if (pa_end == (be64toh(*pte) & PG_FRAME)) 6390 pa_end += PAGE_SIZE; 6391 else { 6392 /* Run ended, update direct map. */ 6393 error = pmap_change_attr_locked( 6394 PHYS_TO_DMAP(pa_start), 6395 pa_end - pa_start, mode, flush); 6396 if (error != 0) 6397 break; 6398 /* Start physical address run. */ 6399 pa_start = be64toh(*pte) & PG_FRAME; 6400 pa_end = pa_start + PAGE_SIZE; 6401 } 6402 } 6403 tmpva += PAGE_SIZE; 6404 } 6405 } 6406 if (error == 0 && pa_start != pa_end && pa_start < dmaplimit) { 6407 pa_end1 = MIN(pa_end, dmaplimit); 6408 if (pa_start != pa_end1) 6409 error = pmap_change_attr_locked(PHYS_TO_DMAP(pa_start), 6410 pa_end1 - pa_start, mode, flush); 6411 } 6412 6413 /* 6414 * Flush CPU caches if required to make sure any data isn't cached that 6415 * shouldn't be, etc. 6416 */ 6417 if (changed) { 6418 pmap_invalidate_all(kernel_pmap); 6419 6420 if (flush) 6421 pmap_invalidate_cache_range(base, tmpva); 6422 } 6423 return (error); 6424 } 6425 6426 /* 6427 * Allocate physical memory for the vm_page array and map it into KVA, 6428 * attempting to back the vm_pages with domain-local memory. 6429 */ 6430 void 6431 mmu_radix_page_array_startup(long pages) 6432 { 6433 #ifdef notyet 6434 pml2_entry_t *l2e; 6435 pml3_entry_t *pde; 6436 pml3_entry_t newl3; 6437 vm_offset_t va; 6438 long pfn; 6439 int domain, i; 6440 #endif 6441 vm_paddr_t pa; 6442 vm_offset_t start, end; 6443 6444 vm_page_array_size = pages; 6445 6446 start = VM_MIN_KERNEL_ADDRESS; 6447 end = start + pages * sizeof(struct vm_page); 6448 6449 pa = vm_phys_early_alloc(-1, end - start); 6450 6451 start = mmu_radix_map(&start, pa, end - start, VM_MEMATTR_DEFAULT); 6452 #ifdef notyet 6453 /* TODO: NUMA vm_page_array. Blocked out until then (copied from amd64). */ 6454 for (va = start; va < end; va += L3_PAGE_SIZE) { 6455 pfn = first_page + (va - start) / sizeof(struct vm_page); 6456 domain = vm_phys_domain(ptoa(pfn)); 6457 l2e = pmap_pml2e(kernel_pmap, va); 6458 if ((be64toh(*l2e) & PG_V) == 0) { 6459 pa = vm_phys_early_alloc(domain, PAGE_SIZE); 6460 dump_add_page(pa); 6461 pagezero(PHYS_TO_DMAP(pa)); 6462 pde_store(l2e, (pml2_entry_t)pa); 6463 } 6464 pde = pmap_l2e_to_l3e(l2e, va); 6465 if ((be64toh(*pde) & PG_V) != 0) 6466 panic("Unexpected pde %p", pde); 6467 pa = vm_phys_early_alloc(domain, L3_PAGE_SIZE); 6468 for (i = 0; i < NPDEPG; i++) 6469 dump_add_page(pa + i * PAGE_SIZE); 6470 newl3 = (pml3_entry_t)(pa | RPTE_EAA_P | RPTE_EAA_R | RPTE_EAA_W); 6471 pte_store(pde, newl3); 6472 } 6473 #endif 6474 vm_page_array = (vm_page_t)start; 6475 } 6476 6477 #ifdef DDB 6478 #include <sys/kdb.h> 6479 #include <ddb/ddb.h> 6480 6481 static void 6482 pmap_pte_walk(pml1_entry_t *l1, vm_offset_t va) 6483 { 6484 pml1_entry_t *l1e; 6485 pml2_entry_t *l2e; 6486 pml3_entry_t *l3e; 6487 pt_entry_t *pte; 6488 6489 l1e = &l1[pmap_pml1e_index(va)]; 6490 db_printf("VA %#016lx l1e %#016lx", va, be64toh(*l1e)); 6491 if ((be64toh(*l1e) & PG_V) == 0) { 6492 db_printf("\n"); 6493 return; 6494 } 6495 l2e = pmap_l1e_to_l2e(l1e, va); 6496 db_printf(" l2e %#016lx", be64toh(*l2e)); 6497 if ((be64toh(*l2e) & PG_V) == 0 || (be64toh(*l2e) & RPTE_LEAF) != 0) { 6498 db_printf("\n"); 6499 return; 6500 } 6501 l3e = pmap_l2e_to_l3e(l2e, va); 6502 db_printf(" l3e %#016lx", be64toh(*l3e)); 6503 if ((be64toh(*l3e) & PG_V) == 0 || (be64toh(*l3e) & RPTE_LEAF) != 0) { 6504 db_printf("\n"); 6505 return; 6506 } 6507 pte = pmap_l3e_to_pte(l3e, va); 6508 db_printf(" pte %#016lx\n", be64toh(*pte)); 6509 } 6510 6511 void 6512 pmap_page_print_mappings(vm_page_t m) 6513 { 6514 pmap_t pmap; 6515 pv_entry_t pv; 6516 6517 db_printf("page %p(%lx)\n", m, m->phys_addr); 6518 /* need to elide locks if running in ddb */ 6519 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) { 6520 db_printf("pv: %p ", pv); 6521 db_printf("va: %#016lx ", pv->pv_va); 6522 pmap = PV_PMAP(pv); 6523 db_printf("pmap %p ", pmap); 6524 if (pmap != NULL) { 6525 db_printf("asid: %lu\n", pmap->pm_pid); 6526 pmap_pte_walk(pmap->pm_pml1, pv->pv_va); 6527 } 6528 } 6529 } 6530 6531 DB_SHOW_COMMAND(pte, pmap_print_pte) 6532 { 6533 vm_offset_t va; 6534 pmap_t pmap; 6535 6536 if (!have_addr) { 6537 db_printf("show pte addr\n"); 6538 return; 6539 } 6540 va = (vm_offset_t)addr; 6541 6542 if (va >= DMAP_MIN_ADDRESS) 6543 pmap = kernel_pmap; 6544 else if (kdb_thread != NULL) 6545 pmap = vmspace_pmap(kdb_thread->td_proc->p_vmspace); 6546 else 6547 pmap = vmspace_pmap(curthread->td_proc->p_vmspace); 6548 6549 pmap_pte_walk(pmap->pm_pml1, va); 6550 } 6551 6552 #endif 6553