1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2018 Matthew Macy 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __FBSDID("$FreeBSD$"); 30 31 #include <sys/param.h> 32 #include <sys/kernel.h> 33 #include <sys/systm.h> 34 #include <sys/conf.h> 35 #include <sys/bitstring.h> 36 #include <sys/queue.h> 37 #include <sys/cpuset.h> 38 #include <sys/endian.h> 39 #include <sys/kerneldump.h> 40 #include <sys/ktr.h> 41 #include <sys/lock.h> 42 #include <sys/syslog.h> 43 #include <sys/msgbuf.h> 44 #include <sys/malloc.h> 45 #include <sys/mman.h> 46 #include <sys/mutex.h> 47 #include <sys/proc.h> 48 #include <sys/rwlock.h> 49 #include <sys/sched.h> 50 #include <sys/sysctl.h> 51 #include <sys/systm.h> 52 #include <sys/vmem.h> 53 #include <sys/vmmeter.h> 54 #include <sys/smp.h> 55 56 #include <sys/kdb.h> 57 58 #include <dev/ofw/openfirm.h> 59 60 #include <vm/vm.h> 61 #include <vm/pmap.h> 62 #include <vm/vm_param.h> 63 #include <vm/vm_kern.h> 64 #include <vm/vm_page.h> 65 #include <vm/vm_map.h> 66 #include <vm/vm_object.h> 67 #include <vm/vm_extern.h> 68 #include <vm/vm_pageout.h> 69 #include <vm/vm_phys.h> 70 #include <vm/vm_reserv.h> 71 #include <vm/vm_dumpset.h> 72 #include <vm/uma.h> 73 74 #include <machine/_inttypes.h> 75 #include <machine/cpu.h> 76 #include <machine/platform.h> 77 #include <machine/frame.h> 78 #include <machine/md_var.h> 79 #include <machine/psl.h> 80 #include <machine/bat.h> 81 #include <machine/hid.h> 82 #include <machine/pte.h> 83 #include <machine/sr.h> 84 #include <machine/trap.h> 85 #include <machine/mmuvar.h> 86 87 #ifdef INVARIANTS 88 #include <vm/uma_dbg.h> 89 #endif 90 91 #define PPC_BITLSHIFT(bit) (sizeof(long)*NBBY - 1 - (bit)) 92 #define PPC_BIT(bit) (1UL << PPC_BITLSHIFT(bit)) 93 #define PPC_BITLSHIFT_VAL(val, bit) ((val) << PPC_BITLSHIFT(bit)) 94 95 #include "opt_ddb.h" 96 #ifdef DDB 97 static void pmap_pte_walk(pml1_entry_t *l1, vm_offset_t va); 98 #endif 99 100 #define PG_W RPTE_WIRED 101 #define PG_V RPTE_VALID 102 #define PG_MANAGED RPTE_MANAGED 103 #define PG_PROMOTED RPTE_PROMOTED 104 #define PG_M RPTE_C 105 #define PG_A RPTE_R 106 #define PG_X RPTE_EAA_X 107 #define PG_RW RPTE_EAA_W 108 #define PG_PTE_CACHE RPTE_ATTR_MASK 109 110 #define RPTE_SHIFT 9 111 #define NLS_MASK ((1UL<<5)-1) 112 #define RPTE_ENTRIES (1UL<<RPTE_SHIFT) 113 #define RPTE_MASK (RPTE_ENTRIES-1) 114 115 #define NLB_SHIFT 0 116 #define NLB_MASK (((1UL<<52)-1) << 8) 117 118 extern int nkpt; 119 extern caddr_t crashdumpmap; 120 121 #define RIC_FLUSH_TLB 0 122 #define RIC_FLUSH_PWC 1 123 #define RIC_FLUSH_ALL 2 124 125 #define POWER9_TLB_SETS_RADIX 128 /* # sets in POWER9 TLB Radix mode */ 126 127 #define PPC_INST_TLBIE 0x7c000264 128 #define PPC_INST_TLBIEL 0x7c000224 129 #define PPC_INST_SLBIA 0x7c0003e4 130 131 #define ___PPC_RA(a) (((a) & 0x1f) << 16) 132 #define ___PPC_RB(b) (((b) & 0x1f) << 11) 133 #define ___PPC_RS(s) (((s) & 0x1f) << 21) 134 #define ___PPC_RT(t) ___PPC_RS(t) 135 #define ___PPC_R(r) (((r) & 0x1) << 16) 136 #define ___PPC_PRS(prs) (((prs) & 0x1) << 17) 137 #define ___PPC_RIC(ric) (((ric) & 0x3) << 18) 138 139 #define PPC_SLBIA(IH) __XSTRING(.long PPC_INST_SLBIA | \ 140 ((IH & 0x7) << 21)) 141 #define PPC_TLBIE_5(rb,rs,ric,prs,r) \ 142 __XSTRING(.long PPC_INST_TLBIE | \ 143 ___PPC_RB(rb) | ___PPC_RS(rs) | \ 144 ___PPC_RIC(ric) | ___PPC_PRS(prs) | \ 145 ___PPC_R(r)) 146 147 #define PPC_TLBIEL(rb,rs,ric,prs,r) \ 148 __XSTRING(.long PPC_INST_TLBIEL | \ 149 ___PPC_RB(rb) | ___PPC_RS(rs) | \ 150 ___PPC_RIC(ric) | ___PPC_PRS(prs) | \ 151 ___PPC_R(r)) 152 153 #define PPC_INVALIDATE_ERAT PPC_SLBIA(7) 154 155 static __inline void 156 ttusync(void) 157 { 158 __asm __volatile("eieio; tlbsync; ptesync" ::: "memory"); 159 } 160 161 #define TLBIEL_INVAL_SEL_MASK 0xc00 /* invalidation selector */ 162 #define TLBIEL_INVAL_PAGE 0x000 /* invalidate a single page */ 163 #define TLBIEL_INVAL_SET_PID 0x400 /* invalidate a set for the current PID */ 164 #define TLBIEL_INVAL_SET_LPID 0x800 /* invalidate a set for current LPID */ 165 #define TLBIEL_INVAL_SET 0xc00 /* invalidate a set for all LPIDs */ 166 167 #define TLBIE_ACTUAL_PAGE_MASK 0xe0 168 #define TLBIE_ACTUAL_PAGE_4K 0x00 169 #define TLBIE_ACTUAL_PAGE_64K 0xa0 170 #define TLBIE_ACTUAL_PAGE_2M 0x20 171 #define TLBIE_ACTUAL_PAGE_1G 0x40 172 173 #define TLBIE_PRS_PARTITION_SCOPE 0x0 174 #define TLBIE_PRS_PROCESS_SCOPE 0x1 175 176 #define TLBIE_RIC_INVALIDATE_TLB 0x0 /* Invalidate just TLB */ 177 #define TLBIE_RIC_INVALIDATE_PWC 0x1 /* Invalidate just PWC */ 178 #define TLBIE_RIC_INVALIDATE_ALL 0x2 /* Invalidate TLB, PWC, 179 * cached {proc, part}tab entries 180 */ 181 #define TLBIE_RIC_INVALIDATE_SEQ 0x3 /* HPT - only: 182 * Invalidate a range of translations 183 */ 184 185 static __always_inline void 186 radix_tlbie(uint8_t ric, uint8_t prs, uint16_t is, uint32_t pid, uint32_t lpid, 187 vm_offset_t va, uint16_t ap) 188 { 189 uint64_t rb, rs; 190 191 MPASS((va & PAGE_MASK) == 0); 192 193 rs = ((uint64_t)pid << 32) | lpid; 194 rb = va | is | ap; 195 __asm __volatile(PPC_TLBIE_5(%0, %1, %2, %3, 1) : : 196 "r" (rb), "r" (rs), "i" (ric), "i" (prs) : "memory"); 197 } 198 199 static __inline void 200 radix_tlbie_fixup(uint32_t pid, vm_offset_t va, int ap) 201 { 202 203 __asm __volatile("ptesync" ::: "memory"); 204 radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE, 205 TLBIEL_INVAL_PAGE, 0, 0, va, ap); 206 __asm __volatile("ptesync" ::: "memory"); 207 radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE, 208 TLBIEL_INVAL_PAGE, pid, 0, va, ap); 209 } 210 211 static __inline void 212 radix_tlbie_invlpg_user_4k(uint32_t pid, vm_offset_t va) 213 { 214 215 radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE, 216 TLBIEL_INVAL_PAGE, pid, 0, va, TLBIE_ACTUAL_PAGE_4K); 217 radix_tlbie_fixup(pid, va, TLBIE_ACTUAL_PAGE_4K); 218 } 219 220 static __inline void 221 radix_tlbie_invlpg_user_2m(uint32_t pid, vm_offset_t va) 222 { 223 224 radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE, 225 TLBIEL_INVAL_PAGE, pid, 0, va, TLBIE_ACTUAL_PAGE_2M); 226 radix_tlbie_fixup(pid, va, TLBIE_ACTUAL_PAGE_2M); 227 } 228 229 static __inline void 230 radix_tlbie_invlpwc_user(uint32_t pid) 231 { 232 233 radix_tlbie(TLBIE_RIC_INVALIDATE_PWC, TLBIE_PRS_PROCESS_SCOPE, 234 TLBIEL_INVAL_SET_PID, pid, 0, 0, 0); 235 } 236 237 static __inline void 238 radix_tlbie_flush_user(uint32_t pid) 239 { 240 241 radix_tlbie(TLBIE_RIC_INVALIDATE_ALL, TLBIE_PRS_PROCESS_SCOPE, 242 TLBIEL_INVAL_SET_PID, pid, 0, 0, 0); 243 } 244 245 static __inline void 246 radix_tlbie_invlpg_kernel_4k(vm_offset_t va) 247 { 248 249 radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE, 250 TLBIEL_INVAL_PAGE, 0, 0, va, TLBIE_ACTUAL_PAGE_4K); 251 radix_tlbie_fixup(0, va, TLBIE_ACTUAL_PAGE_4K); 252 } 253 254 static __inline void 255 radix_tlbie_invlpg_kernel_2m(vm_offset_t va) 256 { 257 258 radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE, 259 TLBIEL_INVAL_PAGE, 0, 0, va, TLBIE_ACTUAL_PAGE_2M); 260 radix_tlbie_fixup(0, va, TLBIE_ACTUAL_PAGE_2M); 261 } 262 263 /* 1GB pages aren't currently supported. */ 264 static __inline __unused void 265 radix_tlbie_invlpg_kernel_1g(vm_offset_t va) 266 { 267 268 radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE, 269 TLBIEL_INVAL_PAGE, 0, 0, va, TLBIE_ACTUAL_PAGE_1G); 270 radix_tlbie_fixup(0, va, TLBIE_ACTUAL_PAGE_1G); 271 } 272 273 static __inline void 274 radix_tlbie_invlpwc_kernel(void) 275 { 276 277 radix_tlbie(TLBIE_RIC_INVALIDATE_PWC, TLBIE_PRS_PROCESS_SCOPE, 278 TLBIEL_INVAL_SET_LPID, 0, 0, 0, 0); 279 } 280 281 static __inline void 282 radix_tlbie_flush_kernel(void) 283 { 284 285 radix_tlbie(TLBIE_RIC_INVALIDATE_ALL, TLBIE_PRS_PROCESS_SCOPE, 286 TLBIEL_INVAL_SET_LPID, 0, 0, 0, 0); 287 } 288 289 static __inline vm_pindex_t 290 pmap_l3e_pindex(vm_offset_t va) 291 { 292 return ((va & PG_FRAME) >> L3_PAGE_SIZE_SHIFT); 293 } 294 295 static __inline vm_pindex_t 296 pmap_pml3e_index(vm_offset_t va) 297 { 298 299 return ((va >> L3_PAGE_SIZE_SHIFT) & RPTE_MASK); 300 } 301 302 static __inline vm_pindex_t 303 pmap_pml2e_index(vm_offset_t va) 304 { 305 return ((va >> L2_PAGE_SIZE_SHIFT) & RPTE_MASK); 306 } 307 308 static __inline vm_pindex_t 309 pmap_pml1e_index(vm_offset_t va) 310 { 311 return ((va & PG_FRAME) >> L1_PAGE_SIZE_SHIFT); 312 } 313 314 /* Return various clipped indexes for a given VA */ 315 static __inline vm_pindex_t 316 pmap_pte_index(vm_offset_t va) 317 { 318 319 return ((va >> PAGE_SHIFT) & RPTE_MASK); 320 } 321 322 /* Return a pointer to the PT slot that corresponds to a VA */ 323 static __inline pt_entry_t * 324 pmap_l3e_to_pte(pt_entry_t *l3e, vm_offset_t va) 325 { 326 pt_entry_t *pte; 327 vm_paddr_t ptepa; 328 329 ptepa = (be64toh(*l3e) & NLB_MASK); 330 pte = (pt_entry_t *)PHYS_TO_DMAP(ptepa); 331 return (&pte[pmap_pte_index(va)]); 332 } 333 334 /* Return a pointer to the PD slot that corresponds to a VA */ 335 static __inline pt_entry_t * 336 pmap_l2e_to_l3e(pt_entry_t *l2e, vm_offset_t va) 337 { 338 pt_entry_t *l3e; 339 vm_paddr_t l3pa; 340 341 l3pa = (be64toh(*l2e) & NLB_MASK); 342 l3e = (pml3_entry_t *)PHYS_TO_DMAP(l3pa); 343 return (&l3e[pmap_pml3e_index(va)]); 344 } 345 346 /* Return a pointer to the PD slot that corresponds to a VA */ 347 static __inline pt_entry_t * 348 pmap_l1e_to_l2e(pt_entry_t *l1e, vm_offset_t va) 349 { 350 pt_entry_t *l2e; 351 vm_paddr_t l2pa; 352 353 l2pa = (be64toh(*l1e) & NLB_MASK); 354 355 l2e = (pml2_entry_t *)PHYS_TO_DMAP(l2pa); 356 return (&l2e[pmap_pml2e_index(va)]); 357 } 358 359 static __inline pml1_entry_t * 360 pmap_pml1e(pmap_t pmap, vm_offset_t va) 361 { 362 363 return (&pmap->pm_pml1[pmap_pml1e_index(va)]); 364 } 365 366 static pt_entry_t * 367 pmap_pml2e(pmap_t pmap, vm_offset_t va) 368 { 369 pt_entry_t *l1e; 370 371 l1e = pmap_pml1e(pmap, va); 372 if (l1e == NULL || (be64toh(*l1e) & RPTE_VALID) == 0) 373 return (NULL); 374 return (pmap_l1e_to_l2e(l1e, va)); 375 } 376 377 static __inline pt_entry_t * 378 pmap_pml3e(pmap_t pmap, vm_offset_t va) 379 { 380 pt_entry_t *l2e; 381 382 l2e = pmap_pml2e(pmap, va); 383 if (l2e == NULL || (be64toh(*l2e) & RPTE_VALID) == 0) 384 return (NULL); 385 return (pmap_l2e_to_l3e(l2e, va)); 386 } 387 388 static __inline pt_entry_t * 389 pmap_pte(pmap_t pmap, vm_offset_t va) 390 { 391 pt_entry_t *l3e; 392 393 l3e = pmap_pml3e(pmap, va); 394 if (l3e == NULL || (be64toh(*l3e) & RPTE_VALID) == 0) 395 return (NULL); 396 return (pmap_l3e_to_pte(l3e, va)); 397 } 398 399 int nkpt = 64; 400 SYSCTL_INT(_machdep, OID_AUTO, nkpt, CTLFLAG_RD, &nkpt, 0, 401 "Number of kernel page table pages allocated on bootup"); 402 403 vm_paddr_t dmaplimit; 404 405 SYSCTL_DECL(_vm_pmap); 406 407 #ifdef INVARIANTS 408 #define VERBOSE_PMAP 0 409 #define VERBOSE_PROTECT 0 410 static int pmap_logging; 411 SYSCTL_INT(_vm_pmap, OID_AUTO, pmap_logging, CTLFLAG_RWTUN, 412 &pmap_logging, 0, "verbose debug logging"); 413 #endif 414 415 static u_int64_t KPTphys; /* phys addr of kernel level 1 */ 416 417 //static vm_paddr_t KERNend; /* phys addr of end of bootstrap data */ 418 419 static vm_offset_t qframe = 0; 420 static struct mtx qframe_mtx; 421 422 void mmu_radix_activate(struct thread *); 423 void mmu_radix_advise(pmap_t, vm_offset_t, vm_offset_t, int); 424 void mmu_radix_align_superpage(vm_object_t, vm_ooffset_t, vm_offset_t *, 425 vm_size_t); 426 void mmu_radix_clear_modify(vm_page_t); 427 void mmu_radix_copy(pmap_t, pmap_t, vm_offset_t, vm_size_t, vm_offset_t); 428 int mmu_radix_decode_kernel_ptr(vm_offset_t, int *, vm_offset_t *); 429 int mmu_radix_enter(pmap_t, vm_offset_t, vm_page_t, vm_prot_t, u_int, int8_t); 430 void mmu_radix_enter_object(pmap_t, vm_offset_t, vm_offset_t, vm_page_t, 431 vm_prot_t); 432 void mmu_radix_enter_quick(pmap_t, vm_offset_t, vm_page_t, vm_prot_t); 433 vm_paddr_t mmu_radix_extract(pmap_t pmap, vm_offset_t va); 434 vm_page_t mmu_radix_extract_and_hold(pmap_t, vm_offset_t, vm_prot_t); 435 void mmu_radix_kenter(vm_offset_t, vm_paddr_t); 436 vm_paddr_t mmu_radix_kextract(vm_offset_t); 437 void mmu_radix_kremove(vm_offset_t); 438 boolean_t mmu_radix_is_modified(vm_page_t); 439 boolean_t mmu_radix_is_prefaultable(pmap_t, vm_offset_t); 440 boolean_t mmu_radix_is_referenced(vm_page_t); 441 void mmu_radix_object_init_pt(pmap_t, vm_offset_t, vm_object_t, 442 vm_pindex_t, vm_size_t); 443 boolean_t mmu_radix_page_exists_quick(pmap_t, vm_page_t); 444 void mmu_radix_page_init(vm_page_t); 445 boolean_t mmu_radix_page_is_mapped(vm_page_t m); 446 void mmu_radix_page_set_memattr(vm_page_t, vm_memattr_t); 447 int mmu_radix_page_wired_mappings(vm_page_t); 448 int mmu_radix_pinit(pmap_t); 449 void mmu_radix_protect(pmap_t, vm_offset_t, vm_offset_t, vm_prot_t); 450 bool mmu_radix_ps_enabled(pmap_t); 451 void mmu_radix_qenter(vm_offset_t, vm_page_t *, int); 452 void mmu_radix_qremove(vm_offset_t, int); 453 vm_offset_t mmu_radix_quick_enter_page(vm_page_t); 454 void mmu_radix_quick_remove_page(vm_offset_t); 455 boolean_t mmu_radix_ts_referenced(vm_page_t); 456 void mmu_radix_release(pmap_t); 457 void mmu_radix_remove(pmap_t, vm_offset_t, vm_offset_t); 458 void mmu_radix_remove_all(vm_page_t); 459 void mmu_radix_remove_pages(pmap_t); 460 void mmu_radix_remove_write(vm_page_t); 461 void mmu_radix_unwire(pmap_t, vm_offset_t, vm_offset_t); 462 void mmu_radix_zero_page(vm_page_t); 463 void mmu_radix_zero_page_area(vm_page_t, int, int); 464 int mmu_radix_change_attr(vm_offset_t, vm_size_t, vm_memattr_t); 465 void mmu_radix_page_array_startup(long pages); 466 467 #include "mmu_oea64.h" 468 469 /* 470 * Kernel MMU interface 471 */ 472 473 static void mmu_radix_bootstrap(vm_offset_t, vm_offset_t); 474 475 static void mmu_radix_copy_page(vm_page_t, vm_page_t); 476 static void mmu_radix_copy_pages(vm_page_t *ma, vm_offset_t a_offset, 477 vm_page_t *mb, vm_offset_t b_offset, int xfersize); 478 static void mmu_radix_growkernel(vm_offset_t); 479 static void mmu_radix_init(void); 480 static int mmu_radix_mincore(pmap_t, vm_offset_t, vm_paddr_t *); 481 static vm_offset_t mmu_radix_map(vm_offset_t *, vm_paddr_t, vm_paddr_t, int); 482 static void mmu_radix_pinit0(pmap_t); 483 484 static void *mmu_radix_mapdev(vm_paddr_t, vm_size_t); 485 static void *mmu_radix_mapdev_attr(vm_paddr_t, vm_size_t, vm_memattr_t); 486 static void mmu_radix_unmapdev(vm_offset_t, vm_size_t); 487 static void mmu_radix_kenter_attr(vm_offset_t, vm_paddr_t, vm_memattr_t ma); 488 static boolean_t mmu_radix_dev_direct_mapped(vm_paddr_t, vm_size_t); 489 static void mmu_radix_dumpsys_map(vm_paddr_t pa, size_t sz, void **va); 490 static void mmu_radix_scan_init(void); 491 static void mmu_radix_cpu_bootstrap(int ap); 492 static void mmu_radix_tlbie_all(void); 493 494 static struct pmap_funcs mmu_radix_methods = { 495 .bootstrap = mmu_radix_bootstrap, 496 .copy_page = mmu_radix_copy_page, 497 .copy_pages = mmu_radix_copy_pages, 498 .cpu_bootstrap = mmu_radix_cpu_bootstrap, 499 .growkernel = mmu_radix_growkernel, 500 .init = mmu_radix_init, 501 .map = mmu_radix_map, 502 .mincore = mmu_radix_mincore, 503 .pinit = mmu_radix_pinit, 504 .pinit0 = mmu_radix_pinit0, 505 506 .mapdev = mmu_radix_mapdev, 507 .mapdev_attr = mmu_radix_mapdev_attr, 508 .unmapdev = mmu_radix_unmapdev, 509 .kenter_attr = mmu_radix_kenter_attr, 510 .dev_direct_mapped = mmu_radix_dev_direct_mapped, 511 .dumpsys_pa_init = mmu_radix_scan_init, 512 .dumpsys_map_chunk = mmu_radix_dumpsys_map, 513 .page_is_mapped = mmu_radix_page_is_mapped, 514 .ps_enabled = mmu_radix_ps_enabled, 515 .object_init_pt = mmu_radix_object_init_pt, 516 .protect = mmu_radix_protect, 517 /* pmap dispatcher interface */ 518 .clear_modify = mmu_radix_clear_modify, 519 .copy = mmu_radix_copy, 520 .enter = mmu_radix_enter, 521 .enter_object = mmu_radix_enter_object, 522 .enter_quick = mmu_radix_enter_quick, 523 .extract = mmu_radix_extract, 524 .extract_and_hold = mmu_radix_extract_and_hold, 525 .is_modified = mmu_radix_is_modified, 526 .is_prefaultable = mmu_radix_is_prefaultable, 527 .is_referenced = mmu_radix_is_referenced, 528 .ts_referenced = mmu_radix_ts_referenced, 529 .page_exists_quick = mmu_radix_page_exists_quick, 530 .page_init = mmu_radix_page_init, 531 .page_wired_mappings = mmu_radix_page_wired_mappings, 532 .qenter = mmu_radix_qenter, 533 .qremove = mmu_radix_qremove, 534 .release = mmu_radix_release, 535 .remove = mmu_radix_remove, 536 .remove_all = mmu_radix_remove_all, 537 .remove_write = mmu_radix_remove_write, 538 .unwire = mmu_radix_unwire, 539 .zero_page = mmu_radix_zero_page, 540 .zero_page_area = mmu_radix_zero_page_area, 541 .activate = mmu_radix_activate, 542 .quick_enter_page = mmu_radix_quick_enter_page, 543 .quick_remove_page = mmu_radix_quick_remove_page, 544 .page_set_memattr = mmu_radix_page_set_memattr, 545 .page_array_startup = mmu_radix_page_array_startup, 546 547 /* Internal interfaces */ 548 .kenter = mmu_radix_kenter, 549 .kextract = mmu_radix_kextract, 550 .kremove = mmu_radix_kremove, 551 .change_attr = mmu_radix_change_attr, 552 .decode_kernel_ptr = mmu_radix_decode_kernel_ptr, 553 554 .tlbie_all = mmu_radix_tlbie_all, 555 }; 556 557 MMU_DEF(mmu_radix, MMU_TYPE_RADIX, mmu_radix_methods); 558 559 static boolean_t pmap_demote_l3e_locked(pmap_t pmap, pml3_entry_t *l3e, vm_offset_t va, 560 struct rwlock **lockp); 561 static boolean_t pmap_demote_l3e(pmap_t pmap, pml3_entry_t *pde, vm_offset_t va); 562 static int pmap_unuse_pt(pmap_t, vm_offset_t, pml3_entry_t, struct spglist *); 563 static int pmap_remove_l3e(pmap_t pmap, pml3_entry_t *pdq, vm_offset_t sva, 564 struct spglist *free, struct rwlock **lockp); 565 static int pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t sva, 566 pml3_entry_t ptepde, struct spglist *free, struct rwlock **lockp); 567 static vm_page_t pmap_remove_pt_page(pmap_t pmap, vm_offset_t va); 568 static bool pmap_remove_page(pmap_t pmap, vm_offset_t va, pml3_entry_t *pde, 569 struct spglist *free); 570 static bool pmap_remove_ptes(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, 571 pml3_entry_t *l3e, struct spglist *free, struct rwlock **lockp); 572 573 static bool pmap_pv_insert_l3e(pmap_t pmap, vm_offset_t va, pml3_entry_t l3e, 574 u_int flags, struct rwlock **lockp); 575 #if VM_NRESERVLEVEL > 0 576 static void pmap_pv_promote_l3e(pmap_t pmap, vm_offset_t va, vm_paddr_t pa, 577 struct rwlock **lockp); 578 #endif 579 static void pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va); 580 static int pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte); 581 static vm_page_t mmu_radix_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, 582 vm_prot_t prot, vm_page_t mpte, struct rwlock **lockp, bool *invalidate); 583 584 static bool pmap_enter_2mpage(pmap_t pmap, vm_offset_t va, vm_page_t m, 585 vm_prot_t prot, struct rwlock **lockp); 586 static int pmap_enter_l3e(pmap_t pmap, vm_offset_t va, pml3_entry_t newpde, 587 u_int flags, vm_page_t m, struct rwlock **lockp); 588 589 static vm_page_t reclaim_pv_chunk(pmap_t locked_pmap, struct rwlock **lockp); 590 static void free_pv_chunk(struct pv_chunk *pc); 591 static vm_page_t _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, struct rwlock **lockp); 592 static vm_page_t pmap_allocl3e(pmap_t pmap, vm_offset_t va, 593 struct rwlock **lockp); 594 static vm_page_t pmap_allocpte(pmap_t pmap, vm_offset_t va, 595 struct rwlock **lockp); 596 static void _pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, 597 struct spglist *free); 598 static boolean_t pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free); 599 600 static void pmap_invalidate_page(pmap_t pmap, vm_offset_t start); 601 static void pmap_invalidate_all(pmap_t pmap); 602 static int pmap_change_attr_locked(vm_offset_t va, vm_size_t size, int mode, bool flush); 603 604 /* 605 * Internal flags for pmap_enter()'s helper functions. 606 */ 607 #define PMAP_ENTER_NORECLAIM 0x1000000 /* Don't reclaim PV entries. */ 608 #define PMAP_ENTER_NOREPLACE 0x2000000 /* Don't replace mappings. */ 609 610 #define UNIMPLEMENTED() panic("%s not implemented", __func__) 611 #define UNTESTED() panic("%s not yet tested", __func__) 612 613 /* Number of supported PID bits */ 614 static unsigned int isa3_pid_bits; 615 616 /* PID to start allocating from */ 617 static unsigned int isa3_base_pid; 618 619 #define PROCTAB_SIZE_SHIFT (isa3_pid_bits + 4) 620 #define PROCTAB_ENTRIES (1ul << isa3_pid_bits) 621 622 /* 623 * Map of physical memory regions. 624 */ 625 static struct mem_region *regions, *pregions; 626 static struct numa_mem_region *numa_pregions; 627 static u_int phys_avail_count; 628 static int regions_sz, pregions_sz, numa_pregions_sz; 629 static struct pate *isa3_parttab; 630 static struct prte *isa3_proctab; 631 static vmem_t *asid_arena; 632 633 extern void bs_remap_earlyboot(void); 634 635 #define RADIX_PGD_SIZE_SHIFT 16 636 #define RADIX_PGD_SIZE (1UL << RADIX_PGD_SIZE_SHIFT) 637 638 #define RADIX_PGD_INDEX_SHIFT (RADIX_PGD_SIZE_SHIFT-3) 639 #define NL2EPG (PAGE_SIZE/sizeof(pml2_entry_t)) 640 #define NL3EPG (PAGE_SIZE/sizeof(pml3_entry_t)) 641 642 #define NUPML1E (RADIX_PGD_SIZE/sizeof(uint64_t)) /* number of userland PML1 pages */ 643 #define NUPDPE (NUPML1E * NL2EPG)/* number of userland PDP pages */ 644 #define NUPDE (NUPDPE * NL3EPG) /* number of userland PD entries */ 645 646 /* POWER9 only permits a 64k partition table size. */ 647 #define PARTTAB_SIZE_SHIFT 16 648 #define PARTTAB_SIZE (1UL << PARTTAB_SIZE_SHIFT) 649 650 #define PARTTAB_HR (1UL << 63) /* host uses radix */ 651 #define PARTTAB_GR (1UL << 63) /* guest uses radix must match host */ 652 653 /* TLB flush actions. Used as argument to tlbiel_all() */ 654 enum { 655 TLB_INVAL_SCOPE_LPID = 0, /* invalidate TLBs for current LPID */ 656 TLB_INVAL_SCOPE_GLOBAL = 1, /* invalidate all TLBs */ 657 }; 658 659 #define NPV_LIST_LOCKS MAXCPU 660 static int pmap_initialized; 661 static vm_paddr_t proctab0pa; 662 static vm_paddr_t parttab_phys; 663 CTASSERT(sizeof(struct pv_chunk) == PAGE_SIZE); 664 665 /* 666 * Data for the pv entry allocation mechanism. 667 * Updates to pv_invl_gen are protected by the pv_list_locks[] 668 * elements, but reads are not. 669 */ 670 static TAILQ_HEAD(pch, pv_chunk) pv_chunks = TAILQ_HEAD_INITIALIZER(pv_chunks); 671 static struct mtx __exclusive_cache_line pv_chunks_mutex; 672 static struct rwlock __exclusive_cache_line pv_list_locks[NPV_LIST_LOCKS]; 673 static struct md_page *pv_table; 674 static struct md_page pv_dummy; 675 676 #ifdef PV_STATS 677 #define PV_STAT(x) do { x ; } while (0) 678 #else 679 #define PV_STAT(x) do { } while (0) 680 #endif 681 682 #define pa_radix_index(pa) ((pa) >> L3_PAGE_SIZE_SHIFT) 683 #define pa_to_pvh(pa) (&pv_table[pa_radix_index(pa)]) 684 685 #define PHYS_TO_PV_LIST_LOCK(pa) \ 686 (&pv_list_locks[pa_radix_index(pa) % NPV_LIST_LOCKS]) 687 688 #define CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa) do { \ 689 struct rwlock **_lockp = (lockp); \ 690 struct rwlock *_new_lock; \ 691 \ 692 _new_lock = PHYS_TO_PV_LIST_LOCK(pa); \ 693 if (_new_lock != *_lockp) { \ 694 if (*_lockp != NULL) \ 695 rw_wunlock(*_lockp); \ 696 *_lockp = _new_lock; \ 697 rw_wlock(*_lockp); \ 698 } \ 699 } while (0) 700 701 #define CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m) \ 702 CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, VM_PAGE_TO_PHYS(m)) 703 704 #define RELEASE_PV_LIST_LOCK(lockp) do { \ 705 struct rwlock **_lockp = (lockp); \ 706 \ 707 if (*_lockp != NULL) { \ 708 rw_wunlock(*_lockp); \ 709 *_lockp = NULL; \ 710 } \ 711 } while (0) 712 713 #define VM_PAGE_TO_PV_LIST_LOCK(m) \ 714 PHYS_TO_PV_LIST_LOCK(VM_PAGE_TO_PHYS(m)) 715 716 /* 717 * We support 52 bits, hence: 718 * bits 52 - 31 = 21, 0b10101 719 * RTS encoding details 720 * bits 0 - 3 of rts -> bits 6 - 8 unsigned long 721 * bits 4 - 5 of rts -> bits 62 - 63 of unsigned long 722 */ 723 #define RTS_SIZE ((0x2UL << 61) | (0x5UL << 5)) 724 725 static int powernv_enabled = 1; 726 727 static __always_inline void 728 tlbiel_radix_set_isa300(uint32_t set, uint32_t is, 729 uint32_t pid, uint32_t ric, uint32_t prs) 730 { 731 uint64_t rb; 732 uint64_t rs; 733 734 rb = PPC_BITLSHIFT_VAL(set, 51) | PPC_BITLSHIFT_VAL(is, 53); 735 rs = PPC_BITLSHIFT_VAL((uint64_t)pid, 31); 736 737 __asm __volatile(PPC_TLBIEL(%0, %1, %2, %3, 1) 738 : : "r"(rb), "r"(rs), "i"(ric), "i"(prs) 739 : "memory"); 740 } 741 742 static void 743 tlbiel_flush_isa3(uint32_t num_sets, uint32_t is) 744 { 745 uint32_t set; 746 747 __asm __volatile("ptesync": : :"memory"); 748 749 /* 750 * Flush the first set of the TLB, and the entire Page Walk Cache 751 * and partition table entries. Then flush the remaining sets of the 752 * TLB. 753 */ 754 tlbiel_radix_set_isa300(0, is, 0, RIC_FLUSH_ALL, 0); 755 for (set = 1; set < num_sets; set++) 756 tlbiel_radix_set_isa300(set, is, 0, RIC_FLUSH_TLB, 0); 757 758 /* Do the same for process scoped entries. */ 759 tlbiel_radix_set_isa300(0, is, 0, RIC_FLUSH_ALL, 1); 760 for (set = 1; set < num_sets; set++) 761 tlbiel_radix_set_isa300(set, is, 0, RIC_FLUSH_TLB, 1); 762 763 __asm __volatile("ptesync": : :"memory"); 764 } 765 766 static void 767 mmu_radix_tlbiel_flush(int scope) 768 { 769 int is; 770 771 MPASS(scope == TLB_INVAL_SCOPE_LPID || 772 scope == TLB_INVAL_SCOPE_GLOBAL); 773 is = scope + 2; 774 775 tlbiel_flush_isa3(POWER9_TLB_SETS_RADIX, is); 776 __asm __volatile(PPC_INVALIDATE_ERAT "; isync" : : :"memory"); 777 } 778 779 static void 780 mmu_radix_tlbie_all() 781 { 782 /* TODO: LPID invalidate */ 783 mmu_radix_tlbiel_flush(TLB_INVAL_SCOPE_GLOBAL); 784 } 785 786 static void 787 mmu_radix_init_amor(void) 788 { 789 /* 790 * In HV mode, we init AMOR (Authority Mask Override Register) so that 791 * the hypervisor and guest can setup IAMR (Instruction Authority Mask 792 * Register), enable key 0 and set it to 1. 793 * 794 * AMOR = 0b1100 .... 0000 (Mask for key 0 is 11) 795 */ 796 mtspr(SPR_AMOR, (3ul << 62)); 797 } 798 799 static void 800 mmu_radix_init_iamr(void) 801 { 802 /* 803 * Radix always uses key0 of the IAMR to determine if an access is 804 * allowed. We set bit 0 (IBM bit 1) of key0, to prevent instruction 805 * fetch. 806 */ 807 mtspr(SPR_IAMR, (1ul << 62)); 808 } 809 810 static void 811 mmu_radix_pid_set(pmap_t pmap) 812 { 813 814 mtspr(SPR_PID, pmap->pm_pid); 815 isync(); 816 } 817 818 /* Quick sort callout for comparing physical addresses. */ 819 static int 820 pa_cmp(const void *a, const void *b) 821 { 822 const vm_paddr_t *pa = a, *pb = b; 823 824 if (*pa < *pb) 825 return (-1); 826 else if (*pa > *pb) 827 return (1); 828 else 829 return (0); 830 } 831 832 #define pte_load_store(ptep, pte) atomic_swap_long(ptep, pte) 833 #define pte_load_clear(ptep) atomic_swap_long(ptep, 0) 834 #define pte_store(ptep, pte) do { \ 835 MPASS((pte) & (RPTE_EAA_R | RPTE_EAA_W | RPTE_EAA_X)); \ 836 *(u_long *)(ptep) = htobe64((u_long)((pte) | PG_V | RPTE_LEAF)); \ 837 } while (0) 838 /* 839 * NB: should only be used for adding directories - not for direct mappings 840 */ 841 #define pde_store(ptep, pa) do { \ 842 *(u_long *)(ptep) = htobe64((u_long)(pa|RPTE_VALID|RPTE_SHIFT)); \ 843 } while (0) 844 845 #define pte_clear(ptep) do { \ 846 *(u_long *)(ptep) = (u_long)(0); \ 847 } while (0) 848 849 #define PMAP_PDE_SUPERPAGE (1 << 8) /* supports 2MB superpages */ 850 851 /* 852 * Promotion to a 2MB (PDE) page mapping requires that the corresponding 4KB 853 * (PTE) page mappings have identical settings for the following fields: 854 */ 855 #define PG_PTE_PROMOTE (PG_X | PG_MANAGED | PG_W | PG_PTE_CACHE | \ 856 PG_M | PG_A | RPTE_EAA_MASK | PG_V) 857 858 static __inline void 859 pmap_resident_count_inc(pmap_t pmap, int count) 860 { 861 862 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 863 pmap->pm_stats.resident_count += count; 864 } 865 866 static __inline void 867 pmap_resident_count_dec(pmap_t pmap, int count) 868 { 869 870 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 871 KASSERT(pmap->pm_stats.resident_count >= count, 872 ("pmap %p resident count underflow %ld %d", pmap, 873 pmap->pm_stats.resident_count, count)); 874 pmap->pm_stats.resident_count -= count; 875 } 876 877 static void 878 pagezero(vm_offset_t va) 879 { 880 va = trunc_page(va); 881 882 bzero((void *)va, PAGE_SIZE); 883 } 884 885 static uint64_t 886 allocpages(int n) 887 { 888 u_int64_t ret; 889 890 ret = moea64_bootstrap_alloc(n * PAGE_SIZE, PAGE_SIZE); 891 for (int i = 0; i < n; i++) 892 pagezero(PHYS_TO_DMAP(ret + i * PAGE_SIZE)); 893 return (ret); 894 } 895 896 static pt_entry_t * 897 kvtopte(vm_offset_t va) 898 { 899 pt_entry_t *l3e; 900 901 l3e = pmap_pml3e(kernel_pmap, va); 902 if ((be64toh(*l3e) & RPTE_VALID) == 0) 903 return (NULL); 904 return (pmap_l3e_to_pte(l3e, va)); 905 } 906 907 void 908 mmu_radix_kenter(vm_offset_t va, vm_paddr_t pa) 909 { 910 pt_entry_t *pte; 911 912 pte = kvtopte(va); 913 MPASS(pte != NULL); 914 *pte = htobe64(pa | RPTE_VALID | RPTE_LEAF | RPTE_EAA_R | \ 915 RPTE_EAA_W | RPTE_EAA_P | PG_M | PG_A); 916 } 917 918 bool 919 mmu_radix_ps_enabled(pmap_t pmap) 920 { 921 return (superpages_enabled && (pmap->pm_flags & PMAP_PDE_SUPERPAGE) != 0); 922 } 923 924 static pt_entry_t * 925 pmap_nofault_pte(pmap_t pmap, vm_offset_t va, int *is_l3e) 926 { 927 pml3_entry_t *l3e; 928 pt_entry_t *pte; 929 930 va &= PG_PS_FRAME; 931 l3e = pmap_pml3e(pmap, va); 932 if (l3e == NULL || (be64toh(*l3e) & PG_V) == 0) 933 return (NULL); 934 935 if (be64toh(*l3e) & RPTE_LEAF) { 936 *is_l3e = 1; 937 return (l3e); 938 } 939 *is_l3e = 0; 940 va &= PG_FRAME; 941 pte = pmap_l3e_to_pte(l3e, va); 942 if (pte == NULL || (be64toh(*pte) & PG_V) == 0) 943 return (NULL); 944 return (pte); 945 } 946 947 int 948 pmap_nofault(pmap_t pmap, vm_offset_t va, vm_prot_t flags) 949 { 950 pt_entry_t *pte; 951 pt_entry_t startpte, origpte, newpte; 952 vm_page_t m; 953 int is_l3e; 954 955 startpte = 0; 956 retry: 957 if ((pte = pmap_nofault_pte(pmap, va, &is_l3e)) == NULL) 958 return (KERN_INVALID_ADDRESS); 959 origpte = newpte = be64toh(*pte); 960 if (startpte == 0) { 961 startpte = origpte; 962 if (((flags & VM_PROT_WRITE) && (startpte & PG_M)) || 963 ((flags & VM_PROT_READ) && (startpte & PG_A))) { 964 pmap_invalidate_all(pmap); 965 #ifdef INVARIANTS 966 if (VERBOSE_PMAP || pmap_logging) 967 printf("%s(%p, %#lx, %#x) (%#lx) -- invalidate all\n", 968 __func__, pmap, va, flags, origpte); 969 #endif 970 return (KERN_FAILURE); 971 } 972 } 973 #ifdef INVARIANTS 974 if (VERBOSE_PMAP || pmap_logging) 975 printf("%s(%p, %#lx, %#x) (%#lx)\n", __func__, pmap, va, 976 flags, origpte); 977 #endif 978 PMAP_LOCK(pmap); 979 if ((pte = pmap_nofault_pte(pmap, va, &is_l3e)) == NULL || 980 be64toh(*pte) != origpte) { 981 PMAP_UNLOCK(pmap); 982 return (KERN_FAILURE); 983 } 984 m = PHYS_TO_VM_PAGE(newpte & PG_FRAME); 985 MPASS(m != NULL); 986 switch (flags) { 987 case VM_PROT_READ: 988 if ((newpte & (RPTE_EAA_R|RPTE_EAA_X)) == 0) 989 goto protfail; 990 newpte |= PG_A; 991 vm_page_aflag_set(m, PGA_REFERENCED); 992 break; 993 case VM_PROT_WRITE: 994 if ((newpte & RPTE_EAA_W) == 0) 995 goto protfail; 996 if (is_l3e) 997 goto protfail; 998 newpte |= PG_M; 999 vm_page_dirty(m); 1000 break; 1001 case VM_PROT_EXECUTE: 1002 if ((newpte & RPTE_EAA_X) == 0) 1003 goto protfail; 1004 newpte |= PG_A; 1005 vm_page_aflag_set(m, PGA_REFERENCED); 1006 break; 1007 } 1008 1009 if (!atomic_cmpset_long(pte, htobe64(origpte), htobe64(newpte))) 1010 goto retry; 1011 ptesync(); 1012 PMAP_UNLOCK(pmap); 1013 if (startpte == newpte) 1014 return (KERN_FAILURE); 1015 return (0); 1016 protfail: 1017 PMAP_UNLOCK(pmap); 1018 return (KERN_PROTECTION_FAILURE); 1019 } 1020 1021 /* 1022 * Returns TRUE if the given page is mapped individually or as part of 1023 * a 2mpage. Otherwise, returns FALSE. 1024 */ 1025 boolean_t 1026 mmu_radix_page_is_mapped(vm_page_t m) 1027 { 1028 struct rwlock *lock; 1029 boolean_t rv; 1030 1031 if ((m->oflags & VPO_UNMANAGED) != 0) 1032 return (FALSE); 1033 lock = VM_PAGE_TO_PV_LIST_LOCK(m); 1034 rw_rlock(lock); 1035 rv = !TAILQ_EMPTY(&m->md.pv_list) || 1036 ((m->flags & PG_FICTITIOUS) == 0 && 1037 !TAILQ_EMPTY(&pa_to_pvh(VM_PAGE_TO_PHYS(m))->pv_list)); 1038 rw_runlock(lock); 1039 return (rv); 1040 } 1041 1042 /* 1043 * Determine the appropriate bits to set in a PTE or PDE for a specified 1044 * caching mode. 1045 */ 1046 static int 1047 pmap_cache_bits(vm_memattr_t ma) 1048 { 1049 if (ma != VM_MEMATTR_DEFAULT) { 1050 switch (ma) { 1051 case VM_MEMATTR_UNCACHEABLE: 1052 return (RPTE_ATTR_GUARDEDIO); 1053 case VM_MEMATTR_CACHEABLE: 1054 return (RPTE_ATTR_MEM); 1055 case VM_MEMATTR_WRITE_BACK: 1056 case VM_MEMATTR_PREFETCHABLE: 1057 case VM_MEMATTR_WRITE_COMBINING: 1058 return (RPTE_ATTR_UNGUARDEDIO); 1059 } 1060 } 1061 return (0); 1062 } 1063 1064 static void 1065 pmap_invalidate_page(pmap_t pmap, vm_offset_t start) 1066 { 1067 ptesync(); 1068 if (pmap == kernel_pmap) 1069 radix_tlbie_invlpg_kernel_4k(start); 1070 else 1071 radix_tlbie_invlpg_user_4k(pmap->pm_pid, start); 1072 ttusync(); 1073 } 1074 1075 static void 1076 pmap_invalidate_page_2m(pmap_t pmap, vm_offset_t start) 1077 { 1078 ptesync(); 1079 if (pmap == kernel_pmap) 1080 radix_tlbie_invlpg_kernel_2m(start); 1081 else 1082 radix_tlbie_invlpg_user_2m(pmap->pm_pid, start); 1083 ttusync(); 1084 } 1085 1086 static void 1087 pmap_invalidate_pwc(pmap_t pmap) 1088 { 1089 ptesync(); 1090 if (pmap == kernel_pmap) 1091 radix_tlbie_invlpwc_kernel(); 1092 else 1093 radix_tlbie_invlpwc_user(pmap->pm_pid); 1094 ttusync(); 1095 } 1096 1097 static void 1098 pmap_invalidate_range(pmap_t pmap, vm_offset_t start, vm_offset_t end) 1099 { 1100 if (((start - end) >> PAGE_SHIFT) > 8) { 1101 pmap_invalidate_all(pmap); 1102 return; 1103 } 1104 ptesync(); 1105 if (pmap == kernel_pmap) { 1106 while (start < end) { 1107 radix_tlbie_invlpg_kernel_4k(start); 1108 start += PAGE_SIZE; 1109 } 1110 } else { 1111 while (start < end) { 1112 radix_tlbie_invlpg_user_4k(pmap->pm_pid, start); 1113 start += PAGE_SIZE; 1114 } 1115 } 1116 ttusync(); 1117 } 1118 1119 static void 1120 pmap_invalidate_all(pmap_t pmap) 1121 { 1122 ptesync(); 1123 if (pmap == kernel_pmap) 1124 radix_tlbie_flush_kernel(); 1125 else 1126 radix_tlbie_flush_user(pmap->pm_pid); 1127 ttusync(); 1128 } 1129 1130 static void 1131 pmap_invalidate_l3e_page(pmap_t pmap, vm_offset_t va, pml3_entry_t l3e) 1132 { 1133 1134 /* 1135 * When the PDE has PG_PROMOTED set, the 2MB page mapping was created 1136 * by a promotion that did not invalidate the 512 4KB page mappings 1137 * that might exist in the TLB. Consequently, at this point, the TLB 1138 * may hold both 4KB and 2MB page mappings for the address range [va, 1139 * va + L3_PAGE_SIZE). Therefore, the entire range must be invalidated here. 1140 * In contrast, when PG_PROMOTED is clear, the TLB will not hold any 1141 * 4KB page mappings for the address range [va, va + L3_PAGE_SIZE), and so a 1142 * single INVLPG suffices to invalidate the 2MB page mapping from the 1143 * TLB. 1144 */ 1145 ptesync(); 1146 if ((l3e & PG_PROMOTED) != 0) 1147 pmap_invalidate_range(pmap, va, va + L3_PAGE_SIZE - 1); 1148 else 1149 pmap_invalidate_page_2m(pmap, va); 1150 1151 pmap_invalidate_pwc(pmap); 1152 } 1153 1154 static __inline struct pv_chunk * 1155 pv_to_chunk(pv_entry_t pv) 1156 { 1157 1158 return ((struct pv_chunk *)((uintptr_t)pv & ~(uintptr_t)PAGE_MASK)); 1159 } 1160 1161 #define PV_PMAP(pv) (pv_to_chunk(pv)->pc_pmap) 1162 1163 #define PC_FREE0 0xfffffffffffffffful 1164 #define PC_FREE1 0x3ffffffffffffffful 1165 1166 static const uint64_t pc_freemask[_NPCM] = { PC_FREE0, PC_FREE1 }; 1167 1168 /* 1169 * Ensure that the number of spare PV entries in the specified pmap meets or 1170 * exceeds the given count, "needed". 1171 * 1172 * The given PV list lock may be released. 1173 */ 1174 static void 1175 reserve_pv_entries(pmap_t pmap, int needed, struct rwlock **lockp) 1176 { 1177 struct pch new_tail; 1178 struct pv_chunk *pc; 1179 vm_page_t m; 1180 int avail, free; 1181 bool reclaimed; 1182 1183 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1184 KASSERT(lockp != NULL, ("reserve_pv_entries: lockp is NULL")); 1185 1186 /* 1187 * Newly allocated PV chunks must be stored in a private list until 1188 * the required number of PV chunks have been allocated. Otherwise, 1189 * reclaim_pv_chunk() could recycle one of these chunks. In 1190 * contrast, these chunks must be added to the pmap upon allocation. 1191 */ 1192 TAILQ_INIT(&new_tail); 1193 retry: 1194 avail = 0; 1195 TAILQ_FOREACH(pc, &pmap->pm_pvchunk, pc_list) { 1196 // if ((cpu_feature2 & CPUID2_POPCNT) == 0) 1197 bit_count((bitstr_t *)pc->pc_map, 0, 1198 sizeof(pc->pc_map) * NBBY, &free); 1199 #if 0 1200 free = popcnt_pc_map_pq(pc->pc_map); 1201 #endif 1202 if (free == 0) 1203 break; 1204 avail += free; 1205 if (avail >= needed) 1206 break; 1207 } 1208 for (reclaimed = false; avail < needed; avail += _NPCPV) { 1209 m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | 1210 VM_ALLOC_WIRED); 1211 if (m == NULL) { 1212 m = reclaim_pv_chunk(pmap, lockp); 1213 if (m == NULL) 1214 goto retry; 1215 reclaimed = true; 1216 } 1217 PV_STAT(atomic_add_int(&pc_chunk_count, 1)); 1218 PV_STAT(atomic_add_int(&pc_chunk_allocs, 1)); 1219 dump_add_page(m->phys_addr); 1220 pc = (void *)PHYS_TO_DMAP(m->phys_addr); 1221 pc->pc_pmap = pmap; 1222 pc->pc_map[0] = PC_FREE0; 1223 pc->pc_map[1] = PC_FREE1; 1224 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list); 1225 TAILQ_INSERT_TAIL(&new_tail, pc, pc_lru); 1226 PV_STAT(atomic_add_int(&pv_entry_spare, _NPCPV)); 1227 1228 /* 1229 * The reclaim might have freed a chunk from the current pmap. 1230 * If that chunk contained available entries, we need to 1231 * re-count the number of available entries. 1232 */ 1233 if (reclaimed) 1234 goto retry; 1235 } 1236 if (!TAILQ_EMPTY(&new_tail)) { 1237 mtx_lock(&pv_chunks_mutex); 1238 TAILQ_CONCAT(&pv_chunks, &new_tail, pc_lru); 1239 mtx_unlock(&pv_chunks_mutex); 1240 } 1241 } 1242 1243 /* 1244 * First find and then remove the pv entry for the specified pmap and virtual 1245 * address from the specified pv list. Returns the pv entry if found and NULL 1246 * otherwise. This operation can be performed on pv lists for either 4KB or 1247 * 2MB page mappings. 1248 */ 1249 static __inline pv_entry_t 1250 pmap_pvh_remove(struct md_page *pvh, pmap_t pmap, vm_offset_t va) 1251 { 1252 pv_entry_t pv; 1253 1254 TAILQ_FOREACH(pv, &pvh->pv_list, pv_link) { 1255 #ifdef INVARIANTS 1256 if (PV_PMAP(pv) == NULL) { 1257 printf("corrupted pv_chunk/pv %p\n", pv); 1258 printf("pv_chunk: %64D\n", pv_to_chunk(pv), ":"); 1259 } 1260 MPASS(PV_PMAP(pv) != NULL); 1261 MPASS(pv->pv_va != 0); 1262 #endif 1263 if (pmap == PV_PMAP(pv) && va == pv->pv_va) { 1264 TAILQ_REMOVE(&pvh->pv_list, pv, pv_link); 1265 pvh->pv_gen++; 1266 break; 1267 } 1268 } 1269 return (pv); 1270 } 1271 1272 /* 1273 * After demotion from a 2MB page mapping to 512 4KB page mappings, 1274 * destroy the pv entry for the 2MB page mapping and reinstantiate the pv 1275 * entries for each of the 4KB page mappings. 1276 */ 1277 static void 1278 pmap_pv_demote_l3e(pmap_t pmap, vm_offset_t va, vm_paddr_t pa, 1279 struct rwlock **lockp) 1280 { 1281 struct md_page *pvh; 1282 struct pv_chunk *pc; 1283 pv_entry_t pv; 1284 vm_offset_t va_last; 1285 vm_page_t m; 1286 int bit, field; 1287 1288 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1289 KASSERT((pa & L3_PAGE_MASK) == 0, 1290 ("pmap_pv_demote_pde: pa is not 2mpage aligned")); 1291 CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa); 1292 1293 /* 1294 * Transfer the 2mpage's pv entry for this mapping to the first 1295 * page's pv list. Once this transfer begins, the pv list lock 1296 * must not be released until the last pv entry is reinstantiated. 1297 */ 1298 pvh = pa_to_pvh(pa); 1299 va = trunc_2mpage(va); 1300 pv = pmap_pvh_remove(pvh, pmap, va); 1301 KASSERT(pv != NULL, ("pmap_pv_demote_pde: pv not found")); 1302 m = PHYS_TO_VM_PAGE(pa); 1303 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_link); 1304 1305 m->md.pv_gen++; 1306 /* Instantiate the remaining NPTEPG - 1 pv entries. */ 1307 PV_STAT(atomic_add_long(&pv_entry_allocs, NPTEPG - 1)); 1308 va_last = va + L3_PAGE_SIZE - PAGE_SIZE; 1309 for (;;) { 1310 pc = TAILQ_FIRST(&pmap->pm_pvchunk); 1311 KASSERT(pc->pc_map[0] != 0 || pc->pc_map[1] != 0 1312 , ("pmap_pv_demote_pde: missing spare")); 1313 for (field = 0; field < _NPCM; field++) { 1314 while (pc->pc_map[field]) { 1315 bit = cnttzd(pc->pc_map[field]); 1316 pc->pc_map[field] &= ~(1ul << bit); 1317 pv = &pc->pc_pventry[field * 64 + bit]; 1318 va += PAGE_SIZE; 1319 pv->pv_va = va; 1320 m++; 1321 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 1322 ("pmap_pv_demote_pde: page %p is not managed", m)); 1323 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_link); 1324 1325 m->md.pv_gen++; 1326 if (va == va_last) 1327 goto out; 1328 } 1329 } 1330 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 1331 TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list); 1332 } 1333 out: 1334 if (pc->pc_map[0] == 0 && pc->pc_map[1] == 0) { 1335 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 1336 TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list); 1337 } 1338 PV_STAT(atomic_add_long(&pv_entry_count, NPTEPG - 1)); 1339 PV_STAT(atomic_subtract_int(&pv_entry_spare, NPTEPG - 1)); 1340 } 1341 1342 static void 1343 reclaim_pv_chunk_leave_pmap(pmap_t pmap, pmap_t locked_pmap) 1344 { 1345 1346 if (pmap == NULL) 1347 return; 1348 pmap_invalidate_all(pmap); 1349 if (pmap != locked_pmap) 1350 PMAP_UNLOCK(pmap); 1351 } 1352 1353 /* 1354 * We are in a serious low memory condition. Resort to 1355 * drastic measures to free some pages so we can allocate 1356 * another pv entry chunk. 1357 * 1358 * Returns NULL if PV entries were reclaimed from the specified pmap. 1359 * 1360 * We do not, however, unmap 2mpages because subsequent accesses will 1361 * allocate per-page pv entries until repromotion occurs, thereby 1362 * exacerbating the shortage of free pv entries. 1363 */ 1364 static int active_reclaims = 0; 1365 static vm_page_t 1366 reclaim_pv_chunk(pmap_t locked_pmap, struct rwlock **lockp) 1367 { 1368 struct pv_chunk *pc, *pc_marker, *pc_marker_end; 1369 struct pv_chunk_header pc_marker_b, pc_marker_end_b; 1370 struct md_page *pvh; 1371 pml3_entry_t *l3e; 1372 pmap_t next_pmap, pmap; 1373 pt_entry_t *pte, tpte; 1374 pv_entry_t pv; 1375 vm_offset_t va; 1376 vm_page_t m, m_pc; 1377 struct spglist free; 1378 uint64_t inuse; 1379 int bit, field, freed; 1380 1381 PMAP_LOCK_ASSERT(locked_pmap, MA_OWNED); 1382 KASSERT(lockp != NULL, ("reclaim_pv_chunk: lockp is NULL")); 1383 pmap = NULL; 1384 m_pc = NULL; 1385 SLIST_INIT(&free); 1386 bzero(&pc_marker_b, sizeof(pc_marker_b)); 1387 bzero(&pc_marker_end_b, sizeof(pc_marker_end_b)); 1388 pc_marker = (struct pv_chunk *)&pc_marker_b; 1389 pc_marker_end = (struct pv_chunk *)&pc_marker_end_b; 1390 1391 mtx_lock(&pv_chunks_mutex); 1392 active_reclaims++; 1393 TAILQ_INSERT_HEAD(&pv_chunks, pc_marker, pc_lru); 1394 TAILQ_INSERT_TAIL(&pv_chunks, pc_marker_end, pc_lru); 1395 while ((pc = TAILQ_NEXT(pc_marker, pc_lru)) != pc_marker_end && 1396 SLIST_EMPTY(&free)) { 1397 next_pmap = pc->pc_pmap; 1398 if (next_pmap == NULL) { 1399 /* 1400 * The next chunk is a marker. However, it is 1401 * not our marker, so active_reclaims must be 1402 * > 1. Consequently, the next_chunk code 1403 * will not rotate the pv_chunks list. 1404 */ 1405 goto next_chunk; 1406 } 1407 mtx_unlock(&pv_chunks_mutex); 1408 1409 /* 1410 * A pv_chunk can only be removed from the pc_lru list 1411 * when both pc_chunks_mutex is owned and the 1412 * corresponding pmap is locked. 1413 */ 1414 if (pmap != next_pmap) { 1415 reclaim_pv_chunk_leave_pmap(pmap, locked_pmap); 1416 pmap = next_pmap; 1417 /* Avoid deadlock and lock recursion. */ 1418 if (pmap > locked_pmap) { 1419 RELEASE_PV_LIST_LOCK(lockp); 1420 PMAP_LOCK(pmap); 1421 mtx_lock(&pv_chunks_mutex); 1422 continue; 1423 } else if (pmap != locked_pmap) { 1424 if (PMAP_TRYLOCK(pmap)) { 1425 mtx_lock(&pv_chunks_mutex); 1426 continue; 1427 } else { 1428 pmap = NULL; /* pmap is not locked */ 1429 mtx_lock(&pv_chunks_mutex); 1430 pc = TAILQ_NEXT(pc_marker, pc_lru); 1431 if (pc == NULL || 1432 pc->pc_pmap != next_pmap) 1433 continue; 1434 goto next_chunk; 1435 } 1436 } 1437 } 1438 1439 /* 1440 * Destroy every non-wired, 4 KB page mapping in the chunk. 1441 */ 1442 freed = 0; 1443 for (field = 0; field < _NPCM; field++) { 1444 for (inuse = ~pc->pc_map[field] & pc_freemask[field]; 1445 inuse != 0; inuse &= ~(1UL << bit)) { 1446 bit = cnttzd(inuse); 1447 pv = &pc->pc_pventry[field * 64 + bit]; 1448 va = pv->pv_va; 1449 l3e = pmap_pml3e(pmap, va); 1450 if ((be64toh(*l3e) & RPTE_LEAF) != 0) 1451 continue; 1452 pte = pmap_l3e_to_pte(l3e, va); 1453 if ((be64toh(*pte) & PG_W) != 0) 1454 continue; 1455 tpte = be64toh(pte_load_clear(pte)); 1456 m = PHYS_TO_VM_PAGE(tpte & PG_FRAME); 1457 if ((tpte & (PG_M | PG_RW)) == (PG_M | PG_RW)) 1458 vm_page_dirty(m); 1459 if ((tpte & PG_A) != 0) 1460 vm_page_aflag_set(m, PGA_REFERENCED); 1461 CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m); 1462 TAILQ_REMOVE(&m->md.pv_list, pv, pv_link); 1463 1464 m->md.pv_gen++; 1465 if (TAILQ_EMPTY(&m->md.pv_list) && 1466 (m->flags & PG_FICTITIOUS) == 0) { 1467 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); 1468 if (TAILQ_EMPTY(&pvh->pv_list)) { 1469 vm_page_aflag_clear(m, 1470 PGA_WRITEABLE); 1471 } 1472 } 1473 pc->pc_map[field] |= 1UL << bit; 1474 pmap_unuse_pt(pmap, va, be64toh(*l3e), &free); 1475 freed++; 1476 } 1477 } 1478 if (freed == 0) { 1479 mtx_lock(&pv_chunks_mutex); 1480 goto next_chunk; 1481 } 1482 /* Every freed mapping is for a 4 KB page. */ 1483 pmap_resident_count_dec(pmap, freed); 1484 PV_STAT(atomic_add_long(&pv_entry_frees, freed)); 1485 PV_STAT(atomic_add_int(&pv_entry_spare, freed)); 1486 PV_STAT(atomic_subtract_long(&pv_entry_count, freed)); 1487 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 1488 if (pc->pc_map[0] == PC_FREE0 && pc->pc_map[1] == PC_FREE1) { 1489 PV_STAT(atomic_subtract_int(&pv_entry_spare, _NPCPV)); 1490 PV_STAT(atomic_subtract_int(&pc_chunk_count, 1)); 1491 PV_STAT(atomic_add_int(&pc_chunk_frees, 1)); 1492 /* Entire chunk is free; return it. */ 1493 m_pc = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pc)); 1494 dump_drop_page(m_pc->phys_addr); 1495 mtx_lock(&pv_chunks_mutex); 1496 TAILQ_REMOVE(&pv_chunks, pc, pc_lru); 1497 break; 1498 } 1499 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list); 1500 mtx_lock(&pv_chunks_mutex); 1501 /* One freed pv entry in locked_pmap is sufficient. */ 1502 if (pmap == locked_pmap) 1503 break; 1504 next_chunk: 1505 TAILQ_REMOVE(&pv_chunks, pc_marker, pc_lru); 1506 TAILQ_INSERT_AFTER(&pv_chunks, pc, pc_marker, pc_lru); 1507 if (active_reclaims == 1 && pmap != NULL) { 1508 /* 1509 * Rotate the pv chunks list so that we do not 1510 * scan the same pv chunks that could not be 1511 * freed (because they contained a wired 1512 * and/or superpage mapping) on every 1513 * invocation of reclaim_pv_chunk(). 1514 */ 1515 while ((pc = TAILQ_FIRST(&pv_chunks)) != pc_marker) { 1516 MPASS(pc->pc_pmap != NULL); 1517 TAILQ_REMOVE(&pv_chunks, pc, pc_lru); 1518 TAILQ_INSERT_TAIL(&pv_chunks, pc, pc_lru); 1519 } 1520 } 1521 } 1522 TAILQ_REMOVE(&pv_chunks, pc_marker, pc_lru); 1523 TAILQ_REMOVE(&pv_chunks, pc_marker_end, pc_lru); 1524 active_reclaims--; 1525 mtx_unlock(&pv_chunks_mutex); 1526 reclaim_pv_chunk_leave_pmap(pmap, locked_pmap); 1527 if (m_pc == NULL && !SLIST_EMPTY(&free)) { 1528 m_pc = SLIST_FIRST(&free); 1529 SLIST_REMOVE_HEAD(&free, plinks.s.ss); 1530 /* Recycle a freed page table page. */ 1531 m_pc->ref_count = 1; 1532 } 1533 vm_page_free_pages_toq(&free, true); 1534 return (m_pc); 1535 } 1536 1537 /* 1538 * free the pv_entry back to the free list 1539 */ 1540 static void 1541 free_pv_entry(pmap_t pmap, pv_entry_t pv) 1542 { 1543 struct pv_chunk *pc; 1544 int idx, field, bit; 1545 1546 #ifdef VERBOSE_PV 1547 if (pmap != kernel_pmap) 1548 printf("%s(%p, %p)\n", __func__, pmap, pv); 1549 #endif 1550 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1551 PV_STAT(atomic_add_long(&pv_entry_frees, 1)); 1552 PV_STAT(atomic_add_int(&pv_entry_spare, 1)); 1553 PV_STAT(atomic_subtract_long(&pv_entry_count, 1)); 1554 pc = pv_to_chunk(pv); 1555 idx = pv - &pc->pc_pventry[0]; 1556 field = idx / 64; 1557 bit = idx % 64; 1558 pc->pc_map[field] |= 1ul << bit; 1559 if (pc->pc_map[0] != PC_FREE0 || pc->pc_map[1] != PC_FREE1) { 1560 /* 98% of the time, pc is already at the head of the list. */ 1561 if (__predict_false(pc != TAILQ_FIRST(&pmap->pm_pvchunk))) { 1562 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 1563 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list); 1564 } 1565 return; 1566 } 1567 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 1568 free_pv_chunk(pc); 1569 } 1570 1571 static void 1572 free_pv_chunk(struct pv_chunk *pc) 1573 { 1574 vm_page_t m; 1575 1576 mtx_lock(&pv_chunks_mutex); 1577 TAILQ_REMOVE(&pv_chunks, pc, pc_lru); 1578 mtx_unlock(&pv_chunks_mutex); 1579 PV_STAT(atomic_subtract_int(&pv_entry_spare, _NPCPV)); 1580 PV_STAT(atomic_subtract_int(&pc_chunk_count, 1)); 1581 PV_STAT(atomic_add_int(&pc_chunk_frees, 1)); 1582 /* entire chunk is free, return it */ 1583 m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pc)); 1584 dump_drop_page(m->phys_addr); 1585 vm_page_unwire_noq(m); 1586 vm_page_free(m); 1587 } 1588 1589 /* 1590 * Returns a new PV entry, allocating a new PV chunk from the system when 1591 * needed. If this PV chunk allocation fails and a PV list lock pointer was 1592 * given, a PV chunk is reclaimed from an arbitrary pmap. Otherwise, NULL is 1593 * returned. 1594 * 1595 * The given PV list lock may be released. 1596 */ 1597 static pv_entry_t 1598 get_pv_entry(pmap_t pmap, struct rwlock **lockp) 1599 { 1600 int bit, field; 1601 pv_entry_t pv; 1602 struct pv_chunk *pc; 1603 vm_page_t m; 1604 1605 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1606 PV_STAT(atomic_add_long(&pv_entry_allocs, 1)); 1607 retry: 1608 pc = TAILQ_FIRST(&pmap->pm_pvchunk); 1609 if (pc != NULL) { 1610 for (field = 0; field < _NPCM; field++) { 1611 if (pc->pc_map[field]) { 1612 bit = cnttzd(pc->pc_map[field]); 1613 break; 1614 } 1615 } 1616 if (field < _NPCM) { 1617 pv = &pc->pc_pventry[field * 64 + bit]; 1618 pc->pc_map[field] &= ~(1ul << bit); 1619 /* If this was the last item, move it to tail */ 1620 if (pc->pc_map[0] == 0 && pc->pc_map[1] == 0) { 1621 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 1622 TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, 1623 pc_list); 1624 } 1625 PV_STAT(atomic_add_long(&pv_entry_count, 1)); 1626 PV_STAT(atomic_subtract_int(&pv_entry_spare, 1)); 1627 MPASS(PV_PMAP(pv) != NULL); 1628 return (pv); 1629 } 1630 } 1631 /* No free items, allocate another chunk */ 1632 m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | 1633 VM_ALLOC_WIRED); 1634 if (m == NULL) { 1635 if (lockp == NULL) { 1636 PV_STAT(pc_chunk_tryfail++); 1637 return (NULL); 1638 } 1639 m = reclaim_pv_chunk(pmap, lockp); 1640 if (m == NULL) 1641 goto retry; 1642 } 1643 PV_STAT(atomic_add_int(&pc_chunk_count, 1)); 1644 PV_STAT(atomic_add_int(&pc_chunk_allocs, 1)); 1645 dump_add_page(m->phys_addr); 1646 pc = (void *)PHYS_TO_DMAP(m->phys_addr); 1647 pc->pc_pmap = pmap; 1648 pc->pc_map[0] = PC_FREE0 & ~1ul; /* preallocated bit 0 */ 1649 pc->pc_map[1] = PC_FREE1; 1650 mtx_lock(&pv_chunks_mutex); 1651 TAILQ_INSERT_TAIL(&pv_chunks, pc, pc_lru); 1652 mtx_unlock(&pv_chunks_mutex); 1653 pv = &pc->pc_pventry[0]; 1654 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list); 1655 PV_STAT(atomic_add_long(&pv_entry_count, 1)); 1656 PV_STAT(atomic_add_int(&pv_entry_spare, _NPCPV - 1)); 1657 MPASS(PV_PMAP(pv) != NULL); 1658 return (pv); 1659 } 1660 1661 #if VM_NRESERVLEVEL > 0 1662 /* 1663 * After promotion from 512 4KB page mappings to a single 2MB page mapping, 1664 * replace the many pv entries for the 4KB page mappings by a single pv entry 1665 * for the 2MB page mapping. 1666 */ 1667 static void 1668 pmap_pv_promote_l3e(pmap_t pmap, vm_offset_t va, vm_paddr_t pa, 1669 struct rwlock **lockp) 1670 { 1671 struct md_page *pvh; 1672 pv_entry_t pv; 1673 vm_offset_t va_last; 1674 vm_page_t m; 1675 1676 KASSERT((pa & L3_PAGE_MASK) == 0, 1677 ("pmap_pv_promote_pde: pa is not 2mpage aligned")); 1678 CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa); 1679 1680 /* 1681 * Transfer the first page's pv entry for this mapping to the 2mpage's 1682 * pv list. Aside from avoiding the cost of a call to get_pv_entry(), 1683 * a transfer avoids the possibility that get_pv_entry() calls 1684 * reclaim_pv_chunk() and that reclaim_pv_chunk() removes one of the 1685 * mappings that is being promoted. 1686 */ 1687 m = PHYS_TO_VM_PAGE(pa); 1688 va = trunc_2mpage(va); 1689 pv = pmap_pvh_remove(&m->md, pmap, va); 1690 KASSERT(pv != NULL, ("pmap_pv_promote_pde: pv not found")); 1691 pvh = pa_to_pvh(pa); 1692 TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_link); 1693 pvh->pv_gen++; 1694 /* Free the remaining NPTEPG - 1 pv entries. */ 1695 va_last = va + L3_PAGE_SIZE - PAGE_SIZE; 1696 do { 1697 m++; 1698 va += PAGE_SIZE; 1699 pmap_pvh_free(&m->md, pmap, va); 1700 } while (va < va_last); 1701 } 1702 #endif /* VM_NRESERVLEVEL > 0 */ 1703 1704 /* 1705 * First find and then destroy the pv entry for the specified pmap and virtual 1706 * address. This operation can be performed on pv lists for either 4KB or 2MB 1707 * page mappings. 1708 */ 1709 static void 1710 pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va) 1711 { 1712 pv_entry_t pv; 1713 1714 pv = pmap_pvh_remove(pvh, pmap, va); 1715 KASSERT(pv != NULL, ("pmap_pvh_free: pv not found")); 1716 free_pv_entry(pmap, pv); 1717 } 1718 1719 /* 1720 * Conditionally create the PV entry for a 4KB page mapping if the required 1721 * memory can be allocated without resorting to reclamation. 1722 */ 1723 static boolean_t 1724 pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, vm_page_t m, 1725 struct rwlock **lockp) 1726 { 1727 pv_entry_t pv; 1728 1729 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1730 /* Pass NULL instead of the lock pointer to disable reclamation. */ 1731 if ((pv = get_pv_entry(pmap, NULL)) != NULL) { 1732 pv->pv_va = va; 1733 CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m); 1734 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_link); 1735 m->md.pv_gen++; 1736 return (TRUE); 1737 } else 1738 return (FALSE); 1739 } 1740 1741 vm_paddr_t phys_avail_debug[2 * VM_PHYSSEG_MAX]; 1742 #ifdef INVARIANTS 1743 static void 1744 validate_addr(vm_paddr_t addr, vm_size_t size) 1745 { 1746 vm_paddr_t end = addr + size; 1747 bool found = false; 1748 1749 for (int i = 0; i < 2 * phys_avail_count; i += 2) { 1750 if (addr >= phys_avail_debug[i] && 1751 end <= phys_avail_debug[i + 1]) { 1752 found = true; 1753 break; 1754 } 1755 } 1756 KASSERT(found, ("%#lx-%#lx outside of initial phys_avail array", 1757 addr, end)); 1758 } 1759 #else 1760 static void validate_addr(vm_paddr_t addr, vm_size_t size) {} 1761 #endif 1762 #define DMAP_PAGE_BITS (RPTE_VALID | RPTE_LEAF | RPTE_EAA_MASK | PG_M | PG_A) 1763 1764 static vm_paddr_t 1765 alloc_pt_page(void) 1766 { 1767 vm_paddr_t page; 1768 1769 page = allocpages(1); 1770 pagezero(PHYS_TO_DMAP(page)); 1771 return (page); 1772 } 1773 1774 static void 1775 mmu_radix_dmap_range(vm_paddr_t start, vm_paddr_t end) 1776 { 1777 pt_entry_t *pte, pteval; 1778 vm_paddr_t page; 1779 1780 if (bootverbose) 1781 printf("%s %lx -> %lx\n", __func__, start, end); 1782 while (start < end) { 1783 pteval = start | DMAP_PAGE_BITS; 1784 pte = pmap_pml1e(kernel_pmap, PHYS_TO_DMAP(start)); 1785 if ((be64toh(*pte) & RPTE_VALID) == 0) { 1786 page = alloc_pt_page(); 1787 pde_store(pte, page); 1788 } 1789 pte = pmap_l1e_to_l2e(pte, PHYS_TO_DMAP(start)); 1790 if ((start & L2_PAGE_MASK) == 0 && 1791 end - start >= L2_PAGE_SIZE) { 1792 start += L2_PAGE_SIZE; 1793 goto done; 1794 } else if ((be64toh(*pte) & RPTE_VALID) == 0) { 1795 page = alloc_pt_page(); 1796 pde_store(pte, page); 1797 } 1798 1799 pte = pmap_l2e_to_l3e(pte, PHYS_TO_DMAP(start)); 1800 if ((start & L3_PAGE_MASK) == 0 && 1801 end - start >= L3_PAGE_SIZE) { 1802 start += L3_PAGE_SIZE; 1803 goto done; 1804 } else if ((be64toh(*pte) & RPTE_VALID) == 0) { 1805 page = alloc_pt_page(); 1806 pde_store(pte, page); 1807 } 1808 pte = pmap_l3e_to_pte(pte, PHYS_TO_DMAP(start)); 1809 start += PAGE_SIZE; 1810 done: 1811 pte_store(pte, pteval); 1812 } 1813 } 1814 1815 static void 1816 mmu_radix_dmap_populate(vm_size_t hwphyssz) 1817 { 1818 vm_paddr_t start, end; 1819 1820 for (int i = 0; i < pregions_sz; i++) { 1821 start = pregions[i].mr_start; 1822 end = start + pregions[i].mr_size; 1823 if (hwphyssz && start >= hwphyssz) 1824 break; 1825 if (hwphyssz && hwphyssz < end) 1826 end = hwphyssz; 1827 mmu_radix_dmap_range(start, end); 1828 } 1829 } 1830 1831 static void 1832 mmu_radix_setup_pagetables(vm_size_t hwphyssz) 1833 { 1834 vm_paddr_t ptpages, pages; 1835 pt_entry_t *pte; 1836 vm_paddr_t l1phys; 1837 1838 bzero(kernel_pmap, sizeof(struct pmap)); 1839 PMAP_LOCK_INIT(kernel_pmap); 1840 1841 ptpages = allocpages(3); 1842 l1phys = moea64_bootstrap_alloc(RADIX_PGD_SIZE, RADIX_PGD_SIZE); 1843 validate_addr(l1phys, RADIX_PGD_SIZE); 1844 if (bootverbose) 1845 printf("l1phys=%lx\n", l1phys); 1846 MPASS((l1phys & (RADIX_PGD_SIZE-1)) == 0); 1847 for (int i = 0; i < RADIX_PGD_SIZE/PAGE_SIZE; i++) 1848 pagezero(PHYS_TO_DMAP(l1phys + i * PAGE_SIZE)); 1849 kernel_pmap->pm_pml1 = (pml1_entry_t *)PHYS_TO_DMAP(l1phys); 1850 1851 mmu_radix_dmap_populate(hwphyssz); 1852 1853 /* 1854 * Create page tables for first 128MB of KVA 1855 */ 1856 pages = ptpages; 1857 pte = pmap_pml1e(kernel_pmap, VM_MIN_KERNEL_ADDRESS); 1858 *pte = htobe64(pages | RPTE_VALID | RPTE_SHIFT); 1859 pages += PAGE_SIZE; 1860 pte = pmap_l1e_to_l2e(pte, VM_MIN_KERNEL_ADDRESS); 1861 *pte = htobe64(pages | RPTE_VALID | RPTE_SHIFT); 1862 pages += PAGE_SIZE; 1863 pte = pmap_l2e_to_l3e(pte, VM_MIN_KERNEL_ADDRESS); 1864 /* 1865 * the kernel page table pages need to be preserved in 1866 * phys_avail and not overlap with previous allocations 1867 */ 1868 pages = allocpages(nkpt); 1869 if (bootverbose) { 1870 printf("phys_avail after dmap populate and nkpt allocation\n"); 1871 for (int j = 0; j < 2 * phys_avail_count; j+=2) 1872 printf("phys_avail[%d]=%08lx - phys_avail[%d]=%08lx\n", 1873 j, phys_avail[j], j + 1, phys_avail[j + 1]); 1874 } 1875 KPTphys = pages; 1876 for (int i = 0; i < nkpt; i++, pte++, pages += PAGE_SIZE) 1877 *pte = htobe64(pages | RPTE_VALID | RPTE_SHIFT); 1878 kernel_vm_end = VM_MIN_KERNEL_ADDRESS + nkpt * L3_PAGE_SIZE; 1879 if (bootverbose) 1880 printf("kernel_pmap pml1 %p\n", kernel_pmap->pm_pml1); 1881 /* 1882 * Add a physical memory segment (vm_phys_seg) corresponding to the 1883 * preallocated kernel page table pages so that vm_page structures 1884 * representing these pages will be created. The vm_page structures 1885 * are required for promotion of the corresponding kernel virtual 1886 * addresses to superpage mappings. 1887 */ 1888 vm_phys_add_seg(KPTphys, KPTphys + ptoa(nkpt)); 1889 } 1890 1891 static void 1892 mmu_radix_early_bootstrap(vm_offset_t start, vm_offset_t end) 1893 { 1894 vm_paddr_t kpstart, kpend; 1895 vm_size_t physsz, hwphyssz; 1896 //uint64_t l2virt; 1897 int rm_pavail, proctab_size; 1898 int i, j; 1899 1900 kpstart = start & ~DMAP_BASE_ADDRESS; 1901 kpend = end & ~DMAP_BASE_ADDRESS; 1902 1903 /* Get physical memory regions from firmware */ 1904 mem_regions(&pregions, &pregions_sz, ®ions, ®ions_sz); 1905 CTR0(KTR_PMAP, "mmu_radix_early_bootstrap: physical memory"); 1906 1907 if (2 * VM_PHYSSEG_MAX < regions_sz) 1908 panic("mmu_radix_early_bootstrap: phys_avail too small"); 1909 1910 if (bootverbose) 1911 for (int i = 0; i < regions_sz; i++) 1912 printf("regions[%d].mr_start=%lx regions[%d].mr_size=%lx\n", 1913 i, regions[i].mr_start, i, regions[i].mr_size); 1914 /* 1915 * XXX workaround a simulator bug 1916 */ 1917 for (int i = 0; i < regions_sz; i++) 1918 if (regions[i].mr_start & PAGE_MASK) { 1919 regions[i].mr_start += PAGE_MASK; 1920 regions[i].mr_start &= ~PAGE_MASK; 1921 regions[i].mr_size &= ~PAGE_MASK; 1922 } 1923 if (bootverbose) 1924 for (int i = 0; i < pregions_sz; i++) 1925 printf("pregions[%d].mr_start=%lx pregions[%d].mr_size=%lx\n", 1926 i, pregions[i].mr_start, i, pregions[i].mr_size); 1927 1928 phys_avail_count = 0; 1929 physsz = 0; 1930 hwphyssz = 0; 1931 TUNABLE_ULONG_FETCH("hw.physmem", (u_long *) &hwphyssz); 1932 for (i = 0, j = 0; i < regions_sz; i++) { 1933 if (bootverbose) 1934 printf("regions[%d].mr_start=%016lx regions[%d].mr_size=%016lx\n", 1935 i, regions[i].mr_start, i, regions[i].mr_size); 1936 1937 if (regions[i].mr_size < PAGE_SIZE) 1938 continue; 1939 1940 if (hwphyssz != 0 && 1941 (physsz + regions[i].mr_size) >= hwphyssz) { 1942 if (physsz < hwphyssz) { 1943 phys_avail[j] = regions[i].mr_start; 1944 phys_avail[j + 1] = regions[i].mr_start + 1945 (hwphyssz - physsz); 1946 physsz = hwphyssz; 1947 phys_avail_count++; 1948 dump_avail[j] = phys_avail[j]; 1949 dump_avail[j + 1] = phys_avail[j + 1]; 1950 } 1951 break; 1952 } 1953 phys_avail[j] = regions[i].mr_start; 1954 phys_avail[j + 1] = regions[i].mr_start + regions[i].mr_size; 1955 dump_avail[j] = phys_avail[j]; 1956 dump_avail[j + 1] = phys_avail[j + 1]; 1957 1958 phys_avail_count++; 1959 physsz += regions[i].mr_size; 1960 j += 2; 1961 } 1962 1963 /* Check for overlap with the kernel and exception vectors */ 1964 rm_pavail = 0; 1965 for (j = 0; j < 2 * phys_avail_count; j+=2) { 1966 if (phys_avail[j] < EXC_LAST) 1967 phys_avail[j] += EXC_LAST; 1968 1969 if (phys_avail[j] >= kpstart && 1970 phys_avail[j + 1] <= kpend) { 1971 phys_avail[j] = phys_avail[j + 1] = ~0; 1972 rm_pavail++; 1973 continue; 1974 } 1975 1976 if (kpstart >= phys_avail[j] && 1977 kpstart < phys_avail[j + 1]) { 1978 if (kpend < phys_avail[j + 1]) { 1979 phys_avail[2 * phys_avail_count] = 1980 (kpend & ~PAGE_MASK) + PAGE_SIZE; 1981 phys_avail[2 * phys_avail_count + 1] = 1982 phys_avail[j + 1]; 1983 phys_avail_count++; 1984 } 1985 1986 phys_avail[j + 1] = kpstart & ~PAGE_MASK; 1987 } 1988 1989 if (kpend >= phys_avail[j] && 1990 kpend < phys_avail[j + 1]) { 1991 if (kpstart > phys_avail[j]) { 1992 phys_avail[2 * phys_avail_count] = phys_avail[j]; 1993 phys_avail[2 * phys_avail_count + 1] = 1994 kpstart & ~PAGE_MASK; 1995 phys_avail_count++; 1996 } 1997 1998 phys_avail[j] = (kpend & ~PAGE_MASK) + 1999 PAGE_SIZE; 2000 } 2001 } 2002 qsort(phys_avail, 2 * phys_avail_count, sizeof(phys_avail[0]), pa_cmp); 2003 for (i = 0; i < 2 * phys_avail_count; i++) 2004 phys_avail_debug[i] = phys_avail[i]; 2005 2006 /* Remove physical available regions marked for removal (~0) */ 2007 if (rm_pavail) { 2008 phys_avail_count -= rm_pavail; 2009 for (i = 2 * phys_avail_count; 2010 i < 2*(phys_avail_count + rm_pavail); i+=2) 2011 phys_avail[i] = phys_avail[i + 1] = 0; 2012 } 2013 if (bootverbose) { 2014 printf("phys_avail ranges after filtering:\n"); 2015 for (j = 0; j < 2 * phys_avail_count; j+=2) 2016 printf("phys_avail[%d]=%08lx - phys_avail[%d]=%08lx\n", 2017 j, phys_avail[j], j + 1, phys_avail[j + 1]); 2018 } 2019 physmem = btoc(physsz); 2020 2021 /* XXX assume we're running non-virtualized and 2022 * we don't support BHYVE 2023 */ 2024 if (isa3_pid_bits == 0) 2025 isa3_pid_bits = 20; 2026 parttab_phys = moea64_bootstrap_alloc(PARTTAB_SIZE, PARTTAB_SIZE); 2027 validate_addr(parttab_phys, PARTTAB_SIZE); 2028 for (int i = 0; i < PARTTAB_SIZE/PAGE_SIZE; i++) 2029 pagezero(PHYS_TO_DMAP(parttab_phys + i * PAGE_SIZE)); 2030 2031 proctab_size = 1UL << PROCTAB_SIZE_SHIFT; 2032 proctab0pa = moea64_bootstrap_alloc(proctab_size, proctab_size); 2033 validate_addr(proctab0pa, proctab_size); 2034 for (int i = 0; i < proctab_size/PAGE_SIZE; i++) 2035 pagezero(PHYS_TO_DMAP(proctab0pa + i * PAGE_SIZE)); 2036 2037 mmu_radix_setup_pagetables(hwphyssz); 2038 } 2039 2040 static void 2041 mmu_radix_late_bootstrap(vm_offset_t start, vm_offset_t end) 2042 { 2043 int i; 2044 vm_paddr_t pa; 2045 void *dpcpu; 2046 vm_offset_t va; 2047 2048 /* 2049 * Set up the Open Firmware pmap and add its mappings if not in real 2050 * mode. 2051 */ 2052 if (bootverbose) 2053 printf("%s enter\n", __func__); 2054 2055 /* 2056 * Calculate the last available physical address, and reserve the 2057 * vm_page_array (upper bound). 2058 */ 2059 Maxmem = 0; 2060 for (i = 0; phys_avail[i + 2] != 0; i += 2) 2061 Maxmem = MAX(Maxmem, powerpc_btop(phys_avail[i + 1])); 2062 2063 /* 2064 * Set the start and end of kva. 2065 */ 2066 virtual_avail = VM_MIN_KERNEL_ADDRESS; 2067 virtual_end = VM_MAX_SAFE_KERNEL_ADDRESS; 2068 2069 /* 2070 * Remap any early IO mappings (console framebuffer, etc.) 2071 */ 2072 bs_remap_earlyboot(); 2073 2074 /* 2075 * Allocate a kernel stack with a guard page for thread0 and map it 2076 * into the kernel page map. 2077 */ 2078 pa = allocpages(kstack_pages); 2079 va = virtual_avail + KSTACK_GUARD_PAGES * PAGE_SIZE; 2080 virtual_avail = va + kstack_pages * PAGE_SIZE; 2081 CTR2(KTR_PMAP, "moea64_bootstrap: kstack0 at %#x (%#x)", pa, va); 2082 thread0.td_kstack = va; 2083 for (i = 0; i < kstack_pages; i++) { 2084 mmu_radix_kenter(va, pa); 2085 pa += PAGE_SIZE; 2086 va += PAGE_SIZE; 2087 } 2088 thread0.td_kstack_pages = kstack_pages; 2089 2090 /* 2091 * Allocate virtual address space for the message buffer. 2092 */ 2093 pa = msgbuf_phys = allocpages((msgbufsize + PAGE_MASK) >> PAGE_SHIFT); 2094 msgbufp = (struct msgbuf *)PHYS_TO_DMAP(pa); 2095 2096 /* 2097 * Allocate virtual address space for the dynamic percpu area. 2098 */ 2099 pa = allocpages(DPCPU_SIZE >> PAGE_SHIFT); 2100 dpcpu = (void *)PHYS_TO_DMAP(pa); 2101 dpcpu_init(dpcpu, curcpu); 2102 2103 crashdumpmap = (caddr_t)virtual_avail; 2104 virtual_avail += MAXDUMPPGS * PAGE_SIZE; 2105 2106 /* 2107 * Reserve some special page table entries/VA space for temporary 2108 * mapping of pages. 2109 */ 2110 } 2111 2112 static void 2113 mmu_parttab_init(void) 2114 { 2115 uint64_t ptcr; 2116 2117 isa3_parttab = (struct pate *)PHYS_TO_DMAP(parttab_phys); 2118 2119 if (bootverbose) 2120 printf("%s parttab: %p\n", __func__, isa3_parttab); 2121 ptcr = parttab_phys | (PARTTAB_SIZE_SHIFT-12); 2122 if (bootverbose) 2123 printf("setting ptcr %lx\n", ptcr); 2124 mtspr(SPR_PTCR, ptcr); 2125 } 2126 2127 static void 2128 mmu_parttab_update(uint64_t lpid, uint64_t pagetab, uint64_t proctab) 2129 { 2130 uint64_t prev; 2131 2132 if (bootverbose) 2133 printf("%s isa3_parttab %p lpid %lx pagetab %lx proctab %lx\n", __func__, isa3_parttab, 2134 lpid, pagetab, proctab); 2135 prev = be64toh(isa3_parttab[lpid].pagetab); 2136 isa3_parttab[lpid].pagetab = htobe64(pagetab); 2137 isa3_parttab[lpid].proctab = htobe64(proctab); 2138 2139 if (prev & PARTTAB_HR) { 2140 __asm __volatile(PPC_TLBIE_5(%0,%1,2,0,1) : : 2141 "r" (TLBIEL_INVAL_SET_LPID), "r" (lpid)); 2142 __asm __volatile(PPC_TLBIE_5(%0,%1,2,1,1) : : 2143 "r" (TLBIEL_INVAL_SET_LPID), "r" (lpid)); 2144 } else { 2145 __asm __volatile(PPC_TLBIE_5(%0,%1,2,0,0) : : 2146 "r" (TLBIEL_INVAL_SET_LPID), "r" (lpid)); 2147 } 2148 ttusync(); 2149 } 2150 2151 static void 2152 mmu_radix_parttab_init(void) 2153 { 2154 uint64_t pagetab; 2155 2156 mmu_parttab_init(); 2157 pagetab = RTS_SIZE | DMAP_TO_PHYS((vm_offset_t)kernel_pmap->pm_pml1) | \ 2158 RADIX_PGD_INDEX_SHIFT | PARTTAB_HR; 2159 mmu_parttab_update(0, pagetab, 0); 2160 } 2161 2162 static void 2163 mmu_radix_proctab_register(vm_paddr_t proctabpa, uint64_t table_size) 2164 { 2165 uint64_t pagetab, proctab; 2166 2167 pagetab = be64toh(isa3_parttab[0].pagetab); 2168 proctab = proctabpa | table_size | PARTTAB_GR; 2169 mmu_parttab_update(0, pagetab, proctab); 2170 } 2171 2172 static void 2173 mmu_radix_proctab_init(void) 2174 { 2175 2176 isa3_base_pid = 1; 2177 2178 isa3_proctab = (void*)PHYS_TO_DMAP(proctab0pa); 2179 isa3_proctab->proctab0 = 2180 htobe64(RTS_SIZE | DMAP_TO_PHYS((vm_offset_t)kernel_pmap->pm_pml1) | 2181 RADIX_PGD_INDEX_SHIFT); 2182 2183 mmu_radix_proctab_register(proctab0pa, PROCTAB_SIZE_SHIFT - 12); 2184 2185 __asm __volatile("ptesync" : : : "memory"); 2186 __asm __volatile(PPC_TLBIE_5(%0,%1,2,1,1) : : 2187 "r" (TLBIEL_INVAL_SET_LPID), "r" (0)); 2188 __asm __volatile("eieio; tlbsync; ptesync" : : : "memory"); 2189 if (bootverbose) 2190 printf("process table %p and kernel radix PDE: %p\n", 2191 isa3_proctab, kernel_pmap->pm_pml1); 2192 mtmsr(mfmsr() | PSL_DR ); 2193 mtmsr(mfmsr() & ~PSL_DR); 2194 kernel_pmap->pm_pid = isa3_base_pid; 2195 isa3_base_pid++; 2196 } 2197 2198 void 2199 mmu_radix_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, 2200 int advice) 2201 { 2202 struct rwlock *lock; 2203 pml1_entry_t *l1e; 2204 pml2_entry_t *l2e; 2205 pml3_entry_t oldl3e, *l3e; 2206 pt_entry_t *pte; 2207 vm_offset_t va, va_next; 2208 vm_page_t m; 2209 boolean_t anychanged; 2210 2211 if (advice != MADV_DONTNEED && advice != MADV_FREE) 2212 return; 2213 anychanged = FALSE; 2214 PMAP_LOCK(pmap); 2215 for (; sva < eva; sva = va_next) { 2216 l1e = pmap_pml1e(pmap, sva); 2217 if ((be64toh(*l1e) & PG_V) == 0) { 2218 va_next = (sva + L1_PAGE_SIZE) & ~L1_PAGE_MASK; 2219 if (va_next < sva) 2220 va_next = eva; 2221 continue; 2222 } 2223 l2e = pmap_l1e_to_l2e(l1e, sva); 2224 if ((be64toh(*l2e) & PG_V) == 0) { 2225 va_next = (sva + L2_PAGE_SIZE) & ~L2_PAGE_MASK; 2226 if (va_next < sva) 2227 va_next = eva; 2228 continue; 2229 } 2230 va_next = (sva + L3_PAGE_SIZE) & ~L3_PAGE_MASK; 2231 if (va_next < sva) 2232 va_next = eva; 2233 l3e = pmap_l2e_to_l3e(l2e, sva); 2234 oldl3e = be64toh(*l3e); 2235 if ((oldl3e & PG_V) == 0) 2236 continue; 2237 else if ((oldl3e & RPTE_LEAF) != 0) { 2238 if ((oldl3e & PG_MANAGED) == 0) 2239 continue; 2240 lock = NULL; 2241 if (!pmap_demote_l3e_locked(pmap, l3e, sva, &lock)) { 2242 if (lock != NULL) 2243 rw_wunlock(lock); 2244 2245 /* 2246 * The large page mapping was destroyed. 2247 */ 2248 continue; 2249 } 2250 2251 /* 2252 * Unless the page mappings are wired, remove the 2253 * mapping to a single page so that a subsequent 2254 * access may repromote. Since the underlying page 2255 * table page is fully populated, this removal never 2256 * frees a page table page. 2257 */ 2258 if ((oldl3e & PG_W) == 0) { 2259 pte = pmap_l3e_to_pte(l3e, sva); 2260 KASSERT((be64toh(*pte) & PG_V) != 0, 2261 ("pmap_advise: invalid PTE")); 2262 pmap_remove_pte(pmap, pte, sva, be64toh(*l3e), NULL, 2263 &lock); 2264 anychanged = TRUE; 2265 } 2266 if (lock != NULL) 2267 rw_wunlock(lock); 2268 } 2269 if (va_next > eva) 2270 va_next = eva; 2271 va = va_next; 2272 for (pte = pmap_l3e_to_pte(l3e, sva); sva != va_next; 2273 pte++, sva += PAGE_SIZE) { 2274 MPASS(pte == pmap_pte(pmap, sva)); 2275 2276 if ((be64toh(*pte) & (PG_MANAGED | PG_V)) != (PG_MANAGED | PG_V)) 2277 goto maybe_invlrng; 2278 else if ((be64toh(*pte) & (PG_M | PG_RW)) == (PG_M | PG_RW)) { 2279 if (advice == MADV_DONTNEED) { 2280 /* 2281 * Future calls to pmap_is_modified() 2282 * can be avoided by making the page 2283 * dirty now. 2284 */ 2285 m = PHYS_TO_VM_PAGE(be64toh(*pte) & PG_FRAME); 2286 vm_page_dirty(m); 2287 } 2288 atomic_clear_long(pte, htobe64(PG_M | PG_A)); 2289 } else if ((be64toh(*pte) & PG_A) != 0) 2290 atomic_clear_long(pte, htobe64(PG_A)); 2291 else 2292 goto maybe_invlrng; 2293 anychanged = TRUE; 2294 continue; 2295 maybe_invlrng: 2296 if (va != va_next) { 2297 anychanged = true; 2298 va = va_next; 2299 } 2300 } 2301 if (va != va_next) 2302 anychanged = true; 2303 } 2304 if (anychanged) 2305 pmap_invalidate_all(pmap); 2306 PMAP_UNLOCK(pmap); 2307 } 2308 2309 /* 2310 * Routines used in machine-dependent code 2311 */ 2312 static void 2313 mmu_radix_bootstrap(vm_offset_t start, vm_offset_t end) 2314 { 2315 uint64_t lpcr; 2316 2317 if (bootverbose) 2318 printf("%s\n", __func__); 2319 hw_direct_map = 1; 2320 mmu_radix_early_bootstrap(start, end); 2321 if (bootverbose) 2322 printf("early bootstrap complete\n"); 2323 if (powernv_enabled) { 2324 lpcr = mfspr(SPR_LPCR); 2325 mtspr(SPR_LPCR, lpcr | LPCR_UPRT | LPCR_HR); 2326 mmu_radix_parttab_init(); 2327 mmu_radix_init_amor(); 2328 if (bootverbose) 2329 printf("powernv init complete\n"); 2330 } 2331 mmu_radix_init_iamr(); 2332 mmu_radix_proctab_init(); 2333 mmu_radix_pid_set(kernel_pmap); 2334 /* XXX assume CPU_FTR_HVMODE */ 2335 mmu_radix_tlbiel_flush(TLB_INVAL_SCOPE_GLOBAL); 2336 2337 mmu_radix_late_bootstrap(start, end); 2338 numa_mem_regions(&numa_pregions, &numa_pregions_sz); 2339 if (bootverbose) 2340 printf("%s done\n", __func__); 2341 pmap_bootstrapped = 1; 2342 dmaplimit = roundup2(powerpc_ptob(Maxmem), L2_PAGE_SIZE); 2343 PCPU_SET(flags, PCPU_GET(flags) | PC_FLAG_NOSRS); 2344 } 2345 2346 static void 2347 mmu_radix_cpu_bootstrap(int ap) 2348 { 2349 uint64_t lpcr; 2350 uint64_t ptcr; 2351 2352 if (powernv_enabled) { 2353 lpcr = mfspr(SPR_LPCR); 2354 mtspr(SPR_LPCR, lpcr | LPCR_UPRT | LPCR_HR); 2355 2356 ptcr = parttab_phys | (PARTTAB_SIZE_SHIFT-12); 2357 mtspr(SPR_PTCR, ptcr); 2358 mmu_radix_init_amor(); 2359 } 2360 mmu_radix_init_iamr(); 2361 mmu_radix_pid_set(kernel_pmap); 2362 mmu_radix_tlbiel_flush(TLB_INVAL_SCOPE_GLOBAL); 2363 } 2364 2365 static SYSCTL_NODE(_vm_pmap, OID_AUTO, l3e, CTLFLAG_RD, 0, 2366 "2MB page mapping counters"); 2367 2368 static u_long pmap_l3e_demotions; 2369 SYSCTL_ULONG(_vm_pmap_l3e, OID_AUTO, demotions, CTLFLAG_RD, 2370 &pmap_l3e_demotions, 0, "2MB page demotions"); 2371 2372 static u_long pmap_l3e_mappings; 2373 SYSCTL_ULONG(_vm_pmap_l3e, OID_AUTO, mappings, CTLFLAG_RD, 2374 &pmap_l3e_mappings, 0, "2MB page mappings"); 2375 2376 static u_long pmap_l3e_p_failures; 2377 SYSCTL_ULONG(_vm_pmap_l3e, OID_AUTO, p_failures, CTLFLAG_RD, 2378 &pmap_l3e_p_failures, 0, "2MB page promotion failures"); 2379 2380 static u_long pmap_l3e_promotions; 2381 SYSCTL_ULONG(_vm_pmap_l3e, OID_AUTO, promotions, CTLFLAG_RD, 2382 &pmap_l3e_promotions, 0, "2MB page promotions"); 2383 2384 static SYSCTL_NODE(_vm_pmap, OID_AUTO, l2e, CTLFLAG_RD, 0, 2385 "1GB page mapping counters"); 2386 2387 static u_long pmap_l2e_demotions; 2388 SYSCTL_ULONG(_vm_pmap_l2e, OID_AUTO, demotions, CTLFLAG_RD, 2389 &pmap_l2e_demotions, 0, "1GB page demotions"); 2390 2391 void 2392 mmu_radix_clear_modify(vm_page_t m) 2393 { 2394 struct md_page *pvh; 2395 pmap_t pmap; 2396 pv_entry_t next_pv, pv; 2397 pml3_entry_t oldl3e, *l3e; 2398 pt_entry_t oldpte, *pte; 2399 struct rwlock *lock; 2400 vm_offset_t va; 2401 int md_gen, pvh_gen; 2402 2403 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 2404 ("pmap_clear_modify: page %p is not managed", m)); 2405 vm_page_assert_busied(m); 2406 CTR2(KTR_PMAP, "%s(%p)", __func__, m); 2407 2408 /* 2409 * If the page is not PGA_WRITEABLE, then no PTEs can have PG_M set. 2410 * If the object containing the page is locked and the page is not 2411 * exclusive busied, then PGA_WRITEABLE cannot be concurrently set. 2412 */ 2413 if ((m->a.flags & PGA_WRITEABLE) == 0) 2414 return; 2415 pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy : 2416 pa_to_pvh(VM_PAGE_TO_PHYS(m)); 2417 lock = VM_PAGE_TO_PV_LIST_LOCK(m); 2418 rw_wlock(lock); 2419 restart: 2420 TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_link, next_pv) { 2421 pmap = PV_PMAP(pv); 2422 if (!PMAP_TRYLOCK(pmap)) { 2423 pvh_gen = pvh->pv_gen; 2424 rw_wunlock(lock); 2425 PMAP_LOCK(pmap); 2426 rw_wlock(lock); 2427 if (pvh_gen != pvh->pv_gen) { 2428 PMAP_UNLOCK(pmap); 2429 goto restart; 2430 } 2431 } 2432 va = pv->pv_va; 2433 l3e = pmap_pml3e(pmap, va); 2434 oldl3e = be64toh(*l3e); 2435 if ((oldl3e & PG_RW) != 0) { 2436 if (pmap_demote_l3e_locked(pmap, l3e, va, &lock)) { 2437 if ((oldl3e & PG_W) == 0) { 2438 /* 2439 * Write protect the mapping to a 2440 * single page so that a subsequent 2441 * write access may repromote. 2442 */ 2443 va += VM_PAGE_TO_PHYS(m) - (oldl3e & 2444 PG_PS_FRAME); 2445 pte = pmap_l3e_to_pte(l3e, va); 2446 oldpte = be64toh(*pte); 2447 if ((oldpte & PG_V) != 0) { 2448 while (!atomic_cmpset_long(pte, 2449 htobe64(oldpte), 2450 htobe64((oldpte | RPTE_EAA_R) & ~(PG_M | PG_RW)))) 2451 oldpte = be64toh(*pte); 2452 vm_page_dirty(m); 2453 pmap_invalidate_page(pmap, va); 2454 } 2455 } 2456 } 2457 } 2458 PMAP_UNLOCK(pmap); 2459 } 2460 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) { 2461 pmap = PV_PMAP(pv); 2462 if (!PMAP_TRYLOCK(pmap)) { 2463 md_gen = m->md.pv_gen; 2464 pvh_gen = pvh->pv_gen; 2465 rw_wunlock(lock); 2466 PMAP_LOCK(pmap); 2467 rw_wlock(lock); 2468 if (pvh_gen != pvh->pv_gen || md_gen != m->md.pv_gen) { 2469 PMAP_UNLOCK(pmap); 2470 goto restart; 2471 } 2472 } 2473 l3e = pmap_pml3e(pmap, pv->pv_va); 2474 KASSERT((be64toh(*l3e) & RPTE_LEAF) == 0, ("pmap_clear_modify: found" 2475 " a 2mpage in page %p's pv list", m)); 2476 pte = pmap_l3e_to_pte(l3e, pv->pv_va); 2477 if ((be64toh(*pte) & (PG_M | PG_RW)) == (PG_M | PG_RW)) { 2478 atomic_clear_long(pte, htobe64(PG_M)); 2479 pmap_invalidate_page(pmap, pv->pv_va); 2480 } 2481 PMAP_UNLOCK(pmap); 2482 } 2483 rw_wunlock(lock); 2484 } 2485 2486 void 2487 mmu_radix_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, 2488 vm_size_t len, vm_offset_t src_addr) 2489 { 2490 struct rwlock *lock; 2491 struct spglist free; 2492 vm_offset_t addr; 2493 vm_offset_t end_addr = src_addr + len; 2494 vm_offset_t va_next; 2495 vm_page_t dst_pdpg, dstmpte, srcmpte; 2496 bool invalidate_all; 2497 2498 CTR6(KTR_PMAP, 2499 "%s(dst_pmap=%p, src_pmap=%p, dst_addr=%lx, len=%lu, src_addr=%lx)\n", 2500 __func__, dst_pmap, src_pmap, dst_addr, len, src_addr); 2501 2502 if (dst_addr != src_addr) 2503 return; 2504 lock = NULL; 2505 invalidate_all = false; 2506 if (dst_pmap < src_pmap) { 2507 PMAP_LOCK(dst_pmap); 2508 PMAP_LOCK(src_pmap); 2509 } else { 2510 PMAP_LOCK(src_pmap); 2511 PMAP_LOCK(dst_pmap); 2512 } 2513 2514 for (addr = src_addr; addr < end_addr; addr = va_next) { 2515 pml1_entry_t *l1e; 2516 pml2_entry_t *l2e; 2517 pml3_entry_t srcptepaddr, *l3e; 2518 pt_entry_t *src_pte, *dst_pte; 2519 2520 l1e = pmap_pml1e(src_pmap, addr); 2521 if ((be64toh(*l1e) & PG_V) == 0) { 2522 va_next = (addr + L1_PAGE_SIZE) & ~L1_PAGE_MASK; 2523 if (va_next < addr) 2524 va_next = end_addr; 2525 continue; 2526 } 2527 2528 l2e = pmap_l1e_to_l2e(l1e, addr); 2529 if ((be64toh(*l2e) & PG_V) == 0) { 2530 va_next = (addr + L2_PAGE_SIZE) & ~L2_PAGE_MASK; 2531 if (va_next < addr) 2532 va_next = end_addr; 2533 continue; 2534 } 2535 2536 va_next = (addr + L3_PAGE_SIZE) & ~L3_PAGE_MASK; 2537 if (va_next < addr) 2538 va_next = end_addr; 2539 2540 l3e = pmap_l2e_to_l3e(l2e, addr); 2541 srcptepaddr = be64toh(*l3e); 2542 if (srcptepaddr == 0) 2543 continue; 2544 2545 if (srcptepaddr & RPTE_LEAF) { 2546 if ((addr & L3_PAGE_MASK) != 0 || 2547 addr + L3_PAGE_SIZE > end_addr) 2548 continue; 2549 dst_pdpg = pmap_allocl3e(dst_pmap, addr, NULL); 2550 if (dst_pdpg == NULL) 2551 break; 2552 l3e = (pml3_entry_t *) 2553 PHYS_TO_DMAP(VM_PAGE_TO_PHYS(dst_pdpg)); 2554 l3e = &l3e[pmap_pml3e_index(addr)]; 2555 if (be64toh(*l3e) == 0 && ((srcptepaddr & PG_MANAGED) == 0 || 2556 pmap_pv_insert_l3e(dst_pmap, addr, srcptepaddr, 2557 PMAP_ENTER_NORECLAIM, &lock))) { 2558 *l3e = htobe64(srcptepaddr & ~PG_W); 2559 pmap_resident_count_inc(dst_pmap, 2560 L3_PAGE_SIZE / PAGE_SIZE); 2561 atomic_add_long(&pmap_l3e_mappings, 1); 2562 } else 2563 dst_pdpg->ref_count--; 2564 continue; 2565 } 2566 2567 srcptepaddr &= PG_FRAME; 2568 srcmpte = PHYS_TO_VM_PAGE(srcptepaddr); 2569 KASSERT(srcmpte->ref_count > 0, 2570 ("pmap_copy: source page table page is unused")); 2571 2572 if (va_next > end_addr) 2573 va_next = end_addr; 2574 2575 src_pte = (pt_entry_t *)PHYS_TO_DMAP(srcptepaddr); 2576 src_pte = &src_pte[pmap_pte_index(addr)]; 2577 dstmpte = NULL; 2578 while (addr < va_next) { 2579 pt_entry_t ptetemp; 2580 ptetemp = be64toh(*src_pte); 2581 /* 2582 * we only virtual copy managed pages 2583 */ 2584 if ((ptetemp & PG_MANAGED) != 0) { 2585 if (dstmpte != NULL && 2586 dstmpte->pindex == pmap_l3e_pindex(addr)) 2587 dstmpte->ref_count++; 2588 else if ((dstmpte = pmap_allocpte(dst_pmap, 2589 addr, NULL)) == NULL) 2590 goto out; 2591 dst_pte = (pt_entry_t *) 2592 PHYS_TO_DMAP(VM_PAGE_TO_PHYS(dstmpte)); 2593 dst_pte = &dst_pte[pmap_pte_index(addr)]; 2594 if (be64toh(*dst_pte) == 0 && 2595 pmap_try_insert_pv_entry(dst_pmap, addr, 2596 PHYS_TO_VM_PAGE(ptetemp & PG_FRAME), 2597 &lock)) { 2598 /* 2599 * Clear the wired, modified, and 2600 * accessed (referenced) bits 2601 * during the copy. 2602 */ 2603 *dst_pte = htobe64(ptetemp & ~(PG_W | PG_M | 2604 PG_A)); 2605 pmap_resident_count_inc(dst_pmap, 1); 2606 } else { 2607 SLIST_INIT(&free); 2608 if (pmap_unwire_ptp(dst_pmap, addr, 2609 dstmpte, &free)) { 2610 /* 2611 * Although "addr" is not 2612 * mapped, paging-structure 2613 * caches could nonetheless 2614 * have entries that refer to 2615 * the freed page table pages. 2616 * Invalidate those entries. 2617 */ 2618 invalidate_all = true; 2619 vm_page_free_pages_toq(&free, 2620 true); 2621 } 2622 goto out; 2623 } 2624 if (dstmpte->ref_count >= srcmpte->ref_count) 2625 break; 2626 } 2627 addr += PAGE_SIZE; 2628 if (__predict_false((addr & L3_PAGE_MASK) == 0)) 2629 src_pte = pmap_pte(src_pmap, addr); 2630 else 2631 src_pte++; 2632 } 2633 } 2634 out: 2635 if (invalidate_all) 2636 pmap_invalidate_all(dst_pmap); 2637 if (lock != NULL) 2638 rw_wunlock(lock); 2639 PMAP_UNLOCK(src_pmap); 2640 PMAP_UNLOCK(dst_pmap); 2641 } 2642 2643 static void 2644 mmu_radix_copy_page(vm_page_t msrc, vm_page_t mdst) 2645 { 2646 vm_offset_t src = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(msrc)); 2647 vm_offset_t dst = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mdst)); 2648 2649 CTR3(KTR_PMAP, "%s(%p, %p)", __func__, src, dst); 2650 /* 2651 * XXX slow 2652 */ 2653 bcopy((void *)src, (void *)dst, PAGE_SIZE); 2654 } 2655 2656 static void 2657 mmu_radix_copy_pages(vm_page_t ma[], vm_offset_t a_offset, vm_page_t mb[], 2658 vm_offset_t b_offset, int xfersize) 2659 { 2660 void *a_cp, *b_cp; 2661 vm_offset_t a_pg_offset, b_pg_offset; 2662 int cnt; 2663 2664 CTR6(KTR_PMAP, "%s(%p, %#x, %p, %#x, %#x)", __func__, ma, 2665 a_offset, mb, b_offset, xfersize); 2666 2667 while (xfersize > 0) { 2668 a_pg_offset = a_offset & PAGE_MASK; 2669 cnt = min(xfersize, PAGE_SIZE - a_pg_offset); 2670 a_cp = (char *)(uintptr_t)PHYS_TO_DMAP( 2671 VM_PAGE_TO_PHYS(ma[a_offset >> PAGE_SHIFT])) + 2672 a_pg_offset; 2673 b_pg_offset = b_offset & PAGE_MASK; 2674 cnt = min(cnt, PAGE_SIZE - b_pg_offset); 2675 b_cp = (char *)(uintptr_t)PHYS_TO_DMAP( 2676 VM_PAGE_TO_PHYS(mb[b_offset >> PAGE_SHIFT])) + 2677 b_pg_offset; 2678 bcopy(a_cp, b_cp, cnt); 2679 a_offset += cnt; 2680 b_offset += cnt; 2681 xfersize -= cnt; 2682 } 2683 } 2684 2685 #if VM_NRESERVLEVEL > 0 2686 /* 2687 * Tries to promote the 512, contiguous 4KB page mappings that are within a 2688 * single page table page (PTP) to a single 2MB page mapping. For promotion 2689 * to occur, two conditions must be met: (1) the 4KB page mappings must map 2690 * aligned, contiguous physical memory and (2) the 4KB page mappings must have 2691 * identical characteristics. 2692 */ 2693 static int 2694 pmap_promote_l3e(pmap_t pmap, pml3_entry_t *pde, vm_offset_t va, 2695 struct rwlock **lockp) 2696 { 2697 pml3_entry_t newpde; 2698 pt_entry_t *firstpte, oldpte, pa, *pte; 2699 vm_page_t mpte; 2700 2701 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 2702 2703 /* 2704 * Examine the first PTE in the specified PTP. Abort if this PTE is 2705 * either invalid, unused, or does not map the first 4KB physical page 2706 * within a 2MB page. 2707 */ 2708 firstpte = (pt_entry_t *)PHYS_TO_DMAP(be64toh(*pde) & PG_FRAME); 2709 setpde: 2710 newpde = *firstpte; 2711 if ((newpde & ((PG_FRAME & L3_PAGE_MASK) | PG_A | PG_V)) != (PG_A | PG_V)) { 2712 CTR2(KTR_PMAP, "pmap_promote_l3e: failure for va %#lx" 2713 " in pmap %p", va, pmap); 2714 goto fail; 2715 } 2716 if ((newpde & (PG_M | PG_RW)) == PG_RW) { 2717 /* 2718 * When PG_M is already clear, PG_RW can be cleared without 2719 * a TLB invalidation. 2720 */ 2721 if (!atomic_cmpset_long(firstpte, htobe64(newpde), htobe64((newpde | RPTE_EAA_R) & ~RPTE_EAA_W))) 2722 goto setpde; 2723 newpde &= ~RPTE_EAA_W; 2724 } 2725 2726 /* 2727 * Examine each of the other PTEs in the specified PTP. Abort if this 2728 * PTE maps an unexpected 4KB physical page or does not have identical 2729 * characteristics to the first PTE. 2730 */ 2731 pa = (newpde & (PG_PS_FRAME | PG_A | PG_V)) + L3_PAGE_SIZE - PAGE_SIZE; 2732 for (pte = firstpte + NPTEPG - 1; pte > firstpte; pte--) { 2733 setpte: 2734 oldpte = be64toh(*pte); 2735 if ((oldpte & (PG_FRAME | PG_A | PG_V)) != pa) { 2736 CTR2(KTR_PMAP, "pmap_promote_l3e: failure for va %#lx" 2737 " in pmap %p", va, pmap); 2738 goto fail; 2739 } 2740 if ((oldpte & (PG_M | PG_RW)) == PG_RW) { 2741 /* 2742 * When PG_M is already clear, PG_RW can be cleared 2743 * without a TLB invalidation. 2744 */ 2745 if (!atomic_cmpset_long(pte, htobe64(oldpte), htobe64((oldpte | RPTE_EAA_R) & ~RPTE_EAA_W))) 2746 goto setpte; 2747 oldpte &= ~RPTE_EAA_W; 2748 CTR2(KTR_PMAP, "pmap_promote_l3e: protect for va %#lx" 2749 " in pmap %p", (oldpte & PG_FRAME & L3_PAGE_MASK) | 2750 (va & ~L3_PAGE_MASK), pmap); 2751 } 2752 if ((oldpte & PG_PTE_PROMOTE) != (newpde & PG_PTE_PROMOTE)) { 2753 CTR2(KTR_PMAP, "pmap_promote_l3e: failure for va %#lx" 2754 " in pmap %p", va, pmap); 2755 goto fail; 2756 } 2757 pa -= PAGE_SIZE; 2758 } 2759 2760 /* 2761 * Save the page table page in its current state until the PDE 2762 * mapping the superpage is demoted by pmap_demote_pde() or 2763 * destroyed by pmap_remove_pde(). 2764 */ 2765 mpte = PHYS_TO_VM_PAGE(be64toh(*pde) & PG_FRAME); 2766 KASSERT(mpte >= vm_page_array && 2767 mpte < &vm_page_array[vm_page_array_size], 2768 ("pmap_promote_l3e: page table page is out of range")); 2769 KASSERT(mpte->pindex == pmap_l3e_pindex(va), 2770 ("pmap_promote_l3e: page table page's pindex is wrong")); 2771 if (pmap_insert_pt_page(pmap, mpte)) { 2772 CTR2(KTR_PMAP, 2773 "pmap_promote_l3e: failure for va %#lx in pmap %p", va, 2774 pmap); 2775 goto fail; 2776 } 2777 2778 /* 2779 * Promote the pv entries. 2780 */ 2781 if ((newpde & PG_MANAGED) != 0) 2782 pmap_pv_promote_l3e(pmap, va, newpde & PG_PS_FRAME, lockp); 2783 2784 pte_store(pde, PG_PROMOTED | newpde); 2785 ptesync(); 2786 atomic_add_long(&pmap_l3e_promotions, 1); 2787 CTR2(KTR_PMAP, "pmap_promote_l3e: success for va %#lx" 2788 " in pmap %p", va, pmap); 2789 return (0); 2790 fail: 2791 atomic_add_long(&pmap_l3e_p_failures, 1); 2792 return (KERN_FAILURE); 2793 } 2794 #endif /* VM_NRESERVLEVEL > 0 */ 2795 2796 int 2797 mmu_radix_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, 2798 vm_prot_t prot, u_int flags, int8_t psind) 2799 { 2800 struct rwlock *lock; 2801 pml3_entry_t *l3e; 2802 pt_entry_t *pte; 2803 pt_entry_t newpte, origpte; 2804 pv_entry_t pv; 2805 vm_paddr_t opa, pa; 2806 vm_page_t mpte, om; 2807 int rv, retrycount; 2808 boolean_t nosleep, invalidate_all, invalidate_page; 2809 2810 va = trunc_page(va); 2811 retrycount = 0; 2812 invalidate_page = invalidate_all = false; 2813 CTR6(KTR_PMAP, "pmap_enter(%p, %#lx, %p, %#x, %#x, %d)", pmap, va, 2814 m, prot, flags, psind); 2815 KASSERT(va <= VM_MAX_KERNEL_ADDRESS, ("pmap_enter: toobig")); 2816 KASSERT((m->oflags & VPO_UNMANAGED) != 0 || !VA_IS_CLEANMAP(va), 2817 ("pmap_enter: managed mapping within the clean submap")); 2818 if ((m->oflags & VPO_UNMANAGED) == 0) 2819 VM_PAGE_OBJECT_BUSY_ASSERT(m); 2820 2821 KASSERT((flags & PMAP_ENTER_RESERVED) == 0, 2822 ("pmap_enter: flags %u has reserved bits set", flags)); 2823 pa = VM_PAGE_TO_PHYS(m); 2824 newpte = (pt_entry_t)(pa | PG_A | PG_V | RPTE_LEAF); 2825 if ((flags & VM_PROT_WRITE) != 0) 2826 newpte |= PG_M; 2827 if ((flags & VM_PROT_READ) != 0) 2828 newpte |= PG_A; 2829 if (prot & VM_PROT_READ) 2830 newpte |= RPTE_EAA_R; 2831 if ((prot & VM_PROT_WRITE) != 0) 2832 newpte |= RPTE_EAA_W; 2833 KASSERT((newpte & (PG_M | PG_RW)) != PG_M, 2834 ("pmap_enter: flags includes VM_PROT_WRITE but prot doesn't")); 2835 2836 if (prot & VM_PROT_EXECUTE) 2837 newpte |= PG_X; 2838 if ((flags & PMAP_ENTER_WIRED) != 0) 2839 newpte |= PG_W; 2840 if (va >= DMAP_MIN_ADDRESS) 2841 newpte |= RPTE_EAA_P; 2842 newpte |= pmap_cache_bits(m->md.mdpg_cache_attrs); 2843 /* 2844 * Set modified bit gratuitously for writeable mappings if 2845 * the page is unmanaged. We do not want to take a fault 2846 * to do the dirty bit accounting for these mappings. 2847 */ 2848 if ((m->oflags & VPO_UNMANAGED) != 0) { 2849 if ((newpte & PG_RW) != 0) 2850 newpte |= PG_M; 2851 } else 2852 newpte |= PG_MANAGED; 2853 2854 lock = NULL; 2855 PMAP_LOCK(pmap); 2856 if (psind == 1) { 2857 /* Assert the required virtual and physical alignment. */ 2858 KASSERT((va & L3_PAGE_MASK) == 0, ("pmap_enter: va unaligned")); 2859 KASSERT(m->psind > 0, ("pmap_enter: m->psind < psind")); 2860 rv = pmap_enter_l3e(pmap, va, newpte | RPTE_LEAF, flags, m, &lock); 2861 goto out; 2862 } 2863 mpte = NULL; 2864 2865 /* 2866 * In the case that a page table page is not 2867 * resident, we are creating it here. 2868 */ 2869 retry: 2870 l3e = pmap_pml3e(pmap, va); 2871 if (l3e != NULL && (be64toh(*l3e) & PG_V) != 0 && ((be64toh(*l3e) & RPTE_LEAF) == 0 || 2872 pmap_demote_l3e_locked(pmap, l3e, va, &lock))) { 2873 pte = pmap_l3e_to_pte(l3e, va); 2874 if (va < VM_MAXUSER_ADDRESS && mpte == NULL) { 2875 mpte = PHYS_TO_VM_PAGE(be64toh(*l3e) & PG_FRAME); 2876 mpte->ref_count++; 2877 } 2878 } else if (va < VM_MAXUSER_ADDRESS) { 2879 /* 2880 * Here if the pte page isn't mapped, or if it has been 2881 * deallocated. 2882 */ 2883 nosleep = (flags & PMAP_ENTER_NOSLEEP) != 0; 2884 mpte = _pmap_allocpte(pmap, pmap_l3e_pindex(va), 2885 nosleep ? NULL : &lock); 2886 if (mpte == NULL && nosleep) { 2887 rv = KERN_RESOURCE_SHORTAGE; 2888 goto out; 2889 } 2890 if (__predict_false(retrycount++ == 6)) 2891 panic("too many retries"); 2892 invalidate_all = true; 2893 goto retry; 2894 } else 2895 panic("pmap_enter: invalid page directory va=%#lx", va); 2896 2897 origpte = be64toh(*pte); 2898 pv = NULL; 2899 2900 /* 2901 * Is the specified virtual address already mapped? 2902 */ 2903 if ((origpte & PG_V) != 0) { 2904 #ifdef INVARIANTS 2905 if (VERBOSE_PMAP || pmap_logging) { 2906 printf("cow fault pmap_enter(%p, %#lx, %p, %#x, %x, %d) --" 2907 " asid=%lu curpid=%d name=%s origpte0x%lx\n", 2908 pmap, va, m, prot, flags, psind, pmap->pm_pid, 2909 curproc->p_pid, curproc->p_comm, origpte); 2910 pmap_pte_walk(pmap->pm_pml1, va); 2911 } 2912 #endif 2913 /* 2914 * Wiring change, just update stats. We don't worry about 2915 * wiring PT pages as they remain resident as long as there 2916 * are valid mappings in them. Hence, if a user page is wired, 2917 * the PT page will be also. 2918 */ 2919 if ((newpte & PG_W) != 0 && (origpte & PG_W) == 0) 2920 pmap->pm_stats.wired_count++; 2921 else if ((newpte & PG_W) == 0 && (origpte & PG_W) != 0) 2922 pmap->pm_stats.wired_count--; 2923 2924 /* 2925 * Remove the extra PT page reference. 2926 */ 2927 if (mpte != NULL) { 2928 mpte->ref_count--; 2929 KASSERT(mpte->ref_count > 0, 2930 ("pmap_enter: missing reference to page table page," 2931 " va: 0x%lx", va)); 2932 } 2933 2934 /* 2935 * Has the physical page changed? 2936 */ 2937 opa = origpte & PG_FRAME; 2938 if (opa == pa) { 2939 /* 2940 * No, might be a protection or wiring change. 2941 */ 2942 if ((origpte & PG_MANAGED) != 0 && 2943 (newpte & PG_RW) != 0) 2944 vm_page_aflag_set(m, PGA_WRITEABLE); 2945 if (((origpte ^ newpte) & ~(PG_M | PG_A)) == 0) { 2946 if ((newpte & (PG_A|PG_M)) != (origpte & (PG_A|PG_M))) { 2947 if (!atomic_cmpset_long(pte, htobe64(origpte), htobe64(newpte))) 2948 goto retry; 2949 if ((newpte & PG_M) != (origpte & PG_M)) 2950 vm_page_dirty(m); 2951 if ((newpte & PG_A) != (origpte & PG_A)) 2952 vm_page_aflag_set(m, PGA_REFERENCED); 2953 ptesync(); 2954 } else 2955 invalidate_all = true; 2956 if (((origpte ^ newpte) & ~(PG_M | PG_A)) == 0) 2957 goto unchanged; 2958 } 2959 goto validate; 2960 } 2961 2962 /* 2963 * The physical page has changed. Temporarily invalidate 2964 * the mapping. This ensures that all threads sharing the 2965 * pmap keep a consistent view of the mapping, which is 2966 * necessary for the correct handling of COW faults. It 2967 * also permits reuse of the old mapping's PV entry, 2968 * avoiding an allocation. 2969 * 2970 * For consistency, handle unmanaged mappings the same way. 2971 */ 2972 origpte = be64toh(pte_load_clear(pte)); 2973 KASSERT((origpte & PG_FRAME) == opa, 2974 ("pmap_enter: unexpected pa update for %#lx", va)); 2975 if ((origpte & PG_MANAGED) != 0) { 2976 om = PHYS_TO_VM_PAGE(opa); 2977 2978 /* 2979 * The pmap lock is sufficient to synchronize with 2980 * concurrent calls to pmap_page_test_mappings() and 2981 * pmap_ts_referenced(). 2982 */ 2983 if ((origpte & (PG_M | PG_RW)) == (PG_M | PG_RW)) 2984 vm_page_dirty(om); 2985 if ((origpte & PG_A) != 0) 2986 vm_page_aflag_set(om, PGA_REFERENCED); 2987 CHANGE_PV_LIST_LOCK_TO_PHYS(&lock, opa); 2988 pv = pmap_pvh_remove(&om->md, pmap, va); 2989 if ((newpte & PG_MANAGED) == 0) 2990 free_pv_entry(pmap, pv); 2991 #ifdef INVARIANTS 2992 else if (origpte & PG_MANAGED) { 2993 if (pv == NULL) { 2994 pmap_page_print_mappings(om); 2995 MPASS(pv != NULL); 2996 } 2997 } 2998 #endif 2999 if ((om->a.flags & PGA_WRITEABLE) != 0 && 3000 TAILQ_EMPTY(&om->md.pv_list) && 3001 ((om->flags & PG_FICTITIOUS) != 0 || 3002 TAILQ_EMPTY(&pa_to_pvh(opa)->pv_list))) 3003 vm_page_aflag_clear(om, PGA_WRITEABLE); 3004 } 3005 if ((origpte & PG_A) != 0) 3006 invalidate_page = true; 3007 origpte = 0; 3008 } else { 3009 if (pmap != kernel_pmap) { 3010 #ifdef INVARIANTS 3011 if (VERBOSE_PMAP || pmap_logging) 3012 printf("pmap_enter(%p, %#lx, %p, %#x, %x, %d) -- asid=%lu curpid=%d name=%s\n", 3013 pmap, va, m, prot, flags, psind, 3014 pmap->pm_pid, curproc->p_pid, 3015 curproc->p_comm); 3016 #endif 3017 } 3018 3019 /* 3020 * Increment the counters. 3021 */ 3022 if ((newpte & PG_W) != 0) 3023 pmap->pm_stats.wired_count++; 3024 pmap_resident_count_inc(pmap, 1); 3025 } 3026 3027 /* 3028 * Enter on the PV list if part of our managed memory. 3029 */ 3030 if ((newpte & PG_MANAGED) != 0) { 3031 if (pv == NULL) { 3032 pv = get_pv_entry(pmap, &lock); 3033 pv->pv_va = va; 3034 } 3035 #ifdef VERBOSE_PV 3036 else 3037 printf("reassigning pv: %p to pmap: %p\n", 3038 pv, pmap); 3039 #endif 3040 CHANGE_PV_LIST_LOCK_TO_PHYS(&lock, pa); 3041 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_link); 3042 m->md.pv_gen++; 3043 if ((newpte & PG_RW) != 0) 3044 vm_page_aflag_set(m, PGA_WRITEABLE); 3045 } 3046 3047 /* 3048 * Update the PTE. 3049 */ 3050 if ((origpte & PG_V) != 0) { 3051 validate: 3052 origpte = be64toh(pte_load_store(pte, htobe64(newpte))); 3053 KASSERT((origpte & PG_FRAME) == pa, 3054 ("pmap_enter: unexpected pa update for %#lx", va)); 3055 if ((newpte & PG_M) == 0 && (origpte & (PG_M | PG_RW)) == 3056 (PG_M | PG_RW)) { 3057 if ((origpte & PG_MANAGED) != 0) 3058 vm_page_dirty(m); 3059 invalidate_page = true; 3060 3061 /* 3062 * Although the PTE may still have PG_RW set, TLB 3063 * invalidation may nonetheless be required because 3064 * the PTE no longer has PG_M set. 3065 */ 3066 } else if ((origpte & PG_X) != 0 || (newpte & PG_X) == 0) { 3067 /* 3068 * Removing capabilities requires invalidation on POWER 3069 */ 3070 invalidate_page = true; 3071 goto unchanged; 3072 } 3073 if ((origpte & PG_A) != 0) 3074 invalidate_page = true; 3075 } else { 3076 pte_store(pte, newpte); 3077 ptesync(); 3078 } 3079 unchanged: 3080 3081 #if VM_NRESERVLEVEL > 0 3082 /* 3083 * If both the page table page and the reservation are fully 3084 * populated, then attempt promotion. 3085 */ 3086 if ((mpte == NULL || mpte->ref_count == NPTEPG) && 3087 mmu_radix_ps_enabled(pmap) && 3088 (m->flags & PG_FICTITIOUS) == 0 && 3089 vm_reserv_level_iffullpop(m) == 0 && 3090 pmap_promote_l3e(pmap, l3e, va, &lock) == 0) 3091 invalidate_all = true; 3092 #endif 3093 if (invalidate_all) 3094 pmap_invalidate_all(pmap); 3095 else if (invalidate_page) 3096 pmap_invalidate_page(pmap, va); 3097 3098 rv = KERN_SUCCESS; 3099 out: 3100 if (lock != NULL) 3101 rw_wunlock(lock); 3102 PMAP_UNLOCK(pmap); 3103 3104 return (rv); 3105 } 3106 3107 /* 3108 * Tries to create a read- and/or execute-only 2MB page mapping. Returns true 3109 * if successful. Returns false if (1) a page table page cannot be allocated 3110 * without sleeping, (2) a mapping already exists at the specified virtual 3111 * address, or (3) a PV entry cannot be allocated without reclaiming another 3112 * PV entry. 3113 */ 3114 static bool 3115 pmap_enter_2mpage(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, 3116 struct rwlock **lockp) 3117 { 3118 pml3_entry_t newpde; 3119 3120 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 3121 newpde = VM_PAGE_TO_PHYS(m) | pmap_cache_bits(m->md.mdpg_cache_attrs) | 3122 RPTE_LEAF | PG_V; 3123 if ((m->oflags & VPO_UNMANAGED) == 0) 3124 newpde |= PG_MANAGED; 3125 if (prot & VM_PROT_EXECUTE) 3126 newpde |= PG_X; 3127 if (prot & VM_PROT_READ) 3128 newpde |= RPTE_EAA_R; 3129 if (va >= DMAP_MIN_ADDRESS) 3130 newpde |= RPTE_EAA_P; 3131 return (pmap_enter_l3e(pmap, va, newpde, PMAP_ENTER_NOSLEEP | 3132 PMAP_ENTER_NOREPLACE | PMAP_ENTER_NORECLAIM, NULL, lockp) == 3133 KERN_SUCCESS); 3134 } 3135 3136 /* 3137 * Tries to create the specified 2MB page mapping. Returns KERN_SUCCESS if 3138 * the mapping was created, and either KERN_FAILURE or KERN_RESOURCE_SHORTAGE 3139 * otherwise. Returns KERN_FAILURE if PMAP_ENTER_NOREPLACE was specified and 3140 * a mapping already exists at the specified virtual address. Returns 3141 * KERN_RESOURCE_SHORTAGE if PMAP_ENTER_NOSLEEP was specified and a page table 3142 * page allocation failed. Returns KERN_RESOURCE_SHORTAGE if 3143 * PMAP_ENTER_NORECLAIM was specified and a PV entry allocation failed. 3144 * 3145 * The parameter "m" is only used when creating a managed, writeable mapping. 3146 */ 3147 static int 3148 pmap_enter_l3e(pmap_t pmap, vm_offset_t va, pml3_entry_t newpde, u_int flags, 3149 vm_page_t m, struct rwlock **lockp) 3150 { 3151 struct spglist free; 3152 pml3_entry_t oldl3e, *l3e; 3153 vm_page_t mt, pdpg; 3154 3155 KASSERT((newpde & (PG_M | PG_RW)) != PG_RW, 3156 ("pmap_enter_pde: newpde is missing PG_M")); 3157 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 3158 3159 if ((pdpg = pmap_allocl3e(pmap, va, (flags & PMAP_ENTER_NOSLEEP) != 0 ? 3160 NULL : lockp)) == NULL) { 3161 CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx" 3162 " in pmap %p", va, pmap); 3163 return (KERN_RESOURCE_SHORTAGE); 3164 } 3165 l3e = (pml3_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pdpg)); 3166 l3e = &l3e[pmap_pml3e_index(va)]; 3167 oldl3e = be64toh(*l3e); 3168 if ((oldl3e & PG_V) != 0) { 3169 KASSERT(pdpg->ref_count > 1, 3170 ("pmap_enter_pde: pdpg's wire count is too low")); 3171 if ((flags & PMAP_ENTER_NOREPLACE) != 0) { 3172 pdpg->ref_count--; 3173 CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx" 3174 " in pmap %p", va, pmap); 3175 return (KERN_FAILURE); 3176 } 3177 /* Break the existing mapping(s). */ 3178 SLIST_INIT(&free); 3179 if ((oldl3e & RPTE_LEAF) != 0) { 3180 /* 3181 * The reference to the PD page that was acquired by 3182 * pmap_allocl3e() ensures that it won't be freed. 3183 * However, if the PDE resulted from a promotion, then 3184 * a reserved PT page could be freed. 3185 */ 3186 (void)pmap_remove_l3e(pmap, l3e, va, &free, lockp); 3187 pmap_invalidate_l3e_page(pmap, va, oldl3e); 3188 } else { 3189 if (pmap_remove_ptes(pmap, va, va + L3_PAGE_SIZE, l3e, 3190 &free, lockp)) 3191 pmap_invalidate_all(pmap); 3192 } 3193 vm_page_free_pages_toq(&free, true); 3194 if (va >= VM_MAXUSER_ADDRESS) { 3195 mt = PHYS_TO_VM_PAGE(be64toh(*l3e) & PG_FRAME); 3196 if (pmap_insert_pt_page(pmap, mt)) { 3197 /* 3198 * XXX Currently, this can't happen because 3199 * we do not perform pmap_enter(psind == 1) 3200 * on the kernel pmap. 3201 */ 3202 panic("pmap_enter_pde: trie insert failed"); 3203 } 3204 } else 3205 KASSERT(be64toh(*l3e) == 0, ("pmap_enter_pde: non-zero pde %p", 3206 l3e)); 3207 } 3208 if ((newpde & PG_MANAGED) != 0) { 3209 /* 3210 * Abort this mapping if its PV entry could not be created. 3211 */ 3212 if (!pmap_pv_insert_l3e(pmap, va, newpde, flags, lockp)) { 3213 SLIST_INIT(&free); 3214 if (pmap_unwire_ptp(pmap, va, pdpg, &free)) { 3215 /* 3216 * Although "va" is not mapped, paging- 3217 * structure caches could nonetheless have 3218 * entries that refer to the freed page table 3219 * pages. Invalidate those entries. 3220 */ 3221 pmap_invalidate_page(pmap, va); 3222 vm_page_free_pages_toq(&free, true); 3223 } 3224 CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx" 3225 " in pmap %p", va, pmap); 3226 return (KERN_RESOURCE_SHORTAGE); 3227 } 3228 if ((newpde & PG_RW) != 0) { 3229 for (mt = m; mt < &m[L3_PAGE_SIZE / PAGE_SIZE]; mt++) 3230 vm_page_aflag_set(mt, PGA_WRITEABLE); 3231 } 3232 } 3233 3234 /* 3235 * Increment counters. 3236 */ 3237 if ((newpde & PG_W) != 0) 3238 pmap->pm_stats.wired_count += L3_PAGE_SIZE / PAGE_SIZE; 3239 pmap_resident_count_inc(pmap, L3_PAGE_SIZE / PAGE_SIZE); 3240 3241 /* 3242 * Map the superpage. (This is not a promoted mapping; there will not 3243 * be any lingering 4KB page mappings in the TLB.) 3244 */ 3245 pte_store(l3e, newpde); 3246 ptesync(); 3247 3248 atomic_add_long(&pmap_l3e_mappings, 1); 3249 CTR2(KTR_PMAP, "pmap_enter_pde: success for va %#lx" 3250 " in pmap %p", va, pmap); 3251 return (KERN_SUCCESS); 3252 } 3253 3254 void 3255 mmu_radix_enter_object(pmap_t pmap, vm_offset_t start, 3256 vm_offset_t end, vm_page_t m_start, vm_prot_t prot) 3257 { 3258 3259 struct rwlock *lock; 3260 vm_offset_t va; 3261 vm_page_t m, mpte; 3262 vm_pindex_t diff, psize; 3263 bool invalidate; 3264 VM_OBJECT_ASSERT_LOCKED(m_start->object); 3265 3266 CTR6(KTR_PMAP, "%s(%p, %#x, %#x, %p, %#x)", __func__, pmap, start, 3267 end, m_start, prot); 3268 3269 invalidate = false; 3270 psize = atop(end - start); 3271 mpte = NULL; 3272 m = m_start; 3273 lock = NULL; 3274 PMAP_LOCK(pmap); 3275 while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) { 3276 va = start + ptoa(diff); 3277 if ((va & L3_PAGE_MASK) == 0 && va + L3_PAGE_SIZE <= end && 3278 m->psind == 1 && mmu_radix_ps_enabled(pmap) && 3279 pmap_enter_2mpage(pmap, va, m, prot, &lock)) 3280 m = &m[L3_PAGE_SIZE / PAGE_SIZE - 1]; 3281 else 3282 mpte = mmu_radix_enter_quick_locked(pmap, va, m, prot, 3283 mpte, &lock, &invalidate); 3284 m = TAILQ_NEXT(m, listq); 3285 } 3286 ptesync(); 3287 if (lock != NULL) 3288 rw_wunlock(lock); 3289 if (invalidate) 3290 pmap_invalidate_all(pmap); 3291 PMAP_UNLOCK(pmap); 3292 } 3293 3294 static vm_page_t 3295 mmu_radix_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, 3296 vm_prot_t prot, vm_page_t mpte, struct rwlock **lockp, bool *invalidate) 3297 { 3298 struct spglist free; 3299 pt_entry_t *pte; 3300 vm_paddr_t pa; 3301 3302 KASSERT(!VA_IS_CLEANMAP(va) || 3303 (m->oflags & VPO_UNMANAGED) != 0, 3304 ("mmu_radix_enter_quick_locked: managed mapping within the clean submap")); 3305 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 3306 3307 /* 3308 * In the case that a page table page is not 3309 * resident, we are creating it here. 3310 */ 3311 if (va < VM_MAXUSER_ADDRESS) { 3312 vm_pindex_t ptepindex; 3313 pml3_entry_t *ptepa; 3314 3315 /* 3316 * Calculate pagetable page index 3317 */ 3318 ptepindex = pmap_l3e_pindex(va); 3319 if (mpte && (mpte->pindex == ptepindex)) { 3320 mpte->ref_count++; 3321 } else { 3322 /* 3323 * Get the page directory entry 3324 */ 3325 ptepa = pmap_pml3e(pmap, va); 3326 3327 /* 3328 * If the page table page is mapped, we just increment 3329 * the hold count, and activate it. Otherwise, we 3330 * attempt to allocate a page table page. If this 3331 * attempt fails, we don't retry. Instead, we give up. 3332 */ 3333 if (ptepa && (be64toh(*ptepa) & PG_V) != 0) { 3334 if (be64toh(*ptepa) & RPTE_LEAF) 3335 return (NULL); 3336 mpte = PHYS_TO_VM_PAGE(be64toh(*ptepa) & PG_FRAME); 3337 mpte->ref_count++; 3338 } else { 3339 /* 3340 * Pass NULL instead of the PV list lock 3341 * pointer, because we don't intend to sleep. 3342 */ 3343 mpte = _pmap_allocpte(pmap, ptepindex, NULL); 3344 if (mpte == NULL) 3345 return (mpte); 3346 } 3347 } 3348 pte = (pt_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mpte)); 3349 pte = &pte[pmap_pte_index(va)]; 3350 } else { 3351 mpte = NULL; 3352 pte = pmap_pte(pmap, va); 3353 } 3354 if (be64toh(*pte)) { 3355 if (mpte != NULL) { 3356 mpte->ref_count--; 3357 mpte = NULL; 3358 } 3359 return (mpte); 3360 } 3361 3362 /* 3363 * Enter on the PV list if part of our managed memory. 3364 */ 3365 if ((m->oflags & VPO_UNMANAGED) == 0 && 3366 !pmap_try_insert_pv_entry(pmap, va, m, lockp)) { 3367 if (mpte != NULL) { 3368 SLIST_INIT(&free); 3369 if (pmap_unwire_ptp(pmap, va, mpte, &free)) { 3370 /* 3371 * Although "va" is not mapped, paging- 3372 * structure caches could nonetheless have 3373 * entries that refer to the freed page table 3374 * pages. Invalidate those entries. 3375 */ 3376 *invalidate = true; 3377 vm_page_free_pages_toq(&free, true); 3378 } 3379 mpte = NULL; 3380 } 3381 return (mpte); 3382 } 3383 3384 /* 3385 * Increment counters 3386 */ 3387 pmap_resident_count_inc(pmap, 1); 3388 3389 pa = VM_PAGE_TO_PHYS(m) | pmap_cache_bits(m->md.mdpg_cache_attrs); 3390 if (prot & VM_PROT_EXECUTE) 3391 pa |= PG_X; 3392 else 3393 pa |= RPTE_EAA_R; 3394 if ((m->oflags & VPO_UNMANAGED) == 0) 3395 pa |= PG_MANAGED; 3396 3397 pte_store(pte, pa); 3398 return (mpte); 3399 } 3400 3401 void 3402 mmu_radix_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, 3403 vm_prot_t prot) 3404 { 3405 struct rwlock *lock; 3406 bool invalidate; 3407 3408 lock = NULL; 3409 invalidate = false; 3410 PMAP_LOCK(pmap); 3411 mmu_radix_enter_quick_locked(pmap, va, m, prot, NULL, &lock, 3412 &invalidate); 3413 ptesync(); 3414 if (lock != NULL) 3415 rw_wunlock(lock); 3416 if (invalidate) 3417 pmap_invalidate_all(pmap); 3418 PMAP_UNLOCK(pmap); 3419 } 3420 3421 vm_paddr_t 3422 mmu_radix_extract(pmap_t pmap, vm_offset_t va) 3423 { 3424 pml3_entry_t *l3e; 3425 pt_entry_t *pte; 3426 vm_paddr_t pa; 3427 3428 l3e = pmap_pml3e(pmap, va); 3429 if (__predict_false(l3e == NULL)) 3430 return (0); 3431 if (be64toh(*l3e) & RPTE_LEAF) { 3432 pa = (be64toh(*l3e) & PG_PS_FRAME) | (va & L3_PAGE_MASK); 3433 pa |= (va & L3_PAGE_MASK); 3434 } else { 3435 /* 3436 * Beware of a concurrent promotion that changes the 3437 * PDE at this point! For example, vtopte() must not 3438 * be used to access the PTE because it would use the 3439 * new PDE. It is, however, safe to use the old PDE 3440 * because the page table page is preserved by the 3441 * promotion. 3442 */ 3443 pte = pmap_l3e_to_pte(l3e, va); 3444 if (__predict_false(pte == NULL)) 3445 return (0); 3446 pa = be64toh(*pte); 3447 pa = (pa & PG_FRAME) | (va & PAGE_MASK); 3448 pa |= (va & PAGE_MASK); 3449 } 3450 return (pa); 3451 } 3452 3453 vm_page_t 3454 mmu_radix_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot) 3455 { 3456 pml3_entry_t l3e, *l3ep; 3457 pt_entry_t pte; 3458 vm_paddr_t pa; 3459 vm_page_t m; 3460 3461 pa = 0; 3462 m = NULL; 3463 CTR4(KTR_PMAP, "%s(%p, %#x, %#x)", __func__, pmap, va, prot); 3464 PMAP_LOCK(pmap); 3465 l3ep = pmap_pml3e(pmap, va); 3466 if (l3ep != NULL && (l3e = be64toh(*l3ep))) { 3467 if (l3e & RPTE_LEAF) { 3468 if ((l3e & PG_RW) || (prot & VM_PROT_WRITE) == 0) 3469 m = PHYS_TO_VM_PAGE((l3e & PG_PS_FRAME) | 3470 (va & L3_PAGE_MASK)); 3471 } else { 3472 /* Native endian PTE, do not pass to pmap functions */ 3473 pte = be64toh(*pmap_l3e_to_pte(l3ep, va)); 3474 if ((pte & PG_V) && 3475 ((pte & PG_RW) || (prot & VM_PROT_WRITE) == 0)) 3476 m = PHYS_TO_VM_PAGE(pte & PG_FRAME); 3477 } 3478 if (m != NULL && !vm_page_wire_mapped(m)) 3479 m = NULL; 3480 } 3481 PMAP_UNLOCK(pmap); 3482 return (m); 3483 } 3484 3485 static void 3486 mmu_radix_growkernel(vm_offset_t addr) 3487 { 3488 vm_paddr_t paddr; 3489 vm_page_t nkpg; 3490 pml3_entry_t *l3e; 3491 pml2_entry_t *l2e; 3492 3493 CTR2(KTR_PMAP, "%s(%#x)", __func__, addr); 3494 if (VM_MIN_KERNEL_ADDRESS < addr && 3495 addr < (VM_MIN_KERNEL_ADDRESS + nkpt * L3_PAGE_SIZE)) 3496 return; 3497 3498 addr = roundup2(addr, L3_PAGE_SIZE); 3499 if (addr - 1 >= vm_map_max(kernel_map)) 3500 addr = vm_map_max(kernel_map); 3501 while (kernel_vm_end < addr) { 3502 l2e = pmap_pml2e(kernel_pmap, kernel_vm_end); 3503 if ((be64toh(*l2e) & PG_V) == 0) { 3504 /* We need a new PDP entry */ 3505 nkpg = vm_page_alloc(NULL, kernel_vm_end >> L2_PAGE_SIZE_SHIFT, 3506 VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ | 3507 VM_ALLOC_WIRED | VM_ALLOC_ZERO); 3508 if (nkpg == NULL) 3509 panic("pmap_growkernel: no memory to grow kernel"); 3510 if ((nkpg->flags & PG_ZERO) == 0) 3511 mmu_radix_zero_page(nkpg); 3512 paddr = VM_PAGE_TO_PHYS(nkpg); 3513 pde_store(l2e, paddr); 3514 continue; /* try again */ 3515 } 3516 l3e = pmap_l2e_to_l3e(l2e, kernel_vm_end); 3517 if ((be64toh(*l3e) & PG_V) != 0) { 3518 kernel_vm_end = (kernel_vm_end + L3_PAGE_SIZE) & ~L3_PAGE_MASK; 3519 if (kernel_vm_end - 1 >= vm_map_max(kernel_map)) { 3520 kernel_vm_end = vm_map_max(kernel_map); 3521 break; 3522 } 3523 continue; 3524 } 3525 3526 nkpg = vm_page_alloc(NULL, pmap_l3e_pindex(kernel_vm_end), 3527 VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | 3528 VM_ALLOC_ZERO); 3529 if (nkpg == NULL) 3530 panic("pmap_growkernel: no memory to grow kernel"); 3531 if ((nkpg->flags & PG_ZERO) == 0) 3532 mmu_radix_zero_page(nkpg); 3533 paddr = VM_PAGE_TO_PHYS(nkpg); 3534 pde_store(l3e, paddr); 3535 3536 kernel_vm_end = (kernel_vm_end + L3_PAGE_SIZE) & ~L3_PAGE_MASK; 3537 if (kernel_vm_end - 1 >= vm_map_max(kernel_map)) { 3538 kernel_vm_end = vm_map_max(kernel_map); 3539 break; 3540 } 3541 } 3542 ptesync(); 3543 } 3544 3545 static MALLOC_DEFINE(M_RADIX_PGD, "radix_pgd", "radix page table root directory"); 3546 static uma_zone_t zone_radix_pgd; 3547 3548 static int 3549 radix_pgd_import(void *arg __unused, void **store, int count, int domain __unused, 3550 int flags) 3551 { 3552 3553 for (int i = 0; i < count; i++) { 3554 vm_page_t m = vm_page_alloc_contig(NULL, 0, 3555 VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | 3556 VM_ALLOC_ZERO | VM_ALLOC_WAITOK, RADIX_PGD_SIZE/PAGE_SIZE, 3557 0, (vm_paddr_t)-1, RADIX_PGD_SIZE, L1_PAGE_SIZE, 3558 VM_MEMATTR_DEFAULT); 3559 /* XXX zero on alloc here so we don't have to later */ 3560 store[i] = (void *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)); 3561 } 3562 return (count); 3563 } 3564 3565 static void 3566 radix_pgd_release(void *arg __unused, void **store, int count) 3567 { 3568 vm_page_t m; 3569 struct spglist free; 3570 int page_count; 3571 3572 SLIST_INIT(&free); 3573 page_count = RADIX_PGD_SIZE/PAGE_SIZE; 3574 3575 for (int i = 0; i < count; i++) { 3576 /* 3577 * XXX selectively remove dmap and KVA entries so we don't 3578 * need to bzero 3579 */ 3580 m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)store[i])); 3581 for (int j = page_count-1; j >= 0; j--) { 3582 vm_page_unwire_noq(&m[j]); 3583 SLIST_INSERT_HEAD(&free, &m[j], plinks.s.ss); 3584 } 3585 vm_page_free_pages_toq(&free, false); 3586 } 3587 } 3588 3589 static void 3590 mmu_radix_init() 3591 { 3592 vm_page_t mpte; 3593 vm_size_t s; 3594 int error, i, pv_npg; 3595 3596 /* XXX is this really needed for POWER? */ 3597 /* L1TF, reserve page @0 unconditionally */ 3598 vm_page_blacklist_add(0, bootverbose); 3599 3600 zone_radix_pgd = uma_zcache_create("radix_pgd_cache", 3601 RADIX_PGD_SIZE, NULL, NULL, 3602 #ifdef INVARIANTS 3603 trash_init, trash_fini, 3604 #else 3605 NULL, NULL, 3606 #endif 3607 radix_pgd_import, radix_pgd_release, 3608 NULL, UMA_ZONE_NOBUCKET); 3609 3610 /* 3611 * Initialize the vm page array entries for the kernel pmap's 3612 * page table pages. 3613 */ 3614 PMAP_LOCK(kernel_pmap); 3615 for (i = 0; i < nkpt; i++) { 3616 mpte = PHYS_TO_VM_PAGE(KPTphys + (i << PAGE_SHIFT)); 3617 KASSERT(mpte >= vm_page_array && 3618 mpte < &vm_page_array[vm_page_array_size], 3619 ("pmap_init: page table page is out of range size: %lu", 3620 vm_page_array_size)); 3621 mpte->pindex = pmap_l3e_pindex(VM_MIN_KERNEL_ADDRESS) + i; 3622 mpte->phys_addr = KPTphys + (i << PAGE_SHIFT); 3623 MPASS(PHYS_TO_VM_PAGE(mpte->phys_addr) == mpte); 3624 //pmap_insert_pt_page(kernel_pmap, mpte); 3625 mpte->ref_count = 1; 3626 } 3627 PMAP_UNLOCK(kernel_pmap); 3628 vm_wire_add(nkpt); 3629 3630 CTR1(KTR_PMAP, "%s()", __func__); 3631 TAILQ_INIT(&pv_dummy.pv_list); 3632 3633 /* 3634 * Are large page mappings enabled? 3635 */ 3636 TUNABLE_INT_FETCH("vm.pmap.superpages_enabled", &superpages_enabled); 3637 if (superpages_enabled) { 3638 KASSERT(MAXPAGESIZES > 1 && pagesizes[1] == 0, 3639 ("pmap_init: can't assign to pagesizes[1]")); 3640 pagesizes[1] = L3_PAGE_SIZE; 3641 } 3642 3643 /* 3644 * Initialize the pv chunk list mutex. 3645 */ 3646 mtx_init(&pv_chunks_mutex, "pmap pv chunk list", NULL, MTX_DEF); 3647 3648 /* 3649 * Initialize the pool of pv list locks. 3650 */ 3651 for (i = 0; i < NPV_LIST_LOCKS; i++) 3652 rw_init(&pv_list_locks[i], "pmap pv list"); 3653 3654 /* 3655 * Calculate the size of the pv head table for superpages. 3656 */ 3657 pv_npg = howmany(vm_phys_segs[vm_phys_nsegs - 1].end, L3_PAGE_SIZE); 3658 3659 /* 3660 * Allocate memory for the pv head table for superpages. 3661 */ 3662 s = (vm_size_t)(pv_npg * sizeof(struct md_page)); 3663 s = round_page(s); 3664 pv_table = (struct md_page *)kmem_malloc(s, M_WAITOK | M_ZERO); 3665 for (i = 0; i < pv_npg; i++) 3666 TAILQ_INIT(&pv_table[i].pv_list); 3667 TAILQ_INIT(&pv_dummy.pv_list); 3668 3669 pmap_initialized = 1; 3670 mtx_init(&qframe_mtx, "qfrmlk", NULL, MTX_SPIN); 3671 error = vmem_alloc(kernel_arena, PAGE_SIZE, M_BESTFIT | M_WAITOK, 3672 (vmem_addr_t *)&qframe); 3673 3674 if (error != 0) 3675 panic("qframe allocation failed"); 3676 asid_arena = vmem_create("ASID", isa3_base_pid + 1, (1<<isa3_pid_bits), 3677 1, 1, M_WAITOK); 3678 } 3679 3680 static boolean_t 3681 pmap_page_test_mappings(vm_page_t m, boolean_t accessed, boolean_t modified) 3682 { 3683 struct rwlock *lock; 3684 pv_entry_t pv; 3685 struct md_page *pvh; 3686 pt_entry_t *pte, mask; 3687 pmap_t pmap; 3688 int md_gen, pvh_gen; 3689 boolean_t rv; 3690 3691 rv = FALSE; 3692 lock = VM_PAGE_TO_PV_LIST_LOCK(m); 3693 rw_rlock(lock); 3694 restart: 3695 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) { 3696 pmap = PV_PMAP(pv); 3697 if (!PMAP_TRYLOCK(pmap)) { 3698 md_gen = m->md.pv_gen; 3699 rw_runlock(lock); 3700 PMAP_LOCK(pmap); 3701 rw_rlock(lock); 3702 if (md_gen != m->md.pv_gen) { 3703 PMAP_UNLOCK(pmap); 3704 goto restart; 3705 } 3706 } 3707 pte = pmap_pte(pmap, pv->pv_va); 3708 mask = 0; 3709 if (modified) 3710 mask |= PG_RW | PG_M; 3711 if (accessed) 3712 mask |= PG_V | PG_A; 3713 rv = (be64toh(*pte) & mask) == mask; 3714 PMAP_UNLOCK(pmap); 3715 if (rv) 3716 goto out; 3717 } 3718 if ((m->flags & PG_FICTITIOUS) == 0) { 3719 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); 3720 TAILQ_FOREACH(pv, &pvh->pv_list, pv_link) { 3721 pmap = PV_PMAP(pv); 3722 if (!PMAP_TRYLOCK(pmap)) { 3723 md_gen = m->md.pv_gen; 3724 pvh_gen = pvh->pv_gen; 3725 rw_runlock(lock); 3726 PMAP_LOCK(pmap); 3727 rw_rlock(lock); 3728 if (md_gen != m->md.pv_gen || 3729 pvh_gen != pvh->pv_gen) { 3730 PMAP_UNLOCK(pmap); 3731 goto restart; 3732 } 3733 } 3734 pte = pmap_pml3e(pmap, pv->pv_va); 3735 mask = 0; 3736 if (modified) 3737 mask |= PG_RW | PG_M; 3738 if (accessed) 3739 mask |= PG_V | PG_A; 3740 rv = (be64toh(*pte) & mask) == mask; 3741 PMAP_UNLOCK(pmap); 3742 if (rv) 3743 goto out; 3744 } 3745 } 3746 out: 3747 rw_runlock(lock); 3748 return (rv); 3749 } 3750 3751 /* 3752 * pmap_is_modified: 3753 * 3754 * Return whether or not the specified physical page was modified 3755 * in any physical maps. 3756 */ 3757 boolean_t 3758 mmu_radix_is_modified(vm_page_t m) 3759 { 3760 3761 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 3762 ("pmap_is_modified: page %p is not managed", m)); 3763 3764 CTR2(KTR_PMAP, "%s(%p)", __func__, m); 3765 /* 3766 * If the page is not busied then this check is racy. 3767 */ 3768 if (!pmap_page_is_write_mapped(m)) 3769 return (FALSE); 3770 return (pmap_page_test_mappings(m, FALSE, TRUE)); 3771 } 3772 3773 boolean_t 3774 mmu_radix_is_prefaultable(pmap_t pmap, vm_offset_t addr) 3775 { 3776 pml3_entry_t *l3e; 3777 pt_entry_t *pte; 3778 boolean_t rv; 3779 3780 CTR3(KTR_PMAP, "%s(%p, %#x)", __func__, pmap, addr); 3781 rv = FALSE; 3782 PMAP_LOCK(pmap); 3783 l3e = pmap_pml3e(pmap, addr); 3784 if (l3e != NULL && (be64toh(*l3e) & (RPTE_LEAF | PG_V)) == PG_V) { 3785 pte = pmap_l3e_to_pte(l3e, addr); 3786 rv = (be64toh(*pte) & PG_V) == 0; 3787 } 3788 PMAP_UNLOCK(pmap); 3789 return (rv); 3790 } 3791 3792 boolean_t 3793 mmu_radix_is_referenced(vm_page_t m) 3794 { 3795 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 3796 ("pmap_is_referenced: page %p is not managed", m)); 3797 CTR2(KTR_PMAP, "%s(%p)", __func__, m); 3798 return (pmap_page_test_mappings(m, TRUE, FALSE)); 3799 } 3800 3801 /* 3802 * pmap_ts_referenced: 3803 * 3804 * Return a count of reference bits for a page, clearing those bits. 3805 * It is not necessary for every reference bit to be cleared, but it 3806 * is necessary that 0 only be returned when there are truly no 3807 * reference bits set. 3808 * 3809 * As an optimization, update the page's dirty field if a modified bit is 3810 * found while counting reference bits. This opportunistic update can be 3811 * performed at low cost and can eliminate the need for some future calls 3812 * to pmap_is_modified(). However, since this function stops after 3813 * finding PMAP_TS_REFERENCED_MAX reference bits, it may not detect some 3814 * dirty pages. Those dirty pages will only be detected by a future call 3815 * to pmap_is_modified(). 3816 * 3817 * A DI block is not needed within this function, because 3818 * invalidations are performed before the PV list lock is 3819 * released. 3820 */ 3821 boolean_t 3822 mmu_radix_ts_referenced(vm_page_t m) 3823 { 3824 struct md_page *pvh; 3825 pv_entry_t pv, pvf; 3826 pmap_t pmap; 3827 struct rwlock *lock; 3828 pml3_entry_t oldl3e, *l3e; 3829 pt_entry_t *pte; 3830 vm_paddr_t pa; 3831 int cleared, md_gen, not_cleared, pvh_gen; 3832 struct spglist free; 3833 3834 CTR2(KTR_PMAP, "%s(%p)", __func__, m); 3835 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 3836 ("pmap_ts_referenced: page %p is not managed", m)); 3837 SLIST_INIT(&free); 3838 cleared = 0; 3839 pa = VM_PAGE_TO_PHYS(m); 3840 lock = PHYS_TO_PV_LIST_LOCK(pa); 3841 pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy : pa_to_pvh(pa); 3842 rw_wlock(lock); 3843 retry: 3844 not_cleared = 0; 3845 if ((pvf = TAILQ_FIRST(&pvh->pv_list)) == NULL) 3846 goto small_mappings; 3847 pv = pvf; 3848 do { 3849 if (pvf == NULL) 3850 pvf = pv; 3851 pmap = PV_PMAP(pv); 3852 if (!PMAP_TRYLOCK(pmap)) { 3853 pvh_gen = pvh->pv_gen; 3854 rw_wunlock(lock); 3855 PMAP_LOCK(pmap); 3856 rw_wlock(lock); 3857 if (pvh_gen != pvh->pv_gen) { 3858 PMAP_UNLOCK(pmap); 3859 goto retry; 3860 } 3861 } 3862 l3e = pmap_pml3e(pmap, pv->pv_va); 3863 oldl3e = be64toh(*l3e); 3864 if ((oldl3e & (PG_M | PG_RW)) == (PG_M | PG_RW)) { 3865 /* 3866 * Although "oldpde" is mapping a 2MB page, because 3867 * this function is called at a 4KB page granularity, 3868 * we only update the 4KB page under test. 3869 */ 3870 vm_page_dirty(m); 3871 } 3872 if ((oldl3e & PG_A) != 0) { 3873 /* 3874 * Since this reference bit is shared by 512 4KB 3875 * pages, it should not be cleared every time it is 3876 * tested. Apply a simple "hash" function on the 3877 * physical page number, the virtual superpage number, 3878 * and the pmap address to select one 4KB page out of 3879 * the 512 on which testing the reference bit will 3880 * result in clearing that reference bit. This 3881 * function is designed to avoid the selection of the 3882 * same 4KB page for every 2MB page mapping. 3883 * 3884 * On demotion, a mapping that hasn't been referenced 3885 * is simply destroyed. To avoid the possibility of a 3886 * subsequent page fault on a demoted wired mapping, 3887 * always leave its reference bit set. Moreover, 3888 * since the superpage is wired, the current state of 3889 * its reference bit won't affect page replacement. 3890 */ 3891 if ((((pa >> PAGE_SHIFT) ^ (pv->pv_va >> L3_PAGE_SIZE_SHIFT) ^ 3892 (uintptr_t)pmap) & (NPTEPG - 1)) == 0 && 3893 (oldl3e & PG_W) == 0) { 3894 atomic_clear_long(l3e, htobe64(PG_A)); 3895 pmap_invalidate_page(pmap, pv->pv_va); 3896 cleared++; 3897 KASSERT(lock == VM_PAGE_TO_PV_LIST_LOCK(m), 3898 ("inconsistent pv lock %p %p for page %p", 3899 lock, VM_PAGE_TO_PV_LIST_LOCK(m), m)); 3900 } else 3901 not_cleared++; 3902 } 3903 PMAP_UNLOCK(pmap); 3904 /* Rotate the PV list if it has more than one entry. */ 3905 if (pv != NULL && TAILQ_NEXT(pv, pv_link) != NULL) { 3906 TAILQ_REMOVE(&pvh->pv_list, pv, pv_link); 3907 TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_link); 3908 pvh->pv_gen++; 3909 } 3910 if (cleared + not_cleared >= PMAP_TS_REFERENCED_MAX) 3911 goto out; 3912 } while ((pv = TAILQ_FIRST(&pvh->pv_list)) != pvf); 3913 small_mappings: 3914 if ((pvf = TAILQ_FIRST(&m->md.pv_list)) == NULL) 3915 goto out; 3916 pv = pvf; 3917 do { 3918 if (pvf == NULL) 3919 pvf = pv; 3920 pmap = PV_PMAP(pv); 3921 if (!PMAP_TRYLOCK(pmap)) { 3922 pvh_gen = pvh->pv_gen; 3923 md_gen = m->md.pv_gen; 3924 rw_wunlock(lock); 3925 PMAP_LOCK(pmap); 3926 rw_wlock(lock); 3927 if (pvh_gen != pvh->pv_gen || md_gen != m->md.pv_gen) { 3928 PMAP_UNLOCK(pmap); 3929 goto retry; 3930 } 3931 } 3932 l3e = pmap_pml3e(pmap, pv->pv_va); 3933 KASSERT((be64toh(*l3e) & RPTE_LEAF) == 0, 3934 ("pmap_ts_referenced: found a 2mpage in page %p's pv list", 3935 m)); 3936 pte = pmap_l3e_to_pte(l3e, pv->pv_va); 3937 if ((be64toh(*pte) & (PG_M | PG_RW)) == (PG_M | PG_RW)) 3938 vm_page_dirty(m); 3939 if ((be64toh(*pte) & PG_A) != 0) { 3940 atomic_clear_long(pte, htobe64(PG_A)); 3941 pmap_invalidate_page(pmap, pv->pv_va); 3942 cleared++; 3943 } 3944 PMAP_UNLOCK(pmap); 3945 /* Rotate the PV list if it has more than one entry. */ 3946 if (pv != NULL && TAILQ_NEXT(pv, pv_link) != NULL) { 3947 TAILQ_REMOVE(&m->md.pv_list, pv, pv_link); 3948 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_link); 3949 m->md.pv_gen++; 3950 } 3951 } while ((pv = TAILQ_FIRST(&m->md.pv_list)) != pvf && cleared + 3952 not_cleared < PMAP_TS_REFERENCED_MAX); 3953 out: 3954 rw_wunlock(lock); 3955 vm_page_free_pages_toq(&free, true); 3956 return (cleared + not_cleared); 3957 } 3958 3959 static vm_offset_t 3960 mmu_radix_map(vm_offset_t *virt __unused, vm_paddr_t start, 3961 vm_paddr_t end, int prot __unused) 3962 { 3963 3964 CTR5(KTR_PMAP, "%s(%p, %#x, %#x, %#x)", __func__, virt, start, end, 3965 prot); 3966 return (PHYS_TO_DMAP(start)); 3967 } 3968 3969 void 3970 mmu_radix_object_init_pt(pmap_t pmap, vm_offset_t addr, 3971 vm_object_t object, vm_pindex_t pindex, vm_size_t size) 3972 { 3973 pml3_entry_t *l3e; 3974 vm_paddr_t pa, ptepa; 3975 vm_page_t p, pdpg; 3976 vm_memattr_t ma; 3977 3978 CTR6(KTR_PMAP, "%s(%p, %#x, %p, %u, %#x)", __func__, pmap, addr, 3979 object, pindex, size); 3980 VM_OBJECT_ASSERT_WLOCKED(object); 3981 KASSERT(object->type == OBJT_DEVICE || object->type == OBJT_SG, 3982 ("pmap_object_init_pt: non-device object")); 3983 /* NB: size can be logically ored with addr here */ 3984 if ((addr & L3_PAGE_MASK) == 0 && (size & L3_PAGE_MASK) == 0) { 3985 if (!mmu_radix_ps_enabled(pmap)) 3986 return; 3987 if (!vm_object_populate(object, pindex, pindex + atop(size))) 3988 return; 3989 p = vm_page_lookup(object, pindex); 3990 KASSERT(p->valid == VM_PAGE_BITS_ALL, 3991 ("pmap_object_init_pt: invalid page %p", p)); 3992 ma = p->md.mdpg_cache_attrs; 3993 3994 /* 3995 * Abort the mapping if the first page is not physically 3996 * aligned to a 2MB page boundary. 3997 */ 3998 ptepa = VM_PAGE_TO_PHYS(p); 3999 if (ptepa & L3_PAGE_MASK) 4000 return; 4001 4002 /* 4003 * Skip the first page. Abort the mapping if the rest of 4004 * the pages are not physically contiguous or have differing 4005 * memory attributes. 4006 */ 4007 p = TAILQ_NEXT(p, listq); 4008 for (pa = ptepa + PAGE_SIZE; pa < ptepa + size; 4009 pa += PAGE_SIZE) { 4010 KASSERT(p->valid == VM_PAGE_BITS_ALL, 4011 ("pmap_object_init_pt: invalid page %p", p)); 4012 if (pa != VM_PAGE_TO_PHYS(p) || 4013 ma != p->md.mdpg_cache_attrs) 4014 return; 4015 p = TAILQ_NEXT(p, listq); 4016 } 4017 4018 PMAP_LOCK(pmap); 4019 for (pa = ptepa | pmap_cache_bits(ma); 4020 pa < ptepa + size; pa += L3_PAGE_SIZE) { 4021 pdpg = pmap_allocl3e(pmap, addr, NULL); 4022 if (pdpg == NULL) { 4023 /* 4024 * The creation of mappings below is only an 4025 * optimization. If a page directory page 4026 * cannot be allocated without blocking, 4027 * continue on to the next mapping rather than 4028 * blocking. 4029 */ 4030 addr += L3_PAGE_SIZE; 4031 continue; 4032 } 4033 l3e = (pml3_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pdpg)); 4034 l3e = &l3e[pmap_pml3e_index(addr)]; 4035 if ((be64toh(*l3e) & PG_V) == 0) { 4036 pa |= PG_M | PG_A | PG_RW; 4037 pte_store(l3e, pa); 4038 pmap_resident_count_inc(pmap, L3_PAGE_SIZE / PAGE_SIZE); 4039 atomic_add_long(&pmap_l3e_mappings, 1); 4040 } else { 4041 /* Continue on if the PDE is already valid. */ 4042 pdpg->ref_count--; 4043 KASSERT(pdpg->ref_count > 0, 4044 ("pmap_object_init_pt: missing reference " 4045 "to page directory page, va: 0x%lx", addr)); 4046 } 4047 addr += L3_PAGE_SIZE; 4048 } 4049 ptesync(); 4050 PMAP_UNLOCK(pmap); 4051 } 4052 } 4053 4054 boolean_t 4055 mmu_radix_page_exists_quick(pmap_t pmap, vm_page_t m) 4056 { 4057 struct md_page *pvh; 4058 struct rwlock *lock; 4059 pv_entry_t pv; 4060 int loops = 0; 4061 boolean_t rv; 4062 4063 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 4064 ("pmap_page_exists_quick: page %p is not managed", m)); 4065 CTR3(KTR_PMAP, "%s(%p, %p)", __func__, pmap, m); 4066 rv = FALSE; 4067 lock = VM_PAGE_TO_PV_LIST_LOCK(m); 4068 rw_rlock(lock); 4069 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) { 4070 if (PV_PMAP(pv) == pmap) { 4071 rv = TRUE; 4072 break; 4073 } 4074 loops++; 4075 if (loops >= 16) 4076 break; 4077 } 4078 if (!rv && loops < 16 && (m->flags & PG_FICTITIOUS) == 0) { 4079 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); 4080 TAILQ_FOREACH(pv, &pvh->pv_list, pv_link) { 4081 if (PV_PMAP(pv) == pmap) { 4082 rv = TRUE; 4083 break; 4084 } 4085 loops++; 4086 if (loops >= 16) 4087 break; 4088 } 4089 } 4090 rw_runlock(lock); 4091 return (rv); 4092 } 4093 4094 void 4095 mmu_radix_page_init(vm_page_t m) 4096 { 4097 4098 CTR2(KTR_PMAP, "%s(%p)", __func__, m); 4099 TAILQ_INIT(&m->md.pv_list); 4100 m->md.mdpg_cache_attrs = VM_MEMATTR_DEFAULT; 4101 } 4102 4103 int 4104 mmu_radix_page_wired_mappings(vm_page_t m) 4105 { 4106 struct rwlock *lock; 4107 struct md_page *pvh; 4108 pmap_t pmap; 4109 pt_entry_t *pte; 4110 pv_entry_t pv; 4111 int count, md_gen, pvh_gen; 4112 4113 if ((m->oflags & VPO_UNMANAGED) != 0) 4114 return (0); 4115 CTR2(KTR_PMAP, "%s(%p)", __func__, m); 4116 lock = VM_PAGE_TO_PV_LIST_LOCK(m); 4117 rw_rlock(lock); 4118 restart: 4119 count = 0; 4120 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) { 4121 pmap = PV_PMAP(pv); 4122 if (!PMAP_TRYLOCK(pmap)) { 4123 md_gen = m->md.pv_gen; 4124 rw_runlock(lock); 4125 PMAP_LOCK(pmap); 4126 rw_rlock(lock); 4127 if (md_gen != m->md.pv_gen) { 4128 PMAP_UNLOCK(pmap); 4129 goto restart; 4130 } 4131 } 4132 pte = pmap_pte(pmap, pv->pv_va); 4133 if ((be64toh(*pte) & PG_W) != 0) 4134 count++; 4135 PMAP_UNLOCK(pmap); 4136 } 4137 if ((m->flags & PG_FICTITIOUS) == 0) { 4138 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); 4139 TAILQ_FOREACH(pv, &pvh->pv_list, pv_link) { 4140 pmap = PV_PMAP(pv); 4141 if (!PMAP_TRYLOCK(pmap)) { 4142 md_gen = m->md.pv_gen; 4143 pvh_gen = pvh->pv_gen; 4144 rw_runlock(lock); 4145 PMAP_LOCK(pmap); 4146 rw_rlock(lock); 4147 if (md_gen != m->md.pv_gen || 4148 pvh_gen != pvh->pv_gen) { 4149 PMAP_UNLOCK(pmap); 4150 goto restart; 4151 } 4152 } 4153 pte = pmap_pml3e(pmap, pv->pv_va); 4154 if ((be64toh(*pte) & PG_W) != 0) 4155 count++; 4156 PMAP_UNLOCK(pmap); 4157 } 4158 } 4159 rw_runlock(lock); 4160 return (count); 4161 } 4162 4163 static void 4164 mmu_radix_update_proctab(int pid, pml1_entry_t l1pa) 4165 { 4166 isa3_proctab[pid].proctab0 = htobe64(RTS_SIZE | l1pa | RADIX_PGD_INDEX_SHIFT); 4167 } 4168 4169 int 4170 mmu_radix_pinit(pmap_t pmap) 4171 { 4172 vmem_addr_t pid; 4173 vm_paddr_t l1pa; 4174 4175 CTR2(KTR_PMAP, "%s(%p)", __func__, pmap); 4176 4177 /* 4178 * allocate the page directory page 4179 */ 4180 pmap->pm_pml1 = uma_zalloc(zone_radix_pgd, M_WAITOK); 4181 4182 for (int j = 0; j < RADIX_PGD_SIZE_SHIFT; j++) 4183 pagezero((vm_offset_t)pmap->pm_pml1 + j * PAGE_SIZE); 4184 pmap->pm_radix.rt_root = 0; 4185 TAILQ_INIT(&pmap->pm_pvchunk); 4186 bzero(&pmap->pm_stats, sizeof pmap->pm_stats); 4187 pmap->pm_flags = PMAP_PDE_SUPERPAGE; 4188 vmem_alloc(asid_arena, 1, M_FIRSTFIT|M_WAITOK, &pid); 4189 4190 pmap->pm_pid = pid; 4191 l1pa = DMAP_TO_PHYS((vm_offset_t)pmap->pm_pml1); 4192 mmu_radix_update_proctab(pid, l1pa); 4193 __asm __volatile("ptesync;isync" : : : "memory"); 4194 4195 return (1); 4196 } 4197 4198 /* 4199 * This routine is called if the desired page table page does not exist. 4200 * 4201 * If page table page allocation fails, this routine may sleep before 4202 * returning NULL. It sleeps only if a lock pointer was given. 4203 * 4204 * Note: If a page allocation fails at page table level two or three, 4205 * one or two pages may be held during the wait, only to be released 4206 * afterwards. This conservative approach is easily argued to avoid 4207 * race conditions. 4208 */ 4209 static vm_page_t 4210 _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, struct rwlock **lockp) 4211 { 4212 vm_page_t m, pdppg, pdpg; 4213 4214 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 4215 4216 /* 4217 * Allocate a page table page. 4218 */ 4219 if ((m = vm_page_alloc(NULL, ptepindex, VM_ALLOC_NOOBJ | 4220 VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) { 4221 if (lockp != NULL) { 4222 RELEASE_PV_LIST_LOCK(lockp); 4223 PMAP_UNLOCK(pmap); 4224 vm_wait(NULL); 4225 PMAP_LOCK(pmap); 4226 } 4227 /* 4228 * Indicate the need to retry. While waiting, the page table 4229 * page may have been allocated. 4230 */ 4231 return (NULL); 4232 } 4233 if ((m->flags & PG_ZERO) == 0) 4234 mmu_radix_zero_page(m); 4235 4236 /* 4237 * Map the pagetable page into the process address space, if 4238 * it isn't already there. 4239 */ 4240 4241 if (ptepindex >= (NUPDE + NUPDPE)) { 4242 pml1_entry_t *l1e; 4243 vm_pindex_t pml1index; 4244 4245 /* Wire up a new PDPE page */ 4246 pml1index = ptepindex - (NUPDE + NUPDPE); 4247 l1e = &pmap->pm_pml1[pml1index]; 4248 pde_store(l1e, VM_PAGE_TO_PHYS(m)); 4249 4250 } else if (ptepindex >= NUPDE) { 4251 vm_pindex_t pml1index; 4252 vm_pindex_t pdpindex; 4253 pml1_entry_t *l1e; 4254 pml2_entry_t *l2e; 4255 4256 /* Wire up a new l2e page */ 4257 pdpindex = ptepindex - NUPDE; 4258 pml1index = pdpindex >> RPTE_SHIFT; 4259 4260 l1e = &pmap->pm_pml1[pml1index]; 4261 if ((be64toh(*l1e) & PG_V) == 0) { 4262 /* Have to allocate a new pdp, recurse */ 4263 if (_pmap_allocpte(pmap, NUPDE + NUPDPE + pml1index, 4264 lockp) == NULL) { 4265 vm_page_unwire_noq(m); 4266 vm_page_free_zero(m); 4267 return (NULL); 4268 } 4269 } else { 4270 /* Add reference to l2e page */ 4271 pdppg = PHYS_TO_VM_PAGE(be64toh(*l1e) & PG_FRAME); 4272 pdppg->ref_count++; 4273 } 4274 l2e = (pml2_entry_t *)PHYS_TO_DMAP(be64toh(*l1e) & PG_FRAME); 4275 4276 /* Now find the pdp page */ 4277 l2e = &l2e[pdpindex & RPTE_MASK]; 4278 pde_store(l2e, VM_PAGE_TO_PHYS(m)); 4279 4280 } else { 4281 vm_pindex_t pml1index; 4282 vm_pindex_t pdpindex; 4283 pml1_entry_t *l1e; 4284 pml2_entry_t *l2e; 4285 pml3_entry_t *l3e; 4286 4287 /* Wire up a new PTE page */ 4288 pdpindex = ptepindex >> RPTE_SHIFT; 4289 pml1index = pdpindex >> RPTE_SHIFT; 4290 4291 /* First, find the pdp and check that its valid. */ 4292 l1e = &pmap->pm_pml1[pml1index]; 4293 if ((be64toh(*l1e) & PG_V) == 0) { 4294 /* Have to allocate a new pd, recurse */ 4295 if (_pmap_allocpte(pmap, NUPDE + pdpindex, 4296 lockp) == NULL) { 4297 vm_page_unwire_noq(m); 4298 vm_page_free_zero(m); 4299 return (NULL); 4300 } 4301 l2e = (pml2_entry_t *)PHYS_TO_DMAP(be64toh(*l1e) & PG_FRAME); 4302 l2e = &l2e[pdpindex & RPTE_MASK]; 4303 } else { 4304 l2e = (pml2_entry_t *)PHYS_TO_DMAP(be64toh(*l1e) & PG_FRAME); 4305 l2e = &l2e[pdpindex & RPTE_MASK]; 4306 if ((be64toh(*l2e) & PG_V) == 0) { 4307 /* Have to allocate a new pd, recurse */ 4308 if (_pmap_allocpte(pmap, NUPDE + pdpindex, 4309 lockp) == NULL) { 4310 vm_page_unwire_noq(m); 4311 vm_page_free_zero(m); 4312 return (NULL); 4313 } 4314 } else { 4315 /* Add reference to the pd page */ 4316 pdpg = PHYS_TO_VM_PAGE(be64toh(*l2e) & PG_FRAME); 4317 pdpg->ref_count++; 4318 } 4319 } 4320 l3e = (pml3_entry_t *)PHYS_TO_DMAP(be64toh(*l2e) & PG_FRAME); 4321 4322 /* Now we know where the page directory page is */ 4323 l3e = &l3e[ptepindex & RPTE_MASK]; 4324 pde_store(l3e, VM_PAGE_TO_PHYS(m)); 4325 } 4326 4327 pmap_resident_count_inc(pmap, 1); 4328 return (m); 4329 } 4330 static vm_page_t 4331 pmap_allocl3e(pmap_t pmap, vm_offset_t va, struct rwlock **lockp) 4332 { 4333 vm_pindex_t pdpindex, ptepindex; 4334 pml2_entry_t *pdpe; 4335 vm_page_t pdpg; 4336 4337 retry: 4338 pdpe = pmap_pml2e(pmap, va); 4339 if (pdpe != NULL && (be64toh(*pdpe) & PG_V) != 0) { 4340 /* Add a reference to the pd page. */ 4341 pdpg = PHYS_TO_VM_PAGE(be64toh(*pdpe) & PG_FRAME); 4342 pdpg->ref_count++; 4343 } else { 4344 /* Allocate a pd page. */ 4345 ptepindex = pmap_l3e_pindex(va); 4346 pdpindex = ptepindex >> RPTE_SHIFT; 4347 pdpg = _pmap_allocpte(pmap, NUPDE + pdpindex, lockp); 4348 if (pdpg == NULL && lockp != NULL) 4349 goto retry; 4350 } 4351 return (pdpg); 4352 } 4353 4354 static vm_page_t 4355 pmap_allocpte(pmap_t pmap, vm_offset_t va, struct rwlock **lockp) 4356 { 4357 vm_pindex_t ptepindex; 4358 pml3_entry_t *pd; 4359 vm_page_t m; 4360 4361 /* 4362 * Calculate pagetable page index 4363 */ 4364 ptepindex = pmap_l3e_pindex(va); 4365 retry: 4366 /* 4367 * Get the page directory entry 4368 */ 4369 pd = pmap_pml3e(pmap, va); 4370 4371 /* 4372 * This supports switching from a 2MB page to a 4373 * normal 4K page. 4374 */ 4375 if (pd != NULL && (be64toh(*pd) & (RPTE_LEAF | PG_V)) == (RPTE_LEAF | PG_V)) { 4376 if (!pmap_demote_l3e_locked(pmap, pd, va, lockp)) { 4377 /* 4378 * Invalidation of the 2MB page mapping may have caused 4379 * the deallocation of the underlying PD page. 4380 */ 4381 pd = NULL; 4382 } 4383 } 4384 4385 /* 4386 * If the page table page is mapped, we just increment the 4387 * hold count, and activate it. 4388 */ 4389 if (pd != NULL && (be64toh(*pd) & PG_V) != 0) { 4390 m = PHYS_TO_VM_PAGE(be64toh(*pd) & PG_FRAME); 4391 m->ref_count++; 4392 } else { 4393 /* 4394 * Here if the pte page isn't mapped, or if it has been 4395 * deallocated. 4396 */ 4397 m = _pmap_allocpte(pmap, ptepindex, lockp); 4398 if (m == NULL && lockp != NULL) 4399 goto retry; 4400 } 4401 return (m); 4402 } 4403 4404 static void 4405 mmu_radix_pinit0(pmap_t pmap) 4406 { 4407 4408 CTR2(KTR_PMAP, "%s(%p)", __func__, pmap); 4409 PMAP_LOCK_INIT(pmap); 4410 pmap->pm_pml1 = kernel_pmap->pm_pml1; 4411 pmap->pm_pid = kernel_pmap->pm_pid; 4412 4413 pmap->pm_radix.rt_root = 0; 4414 TAILQ_INIT(&pmap->pm_pvchunk); 4415 bzero(&pmap->pm_stats, sizeof pmap->pm_stats); 4416 kernel_pmap->pm_flags = 4417 pmap->pm_flags = PMAP_PDE_SUPERPAGE; 4418 } 4419 /* 4420 * pmap_protect_l3e: do the things to protect a 2mpage in a process 4421 */ 4422 static boolean_t 4423 pmap_protect_l3e(pmap_t pmap, pt_entry_t *l3e, vm_offset_t sva, vm_prot_t prot) 4424 { 4425 pt_entry_t newpde, oldpde; 4426 vm_offset_t eva, va; 4427 vm_page_t m; 4428 boolean_t anychanged; 4429 4430 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 4431 KASSERT((sva & L3_PAGE_MASK) == 0, 4432 ("pmap_protect_l3e: sva is not 2mpage aligned")); 4433 anychanged = FALSE; 4434 retry: 4435 oldpde = newpde = be64toh(*l3e); 4436 if ((oldpde & (PG_MANAGED | PG_M | PG_RW)) == 4437 (PG_MANAGED | PG_M | PG_RW)) { 4438 eva = sva + L3_PAGE_SIZE; 4439 for (va = sva, m = PHYS_TO_VM_PAGE(oldpde & PG_PS_FRAME); 4440 va < eva; va += PAGE_SIZE, m++) 4441 vm_page_dirty(m); 4442 } 4443 if ((prot & VM_PROT_WRITE) == 0) { 4444 newpde &= ~(PG_RW | PG_M); 4445 newpde |= RPTE_EAA_R; 4446 } 4447 if (prot & VM_PROT_EXECUTE) 4448 newpde |= PG_X; 4449 if (newpde != oldpde) { 4450 /* 4451 * As an optimization to future operations on this PDE, clear 4452 * PG_PROMOTED. The impending invalidation will remove any 4453 * lingering 4KB page mappings from the TLB. 4454 */ 4455 if (!atomic_cmpset_long(l3e, htobe64(oldpde), htobe64(newpde & ~PG_PROMOTED))) 4456 goto retry; 4457 anychanged = TRUE; 4458 } 4459 return (anychanged); 4460 } 4461 4462 void 4463 mmu_radix_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, 4464 vm_prot_t prot) 4465 { 4466 vm_offset_t va_next; 4467 pml1_entry_t *l1e; 4468 pml2_entry_t *l2e; 4469 pml3_entry_t ptpaddr, *l3e; 4470 pt_entry_t *pte; 4471 boolean_t anychanged; 4472 4473 CTR5(KTR_PMAP, "%s(%p, %#x, %#x, %#x)", __func__, pmap, sva, eva, 4474 prot); 4475 4476 KASSERT((prot & ~VM_PROT_ALL) == 0, ("invalid prot %x", prot)); 4477 if (prot == VM_PROT_NONE) { 4478 mmu_radix_remove(pmap, sva, eva); 4479 return; 4480 } 4481 4482 if ((prot & (VM_PROT_WRITE|VM_PROT_EXECUTE)) == 4483 (VM_PROT_WRITE|VM_PROT_EXECUTE)) 4484 return; 4485 4486 #ifdef INVARIANTS 4487 if (VERBOSE_PROTECT || pmap_logging) 4488 printf("pmap_protect(%p, %#lx, %#lx, %x) - asid: %lu\n", 4489 pmap, sva, eva, prot, pmap->pm_pid); 4490 #endif 4491 anychanged = FALSE; 4492 4493 PMAP_LOCK(pmap); 4494 for (; sva < eva; sva = va_next) { 4495 l1e = pmap_pml1e(pmap, sva); 4496 if ((be64toh(*l1e) & PG_V) == 0) { 4497 va_next = (sva + L1_PAGE_SIZE) & ~L1_PAGE_MASK; 4498 if (va_next < sva) 4499 va_next = eva; 4500 continue; 4501 } 4502 4503 l2e = pmap_l1e_to_l2e(l1e, sva); 4504 if ((be64toh(*l2e) & PG_V) == 0) { 4505 va_next = (sva + L2_PAGE_SIZE) & ~L2_PAGE_MASK; 4506 if (va_next < sva) 4507 va_next = eva; 4508 continue; 4509 } 4510 4511 va_next = (sva + L3_PAGE_SIZE) & ~L3_PAGE_MASK; 4512 if (va_next < sva) 4513 va_next = eva; 4514 4515 l3e = pmap_l2e_to_l3e(l2e, sva); 4516 ptpaddr = be64toh(*l3e); 4517 4518 /* 4519 * Weed out invalid mappings. 4520 */ 4521 if (ptpaddr == 0) 4522 continue; 4523 4524 /* 4525 * Check for large page. 4526 */ 4527 if ((ptpaddr & RPTE_LEAF) != 0) { 4528 /* 4529 * Are we protecting the entire large page? If not, 4530 * demote the mapping and fall through. 4531 */ 4532 if (sva + L3_PAGE_SIZE == va_next && eva >= va_next) { 4533 if (pmap_protect_l3e(pmap, l3e, sva, prot)) 4534 anychanged = TRUE; 4535 continue; 4536 } else if (!pmap_demote_l3e(pmap, l3e, sva)) { 4537 /* 4538 * The large page mapping was destroyed. 4539 */ 4540 continue; 4541 } 4542 } 4543 4544 if (va_next > eva) 4545 va_next = eva; 4546 4547 for (pte = pmap_l3e_to_pte(l3e, sva); sva != va_next; pte++, 4548 sva += PAGE_SIZE) { 4549 pt_entry_t obits, pbits; 4550 vm_page_t m; 4551 4552 retry: 4553 MPASS(pte == pmap_pte(pmap, sva)); 4554 obits = pbits = be64toh(*pte); 4555 if ((pbits & PG_V) == 0) 4556 continue; 4557 4558 if ((prot & VM_PROT_WRITE) == 0) { 4559 if ((pbits & (PG_MANAGED | PG_M | PG_RW)) == 4560 (PG_MANAGED | PG_M | PG_RW)) { 4561 m = PHYS_TO_VM_PAGE(pbits & PG_FRAME); 4562 vm_page_dirty(m); 4563 } 4564 pbits &= ~(PG_RW | PG_M); 4565 pbits |= RPTE_EAA_R; 4566 } 4567 if (prot & VM_PROT_EXECUTE) 4568 pbits |= PG_X; 4569 4570 if (pbits != obits) { 4571 if (!atomic_cmpset_long(pte, htobe64(obits), htobe64(pbits))) 4572 goto retry; 4573 if (obits & (PG_A|PG_M)) { 4574 anychanged = TRUE; 4575 #ifdef INVARIANTS 4576 if (VERBOSE_PROTECT || pmap_logging) 4577 printf("%#lx %#lx -> %#lx\n", 4578 sva, obits, pbits); 4579 #endif 4580 } 4581 } 4582 } 4583 } 4584 if (anychanged) 4585 pmap_invalidate_all(pmap); 4586 PMAP_UNLOCK(pmap); 4587 } 4588 4589 void 4590 mmu_radix_qenter(vm_offset_t sva, vm_page_t *ma, int count) 4591 { 4592 4593 CTR4(KTR_PMAP, "%s(%#x, %p, %d)", __func__, sva, ma, count); 4594 pt_entry_t oldpte, pa, *pte; 4595 vm_page_t m; 4596 uint64_t cache_bits, attr_bits; 4597 vm_offset_t va; 4598 4599 oldpte = 0; 4600 attr_bits = RPTE_EAA_R | RPTE_EAA_W | RPTE_EAA_P | PG_M | PG_A; 4601 va = sva; 4602 pte = kvtopte(va); 4603 while (va < sva + PAGE_SIZE * count) { 4604 if (__predict_false((va & L3_PAGE_MASK) == 0)) 4605 pte = kvtopte(va); 4606 MPASS(pte == pmap_pte(kernel_pmap, va)); 4607 4608 /* 4609 * XXX there has to be a more efficient way than traversing 4610 * the page table every time - but go for correctness for 4611 * today 4612 */ 4613 4614 m = *ma++; 4615 cache_bits = pmap_cache_bits(m->md.mdpg_cache_attrs); 4616 pa = VM_PAGE_TO_PHYS(m) | cache_bits | attr_bits; 4617 if (be64toh(*pte) != pa) { 4618 oldpte |= be64toh(*pte); 4619 pte_store(pte, pa); 4620 } 4621 va += PAGE_SIZE; 4622 pte++; 4623 } 4624 if (__predict_false((oldpte & RPTE_VALID) != 0)) 4625 pmap_invalidate_range(kernel_pmap, sva, sva + count * 4626 PAGE_SIZE); 4627 else 4628 ptesync(); 4629 } 4630 4631 void 4632 mmu_radix_qremove(vm_offset_t sva, int count) 4633 { 4634 vm_offset_t va; 4635 pt_entry_t *pte; 4636 4637 CTR3(KTR_PMAP, "%s(%#x, %d)", __func__, sva, count); 4638 KASSERT(sva >= VM_MIN_KERNEL_ADDRESS, ("usermode or dmap va %lx", sva)); 4639 4640 va = sva; 4641 pte = kvtopte(va); 4642 while (va < sva + PAGE_SIZE * count) { 4643 if (__predict_false((va & L3_PAGE_MASK) == 0)) 4644 pte = kvtopte(va); 4645 pte_clear(pte); 4646 pte++; 4647 va += PAGE_SIZE; 4648 } 4649 pmap_invalidate_range(kernel_pmap, sva, va); 4650 } 4651 4652 /*************************************************** 4653 * Page table page management routines..... 4654 ***************************************************/ 4655 /* 4656 * Schedule the specified unused page table page to be freed. Specifically, 4657 * add the page to the specified list of pages that will be released to the 4658 * physical memory manager after the TLB has been updated. 4659 */ 4660 static __inline void 4661 pmap_add_delayed_free_list(vm_page_t m, struct spglist *free, 4662 boolean_t set_PG_ZERO) 4663 { 4664 4665 if (set_PG_ZERO) 4666 m->flags |= PG_ZERO; 4667 else 4668 m->flags &= ~PG_ZERO; 4669 SLIST_INSERT_HEAD(free, m, plinks.s.ss); 4670 } 4671 4672 /* 4673 * Inserts the specified page table page into the specified pmap's collection 4674 * of idle page table pages. Each of a pmap's page table pages is responsible 4675 * for mapping a distinct range of virtual addresses. The pmap's collection is 4676 * ordered by this virtual address range. 4677 */ 4678 static __inline int 4679 pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte) 4680 { 4681 4682 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 4683 return (vm_radix_insert(&pmap->pm_radix, mpte)); 4684 } 4685 4686 /* 4687 * Removes the page table page mapping the specified virtual address from the 4688 * specified pmap's collection of idle page table pages, and returns it. 4689 * Otherwise, returns NULL if there is no page table page corresponding to the 4690 * specified virtual address. 4691 */ 4692 static __inline vm_page_t 4693 pmap_remove_pt_page(pmap_t pmap, vm_offset_t va) 4694 { 4695 4696 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 4697 return (vm_radix_remove(&pmap->pm_radix, pmap_l3e_pindex(va))); 4698 } 4699 4700 /* 4701 * Decrements a page table page's wire count, which is used to record the 4702 * number of valid page table entries within the page. If the wire count 4703 * drops to zero, then the page table page is unmapped. Returns TRUE if the 4704 * page table page was unmapped and FALSE otherwise. 4705 */ 4706 static inline boolean_t 4707 pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free) 4708 { 4709 4710 --m->ref_count; 4711 if (m->ref_count == 0) { 4712 _pmap_unwire_ptp(pmap, va, m, free); 4713 return (TRUE); 4714 } else 4715 return (FALSE); 4716 } 4717 4718 static void 4719 _pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free) 4720 { 4721 4722 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 4723 /* 4724 * unmap the page table page 4725 */ 4726 if (m->pindex >= (NUPDE + NUPDPE)) { 4727 /* PDP page */ 4728 pml1_entry_t *pml1; 4729 pml1 = pmap_pml1e(pmap, va); 4730 *pml1 = 0; 4731 } else if (m->pindex >= NUPDE) { 4732 /* PD page */ 4733 pml2_entry_t *l2e; 4734 l2e = pmap_pml2e(pmap, va); 4735 *l2e = 0; 4736 } else { 4737 /* PTE page */ 4738 pml3_entry_t *l3e; 4739 l3e = pmap_pml3e(pmap, va); 4740 *l3e = 0; 4741 } 4742 pmap_resident_count_dec(pmap, 1); 4743 if (m->pindex < NUPDE) { 4744 /* We just released a PT, unhold the matching PD */ 4745 vm_page_t pdpg; 4746 4747 pdpg = PHYS_TO_VM_PAGE(be64toh(*pmap_pml2e(pmap, va)) & PG_FRAME); 4748 pmap_unwire_ptp(pmap, va, pdpg, free); 4749 } 4750 if (m->pindex >= NUPDE && m->pindex < (NUPDE + NUPDPE)) { 4751 /* We just released a PD, unhold the matching PDP */ 4752 vm_page_t pdppg; 4753 4754 pdppg = PHYS_TO_VM_PAGE(be64toh(*pmap_pml1e(pmap, va)) & PG_FRAME); 4755 pmap_unwire_ptp(pmap, va, pdppg, free); 4756 } 4757 4758 /* 4759 * Put page on a list so that it is released after 4760 * *ALL* TLB shootdown is done 4761 */ 4762 pmap_add_delayed_free_list(m, free, TRUE); 4763 } 4764 4765 /* 4766 * After removing a page table entry, this routine is used to 4767 * conditionally free the page, and manage the hold/wire counts. 4768 */ 4769 static int 4770 pmap_unuse_pt(pmap_t pmap, vm_offset_t va, pml3_entry_t ptepde, 4771 struct spglist *free) 4772 { 4773 vm_page_t mpte; 4774 4775 if (va >= VM_MAXUSER_ADDRESS) 4776 return (0); 4777 KASSERT(ptepde != 0, ("pmap_unuse_pt: ptepde != 0")); 4778 mpte = PHYS_TO_VM_PAGE(ptepde & PG_FRAME); 4779 return (pmap_unwire_ptp(pmap, va, mpte, free)); 4780 } 4781 4782 void 4783 mmu_radix_release(pmap_t pmap) 4784 { 4785 4786 CTR2(KTR_PMAP, "%s(%p)", __func__, pmap); 4787 KASSERT(pmap->pm_stats.resident_count == 0, 4788 ("pmap_release: pmap resident count %ld != 0", 4789 pmap->pm_stats.resident_count)); 4790 KASSERT(vm_radix_is_empty(&pmap->pm_radix), 4791 ("pmap_release: pmap has reserved page table page(s)")); 4792 4793 pmap_invalidate_all(pmap); 4794 isa3_proctab[pmap->pm_pid].proctab0 = 0; 4795 uma_zfree(zone_radix_pgd, pmap->pm_pml1); 4796 vmem_free(asid_arena, pmap->pm_pid, 1); 4797 } 4798 4799 /* 4800 * Create the PV entry for a 2MB page mapping. Always returns true unless the 4801 * flag PMAP_ENTER_NORECLAIM is specified. If that flag is specified, returns 4802 * false if the PV entry cannot be allocated without resorting to reclamation. 4803 */ 4804 static bool 4805 pmap_pv_insert_l3e(pmap_t pmap, vm_offset_t va, pml3_entry_t pde, u_int flags, 4806 struct rwlock **lockp) 4807 { 4808 struct md_page *pvh; 4809 pv_entry_t pv; 4810 vm_paddr_t pa; 4811 4812 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 4813 /* Pass NULL instead of the lock pointer to disable reclamation. */ 4814 if ((pv = get_pv_entry(pmap, (flags & PMAP_ENTER_NORECLAIM) != 0 ? 4815 NULL : lockp)) == NULL) 4816 return (false); 4817 pv->pv_va = va; 4818 pa = pde & PG_PS_FRAME; 4819 CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa); 4820 pvh = pa_to_pvh(pa); 4821 TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_link); 4822 pvh->pv_gen++; 4823 return (true); 4824 } 4825 4826 /* 4827 * Fills a page table page with mappings to consecutive physical pages. 4828 */ 4829 static void 4830 pmap_fill_ptp(pt_entry_t *firstpte, pt_entry_t newpte) 4831 { 4832 pt_entry_t *pte; 4833 4834 for (pte = firstpte; pte < firstpte + NPTEPG; pte++) { 4835 *pte = htobe64(newpte); 4836 newpte += PAGE_SIZE; 4837 } 4838 } 4839 4840 static boolean_t 4841 pmap_demote_l3e(pmap_t pmap, pml3_entry_t *pde, vm_offset_t va) 4842 { 4843 struct rwlock *lock; 4844 boolean_t rv; 4845 4846 lock = NULL; 4847 rv = pmap_demote_l3e_locked(pmap, pde, va, &lock); 4848 if (lock != NULL) 4849 rw_wunlock(lock); 4850 return (rv); 4851 } 4852 4853 static boolean_t 4854 pmap_demote_l3e_locked(pmap_t pmap, pml3_entry_t *l3e, vm_offset_t va, 4855 struct rwlock **lockp) 4856 { 4857 pml3_entry_t oldpde; 4858 pt_entry_t *firstpte; 4859 vm_paddr_t mptepa; 4860 vm_page_t mpte; 4861 struct spglist free; 4862 vm_offset_t sva; 4863 4864 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 4865 oldpde = be64toh(*l3e); 4866 KASSERT((oldpde & (RPTE_LEAF | PG_V)) == (RPTE_LEAF | PG_V), 4867 ("pmap_demote_l3e: oldpde is missing RPTE_LEAF and/or PG_V %lx", 4868 oldpde)); 4869 if ((oldpde & PG_A) == 0 || (mpte = pmap_remove_pt_page(pmap, va)) == 4870 NULL) { 4871 KASSERT((oldpde & PG_W) == 0, 4872 ("pmap_demote_l3e: page table page for a wired mapping" 4873 " is missing")); 4874 4875 /* 4876 * Invalidate the 2MB page mapping and return "failure" if the 4877 * mapping was never accessed or the allocation of the new 4878 * page table page fails. If the 2MB page mapping belongs to 4879 * the direct map region of the kernel's address space, then 4880 * the page allocation request specifies the highest possible 4881 * priority (VM_ALLOC_INTERRUPT). Otherwise, the priority is 4882 * normal. Page table pages are preallocated for every other 4883 * part of the kernel address space, so the direct map region 4884 * is the only part of the kernel address space that must be 4885 * handled here. 4886 */ 4887 if ((oldpde & PG_A) == 0 || (mpte = vm_page_alloc(NULL, 4888 pmap_l3e_pindex(va), (va >= DMAP_MIN_ADDRESS && va < 4889 DMAP_MAX_ADDRESS ? VM_ALLOC_INTERRUPT : VM_ALLOC_NORMAL) | 4890 VM_ALLOC_NOOBJ | VM_ALLOC_WIRED)) == NULL) { 4891 SLIST_INIT(&free); 4892 sva = trunc_2mpage(va); 4893 pmap_remove_l3e(pmap, l3e, sva, &free, lockp); 4894 pmap_invalidate_l3e_page(pmap, sva, oldpde); 4895 vm_page_free_pages_toq(&free, true); 4896 CTR2(KTR_PMAP, "pmap_demote_l3e: failure for va %#lx" 4897 " in pmap %p", va, pmap); 4898 return (FALSE); 4899 } 4900 if (va < VM_MAXUSER_ADDRESS) 4901 pmap_resident_count_inc(pmap, 1); 4902 } 4903 mptepa = VM_PAGE_TO_PHYS(mpte); 4904 firstpte = (pt_entry_t *)PHYS_TO_DMAP(mptepa); 4905 KASSERT((oldpde & PG_A) != 0, 4906 ("pmap_demote_l3e: oldpde is missing PG_A")); 4907 KASSERT((oldpde & (PG_M | PG_RW)) != PG_RW, 4908 ("pmap_demote_l3e: oldpde is missing PG_M")); 4909 4910 /* 4911 * If the page table page is new, initialize it. 4912 */ 4913 if (mpte->ref_count == 1) { 4914 mpte->ref_count = NPTEPG; 4915 pmap_fill_ptp(firstpte, oldpde); 4916 } 4917 4918 KASSERT((be64toh(*firstpte) & PG_FRAME) == (oldpde & PG_FRAME), 4919 ("pmap_demote_l3e: firstpte and newpte map different physical" 4920 " addresses")); 4921 4922 /* 4923 * If the mapping has changed attributes, update the page table 4924 * entries. 4925 */ 4926 if ((be64toh(*firstpte) & PG_PTE_PROMOTE) != (oldpde & PG_PTE_PROMOTE)) 4927 pmap_fill_ptp(firstpte, oldpde); 4928 4929 /* 4930 * The spare PV entries must be reserved prior to demoting the 4931 * mapping, that is, prior to changing the PDE. Otherwise, the state 4932 * of the PDE and the PV lists will be inconsistent, which can result 4933 * in reclaim_pv_chunk() attempting to remove a PV entry from the 4934 * wrong PV list and pmap_pv_demote_l3e() failing to find the expected 4935 * PV entry for the 2MB page mapping that is being demoted. 4936 */ 4937 if ((oldpde & PG_MANAGED) != 0) 4938 reserve_pv_entries(pmap, NPTEPG - 1, lockp); 4939 4940 /* 4941 * Demote the mapping. This pmap is locked. The old PDE has 4942 * PG_A set. If the old PDE has PG_RW set, it also has PG_M 4943 * set. Thus, there is no danger of a race with another 4944 * processor changing the setting of PG_A and/or PG_M between 4945 * the read above and the store below. 4946 */ 4947 pde_store(l3e, mptepa); 4948 pmap_invalidate_l3e_page(pmap, trunc_2mpage(va), oldpde); 4949 /* 4950 * Demote the PV entry. 4951 */ 4952 if ((oldpde & PG_MANAGED) != 0) 4953 pmap_pv_demote_l3e(pmap, va, oldpde & PG_PS_FRAME, lockp); 4954 4955 atomic_add_long(&pmap_l3e_demotions, 1); 4956 CTR2(KTR_PMAP, "pmap_demote_l3e: success for va %#lx" 4957 " in pmap %p", va, pmap); 4958 return (TRUE); 4959 } 4960 4961 /* 4962 * pmap_remove_kernel_pde: Remove a kernel superpage mapping. 4963 */ 4964 static void 4965 pmap_remove_kernel_l3e(pmap_t pmap, pml3_entry_t *l3e, vm_offset_t va) 4966 { 4967 vm_paddr_t mptepa; 4968 vm_page_t mpte; 4969 4970 KASSERT(pmap == kernel_pmap, ("pmap %p is not kernel_pmap", pmap)); 4971 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 4972 mpte = pmap_remove_pt_page(pmap, va); 4973 if (mpte == NULL) 4974 panic("pmap_remove_kernel_pde: Missing pt page."); 4975 4976 mptepa = VM_PAGE_TO_PHYS(mpte); 4977 4978 /* 4979 * Initialize the page table page. 4980 */ 4981 pagezero(PHYS_TO_DMAP(mptepa)); 4982 4983 /* 4984 * Demote the mapping. 4985 */ 4986 pde_store(l3e, mptepa); 4987 ptesync(); 4988 } 4989 4990 /* 4991 * pmap_remove_l3e: do the things to unmap a superpage in a process 4992 */ 4993 static int 4994 pmap_remove_l3e(pmap_t pmap, pml3_entry_t *pdq, vm_offset_t sva, 4995 struct spglist *free, struct rwlock **lockp) 4996 { 4997 struct md_page *pvh; 4998 pml3_entry_t oldpde; 4999 vm_offset_t eva, va; 5000 vm_page_t m, mpte; 5001 5002 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 5003 KASSERT((sva & L3_PAGE_MASK) == 0, 5004 ("pmap_remove_l3e: sva is not 2mpage aligned")); 5005 oldpde = be64toh(pte_load_clear(pdq)); 5006 if (oldpde & PG_W) 5007 pmap->pm_stats.wired_count -= (L3_PAGE_SIZE / PAGE_SIZE); 5008 pmap_resident_count_dec(pmap, L3_PAGE_SIZE / PAGE_SIZE); 5009 if (oldpde & PG_MANAGED) { 5010 CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, oldpde & PG_PS_FRAME); 5011 pvh = pa_to_pvh(oldpde & PG_PS_FRAME); 5012 pmap_pvh_free(pvh, pmap, sva); 5013 eva = sva + L3_PAGE_SIZE; 5014 for (va = sva, m = PHYS_TO_VM_PAGE(oldpde & PG_PS_FRAME); 5015 va < eva; va += PAGE_SIZE, m++) { 5016 if ((oldpde & (PG_M | PG_RW)) == (PG_M | PG_RW)) 5017 vm_page_dirty(m); 5018 if (oldpde & PG_A) 5019 vm_page_aflag_set(m, PGA_REFERENCED); 5020 if (TAILQ_EMPTY(&m->md.pv_list) && 5021 TAILQ_EMPTY(&pvh->pv_list)) 5022 vm_page_aflag_clear(m, PGA_WRITEABLE); 5023 } 5024 } 5025 if (pmap == kernel_pmap) { 5026 pmap_remove_kernel_l3e(pmap, pdq, sva); 5027 } else { 5028 mpte = pmap_remove_pt_page(pmap, sva); 5029 if (mpte != NULL) { 5030 pmap_resident_count_dec(pmap, 1); 5031 KASSERT(mpte->ref_count == NPTEPG, 5032 ("pmap_remove_l3e: pte page wire count error")); 5033 mpte->ref_count = 0; 5034 pmap_add_delayed_free_list(mpte, free, FALSE); 5035 } 5036 } 5037 return (pmap_unuse_pt(pmap, sva, be64toh(*pmap_pml2e(pmap, sva)), free)); 5038 } 5039 5040 /* 5041 * pmap_remove_pte: do the things to unmap a page in a process 5042 */ 5043 static int 5044 pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t va, 5045 pml3_entry_t ptepde, struct spglist *free, struct rwlock **lockp) 5046 { 5047 struct md_page *pvh; 5048 pt_entry_t oldpte; 5049 vm_page_t m; 5050 5051 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 5052 oldpte = be64toh(pte_load_clear(ptq)); 5053 if (oldpte & RPTE_WIRED) 5054 pmap->pm_stats.wired_count -= 1; 5055 pmap_resident_count_dec(pmap, 1); 5056 if (oldpte & RPTE_MANAGED) { 5057 m = PHYS_TO_VM_PAGE(oldpte & PG_FRAME); 5058 if ((oldpte & (PG_M | PG_RW)) == (PG_M | PG_RW)) 5059 vm_page_dirty(m); 5060 if (oldpte & PG_A) 5061 vm_page_aflag_set(m, PGA_REFERENCED); 5062 CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m); 5063 pmap_pvh_free(&m->md, pmap, va); 5064 if (TAILQ_EMPTY(&m->md.pv_list) && 5065 (m->flags & PG_FICTITIOUS) == 0) { 5066 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); 5067 if (TAILQ_EMPTY(&pvh->pv_list)) 5068 vm_page_aflag_clear(m, PGA_WRITEABLE); 5069 } 5070 } 5071 return (pmap_unuse_pt(pmap, va, ptepde, free)); 5072 } 5073 5074 /* 5075 * Remove a single page from a process address space 5076 */ 5077 static bool 5078 pmap_remove_page(pmap_t pmap, vm_offset_t va, pml3_entry_t *l3e, 5079 struct spglist *free) 5080 { 5081 struct rwlock *lock; 5082 pt_entry_t *pte; 5083 bool invalidate_all; 5084 5085 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 5086 if ((be64toh(*l3e) & RPTE_VALID) == 0) { 5087 return (false); 5088 } 5089 pte = pmap_l3e_to_pte(l3e, va); 5090 if ((be64toh(*pte) & RPTE_VALID) == 0) { 5091 return (false); 5092 } 5093 lock = NULL; 5094 5095 invalidate_all = pmap_remove_pte(pmap, pte, va, be64toh(*l3e), free, &lock); 5096 if (lock != NULL) 5097 rw_wunlock(lock); 5098 if (!invalidate_all) 5099 pmap_invalidate_page(pmap, va); 5100 return (invalidate_all); 5101 } 5102 5103 /* 5104 * Removes the specified range of addresses from the page table page. 5105 */ 5106 static bool 5107 pmap_remove_ptes(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, 5108 pml3_entry_t *l3e, struct spglist *free, struct rwlock **lockp) 5109 { 5110 pt_entry_t *pte; 5111 vm_offset_t va; 5112 bool anyvalid; 5113 5114 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 5115 anyvalid = false; 5116 va = eva; 5117 for (pte = pmap_l3e_to_pte(l3e, sva); sva != eva; pte++, 5118 sva += PAGE_SIZE) { 5119 MPASS(pte == pmap_pte(pmap, sva)); 5120 if (*pte == 0) { 5121 if (va != eva) { 5122 anyvalid = true; 5123 va = eva; 5124 } 5125 continue; 5126 } 5127 if (va == eva) 5128 va = sva; 5129 if (pmap_remove_pte(pmap, pte, sva, be64toh(*l3e), free, lockp)) { 5130 anyvalid = true; 5131 sva += PAGE_SIZE; 5132 break; 5133 } 5134 } 5135 if (anyvalid) 5136 pmap_invalidate_all(pmap); 5137 else if (va != eva) 5138 pmap_invalidate_range(pmap, va, sva); 5139 return (anyvalid); 5140 } 5141 5142 void 5143 mmu_radix_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) 5144 { 5145 struct rwlock *lock; 5146 vm_offset_t va_next; 5147 pml1_entry_t *l1e; 5148 pml2_entry_t *l2e; 5149 pml3_entry_t ptpaddr, *l3e; 5150 struct spglist free; 5151 bool anyvalid; 5152 5153 CTR4(KTR_PMAP, "%s(%p, %#x, %#x)", __func__, pmap, sva, eva); 5154 5155 /* 5156 * Perform an unsynchronized read. This is, however, safe. 5157 */ 5158 if (pmap->pm_stats.resident_count == 0) 5159 return; 5160 5161 anyvalid = false; 5162 SLIST_INIT(&free); 5163 5164 /* XXX something fishy here */ 5165 sva = (sva + PAGE_MASK) & ~PAGE_MASK; 5166 eva = (eva + PAGE_MASK) & ~PAGE_MASK; 5167 5168 PMAP_LOCK(pmap); 5169 5170 /* 5171 * special handling of removing one page. a very 5172 * common operation and easy to short circuit some 5173 * code. 5174 */ 5175 if (sva + PAGE_SIZE == eva) { 5176 l3e = pmap_pml3e(pmap, sva); 5177 if (l3e && (be64toh(*l3e) & RPTE_LEAF) == 0) { 5178 anyvalid = pmap_remove_page(pmap, sva, l3e, &free); 5179 goto out; 5180 } 5181 } 5182 5183 lock = NULL; 5184 for (; sva < eva; sva = va_next) { 5185 if (pmap->pm_stats.resident_count == 0) 5186 break; 5187 l1e = pmap_pml1e(pmap, sva); 5188 if (l1e == NULL || (be64toh(*l1e) & PG_V) == 0) { 5189 va_next = (sva + L1_PAGE_SIZE) & ~L1_PAGE_MASK; 5190 if (va_next < sva) 5191 va_next = eva; 5192 continue; 5193 } 5194 5195 l2e = pmap_l1e_to_l2e(l1e, sva); 5196 if (l2e == NULL || (be64toh(*l2e) & PG_V) == 0) { 5197 va_next = (sva + L2_PAGE_SIZE) & ~L2_PAGE_MASK; 5198 if (va_next < sva) 5199 va_next = eva; 5200 continue; 5201 } 5202 5203 /* 5204 * Calculate index for next page table. 5205 */ 5206 va_next = (sva + L3_PAGE_SIZE) & ~L3_PAGE_MASK; 5207 if (va_next < sva) 5208 va_next = eva; 5209 5210 l3e = pmap_l2e_to_l3e(l2e, sva); 5211 ptpaddr = be64toh(*l3e); 5212 5213 /* 5214 * Weed out invalid mappings. 5215 */ 5216 if (ptpaddr == 0) 5217 continue; 5218 5219 /* 5220 * Check for large page. 5221 */ 5222 if ((ptpaddr & RPTE_LEAF) != 0) { 5223 /* 5224 * Are we removing the entire large page? If not, 5225 * demote the mapping and fall through. 5226 */ 5227 if (sva + L3_PAGE_SIZE == va_next && eva >= va_next) { 5228 pmap_remove_l3e(pmap, l3e, sva, &free, &lock); 5229 anyvalid = true; 5230 continue; 5231 } else if (!pmap_demote_l3e_locked(pmap, l3e, sva, 5232 &lock)) { 5233 /* The large page mapping was destroyed. */ 5234 continue; 5235 } else 5236 ptpaddr = be64toh(*l3e); 5237 } 5238 5239 /* 5240 * Limit our scan to either the end of the va represented 5241 * by the current page table page, or to the end of the 5242 * range being removed. 5243 */ 5244 if (va_next > eva) 5245 va_next = eva; 5246 5247 if (pmap_remove_ptes(pmap, sva, va_next, l3e, &free, &lock)) 5248 anyvalid = true; 5249 } 5250 if (lock != NULL) 5251 rw_wunlock(lock); 5252 out: 5253 if (anyvalid) 5254 pmap_invalidate_all(pmap); 5255 PMAP_UNLOCK(pmap); 5256 vm_page_free_pages_toq(&free, true); 5257 } 5258 5259 void 5260 mmu_radix_remove_all(vm_page_t m) 5261 { 5262 struct md_page *pvh; 5263 pv_entry_t pv; 5264 pmap_t pmap; 5265 struct rwlock *lock; 5266 pt_entry_t *pte, tpte; 5267 pml3_entry_t *l3e; 5268 vm_offset_t va; 5269 struct spglist free; 5270 int pvh_gen, md_gen; 5271 5272 CTR2(KTR_PMAP, "%s(%p)", __func__, m); 5273 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 5274 ("pmap_remove_all: page %p is not managed", m)); 5275 SLIST_INIT(&free); 5276 lock = VM_PAGE_TO_PV_LIST_LOCK(m); 5277 pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy : 5278 pa_to_pvh(VM_PAGE_TO_PHYS(m)); 5279 retry: 5280 rw_wlock(lock); 5281 while ((pv = TAILQ_FIRST(&pvh->pv_list)) != NULL) { 5282 pmap = PV_PMAP(pv); 5283 if (!PMAP_TRYLOCK(pmap)) { 5284 pvh_gen = pvh->pv_gen; 5285 rw_wunlock(lock); 5286 PMAP_LOCK(pmap); 5287 rw_wlock(lock); 5288 if (pvh_gen != pvh->pv_gen) { 5289 rw_wunlock(lock); 5290 PMAP_UNLOCK(pmap); 5291 goto retry; 5292 } 5293 } 5294 va = pv->pv_va; 5295 l3e = pmap_pml3e(pmap, va); 5296 (void)pmap_demote_l3e_locked(pmap, l3e, va, &lock); 5297 PMAP_UNLOCK(pmap); 5298 } 5299 while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) { 5300 pmap = PV_PMAP(pv); 5301 if (!PMAP_TRYLOCK(pmap)) { 5302 pvh_gen = pvh->pv_gen; 5303 md_gen = m->md.pv_gen; 5304 rw_wunlock(lock); 5305 PMAP_LOCK(pmap); 5306 rw_wlock(lock); 5307 if (pvh_gen != pvh->pv_gen || md_gen != m->md.pv_gen) { 5308 rw_wunlock(lock); 5309 PMAP_UNLOCK(pmap); 5310 goto retry; 5311 } 5312 } 5313 pmap_resident_count_dec(pmap, 1); 5314 l3e = pmap_pml3e(pmap, pv->pv_va); 5315 KASSERT((be64toh(*l3e) & RPTE_LEAF) == 0, ("pmap_remove_all: found" 5316 " a 2mpage in page %p's pv list", m)); 5317 pte = pmap_l3e_to_pte(l3e, pv->pv_va); 5318 tpte = be64toh(pte_load_clear(pte)); 5319 if (tpte & PG_W) 5320 pmap->pm_stats.wired_count--; 5321 if (tpte & PG_A) 5322 vm_page_aflag_set(m, PGA_REFERENCED); 5323 5324 /* 5325 * Update the vm_page_t clean and reference bits. 5326 */ 5327 if ((tpte & (PG_M | PG_RW)) == (PG_M | PG_RW)) 5328 vm_page_dirty(m); 5329 pmap_unuse_pt(pmap, pv->pv_va, be64toh(*l3e), &free); 5330 pmap_invalidate_page(pmap, pv->pv_va); 5331 TAILQ_REMOVE(&m->md.pv_list, pv, pv_link); 5332 m->md.pv_gen++; 5333 free_pv_entry(pmap, pv); 5334 PMAP_UNLOCK(pmap); 5335 } 5336 vm_page_aflag_clear(m, PGA_WRITEABLE); 5337 rw_wunlock(lock); 5338 vm_page_free_pages_toq(&free, true); 5339 } 5340 5341 /* 5342 * Destroy all managed, non-wired mappings in the given user-space 5343 * pmap. This pmap cannot be active on any processor besides the 5344 * caller. 5345 * 5346 * This function cannot be applied to the kernel pmap. Moreover, it 5347 * is not intended for general use. It is only to be used during 5348 * process termination. Consequently, it can be implemented in ways 5349 * that make it faster than pmap_remove(). First, it can more quickly 5350 * destroy mappings by iterating over the pmap's collection of PV 5351 * entries, rather than searching the page table. Second, it doesn't 5352 * have to test and clear the page table entries atomically, because 5353 * no processor is currently accessing the user address space. In 5354 * particular, a page table entry's dirty bit won't change state once 5355 * this function starts. 5356 * 5357 * Although this function destroys all of the pmap's managed, 5358 * non-wired mappings, it can delay and batch the invalidation of TLB 5359 * entries without calling pmap_delayed_invl_started() and 5360 * pmap_delayed_invl_finished(). Because the pmap is not active on 5361 * any other processor, none of these TLB entries will ever be used 5362 * before their eventual invalidation. Consequently, there is no need 5363 * for either pmap_remove_all() or pmap_remove_write() to wait for 5364 * that eventual TLB invalidation. 5365 */ 5366 5367 void 5368 mmu_radix_remove_pages(pmap_t pmap) 5369 { 5370 5371 CTR2(KTR_PMAP, "%s(%p)", __func__, pmap); 5372 pml3_entry_t ptel3e; 5373 pt_entry_t *pte, tpte; 5374 struct spglist free; 5375 vm_page_t m, mpte, mt; 5376 pv_entry_t pv; 5377 struct md_page *pvh; 5378 struct pv_chunk *pc, *npc; 5379 struct rwlock *lock; 5380 int64_t bit; 5381 uint64_t inuse, bitmask; 5382 int allfree, field, freed, idx; 5383 boolean_t superpage; 5384 vm_paddr_t pa; 5385 5386 /* 5387 * Assert that the given pmap is only active on the current 5388 * CPU. Unfortunately, we cannot block another CPU from 5389 * activating the pmap while this function is executing. 5390 */ 5391 KASSERT(pmap->pm_pid == mfspr(SPR_PID), 5392 ("non-current asid %lu - expected %lu", pmap->pm_pid, 5393 mfspr(SPR_PID))); 5394 5395 lock = NULL; 5396 5397 SLIST_INIT(&free); 5398 PMAP_LOCK(pmap); 5399 TAILQ_FOREACH_SAFE(pc, &pmap->pm_pvchunk, pc_list, npc) { 5400 allfree = 1; 5401 freed = 0; 5402 for (field = 0; field < _NPCM; field++) { 5403 inuse = ~pc->pc_map[field] & pc_freemask[field]; 5404 while (inuse != 0) { 5405 bit = cnttzd(inuse); 5406 bitmask = 1UL << bit; 5407 idx = field * 64 + bit; 5408 pv = &pc->pc_pventry[idx]; 5409 inuse &= ~bitmask; 5410 5411 pte = pmap_pml2e(pmap, pv->pv_va); 5412 ptel3e = be64toh(*pte); 5413 pte = pmap_l2e_to_l3e(pte, pv->pv_va); 5414 tpte = be64toh(*pte); 5415 if ((tpte & (RPTE_LEAF | PG_V)) == PG_V) { 5416 superpage = FALSE; 5417 ptel3e = tpte; 5418 pte = (pt_entry_t *)PHYS_TO_DMAP(tpte & 5419 PG_FRAME); 5420 pte = &pte[pmap_pte_index(pv->pv_va)]; 5421 tpte = be64toh(*pte); 5422 } else { 5423 /* 5424 * Keep track whether 'tpte' is a 5425 * superpage explicitly instead of 5426 * relying on RPTE_LEAF being set. 5427 * 5428 * This is because RPTE_LEAF is numerically 5429 * identical to PG_PTE_PAT and thus a 5430 * regular page could be mistaken for 5431 * a superpage. 5432 */ 5433 superpage = TRUE; 5434 } 5435 5436 if ((tpte & PG_V) == 0) { 5437 panic("bad pte va %lx pte %lx", 5438 pv->pv_va, tpte); 5439 } 5440 5441 /* 5442 * We cannot remove wired pages from a process' mapping at this time 5443 */ 5444 if (tpte & PG_W) { 5445 allfree = 0; 5446 continue; 5447 } 5448 5449 if (superpage) 5450 pa = tpte & PG_PS_FRAME; 5451 else 5452 pa = tpte & PG_FRAME; 5453 5454 m = PHYS_TO_VM_PAGE(pa); 5455 KASSERT(m->phys_addr == pa, 5456 ("vm_page_t %p phys_addr mismatch %016jx %016jx", 5457 m, (uintmax_t)m->phys_addr, 5458 (uintmax_t)tpte)); 5459 5460 KASSERT((m->flags & PG_FICTITIOUS) != 0 || 5461 m < &vm_page_array[vm_page_array_size], 5462 ("pmap_remove_pages: bad tpte %#jx", 5463 (uintmax_t)tpte)); 5464 5465 pte_clear(pte); 5466 5467 /* 5468 * Update the vm_page_t clean/reference bits. 5469 */ 5470 if ((tpte & (PG_M | PG_RW)) == (PG_M | PG_RW)) { 5471 if (superpage) { 5472 for (mt = m; mt < &m[L3_PAGE_SIZE / PAGE_SIZE]; mt++) 5473 vm_page_dirty(mt); 5474 } else 5475 vm_page_dirty(m); 5476 } 5477 5478 CHANGE_PV_LIST_LOCK_TO_VM_PAGE(&lock, m); 5479 5480 /* Mark free */ 5481 pc->pc_map[field] |= bitmask; 5482 if (superpage) { 5483 pmap_resident_count_dec(pmap, L3_PAGE_SIZE / PAGE_SIZE); 5484 pvh = pa_to_pvh(tpte & PG_PS_FRAME); 5485 TAILQ_REMOVE(&pvh->pv_list, pv, pv_link); 5486 pvh->pv_gen++; 5487 if (TAILQ_EMPTY(&pvh->pv_list)) { 5488 for (mt = m; mt < &m[L3_PAGE_SIZE / PAGE_SIZE]; mt++) 5489 if ((mt->a.flags & PGA_WRITEABLE) != 0 && 5490 TAILQ_EMPTY(&mt->md.pv_list)) 5491 vm_page_aflag_clear(mt, PGA_WRITEABLE); 5492 } 5493 mpte = pmap_remove_pt_page(pmap, pv->pv_va); 5494 if (mpte != NULL) { 5495 pmap_resident_count_dec(pmap, 1); 5496 KASSERT(mpte->ref_count == NPTEPG, 5497 ("pmap_remove_pages: pte page wire count error")); 5498 mpte->ref_count = 0; 5499 pmap_add_delayed_free_list(mpte, &free, FALSE); 5500 } 5501 } else { 5502 pmap_resident_count_dec(pmap, 1); 5503 #ifdef VERBOSE_PV 5504 printf("freeing pv (%p, %p)\n", 5505 pmap, pv); 5506 #endif 5507 TAILQ_REMOVE(&m->md.pv_list, pv, pv_link); 5508 m->md.pv_gen++; 5509 if ((m->a.flags & PGA_WRITEABLE) != 0 && 5510 TAILQ_EMPTY(&m->md.pv_list) && 5511 (m->flags & PG_FICTITIOUS) == 0) { 5512 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); 5513 if (TAILQ_EMPTY(&pvh->pv_list)) 5514 vm_page_aflag_clear(m, PGA_WRITEABLE); 5515 } 5516 } 5517 pmap_unuse_pt(pmap, pv->pv_va, ptel3e, &free); 5518 freed++; 5519 } 5520 } 5521 PV_STAT(atomic_add_long(&pv_entry_frees, freed)); 5522 PV_STAT(atomic_add_int(&pv_entry_spare, freed)); 5523 PV_STAT(atomic_subtract_long(&pv_entry_count, freed)); 5524 if (allfree) { 5525 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 5526 free_pv_chunk(pc); 5527 } 5528 } 5529 if (lock != NULL) 5530 rw_wunlock(lock); 5531 pmap_invalidate_all(pmap); 5532 PMAP_UNLOCK(pmap); 5533 vm_page_free_pages_toq(&free, true); 5534 } 5535 5536 void 5537 mmu_radix_remove_write(vm_page_t m) 5538 { 5539 struct md_page *pvh; 5540 pmap_t pmap; 5541 struct rwlock *lock; 5542 pv_entry_t next_pv, pv; 5543 pml3_entry_t *l3e; 5544 pt_entry_t oldpte, *pte; 5545 int pvh_gen, md_gen; 5546 5547 CTR2(KTR_PMAP, "%s(%p)", __func__, m); 5548 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 5549 ("pmap_remove_write: page %p is not managed", m)); 5550 vm_page_assert_busied(m); 5551 5552 if (!pmap_page_is_write_mapped(m)) 5553 return; 5554 lock = VM_PAGE_TO_PV_LIST_LOCK(m); 5555 pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy : 5556 pa_to_pvh(VM_PAGE_TO_PHYS(m)); 5557 retry_pv_loop: 5558 rw_wlock(lock); 5559 TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_link, next_pv) { 5560 pmap = PV_PMAP(pv); 5561 if (!PMAP_TRYLOCK(pmap)) { 5562 pvh_gen = pvh->pv_gen; 5563 rw_wunlock(lock); 5564 PMAP_LOCK(pmap); 5565 rw_wlock(lock); 5566 if (pvh_gen != pvh->pv_gen) { 5567 PMAP_UNLOCK(pmap); 5568 rw_wunlock(lock); 5569 goto retry_pv_loop; 5570 } 5571 } 5572 l3e = pmap_pml3e(pmap, pv->pv_va); 5573 if ((be64toh(*l3e) & PG_RW) != 0) 5574 (void)pmap_demote_l3e_locked(pmap, l3e, pv->pv_va, &lock); 5575 KASSERT(lock == VM_PAGE_TO_PV_LIST_LOCK(m), 5576 ("inconsistent pv lock %p %p for page %p", 5577 lock, VM_PAGE_TO_PV_LIST_LOCK(m), m)); 5578 PMAP_UNLOCK(pmap); 5579 } 5580 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) { 5581 pmap = PV_PMAP(pv); 5582 if (!PMAP_TRYLOCK(pmap)) { 5583 pvh_gen = pvh->pv_gen; 5584 md_gen = m->md.pv_gen; 5585 rw_wunlock(lock); 5586 PMAP_LOCK(pmap); 5587 rw_wlock(lock); 5588 if (pvh_gen != pvh->pv_gen || 5589 md_gen != m->md.pv_gen) { 5590 PMAP_UNLOCK(pmap); 5591 rw_wunlock(lock); 5592 goto retry_pv_loop; 5593 } 5594 } 5595 l3e = pmap_pml3e(pmap, pv->pv_va); 5596 KASSERT((be64toh(*l3e) & RPTE_LEAF) == 0, 5597 ("pmap_remove_write: found a 2mpage in page %p's pv list", 5598 m)); 5599 pte = pmap_l3e_to_pte(l3e, pv->pv_va); 5600 retry: 5601 oldpte = be64toh(*pte); 5602 if (oldpte & PG_RW) { 5603 if (!atomic_cmpset_long(pte, htobe64(oldpte), 5604 htobe64((oldpte | RPTE_EAA_R) & ~(PG_RW | PG_M)))) 5605 goto retry; 5606 if ((oldpte & PG_M) != 0) 5607 vm_page_dirty(m); 5608 pmap_invalidate_page(pmap, pv->pv_va); 5609 } 5610 PMAP_UNLOCK(pmap); 5611 } 5612 rw_wunlock(lock); 5613 vm_page_aflag_clear(m, PGA_WRITEABLE); 5614 } 5615 5616 /* 5617 * Clear the wired attribute from the mappings for the specified range of 5618 * addresses in the given pmap. Every valid mapping within that range 5619 * must have the wired attribute set. In contrast, invalid mappings 5620 * cannot have the wired attribute set, so they are ignored. 5621 * 5622 * The wired attribute of the page table entry is not a hardware 5623 * feature, so there is no need to invalidate any TLB entries. 5624 * Since pmap_demote_l3e() for the wired entry must never fail, 5625 * pmap_delayed_invl_started()/finished() calls around the 5626 * function are not needed. 5627 */ 5628 void 5629 mmu_radix_unwire(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) 5630 { 5631 vm_offset_t va_next; 5632 pml1_entry_t *l1e; 5633 pml2_entry_t *l2e; 5634 pml3_entry_t *l3e; 5635 pt_entry_t *pte; 5636 5637 CTR4(KTR_PMAP, "%s(%p, %#x, %#x)", __func__, pmap, sva, eva); 5638 PMAP_LOCK(pmap); 5639 for (; sva < eva; sva = va_next) { 5640 l1e = pmap_pml1e(pmap, sva); 5641 if ((be64toh(*l1e) & PG_V) == 0) { 5642 va_next = (sva + L1_PAGE_SIZE) & ~L1_PAGE_MASK; 5643 if (va_next < sva) 5644 va_next = eva; 5645 continue; 5646 } 5647 l2e = pmap_l1e_to_l2e(l1e, sva); 5648 if ((be64toh(*l2e) & PG_V) == 0) { 5649 va_next = (sva + L2_PAGE_SIZE) & ~L2_PAGE_MASK; 5650 if (va_next < sva) 5651 va_next = eva; 5652 continue; 5653 } 5654 va_next = (sva + L3_PAGE_SIZE) & ~L3_PAGE_MASK; 5655 if (va_next < sva) 5656 va_next = eva; 5657 l3e = pmap_l2e_to_l3e(l2e, sva); 5658 if ((be64toh(*l3e) & PG_V) == 0) 5659 continue; 5660 if ((be64toh(*l3e) & RPTE_LEAF) != 0) { 5661 if ((be64toh(*l3e) & PG_W) == 0) 5662 panic("pmap_unwire: pde %#jx is missing PG_W", 5663 (uintmax_t)(be64toh(*l3e))); 5664 5665 /* 5666 * Are we unwiring the entire large page? If not, 5667 * demote the mapping and fall through. 5668 */ 5669 if (sva + L3_PAGE_SIZE == va_next && eva >= va_next) { 5670 atomic_clear_long(l3e, htobe64(PG_W)); 5671 pmap->pm_stats.wired_count -= L3_PAGE_SIZE / 5672 PAGE_SIZE; 5673 continue; 5674 } else if (!pmap_demote_l3e(pmap, l3e, sva)) 5675 panic("pmap_unwire: demotion failed"); 5676 } 5677 if (va_next > eva) 5678 va_next = eva; 5679 for (pte = pmap_l3e_to_pte(l3e, sva); sva != va_next; pte++, 5680 sva += PAGE_SIZE) { 5681 MPASS(pte == pmap_pte(pmap, sva)); 5682 if ((be64toh(*pte) & PG_V) == 0) 5683 continue; 5684 if ((be64toh(*pte) & PG_W) == 0) 5685 panic("pmap_unwire: pte %#jx is missing PG_W", 5686 (uintmax_t)(be64toh(*pte))); 5687 5688 /* 5689 * PG_W must be cleared atomically. Although the pmap 5690 * lock synchronizes access to PG_W, another processor 5691 * could be setting PG_M and/or PG_A concurrently. 5692 */ 5693 atomic_clear_long(pte, htobe64(PG_W)); 5694 pmap->pm_stats.wired_count--; 5695 } 5696 } 5697 PMAP_UNLOCK(pmap); 5698 } 5699 5700 void 5701 mmu_radix_zero_page(vm_page_t m) 5702 { 5703 vm_offset_t addr; 5704 5705 CTR2(KTR_PMAP, "%s(%p)", __func__, m); 5706 addr = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)); 5707 pagezero(addr); 5708 } 5709 5710 void 5711 mmu_radix_zero_page_area(vm_page_t m, int off, int size) 5712 { 5713 caddr_t addr; 5714 5715 CTR4(KTR_PMAP, "%s(%p, %d, %d)", __func__, m, off, size); 5716 MPASS(off + size <= PAGE_SIZE); 5717 addr = (caddr_t)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)); 5718 memset(addr + off, 0, size); 5719 } 5720 5721 static int 5722 mmu_radix_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *locked_pa) 5723 { 5724 pml3_entry_t *l3ep; 5725 pt_entry_t pte; 5726 vm_paddr_t pa; 5727 int val; 5728 5729 CTR3(KTR_PMAP, "%s(%p, %#x)", __func__, pmap, addr); 5730 PMAP_LOCK(pmap); 5731 5732 l3ep = pmap_pml3e(pmap, addr); 5733 if (l3ep != NULL && (be64toh(*l3ep) & PG_V)) { 5734 if (be64toh(*l3ep) & RPTE_LEAF) { 5735 pte = be64toh(*l3ep); 5736 /* Compute the physical address of the 4KB page. */ 5737 pa = ((be64toh(*l3ep) & PG_PS_FRAME) | (addr & L3_PAGE_MASK)) & 5738 PG_FRAME; 5739 val = MINCORE_PSIND(1); 5740 } else { 5741 /* Native endian PTE, do not pass to functions */ 5742 pte = be64toh(*pmap_l3e_to_pte(l3ep, addr)); 5743 pa = pte & PG_FRAME; 5744 val = 0; 5745 } 5746 } else { 5747 pte = 0; 5748 pa = 0; 5749 val = 0; 5750 } 5751 if ((pte & PG_V) != 0) { 5752 val |= MINCORE_INCORE; 5753 if ((pte & (PG_M | PG_RW)) == (PG_M | PG_RW)) 5754 val |= MINCORE_MODIFIED | MINCORE_MODIFIED_OTHER; 5755 if ((pte & PG_A) != 0) 5756 val |= MINCORE_REFERENCED | MINCORE_REFERENCED_OTHER; 5757 } 5758 if ((val & (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER)) != 5759 (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER) && 5760 (pte & (PG_MANAGED | PG_V)) == (PG_MANAGED | PG_V)) { 5761 *locked_pa = pa; 5762 } 5763 PMAP_UNLOCK(pmap); 5764 return (val); 5765 } 5766 5767 void 5768 mmu_radix_activate(struct thread *td) 5769 { 5770 pmap_t pmap; 5771 uint32_t curpid; 5772 5773 CTR2(KTR_PMAP, "%s(%p)", __func__, td); 5774 critical_enter(); 5775 pmap = vmspace_pmap(td->td_proc->p_vmspace); 5776 curpid = mfspr(SPR_PID); 5777 if (pmap->pm_pid > isa3_base_pid && 5778 curpid != pmap->pm_pid) { 5779 mmu_radix_pid_set(pmap); 5780 } 5781 critical_exit(); 5782 } 5783 5784 /* 5785 * Increase the starting virtual address of the given mapping if a 5786 * different alignment might result in more superpage mappings. 5787 */ 5788 void 5789 mmu_radix_align_superpage(vm_object_t object, vm_ooffset_t offset, 5790 vm_offset_t *addr, vm_size_t size) 5791 { 5792 5793 CTR5(KTR_PMAP, "%s(%p, %#x, %p, %#x)", __func__, object, offset, addr, 5794 size); 5795 vm_offset_t superpage_offset; 5796 5797 if (size < L3_PAGE_SIZE) 5798 return; 5799 if (object != NULL && (object->flags & OBJ_COLORED) != 0) 5800 offset += ptoa(object->pg_color); 5801 superpage_offset = offset & L3_PAGE_MASK; 5802 if (size - ((L3_PAGE_SIZE - superpage_offset) & L3_PAGE_MASK) < L3_PAGE_SIZE || 5803 (*addr & L3_PAGE_MASK) == superpage_offset) 5804 return; 5805 if ((*addr & L3_PAGE_MASK) < superpage_offset) 5806 *addr = (*addr & ~L3_PAGE_MASK) + superpage_offset; 5807 else 5808 *addr = ((*addr + L3_PAGE_MASK) & ~L3_PAGE_MASK) + superpage_offset; 5809 } 5810 5811 static void * 5812 mmu_radix_mapdev_attr(vm_paddr_t pa, vm_size_t size, vm_memattr_t attr) 5813 { 5814 vm_offset_t va, tmpva, ppa, offset; 5815 5816 ppa = trunc_page(pa); 5817 offset = pa & PAGE_MASK; 5818 size = roundup2(offset + size, PAGE_SIZE); 5819 if (pa < powerpc_ptob(Maxmem)) 5820 panic("bad pa: %#lx less than Maxmem %#lx\n", 5821 pa, powerpc_ptob(Maxmem)); 5822 va = kva_alloc(size); 5823 if (bootverbose) 5824 printf("%s(%#lx, %lu, %d)\n", __func__, pa, size, attr); 5825 KASSERT(size > 0, ("%s(%#lx, %lu, %d)", __func__, pa, size, attr)); 5826 5827 if (!va) 5828 panic("%s: Couldn't alloc kernel virtual memory", __func__); 5829 5830 for (tmpva = va; size > 0;) { 5831 mmu_radix_kenter_attr(tmpva, ppa, attr); 5832 size -= PAGE_SIZE; 5833 tmpva += PAGE_SIZE; 5834 ppa += PAGE_SIZE; 5835 } 5836 ptesync(); 5837 5838 return ((void *)(va + offset)); 5839 } 5840 5841 static void * 5842 mmu_radix_mapdev(vm_paddr_t pa, vm_size_t size) 5843 { 5844 5845 CTR3(KTR_PMAP, "%s(%#x, %#x)", __func__, pa, size); 5846 5847 return (mmu_radix_mapdev_attr(pa, size, VM_MEMATTR_DEFAULT)); 5848 } 5849 5850 void 5851 mmu_radix_page_set_memattr(vm_page_t m, vm_memattr_t ma) 5852 { 5853 5854 CTR3(KTR_PMAP, "%s(%p, %#x)", __func__, m, ma); 5855 m->md.mdpg_cache_attrs = ma; 5856 5857 /* 5858 * If "m" is a normal page, update its direct mapping. This update 5859 * can be relied upon to perform any cache operations that are 5860 * required for data coherence. 5861 */ 5862 if ((m->flags & PG_FICTITIOUS) == 0 && 5863 mmu_radix_change_attr(PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)), 5864 PAGE_SIZE, m->md.mdpg_cache_attrs)) 5865 panic("memory attribute change on the direct map failed"); 5866 } 5867 5868 static void 5869 mmu_radix_unmapdev(vm_offset_t va, vm_size_t size) 5870 { 5871 vm_offset_t offset; 5872 5873 CTR3(KTR_PMAP, "%s(%#x, %#x)", __func__, va, size); 5874 /* If we gave a direct map region in pmap_mapdev, do nothing */ 5875 if (va >= DMAP_MIN_ADDRESS && va < DMAP_MAX_ADDRESS) 5876 return; 5877 5878 offset = va & PAGE_MASK; 5879 size = round_page(offset + size); 5880 va = trunc_page(va); 5881 5882 if (pmap_initialized) { 5883 mmu_radix_qremove(va, atop(size)); 5884 kva_free(va, size); 5885 } 5886 } 5887 5888 static __inline void 5889 pmap_pte_attr(pt_entry_t *pte, uint64_t cache_bits, uint64_t mask) 5890 { 5891 uint64_t opte, npte; 5892 5893 /* 5894 * The cache mode bits are all in the low 32-bits of the 5895 * PTE, so we can just spin on updating the low 32-bits. 5896 */ 5897 do { 5898 opte = be64toh(*pte); 5899 npte = opte & ~mask; 5900 npte |= cache_bits; 5901 } while (npte != opte && !atomic_cmpset_long(pte, htobe64(opte), htobe64(npte))); 5902 } 5903 5904 /* 5905 * Tries to demote a 1GB page mapping. 5906 */ 5907 static boolean_t 5908 pmap_demote_l2e(pmap_t pmap, pml2_entry_t *l2e, vm_offset_t va) 5909 { 5910 pml2_entry_t oldpdpe; 5911 pml3_entry_t *firstpde, newpde, *pde; 5912 vm_paddr_t pdpgpa; 5913 vm_page_t pdpg; 5914 5915 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 5916 oldpdpe = be64toh(*l2e); 5917 KASSERT((oldpdpe & (RPTE_LEAF | PG_V)) == (RPTE_LEAF | PG_V), 5918 ("pmap_demote_pdpe: oldpdpe is missing PG_PS and/or PG_V")); 5919 pdpg = vm_page_alloc(NULL, va >> L2_PAGE_SIZE_SHIFT, 5920 VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED); 5921 if (pdpg == NULL) { 5922 CTR2(KTR_PMAP, "pmap_demote_pdpe: failure for va %#lx" 5923 " in pmap %p", va, pmap); 5924 return (FALSE); 5925 } 5926 pdpgpa = VM_PAGE_TO_PHYS(pdpg); 5927 firstpde = (pml3_entry_t *)PHYS_TO_DMAP(pdpgpa); 5928 KASSERT((oldpdpe & PG_A) != 0, 5929 ("pmap_demote_pdpe: oldpdpe is missing PG_A")); 5930 KASSERT((oldpdpe & (PG_M | PG_RW)) != PG_RW, 5931 ("pmap_demote_pdpe: oldpdpe is missing PG_M")); 5932 newpde = oldpdpe; 5933 5934 /* 5935 * Initialize the page directory page. 5936 */ 5937 for (pde = firstpde; pde < firstpde + NPDEPG; pde++) { 5938 *pde = htobe64(newpde); 5939 newpde += L3_PAGE_SIZE; 5940 } 5941 5942 /* 5943 * Demote the mapping. 5944 */ 5945 pde_store(l2e, pdpgpa); 5946 5947 /* 5948 * Flush PWC --- XXX revisit 5949 */ 5950 pmap_invalidate_all(pmap); 5951 5952 pmap_l2e_demotions++; 5953 CTR2(KTR_PMAP, "pmap_demote_pdpe: success for va %#lx" 5954 " in pmap %p", va, pmap); 5955 return (TRUE); 5956 } 5957 5958 vm_paddr_t 5959 mmu_radix_kextract(vm_offset_t va) 5960 { 5961 pml3_entry_t l3e; 5962 vm_paddr_t pa; 5963 5964 CTR2(KTR_PMAP, "%s(%#x)", __func__, va); 5965 if (va >= DMAP_MIN_ADDRESS && va < DMAP_MAX_ADDRESS) { 5966 pa = DMAP_TO_PHYS(va); 5967 } else { 5968 /* Big-endian PTE on stack */ 5969 l3e = *pmap_pml3e(kernel_pmap, va); 5970 if (be64toh(l3e) & RPTE_LEAF) { 5971 pa = (be64toh(l3e) & PG_PS_FRAME) | (va & L3_PAGE_MASK); 5972 pa |= (va & L3_PAGE_MASK); 5973 } else { 5974 /* 5975 * Beware of a concurrent promotion that changes the 5976 * PDE at this point! For example, vtopte() must not 5977 * be used to access the PTE because it would use the 5978 * new PDE. It is, however, safe to use the old PDE 5979 * because the page table page is preserved by the 5980 * promotion. 5981 */ 5982 pa = be64toh(*pmap_l3e_to_pte(&l3e, va)); 5983 pa = (pa & PG_FRAME) | (va & PAGE_MASK); 5984 pa |= (va & PAGE_MASK); 5985 } 5986 } 5987 return (pa); 5988 } 5989 5990 static pt_entry_t 5991 mmu_radix_calc_wimg(vm_paddr_t pa, vm_memattr_t ma) 5992 { 5993 5994 if (ma != VM_MEMATTR_DEFAULT) { 5995 return pmap_cache_bits(ma); 5996 } 5997 5998 /* 5999 * Assume the page is cache inhibited and access is guarded unless 6000 * it's in our available memory array. 6001 */ 6002 for (int i = 0; i < pregions_sz; i++) { 6003 if ((pa >= pregions[i].mr_start) && 6004 (pa < (pregions[i].mr_start + pregions[i].mr_size))) 6005 return (RPTE_ATTR_MEM); 6006 } 6007 return (RPTE_ATTR_GUARDEDIO); 6008 } 6009 6010 static void 6011 mmu_radix_kenter_attr(vm_offset_t va, vm_paddr_t pa, vm_memattr_t ma) 6012 { 6013 pt_entry_t *pte, pteval; 6014 uint64_t cache_bits; 6015 6016 pte = kvtopte(va); 6017 MPASS(pte != NULL); 6018 pteval = pa | RPTE_EAA_R | RPTE_EAA_W | RPTE_EAA_P | PG_M | PG_A; 6019 cache_bits = mmu_radix_calc_wimg(pa, ma); 6020 pte_store(pte, pteval | cache_bits); 6021 } 6022 6023 void 6024 mmu_radix_kremove(vm_offset_t va) 6025 { 6026 pt_entry_t *pte; 6027 6028 CTR2(KTR_PMAP, "%s(%#x)", __func__, va); 6029 6030 pte = kvtopte(va); 6031 pte_clear(pte); 6032 } 6033 6034 int 6035 mmu_radix_decode_kernel_ptr(vm_offset_t addr, 6036 int *is_user, vm_offset_t *decoded) 6037 { 6038 6039 CTR2(KTR_PMAP, "%s(%#jx)", __func__, (uintmax_t)addr); 6040 *decoded = addr; 6041 *is_user = (addr < VM_MAXUSER_ADDRESS); 6042 return (0); 6043 } 6044 6045 static boolean_t 6046 mmu_radix_dev_direct_mapped(vm_paddr_t pa, vm_size_t size) 6047 { 6048 6049 CTR3(KTR_PMAP, "%s(%#x, %#x)", __func__, pa, size); 6050 return (mem_valid(pa, size)); 6051 } 6052 6053 static void 6054 mmu_radix_scan_init() 6055 { 6056 6057 CTR1(KTR_PMAP, "%s()", __func__); 6058 UNIMPLEMENTED(); 6059 } 6060 6061 static void 6062 mmu_radix_dumpsys_map(vm_paddr_t pa, size_t sz, 6063 void **va) 6064 { 6065 CTR4(KTR_PMAP, "%s(%#jx, %#zx, %p)", __func__, (uintmax_t)pa, sz, va); 6066 UNIMPLEMENTED(); 6067 } 6068 6069 vm_offset_t 6070 mmu_radix_quick_enter_page(vm_page_t m) 6071 { 6072 vm_paddr_t paddr; 6073 6074 CTR2(KTR_PMAP, "%s(%p)", __func__, m); 6075 paddr = VM_PAGE_TO_PHYS(m); 6076 return (PHYS_TO_DMAP(paddr)); 6077 } 6078 6079 void 6080 mmu_radix_quick_remove_page(vm_offset_t addr __unused) 6081 { 6082 /* no work to do here */ 6083 CTR2(KTR_PMAP, "%s(%#x)", __func__, addr); 6084 } 6085 6086 static void 6087 pmap_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva) 6088 { 6089 cpu_flush_dcache((void *)sva, eva - sva); 6090 } 6091 6092 int 6093 mmu_radix_change_attr(vm_offset_t va, vm_size_t size, 6094 vm_memattr_t mode) 6095 { 6096 int error; 6097 6098 CTR4(KTR_PMAP, "%s(%#x, %#zx, %d)", __func__, va, size, mode); 6099 PMAP_LOCK(kernel_pmap); 6100 error = pmap_change_attr_locked(va, size, mode, true); 6101 PMAP_UNLOCK(kernel_pmap); 6102 return (error); 6103 } 6104 6105 static int 6106 pmap_change_attr_locked(vm_offset_t va, vm_size_t size, int mode, bool flush) 6107 { 6108 vm_offset_t base, offset, tmpva; 6109 vm_paddr_t pa_start, pa_end, pa_end1; 6110 pml2_entry_t *l2e; 6111 pml3_entry_t *l3e; 6112 pt_entry_t *pte; 6113 int cache_bits, error; 6114 boolean_t changed; 6115 6116 PMAP_LOCK_ASSERT(kernel_pmap, MA_OWNED); 6117 base = trunc_page(va); 6118 offset = va & PAGE_MASK; 6119 size = round_page(offset + size); 6120 6121 /* 6122 * Only supported on kernel virtual addresses, including the direct 6123 * map but excluding the recursive map. 6124 */ 6125 if (base < DMAP_MIN_ADDRESS) 6126 return (EINVAL); 6127 6128 cache_bits = pmap_cache_bits(mode); 6129 changed = FALSE; 6130 6131 /* 6132 * Pages that aren't mapped aren't supported. Also break down 2MB pages 6133 * into 4KB pages if required. 6134 */ 6135 for (tmpva = base; tmpva < base + size; ) { 6136 l2e = pmap_pml2e(kernel_pmap, tmpva); 6137 if (l2e == NULL || *l2e == 0) 6138 return (EINVAL); 6139 if (be64toh(*l2e) & RPTE_LEAF) { 6140 /* 6141 * If the current 1GB page already has the required 6142 * memory type, then we need not demote this page. Just 6143 * increment tmpva to the next 1GB page frame. 6144 */ 6145 if ((be64toh(*l2e) & RPTE_ATTR_MASK) == cache_bits) { 6146 tmpva = trunc_1gpage(tmpva) + L2_PAGE_SIZE; 6147 continue; 6148 } 6149 6150 /* 6151 * If the current offset aligns with a 1GB page frame 6152 * and there is at least 1GB left within the range, then 6153 * we need not break down this page into 2MB pages. 6154 */ 6155 if ((tmpva & L2_PAGE_MASK) == 0 && 6156 tmpva + L2_PAGE_MASK < base + size) { 6157 tmpva += L2_PAGE_MASK; 6158 continue; 6159 } 6160 if (!pmap_demote_l2e(kernel_pmap, l2e, tmpva)) 6161 return (ENOMEM); 6162 } 6163 l3e = pmap_l2e_to_l3e(l2e, tmpva); 6164 KASSERT(l3e != NULL, ("no l3e entry for %#lx in %p\n", 6165 tmpva, l2e)); 6166 if (*l3e == 0) 6167 return (EINVAL); 6168 if (be64toh(*l3e) & RPTE_LEAF) { 6169 /* 6170 * If the current 2MB page already has the required 6171 * memory type, then we need not demote this page. Just 6172 * increment tmpva to the next 2MB page frame. 6173 */ 6174 if ((be64toh(*l3e) & RPTE_ATTR_MASK) == cache_bits) { 6175 tmpva = trunc_2mpage(tmpva) + L3_PAGE_SIZE; 6176 continue; 6177 } 6178 6179 /* 6180 * If the current offset aligns with a 2MB page frame 6181 * and there is at least 2MB left within the range, then 6182 * we need not break down this page into 4KB pages. 6183 */ 6184 if ((tmpva & L3_PAGE_MASK) == 0 && 6185 tmpva + L3_PAGE_MASK < base + size) { 6186 tmpva += L3_PAGE_SIZE; 6187 continue; 6188 } 6189 if (!pmap_demote_l3e(kernel_pmap, l3e, tmpva)) 6190 return (ENOMEM); 6191 } 6192 pte = pmap_l3e_to_pte(l3e, tmpva); 6193 if (*pte == 0) 6194 return (EINVAL); 6195 tmpva += PAGE_SIZE; 6196 } 6197 error = 0; 6198 6199 /* 6200 * Ok, all the pages exist, so run through them updating their 6201 * cache mode if required. 6202 */ 6203 pa_start = pa_end = 0; 6204 for (tmpva = base; tmpva < base + size; ) { 6205 l2e = pmap_pml2e(kernel_pmap, tmpva); 6206 if (be64toh(*l2e) & RPTE_LEAF) { 6207 if ((be64toh(*l2e) & RPTE_ATTR_MASK) != cache_bits) { 6208 pmap_pte_attr(l2e, cache_bits, 6209 RPTE_ATTR_MASK); 6210 changed = TRUE; 6211 } 6212 if (tmpva >= VM_MIN_KERNEL_ADDRESS && 6213 (*l2e & PG_PS_FRAME) < dmaplimit) { 6214 if (pa_start == pa_end) { 6215 /* Start physical address run. */ 6216 pa_start = be64toh(*l2e) & PG_PS_FRAME; 6217 pa_end = pa_start + L2_PAGE_SIZE; 6218 } else if (pa_end == (be64toh(*l2e) & PG_PS_FRAME)) 6219 pa_end += L2_PAGE_SIZE; 6220 else { 6221 /* Run ended, update direct map. */ 6222 error = pmap_change_attr_locked( 6223 PHYS_TO_DMAP(pa_start), 6224 pa_end - pa_start, mode, flush); 6225 if (error != 0) 6226 break; 6227 /* Start physical address run. */ 6228 pa_start = be64toh(*l2e) & PG_PS_FRAME; 6229 pa_end = pa_start + L2_PAGE_SIZE; 6230 } 6231 } 6232 tmpva = trunc_1gpage(tmpva) + L2_PAGE_SIZE; 6233 continue; 6234 } 6235 l3e = pmap_l2e_to_l3e(l2e, tmpva); 6236 if (be64toh(*l3e) & RPTE_LEAF) { 6237 if ((be64toh(*l3e) & RPTE_ATTR_MASK) != cache_bits) { 6238 pmap_pte_attr(l3e, cache_bits, 6239 RPTE_ATTR_MASK); 6240 changed = TRUE; 6241 } 6242 if (tmpva >= VM_MIN_KERNEL_ADDRESS && 6243 (be64toh(*l3e) & PG_PS_FRAME) < dmaplimit) { 6244 if (pa_start == pa_end) { 6245 /* Start physical address run. */ 6246 pa_start = be64toh(*l3e) & PG_PS_FRAME; 6247 pa_end = pa_start + L3_PAGE_SIZE; 6248 } else if (pa_end == (be64toh(*l3e) & PG_PS_FRAME)) 6249 pa_end += L3_PAGE_SIZE; 6250 else { 6251 /* Run ended, update direct map. */ 6252 error = pmap_change_attr_locked( 6253 PHYS_TO_DMAP(pa_start), 6254 pa_end - pa_start, mode, flush); 6255 if (error != 0) 6256 break; 6257 /* Start physical address run. */ 6258 pa_start = be64toh(*l3e) & PG_PS_FRAME; 6259 pa_end = pa_start + L3_PAGE_SIZE; 6260 } 6261 } 6262 tmpva = trunc_2mpage(tmpva) + L3_PAGE_SIZE; 6263 } else { 6264 pte = pmap_l3e_to_pte(l3e, tmpva); 6265 if ((be64toh(*pte) & RPTE_ATTR_MASK) != cache_bits) { 6266 pmap_pte_attr(pte, cache_bits, 6267 RPTE_ATTR_MASK); 6268 changed = TRUE; 6269 } 6270 if (tmpva >= VM_MIN_KERNEL_ADDRESS && 6271 (be64toh(*pte) & PG_FRAME) < dmaplimit) { 6272 if (pa_start == pa_end) { 6273 /* Start physical address run. */ 6274 pa_start = be64toh(*pte) & PG_FRAME; 6275 pa_end = pa_start + PAGE_SIZE; 6276 } else if (pa_end == (be64toh(*pte) & PG_FRAME)) 6277 pa_end += PAGE_SIZE; 6278 else { 6279 /* Run ended, update direct map. */ 6280 error = pmap_change_attr_locked( 6281 PHYS_TO_DMAP(pa_start), 6282 pa_end - pa_start, mode, flush); 6283 if (error != 0) 6284 break; 6285 /* Start physical address run. */ 6286 pa_start = be64toh(*pte) & PG_FRAME; 6287 pa_end = pa_start + PAGE_SIZE; 6288 } 6289 } 6290 tmpva += PAGE_SIZE; 6291 } 6292 } 6293 if (error == 0 && pa_start != pa_end && pa_start < dmaplimit) { 6294 pa_end1 = MIN(pa_end, dmaplimit); 6295 if (pa_start != pa_end1) 6296 error = pmap_change_attr_locked(PHYS_TO_DMAP(pa_start), 6297 pa_end1 - pa_start, mode, flush); 6298 } 6299 6300 /* 6301 * Flush CPU caches if required to make sure any data isn't cached that 6302 * shouldn't be, etc. 6303 */ 6304 if (changed) { 6305 pmap_invalidate_all(kernel_pmap); 6306 6307 if (flush) 6308 pmap_invalidate_cache_range(base, tmpva); 6309 } 6310 return (error); 6311 } 6312 6313 /* 6314 * Allocate physical memory for the vm_page array and map it into KVA, 6315 * attempting to back the vm_pages with domain-local memory. 6316 */ 6317 void 6318 mmu_radix_page_array_startup(long pages) 6319 { 6320 #ifdef notyet 6321 pml2_entry_t *l2e; 6322 pml3_entry_t *pde; 6323 pml3_entry_t newl3; 6324 vm_offset_t va; 6325 long pfn; 6326 int domain, i; 6327 #endif 6328 vm_paddr_t pa; 6329 vm_offset_t start, end; 6330 6331 vm_page_array_size = pages; 6332 6333 start = VM_MIN_KERNEL_ADDRESS; 6334 end = start + pages * sizeof(struct vm_page); 6335 6336 pa = vm_phys_early_alloc(0, end - start); 6337 6338 start = mmu_radix_map(&start, pa, end - start, VM_MEMATTR_DEFAULT); 6339 #ifdef notyet 6340 /* TODO: NUMA vm_page_array. Blocked out until then (copied from amd64). */ 6341 for (va = start; va < end; va += L3_PAGE_SIZE) { 6342 pfn = first_page + (va - start) / sizeof(struct vm_page); 6343 domain = vm_phys_domain(ptoa(pfn)); 6344 l2e = pmap_pml2e(kernel_pmap, va); 6345 if ((be64toh(*l2e) & PG_V) == 0) { 6346 pa = vm_phys_early_alloc(domain, PAGE_SIZE); 6347 dump_add_page(pa); 6348 pagezero(PHYS_TO_DMAP(pa)); 6349 pde_store(l2e, (pml2_entry_t)pa); 6350 } 6351 pde = pmap_l2e_to_l3e(l2e, va); 6352 if ((be64toh(*pde) & PG_V) != 0) 6353 panic("Unexpected pde %p", pde); 6354 pa = vm_phys_early_alloc(domain, L3_PAGE_SIZE); 6355 for (i = 0; i < NPDEPG; i++) 6356 dump_add_page(pa + i * PAGE_SIZE); 6357 newl3 = (pml3_entry_t)(pa | RPTE_EAA_P | RPTE_EAA_R | RPTE_EAA_W); 6358 pte_store(pde, newl3); 6359 } 6360 #endif 6361 vm_page_array = (vm_page_t)start; 6362 } 6363 6364 #ifdef DDB 6365 #include <sys/kdb.h> 6366 #include <ddb/ddb.h> 6367 6368 static void 6369 pmap_pte_walk(pml1_entry_t *l1, vm_offset_t va) 6370 { 6371 pml1_entry_t *l1e; 6372 pml2_entry_t *l2e; 6373 pml3_entry_t *l3e; 6374 pt_entry_t *pte; 6375 6376 l1e = &l1[pmap_pml1e_index(va)]; 6377 db_printf("VA %#016lx l1e %#016lx", va, be64toh(*l1e)); 6378 if ((be64toh(*l1e) & PG_V) == 0) { 6379 db_printf("\n"); 6380 return; 6381 } 6382 l2e = pmap_l1e_to_l2e(l1e, va); 6383 db_printf(" l2e %#016lx", be64toh(*l2e)); 6384 if ((be64toh(*l2e) & PG_V) == 0 || (be64toh(*l2e) & RPTE_LEAF) != 0) { 6385 db_printf("\n"); 6386 return; 6387 } 6388 l3e = pmap_l2e_to_l3e(l2e, va); 6389 db_printf(" l3e %#016lx", be64toh(*l3e)); 6390 if ((be64toh(*l3e) & PG_V) == 0 || (be64toh(*l3e) & RPTE_LEAF) != 0) { 6391 db_printf("\n"); 6392 return; 6393 } 6394 pte = pmap_l3e_to_pte(l3e, va); 6395 db_printf(" pte %#016lx\n", be64toh(*pte)); 6396 } 6397 6398 void 6399 pmap_page_print_mappings(vm_page_t m) 6400 { 6401 pmap_t pmap; 6402 pv_entry_t pv; 6403 6404 db_printf("page %p(%lx)\n", m, m->phys_addr); 6405 /* need to elide locks if running in ddb */ 6406 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) { 6407 db_printf("pv: %p ", pv); 6408 db_printf("va: %#016lx ", pv->pv_va); 6409 pmap = PV_PMAP(pv); 6410 db_printf("pmap %p ", pmap); 6411 if (pmap != NULL) { 6412 db_printf("asid: %lu\n", pmap->pm_pid); 6413 pmap_pte_walk(pmap->pm_pml1, pv->pv_va); 6414 } 6415 } 6416 } 6417 6418 DB_SHOW_COMMAND(pte, pmap_print_pte) 6419 { 6420 vm_offset_t va; 6421 pmap_t pmap; 6422 6423 if (!have_addr) { 6424 db_printf("show pte addr\n"); 6425 return; 6426 } 6427 va = (vm_offset_t)addr; 6428 6429 if (va >= DMAP_MIN_ADDRESS) 6430 pmap = kernel_pmap; 6431 else if (kdb_thread != NULL) 6432 pmap = vmspace_pmap(kdb_thread->td_proc->p_vmspace); 6433 else 6434 pmap = vmspace_pmap(curthread->td_proc->p_vmspace); 6435 6436 pmap_pte_walk(pmap->pm_pml1, va); 6437 } 6438 6439 #endif 6440