1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2018 Matthew Macy 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __FBSDID("$FreeBSD$"); 30 31 #include <sys/param.h> 32 #include <sys/kernel.h> 33 #include <sys/systm.h> 34 #include <sys/conf.h> 35 #include <sys/bitstring.h> 36 #include <sys/queue.h> 37 #include <sys/cpuset.h> 38 #include <sys/endian.h> 39 #include <sys/kerneldump.h> 40 #include <sys/ktr.h> 41 #include <sys/lock.h> 42 #include <sys/syslog.h> 43 #include <sys/msgbuf.h> 44 #include <sys/malloc.h> 45 #include <sys/mman.h> 46 #include <sys/mutex.h> 47 #include <sys/proc.h> 48 #include <sys/rwlock.h> 49 #include <sys/sched.h> 50 #include <sys/sysctl.h> 51 #include <sys/systm.h> 52 #include <sys/vmem.h> 53 #include <sys/vmmeter.h> 54 #include <sys/smp.h> 55 56 #include <sys/kdb.h> 57 58 #include <dev/ofw/openfirm.h> 59 60 #include <vm/vm.h> 61 #include <vm/pmap.h> 62 #include <vm/vm_param.h> 63 #include <vm/vm_kern.h> 64 #include <vm/vm_page.h> 65 #include <vm/vm_map.h> 66 #include <vm/vm_object.h> 67 #include <vm/vm_extern.h> 68 #include <vm/vm_pageout.h> 69 #include <vm/vm_phys.h> 70 #include <vm/vm_reserv.h> 71 #include <vm/vm_dumpset.h> 72 #include <vm/uma.h> 73 74 #include <machine/_inttypes.h> 75 #include <machine/cpu.h> 76 #include <machine/platform.h> 77 #include <machine/frame.h> 78 #include <machine/md_var.h> 79 #include <machine/psl.h> 80 #include <machine/bat.h> 81 #include <machine/hid.h> 82 #include <machine/pte.h> 83 #include <machine/sr.h> 84 #include <machine/trap.h> 85 #include <machine/mmuvar.h> 86 87 #ifdef INVARIANTS 88 #include <vm/uma_dbg.h> 89 #endif 90 91 #define PPC_BITLSHIFT(bit) (sizeof(long)*NBBY - 1 - (bit)) 92 #define PPC_BIT(bit) (1UL << PPC_BITLSHIFT(bit)) 93 #define PPC_BITLSHIFT_VAL(val, bit) ((val) << PPC_BITLSHIFT(bit)) 94 95 #include "opt_ddb.h" 96 #ifdef DDB 97 static void pmap_pte_walk(pml1_entry_t *l1, vm_offset_t va); 98 #endif 99 100 #define PG_W RPTE_WIRED 101 #define PG_V RPTE_VALID 102 #define PG_MANAGED RPTE_MANAGED 103 #define PG_PROMOTED RPTE_PROMOTED 104 #define PG_M RPTE_C 105 #define PG_A RPTE_R 106 #define PG_X RPTE_EAA_X 107 #define PG_RW RPTE_EAA_W 108 #define PG_PTE_CACHE RPTE_ATTR_MASK 109 110 #define RPTE_SHIFT 9 111 #define NLS_MASK ((1UL<<5)-1) 112 #define RPTE_ENTRIES (1UL<<RPTE_SHIFT) 113 #define RPTE_MASK (RPTE_ENTRIES-1) 114 115 #define NLB_SHIFT 0 116 #define NLB_MASK (((1UL<<52)-1) << 8) 117 118 extern int nkpt; 119 extern caddr_t crashdumpmap; 120 121 #define RIC_FLUSH_TLB 0 122 #define RIC_FLUSH_PWC 1 123 #define RIC_FLUSH_ALL 2 124 125 #define POWER9_TLB_SETS_RADIX 128 /* # sets in POWER9 TLB Radix mode */ 126 127 #define PPC_INST_TLBIE 0x7c000264 128 #define PPC_INST_TLBIEL 0x7c000224 129 #define PPC_INST_SLBIA 0x7c0003e4 130 131 #define ___PPC_RA(a) (((a) & 0x1f) << 16) 132 #define ___PPC_RB(b) (((b) & 0x1f) << 11) 133 #define ___PPC_RS(s) (((s) & 0x1f) << 21) 134 #define ___PPC_RT(t) ___PPC_RS(t) 135 #define ___PPC_R(r) (((r) & 0x1) << 16) 136 #define ___PPC_PRS(prs) (((prs) & 0x1) << 17) 137 #define ___PPC_RIC(ric) (((ric) & 0x3) << 18) 138 139 #define PPC_SLBIA(IH) __XSTRING(.long PPC_INST_SLBIA | \ 140 ((IH & 0x7) << 21)) 141 #define PPC_TLBIE_5(rb,rs,ric,prs,r) \ 142 __XSTRING(.long PPC_INST_TLBIE | \ 143 ___PPC_RB(rb) | ___PPC_RS(rs) | \ 144 ___PPC_RIC(ric) | ___PPC_PRS(prs) | \ 145 ___PPC_R(r)) 146 147 #define PPC_TLBIEL(rb,rs,ric,prs,r) \ 148 __XSTRING(.long PPC_INST_TLBIEL | \ 149 ___PPC_RB(rb) | ___PPC_RS(rs) | \ 150 ___PPC_RIC(ric) | ___PPC_PRS(prs) | \ 151 ___PPC_R(r)) 152 153 #define PPC_INVALIDATE_ERAT PPC_SLBIA(7) 154 155 static __inline void 156 ttusync(void) 157 { 158 __asm __volatile("eieio; tlbsync; ptesync" ::: "memory"); 159 } 160 161 #define TLBIEL_INVAL_SEL_MASK 0xc00 /* invalidation selector */ 162 #define TLBIEL_INVAL_PAGE 0x000 /* invalidate a single page */ 163 #define TLBIEL_INVAL_SET_PID 0x400 /* invalidate a set for the current PID */ 164 #define TLBIEL_INVAL_SET_LPID 0x800 /* invalidate a set for current LPID */ 165 #define TLBIEL_INVAL_SET 0xc00 /* invalidate a set for all LPIDs */ 166 167 #define TLBIE_ACTUAL_PAGE_MASK 0xe0 168 #define TLBIE_ACTUAL_PAGE_4K 0x00 169 #define TLBIE_ACTUAL_PAGE_64K 0xa0 170 #define TLBIE_ACTUAL_PAGE_2M 0x20 171 #define TLBIE_ACTUAL_PAGE_1G 0x40 172 173 #define TLBIE_PRS_PARTITION_SCOPE 0x0 174 #define TLBIE_PRS_PROCESS_SCOPE 0x1 175 176 #define TLBIE_RIC_INVALIDATE_TLB 0x0 /* Invalidate just TLB */ 177 #define TLBIE_RIC_INVALIDATE_PWC 0x1 /* Invalidate just PWC */ 178 #define TLBIE_RIC_INVALIDATE_ALL 0x2 /* Invalidate TLB, PWC, 179 * cached {proc, part}tab entries 180 */ 181 #define TLBIE_RIC_INVALIDATE_SEQ 0x3 /* HPT - only: 182 * Invalidate a range of translations 183 */ 184 185 static __always_inline void 186 radix_tlbie(uint8_t ric, uint8_t prs, uint16_t is, uint32_t pid, uint32_t lpid, 187 vm_offset_t va, uint16_t ap) 188 { 189 uint64_t rb, rs; 190 191 MPASS((va & PAGE_MASK) == 0); 192 193 rs = ((uint64_t)pid << 32) | lpid; 194 rb = va | is | ap; 195 __asm __volatile(PPC_TLBIE_5(%0, %1, %2, %3, 1) : : 196 "r" (rb), "r" (rs), "i" (ric), "i" (prs)); 197 } 198 199 static __inline void 200 radix_tlbie_invlpg_user_4k(uint32_t pid, vm_offset_t va) 201 { 202 203 radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE, 204 TLBIEL_INVAL_PAGE, pid, 0, va, TLBIE_ACTUAL_PAGE_4K); 205 } 206 207 static __inline void 208 radix_tlbie_invlpg_user_2m(uint32_t pid, vm_offset_t va) 209 { 210 211 radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE, 212 TLBIEL_INVAL_PAGE, pid, 0, va, TLBIE_ACTUAL_PAGE_2M); 213 } 214 215 static __inline void 216 radix_tlbie_invlpwc_user(uint32_t pid) 217 { 218 219 radix_tlbie(TLBIE_RIC_INVALIDATE_PWC, TLBIE_PRS_PROCESS_SCOPE, 220 TLBIEL_INVAL_SET_PID, pid, 0, 0, 0); 221 } 222 223 static __inline void 224 radix_tlbie_flush_user(uint32_t pid) 225 { 226 227 radix_tlbie(TLBIE_RIC_INVALIDATE_ALL, TLBIE_PRS_PROCESS_SCOPE, 228 TLBIEL_INVAL_SET_PID, pid, 0, 0, 0); 229 } 230 231 static __inline void 232 radix_tlbie_invlpg_kernel_4k(vm_offset_t va) 233 { 234 235 radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE, 236 TLBIEL_INVAL_PAGE, 0, 0, va, TLBIE_ACTUAL_PAGE_4K); 237 } 238 239 static __inline void 240 radix_tlbie_invlpg_kernel_2m(vm_offset_t va) 241 { 242 243 radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE, 244 TLBIEL_INVAL_PAGE, 0, 0, va, TLBIE_ACTUAL_PAGE_2M); 245 } 246 247 /* 1GB pages aren't currently supported. */ 248 static __inline __unused void 249 radix_tlbie_invlpg_kernel_1g(vm_offset_t va) 250 { 251 252 radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE, 253 TLBIEL_INVAL_PAGE, 0, 0, va, TLBIE_ACTUAL_PAGE_1G); 254 } 255 256 static __inline void 257 radix_tlbie_invlpwc_kernel(void) 258 { 259 260 radix_tlbie(TLBIE_RIC_INVALIDATE_PWC, TLBIE_PRS_PROCESS_SCOPE, 261 TLBIEL_INVAL_SET_LPID, 0, 0, 0, 0); 262 } 263 264 static __inline void 265 radix_tlbie_flush_kernel(void) 266 { 267 268 radix_tlbie(TLBIE_RIC_INVALIDATE_ALL, TLBIE_PRS_PROCESS_SCOPE, 269 TLBIEL_INVAL_SET_LPID, 0, 0, 0, 0); 270 } 271 272 static __inline vm_pindex_t 273 pmap_l3e_pindex(vm_offset_t va) 274 { 275 return ((va & PG_FRAME) >> L3_PAGE_SIZE_SHIFT); 276 } 277 278 static __inline vm_pindex_t 279 pmap_pml3e_index(vm_offset_t va) 280 { 281 282 return ((va >> L3_PAGE_SIZE_SHIFT) & RPTE_MASK); 283 } 284 285 static __inline vm_pindex_t 286 pmap_pml2e_index(vm_offset_t va) 287 { 288 return ((va >> L2_PAGE_SIZE_SHIFT) & RPTE_MASK); 289 } 290 291 static __inline vm_pindex_t 292 pmap_pml1e_index(vm_offset_t va) 293 { 294 return ((va & PG_FRAME) >> L1_PAGE_SIZE_SHIFT); 295 } 296 297 /* Return various clipped indexes for a given VA */ 298 static __inline vm_pindex_t 299 pmap_pte_index(vm_offset_t va) 300 { 301 302 return ((va >> PAGE_SHIFT) & RPTE_MASK); 303 } 304 305 /* Return a pointer to the PT slot that corresponds to a VA */ 306 static __inline pt_entry_t * 307 pmap_l3e_to_pte(pt_entry_t *l3e, vm_offset_t va) 308 { 309 pt_entry_t *pte; 310 vm_paddr_t ptepa; 311 312 ptepa = (be64toh(*l3e) & NLB_MASK); 313 pte = (pt_entry_t *)PHYS_TO_DMAP(ptepa); 314 return (&pte[pmap_pte_index(va)]); 315 } 316 317 /* Return a pointer to the PD slot that corresponds to a VA */ 318 static __inline pt_entry_t * 319 pmap_l2e_to_l3e(pt_entry_t *l2e, vm_offset_t va) 320 { 321 pt_entry_t *l3e; 322 vm_paddr_t l3pa; 323 324 l3pa = (be64toh(*l2e) & NLB_MASK); 325 l3e = (pml3_entry_t *)PHYS_TO_DMAP(l3pa); 326 return (&l3e[pmap_pml3e_index(va)]); 327 } 328 329 /* Return a pointer to the PD slot that corresponds to a VA */ 330 static __inline pt_entry_t * 331 pmap_l1e_to_l2e(pt_entry_t *l1e, vm_offset_t va) 332 { 333 pt_entry_t *l2e; 334 vm_paddr_t l2pa; 335 336 l2pa = (be64toh(*l1e) & NLB_MASK); 337 338 l2e = (pml2_entry_t *)PHYS_TO_DMAP(l2pa); 339 return (&l2e[pmap_pml2e_index(va)]); 340 } 341 342 static __inline pml1_entry_t * 343 pmap_pml1e(pmap_t pmap, vm_offset_t va) 344 { 345 346 return (&pmap->pm_pml1[pmap_pml1e_index(va)]); 347 } 348 349 static pt_entry_t * 350 pmap_pml2e(pmap_t pmap, vm_offset_t va) 351 { 352 pt_entry_t *l1e; 353 354 l1e = pmap_pml1e(pmap, va); 355 if (l1e == NULL || (be64toh(*l1e) & RPTE_VALID) == 0) 356 return (NULL); 357 return (pmap_l1e_to_l2e(l1e, va)); 358 } 359 360 static __inline pt_entry_t * 361 pmap_pml3e(pmap_t pmap, vm_offset_t va) 362 { 363 pt_entry_t *l2e; 364 365 l2e = pmap_pml2e(pmap, va); 366 if (l2e == NULL || (be64toh(*l2e) & RPTE_VALID) == 0) 367 return (NULL); 368 return (pmap_l2e_to_l3e(l2e, va)); 369 } 370 371 static __inline pt_entry_t * 372 pmap_pte(pmap_t pmap, vm_offset_t va) 373 { 374 pt_entry_t *l3e; 375 376 l3e = pmap_pml3e(pmap, va); 377 if (l3e == NULL || (be64toh(*l3e) & RPTE_VALID) == 0) 378 return (NULL); 379 return (pmap_l3e_to_pte(l3e, va)); 380 } 381 382 int nkpt = 64; 383 SYSCTL_INT(_machdep, OID_AUTO, nkpt, CTLFLAG_RD, &nkpt, 0, 384 "Number of kernel page table pages allocated on bootup"); 385 386 vm_paddr_t dmaplimit; 387 388 SYSCTL_NODE(_vm, OID_AUTO, pmap, CTLFLAG_RD, 0, "VM/pmap parameters"); 389 390 static int pg_ps_enabled = 1; 391 SYSCTL_INT(_vm_pmap, OID_AUTO, pg_ps_enabled, CTLFLAG_RDTUN | CTLFLAG_NOFETCH, 392 &pg_ps_enabled, 0, "Are large page mappings enabled?"); 393 #ifdef INVARIANTS 394 #define VERBOSE_PMAP 0 395 #define VERBOSE_PROTECT 0 396 static int pmap_logging; 397 SYSCTL_INT(_vm_pmap, OID_AUTO, pmap_logging, CTLFLAG_RWTUN, 398 &pmap_logging, 0, "verbose debug logging"); 399 #endif 400 401 static u_int64_t KPTphys; /* phys addr of kernel level 1 */ 402 403 //static vm_paddr_t KERNend; /* phys addr of end of bootstrap data */ 404 405 static vm_offset_t qframe = 0; 406 static struct mtx qframe_mtx; 407 408 void mmu_radix_activate(struct thread *); 409 void mmu_radix_advise(pmap_t, vm_offset_t, vm_offset_t, int); 410 void mmu_radix_align_superpage(vm_object_t, vm_ooffset_t, vm_offset_t *, 411 vm_size_t); 412 void mmu_radix_clear_modify(vm_page_t); 413 void mmu_radix_copy(pmap_t, pmap_t, vm_offset_t, vm_size_t, vm_offset_t); 414 int mmu_radix_decode_kernel_ptr(vm_offset_t, int *, vm_offset_t *); 415 int mmu_radix_enter(pmap_t, vm_offset_t, vm_page_t, vm_prot_t, u_int, int8_t); 416 void mmu_radix_enter_object(pmap_t, vm_offset_t, vm_offset_t, vm_page_t, 417 vm_prot_t); 418 void mmu_radix_enter_quick(pmap_t, vm_offset_t, vm_page_t, vm_prot_t); 419 vm_paddr_t mmu_radix_extract(pmap_t pmap, vm_offset_t va); 420 vm_page_t mmu_radix_extract_and_hold(pmap_t, vm_offset_t, vm_prot_t); 421 void mmu_radix_kenter(vm_offset_t, vm_paddr_t); 422 vm_paddr_t mmu_radix_kextract(vm_offset_t); 423 void mmu_radix_kremove(vm_offset_t); 424 boolean_t mmu_radix_is_modified(vm_page_t); 425 boolean_t mmu_radix_is_prefaultable(pmap_t, vm_offset_t); 426 boolean_t mmu_radix_is_referenced(vm_page_t); 427 void mmu_radix_object_init_pt(pmap_t, vm_offset_t, vm_object_t, 428 vm_pindex_t, vm_size_t); 429 boolean_t mmu_radix_page_exists_quick(pmap_t, vm_page_t); 430 void mmu_radix_page_init(vm_page_t); 431 boolean_t mmu_radix_page_is_mapped(vm_page_t m); 432 void mmu_radix_page_set_memattr(vm_page_t, vm_memattr_t); 433 int mmu_radix_page_wired_mappings(vm_page_t); 434 int mmu_radix_pinit(pmap_t); 435 void mmu_radix_protect(pmap_t, vm_offset_t, vm_offset_t, vm_prot_t); 436 bool mmu_radix_ps_enabled(pmap_t); 437 void mmu_radix_qenter(vm_offset_t, vm_page_t *, int); 438 void mmu_radix_qremove(vm_offset_t, int); 439 vm_offset_t mmu_radix_quick_enter_page(vm_page_t); 440 void mmu_radix_quick_remove_page(vm_offset_t); 441 boolean_t mmu_radix_ts_referenced(vm_page_t); 442 void mmu_radix_release(pmap_t); 443 void mmu_radix_remove(pmap_t, vm_offset_t, vm_offset_t); 444 void mmu_radix_remove_all(vm_page_t); 445 void mmu_radix_remove_pages(pmap_t); 446 void mmu_radix_remove_write(vm_page_t); 447 void mmu_radix_unwire(pmap_t, vm_offset_t, vm_offset_t); 448 void mmu_radix_zero_page(vm_page_t); 449 void mmu_radix_zero_page_area(vm_page_t, int, int); 450 int mmu_radix_change_attr(vm_offset_t, vm_size_t, vm_memattr_t); 451 void mmu_radix_page_array_startup(long pages); 452 453 #include "mmu_oea64.h" 454 455 /* 456 * Kernel MMU interface 457 */ 458 459 static void mmu_radix_bootstrap(vm_offset_t, vm_offset_t); 460 461 static void mmu_radix_copy_page(vm_page_t, vm_page_t); 462 static void mmu_radix_copy_pages(vm_page_t *ma, vm_offset_t a_offset, 463 vm_page_t *mb, vm_offset_t b_offset, int xfersize); 464 static void mmu_radix_growkernel(vm_offset_t); 465 static void mmu_radix_init(void); 466 static int mmu_radix_mincore(pmap_t, vm_offset_t, vm_paddr_t *); 467 static vm_offset_t mmu_radix_map(vm_offset_t *, vm_paddr_t, vm_paddr_t, int); 468 static void mmu_radix_pinit0(pmap_t); 469 470 static void *mmu_radix_mapdev(vm_paddr_t, vm_size_t); 471 static void *mmu_radix_mapdev_attr(vm_paddr_t, vm_size_t, vm_memattr_t); 472 static void mmu_radix_unmapdev(vm_offset_t, vm_size_t); 473 static void mmu_radix_kenter_attr(vm_offset_t, vm_paddr_t, vm_memattr_t ma); 474 static boolean_t mmu_radix_dev_direct_mapped(vm_paddr_t, vm_size_t); 475 static void mmu_radix_dumpsys_map(vm_paddr_t pa, size_t sz, void **va); 476 static void mmu_radix_scan_init(void); 477 static void mmu_radix_cpu_bootstrap(int ap); 478 static void mmu_radix_tlbie_all(void); 479 480 static struct pmap_funcs mmu_radix_methods = { 481 .bootstrap = mmu_radix_bootstrap, 482 .copy_page = mmu_radix_copy_page, 483 .copy_pages = mmu_radix_copy_pages, 484 .cpu_bootstrap = mmu_radix_cpu_bootstrap, 485 .growkernel = mmu_radix_growkernel, 486 .init = mmu_radix_init, 487 .map = mmu_radix_map, 488 .mincore = mmu_radix_mincore, 489 .pinit = mmu_radix_pinit, 490 .pinit0 = mmu_radix_pinit0, 491 492 .mapdev = mmu_radix_mapdev, 493 .mapdev_attr = mmu_radix_mapdev_attr, 494 .unmapdev = mmu_radix_unmapdev, 495 .kenter_attr = mmu_radix_kenter_attr, 496 .dev_direct_mapped = mmu_radix_dev_direct_mapped, 497 .dumpsys_pa_init = mmu_radix_scan_init, 498 .dumpsys_map_chunk = mmu_radix_dumpsys_map, 499 .page_is_mapped = mmu_radix_page_is_mapped, 500 .ps_enabled = mmu_radix_ps_enabled, 501 .object_init_pt = mmu_radix_object_init_pt, 502 .protect = mmu_radix_protect, 503 /* pmap dispatcher interface */ 504 .clear_modify = mmu_radix_clear_modify, 505 .copy = mmu_radix_copy, 506 .enter = mmu_radix_enter, 507 .enter_object = mmu_radix_enter_object, 508 .enter_quick = mmu_radix_enter_quick, 509 .extract = mmu_radix_extract, 510 .extract_and_hold = mmu_radix_extract_and_hold, 511 .is_modified = mmu_radix_is_modified, 512 .is_prefaultable = mmu_radix_is_prefaultable, 513 .is_referenced = mmu_radix_is_referenced, 514 .ts_referenced = mmu_radix_ts_referenced, 515 .page_exists_quick = mmu_radix_page_exists_quick, 516 .page_init = mmu_radix_page_init, 517 .page_wired_mappings = mmu_radix_page_wired_mappings, 518 .qenter = mmu_radix_qenter, 519 .qremove = mmu_radix_qremove, 520 .release = mmu_radix_release, 521 .remove = mmu_radix_remove, 522 .remove_all = mmu_radix_remove_all, 523 .remove_write = mmu_radix_remove_write, 524 .unwire = mmu_radix_unwire, 525 .zero_page = mmu_radix_zero_page, 526 .zero_page_area = mmu_radix_zero_page_area, 527 .activate = mmu_radix_activate, 528 .quick_enter_page = mmu_radix_quick_enter_page, 529 .quick_remove_page = mmu_radix_quick_remove_page, 530 .page_set_memattr = mmu_radix_page_set_memattr, 531 .page_array_startup = mmu_radix_page_array_startup, 532 533 /* Internal interfaces */ 534 .kenter = mmu_radix_kenter, 535 .kextract = mmu_radix_kextract, 536 .kremove = mmu_radix_kremove, 537 .change_attr = mmu_radix_change_attr, 538 .decode_kernel_ptr = mmu_radix_decode_kernel_ptr, 539 540 .tlbie_all = mmu_radix_tlbie_all, 541 }; 542 543 MMU_DEF(mmu_radix, MMU_TYPE_RADIX, mmu_radix_methods); 544 545 static boolean_t pmap_demote_l3e_locked(pmap_t pmap, pml3_entry_t *l3e, vm_offset_t va, 546 struct rwlock **lockp); 547 static boolean_t pmap_demote_l3e(pmap_t pmap, pml3_entry_t *pde, vm_offset_t va); 548 static int pmap_unuse_pt(pmap_t, vm_offset_t, pml3_entry_t, struct spglist *); 549 static int pmap_remove_l3e(pmap_t pmap, pml3_entry_t *pdq, vm_offset_t sva, 550 struct spglist *free, struct rwlock **lockp); 551 static int pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t sva, 552 pml3_entry_t ptepde, struct spglist *free, struct rwlock **lockp); 553 static vm_page_t pmap_remove_pt_page(pmap_t pmap, vm_offset_t va); 554 static bool pmap_remove_page(pmap_t pmap, vm_offset_t va, pml3_entry_t *pde, 555 struct spglist *free); 556 static bool pmap_remove_ptes(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, 557 pml3_entry_t *l3e, struct spglist *free, struct rwlock **lockp); 558 559 static bool pmap_pv_insert_l3e(pmap_t pmap, vm_offset_t va, pml3_entry_t l3e, 560 u_int flags, struct rwlock **lockp); 561 #if VM_NRESERVLEVEL > 0 562 static void pmap_pv_promote_l3e(pmap_t pmap, vm_offset_t va, vm_paddr_t pa, 563 struct rwlock **lockp); 564 #endif 565 static void pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va); 566 static int pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte); 567 static vm_page_t mmu_radix_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, 568 vm_prot_t prot, vm_page_t mpte, struct rwlock **lockp, bool *invalidate); 569 570 static bool pmap_enter_2mpage(pmap_t pmap, vm_offset_t va, vm_page_t m, 571 vm_prot_t prot, struct rwlock **lockp); 572 static int pmap_enter_l3e(pmap_t pmap, vm_offset_t va, pml3_entry_t newpde, 573 u_int flags, vm_page_t m, struct rwlock **lockp); 574 575 static vm_page_t reclaim_pv_chunk(pmap_t locked_pmap, struct rwlock **lockp); 576 static void free_pv_chunk(struct pv_chunk *pc); 577 static vm_page_t _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, struct rwlock **lockp); 578 static vm_page_t pmap_allocl3e(pmap_t pmap, vm_offset_t va, 579 struct rwlock **lockp); 580 static vm_page_t pmap_allocpte(pmap_t pmap, vm_offset_t va, 581 struct rwlock **lockp); 582 static void _pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, 583 struct spglist *free); 584 static boolean_t pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free); 585 586 static void pmap_invalidate_page(pmap_t pmap, vm_offset_t start); 587 static void pmap_invalidate_all(pmap_t pmap); 588 static int pmap_change_attr_locked(vm_offset_t va, vm_size_t size, int mode, bool flush); 589 590 /* 591 * Internal flags for pmap_enter()'s helper functions. 592 */ 593 #define PMAP_ENTER_NORECLAIM 0x1000000 /* Don't reclaim PV entries. */ 594 #define PMAP_ENTER_NOREPLACE 0x2000000 /* Don't replace mappings. */ 595 596 #define UNIMPLEMENTED() panic("%s not implemented", __func__) 597 #define UNTESTED() panic("%s not yet tested", __func__) 598 599 /* Number of supported PID bits */ 600 static unsigned int isa3_pid_bits; 601 602 /* PID to start allocating from */ 603 static unsigned int isa3_base_pid; 604 605 #define PROCTAB_SIZE_SHIFT (isa3_pid_bits + 4) 606 #define PROCTAB_ENTRIES (1ul << isa3_pid_bits) 607 608 /* 609 * Map of physical memory regions. 610 */ 611 static struct mem_region *regions, *pregions; 612 static struct numa_mem_region *numa_pregions; 613 static u_int phys_avail_count; 614 static int regions_sz, pregions_sz, numa_pregions_sz; 615 static struct pate *isa3_parttab; 616 static struct prte *isa3_proctab; 617 static vmem_t *asid_arena; 618 619 extern void bs_remap_earlyboot(void); 620 621 #define RADIX_PGD_SIZE_SHIFT 16 622 #define RADIX_PGD_SIZE (1UL << RADIX_PGD_SIZE_SHIFT) 623 624 #define RADIX_PGD_INDEX_SHIFT (RADIX_PGD_SIZE_SHIFT-3) 625 #define NL2EPG (PAGE_SIZE/sizeof(pml2_entry_t)) 626 #define NL3EPG (PAGE_SIZE/sizeof(pml3_entry_t)) 627 628 #define NUPML1E (RADIX_PGD_SIZE/sizeof(uint64_t)) /* number of userland PML1 pages */ 629 #define NUPDPE (NUPML1E * NL2EPG)/* number of userland PDP pages */ 630 #define NUPDE (NUPDPE * NL3EPG) /* number of userland PD entries */ 631 632 /* POWER9 only permits a 64k partition table size. */ 633 #define PARTTAB_SIZE_SHIFT 16 634 #define PARTTAB_SIZE (1UL << PARTTAB_SIZE_SHIFT) 635 636 #define PARTTAB_HR (1UL << 63) /* host uses radix */ 637 #define PARTTAB_GR (1UL << 63) /* guest uses radix must match host */ 638 639 /* TLB flush actions. Used as argument to tlbiel_all() */ 640 enum { 641 TLB_INVAL_SCOPE_LPID = 0, /* invalidate TLBs for current LPID */ 642 TLB_INVAL_SCOPE_GLOBAL = 1, /* invalidate all TLBs */ 643 }; 644 645 #define NPV_LIST_LOCKS MAXCPU 646 static int pmap_initialized; 647 static vm_paddr_t proctab0pa; 648 static vm_paddr_t parttab_phys; 649 CTASSERT(sizeof(struct pv_chunk) == PAGE_SIZE); 650 651 /* 652 * Data for the pv entry allocation mechanism. 653 * Updates to pv_invl_gen are protected by the pv_list_locks[] 654 * elements, but reads are not. 655 */ 656 static TAILQ_HEAD(pch, pv_chunk) pv_chunks = TAILQ_HEAD_INITIALIZER(pv_chunks); 657 static struct mtx __exclusive_cache_line pv_chunks_mutex; 658 static struct rwlock __exclusive_cache_line pv_list_locks[NPV_LIST_LOCKS]; 659 static struct md_page *pv_table; 660 static struct md_page pv_dummy; 661 662 #ifdef PV_STATS 663 #define PV_STAT(x) do { x ; } while (0) 664 #else 665 #define PV_STAT(x) do { } while (0) 666 #endif 667 668 #define pa_radix_index(pa) ((pa) >> L3_PAGE_SIZE_SHIFT) 669 #define pa_to_pvh(pa) (&pv_table[pa_radix_index(pa)]) 670 671 #define PHYS_TO_PV_LIST_LOCK(pa) \ 672 (&pv_list_locks[pa_radix_index(pa) % NPV_LIST_LOCKS]) 673 674 #define CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa) do { \ 675 struct rwlock **_lockp = (lockp); \ 676 struct rwlock *_new_lock; \ 677 \ 678 _new_lock = PHYS_TO_PV_LIST_LOCK(pa); \ 679 if (_new_lock != *_lockp) { \ 680 if (*_lockp != NULL) \ 681 rw_wunlock(*_lockp); \ 682 *_lockp = _new_lock; \ 683 rw_wlock(*_lockp); \ 684 } \ 685 } while (0) 686 687 #define CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m) \ 688 CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, VM_PAGE_TO_PHYS(m)) 689 690 #define RELEASE_PV_LIST_LOCK(lockp) do { \ 691 struct rwlock **_lockp = (lockp); \ 692 \ 693 if (*_lockp != NULL) { \ 694 rw_wunlock(*_lockp); \ 695 *_lockp = NULL; \ 696 } \ 697 } while (0) 698 699 #define VM_PAGE_TO_PV_LIST_LOCK(m) \ 700 PHYS_TO_PV_LIST_LOCK(VM_PAGE_TO_PHYS(m)) 701 702 /* 703 * We support 52 bits, hence: 704 * bits 52 - 31 = 21, 0b10101 705 * RTS encoding details 706 * bits 0 - 3 of rts -> bits 6 - 8 unsigned long 707 * bits 4 - 5 of rts -> bits 62 - 63 of unsigned long 708 */ 709 #define RTS_SIZE ((0x2UL << 61) | (0x5UL << 5)) 710 711 static int powernv_enabled = 1; 712 713 static __always_inline void 714 tlbiel_radix_set_isa300(uint32_t set, uint32_t is, 715 uint32_t pid, uint32_t ric, uint32_t prs) 716 { 717 uint64_t rb; 718 uint64_t rs; 719 720 rb = PPC_BITLSHIFT_VAL(set, 51) | PPC_BITLSHIFT_VAL(is, 53); 721 rs = PPC_BITLSHIFT_VAL((uint64_t)pid, 31); 722 723 __asm __volatile(PPC_TLBIEL(%0, %1, %2, %3, 1) 724 : : "r"(rb), "r"(rs), "i"(ric), "i"(prs) 725 : "memory"); 726 } 727 728 static void 729 tlbiel_flush_isa3(uint32_t num_sets, uint32_t is) 730 { 731 uint32_t set; 732 733 __asm __volatile("ptesync": : :"memory"); 734 735 /* 736 * Flush the first set of the TLB, and the entire Page Walk Cache 737 * and partition table entries. Then flush the remaining sets of the 738 * TLB. 739 */ 740 tlbiel_radix_set_isa300(0, is, 0, RIC_FLUSH_ALL, 0); 741 for (set = 1; set < num_sets; set++) 742 tlbiel_radix_set_isa300(set, is, 0, RIC_FLUSH_TLB, 0); 743 744 /* Do the same for process scoped entries. */ 745 tlbiel_radix_set_isa300(0, is, 0, RIC_FLUSH_ALL, 1); 746 for (set = 1; set < num_sets; set++) 747 tlbiel_radix_set_isa300(set, is, 0, RIC_FLUSH_TLB, 1); 748 749 __asm __volatile("ptesync": : :"memory"); 750 } 751 752 static void 753 mmu_radix_tlbiel_flush(int scope) 754 { 755 int is; 756 757 MPASS(scope == TLB_INVAL_SCOPE_LPID || 758 scope == TLB_INVAL_SCOPE_GLOBAL); 759 is = scope + 2; 760 761 tlbiel_flush_isa3(POWER9_TLB_SETS_RADIX, is); 762 __asm __volatile(PPC_INVALIDATE_ERAT "; isync" : : :"memory"); 763 } 764 765 static void 766 mmu_radix_tlbie_all() 767 { 768 /* TODO: LPID invalidate */ 769 mmu_radix_tlbiel_flush(TLB_INVAL_SCOPE_GLOBAL); 770 } 771 772 static void 773 mmu_radix_init_amor(void) 774 { 775 /* 776 * In HV mode, we init AMOR (Authority Mask Override Register) so that 777 * the hypervisor and guest can setup IAMR (Instruction Authority Mask 778 * Register), enable key 0 and set it to 1. 779 * 780 * AMOR = 0b1100 .... 0000 (Mask for key 0 is 11) 781 */ 782 mtspr(SPR_AMOR, (3ul << 62)); 783 } 784 785 static void 786 mmu_radix_init_iamr(void) 787 { 788 /* 789 * Radix always uses key0 of the IAMR to determine if an access is 790 * allowed. We set bit 0 (IBM bit 1) of key0, to prevent instruction 791 * fetch. 792 */ 793 mtspr(SPR_IAMR, (1ul << 62)); 794 } 795 796 static void 797 mmu_radix_pid_set(pmap_t pmap) 798 { 799 800 mtspr(SPR_PID, pmap->pm_pid); 801 isync(); 802 } 803 804 /* Quick sort callout for comparing physical addresses. */ 805 static int 806 pa_cmp(const void *a, const void *b) 807 { 808 const vm_paddr_t *pa = a, *pb = b; 809 810 if (*pa < *pb) 811 return (-1); 812 else if (*pa > *pb) 813 return (1); 814 else 815 return (0); 816 } 817 818 #define pte_load_store(ptep, pte) atomic_swap_long(ptep, pte) 819 #define pte_load_clear(ptep) atomic_swap_long(ptep, 0) 820 #define pte_store(ptep, pte) do { \ 821 MPASS((pte) & (RPTE_EAA_R | RPTE_EAA_W | RPTE_EAA_X)); \ 822 *(u_long *)(ptep) = htobe64((u_long)((pte) | PG_V | RPTE_LEAF)); \ 823 } while (0) 824 /* 825 * NB: should only be used for adding directories - not for direct mappings 826 */ 827 #define pde_store(ptep, pa) do { \ 828 *(u_long *)(ptep) = htobe64((u_long)(pa|RPTE_VALID|RPTE_SHIFT)); \ 829 } while (0) 830 831 #define pte_clear(ptep) do { \ 832 *(u_long *)(ptep) = (u_long)(0); \ 833 } while (0) 834 835 #define PMAP_PDE_SUPERPAGE (1 << 8) /* supports 2MB superpages */ 836 837 /* 838 * Promotion to a 2MB (PDE) page mapping requires that the corresponding 4KB 839 * (PTE) page mappings have identical settings for the following fields: 840 */ 841 #define PG_PTE_PROMOTE (PG_X | PG_MANAGED | PG_W | PG_PTE_CACHE | \ 842 PG_M | PG_A | RPTE_EAA_MASK | PG_V) 843 844 static __inline void 845 pmap_resident_count_inc(pmap_t pmap, int count) 846 { 847 848 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 849 pmap->pm_stats.resident_count += count; 850 } 851 852 static __inline void 853 pmap_resident_count_dec(pmap_t pmap, int count) 854 { 855 856 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 857 KASSERT(pmap->pm_stats.resident_count >= count, 858 ("pmap %p resident count underflow %ld %d", pmap, 859 pmap->pm_stats.resident_count, count)); 860 pmap->pm_stats.resident_count -= count; 861 } 862 863 static void 864 pagezero(vm_offset_t va) 865 { 866 va = trunc_page(va); 867 868 bzero((void *)va, PAGE_SIZE); 869 } 870 871 static uint64_t 872 allocpages(int n) 873 { 874 u_int64_t ret; 875 876 ret = moea64_bootstrap_alloc(n * PAGE_SIZE, PAGE_SIZE); 877 for (int i = 0; i < n; i++) 878 pagezero(PHYS_TO_DMAP(ret + i * PAGE_SIZE)); 879 return (ret); 880 } 881 882 static pt_entry_t * 883 kvtopte(vm_offset_t va) 884 { 885 pt_entry_t *l3e; 886 887 l3e = pmap_pml3e(kernel_pmap, va); 888 if ((be64toh(*l3e) & RPTE_VALID) == 0) 889 return (NULL); 890 return (pmap_l3e_to_pte(l3e, va)); 891 } 892 893 void 894 mmu_radix_kenter(vm_offset_t va, vm_paddr_t pa) 895 { 896 pt_entry_t *pte; 897 898 pte = kvtopte(va); 899 MPASS(pte != NULL); 900 *pte = htobe64(pa | RPTE_VALID | RPTE_LEAF | RPTE_EAA_R | \ 901 RPTE_EAA_W | RPTE_EAA_P | PG_M | PG_A); 902 } 903 904 bool 905 mmu_radix_ps_enabled(pmap_t pmap) 906 { 907 return (pg_ps_enabled && (pmap->pm_flags & PMAP_PDE_SUPERPAGE) != 0); 908 } 909 910 static pt_entry_t * 911 pmap_nofault_pte(pmap_t pmap, vm_offset_t va, int *is_l3e) 912 { 913 pml3_entry_t *l3e; 914 pt_entry_t *pte; 915 916 va &= PG_PS_FRAME; 917 l3e = pmap_pml3e(pmap, va); 918 if (l3e == NULL || (be64toh(*l3e) & PG_V) == 0) 919 return (NULL); 920 921 if (be64toh(*l3e) & RPTE_LEAF) { 922 *is_l3e = 1; 923 return (l3e); 924 } 925 *is_l3e = 0; 926 va &= PG_FRAME; 927 pte = pmap_l3e_to_pte(l3e, va); 928 if (pte == NULL || (be64toh(*pte) & PG_V) == 0) 929 return (NULL); 930 return (pte); 931 } 932 933 int 934 pmap_nofault(pmap_t pmap, vm_offset_t va, vm_prot_t flags) 935 { 936 pt_entry_t *pte; 937 pt_entry_t startpte, origpte, newpte; 938 vm_page_t m; 939 int is_l3e; 940 941 startpte = 0; 942 retry: 943 if ((pte = pmap_nofault_pte(pmap, va, &is_l3e)) == NULL) 944 return (KERN_INVALID_ADDRESS); 945 origpte = newpte = be64toh(*pte); 946 if (startpte == 0) { 947 startpte = origpte; 948 if (((flags & VM_PROT_WRITE) && (startpte & PG_M)) || 949 ((flags & VM_PROT_READ) && (startpte & PG_A))) { 950 pmap_invalidate_all(pmap); 951 #ifdef INVARIANTS 952 if (VERBOSE_PMAP || pmap_logging) 953 printf("%s(%p, %#lx, %#x) (%#lx) -- invalidate all\n", 954 __func__, pmap, va, flags, origpte); 955 #endif 956 return (KERN_FAILURE); 957 } 958 } 959 #ifdef INVARIANTS 960 if (VERBOSE_PMAP || pmap_logging) 961 printf("%s(%p, %#lx, %#x) (%#lx)\n", __func__, pmap, va, 962 flags, origpte); 963 #endif 964 PMAP_LOCK(pmap); 965 if ((pte = pmap_nofault_pte(pmap, va, &is_l3e)) == NULL || 966 be64toh(*pte) != origpte) { 967 PMAP_UNLOCK(pmap); 968 return (KERN_FAILURE); 969 } 970 m = PHYS_TO_VM_PAGE(newpte & PG_FRAME); 971 MPASS(m != NULL); 972 switch (flags) { 973 case VM_PROT_READ: 974 if ((newpte & (RPTE_EAA_R|RPTE_EAA_X)) == 0) 975 goto protfail; 976 newpte |= PG_A; 977 vm_page_aflag_set(m, PGA_REFERENCED); 978 break; 979 case VM_PROT_WRITE: 980 if ((newpte & RPTE_EAA_W) == 0) 981 goto protfail; 982 if (is_l3e) 983 goto protfail; 984 newpte |= PG_M; 985 vm_page_dirty(m); 986 break; 987 case VM_PROT_EXECUTE: 988 if ((newpte & RPTE_EAA_X) == 0) 989 goto protfail; 990 newpte |= PG_A; 991 vm_page_aflag_set(m, PGA_REFERENCED); 992 break; 993 } 994 995 if (!atomic_cmpset_long(pte, htobe64(origpte), htobe64(newpte))) 996 goto retry; 997 ptesync(); 998 PMAP_UNLOCK(pmap); 999 if (startpte == newpte) 1000 return (KERN_FAILURE); 1001 return (0); 1002 protfail: 1003 PMAP_UNLOCK(pmap); 1004 return (KERN_PROTECTION_FAILURE); 1005 } 1006 1007 /* 1008 * Returns TRUE if the given page is mapped individually or as part of 1009 * a 2mpage. Otherwise, returns FALSE. 1010 */ 1011 boolean_t 1012 mmu_radix_page_is_mapped(vm_page_t m) 1013 { 1014 struct rwlock *lock; 1015 boolean_t rv; 1016 1017 if ((m->oflags & VPO_UNMANAGED) != 0) 1018 return (FALSE); 1019 lock = VM_PAGE_TO_PV_LIST_LOCK(m); 1020 rw_rlock(lock); 1021 rv = !TAILQ_EMPTY(&m->md.pv_list) || 1022 ((m->flags & PG_FICTITIOUS) == 0 && 1023 !TAILQ_EMPTY(&pa_to_pvh(VM_PAGE_TO_PHYS(m))->pv_list)); 1024 rw_runlock(lock); 1025 return (rv); 1026 } 1027 1028 /* 1029 * Determine the appropriate bits to set in a PTE or PDE for a specified 1030 * caching mode. 1031 */ 1032 static int 1033 pmap_cache_bits(vm_memattr_t ma) 1034 { 1035 if (ma != VM_MEMATTR_DEFAULT) { 1036 switch (ma) { 1037 case VM_MEMATTR_UNCACHEABLE: 1038 return (RPTE_ATTR_GUARDEDIO); 1039 case VM_MEMATTR_CACHEABLE: 1040 return (RPTE_ATTR_MEM); 1041 case VM_MEMATTR_WRITE_BACK: 1042 case VM_MEMATTR_PREFETCHABLE: 1043 case VM_MEMATTR_WRITE_COMBINING: 1044 return (RPTE_ATTR_UNGUARDEDIO); 1045 } 1046 } 1047 return (0); 1048 } 1049 1050 static void 1051 pmap_invalidate_page(pmap_t pmap, vm_offset_t start) 1052 { 1053 ptesync(); 1054 if (pmap == kernel_pmap) 1055 radix_tlbie_invlpg_kernel_4k(start); 1056 else 1057 radix_tlbie_invlpg_user_4k(pmap->pm_pid, start); 1058 ttusync(); 1059 } 1060 1061 static void 1062 pmap_invalidate_page_2m(pmap_t pmap, vm_offset_t start) 1063 { 1064 ptesync(); 1065 if (pmap == kernel_pmap) 1066 radix_tlbie_invlpg_kernel_2m(start); 1067 else 1068 radix_tlbie_invlpg_user_2m(pmap->pm_pid, start); 1069 ttusync(); 1070 } 1071 1072 static void 1073 pmap_invalidate_pwc(pmap_t pmap) 1074 { 1075 ptesync(); 1076 if (pmap == kernel_pmap) 1077 radix_tlbie_invlpwc_kernel(); 1078 else 1079 radix_tlbie_invlpwc_user(pmap->pm_pid); 1080 ttusync(); 1081 } 1082 1083 static void 1084 pmap_invalidate_range(pmap_t pmap, vm_offset_t start, vm_offset_t end) 1085 { 1086 if (((start - end) >> PAGE_SHIFT) > 8) { 1087 pmap_invalidate_all(pmap); 1088 return; 1089 } 1090 ptesync(); 1091 if (pmap == kernel_pmap) { 1092 while (start < end) { 1093 radix_tlbie_invlpg_kernel_4k(start); 1094 start += PAGE_SIZE; 1095 } 1096 } else { 1097 while (start < end) { 1098 radix_tlbie_invlpg_user_4k(pmap->pm_pid, start); 1099 start += PAGE_SIZE; 1100 } 1101 } 1102 ttusync(); 1103 } 1104 1105 static void 1106 pmap_invalidate_all(pmap_t pmap) 1107 { 1108 ptesync(); 1109 if (pmap == kernel_pmap) 1110 radix_tlbie_flush_kernel(); 1111 else 1112 radix_tlbie_flush_user(pmap->pm_pid); 1113 ttusync(); 1114 } 1115 1116 static void 1117 pmap_invalidate_l3e_page(pmap_t pmap, vm_offset_t va, pml3_entry_t l3e) 1118 { 1119 1120 /* 1121 * When the PDE has PG_PROMOTED set, the 2MB page mapping was created 1122 * by a promotion that did not invalidate the 512 4KB page mappings 1123 * that might exist in the TLB. Consequently, at this point, the TLB 1124 * may hold both 4KB and 2MB page mappings for the address range [va, 1125 * va + L3_PAGE_SIZE). Therefore, the entire range must be invalidated here. 1126 * In contrast, when PG_PROMOTED is clear, the TLB will not hold any 1127 * 4KB page mappings for the address range [va, va + L3_PAGE_SIZE), and so a 1128 * single INVLPG suffices to invalidate the 2MB page mapping from the 1129 * TLB. 1130 */ 1131 ptesync(); 1132 if ((l3e & PG_PROMOTED) != 0) 1133 pmap_invalidate_range(pmap, va, va + L3_PAGE_SIZE - 1); 1134 else 1135 pmap_invalidate_page_2m(pmap, va); 1136 1137 pmap_invalidate_pwc(pmap); 1138 } 1139 1140 static __inline struct pv_chunk * 1141 pv_to_chunk(pv_entry_t pv) 1142 { 1143 1144 return ((struct pv_chunk *)((uintptr_t)pv & ~(uintptr_t)PAGE_MASK)); 1145 } 1146 1147 #define PV_PMAP(pv) (pv_to_chunk(pv)->pc_pmap) 1148 1149 #define PC_FREE0 0xfffffffffffffffful 1150 #define PC_FREE1 0x3ffffffffffffffful 1151 1152 static const uint64_t pc_freemask[_NPCM] = { PC_FREE0, PC_FREE1 }; 1153 1154 /* 1155 * Ensure that the number of spare PV entries in the specified pmap meets or 1156 * exceeds the given count, "needed". 1157 * 1158 * The given PV list lock may be released. 1159 */ 1160 static void 1161 reserve_pv_entries(pmap_t pmap, int needed, struct rwlock **lockp) 1162 { 1163 struct pch new_tail; 1164 struct pv_chunk *pc; 1165 vm_page_t m; 1166 int avail, free; 1167 bool reclaimed; 1168 1169 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1170 KASSERT(lockp != NULL, ("reserve_pv_entries: lockp is NULL")); 1171 1172 /* 1173 * Newly allocated PV chunks must be stored in a private list until 1174 * the required number of PV chunks have been allocated. Otherwise, 1175 * reclaim_pv_chunk() could recycle one of these chunks. In 1176 * contrast, these chunks must be added to the pmap upon allocation. 1177 */ 1178 TAILQ_INIT(&new_tail); 1179 retry: 1180 avail = 0; 1181 TAILQ_FOREACH(pc, &pmap->pm_pvchunk, pc_list) { 1182 // if ((cpu_feature2 & CPUID2_POPCNT) == 0) 1183 bit_count((bitstr_t *)pc->pc_map, 0, 1184 sizeof(pc->pc_map) * NBBY, &free); 1185 #if 0 1186 free = popcnt_pc_map_pq(pc->pc_map); 1187 #endif 1188 if (free == 0) 1189 break; 1190 avail += free; 1191 if (avail >= needed) 1192 break; 1193 } 1194 for (reclaimed = false; avail < needed; avail += _NPCPV) { 1195 m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | 1196 VM_ALLOC_WIRED); 1197 if (m == NULL) { 1198 m = reclaim_pv_chunk(pmap, lockp); 1199 if (m == NULL) 1200 goto retry; 1201 reclaimed = true; 1202 } 1203 PV_STAT(atomic_add_int(&pc_chunk_count, 1)); 1204 PV_STAT(atomic_add_int(&pc_chunk_allocs, 1)); 1205 pc = (void *)PHYS_TO_DMAP(m->phys_addr); 1206 pc->pc_pmap = pmap; 1207 pc->pc_map[0] = PC_FREE0; 1208 pc->pc_map[1] = PC_FREE1; 1209 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list); 1210 TAILQ_INSERT_TAIL(&new_tail, pc, pc_lru); 1211 PV_STAT(atomic_add_int(&pv_entry_spare, _NPCPV)); 1212 1213 /* 1214 * The reclaim might have freed a chunk from the current pmap. 1215 * If that chunk contained available entries, we need to 1216 * re-count the number of available entries. 1217 */ 1218 if (reclaimed) 1219 goto retry; 1220 } 1221 if (!TAILQ_EMPTY(&new_tail)) { 1222 mtx_lock(&pv_chunks_mutex); 1223 TAILQ_CONCAT(&pv_chunks, &new_tail, pc_lru); 1224 mtx_unlock(&pv_chunks_mutex); 1225 } 1226 } 1227 1228 /* 1229 * First find and then remove the pv entry for the specified pmap and virtual 1230 * address from the specified pv list. Returns the pv entry if found and NULL 1231 * otherwise. This operation can be performed on pv lists for either 4KB or 1232 * 2MB page mappings. 1233 */ 1234 static __inline pv_entry_t 1235 pmap_pvh_remove(struct md_page *pvh, pmap_t pmap, vm_offset_t va) 1236 { 1237 pv_entry_t pv; 1238 1239 TAILQ_FOREACH(pv, &pvh->pv_list, pv_link) { 1240 #ifdef INVARIANTS 1241 if (PV_PMAP(pv) == NULL) { 1242 printf("corrupted pv_chunk/pv %p\n", pv); 1243 printf("pv_chunk: %64D\n", pv_to_chunk(pv), ":"); 1244 } 1245 MPASS(PV_PMAP(pv) != NULL); 1246 MPASS(pv->pv_va != 0); 1247 #endif 1248 if (pmap == PV_PMAP(pv) && va == pv->pv_va) { 1249 TAILQ_REMOVE(&pvh->pv_list, pv, pv_link); 1250 pvh->pv_gen++; 1251 break; 1252 } 1253 } 1254 return (pv); 1255 } 1256 1257 /* 1258 * After demotion from a 2MB page mapping to 512 4KB page mappings, 1259 * destroy the pv entry for the 2MB page mapping and reinstantiate the pv 1260 * entries for each of the 4KB page mappings. 1261 */ 1262 static void 1263 pmap_pv_demote_l3e(pmap_t pmap, vm_offset_t va, vm_paddr_t pa, 1264 struct rwlock **lockp) 1265 { 1266 struct md_page *pvh; 1267 struct pv_chunk *pc; 1268 pv_entry_t pv; 1269 vm_offset_t va_last; 1270 vm_page_t m; 1271 int bit, field; 1272 1273 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1274 KASSERT((pa & L3_PAGE_MASK) == 0, 1275 ("pmap_pv_demote_pde: pa is not 2mpage aligned")); 1276 CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa); 1277 1278 /* 1279 * Transfer the 2mpage's pv entry for this mapping to the first 1280 * page's pv list. Once this transfer begins, the pv list lock 1281 * must not be released until the last pv entry is reinstantiated. 1282 */ 1283 pvh = pa_to_pvh(pa); 1284 va = trunc_2mpage(va); 1285 pv = pmap_pvh_remove(pvh, pmap, va); 1286 KASSERT(pv != NULL, ("pmap_pv_demote_pde: pv not found")); 1287 m = PHYS_TO_VM_PAGE(pa); 1288 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_link); 1289 1290 m->md.pv_gen++; 1291 /* Instantiate the remaining NPTEPG - 1 pv entries. */ 1292 PV_STAT(atomic_add_long(&pv_entry_allocs, NPTEPG - 1)); 1293 va_last = va + L3_PAGE_SIZE - PAGE_SIZE; 1294 for (;;) { 1295 pc = TAILQ_FIRST(&pmap->pm_pvchunk); 1296 KASSERT(pc->pc_map[0] != 0 || pc->pc_map[1] != 0 1297 , ("pmap_pv_demote_pde: missing spare")); 1298 for (field = 0; field < _NPCM; field++) { 1299 while (pc->pc_map[field]) { 1300 bit = cnttzd(pc->pc_map[field]); 1301 pc->pc_map[field] &= ~(1ul << bit); 1302 pv = &pc->pc_pventry[field * 64 + bit]; 1303 va += PAGE_SIZE; 1304 pv->pv_va = va; 1305 m++; 1306 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 1307 ("pmap_pv_demote_pde: page %p is not managed", m)); 1308 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_link); 1309 1310 m->md.pv_gen++; 1311 if (va == va_last) 1312 goto out; 1313 } 1314 } 1315 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 1316 TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list); 1317 } 1318 out: 1319 if (pc->pc_map[0] == 0 && pc->pc_map[1] == 0) { 1320 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 1321 TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list); 1322 } 1323 PV_STAT(atomic_add_long(&pv_entry_count, NPTEPG - 1)); 1324 PV_STAT(atomic_subtract_int(&pv_entry_spare, NPTEPG - 1)); 1325 } 1326 1327 static void 1328 reclaim_pv_chunk_leave_pmap(pmap_t pmap, pmap_t locked_pmap) 1329 { 1330 1331 if (pmap == NULL) 1332 return; 1333 pmap_invalidate_all(pmap); 1334 if (pmap != locked_pmap) 1335 PMAP_UNLOCK(pmap); 1336 } 1337 1338 /* 1339 * We are in a serious low memory condition. Resort to 1340 * drastic measures to free some pages so we can allocate 1341 * another pv entry chunk. 1342 * 1343 * Returns NULL if PV entries were reclaimed from the specified pmap. 1344 * 1345 * We do not, however, unmap 2mpages because subsequent accesses will 1346 * allocate per-page pv entries until repromotion occurs, thereby 1347 * exacerbating the shortage of free pv entries. 1348 */ 1349 static int active_reclaims = 0; 1350 static vm_page_t 1351 reclaim_pv_chunk(pmap_t locked_pmap, struct rwlock **lockp) 1352 { 1353 struct pv_chunk *pc, *pc_marker, *pc_marker_end; 1354 struct pv_chunk_header pc_marker_b, pc_marker_end_b; 1355 struct md_page *pvh; 1356 pml3_entry_t *l3e; 1357 pmap_t next_pmap, pmap; 1358 pt_entry_t *pte, tpte; 1359 pv_entry_t pv; 1360 vm_offset_t va; 1361 vm_page_t m, m_pc; 1362 struct spglist free; 1363 uint64_t inuse; 1364 int bit, field, freed; 1365 1366 PMAP_LOCK_ASSERT(locked_pmap, MA_OWNED); 1367 KASSERT(lockp != NULL, ("reclaim_pv_chunk: lockp is NULL")); 1368 pmap = NULL; 1369 m_pc = NULL; 1370 SLIST_INIT(&free); 1371 bzero(&pc_marker_b, sizeof(pc_marker_b)); 1372 bzero(&pc_marker_end_b, sizeof(pc_marker_end_b)); 1373 pc_marker = (struct pv_chunk *)&pc_marker_b; 1374 pc_marker_end = (struct pv_chunk *)&pc_marker_end_b; 1375 1376 mtx_lock(&pv_chunks_mutex); 1377 active_reclaims++; 1378 TAILQ_INSERT_HEAD(&pv_chunks, pc_marker, pc_lru); 1379 TAILQ_INSERT_TAIL(&pv_chunks, pc_marker_end, pc_lru); 1380 while ((pc = TAILQ_NEXT(pc_marker, pc_lru)) != pc_marker_end && 1381 SLIST_EMPTY(&free)) { 1382 next_pmap = pc->pc_pmap; 1383 if (next_pmap == NULL) { 1384 /* 1385 * The next chunk is a marker. However, it is 1386 * not our marker, so active_reclaims must be 1387 * > 1. Consequently, the next_chunk code 1388 * will not rotate the pv_chunks list. 1389 */ 1390 goto next_chunk; 1391 } 1392 mtx_unlock(&pv_chunks_mutex); 1393 1394 /* 1395 * A pv_chunk can only be removed from the pc_lru list 1396 * when both pc_chunks_mutex is owned and the 1397 * corresponding pmap is locked. 1398 */ 1399 if (pmap != next_pmap) { 1400 reclaim_pv_chunk_leave_pmap(pmap, locked_pmap); 1401 pmap = next_pmap; 1402 /* Avoid deadlock and lock recursion. */ 1403 if (pmap > locked_pmap) { 1404 RELEASE_PV_LIST_LOCK(lockp); 1405 PMAP_LOCK(pmap); 1406 mtx_lock(&pv_chunks_mutex); 1407 continue; 1408 } else if (pmap != locked_pmap) { 1409 if (PMAP_TRYLOCK(pmap)) { 1410 mtx_lock(&pv_chunks_mutex); 1411 continue; 1412 } else { 1413 pmap = NULL; /* pmap is not locked */ 1414 mtx_lock(&pv_chunks_mutex); 1415 pc = TAILQ_NEXT(pc_marker, pc_lru); 1416 if (pc == NULL || 1417 pc->pc_pmap != next_pmap) 1418 continue; 1419 goto next_chunk; 1420 } 1421 } 1422 } 1423 1424 /* 1425 * Destroy every non-wired, 4 KB page mapping in the chunk. 1426 */ 1427 freed = 0; 1428 for (field = 0; field < _NPCM; field++) { 1429 for (inuse = ~pc->pc_map[field] & pc_freemask[field]; 1430 inuse != 0; inuse &= ~(1UL << bit)) { 1431 bit = cnttzd(inuse); 1432 pv = &pc->pc_pventry[field * 64 + bit]; 1433 va = pv->pv_va; 1434 l3e = pmap_pml3e(pmap, va); 1435 if ((be64toh(*l3e) & RPTE_LEAF) != 0) 1436 continue; 1437 pte = pmap_l3e_to_pte(l3e, va); 1438 if ((be64toh(*pte) & PG_W) != 0) 1439 continue; 1440 tpte = be64toh(pte_load_clear(pte)); 1441 m = PHYS_TO_VM_PAGE(tpte & PG_FRAME); 1442 if ((tpte & (PG_M | PG_RW)) == (PG_M | PG_RW)) 1443 vm_page_dirty(m); 1444 if ((tpte & PG_A) != 0) 1445 vm_page_aflag_set(m, PGA_REFERENCED); 1446 CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m); 1447 TAILQ_REMOVE(&m->md.pv_list, pv, pv_link); 1448 1449 m->md.pv_gen++; 1450 if (TAILQ_EMPTY(&m->md.pv_list) && 1451 (m->flags & PG_FICTITIOUS) == 0) { 1452 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); 1453 if (TAILQ_EMPTY(&pvh->pv_list)) { 1454 vm_page_aflag_clear(m, 1455 PGA_WRITEABLE); 1456 } 1457 } 1458 pc->pc_map[field] |= 1UL << bit; 1459 pmap_unuse_pt(pmap, va, be64toh(*l3e), &free); 1460 freed++; 1461 } 1462 } 1463 if (freed == 0) { 1464 mtx_lock(&pv_chunks_mutex); 1465 goto next_chunk; 1466 } 1467 /* Every freed mapping is for a 4 KB page. */ 1468 pmap_resident_count_dec(pmap, freed); 1469 PV_STAT(atomic_add_long(&pv_entry_frees, freed)); 1470 PV_STAT(atomic_add_int(&pv_entry_spare, freed)); 1471 PV_STAT(atomic_subtract_long(&pv_entry_count, freed)); 1472 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 1473 if (pc->pc_map[0] == PC_FREE0 && pc->pc_map[1] == PC_FREE1) { 1474 PV_STAT(atomic_subtract_int(&pv_entry_spare, _NPCPV)); 1475 PV_STAT(atomic_subtract_int(&pc_chunk_count, 1)); 1476 PV_STAT(atomic_add_int(&pc_chunk_frees, 1)); 1477 /* Entire chunk is free; return it. */ 1478 m_pc = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pc)); 1479 mtx_lock(&pv_chunks_mutex); 1480 TAILQ_REMOVE(&pv_chunks, pc, pc_lru); 1481 break; 1482 } 1483 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list); 1484 mtx_lock(&pv_chunks_mutex); 1485 /* One freed pv entry in locked_pmap is sufficient. */ 1486 if (pmap == locked_pmap) 1487 break; 1488 next_chunk: 1489 TAILQ_REMOVE(&pv_chunks, pc_marker, pc_lru); 1490 TAILQ_INSERT_AFTER(&pv_chunks, pc, pc_marker, pc_lru); 1491 if (active_reclaims == 1 && pmap != NULL) { 1492 /* 1493 * Rotate the pv chunks list so that we do not 1494 * scan the same pv chunks that could not be 1495 * freed (because they contained a wired 1496 * and/or superpage mapping) on every 1497 * invocation of reclaim_pv_chunk(). 1498 */ 1499 while ((pc = TAILQ_FIRST(&pv_chunks)) != pc_marker) { 1500 MPASS(pc->pc_pmap != NULL); 1501 TAILQ_REMOVE(&pv_chunks, pc, pc_lru); 1502 TAILQ_INSERT_TAIL(&pv_chunks, pc, pc_lru); 1503 } 1504 } 1505 } 1506 TAILQ_REMOVE(&pv_chunks, pc_marker, pc_lru); 1507 TAILQ_REMOVE(&pv_chunks, pc_marker_end, pc_lru); 1508 active_reclaims--; 1509 mtx_unlock(&pv_chunks_mutex); 1510 reclaim_pv_chunk_leave_pmap(pmap, locked_pmap); 1511 if (m_pc == NULL && !SLIST_EMPTY(&free)) { 1512 m_pc = SLIST_FIRST(&free); 1513 SLIST_REMOVE_HEAD(&free, plinks.s.ss); 1514 /* Recycle a freed page table page. */ 1515 m_pc->ref_count = 1; 1516 } 1517 vm_page_free_pages_toq(&free, true); 1518 return (m_pc); 1519 } 1520 1521 /* 1522 * free the pv_entry back to the free list 1523 */ 1524 static void 1525 free_pv_entry(pmap_t pmap, pv_entry_t pv) 1526 { 1527 struct pv_chunk *pc; 1528 int idx, field, bit; 1529 1530 #ifdef VERBOSE_PV 1531 if (pmap != kernel_pmap) 1532 printf("%s(%p, %p)\n", __func__, pmap, pv); 1533 #endif 1534 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1535 PV_STAT(atomic_add_long(&pv_entry_frees, 1)); 1536 PV_STAT(atomic_add_int(&pv_entry_spare, 1)); 1537 PV_STAT(atomic_subtract_long(&pv_entry_count, 1)); 1538 pc = pv_to_chunk(pv); 1539 idx = pv - &pc->pc_pventry[0]; 1540 field = idx / 64; 1541 bit = idx % 64; 1542 pc->pc_map[field] |= 1ul << bit; 1543 if (pc->pc_map[0] != PC_FREE0 || pc->pc_map[1] != PC_FREE1) { 1544 /* 98% of the time, pc is already at the head of the list. */ 1545 if (__predict_false(pc != TAILQ_FIRST(&pmap->pm_pvchunk))) { 1546 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 1547 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list); 1548 } 1549 return; 1550 } 1551 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 1552 free_pv_chunk(pc); 1553 } 1554 1555 static void 1556 free_pv_chunk(struct pv_chunk *pc) 1557 { 1558 vm_page_t m; 1559 1560 mtx_lock(&pv_chunks_mutex); 1561 TAILQ_REMOVE(&pv_chunks, pc, pc_lru); 1562 mtx_unlock(&pv_chunks_mutex); 1563 PV_STAT(atomic_subtract_int(&pv_entry_spare, _NPCPV)); 1564 PV_STAT(atomic_subtract_int(&pc_chunk_count, 1)); 1565 PV_STAT(atomic_add_int(&pc_chunk_frees, 1)); 1566 /* entire chunk is free, return it */ 1567 m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pc)); 1568 vm_page_unwire_noq(m); 1569 vm_page_free(m); 1570 } 1571 1572 /* 1573 * Returns a new PV entry, allocating a new PV chunk from the system when 1574 * needed. If this PV chunk allocation fails and a PV list lock pointer was 1575 * given, a PV chunk is reclaimed from an arbitrary pmap. Otherwise, NULL is 1576 * returned. 1577 * 1578 * The given PV list lock may be released. 1579 */ 1580 static pv_entry_t 1581 get_pv_entry(pmap_t pmap, struct rwlock **lockp) 1582 { 1583 int bit, field; 1584 pv_entry_t pv; 1585 struct pv_chunk *pc; 1586 vm_page_t m; 1587 1588 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1589 PV_STAT(atomic_add_long(&pv_entry_allocs, 1)); 1590 retry: 1591 pc = TAILQ_FIRST(&pmap->pm_pvchunk); 1592 if (pc != NULL) { 1593 for (field = 0; field < _NPCM; field++) { 1594 if (pc->pc_map[field]) { 1595 bit = cnttzd(pc->pc_map[field]); 1596 break; 1597 } 1598 } 1599 if (field < _NPCM) { 1600 pv = &pc->pc_pventry[field * 64 + bit]; 1601 pc->pc_map[field] &= ~(1ul << bit); 1602 /* If this was the last item, move it to tail */ 1603 if (pc->pc_map[0] == 0 && pc->pc_map[1] == 0) { 1604 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 1605 TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, 1606 pc_list); 1607 } 1608 PV_STAT(atomic_add_long(&pv_entry_count, 1)); 1609 PV_STAT(atomic_subtract_int(&pv_entry_spare, 1)); 1610 MPASS(PV_PMAP(pv) != NULL); 1611 return (pv); 1612 } 1613 } 1614 /* No free items, allocate another chunk */ 1615 m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | 1616 VM_ALLOC_WIRED); 1617 if (m == NULL) { 1618 if (lockp == NULL) { 1619 PV_STAT(pc_chunk_tryfail++); 1620 return (NULL); 1621 } 1622 m = reclaim_pv_chunk(pmap, lockp); 1623 if (m == NULL) 1624 goto retry; 1625 } 1626 PV_STAT(atomic_add_int(&pc_chunk_count, 1)); 1627 PV_STAT(atomic_add_int(&pc_chunk_allocs, 1)); 1628 pc = (void *)PHYS_TO_DMAP(m->phys_addr); 1629 pc->pc_pmap = pmap; 1630 pc->pc_map[0] = PC_FREE0 & ~1ul; /* preallocated bit 0 */ 1631 pc->pc_map[1] = PC_FREE1; 1632 mtx_lock(&pv_chunks_mutex); 1633 TAILQ_INSERT_TAIL(&pv_chunks, pc, pc_lru); 1634 mtx_unlock(&pv_chunks_mutex); 1635 pv = &pc->pc_pventry[0]; 1636 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list); 1637 PV_STAT(atomic_add_long(&pv_entry_count, 1)); 1638 PV_STAT(atomic_add_int(&pv_entry_spare, _NPCPV - 1)); 1639 MPASS(PV_PMAP(pv) != NULL); 1640 return (pv); 1641 } 1642 1643 #if VM_NRESERVLEVEL > 0 1644 /* 1645 * After promotion from 512 4KB page mappings to a single 2MB page mapping, 1646 * replace the many pv entries for the 4KB page mappings by a single pv entry 1647 * for the 2MB page mapping. 1648 */ 1649 static void 1650 pmap_pv_promote_l3e(pmap_t pmap, vm_offset_t va, vm_paddr_t pa, 1651 struct rwlock **lockp) 1652 { 1653 struct md_page *pvh; 1654 pv_entry_t pv; 1655 vm_offset_t va_last; 1656 vm_page_t m; 1657 1658 KASSERT((pa & L3_PAGE_MASK) == 0, 1659 ("pmap_pv_promote_pde: pa is not 2mpage aligned")); 1660 CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa); 1661 1662 /* 1663 * Transfer the first page's pv entry for this mapping to the 2mpage's 1664 * pv list. Aside from avoiding the cost of a call to get_pv_entry(), 1665 * a transfer avoids the possibility that get_pv_entry() calls 1666 * reclaim_pv_chunk() and that reclaim_pv_chunk() removes one of the 1667 * mappings that is being promoted. 1668 */ 1669 m = PHYS_TO_VM_PAGE(pa); 1670 va = trunc_2mpage(va); 1671 pv = pmap_pvh_remove(&m->md, pmap, va); 1672 KASSERT(pv != NULL, ("pmap_pv_promote_pde: pv not found")); 1673 pvh = pa_to_pvh(pa); 1674 TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_link); 1675 pvh->pv_gen++; 1676 /* Free the remaining NPTEPG - 1 pv entries. */ 1677 va_last = va + L3_PAGE_SIZE - PAGE_SIZE; 1678 do { 1679 m++; 1680 va += PAGE_SIZE; 1681 pmap_pvh_free(&m->md, pmap, va); 1682 } while (va < va_last); 1683 } 1684 #endif /* VM_NRESERVLEVEL > 0 */ 1685 1686 /* 1687 * First find and then destroy the pv entry for the specified pmap and virtual 1688 * address. This operation can be performed on pv lists for either 4KB or 2MB 1689 * page mappings. 1690 */ 1691 static void 1692 pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va) 1693 { 1694 pv_entry_t pv; 1695 1696 pv = pmap_pvh_remove(pvh, pmap, va); 1697 KASSERT(pv != NULL, ("pmap_pvh_free: pv not found")); 1698 free_pv_entry(pmap, pv); 1699 } 1700 1701 /* 1702 * Conditionally create the PV entry for a 4KB page mapping if the required 1703 * memory can be allocated without resorting to reclamation. 1704 */ 1705 static boolean_t 1706 pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, vm_page_t m, 1707 struct rwlock **lockp) 1708 { 1709 pv_entry_t pv; 1710 1711 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1712 /* Pass NULL instead of the lock pointer to disable reclamation. */ 1713 if ((pv = get_pv_entry(pmap, NULL)) != NULL) { 1714 pv->pv_va = va; 1715 CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m); 1716 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_link); 1717 m->md.pv_gen++; 1718 return (TRUE); 1719 } else 1720 return (FALSE); 1721 } 1722 1723 vm_paddr_t phys_avail_debug[2 * VM_PHYSSEG_MAX]; 1724 #ifdef INVARIANTS 1725 static void 1726 validate_addr(vm_paddr_t addr, vm_size_t size) 1727 { 1728 vm_paddr_t end = addr + size; 1729 bool found = false; 1730 1731 for (int i = 0; i < 2 * phys_avail_count; i += 2) { 1732 if (addr >= phys_avail_debug[i] && 1733 end <= phys_avail_debug[i + 1]) { 1734 found = true; 1735 break; 1736 } 1737 } 1738 KASSERT(found, ("%#lx-%#lx outside of initial phys_avail array", 1739 addr, end)); 1740 } 1741 #else 1742 static void validate_addr(vm_paddr_t addr, vm_size_t size) {} 1743 #endif 1744 #define DMAP_PAGE_BITS (RPTE_VALID | RPTE_LEAF | RPTE_EAA_MASK | PG_M | PG_A) 1745 1746 static vm_paddr_t 1747 alloc_pt_page(void) 1748 { 1749 vm_paddr_t page; 1750 1751 page = allocpages(1); 1752 pagezero(PHYS_TO_DMAP(page)); 1753 return (page); 1754 } 1755 1756 static void 1757 mmu_radix_dmap_range(vm_paddr_t start, vm_paddr_t end) 1758 { 1759 pt_entry_t *pte, pteval; 1760 vm_paddr_t page; 1761 1762 if (bootverbose) 1763 printf("%s %lx -> %lx\n", __func__, start, end); 1764 while (start < end) { 1765 pteval = start | DMAP_PAGE_BITS; 1766 pte = pmap_pml1e(kernel_pmap, PHYS_TO_DMAP(start)); 1767 if ((be64toh(*pte) & RPTE_VALID) == 0) { 1768 page = alloc_pt_page(); 1769 pde_store(pte, page); 1770 } 1771 pte = pmap_l1e_to_l2e(pte, PHYS_TO_DMAP(start)); 1772 if ((start & L2_PAGE_MASK) == 0 && 1773 end - start >= L2_PAGE_SIZE) { 1774 start += L2_PAGE_SIZE; 1775 goto done; 1776 } else if ((be64toh(*pte) & RPTE_VALID) == 0) { 1777 page = alloc_pt_page(); 1778 pde_store(pte, page); 1779 } 1780 1781 pte = pmap_l2e_to_l3e(pte, PHYS_TO_DMAP(start)); 1782 if ((start & L3_PAGE_MASK) == 0 && 1783 end - start >= L3_PAGE_SIZE) { 1784 start += L3_PAGE_SIZE; 1785 goto done; 1786 } else if ((be64toh(*pte) & RPTE_VALID) == 0) { 1787 page = alloc_pt_page(); 1788 pde_store(pte, page); 1789 } 1790 pte = pmap_l3e_to_pte(pte, PHYS_TO_DMAP(start)); 1791 start += PAGE_SIZE; 1792 done: 1793 pte_store(pte, pteval); 1794 } 1795 } 1796 1797 static void 1798 mmu_radix_dmap_populate(vm_size_t hwphyssz) 1799 { 1800 vm_paddr_t start, end; 1801 1802 for (int i = 0; i < pregions_sz; i++) { 1803 start = pregions[i].mr_start; 1804 end = start + pregions[i].mr_size; 1805 if (hwphyssz && start >= hwphyssz) 1806 break; 1807 if (hwphyssz && hwphyssz < end) 1808 end = hwphyssz; 1809 mmu_radix_dmap_range(start, end); 1810 } 1811 } 1812 1813 static void 1814 mmu_radix_setup_pagetables(vm_size_t hwphyssz) 1815 { 1816 vm_paddr_t ptpages, pages; 1817 pt_entry_t *pte; 1818 vm_paddr_t l1phys; 1819 1820 bzero(kernel_pmap, sizeof(struct pmap)); 1821 PMAP_LOCK_INIT(kernel_pmap); 1822 1823 ptpages = allocpages(3); 1824 l1phys = moea64_bootstrap_alloc(RADIX_PGD_SIZE, RADIX_PGD_SIZE); 1825 validate_addr(l1phys, RADIX_PGD_SIZE); 1826 if (bootverbose) 1827 printf("l1phys=%lx\n", l1phys); 1828 MPASS((l1phys & (RADIX_PGD_SIZE-1)) == 0); 1829 for (int i = 0; i < RADIX_PGD_SIZE/PAGE_SIZE; i++) 1830 pagezero(PHYS_TO_DMAP(l1phys + i * PAGE_SIZE)); 1831 kernel_pmap->pm_pml1 = (pml1_entry_t *)PHYS_TO_DMAP(l1phys); 1832 1833 mmu_radix_dmap_populate(hwphyssz); 1834 1835 /* 1836 * Create page tables for first 128MB of KVA 1837 */ 1838 pages = ptpages; 1839 pte = pmap_pml1e(kernel_pmap, VM_MIN_KERNEL_ADDRESS); 1840 *pte = htobe64(pages | RPTE_VALID | RPTE_SHIFT); 1841 pages += PAGE_SIZE; 1842 pte = pmap_l1e_to_l2e(pte, VM_MIN_KERNEL_ADDRESS); 1843 *pte = htobe64(pages | RPTE_VALID | RPTE_SHIFT); 1844 pages += PAGE_SIZE; 1845 pte = pmap_l2e_to_l3e(pte, VM_MIN_KERNEL_ADDRESS); 1846 /* 1847 * the kernel page table pages need to be preserved in 1848 * phys_avail and not overlap with previous allocations 1849 */ 1850 pages = allocpages(nkpt); 1851 if (bootverbose) { 1852 printf("phys_avail after dmap populate and nkpt allocation\n"); 1853 for (int j = 0; j < 2 * phys_avail_count; j+=2) 1854 printf("phys_avail[%d]=%08lx - phys_avail[%d]=%08lx\n", 1855 j, phys_avail[j], j + 1, phys_avail[j + 1]); 1856 } 1857 KPTphys = pages; 1858 for (int i = 0; i < nkpt; i++, pte++, pages += PAGE_SIZE) 1859 *pte = htobe64(pages | RPTE_VALID | RPTE_SHIFT); 1860 kernel_vm_end = VM_MIN_KERNEL_ADDRESS + nkpt * L3_PAGE_SIZE; 1861 if (bootverbose) 1862 printf("kernel_pmap pml1 %p\n", kernel_pmap->pm_pml1); 1863 /* 1864 * Add a physical memory segment (vm_phys_seg) corresponding to the 1865 * preallocated kernel page table pages so that vm_page structures 1866 * representing these pages will be created. The vm_page structures 1867 * are required for promotion of the corresponding kernel virtual 1868 * addresses to superpage mappings. 1869 */ 1870 vm_phys_add_seg(KPTphys, KPTphys + ptoa(nkpt)); 1871 } 1872 1873 static void 1874 mmu_radix_early_bootstrap(vm_offset_t start, vm_offset_t end) 1875 { 1876 vm_paddr_t kpstart, kpend; 1877 vm_size_t physsz, hwphyssz; 1878 //uint64_t l2virt; 1879 int rm_pavail, proctab_size; 1880 int i, j; 1881 1882 kpstart = start & ~DMAP_BASE_ADDRESS; 1883 kpend = end & ~DMAP_BASE_ADDRESS; 1884 1885 /* Get physical memory regions from firmware */ 1886 mem_regions(&pregions, &pregions_sz, ®ions, ®ions_sz); 1887 CTR0(KTR_PMAP, "mmu_radix_early_bootstrap: physical memory"); 1888 1889 if (2 * VM_PHYSSEG_MAX < regions_sz) 1890 panic("mmu_radix_early_bootstrap: phys_avail too small"); 1891 1892 if (bootverbose) 1893 for (int i = 0; i < regions_sz; i++) 1894 printf("regions[%d].mr_start=%lx regions[%d].mr_size=%lx\n", 1895 i, regions[i].mr_start, i, regions[i].mr_size); 1896 /* 1897 * XXX workaround a simulator bug 1898 */ 1899 for (int i = 0; i < regions_sz; i++) 1900 if (regions[i].mr_start & PAGE_MASK) { 1901 regions[i].mr_start += PAGE_MASK; 1902 regions[i].mr_start &= ~PAGE_MASK; 1903 regions[i].mr_size &= ~PAGE_MASK; 1904 } 1905 if (bootverbose) 1906 for (int i = 0; i < pregions_sz; i++) 1907 printf("pregions[%d].mr_start=%lx pregions[%d].mr_size=%lx\n", 1908 i, pregions[i].mr_start, i, pregions[i].mr_size); 1909 1910 phys_avail_count = 0; 1911 physsz = 0; 1912 hwphyssz = 0; 1913 TUNABLE_ULONG_FETCH("hw.physmem", (u_long *) &hwphyssz); 1914 for (i = 0, j = 0; i < regions_sz; i++) { 1915 if (bootverbose) 1916 printf("regions[%d].mr_start=%016lx regions[%d].mr_size=%016lx\n", 1917 i, regions[i].mr_start, i, regions[i].mr_size); 1918 1919 if (regions[i].mr_size < PAGE_SIZE) 1920 continue; 1921 1922 if (hwphyssz != 0 && 1923 (physsz + regions[i].mr_size) >= hwphyssz) { 1924 if (physsz < hwphyssz) { 1925 phys_avail[j] = regions[i].mr_start; 1926 phys_avail[j + 1] = regions[i].mr_start + 1927 (hwphyssz - physsz); 1928 physsz = hwphyssz; 1929 phys_avail_count++; 1930 dump_avail[j] = phys_avail[j]; 1931 dump_avail[j + 1] = phys_avail[j + 1]; 1932 } 1933 break; 1934 } 1935 phys_avail[j] = regions[i].mr_start; 1936 phys_avail[j + 1] = regions[i].mr_start + regions[i].mr_size; 1937 dump_avail[j] = phys_avail[j]; 1938 dump_avail[j + 1] = phys_avail[j + 1]; 1939 1940 phys_avail_count++; 1941 physsz += regions[i].mr_size; 1942 j += 2; 1943 } 1944 1945 /* Check for overlap with the kernel and exception vectors */ 1946 rm_pavail = 0; 1947 for (j = 0; j < 2 * phys_avail_count; j+=2) { 1948 if (phys_avail[j] < EXC_LAST) 1949 phys_avail[j] += EXC_LAST; 1950 1951 if (phys_avail[j] >= kpstart && 1952 phys_avail[j + 1] <= kpend) { 1953 phys_avail[j] = phys_avail[j + 1] = ~0; 1954 rm_pavail++; 1955 continue; 1956 } 1957 1958 if (kpstart >= phys_avail[j] && 1959 kpstart < phys_avail[j + 1]) { 1960 if (kpend < phys_avail[j + 1]) { 1961 phys_avail[2 * phys_avail_count] = 1962 (kpend & ~PAGE_MASK) + PAGE_SIZE; 1963 phys_avail[2 * phys_avail_count + 1] = 1964 phys_avail[j + 1]; 1965 phys_avail_count++; 1966 } 1967 1968 phys_avail[j + 1] = kpstart & ~PAGE_MASK; 1969 } 1970 1971 if (kpend >= phys_avail[j] && 1972 kpend < phys_avail[j + 1]) { 1973 if (kpstart > phys_avail[j]) { 1974 phys_avail[2 * phys_avail_count] = phys_avail[j]; 1975 phys_avail[2 * phys_avail_count + 1] = 1976 kpstart & ~PAGE_MASK; 1977 phys_avail_count++; 1978 } 1979 1980 phys_avail[j] = (kpend & ~PAGE_MASK) + 1981 PAGE_SIZE; 1982 } 1983 } 1984 qsort(phys_avail, 2 * phys_avail_count, sizeof(phys_avail[0]), pa_cmp); 1985 for (i = 0; i < 2 * phys_avail_count; i++) 1986 phys_avail_debug[i] = phys_avail[i]; 1987 1988 /* Remove physical available regions marked for removal (~0) */ 1989 if (rm_pavail) { 1990 phys_avail_count -= rm_pavail; 1991 for (i = 2 * phys_avail_count; 1992 i < 2*(phys_avail_count + rm_pavail); i+=2) 1993 phys_avail[i] = phys_avail[i + 1] = 0; 1994 } 1995 if (bootverbose) { 1996 printf("phys_avail ranges after filtering:\n"); 1997 for (j = 0; j < 2 * phys_avail_count; j+=2) 1998 printf("phys_avail[%d]=%08lx - phys_avail[%d]=%08lx\n", 1999 j, phys_avail[j], j + 1, phys_avail[j + 1]); 2000 } 2001 physmem = btoc(physsz); 2002 2003 /* XXX assume we're running non-virtualized and 2004 * we don't support BHYVE 2005 */ 2006 if (isa3_pid_bits == 0) 2007 isa3_pid_bits = 20; 2008 parttab_phys = moea64_bootstrap_alloc(PARTTAB_SIZE, PARTTAB_SIZE); 2009 validate_addr(parttab_phys, PARTTAB_SIZE); 2010 for (int i = 0; i < PARTTAB_SIZE/PAGE_SIZE; i++) 2011 pagezero(PHYS_TO_DMAP(parttab_phys + i * PAGE_SIZE)); 2012 2013 proctab_size = 1UL << PROCTAB_SIZE_SHIFT; 2014 proctab0pa = moea64_bootstrap_alloc(proctab_size, proctab_size); 2015 validate_addr(proctab0pa, proctab_size); 2016 for (int i = 0; i < proctab_size/PAGE_SIZE; i++) 2017 pagezero(PHYS_TO_DMAP(proctab0pa + i * PAGE_SIZE)); 2018 2019 mmu_radix_setup_pagetables(hwphyssz); 2020 } 2021 2022 static void 2023 mmu_radix_late_bootstrap(vm_offset_t start, vm_offset_t end) 2024 { 2025 int i; 2026 vm_paddr_t pa; 2027 void *dpcpu; 2028 vm_offset_t va; 2029 2030 /* 2031 * Set up the Open Firmware pmap and add its mappings if not in real 2032 * mode. 2033 */ 2034 if (bootverbose) 2035 printf("%s enter\n", __func__); 2036 2037 /* 2038 * Calculate the last available physical address, and reserve the 2039 * vm_page_array (upper bound). 2040 */ 2041 Maxmem = 0; 2042 for (i = 0; phys_avail[i + 2] != 0; i += 2) 2043 Maxmem = MAX(Maxmem, powerpc_btop(phys_avail[i + 1])); 2044 2045 /* 2046 * Set the start and end of kva. 2047 */ 2048 virtual_avail = VM_MIN_KERNEL_ADDRESS; 2049 virtual_end = VM_MAX_SAFE_KERNEL_ADDRESS; 2050 2051 /* 2052 * Remap any early IO mappings (console framebuffer, etc.) 2053 */ 2054 bs_remap_earlyboot(); 2055 2056 /* 2057 * Allocate a kernel stack with a guard page for thread0 and map it 2058 * into the kernel page map. 2059 */ 2060 pa = allocpages(kstack_pages); 2061 va = virtual_avail + KSTACK_GUARD_PAGES * PAGE_SIZE; 2062 virtual_avail = va + kstack_pages * PAGE_SIZE; 2063 CTR2(KTR_PMAP, "moea64_bootstrap: kstack0 at %#x (%#x)", pa, va); 2064 thread0.td_kstack = va; 2065 for (i = 0; i < kstack_pages; i++) { 2066 mmu_radix_kenter(va, pa); 2067 pa += PAGE_SIZE; 2068 va += PAGE_SIZE; 2069 } 2070 thread0.td_kstack_pages = kstack_pages; 2071 2072 /* 2073 * Allocate virtual address space for the message buffer. 2074 */ 2075 pa = msgbuf_phys = allocpages((msgbufsize + PAGE_MASK) >> PAGE_SHIFT); 2076 msgbufp = (struct msgbuf *)PHYS_TO_DMAP(pa); 2077 2078 /* 2079 * Allocate virtual address space for the dynamic percpu area. 2080 */ 2081 pa = allocpages(DPCPU_SIZE >> PAGE_SHIFT); 2082 dpcpu = (void *)PHYS_TO_DMAP(pa); 2083 dpcpu_init(dpcpu, curcpu); 2084 /* 2085 * Reserve some special page table entries/VA space for temporary 2086 * mapping of pages. 2087 */ 2088 } 2089 2090 static void 2091 mmu_parttab_init(void) 2092 { 2093 uint64_t ptcr; 2094 2095 isa3_parttab = (struct pate *)PHYS_TO_DMAP(parttab_phys); 2096 2097 if (bootverbose) 2098 printf("%s parttab: %p\n", __func__, isa3_parttab); 2099 ptcr = parttab_phys | (PARTTAB_SIZE_SHIFT-12); 2100 if (bootverbose) 2101 printf("setting ptcr %lx\n", ptcr); 2102 mtspr(SPR_PTCR, ptcr); 2103 } 2104 2105 static void 2106 mmu_parttab_update(uint64_t lpid, uint64_t pagetab, uint64_t proctab) 2107 { 2108 uint64_t prev; 2109 2110 if (bootverbose) 2111 printf("%s isa3_parttab %p lpid %lx pagetab %lx proctab %lx\n", __func__, isa3_parttab, 2112 lpid, pagetab, proctab); 2113 prev = be64toh(isa3_parttab[lpid].pagetab); 2114 isa3_parttab[lpid].pagetab = htobe64(pagetab); 2115 isa3_parttab[lpid].proctab = htobe64(proctab); 2116 2117 if (prev & PARTTAB_HR) { 2118 __asm __volatile(PPC_TLBIE_5(%0,%1,2,0,1) : : 2119 "r" (TLBIEL_INVAL_SET_LPID), "r" (lpid)); 2120 __asm __volatile(PPC_TLBIE_5(%0,%1,2,1,1) : : 2121 "r" (TLBIEL_INVAL_SET_LPID), "r" (lpid)); 2122 } else { 2123 __asm __volatile(PPC_TLBIE_5(%0,%1,2,0,0) : : 2124 "r" (TLBIEL_INVAL_SET_LPID), "r" (lpid)); 2125 } 2126 ttusync(); 2127 } 2128 2129 static void 2130 mmu_radix_parttab_init(void) 2131 { 2132 uint64_t pagetab; 2133 2134 mmu_parttab_init(); 2135 pagetab = RTS_SIZE | DMAP_TO_PHYS((vm_offset_t)kernel_pmap->pm_pml1) | \ 2136 RADIX_PGD_INDEX_SHIFT | PARTTAB_HR; 2137 mmu_parttab_update(0, pagetab, 0); 2138 } 2139 2140 static void 2141 mmu_radix_proctab_register(vm_paddr_t proctabpa, uint64_t table_size) 2142 { 2143 uint64_t pagetab, proctab; 2144 2145 pagetab = be64toh(isa3_parttab[0].pagetab); 2146 proctab = proctabpa | table_size | PARTTAB_GR; 2147 mmu_parttab_update(0, pagetab, proctab); 2148 } 2149 2150 static void 2151 mmu_radix_proctab_init(void) 2152 { 2153 2154 isa3_base_pid = 1; 2155 2156 isa3_proctab = (void*)PHYS_TO_DMAP(proctab0pa); 2157 isa3_proctab->proctab0 = 2158 htobe64(RTS_SIZE | DMAP_TO_PHYS((vm_offset_t)kernel_pmap->pm_pml1) | 2159 RADIX_PGD_INDEX_SHIFT); 2160 2161 mmu_radix_proctab_register(proctab0pa, PROCTAB_SIZE_SHIFT - 12); 2162 2163 __asm __volatile("ptesync" : : : "memory"); 2164 __asm __volatile(PPC_TLBIE_5(%0,%1,2,1,1) : : 2165 "r" (TLBIEL_INVAL_SET_LPID), "r" (0)); 2166 __asm __volatile("eieio; tlbsync; ptesync" : : : "memory"); 2167 if (bootverbose) 2168 printf("process table %p and kernel radix PDE: %p\n", 2169 isa3_proctab, kernel_pmap->pm_pml1); 2170 mtmsr(mfmsr() | PSL_DR ); 2171 mtmsr(mfmsr() & ~PSL_DR); 2172 kernel_pmap->pm_pid = isa3_base_pid; 2173 isa3_base_pid++; 2174 } 2175 2176 void 2177 mmu_radix_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, 2178 int advice) 2179 { 2180 struct rwlock *lock; 2181 pml1_entry_t *l1e; 2182 pml2_entry_t *l2e; 2183 pml3_entry_t oldl3e, *l3e; 2184 pt_entry_t *pte; 2185 vm_offset_t va, va_next; 2186 vm_page_t m; 2187 boolean_t anychanged; 2188 2189 if (advice != MADV_DONTNEED && advice != MADV_FREE) 2190 return; 2191 anychanged = FALSE; 2192 PMAP_LOCK(pmap); 2193 for (; sva < eva; sva = va_next) { 2194 l1e = pmap_pml1e(pmap, sva); 2195 if ((be64toh(*l1e) & PG_V) == 0) { 2196 va_next = (sva + L1_PAGE_SIZE) & ~L1_PAGE_MASK; 2197 if (va_next < sva) 2198 va_next = eva; 2199 continue; 2200 } 2201 l2e = pmap_l1e_to_l2e(l1e, sva); 2202 if ((be64toh(*l2e) & PG_V) == 0) { 2203 va_next = (sva + L2_PAGE_SIZE) & ~L2_PAGE_MASK; 2204 if (va_next < sva) 2205 va_next = eva; 2206 continue; 2207 } 2208 va_next = (sva + L3_PAGE_SIZE) & ~L3_PAGE_MASK; 2209 if (va_next < sva) 2210 va_next = eva; 2211 l3e = pmap_l2e_to_l3e(l2e, sva); 2212 oldl3e = be64toh(*l3e); 2213 if ((oldl3e & PG_V) == 0) 2214 continue; 2215 else if ((oldl3e & RPTE_LEAF) != 0) { 2216 if ((oldl3e & PG_MANAGED) == 0) 2217 continue; 2218 lock = NULL; 2219 if (!pmap_demote_l3e_locked(pmap, l3e, sva, &lock)) { 2220 if (lock != NULL) 2221 rw_wunlock(lock); 2222 2223 /* 2224 * The large page mapping was destroyed. 2225 */ 2226 continue; 2227 } 2228 2229 /* 2230 * Unless the page mappings are wired, remove the 2231 * mapping to a single page so that a subsequent 2232 * access may repromote. Since the underlying page 2233 * table page is fully populated, this removal never 2234 * frees a page table page. 2235 */ 2236 if ((oldl3e & PG_W) == 0) { 2237 pte = pmap_l3e_to_pte(l3e, sva); 2238 KASSERT((be64toh(*pte) & PG_V) != 0, 2239 ("pmap_advise: invalid PTE")); 2240 pmap_remove_pte(pmap, pte, sva, be64toh(*l3e), NULL, 2241 &lock); 2242 anychanged = TRUE; 2243 } 2244 if (lock != NULL) 2245 rw_wunlock(lock); 2246 } 2247 if (va_next > eva) 2248 va_next = eva; 2249 va = va_next; 2250 for (pte = pmap_l3e_to_pte(l3e, sva); sva != va_next; 2251 pte++, sva += PAGE_SIZE) { 2252 MPASS(pte == pmap_pte(pmap, sva)); 2253 2254 if ((be64toh(*pte) & (PG_MANAGED | PG_V)) != (PG_MANAGED | PG_V)) 2255 goto maybe_invlrng; 2256 else if ((be64toh(*pte) & (PG_M | PG_RW)) == (PG_M | PG_RW)) { 2257 if (advice == MADV_DONTNEED) { 2258 /* 2259 * Future calls to pmap_is_modified() 2260 * can be avoided by making the page 2261 * dirty now. 2262 */ 2263 m = PHYS_TO_VM_PAGE(be64toh(*pte) & PG_FRAME); 2264 vm_page_dirty(m); 2265 } 2266 atomic_clear_long(pte, htobe64(PG_M | PG_A)); 2267 } else if ((be64toh(*pte) & PG_A) != 0) 2268 atomic_clear_long(pte, htobe64(PG_A)); 2269 else 2270 goto maybe_invlrng; 2271 anychanged = TRUE; 2272 continue; 2273 maybe_invlrng: 2274 if (va != va_next) { 2275 anychanged = true; 2276 va = va_next; 2277 } 2278 } 2279 if (va != va_next) 2280 anychanged = true; 2281 } 2282 if (anychanged) 2283 pmap_invalidate_all(pmap); 2284 PMAP_UNLOCK(pmap); 2285 } 2286 2287 /* 2288 * Routines used in machine-dependent code 2289 */ 2290 static void 2291 mmu_radix_bootstrap(vm_offset_t start, vm_offset_t end) 2292 { 2293 uint64_t lpcr; 2294 2295 if (bootverbose) 2296 printf("%s\n", __func__); 2297 hw_direct_map = 1; 2298 mmu_radix_early_bootstrap(start, end); 2299 if (bootverbose) 2300 printf("early bootstrap complete\n"); 2301 if (powernv_enabled) { 2302 lpcr = mfspr(SPR_LPCR); 2303 mtspr(SPR_LPCR, lpcr | LPCR_UPRT | LPCR_HR); 2304 mmu_radix_parttab_init(); 2305 mmu_radix_init_amor(); 2306 if (bootverbose) 2307 printf("powernv init complete\n"); 2308 } 2309 mmu_radix_init_iamr(); 2310 mmu_radix_proctab_init(); 2311 mmu_radix_pid_set(kernel_pmap); 2312 /* XXX assume CPU_FTR_HVMODE */ 2313 mmu_radix_tlbiel_flush(TLB_INVAL_SCOPE_GLOBAL); 2314 2315 mmu_radix_late_bootstrap(start, end); 2316 numa_mem_regions(&numa_pregions, &numa_pregions_sz); 2317 if (bootverbose) 2318 printf("%s done\n", __func__); 2319 pmap_bootstrapped = 1; 2320 dmaplimit = roundup2(powerpc_ptob(Maxmem), L2_PAGE_SIZE); 2321 PCPU_SET(flags, PCPU_GET(flags) | PC_FLAG_NOSRS); 2322 } 2323 2324 static void 2325 mmu_radix_cpu_bootstrap(int ap) 2326 { 2327 uint64_t lpcr; 2328 uint64_t ptcr; 2329 2330 if (powernv_enabled) { 2331 lpcr = mfspr(SPR_LPCR); 2332 mtspr(SPR_LPCR, lpcr | LPCR_UPRT | LPCR_HR); 2333 2334 ptcr = parttab_phys | (PARTTAB_SIZE_SHIFT-12); 2335 mtspr(SPR_PTCR, ptcr); 2336 mmu_radix_init_amor(); 2337 } 2338 mmu_radix_init_iamr(); 2339 mmu_radix_pid_set(kernel_pmap); 2340 mmu_radix_tlbiel_flush(TLB_INVAL_SCOPE_GLOBAL); 2341 } 2342 2343 static SYSCTL_NODE(_vm_pmap, OID_AUTO, l3e, CTLFLAG_RD, 0, 2344 "2MB page mapping counters"); 2345 2346 static u_long pmap_l3e_demotions; 2347 SYSCTL_ULONG(_vm_pmap_l3e, OID_AUTO, demotions, CTLFLAG_RD, 2348 &pmap_l3e_demotions, 0, "2MB page demotions"); 2349 2350 static u_long pmap_l3e_mappings; 2351 SYSCTL_ULONG(_vm_pmap_l3e, OID_AUTO, mappings, CTLFLAG_RD, 2352 &pmap_l3e_mappings, 0, "2MB page mappings"); 2353 2354 static u_long pmap_l3e_p_failures; 2355 SYSCTL_ULONG(_vm_pmap_l3e, OID_AUTO, p_failures, CTLFLAG_RD, 2356 &pmap_l3e_p_failures, 0, "2MB page promotion failures"); 2357 2358 static u_long pmap_l3e_promotions; 2359 SYSCTL_ULONG(_vm_pmap_l3e, OID_AUTO, promotions, CTLFLAG_RD, 2360 &pmap_l3e_promotions, 0, "2MB page promotions"); 2361 2362 static SYSCTL_NODE(_vm_pmap, OID_AUTO, l2e, CTLFLAG_RD, 0, 2363 "1GB page mapping counters"); 2364 2365 static u_long pmap_l2e_demotions; 2366 SYSCTL_ULONG(_vm_pmap_l2e, OID_AUTO, demotions, CTLFLAG_RD, 2367 &pmap_l2e_demotions, 0, "1GB page demotions"); 2368 2369 void 2370 mmu_radix_clear_modify(vm_page_t m) 2371 { 2372 struct md_page *pvh; 2373 pmap_t pmap; 2374 pv_entry_t next_pv, pv; 2375 pml3_entry_t oldl3e, *l3e; 2376 pt_entry_t oldpte, *pte; 2377 struct rwlock *lock; 2378 vm_offset_t va; 2379 int md_gen, pvh_gen; 2380 2381 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 2382 ("pmap_clear_modify: page %p is not managed", m)); 2383 vm_page_assert_busied(m); 2384 CTR2(KTR_PMAP, "%s(%p)", __func__, m); 2385 2386 /* 2387 * If the page is not PGA_WRITEABLE, then no PTEs can have PG_M set. 2388 * If the object containing the page is locked and the page is not 2389 * exclusive busied, then PGA_WRITEABLE cannot be concurrently set. 2390 */ 2391 if ((m->a.flags & PGA_WRITEABLE) == 0) 2392 return; 2393 pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy : 2394 pa_to_pvh(VM_PAGE_TO_PHYS(m)); 2395 lock = VM_PAGE_TO_PV_LIST_LOCK(m); 2396 rw_wlock(lock); 2397 restart: 2398 TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_link, next_pv) { 2399 pmap = PV_PMAP(pv); 2400 if (!PMAP_TRYLOCK(pmap)) { 2401 pvh_gen = pvh->pv_gen; 2402 rw_wunlock(lock); 2403 PMAP_LOCK(pmap); 2404 rw_wlock(lock); 2405 if (pvh_gen != pvh->pv_gen) { 2406 PMAP_UNLOCK(pmap); 2407 goto restart; 2408 } 2409 } 2410 va = pv->pv_va; 2411 l3e = pmap_pml3e(pmap, va); 2412 oldl3e = be64toh(*l3e); 2413 if ((oldl3e & PG_RW) != 0) { 2414 if (pmap_demote_l3e_locked(pmap, l3e, va, &lock)) { 2415 if ((oldl3e & PG_W) == 0) { 2416 /* 2417 * Write protect the mapping to a 2418 * single page so that a subsequent 2419 * write access may repromote. 2420 */ 2421 va += VM_PAGE_TO_PHYS(m) - (oldl3e & 2422 PG_PS_FRAME); 2423 pte = pmap_l3e_to_pte(l3e, va); 2424 oldpte = be64toh(*pte); 2425 if ((oldpte & PG_V) != 0) { 2426 while (!atomic_cmpset_long(pte, 2427 htobe64(oldpte), 2428 htobe64((oldpte | RPTE_EAA_R) & ~(PG_M | PG_RW)))) 2429 oldpte = be64toh(*pte); 2430 vm_page_dirty(m); 2431 pmap_invalidate_page(pmap, va); 2432 } 2433 } 2434 } 2435 } 2436 PMAP_UNLOCK(pmap); 2437 } 2438 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) { 2439 pmap = PV_PMAP(pv); 2440 if (!PMAP_TRYLOCK(pmap)) { 2441 md_gen = m->md.pv_gen; 2442 pvh_gen = pvh->pv_gen; 2443 rw_wunlock(lock); 2444 PMAP_LOCK(pmap); 2445 rw_wlock(lock); 2446 if (pvh_gen != pvh->pv_gen || md_gen != m->md.pv_gen) { 2447 PMAP_UNLOCK(pmap); 2448 goto restart; 2449 } 2450 } 2451 l3e = pmap_pml3e(pmap, pv->pv_va); 2452 KASSERT((be64toh(*l3e) & RPTE_LEAF) == 0, ("pmap_clear_modify: found" 2453 " a 2mpage in page %p's pv list", m)); 2454 pte = pmap_l3e_to_pte(l3e, pv->pv_va); 2455 if ((be64toh(*pte) & (PG_M | PG_RW)) == (PG_M | PG_RW)) { 2456 atomic_clear_long(pte, htobe64(PG_M)); 2457 pmap_invalidate_page(pmap, pv->pv_va); 2458 } 2459 PMAP_UNLOCK(pmap); 2460 } 2461 rw_wunlock(lock); 2462 } 2463 2464 void 2465 mmu_radix_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, 2466 vm_size_t len, vm_offset_t src_addr) 2467 { 2468 struct rwlock *lock; 2469 struct spglist free; 2470 vm_offset_t addr; 2471 vm_offset_t end_addr = src_addr + len; 2472 vm_offset_t va_next; 2473 vm_page_t dst_pdpg, dstmpte, srcmpte; 2474 bool invalidate_all; 2475 2476 CTR6(KTR_PMAP, 2477 "%s(dst_pmap=%p, src_pmap=%p, dst_addr=%lx, len=%lu, src_addr=%lx)\n", 2478 __func__, dst_pmap, src_pmap, dst_addr, len, src_addr); 2479 2480 if (dst_addr != src_addr) 2481 return; 2482 lock = NULL; 2483 invalidate_all = false; 2484 if (dst_pmap < src_pmap) { 2485 PMAP_LOCK(dst_pmap); 2486 PMAP_LOCK(src_pmap); 2487 } else { 2488 PMAP_LOCK(src_pmap); 2489 PMAP_LOCK(dst_pmap); 2490 } 2491 2492 for (addr = src_addr; addr < end_addr; addr = va_next) { 2493 pml1_entry_t *l1e; 2494 pml2_entry_t *l2e; 2495 pml3_entry_t srcptepaddr, *l3e; 2496 pt_entry_t *src_pte, *dst_pte; 2497 2498 l1e = pmap_pml1e(src_pmap, addr); 2499 if ((be64toh(*l1e) & PG_V) == 0) { 2500 va_next = (addr + L1_PAGE_SIZE) & ~L1_PAGE_MASK; 2501 if (va_next < addr) 2502 va_next = end_addr; 2503 continue; 2504 } 2505 2506 l2e = pmap_l1e_to_l2e(l1e, addr); 2507 if ((be64toh(*l2e) & PG_V) == 0) { 2508 va_next = (addr + L2_PAGE_SIZE) & ~L2_PAGE_MASK; 2509 if (va_next < addr) 2510 va_next = end_addr; 2511 continue; 2512 } 2513 2514 va_next = (addr + L3_PAGE_SIZE) & ~L3_PAGE_MASK; 2515 if (va_next < addr) 2516 va_next = end_addr; 2517 2518 l3e = pmap_l2e_to_l3e(l2e, addr); 2519 srcptepaddr = be64toh(*l3e); 2520 if (srcptepaddr == 0) 2521 continue; 2522 2523 if (srcptepaddr & RPTE_LEAF) { 2524 if ((addr & L3_PAGE_MASK) != 0 || 2525 addr + L3_PAGE_SIZE > end_addr) 2526 continue; 2527 dst_pdpg = pmap_allocl3e(dst_pmap, addr, NULL); 2528 if (dst_pdpg == NULL) 2529 break; 2530 l3e = (pml3_entry_t *) 2531 PHYS_TO_DMAP(VM_PAGE_TO_PHYS(dst_pdpg)); 2532 l3e = &l3e[pmap_pml3e_index(addr)]; 2533 if (be64toh(*l3e) == 0 && ((srcptepaddr & PG_MANAGED) == 0 || 2534 pmap_pv_insert_l3e(dst_pmap, addr, srcptepaddr, 2535 PMAP_ENTER_NORECLAIM, &lock))) { 2536 *l3e = htobe64(srcptepaddr & ~PG_W); 2537 pmap_resident_count_inc(dst_pmap, 2538 L3_PAGE_SIZE / PAGE_SIZE); 2539 atomic_add_long(&pmap_l3e_mappings, 1); 2540 } else 2541 dst_pdpg->ref_count--; 2542 continue; 2543 } 2544 2545 srcptepaddr &= PG_FRAME; 2546 srcmpte = PHYS_TO_VM_PAGE(srcptepaddr); 2547 KASSERT(srcmpte->ref_count > 0, 2548 ("pmap_copy: source page table page is unused")); 2549 2550 if (va_next > end_addr) 2551 va_next = end_addr; 2552 2553 src_pte = (pt_entry_t *)PHYS_TO_DMAP(srcptepaddr); 2554 src_pte = &src_pte[pmap_pte_index(addr)]; 2555 dstmpte = NULL; 2556 while (addr < va_next) { 2557 pt_entry_t ptetemp; 2558 ptetemp = be64toh(*src_pte); 2559 /* 2560 * we only virtual copy managed pages 2561 */ 2562 if ((ptetemp & PG_MANAGED) != 0) { 2563 if (dstmpte != NULL && 2564 dstmpte->pindex == pmap_l3e_pindex(addr)) 2565 dstmpte->ref_count++; 2566 else if ((dstmpte = pmap_allocpte(dst_pmap, 2567 addr, NULL)) == NULL) 2568 goto out; 2569 dst_pte = (pt_entry_t *) 2570 PHYS_TO_DMAP(VM_PAGE_TO_PHYS(dstmpte)); 2571 dst_pte = &dst_pte[pmap_pte_index(addr)]; 2572 if (be64toh(*dst_pte) == 0 && 2573 pmap_try_insert_pv_entry(dst_pmap, addr, 2574 PHYS_TO_VM_PAGE(ptetemp & PG_FRAME), 2575 &lock)) { 2576 /* 2577 * Clear the wired, modified, and 2578 * accessed (referenced) bits 2579 * during the copy. 2580 */ 2581 *dst_pte = htobe64(ptetemp & ~(PG_W | PG_M | 2582 PG_A)); 2583 pmap_resident_count_inc(dst_pmap, 1); 2584 } else { 2585 SLIST_INIT(&free); 2586 if (pmap_unwire_ptp(dst_pmap, addr, 2587 dstmpte, &free)) { 2588 /* 2589 * Although "addr" is not 2590 * mapped, paging-structure 2591 * caches could nonetheless 2592 * have entries that refer to 2593 * the freed page table pages. 2594 * Invalidate those entries. 2595 */ 2596 invalidate_all = true; 2597 vm_page_free_pages_toq(&free, 2598 true); 2599 } 2600 goto out; 2601 } 2602 if (dstmpte->ref_count >= srcmpte->ref_count) 2603 break; 2604 } 2605 addr += PAGE_SIZE; 2606 if (__predict_false((addr & L3_PAGE_MASK) == 0)) 2607 src_pte = pmap_pte(src_pmap, addr); 2608 else 2609 src_pte++; 2610 } 2611 } 2612 out: 2613 if (invalidate_all) 2614 pmap_invalidate_all(dst_pmap); 2615 if (lock != NULL) 2616 rw_wunlock(lock); 2617 PMAP_UNLOCK(src_pmap); 2618 PMAP_UNLOCK(dst_pmap); 2619 } 2620 2621 static void 2622 mmu_radix_copy_page(vm_page_t msrc, vm_page_t mdst) 2623 { 2624 vm_offset_t src = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(msrc)); 2625 vm_offset_t dst = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mdst)); 2626 2627 CTR3(KTR_PMAP, "%s(%p, %p)", __func__, src, dst); 2628 /* 2629 * XXX slow 2630 */ 2631 bcopy((void *)src, (void *)dst, PAGE_SIZE); 2632 } 2633 2634 static void 2635 mmu_radix_copy_pages(vm_page_t ma[], vm_offset_t a_offset, vm_page_t mb[], 2636 vm_offset_t b_offset, int xfersize) 2637 { 2638 void *a_cp, *b_cp; 2639 vm_offset_t a_pg_offset, b_pg_offset; 2640 int cnt; 2641 2642 CTR6(KTR_PMAP, "%s(%p, %#x, %p, %#x, %#x)", __func__, ma, 2643 a_offset, mb, b_offset, xfersize); 2644 2645 while (xfersize > 0) { 2646 a_pg_offset = a_offset & PAGE_MASK; 2647 cnt = min(xfersize, PAGE_SIZE - a_pg_offset); 2648 a_cp = (char *)(uintptr_t)PHYS_TO_DMAP( 2649 VM_PAGE_TO_PHYS(ma[a_offset >> PAGE_SHIFT])) + 2650 a_pg_offset; 2651 b_pg_offset = b_offset & PAGE_MASK; 2652 cnt = min(cnt, PAGE_SIZE - b_pg_offset); 2653 b_cp = (char *)(uintptr_t)PHYS_TO_DMAP( 2654 VM_PAGE_TO_PHYS(mb[b_offset >> PAGE_SHIFT])) + 2655 b_pg_offset; 2656 bcopy(a_cp, b_cp, cnt); 2657 a_offset += cnt; 2658 b_offset += cnt; 2659 xfersize -= cnt; 2660 } 2661 } 2662 2663 #if VM_NRESERVLEVEL > 0 2664 /* 2665 * Tries to promote the 512, contiguous 4KB page mappings that are within a 2666 * single page table page (PTP) to a single 2MB page mapping. For promotion 2667 * to occur, two conditions must be met: (1) the 4KB page mappings must map 2668 * aligned, contiguous physical memory and (2) the 4KB page mappings must have 2669 * identical characteristics. 2670 */ 2671 static int 2672 pmap_promote_l3e(pmap_t pmap, pml3_entry_t *pde, vm_offset_t va, 2673 struct rwlock **lockp) 2674 { 2675 pml3_entry_t newpde; 2676 pt_entry_t *firstpte, oldpte, pa, *pte; 2677 vm_page_t mpte; 2678 2679 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 2680 2681 /* 2682 * Examine the first PTE in the specified PTP. Abort if this PTE is 2683 * either invalid, unused, or does not map the first 4KB physical page 2684 * within a 2MB page. 2685 */ 2686 firstpte = (pt_entry_t *)PHYS_TO_DMAP(be64toh(*pde) & PG_FRAME); 2687 setpde: 2688 newpde = *firstpte; 2689 if ((newpde & ((PG_FRAME & L3_PAGE_MASK) | PG_A | PG_V)) != (PG_A | PG_V)) { 2690 CTR2(KTR_PMAP, "pmap_promote_l3e: failure for va %#lx" 2691 " in pmap %p", va, pmap); 2692 goto fail; 2693 } 2694 if ((newpde & (PG_M | PG_RW)) == PG_RW) { 2695 /* 2696 * When PG_M is already clear, PG_RW can be cleared without 2697 * a TLB invalidation. 2698 */ 2699 if (!atomic_cmpset_long(firstpte, htobe64(newpde), htobe64((newpde | RPTE_EAA_R) & ~RPTE_EAA_W))) 2700 goto setpde; 2701 newpde &= ~RPTE_EAA_W; 2702 } 2703 2704 /* 2705 * Examine each of the other PTEs in the specified PTP. Abort if this 2706 * PTE maps an unexpected 4KB physical page or does not have identical 2707 * characteristics to the first PTE. 2708 */ 2709 pa = (newpde & (PG_PS_FRAME | PG_A | PG_V)) + L3_PAGE_SIZE - PAGE_SIZE; 2710 for (pte = firstpte + NPTEPG - 1; pte > firstpte; pte--) { 2711 setpte: 2712 oldpte = be64toh(*pte); 2713 if ((oldpte & (PG_FRAME | PG_A | PG_V)) != pa) { 2714 CTR2(KTR_PMAP, "pmap_promote_l3e: failure for va %#lx" 2715 " in pmap %p", va, pmap); 2716 goto fail; 2717 } 2718 if ((oldpte & (PG_M | PG_RW)) == PG_RW) { 2719 /* 2720 * When PG_M is already clear, PG_RW can be cleared 2721 * without a TLB invalidation. 2722 */ 2723 if (!atomic_cmpset_long(pte, htobe64(oldpte), htobe64((oldpte | RPTE_EAA_R) & ~RPTE_EAA_W))) 2724 goto setpte; 2725 oldpte &= ~RPTE_EAA_W; 2726 CTR2(KTR_PMAP, "pmap_promote_l3e: protect for va %#lx" 2727 " in pmap %p", (oldpte & PG_FRAME & L3_PAGE_MASK) | 2728 (va & ~L3_PAGE_MASK), pmap); 2729 } 2730 if ((oldpte & PG_PTE_PROMOTE) != (newpde & PG_PTE_PROMOTE)) { 2731 CTR2(KTR_PMAP, "pmap_promote_l3e: failure for va %#lx" 2732 " in pmap %p", va, pmap); 2733 goto fail; 2734 } 2735 pa -= PAGE_SIZE; 2736 } 2737 2738 /* 2739 * Save the page table page in its current state until the PDE 2740 * mapping the superpage is demoted by pmap_demote_pde() or 2741 * destroyed by pmap_remove_pde(). 2742 */ 2743 mpte = PHYS_TO_VM_PAGE(be64toh(*pde) & PG_FRAME); 2744 KASSERT(mpte >= vm_page_array && 2745 mpte < &vm_page_array[vm_page_array_size], 2746 ("pmap_promote_l3e: page table page is out of range")); 2747 KASSERT(mpte->pindex == pmap_l3e_pindex(va), 2748 ("pmap_promote_l3e: page table page's pindex is wrong")); 2749 if (pmap_insert_pt_page(pmap, mpte)) { 2750 CTR2(KTR_PMAP, 2751 "pmap_promote_l3e: failure for va %#lx in pmap %p", va, 2752 pmap); 2753 goto fail; 2754 } 2755 2756 /* 2757 * Promote the pv entries. 2758 */ 2759 if ((newpde & PG_MANAGED) != 0) 2760 pmap_pv_promote_l3e(pmap, va, newpde & PG_PS_FRAME, lockp); 2761 2762 pte_store(pde, PG_PROMOTED | newpde); 2763 atomic_add_long(&pmap_l3e_promotions, 1); 2764 CTR2(KTR_PMAP, "pmap_promote_l3e: success for va %#lx" 2765 " in pmap %p", va, pmap); 2766 return (0); 2767 fail: 2768 atomic_add_long(&pmap_l3e_p_failures, 1); 2769 return (KERN_FAILURE); 2770 } 2771 #endif /* VM_NRESERVLEVEL > 0 */ 2772 2773 int 2774 mmu_radix_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, 2775 vm_prot_t prot, u_int flags, int8_t psind) 2776 { 2777 struct rwlock *lock; 2778 pml3_entry_t *l3e; 2779 pt_entry_t *pte; 2780 pt_entry_t newpte, origpte; 2781 pv_entry_t pv; 2782 vm_paddr_t opa, pa; 2783 vm_page_t mpte, om; 2784 int rv, retrycount; 2785 boolean_t nosleep, invalidate_all, invalidate_page; 2786 2787 va = trunc_page(va); 2788 retrycount = 0; 2789 invalidate_page = invalidate_all = false; 2790 CTR6(KTR_PMAP, "pmap_enter(%p, %#lx, %p, %#x, %#x, %d)", pmap, va, 2791 m, prot, flags, psind); 2792 KASSERT(va <= VM_MAX_KERNEL_ADDRESS, ("pmap_enter: toobig")); 2793 KASSERT((m->oflags & VPO_UNMANAGED) != 0 || va < kmi.clean_sva || 2794 va >= kmi.clean_eva, 2795 ("pmap_enter: managed mapping within the clean submap")); 2796 if ((m->oflags & VPO_UNMANAGED) == 0) 2797 VM_PAGE_OBJECT_BUSY_ASSERT(m); 2798 2799 KASSERT((flags & PMAP_ENTER_RESERVED) == 0, 2800 ("pmap_enter: flags %u has reserved bits set", flags)); 2801 pa = VM_PAGE_TO_PHYS(m); 2802 newpte = (pt_entry_t)(pa | PG_A | PG_V | RPTE_LEAF); 2803 if ((flags & VM_PROT_WRITE) != 0) 2804 newpte |= PG_M; 2805 if ((flags & VM_PROT_READ) != 0) 2806 newpte |= PG_A; 2807 if (prot & VM_PROT_READ) 2808 newpte |= RPTE_EAA_R; 2809 if ((prot & VM_PROT_WRITE) != 0) 2810 newpte |= RPTE_EAA_W; 2811 KASSERT((newpte & (PG_M | PG_RW)) != PG_M, 2812 ("pmap_enter: flags includes VM_PROT_WRITE but prot doesn't")); 2813 2814 if (prot & VM_PROT_EXECUTE) 2815 newpte |= PG_X; 2816 if ((flags & PMAP_ENTER_WIRED) != 0) 2817 newpte |= PG_W; 2818 if (va >= DMAP_MIN_ADDRESS) 2819 newpte |= RPTE_EAA_P; 2820 newpte |= pmap_cache_bits(m->md.mdpg_cache_attrs); 2821 /* 2822 * Set modified bit gratuitously for writeable mappings if 2823 * the page is unmanaged. We do not want to take a fault 2824 * to do the dirty bit accounting for these mappings. 2825 */ 2826 if ((m->oflags & VPO_UNMANAGED) != 0) { 2827 if ((newpte & PG_RW) != 0) 2828 newpte |= PG_M; 2829 } else 2830 newpte |= PG_MANAGED; 2831 2832 lock = NULL; 2833 PMAP_LOCK(pmap); 2834 if (psind == 1) { 2835 /* Assert the required virtual and physical alignment. */ 2836 KASSERT((va & L3_PAGE_MASK) == 0, ("pmap_enter: va unaligned")); 2837 KASSERT(m->psind > 0, ("pmap_enter: m->psind < psind")); 2838 rv = pmap_enter_l3e(pmap, va, newpte | RPTE_LEAF, flags, m, &lock); 2839 goto out; 2840 } 2841 mpte = NULL; 2842 2843 /* 2844 * In the case that a page table page is not 2845 * resident, we are creating it here. 2846 */ 2847 retry: 2848 l3e = pmap_pml3e(pmap, va); 2849 if (l3e != NULL && (be64toh(*l3e) & PG_V) != 0 && ((be64toh(*l3e) & RPTE_LEAF) == 0 || 2850 pmap_demote_l3e_locked(pmap, l3e, va, &lock))) { 2851 pte = pmap_l3e_to_pte(l3e, va); 2852 if (va < VM_MAXUSER_ADDRESS && mpte == NULL) { 2853 mpte = PHYS_TO_VM_PAGE(be64toh(*l3e) & PG_FRAME); 2854 mpte->ref_count++; 2855 } 2856 } else if (va < VM_MAXUSER_ADDRESS) { 2857 /* 2858 * Here if the pte page isn't mapped, or if it has been 2859 * deallocated. 2860 */ 2861 nosleep = (flags & PMAP_ENTER_NOSLEEP) != 0; 2862 mpte = _pmap_allocpte(pmap, pmap_l3e_pindex(va), 2863 nosleep ? NULL : &lock); 2864 if (mpte == NULL && nosleep) { 2865 rv = KERN_RESOURCE_SHORTAGE; 2866 goto out; 2867 } 2868 if (__predict_false(retrycount++ == 6)) 2869 panic("too many retries"); 2870 invalidate_all = true; 2871 goto retry; 2872 } else 2873 panic("pmap_enter: invalid page directory va=%#lx", va); 2874 2875 origpte = be64toh(*pte); 2876 pv = NULL; 2877 2878 /* 2879 * Is the specified virtual address already mapped? 2880 */ 2881 if ((origpte & PG_V) != 0) { 2882 #ifdef INVARIANTS 2883 if (VERBOSE_PMAP || pmap_logging) { 2884 printf("cow fault pmap_enter(%p, %#lx, %p, %#x, %x, %d) --" 2885 " asid=%lu curpid=%d name=%s origpte0x%lx\n", 2886 pmap, va, m, prot, flags, psind, pmap->pm_pid, 2887 curproc->p_pid, curproc->p_comm, origpte); 2888 pmap_pte_walk(pmap->pm_pml1, va); 2889 } 2890 #endif 2891 /* 2892 * Wiring change, just update stats. We don't worry about 2893 * wiring PT pages as they remain resident as long as there 2894 * are valid mappings in them. Hence, if a user page is wired, 2895 * the PT page will be also. 2896 */ 2897 if ((newpte & PG_W) != 0 && (origpte & PG_W) == 0) 2898 pmap->pm_stats.wired_count++; 2899 else if ((newpte & PG_W) == 0 && (origpte & PG_W) != 0) 2900 pmap->pm_stats.wired_count--; 2901 2902 /* 2903 * Remove the extra PT page reference. 2904 */ 2905 if (mpte != NULL) { 2906 mpte->ref_count--; 2907 KASSERT(mpte->ref_count > 0, 2908 ("pmap_enter: missing reference to page table page," 2909 " va: 0x%lx", va)); 2910 } 2911 2912 /* 2913 * Has the physical page changed? 2914 */ 2915 opa = origpte & PG_FRAME; 2916 if (opa == pa) { 2917 /* 2918 * No, might be a protection or wiring change. 2919 */ 2920 if ((origpte & PG_MANAGED) != 0 && 2921 (newpte & PG_RW) != 0) 2922 vm_page_aflag_set(m, PGA_WRITEABLE); 2923 if (((origpte ^ newpte) & ~(PG_M | PG_A)) == 0) { 2924 if ((newpte & (PG_A|PG_M)) != (origpte & (PG_A|PG_M))) { 2925 if (!atomic_cmpset_long(pte, htobe64(origpte), htobe64(newpte))) 2926 goto retry; 2927 if ((newpte & PG_M) != (origpte & PG_M)) 2928 vm_page_dirty(m); 2929 if ((newpte & PG_A) != (origpte & PG_A)) 2930 vm_page_aflag_set(m, PGA_REFERENCED); 2931 ptesync(); 2932 } else 2933 invalidate_all = true; 2934 if (((origpte ^ newpte) & ~(PG_M | PG_A)) == 0) 2935 goto unchanged; 2936 } 2937 goto validate; 2938 } 2939 2940 /* 2941 * The physical page has changed. Temporarily invalidate 2942 * the mapping. This ensures that all threads sharing the 2943 * pmap keep a consistent view of the mapping, which is 2944 * necessary for the correct handling of COW faults. It 2945 * also permits reuse of the old mapping's PV entry, 2946 * avoiding an allocation. 2947 * 2948 * For consistency, handle unmanaged mappings the same way. 2949 */ 2950 origpte = be64toh(pte_load_clear(pte)); 2951 KASSERT((origpte & PG_FRAME) == opa, 2952 ("pmap_enter: unexpected pa update for %#lx", va)); 2953 if ((origpte & PG_MANAGED) != 0) { 2954 om = PHYS_TO_VM_PAGE(opa); 2955 2956 /* 2957 * The pmap lock is sufficient to synchronize with 2958 * concurrent calls to pmap_page_test_mappings() and 2959 * pmap_ts_referenced(). 2960 */ 2961 if ((origpte & (PG_M | PG_RW)) == (PG_M | PG_RW)) 2962 vm_page_dirty(om); 2963 if ((origpte & PG_A) != 0) 2964 vm_page_aflag_set(om, PGA_REFERENCED); 2965 CHANGE_PV_LIST_LOCK_TO_PHYS(&lock, opa); 2966 pv = pmap_pvh_remove(&om->md, pmap, va); 2967 if ((newpte & PG_MANAGED) == 0) 2968 free_pv_entry(pmap, pv); 2969 #ifdef INVARIANTS 2970 else if (origpte & PG_MANAGED) { 2971 if (pv == NULL) { 2972 pmap_page_print_mappings(om); 2973 MPASS(pv != NULL); 2974 } 2975 } 2976 #endif 2977 if ((om->a.flags & PGA_WRITEABLE) != 0 && 2978 TAILQ_EMPTY(&om->md.pv_list) && 2979 ((om->flags & PG_FICTITIOUS) != 0 || 2980 TAILQ_EMPTY(&pa_to_pvh(opa)->pv_list))) 2981 vm_page_aflag_clear(om, PGA_WRITEABLE); 2982 } 2983 if ((origpte & PG_A) != 0) 2984 invalidate_page = true; 2985 origpte = 0; 2986 } else { 2987 if (pmap != kernel_pmap) { 2988 #ifdef INVARIANTS 2989 if (VERBOSE_PMAP || pmap_logging) 2990 printf("pmap_enter(%p, %#lx, %p, %#x, %x, %d) -- asid=%lu curpid=%d name=%s\n", 2991 pmap, va, m, prot, flags, psind, 2992 pmap->pm_pid, curproc->p_pid, 2993 curproc->p_comm); 2994 #endif 2995 } 2996 2997 /* 2998 * Increment the counters. 2999 */ 3000 if ((newpte & PG_W) != 0) 3001 pmap->pm_stats.wired_count++; 3002 pmap_resident_count_inc(pmap, 1); 3003 } 3004 3005 /* 3006 * Enter on the PV list if part of our managed memory. 3007 */ 3008 if ((newpte & PG_MANAGED) != 0) { 3009 if (pv == NULL) { 3010 pv = get_pv_entry(pmap, &lock); 3011 pv->pv_va = va; 3012 } 3013 #ifdef VERBOSE_PV 3014 else 3015 printf("reassigning pv: %p to pmap: %p\n", 3016 pv, pmap); 3017 #endif 3018 CHANGE_PV_LIST_LOCK_TO_PHYS(&lock, pa); 3019 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_link); 3020 m->md.pv_gen++; 3021 if ((newpte & PG_RW) != 0) 3022 vm_page_aflag_set(m, PGA_WRITEABLE); 3023 } 3024 3025 /* 3026 * Update the PTE. 3027 */ 3028 if ((origpte & PG_V) != 0) { 3029 validate: 3030 origpte = be64toh(pte_load_store(pte, htobe64(newpte))); 3031 KASSERT((origpte & PG_FRAME) == pa, 3032 ("pmap_enter: unexpected pa update for %#lx", va)); 3033 if ((newpte & PG_M) == 0 && (origpte & (PG_M | PG_RW)) == 3034 (PG_M | PG_RW)) { 3035 if ((origpte & PG_MANAGED) != 0) 3036 vm_page_dirty(m); 3037 invalidate_page = true; 3038 3039 /* 3040 * Although the PTE may still have PG_RW set, TLB 3041 * invalidation may nonetheless be required because 3042 * the PTE no longer has PG_M set. 3043 */ 3044 } else if ((origpte & PG_X) != 0 || (newpte & PG_X) == 0) { 3045 /* 3046 * Removing capabilities requires invalidation on POWER 3047 */ 3048 invalidate_page = true; 3049 goto unchanged; 3050 } 3051 if ((origpte & PG_A) != 0) 3052 invalidate_page = true; 3053 } else { 3054 pte_store(pte, newpte); 3055 ptesync(); 3056 } 3057 unchanged: 3058 3059 #if VM_NRESERVLEVEL > 0 3060 /* 3061 * If both the page table page and the reservation are fully 3062 * populated, then attempt promotion. 3063 */ 3064 if ((mpte == NULL || mpte->ref_count == NPTEPG) && 3065 mmu_radix_ps_enabled(pmap) && 3066 (m->flags & PG_FICTITIOUS) == 0 && 3067 vm_reserv_level_iffullpop(m) == 0 && 3068 pmap_promote_l3e(pmap, l3e, va, &lock) == 0) 3069 invalidate_all = true; 3070 #endif 3071 if (invalidate_all) 3072 pmap_invalidate_all(pmap); 3073 else if (invalidate_page) 3074 pmap_invalidate_page(pmap, va); 3075 3076 rv = KERN_SUCCESS; 3077 out: 3078 if (lock != NULL) 3079 rw_wunlock(lock); 3080 PMAP_UNLOCK(pmap); 3081 3082 return (rv); 3083 } 3084 3085 /* 3086 * Tries to create a read- and/or execute-only 2MB page mapping. Returns true 3087 * if successful. Returns false if (1) a page table page cannot be allocated 3088 * without sleeping, (2) a mapping already exists at the specified virtual 3089 * address, or (3) a PV entry cannot be allocated without reclaiming another 3090 * PV entry. 3091 */ 3092 static bool 3093 pmap_enter_2mpage(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, 3094 struct rwlock **lockp) 3095 { 3096 pml3_entry_t newpde; 3097 3098 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 3099 newpde = VM_PAGE_TO_PHYS(m) | pmap_cache_bits(m->md.mdpg_cache_attrs) | 3100 RPTE_LEAF | PG_V; 3101 if ((m->oflags & VPO_UNMANAGED) == 0) 3102 newpde |= PG_MANAGED; 3103 if (prot & VM_PROT_EXECUTE) 3104 newpde |= PG_X; 3105 if (prot & VM_PROT_READ) 3106 newpde |= RPTE_EAA_R; 3107 if (va >= DMAP_MIN_ADDRESS) 3108 newpde |= RPTE_EAA_P; 3109 return (pmap_enter_l3e(pmap, va, newpde, PMAP_ENTER_NOSLEEP | 3110 PMAP_ENTER_NOREPLACE | PMAP_ENTER_NORECLAIM, NULL, lockp) == 3111 KERN_SUCCESS); 3112 } 3113 3114 /* 3115 * Tries to create the specified 2MB page mapping. Returns KERN_SUCCESS if 3116 * the mapping was created, and either KERN_FAILURE or KERN_RESOURCE_SHORTAGE 3117 * otherwise. Returns KERN_FAILURE if PMAP_ENTER_NOREPLACE was specified and 3118 * a mapping already exists at the specified virtual address. Returns 3119 * KERN_RESOURCE_SHORTAGE if PMAP_ENTER_NOSLEEP was specified and a page table 3120 * page allocation failed. Returns KERN_RESOURCE_SHORTAGE if 3121 * PMAP_ENTER_NORECLAIM was specified and a PV entry allocation failed. 3122 * 3123 * The parameter "m" is only used when creating a managed, writeable mapping. 3124 */ 3125 static int 3126 pmap_enter_l3e(pmap_t pmap, vm_offset_t va, pml3_entry_t newpde, u_int flags, 3127 vm_page_t m, struct rwlock **lockp) 3128 { 3129 struct spglist free; 3130 pml3_entry_t oldl3e, *l3e; 3131 vm_page_t mt, pdpg; 3132 3133 KASSERT((newpde & (PG_M | PG_RW)) != PG_RW, 3134 ("pmap_enter_pde: newpde is missing PG_M")); 3135 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 3136 3137 if ((pdpg = pmap_allocl3e(pmap, va, (flags & PMAP_ENTER_NOSLEEP) != 0 ? 3138 NULL : lockp)) == NULL) { 3139 CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx" 3140 " in pmap %p", va, pmap); 3141 return (KERN_RESOURCE_SHORTAGE); 3142 } 3143 l3e = (pml3_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pdpg)); 3144 l3e = &l3e[pmap_pml3e_index(va)]; 3145 oldl3e = be64toh(*l3e); 3146 if ((oldl3e & PG_V) != 0) { 3147 KASSERT(pdpg->ref_count > 1, 3148 ("pmap_enter_pde: pdpg's wire count is too low")); 3149 if ((flags & PMAP_ENTER_NOREPLACE) != 0) { 3150 pdpg->ref_count--; 3151 CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx" 3152 " in pmap %p", va, pmap); 3153 return (KERN_FAILURE); 3154 } 3155 /* Break the existing mapping(s). */ 3156 SLIST_INIT(&free); 3157 if ((oldl3e & RPTE_LEAF) != 0) { 3158 /* 3159 * The reference to the PD page that was acquired by 3160 * pmap_allocl3e() ensures that it won't be freed. 3161 * However, if the PDE resulted from a promotion, then 3162 * a reserved PT page could be freed. 3163 */ 3164 (void)pmap_remove_l3e(pmap, l3e, va, &free, lockp); 3165 } else { 3166 if (pmap_remove_ptes(pmap, va, va + L3_PAGE_SIZE, l3e, 3167 &free, lockp)) 3168 pmap_invalidate_all(pmap); 3169 } 3170 vm_page_free_pages_toq(&free, true); 3171 if (va >= VM_MAXUSER_ADDRESS) { 3172 mt = PHYS_TO_VM_PAGE(be64toh(*l3e) & PG_FRAME); 3173 if (pmap_insert_pt_page(pmap, mt)) { 3174 /* 3175 * XXX Currently, this can't happen because 3176 * we do not perform pmap_enter(psind == 1) 3177 * on the kernel pmap. 3178 */ 3179 panic("pmap_enter_pde: trie insert failed"); 3180 } 3181 } else 3182 KASSERT(be64toh(*l3e) == 0, ("pmap_enter_pde: non-zero pde %p", 3183 l3e)); 3184 } 3185 if ((newpde & PG_MANAGED) != 0) { 3186 /* 3187 * Abort this mapping if its PV entry could not be created. 3188 */ 3189 if (!pmap_pv_insert_l3e(pmap, va, newpde, flags, lockp)) { 3190 SLIST_INIT(&free); 3191 if (pmap_unwire_ptp(pmap, va, pdpg, &free)) { 3192 /* 3193 * Although "va" is not mapped, paging- 3194 * structure caches could nonetheless have 3195 * entries that refer to the freed page table 3196 * pages. Invalidate those entries. 3197 */ 3198 pmap_invalidate_page(pmap, va); 3199 vm_page_free_pages_toq(&free, true); 3200 } 3201 CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx" 3202 " in pmap %p", va, pmap); 3203 return (KERN_RESOURCE_SHORTAGE); 3204 } 3205 if ((newpde & PG_RW) != 0) { 3206 for (mt = m; mt < &m[L3_PAGE_SIZE / PAGE_SIZE]; mt++) 3207 vm_page_aflag_set(mt, PGA_WRITEABLE); 3208 } 3209 } 3210 3211 /* 3212 * Increment counters. 3213 */ 3214 if ((newpde & PG_W) != 0) 3215 pmap->pm_stats.wired_count += L3_PAGE_SIZE / PAGE_SIZE; 3216 pmap_resident_count_inc(pmap, L3_PAGE_SIZE / PAGE_SIZE); 3217 3218 /* 3219 * Map the superpage. (This is not a promoted mapping; there will not 3220 * be any lingering 4KB page mappings in the TLB.) 3221 */ 3222 pte_store(l3e, newpde); 3223 3224 atomic_add_long(&pmap_l3e_mappings, 1); 3225 CTR2(KTR_PMAP, "pmap_enter_pde: success for va %#lx" 3226 " in pmap %p", va, pmap); 3227 return (KERN_SUCCESS); 3228 } 3229 3230 void 3231 mmu_radix_enter_object(pmap_t pmap, vm_offset_t start, 3232 vm_offset_t end, vm_page_t m_start, vm_prot_t prot) 3233 { 3234 3235 struct rwlock *lock; 3236 vm_offset_t va; 3237 vm_page_t m, mpte; 3238 vm_pindex_t diff, psize; 3239 bool invalidate; 3240 VM_OBJECT_ASSERT_LOCKED(m_start->object); 3241 3242 CTR6(KTR_PMAP, "%s(%p, %#x, %#x, %p, %#x)", __func__, pmap, start, 3243 end, m_start, prot); 3244 3245 invalidate = false; 3246 psize = atop(end - start); 3247 mpte = NULL; 3248 m = m_start; 3249 lock = NULL; 3250 PMAP_LOCK(pmap); 3251 while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) { 3252 va = start + ptoa(diff); 3253 if ((va & L3_PAGE_MASK) == 0 && va + L3_PAGE_SIZE <= end && 3254 m->psind == 1 && mmu_radix_ps_enabled(pmap) && 3255 pmap_enter_2mpage(pmap, va, m, prot, &lock)) 3256 m = &m[L3_PAGE_SIZE / PAGE_SIZE - 1]; 3257 else 3258 mpte = mmu_radix_enter_quick_locked(pmap, va, m, prot, 3259 mpte, &lock, &invalidate); 3260 m = TAILQ_NEXT(m, listq); 3261 } 3262 ptesync(); 3263 if (lock != NULL) 3264 rw_wunlock(lock); 3265 if (invalidate) 3266 pmap_invalidate_all(pmap); 3267 PMAP_UNLOCK(pmap); 3268 } 3269 3270 static vm_page_t 3271 mmu_radix_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, 3272 vm_prot_t prot, vm_page_t mpte, struct rwlock **lockp, bool *invalidate) 3273 { 3274 struct spglist free; 3275 pt_entry_t *pte; 3276 vm_paddr_t pa; 3277 3278 KASSERT(va < kmi.clean_sva || va >= kmi.clean_eva || 3279 (m->oflags & VPO_UNMANAGED) != 0, 3280 ("mmu_radix_enter_quick_locked: managed mapping within the clean submap")); 3281 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 3282 3283 /* 3284 * In the case that a page table page is not 3285 * resident, we are creating it here. 3286 */ 3287 if (va < VM_MAXUSER_ADDRESS) { 3288 vm_pindex_t ptepindex; 3289 pml3_entry_t *ptepa; 3290 3291 /* 3292 * Calculate pagetable page index 3293 */ 3294 ptepindex = pmap_l3e_pindex(va); 3295 if (mpte && (mpte->pindex == ptepindex)) { 3296 mpte->ref_count++; 3297 } else { 3298 /* 3299 * Get the page directory entry 3300 */ 3301 ptepa = pmap_pml3e(pmap, va); 3302 3303 /* 3304 * If the page table page is mapped, we just increment 3305 * the hold count, and activate it. Otherwise, we 3306 * attempt to allocate a page table page. If this 3307 * attempt fails, we don't retry. Instead, we give up. 3308 */ 3309 if (ptepa && (be64toh(*ptepa) & PG_V) != 0) { 3310 if (be64toh(*ptepa) & RPTE_LEAF) 3311 return (NULL); 3312 mpte = PHYS_TO_VM_PAGE(be64toh(*ptepa) & PG_FRAME); 3313 mpte->ref_count++; 3314 } else { 3315 /* 3316 * Pass NULL instead of the PV list lock 3317 * pointer, because we don't intend to sleep. 3318 */ 3319 mpte = _pmap_allocpte(pmap, ptepindex, NULL); 3320 if (mpte == NULL) 3321 return (mpte); 3322 } 3323 } 3324 pte = (pt_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mpte)); 3325 pte = &pte[pmap_pte_index(va)]; 3326 } else { 3327 mpte = NULL; 3328 pte = pmap_pte(pmap, va); 3329 } 3330 if (be64toh(*pte)) { 3331 if (mpte != NULL) { 3332 mpte->ref_count--; 3333 mpte = NULL; 3334 } 3335 return (mpte); 3336 } 3337 3338 /* 3339 * Enter on the PV list if part of our managed memory. 3340 */ 3341 if ((m->oflags & VPO_UNMANAGED) == 0 && 3342 !pmap_try_insert_pv_entry(pmap, va, m, lockp)) { 3343 if (mpte != NULL) { 3344 SLIST_INIT(&free); 3345 if (pmap_unwire_ptp(pmap, va, mpte, &free)) { 3346 /* 3347 * Although "va" is not mapped, paging- 3348 * structure caches could nonetheless have 3349 * entries that refer to the freed page table 3350 * pages. Invalidate those entries. 3351 */ 3352 *invalidate = true; 3353 vm_page_free_pages_toq(&free, true); 3354 } 3355 mpte = NULL; 3356 } 3357 return (mpte); 3358 } 3359 3360 /* 3361 * Increment counters 3362 */ 3363 pmap_resident_count_inc(pmap, 1); 3364 3365 pa = VM_PAGE_TO_PHYS(m) | pmap_cache_bits(m->md.mdpg_cache_attrs); 3366 if (prot & VM_PROT_EXECUTE) 3367 pa |= PG_X; 3368 else 3369 pa |= RPTE_EAA_R; 3370 if ((m->oflags & VPO_UNMANAGED) == 0) 3371 pa |= PG_MANAGED; 3372 3373 pte_store(pte, pa); 3374 return (mpte); 3375 } 3376 3377 void 3378 mmu_radix_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, 3379 vm_prot_t prot) 3380 { 3381 struct rwlock *lock; 3382 bool invalidate; 3383 3384 lock = NULL; 3385 invalidate = false; 3386 PMAP_LOCK(pmap); 3387 mmu_radix_enter_quick_locked(pmap, va, m, prot, NULL, &lock, 3388 &invalidate); 3389 ptesync(); 3390 if (lock != NULL) 3391 rw_wunlock(lock); 3392 if (invalidate) 3393 pmap_invalidate_all(pmap); 3394 PMAP_UNLOCK(pmap); 3395 } 3396 3397 vm_paddr_t 3398 mmu_radix_extract(pmap_t pmap, vm_offset_t va) 3399 { 3400 pml3_entry_t *l3e; 3401 pt_entry_t *pte; 3402 vm_paddr_t pa; 3403 3404 l3e = pmap_pml3e(pmap, va); 3405 if (__predict_false(l3e == NULL)) 3406 return (0); 3407 if (be64toh(*l3e) & RPTE_LEAF) { 3408 pa = (be64toh(*l3e) & PG_PS_FRAME) | (va & L3_PAGE_MASK); 3409 pa |= (va & L3_PAGE_MASK); 3410 } else { 3411 /* 3412 * Beware of a concurrent promotion that changes the 3413 * PDE at this point! For example, vtopte() must not 3414 * be used to access the PTE because it would use the 3415 * new PDE. It is, however, safe to use the old PDE 3416 * because the page table page is preserved by the 3417 * promotion. 3418 */ 3419 pte = pmap_l3e_to_pte(l3e, va); 3420 if (__predict_false(pte == NULL)) 3421 return (0); 3422 pa = be64toh(*pte); 3423 pa = (pa & PG_FRAME) | (va & PAGE_MASK); 3424 pa |= (va & PAGE_MASK); 3425 } 3426 return (pa); 3427 } 3428 3429 vm_page_t 3430 mmu_radix_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot) 3431 { 3432 pml3_entry_t l3e, *l3ep; 3433 pt_entry_t pte; 3434 vm_paddr_t pa; 3435 vm_page_t m; 3436 3437 pa = 0; 3438 m = NULL; 3439 CTR4(KTR_PMAP, "%s(%p, %#x, %#x)", __func__, pmap, va, prot); 3440 PMAP_LOCK(pmap); 3441 l3ep = pmap_pml3e(pmap, va); 3442 if (l3ep != NULL && (l3e = be64toh(*l3ep))) { 3443 if (l3e & RPTE_LEAF) { 3444 if ((l3e & PG_RW) || (prot & VM_PROT_WRITE) == 0) 3445 m = PHYS_TO_VM_PAGE((l3e & PG_PS_FRAME) | 3446 (va & L3_PAGE_MASK)); 3447 } else { 3448 /* Native endian PTE, do not pass to pmap functions */ 3449 pte = be64toh(*pmap_l3e_to_pte(l3ep, va)); 3450 if ((pte & PG_V) && 3451 ((pte & PG_RW) || (prot & VM_PROT_WRITE) == 0)) 3452 m = PHYS_TO_VM_PAGE(pte & PG_FRAME); 3453 } 3454 if (m != NULL && !vm_page_wire_mapped(m)) 3455 m = NULL; 3456 } 3457 PMAP_UNLOCK(pmap); 3458 return (m); 3459 } 3460 3461 static void 3462 mmu_radix_growkernel(vm_offset_t addr) 3463 { 3464 vm_paddr_t paddr; 3465 vm_page_t nkpg; 3466 pml3_entry_t *l3e; 3467 pml2_entry_t *l2e; 3468 3469 CTR2(KTR_PMAP, "%s(%#x)", __func__, addr); 3470 if (VM_MIN_KERNEL_ADDRESS < addr && 3471 addr < (VM_MIN_KERNEL_ADDRESS + nkpt * L3_PAGE_SIZE)) 3472 return; 3473 3474 addr = roundup2(addr, L3_PAGE_SIZE); 3475 if (addr - 1 >= vm_map_max(kernel_map)) 3476 addr = vm_map_max(kernel_map); 3477 while (kernel_vm_end < addr) { 3478 l2e = pmap_pml2e(kernel_pmap, kernel_vm_end); 3479 if ((be64toh(*l2e) & PG_V) == 0) { 3480 /* We need a new PDP entry */ 3481 nkpg = vm_page_alloc(NULL, kernel_vm_end >> L2_PAGE_SIZE_SHIFT, 3482 VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ | 3483 VM_ALLOC_WIRED | VM_ALLOC_ZERO); 3484 if (nkpg == NULL) 3485 panic("pmap_growkernel: no memory to grow kernel"); 3486 if ((nkpg->flags & PG_ZERO) == 0) 3487 mmu_radix_zero_page(nkpg); 3488 paddr = VM_PAGE_TO_PHYS(nkpg); 3489 pde_store(l2e, paddr); 3490 continue; /* try again */ 3491 } 3492 l3e = pmap_l2e_to_l3e(l2e, kernel_vm_end); 3493 if ((be64toh(*l3e) & PG_V) != 0) { 3494 kernel_vm_end = (kernel_vm_end + L3_PAGE_SIZE) & ~L3_PAGE_MASK; 3495 if (kernel_vm_end - 1 >= vm_map_max(kernel_map)) { 3496 kernel_vm_end = vm_map_max(kernel_map); 3497 break; 3498 } 3499 continue; 3500 } 3501 3502 nkpg = vm_page_alloc(NULL, pmap_l3e_pindex(kernel_vm_end), 3503 VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | 3504 VM_ALLOC_ZERO); 3505 if (nkpg == NULL) 3506 panic("pmap_growkernel: no memory to grow kernel"); 3507 if ((nkpg->flags & PG_ZERO) == 0) 3508 mmu_radix_zero_page(nkpg); 3509 paddr = VM_PAGE_TO_PHYS(nkpg); 3510 pde_store(l3e, paddr); 3511 3512 kernel_vm_end = (kernel_vm_end + L3_PAGE_SIZE) & ~L3_PAGE_MASK; 3513 if (kernel_vm_end - 1 >= vm_map_max(kernel_map)) { 3514 kernel_vm_end = vm_map_max(kernel_map); 3515 break; 3516 } 3517 } 3518 ptesync(); 3519 } 3520 3521 static MALLOC_DEFINE(M_RADIX_PGD, "radix_pgd", "radix page table root directory"); 3522 static uma_zone_t zone_radix_pgd; 3523 3524 static int 3525 radix_pgd_import(void *arg __unused, void **store, int count, int domain __unused, 3526 int flags) 3527 { 3528 3529 for (int i = 0; i < count; i++) { 3530 vm_page_t m = vm_page_alloc_contig(NULL, 0, 3531 VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | 3532 VM_ALLOC_ZERO | VM_ALLOC_WAITOK, RADIX_PGD_SIZE/PAGE_SIZE, 3533 0, (vm_paddr_t)-1, RADIX_PGD_SIZE, L1_PAGE_SIZE, 3534 VM_MEMATTR_DEFAULT); 3535 /* XXX zero on alloc here so we don't have to later */ 3536 store[i] = (void *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)); 3537 } 3538 return (count); 3539 } 3540 3541 static void 3542 radix_pgd_release(void *arg __unused, void **store, int count) 3543 { 3544 vm_page_t m; 3545 struct spglist free; 3546 int page_count; 3547 3548 SLIST_INIT(&free); 3549 page_count = RADIX_PGD_SIZE/PAGE_SIZE; 3550 3551 for (int i = 0; i < count; i++) { 3552 /* 3553 * XXX selectively remove dmap and KVA entries so we don't 3554 * need to bzero 3555 */ 3556 m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)store[i])); 3557 for (int j = page_count-1; j >= 0; j--) { 3558 vm_page_unwire_noq(&m[j]); 3559 SLIST_INSERT_HEAD(&free, &m[j], plinks.s.ss); 3560 } 3561 vm_page_free_pages_toq(&free, false); 3562 } 3563 } 3564 3565 static void 3566 mmu_radix_init() 3567 { 3568 vm_page_t mpte; 3569 vm_size_t s; 3570 int error, i, pv_npg; 3571 3572 /* XXX is this really needed for POWER? */ 3573 /* L1TF, reserve page @0 unconditionally */ 3574 vm_page_blacklist_add(0, bootverbose); 3575 3576 zone_radix_pgd = uma_zcache_create("radix_pgd_cache", 3577 RADIX_PGD_SIZE, NULL, NULL, 3578 #ifdef INVARIANTS 3579 trash_init, trash_fini, 3580 #else 3581 NULL, NULL, 3582 #endif 3583 radix_pgd_import, radix_pgd_release, 3584 NULL, UMA_ZONE_NOBUCKET); 3585 3586 /* 3587 * Initialize the vm page array entries for the kernel pmap's 3588 * page table pages. 3589 */ 3590 PMAP_LOCK(kernel_pmap); 3591 for (i = 0; i < nkpt; i++) { 3592 mpte = PHYS_TO_VM_PAGE(KPTphys + (i << PAGE_SHIFT)); 3593 KASSERT(mpte >= vm_page_array && 3594 mpte < &vm_page_array[vm_page_array_size], 3595 ("pmap_init: page table page is out of range size: %lu", 3596 vm_page_array_size)); 3597 mpte->pindex = pmap_l3e_pindex(VM_MIN_KERNEL_ADDRESS) + i; 3598 mpte->phys_addr = KPTphys + (i << PAGE_SHIFT); 3599 MPASS(PHYS_TO_VM_PAGE(mpte->phys_addr) == mpte); 3600 //pmap_insert_pt_page(kernel_pmap, mpte); 3601 mpte->ref_count = 1; 3602 } 3603 PMAP_UNLOCK(kernel_pmap); 3604 vm_wire_add(nkpt); 3605 3606 CTR1(KTR_PMAP, "%s()", __func__); 3607 TAILQ_INIT(&pv_dummy.pv_list); 3608 3609 /* 3610 * Are large page mappings enabled? 3611 */ 3612 TUNABLE_INT_FETCH("vm.pmap.pg_ps_enabled", &pg_ps_enabled); 3613 if (pg_ps_enabled) { 3614 KASSERT(MAXPAGESIZES > 1 && pagesizes[1] == 0, 3615 ("pmap_init: can't assign to pagesizes[1]")); 3616 pagesizes[1] = L3_PAGE_SIZE; 3617 } 3618 3619 /* 3620 * Initialize the pv chunk list mutex. 3621 */ 3622 mtx_init(&pv_chunks_mutex, "pmap pv chunk list", NULL, MTX_DEF); 3623 3624 /* 3625 * Initialize the pool of pv list locks. 3626 */ 3627 for (i = 0; i < NPV_LIST_LOCKS; i++) 3628 rw_init(&pv_list_locks[i], "pmap pv list"); 3629 3630 /* 3631 * Calculate the size of the pv head table for superpages. 3632 */ 3633 pv_npg = howmany(vm_phys_segs[vm_phys_nsegs - 1].end, L3_PAGE_SIZE); 3634 3635 /* 3636 * Allocate memory for the pv head table for superpages. 3637 */ 3638 s = (vm_size_t)(pv_npg * sizeof(struct md_page)); 3639 s = round_page(s); 3640 pv_table = (struct md_page *)kmem_malloc(s, M_WAITOK | M_ZERO); 3641 for (i = 0; i < pv_npg; i++) 3642 TAILQ_INIT(&pv_table[i].pv_list); 3643 TAILQ_INIT(&pv_dummy.pv_list); 3644 3645 pmap_initialized = 1; 3646 mtx_init(&qframe_mtx, "qfrmlk", NULL, MTX_SPIN); 3647 error = vmem_alloc(kernel_arena, PAGE_SIZE, M_BESTFIT | M_WAITOK, 3648 (vmem_addr_t *)&qframe); 3649 3650 if (error != 0) 3651 panic("qframe allocation failed"); 3652 asid_arena = vmem_create("ASID", isa3_base_pid + 1, (1<<isa3_pid_bits), 3653 1, 1, M_WAITOK); 3654 } 3655 3656 static boolean_t 3657 pmap_page_test_mappings(vm_page_t m, boolean_t accessed, boolean_t modified) 3658 { 3659 struct rwlock *lock; 3660 pv_entry_t pv; 3661 struct md_page *pvh; 3662 pt_entry_t *pte, mask; 3663 pmap_t pmap; 3664 int md_gen, pvh_gen; 3665 boolean_t rv; 3666 3667 rv = FALSE; 3668 lock = VM_PAGE_TO_PV_LIST_LOCK(m); 3669 rw_rlock(lock); 3670 restart: 3671 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) { 3672 pmap = PV_PMAP(pv); 3673 if (!PMAP_TRYLOCK(pmap)) { 3674 md_gen = m->md.pv_gen; 3675 rw_runlock(lock); 3676 PMAP_LOCK(pmap); 3677 rw_rlock(lock); 3678 if (md_gen != m->md.pv_gen) { 3679 PMAP_UNLOCK(pmap); 3680 goto restart; 3681 } 3682 } 3683 pte = pmap_pte(pmap, pv->pv_va); 3684 mask = 0; 3685 if (modified) 3686 mask |= PG_RW | PG_M; 3687 if (accessed) 3688 mask |= PG_V | PG_A; 3689 rv = (be64toh(*pte) & mask) == mask; 3690 PMAP_UNLOCK(pmap); 3691 if (rv) 3692 goto out; 3693 } 3694 if ((m->flags & PG_FICTITIOUS) == 0) { 3695 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); 3696 TAILQ_FOREACH(pv, &pvh->pv_list, pv_link) { 3697 pmap = PV_PMAP(pv); 3698 if (!PMAP_TRYLOCK(pmap)) { 3699 md_gen = m->md.pv_gen; 3700 pvh_gen = pvh->pv_gen; 3701 rw_runlock(lock); 3702 PMAP_LOCK(pmap); 3703 rw_rlock(lock); 3704 if (md_gen != m->md.pv_gen || 3705 pvh_gen != pvh->pv_gen) { 3706 PMAP_UNLOCK(pmap); 3707 goto restart; 3708 } 3709 } 3710 pte = pmap_pml3e(pmap, pv->pv_va); 3711 mask = 0; 3712 if (modified) 3713 mask |= PG_RW | PG_M; 3714 if (accessed) 3715 mask |= PG_V | PG_A; 3716 rv = (be64toh(*pte) & mask) == mask; 3717 PMAP_UNLOCK(pmap); 3718 if (rv) 3719 goto out; 3720 } 3721 } 3722 out: 3723 rw_runlock(lock); 3724 return (rv); 3725 } 3726 3727 /* 3728 * pmap_is_modified: 3729 * 3730 * Return whether or not the specified physical page was modified 3731 * in any physical maps. 3732 */ 3733 boolean_t 3734 mmu_radix_is_modified(vm_page_t m) 3735 { 3736 3737 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 3738 ("pmap_is_modified: page %p is not managed", m)); 3739 3740 CTR2(KTR_PMAP, "%s(%p)", __func__, m); 3741 /* 3742 * If the page is not busied then this check is racy. 3743 */ 3744 if (!pmap_page_is_write_mapped(m)) 3745 return (FALSE); 3746 return (pmap_page_test_mappings(m, FALSE, TRUE)); 3747 } 3748 3749 boolean_t 3750 mmu_radix_is_prefaultable(pmap_t pmap, vm_offset_t addr) 3751 { 3752 pml3_entry_t *l3e; 3753 pt_entry_t *pte; 3754 boolean_t rv; 3755 3756 CTR3(KTR_PMAP, "%s(%p, %#x)", __func__, pmap, addr); 3757 rv = FALSE; 3758 PMAP_LOCK(pmap); 3759 l3e = pmap_pml3e(pmap, addr); 3760 if (l3e != NULL && (be64toh(*l3e) & (RPTE_LEAF | PG_V)) == PG_V) { 3761 pte = pmap_l3e_to_pte(l3e, addr); 3762 rv = (be64toh(*pte) & PG_V) == 0; 3763 } 3764 PMAP_UNLOCK(pmap); 3765 return (rv); 3766 } 3767 3768 boolean_t 3769 mmu_radix_is_referenced(vm_page_t m) 3770 { 3771 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 3772 ("pmap_is_referenced: page %p is not managed", m)); 3773 CTR2(KTR_PMAP, "%s(%p)", __func__, m); 3774 return (pmap_page_test_mappings(m, TRUE, FALSE)); 3775 } 3776 3777 /* 3778 * pmap_ts_referenced: 3779 * 3780 * Return a count of reference bits for a page, clearing those bits. 3781 * It is not necessary for every reference bit to be cleared, but it 3782 * is necessary that 0 only be returned when there are truly no 3783 * reference bits set. 3784 * 3785 * As an optimization, update the page's dirty field if a modified bit is 3786 * found while counting reference bits. This opportunistic update can be 3787 * performed at low cost and can eliminate the need for some future calls 3788 * to pmap_is_modified(). However, since this function stops after 3789 * finding PMAP_TS_REFERENCED_MAX reference bits, it may not detect some 3790 * dirty pages. Those dirty pages will only be detected by a future call 3791 * to pmap_is_modified(). 3792 * 3793 * A DI block is not needed within this function, because 3794 * invalidations are performed before the PV list lock is 3795 * released. 3796 */ 3797 boolean_t 3798 mmu_radix_ts_referenced(vm_page_t m) 3799 { 3800 struct md_page *pvh; 3801 pv_entry_t pv, pvf; 3802 pmap_t pmap; 3803 struct rwlock *lock; 3804 pml3_entry_t oldl3e, *l3e; 3805 pt_entry_t *pte; 3806 vm_paddr_t pa; 3807 int cleared, md_gen, not_cleared, pvh_gen; 3808 struct spglist free; 3809 3810 CTR2(KTR_PMAP, "%s(%p)", __func__, m); 3811 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 3812 ("pmap_ts_referenced: page %p is not managed", m)); 3813 SLIST_INIT(&free); 3814 cleared = 0; 3815 pa = VM_PAGE_TO_PHYS(m); 3816 lock = PHYS_TO_PV_LIST_LOCK(pa); 3817 pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy : pa_to_pvh(pa); 3818 rw_wlock(lock); 3819 retry: 3820 not_cleared = 0; 3821 if ((pvf = TAILQ_FIRST(&pvh->pv_list)) == NULL) 3822 goto small_mappings; 3823 pv = pvf; 3824 do { 3825 if (pvf == NULL) 3826 pvf = pv; 3827 pmap = PV_PMAP(pv); 3828 if (!PMAP_TRYLOCK(pmap)) { 3829 pvh_gen = pvh->pv_gen; 3830 rw_wunlock(lock); 3831 PMAP_LOCK(pmap); 3832 rw_wlock(lock); 3833 if (pvh_gen != pvh->pv_gen) { 3834 PMAP_UNLOCK(pmap); 3835 goto retry; 3836 } 3837 } 3838 l3e = pmap_pml3e(pmap, pv->pv_va); 3839 oldl3e = be64toh(*l3e); 3840 if ((oldl3e & (PG_M | PG_RW)) == (PG_M | PG_RW)) { 3841 /* 3842 * Although "oldpde" is mapping a 2MB page, because 3843 * this function is called at a 4KB page granularity, 3844 * we only update the 4KB page under test. 3845 */ 3846 vm_page_dirty(m); 3847 } 3848 if ((oldl3e & PG_A) != 0) { 3849 /* 3850 * Since this reference bit is shared by 512 4KB 3851 * pages, it should not be cleared every time it is 3852 * tested. Apply a simple "hash" function on the 3853 * physical page number, the virtual superpage number, 3854 * and the pmap address to select one 4KB page out of 3855 * the 512 on which testing the reference bit will 3856 * result in clearing that reference bit. This 3857 * function is designed to avoid the selection of the 3858 * same 4KB page for every 2MB page mapping. 3859 * 3860 * On demotion, a mapping that hasn't been referenced 3861 * is simply destroyed. To avoid the possibility of a 3862 * subsequent page fault on a demoted wired mapping, 3863 * always leave its reference bit set. Moreover, 3864 * since the superpage is wired, the current state of 3865 * its reference bit won't affect page replacement. 3866 */ 3867 if ((((pa >> PAGE_SHIFT) ^ (pv->pv_va >> L3_PAGE_SIZE_SHIFT) ^ 3868 (uintptr_t)pmap) & (NPTEPG - 1)) == 0 && 3869 (oldl3e & PG_W) == 0) { 3870 atomic_clear_long(l3e, htobe64(PG_A)); 3871 pmap_invalidate_page(pmap, pv->pv_va); 3872 cleared++; 3873 KASSERT(lock == VM_PAGE_TO_PV_LIST_LOCK(m), 3874 ("inconsistent pv lock %p %p for page %p", 3875 lock, VM_PAGE_TO_PV_LIST_LOCK(m), m)); 3876 } else 3877 not_cleared++; 3878 } 3879 PMAP_UNLOCK(pmap); 3880 /* Rotate the PV list if it has more than one entry. */ 3881 if (pv != NULL && TAILQ_NEXT(pv, pv_link) != NULL) { 3882 TAILQ_REMOVE(&pvh->pv_list, pv, pv_link); 3883 TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_link); 3884 pvh->pv_gen++; 3885 } 3886 if (cleared + not_cleared >= PMAP_TS_REFERENCED_MAX) 3887 goto out; 3888 } while ((pv = TAILQ_FIRST(&pvh->pv_list)) != pvf); 3889 small_mappings: 3890 if ((pvf = TAILQ_FIRST(&m->md.pv_list)) == NULL) 3891 goto out; 3892 pv = pvf; 3893 do { 3894 if (pvf == NULL) 3895 pvf = pv; 3896 pmap = PV_PMAP(pv); 3897 if (!PMAP_TRYLOCK(pmap)) { 3898 pvh_gen = pvh->pv_gen; 3899 md_gen = m->md.pv_gen; 3900 rw_wunlock(lock); 3901 PMAP_LOCK(pmap); 3902 rw_wlock(lock); 3903 if (pvh_gen != pvh->pv_gen || md_gen != m->md.pv_gen) { 3904 PMAP_UNLOCK(pmap); 3905 goto retry; 3906 } 3907 } 3908 l3e = pmap_pml3e(pmap, pv->pv_va); 3909 KASSERT((be64toh(*l3e) & RPTE_LEAF) == 0, 3910 ("pmap_ts_referenced: found a 2mpage in page %p's pv list", 3911 m)); 3912 pte = pmap_l3e_to_pte(l3e, pv->pv_va); 3913 if ((be64toh(*pte) & (PG_M | PG_RW)) == (PG_M | PG_RW)) 3914 vm_page_dirty(m); 3915 if ((be64toh(*pte) & PG_A) != 0) { 3916 atomic_clear_long(pte, htobe64(PG_A)); 3917 pmap_invalidate_page(pmap, pv->pv_va); 3918 cleared++; 3919 } 3920 PMAP_UNLOCK(pmap); 3921 /* Rotate the PV list if it has more than one entry. */ 3922 if (pv != NULL && TAILQ_NEXT(pv, pv_link) != NULL) { 3923 TAILQ_REMOVE(&m->md.pv_list, pv, pv_link); 3924 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_link); 3925 m->md.pv_gen++; 3926 } 3927 } while ((pv = TAILQ_FIRST(&m->md.pv_list)) != pvf && cleared + 3928 not_cleared < PMAP_TS_REFERENCED_MAX); 3929 out: 3930 rw_wunlock(lock); 3931 vm_page_free_pages_toq(&free, true); 3932 return (cleared + not_cleared); 3933 } 3934 3935 static vm_offset_t 3936 mmu_radix_map(vm_offset_t *virt __unused, vm_paddr_t start, 3937 vm_paddr_t end, int prot __unused) 3938 { 3939 3940 CTR5(KTR_PMAP, "%s(%p, %#x, %#x, %#x)", __func__, virt, start, end, 3941 prot); 3942 return (PHYS_TO_DMAP(start)); 3943 } 3944 3945 void 3946 mmu_radix_object_init_pt(pmap_t pmap, vm_offset_t addr, 3947 vm_object_t object, vm_pindex_t pindex, vm_size_t size) 3948 { 3949 pml3_entry_t *l3e; 3950 vm_paddr_t pa, ptepa; 3951 vm_page_t p, pdpg; 3952 vm_memattr_t ma; 3953 3954 CTR6(KTR_PMAP, "%s(%p, %#x, %p, %u, %#x)", __func__, pmap, addr, 3955 object, pindex, size); 3956 VM_OBJECT_ASSERT_WLOCKED(object); 3957 KASSERT(object->type == OBJT_DEVICE || object->type == OBJT_SG, 3958 ("pmap_object_init_pt: non-device object")); 3959 /* NB: size can be logically ored with addr here */ 3960 if ((addr & L3_PAGE_MASK) == 0 && (size & L3_PAGE_MASK) == 0) { 3961 if (!mmu_radix_ps_enabled(pmap)) 3962 return; 3963 if (!vm_object_populate(object, pindex, pindex + atop(size))) 3964 return; 3965 p = vm_page_lookup(object, pindex); 3966 KASSERT(p->valid == VM_PAGE_BITS_ALL, 3967 ("pmap_object_init_pt: invalid page %p", p)); 3968 ma = p->md.mdpg_cache_attrs; 3969 3970 /* 3971 * Abort the mapping if the first page is not physically 3972 * aligned to a 2MB page boundary. 3973 */ 3974 ptepa = VM_PAGE_TO_PHYS(p); 3975 if (ptepa & L3_PAGE_MASK) 3976 return; 3977 3978 /* 3979 * Skip the first page. Abort the mapping if the rest of 3980 * the pages are not physically contiguous or have differing 3981 * memory attributes. 3982 */ 3983 p = TAILQ_NEXT(p, listq); 3984 for (pa = ptepa + PAGE_SIZE; pa < ptepa + size; 3985 pa += PAGE_SIZE) { 3986 KASSERT(p->valid == VM_PAGE_BITS_ALL, 3987 ("pmap_object_init_pt: invalid page %p", p)); 3988 if (pa != VM_PAGE_TO_PHYS(p) || 3989 ma != p->md.mdpg_cache_attrs) 3990 return; 3991 p = TAILQ_NEXT(p, listq); 3992 } 3993 3994 PMAP_LOCK(pmap); 3995 for (pa = ptepa | pmap_cache_bits(ma); 3996 pa < ptepa + size; pa += L3_PAGE_SIZE) { 3997 pdpg = pmap_allocl3e(pmap, addr, NULL); 3998 if (pdpg == NULL) { 3999 /* 4000 * The creation of mappings below is only an 4001 * optimization. If a page directory page 4002 * cannot be allocated without blocking, 4003 * continue on to the next mapping rather than 4004 * blocking. 4005 */ 4006 addr += L3_PAGE_SIZE; 4007 continue; 4008 } 4009 l3e = (pml3_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pdpg)); 4010 l3e = &l3e[pmap_pml3e_index(addr)]; 4011 if ((be64toh(*l3e) & PG_V) == 0) { 4012 pa |= PG_M | PG_A | PG_RW; 4013 pte_store(l3e, pa); 4014 pmap_resident_count_inc(pmap, L3_PAGE_SIZE / PAGE_SIZE); 4015 atomic_add_long(&pmap_l3e_mappings, 1); 4016 } else { 4017 /* Continue on if the PDE is already valid. */ 4018 pdpg->ref_count--; 4019 KASSERT(pdpg->ref_count > 0, 4020 ("pmap_object_init_pt: missing reference " 4021 "to page directory page, va: 0x%lx", addr)); 4022 } 4023 addr += L3_PAGE_SIZE; 4024 } 4025 ptesync(); 4026 PMAP_UNLOCK(pmap); 4027 } 4028 } 4029 4030 boolean_t 4031 mmu_radix_page_exists_quick(pmap_t pmap, vm_page_t m) 4032 { 4033 struct md_page *pvh; 4034 struct rwlock *lock; 4035 pv_entry_t pv; 4036 int loops = 0; 4037 boolean_t rv; 4038 4039 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 4040 ("pmap_page_exists_quick: page %p is not managed", m)); 4041 CTR3(KTR_PMAP, "%s(%p, %p)", __func__, pmap, m); 4042 rv = FALSE; 4043 lock = VM_PAGE_TO_PV_LIST_LOCK(m); 4044 rw_rlock(lock); 4045 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) { 4046 if (PV_PMAP(pv) == pmap) { 4047 rv = TRUE; 4048 break; 4049 } 4050 loops++; 4051 if (loops >= 16) 4052 break; 4053 } 4054 if (!rv && loops < 16 && (m->flags & PG_FICTITIOUS) == 0) { 4055 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); 4056 TAILQ_FOREACH(pv, &pvh->pv_list, pv_link) { 4057 if (PV_PMAP(pv) == pmap) { 4058 rv = TRUE; 4059 break; 4060 } 4061 loops++; 4062 if (loops >= 16) 4063 break; 4064 } 4065 } 4066 rw_runlock(lock); 4067 return (rv); 4068 } 4069 4070 void 4071 mmu_radix_page_init(vm_page_t m) 4072 { 4073 4074 CTR2(KTR_PMAP, "%s(%p)", __func__, m); 4075 TAILQ_INIT(&m->md.pv_list); 4076 m->md.mdpg_cache_attrs = VM_MEMATTR_DEFAULT; 4077 } 4078 4079 int 4080 mmu_radix_page_wired_mappings(vm_page_t m) 4081 { 4082 struct rwlock *lock; 4083 struct md_page *pvh; 4084 pmap_t pmap; 4085 pt_entry_t *pte; 4086 pv_entry_t pv; 4087 int count, md_gen, pvh_gen; 4088 4089 if ((m->oflags & VPO_UNMANAGED) != 0) 4090 return (0); 4091 CTR2(KTR_PMAP, "%s(%p)", __func__, m); 4092 lock = VM_PAGE_TO_PV_LIST_LOCK(m); 4093 rw_rlock(lock); 4094 restart: 4095 count = 0; 4096 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) { 4097 pmap = PV_PMAP(pv); 4098 if (!PMAP_TRYLOCK(pmap)) { 4099 md_gen = m->md.pv_gen; 4100 rw_runlock(lock); 4101 PMAP_LOCK(pmap); 4102 rw_rlock(lock); 4103 if (md_gen != m->md.pv_gen) { 4104 PMAP_UNLOCK(pmap); 4105 goto restart; 4106 } 4107 } 4108 pte = pmap_pte(pmap, pv->pv_va); 4109 if ((be64toh(*pte) & PG_W) != 0) 4110 count++; 4111 PMAP_UNLOCK(pmap); 4112 } 4113 if ((m->flags & PG_FICTITIOUS) == 0) { 4114 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); 4115 TAILQ_FOREACH(pv, &pvh->pv_list, pv_link) { 4116 pmap = PV_PMAP(pv); 4117 if (!PMAP_TRYLOCK(pmap)) { 4118 md_gen = m->md.pv_gen; 4119 pvh_gen = pvh->pv_gen; 4120 rw_runlock(lock); 4121 PMAP_LOCK(pmap); 4122 rw_rlock(lock); 4123 if (md_gen != m->md.pv_gen || 4124 pvh_gen != pvh->pv_gen) { 4125 PMAP_UNLOCK(pmap); 4126 goto restart; 4127 } 4128 } 4129 pte = pmap_pml3e(pmap, pv->pv_va); 4130 if ((be64toh(*pte) & PG_W) != 0) 4131 count++; 4132 PMAP_UNLOCK(pmap); 4133 } 4134 } 4135 rw_runlock(lock); 4136 return (count); 4137 } 4138 4139 static void 4140 mmu_radix_update_proctab(int pid, pml1_entry_t l1pa) 4141 { 4142 isa3_proctab[pid].proctab0 = htobe64(RTS_SIZE | l1pa | RADIX_PGD_INDEX_SHIFT); 4143 } 4144 4145 int 4146 mmu_radix_pinit(pmap_t pmap) 4147 { 4148 vmem_addr_t pid; 4149 vm_paddr_t l1pa; 4150 4151 CTR2(KTR_PMAP, "%s(%p)", __func__, pmap); 4152 4153 /* 4154 * allocate the page directory page 4155 */ 4156 pmap->pm_pml1 = uma_zalloc(zone_radix_pgd, M_WAITOK); 4157 4158 for (int j = 0; j < RADIX_PGD_SIZE_SHIFT; j++) 4159 pagezero((vm_offset_t)pmap->pm_pml1 + j * PAGE_SIZE); 4160 pmap->pm_radix.rt_root = 0; 4161 TAILQ_INIT(&pmap->pm_pvchunk); 4162 bzero(&pmap->pm_stats, sizeof pmap->pm_stats); 4163 pmap->pm_flags = PMAP_PDE_SUPERPAGE; 4164 vmem_alloc(asid_arena, 1, M_FIRSTFIT|M_WAITOK, &pid); 4165 4166 pmap->pm_pid = pid; 4167 l1pa = DMAP_TO_PHYS((vm_offset_t)pmap->pm_pml1); 4168 mmu_radix_update_proctab(pid, l1pa); 4169 __asm __volatile("ptesync;isync" : : : "memory"); 4170 4171 return (1); 4172 } 4173 4174 /* 4175 * This routine is called if the desired page table page does not exist. 4176 * 4177 * If page table page allocation fails, this routine may sleep before 4178 * returning NULL. It sleeps only if a lock pointer was given. 4179 * 4180 * Note: If a page allocation fails at page table level two or three, 4181 * one or two pages may be held during the wait, only to be released 4182 * afterwards. This conservative approach is easily argued to avoid 4183 * race conditions. 4184 */ 4185 static vm_page_t 4186 _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, struct rwlock **lockp) 4187 { 4188 vm_page_t m, pdppg, pdpg; 4189 4190 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 4191 4192 /* 4193 * Allocate a page table page. 4194 */ 4195 if ((m = vm_page_alloc(NULL, ptepindex, VM_ALLOC_NOOBJ | 4196 VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) { 4197 if (lockp != NULL) { 4198 RELEASE_PV_LIST_LOCK(lockp); 4199 PMAP_UNLOCK(pmap); 4200 vm_wait(NULL); 4201 PMAP_LOCK(pmap); 4202 } 4203 /* 4204 * Indicate the need to retry. While waiting, the page table 4205 * page may have been allocated. 4206 */ 4207 return (NULL); 4208 } 4209 if ((m->flags & PG_ZERO) == 0) 4210 mmu_radix_zero_page(m); 4211 4212 /* 4213 * Map the pagetable page into the process address space, if 4214 * it isn't already there. 4215 */ 4216 4217 if (ptepindex >= (NUPDE + NUPDPE)) { 4218 pml1_entry_t *l1e; 4219 vm_pindex_t pml1index; 4220 4221 /* Wire up a new PDPE page */ 4222 pml1index = ptepindex - (NUPDE + NUPDPE); 4223 l1e = &pmap->pm_pml1[pml1index]; 4224 pde_store(l1e, VM_PAGE_TO_PHYS(m)); 4225 4226 } else if (ptepindex >= NUPDE) { 4227 vm_pindex_t pml1index; 4228 vm_pindex_t pdpindex; 4229 pml1_entry_t *l1e; 4230 pml2_entry_t *l2e; 4231 4232 /* Wire up a new l2e page */ 4233 pdpindex = ptepindex - NUPDE; 4234 pml1index = pdpindex >> RPTE_SHIFT; 4235 4236 l1e = &pmap->pm_pml1[pml1index]; 4237 if ((be64toh(*l1e) & PG_V) == 0) { 4238 /* Have to allocate a new pdp, recurse */ 4239 if (_pmap_allocpte(pmap, NUPDE + NUPDPE + pml1index, 4240 lockp) == NULL) { 4241 vm_page_unwire_noq(m); 4242 vm_page_free_zero(m); 4243 return (NULL); 4244 } 4245 } else { 4246 /* Add reference to l2e page */ 4247 pdppg = PHYS_TO_VM_PAGE(be64toh(*l1e) & PG_FRAME); 4248 pdppg->ref_count++; 4249 } 4250 l2e = (pml2_entry_t *)PHYS_TO_DMAP(be64toh(*l1e) & PG_FRAME); 4251 4252 /* Now find the pdp page */ 4253 l2e = &l2e[pdpindex & RPTE_MASK]; 4254 pde_store(l2e, VM_PAGE_TO_PHYS(m)); 4255 4256 } else { 4257 vm_pindex_t pml1index; 4258 vm_pindex_t pdpindex; 4259 pml1_entry_t *l1e; 4260 pml2_entry_t *l2e; 4261 pml3_entry_t *l3e; 4262 4263 /* Wire up a new PTE page */ 4264 pdpindex = ptepindex >> RPTE_SHIFT; 4265 pml1index = pdpindex >> RPTE_SHIFT; 4266 4267 /* First, find the pdp and check that its valid. */ 4268 l1e = &pmap->pm_pml1[pml1index]; 4269 if ((be64toh(*l1e) & PG_V) == 0) { 4270 /* Have to allocate a new pd, recurse */ 4271 if (_pmap_allocpte(pmap, NUPDE + pdpindex, 4272 lockp) == NULL) { 4273 vm_page_unwire_noq(m); 4274 vm_page_free_zero(m); 4275 return (NULL); 4276 } 4277 l2e = (pml2_entry_t *)PHYS_TO_DMAP(be64toh(*l1e) & PG_FRAME); 4278 l2e = &l2e[pdpindex & RPTE_MASK]; 4279 } else { 4280 l2e = (pml2_entry_t *)PHYS_TO_DMAP(be64toh(*l1e) & PG_FRAME); 4281 l2e = &l2e[pdpindex & RPTE_MASK]; 4282 if ((be64toh(*l2e) & PG_V) == 0) { 4283 /* Have to allocate a new pd, recurse */ 4284 if (_pmap_allocpte(pmap, NUPDE + pdpindex, 4285 lockp) == NULL) { 4286 vm_page_unwire_noq(m); 4287 vm_page_free_zero(m); 4288 return (NULL); 4289 } 4290 } else { 4291 /* Add reference to the pd page */ 4292 pdpg = PHYS_TO_VM_PAGE(be64toh(*l2e) & PG_FRAME); 4293 pdpg->ref_count++; 4294 } 4295 } 4296 l3e = (pml3_entry_t *)PHYS_TO_DMAP(be64toh(*l2e) & PG_FRAME); 4297 4298 /* Now we know where the page directory page is */ 4299 l3e = &l3e[ptepindex & RPTE_MASK]; 4300 pde_store(l3e, VM_PAGE_TO_PHYS(m)); 4301 } 4302 4303 pmap_resident_count_inc(pmap, 1); 4304 return (m); 4305 } 4306 static vm_page_t 4307 pmap_allocl3e(pmap_t pmap, vm_offset_t va, struct rwlock **lockp) 4308 { 4309 vm_pindex_t pdpindex, ptepindex; 4310 pml2_entry_t *pdpe; 4311 vm_page_t pdpg; 4312 4313 retry: 4314 pdpe = pmap_pml2e(pmap, va); 4315 if (pdpe != NULL && (be64toh(*pdpe) & PG_V) != 0) { 4316 /* Add a reference to the pd page. */ 4317 pdpg = PHYS_TO_VM_PAGE(be64toh(*pdpe) & PG_FRAME); 4318 pdpg->ref_count++; 4319 } else { 4320 /* Allocate a pd page. */ 4321 ptepindex = pmap_l3e_pindex(va); 4322 pdpindex = ptepindex >> RPTE_SHIFT; 4323 pdpg = _pmap_allocpte(pmap, NUPDE + pdpindex, lockp); 4324 if (pdpg == NULL && lockp != NULL) 4325 goto retry; 4326 } 4327 return (pdpg); 4328 } 4329 4330 static vm_page_t 4331 pmap_allocpte(pmap_t pmap, vm_offset_t va, struct rwlock **lockp) 4332 { 4333 vm_pindex_t ptepindex; 4334 pml3_entry_t *pd; 4335 vm_page_t m; 4336 4337 /* 4338 * Calculate pagetable page index 4339 */ 4340 ptepindex = pmap_l3e_pindex(va); 4341 retry: 4342 /* 4343 * Get the page directory entry 4344 */ 4345 pd = pmap_pml3e(pmap, va); 4346 4347 /* 4348 * This supports switching from a 2MB page to a 4349 * normal 4K page. 4350 */ 4351 if (pd != NULL && (be64toh(*pd) & (RPTE_LEAF | PG_V)) == (RPTE_LEAF | PG_V)) { 4352 if (!pmap_demote_l3e_locked(pmap, pd, va, lockp)) { 4353 /* 4354 * Invalidation of the 2MB page mapping may have caused 4355 * the deallocation of the underlying PD page. 4356 */ 4357 pd = NULL; 4358 } 4359 } 4360 4361 /* 4362 * If the page table page is mapped, we just increment the 4363 * hold count, and activate it. 4364 */ 4365 if (pd != NULL && (be64toh(*pd) & PG_V) != 0) { 4366 m = PHYS_TO_VM_PAGE(be64toh(*pd) & PG_FRAME); 4367 m->ref_count++; 4368 } else { 4369 /* 4370 * Here if the pte page isn't mapped, or if it has been 4371 * deallocated. 4372 */ 4373 m = _pmap_allocpte(pmap, ptepindex, lockp); 4374 if (m == NULL && lockp != NULL) 4375 goto retry; 4376 } 4377 return (m); 4378 } 4379 4380 static void 4381 mmu_radix_pinit0(pmap_t pmap) 4382 { 4383 4384 CTR2(KTR_PMAP, "%s(%p)", __func__, pmap); 4385 PMAP_LOCK_INIT(pmap); 4386 pmap->pm_pml1 = kernel_pmap->pm_pml1; 4387 pmap->pm_pid = kernel_pmap->pm_pid; 4388 4389 pmap->pm_radix.rt_root = 0; 4390 TAILQ_INIT(&pmap->pm_pvchunk); 4391 bzero(&pmap->pm_stats, sizeof pmap->pm_stats); 4392 kernel_pmap->pm_flags = 4393 pmap->pm_flags = PMAP_PDE_SUPERPAGE; 4394 } 4395 /* 4396 * pmap_protect_l3e: do the things to protect a 2mpage in a process 4397 */ 4398 static boolean_t 4399 pmap_protect_l3e(pmap_t pmap, pt_entry_t *l3e, vm_offset_t sva, vm_prot_t prot) 4400 { 4401 pt_entry_t newpde, oldpde; 4402 vm_offset_t eva, va; 4403 vm_page_t m; 4404 boolean_t anychanged; 4405 4406 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 4407 KASSERT((sva & L3_PAGE_MASK) == 0, 4408 ("pmap_protect_l3e: sva is not 2mpage aligned")); 4409 anychanged = FALSE; 4410 retry: 4411 oldpde = newpde = be64toh(*l3e); 4412 if ((oldpde & (PG_MANAGED | PG_M | PG_RW)) == 4413 (PG_MANAGED | PG_M | PG_RW)) { 4414 eva = sva + L3_PAGE_SIZE; 4415 for (va = sva, m = PHYS_TO_VM_PAGE(oldpde & PG_PS_FRAME); 4416 va < eva; va += PAGE_SIZE, m++) 4417 vm_page_dirty(m); 4418 } 4419 if ((prot & VM_PROT_WRITE) == 0) { 4420 newpde &= ~(PG_RW | PG_M); 4421 newpde |= RPTE_EAA_R; 4422 } 4423 if (prot & VM_PROT_EXECUTE) 4424 newpde |= PG_X; 4425 if (newpde != oldpde) { 4426 /* 4427 * As an optimization to future operations on this PDE, clear 4428 * PG_PROMOTED. The impending invalidation will remove any 4429 * lingering 4KB page mappings from the TLB. 4430 */ 4431 if (!atomic_cmpset_long(l3e, htobe64(oldpde), htobe64(newpde & ~PG_PROMOTED))) 4432 goto retry; 4433 anychanged = TRUE; 4434 } 4435 return (anychanged); 4436 } 4437 4438 void 4439 mmu_radix_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, 4440 vm_prot_t prot) 4441 { 4442 vm_offset_t va_next; 4443 pml1_entry_t *l1e; 4444 pml2_entry_t *l2e; 4445 pml3_entry_t ptpaddr, *l3e; 4446 pt_entry_t *pte; 4447 boolean_t anychanged; 4448 4449 CTR5(KTR_PMAP, "%s(%p, %#x, %#x, %#x)", __func__, pmap, sva, eva, 4450 prot); 4451 4452 KASSERT((prot & ~VM_PROT_ALL) == 0, ("invalid prot %x", prot)); 4453 if (prot == VM_PROT_NONE) { 4454 mmu_radix_remove(pmap, sva, eva); 4455 return; 4456 } 4457 4458 if ((prot & (VM_PROT_WRITE|VM_PROT_EXECUTE)) == 4459 (VM_PROT_WRITE|VM_PROT_EXECUTE)) 4460 return; 4461 4462 #ifdef INVARIANTS 4463 if (VERBOSE_PROTECT || pmap_logging) 4464 printf("pmap_protect(%p, %#lx, %#lx, %x) - asid: %lu\n", 4465 pmap, sva, eva, prot, pmap->pm_pid); 4466 #endif 4467 anychanged = FALSE; 4468 4469 PMAP_LOCK(pmap); 4470 for (; sva < eva; sva = va_next) { 4471 l1e = pmap_pml1e(pmap, sva); 4472 if ((be64toh(*l1e) & PG_V) == 0) { 4473 va_next = (sva + L1_PAGE_SIZE) & ~L1_PAGE_MASK; 4474 if (va_next < sva) 4475 va_next = eva; 4476 continue; 4477 } 4478 4479 l2e = pmap_l1e_to_l2e(l1e, sva); 4480 if ((be64toh(*l2e) & PG_V) == 0) { 4481 va_next = (sva + L2_PAGE_SIZE) & ~L2_PAGE_MASK; 4482 if (va_next < sva) 4483 va_next = eva; 4484 continue; 4485 } 4486 4487 va_next = (sva + L3_PAGE_SIZE) & ~L3_PAGE_MASK; 4488 if (va_next < sva) 4489 va_next = eva; 4490 4491 l3e = pmap_l2e_to_l3e(l2e, sva); 4492 ptpaddr = be64toh(*l3e); 4493 4494 /* 4495 * Weed out invalid mappings. 4496 */ 4497 if (ptpaddr == 0) 4498 continue; 4499 4500 /* 4501 * Check for large page. 4502 */ 4503 if ((ptpaddr & RPTE_LEAF) != 0) { 4504 /* 4505 * Are we protecting the entire large page? If not, 4506 * demote the mapping and fall through. 4507 */ 4508 if (sva + L3_PAGE_SIZE == va_next && eva >= va_next) { 4509 if (pmap_protect_l3e(pmap, l3e, sva, prot)) 4510 anychanged = TRUE; 4511 continue; 4512 } else if (!pmap_demote_l3e(pmap, l3e, sva)) { 4513 /* 4514 * The large page mapping was destroyed. 4515 */ 4516 continue; 4517 } 4518 } 4519 4520 if (va_next > eva) 4521 va_next = eva; 4522 4523 for (pte = pmap_l3e_to_pte(l3e, sva); sva != va_next; pte++, 4524 sva += PAGE_SIZE) { 4525 pt_entry_t obits, pbits; 4526 vm_page_t m; 4527 4528 retry: 4529 MPASS(pte == pmap_pte(pmap, sva)); 4530 obits = pbits = be64toh(*pte); 4531 if ((pbits & PG_V) == 0) 4532 continue; 4533 4534 if ((prot & VM_PROT_WRITE) == 0) { 4535 if ((pbits & (PG_MANAGED | PG_M | PG_RW)) == 4536 (PG_MANAGED | PG_M | PG_RW)) { 4537 m = PHYS_TO_VM_PAGE(pbits & PG_FRAME); 4538 vm_page_dirty(m); 4539 } 4540 pbits &= ~(PG_RW | PG_M); 4541 pbits |= RPTE_EAA_R; 4542 } 4543 if (prot & VM_PROT_EXECUTE) 4544 pbits |= PG_X; 4545 4546 if (pbits != obits) { 4547 if (!atomic_cmpset_long(pte, htobe64(obits), htobe64(pbits))) 4548 goto retry; 4549 if (obits & (PG_A|PG_M)) { 4550 anychanged = TRUE; 4551 #ifdef INVARIANTS 4552 if (VERBOSE_PROTECT || pmap_logging) 4553 printf("%#lx %#lx -> %#lx\n", 4554 sva, obits, pbits); 4555 #endif 4556 } 4557 } 4558 } 4559 } 4560 if (anychanged) 4561 pmap_invalidate_all(pmap); 4562 PMAP_UNLOCK(pmap); 4563 } 4564 4565 void 4566 mmu_radix_qenter(vm_offset_t sva, vm_page_t *ma, int count) 4567 { 4568 4569 CTR4(KTR_PMAP, "%s(%#x, %p, %d)", __func__, sva, ma, count); 4570 pt_entry_t oldpte, pa, *pte; 4571 vm_page_t m; 4572 uint64_t cache_bits, attr_bits; 4573 vm_offset_t va; 4574 4575 oldpte = 0; 4576 attr_bits = RPTE_EAA_R | RPTE_EAA_W | RPTE_EAA_P | PG_M | PG_A; 4577 va = sva; 4578 pte = kvtopte(va); 4579 while (va < sva + PAGE_SIZE * count) { 4580 if (__predict_false((va & L3_PAGE_MASK) == 0)) 4581 pte = kvtopte(va); 4582 MPASS(pte == pmap_pte(kernel_pmap, va)); 4583 4584 /* 4585 * XXX there has to be a more efficient way than traversing 4586 * the page table every time - but go for correctness for 4587 * today 4588 */ 4589 4590 m = *ma++; 4591 cache_bits = pmap_cache_bits(m->md.mdpg_cache_attrs); 4592 pa = VM_PAGE_TO_PHYS(m) | cache_bits | attr_bits; 4593 if (be64toh(*pte) != pa) { 4594 oldpte |= be64toh(*pte); 4595 pte_store(pte, pa); 4596 } 4597 va += PAGE_SIZE; 4598 pte++; 4599 } 4600 if (__predict_false((oldpte & RPTE_VALID) != 0)) 4601 pmap_invalidate_range(kernel_pmap, sva, sva + count * 4602 PAGE_SIZE); 4603 else 4604 ptesync(); 4605 } 4606 4607 void 4608 mmu_radix_qremove(vm_offset_t sva, int count) 4609 { 4610 vm_offset_t va; 4611 pt_entry_t *pte; 4612 4613 CTR3(KTR_PMAP, "%s(%#x, %d)", __func__, sva, count); 4614 KASSERT(sva >= VM_MIN_KERNEL_ADDRESS, ("usermode or dmap va %lx", sva)); 4615 4616 va = sva; 4617 pte = kvtopte(va); 4618 while (va < sva + PAGE_SIZE * count) { 4619 if (__predict_false((va & L3_PAGE_MASK) == 0)) 4620 pte = kvtopte(va); 4621 pte_clear(pte); 4622 pte++; 4623 va += PAGE_SIZE; 4624 } 4625 pmap_invalidate_range(kernel_pmap, sva, va); 4626 } 4627 4628 /*************************************************** 4629 * Page table page management routines..... 4630 ***************************************************/ 4631 /* 4632 * Schedule the specified unused page table page to be freed. Specifically, 4633 * add the page to the specified list of pages that will be released to the 4634 * physical memory manager after the TLB has been updated. 4635 */ 4636 static __inline void 4637 pmap_add_delayed_free_list(vm_page_t m, struct spglist *free, 4638 boolean_t set_PG_ZERO) 4639 { 4640 4641 if (set_PG_ZERO) 4642 m->flags |= PG_ZERO; 4643 else 4644 m->flags &= ~PG_ZERO; 4645 SLIST_INSERT_HEAD(free, m, plinks.s.ss); 4646 } 4647 4648 /* 4649 * Inserts the specified page table page into the specified pmap's collection 4650 * of idle page table pages. Each of a pmap's page table pages is responsible 4651 * for mapping a distinct range of virtual addresses. The pmap's collection is 4652 * ordered by this virtual address range. 4653 */ 4654 static __inline int 4655 pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte) 4656 { 4657 4658 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 4659 return (vm_radix_insert(&pmap->pm_radix, mpte)); 4660 } 4661 4662 /* 4663 * Removes the page table page mapping the specified virtual address from the 4664 * specified pmap's collection of idle page table pages, and returns it. 4665 * Otherwise, returns NULL if there is no page table page corresponding to the 4666 * specified virtual address. 4667 */ 4668 static __inline vm_page_t 4669 pmap_remove_pt_page(pmap_t pmap, vm_offset_t va) 4670 { 4671 4672 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 4673 return (vm_radix_remove(&pmap->pm_radix, pmap_l3e_pindex(va))); 4674 } 4675 4676 /* 4677 * Decrements a page table page's wire count, which is used to record the 4678 * number of valid page table entries within the page. If the wire count 4679 * drops to zero, then the page table page is unmapped. Returns TRUE if the 4680 * page table page was unmapped and FALSE otherwise. 4681 */ 4682 static inline boolean_t 4683 pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free) 4684 { 4685 4686 --m->ref_count; 4687 if (m->ref_count == 0) { 4688 _pmap_unwire_ptp(pmap, va, m, free); 4689 return (TRUE); 4690 } else 4691 return (FALSE); 4692 } 4693 4694 static void 4695 _pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free) 4696 { 4697 4698 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 4699 /* 4700 * unmap the page table page 4701 */ 4702 if (m->pindex >= (NUPDE + NUPDPE)) { 4703 /* PDP page */ 4704 pml1_entry_t *pml1; 4705 pml1 = pmap_pml1e(pmap, va); 4706 *pml1 = 0; 4707 } else if (m->pindex >= NUPDE) { 4708 /* PD page */ 4709 pml2_entry_t *l2e; 4710 l2e = pmap_pml2e(pmap, va); 4711 *l2e = 0; 4712 } else { 4713 /* PTE page */ 4714 pml3_entry_t *l3e; 4715 l3e = pmap_pml3e(pmap, va); 4716 *l3e = 0; 4717 } 4718 pmap_resident_count_dec(pmap, 1); 4719 if (m->pindex < NUPDE) { 4720 /* We just released a PT, unhold the matching PD */ 4721 vm_page_t pdpg; 4722 4723 pdpg = PHYS_TO_VM_PAGE(be64toh(*pmap_pml2e(pmap, va)) & PG_FRAME); 4724 pmap_unwire_ptp(pmap, va, pdpg, free); 4725 } 4726 if (m->pindex >= NUPDE && m->pindex < (NUPDE + NUPDPE)) { 4727 /* We just released a PD, unhold the matching PDP */ 4728 vm_page_t pdppg; 4729 4730 pdppg = PHYS_TO_VM_PAGE(be64toh(*pmap_pml1e(pmap, va)) & PG_FRAME); 4731 pmap_unwire_ptp(pmap, va, pdppg, free); 4732 } 4733 4734 /* 4735 * Put page on a list so that it is released after 4736 * *ALL* TLB shootdown is done 4737 */ 4738 pmap_add_delayed_free_list(m, free, TRUE); 4739 } 4740 4741 /* 4742 * After removing a page table entry, this routine is used to 4743 * conditionally free the page, and manage the hold/wire counts. 4744 */ 4745 static int 4746 pmap_unuse_pt(pmap_t pmap, vm_offset_t va, pml3_entry_t ptepde, 4747 struct spglist *free) 4748 { 4749 vm_page_t mpte; 4750 4751 if (va >= VM_MAXUSER_ADDRESS) 4752 return (0); 4753 KASSERT(ptepde != 0, ("pmap_unuse_pt: ptepde != 0")); 4754 mpte = PHYS_TO_VM_PAGE(ptepde & PG_FRAME); 4755 return (pmap_unwire_ptp(pmap, va, mpte, free)); 4756 } 4757 4758 void 4759 mmu_radix_release(pmap_t pmap) 4760 { 4761 4762 CTR2(KTR_PMAP, "%s(%p)", __func__, pmap); 4763 KASSERT(pmap->pm_stats.resident_count == 0, 4764 ("pmap_release: pmap resident count %ld != 0", 4765 pmap->pm_stats.resident_count)); 4766 KASSERT(vm_radix_is_empty(&pmap->pm_radix), 4767 ("pmap_release: pmap has reserved page table page(s)")); 4768 4769 pmap_invalidate_all(pmap); 4770 isa3_proctab[pmap->pm_pid].proctab0 = 0; 4771 uma_zfree(zone_radix_pgd, pmap->pm_pml1); 4772 vmem_free(asid_arena, pmap->pm_pid, 1); 4773 } 4774 4775 /* 4776 * Create the PV entry for a 2MB page mapping. Always returns true unless the 4777 * flag PMAP_ENTER_NORECLAIM is specified. If that flag is specified, returns 4778 * false if the PV entry cannot be allocated without resorting to reclamation. 4779 */ 4780 static bool 4781 pmap_pv_insert_l3e(pmap_t pmap, vm_offset_t va, pml3_entry_t pde, u_int flags, 4782 struct rwlock **lockp) 4783 { 4784 struct md_page *pvh; 4785 pv_entry_t pv; 4786 vm_paddr_t pa; 4787 4788 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 4789 /* Pass NULL instead of the lock pointer to disable reclamation. */ 4790 if ((pv = get_pv_entry(pmap, (flags & PMAP_ENTER_NORECLAIM) != 0 ? 4791 NULL : lockp)) == NULL) 4792 return (false); 4793 pv->pv_va = va; 4794 pa = pde & PG_PS_FRAME; 4795 CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa); 4796 pvh = pa_to_pvh(pa); 4797 TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_link); 4798 pvh->pv_gen++; 4799 return (true); 4800 } 4801 4802 /* 4803 * Fills a page table page with mappings to consecutive physical pages. 4804 */ 4805 static void 4806 pmap_fill_ptp(pt_entry_t *firstpte, pt_entry_t newpte) 4807 { 4808 pt_entry_t *pte; 4809 4810 for (pte = firstpte; pte < firstpte + NPTEPG; pte++) { 4811 *pte = htobe64(newpte); 4812 newpte += PAGE_SIZE; 4813 } 4814 } 4815 4816 static boolean_t 4817 pmap_demote_l3e(pmap_t pmap, pml3_entry_t *pde, vm_offset_t va) 4818 { 4819 struct rwlock *lock; 4820 boolean_t rv; 4821 4822 lock = NULL; 4823 rv = pmap_demote_l3e_locked(pmap, pde, va, &lock); 4824 if (lock != NULL) 4825 rw_wunlock(lock); 4826 return (rv); 4827 } 4828 4829 static boolean_t 4830 pmap_demote_l3e_locked(pmap_t pmap, pml3_entry_t *l3e, vm_offset_t va, 4831 struct rwlock **lockp) 4832 { 4833 pml3_entry_t oldpde; 4834 pt_entry_t *firstpte; 4835 vm_paddr_t mptepa; 4836 vm_page_t mpte; 4837 struct spglist free; 4838 vm_offset_t sva; 4839 4840 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 4841 oldpde = be64toh(*l3e); 4842 KASSERT((oldpde & (RPTE_LEAF | PG_V)) == (RPTE_LEAF | PG_V), 4843 ("pmap_demote_l3e: oldpde is missing RPTE_LEAF and/or PG_V %lx", 4844 oldpde)); 4845 if ((oldpde & PG_A) == 0 || (mpte = pmap_remove_pt_page(pmap, va)) == 4846 NULL) { 4847 KASSERT((oldpde & PG_W) == 0, 4848 ("pmap_demote_l3e: page table page for a wired mapping" 4849 " is missing")); 4850 4851 /* 4852 * Invalidate the 2MB page mapping and return "failure" if the 4853 * mapping was never accessed or the allocation of the new 4854 * page table page fails. If the 2MB page mapping belongs to 4855 * the direct map region of the kernel's address space, then 4856 * the page allocation request specifies the highest possible 4857 * priority (VM_ALLOC_INTERRUPT). Otherwise, the priority is 4858 * normal. Page table pages are preallocated for every other 4859 * part of the kernel address space, so the direct map region 4860 * is the only part of the kernel address space that must be 4861 * handled here. 4862 */ 4863 if ((oldpde & PG_A) == 0 || (mpte = vm_page_alloc(NULL, 4864 pmap_l3e_pindex(va), (va >= DMAP_MIN_ADDRESS && va < 4865 DMAP_MAX_ADDRESS ? VM_ALLOC_INTERRUPT : VM_ALLOC_NORMAL) | 4866 VM_ALLOC_NOOBJ | VM_ALLOC_WIRED)) == NULL) { 4867 SLIST_INIT(&free); 4868 sva = trunc_2mpage(va); 4869 pmap_remove_l3e(pmap, l3e, sva, &free, lockp); 4870 pmap_invalidate_l3e_page(pmap, sva, oldpde); 4871 vm_page_free_pages_toq(&free, true); 4872 CTR2(KTR_PMAP, "pmap_demote_l3e: failure for va %#lx" 4873 " in pmap %p", va, pmap); 4874 return (FALSE); 4875 } 4876 if (va < VM_MAXUSER_ADDRESS) 4877 pmap_resident_count_inc(pmap, 1); 4878 } 4879 mptepa = VM_PAGE_TO_PHYS(mpte); 4880 firstpte = (pt_entry_t *)PHYS_TO_DMAP(mptepa); 4881 KASSERT((oldpde & PG_A) != 0, 4882 ("pmap_demote_l3e: oldpde is missing PG_A")); 4883 KASSERT((oldpde & (PG_M | PG_RW)) != PG_RW, 4884 ("pmap_demote_l3e: oldpde is missing PG_M")); 4885 4886 /* 4887 * If the page table page is new, initialize it. 4888 */ 4889 if (mpte->ref_count == 1) { 4890 mpte->ref_count = NPTEPG; 4891 pmap_fill_ptp(firstpte, oldpde); 4892 } 4893 4894 KASSERT((be64toh(*firstpte) & PG_FRAME) == (oldpde & PG_FRAME), 4895 ("pmap_demote_l3e: firstpte and newpte map different physical" 4896 " addresses")); 4897 4898 /* 4899 * If the mapping has changed attributes, update the page table 4900 * entries. 4901 */ 4902 if ((be64toh(*firstpte) & PG_PTE_PROMOTE) != (oldpde & PG_PTE_PROMOTE)) 4903 pmap_fill_ptp(firstpte, oldpde); 4904 4905 /* 4906 * The spare PV entries must be reserved prior to demoting the 4907 * mapping, that is, prior to changing the PDE. Otherwise, the state 4908 * of the PDE and the PV lists will be inconsistent, which can result 4909 * in reclaim_pv_chunk() attempting to remove a PV entry from the 4910 * wrong PV list and pmap_pv_demote_l3e() failing to find the expected 4911 * PV entry for the 2MB page mapping that is being demoted. 4912 */ 4913 if ((oldpde & PG_MANAGED) != 0) 4914 reserve_pv_entries(pmap, NPTEPG - 1, lockp); 4915 4916 /* 4917 * Demote the mapping. This pmap is locked. The old PDE has 4918 * PG_A set. If the old PDE has PG_RW set, it also has PG_M 4919 * set. Thus, there is no danger of a race with another 4920 * processor changing the setting of PG_A and/or PG_M between 4921 * the read above and the store below. 4922 */ 4923 pde_store(l3e, mptepa); 4924 ptesync(); 4925 /* 4926 * Demote the PV entry. 4927 */ 4928 if ((oldpde & PG_MANAGED) != 0) 4929 pmap_pv_demote_l3e(pmap, va, oldpde & PG_PS_FRAME, lockp); 4930 4931 atomic_add_long(&pmap_l3e_demotions, 1); 4932 CTR2(KTR_PMAP, "pmap_demote_l3e: success for va %#lx" 4933 " in pmap %p", va, pmap); 4934 return (TRUE); 4935 } 4936 4937 /* 4938 * pmap_remove_kernel_pde: Remove a kernel superpage mapping. 4939 */ 4940 static void 4941 pmap_remove_kernel_l3e(pmap_t pmap, pml3_entry_t *l3e, vm_offset_t va) 4942 { 4943 vm_paddr_t mptepa; 4944 vm_page_t mpte; 4945 4946 KASSERT(pmap == kernel_pmap, ("pmap %p is not kernel_pmap", pmap)); 4947 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 4948 mpte = pmap_remove_pt_page(pmap, va); 4949 if (mpte == NULL) 4950 panic("pmap_remove_kernel_pde: Missing pt page."); 4951 4952 mptepa = VM_PAGE_TO_PHYS(mpte); 4953 4954 /* 4955 * Initialize the page table page. 4956 */ 4957 pagezero(PHYS_TO_DMAP(mptepa)); 4958 4959 /* 4960 * Demote the mapping. 4961 */ 4962 pde_store(l3e, mptepa); 4963 ptesync(); 4964 } 4965 4966 /* 4967 * pmap_remove_l3e: do the things to unmap a superpage in a process 4968 */ 4969 static int 4970 pmap_remove_l3e(pmap_t pmap, pml3_entry_t *pdq, vm_offset_t sva, 4971 struct spglist *free, struct rwlock **lockp) 4972 { 4973 struct md_page *pvh; 4974 pml3_entry_t oldpde; 4975 vm_offset_t eva, va; 4976 vm_page_t m, mpte; 4977 4978 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 4979 KASSERT((sva & L3_PAGE_MASK) == 0, 4980 ("pmap_remove_l3e: sva is not 2mpage aligned")); 4981 oldpde = be64toh(pte_load_clear(pdq)); 4982 if (oldpde & PG_W) 4983 pmap->pm_stats.wired_count -= (L3_PAGE_SIZE / PAGE_SIZE); 4984 pmap_resident_count_dec(pmap, L3_PAGE_SIZE / PAGE_SIZE); 4985 if (oldpde & PG_MANAGED) { 4986 CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, oldpde & PG_PS_FRAME); 4987 pvh = pa_to_pvh(oldpde & PG_PS_FRAME); 4988 pmap_pvh_free(pvh, pmap, sva); 4989 eva = sva + L3_PAGE_SIZE; 4990 for (va = sva, m = PHYS_TO_VM_PAGE(oldpde & PG_PS_FRAME); 4991 va < eva; va += PAGE_SIZE, m++) { 4992 if ((oldpde & (PG_M | PG_RW)) == (PG_M | PG_RW)) 4993 vm_page_dirty(m); 4994 if (oldpde & PG_A) 4995 vm_page_aflag_set(m, PGA_REFERENCED); 4996 if (TAILQ_EMPTY(&m->md.pv_list) && 4997 TAILQ_EMPTY(&pvh->pv_list)) 4998 vm_page_aflag_clear(m, PGA_WRITEABLE); 4999 } 5000 } 5001 if (pmap == kernel_pmap) { 5002 pmap_remove_kernel_l3e(pmap, pdq, sva); 5003 } else { 5004 mpte = pmap_remove_pt_page(pmap, sva); 5005 if (mpte != NULL) { 5006 pmap_resident_count_dec(pmap, 1); 5007 KASSERT(mpte->ref_count == NPTEPG, 5008 ("pmap_remove_l3e: pte page wire count error")); 5009 mpte->ref_count = 0; 5010 pmap_add_delayed_free_list(mpte, free, FALSE); 5011 } 5012 } 5013 return (pmap_unuse_pt(pmap, sva, be64toh(*pmap_pml2e(pmap, sva)), free)); 5014 } 5015 5016 /* 5017 * pmap_remove_pte: do the things to unmap a page in a process 5018 */ 5019 static int 5020 pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t va, 5021 pml3_entry_t ptepde, struct spglist *free, struct rwlock **lockp) 5022 { 5023 struct md_page *pvh; 5024 pt_entry_t oldpte; 5025 vm_page_t m; 5026 5027 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 5028 oldpte = be64toh(pte_load_clear(ptq)); 5029 if (oldpte & RPTE_WIRED) 5030 pmap->pm_stats.wired_count -= 1; 5031 pmap_resident_count_dec(pmap, 1); 5032 if (oldpte & RPTE_MANAGED) { 5033 m = PHYS_TO_VM_PAGE(oldpte & PG_FRAME); 5034 if ((oldpte & (PG_M | PG_RW)) == (PG_M | PG_RW)) 5035 vm_page_dirty(m); 5036 if (oldpte & PG_A) 5037 vm_page_aflag_set(m, PGA_REFERENCED); 5038 CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m); 5039 pmap_pvh_free(&m->md, pmap, va); 5040 if (TAILQ_EMPTY(&m->md.pv_list) && 5041 (m->flags & PG_FICTITIOUS) == 0) { 5042 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); 5043 if (TAILQ_EMPTY(&pvh->pv_list)) 5044 vm_page_aflag_clear(m, PGA_WRITEABLE); 5045 } 5046 } 5047 return (pmap_unuse_pt(pmap, va, ptepde, free)); 5048 } 5049 5050 /* 5051 * Remove a single page from a process address space 5052 */ 5053 static bool 5054 pmap_remove_page(pmap_t pmap, vm_offset_t va, pml3_entry_t *l3e, 5055 struct spglist *free) 5056 { 5057 struct rwlock *lock; 5058 pt_entry_t *pte; 5059 bool invalidate_all; 5060 5061 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 5062 if ((be64toh(*l3e) & RPTE_VALID) == 0) { 5063 return (false); 5064 } 5065 pte = pmap_l3e_to_pte(l3e, va); 5066 if ((be64toh(*pte) & RPTE_VALID) == 0) { 5067 return (false); 5068 } 5069 lock = NULL; 5070 5071 invalidate_all = pmap_remove_pte(pmap, pte, va, be64toh(*l3e), free, &lock); 5072 if (lock != NULL) 5073 rw_wunlock(lock); 5074 if (!invalidate_all) 5075 pmap_invalidate_page(pmap, va); 5076 return (invalidate_all); 5077 } 5078 5079 /* 5080 * Removes the specified range of addresses from the page table page. 5081 */ 5082 static bool 5083 pmap_remove_ptes(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, 5084 pml3_entry_t *l3e, struct spglist *free, struct rwlock **lockp) 5085 { 5086 pt_entry_t *pte; 5087 vm_offset_t va; 5088 bool anyvalid; 5089 5090 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 5091 anyvalid = false; 5092 va = eva; 5093 for (pte = pmap_l3e_to_pte(l3e, sva); sva != eva; pte++, 5094 sva += PAGE_SIZE) { 5095 MPASS(pte == pmap_pte(pmap, sva)); 5096 if (*pte == 0) { 5097 if (va != eva) { 5098 anyvalid = true; 5099 va = eva; 5100 } 5101 continue; 5102 } 5103 if (va == eva) 5104 va = sva; 5105 if (pmap_remove_pte(pmap, pte, sva, be64toh(*l3e), free, lockp)) { 5106 anyvalid = true; 5107 sva += PAGE_SIZE; 5108 break; 5109 } 5110 } 5111 if (anyvalid) 5112 pmap_invalidate_all(pmap); 5113 else if (va != eva) 5114 pmap_invalidate_range(pmap, va, sva); 5115 return (anyvalid); 5116 } 5117 5118 void 5119 mmu_radix_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) 5120 { 5121 struct rwlock *lock; 5122 vm_offset_t va_next; 5123 pml1_entry_t *l1e; 5124 pml2_entry_t *l2e; 5125 pml3_entry_t ptpaddr, *l3e; 5126 struct spglist free; 5127 bool anyvalid; 5128 5129 CTR4(KTR_PMAP, "%s(%p, %#x, %#x)", __func__, pmap, sva, eva); 5130 5131 /* 5132 * Perform an unsynchronized read. This is, however, safe. 5133 */ 5134 if (pmap->pm_stats.resident_count == 0) 5135 return; 5136 5137 anyvalid = false; 5138 SLIST_INIT(&free); 5139 5140 /* XXX something fishy here */ 5141 sva = (sva + PAGE_MASK) & ~PAGE_MASK; 5142 eva = (eva + PAGE_MASK) & ~PAGE_MASK; 5143 5144 PMAP_LOCK(pmap); 5145 5146 /* 5147 * special handling of removing one page. a very 5148 * common operation and easy to short circuit some 5149 * code. 5150 */ 5151 if (sva + PAGE_SIZE == eva) { 5152 l3e = pmap_pml3e(pmap, sva); 5153 if (l3e && (be64toh(*l3e) & RPTE_LEAF) == 0) { 5154 anyvalid = pmap_remove_page(pmap, sva, l3e, &free); 5155 goto out; 5156 } 5157 } 5158 5159 lock = NULL; 5160 for (; sva < eva; sva = va_next) { 5161 if (pmap->pm_stats.resident_count == 0) 5162 break; 5163 l1e = pmap_pml1e(pmap, sva); 5164 if (l1e == NULL || (be64toh(*l1e) & PG_V) == 0) { 5165 va_next = (sva + L1_PAGE_SIZE) & ~L1_PAGE_MASK; 5166 if (va_next < sva) 5167 va_next = eva; 5168 continue; 5169 } 5170 5171 l2e = pmap_l1e_to_l2e(l1e, sva); 5172 if (l2e == NULL || (be64toh(*l2e) & PG_V) == 0) { 5173 va_next = (sva + L2_PAGE_SIZE) & ~L2_PAGE_MASK; 5174 if (va_next < sva) 5175 va_next = eva; 5176 continue; 5177 } 5178 5179 /* 5180 * Calculate index for next page table. 5181 */ 5182 va_next = (sva + L3_PAGE_SIZE) & ~L3_PAGE_MASK; 5183 if (va_next < sva) 5184 va_next = eva; 5185 5186 l3e = pmap_l2e_to_l3e(l2e, sva); 5187 ptpaddr = be64toh(*l3e); 5188 5189 /* 5190 * Weed out invalid mappings. 5191 */ 5192 if (ptpaddr == 0) 5193 continue; 5194 5195 /* 5196 * Check for large page. 5197 */ 5198 if ((ptpaddr & RPTE_LEAF) != 0) { 5199 /* 5200 * Are we removing the entire large page? If not, 5201 * demote the mapping and fall through. 5202 */ 5203 if (sva + L3_PAGE_SIZE == va_next && eva >= va_next) { 5204 pmap_remove_l3e(pmap, l3e, sva, &free, &lock); 5205 continue; 5206 } else if (!pmap_demote_l3e_locked(pmap, l3e, sva, 5207 &lock)) { 5208 /* The large page mapping was destroyed. */ 5209 continue; 5210 } else 5211 ptpaddr = be64toh(*l3e); 5212 } 5213 5214 /* 5215 * Limit our scan to either the end of the va represented 5216 * by the current page table page, or to the end of the 5217 * range being removed. 5218 */ 5219 if (va_next > eva) 5220 va_next = eva; 5221 5222 if (pmap_remove_ptes(pmap, sva, va_next, l3e, &free, &lock)) 5223 anyvalid = true; 5224 } 5225 if (lock != NULL) 5226 rw_wunlock(lock); 5227 out: 5228 if (anyvalid) 5229 pmap_invalidate_all(pmap); 5230 PMAP_UNLOCK(pmap); 5231 vm_page_free_pages_toq(&free, true); 5232 } 5233 5234 void 5235 mmu_radix_remove_all(vm_page_t m) 5236 { 5237 struct md_page *pvh; 5238 pv_entry_t pv; 5239 pmap_t pmap; 5240 struct rwlock *lock; 5241 pt_entry_t *pte, tpte; 5242 pml3_entry_t *l3e; 5243 vm_offset_t va; 5244 struct spglist free; 5245 int pvh_gen, md_gen; 5246 5247 CTR2(KTR_PMAP, "%s(%p)", __func__, m); 5248 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 5249 ("pmap_remove_all: page %p is not managed", m)); 5250 SLIST_INIT(&free); 5251 lock = VM_PAGE_TO_PV_LIST_LOCK(m); 5252 pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy : 5253 pa_to_pvh(VM_PAGE_TO_PHYS(m)); 5254 retry: 5255 rw_wlock(lock); 5256 while ((pv = TAILQ_FIRST(&pvh->pv_list)) != NULL) { 5257 pmap = PV_PMAP(pv); 5258 if (!PMAP_TRYLOCK(pmap)) { 5259 pvh_gen = pvh->pv_gen; 5260 rw_wunlock(lock); 5261 PMAP_LOCK(pmap); 5262 rw_wlock(lock); 5263 if (pvh_gen != pvh->pv_gen) { 5264 rw_wunlock(lock); 5265 PMAP_UNLOCK(pmap); 5266 goto retry; 5267 } 5268 } 5269 va = pv->pv_va; 5270 l3e = pmap_pml3e(pmap, va); 5271 (void)pmap_demote_l3e_locked(pmap, l3e, va, &lock); 5272 PMAP_UNLOCK(pmap); 5273 } 5274 while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) { 5275 pmap = PV_PMAP(pv); 5276 if (!PMAP_TRYLOCK(pmap)) { 5277 pvh_gen = pvh->pv_gen; 5278 md_gen = m->md.pv_gen; 5279 rw_wunlock(lock); 5280 PMAP_LOCK(pmap); 5281 rw_wlock(lock); 5282 if (pvh_gen != pvh->pv_gen || md_gen != m->md.pv_gen) { 5283 rw_wunlock(lock); 5284 PMAP_UNLOCK(pmap); 5285 goto retry; 5286 } 5287 } 5288 pmap_resident_count_dec(pmap, 1); 5289 l3e = pmap_pml3e(pmap, pv->pv_va); 5290 KASSERT((be64toh(*l3e) & RPTE_LEAF) == 0, ("pmap_remove_all: found" 5291 " a 2mpage in page %p's pv list", m)); 5292 pte = pmap_l3e_to_pte(l3e, pv->pv_va); 5293 tpte = be64toh(pte_load_clear(pte)); 5294 if (tpte & PG_W) 5295 pmap->pm_stats.wired_count--; 5296 if (tpte & PG_A) 5297 vm_page_aflag_set(m, PGA_REFERENCED); 5298 5299 /* 5300 * Update the vm_page_t clean and reference bits. 5301 */ 5302 if ((tpte & (PG_M | PG_RW)) == (PG_M | PG_RW)) 5303 vm_page_dirty(m); 5304 pmap_unuse_pt(pmap, pv->pv_va, be64toh(*l3e), &free); 5305 pmap_invalidate_page(pmap, pv->pv_va); 5306 TAILQ_REMOVE(&m->md.pv_list, pv, pv_link); 5307 m->md.pv_gen++; 5308 free_pv_entry(pmap, pv); 5309 PMAP_UNLOCK(pmap); 5310 } 5311 vm_page_aflag_clear(m, PGA_WRITEABLE); 5312 rw_wunlock(lock); 5313 vm_page_free_pages_toq(&free, true); 5314 } 5315 5316 /* 5317 * Destroy all managed, non-wired mappings in the given user-space 5318 * pmap. This pmap cannot be active on any processor besides the 5319 * caller. 5320 * 5321 * This function cannot be applied to the kernel pmap. Moreover, it 5322 * is not intended for general use. It is only to be used during 5323 * process termination. Consequently, it can be implemented in ways 5324 * that make it faster than pmap_remove(). First, it can more quickly 5325 * destroy mappings by iterating over the pmap's collection of PV 5326 * entries, rather than searching the page table. Second, it doesn't 5327 * have to test and clear the page table entries atomically, because 5328 * no processor is currently accessing the user address space. In 5329 * particular, a page table entry's dirty bit won't change state once 5330 * this function starts. 5331 * 5332 * Although this function destroys all of the pmap's managed, 5333 * non-wired mappings, it can delay and batch the invalidation of TLB 5334 * entries without calling pmap_delayed_invl_started() and 5335 * pmap_delayed_invl_finished(). Because the pmap is not active on 5336 * any other processor, none of these TLB entries will ever be used 5337 * before their eventual invalidation. Consequently, there is no need 5338 * for either pmap_remove_all() or pmap_remove_write() to wait for 5339 * that eventual TLB invalidation. 5340 */ 5341 5342 void 5343 mmu_radix_remove_pages(pmap_t pmap) 5344 { 5345 5346 CTR2(KTR_PMAP, "%s(%p)", __func__, pmap); 5347 pml3_entry_t ptel3e; 5348 pt_entry_t *pte, tpte; 5349 struct spglist free; 5350 vm_page_t m, mpte, mt; 5351 pv_entry_t pv; 5352 struct md_page *pvh; 5353 struct pv_chunk *pc, *npc; 5354 struct rwlock *lock; 5355 int64_t bit; 5356 uint64_t inuse, bitmask; 5357 int allfree, field, freed, idx; 5358 boolean_t superpage; 5359 vm_paddr_t pa; 5360 5361 /* 5362 * Assert that the given pmap is only active on the current 5363 * CPU. Unfortunately, we cannot block another CPU from 5364 * activating the pmap while this function is executing. 5365 */ 5366 KASSERT(pmap->pm_pid == mfspr(SPR_PID), 5367 ("non-current asid %lu - expected %lu", pmap->pm_pid, 5368 mfspr(SPR_PID))); 5369 5370 lock = NULL; 5371 5372 SLIST_INIT(&free); 5373 PMAP_LOCK(pmap); 5374 TAILQ_FOREACH_SAFE(pc, &pmap->pm_pvchunk, pc_list, npc) { 5375 allfree = 1; 5376 freed = 0; 5377 for (field = 0; field < _NPCM; field++) { 5378 inuse = ~pc->pc_map[field] & pc_freemask[field]; 5379 while (inuse != 0) { 5380 bit = cnttzd(inuse); 5381 bitmask = 1UL << bit; 5382 idx = field * 64 + bit; 5383 pv = &pc->pc_pventry[idx]; 5384 inuse &= ~bitmask; 5385 5386 pte = pmap_pml2e(pmap, pv->pv_va); 5387 ptel3e = be64toh(*pte); 5388 pte = pmap_l2e_to_l3e(pte, pv->pv_va); 5389 tpte = be64toh(*pte); 5390 if ((tpte & (RPTE_LEAF | PG_V)) == PG_V) { 5391 superpage = FALSE; 5392 ptel3e = tpte; 5393 pte = (pt_entry_t *)PHYS_TO_DMAP(tpte & 5394 PG_FRAME); 5395 pte = &pte[pmap_pte_index(pv->pv_va)]; 5396 tpte = be64toh(*pte); 5397 } else { 5398 /* 5399 * Keep track whether 'tpte' is a 5400 * superpage explicitly instead of 5401 * relying on RPTE_LEAF being set. 5402 * 5403 * This is because RPTE_LEAF is numerically 5404 * identical to PG_PTE_PAT and thus a 5405 * regular page could be mistaken for 5406 * a superpage. 5407 */ 5408 superpage = TRUE; 5409 } 5410 5411 if ((tpte & PG_V) == 0) { 5412 panic("bad pte va %lx pte %lx", 5413 pv->pv_va, tpte); 5414 } 5415 5416 /* 5417 * We cannot remove wired pages from a process' mapping at this time 5418 */ 5419 if (tpte & PG_W) { 5420 allfree = 0; 5421 continue; 5422 } 5423 5424 if (superpage) 5425 pa = tpte & PG_PS_FRAME; 5426 else 5427 pa = tpte & PG_FRAME; 5428 5429 m = PHYS_TO_VM_PAGE(pa); 5430 KASSERT(m->phys_addr == pa, 5431 ("vm_page_t %p phys_addr mismatch %016jx %016jx", 5432 m, (uintmax_t)m->phys_addr, 5433 (uintmax_t)tpte)); 5434 5435 KASSERT((m->flags & PG_FICTITIOUS) != 0 || 5436 m < &vm_page_array[vm_page_array_size], 5437 ("pmap_remove_pages: bad tpte %#jx", 5438 (uintmax_t)tpte)); 5439 5440 pte_clear(pte); 5441 5442 /* 5443 * Update the vm_page_t clean/reference bits. 5444 */ 5445 if ((tpte & (PG_M | PG_RW)) == (PG_M | PG_RW)) { 5446 if (superpage) { 5447 for (mt = m; mt < &m[L3_PAGE_SIZE / PAGE_SIZE]; mt++) 5448 vm_page_dirty(mt); 5449 } else 5450 vm_page_dirty(m); 5451 } 5452 5453 CHANGE_PV_LIST_LOCK_TO_VM_PAGE(&lock, m); 5454 5455 /* Mark free */ 5456 pc->pc_map[field] |= bitmask; 5457 if (superpage) { 5458 pmap_resident_count_dec(pmap, L3_PAGE_SIZE / PAGE_SIZE); 5459 pvh = pa_to_pvh(tpte & PG_PS_FRAME); 5460 TAILQ_REMOVE(&pvh->pv_list, pv, pv_link); 5461 pvh->pv_gen++; 5462 if (TAILQ_EMPTY(&pvh->pv_list)) { 5463 for (mt = m; mt < &m[L3_PAGE_SIZE / PAGE_SIZE]; mt++) 5464 if ((mt->a.flags & PGA_WRITEABLE) != 0 && 5465 TAILQ_EMPTY(&mt->md.pv_list)) 5466 vm_page_aflag_clear(mt, PGA_WRITEABLE); 5467 } 5468 mpte = pmap_remove_pt_page(pmap, pv->pv_va); 5469 if (mpte != NULL) { 5470 pmap_resident_count_dec(pmap, 1); 5471 KASSERT(mpte->ref_count == NPTEPG, 5472 ("pmap_remove_pages: pte page wire count error")); 5473 mpte->ref_count = 0; 5474 pmap_add_delayed_free_list(mpte, &free, FALSE); 5475 } 5476 } else { 5477 pmap_resident_count_dec(pmap, 1); 5478 #ifdef VERBOSE_PV 5479 printf("freeing pv (%p, %p)\n", 5480 pmap, pv); 5481 #endif 5482 TAILQ_REMOVE(&m->md.pv_list, pv, pv_link); 5483 m->md.pv_gen++; 5484 if ((m->a.flags & PGA_WRITEABLE) != 0 && 5485 TAILQ_EMPTY(&m->md.pv_list) && 5486 (m->flags & PG_FICTITIOUS) == 0) { 5487 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); 5488 if (TAILQ_EMPTY(&pvh->pv_list)) 5489 vm_page_aflag_clear(m, PGA_WRITEABLE); 5490 } 5491 } 5492 pmap_unuse_pt(pmap, pv->pv_va, ptel3e, &free); 5493 freed++; 5494 } 5495 } 5496 PV_STAT(atomic_add_long(&pv_entry_frees, freed)); 5497 PV_STAT(atomic_add_int(&pv_entry_spare, freed)); 5498 PV_STAT(atomic_subtract_long(&pv_entry_count, freed)); 5499 if (allfree) { 5500 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 5501 free_pv_chunk(pc); 5502 } 5503 } 5504 if (lock != NULL) 5505 rw_wunlock(lock); 5506 pmap_invalidate_all(pmap); 5507 PMAP_UNLOCK(pmap); 5508 vm_page_free_pages_toq(&free, true); 5509 } 5510 5511 void 5512 mmu_radix_remove_write(vm_page_t m) 5513 { 5514 struct md_page *pvh; 5515 pmap_t pmap; 5516 struct rwlock *lock; 5517 pv_entry_t next_pv, pv; 5518 pml3_entry_t *l3e; 5519 pt_entry_t oldpte, *pte; 5520 int pvh_gen, md_gen; 5521 5522 CTR2(KTR_PMAP, "%s(%p)", __func__, m); 5523 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 5524 ("pmap_remove_write: page %p is not managed", m)); 5525 vm_page_assert_busied(m); 5526 5527 if (!pmap_page_is_write_mapped(m)) 5528 return; 5529 lock = VM_PAGE_TO_PV_LIST_LOCK(m); 5530 pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy : 5531 pa_to_pvh(VM_PAGE_TO_PHYS(m)); 5532 retry_pv_loop: 5533 rw_wlock(lock); 5534 TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_link, next_pv) { 5535 pmap = PV_PMAP(pv); 5536 if (!PMAP_TRYLOCK(pmap)) { 5537 pvh_gen = pvh->pv_gen; 5538 rw_wunlock(lock); 5539 PMAP_LOCK(pmap); 5540 rw_wlock(lock); 5541 if (pvh_gen != pvh->pv_gen) { 5542 PMAP_UNLOCK(pmap); 5543 rw_wunlock(lock); 5544 goto retry_pv_loop; 5545 } 5546 } 5547 l3e = pmap_pml3e(pmap, pv->pv_va); 5548 if ((be64toh(*l3e) & PG_RW) != 0) 5549 (void)pmap_demote_l3e_locked(pmap, l3e, pv->pv_va, &lock); 5550 KASSERT(lock == VM_PAGE_TO_PV_LIST_LOCK(m), 5551 ("inconsistent pv lock %p %p for page %p", 5552 lock, VM_PAGE_TO_PV_LIST_LOCK(m), m)); 5553 PMAP_UNLOCK(pmap); 5554 } 5555 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) { 5556 pmap = PV_PMAP(pv); 5557 if (!PMAP_TRYLOCK(pmap)) { 5558 pvh_gen = pvh->pv_gen; 5559 md_gen = m->md.pv_gen; 5560 rw_wunlock(lock); 5561 PMAP_LOCK(pmap); 5562 rw_wlock(lock); 5563 if (pvh_gen != pvh->pv_gen || 5564 md_gen != m->md.pv_gen) { 5565 PMAP_UNLOCK(pmap); 5566 rw_wunlock(lock); 5567 goto retry_pv_loop; 5568 } 5569 } 5570 l3e = pmap_pml3e(pmap, pv->pv_va); 5571 KASSERT((be64toh(*l3e) & RPTE_LEAF) == 0, 5572 ("pmap_remove_write: found a 2mpage in page %p's pv list", 5573 m)); 5574 pte = pmap_l3e_to_pte(l3e, pv->pv_va); 5575 retry: 5576 oldpte = be64toh(*pte); 5577 if (oldpte & PG_RW) { 5578 if (!atomic_cmpset_long(pte, htobe64(oldpte), 5579 htobe64((oldpte | RPTE_EAA_R) & ~(PG_RW | PG_M)))) 5580 goto retry; 5581 if ((oldpte & PG_M) != 0) 5582 vm_page_dirty(m); 5583 pmap_invalidate_page(pmap, pv->pv_va); 5584 } 5585 PMAP_UNLOCK(pmap); 5586 } 5587 rw_wunlock(lock); 5588 vm_page_aflag_clear(m, PGA_WRITEABLE); 5589 } 5590 5591 /* 5592 * Clear the wired attribute from the mappings for the specified range of 5593 * addresses in the given pmap. Every valid mapping within that range 5594 * must have the wired attribute set. In contrast, invalid mappings 5595 * cannot have the wired attribute set, so they are ignored. 5596 * 5597 * The wired attribute of the page table entry is not a hardware 5598 * feature, so there is no need to invalidate any TLB entries. 5599 * Since pmap_demote_l3e() for the wired entry must never fail, 5600 * pmap_delayed_invl_started()/finished() calls around the 5601 * function are not needed. 5602 */ 5603 void 5604 mmu_radix_unwire(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) 5605 { 5606 vm_offset_t va_next; 5607 pml1_entry_t *l1e; 5608 pml2_entry_t *l2e; 5609 pml3_entry_t *l3e; 5610 pt_entry_t *pte; 5611 5612 CTR4(KTR_PMAP, "%s(%p, %#x, %#x)", __func__, pmap, sva, eva); 5613 PMAP_LOCK(pmap); 5614 for (; sva < eva; sva = va_next) { 5615 l1e = pmap_pml1e(pmap, sva); 5616 if ((be64toh(*l1e) & PG_V) == 0) { 5617 va_next = (sva + L1_PAGE_SIZE) & ~L1_PAGE_MASK; 5618 if (va_next < sva) 5619 va_next = eva; 5620 continue; 5621 } 5622 l2e = pmap_l1e_to_l2e(l1e, sva); 5623 if ((be64toh(*l2e) & PG_V) == 0) { 5624 va_next = (sva + L2_PAGE_SIZE) & ~L2_PAGE_MASK; 5625 if (va_next < sva) 5626 va_next = eva; 5627 continue; 5628 } 5629 va_next = (sva + L3_PAGE_SIZE) & ~L3_PAGE_MASK; 5630 if (va_next < sva) 5631 va_next = eva; 5632 l3e = pmap_l2e_to_l3e(l2e, sva); 5633 if ((be64toh(*l3e) & PG_V) == 0) 5634 continue; 5635 if ((be64toh(*l3e) & RPTE_LEAF) != 0) { 5636 if ((be64toh(*l3e) & PG_W) == 0) 5637 panic("pmap_unwire: pde %#jx is missing PG_W", 5638 (uintmax_t)(be64toh(*l3e))); 5639 5640 /* 5641 * Are we unwiring the entire large page? If not, 5642 * demote the mapping and fall through. 5643 */ 5644 if (sva + L3_PAGE_SIZE == va_next && eva >= va_next) { 5645 atomic_clear_long(l3e, htobe64(PG_W)); 5646 pmap->pm_stats.wired_count -= L3_PAGE_SIZE / 5647 PAGE_SIZE; 5648 continue; 5649 } else if (!pmap_demote_l3e(pmap, l3e, sva)) 5650 panic("pmap_unwire: demotion failed"); 5651 } 5652 if (va_next > eva) 5653 va_next = eva; 5654 for (pte = pmap_l3e_to_pte(l3e, sva); sva != va_next; pte++, 5655 sva += PAGE_SIZE) { 5656 MPASS(pte == pmap_pte(pmap, sva)); 5657 if ((be64toh(*pte) & PG_V) == 0) 5658 continue; 5659 if ((be64toh(*pte) & PG_W) == 0) 5660 panic("pmap_unwire: pte %#jx is missing PG_W", 5661 (uintmax_t)(be64toh(*pte))); 5662 5663 /* 5664 * PG_W must be cleared atomically. Although the pmap 5665 * lock synchronizes access to PG_W, another processor 5666 * could be setting PG_M and/or PG_A concurrently. 5667 */ 5668 atomic_clear_long(pte, htobe64(PG_W)); 5669 pmap->pm_stats.wired_count--; 5670 } 5671 } 5672 PMAP_UNLOCK(pmap); 5673 } 5674 5675 void 5676 mmu_radix_zero_page(vm_page_t m) 5677 { 5678 vm_offset_t addr; 5679 5680 CTR2(KTR_PMAP, "%s(%p)", __func__, m); 5681 addr = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)); 5682 pagezero(addr); 5683 } 5684 5685 void 5686 mmu_radix_zero_page_area(vm_page_t m, int off, int size) 5687 { 5688 caddr_t addr; 5689 5690 CTR4(KTR_PMAP, "%s(%p, %d, %d)", __func__, m, off, size); 5691 MPASS(off + size <= PAGE_SIZE); 5692 addr = (caddr_t)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)); 5693 memset(addr + off, 0, size); 5694 } 5695 5696 static int 5697 mmu_radix_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *locked_pa) 5698 { 5699 pml3_entry_t *l3ep; 5700 pt_entry_t pte; 5701 vm_paddr_t pa; 5702 int val; 5703 5704 CTR3(KTR_PMAP, "%s(%p, %#x)", __func__, pmap, addr); 5705 PMAP_LOCK(pmap); 5706 5707 l3ep = pmap_pml3e(pmap, addr); 5708 if (l3ep != NULL && (be64toh(*l3ep) & PG_V)) { 5709 if (be64toh(*l3ep) & RPTE_LEAF) { 5710 pte = be64toh(*l3ep); 5711 /* Compute the physical address of the 4KB page. */ 5712 pa = ((be64toh(*l3ep) & PG_PS_FRAME) | (addr & L3_PAGE_MASK)) & 5713 PG_FRAME; 5714 val = MINCORE_PSIND(1); 5715 } else { 5716 /* Native endian PTE, do not pass to functions */ 5717 pte = be64toh(*pmap_l3e_to_pte(l3ep, addr)); 5718 pa = pte & PG_FRAME; 5719 val = 0; 5720 } 5721 } else { 5722 pte = 0; 5723 pa = 0; 5724 val = 0; 5725 } 5726 if ((pte & PG_V) != 0) { 5727 val |= MINCORE_INCORE; 5728 if ((pte & (PG_M | PG_RW)) == (PG_M | PG_RW)) 5729 val |= MINCORE_MODIFIED | MINCORE_MODIFIED_OTHER; 5730 if ((pte & PG_A) != 0) 5731 val |= MINCORE_REFERENCED | MINCORE_REFERENCED_OTHER; 5732 } 5733 if ((val & (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER)) != 5734 (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER) && 5735 (pte & (PG_MANAGED | PG_V)) == (PG_MANAGED | PG_V)) { 5736 *locked_pa = pa; 5737 } 5738 PMAP_UNLOCK(pmap); 5739 return (val); 5740 } 5741 5742 void 5743 mmu_radix_activate(struct thread *td) 5744 { 5745 pmap_t pmap; 5746 uint32_t curpid; 5747 5748 CTR2(KTR_PMAP, "%s(%p)", __func__, td); 5749 critical_enter(); 5750 pmap = vmspace_pmap(td->td_proc->p_vmspace); 5751 curpid = mfspr(SPR_PID); 5752 if (pmap->pm_pid > isa3_base_pid && 5753 curpid != pmap->pm_pid) { 5754 mmu_radix_pid_set(pmap); 5755 } 5756 critical_exit(); 5757 } 5758 5759 /* 5760 * Increase the starting virtual address of the given mapping if a 5761 * different alignment might result in more superpage mappings. 5762 */ 5763 void 5764 mmu_radix_align_superpage(vm_object_t object, vm_ooffset_t offset, 5765 vm_offset_t *addr, vm_size_t size) 5766 { 5767 5768 CTR5(KTR_PMAP, "%s(%p, %#x, %p, %#x)", __func__, object, offset, addr, 5769 size); 5770 vm_offset_t superpage_offset; 5771 5772 if (size < L3_PAGE_SIZE) 5773 return; 5774 if (object != NULL && (object->flags & OBJ_COLORED) != 0) 5775 offset += ptoa(object->pg_color); 5776 superpage_offset = offset & L3_PAGE_MASK; 5777 if (size - ((L3_PAGE_SIZE - superpage_offset) & L3_PAGE_MASK) < L3_PAGE_SIZE || 5778 (*addr & L3_PAGE_MASK) == superpage_offset) 5779 return; 5780 if ((*addr & L3_PAGE_MASK) < superpage_offset) 5781 *addr = (*addr & ~L3_PAGE_MASK) + superpage_offset; 5782 else 5783 *addr = ((*addr + L3_PAGE_MASK) & ~L3_PAGE_MASK) + superpage_offset; 5784 } 5785 5786 static void * 5787 mmu_radix_mapdev_attr(vm_paddr_t pa, vm_size_t size, vm_memattr_t attr) 5788 { 5789 vm_offset_t va, tmpva, ppa, offset; 5790 5791 ppa = trunc_page(pa); 5792 offset = pa & PAGE_MASK; 5793 size = roundup2(offset + size, PAGE_SIZE); 5794 if (pa < powerpc_ptob(Maxmem)) 5795 panic("bad pa: %#lx less than Maxmem %#lx\n", 5796 pa, powerpc_ptob(Maxmem)); 5797 va = kva_alloc(size); 5798 if (bootverbose) 5799 printf("%s(%#lx, %lu, %d)\n", __func__, pa, size, attr); 5800 KASSERT(size > 0, ("%s(%#lx, %lu, %d)", __func__, pa, size, attr)); 5801 5802 if (!va) 5803 panic("%s: Couldn't alloc kernel virtual memory", __func__); 5804 5805 for (tmpva = va; size > 0;) { 5806 mmu_radix_kenter_attr(tmpva, ppa, attr); 5807 size -= PAGE_SIZE; 5808 tmpva += PAGE_SIZE; 5809 ppa += PAGE_SIZE; 5810 } 5811 ptesync(); 5812 5813 return ((void *)(va + offset)); 5814 } 5815 5816 static void * 5817 mmu_radix_mapdev(vm_paddr_t pa, vm_size_t size) 5818 { 5819 5820 CTR3(KTR_PMAP, "%s(%#x, %#x)", __func__, pa, size); 5821 5822 return (mmu_radix_mapdev_attr(pa, size, VM_MEMATTR_DEFAULT)); 5823 } 5824 5825 void 5826 mmu_radix_page_set_memattr(vm_page_t m, vm_memattr_t ma) 5827 { 5828 5829 CTR3(KTR_PMAP, "%s(%p, %#x)", __func__, m, ma); 5830 m->md.mdpg_cache_attrs = ma; 5831 5832 /* 5833 * If "m" is a normal page, update its direct mapping. This update 5834 * can be relied upon to perform any cache operations that are 5835 * required for data coherence. 5836 */ 5837 if ((m->flags & PG_FICTITIOUS) == 0 && 5838 mmu_radix_change_attr(PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)), 5839 PAGE_SIZE, m->md.mdpg_cache_attrs)) 5840 panic("memory attribute change on the direct map failed"); 5841 } 5842 5843 static void 5844 mmu_radix_unmapdev(vm_offset_t va, vm_size_t size) 5845 { 5846 vm_offset_t offset; 5847 5848 CTR3(KTR_PMAP, "%s(%#x, %#x)", __func__, va, size); 5849 /* If we gave a direct map region in pmap_mapdev, do nothing */ 5850 if (va >= DMAP_MIN_ADDRESS && va < DMAP_MAX_ADDRESS) 5851 return; 5852 5853 offset = va & PAGE_MASK; 5854 size = round_page(offset + size); 5855 va = trunc_page(va); 5856 5857 if (pmap_initialized) { 5858 mmu_radix_qremove(va, atop(size)); 5859 kva_free(va, size); 5860 } 5861 } 5862 5863 static __inline void 5864 pmap_pte_attr(pt_entry_t *pte, uint64_t cache_bits, uint64_t mask) 5865 { 5866 uint64_t opte, npte; 5867 5868 /* 5869 * The cache mode bits are all in the low 32-bits of the 5870 * PTE, so we can just spin on updating the low 32-bits. 5871 */ 5872 do { 5873 opte = be64toh(*pte); 5874 npte = opte & ~mask; 5875 npte |= cache_bits; 5876 } while (npte != opte && !atomic_cmpset_long(pte, htobe64(opte), htobe64(npte))); 5877 } 5878 5879 /* 5880 * Tries to demote a 1GB page mapping. 5881 */ 5882 static boolean_t 5883 pmap_demote_l2e(pmap_t pmap, pml2_entry_t *l2e, vm_offset_t va) 5884 { 5885 pml2_entry_t oldpdpe; 5886 pml3_entry_t *firstpde, newpde, *pde; 5887 vm_paddr_t pdpgpa; 5888 vm_page_t pdpg; 5889 5890 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 5891 oldpdpe = be64toh(*l2e); 5892 KASSERT((oldpdpe & (RPTE_LEAF | PG_V)) == (RPTE_LEAF | PG_V), 5893 ("pmap_demote_pdpe: oldpdpe is missing PG_PS and/or PG_V")); 5894 pdpg = vm_page_alloc(NULL, va >> L2_PAGE_SIZE_SHIFT, 5895 VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED); 5896 if (pdpg == NULL) { 5897 CTR2(KTR_PMAP, "pmap_demote_pdpe: failure for va %#lx" 5898 " in pmap %p", va, pmap); 5899 return (FALSE); 5900 } 5901 pdpgpa = VM_PAGE_TO_PHYS(pdpg); 5902 firstpde = (pml3_entry_t *)PHYS_TO_DMAP(pdpgpa); 5903 KASSERT((oldpdpe & PG_A) != 0, 5904 ("pmap_demote_pdpe: oldpdpe is missing PG_A")); 5905 KASSERT((oldpdpe & (PG_M | PG_RW)) != PG_RW, 5906 ("pmap_demote_pdpe: oldpdpe is missing PG_M")); 5907 newpde = oldpdpe; 5908 5909 /* 5910 * Initialize the page directory page. 5911 */ 5912 for (pde = firstpde; pde < firstpde + NPDEPG; pde++) { 5913 *pde = htobe64(newpde); 5914 newpde += L3_PAGE_SIZE; 5915 } 5916 5917 /* 5918 * Demote the mapping. 5919 */ 5920 pde_store(l2e, pdpgpa); 5921 5922 /* 5923 * Flush PWC --- XXX revisit 5924 */ 5925 pmap_invalidate_all(pmap); 5926 5927 pmap_l2e_demotions++; 5928 CTR2(KTR_PMAP, "pmap_demote_pdpe: success for va %#lx" 5929 " in pmap %p", va, pmap); 5930 return (TRUE); 5931 } 5932 5933 vm_paddr_t 5934 mmu_radix_kextract(vm_offset_t va) 5935 { 5936 pml3_entry_t l3e; 5937 vm_paddr_t pa; 5938 5939 CTR2(KTR_PMAP, "%s(%#x)", __func__, va); 5940 if (va >= DMAP_MIN_ADDRESS && va < DMAP_MAX_ADDRESS) { 5941 pa = DMAP_TO_PHYS(va); 5942 } else { 5943 /* Big-endian PTE on stack */ 5944 l3e = *pmap_pml3e(kernel_pmap, va); 5945 if (be64toh(l3e) & RPTE_LEAF) { 5946 pa = (be64toh(l3e) & PG_PS_FRAME) | (va & L3_PAGE_MASK); 5947 pa |= (va & L3_PAGE_MASK); 5948 } else { 5949 /* 5950 * Beware of a concurrent promotion that changes the 5951 * PDE at this point! For example, vtopte() must not 5952 * be used to access the PTE because it would use the 5953 * new PDE. It is, however, safe to use the old PDE 5954 * because the page table page is preserved by the 5955 * promotion. 5956 */ 5957 pa = be64toh(*pmap_l3e_to_pte(&l3e, va)); 5958 pa = (pa & PG_FRAME) | (va & PAGE_MASK); 5959 pa |= (va & PAGE_MASK); 5960 } 5961 } 5962 return (pa); 5963 } 5964 5965 static pt_entry_t 5966 mmu_radix_calc_wimg(vm_paddr_t pa, vm_memattr_t ma) 5967 { 5968 5969 if (ma != VM_MEMATTR_DEFAULT) { 5970 return pmap_cache_bits(ma); 5971 } 5972 5973 /* 5974 * Assume the page is cache inhibited and access is guarded unless 5975 * it's in our available memory array. 5976 */ 5977 for (int i = 0; i < pregions_sz; i++) { 5978 if ((pa >= pregions[i].mr_start) && 5979 (pa < (pregions[i].mr_start + pregions[i].mr_size))) 5980 return (RPTE_ATTR_MEM); 5981 } 5982 return (RPTE_ATTR_GUARDEDIO); 5983 } 5984 5985 static void 5986 mmu_radix_kenter_attr(vm_offset_t va, vm_paddr_t pa, vm_memattr_t ma) 5987 { 5988 pt_entry_t *pte, pteval; 5989 uint64_t cache_bits; 5990 5991 pte = kvtopte(va); 5992 MPASS(pte != NULL); 5993 pteval = pa | RPTE_EAA_R | RPTE_EAA_W | RPTE_EAA_P | PG_M | PG_A; 5994 cache_bits = mmu_radix_calc_wimg(pa, ma); 5995 pte_store(pte, pteval | cache_bits); 5996 } 5997 5998 void 5999 mmu_radix_kremove(vm_offset_t va) 6000 { 6001 pt_entry_t *pte; 6002 6003 CTR2(KTR_PMAP, "%s(%#x)", __func__, va); 6004 6005 pte = kvtopte(va); 6006 pte_clear(pte); 6007 } 6008 6009 int 6010 mmu_radix_decode_kernel_ptr(vm_offset_t addr, 6011 int *is_user, vm_offset_t *decoded) 6012 { 6013 6014 CTR2(KTR_PMAP, "%s(%#jx)", __func__, (uintmax_t)addr); 6015 *decoded = addr; 6016 *is_user = (addr < VM_MAXUSER_ADDRESS); 6017 return (0); 6018 } 6019 6020 static boolean_t 6021 mmu_radix_dev_direct_mapped(vm_paddr_t pa, vm_size_t size) 6022 { 6023 6024 CTR3(KTR_PMAP, "%s(%#x, %#x)", __func__, pa, size); 6025 return (mem_valid(pa, size)); 6026 } 6027 6028 static void 6029 mmu_radix_scan_init() 6030 { 6031 6032 CTR1(KTR_PMAP, "%s()", __func__); 6033 UNIMPLEMENTED(); 6034 } 6035 6036 static void 6037 mmu_radix_dumpsys_map(vm_paddr_t pa, size_t sz, 6038 void **va) 6039 { 6040 CTR4(KTR_PMAP, "%s(%#jx, %#zx, %p)", __func__, (uintmax_t)pa, sz, va); 6041 UNIMPLEMENTED(); 6042 } 6043 6044 vm_offset_t 6045 mmu_radix_quick_enter_page(vm_page_t m) 6046 { 6047 vm_paddr_t paddr; 6048 6049 CTR2(KTR_PMAP, "%s(%p)", __func__, m); 6050 paddr = VM_PAGE_TO_PHYS(m); 6051 return (PHYS_TO_DMAP(paddr)); 6052 } 6053 6054 void 6055 mmu_radix_quick_remove_page(vm_offset_t addr __unused) 6056 { 6057 /* no work to do here */ 6058 CTR2(KTR_PMAP, "%s(%#x)", __func__, addr); 6059 } 6060 6061 static void 6062 pmap_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva) 6063 { 6064 cpu_flush_dcache((void *)sva, eva - sva); 6065 } 6066 6067 int 6068 mmu_radix_change_attr(vm_offset_t va, vm_size_t size, 6069 vm_memattr_t mode) 6070 { 6071 int error; 6072 6073 CTR4(KTR_PMAP, "%s(%#x, %#zx, %d)", __func__, va, size, mode); 6074 PMAP_LOCK(kernel_pmap); 6075 error = pmap_change_attr_locked(va, size, mode, true); 6076 PMAP_UNLOCK(kernel_pmap); 6077 return (error); 6078 } 6079 6080 static int 6081 pmap_change_attr_locked(vm_offset_t va, vm_size_t size, int mode, bool flush) 6082 { 6083 vm_offset_t base, offset, tmpva; 6084 vm_paddr_t pa_start, pa_end, pa_end1; 6085 pml2_entry_t *l2e; 6086 pml3_entry_t *l3e; 6087 pt_entry_t *pte; 6088 int cache_bits, error; 6089 boolean_t changed; 6090 6091 PMAP_LOCK_ASSERT(kernel_pmap, MA_OWNED); 6092 base = trunc_page(va); 6093 offset = va & PAGE_MASK; 6094 size = round_page(offset + size); 6095 6096 /* 6097 * Only supported on kernel virtual addresses, including the direct 6098 * map but excluding the recursive map. 6099 */ 6100 if (base < DMAP_MIN_ADDRESS) 6101 return (EINVAL); 6102 6103 cache_bits = pmap_cache_bits(mode); 6104 changed = FALSE; 6105 6106 /* 6107 * Pages that aren't mapped aren't supported. Also break down 2MB pages 6108 * into 4KB pages if required. 6109 */ 6110 for (tmpva = base; tmpva < base + size; ) { 6111 l2e = pmap_pml2e(kernel_pmap, tmpva); 6112 if (l2e == NULL || *l2e == 0) 6113 return (EINVAL); 6114 if (be64toh(*l2e) & RPTE_LEAF) { 6115 /* 6116 * If the current 1GB page already has the required 6117 * memory type, then we need not demote this page. Just 6118 * increment tmpva to the next 1GB page frame. 6119 */ 6120 if ((be64toh(*l2e) & RPTE_ATTR_MASK) == cache_bits) { 6121 tmpva = trunc_1gpage(tmpva) + L2_PAGE_SIZE; 6122 continue; 6123 } 6124 6125 /* 6126 * If the current offset aligns with a 1GB page frame 6127 * and there is at least 1GB left within the range, then 6128 * we need not break down this page into 2MB pages. 6129 */ 6130 if ((tmpva & L2_PAGE_MASK) == 0 && 6131 tmpva + L2_PAGE_MASK < base + size) { 6132 tmpva += L2_PAGE_MASK; 6133 continue; 6134 } 6135 if (!pmap_demote_l2e(kernel_pmap, l2e, tmpva)) 6136 return (ENOMEM); 6137 } 6138 l3e = pmap_l2e_to_l3e(l2e, tmpva); 6139 KASSERT(l3e != NULL, ("no l3e entry for %#lx in %p\n", 6140 tmpva, l2e)); 6141 if (*l3e == 0) 6142 return (EINVAL); 6143 if (be64toh(*l3e) & RPTE_LEAF) { 6144 /* 6145 * If the current 2MB page already has the required 6146 * memory type, then we need not demote this page. Just 6147 * increment tmpva to the next 2MB page frame. 6148 */ 6149 if ((be64toh(*l3e) & RPTE_ATTR_MASK) == cache_bits) { 6150 tmpva = trunc_2mpage(tmpva) + L3_PAGE_SIZE; 6151 continue; 6152 } 6153 6154 /* 6155 * If the current offset aligns with a 2MB page frame 6156 * and there is at least 2MB left within the range, then 6157 * we need not break down this page into 4KB pages. 6158 */ 6159 if ((tmpva & L3_PAGE_MASK) == 0 && 6160 tmpva + L3_PAGE_MASK < base + size) { 6161 tmpva += L3_PAGE_SIZE; 6162 continue; 6163 } 6164 if (!pmap_demote_l3e(kernel_pmap, l3e, tmpva)) 6165 return (ENOMEM); 6166 } 6167 pte = pmap_l3e_to_pte(l3e, tmpva); 6168 if (*pte == 0) 6169 return (EINVAL); 6170 tmpva += PAGE_SIZE; 6171 } 6172 error = 0; 6173 6174 /* 6175 * Ok, all the pages exist, so run through them updating their 6176 * cache mode if required. 6177 */ 6178 pa_start = pa_end = 0; 6179 for (tmpva = base; tmpva < base + size; ) { 6180 l2e = pmap_pml2e(kernel_pmap, tmpva); 6181 if (be64toh(*l2e) & RPTE_LEAF) { 6182 if ((be64toh(*l2e) & RPTE_ATTR_MASK) != cache_bits) { 6183 pmap_pte_attr(l2e, cache_bits, 6184 RPTE_ATTR_MASK); 6185 changed = TRUE; 6186 } 6187 if (tmpva >= VM_MIN_KERNEL_ADDRESS && 6188 (*l2e & PG_PS_FRAME) < dmaplimit) { 6189 if (pa_start == pa_end) { 6190 /* Start physical address run. */ 6191 pa_start = be64toh(*l2e) & PG_PS_FRAME; 6192 pa_end = pa_start + L2_PAGE_SIZE; 6193 } else if (pa_end == (be64toh(*l2e) & PG_PS_FRAME)) 6194 pa_end += L2_PAGE_SIZE; 6195 else { 6196 /* Run ended, update direct map. */ 6197 error = pmap_change_attr_locked( 6198 PHYS_TO_DMAP(pa_start), 6199 pa_end - pa_start, mode, flush); 6200 if (error != 0) 6201 break; 6202 /* Start physical address run. */ 6203 pa_start = be64toh(*l2e) & PG_PS_FRAME; 6204 pa_end = pa_start + L2_PAGE_SIZE; 6205 } 6206 } 6207 tmpva = trunc_1gpage(tmpva) + L2_PAGE_SIZE; 6208 continue; 6209 } 6210 l3e = pmap_l2e_to_l3e(l2e, tmpva); 6211 if (be64toh(*l3e) & RPTE_LEAF) { 6212 if ((be64toh(*l3e) & RPTE_ATTR_MASK) != cache_bits) { 6213 pmap_pte_attr(l3e, cache_bits, 6214 RPTE_ATTR_MASK); 6215 changed = TRUE; 6216 } 6217 if (tmpva >= VM_MIN_KERNEL_ADDRESS && 6218 (be64toh(*l3e) & PG_PS_FRAME) < dmaplimit) { 6219 if (pa_start == pa_end) { 6220 /* Start physical address run. */ 6221 pa_start = be64toh(*l3e) & PG_PS_FRAME; 6222 pa_end = pa_start + L3_PAGE_SIZE; 6223 } else if (pa_end == (be64toh(*l3e) & PG_PS_FRAME)) 6224 pa_end += L3_PAGE_SIZE; 6225 else { 6226 /* Run ended, update direct map. */ 6227 error = pmap_change_attr_locked( 6228 PHYS_TO_DMAP(pa_start), 6229 pa_end - pa_start, mode, flush); 6230 if (error != 0) 6231 break; 6232 /* Start physical address run. */ 6233 pa_start = be64toh(*l3e) & PG_PS_FRAME; 6234 pa_end = pa_start + L3_PAGE_SIZE; 6235 } 6236 } 6237 tmpva = trunc_2mpage(tmpva) + L3_PAGE_SIZE; 6238 } else { 6239 pte = pmap_l3e_to_pte(l3e, tmpva); 6240 if ((be64toh(*pte) & RPTE_ATTR_MASK) != cache_bits) { 6241 pmap_pte_attr(pte, cache_bits, 6242 RPTE_ATTR_MASK); 6243 changed = TRUE; 6244 } 6245 if (tmpva >= VM_MIN_KERNEL_ADDRESS && 6246 (be64toh(*pte) & PG_FRAME) < dmaplimit) { 6247 if (pa_start == pa_end) { 6248 /* Start physical address run. */ 6249 pa_start = be64toh(*pte) & PG_FRAME; 6250 pa_end = pa_start + PAGE_SIZE; 6251 } else if (pa_end == (be64toh(*pte) & PG_FRAME)) 6252 pa_end += PAGE_SIZE; 6253 else { 6254 /* Run ended, update direct map. */ 6255 error = pmap_change_attr_locked( 6256 PHYS_TO_DMAP(pa_start), 6257 pa_end - pa_start, mode, flush); 6258 if (error != 0) 6259 break; 6260 /* Start physical address run. */ 6261 pa_start = be64toh(*pte) & PG_FRAME; 6262 pa_end = pa_start + PAGE_SIZE; 6263 } 6264 } 6265 tmpva += PAGE_SIZE; 6266 } 6267 } 6268 if (error == 0 && pa_start != pa_end && pa_start < dmaplimit) { 6269 pa_end1 = MIN(pa_end, dmaplimit); 6270 if (pa_start != pa_end1) 6271 error = pmap_change_attr_locked(PHYS_TO_DMAP(pa_start), 6272 pa_end1 - pa_start, mode, flush); 6273 } 6274 6275 /* 6276 * Flush CPU caches if required to make sure any data isn't cached that 6277 * shouldn't be, etc. 6278 */ 6279 if (changed) { 6280 pmap_invalidate_all(kernel_pmap); 6281 6282 if (flush) 6283 pmap_invalidate_cache_range(base, tmpva); 6284 } 6285 return (error); 6286 } 6287 6288 /* 6289 * Allocate physical memory for the vm_page array and map it into KVA, 6290 * attempting to back the vm_pages with domain-local memory. 6291 */ 6292 void 6293 mmu_radix_page_array_startup(long pages) 6294 { 6295 #ifdef notyet 6296 pml2_entry_t *l2e; 6297 pml3_entry_t *pde; 6298 pml3_entry_t newl3; 6299 vm_offset_t va; 6300 long pfn; 6301 int domain, i; 6302 #endif 6303 vm_paddr_t pa; 6304 vm_offset_t start, end; 6305 6306 vm_page_array_size = pages; 6307 6308 start = VM_MIN_KERNEL_ADDRESS; 6309 end = start + pages * sizeof(struct vm_page); 6310 6311 pa = vm_phys_early_alloc(0, end - start); 6312 6313 start = mmu_radix_map(&start, pa, end - start, VM_MEMATTR_DEFAULT); 6314 #ifdef notyet 6315 /* TODO: NUMA vm_page_array. Blocked out until then (copied from amd64). */ 6316 for (va = start; va < end; va += L3_PAGE_SIZE) { 6317 pfn = first_page + (va - start) / sizeof(struct vm_page); 6318 domain = _vm_phys_domain(ptoa(pfn)); 6319 l2e = pmap_pml2e(kernel_pmap, va); 6320 if ((be64toh(*l2e) & PG_V) == 0) { 6321 pa = vm_phys_early_alloc(domain, PAGE_SIZE); 6322 dump_add_page(pa); 6323 pagezero(PHYS_TO_DMAP(pa)); 6324 pde_store(l2e, (pml2_entry_t)pa); 6325 } 6326 pde = pmap_l2e_to_l3e(l2e, va); 6327 if ((be64toh(*pde) & PG_V) != 0) 6328 panic("Unexpected pde %p", pde); 6329 pa = vm_phys_early_alloc(domain, L3_PAGE_SIZE); 6330 for (i = 0; i < NPDEPG; i++) 6331 dump_add_page(pa + i * PAGE_SIZE); 6332 newl3 = (pml3_entry_t)(pa | RPTE_EAA_P | RPTE_EAA_R | RPTE_EAA_W); 6333 pte_store(pde, newl3); 6334 } 6335 #endif 6336 vm_page_array = (vm_page_t)start; 6337 } 6338 6339 #ifdef DDB 6340 #include <sys/kdb.h> 6341 #include <ddb/ddb.h> 6342 6343 static void 6344 pmap_pte_walk(pml1_entry_t *l1, vm_offset_t va) 6345 { 6346 pml1_entry_t *l1e; 6347 pml2_entry_t *l2e; 6348 pml3_entry_t *l3e; 6349 pt_entry_t *pte; 6350 6351 l1e = &l1[pmap_pml1e_index(va)]; 6352 db_printf("VA %#016lx l1e %#016lx", va, be64toh(*l1e)); 6353 if ((be64toh(*l1e) & PG_V) == 0) { 6354 db_printf("\n"); 6355 return; 6356 } 6357 l2e = pmap_l1e_to_l2e(l1e, va); 6358 db_printf(" l2e %#016lx", be64toh(*l2e)); 6359 if ((be64toh(*l2e) & PG_V) == 0 || (be64toh(*l2e) & RPTE_LEAF) != 0) { 6360 db_printf("\n"); 6361 return; 6362 } 6363 l3e = pmap_l2e_to_l3e(l2e, va); 6364 db_printf(" l3e %#016lx", be64toh(*l3e)); 6365 if ((be64toh(*l3e) & PG_V) == 0 || (be64toh(*l3e) & RPTE_LEAF) != 0) { 6366 db_printf("\n"); 6367 return; 6368 } 6369 pte = pmap_l3e_to_pte(l3e, va); 6370 db_printf(" pte %#016lx\n", be64toh(*pte)); 6371 } 6372 6373 void 6374 pmap_page_print_mappings(vm_page_t m) 6375 { 6376 pmap_t pmap; 6377 pv_entry_t pv; 6378 6379 db_printf("page %p(%lx)\n", m, m->phys_addr); 6380 /* need to elide locks if running in ddb */ 6381 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) { 6382 db_printf("pv: %p ", pv); 6383 db_printf("va: %#016lx ", pv->pv_va); 6384 pmap = PV_PMAP(pv); 6385 db_printf("pmap %p ", pmap); 6386 if (pmap != NULL) { 6387 db_printf("asid: %lu\n", pmap->pm_pid); 6388 pmap_pte_walk(pmap->pm_pml1, pv->pv_va); 6389 } 6390 } 6391 } 6392 6393 DB_SHOW_COMMAND(pte, pmap_print_pte) 6394 { 6395 vm_offset_t va; 6396 pmap_t pmap; 6397 6398 if (!have_addr) { 6399 db_printf("show pte addr\n"); 6400 return; 6401 } 6402 va = (vm_offset_t)addr; 6403 6404 if (va >= DMAP_MIN_ADDRESS) 6405 pmap = kernel_pmap; 6406 else if (kdb_thread != NULL) 6407 pmap = vmspace_pmap(kdb_thread->td_proc->p_vmspace); 6408 else 6409 pmap = vmspace_pmap(curthread->td_proc->p_vmspace); 6410 6411 pmap_pte_walk(pmap->pm_pml1, va); 6412 } 6413 6414 #endif 6415