1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2018 Matthew Macy 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __FBSDID("$FreeBSD$"); 30 31 #include <sys/param.h> 32 #include <sys/kernel.h> 33 #include <sys/systm.h> 34 #include <sys/conf.h> 35 #include <sys/bitstring.h> 36 #include <sys/queue.h> 37 #include <sys/cpuset.h> 38 #include <sys/endian.h> 39 #include <sys/kerneldump.h> 40 #include <sys/ktr.h> 41 #include <sys/lock.h> 42 #include <sys/syslog.h> 43 #include <sys/msgbuf.h> 44 #include <sys/malloc.h> 45 #include <sys/mman.h> 46 #include <sys/mutex.h> 47 #include <sys/proc.h> 48 #include <sys/rwlock.h> 49 #include <sys/sched.h> 50 #include <sys/sysctl.h> 51 #include <sys/systm.h> 52 #include <sys/vmem.h> 53 #include <sys/vmmeter.h> 54 #include <sys/smp.h> 55 56 #include <sys/kdb.h> 57 58 #include <dev/ofw/openfirm.h> 59 60 #include <vm/vm.h> 61 #include <vm/pmap.h> 62 #include <vm/vm_param.h> 63 #include <vm/vm_kern.h> 64 #include <vm/vm_page.h> 65 #include <vm/vm_map.h> 66 #include <vm/vm_object.h> 67 #include <vm/vm_extern.h> 68 #include <vm/vm_pageout.h> 69 #include <vm/vm_phys.h> 70 #include <vm/vm_reserv.h> 71 #include <vm/vm_dumpset.h> 72 #include <vm/uma.h> 73 74 #include <machine/_inttypes.h> 75 #include <machine/cpu.h> 76 #include <machine/platform.h> 77 #include <machine/frame.h> 78 #include <machine/md_var.h> 79 #include <machine/psl.h> 80 #include <machine/bat.h> 81 #include <machine/hid.h> 82 #include <machine/pte.h> 83 #include <machine/sr.h> 84 #include <machine/trap.h> 85 #include <machine/mmuvar.h> 86 87 #ifdef INVARIANTS 88 #include <vm/uma_dbg.h> 89 #endif 90 91 #define PPC_BITLSHIFT(bit) (sizeof(long)*NBBY - 1 - (bit)) 92 #define PPC_BIT(bit) (1UL << PPC_BITLSHIFT(bit)) 93 #define PPC_BITLSHIFT_VAL(val, bit) ((val) << PPC_BITLSHIFT(bit)) 94 95 #include "opt_ddb.h" 96 #ifdef DDB 97 static void pmap_pte_walk(pml1_entry_t *l1, vm_offset_t va); 98 #endif 99 100 #define PG_W RPTE_WIRED 101 #define PG_V RPTE_VALID 102 #define PG_MANAGED RPTE_MANAGED 103 #define PG_PROMOTED RPTE_PROMOTED 104 #define PG_M RPTE_C 105 #define PG_A RPTE_R 106 #define PG_X RPTE_EAA_X 107 #define PG_RW RPTE_EAA_W 108 #define PG_PTE_CACHE RPTE_ATTR_MASK 109 110 #define RPTE_SHIFT 9 111 #define NLS_MASK ((1UL<<5)-1) 112 #define RPTE_ENTRIES (1UL<<RPTE_SHIFT) 113 #define RPTE_MASK (RPTE_ENTRIES-1) 114 115 #define NLB_SHIFT 0 116 #define NLB_MASK (((1UL<<52)-1) << 8) 117 118 extern int nkpt; 119 extern caddr_t crashdumpmap; 120 121 #define RIC_FLUSH_TLB 0 122 #define RIC_FLUSH_PWC 1 123 #define RIC_FLUSH_ALL 2 124 125 #define POWER9_TLB_SETS_RADIX 128 /* # sets in POWER9 TLB Radix mode */ 126 127 #define PPC_INST_TLBIE 0x7c000264 128 #define PPC_INST_TLBIEL 0x7c000224 129 #define PPC_INST_SLBIA 0x7c0003e4 130 131 #define ___PPC_RA(a) (((a) & 0x1f) << 16) 132 #define ___PPC_RB(b) (((b) & 0x1f) << 11) 133 #define ___PPC_RS(s) (((s) & 0x1f) << 21) 134 #define ___PPC_RT(t) ___PPC_RS(t) 135 #define ___PPC_R(r) (((r) & 0x1) << 16) 136 #define ___PPC_PRS(prs) (((prs) & 0x1) << 17) 137 #define ___PPC_RIC(ric) (((ric) & 0x3) << 18) 138 139 #define PPC_SLBIA(IH) __XSTRING(.long PPC_INST_SLBIA | \ 140 ((IH & 0x7) << 21)) 141 #define PPC_TLBIE_5(rb,rs,ric,prs,r) \ 142 __XSTRING(.long PPC_INST_TLBIE | \ 143 ___PPC_RB(rb) | ___PPC_RS(rs) | \ 144 ___PPC_RIC(ric) | ___PPC_PRS(prs) | \ 145 ___PPC_R(r)) 146 147 #define PPC_TLBIEL(rb,rs,ric,prs,r) \ 148 __XSTRING(.long PPC_INST_TLBIEL | \ 149 ___PPC_RB(rb) | ___PPC_RS(rs) | \ 150 ___PPC_RIC(ric) | ___PPC_PRS(prs) | \ 151 ___PPC_R(r)) 152 153 #define PPC_INVALIDATE_ERAT PPC_SLBIA(7) 154 155 static __inline void 156 ttusync(void) 157 { 158 __asm __volatile("eieio; tlbsync; ptesync" ::: "memory"); 159 } 160 161 #define TLBIEL_INVAL_SEL_MASK 0xc00 /* invalidation selector */ 162 #define TLBIEL_INVAL_PAGE 0x000 /* invalidate a single page */ 163 #define TLBIEL_INVAL_SET_PID 0x400 /* invalidate a set for the current PID */ 164 #define TLBIEL_INVAL_SET_LPID 0x800 /* invalidate a set for current LPID */ 165 #define TLBIEL_INVAL_SET 0xc00 /* invalidate a set for all LPIDs */ 166 167 #define TLBIE_ACTUAL_PAGE_MASK 0xe0 168 #define TLBIE_ACTUAL_PAGE_4K 0x00 169 #define TLBIE_ACTUAL_PAGE_64K 0xa0 170 #define TLBIE_ACTUAL_PAGE_2M 0x20 171 #define TLBIE_ACTUAL_PAGE_1G 0x40 172 173 #define TLBIE_PRS_PARTITION_SCOPE 0x0 174 #define TLBIE_PRS_PROCESS_SCOPE 0x1 175 176 #define TLBIE_RIC_INVALIDATE_TLB 0x0 /* Invalidate just TLB */ 177 #define TLBIE_RIC_INVALIDATE_PWC 0x1 /* Invalidate just PWC */ 178 #define TLBIE_RIC_INVALIDATE_ALL 0x2 /* Invalidate TLB, PWC, 179 * cached {proc, part}tab entries 180 */ 181 #define TLBIE_RIC_INVALIDATE_SEQ 0x3 /* HPT - only: 182 * Invalidate a range of translations 183 */ 184 185 static __always_inline void 186 radix_tlbie(uint8_t ric, uint8_t prs, uint16_t is, uint32_t pid, uint32_t lpid, 187 vm_offset_t va, uint16_t ap) 188 { 189 uint64_t rb, rs; 190 191 MPASS((va & PAGE_MASK) == 0); 192 193 rs = ((uint64_t)pid << 32) | lpid; 194 rb = va | is | ap; 195 __asm __volatile(PPC_TLBIE_5(%0, %1, %2, %3, 1) : : 196 "r" (rb), "r" (rs), "i" (ric), "i" (prs)); 197 } 198 199 static __inline void 200 radix_tlbie_invlpg_user_4k(uint32_t pid, vm_offset_t va) 201 { 202 203 radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE, 204 TLBIEL_INVAL_PAGE, pid, 0, va, TLBIE_ACTUAL_PAGE_4K); 205 } 206 207 static __inline void 208 radix_tlbie_invlpg_user_2m(uint32_t pid, vm_offset_t va) 209 { 210 211 radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE, 212 TLBIEL_INVAL_PAGE, pid, 0, va, TLBIE_ACTUAL_PAGE_2M); 213 } 214 215 static __inline void 216 radix_tlbie_invlpwc_user(uint32_t pid) 217 { 218 219 radix_tlbie(TLBIE_RIC_INVALIDATE_PWC, TLBIE_PRS_PROCESS_SCOPE, 220 TLBIEL_INVAL_SET_PID, pid, 0, 0, 0); 221 } 222 223 static __inline void 224 radix_tlbie_flush_user(uint32_t pid) 225 { 226 227 radix_tlbie(TLBIE_RIC_INVALIDATE_ALL, TLBIE_PRS_PROCESS_SCOPE, 228 TLBIEL_INVAL_SET_PID, pid, 0, 0, 0); 229 } 230 231 static __inline void 232 radix_tlbie_invlpg_kernel_4k(vm_offset_t va) 233 { 234 235 radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE, 236 TLBIEL_INVAL_PAGE, 0, 0, va, TLBIE_ACTUAL_PAGE_4K); 237 } 238 239 static __inline void 240 radix_tlbie_invlpg_kernel_2m(vm_offset_t va) 241 { 242 243 radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE, 244 TLBIEL_INVAL_PAGE, 0, 0, va, TLBIE_ACTUAL_PAGE_2M); 245 } 246 247 /* 1GB pages aren't currently supported. */ 248 static __inline __unused void 249 radix_tlbie_invlpg_kernel_1g(vm_offset_t va) 250 { 251 252 radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE, 253 TLBIEL_INVAL_PAGE, 0, 0, va, TLBIE_ACTUAL_PAGE_1G); 254 } 255 256 static __inline void 257 radix_tlbie_invlpwc_kernel(void) 258 { 259 260 radix_tlbie(TLBIE_RIC_INVALIDATE_PWC, TLBIE_PRS_PROCESS_SCOPE, 261 TLBIEL_INVAL_SET_LPID, 0, 0, 0, 0); 262 } 263 264 static __inline void 265 radix_tlbie_flush_kernel(void) 266 { 267 268 radix_tlbie(TLBIE_RIC_INVALIDATE_ALL, TLBIE_PRS_PROCESS_SCOPE, 269 TLBIEL_INVAL_SET_LPID, 0, 0, 0, 0); 270 } 271 272 static __inline vm_pindex_t 273 pmap_l3e_pindex(vm_offset_t va) 274 { 275 return ((va & PG_FRAME) >> L3_PAGE_SIZE_SHIFT); 276 } 277 278 static __inline vm_pindex_t 279 pmap_pml3e_index(vm_offset_t va) 280 { 281 282 return ((va >> L3_PAGE_SIZE_SHIFT) & RPTE_MASK); 283 } 284 285 static __inline vm_pindex_t 286 pmap_pml2e_index(vm_offset_t va) 287 { 288 return ((va >> L2_PAGE_SIZE_SHIFT) & RPTE_MASK); 289 } 290 291 static __inline vm_pindex_t 292 pmap_pml1e_index(vm_offset_t va) 293 { 294 return ((va & PG_FRAME) >> L1_PAGE_SIZE_SHIFT); 295 } 296 297 /* Return various clipped indexes for a given VA */ 298 static __inline vm_pindex_t 299 pmap_pte_index(vm_offset_t va) 300 { 301 302 return ((va >> PAGE_SHIFT) & RPTE_MASK); 303 } 304 305 /* Return a pointer to the PT slot that corresponds to a VA */ 306 static __inline pt_entry_t * 307 pmap_l3e_to_pte(pt_entry_t *l3e, vm_offset_t va) 308 { 309 pt_entry_t *pte; 310 vm_paddr_t ptepa; 311 312 ptepa = (be64toh(*l3e) & NLB_MASK); 313 pte = (pt_entry_t *)PHYS_TO_DMAP(ptepa); 314 return (&pte[pmap_pte_index(va)]); 315 } 316 317 /* Return a pointer to the PD slot that corresponds to a VA */ 318 static __inline pt_entry_t * 319 pmap_l2e_to_l3e(pt_entry_t *l2e, vm_offset_t va) 320 { 321 pt_entry_t *l3e; 322 vm_paddr_t l3pa; 323 324 l3pa = (be64toh(*l2e) & NLB_MASK); 325 l3e = (pml3_entry_t *)PHYS_TO_DMAP(l3pa); 326 return (&l3e[pmap_pml3e_index(va)]); 327 } 328 329 /* Return a pointer to the PD slot that corresponds to a VA */ 330 static __inline pt_entry_t * 331 pmap_l1e_to_l2e(pt_entry_t *l1e, vm_offset_t va) 332 { 333 pt_entry_t *l2e; 334 vm_paddr_t l2pa; 335 336 l2pa = (be64toh(*l1e) & NLB_MASK); 337 338 l2e = (pml2_entry_t *)PHYS_TO_DMAP(l2pa); 339 return (&l2e[pmap_pml2e_index(va)]); 340 } 341 342 static __inline pml1_entry_t * 343 pmap_pml1e(pmap_t pmap, vm_offset_t va) 344 { 345 346 return (&pmap->pm_pml1[pmap_pml1e_index(va)]); 347 } 348 349 static pt_entry_t * 350 pmap_pml2e(pmap_t pmap, vm_offset_t va) 351 { 352 pt_entry_t *l1e; 353 354 l1e = pmap_pml1e(pmap, va); 355 if (l1e == NULL || (be64toh(*l1e) & RPTE_VALID) == 0) 356 return (NULL); 357 return (pmap_l1e_to_l2e(l1e, va)); 358 } 359 360 static __inline pt_entry_t * 361 pmap_pml3e(pmap_t pmap, vm_offset_t va) 362 { 363 pt_entry_t *l2e; 364 365 l2e = pmap_pml2e(pmap, va); 366 if (l2e == NULL || (be64toh(*l2e) & RPTE_VALID) == 0) 367 return (NULL); 368 return (pmap_l2e_to_l3e(l2e, va)); 369 } 370 371 static __inline pt_entry_t * 372 pmap_pte(pmap_t pmap, vm_offset_t va) 373 { 374 pt_entry_t *l3e; 375 376 l3e = pmap_pml3e(pmap, va); 377 if (l3e == NULL || (be64toh(*l3e) & RPTE_VALID) == 0) 378 return (NULL); 379 return (pmap_l3e_to_pte(l3e, va)); 380 } 381 382 int nkpt = 64; 383 SYSCTL_INT(_machdep, OID_AUTO, nkpt, CTLFLAG_RD, &nkpt, 0, 384 "Number of kernel page table pages allocated on bootup"); 385 386 vm_paddr_t dmaplimit; 387 388 SYSCTL_DECL(_vm_pmap); 389 390 #ifdef INVARIANTS 391 #define VERBOSE_PMAP 0 392 #define VERBOSE_PROTECT 0 393 static int pmap_logging; 394 SYSCTL_INT(_vm_pmap, OID_AUTO, pmap_logging, CTLFLAG_RWTUN, 395 &pmap_logging, 0, "verbose debug logging"); 396 #endif 397 398 static u_int64_t KPTphys; /* phys addr of kernel level 1 */ 399 400 //static vm_paddr_t KERNend; /* phys addr of end of bootstrap data */ 401 402 static vm_offset_t qframe = 0; 403 static struct mtx qframe_mtx; 404 405 void mmu_radix_activate(struct thread *); 406 void mmu_radix_advise(pmap_t, vm_offset_t, vm_offset_t, int); 407 void mmu_radix_align_superpage(vm_object_t, vm_ooffset_t, vm_offset_t *, 408 vm_size_t); 409 void mmu_radix_clear_modify(vm_page_t); 410 void mmu_radix_copy(pmap_t, pmap_t, vm_offset_t, vm_size_t, vm_offset_t); 411 int mmu_radix_decode_kernel_ptr(vm_offset_t, int *, vm_offset_t *); 412 int mmu_radix_enter(pmap_t, vm_offset_t, vm_page_t, vm_prot_t, u_int, int8_t); 413 void mmu_radix_enter_object(pmap_t, vm_offset_t, vm_offset_t, vm_page_t, 414 vm_prot_t); 415 void mmu_radix_enter_quick(pmap_t, vm_offset_t, vm_page_t, vm_prot_t); 416 vm_paddr_t mmu_radix_extract(pmap_t pmap, vm_offset_t va); 417 vm_page_t mmu_radix_extract_and_hold(pmap_t, vm_offset_t, vm_prot_t); 418 void mmu_radix_kenter(vm_offset_t, vm_paddr_t); 419 vm_paddr_t mmu_radix_kextract(vm_offset_t); 420 void mmu_radix_kremove(vm_offset_t); 421 boolean_t mmu_radix_is_modified(vm_page_t); 422 boolean_t mmu_radix_is_prefaultable(pmap_t, vm_offset_t); 423 boolean_t mmu_radix_is_referenced(vm_page_t); 424 void mmu_radix_object_init_pt(pmap_t, vm_offset_t, vm_object_t, 425 vm_pindex_t, vm_size_t); 426 boolean_t mmu_radix_page_exists_quick(pmap_t, vm_page_t); 427 void mmu_radix_page_init(vm_page_t); 428 boolean_t mmu_radix_page_is_mapped(vm_page_t m); 429 void mmu_radix_page_set_memattr(vm_page_t, vm_memattr_t); 430 int mmu_radix_page_wired_mappings(vm_page_t); 431 int mmu_radix_pinit(pmap_t); 432 void mmu_radix_protect(pmap_t, vm_offset_t, vm_offset_t, vm_prot_t); 433 bool mmu_radix_ps_enabled(pmap_t); 434 void mmu_radix_qenter(vm_offset_t, vm_page_t *, int); 435 void mmu_radix_qremove(vm_offset_t, int); 436 vm_offset_t mmu_radix_quick_enter_page(vm_page_t); 437 void mmu_radix_quick_remove_page(vm_offset_t); 438 boolean_t mmu_radix_ts_referenced(vm_page_t); 439 void mmu_radix_release(pmap_t); 440 void mmu_radix_remove(pmap_t, vm_offset_t, vm_offset_t); 441 void mmu_radix_remove_all(vm_page_t); 442 void mmu_radix_remove_pages(pmap_t); 443 void mmu_radix_remove_write(vm_page_t); 444 void mmu_radix_unwire(pmap_t, vm_offset_t, vm_offset_t); 445 void mmu_radix_zero_page(vm_page_t); 446 void mmu_radix_zero_page_area(vm_page_t, int, int); 447 int mmu_radix_change_attr(vm_offset_t, vm_size_t, vm_memattr_t); 448 void mmu_radix_page_array_startup(long pages); 449 450 #include "mmu_oea64.h" 451 452 /* 453 * Kernel MMU interface 454 */ 455 456 static void mmu_radix_bootstrap(vm_offset_t, vm_offset_t); 457 458 static void mmu_radix_copy_page(vm_page_t, vm_page_t); 459 static void mmu_radix_copy_pages(vm_page_t *ma, vm_offset_t a_offset, 460 vm_page_t *mb, vm_offset_t b_offset, int xfersize); 461 static void mmu_radix_growkernel(vm_offset_t); 462 static void mmu_radix_init(void); 463 static int mmu_radix_mincore(pmap_t, vm_offset_t, vm_paddr_t *); 464 static vm_offset_t mmu_radix_map(vm_offset_t *, vm_paddr_t, vm_paddr_t, int); 465 static void mmu_radix_pinit0(pmap_t); 466 467 static void *mmu_radix_mapdev(vm_paddr_t, vm_size_t); 468 static void *mmu_radix_mapdev_attr(vm_paddr_t, vm_size_t, vm_memattr_t); 469 static void mmu_radix_unmapdev(vm_offset_t, vm_size_t); 470 static void mmu_radix_kenter_attr(vm_offset_t, vm_paddr_t, vm_memattr_t ma); 471 static boolean_t mmu_radix_dev_direct_mapped(vm_paddr_t, vm_size_t); 472 static void mmu_radix_dumpsys_map(vm_paddr_t pa, size_t sz, void **va); 473 static void mmu_radix_scan_init(void); 474 static void mmu_radix_cpu_bootstrap(int ap); 475 static void mmu_radix_tlbie_all(void); 476 477 static struct pmap_funcs mmu_radix_methods = { 478 .bootstrap = mmu_radix_bootstrap, 479 .copy_page = mmu_radix_copy_page, 480 .copy_pages = mmu_radix_copy_pages, 481 .cpu_bootstrap = mmu_radix_cpu_bootstrap, 482 .growkernel = mmu_radix_growkernel, 483 .init = mmu_radix_init, 484 .map = mmu_radix_map, 485 .mincore = mmu_radix_mincore, 486 .pinit = mmu_radix_pinit, 487 .pinit0 = mmu_radix_pinit0, 488 489 .mapdev = mmu_radix_mapdev, 490 .mapdev_attr = mmu_radix_mapdev_attr, 491 .unmapdev = mmu_radix_unmapdev, 492 .kenter_attr = mmu_radix_kenter_attr, 493 .dev_direct_mapped = mmu_radix_dev_direct_mapped, 494 .dumpsys_pa_init = mmu_radix_scan_init, 495 .dumpsys_map_chunk = mmu_radix_dumpsys_map, 496 .page_is_mapped = mmu_radix_page_is_mapped, 497 .ps_enabled = mmu_radix_ps_enabled, 498 .object_init_pt = mmu_radix_object_init_pt, 499 .protect = mmu_radix_protect, 500 /* pmap dispatcher interface */ 501 .clear_modify = mmu_radix_clear_modify, 502 .copy = mmu_radix_copy, 503 .enter = mmu_radix_enter, 504 .enter_object = mmu_radix_enter_object, 505 .enter_quick = mmu_radix_enter_quick, 506 .extract = mmu_radix_extract, 507 .extract_and_hold = mmu_radix_extract_and_hold, 508 .is_modified = mmu_radix_is_modified, 509 .is_prefaultable = mmu_radix_is_prefaultable, 510 .is_referenced = mmu_radix_is_referenced, 511 .ts_referenced = mmu_radix_ts_referenced, 512 .page_exists_quick = mmu_radix_page_exists_quick, 513 .page_init = mmu_radix_page_init, 514 .page_wired_mappings = mmu_radix_page_wired_mappings, 515 .qenter = mmu_radix_qenter, 516 .qremove = mmu_radix_qremove, 517 .release = mmu_radix_release, 518 .remove = mmu_radix_remove, 519 .remove_all = mmu_radix_remove_all, 520 .remove_write = mmu_radix_remove_write, 521 .unwire = mmu_radix_unwire, 522 .zero_page = mmu_radix_zero_page, 523 .zero_page_area = mmu_radix_zero_page_area, 524 .activate = mmu_radix_activate, 525 .quick_enter_page = mmu_radix_quick_enter_page, 526 .quick_remove_page = mmu_radix_quick_remove_page, 527 .page_set_memattr = mmu_radix_page_set_memattr, 528 .page_array_startup = mmu_radix_page_array_startup, 529 530 /* Internal interfaces */ 531 .kenter = mmu_radix_kenter, 532 .kextract = mmu_radix_kextract, 533 .kremove = mmu_radix_kremove, 534 .change_attr = mmu_radix_change_attr, 535 .decode_kernel_ptr = mmu_radix_decode_kernel_ptr, 536 537 .tlbie_all = mmu_radix_tlbie_all, 538 }; 539 540 MMU_DEF(mmu_radix, MMU_TYPE_RADIX, mmu_radix_methods); 541 542 static boolean_t pmap_demote_l3e_locked(pmap_t pmap, pml3_entry_t *l3e, vm_offset_t va, 543 struct rwlock **lockp); 544 static boolean_t pmap_demote_l3e(pmap_t pmap, pml3_entry_t *pde, vm_offset_t va); 545 static int pmap_unuse_pt(pmap_t, vm_offset_t, pml3_entry_t, struct spglist *); 546 static int pmap_remove_l3e(pmap_t pmap, pml3_entry_t *pdq, vm_offset_t sva, 547 struct spglist *free, struct rwlock **lockp); 548 static int pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t sva, 549 pml3_entry_t ptepde, struct spglist *free, struct rwlock **lockp); 550 static vm_page_t pmap_remove_pt_page(pmap_t pmap, vm_offset_t va); 551 static bool pmap_remove_page(pmap_t pmap, vm_offset_t va, pml3_entry_t *pde, 552 struct spglist *free); 553 static bool pmap_remove_ptes(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, 554 pml3_entry_t *l3e, struct spglist *free, struct rwlock **lockp); 555 556 static bool pmap_pv_insert_l3e(pmap_t pmap, vm_offset_t va, pml3_entry_t l3e, 557 u_int flags, struct rwlock **lockp); 558 #if VM_NRESERVLEVEL > 0 559 static void pmap_pv_promote_l3e(pmap_t pmap, vm_offset_t va, vm_paddr_t pa, 560 struct rwlock **lockp); 561 #endif 562 static void pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va); 563 static int pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte); 564 static vm_page_t mmu_radix_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, 565 vm_prot_t prot, vm_page_t mpte, struct rwlock **lockp, bool *invalidate); 566 567 static bool pmap_enter_2mpage(pmap_t pmap, vm_offset_t va, vm_page_t m, 568 vm_prot_t prot, struct rwlock **lockp); 569 static int pmap_enter_l3e(pmap_t pmap, vm_offset_t va, pml3_entry_t newpde, 570 u_int flags, vm_page_t m, struct rwlock **lockp); 571 572 static vm_page_t reclaim_pv_chunk(pmap_t locked_pmap, struct rwlock **lockp); 573 static void free_pv_chunk(struct pv_chunk *pc); 574 static vm_page_t _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, struct rwlock **lockp); 575 static vm_page_t pmap_allocl3e(pmap_t pmap, vm_offset_t va, 576 struct rwlock **lockp); 577 static vm_page_t pmap_allocpte(pmap_t pmap, vm_offset_t va, 578 struct rwlock **lockp); 579 static void _pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, 580 struct spglist *free); 581 static boolean_t pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free); 582 583 static void pmap_invalidate_page(pmap_t pmap, vm_offset_t start); 584 static void pmap_invalidate_all(pmap_t pmap); 585 static int pmap_change_attr_locked(vm_offset_t va, vm_size_t size, int mode, bool flush); 586 587 /* 588 * Internal flags for pmap_enter()'s helper functions. 589 */ 590 #define PMAP_ENTER_NORECLAIM 0x1000000 /* Don't reclaim PV entries. */ 591 #define PMAP_ENTER_NOREPLACE 0x2000000 /* Don't replace mappings. */ 592 593 #define UNIMPLEMENTED() panic("%s not implemented", __func__) 594 #define UNTESTED() panic("%s not yet tested", __func__) 595 596 /* Number of supported PID bits */ 597 static unsigned int isa3_pid_bits; 598 599 /* PID to start allocating from */ 600 static unsigned int isa3_base_pid; 601 602 #define PROCTAB_SIZE_SHIFT (isa3_pid_bits + 4) 603 #define PROCTAB_ENTRIES (1ul << isa3_pid_bits) 604 605 /* 606 * Map of physical memory regions. 607 */ 608 static struct mem_region *regions, *pregions; 609 static struct numa_mem_region *numa_pregions; 610 static u_int phys_avail_count; 611 static int regions_sz, pregions_sz, numa_pregions_sz; 612 static struct pate *isa3_parttab; 613 static struct prte *isa3_proctab; 614 static vmem_t *asid_arena; 615 616 extern void bs_remap_earlyboot(void); 617 618 #define RADIX_PGD_SIZE_SHIFT 16 619 #define RADIX_PGD_SIZE (1UL << RADIX_PGD_SIZE_SHIFT) 620 621 #define RADIX_PGD_INDEX_SHIFT (RADIX_PGD_SIZE_SHIFT-3) 622 #define NL2EPG (PAGE_SIZE/sizeof(pml2_entry_t)) 623 #define NL3EPG (PAGE_SIZE/sizeof(pml3_entry_t)) 624 625 #define NUPML1E (RADIX_PGD_SIZE/sizeof(uint64_t)) /* number of userland PML1 pages */ 626 #define NUPDPE (NUPML1E * NL2EPG)/* number of userland PDP pages */ 627 #define NUPDE (NUPDPE * NL3EPG) /* number of userland PD entries */ 628 629 /* POWER9 only permits a 64k partition table size. */ 630 #define PARTTAB_SIZE_SHIFT 16 631 #define PARTTAB_SIZE (1UL << PARTTAB_SIZE_SHIFT) 632 633 #define PARTTAB_HR (1UL << 63) /* host uses radix */ 634 #define PARTTAB_GR (1UL << 63) /* guest uses radix must match host */ 635 636 /* TLB flush actions. Used as argument to tlbiel_all() */ 637 enum { 638 TLB_INVAL_SCOPE_LPID = 0, /* invalidate TLBs for current LPID */ 639 TLB_INVAL_SCOPE_GLOBAL = 1, /* invalidate all TLBs */ 640 }; 641 642 #define NPV_LIST_LOCKS MAXCPU 643 static int pmap_initialized; 644 static vm_paddr_t proctab0pa; 645 static vm_paddr_t parttab_phys; 646 CTASSERT(sizeof(struct pv_chunk) == PAGE_SIZE); 647 648 /* 649 * Data for the pv entry allocation mechanism. 650 * Updates to pv_invl_gen are protected by the pv_list_locks[] 651 * elements, but reads are not. 652 */ 653 static TAILQ_HEAD(pch, pv_chunk) pv_chunks = TAILQ_HEAD_INITIALIZER(pv_chunks); 654 static struct mtx __exclusive_cache_line pv_chunks_mutex; 655 static struct rwlock __exclusive_cache_line pv_list_locks[NPV_LIST_LOCKS]; 656 static struct md_page *pv_table; 657 static struct md_page pv_dummy; 658 659 #ifdef PV_STATS 660 #define PV_STAT(x) do { x ; } while (0) 661 #else 662 #define PV_STAT(x) do { } while (0) 663 #endif 664 665 #define pa_radix_index(pa) ((pa) >> L3_PAGE_SIZE_SHIFT) 666 #define pa_to_pvh(pa) (&pv_table[pa_radix_index(pa)]) 667 668 #define PHYS_TO_PV_LIST_LOCK(pa) \ 669 (&pv_list_locks[pa_radix_index(pa) % NPV_LIST_LOCKS]) 670 671 #define CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa) do { \ 672 struct rwlock **_lockp = (lockp); \ 673 struct rwlock *_new_lock; \ 674 \ 675 _new_lock = PHYS_TO_PV_LIST_LOCK(pa); \ 676 if (_new_lock != *_lockp) { \ 677 if (*_lockp != NULL) \ 678 rw_wunlock(*_lockp); \ 679 *_lockp = _new_lock; \ 680 rw_wlock(*_lockp); \ 681 } \ 682 } while (0) 683 684 #define CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m) \ 685 CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, VM_PAGE_TO_PHYS(m)) 686 687 #define RELEASE_PV_LIST_LOCK(lockp) do { \ 688 struct rwlock **_lockp = (lockp); \ 689 \ 690 if (*_lockp != NULL) { \ 691 rw_wunlock(*_lockp); \ 692 *_lockp = NULL; \ 693 } \ 694 } while (0) 695 696 #define VM_PAGE_TO_PV_LIST_LOCK(m) \ 697 PHYS_TO_PV_LIST_LOCK(VM_PAGE_TO_PHYS(m)) 698 699 /* 700 * We support 52 bits, hence: 701 * bits 52 - 31 = 21, 0b10101 702 * RTS encoding details 703 * bits 0 - 3 of rts -> bits 6 - 8 unsigned long 704 * bits 4 - 5 of rts -> bits 62 - 63 of unsigned long 705 */ 706 #define RTS_SIZE ((0x2UL << 61) | (0x5UL << 5)) 707 708 static int powernv_enabled = 1; 709 710 static __always_inline void 711 tlbiel_radix_set_isa300(uint32_t set, uint32_t is, 712 uint32_t pid, uint32_t ric, uint32_t prs) 713 { 714 uint64_t rb; 715 uint64_t rs; 716 717 rb = PPC_BITLSHIFT_VAL(set, 51) | PPC_BITLSHIFT_VAL(is, 53); 718 rs = PPC_BITLSHIFT_VAL((uint64_t)pid, 31); 719 720 __asm __volatile(PPC_TLBIEL(%0, %1, %2, %3, 1) 721 : : "r"(rb), "r"(rs), "i"(ric), "i"(prs) 722 : "memory"); 723 } 724 725 static void 726 tlbiel_flush_isa3(uint32_t num_sets, uint32_t is) 727 { 728 uint32_t set; 729 730 __asm __volatile("ptesync": : :"memory"); 731 732 /* 733 * Flush the first set of the TLB, and the entire Page Walk Cache 734 * and partition table entries. Then flush the remaining sets of the 735 * TLB. 736 */ 737 tlbiel_radix_set_isa300(0, is, 0, RIC_FLUSH_ALL, 0); 738 for (set = 1; set < num_sets; set++) 739 tlbiel_radix_set_isa300(set, is, 0, RIC_FLUSH_TLB, 0); 740 741 /* Do the same for process scoped entries. */ 742 tlbiel_radix_set_isa300(0, is, 0, RIC_FLUSH_ALL, 1); 743 for (set = 1; set < num_sets; set++) 744 tlbiel_radix_set_isa300(set, is, 0, RIC_FLUSH_TLB, 1); 745 746 __asm __volatile("ptesync": : :"memory"); 747 } 748 749 static void 750 mmu_radix_tlbiel_flush(int scope) 751 { 752 int is; 753 754 MPASS(scope == TLB_INVAL_SCOPE_LPID || 755 scope == TLB_INVAL_SCOPE_GLOBAL); 756 is = scope + 2; 757 758 tlbiel_flush_isa3(POWER9_TLB_SETS_RADIX, is); 759 __asm __volatile(PPC_INVALIDATE_ERAT "; isync" : : :"memory"); 760 } 761 762 static void 763 mmu_radix_tlbie_all() 764 { 765 /* TODO: LPID invalidate */ 766 mmu_radix_tlbiel_flush(TLB_INVAL_SCOPE_GLOBAL); 767 } 768 769 static void 770 mmu_radix_init_amor(void) 771 { 772 /* 773 * In HV mode, we init AMOR (Authority Mask Override Register) so that 774 * the hypervisor and guest can setup IAMR (Instruction Authority Mask 775 * Register), enable key 0 and set it to 1. 776 * 777 * AMOR = 0b1100 .... 0000 (Mask for key 0 is 11) 778 */ 779 mtspr(SPR_AMOR, (3ul << 62)); 780 } 781 782 static void 783 mmu_radix_init_iamr(void) 784 { 785 /* 786 * Radix always uses key0 of the IAMR to determine if an access is 787 * allowed. We set bit 0 (IBM bit 1) of key0, to prevent instruction 788 * fetch. 789 */ 790 mtspr(SPR_IAMR, (1ul << 62)); 791 } 792 793 static void 794 mmu_radix_pid_set(pmap_t pmap) 795 { 796 797 mtspr(SPR_PID, pmap->pm_pid); 798 isync(); 799 } 800 801 /* Quick sort callout for comparing physical addresses. */ 802 static int 803 pa_cmp(const void *a, const void *b) 804 { 805 const vm_paddr_t *pa = a, *pb = b; 806 807 if (*pa < *pb) 808 return (-1); 809 else if (*pa > *pb) 810 return (1); 811 else 812 return (0); 813 } 814 815 #define pte_load_store(ptep, pte) atomic_swap_long(ptep, pte) 816 #define pte_load_clear(ptep) atomic_swap_long(ptep, 0) 817 #define pte_store(ptep, pte) do { \ 818 MPASS((pte) & (RPTE_EAA_R | RPTE_EAA_W | RPTE_EAA_X)); \ 819 *(u_long *)(ptep) = htobe64((u_long)((pte) | PG_V | RPTE_LEAF)); \ 820 } while (0) 821 /* 822 * NB: should only be used for adding directories - not for direct mappings 823 */ 824 #define pde_store(ptep, pa) do { \ 825 *(u_long *)(ptep) = htobe64((u_long)(pa|RPTE_VALID|RPTE_SHIFT)); \ 826 } while (0) 827 828 #define pte_clear(ptep) do { \ 829 *(u_long *)(ptep) = (u_long)(0); \ 830 } while (0) 831 832 #define PMAP_PDE_SUPERPAGE (1 << 8) /* supports 2MB superpages */ 833 834 /* 835 * Promotion to a 2MB (PDE) page mapping requires that the corresponding 4KB 836 * (PTE) page mappings have identical settings for the following fields: 837 */ 838 #define PG_PTE_PROMOTE (PG_X | PG_MANAGED | PG_W | PG_PTE_CACHE | \ 839 PG_M | PG_A | RPTE_EAA_MASK | PG_V) 840 841 static __inline void 842 pmap_resident_count_inc(pmap_t pmap, int count) 843 { 844 845 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 846 pmap->pm_stats.resident_count += count; 847 } 848 849 static __inline void 850 pmap_resident_count_dec(pmap_t pmap, int count) 851 { 852 853 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 854 KASSERT(pmap->pm_stats.resident_count >= count, 855 ("pmap %p resident count underflow %ld %d", pmap, 856 pmap->pm_stats.resident_count, count)); 857 pmap->pm_stats.resident_count -= count; 858 } 859 860 static void 861 pagezero(vm_offset_t va) 862 { 863 va = trunc_page(va); 864 865 bzero((void *)va, PAGE_SIZE); 866 } 867 868 static uint64_t 869 allocpages(int n) 870 { 871 u_int64_t ret; 872 873 ret = moea64_bootstrap_alloc(n * PAGE_SIZE, PAGE_SIZE); 874 for (int i = 0; i < n; i++) 875 pagezero(PHYS_TO_DMAP(ret + i * PAGE_SIZE)); 876 return (ret); 877 } 878 879 static pt_entry_t * 880 kvtopte(vm_offset_t va) 881 { 882 pt_entry_t *l3e; 883 884 l3e = pmap_pml3e(kernel_pmap, va); 885 if ((be64toh(*l3e) & RPTE_VALID) == 0) 886 return (NULL); 887 return (pmap_l3e_to_pte(l3e, va)); 888 } 889 890 void 891 mmu_radix_kenter(vm_offset_t va, vm_paddr_t pa) 892 { 893 pt_entry_t *pte; 894 895 pte = kvtopte(va); 896 MPASS(pte != NULL); 897 *pte = htobe64(pa | RPTE_VALID | RPTE_LEAF | RPTE_EAA_R | \ 898 RPTE_EAA_W | RPTE_EAA_P | PG_M | PG_A); 899 } 900 901 bool 902 mmu_radix_ps_enabled(pmap_t pmap) 903 { 904 return (superpages_enabled && (pmap->pm_flags & PMAP_PDE_SUPERPAGE) != 0); 905 } 906 907 static pt_entry_t * 908 pmap_nofault_pte(pmap_t pmap, vm_offset_t va, int *is_l3e) 909 { 910 pml3_entry_t *l3e; 911 pt_entry_t *pte; 912 913 va &= PG_PS_FRAME; 914 l3e = pmap_pml3e(pmap, va); 915 if (l3e == NULL || (be64toh(*l3e) & PG_V) == 0) 916 return (NULL); 917 918 if (be64toh(*l3e) & RPTE_LEAF) { 919 *is_l3e = 1; 920 return (l3e); 921 } 922 *is_l3e = 0; 923 va &= PG_FRAME; 924 pte = pmap_l3e_to_pte(l3e, va); 925 if (pte == NULL || (be64toh(*pte) & PG_V) == 0) 926 return (NULL); 927 return (pte); 928 } 929 930 int 931 pmap_nofault(pmap_t pmap, vm_offset_t va, vm_prot_t flags) 932 { 933 pt_entry_t *pte; 934 pt_entry_t startpte, origpte, newpte; 935 vm_page_t m; 936 int is_l3e; 937 938 startpte = 0; 939 retry: 940 if ((pte = pmap_nofault_pte(pmap, va, &is_l3e)) == NULL) 941 return (KERN_INVALID_ADDRESS); 942 origpte = newpte = be64toh(*pte); 943 if (startpte == 0) { 944 startpte = origpte; 945 if (((flags & VM_PROT_WRITE) && (startpte & PG_M)) || 946 ((flags & VM_PROT_READ) && (startpte & PG_A))) { 947 pmap_invalidate_all(pmap); 948 #ifdef INVARIANTS 949 if (VERBOSE_PMAP || pmap_logging) 950 printf("%s(%p, %#lx, %#x) (%#lx) -- invalidate all\n", 951 __func__, pmap, va, flags, origpte); 952 #endif 953 return (KERN_FAILURE); 954 } 955 } 956 #ifdef INVARIANTS 957 if (VERBOSE_PMAP || pmap_logging) 958 printf("%s(%p, %#lx, %#x) (%#lx)\n", __func__, pmap, va, 959 flags, origpte); 960 #endif 961 PMAP_LOCK(pmap); 962 if ((pte = pmap_nofault_pte(pmap, va, &is_l3e)) == NULL || 963 be64toh(*pte) != origpte) { 964 PMAP_UNLOCK(pmap); 965 return (KERN_FAILURE); 966 } 967 m = PHYS_TO_VM_PAGE(newpte & PG_FRAME); 968 MPASS(m != NULL); 969 switch (flags) { 970 case VM_PROT_READ: 971 if ((newpte & (RPTE_EAA_R|RPTE_EAA_X)) == 0) 972 goto protfail; 973 newpte |= PG_A; 974 vm_page_aflag_set(m, PGA_REFERENCED); 975 break; 976 case VM_PROT_WRITE: 977 if ((newpte & RPTE_EAA_W) == 0) 978 goto protfail; 979 if (is_l3e) 980 goto protfail; 981 newpte |= PG_M; 982 vm_page_dirty(m); 983 break; 984 case VM_PROT_EXECUTE: 985 if ((newpte & RPTE_EAA_X) == 0) 986 goto protfail; 987 newpte |= PG_A; 988 vm_page_aflag_set(m, PGA_REFERENCED); 989 break; 990 } 991 992 if (!atomic_cmpset_long(pte, htobe64(origpte), htobe64(newpte))) 993 goto retry; 994 ptesync(); 995 PMAP_UNLOCK(pmap); 996 if (startpte == newpte) 997 return (KERN_FAILURE); 998 return (0); 999 protfail: 1000 PMAP_UNLOCK(pmap); 1001 return (KERN_PROTECTION_FAILURE); 1002 } 1003 1004 /* 1005 * Returns TRUE if the given page is mapped individually or as part of 1006 * a 2mpage. Otherwise, returns FALSE. 1007 */ 1008 boolean_t 1009 mmu_radix_page_is_mapped(vm_page_t m) 1010 { 1011 struct rwlock *lock; 1012 boolean_t rv; 1013 1014 if ((m->oflags & VPO_UNMANAGED) != 0) 1015 return (FALSE); 1016 lock = VM_PAGE_TO_PV_LIST_LOCK(m); 1017 rw_rlock(lock); 1018 rv = !TAILQ_EMPTY(&m->md.pv_list) || 1019 ((m->flags & PG_FICTITIOUS) == 0 && 1020 !TAILQ_EMPTY(&pa_to_pvh(VM_PAGE_TO_PHYS(m))->pv_list)); 1021 rw_runlock(lock); 1022 return (rv); 1023 } 1024 1025 /* 1026 * Determine the appropriate bits to set in a PTE or PDE for a specified 1027 * caching mode. 1028 */ 1029 static int 1030 pmap_cache_bits(vm_memattr_t ma) 1031 { 1032 if (ma != VM_MEMATTR_DEFAULT) { 1033 switch (ma) { 1034 case VM_MEMATTR_UNCACHEABLE: 1035 return (RPTE_ATTR_GUARDEDIO); 1036 case VM_MEMATTR_CACHEABLE: 1037 return (RPTE_ATTR_MEM); 1038 case VM_MEMATTR_WRITE_BACK: 1039 case VM_MEMATTR_PREFETCHABLE: 1040 case VM_MEMATTR_WRITE_COMBINING: 1041 return (RPTE_ATTR_UNGUARDEDIO); 1042 } 1043 } 1044 return (0); 1045 } 1046 1047 static void 1048 pmap_invalidate_page(pmap_t pmap, vm_offset_t start) 1049 { 1050 ptesync(); 1051 if (pmap == kernel_pmap) 1052 radix_tlbie_invlpg_kernel_4k(start); 1053 else 1054 radix_tlbie_invlpg_user_4k(pmap->pm_pid, start); 1055 ttusync(); 1056 } 1057 1058 static void 1059 pmap_invalidate_page_2m(pmap_t pmap, vm_offset_t start) 1060 { 1061 ptesync(); 1062 if (pmap == kernel_pmap) 1063 radix_tlbie_invlpg_kernel_2m(start); 1064 else 1065 radix_tlbie_invlpg_user_2m(pmap->pm_pid, start); 1066 ttusync(); 1067 } 1068 1069 static void 1070 pmap_invalidate_pwc(pmap_t pmap) 1071 { 1072 ptesync(); 1073 if (pmap == kernel_pmap) 1074 radix_tlbie_invlpwc_kernel(); 1075 else 1076 radix_tlbie_invlpwc_user(pmap->pm_pid); 1077 ttusync(); 1078 } 1079 1080 static void 1081 pmap_invalidate_range(pmap_t pmap, vm_offset_t start, vm_offset_t end) 1082 { 1083 if (((start - end) >> PAGE_SHIFT) > 8) { 1084 pmap_invalidate_all(pmap); 1085 return; 1086 } 1087 ptesync(); 1088 if (pmap == kernel_pmap) { 1089 while (start < end) { 1090 radix_tlbie_invlpg_kernel_4k(start); 1091 start += PAGE_SIZE; 1092 } 1093 } else { 1094 while (start < end) { 1095 radix_tlbie_invlpg_user_4k(pmap->pm_pid, start); 1096 start += PAGE_SIZE; 1097 } 1098 } 1099 ttusync(); 1100 } 1101 1102 static void 1103 pmap_invalidate_all(pmap_t pmap) 1104 { 1105 ptesync(); 1106 if (pmap == kernel_pmap) 1107 radix_tlbie_flush_kernel(); 1108 else 1109 radix_tlbie_flush_user(pmap->pm_pid); 1110 ttusync(); 1111 } 1112 1113 static void 1114 pmap_invalidate_l3e_page(pmap_t pmap, vm_offset_t va, pml3_entry_t l3e) 1115 { 1116 1117 /* 1118 * When the PDE has PG_PROMOTED set, the 2MB page mapping was created 1119 * by a promotion that did not invalidate the 512 4KB page mappings 1120 * that might exist in the TLB. Consequently, at this point, the TLB 1121 * may hold both 4KB and 2MB page mappings for the address range [va, 1122 * va + L3_PAGE_SIZE). Therefore, the entire range must be invalidated here. 1123 * In contrast, when PG_PROMOTED is clear, the TLB will not hold any 1124 * 4KB page mappings for the address range [va, va + L3_PAGE_SIZE), and so a 1125 * single INVLPG suffices to invalidate the 2MB page mapping from the 1126 * TLB. 1127 */ 1128 ptesync(); 1129 if ((l3e & PG_PROMOTED) != 0) 1130 pmap_invalidate_range(pmap, va, va + L3_PAGE_SIZE - 1); 1131 else 1132 pmap_invalidate_page_2m(pmap, va); 1133 1134 pmap_invalidate_pwc(pmap); 1135 } 1136 1137 static __inline struct pv_chunk * 1138 pv_to_chunk(pv_entry_t pv) 1139 { 1140 1141 return ((struct pv_chunk *)((uintptr_t)pv & ~(uintptr_t)PAGE_MASK)); 1142 } 1143 1144 #define PV_PMAP(pv) (pv_to_chunk(pv)->pc_pmap) 1145 1146 #define PC_FREE0 0xfffffffffffffffful 1147 #define PC_FREE1 0x3ffffffffffffffful 1148 1149 static const uint64_t pc_freemask[_NPCM] = { PC_FREE0, PC_FREE1 }; 1150 1151 /* 1152 * Ensure that the number of spare PV entries in the specified pmap meets or 1153 * exceeds the given count, "needed". 1154 * 1155 * The given PV list lock may be released. 1156 */ 1157 static void 1158 reserve_pv_entries(pmap_t pmap, int needed, struct rwlock **lockp) 1159 { 1160 struct pch new_tail; 1161 struct pv_chunk *pc; 1162 vm_page_t m; 1163 int avail, free; 1164 bool reclaimed; 1165 1166 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1167 KASSERT(lockp != NULL, ("reserve_pv_entries: lockp is NULL")); 1168 1169 /* 1170 * Newly allocated PV chunks must be stored in a private list until 1171 * the required number of PV chunks have been allocated. Otherwise, 1172 * reclaim_pv_chunk() could recycle one of these chunks. In 1173 * contrast, these chunks must be added to the pmap upon allocation. 1174 */ 1175 TAILQ_INIT(&new_tail); 1176 retry: 1177 avail = 0; 1178 TAILQ_FOREACH(pc, &pmap->pm_pvchunk, pc_list) { 1179 // if ((cpu_feature2 & CPUID2_POPCNT) == 0) 1180 bit_count((bitstr_t *)pc->pc_map, 0, 1181 sizeof(pc->pc_map) * NBBY, &free); 1182 #if 0 1183 free = popcnt_pc_map_pq(pc->pc_map); 1184 #endif 1185 if (free == 0) 1186 break; 1187 avail += free; 1188 if (avail >= needed) 1189 break; 1190 } 1191 for (reclaimed = false; avail < needed; avail += _NPCPV) { 1192 m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | 1193 VM_ALLOC_WIRED); 1194 if (m == NULL) { 1195 m = reclaim_pv_chunk(pmap, lockp); 1196 if (m == NULL) 1197 goto retry; 1198 reclaimed = true; 1199 } 1200 PV_STAT(atomic_add_int(&pc_chunk_count, 1)); 1201 PV_STAT(atomic_add_int(&pc_chunk_allocs, 1)); 1202 pc = (void *)PHYS_TO_DMAP(m->phys_addr); 1203 pc->pc_pmap = pmap; 1204 pc->pc_map[0] = PC_FREE0; 1205 pc->pc_map[1] = PC_FREE1; 1206 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list); 1207 TAILQ_INSERT_TAIL(&new_tail, pc, pc_lru); 1208 PV_STAT(atomic_add_int(&pv_entry_spare, _NPCPV)); 1209 1210 /* 1211 * The reclaim might have freed a chunk from the current pmap. 1212 * If that chunk contained available entries, we need to 1213 * re-count the number of available entries. 1214 */ 1215 if (reclaimed) 1216 goto retry; 1217 } 1218 if (!TAILQ_EMPTY(&new_tail)) { 1219 mtx_lock(&pv_chunks_mutex); 1220 TAILQ_CONCAT(&pv_chunks, &new_tail, pc_lru); 1221 mtx_unlock(&pv_chunks_mutex); 1222 } 1223 } 1224 1225 /* 1226 * First find and then remove the pv entry for the specified pmap and virtual 1227 * address from the specified pv list. Returns the pv entry if found and NULL 1228 * otherwise. This operation can be performed on pv lists for either 4KB or 1229 * 2MB page mappings. 1230 */ 1231 static __inline pv_entry_t 1232 pmap_pvh_remove(struct md_page *pvh, pmap_t pmap, vm_offset_t va) 1233 { 1234 pv_entry_t pv; 1235 1236 TAILQ_FOREACH(pv, &pvh->pv_list, pv_link) { 1237 #ifdef INVARIANTS 1238 if (PV_PMAP(pv) == NULL) { 1239 printf("corrupted pv_chunk/pv %p\n", pv); 1240 printf("pv_chunk: %64D\n", pv_to_chunk(pv), ":"); 1241 } 1242 MPASS(PV_PMAP(pv) != NULL); 1243 MPASS(pv->pv_va != 0); 1244 #endif 1245 if (pmap == PV_PMAP(pv) && va == pv->pv_va) { 1246 TAILQ_REMOVE(&pvh->pv_list, pv, pv_link); 1247 pvh->pv_gen++; 1248 break; 1249 } 1250 } 1251 return (pv); 1252 } 1253 1254 /* 1255 * After demotion from a 2MB page mapping to 512 4KB page mappings, 1256 * destroy the pv entry for the 2MB page mapping and reinstantiate the pv 1257 * entries for each of the 4KB page mappings. 1258 */ 1259 static void 1260 pmap_pv_demote_l3e(pmap_t pmap, vm_offset_t va, vm_paddr_t pa, 1261 struct rwlock **lockp) 1262 { 1263 struct md_page *pvh; 1264 struct pv_chunk *pc; 1265 pv_entry_t pv; 1266 vm_offset_t va_last; 1267 vm_page_t m; 1268 int bit, field; 1269 1270 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1271 KASSERT((pa & L3_PAGE_MASK) == 0, 1272 ("pmap_pv_demote_pde: pa is not 2mpage aligned")); 1273 CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa); 1274 1275 /* 1276 * Transfer the 2mpage's pv entry for this mapping to the first 1277 * page's pv list. Once this transfer begins, the pv list lock 1278 * must not be released until the last pv entry is reinstantiated. 1279 */ 1280 pvh = pa_to_pvh(pa); 1281 va = trunc_2mpage(va); 1282 pv = pmap_pvh_remove(pvh, pmap, va); 1283 KASSERT(pv != NULL, ("pmap_pv_demote_pde: pv not found")); 1284 m = PHYS_TO_VM_PAGE(pa); 1285 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_link); 1286 1287 m->md.pv_gen++; 1288 /* Instantiate the remaining NPTEPG - 1 pv entries. */ 1289 PV_STAT(atomic_add_long(&pv_entry_allocs, NPTEPG - 1)); 1290 va_last = va + L3_PAGE_SIZE - PAGE_SIZE; 1291 for (;;) { 1292 pc = TAILQ_FIRST(&pmap->pm_pvchunk); 1293 KASSERT(pc->pc_map[0] != 0 || pc->pc_map[1] != 0 1294 , ("pmap_pv_demote_pde: missing spare")); 1295 for (field = 0; field < _NPCM; field++) { 1296 while (pc->pc_map[field]) { 1297 bit = cnttzd(pc->pc_map[field]); 1298 pc->pc_map[field] &= ~(1ul << bit); 1299 pv = &pc->pc_pventry[field * 64 + bit]; 1300 va += PAGE_SIZE; 1301 pv->pv_va = va; 1302 m++; 1303 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 1304 ("pmap_pv_demote_pde: page %p is not managed", m)); 1305 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_link); 1306 1307 m->md.pv_gen++; 1308 if (va == va_last) 1309 goto out; 1310 } 1311 } 1312 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 1313 TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list); 1314 } 1315 out: 1316 if (pc->pc_map[0] == 0 && pc->pc_map[1] == 0) { 1317 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 1318 TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list); 1319 } 1320 PV_STAT(atomic_add_long(&pv_entry_count, NPTEPG - 1)); 1321 PV_STAT(atomic_subtract_int(&pv_entry_spare, NPTEPG - 1)); 1322 } 1323 1324 static void 1325 reclaim_pv_chunk_leave_pmap(pmap_t pmap, pmap_t locked_pmap) 1326 { 1327 1328 if (pmap == NULL) 1329 return; 1330 pmap_invalidate_all(pmap); 1331 if (pmap != locked_pmap) 1332 PMAP_UNLOCK(pmap); 1333 } 1334 1335 /* 1336 * We are in a serious low memory condition. Resort to 1337 * drastic measures to free some pages so we can allocate 1338 * another pv entry chunk. 1339 * 1340 * Returns NULL if PV entries were reclaimed from the specified pmap. 1341 * 1342 * We do not, however, unmap 2mpages because subsequent accesses will 1343 * allocate per-page pv entries until repromotion occurs, thereby 1344 * exacerbating the shortage of free pv entries. 1345 */ 1346 static int active_reclaims = 0; 1347 static vm_page_t 1348 reclaim_pv_chunk(pmap_t locked_pmap, struct rwlock **lockp) 1349 { 1350 struct pv_chunk *pc, *pc_marker, *pc_marker_end; 1351 struct pv_chunk_header pc_marker_b, pc_marker_end_b; 1352 struct md_page *pvh; 1353 pml3_entry_t *l3e; 1354 pmap_t next_pmap, pmap; 1355 pt_entry_t *pte, tpte; 1356 pv_entry_t pv; 1357 vm_offset_t va; 1358 vm_page_t m, m_pc; 1359 struct spglist free; 1360 uint64_t inuse; 1361 int bit, field, freed; 1362 1363 PMAP_LOCK_ASSERT(locked_pmap, MA_OWNED); 1364 KASSERT(lockp != NULL, ("reclaim_pv_chunk: lockp is NULL")); 1365 pmap = NULL; 1366 m_pc = NULL; 1367 SLIST_INIT(&free); 1368 bzero(&pc_marker_b, sizeof(pc_marker_b)); 1369 bzero(&pc_marker_end_b, sizeof(pc_marker_end_b)); 1370 pc_marker = (struct pv_chunk *)&pc_marker_b; 1371 pc_marker_end = (struct pv_chunk *)&pc_marker_end_b; 1372 1373 mtx_lock(&pv_chunks_mutex); 1374 active_reclaims++; 1375 TAILQ_INSERT_HEAD(&pv_chunks, pc_marker, pc_lru); 1376 TAILQ_INSERT_TAIL(&pv_chunks, pc_marker_end, pc_lru); 1377 while ((pc = TAILQ_NEXT(pc_marker, pc_lru)) != pc_marker_end && 1378 SLIST_EMPTY(&free)) { 1379 next_pmap = pc->pc_pmap; 1380 if (next_pmap == NULL) { 1381 /* 1382 * The next chunk is a marker. However, it is 1383 * not our marker, so active_reclaims must be 1384 * > 1. Consequently, the next_chunk code 1385 * will not rotate the pv_chunks list. 1386 */ 1387 goto next_chunk; 1388 } 1389 mtx_unlock(&pv_chunks_mutex); 1390 1391 /* 1392 * A pv_chunk can only be removed from the pc_lru list 1393 * when both pc_chunks_mutex is owned and the 1394 * corresponding pmap is locked. 1395 */ 1396 if (pmap != next_pmap) { 1397 reclaim_pv_chunk_leave_pmap(pmap, locked_pmap); 1398 pmap = next_pmap; 1399 /* Avoid deadlock and lock recursion. */ 1400 if (pmap > locked_pmap) { 1401 RELEASE_PV_LIST_LOCK(lockp); 1402 PMAP_LOCK(pmap); 1403 mtx_lock(&pv_chunks_mutex); 1404 continue; 1405 } else if (pmap != locked_pmap) { 1406 if (PMAP_TRYLOCK(pmap)) { 1407 mtx_lock(&pv_chunks_mutex); 1408 continue; 1409 } else { 1410 pmap = NULL; /* pmap is not locked */ 1411 mtx_lock(&pv_chunks_mutex); 1412 pc = TAILQ_NEXT(pc_marker, pc_lru); 1413 if (pc == NULL || 1414 pc->pc_pmap != next_pmap) 1415 continue; 1416 goto next_chunk; 1417 } 1418 } 1419 } 1420 1421 /* 1422 * Destroy every non-wired, 4 KB page mapping in the chunk. 1423 */ 1424 freed = 0; 1425 for (field = 0; field < _NPCM; field++) { 1426 for (inuse = ~pc->pc_map[field] & pc_freemask[field]; 1427 inuse != 0; inuse &= ~(1UL << bit)) { 1428 bit = cnttzd(inuse); 1429 pv = &pc->pc_pventry[field * 64 + bit]; 1430 va = pv->pv_va; 1431 l3e = pmap_pml3e(pmap, va); 1432 if ((be64toh(*l3e) & RPTE_LEAF) != 0) 1433 continue; 1434 pte = pmap_l3e_to_pte(l3e, va); 1435 if ((be64toh(*pte) & PG_W) != 0) 1436 continue; 1437 tpte = be64toh(pte_load_clear(pte)); 1438 m = PHYS_TO_VM_PAGE(tpte & PG_FRAME); 1439 if ((tpte & (PG_M | PG_RW)) == (PG_M | PG_RW)) 1440 vm_page_dirty(m); 1441 if ((tpte & PG_A) != 0) 1442 vm_page_aflag_set(m, PGA_REFERENCED); 1443 CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m); 1444 TAILQ_REMOVE(&m->md.pv_list, pv, pv_link); 1445 1446 m->md.pv_gen++; 1447 if (TAILQ_EMPTY(&m->md.pv_list) && 1448 (m->flags & PG_FICTITIOUS) == 0) { 1449 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); 1450 if (TAILQ_EMPTY(&pvh->pv_list)) { 1451 vm_page_aflag_clear(m, 1452 PGA_WRITEABLE); 1453 } 1454 } 1455 pc->pc_map[field] |= 1UL << bit; 1456 pmap_unuse_pt(pmap, va, be64toh(*l3e), &free); 1457 freed++; 1458 } 1459 } 1460 if (freed == 0) { 1461 mtx_lock(&pv_chunks_mutex); 1462 goto next_chunk; 1463 } 1464 /* Every freed mapping is for a 4 KB page. */ 1465 pmap_resident_count_dec(pmap, freed); 1466 PV_STAT(atomic_add_long(&pv_entry_frees, freed)); 1467 PV_STAT(atomic_add_int(&pv_entry_spare, freed)); 1468 PV_STAT(atomic_subtract_long(&pv_entry_count, freed)); 1469 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 1470 if (pc->pc_map[0] == PC_FREE0 && pc->pc_map[1] == PC_FREE1) { 1471 PV_STAT(atomic_subtract_int(&pv_entry_spare, _NPCPV)); 1472 PV_STAT(atomic_subtract_int(&pc_chunk_count, 1)); 1473 PV_STAT(atomic_add_int(&pc_chunk_frees, 1)); 1474 /* Entire chunk is free; return it. */ 1475 m_pc = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pc)); 1476 mtx_lock(&pv_chunks_mutex); 1477 TAILQ_REMOVE(&pv_chunks, pc, pc_lru); 1478 break; 1479 } 1480 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list); 1481 mtx_lock(&pv_chunks_mutex); 1482 /* One freed pv entry in locked_pmap is sufficient. */ 1483 if (pmap == locked_pmap) 1484 break; 1485 next_chunk: 1486 TAILQ_REMOVE(&pv_chunks, pc_marker, pc_lru); 1487 TAILQ_INSERT_AFTER(&pv_chunks, pc, pc_marker, pc_lru); 1488 if (active_reclaims == 1 && pmap != NULL) { 1489 /* 1490 * Rotate the pv chunks list so that we do not 1491 * scan the same pv chunks that could not be 1492 * freed (because they contained a wired 1493 * and/or superpage mapping) on every 1494 * invocation of reclaim_pv_chunk(). 1495 */ 1496 while ((pc = TAILQ_FIRST(&pv_chunks)) != pc_marker) { 1497 MPASS(pc->pc_pmap != NULL); 1498 TAILQ_REMOVE(&pv_chunks, pc, pc_lru); 1499 TAILQ_INSERT_TAIL(&pv_chunks, pc, pc_lru); 1500 } 1501 } 1502 } 1503 TAILQ_REMOVE(&pv_chunks, pc_marker, pc_lru); 1504 TAILQ_REMOVE(&pv_chunks, pc_marker_end, pc_lru); 1505 active_reclaims--; 1506 mtx_unlock(&pv_chunks_mutex); 1507 reclaim_pv_chunk_leave_pmap(pmap, locked_pmap); 1508 if (m_pc == NULL && !SLIST_EMPTY(&free)) { 1509 m_pc = SLIST_FIRST(&free); 1510 SLIST_REMOVE_HEAD(&free, plinks.s.ss); 1511 /* Recycle a freed page table page. */ 1512 m_pc->ref_count = 1; 1513 } 1514 vm_page_free_pages_toq(&free, true); 1515 return (m_pc); 1516 } 1517 1518 /* 1519 * free the pv_entry back to the free list 1520 */ 1521 static void 1522 free_pv_entry(pmap_t pmap, pv_entry_t pv) 1523 { 1524 struct pv_chunk *pc; 1525 int idx, field, bit; 1526 1527 #ifdef VERBOSE_PV 1528 if (pmap != kernel_pmap) 1529 printf("%s(%p, %p)\n", __func__, pmap, pv); 1530 #endif 1531 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1532 PV_STAT(atomic_add_long(&pv_entry_frees, 1)); 1533 PV_STAT(atomic_add_int(&pv_entry_spare, 1)); 1534 PV_STAT(atomic_subtract_long(&pv_entry_count, 1)); 1535 pc = pv_to_chunk(pv); 1536 idx = pv - &pc->pc_pventry[0]; 1537 field = idx / 64; 1538 bit = idx % 64; 1539 pc->pc_map[field] |= 1ul << bit; 1540 if (pc->pc_map[0] != PC_FREE0 || pc->pc_map[1] != PC_FREE1) { 1541 /* 98% of the time, pc is already at the head of the list. */ 1542 if (__predict_false(pc != TAILQ_FIRST(&pmap->pm_pvchunk))) { 1543 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 1544 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list); 1545 } 1546 return; 1547 } 1548 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 1549 free_pv_chunk(pc); 1550 } 1551 1552 static void 1553 free_pv_chunk(struct pv_chunk *pc) 1554 { 1555 vm_page_t m; 1556 1557 mtx_lock(&pv_chunks_mutex); 1558 TAILQ_REMOVE(&pv_chunks, pc, pc_lru); 1559 mtx_unlock(&pv_chunks_mutex); 1560 PV_STAT(atomic_subtract_int(&pv_entry_spare, _NPCPV)); 1561 PV_STAT(atomic_subtract_int(&pc_chunk_count, 1)); 1562 PV_STAT(atomic_add_int(&pc_chunk_frees, 1)); 1563 /* entire chunk is free, return it */ 1564 m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pc)); 1565 vm_page_unwire_noq(m); 1566 vm_page_free(m); 1567 } 1568 1569 /* 1570 * Returns a new PV entry, allocating a new PV chunk from the system when 1571 * needed. If this PV chunk allocation fails and a PV list lock pointer was 1572 * given, a PV chunk is reclaimed from an arbitrary pmap. Otherwise, NULL is 1573 * returned. 1574 * 1575 * The given PV list lock may be released. 1576 */ 1577 static pv_entry_t 1578 get_pv_entry(pmap_t pmap, struct rwlock **lockp) 1579 { 1580 int bit, field; 1581 pv_entry_t pv; 1582 struct pv_chunk *pc; 1583 vm_page_t m; 1584 1585 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1586 PV_STAT(atomic_add_long(&pv_entry_allocs, 1)); 1587 retry: 1588 pc = TAILQ_FIRST(&pmap->pm_pvchunk); 1589 if (pc != NULL) { 1590 for (field = 0; field < _NPCM; field++) { 1591 if (pc->pc_map[field]) { 1592 bit = cnttzd(pc->pc_map[field]); 1593 break; 1594 } 1595 } 1596 if (field < _NPCM) { 1597 pv = &pc->pc_pventry[field * 64 + bit]; 1598 pc->pc_map[field] &= ~(1ul << bit); 1599 /* If this was the last item, move it to tail */ 1600 if (pc->pc_map[0] == 0 && pc->pc_map[1] == 0) { 1601 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 1602 TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, 1603 pc_list); 1604 } 1605 PV_STAT(atomic_add_long(&pv_entry_count, 1)); 1606 PV_STAT(atomic_subtract_int(&pv_entry_spare, 1)); 1607 MPASS(PV_PMAP(pv) != NULL); 1608 return (pv); 1609 } 1610 } 1611 /* No free items, allocate another chunk */ 1612 m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | 1613 VM_ALLOC_WIRED); 1614 if (m == NULL) { 1615 if (lockp == NULL) { 1616 PV_STAT(pc_chunk_tryfail++); 1617 return (NULL); 1618 } 1619 m = reclaim_pv_chunk(pmap, lockp); 1620 if (m == NULL) 1621 goto retry; 1622 } 1623 PV_STAT(atomic_add_int(&pc_chunk_count, 1)); 1624 PV_STAT(atomic_add_int(&pc_chunk_allocs, 1)); 1625 pc = (void *)PHYS_TO_DMAP(m->phys_addr); 1626 pc->pc_pmap = pmap; 1627 pc->pc_map[0] = PC_FREE0 & ~1ul; /* preallocated bit 0 */ 1628 pc->pc_map[1] = PC_FREE1; 1629 mtx_lock(&pv_chunks_mutex); 1630 TAILQ_INSERT_TAIL(&pv_chunks, pc, pc_lru); 1631 mtx_unlock(&pv_chunks_mutex); 1632 pv = &pc->pc_pventry[0]; 1633 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list); 1634 PV_STAT(atomic_add_long(&pv_entry_count, 1)); 1635 PV_STAT(atomic_add_int(&pv_entry_spare, _NPCPV - 1)); 1636 MPASS(PV_PMAP(pv) != NULL); 1637 return (pv); 1638 } 1639 1640 #if VM_NRESERVLEVEL > 0 1641 /* 1642 * After promotion from 512 4KB page mappings to a single 2MB page mapping, 1643 * replace the many pv entries for the 4KB page mappings by a single pv entry 1644 * for the 2MB page mapping. 1645 */ 1646 static void 1647 pmap_pv_promote_l3e(pmap_t pmap, vm_offset_t va, vm_paddr_t pa, 1648 struct rwlock **lockp) 1649 { 1650 struct md_page *pvh; 1651 pv_entry_t pv; 1652 vm_offset_t va_last; 1653 vm_page_t m; 1654 1655 KASSERT((pa & L3_PAGE_MASK) == 0, 1656 ("pmap_pv_promote_pde: pa is not 2mpage aligned")); 1657 CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa); 1658 1659 /* 1660 * Transfer the first page's pv entry for this mapping to the 2mpage's 1661 * pv list. Aside from avoiding the cost of a call to get_pv_entry(), 1662 * a transfer avoids the possibility that get_pv_entry() calls 1663 * reclaim_pv_chunk() and that reclaim_pv_chunk() removes one of the 1664 * mappings that is being promoted. 1665 */ 1666 m = PHYS_TO_VM_PAGE(pa); 1667 va = trunc_2mpage(va); 1668 pv = pmap_pvh_remove(&m->md, pmap, va); 1669 KASSERT(pv != NULL, ("pmap_pv_promote_pde: pv not found")); 1670 pvh = pa_to_pvh(pa); 1671 TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_link); 1672 pvh->pv_gen++; 1673 /* Free the remaining NPTEPG - 1 pv entries. */ 1674 va_last = va + L3_PAGE_SIZE - PAGE_SIZE; 1675 do { 1676 m++; 1677 va += PAGE_SIZE; 1678 pmap_pvh_free(&m->md, pmap, va); 1679 } while (va < va_last); 1680 } 1681 #endif /* VM_NRESERVLEVEL > 0 */ 1682 1683 /* 1684 * First find and then destroy the pv entry for the specified pmap and virtual 1685 * address. This operation can be performed on pv lists for either 4KB or 2MB 1686 * page mappings. 1687 */ 1688 static void 1689 pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va) 1690 { 1691 pv_entry_t pv; 1692 1693 pv = pmap_pvh_remove(pvh, pmap, va); 1694 KASSERT(pv != NULL, ("pmap_pvh_free: pv not found")); 1695 free_pv_entry(pmap, pv); 1696 } 1697 1698 /* 1699 * Conditionally create the PV entry for a 4KB page mapping if the required 1700 * memory can be allocated without resorting to reclamation. 1701 */ 1702 static boolean_t 1703 pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, vm_page_t m, 1704 struct rwlock **lockp) 1705 { 1706 pv_entry_t pv; 1707 1708 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1709 /* Pass NULL instead of the lock pointer to disable reclamation. */ 1710 if ((pv = get_pv_entry(pmap, NULL)) != NULL) { 1711 pv->pv_va = va; 1712 CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m); 1713 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_link); 1714 m->md.pv_gen++; 1715 return (TRUE); 1716 } else 1717 return (FALSE); 1718 } 1719 1720 vm_paddr_t phys_avail_debug[2 * VM_PHYSSEG_MAX]; 1721 #ifdef INVARIANTS 1722 static void 1723 validate_addr(vm_paddr_t addr, vm_size_t size) 1724 { 1725 vm_paddr_t end = addr + size; 1726 bool found = false; 1727 1728 for (int i = 0; i < 2 * phys_avail_count; i += 2) { 1729 if (addr >= phys_avail_debug[i] && 1730 end <= phys_avail_debug[i + 1]) { 1731 found = true; 1732 break; 1733 } 1734 } 1735 KASSERT(found, ("%#lx-%#lx outside of initial phys_avail array", 1736 addr, end)); 1737 } 1738 #else 1739 static void validate_addr(vm_paddr_t addr, vm_size_t size) {} 1740 #endif 1741 #define DMAP_PAGE_BITS (RPTE_VALID | RPTE_LEAF | RPTE_EAA_MASK | PG_M | PG_A) 1742 1743 static vm_paddr_t 1744 alloc_pt_page(void) 1745 { 1746 vm_paddr_t page; 1747 1748 page = allocpages(1); 1749 pagezero(PHYS_TO_DMAP(page)); 1750 return (page); 1751 } 1752 1753 static void 1754 mmu_radix_dmap_range(vm_paddr_t start, vm_paddr_t end) 1755 { 1756 pt_entry_t *pte, pteval; 1757 vm_paddr_t page; 1758 1759 if (bootverbose) 1760 printf("%s %lx -> %lx\n", __func__, start, end); 1761 while (start < end) { 1762 pteval = start | DMAP_PAGE_BITS; 1763 pte = pmap_pml1e(kernel_pmap, PHYS_TO_DMAP(start)); 1764 if ((be64toh(*pte) & RPTE_VALID) == 0) { 1765 page = alloc_pt_page(); 1766 pde_store(pte, page); 1767 } 1768 pte = pmap_l1e_to_l2e(pte, PHYS_TO_DMAP(start)); 1769 if ((start & L2_PAGE_MASK) == 0 && 1770 end - start >= L2_PAGE_SIZE) { 1771 start += L2_PAGE_SIZE; 1772 goto done; 1773 } else if ((be64toh(*pte) & RPTE_VALID) == 0) { 1774 page = alloc_pt_page(); 1775 pde_store(pte, page); 1776 } 1777 1778 pte = pmap_l2e_to_l3e(pte, PHYS_TO_DMAP(start)); 1779 if ((start & L3_PAGE_MASK) == 0 && 1780 end - start >= L3_PAGE_SIZE) { 1781 start += L3_PAGE_SIZE; 1782 goto done; 1783 } else if ((be64toh(*pte) & RPTE_VALID) == 0) { 1784 page = alloc_pt_page(); 1785 pde_store(pte, page); 1786 } 1787 pte = pmap_l3e_to_pte(pte, PHYS_TO_DMAP(start)); 1788 start += PAGE_SIZE; 1789 done: 1790 pte_store(pte, pteval); 1791 } 1792 } 1793 1794 static void 1795 mmu_radix_dmap_populate(vm_size_t hwphyssz) 1796 { 1797 vm_paddr_t start, end; 1798 1799 for (int i = 0; i < pregions_sz; i++) { 1800 start = pregions[i].mr_start; 1801 end = start + pregions[i].mr_size; 1802 if (hwphyssz && start >= hwphyssz) 1803 break; 1804 if (hwphyssz && hwphyssz < end) 1805 end = hwphyssz; 1806 mmu_radix_dmap_range(start, end); 1807 } 1808 } 1809 1810 static void 1811 mmu_radix_setup_pagetables(vm_size_t hwphyssz) 1812 { 1813 vm_paddr_t ptpages, pages; 1814 pt_entry_t *pte; 1815 vm_paddr_t l1phys; 1816 1817 bzero(kernel_pmap, sizeof(struct pmap)); 1818 PMAP_LOCK_INIT(kernel_pmap); 1819 1820 ptpages = allocpages(3); 1821 l1phys = moea64_bootstrap_alloc(RADIX_PGD_SIZE, RADIX_PGD_SIZE); 1822 validate_addr(l1phys, RADIX_PGD_SIZE); 1823 if (bootverbose) 1824 printf("l1phys=%lx\n", l1phys); 1825 MPASS((l1phys & (RADIX_PGD_SIZE-1)) == 0); 1826 for (int i = 0; i < RADIX_PGD_SIZE/PAGE_SIZE; i++) 1827 pagezero(PHYS_TO_DMAP(l1phys + i * PAGE_SIZE)); 1828 kernel_pmap->pm_pml1 = (pml1_entry_t *)PHYS_TO_DMAP(l1phys); 1829 1830 mmu_radix_dmap_populate(hwphyssz); 1831 1832 /* 1833 * Create page tables for first 128MB of KVA 1834 */ 1835 pages = ptpages; 1836 pte = pmap_pml1e(kernel_pmap, VM_MIN_KERNEL_ADDRESS); 1837 *pte = htobe64(pages | RPTE_VALID | RPTE_SHIFT); 1838 pages += PAGE_SIZE; 1839 pte = pmap_l1e_to_l2e(pte, VM_MIN_KERNEL_ADDRESS); 1840 *pte = htobe64(pages | RPTE_VALID | RPTE_SHIFT); 1841 pages += PAGE_SIZE; 1842 pte = pmap_l2e_to_l3e(pte, VM_MIN_KERNEL_ADDRESS); 1843 /* 1844 * the kernel page table pages need to be preserved in 1845 * phys_avail and not overlap with previous allocations 1846 */ 1847 pages = allocpages(nkpt); 1848 if (bootverbose) { 1849 printf("phys_avail after dmap populate and nkpt allocation\n"); 1850 for (int j = 0; j < 2 * phys_avail_count; j+=2) 1851 printf("phys_avail[%d]=%08lx - phys_avail[%d]=%08lx\n", 1852 j, phys_avail[j], j + 1, phys_avail[j + 1]); 1853 } 1854 KPTphys = pages; 1855 for (int i = 0; i < nkpt; i++, pte++, pages += PAGE_SIZE) 1856 *pte = htobe64(pages | RPTE_VALID | RPTE_SHIFT); 1857 kernel_vm_end = VM_MIN_KERNEL_ADDRESS + nkpt * L3_PAGE_SIZE; 1858 if (bootverbose) 1859 printf("kernel_pmap pml1 %p\n", kernel_pmap->pm_pml1); 1860 /* 1861 * Add a physical memory segment (vm_phys_seg) corresponding to the 1862 * preallocated kernel page table pages so that vm_page structures 1863 * representing these pages will be created. The vm_page structures 1864 * are required for promotion of the corresponding kernel virtual 1865 * addresses to superpage mappings. 1866 */ 1867 vm_phys_add_seg(KPTphys, KPTphys + ptoa(nkpt)); 1868 } 1869 1870 static void 1871 mmu_radix_early_bootstrap(vm_offset_t start, vm_offset_t end) 1872 { 1873 vm_paddr_t kpstart, kpend; 1874 vm_size_t physsz, hwphyssz; 1875 //uint64_t l2virt; 1876 int rm_pavail, proctab_size; 1877 int i, j; 1878 1879 kpstart = start & ~DMAP_BASE_ADDRESS; 1880 kpend = end & ~DMAP_BASE_ADDRESS; 1881 1882 /* Get physical memory regions from firmware */ 1883 mem_regions(&pregions, &pregions_sz, ®ions, ®ions_sz); 1884 CTR0(KTR_PMAP, "mmu_radix_early_bootstrap: physical memory"); 1885 1886 if (2 * VM_PHYSSEG_MAX < regions_sz) 1887 panic("mmu_radix_early_bootstrap: phys_avail too small"); 1888 1889 if (bootverbose) 1890 for (int i = 0; i < regions_sz; i++) 1891 printf("regions[%d].mr_start=%lx regions[%d].mr_size=%lx\n", 1892 i, regions[i].mr_start, i, regions[i].mr_size); 1893 /* 1894 * XXX workaround a simulator bug 1895 */ 1896 for (int i = 0; i < regions_sz; i++) 1897 if (regions[i].mr_start & PAGE_MASK) { 1898 regions[i].mr_start += PAGE_MASK; 1899 regions[i].mr_start &= ~PAGE_MASK; 1900 regions[i].mr_size &= ~PAGE_MASK; 1901 } 1902 if (bootverbose) 1903 for (int i = 0; i < pregions_sz; i++) 1904 printf("pregions[%d].mr_start=%lx pregions[%d].mr_size=%lx\n", 1905 i, pregions[i].mr_start, i, pregions[i].mr_size); 1906 1907 phys_avail_count = 0; 1908 physsz = 0; 1909 hwphyssz = 0; 1910 TUNABLE_ULONG_FETCH("hw.physmem", (u_long *) &hwphyssz); 1911 for (i = 0, j = 0; i < regions_sz; i++) { 1912 if (bootverbose) 1913 printf("regions[%d].mr_start=%016lx regions[%d].mr_size=%016lx\n", 1914 i, regions[i].mr_start, i, regions[i].mr_size); 1915 1916 if (regions[i].mr_size < PAGE_SIZE) 1917 continue; 1918 1919 if (hwphyssz != 0 && 1920 (physsz + regions[i].mr_size) >= hwphyssz) { 1921 if (physsz < hwphyssz) { 1922 phys_avail[j] = regions[i].mr_start; 1923 phys_avail[j + 1] = regions[i].mr_start + 1924 (hwphyssz - physsz); 1925 physsz = hwphyssz; 1926 phys_avail_count++; 1927 dump_avail[j] = phys_avail[j]; 1928 dump_avail[j + 1] = phys_avail[j + 1]; 1929 } 1930 break; 1931 } 1932 phys_avail[j] = regions[i].mr_start; 1933 phys_avail[j + 1] = regions[i].mr_start + regions[i].mr_size; 1934 dump_avail[j] = phys_avail[j]; 1935 dump_avail[j + 1] = phys_avail[j + 1]; 1936 1937 phys_avail_count++; 1938 physsz += regions[i].mr_size; 1939 j += 2; 1940 } 1941 1942 /* Check for overlap with the kernel and exception vectors */ 1943 rm_pavail = 0; 1944 for (j = 0; j < 2 * phys_avail_count; j+=2) { 1945 if (phys_avail[j] < EXC_LAST) 1946 phys_avail[j] += EXC_LAST; 1947 1948 if (phys_avail[j] >= kpstart && 1949 phys_avail[j + 1] <= kpend) { 1950 phys_avail[j] = phys_avail[j + 1] = ~0; 1951 rm_pavail++; 1952 continue; 1953 } 1954 1955 if (kpstart >= phys_avail[j] && 1956 kpstart < phys_avail[j + 1]) { 1957 if (kpend < phys_avail[j + 1]) { 1958 phys_avail[2 * phys_avail_count] = 1959 (kpend & ~PAGE_MASK) + PAGE_SIZE; 1960 phys_avail[2 * phys_avail_count + 1] = 1961 phys_avail[j + 1]; 1962 phys_avail_count++; 1963 } 1964 1965 phys_avail[j + 1] = kpstart & ~PAGE_MASK; 1966 } 1967 1968 if (kpend >= phys_avail[j] && 1969 kpend < phys_avail[j + 1]) { 1970 if (kpstart > phys_avail[j]) { 1971 phys_avail[2 * phys_avail_count] = phys_avail[j]; 1972 phys_avail[2 * phys_avail_count + 1] = 1973 kpstart & ~PAGE_MASK; 1974 phys_avail_count++; 1975 } 1976 1977 phys_avail[j] = (kpend & ~PAGE_MASK) + 1978 PAGE_SIZE; 1979 } 1980 } 1981 qsort(phys_avail, 2 * phys_avail_count, sizeof(phys_avail[0]), pa_cmp); 1982 for (i = 0; i < 2 * phys_avail_count; i++) 1983 phys_avail_debug[i] = phys_avail[i]; 1984 1985 /* Remove physical available regions marked for removal (~0) */ 1986 if (rm_pavail) { 1987 phys_avail_count -= rm_pavail; 1988 for (i = 2 * phys_avail_count; 1989 i < 2*(phys_avail_count + rm_pavail); i+=2) 1990 phys_avail[i] = phys_avail[i + 1] = 0; 1991 } 1992 if (bootverbose) { 1993 printf("phys_avail ranges after filtering:\n"); 1994 for (j = 0; j < 2 * phys_avail_count; j+=2) 1995 printf("phys_avail[%d]=%08lx - phys_avail[%d]=%08lx\n", 1996 j, phys_avail[j], j + 1, phys_avail[j + 1]); 1997 } 1998 physmem = btoc(physsz); 1999 2000 /* XXX assume we're running non-virtualized and 2001 * we don't support BHYVE 2002 */ 2003 if (isa3_pid_bits == 0) 2004 isa3_pid_bits = 20; 2005 parttab_phys = moea64_bootstrap_alloc(PARTTAB_SIZE, PARTTAB_SIZE); 2006 validate_addr(parttab_phys, PARTTAB_SIZE); 2007 for (int i = 0; i < PARTTAB_SIZE/PAGE_SIZE; i++) 2008 pagezero(PHYS_TO_DMAP(parttab_phys + i * PAGE_SIZE)); 2009 2010 proctab_size = 1UL << PROCTAB_SIZE_SHIFT; 2011 proctab0pa = moea64_bootstrap_alloc(proctab_size, proctab_size); 2012 validate_addr(proctab0pa, proctab_size); 2013 for (int i = 0; i < proctab_size/PAGE_SIZE; i++) 2014 pagezero(PHYS_TO_DMAP(proctab0pa + i * PAGE_SIZE)); 2015 2016 mmu_radix_setup_pagetables(hwphyssz); 2017 } 2018 2019 static void 2020 mmu_radix_late_bootstrap(vm_offset_t start, vm_offset_t end) 2021 { 2022 int i; 2023 vm_paddr_t pa; 2024 void *dpcpu; 2025 vm_offset_t va; 2026 2027 /* 2028 * Set up the Open Firmware pmap and add its mappings if not in real 2029 * mode. 2030 */ 2031 if (bootverbose) 2032 printf("%s enter\n", __func__); 2033 2034 /* 2035 * Calculate the last available physical address, and reserve the 2036 * vm_page_array (upper bound). 2037 */ 2038 Maxmem = 0; 2039 for (i = 0; phys_avail[i + 2] != 0; i += 2) 2040 Maxmem = MAX(Maxmem, powerpc_btop(phys_avail[i + 1])); 2041 2042 /* 2043 * Set the start and end of kva. 2044 */ 2045 virtual_avail = VM_MIN_KERNEL_ADDRESS; 2046 virtual_end = VM_MAX_SAFE_KERNEL_ADDRESS; 2047 2048 /* 2049 * Remap any early IO mappings (console framebuffer, etc.) 2050 */ 2051 bs_remap_earlyboot(); 2052 2053 /* 2054 * Allocate a kernel stack with a guard page for thread0 and map it 2055 * into the kernel page map. 2056 */ 2057 pa = allocpages(kstack_pages); 2058 va = virtual_avail + KSTACK_GUARD_PAGES * PAGE_SIZE; 2059 virtual_avail = va + kstack_pages * PAGE_SIZE; 2060 CTR2(KTR_PMAP, "moea64_bootstrap: kstack0 at %#x (%#x)", pa, va); 2061 thread0.td_kstack = va; 2062 for (i = 0; i < kstack_pages; i++) { 2063 mmu_radix_kenter(va, pa); 2064 pa += PAGE_SIZE; 2065 va += PAGE_SIZE; 2066 } 2067 thread0.td_kstack_pages = kstack_pages; 2068 2069 /* 2070 * Allocate virtual address space for the message buffer. 2071 */ 2072 pa = msgbuf_phys = allocpages((msgbufsize + PAGE_MASK) >> PAGE_SHIFT); 2073 msgbufp = (struct msgbuf *)PHYS_TO_DMAP(pa); 2074 2075 /* 2076 * Allocate virtual address space for the dynamic percpu area. 2077 */ 2078 pa = allocpages(DPCPU_SIZE >> PAGE_SHIFT); 2079 dpcpu = (void *)PHYS_TO_DMAP(pa); 2080 dpcpu_init(dpcpu, curcpu); 2081 /* 2082 * Reserve some special page table entries/VA space for temporary 2083 * mapping of pages. 2084 */ 2085 } 2086 2087 static void 2088 mmu_parttab_init(void) 2089 { 2090 uint64_t ptcr; 2091 2092 isa3_parttab = (struct pate *)PHYS_TO_DMAP(parttab_phys); 2093 2094 if (bootverbose) 2095 printf("%s parttab: %p\n", __func__, isa3_parttab); 2096 ptcr = parttab_phys | (PARTTAB_SIZE_SHIFT-12); 2097 if (bootverbose) 2098 printf("setting ptcr %lx\n", ptcr); 2099 mtspr(SPR_PTCR, ptcr); 2100 } 2101 2102 static void 2103 mmu_parttab_update(uint64_t lpid, uint64_t pagetab, uint64_t proctab) 2104 { 2105 uint64_t prev; 2106 2107 if (bootverbose) 2108 printf("%s isa3_parttab %p lpid %lx pagetab %lx proctab %lx\n", __func__, isa3_parttab, 2109 lpid, pagetab, proctab); 2110 prev = be64toh(isa3_parttab[lpid].pagetab); 2111 isa3_parttab[lpid].pagetab = htobe64(pagetab); 2112 isa3_parttab[lpid].proctab = htobe64(proctab); 2113 2114 if (prev & PARTTAB_HR) { 2115 __asm __volatile(PPC_TLBIE_5(%0,%1,2,0,1) : : 2116 "r" (TLBIEL_INVAL_SET_LPID), "r" (lpid)); 2117 __asm __volatile(PPC_TLBIE_5(%0,%1,2,1,1) : : 2118 "r" (TLBIEL_INVAL_SET_LPID), "r" (lpid)); 2119 } else { 2120 __asm __volatile(PPC_TLBIE_5(%0,%1,2,0,0) : : 2121 "r" (TLBIEL_INVAL_SET_LPID), "r" (lpid)); 2122 } 2123 ttusync(); 2124 } 2125 2126 static void 2127 mmu_radix_parttab_init(void) 2128 { 2129 uint64_t pagetab; 2130 2131 mmu_parttab_init(); 2132 pagetab = RTS_SIZE | DMAP_TO_PHYS((vm_offset_t)kernel_pmap->pm_pml1) | \ 2133 RADIX_PGD_INDEX_SHIFT | PARTTAB_HR; 2134 mmu_parttab_update(0, pagetab, 0); 2135 } 2136 2137 static void 2138 mmu_radix_proctab_register(vm_paddr_t proctabpa, uint64_t table_size) 2139 { 2140 uint64_t pagetab, proctab; 2141 2142 pagetab = be64toh(isa3_parttab[0].pagetab); 2143 proctab = proctabpa | table_size | PARTTAB_GR; 2144 mmu_parttab_update(0, pagetab, proctab); 2145 } 2146 2147 static void 2148 mmu_radix_proctab_init(void) 2149 { 2150 2151 isa3_base_pid = 1; 2152 2153 isa3_proctab = (void*)PHYS_TO_DMAP(proctab0pa); 2154 isa3_proctab->proctab0 = 2155 htobe64(RTS_SIZE | DMAP_TO_PHYS((vm_offset_t)kernel_pmap->pm_pml1) | 2156 RADIX_PGD_INDEX_SHIFT); 2157 2158 mmu_radix_proctab_register(proctab0pa, PROCTAB_SIZE_SHIFT - 12); 2159 2160 __asm __volatile("ptesync" : : : "memory"); 2161 __asm __volatile(PPC_TLBIE_5(%0,%1,2,1,1) : : 2162 "r" (TLBIEL_INVAL_SET_LPID), "r" (0)); 2163 __asm __volatile("eieio; tlbsync; ptesync" : : : "memory"); 2164 if (bootverbose) 2165 printf("process table %p and kernel radix PDE: %p\n", 2166 isa3_proctab, kernel_pmap->pm_pml1); 2167 mtmsr(mfmsr() | PSL_DR ); 2168 mtmsr(mfmsr() & ~PSL_DR); 2169 kernel_pmap->pm_pid = isa3_base_pid; 2170 isa3_base_pid++; 2171 } 2172 2173 void 2174 mmu_radix_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, 2175 int advice) 2176 { 2177 struct rwlock *lock; 2178 pml1_entry_t *l1e; 2179 pml2_entry_t *l2e; 2180 pml3_entry_t oldl3e, *l3e; 2181 pt_entry_t *pte; 2182 vm_offset_t va, va_next; 2183 vm_page_t m; 2184 boolean_t anychanged; 2185 2186 if (advice != MADV_DONTNEED && advice != MADV_FREE) 2187 return; 2188 anychanged = FALSE; 2189 PMAP_LOCK(pmap); 2190 for (; sva < eva; sva = va_next) { 2191 l1e = pmap_pml1e(pmap, sva); 2192 if ((be64toh(*l1e) & PG_V) == 0) { 2193 va_next = (sva + L1_PAGE_SIZE) & ~L1_PAGE_MASK; 2194 if (va_next < sva) 2195 va_next = eva; 2196 continue; 2197 } 2198 l2e = pmap_l1e_to_l2e(l1e, sva); 2199 if ((be64toh(*l2e) & PG_V) == 0) { 2200 va_next = (sva + L2_PAGE_SIZE) & ~L2_PAGE_MASK; 2201 if (va_next < sva) 2202 va_next = eva; 2203 continue; 2204 } 2205 va_next = (sva + L3_PAGE_SIZE) & ~L3_PAGE_MASK; 2206 if (va_next < sva) 2207 va_next = eva; 2208 l3e = pmap_l2e_to_l3e(l2e, sva); 2209 oldl3e = be64toh(*l3e); 2210 if ((oldl3e & PG_V) == 0) 2211 continue; 2212 else if ((oldl3e & RPTE_LEAF) != 0) { 2213 if ((oldl3e & PG_MANAGED) == 0) 2214 continue; 2215 lock = NULL; 2216 if (!pmap_demote_l3e_locked(pmap, l3e, sva, &lock)) { 2217 if (lock != NULL) 2218 rw_wunlock(lock); 2219 2220 /* 2221 * The large page mapping was destroyed. 2222 */ 2223 continue; 2224 } 2225 2226 /* 2227 * Unless the page mappings are wired, remove the 2228 * mapping to a single page so that a subsequent 2229 * access may repromote. Since the underlying page 2230 * table page is fully populated, this removal never 2231 * frees a page table page. 2232 */ 2233 if ((oldl3e & PG_W) == 0) { 2234 pte = pmap_l3e_to_pte(l3e, sva); 2235 KASSERT((be64toh(*pte) & PG_V) != 0, 2236 ("pmap_advise: invalid PTE")); 2237 pmap_remove_pte(pmap, pte, sva, be64toh(*l3e), NULL, 2238 &lock); 2239 anychanged = TRUE; 2240 } 2241 if (lock != NULL) 2242 rw_wunlock(lock); 2243 } 2244 if (va_next > eva) 2245 va_next = eva; 2246 va = va_next; 2247 for (pte = pmap_l3e_to_pte(l3e, sva); sva != va_next; 2248 pte++, sva += PAGE_SIZE) { 2249 MPASS(pte == pmap_pte(pmap, sva)); 2250 2251 if ((be64toh(*pte) & (PG_MANAGED | PG_V)) != (PG_MANAGED | PG_V)) 2252 goto maybe_invlrng; 2253 else if ((be64toh(*pte) & (PG_M | PG_RW)) == (PG_M | PG_RW)) { 2254 if (advice == MADV_DONTNEED) { 2255 /* 2256 * Future calls to pmap_is_modified() 2257 * can be avoided by making the page 2258 * dirty now. 2259 */ 2260 m = PHYS_TO_VM_PAGE(be64toh(*pte) & PG_FRAME); 2261 vm_page_dirty(m); 2262 } 2263 atomic_clear_long(pte, htobe64(PG_M | PG_A)); 2264 } else if ((be64toh(*pte) & PG_A) != 0) 2265 atomic_clear_long(pte, htobe64(PG_A)); 2266 else 2267 goto maybe_invlrng; 2268 anychanged = TRUE; 2269 continue; 2270 maybe_invlrng: 2271 if (va != va_next) { 2272 anychanged = true; 2273 va = va_next; 2274 } 2275 } 2276 if (va != va_next) 2277 anychanged = true; 2278 } 2279 if (anychanged) 2280 pmap_invalidate_all(pmap); 2281 PMAP_UNLOCK(pmap); 2282 } 2283 2284 /* 2285 * Routines used in machine-dependent code 2286 */ 2287 static void 2288 mmu_radix_bootstrap(vm_offset_t start, vm_offset_t end) 2289 { 2290 uint64_t lpcr; 2291 2292 if (bootverbose) 2293 printf("%s\n", __func__); 2294 hw_direct_map = 1; 2295 mmu_radix_early_bootstrap(start, end); 2296 if (bootverbose) 2297 printf("early bootstrap complete\n"); 2298 if (powernv_enabled) { 2299 lpcr = mfspr(SPR_LPCR); 2300 mtspr(SPR_LPCR, lpcr | LPCR_UPRT | LPCR_HR); 2301 mmu_radix_parttab_init(); 2302 mmu_radix_init_amor(); 2303 if (bootverbose) 2304 printf("powernv init complete\n"); 2305 } 2306 mmu_radix_init_iamr(); 2307 mmu_radix_proctab_init(); 2308 mmu_radix_pid_set(kernel_pmap); 2309 /* XXX assume CPU_FTR_HVMODE */ 2310 mmu_radix_tlbiel_flush(TLB_INVAL_SCOPE_GLOBAL); 2311 2312 mmu_radix_late_bootstrap(start, end); 2313 numa_mem_regions(&numa_pregions, &numa_pregions_sz); 2314 if (bootverbose) 2315 printf("%s done\n", __func__); 2316 pmap_bootstrapped = 1; 2317 dmaplimit = roundup2(powerpc_ptob(Maxmem), L2_PAGE_SIZE); 2318 PCPU_SET(flags, PCPU_GET(flags) | PC_FLAG_NOSRS); 2319 } 2320 2321 static void 2322 mmu_radix_cpu_bootstrap(int ap) 2323 { 2324 uint64_t lpcr; 2325 uint64_t ptcr; 2326 2327 if (powernv_enabled) { 2328 lpcr = mfspr(SPR_LPCR); 2329 mtspr(SPR_LPCR, lpcr | LPCR_UPRT | LPCR_HR); 2330 2331 ptcr = parttab_phys | (PARTTAB_SIZE_SHIFT-12); 2332 mtspr(SPR_PTCR, ptcr); 2333 mmu_radix_init_amor(); 2334 } 2335 mmu_radix_init_iamr(); 2336 mmu_radix_pid_set(kernel_pmap); 2337 mmu_radix_tlbiel_flush(TLB_INVAL_SCOPE_GLOBAL); 2338 } 2339 2340 static SYSCTL_NODE(_vm_pmap, OID_AUTO, l3e, CTLFLAG_RD, 0, 2341 "2MB page mapping counters"); 2342 2343 static u_long pmap_l3e_demotions; 2344 SYSCTL_ULONG(_vm_pmap_l3e, OID_AUTO, demotions, CTLFLAG_RD, 2345 &pmap_l3e_demotions, 0, "2MB page demotions"); 2346 2347 static u_long pmap_l3e_mappings; 2348 SYSCTL_ULONG(_vm_pmap_l3e, OID_AUTO, mappings, CTLFLAG_RD, 2349 &pmap_l3e_mappings, 0, "2MB page mappings"); 2350 2351 static u_long pmap_l3e_p_failures; 2352 SYSCTL_ULONG(_vm_pmap_l3e, OID_AUTO, p_failures, CTLFLAG_RD, 2353 &pmap_l3e_p_failures, 0, "2MB page promotion failures"); 2354 2355 static u_long pmap_l3e_promotions; 2356 SYSCTL_ULONG(_vm_pmap_l3e, OID_AUTO, promotions, CTLFLAG_RD, 2357 &pmap_l3e_promotions, 0, "2MB page promotions"); 2358 2359 static SYSCTL_NODE(_vm_pmap, OID_AUTO, l2e, CTLFLAG_RD, 0, 2360 "1GB page mapping counters"); 2361 2362 static u_long pmap_l2e_demotions; 2363 SYSCTL_ULONG(_vm_pmap_l2e, OID_AUTO, demotions, CTLFLAG_RD, 2364 &pmap_l2e_demotions, 0, "1GB page demotions"); 2365 2366 void 2367 mmu_radix_clear_modify(vm_page_t m) 2368 { 2369 struct md_page *pvh; 2370 pmap_t pmap; 2371 pv_entry_t next_pv, pv; 2372 pml3_entry_t oldl3e, *l3e; 2373 pt_entry_t oldpte, *pte; 2374 struct rwlock *lock; 2375 vm_offset_t va; 2376 int md_gen, pvh_gen; 2377 2378 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 2379 ("pmap_clear_modify: page %p is not managed", m)); 2380 vm_page_assert_busied(m); 2381 CTR2(KTR_PMAP, "%s(%p)", __func__, m); 2382 2383 /* 2384 * If the page is not PGA_WRITEABLE, then no PTEs can have PG_M set. 2385 * If the object containing the page is locked and the page is not 2386 * exclusive busied, then PGA_WRITEABLE cannot be concurrently set. 2387 */ 2388 if ((m->a.flags & PGA_WRITEABLE) == 0) 2389 return; 2390 pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy : 2391 pa_to_pvh(VM_PAGE_TO_PHYS(m)); 2392 lock = VM_PAGE_TO_PV_LIST_LOCK(m); 2393 rw_wlock(lock); 2394 restart: 2395 TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_link, next_pv) { 2396 pmap = PV_PMAP(pv); 2397 if (!PMAP_TRYLOCK(pmap)) { 2398 pvh_gen = pvh->pv_gen; 2399 rw_wunlock(lock); 2400 PMAP_LOCK(pmap); 2401 rw_wlock(lock); 2402 if (pvh_gen != pvh->pv_gen) { 2403 PMAP_UNLOCK(pmap); 2404 goto restart; 2405 } 2406 } 2407 va = pv->pv_va; 2408 l3e = pmap_pml3e(pmap, va); 2409 oldl3e = be64toh(*l3e); 2410 if ((oldl3e & PG_RW) != 0) { 2411 if (pmap_demote_l3e_locked(pmap, l3e, va, &lock)) { 2412 if ((oldl3e & PG_W) == 0) { 2413 /* 2414 * Write protect the mapping to a 2415 * single page so that a subsequent 2416 * write access may repromote. 2417 */ 2418 va += VM_PAGE_TO_PHYS(m) - (oldl3e & 2419 PG_PS_FRAME); 2420 pte = pmap_l3e_to_pte(l3e, va); 2421 oldpte = be64toh(*pte); 2422 if ((oldpte & PG_V) != 0) { 2423 while (!atomic_cmpset_long(pte, 2424 htobe64(oldpte), 2425 htobe64((oldpte | RPTE_EAA_R) & ~(PG_M | PG_RW)))) 2426 oldpte = be64toh(*pte); 2427 vm_page_dirty(m); 2428 pmap_invalidate_page(pmap, va); 2429 } 2430 } 2431 } 2432 } 2433 PMAP_UNLOCK(pmap); 2434 } 2435 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) { 2436 pmap = PV_PMAP(pv); 2437 if (!PMAP_TRYLOCK(pmap)) { 2438 md_gen = m->md.pv_gen; 2439 pvh_gen = pvh->pv_gen; 2440 rw_wunlock(lock); 2441 PMAP_LOCK(pmap); 2442 rw_wlock(lock); 2443 if (pvh_gen != pvh->pv_gen || md_gen != m->md.pv_gen) { 2444 PMAP_UNLOCK(pmap); 2445 goto restart; 2446 } 2447 } 2448 l3e = pmap_pml3e(pmap, pv->pv_va); 2449 KASSERT((be64toh(*l3e) & RPTE_LEAF) == 0, ("pmap_clear_modify: found" 2450 " a 2mpage in page %p's pv list", m)); 2451 pte = pmap_l3e_to_pte(l3e, pv->pv_va); 2452 if ((be64toh(*pte) & (PG_M | PG_RW)) == (PG_M | PG_RW)) { 2453 atomic_clear_long(pte, htobe64(PG_M)); 2454 pmap_invalidate_page(pmap, pv->pv_va); 2455 } 2456 PMAP_UNLOCK(pmap); 2457 } 2458 rw_wunlock(lock); 2459 } 2460 2461 void 2462 mmu_radix_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, 2463 vm_size_t len, vm_offset_t src_addr) 2464 { 2465 struct rwlock *lock; 2466 struct spglist free; 2467 vm_offset_t addr; 2468 vm_offset_t end_addr = src_addr + len; 2469 vm_offset_t va_next; 2470 vm_page_t dst_pdpg, dstmpte, srcmpte; 2471 bool invalidate_all; 2472 2473 CTR6(KTR_PMAP, 2474 "%s(dst_pmap=%p, src_pmap=%p, dst_addr=%lx, len=%lu, src_addr=%lx)\n", 2475 __func__, dst_pmap, src_pmap, dst_addr, len, src_addr); 2476 2477 if (dst_addr != src_addr) 2478 return; 2479 lock = NULL; 2480 invalidate_all = false; 2481 if (dst_pmap < src_pmap) { 2482 PMAP_LOCK(dst_pmap); 2483 PMAP_LOCK(src_pmap); 2484 } else { 2485 PMAP_LOCK(src_pmap); 2486 PMAP_LOCK(dst_pmap); 2487 } 2488 2489 for (addr = src_addr; addr < end_addr; addr = va_next) { 2490 pml1_entry_t *l1e; 2491 pml2_entry_t *l2e; 2492 pml3_entry_t srcptepaddr, *l3e; 2493 pt_entry_t *src_pte, *dst_pte; 2494 2495 l1e = pmap_pml1e(src_pmap, addr); 2496 if ((be64toh(*l1e) & PG_V) == 0) { 2497 va_next = (addr + L1_PAGE_SIZE) & ~L1_PAGE_MASK; 2498 if (va_next < addr) 2499 va_next = end_addr; 2500 continue; 2501 } 2502 2503 l2e = pmap_l1e_to_l2e(l1e, addr); 2504 if ((be64toh(*l2e) & PG_V) == 0) { 2505 va_next = (addr + L2_PAGE_SIZE) & ~L2_PAGE_MASK; 2506 if (va_next < addr) 2507 va_next = end_addr; 2508 continue; 2509 } 2510 2511 va_next = (addr + L3_PAGE_SIZE) & ~L3_PAGE_MASK; 2512 if (va_next < addr) 2513 va_next = end_addr; 2514 2515 l3e = pmap_l2e_to_l3e(l2e, addr); 2516 srcptepaddr = be64toh(*l3e); 2517 if (srcptepaddr == 0) 2518 continue; 2519 2520 if (srcptepaddr & RPTE_LEAF) { 2521 if ((addr & L3_PAGE_MASK) != 0 || 2522 addr + L3_PAGE_SIZE > end_addr) 2523 continue; 2524 dst_pdpg = pmap_allocl3e(dst_pmap, addr, NULL); 2525 if (dst_pdpg == NULL) 2526 break; 2527 l3e = (pml3_entry_t *) 2528 PHYS_TO_DMAP(VM_PAGE_TO_PHYS(dst_pdpg)); 2529 l3e = &l3e[pmap_pml3e_index(addr)]; 2530 if (be64toh(*l3e) == 0 && ((srcptepaddr & PG_MANAGED) == 0 || 2531 pmap_pv_insert_l3e(dst_pmap, addr, srcptepaddr, 2532 PMAP_ENTER_NORECLAIM, &lock))) { 2533 *l3e = htobe64(srcptepaddr & ~PG_W); 2534 pmap_resident_count_inc(dst_pmap, 2535 L3_PAGE_SIZE / PAGE_SIZE); 2536 atomic_add_long(&pmap_l3e_mappings, 1); 2537 } else 2538 dst_pdpg->ref_count--; 2539 continue; 2540 } 2541 2542 srcptepaddr &= PG_FRAME; 2543 srcmpte = PHYS_TO_VM_PAGE(srcptepaddr); 2544 KASSERT(srcmpte->ref_count > 0, 2545 ("pmap_copy: source page table page is unused")); 2546 2547 if (va_next > end_addr) 2548 va_next = end_addr; 2549 2550 src_pte = (pt_entry_t *)PHYS_TO_DMAP(srcptepaddr); 2551 src_pte = &src_pte[pmap_pte_index(addr)]; 2552 dstmpte = NULL; 2553 while (addr < va_next) { 2554 pt_entry_t ptetemp; 2555 ptetemp = be64toh(*src_pte); 2556 /* 2557 * we only virtual copy managed pages 2558 */ 2559 if ((ptetemp & PG_MANAGED) != 0) { 2560 if (dstmpte != NULL && 2561 dstmpte->pindex == pmap_l3e_pindex(addr)) 2562 dstmpte->ref_count++; 2563 else if ((dstmpte = pmap_allocpte(dst_pmap, 2564 addr, NULL)) == NULL) 2565 goto out; 2566 dst_pte = (pt_entry_t *) 2567 PHYS_TO_DMAP(VM_PAGE_TO_PHYS(dstmpte)); 2568 dst_pte = &dst_pte[pmap_pte_index(addr)]; 2569 if (be64toh(*dst_pte) == 0 && 2570 pmap_try_insert_pv_entry(dst_pmap, addr, 2571 PHYS_TO_VM_PAGE(ptetemp & PG_FRAME), 2572 &lock)) { 2573 /* 2574 * Clear the wired, modified, and 2575 * accessed (referenced) bits 2576 * during the copy. 2577 */ 2578 *dst_pte = htobe64(ptetemp & ~(PG_W | PG_M | 2579 PG_A)); 2580 pmap_resident_count_inc(dst_pmap, 1); 2581 } else { 2582 SLIST_INIT(&free); 2583 if (pmap_unwire_ptp(dst_pmap, addr, 2584 dstmpte, &free)) { 2585 /* 2586 * Although "addr" is not 2587 * mapped, paging-structure 2588 * caches could nonetheless 2589 * have entries that refer to 2590 * the freed page table pages. 2591 * Invalidate those entries. 2592 */ 2593 invalidate_all = true; 2594 vm_page_free_pages_toq(&free, 2595 true); 2596 } 2597 goto out; 2598 } 2599 if (dstmpte->ref_count >= srcmpte->ref_count) 2600 break; 2601 } 2602 addr += PAGE_SIZE; 2603 if (__predict_false((addr & L3_PAGE_MASK) == 0)) 2604 src_pte = pmap_pte(src_pmap, addr); 2605 else 2606 src_pte++; 2607 } 2608 } 2609 out: 2610 if (invalidate_all) 2611 pmap_invalidate_all(dst_pmap); 2612 if (lock != NULL) 2613 rw_wunlock(lock); 2614 PMAP_UNLOCK(src_pmap); 2615 PMAP_UNLOCK(dst_pmap); 2616 } 2617 2618 static void 2619 mmu_radix_copy_page(vm_page_t msrc, vm_page_t mdst) 2620 { 2621 vm_offset_t src = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(msrc)); 2622 vm_offset_t dst = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mdst)); 2623 2624 CTR3(KTR_PMAP, "%s(%p, %p)", __func__, src, dst); 2625 /* 2626 * XXX slow 2627 */ 2628 bcopy((void *)src, (void *)dst, PAGE_SIZE); 2629 } 2630 2631 static void 2632 mmu_radix_copy_pages(vm_page_t ma[], vm_offset_t a_offset, vm_page_t mb[], 2633 vm_offset_t b_offset, int xfersize) 2634 { 2635 void *a_cp, *b_cp; 2636 vm_offset_t a_pg_offset, b_pg_offset; 2637 int cnt; 2638 2639 CTR6(KTR_PMAP, "%s(%p, %#x, %p, %#x, %#x)", __func__, ma, 2640 a_offset, mb, b_offset, xfersize); 2641 2642 while (xfersize > 0) { 2643 a_pg_offset = a_offset & PAGE_MASK; 2644 cnt = min(xfersize, PAGE_SIZE - a_pg_offset); 2645 a_cp = (char *)(uintptr_t)PHYS_TO_DMAP( 2646 VM_PAGE_TO_PHYS(ma[a_offset >> PAGE_SHIFT])) + 2647 a_pg_offset; 2648 b_pg_offset = b_offset & PAGE_MASK; 2649 cnt = min(cnt, PAGE_SIZE - b_pg_offset); 2650 b_cp = (char *)(uintptr_t)PHYS_TO_DMAP( 2651 VM_PAGE_TO_PHYS(mb[b_offset >> PAGE_SHIFT])) + 2652 b_pg_offset; 2653 bcopy(a_cp, b_cp, cnt); 2654 a_offset += cnt; 2655 b_offset += cnt; 2656 xfersize -= cnt; 2657 } 2658 } 2659 2660 #if VM_NRESERVLEVEL > 0 2661 /* 2662 * Tries to promote the 512, contiguous 4KB page mappings that are within a 2663 * single page table page (PTP) to a single 2MB page mapping. For promotion 2664 * to occur, two conditions must be met: (1) the 4KB page mappings must map 2665 * aligned, contiguous physical memory and (2) the 4KB page mappings must have 2666 * identical characteristics. 2667 */ 2668 static int 2669 pmap_promote_l3e(pmap_t pmap, pml3_entry_t *pde, vm_offset_t va, 2670 struct rwlock **lockp) 2671 { 2672 pml3_entry_t newpde; 2673 pt_entry_t *firstpte, oldpte, pa, *pte; 2674 vm_page_t mpte; 2675 2676 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 2677 2678 /* 2679 * Examine the first PTE in the specified PTP. Abort if this PTE is 2680 * either invalid, unused, or does not map the first 4KB physical page 2681 * within a 2MB page. 2682 */ 2683 firstpte = (pt_entry_t *)PHYS_TO_DMAP(be64toh(*pde) & PG_FRAME); 2684 setpde: 2685 newpde = *firstpte; 2686 if ((newpde & ((PG_FRAME & L3_PAGE_MASK) | PG_A | PG_V)) != (PG_A | PG_V)) { 2687 CTR2(KTR_PMAP, "pmap_promote_l3e: failure for va %#lx" 2688 " in pmap %p", va, pmap); 2689 goto fail; 2690 } 2691 if ((newpde & (PG_M | PG_RW)) == PG_RW) { 2692 /* 2693 * When PG_M is already clear, PG_RW can be cleared without 2694 * a TLB invalidation. 2695 */ 2696 if (!atomic_cmpset_long(firstpte, htobe64(newpde), htobe64((newpde | RPTE_EAA_R) & ~RPTE_EAA_W))) 2697 goto setpde; 2698 newpde &= ~RPTE_EAA_W; 2699 } 2700 2701 /* 2702 * Examine each of the other PTEs in the specified PTP. Abort if this 2703 * PTE maps an unexpected 4KB physical page or does not have identical 2704 * characteristics to the first PTE. 2705 */ 2706 pa = (newpde & (PG_PS_FRAME | PG_A | PG_V)) + L3_PAGE_SIZE - PAGE_SIZE; 2707 for (pte = firstpte + NPTEPG - 1; pte > firstpte; pte--) { 2708 setpte: 2709 oldpte = be64toh(*pte); 2710 if ((oldpte & (PG_FRAME | PG_A | PG_V)) != pa) { 2711 CTR2(KTR_PMAP, "pmap_promote_l3e: failure for va %#lx" 2712 " in pmap %p", va, pmap); 2713 goto fail; 2714 } 2715 if ((oldpte & (PG_M | PG_RW)) == PG_RW) { 2716 /* 2717 * When PG_M is already clear, PG_RW can be cleared 2718 * without a TLB invalidation. 2719 */ 2720 if (!atomic_cmpset_long(pte, htobe64(oldpte), htobe64((oldpte | RPTE_EAA_R) & ~RPTE_EAA_W))) 2721 goto setpte; 2722 oldpte &= ~RPTE_EAA_W; 2723 CTR2(KTR_PMAP, "pmap_promote_l3e: protect for va %#lx" 2724 " in pmap %p", (oldpte & PG_FRAME & L3_PAGE_MASK) | 2725 (va & ~L3_PAGE_MASK), pmap); 2726 } 2727 if ((oldpte & PG_PTE_PROMOTE) != (newpde & PG_PTE_PROMOTE)) { 2728 CTR2(KTR_PMAP, "pmap_promote_l3e: failure for va %#lx" 2729 " in pmap %p", va, pmap); 2730 goto fail; 2731 } 2732 pa -= PAGE_SIZE; 2733 } 2734 2735 /* 2736 * Save the page table page in its current state until the PDE 2737 * mapping the superpage is demoted by pmap_demote_pde() or 2738 * destroyed by pmap_remove_pde(). 2739 */ 2740 mpte = PHYS_TO_VM_PAGE(be64toh(*pde) & PG_FRAME); 2741 KASSERT(mpte >= vm_page_array && 2742 mpte < &vm_page_array[vm_page_array_size], 2743 ("pmap_promote_l3e: page table page is out of range")); 2744 KASSERT(mpte->pindex == pmap_l3e_pindex(va), 2745 ("pmap_promote_l3e: page table page's pindex is wrong")); 2746 if (pmap_insert_pt_page(pmap, mpte)) { 2747 CTR2(KTR_PMAP, 2748 "pmap_promote_l3e: failure for va %#lx in pmap %p", va, 2749 pmap); 2750 goto fail; 2751 } 2752 2753 /* 2754 * Promote the pv entries. 2755 */ 2756 if ((newpde & PG_MANAGED) != 0) 2757 pmap_pv_promote_l3e(pmap, va, newpde & PG_PS_FRAME, lockp); 2758 2759 pte_store(pde, PG_PROMOTED | newpde); 2760 atomic_add_long(&pmap_l3e_promotions, 1); 2761 CTR2(KTR_PMAP, "pmap_promote_l3e: success for va %#lx" 2762 " in pmap %p", va, pmap); 2763 return (0); 2764 fail: 2765 atomic_add_long(&pmap_l3e_p_failures, 1); 2766 return (KERN_FAILURE); 2767 } 2768 #endif /* VM_NRESERVLEVEL > 0 */ 2769 2770 int 2771 mmu_radix_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, 2772 vm_prot_t prot, u_int flags, int8_t psind) 2773 { 2774 struct rwlock *lock; 2775 pml3_entry_t *l3e; 2776 pt_entry_t *pte; 2777 pt_entry_t newpte, origpte; 2778 pv_entry_t pv; 2779 vm_paddr_t opa, pa; 2780 vm_page_t mpte, om; 2781 int rv, retrycount; 2782 boolean_t nosleep, invalidate_all, invalidate_page; 2783 2784 va = trunc_page(va); 2785 retrycount = 0; 2786 invalidate_page = invalidate_all = false; 2787 CTR6(KTR_PMAP, "pmap_enter(%p, %#lx, %p, %#x, %#x, %d)", pmap, va, 2788 m, prot, flags, psind); 2789 KASSERT(va <= VM_MAX_KERNEL_ADDRESS, ("pmap_enter: toobig")); 2790 KASSERT((m->oflags & VPO_UNMANAGED) != 0 || va < kmi.clean_sva || 2791 va >= kmi.clean_eva, 2792 ("pmap_enter: managed mapping within the clean submap")); 2793 if ((m->oflags & VPO_UNMANAGED) == 0) 2794 VM_PAGE_OBJECT_BUSY_ASSERT(m); 2795 2796 KASSERT((flags & PMAP_ENTER_RESERVED) == 0, 2797 ("pmap_enter: flags %u has reserved bits set", flags)); 2798 pa = VM_PAGE_TO_PHYS(m); 2799 newpte = (pt_entry_t)(pa | PG_A | PG_V | RPTE_LEAF); 2800 if ((flags & VM_PROT_WRITE) != 0) 2801 newpte |= PG_M; 2802 if ((flags & VM_PROT_READ) != 0) 2803 newpte |= PG_A; 2804 if (prot & VM_PROT_READ) 2805 newpte |= RPTE_EAA_R; 2806 if ((prot & VM_PROT_WRITE) != 0) 2807 newpte |= RPTE_EAA_W; 2808 KASSERT((newpte & (PG_M | PG_RW)) != PG_M, 2809 ("pmap_enter: flags includes VM_PROT_WRITE but prot doesn't")); 2810 2811 if (prot & VM_PROT_EXECUTE) 2812 newpte |= PG_X; 2813 if ((flags & PMAP_ENTER_WIRED) != 0) 2814 newpte |= PG_W; 2815 if (va >= DMAP_MIN_ADDRESS) 2816 newpte |= RPTE_EAA_P; 2817 newpte |= pmap_cache_bits(m->md.mdpg_cache_attrs); 2818 /* 2819 * Set modified bit gratuitously for writeable mappings if 2820 * the page is unmanaged. We do not want to take a fault 2821 * to do the dirty bit accounting for these mappings. 2822 */ 2823 if ((m->oflags & VPO_UNMANAGED) != 0) { 2824 if ((newpte & PG_RW) != 0) 2825 newpte |= PG_M; 2826 } else 2827 newpte |= PG_MANAGED; 2828 2829 lock = NULL; 2830 PMAP_LOCK(pmap); 2831 if (psind == 1) { 2832 /* Assert the required virtual and physical alignment. */ 2833 KASSERT((va & L3_PAGE_MASK) == 0, ("pmap_enter: va unaligned")); 2834 KASSERT(m->psind > 0, ("pmap_enter: m->psind < psind")); 2835 rv = pmap_enter_l3e(pmap, va, newpte | RPTE_LEAF, flags, m, &lock); 2836 goto out; 2837 } 2838 mpte = NULL; 2839 2840 /* 2841 * In the case that a page table page is not 2842 * resident, we are creating it here. 2843 */ 2844 retry: 2845 l3e = pmap_pml3e(pmap, va); 2846 if (l3e != NULL && (be64toh(*l3e) & PG_V) != 0 && ((be64toh(*l3e) & RPTE_LEAF) == 0 || 2847 pmap_demote_l3e_locked(pmap, l3e, va, &lock))) { 2848 pte = pmap_l3e_to_pte(l3e, va); 2849 if (va < VM_MAXUSER_ADDRESS && mpte == NULL) { 2850 mpte = PHYS_TO_VM_PAGE(be64toh(*l3e) & PG_FRAME); 2851 mpte->ref_count++; 2852 } 2853 } else if (va < VM_MAXUSER_ADDRESS) { 2854 /* 2855 * Here if the pte page isn't mapped, or if it has been 2856 * deallocated. 2857 */ 2858 nosleep = (flags & PMAP_ENTER_NOSLEEP) != 0; 2859 mpte = _pmap_allocpte(pmap, pmap_l3e_pindex(va), 2860 nosleep ? NULL : &lock); 2861 if (mpte == NULL && nosleep) { 2862 rv = KERN_RESOURCE_SHORTAGE; 2863 goto out; 2864 } 2865 if (__predict_false(retrycount++ == 6)) 2866 panic("too many retries"); 2867 invalidate_all = true; 2868 goto retry; 2869 } else 2870 panic("pmap_enter: invalid page directory va=%#lx", va); 2871 2872 origpte = be64toh(*pte); 2873 pv = NULL; 2874 2875 /* 2876 * Is the specified virtual address already mapped? 2877 */ 2878 if ((origpte & PG_V) != 0) { 2879 #ifdef INVARIANTS 2880 if (VERBOSE_PMAP || pmap_logging) { 2881 printf("cow fault pmap_enter(%p, %#lx, %p, %#x, %x, %d) --" 2882 " asid=%lu curpid=%d name=%s origpte0x%lx\n", 2883 pmap, va, m, prot, flags, psind, pmap->pm_pid, 2884 curproc->p_pid, curproc->p_comm, origpte); 2885 pmap_pte_walk(pmap->pm_pml1, va); 2886 } 2887 #endif 2888 /* 2889 * Wiring change, just update stats. We don't worry about 2890 * wiring PT pages as they remain resident as long as there 2891 * are valid mappings in them. Hence, if a user page is wired, 2892 * the PT page will be also. 2893 */ 2894 if ((newpte & PG_W) != 0 && (origpte & PG_W) == 0) 2895 pmap->pm_stats.wired_count++; 2896 else if ((newpte & PG_W) == 0 && (origpte & PG_W) != 0) 2897 pmap->pm_stats.wired_count--; 2898 2899 /* 2900 * Remove the extra PT page reference. 2901 */ 2902 if (mpte != NULL) { 2903 mpte->ref_count--; 2904 KASSERT(mpte->ref_count > 0, 2905 ("pmap_enter: missing reference to page table page," 2906 " va: 0x%lx", va)); 2907 } 2908 2909 /* 2910 * Has the physical page changed? 2911 */ 2912 opa = origpte & PG_FRAME; 2913 if (opa == pa) { 2914 /* 2915 * No, might be a protection or wiring change. 2916 */ 2917 if ((origpte & PG_MANAGED) != 0 && 2918 (newpte & PG_RW) != 0) 2919 vm_page_aflag_set(m, PGA_WRITEABLE); 2920 if (((origpte ^ newpte) & ~(PG_M | PG_A)) == 0) { 2921 if ((newpte & (PG_A|PG_M)) != (origpte & (PG_A|PG_M))) { 2922 if (!atomic_cmpset_long(pte, htobe64(origpte), htobe64(newpte))) 2923 goto retry; 2924 if ((newpte & PG_M) != (origpte & PG_M)) 2925 vm_page_dirty(m); 2926 if ((newpte & PG_A) != (origpte & PG_A)) 2927 vm_page_aflag_set(m, PGA_REFERENCED); 2928 ptesync(); 2929 } else 2930 invalidate_all = true; 2931 if (((origpte ^ newpte) & ~(PG_M | PG_A)) == 0) 2932 goto unchanged; 2933 } 2934 goto validate; 2935 } 2936 2937 /* 2938 * The physical page has changed. Temporarily invalidate 2939 * the mapping. This ensures that all threads sharing the 2940 * pmap keep a consistent view of the mapping, which is 2941 * necessary for the correct handling of COW faults. It 2942 * also permits reuse of the old mapping's PV entry, 2943 * avoiding an allocation. 2944 * 2945 * For consistency, handle unmanaged mappings the same way. 2946 */ 2947 origpte = be64toh(pte_load_clear(pte)); 2948 KASSERT((origpte & PG_FRAME) == opa, 2949 ("pmap_enter: unexpected pa update for %#lx", va)); 2950 if ((origpte & PG_MANAGED) != 0) { 2951 om = PHYS_TO_VM_PAGE(opa); 2952 2953 /* 2954 * The pmap lock is sufficient to synchronize with 2955 * concurrent calls to pmap_page_test_mappings() and 2956 * pmap_ts_referenced(). 2957 */ 2958 if ((origpte & (PG_M | PG_RW)) == (PG_M | PG_RW)) 2959 vm_page_dirty(om); 2960 if ((origpte & PG_A) != 0) 2961 vm_page_aflag_set(om, PGA_REFERENCED); 2962 CHANGE_PV_LIST_LOCK_TO_PHYS(&lock, opa); 2963 pv = pmap_pvh_remove(&om->md, pmap, va); 2964 if ((newpte & PG_MANAGED) == 0) 2965 free_pv_entry(pmap, pv); 2966 #ifdef INVARIANTS 2967 else if (origpte & PG_MANAGED) { 2968 if (pv == NULL) { 2969 pmap_page_print_mappings(om); 2970 MPASS(pv != NULL); 2971 } 2972 } 2973 #endif 2974 if ((om->a.flags & PGA_WRITEABLE) != 0 && 2975 TAILQ_EMPTY(&om->md.pv_list) && 2976 ((om->flags & PG_FICTITIOUS) != 0 || 2977 TAILQ_EMPTY(&pa_to_pvh(opa)->pv_list))) 2978 vm_page_aflag_clear(om, PGA_WRITEABLE); 2979 } 2980 if ((origpte & PG_A) != 0) 2981 invalidate_page = true; 2982 origpte = 0; 2983 } else { 2984 if (pmap != kernel_pmap) { 2985 #ifdef INVARIANTS 2986 if (VERBOSE_PMAP || pmap_logging) 2987 printf("pmap_enter(%p, %#lx, %p, %#x, %x, %d) -- asid=%lu curpid=%d name=%s\n", 2988 pmap, va, m, prot, flags, psind, 2989 pmap->pm_pid, curproc->p_pid, 2990 curproc->p_comm); 2991 #endif 2992 } 2993 2994 /* 2995 * Increment the counters. 2996 */ 2997 if ((newpte & PG_W) != 0) 2998 pmap->pm_stats.wired_count++; 2999 pmap_resident_count_inc(pmap, 1); 3000 } 3001 3002 /* 3003 * Enter on the PV list if part of our managed memory. 3004 */ 3005 if ((newpte & PG_MANAGED) != 0) { 3006 if (pv == NULL) { 3007 pv = get_pv_entry(pmap, &lock); 3008 pv->pv_va = va; 3009 } 3010 #ifdef VERBOSE_PV 3011 else 3012 printf("reassigning pv: %p to pmap: %p\n", 3013 pv, pmap); 3014 #endif 3015 CHANGE_PV_LIST_LOCK_TO_PHYS(&lock, pa); 3016 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_link); 3017 m->md.pv_gen++; 3018 if ((newpte & PG_RW) != 0) 3019 vm_page_aflag_set(m, PGA_WRITEABLE); 3020 } 3021 3022 /* 3023 * Update the PTE. 3024 */ 3025 if ((origpte & PG_V) != 0) { 3026 validate: 3027 origpte = be64toh(pte_load_store(pte, htobe64(newpte))); 3028 KASSERT((origpte & PG_FRAME) == pa, 3029 ("pmap_enter: unexpected pa update for %#lx", va)); 3030 if ((newpte & PG_M) == 0 && (origpte & (PG_M | PG_RW)) == 3031 (PG_M | PG_RW)) { 3032 if ((origpte & PG_MANAGED) != 0) 3033 vm_page_dirty(m); 3034 invalidate_page = true; 3035 3036 /* 3037 * Although the PTE may still have PG_RW set, TLB 3038 * invalidation may nonetheless be required because 3039 * the PTE no longer has PG_M set. 3040 */ 3041 } else if ((origpte & PG_X) != 0 || (newpte & PG_X) == 0) { 3042 /* 3043 * Removing capabilities requires invalidation on POWER 3044 */ 3045 invalidate_page = true; 3046 goto unchanged; 3047 } 3048 if ((origpte & PG_A) != 0) 3049 invalidate_page = true; 3050 } else { 3051 pte_store(pte, newpte); 3052 ptesync(); 3053 } 3054 unchanged: 3055 3056 #if VM_NRESERVLEVEL > 0 3057 /* 3058 * If both the page table page and the reservation are fully 3059 * populated, then attempt promotion. 3060 */ 3061 if ((mpte == NULL || mpte->ref_count == NPTEPG) && 3062 mmu_radix_ps_enabled(pmap) && 3063 (m->flags & PG_FICTITIOUS) == 0 && 3064 vm_reserv_level_iffullpop(m) == 0 && 3065 pmap_promote_l3e(pmap, l3e, va, &lock) == 0) 3066 invalidate_all = true; 3067 #endif 3068 if (invalidate_all) 3069 pmap_invalidate_all(pmap); 3070 else if (invalidate_page) 3071 pmap_invalidate_page(pmap, va); 3072 3073 rv = KERN_SUCCESS; 3074 out: 3075 if (lock != NULL) 3076 rw_wunlock(lock); 3077 PMAP_UNLOCK(pmap); 3078 3079 return (rv); 3080 } 3081 3082 /* 3083 * Tries to create a read- and/or execute-only 2MB page mapping. Returns true 3084 * if successful. Returns false if (1) a page table page cannot be allocated 3085 * without sleeping, (2) a mapping already exists at the specified virtual 3086 * address, or (3) a PV entry cannot be allocated without reclaiming another 3087 * PV entry. 3088 */ 3089 static bool 3090 pmap_enter_2mpage(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, 3091 struct rwlock **lockp) 3092 { 3093 pml3_entry_t newpde; 3094 3095 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 3096 newpde = VM_PAGE_TO_PHYS(m) | pmap_cache_bits(m->md.mdpg_cache_attrs) | 3097 RPTE_LEAF | PG_V; 3098 if ((m->oflags & VPO_UNMANAGED) == 0) 3099 newpde |= PG_MANAGED; 3100 if (prot & VM_PROT_EXECUTE) 3101 newpde |= PG_X; 3102 if (prot & VM_PROT_READ) 3103 newpde |= RPTE_EAA_R; 3104 if (va >= DMAP_MIN_ADDRESS) 3105 newpde |= RPTE_EAA_P; 3106 return (pmap_enter_l3e(pmap, va, newpde, PMAP_ENTER_NOSLEEP | 3107 PMAP_ENTER_NOREPLACE | PMAP_ENTER_NORECLAIM, NULL, lockp) == 3108 KERN_SUCCESS); 3109 } 3110 3111 /* 3112 * Tries to create the specified 2MB page mapping. Returns KERN_SUCCESS if 3113 * the mapping was created, and either KERN_FAILURE or KERN_RESOURCE_SHORTAGE 3114 * otherwise. Returns KERN_FAILURE if PMAP_ENTER_NOREPLACE was specified and 3115 * a mapping already exists at the specified virtual address. Returns 3116 * KERN_RESOURCE_SHORTAGE if PMAP_ENTER_NOSLEEP was specified and a page table 3117 * page allocation failed. Returns KERN_RESOURCE_SHORTAGE if 3118 * PMAP_ENTER_NORECLAIM was specified and a PV entry allocation failed. 3119 * 3120 * The parameter "m" is only used when creating a managed, writeable mapping. 3121 */ 3122 static int 3123 pmap_enter_l3e(pmap_t pmap, vm_offset_t va, pml3_entry_t newpde, u_int flags, 3124 vm_page_t m, struct rwlock **lockp) 3125 { 3126 struct spglist free; 3127 pml3_entry_t oldl3e, *l3e; 3128 vm_page_t mt, pdpg; 3129 3130 KASSERT((newpde & (PG_M | PG_RW)) != PG_RW, 3131 ("pmap_enter_pde: newpde is missing PG_M")); 3132 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 3133 3134 if ((pdpg = pmap_allocl3e(pmap, va, (flags & PMAP_ENTER_NOSLEEP) != 0 ? 3135 NULL : lockp)) == NULL) { 3136 CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx" 3137 " in pmap %p", va, pmap); 3138 return (KERN_RESOURCE_SHORTAGE); 3139 } 3140 l3e = (pml3_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pdpg)); 3141 l3e = &l3e[pmap_pml3e_index(va)]; 3142 oldl3e = be64toh(*l3e); 3143 if ((oldl3e & PG_V) != 0) { 3144 KASSERT(pdpg->ref_count > 1, 3145 ("pmap_enter_pde: pdpg's wire count is too low")); 3146 if ((flags & PMAP_ENTER_NOREPLACE) != 0) { 3147 pdpg->ref_count--; 3148 CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx" 3149 " in pmap %p", va, pmap); 3150 return (KERN_FAILURE); 3151 } 3152 /* Break the existing mapping(s). */ 3153 SLIST_INIT(&free); 3154 if ((oldl3e & RPTE_LEAF) != 0) { 3155 /* 3156 * The reference to the PD page that was acquired by 3157 * pmap_allocl3e() ensures that it won't be freed. 3158 * However, if the PDE resulted from a promotion, then 3159 * a reserved PT page could be freed. 3160 */ 3161 (void)pmap_remove_l3e(pmap, l3e, va, &free, lockp); 3162 } else { 3163 if (pmap_remove_ptes(pmap, va, va + L3_PAGE_SIZE, l3e, 3164 &free, lockp)) 3165 pmap_invalidate_all(pmap); 3166 } 3167 vm_page_free_pages_toq(&free, true); 3168 if (va >= VM_MAXUSER_ADDRESS) { 3169 mt = PHYS_TO_VM_PAGE(be64toh(*l3e) & PG_FRAME); 3170 if (pmap_insert_pt_page(pmap, mt)) { 3171 /* 3172 * XXX Currently, this can't happen because 3173 * we do not perform pmap_enter(psind == 1) 3174 * on the kernel pmap. 3175 */ 3176 panic("pmap_enter_pde: trie insert failed"); 3177 } 3178 } else 3179 KASSERT(be64toh(*l3e) == 0, ("pmap_enter_pde: non-zero pde %p", 3180 l3e)); 3181 } 3182 if ((newpde & PG_MANAGED) != 0) { 3183 /* 3184 * Abort this mapping if its PV entry could not be created. 3185 */ 3186 if (!pmap_pv_insert_l3e(pmap, va, newpde, flags, lockp)) { 3187 SLIST_INIT(&free); 3188 if (pmap_unwire_ptp(pmap, va, pdpg, &free)) { 3189 /* 3190 * Although "va" is not mapped, paging- 3191 * structure caches could nonetheless have 3192 * entries that refer to the freed page table 3193 * pages. Invalidate those entries. 3194 */ 3195 pmap_invalidate_page(pmap, va); 3196 vm_page_free_pages_toq(&free, true); 3197 } 3198 CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx" 3199 " in pmap %p", va, pmap); 3200 return (KERN_RESOURCE_SHORTAGE); 3201 } 3202 if ((newpde & PG_RW) != 0) { 3203 for (mt = m; mt < &m[L3_PAGE_SIZE / PAGE_SIZE]; mt++) 3204 vm_page_aflag_set(mt, PGA_WRITEABLE); 3205 } 3206 } 3207 3208 /* 3209 * Increment counters. 3210 */ 3211 if ((newpde & PG_W) != 0) 3212 pmap->pm_stats.wired_count += L3_PAGE_SIZE / PAGE_SIZE; 3213 pmap_resident_count_inc(pmap, L3_PAGE_SIZE / PAGE_SIZE); 3214 3215 /* 3216 * Map the superpage. (This is not a promoted mapping; there will not 3217 * be any lingering 4KB page mappings in the TLB.) 3218 */ 3219 pte_store(l3e, newpde); 3220 3221 atomic_add_long(&pmap_l3e_mappings, 1); 3222 CTR2(KTR_PMAP, "pmap_enter_pde: success for va %#lx" 3223 " in pmap %p", va, pmap); 3224 return (KERN_SUCCESS); 3225 } 3226 3227 void 3228 mmu_radix_enter_object(pmap_t pmap, vm_offset_t start, 3229 vm_offset_t end, vm_page_t m_start, vm_prot_t prot) 3230 { 3231 3232 struct rwlock *lock; 3233 vm_offset_t va; 3234 vm_page_t m, mpte; 3235 vm_pindex_t diff, psize; 3236 bool invalidate; 3237 VM_OBJECT_ASSERT_LOCKED(m_start->object); 3238 3239 CTR6(KTR_PMAP, "%s(%p, %#x, %#x, %p, %#x)", __func__, pmap, start, 3240 end, m_start, prot); 3241 3242 invalidate = false; 3243 psize = atop(end - start); 3244 mpte = NULL; 3245 m = m_start; 3246 lock = NULL; 3247 PMAP_LOCK(pmap); 3248 while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) { 3249 va = start + ptoa(diff); 3250 if ((va & L3_PAGE_MASK) == 0 && va + L3_PAGE_SIZE <= end && 3251 m->psind == 1 && mmu_radix_ps_enabled(pmap) && 3252 pmap_enter_2mpage(pmap, va, m, prot, &lock)) 3253 m = &m[L3_PAGE_SIZE / PAGE_SIZE - 1]; 3254 else 3255 mpte = mmu_radix_enter_quick_locked(pmap, va, m, prot, 3256 mpte, &lock, &invalidate); 3257 m = TAILQ_NEXT(m, listq); 3258 } 3259 ptesync(); 3260 if (lock != NULL) 3261 rw_wunlock(lock); 3262 if (invalidate) 3263 pmap_invalidate_all(pmap); 3264 PMAP_UNLOCK(pmap); 3265 } 3266 3267 static vm_page_t 3268 mmu_radix_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, 3269 vm_prot_t prot, vm_page_t mpte, struct rwlock **lockp, bool *invalidate) 3270 { 3271 struct spglist free; 3272 pt_entry_t *pte; 3273 vm_paddr_t pa; 3274 3275 KASSERT(va < kmi.clean_sva || va >= kmi.clean_eva || 3276 (m->oflags & VPO_UNMANAGED) != 0, 3277 ("mmu_radix_enter_quick_locked: managed mapping within the clean submap")); 3278 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 3279 3280 /* 3281 * In the case that a page table page is not 3282 * resident, we are creating it here. 3283 */ 3284 if (va < VM_MAXUSER_ADDRESS) { 3285 vm_pindex_t ptepindex; 3286 pml3_entry_t *ptepa; 3287 3288 /* 3289 * Calculate pagetable page index 3290 */ 3291 ptepindex = pmap_l3e_pindex(va); 3292 if (mpte && (mpte->pindex == ptepindex)) { 3293 mpte->ref_count++; 3294 } else { 3295 /* 3296 * Get the page directory entry 3297 */ 3298 ptepa = pmap_pml3e(pmap, va); 3299 3300 /* 3301 * If the page table page is mapped, we just increment 3302 * the hold count, and activate it. Otherwise, we 3303 * attempt to allocate a page table page. If this 3304 * attempt fails, we don't retry. Instead, we give up. 3305 */ 3306 if (ptepa && (be64toh(*ptepa) & PG_V) != 0) { 3307 if (be64toh(*ptepa) & RPTE_LEAF) 3308 return (NULL); 3309 mpte = PHYS_TO_VM_PAGE(be64toh(*ptepa) & PG_FRAME); 3310 mpte->ref_count++; 3311 } else { 3312 /* 3313 * Pass NULL instead of the PV list lock 3314 * pointer, because we don't intend to sleep. 3315 */ 3316 mpte = _pmap_allocpte(pmap, ptepindex, NULL); 3317 if (mpte == NULL) 3318 return (mpte); 3319 } 3320 } 3321 pte = (pt_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mpte)); 3322 pte = &pte[pmap_pte_index(va)]; 3323 } else { 3324 mpte = NULL; 3325 pte = pmap_pte(pmap, va); 3326 } 3327 if (be64toh(*pte)) { 3328 if (mpte != NULL) { 3329 mpte->ref_count--; 3330 mpte = NULL; 3331 } 3332 return (mpte); 3333 } 3334 3335 /* 3336 * Enter on the PV list if part of our managed memory. 3337 */ 3338 if ((m->oflags & VPO_UNMANAGED) == 0 && 3339 !pmap_try_insert_pv_entry(pmap, va, m, lockp)) { 3340 if (mpte != NULL) { 3341 SLIST_INIT(&free); 3342 if (pmap_unwire_ptp(pmap, va, mpte, &free)) { 3343 /* 3344 * Although "va" is not mapped, paging- 3345 * structure caches could nonetheless have 3346 * entries that refer to the freed page table 3347 * pages. Invalidate those entries. 3348 */ 3349 *invalidate = true; 3350 vm_page_free_pages_toq(&free, true); 3351 } 3352 mpte = NULL; 3353 } 3354 return (mpte); 3355 } 3356 3357 /* 3358 * Increment counters 3359 */ 3360 pmap_resident_count_inc(pmap, 1); 3361 3362 pa = VM_PAGE_TO_PHYS(m) | pmap_cache_bits(m->md.mdpg_cache_attrs); 3363 if (prot & VM_PROT_EXECUTE) 3364 pa |= PG_X; 3365 else 3366 pa |= RPTE_EAA_R; 3367 if ((m->oflags & VPO_UNMANAGED) == 0) 3368 pa |= PG_MANAGED; 3369 3370 pte_store(pte, pa); 3371 return (mpte); 3372 } 3373 3374 void 3375 mmu_radix_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, 3376 vm_prot_t prot) 3377 { 3378 struct rwlock *lock; 3379 bool invalidate; 3380 3381 lock = NULL; 3382 invalidate = false; 3383 PMAP_LOCK(pmap); 3384 mmu_radix_enter_quick_locked(pmap, va, m, prot, NULL, &lock, 3385 &invalidate); 3386 ptesync(); 3387 if (lock != NULL) 3388 rw_wunlock(lock); 3389 if (invalidate) 3390 pmap_invalidate_all(pmap); 3391 PMAP_UNLOCK(pmap); 3392 } 3393 3394 vm_paddr_t 3395 mmu_radix_extract(pmap_t pmap, vm_offset_t va) 3396 { 3397 pml3_entry_t *l3e; 3398 pt_entry_t *pte; 3399 vm_paddr_t pa; 3400 3401 l3e = pmap_pml3e(pmap, va); 3402 if (__predict_false(l3e == NULL)) 3403 return (0); 3404 if (be64toh(*l3e) & RPTE_LEAF) { 3405 pa = (be64toh(*l3e) & PG_PS_FRAME) | (va & L3_PAGE_MASK); 3406 pa |= (va & L3_PAGE_MASK); 3407 } else { 3408 /* 3409 * Beware of a concurrent promotion that changes the 3410 * PDE at this point! For example, vtopte() must not 3411 * be used to access the PTE because it would use the 3412 * new PDE. It is, however, safe to use the old PDE 3413 * because the page table page is preserved by the 3414 * promotion. 3415 */ 3416 pte = pmap_l3e_to_pte(l3e, va); 3417 if (__predict_false(pte == NULL)) 3418 return (0); 3419 pa = be64toh(*pte); 3420 pa = (pa & PG_FRAME) | (va & PAGE_MASK); 3421 pa |= (va & PAGE_MASK); 3422 } 3423 return (pa); 3424 } 3425 3426 vm_page_t 3427 mmu_radix_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot) 3428 { 3429 pml3_entry_t l3e, *l3ep; 3430 pt_entry_t pte; 3431 vm_paddr_t pa; 3432 vm_page_t m; 3433 3434 pa = 0; 3435 m = NULL; 3436 CTR4(KTR_PMAP, "%s(%p, %#x, %#x)", __func__, pmap, va, prot); 3437 PMAP_LOCK(pmap); 3438 l3ep = pmap_pml3e(pmap, va); 3439 if (l3ep != NULL && (l3e = be64toh(*l3ep))) { 3440 if (l3e & RPTE_LEAF) { 3441 if ((l3e & PG_RW) || (prot & VM_PROT_WRITE) == 0) 3442 m = PHYS_TO_VM_PAGE((l3e & PG_PS_FRAME) | 3443 (va & L3_PAGE_MASK)); 3444 } else { 3445 /* Native endian PTE, do not pass to pmap functions */ 3446 pte = be64toh(*pmap_l3e_to_pte(l3ep, va)); 3447 if ((pte & PG_V) && 3448 ((pte & PG_RW) || (prot & VM_PROT_WRITE) == 0)) 3449 m = PHYS_TO_VM_PAGE(pte & PG_FRAME); 3450 } 3451 if (m != NULL && !vm_page_wire_mapped(m)) 3452 m = NULL; 3453 } 3454 PMAP_UNLOCK(pmap); 3455 return (m); 3456 } 3457 3458 static void 3459 mmu_radix_growkernel(vm_offset_t addr) 3460 { 3461 vm_paddr_t paddr; 3462 vm_page_t nkpg; 3463 pml3_entry_t *l3e; 3464 pml2_entry_t *l2e; 3465 3466 CTR2(KTR_PMAP, "%s(%#x)", __func__, addr); 3467 if (VM_MIN_KERNEL_ADDRESS < addr && 3468 addr < (VM_MIN_KERNEL_ADDRESS + nkpt * L3_PAGE_SIZE)) 3469 return; 3470 3471 addr = roundup2(addr, L3_PAGE_SIZE); 3472 if (addr - 1 >= vm_map_max(kernel_map)) 3473 addr = vm_map_max(kernel_map); 3474 while (kernel_vm_end < addr) { 3475 l2e = pmap_pml2e(kernel_pmap, kernel_vm_end); 3476 if ((be64toh(*l2e) & PG_V) == 0) { 3477 /* We need a new PDP entry */ 3478 nkpg = vm_page_alloc(NULL, kernel_vm_end >> L2_PAGE_SIZE_SHIFT, 3479 VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ | 3480 VM_ALLOC_WIRED | VM_ALLOC_ZERO); 3481 if (nkpg == NULL) 3482 panic("pmap_growkernel: no memory to grow kernel"); 3483 if ((nkpg->flags & PG_ZERO) == 0) 3484 mmu_radix_zero_page(nkpg); 3485 paddr = VM_PAGE_TO_PHYS(nkpg); 3486 pde_store(l2e, paddr); 3487 continue; /* try again */ 3488 } 3489 l3e = pmap_l2e_to_l3e(l2e, kernel_vm_end); 3490 if ((be64toh(*l3e) & PG_V) != 0) { 3491 kernel_vm_end = (kernel_vm_end + L3_PAGE_SIZE) & ~L3_PAGE_MASK; 3492 if (kernel_vm_end - 1 >= vm_map_max(kernel_map)) { 3493 kernel_vm_end = vm_map_max(kernel_map); 3494 break; 3495 } 3496 continue; 3497 } 3498 3499 nkpg = vm_page_alloc(NULL, pmap_l3e_pindex(kernel_vm_end), 3500 VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | 3501 VM_ALLOC_ZERO); 3502 if (nkpg == NULL) 3503 panic("pmap_growkernel: no memory to grow kernel"); 3504 if ((nkpg->flags & PG_ZERO) == 0) 3505 mmu_radix_zero_page(nkpg); 3506 paddr = VM_PAGE_TO_PHYS(nkpg); 3507 pde_store(l3e, paddr); 3508 3509 kernel_vm_end = (kernel_vm_end + L3_PAGE_SIZE) & ~L3_PAGE_MASK; 3510 if (kernel_vm_end - 1 >= vm_map_max(kernel_map)) { 3511 kernel_vm_end = vm_map_max(kernel_map); 3512 break; 3513 } 3514 } 3515 ptesync(); 3516 } 3517 3518 static MALLOC_DEFINE(M_RADIX_PGD, "radix_pgd", "radix page table root directory"); 3519 static uma_zone_t zone_radix_pgd; 3520 3521 static int 3522 radix_pgd_import(void *arg __unused, void **store, int count, int domain __unused, 3523 int flags) 3524 { 3525 3526 for (int i = 0; i < count; i++) { 3527 vm_page_t m = vm_page_alloc_contig(NULL, 0, 3528 VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | 3529 VM_ALLOC_ZERO | VM_ALLOC_WAITOK, RADIX_PGD_SIZE/PAGE_SIZE, 3530 0, (vm_paddr_t)-1, RADIX_PGD_SIZE, L1_PAGE_SIZE, 3531 VM_MEMATTR_DEFAULT); 3532 /* XXX zero on alloc here so we don't have to later */ 3533 store[i] = (void *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)); 3534 } 3535 return (count); 3536 } 3537 3538 static void 3539 radix_pgd_release(void *arg __unused, void **store, int count) 3540 { 3541 vm_page_t m; 3542 struct spglist free; 3543 int page_count; 3544 3545 SLIST_INIT(&free); 3546 page_count = RADIX_PGD_SIZE/PAGE_SIZE; 3547 3548 for (int i = 0; i < count; i++) { 3549 /* 3550 * XXX selectively remove dmap and KVA entries so we don't 3551 * need to bzero 3552 */ 3553 m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)store[i])); 3554 for (int j = page_count-1; j >= 0; j--) { 3555 vm_page_unwire_noq(&m[j]); 3556 SLIST_INSERT_HEAD(&free, &m[j], plinks.s.ss); 3557 } 3558 vm_page_free_pages_toq(&free, false); 3559 } 3560 } 3561 3562 static void 3563 mmu_radix_init() 3564 { 3565 vm_page_t mpte; 3566 vm_size_t s; 3567 int error, i, pv_npg; 3568 3569 /* XXX is this really needed for POWER? */ 3570 /* L1TF, reserve page @0 unconditionally */ 3571 vm_page_blacklist_add(0, bootverbose); 3572 3573 zone_radix_pgd = uma_zcache_create("radix_pgd_cache", 3574 RADIX_PGD_SIZE, NULL, NULL, 3575 #ifdef INVARIANTS 3576 trash_init, trash_fini, 3577 #else 3578 NULL, NULL, 3579 #endif 3580 radix_pgd_import, radix_pgd_release, 3581 NULL, UMA_ZONE_NOBUCKET); 3582 3583 /* 3584 * Initialize the vm page array entries for the kernel pmap's 3585 * page table pages. 3586 */ 3587 PMAP_LOCK(kernel_pmap); 3588 for (i = 0; i < nkpt; i++) { 3589 mpte = PHYS_TO_VM_PAGE(KPTphys + (i << PAGE_SHIFT)); 3590 KASSERT(mpte >= vm_page_array && 3591 mpte < &vm_page_array[vm_page_array_size], 3592 ("pmap_init: page table page is out of range size: %lu", 3593 vm_page_array_size)); 3594 mpte->pindex = pmap_l3e_pindex(VM_MIN_KERNEL_ADDRESS) + i; 3595 mpte->phys_addr = KPTphys + (i << PAGE_SHIFT); 3596 MPASS(PHYS_TO_VM_PAGE(mpte->phys_addr) == mpte); 3597 //pmap_insert_pt_page(kernel_pmap, mpte); 3598 mpte->ref_count = 1; 3599 } 3600 PMAP_UNLOCK(kernel_pmap); 3601 vm_wire_add(nkpt); 3602 3603 CTR1(KTR_PMAP, "%s()", __func__); 3604 TAILQ_INIT(&pv_dummy.pv_list); 3605 3606 /* 3607 * Are large page mappings enabled? 3608 */ 3609 TUNABLE_INT_FETCH("vm.pmap.superpages_enabled", &superpages_enabled); 3610 if (superpages_enabled) { 3611 KASSERT(MAXPAGESIZES > 1 && pagesizes[1] == 0, 3612 ("pmap_init: can't assign to pagesizes[1]")); 3613 pagesizes[1] = L3_PAGE_SIZE; 3614 } 3615 3616 /* 3617 * Initialize the pv chunk list mutex. 3618 */ 3619 mtx_init(&pv_chunks_mutex, "pmap pv chunk list", NULL, MTX_DEF); 3620 3621 /* 3622 * Initialize the pool of pv list locks. 3623 */ 3624 for (i = 0; i < NPV_LIST_LOCKS; i++) 3625 rw_init(&pv_list_locks[i], "pmap pv list"); 3626 3627 /* 3628 * Calculate the size of the pv head table for superpages. 3629 */ 3630 pv_npg = howmany(vm_phys_segs[vm_phys_nsegs - 1].end, L3_PAGE_SIZE); 3631 3632 /* 3633 * Allocate memory for the pv head table for superpages. 3634 */ 3635 s = (vm_size_t)(pv_npg * sizeof(struct md_page)); 3636 s = round_page(s); 3637 pv_table = (struct md_page *)kmem_malloc(s, M_WAITOK | M_ZERO); 3638 for (i = 0; i < pv_npg; i++) 3639 TAILQ_INIT(&pv_table[i].pv_list); 3640 TAILQ_INIT(&pv_dummy.pv_list); 3641 3642 pmap_initialized = 1; 3643 mtx_init(&qframe_mtx, "qfrmlk", NULL, MTX_SPIN); 3644 error = vmem_alloc(kernel_arena, PAGE_SIZE, M_BESTFIT | M_WAITOK, 3645 (vmem_addr_t *)&qframe); 3646 3647 if (error != 0) 3648 panic("qframe allocation failed"); 3649 asid_arena = vmem_create("ASID", isa3_base_pid + 1, (1<<isa3_pid_bits), 3650 1, 1, M_WAITOK); 3651 } 3652 3653 static boolean_t 3654 pmap_page_test_mappings(vm_page_t m, boolean_t accessed, boolean_t modified) 3655 { 3656 struct rwlock *lock; 3657 pv_entry_t pv; 3658 struct md_page *pvh; 3659 pt_entry_t *pte, mask; 3660 pmap_t pmap; 3661 int md_gen, pvh_gen; 3662 boolean_t rv; 3663 3664 rv = FALSE; 3665 lock = VM_PAGE_TO_PV_LIST_LOCK(m); 3666 rw_rlock(lock); 3667 restart: 3668 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) { 3669 pmap = PV_PMAP(pv); 3670 if (!PMAP_TRYLOCK(pmap)) { 3671 md_gen = m->md.pv_gen; 3672 rw_runlock(lock); 3673 PMAP_LOCK(pmap); 3674 rw_rlock(lock); 3675 if (md_gen != m->md.pv_gen) { 3676 PMAP_UNLOCK(pmap); 3677 goto restart; 3678 } 3679 } 3680 pte = pmap_pte(pmap, pv->pv_va); 3681 mask = 0; 3682 if (modified) 3683 mask |= PG_RW | PG_M; 3684 if (accessed) 3685 mask |= PG_V | PG_A; 3686 rv = (be64toh(*pte) & mask) == mask; 3687 PMAP_UNLOCK(pmap); 3688 if (rv) 3689 goto out; 3690 } 3691 if ((m->flags & PG_FICTITIOUS) == 0) { 3692 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); 3693 TAILQ_FOREACH(pv, &pvh->pv_list, pv_link) { 3694 pmap = PV_PMAP(pv); 3695 if (!PMAP_TRYLOCK(pmap)) { 3696 md_gen = m->md.pv_gen; 3697 pvh_gen = pvh->pv_gen; 3698 rw_runlock(lock); 3699 PMAP_LOCK(pmap); 3700 rw_rlock(lock); 3701 if (md_gen != m->md.pv_gen || 3702 pvh_gen != pvh->pv_gen) { 3703 PMAP_UNLOCK(pmap); 3704 goto restart; 3705 } 3706 } 3707 pte = pmap_pml3e(pmap, pv->pv_va); 3708 mask = 0; 3709 if (modified) 3710 mask |= PG_RW | PG_M; 3711 if (accessed) 3712 mask |= PG_V | PG_A; 3713 rv = (be64toh(*pte) & mask) == mask; 3714 PMAP_UNLOCK(pmap); 3715 if (rv) 3716 goto out; 3717 } 3718 } 3719 out: 3720 rw_runlock(lock); 3721 return (rv); 3722 } 3723 3724 /* 3725 * pmap_is_modified: 3726 * 3727 * Return whether or not the specified physical page was modified 3728 * in any physical maps. 3729 */ 3730 boolean_t 3731 mmu_radix_is_modified(vm_page_t m) 3732 { 3733 3734 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 3735 ("pmap_is_modified: page %p is not managed", m)); 3736 3737 CTR2(KTR_PMAP, "%s(%p)", __func__, m); 3738 /* 3739 * If the page is not busied then this check is racy. 3740 */ 3741 if (!pmap_page_is_write_mapped(m)) 3742 return (FALSE); 3743 return (pmap_page_test_mappings(m, FALSE, TRUE)); 3744 } 3745 3746 boolean_t 3747 mmu_radix_is_prefaultable(pmap_t pmap, vm_offset_t addr) 3748 { 3749 pml3_entry_t *l3e; 3750 pt_entry_t *pte; 3751 boolean_t rv; 3752 3753 CTR3(KTR_PMAP, "%s(%p, %#x)", __func__, pmap, addr); 3754 rv = FALSE; 3755 PMAP_LOCK(pmap); 3756 l3e = pmap_pml3e(pmap, addr); 3757 if (l3e != NULL && (be64toh(*l3e) & (RPTE_LEAF | PG_V)) == PG_V) { 3758 pte = pmap_l3e_to_pte(l3e, addr); 3759 rv = (be64toh(*pte) & PG_V) == 0; 3760 } 3761 PMAP_UNLOCK(pmap); 3762 return (rv); 3763 } 3764 3765 boolean_t 3766 mmu_radix_is_referenced(vm_page_t m) 3767 { 3768 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 3769 ("pmap_is_referenced: page %p is not managed", m)); 3770 CTR2(KTR_PMAP, "%s(%p)", __func__, m); 3771 return (pmap_page_test_mappings(m, TRUE, FALSE)); 3772 } 3773 3774 /* 3775 * pmap_ts_referenced: 3776 * 3777 * Return a count of reference bits for a page, clearing those bits. 3778 * It is not necessary for every reference bit to be cleared, but it 3779 * is necessary that 0 only be returned when there are truly no 3780 * reference bits set. 3781 * 3782 * As an optimization, update the page's dirty field if a modified bit is 3783 * found while counting reference bits. This opportunistic update can be 3784 * performed at low cost and can eliminate the need for some future calls 3785 * to pmap_is_modified(). However, since this function stops after 3786 * finding PMAP_TS_REFERENCED_MAX reference bits, it may not detect some 3787 * dirty pages. Those dirty pages will only be detected by a future call 3788 * to pmap_is_modified(). 3789 * 3790 * A DI block is not needed within this function, because 3791 * invalidations are performed before the PV list lock is 3792 * released. 3793 */ 3794 boolean_t 3795 mmu_radix_ts_referenced(vm_page_t m) 3796 { 3797 struct md_page *pvh; 3798 pv_entry_t pv, pvf; 3799 pmap_t pmap; 3800 struct rwlock *lock; 3801 pml3_entry_t oldl3e, *l3e; 3802 pt_entry_t *pte; 3803 vm_paddr_t pa; 3804 int cleared, md_gen, not_cleared, pvh_gen; 3805 struct spglist free; 3806 3807 CTR2(KTR_PMAP, "%s(%p)", __func__, m); 3808 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 3809 ("pmap_ts_referenced: page %p is not managed", m)); 3810 SLIST_INIT(&free); 3811 cleared = 0; 3812 pa = VM_PAGE_TO_PHYS(m); 3813 lock = PHYS_TO_PV_LIST_LOCK(pa); 3814 pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy : pa_to_pvh(pa); 3815 rw_wlock(lock); 3816 retry: 3817 not_cleared = 0; 3818 if ((pvf = TAILQ_FIRST(&pvh->pv_list)) == NULL) 3819 goto small_mappings; 3820 pv = pvf; 3821 do { 3822 if (pvf == NULL) 3823 pvf = pv; 3824 pmap = PV_PMAP(pv); 3825 if (!PMAP_TRYLOCK(pmap)) { 3826 pvh_gen = pvh->pv_gen; 3827 rw_wunlock(lock); 3828 PMAP_LOCK(pmap); 3829 rw_wlock(lock); 3830 if (pvh_gen != pvh->pv_gen) { 3831 PMAP_UNLOCK(pmap); 3832 goto retry; 3833 } 3834 } 3835 l3e = pmap_pml3e(pmap, pv->pv_va); 3836 oldl3e = be64toh(*l3e); 3837 if ((oldl3e & (PG_M | PG_RW)) == (PG_M | PG_RW)) { 3838 /* 3839 * Although "oldpde" is mapping a 2MB page, because 3840 * this function is called at a 4KB page granularity, 3841 * we only update the 4KB page under test. 3842 */ 3843 vm_page_dirty(m); 3844 } 3845 if ((oldl3e & PG_A) != 0) { 3846 /* 3847 * Since this reference bit is shared by 512 4KB 3848 * pages, it should not be cleared every time it is 3849 * tested. Apply a simple "hash" function on the 3850 * physical page number, the virtual superpage number, 3851 * and the pmap address to select one 4KB page out of 3852 * the 512 on which testing the reference bit will 3853 * result in clearing that reference bit. This 3854 * function is designed to avoid the selection of the 3855 * same 4KB page for every 2MB page mapping. 3856 * 3857 * On demotion, a mapping that hasn't been referenced 3858 * is simply destroyed. To avoid the possibility of a 3859 * subsequent page fault on a demoted wired mapping, 3860 * always leave its reference bit set. Moreover, 3861 * since the superpage is wired, the current state of 3862 * its reference bit won't affect page replacement. 3863 */ 3864 if ((((pa >> PAGE_SHIFT) ^ (pv->pv_va >> L3_PAGE_SIZE_SHIFT) ^ 3865 (uintptr_t)pmap) & (NPTEPG - 1)) == 0 && 3866 (oldl3e & PG_W) == 0) { 3867 atomic_clear_long(l3e, htobe64(PG_A)); 3868 pmap_invalidate_page(pmap, pv->pv_va); 3869 cleared++; 3870 KASSERT(lock == VM_PAGE_TO_PV_LIST_LOCK(m), 3871 ("inconsistent pv lock %p %p for page %p", 3872 lock, VM_PAGE_TO_PV_LIST_LOCK(m), m)); 3873 } else 3874 not_cleared++; 3875 } 3876 PMAP_UNLOCK(pmap); 3877 /* Rotate the PV list if it has more than one entry. */ 3878 if (pv != NULL && TAILQ_NEXT(pv, pv_link) != NULL) { 3879 TAILQ_REMOVE(&pvh->pv_list, pv, pv_link); 3880 TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_link); 3881 pvh->pv_gen++; 3882 } 3883 if (cleared + not_cleared >= PMAP_TS_REFERENCED_MAX) 3884 goto out; 3885 } while ((pv = TAILQ_FIRST(&pvh->pv_list)) != pvf); 3886 small_mappings: 3887 if ((pvf = TAILQ_FIRST(&m->md.pv_list)) == NULL) 3888 goto out; 3889 pv = pvf; 3890 do { 3891 if (pvf == NULL) 3892 pvf = pv; 3893 pmap = PV_PMAP(pv); 3894 if (!PMAP_TRYLOCK(pmap)) { 3895 pvh_gen = pvh->pv_gen; 3896 md_gen = m->md.pv_gen; 3897 rw_wunlock(lock); 3898 PMAP_LOCK(pmap); 3899 rw_wlock(lock); 3900 if (pvh_gen != pvh->pv_gen || md_gen != m->md.pv_gen) { 3901 PMAP_UNLOCK(pmap); 3902 goto retry; 3903 } 3904 } 3905 l3e = pmap_pml3e(pmap, pv->pv_va); 3906 KASSERT((be64toh(*l3e) & RPTE_LEAF) == 0, 3907 ("pmap_ts_referenced: found a 2mpage in page %p's pv list", 3908 m)); 3909 pte = pmap_l3e_to_pte(l3e, pv->pv_va); 3910 if ((be64toh(*pte) & (PG_M | PG_RW)) == (PG_M | PG_RW)) 3911 vm_page_dirty(m); 3912 if ((be64toh(*pte) & PG_A) != 0) { 3913 atomic_clear_long(pte, htobe64(PG_A)); 3914 pmap_invalidate_page(pmap, pv->pv_va); 3915 cleared++; 3916 } 3917 PMAP_UNLOCK(pmap); 3918 /* Rotate the PV list if it has more than one entry. */ 3919 if (pv != NULL && TAILQ_NEXT(pv, pv_link) != NULL) { 3920 TAILQ_REMOVE(&m->md.pv_list, pv, pv_link); 3921 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_link); 3922 m->md.pv_gen++; 3923 } 3924 } while ((pv = TAILQ_FIRST(&m->md.pv_list)) != pvf && cleared + 3925 not_cleared < PMAP_TS_REFERENCED_MAX); 3926 out: 3927 rw_wunlock(lock); 3928 vm_page_free_pages_toq(&free, true); 3929 return (cleared + not_cleared); 3930 } 3931 3932 static vm_offset_t 3933 mmu_radix_map(vm_offset_t *virt __unused, vm_paddr_t start, 3934 vm_paddr_t end, int prot __unused) 3935 { 3936 3937 CTR5(KTR_PMAP, "%s(%p, %#x, %#x, %#x)", __func__, virt, start, end, 3938 prot); 3939 return (PHYS_TO_DMAP(start)); 3940 } 3941 3942 void 3943 mmu_radix_object_init_pt(pmap_t pmap, vm_offset_t addr, 3944 vm_object_t object, vm_pindex_t pindex, vm_size_t size) 3945 { 3946 pml3_entry_t *l3e; 3947 vm_paddr_t pa, ptepa; 3948 vm_page_t p, pdpg; 3949 vm_memattr_t ma; 3950 3951 CTR6(KTR_PMAP, "%s(%p, %#x, %p, %u, %#x)", __func__, pmap, addr, 3952 object, pindex, size); 3953 VM_OBJECT_ASSERT_WLOCKED(object); 3954 KASSERT(object->type == OBJT_DEVICE || object->type == OBJT_SG, 3955 ("pmap_object_init_pt: non-device object")); 3956 /* NB: size can be logically ored with addr here */ 3957 if ((addr & L3_PAGE_MASK) == 0 && (size & L3_PAGE_MASK) == 0) { 3958 if (!mmu_radix_ps_enabled(pmap)) 3959 return; 3960 if (!vm_object_populate(object, pindex, pindex + atop(size))) 3961 return; 3962 p = vm_page_lookup(object, pindex); 3963 KASSERT(p->valid == VM_PAGE_BITS_ALL, 3964 ("pmap_object_init_pt: invalid page %p", p)); 3965 ma = p->md.mdpg_cache_attrs; 3966 3967 /* 3968 * Abort the mapping if the first page is not physically 3969 * aligned to a 2MB page boundary. 3970 */ 3971 ptepa = VM_PAGE_TO_PHYS(p); 3972 if (ptepa & L3_PAGE_MASK) 3973 return; 3974 3975 /* 3976 * Skip the first page. Abort the mapping if the rest of 3977 * the pages are not physically contiguous or have differing 3978 * memory attributes. 3979 */ 3980 p = TAILQ_NEXT(p, listq); 3981 for (pa = ptepa + PAGE_SIZE; pa < ptepa + size; 3982 pa += PAGE_SIZE) { 3983 KASSERT(p->valid == VM_PAGE_BITS_ALL, 3984 ("pmap_object_init_pt: invalid page %p", p)); 3985 if (pa != VM_PAGE_TO_PHYS(p) || 3986 ma != p->md.mdpg_cache_attrs) 3987 return; 3988 p = TAILQ_NEXT(p, listq); 3989 } 3990 3991 PMAP_LOCK(pmap); 3992 for (pa = ptepa | pmap_cache_bits(ma); 3993 pa < ptepa + size; pa += L3_PAGE_SIZE) { 3994 pdpg = pmap_allocl3e(pmap, addr, NULL); 3995 if (pdpg == NULL) { 3996 /* 3997 * The creation of mappings below is only an 3998 * optimization. If a page directory page 3999 * cannot be allocated without blocking, 4000 * continue on to the next mapping rather than 4001 * blocking. 4002 */ 4003 addr += L3_PAGE_SIZE; 4004 continue; 4005 } 4006 l3e = (pml3_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pdpg)); 4007 l3e = &l3e[pmap_pml3e_index(addr)]; 4008 if ((be64toh(*l3e) & PG_V) == 0) { 4009 pa |= PG_M | PG_A | PG_RW; 4010 pte_store(l3e, pa); 4011 pmap_resident_count_inc(pmap, L3_PAGE_SIZE / PAGE_SIZE); 4012 atomic_add_long(&pmap_l3e_mappings, 1); 4013 } else { 4014 /* Continue on if the PDE is already valid. */ 4015 pdpg->ref_count--; 4016 KASSERT(pdpg->ref_count > 0, 4017 ("pmap_object_init_pt: missing reference " 4018 "to page directory page, va: 0x%lx", addr)); 4019 } 4020 addr += L3_PAGE_SIZE; 4021 } 4022 ptesync(); 4023 PMAP_UNLOCK(pmap); 4024 } 4025 } 4026 4027 boolean_t 4028 mmu_radix_page_exists_quick(pmap_t pmap, vm_page_t m) 4029 { 4030 struct md_page *pvh; 4031 struct rwlock *lock; 4032 pv_entry_t pv; 4033 int loops = 0; 4034 boolean_t rv; 4035 4036 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 4037 ("pmap_page_exists_quick: page %p is not managed", m)); 4038 CTR3(KTR_PMAP, "%s(%p, %p)", __func__, pmap, m); 4039 rv = FALSE; 4040 lock = VM_PAGE_TO_PV_LIST_LOCK(m); 4041 rw_rlock(lock); 4042 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) { 4043 if (PV_PMAP(pv) == pmap) { 4044 rv = TRUE; 4045 break; 4046 } 4047 loops++; 4048 if (loops >= 16) 4049 break; 4050 } 4051 if (!rv && loops < 16 && (m->flags & PG_FICTITIOUS) == 0) { 4052 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); 4053 TAILQ_FOREACH(pv, &pvh->pv_list, pv_link) { 4054 if (PV_PMAP(pv) == pmap) { 4055 rv = TRUE; 4056 break; 4057 } 4058 loops++; 4059 if (loops >= 16) 4060 break; 4061 } 4062 } 4063 rw_runlock(lock); 4064 return (rv); 4065 } 4066 4067 void 4068 mmu_radix_page_init(vm_page_t m) 4069 { 4070 4071 CTR2(KTR_PMAP, "%s(%p)", __func__, m); 4072 TAILQ_INIT(&m->md.pv_list); 4073 m->md.mdpg_cache_attrs = VM_MEMATTR_DEFAULT; 4074 } 4075 4076 int 4077 mmu_radix_page_wired_mappings(vm_page_t m) 4078 { 4079 struct rwlock *lock; 4080 struct md_page *pvh; 4081 pmap_t pmap; 4082 pt_entry_t *pte; 4083 pv_entry_t pv; 4084 int count, md_gen, pvh_gen; 4085 4086 if ((m->oflags & VPO_UNMANAGED) != 0) 4087 return (0); 4088 CTR2(KTR_PMAP, "%s(%p)", __func__, m); 4089 lock = VM_PAGE_TO_PV_LIST_LOCK(m); 4090 rw_rlock(lock); 4091 restart: 4092 count = 0; 4093 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) { 4094 pmap = PV_PMAP(pv); 4095 if (!PMAP_TRYLOCK(pmap)) { 4096 md_gen = m->md.pv_gen; 4097 rw_runlock(lock); 4098 PMAP_LOCK(pmap); 4099 rw_rlock(lock); 4100 if (md_gen != m->md.pv_gen) { 4101 PMAP_UNLOCK(pmap); 4102 goto restart; 4103 } 4104 } 4105 pte = pmap_pte(pmap, pv->pv_va); 4106 if ((be64toh(*pte) & PG_W) != 0) 4107 count++; 4108 PMAP_UNLOCK(pmap); 4109 } 4110 if ((m->flags & PG_FICTITIOUS) == 0) { 4111 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); 4112 TAILQ_FOREACH(pv, &pvh->pv_list, pv_link) { 4113 pmap = PV_PMAP(pv); 4114 if (!PMAP_TRYLOCK(pmap)) { 4115 md_gen = m->md.pv_gen; 4116 pvh_gen = pvh->pv_gen; 4117 rw_runlock(lock); 4118 PMAP_LOCK(pmap); 4119 rw_rlock(lock); 4120 if (md_gen != m->md.pv_gen || 4121 pvh_gen != pvh->pv_gen) { 4122 PMAP_UNLOCK(pmap); 4123 goto restart; 4124 } 4125 } 4126 pte = pmap_pml3e(pmap, pv->pv_va); 4127 if ((be64toh(*pte) & PG_W) != 0) 4128 count++; 4129 PMAP_UNLOCK(pmap); 4130 } 4131 } 4132 rw_runlock(lock); 4133 return (count); 4134 } 4135 4136 static void 4137 mmu_radix_update_proctab(int pid, pml1_entry_t l1pa) 4138 { 4139 isa3_proctab[pid].proctab0 = htobe64(RTS_SIZE | l1pa | RADIX_PGD_INDEX_SHIFT); 4140 } 4141 4142 int 4143 mmu_radix_pinit(pmap_t pmap) 4144 { 4145 vmem_addr_t pid; 4146 vm_paddr_t l1pa; 4147 4148 CTR2(KTR_PMAP, "%s(%p)", __func__, pmap); 4149 4150 /* 4151 * allocate the page directory page 4152 */ 4153 pmap->pm_pml1 = uma_zalloc(zone_radix_pgd, M_WAITOK); 4154 4155 for (int j = 0; j < RADIX_PGD_SIZE_SHIFT; j++) 4156 pagezero((vm_offset_t)pmap->pm_pml1 + j * PAGE_SIZE); 4157 pmap->pm_radix.rt_root = 0; 4158 TAILQ_INIT(&pmap->pm_pvchunk); 4159 bzero(&pmap->pm_stats, sizeof pmap->pm_stats); 4160 pmap->pm_flags = PMAP_PDE_SUPERPAGE; 4161 vmem_alloc(asid_arena, 1, M_FIRSTFIT|M_WAITOK, &pid); 4162 4163 pmap->pm_pid = pid; 4164 l1pa = DMAP_TO_PHYS((vm_offset_t)pmap->pm_pml1); 4165 mmu_radix_update_proctab(pid, l1pa); 4166 __asm __volatile("ptesync;isync" : : : "memory"); 4167 4168 return (1); 4169 } 4170 4171 /* 4172 * This routine is called if the desired page table page does not exist. 4173 * 4174 * If page table page allocation fails, this routine may sleep before 4175 * returning NULL. It sleeps only if a lock pointer was given. 4176 * 4177 * Note: If a page allocation fails at page table level two or three, 4178 * one or two pages may be held during the wait, only to be released 4179 * afterwards. This conservative approach is easily argued to avoid 4180 * race conditions. 4181 */ 4182 static vm_page_t 4183 _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, struct rwlock **lockp) 4184 { 4185 vm_page_t m, pdppg, pdpg; 4186 4187 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 4188 4189 /* 4190 * Allocate a page table page. 4191 */ 4192 if ((m = vm_page_alloc(NULL, ptepindex, VM_ALLOC_NOOBJ | 4193 VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) { 4194 if (lockp != NULL) { 4195 RELEASE_PV_LIST_LOCK(lockp); 4196 PMAP_UNLOCK(pmap); 4197 vm_wait(NULL); 4198 PMAP_LOCK(pmap); 4199 } 4200 /* 4201 * Indicate the need to retry. While waiting, the page table 4202 * page may have been allocated. 4203 */ 4204 return (NULL); 4205 } 4206 if ((m->flags & PG_ZERO) == 0) 4207 mmu_radix_zero_page(m); 4208 4209 /* 4210 * Map the pagetable page into the process address space, if 4211 * it isn't already there. 4212 */ 4213 4214 if (ptepindex >= (NUPDE + NUPDPE)) { 4215 pml1_entry_t *l1e; 4216 vm_pindex_t pml1index; 4217 4218 /* Wire up a new PDPE page */ 4219 pml1index = ptepindex - (NUPDE + NUPDPE); 4220 l1e = &pmap->pm_pml1[pml1index]; 4221 pde_store(l1e, VM_PAGE_TO_PHYS(m)); 4222 4223 } else if (ptepindex >= NUPDE) { 4224 vm_pindex_t pml1index; 4225 vm_pindex_t pdpindex; 4226 pml1_entry_t *l1e; 4227 pml2_entry_t *l2e; 4228 4229 /* Wire up a new l2e page */ 4230 pdpindex = ptepindex - NUPDE; 4231 pml1index = pdpindex >> RPTE_SHIFT; 4232 4233 l1e = &pmap->pm_pml1[pml1index]; 4234 if ((be64toh(*l1e) & PG_V) == 0) { 4235 /* Have to allocate a new pdp, recurse */ 4236 if (_pmap_allocpte(pmap, NUPDE + NUPDPE + pml1index, 4237 lockp) == NULL) { 4238 vm_page_unwire_noq(m); 4239 vm_page_free_zero(m); 4240 return (NULL); 4241 } 4242 } else { 4243 /* Add reference to l2e page */ 4244 pdppg = PHYS_TO_VM_PAGE(be64toh(*l1e) & PG_FRAME); 4245 pdppg->ref_count++; 4246 } 4247 l2e = (pml2_entry_t *)PHYS_TO_DMAP(be64toh(*l1e) & PG_FRAME); 4248 4249 /* Now find the pdp page */ 4250 l2e = &l2e[pdpindex & RPTE_MASK]; 4251 pde_store(l2e, VM_PAGE_TO_PHYS(m)); 4252 4253 } else { 4254 vm_pindex_t pml1index; 4255 vm_pindex_t pdpindex; 4256 pml1_entry_t *l1e; 4257 pml2_entry_t *l2e; 4258 pml3_entry_t *l3e; 4259 4260 /* Wire up a new PTE page */ 4261 pdpindex = ptepindex >> RPTE_SHIFT; 4262 pml1index = pdpindex >> RPTE_SHIFT; 4263 4264 /* First, find the pdp and check that its valid. */ 4265 l1e = &pmap->pm_pml1[pml1index]; 4266 if ((be64toh(*l1e) & PG_V) == 0) { 4267 /* Have to allocate a new pd, recurse */ 4268 if (_pmap_allocpte(pmap, NUPDE + pdpindex, 4269 lockp) == NULL) { 4270 vm_page_unwire_noq(m); 4271 vm_page_free_zero(m); 4272 return (NULL); 4273 } 4274 l2e = (pml2_entry_t *)PHYS_TO_DMAP(be64toh(*l1e) & PG_FRAME); 4275 l2e = &l2e[pdpindex & RPTE_MASK]; 4276 } else { 4277 l2e = (pml2_entry_t *)PHYS_TO_DMAP(be64toh(*l1e) & PG_FRAME); 4278 l2e = &l2e[pdpindex & RPTE_MASK]; 4279 if ((be64toh(*l2e) & PG_V) == 0) { 4280 /* Have to allocate a new pd, recurse */ 4281 if (_pmap_allocpte(pmap, NUPDE + pdpindex, 4282 lockp) == NULL) { 4283 vm_page_unwire_noq(m); 4284 vm_page_free_zero(m); 4285 return (NULL); 4286 } 4287 } else { 4288 /* Add reference to the pd page */ 4289 pdpg = PHYS_TO_VM_PAGE(be64toh(*l2e) & PG_FRAME); 4290 pdpg->ref_count++; 4291 } 4292 } 4293 l3e = (pml3_entry_t *)PHYS_TO_DMAP(be64toh(*l2e) & PG_FRAME); 4294 4295 /* Now we know where the page directory page is */ 4296 l3e = &l3e[ptepindex & RPTE_MASK]; 4297 pde_store(l3e, VM_PAGE_TO_PHYS(m)); 4298 } 4299 4300 pmap_resident_count_inc(pmap, 1); 4301 return (m); 4302 } 4303 static vm_page_t 4304 pmap_allocl3e(pmap_t pmap, vm_offset_t va, struct rwlock **lockp) 4305 { 4306 vm_pindex_t pdpindex, ptepindex; 4307 pml2_entry_t *pdpe; 4308 vm_page_t pdpg; 4309 4310 retry: 4311 pdpe = pmap_pml2e(pmap, va); 4312 if (pdpe != NULL && (be64toh(*pdpe) & PG_V) != 0) { 4313 /* Add a reference to the pd page. */ 4314 pdpg = PHYS_TO_VM_PAGE(be64toh(*pdpe) & PG_FRAME); 4315 pdpg->ref_count++; 4316 } else { 4317 /* Allocate a pd page. */ 4318 ptepindex = pmap_l3e_pindex(va); 4319 pdpindex = ptepindex >> RPTE_SHIFT; 4320 pdpg = _pmap_allocpte(pmap, NUPDE + pdpindex, lockp); 4321 if (pdpg == NULL && lockp != NULL) 4322 goto retry; 4323 } 4324 return (pdpg); 4325 } 4326 4327 static vm_page_t 4328 pmap_allocpte(pmap_t pmap, vm_offset_t va, struct rwlock **lockp) 4329 { 4330 vm_pindex_t ptepindex; 4331 pml3_entry_t *pd; 4332 vm_page_t m; 4333 4334 /* 4335 * Calculate pagetable page index 4336 */ 4337 ptepindex = pmap_l3e_pindex(va); 4338 retry: 4339 /* 4340 * Get the page directory entry 4341 */ 4342 pd = pmap_pml3e(pmap, va); 4343 4344 /* 4345 * This supports switching from a 2MB page to a 4346 * normal 4K page. 4347 */ 4348 if (pd != NULL && (be64toh(*pd) & (RPTE_LEAF | PG_V)) == (RPTE_LEAF | PG_V)) { 4349 if (!pmap_demote_l3e_locked(pmap, pd, va, lockp)) { 4350 /* 4351 * Invalidation of the 2MB page mapping may have caused 4352 * the deallocation of the underlying PD page. 4353 */ 4354 pd = NULL; 4355 } 4356 } 4357 4358 /* 4359 * If the page table page is mapped, we just increment the 4360 * hold count, and activate it. 4361 */ 4362 if (pd != NULL && (be64toh(*pd) & PG_V) != 0) { 4363 m = PHYS_TO_VM_PAGE(be64toh(*pd) & PG_FRAME); 4364 m->ref_count++; 4365 } else { 4366 /* 4367 * Here if the pte page isn't mapped, or if it has been 4368 * deallocated. 4369 */ 4370 m = _pmap_allocpte(pmap, ptepindex, lockp); 4371 if (m == NULL && lockp != NULL) 4372 goto retry; 4373 } 4374 return (m); 4375 } 4376 4377 static void 4378 mmu_radix_pinit0(pmap_t pmap) 4379 { 4380 4381 CTR2(KTR_PMAP, "%s(%p)", __func__, pmap); 4382 PMAP_LOCK_INIT(pmap); 4383 pmap->pm_pml1 = kernel_pmap->pm_pml1; 4384 pmap->pm_pid = kernel_pmap->pm_pid; 4385 4386 pmap->pm_radix.rt_root = 0; 4387 TAILQ_INIT(&pmap->pm_pvchunk); 4388 bzero(&pmap->pm_stats, sizeof pmap->pm_stats); 4389 kernel_pmap->pm_flags = 4390 pmap->pm_flags = PMAP_PDE_SUPERPAGE; 4391 } 4392 /* 4393 * pmap_protect_l3e: do the things to protect a 2mpage in a process 4394 */ 4395 static boolean_t 4396 pmap_protect_l3e(pmap_t pmap, pt_entry_t *l3e, vm_offset_t sva, vm_prot_t prot) 4397 { 4398 pt_entry_t newpde, oldpde; 4399 vm_offset_t eva, va; 4400 vm_page_t m; 4401 boolean_t anychanged; 4402 4403 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 4404 KASSERT((sva & L3_PAGE_MASK) == 0, 4405 ("pmap_protect_l3e: sva is not 2mpage aligned")); 4406 anychanged = FALSE; 4407 retry: 4408 oldpde = newpde = be64toh(*l3e); 4409 if ((oldpde & (PG_MANAGED | PG_M | PG_RW)) == 4410 (PG_MANAGED | PG_M | PG_RW)) { 4411 eva = sva + L3_PAGE_SIZE; 4412 for (va = sva, m = PHYS_TO_VM_PAGE(oldpde & PG_PS_FRAME); 4413 va < eva; va += PAGE_SIZE, m++) 4414 vm_page_dirty(m); 4415 } 4416 if ((prot & VM_PROT_WRITE) == 0) { 4417 newpde &= ~(PG_RW | PG_M); 4418 newpde |= RPTE_EAA_R; 4419 } 4420 if (prot & VM_PROT_EXECUTE) 4421 newpde |= PG_X; 4422 if (newpde != oldpde) { 4423 /* 4424 * As an optimization to future operations on this PDE, clear 4425 * PG_PROMOTED. The impending invalidation will remove any 4426 * lingering 4KB page mappings from the TLB. 4427 */ 4428 if (!atomic_cmpset_long(l3e, htobe64(oldpde), htobe64(newpde & ~PG_PROMOTED))) 4429 goto retry; 4430 anychanged = TRUE; 4431 } 4432 return (anychanged); 4433 } 4434 4435 void 4436 mmu_radix_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, 4437 vm_prot_t prot) 4438 { 4439 vm_offset_t va_next; 4440 pml1_entry_t *l1e; 4441 pml2_entry_t *l2e; 4442 pml3_entry_t ptpaddr, *l3e; 4443 pt_entry_t *pte; 4444 boolean_t anychanged; 4445 4446 CTR5(KTR_PMAP, "%s(%p, %#x, %#x, %#x)", __func__, pmap, sva, eva, 4447 prot); 4448 4449 KASSERT((prot & ~VM_PROT_ALL) == 0, ("invalid prot %x", prot)); 4450 if (prot == VM_PROT_NONE) { 4451 mmu_radix_remove(pmap, sva, eva); 4452 return; 4453 } 4454 4455 if ((prot & (VM_PROT_WRITE|VM_PROT_EXECUTE)) == 4456 (VM_PROT_WRITE|VM_PROT_EXECUTE)) 4457 return; 4458 4459 #ifdef INVARIANTS 4460 if (VERBOSE_PROTECT || pmap_logging) 4461 printf("pmap_protect(%p, %#lx, %#lx, %x) - asid: %lu\n", 4462 pmap, sva, eva, prot, pmap->pm_pid); 4463 #endif 4464 anychanged = FALSE; 4465 4466 PMAP_LOCK(pmap); 4467 for (; sva < eva; sva = va_next) { 4468 l1e = pmap_pml1e(pmap, sva); 4469 if ((be64toh(*l1e) & PG_V) == 0) { 4470 va_next = (sva + L1_PAGE_SIZE) & ~L1_PAGE_MASK; 4471 if (va_next < sva) 4472 va_next = eva; 4473 continue; 4474 } 4475 4476 l2e = pmap_l1e_to_l2e(l1e, sva); 4477 if ((be64toh(*l2e) & PG_V) == 0) { 4478 va_next = (sva + L2_PAGE_SIZE) & ~L2_PAGE_MASK; 4479 if (va_next < sva) 4480 va_next = eva; 4481 continue; 4482 } 4483 4484 va_next = (sva + L3_PAGE_SIZE) & ~L3_PAGE_MASK; 4485 if (va_next < sva) 4486 va_next = eva; 4487 4488 l3e = pmap_l2e_to_l3e(l2e, sva); 4489 ptpaddr = be64toh(*l3e); 4490 4491 /* 4492 * Weed out invalid mappings. 4493 */ 4494 if (ptpaddr == 0) 4495 continue; 4496 4497 /* 4498 * Check for large page. 4499 */ 4500 if ((ptpaddr & RPTE_LEAF) != 0) { 4501 /* 4502 * Are we protecting the entire large page? If not, 4503 * demote the mapping and fall through. 4504 */ 4505 if (sva + L3_PAGE_SIZE == va_next && eva >= va_next) { 4506 if (pmap_protect_l3e(pmap, l3e, sva, prot)) 4507 anychanged = TRUE; 4508 continue; 4509 } else if (!pmap_demote_l3e(pmap, l3e, sva)) { 4510 /* 4511 * The large page mapping was destroyed. 4512 */ 4513 continue; 4514 } 4515 } 4516 4517 if (va_next > eva) 4518 va_next = eva; 4519 4520 for (pte = pmap_l3e_to_pte(l3e, sva); sva != va_next; pte++, 4521 sva += PAGE_SIZE) { 4522 pt_entry_t obits, pbits; 4523 vm_page_t m; 4524 4525 retry: 4526 MPASS(pte == pmap_pte(pmap, sva)); 4527 obits = pbits = be64toh(*pte); 4528 if ((pbits & PG_V) == 0) 4529 continue; 4530 4531 if ((prot & VM_PROT_WRITE) == 0) { 4532 if ((pbits & (PG_MANAGED | PG_M | PG_RW)) == 4533 (PG_MANAGED | PG_M | PG_RW)) { 4534 m = PHYS_TO_VM_PAGE(pbits & PG_FRAME); 4535 vm_page_dirty(m); 4536 } 4537 pbits &= ~(PG_RW | PG_M); 4538 pbits |= RPTE_EAA_R; 4539 } 4540 if (prot & VM_PROT_EXECUTE) 4541 pbits |= PG_X; 4542 4543 if (pbits != obits) { 4544 if (!atomic_cmpset_long(pte, htobe64(obits), htobe64(pbits))) 4545 goto retry; 4546 if (obits & (PG_A|PG_M)) { 4547 anychanged = TRUE; 4548 #ifdef INVARIANTS 4549 if (VERBOSE_PROTECT || pmap_logging) 4550 printf("%#lx %#lx -> %#lx\n", 4551 sva, obits, pbits); 4552 #endif 4553 } 4554 } 4555 } 4556 } 4557 if (anychanged) 4558 pmap_invalidate_all(pmap); 4559 PMAP_UNLOCK(pmap); 4560 } 4561 4562 void 4563 mmu_radix_qenter(vm_offset_t sva, vm_page_t *ma, int count) 4564 { 4565 4566 CTR4(KTR_PMAP, "%s(%#x, %p, %d)", __func__, sva, ma, count); 4567 pt_entry_t oldpte, pa, *pte; 4568 vm_page_t m; 4569 uint64_t cache_bits, attr_bits; 4570 vm_offset_t va; 4571 4572 oldpte = 0; 4573 attr_bits = RPTE_EAA_R | RPTE_EAA_W | RPTE_EAA_P | PG_M | PG_A; 4574 va = sva; 4575 pte = kvtopte(va); 4576 while (va < sva + PAGE_SIZE * count) { 4577 if (__predict_false((va & L3_PAGE_MASK) == 0)) 4578 pte = kvtopte(va); 4579 MPASS(pte == pmap_pte(kernel_pmap, va)); 4580 4581 /* 4582 * XXX there has to be a more efficient way than traversing 4583 * the page table every time - but go for correctness for 4584 * today 4585 */ 4586 4587 m = *ma++; 4588 cache_bits = pmap_cache_bits(m->md.mdpg_cache_attrs); 4589 pa = VM_PAGE_TO_PHYS(m) | cache_bits | attr_bits; 4590 if (be64toh(*pte) != pa) { 4591 oldpte |= be64toh(*pte); 4592 pte_store(pte, pa); 4593 } 4594 va += PAGE_SIZE; 4595 pte++; 4596 } 4597 if (__predict_false((oldpte & RPTE_VALID) != 0)) 4598 pmap_invalidate_range(kernel_pmap, sva, sva + count * 4599 PAGE_SIZE); 4600 else 4601 ptesync(); 4602 } 4603 4604 void 4605 mmu_radix_qremove(vm_offset_t sva, int count) 4606 { 4607 vm_offset_t va; 4608 pt_entry_t *pte; 4609 4610 CTR3(KTR_PMAP, "%s(%#x, %d)", __func__, sva, count); 4611 KASSERT(sva >= VM_MIN_KERNEL_ADDRESS, ("usermode or dmap va %lx", sva)); 4612 4613 va = sva; 4614 pte = kvtopte(va); 4615 while (va < sva + PAGE_SIZE * count) { 4616 if (__predict_false((va & L3_PAGE_MASK) == 0)) 4617 pte = kvtopte(va); 4618 pte_clear(pte); 4619 pte++; 4620 va += PAGE_SIZE; 4621 } 4622 pmap_invalidate_range(kernel_pmap, sva, va); 4623 } 4624 4625 /*************************************************** 4626 * Page table page management routines..... 4627 ***************************************************/ 4628 /* 4629 * Schedule the specified unused page table page to be freed. Specifically, 4630 * add the page to the specified list of pages that will be released to the 4631 * physical memory manager after the TLB has been updated. 4632 */ 4633 static __inline void 4634 pmap_add_delayed_free_list(vm_page_t m, struct spglist *free, 4635 boolean_t set_PG_ZERO) 4636 { 4637 4638 if (set_PG_ZERO) 4639 m->flags |= PG_ZERO; 4640 else 4641 m->flags &= ~PG_ZERO; 4642 SLIST_INSERT_HEAD(free, m, plinks.s.ss); 4643 } 4644 4645 /* 4646 * Inserts the specified page table page into the specified pmap's collection 4647 * of idle page table pages. Each of a pmap's page table pages is responsible 4648 * for mapping a distinct range of virtual addresses. The pmap's collection is 4649 * ordered by this virtual address range. 4650 */ 4651 static __inline int 4652 pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte) 4653 { 4654 4655 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 4656 return (vm_radix_insert(&pmap->pm_radix, mpte)); 4657 } 4658 4659 /* 4660 * Removes the page table page mapping the specified virtual address from the 4661 * specified pmap's collection of idle page table pages, and returns it. 4662 * Otherwise, returns NULL if there is no page table page corresponding to the 4663 * specified virtual address. 4664 */ 4665 static __inline vm_page_t 4666 pmap_remove_pt_page(pmap_t pmap, vm_offset_t va) 4667 { 4668 4669 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 4670 return (vm_radix_remove(&pmap->pm_radix, pmap_l3e_pindex(va))); 4671 } 4672 4673 /* 4674 * Decrements a page table page's wire count, which is used to record the 4675 * number of valid page table entries within the page. If the wire count 4676 * drops to zero, then the page table page is unmapped. Returns TRUE if the 4677 * page table page was unmapped and FALSE otherwise. 4678 */ 4679 static inline boolean_t 4680 pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free) 4681 { 4682 4683 --m->ref_count; 4684 if (m->ref_count == 0) { 4685 _pmap_unwire_ptp(pmap, va, m, free); 4686 return (TRUE); 4687 } else 4688 return (FALSE); 4689 } 4690 4691 static void 4692 _pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free) 4693 { 4694 4695 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 4696 /* 4697 * unmap the page table page 4698 */ 4699 if (m->pindex >= (NUPDE + NUPDPE)) { 4700 /* PDP page */ 4701 pml1_entry_t *pml1; 4702 pml1 = pmap_pml1e(pmap, va); 4703 *pml1 = 0; 4704 } else if (m->pindex >= NUPDE) { 4705 /* PD page */ 4706 pml2_entry_t *l2e; 4707 l2e = pmap_pml2e(pmap, va); 4708 *l2e = 0; 4709 } else { 4710 /* PTE page */ 4711 pml3_entry_t *l3e; 4712 l3e = pmap_pml3e(pmap, va); 4713 *l3e = 0; 4714 } 4715 pmap_resident_count_dec(pmap, 1); 4716 if (m->pindex < NUPDE) { 4717 /* We just released a PT, unhold the matching PD */ 4718 vm_page_t pdpg; 4719 4720 pdpg = PHYS_TO_VM_PAGE(be64toh(*pmap_pml2e(pmap, va)) & PG_FRAME); 4721 pmap_unwire_ptp(pmap, va, pdpg, free); 4722 } 4723 if (m->pindex >= NUPDE && m->pindex < (NUPDE + NUPDPE)) { 4724 /* We just released a PD, unhold the matching PDP */ 4725 vm_page_t pdppg; 4726 4727 pdppg = PHYS_TO_VM_PAGE(be64toh(*pmap_pml1e(pmap, va)) & PG_FRAME); 4728 pmap_unwire_ptp(pmap, va, pdppg, free); 4729 } 4730 4731 /* 4732 * Put page on a list so that it is released after 4733 * *ALL* TLB shootdown is done 4734 */ 4735 pmap_add_delayed_free_list(m, free, TRUE); 4736 } 4737 4738 /* 4739 * After removing a page table entry, this routine is used to 4740 * conditionally free the page, and manage the hold/wire counts. 4741 */ 4742 static int 4743 pmap_unuse_pt(pmap_t pmap, vm_offset_t va, pml3_entry_t ptepde, 4744 struct spglist *free) 4745 { 4746 vm_page_t mpte; 4747 4748 if (va >= VM_MAXUSER_ADDRESS) 4749 return (0); 4750 KASSERT(ptepde != 0, ("pmap_unuse_pt: ptepde != 0")); 4751 mpte = PHYS_TO_VM_PAGE(ptepde & PG_FRAME); 4752 return (pmap_unwire_ptp(pmap, va, mpte, free)); 4753 } 4754 4755 void 4756 mmu_radix_release(pmap_t pmap) 4757 { 4758 4759 CTR2(KTR_PMAP, "%s(%p)", __func__, pmap); 4760 KASSERT(pmap->pm_stats.resident_count == 0, 4761 ("pmap_release: pmap resident count %ld != 0", 4762 pmap->pm_stats.resident_count)); 4763 KASSERT(vm_radix_is_empty(&pmap->pm_radix), 4764 ("pmap_release: pmap has reserved page table page(s)")); 4765 4766 pmap_invalidate_all(pmap); 4767 isa3_proctab[pmap->pm_pid].proctab0 = 0; 4768 uma_zfree(zone_radix_pgd, pmap->pm_pml1); 4769 vmem_free(asid_arena, pmap->pm_pid, 1); 4770 } 4771 4772 /* 4773 * Create the PV entry for a 2MB page mapping. Always returns true unless the 4774 * flag PMAP_ENTER_NORECLAIM is specified. If that flag is specified, returns 4775 * false if the PV entry cannot be allocated without resorting to reclamation. 4776 */ 4777 static bool 4778 pmap_pv_insert_l3e(pmap_t pmap, vm_offset_t va, pml3_entry_t pde, u_int flags, 4779 struct rwlock **lockp) 4780 { 4781 struct md_page *pvh; 4782 pv_entry_t pv; 4783 vm_paddr_t pa; 4784 4785 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 4786 /* Pass NULL instead of the lock pointer to disable reclamation. */ 4787 if ((pv = get_pv_entry(pmap, (flags & PMAP_ENTER_NORECLAIM) != 0 ? 4788 NULL : lockp)) == NULL) 4789 return (false); 4790 pv->pv_va = va; 4791 pa = pde & PG_PS_FRAME; 4792 CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa); 4793 pvh = pa_to_pvh(pa); 4794 TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_link); 4795 pvh->pv_gen++; 4796 return (true); 4797 } 4798 4799 /* 4800 * Fills a page table page with mappings to consecutive physical pages. 4801 */ 4802 static void 4803 pmap_fill_ptp(pt_entry_t *firstpte, pt_entry_t newpte) 4804 { 4805 pt_entry_t *pte; 4806 4807 for (pte = firstpte; pte < firstpte + NPTEPG; pte++) { 4808 *pte = htobe64(newpte); 4809 newpte += PAGE_SIZE; 4810 } 4811 } 4812 4813 static boolean_t 4814 pmap_demote_l3e(pmap_t pmap, pml3_entry_t *pde, vm_offset_t va) 4815 { 4816 struct rwlock *lock; 4817 boolean_t rv; 4818 4819 lock = NULL; 4820 rv = pmap_demote_l3e_locked(pmap, pde, va, &lock); 4821 if (lock != NULL) 4822 rw_wunlock(lock); 4823 return (rv); 4824 } 4825 4826 static boolean_t 4827 pmap_demote_l3e_locked(pmap_t pmap, pml3_entry_t *l3e, vm_offset_t va, 4828 struct rwlock **lockp) 4829 { 4830 pml3_entry_t oldpde; 4831 pt_entry_t *firstpte; 4832 vm_paddr_t mptepa; 4833 vm_page_t mpte; 4834 struct spglist free; 4835 vm_offset_t sva; 4836 4837 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 4838 oldpde = be64toh(*l3e); 4839 KASSERT((oldpde & (RPTE_LEAF | PG_V)) == (RPTE_LEAF | PG_V), 4840 ("pmap_demote_l3e: oldpde is missing RPTE_LEAF and/or PG_V %lx", 4841 oldpde)); 4842 if ((oldpde & PG_A) == 0 || (mpte = pmap_remove_pt_page(pmap, va)) == 4843 NULL) { 4844 KASSERT((oldpde & PG_W) == 0, 4845 ("pmap_demote_l3e: page table page for a wired mapping" 4846 " is missing")); 4847 4848 /* 4849 * Invalidate the 2MB page mapping and return "failure" if the 4850 * mapping was never accessed or the allocation of the new 4851 * page table page fails. If the 2MB page mapping belongs to 4852 * the direct map region of the kernel's address space, then 4853 * the page allocation request specifies the highest possible 4854 * priority (VM_ALLOC_INTERRUPT). Otherwise, the priority is 4855 * normal. Page table pages are preallocated for every other 4856 * part of the kernel address space, so the direct map region 4857 * is the only part of the kernel address space that must be 4858 * handled here. 4859 */ 4860 if ((oldpde & PG_A) == 0 || (mpte = vm_page_alloc(NULL, 4861 pmap_l3e_pindex(va), (va >= DMAP_MIN_ADDRESS && va < 4862 DMAP_MAX_ADDRESS ? VM_ALLOC_INTERRUPT : VM_ALLOC_NORMAL) | 4863 VM_ALLOC_NOOBJ | VM_ALLOC_WIRED)) == NULL) { 4864 SLIST_INIT(&free); 4865 sva = trunc_2mpage(va); 4866 pmap_remove_l3e(pmap, l3e, sva, &free, lockp); 4867 pmap_invalidate_l3e_page(pmap, sva, oldpde); 4868 vm_page_free_pages_toq(&free, true); 4869 CTR2(KTR_PMAP, "pmap_demote_l3e: failure for va %#lx" 4870 " in pmap %p", va, pmap); 4871 return (FALSE); 4872 } 4873 if (va < VM_MAXUSER_ADDRESS) 4874 pmap_resident_count_inc(pmap, 1); 4875 } 4876 mptepa = VM_PAGE_TO_PHYS(mpte); 4877 firstpte = (pt_entry_t *)PHYS_TO_DMAP(mptepa); 4878 KASSERT((oldpde & PG_A) != 0, 4879 ("pmap_demote_l3e: oldpde is missing PG_A")); 4880 KASSERT((oldpde & (PG_M | PG_RW)) != PG_RW, 4881 ("pmap_demote_l3e: oldpde is missing PG_M")); 4882 4883 /* 4884 * If the page table page is new, initialize it. 4885 */ 4886 if (mpte->ref_count == 1) { 4887 mpte->ref_count = NPTEPG; 4888 pmap_fill_ptp(firstpte, oldpde); 4889 } 4890 4891 KASSERT((be64toh(*firstpte) & PG_FRAME) == (oldpde & PG_FRAME), 4892 ("pmap_demote_l3e: firstpte and newpte map different physical" 4893 " addresses")); 4894 4895 /* 4896 * If the mapping has changed attributes, update the page table 4897 * entries. 4898 */ 4899 if ((be64toh(*firstpte) & PG_PTE_PROMOTE) != (oldpde & PG_PTE_PROMOTE)) 4900 pmap_fill_ptp(firstpte, oldpde); 4901 4902 /* 4903 * The spare PV entries must be reserved prior to demoting the 4904 * mapping, that is, prior to changing the PDE. Otherwise, the state 4905 * of the PDE and the PV lists will be inconsistent, which can result 4906 * in reclaim_pv_chunk() attempting to remove a PV entry from the 4907 * wrong PV list and pmap_pv_demote_l3e() failing to find the expected 4908 * PV entry for the 2MB page mapping that is being demoted. 4909 */ 4910 if ((oldpde & PG_MANAGED) != 0) 4911 reserve_pv_entries(pmap, NPTEPG - 1, lockp); 4912 4913 /* 4914 * Demote the mapping. This pmap is locked. The old PDE has 4915 * PG_A set. If the old PDE has PG_RW set, it also has PG_M 4916 * set. Thus, there is no danger of a race with another 4917 * processor changing the setting of PG_A and/or PG_M between 4918 * the read above and the store below. 4919 */ 4920 pde_store(l3e, mptepa); 4921 ptesync(); 4922 /* 4923 * Demote the PV entry. 4924 */ 4925 if ((oldpde & PG_MANAGED) != 0) 4926 pmap_pv_demote_l3e(pmap, va, oldpde & PG_PS_FRAME, lockp); 4927 4928 atomic_add_long(&pmap_l3e_demotions, 1); 4929 CTR2(KTR_PMAP, "pmap_demote_l3e: success for va %#lx" 4930 " in pmap %p", va, pmap); 4931 return (TRUE); 4932 } 4933 4934 /* 4935 * pmap_remove_kernel_pde: Remove a kernel superpage mapping. 4936 */ 4937 static void 4938 pmap_remove_kernel_l3e(pmap_t pmap, pml3_entry_t *l3e, vm_offset_t va) 4939 { 4940 vm_paddr_t mptepa; 4941 vm_page_t mpte; 4942 4943 KASSERT(pmap == kernel_pmap, ("pmap %p is not kernel_pmap", pmap)); 4944 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 4945 mpte = pmap_remove_pt_page(pmap, va); 4946 if (mpte == NULL) 4947 panic("pmap_remove_kernel_pde: Missing pt page."); 4948 4949 mptepa = VM_PAGE_TO_PHYS(mpte); 4950 4951 /* 4952 * Initialize the page table page. 4953 */ 4954 pagezero(PHYS_TO_DMAP(mptepa)); 4955 4956 /* 4957 * Demote the mapping. 4958 */ 4959 pde_store(l3e, mptepa); 4960 ptesync(); 4961 } 4962 4963 /* 4964 * pmap_remove_l3e: do the things to unmap a superpage in a process 4965 */ 4966 static int 4967 pmap_remove_l3e(pmap_t pmap, pml3_entry_t *pdq, vm_offset_t sva, 4968 struct spglist *free, struct rwlock **lockp) 4969 { 4970 struct md_page *pvh; 4971 pml3_entry_t oldpde; 4972 vm_offset_t eva, va; 4973 vm_page_t m, mpte; 4974 4975 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 4976 KASSERT((sva & L3_PAGE_MASK) == 0, 4977 ("pmap_remove_l3e: sva is not 2mpage aligned")); 4978 oldpde = be64toh(pte_load_clear(pdq)); 4979 if (oldpde & PG_W) 4980 pmap->pm_stats.wired_count -= (L3_PAGE_SIZE / PAGE_SIZE); 4981 pmap_resident_count_dec(pmap, L3_PAGE_SIZE / PAGE_SIZE); 4982 if (oldpde & PG_MANAGED) { 4983 CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, oldpde & PG_PS_FRAME); 4984 pvh = pa_to_pvh(oldpde & PG_PS_FRAME); 4985 pmap_pvh_free(pvh, pmap, sva); 4986 eva = sva + L3_PAGE_SIZE; 4987 for (va = sva, m = PHYS_TO_VM_PAGE(oldpde & PG_PS_FRAME); 4988 va < eva; va += PAGE_SIZE, m++) { 4989 if ((oldpde & (PG_M | PG_RW)) == (PG_M | PG_RW)) 4990 vm_page_dirty(m); 4991 if (oldpde & PG_A) 4992 vm_page_aflag_set(m, PGA_REFERENCED); 4993 if (TAILQ_EMPTY(&m->md.pv_list) && 4994 TAILQ_EMPTY(&pvh->pv_list)) 4995 vm_page_aflag_clear(m, PGA_WRITEABLE); 4996 } 4997 } 4998 if (pmap == kernel_pmap) { 4999 pmap_remove_kernel_l3e(pmap, pdq, sva); 5000 } else { 5001 mpte = pmap_remove_pt_page(pmap, sva); 5002 if (mpte != NULL) { 5003 pmap_resident_count_dec(pmap, 1); 5004 KASSERT(mpte->ref_count == NPTEPG, 5005 ("pmap_remove_l3e: pte page wire count error")); 5006 mpte->ref_count = 0; 5007 pmap_add_delayed_free_list(mpte, free, FALSE); 5008 } 5009 } 5010 return (pmap_unuse_pt(pmap, sva, be64toh(*pmap_pml2e(pmap, sva)), free)); 5011 } 5012 5013 /* 5014 * pmap_remove_pte: do the things to unmap a page in a process 5015 */ 5016 static int 5017 pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t va, 5018 pml3_entry_t ptepde, struct spglist *free, struct rwlock **lockp) 5019 { 5020 struct md_page *pvh; 5021 pt_entry_t oldpte; 5022 vm_page_t m; 5023 5024 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 5025 oldpte = be64toh(pte_load_clear(ptq)); 5026 if (oldpte & RPTE_WIRED) 5027 pmap->pm_stats.wired_count -= 1; 5028 pmap_resident_count_dec(pmap, 1); 5029 if (oldpte & RPTE_MANAGED) { 5030 m = PHYS_TO_VM_PAGE(oldpte & PG_FRAME); 5031 if ((oldpte & (PG_M | PG_RW)) == (PG_M | PG_RW)) 5032 vm_page_dirty(m); 5033 if (oldpte & PG_A) 5034 vm_page_aflag_set(m, PGA_REFERENCED); 5035 CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m); 5036 pmap_pvh_free(&m->md, pmap, va); 5037 if (TAILQ_EMPTY(&m->md.pv_list) && 5038 (m->flags & PG_FICTITIOUS) == 0) { 5039 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); 5040 if (TAILQ_EMPTY(&pvh->pv_list)) 5041 vm_page_aflag_clear(m, PGA_WRITEABLE); 5042 } 5043 } 5044 return (pmap_unuse_pt(pmap, va, ptepde, free)); 5045 } 5046 5047 /* 5048 * Remove a single page from a process address space 5049 */ 5050 static bool 5051 pmap_remove_page(pmap_t pmap, vm_offset_t va, pml3_entry_t *l3e, 5052 struct spglist *free) 5053 { 5054 struct rwlock *lock; 5055 pt_entry_t *pte; 5056 bool invalidate_all; 5057 5058 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 5059 if ((be64toh(*l3e) & RPTE_VALID) == 0) { 5060 return (false); 5061 } 5062 pte = pmap_l3e_to_pte(l3e, va); 5063 if ((be64toh(*pte) & RPTE_VALID) == 0) { 5064 return (false); 5065 } 5066 lock = NULL; 5067 5068 invalidate_all = pmap_remove_pte(pmap, pte, va, be64toh(*l3e), free, &lock); 5069 if (lock != NULL) 5070 rw_wunlock(lock); 5071 if (!invalidate_all) 5072 pmap_invalidate_page(pmap, va); 5073 return (invalidate_all); 5074 } 5075 5076 /* 5077 * Removes the specified range of addresses from the page table page. 5078 */ 5079 static bool 5080 pmap_remove_ptes(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, 5081 pml3_entry_t *l3e, struct spglist *free, struct rwlock **lockp) 5082 { 5083 pt_entry_t *pte; 5084 vm_offset_t va; 5085 bool anyvalid; 5086 5087 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 5088 anyvalid = false; 5089 va = eva; 5090 for (pte = pmap_l3e_to_pte(l3e, sva); sva != eva; pte++, 5091 sva += PAGE_SIZE) { 5092 MPASS(pte == pmap_pte(pmap, sva)); 5093 if (*pte == 0) { 5094 if (va != eva) { 5095 anyvalid = true; 5096 va = eva; 5097 } 5098 continue; 5099 } 5100 if (va == eva) 5101 va = sva; 5102 if (pmap_remove_pte(pmap, pte, sva, be64toh(*l3e), free, lockp)) { 5103 anyvalid = true; 5104 sva += PAGE_SIZE; 5105 break; 5106 } 5107 } 5108 if (anyvalid) 5109 pmap_invalidate_all(pmap); 5110 else if (va != eva) 5111 pmap_invalidate_range(pmap, va, sva); 5112 return (anyvalid); 5113 } 5114 5115 void 5116 mmu_radix_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) 5117 { 5118 struct rwlock *lock; 5119 vm_offset_t va_next; 5120 pml1_entry_t *l1e; 5121 pml2_entry_t *l2e; 5122 pml3_entry_t ptpaddr, *l3e; 5123 struct spglist free; 5124 bool anyvalid; 5125 5126 CTR4(KTR_PMAP, "%s(%p, %#x, %#x)", __func__, pmap, sva, eva); 5127 5128 /* 5129 * Perform an unsynchronized read. This is, however, safe. 5130 */ 5131 if (pmap->pm_stats.resident_count == 0) 5132 return; 5133 5134 anyvalid = false; 5135 SLIST_INIT(&free); 5136 5137 /* XXX something fishy here */ 5138 sva = (sva + PAGE_MASK) & ~PAGE_MASK; 5139 eva = (eva + PAGE_MASK) & ~PAGE_MASK; 5140 5141 PMAP_LOCK(pmap); 5142 5143 /* 5144 * special handling of removing one page. a very 5145 * common operation and easy to short circuit some 5146 * code. 5147 */ 5148 if (sva + PAGE_SIZE == eva) { 5149 l3e = pmap_pml3e(pmap, sva); 5150 if (l3e && (be64toh(*l3e) & RPTE_LEAF) == 0) { 5151 anyvalid = pmap_remove_page(pmap, sva, l3e, &free); 5152 goto out; 5153 } 5154 } 5155 5156 lock = NULL; 5157 for (; sva < eva; sva = va_next) { 5158 if (pmap->pm_stats.resident_count == 0) 5159 break; 5160 l1e = pmap_pml1e(pmap, sva); 5161 if (l1e == NULL || (be64toh(*l1e) & PG_V) == 0) { 5162 va_next = (sva + L1_PAGE_SIZE) & ~L1_PAGE_MASK; 5163 if (va_next < sva) 5164 va_next = eva; 5165 continue; 5166 } 5167 5168 l2e = pmap_l1e_to_l2e(l1e, sva); 5169 if (l2e == NULL || (be64toh(*l2e) & PG_V) == 0) { 5170 va_next = (sva + L2_PAGE_SIZE) & ~L2_PAGE_MASK; 5171 if (va_next < sva) 5172 va_next = eva; 5173 continue; 5174 } 5175 5176 /* 5177 * Calculate index for next page table. 5178 */ 5179 va_next = (sva + L3_PAGE_SIZE) & ~L3_PAGE_MASK; 5180 if (va_next < sva) 5181 va_next = eva; 5182 5183 l3e = pmap_l2e_to_l3e(l2e, sva); 5184 ptpaddr = be64toh(*l3e); 5185 5186 /* 5187 * Weed out invalid mappings. 5188 */ 5189 if (ptpaddr == 0) 5190 continue; 5191 5192 /* 5193 * Check for large page. 5194 */ 5195 if ((ptpaddr & RPTE_LEAF) != 0) { 5196 /* 5197 * Are we removing the entire large page? If not, 5198 * demote the mapping and fall through. 5199 */ 5200 if (sva + L3_PAGE_SIZE == va_next && eva >= va_next) { 5201 pmap_remove_l3e(pmap, l3e, sva, &free, &lock); 5202 continue; 5203 } else if (!pmap_demote_l3e_locked(pmap, l3e, sva, 5204 &lock)) { 5205 /* The large page mapping was destroyed. */ 5206 continue; 5207 } else 5208 ptpaddr = be64toh(*l3e); 5209 } 5210 5211 /* 5212 * Limit our scan to either the end of the va represented 5213 * by the current page table page, or to the end of the 5214 * range being removed. 5215 */ 5216 if (va_next > eva) 5217 va_next = eva; 5218 5219 if (pmap_remove_ptes(pmap, sva, va_next, l3e, &free, &lock)) 5220 anyvalid = true; 5221 } 5222 if (lock != NULL) 5223 rw_wunlock(lock); 5224 out: 5225 if (anyvalid) 5226 pmap_invalidate_all(pmap); 5227 PMAP_UNLOCK(pmap); 5228 vm_page_free_pages_toq(&free, true); 5229 } 5230 5231 void 5232 mmu_radix_remove_all(vm_page_t m) 5233 { 5234 struct md_page *pvh; 5235 pv_entry_t pv; 5236 pmap_t pmap; 5237 struct rwlock *lock; 5238 pt_entry_t *pte, tpte; 5239 pml3_entry_t *l3e; 5240 vm_offset_t va; 5241 struct spglist free; 5242 int pvh_gen, md_gen; 5243 5244 CTR2(KTR_PMAP, "%s(%p)", __func__, m); 5245 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 5246 ("pmap_remove_all: page %p is not managed", m)); 5247 SLIST_INIT(&free); 5248 lock = VM_PAGE_TO_PV_LIST_LOCK(m); 5249 pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy : 5250 pa_to_pvh(VM_PAGE_TO_PHYS(m)); 5251 retry: 5252 rw_wlock(lock); 5253 while ((pv = TAILQ_FIRST(&pvh->pv_list)) != NULL) { 5254 pmap = PV_PMAP(pv); 5255 if (!PMAP_TRYLOCK(pmap)) { 5256 pvh_gen = pvh->pv_gen; 5257 rw_wunlock(lock); 5258 PMAP_LOCK(pmap); 5259 rw_wlock(lock); 5260 if (pvh_gen != pvh->pv_gen) { 5261 rw_wunlock(lock); 5262 PMAP_UNLOCK(pmap); 5263 goto retry; 5264 } 5265 } 5266 va = pv->pv_va; 5267 l3e = pmap_pml3e(pmap, va); 5268 (void)pmap_demote_l3e_locked(pmap, l3e, va, &lock); 5269 PMAP_UNLOCK(pmap); 5270 } 5271 while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) { 5272 pmap = PV_PMAP(pv); 5273 if (!PMAP_TRYLOCK(pmap)) { 5274 pvh_gen = pvh->pv_gen; 5275 md_gen = m->md.pv_gen; 5276 rw_wunlock(lock); 5277 PMAP_LOCK(pmap); 5278 rw_wlock(lock); 5279 if (pvh_gen != pvh->pv_gen || md_gen != m->md.pv_gen) { 5280 rw_wunlock(lock); 5281 PMAP_UNLOCK(pmap); 5282 goto retry; 5283 } 5284 } 5285 pmap_resident_count_dec(pmap, 1); 5286 l3e = pmap_pml3e(pmap, pv->pv_va); 5287 KASSERT((be64toh(*l3e) & RPTE_LEAF) == 0, ("pmap_remove_all: found" 5288 " a 2mpage in page %p's pv list", m)); 5289 pte = pmap_l3e_to_pte(l3e, pv->pv_va); 5290 tpte = be64toh(pte_load_clear(pte)); 5291 if (tpte & PG_W) 5292 pmap->pm_stats.wired_count--; 5293 if (tpte & PG_A) 5294 vm_page_aflag_set(m, PGA_REFERENCED); 5295 5296 /* 5297 * Update the vm_page_t clean and reference bits. 5298 */ 5299 if ((tpte & (PG_M | PG_RW)) == (PG_M | PG_RW)) 5300 vm_page_dirty(m); 5301 pmap_unuse_pt(pmap, pv->pv_va, be64toh(*l3e), &free); 5302 pmap_invalidate_page(pmap, pv->pv_va); 5303 TAILQ_REMOVE(&m->md.pv_list, pv, pv_link); 5304 m->md.pv_gen++; 5305 free_pv_entry(pmap, pv); 5306 PMAP_UNLOCK(pmap); 5307 } 5308 vm_page_aflag_clear(m, PGA_WRITEABLE); 5309 rw_wunlock(lock); 5310 vm_page_free_pages_toq(&free, true); 5311 } 5312 5313 /* 5314 * Destroy all managed, non-wired mappings in the given user-space 5315 * pmap. This pmap cannot be active on any processor besides the 5316 * caller. 5317 * 5318 * This function cannot be applied to the kernel pmap. Moreover, it 5319 * is not intended for general use. It is only to be used during 5320 * process termination. Consequently, it can be implemented in ways 5321 * that make it faster than pmap_remove(). First, it can more quickly 5322 * destroy mappings by iterating over the pmap's collection of PV 5323 * entries, rather than searching the page table. Second, it doesn't 5324 * have to test and clear the page table entries atomically, because 5325 * no processor is currently accessing the user address space. In 5326 * particular, a page table entry's dirty bit won't change state once 5327 * this function starts. 5328 * 5329 * Although this function destroys all of the pmap's managed, 5330 * non-wired mappings, it can delay and batch the invalidation of TLB 5331 * entries without calling pmap_delayed_invl_started() and 5332 * pmap_delayed_invl_finished(). Because the pmap is not active on 5333 * any other processor, none of these TLB entries will ever be used 5334 * before their eventual invalidation. Consequently, there is no need 5335 * for either pmap_remove_all() or pmap_remove_write() to wait for 5336 * that eventual TLB invalidation. 5337 */ 5338 5339 void 5340 mmu_radix_remove_pages(pmap_t pmap) 5341 { 5342 5343 CTR2(KTR_PMAP, "%s(%p)", __func__, pmap); 5344 pml3_entry_t ptel3e; 5345 pt_entry_t *pte, tpte; 5346 struct spglist free; 5347 vm_page_t m, mpte, mt; 5348 pv_entry_t pv; 5349 struct md_page *pvh; 5350 struct pv_chunk *pc, *npc; 5351 struct rwlock *lock; 5352 int64_t bit; 5353 uint64_t inuse, bitmask; 5354 int allfree, field, freed, idx; 5355 boolean_t superpage; 5356 vm_paddr_t pa; 5357 5358 /* 5359 * Assert that the given pmap is only active on the current 5360 * CPU. Unfortunately, we cannot block another CPU from 5361 * activating the pmap while this function is executing. 5362 */ 5363 KASSERT(pmap->pm_pid == mfspr(SPR_PID), 5364 ("non-current asid %lu - expected %lu", pmap->pm_pid, 5365 mfspr(SPR_PID))); 5366 5367 lock = NULL; 5368 5369 SLIST_INIT(&free); 5370 PMAP_LOCK(pmap); 5371 TAILQ_FOREACH_SAFE(pc, &pmap->pm_pvchunk, pc_list, npc) { 5372 allfree = 1; 5373 freed = 0; 5374 for (field = 0; field < _NPCM; field++) { 5375 inuse = ~pc->pc_map[field] & pc_freemask[field]; 5376 while (inuse != 0) { 5377 bit = cnttzd(inuse); 5378 bitmask = 1UL << bit; 5379 idx = field * 64 + bit; 5380 pv = &pc->pc_pventry[idx]; 5381 inuse &= ~bitmask; 5382 5383 pte = pmap_pml2e(pmap, pv->pv_va); 5384 ptel3e = be64toh(*pte); 5385 pte = pmap_l2e_to_l3e(pte, pv->pv_va); 5386 tpte = be64toh(*pte); 5387 if ((tpte & (RPTE_LEAF | PG_V)) == PG_V) { 5388 superpage = FALSE; 5389 ptel3e = tpte; 5390 pte = (pt_entry_t *)PHYS_TO_DMAP(tpte & 5391 PG_FRAME); 5392 pte = &pte[pmap_pte_index(pv->pv_va)]; 5393 tpte = be64toh(*pte); 5394 } else { 5395 /* 5396 * Keep track whether 'tpte' is a 5397 * superpage explicitly instead of 5398 * relying on RPTE_LEAF being set. 5399 * 5400 * This is because RPTE_LEAF is numerically 5401 * identical to PG_PTE_PAT and thus a 5402 * regular page could be mistaken for 5403 * a superpage. 5404 */ 5405 superpage = TRUE; 5406 } 5407 5408 if ((tpte & PG_V) == 0) { 5409 panic("bad pte va %lx pte %lx", 5410 pv->pv_va, tpte); 5411 } 5412 5413 /* 5414 * We cannot remove wired pages from a process' mapping at this time 5415 */ 5416 if (tpte & PG_W) { 5417 allfree = 0; 5418 continue; 5419 } 5420 5421 if (superpage) 5422 pa = tpte & PG_PS_FRAME; 5423 else 5424 pa = tpte & PG_FRAME; 5425 5426 m = PHYS_TO_VM_PAGE(pa); 5427 KASSERT(m->phys_addr == pa, 5428 ("vm_page_t %p phys_addr mismatch %016jx %016jx", 5429 m, (uintmax_t)m->phys_addr, 5430 (uintmax_t)tpte)); 5431 5432 KASSERT((m->flags & PG_FICTITIOUS) != 0 || 5433 m < &vm_page_array[vm_page_array_size], 5434 ("pmap_remove_pages: bad tpte %#jx", 5435 (uintmax_t)tpte)); 5436 5437 pte_clear(pte); 5438 5439 /* 5440 * Update the vm_page_t clean/reference bits. 5441 */ 5442 if ((tpte & (PG_M | PG_RW)) == (PG_M | PG_RW)) { 5443 if (superpage) { 5444 for (mt = m; mt < &m[L3_PAGE_SIZE / PAGE_SIZE]; mt++) 5445 vm_page_dirty(mt); 5446 } else 5447 vm_page_dirty(m); 5448 } 5449 5450 CHANGE_PV_LIST_LOCK_TO_VM_PAGE(&lock, m); 5451 5452 /* Mark free */ 5453 pc->pc_map[field] |= bitmask; 5454 if (superpage) { 5455 pmap_resident_count_dec(pmap, L3_PAGE_SIZE / PAGE_SIZE); 5456 pvh = pa_to_pvh(tpte & PG_PS_FRAME); 5457 TAILQ_REMOVE(&pvh->pv_list, pv, pv_link); 5458 pvh->pv_gen++; 5459 if (TAILQ_EMPTY(&pvh->pv_list)) { 5460 for (mt = m; mt < &m[L3_PAGE_SIZE / PAGE_SIZE]; mt++) 5461 if ((mt->a.flags & PGA_WRITEABLE) != 0 && 5462 TAILQ_EMPTY(&mt->md.pv_list)) 5463 vm_page_aflag_clear(mt, PGA_WRITEABLE); 5464 } 5465 mpte = pmap_remove_pt_page(pmap, pv->pv_va); 5466 if (mpte != NULL) { 5467 pmap_resident_count_dec(pmap, 1); 5468 KASSERT(mpte->ref_count == NPTEPG, 5469 ("pmap_remove_pages: pte page wire count error")); 5470 mpte->ref_count = 0; 5471 pmap_add_delayed_free_list(mpte, &free, FALSE); 5472 } 5473 } else { 5474 pmap_resident_count_dec(pmap, 1); 5475 #ifdef VERBOSE_PV 5476 printf("freeing pv (%p, %p)\n", 5477 pmap, pv); 5478 #endif 5479 TAILQ_REMOVE(&m->md.pv_list, pv, pv_link); 5480 m->md.pv_gen++; 5481 if ((m->a.flags & PGA_WRITEABLE) != 0 && 5482 TAILQ_EMPTY(&m->md.pv_list) && 5483 (m->flags & PG_FICTITIOUS) == 0) { 5484 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); 5485 if (TAILQ_EMPTY(&pvh->pv_list)) 5486 vm_page_aflag_clear(m, PGA_WRITEABLE); 5487 } 5488 } 5489 pmap_unuse_pt(pmap, pv->pv_va, ptel3e, &free); 5490 freed++; 5491 } 5492 } 5493 PV_STAT(atomic_add_long(&pv_entry_frees, freed)); 5494 PV_STAT(atomic_add_int(&pv_entry_spare, freed)); 5495 PV_STAT(atomic_subtract_long(&pv_entry_count, freed)); 5496 if (allfree) { 5497 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 5498 free_pv_chunk(pc); 5499 } 5500 } 5501 if (lock != NULL) 5502 rw_wunlock(lock); 5503 pmap_invalidate_all(pmap); 5504 PMAP_UNLOCK(pmap); 5505 vm_page_free_pages_toq(&free, true); 5506 } 5507 5508 void 5509 mmu_radix_remove_write(vm_page_t m) 5510 { 5511 struct md_page *pvh; 5512 pmap_t pmap; 5513 struct rwlock *lock; 5514 pv_entry_t next_pv, pv; 5515 pml3_entry_t *l3e; 5516 pt_entry_t oldpte, *pte; 5517 int pvh_gen, md_gen; 5518 5519 CTR2(KTR_PMAP, "%s(%p)", __func__, m); 5520 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 5521 ("pmap_remove_write: page %p is not managed", m)); 5522 vm_page_assert_busied(m); 5523 5524 if (!pmap_page_is_write_mapped(m)) 5525 return; 5526 lock = VM_PAGE_TO_PV_LIST_LOCK(m); 5527 pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy : 5528 pa_to_pvh(VM_PAGE_TO_PHYS(m)); 5529 retry_pv_loop: 5530 rw_wlock(lock); 5531 TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_link, next_pv) { 5532 pmap = PV_PMAP(pv); 5533 if (!PMAP_TRYLOCK(pmap)) { 5534 pvh_gen = pvh->pv_gen; 5535 rw_wunlock(lock); 5536 PMAP_LOCK(pmap); 5537 rw_wlock(lock); 5538 if (pvh_gen != pvh->pv_gen) { 5539 PMAP_UNLOCK(pmap); 5540 rw_wunlock(lock); 5541 goto retry_pv_loop; 5542 } 5543 } 5544 l3e = pmap_pml3e(pmap, pv->pv_va); 5545 if ((be64toh(*l3e) & PG_RW) != 0) 5546 (void)pmap_demote_l3e_locked(pmap, l3e, pv->pv_va, &lock); 5547 KASSERT(lock == VM_PAGE_TO_PV_LIST_LOCK(m), 5548 ("inconsistent pv lock %p %p for page %p", 5549 lock, VM_PAGE_TO_PV_LIST_LOCK(m), m)); 5550 PMAP_UNLOCK(pmap); 5551 } 5552 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) { 5553 pmap = PV_PMAP(pv); 5554 if (!PMAP_TRYLOCK(pmap)) { 5555 pvh_gen = pvh->pv_gen; 5556 md_gen = m->md.pv_gen; 5557 rw_wunlock(lock); 5558 PMAP_LOCK(pmap); 5559 rw_wlock(lock); 5560 if (pvh_gen != pvh->pv_gen || 5561 md_gen != m->md.pv_gen) { 5562 PMAP_UNLOCK(pmap); 5563 rw_wunlock(lock); 5564 goto retry_pv_loop; 5565 } 5566 } 5567 l3e = pmap_pml3e(pmap, pv->pv_va); 5568 KASSERT((be64toh(*l3e) & RPTE_LEAF) == 0, 5569 ("pmap_remove_write: found a 2mpage in page %p's pv list", 5570 m)); 5571 pte = pmap_l3e_to_pte(l3e, pv->pv_va); 5572 retry: 5573 oldpte = be64toh(*pte); 5574 if (oldpte & PG_RW) { 5575 if (!atomic_cmpset_long(pte, htobe64(oldpte), 5576 htobe64((oldpte | RPTE_EAA_R) & ~(PG_RW | PG_M)))) 5577 goto retry; 5578 if ((oldpte & PG_M) != 0) 5579 vm_page_dirty(m); 5580 pmap_invalidate_page(pmap, pv->pv_va); 5581 } 5582 PMAP_UNLOCK(pmap); 5583 } 5584 rw_wunlock(lock); 5585 vm_page_aflag_clear(m, PGA_WRITEABLE); 5586 } 5587 5588 /* 5589 * Clear the wired attribute from the mappings for the specified range of 5590 * addresses in the given pmap. Every valid mapping within that range 5591 * must have the wired attribute set. In contrast, invalid mappings 5592 * cannot have the wired attribute set, so they are ignored. 5593 * 5594 * The wired attribute of the page table entry is not a hardware 5595 * feature, so there is no need to invalidate any TLB entries. 5596 * Since pmap_demote_l3e() for the wired entry must never fail, 5597 * pmap_delayed_invl_started()/finished() calls around the 5598 * function are not needed. 5599 */ 5600 void 5601 mmu_radix_unwire(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) 5602 { 5603 vm_offset_t va_next; 5604 pml1_entry_t *l1e; 5605 pml2_entry_t *l2e; 5606 pml3_entry_t *l3e; 5607 pt_entry_t *pte; 5608 5609 CTR4(KTR_PMAP, "%s(%p, %#x, %#x)", __func__, pmap, sva, eva); 5610 PMAP_LOCK(pmap); 5611 for (; sva < eva; sva = va_next) { 5612 l1e = pmap_pml1e(pmap, sva); 5613 if ((be64toh(*l1e) & PG_V) == 0) { 5614 va_next = (sva + L1_PAGE_SIZE) & ~L1_PAGE_MASK; 5615 if (va_next < sva) 5616 va_next = eva; 5617 continue; 5618 } 5619 l2e = pmap_l1e_to_l2e(l1e, sva); 5620 if ((be64toh(*l2e) & PG_V) == 0) { 5621 va_next = (sva + L2_PAGE_SIZE) & ~L2_PAGE_MASK; 5622 if (va_next < sva) 5623 va_next = eva; 5624 continue; 5625 } 5626 va_next = (sva + L3_PAGE_SIZE) & ~L3_PAGE_MASK; 5627 if (va_next < sva) 5628 va_next = eva; 5629 l3e = pmap_l2e_to_l3e(l2e, sva); 5630 if ((be64toh(*l3e) & PG_V) == 0) 5631 continue; 5632 if ((be64toh(*l3e) & RPTE_LEAF) != 0) { 5633 if ((be64toh(*l3e) & PG_W) == 0) 5634 panic("pmap_unwire: pde %#jx is missing PG_W", 5635 (uintmax_t)(be64toh(*l3e))); 5636 5637 /* 5638 * Are we unwiring the entire large page? If not, 5639 * demote the mapping and fall through. 5640 */ 5641 if (sva + L3_PAGE_SIZE == va_next && eva >= va_next) { 5642 atomic_clear_long(l3e, htobe64(PG_W)); 5643 pmap->pm_stats.wired_count -= L3_PAGE_SIZE / 5644 PAGE_SIZE; 5645 continue; 5646 } else if (!pmap_demote_l3e(pmap, l3e, sva)) 5647 panic("pmap_unwire: demotion failed"); 5648 } 5649 if (va_next > eva) 5650 va_next = eva; 5651 for (pte = pmap_l3e_to_pte(l3e, sva); sva != va_next; pte++, 5652 sva += PAGE_SIZE) { 5653 MPASS(pte == pmap_pte(pmap, sva)); 5654 if ((be64toh(*pte) & PG_V) == 0) 5655 continue; 5656 if ((be64toh(*pte) & PG_W) == 0) 5657 panic("pmap_unwire: pte %#jx is missing PG_W", 5658 (uintmax_t)(be64toh(*pte))); 5659 5660 /* 5661 * PG_W must be cleared atomically. Although the pmap 5662 * lock synchronizes access to PG_W, another processor 5663 * could be setting PG_M and/or PG_A concurrently. 5664 */ 5665 atomic_clear_long(pte, htobe64(PG_W)); 5666 pmap->pm_stats.wired_count--; 5667 } 5668 } 5669 PMAP_UNLOCK(pmap); 5670 } 5671 5672 void 5673 mmu_radix_zero_page(vm_page_t m) 5674 { 5675 vm_offset_t addr; 5676 5677 CTR2(KTR_PMAP, "%s(%p)", __func__, m); 5678 addr = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)); 5679 pagezero(addr); 5680 } 5681 5682 void 5683 mmu_radix_zero_page_area(vm_page_t m, int off, int size) 5684 { 5685 caddr_t addr; 5686 5687 CTR4(KTR_PMAP, "%s(%p, %d, %d)", __func__, m, off, size); 5688 MPASS(off + size <= PAGE_SIZE); 5689 addr = (caddr_t)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)); 5690 memset(addr + off, 0, size); 5691 } 5692 5693 static int 5694 mmu_radix_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *locked_pa) 5695 { 5696 pml3_entry_t *l3ep; 5697 pt_entry_t pte; 5698 vm_paddr_t pa; 5699 int val; 5700 5701 CTR3(KTR_PMAP, "%s(%p, %#x)", __func__, pmap, addr); 5702 PMAP_LOCK(pmap); 5703 5704 l3ep = pmap_pml3e(pmap, addr); 5705 if (l3ep != NULL && (be64toh(*l3ep) & PG_V)) { 5706 if (be64toh(*l3ep) & RPTE_LEAF) { 5707 pte = be64toh(*l3ep); 5708 /* Compute the physical address of the 4KB page. */ 5709 pa = ((be64toh(*l3ep) & PG_PS_FRAME) | (addr & L3_PAGE_MASK)) & 5710 PG_FRAME; 5711 val = MINCORE_PSIND(1); 5712 } else { 5713 /* Native endian PTE, do not pass to functions */ 5714 pte = be64toh(*pmap_l3e_to_pte(l3ep, addr)); 5715 pa = pte & PG_FRAME; 5716 val = 0; 5717 } 5718 } else { 5719 pte = 0; 5720 pa = 0; 5721 val = 0; 5722 } 5723 if ((pte & PG_V) != 0) { 5724 val |= MINCORE_INCORE; 5725 if ((pte & (PG_M | PG_RW)) == (PG_M | PG_RW)) 5726 val |= MINCORE_MODIFIED | MINCORE_MODIFIED_OTHER; 5727 if ((pte & PG_A) != 0) 5728 val |= MINCORE_REFERENCED | MINCORE_REFERENCED_OTHER; 5729 } 5730 if ((val & (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER)) != 5731 (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER) && 5732 (pte & (PG_MANAGED | PG_V)) == (PG_MANAGED | PG_V)) { 5733 *locked_pa = pa; 5734 } 5735 PMAP_UNLOCK(pmap); 5736 return (val); 5737 } 5738 5739 void 5740 mmu_radix_activate(struct thread *td) 5741 { 5742 pmap_t pmap; 5743 uint32_t curpid; 5744 5745 CTR2(KTR_PMAP, "%s(%p)", __func__, td); 5746 critical_enter(); 5747 pmap = vmspace_pmap(td->td_proc->p_vmspace); 5748 curpid = mfspr(SPR_PID); 5749 if (pmap->pm_pid > isa3_base_pid && 5750 curpid != pmap->pm_pid) { 5751 mmu_radix_pid_set(pmap); 5752 } 5753 critical_exit(); 5754 } 5755 5756 /* 5757 * Increase the starting virtual address of the given mapping if a 5758 * different alignment might result in more superpage mappings. 5759 */ 5760 void 5761 mmu_radix_align_superpage(vm_object_t object, vm_ooffset_t offset, 5762 vm_offset_t *addr, vm_size_t size) 5763 { 5764 5765 CTR5(KTR_PMAP, "%s(%p, %#x, %p, %#x)", __func__, object, offset, addr, 5766 size); 5767 vm_offset_t superpage_offset; 5768 5769 if (size < L3_PAGE_SIZE) 5770 return; 5771 if (object != NULL && (object->flags & OBJ_COLORED) != 0) 5772 offset += ptoa(object->pg_color); 5773 superpage_offset = offset & L3_PAGE_MASK; 5774 if (size - ((L3_PAGE_SIZE - superpage_offset) & L3_PAGE_MASK) < L3_PAGE_SIZE || 5775 (*addr & L3_PAGE_MASK) == superpage_offset) 5776 return; 5777 if ((*addr & L3_PAGE_MASK) < superpage_offset) 5778 *addr = (*addr & ~L3_PAGE_MASK) + superpage_offset; 5779 else 5780 *addr = ((*addr + L3_PAGE_MASK) & ~L3_PAGE_MASK) + superpage_offset; 5781 } 5782 5783 static void * 5784 mmu_radix_mapdev_attr(vm_paddr_t pa, vm_size_t size, vm_memattr_t attr) 5785 { 5786 vm_offset_t va, tmpva, ppa, offset; 5787 5788 ppa = trunc_page(pa); 5789 offset = pa & PAGE_MASK; 5790 size = roundup2(offset + size, PAGE_SIZE); 5791 if (pa < powerpc_ptob(Maxmem)) 5792 panic("bad pa: %#lx less than Maxmem %#lx\n", 5793 pa, powerpc_ptob(Maxmem)); 5794 va = kva_alloc(size); 5795 if (bootverbose) 5796 printf("%s(%#lx, %lu, %d)\n", __func__, pa, size, attr); 5797 KASSERT(size > 0, ("%s(%#lx, %lu, %d)", __func__, pa, size, attr)); 5798 5799 if (!va) 5800 panic("%s: Couldn't alloc kernel virtual memory", __func__); 5801 5802 for (tmpva = va; size > 0;) { 5803 mmu_radix_kenter_attr(tmpva, ppa, attr); 5804 size -= PAGE_SIZE; 5805 tmpva += PAGE_SIZE; 5806 ppa += PAGE_SIZE; 5807 } 5808 ptesync(); 5809 5810 return ((void *)(va + offset)); 5811 } 5812 5813 static void * 5814 mmu_radix_mapdev(vm_paddr_t pa, vm_size_t size) 5815 { 5816 5817 CTR3(KTR_PMAP, "%s(%#x, %#x)", __func__, pa, size); 5818 5819 return (mmu_radix_mapdev_attr(pa, size, VM_MEMATTR_DEFAULT)); 5820 } 5821 5822 void 5823 mmu_radix_page_set_memattr(vm_page_t m, vm_memattr_t ma) 5824 { 5825 5826 CTR3(KTR_PMAP, "%s(%p, %#x)", __func__, m, ma); 5827 m->md.mdpg_cache_attrs = ma; 5828 5829 /* 5830 * If "m" is a normal page, update its direct mapping. This update 5831 * can be relied upon to perform any cache operations that are 5832 * required for data coherence. 5833 */ 5834 if ((m->flags & PG_FICTITIOUS) == 0 && 5835 mmu_radix_change_attr(PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)), 5836 PAGE_SIZE, m->md.mdpg_cache_attrs)) 5837 panic("memory attribute change on the direct map failed"); 5838 } 5839 5840 static void 5841 mmu_radix_unmapdev(vm_offset_t va, vm_size_t size) 5842 { 5843 vm_offset_t offset; 5844 5845 CTR3(KTR_PMAP, "%s(%#x, %#x)", __func__, va, size); 5846 /* If we gave a direct map region in pmap_mapdev, do nothing */ 5847 if (va >= DMAP_MIN_ADDRESS && va < DMAP_MAX_ADDRESS) 5848 return; 5849 5850 offset = va & PAGE_MASK; 5851 size = round_page(offset + size); 5852 va = trunc_page(va); 5853 5854 if (pmap_initialized) { 5855 mmu_radix_qremove(va, atop(size)); 5856 kva_free(va, size); 5857 } 5858 } 5859 5860 static __inline void 5861 pmap_pte_attr(pt_entry_t *pte, uint64_t cache_bits, uint64_t mask) 5862 { 5863 uint64_t opte, npte; 5864 5865 /* 5866 * The cache mode bits are all in the low 32-bits of the 5867 * PTE, so we can just spin on updating the low 32-bits. 5868 */ 5869 do { 5870 opte = be64toh(*pte); 5871 npte = opte & ~mask; 5872 npte |= cache_bits; 5873 } while (npte != opte && !atomic_cmpset_long(pte, htobe64(opte), htobe64(npte))); 5874 } 5875 5876 /* 5877 * Tries to demote a 1GB page mapping. 5878 */ 5879 static boolean_t 5880 pmap_demote_l2e(pmap_t pmap, pml2_entry_t *l2e, vm_offset_t va) 5881 { 5882 pml2_entry_t oldpdpe; 5883 pml3_entry_t *firstpde, newpde, *pde; 5884 vm_paddr_t pdpgpa; 5885 vm_page_t pdpg; 5886 5887 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 5888 oldpdpe = be64toh(*l2e); 5889 KASSERT((oldpdpe & (RPTE_LEAF | PG_V)) == (RPTE_LEAF | PG_V), 5890 ("pmap_demote_pdpe: oldpdpe is missing PG_PS and/or PG_V")); 5891 pdpg = vm_page_alloc(NULL, va >> L2_PAGE_SIZE_SHIFT, 5892 VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED); 5893 if (pdpg == NULL) { 5894 CTR2(KTR_PMAP, "pmap_demote_pdpe: failure for va %#lx" 5895 " in pmap %p", va, pmap); 5896 return (FALSE); 5897 } 5898 pdpgpa = VM_PAGE_TO_PHYS(pdpg); 5899 firstpde = (pml3_entry_t *)PHYS_TO_DMAP(pdpgpa); 5900 KASSERT((oldpdpe & PG_A) != 0, 5901 ("pmap_demote_pdpe: oldpdpe is missing PG_A")); 5902 KASSERT((oldpdpe & (PG_M | PG_RW)) != PG_RW, 5903 ("pmap_demote_pdpe: oldpdpe is missing PG_M")); 5904 newpde = oldpdpe; 5905 5906 /* 5907 * Initialize the page directory page. 5908 */ 5909 for (pde = firstpde; pde < firstpde + NPDEPG; pde++) { 5910 *pde = htobe64(newpde); 5911 newpde += L3_PAGE_SIZE; 5912 } 5913 5914 /* 5915 * Demote the mapping. 5916 */ 5917 pde_store(l2e, pdpgpa); 5918 5919 /* 5920 * Flush PWC --- XXX revisit 5921 */ 5922 pmap_invalidate_all(pmap); 5923 5924 pmap_l2e_demotions++; 5925 CTR2(KTR_PMAP, "pmap_demote_pdpe: success for va %#lx" 5926 " in pmap %p", va, pmap); 5927 return (TRUE); 5928 } 5929 5930 vm_paddr_t 5931 mmu_radix_kextract(vm_offset_t va) 5932 { 5933 pml3_entry_t l3e; 5934 vm_paddr_t pa; 5935 5936 CTR2(KTR_PMAP, "%s(%#x)", __func__, va); 5937 if (va >= DMAP_MIN_ADDRESS && va < DMAP_MAX_ADDRESS) { 5938 pa = DMAP_TO_PHYS(va); 5939 } else { 5940 /* Big-endian PTE on stack */ 5941 l3e = *pmap_pml3e(kernel_pmap, va); 5942 if (be64toh(l3e) & RPTE_LEAF) { 5943 pa = (be64toh(l3e) & PG_PS_FRAME) | (va & L3_PAGE_MASK); 5944 pa |= (va & L3_PAGE_MASK); 5945 } else { 5946 /* 5947 * Beware of a concurrent promotion that changes the 5948 * PDE at this point! For example, vtopte() must not 5949 * be used to access the PTE because it would use the 5950 * new PDE. It is, however, safe to use the old PDE 5951 * because the page table page is preserved by the 5952 * promotion. 5953 */ 5954 pa = be64toh(*pmap_l3e_to_pte(&l3e, va)); 5955 pa = (pa & PG_FRAME) | (va & PAGE_MASK); 5956 pa |= (va & PAGE_MASK); 5957 } 5958 } 5959 return (pa); 5960 } 5961 5962 static pt_entry_t 5963 mmu_radix_calc_wimg(vm_paddr_t pa, vm_memattr_t ma) 5964 { 5965 5966 if (ma != VM_MEMATTR_DEFAULT) { 5967 return pmap_cache_bits(ma); 5968 } 5969 5970 /* 5971 * Assume the page is cache inhibited and access is guarded unless 5972 * it's in our available memory array. 5973 */ 5974 for (int i = 0; i < pregions_sz; i++) { 5975 if ((pa >= pregions[i].mr_start) && 5976 (pa < (pregions[i].mr_start + pregions[i].mr_size))) 5977 return (RPTE_ATTR_MEM); 5978 } 5979 return (RPTE_ATTR_GUARDEDIO); 5980 } 5981 5982 static void 5983 mmu_radix_kenter_attr(vm_offset_t va, vm_paddr_t pa, vm_memattr_t ma) 5984 { 5985 pt_entry_t *pte, pteval; 5986 uint64_t cache_bits; 5987 5988 pte = kvtopte(va); 5989 MPASS(pte != NULL); 5990 pteval = pa | RPTE_EAA_R | RPTE_EAA_W | RPTE_EAA_P | PG_M | PG_A; 5991 cache_bits = mmu_radix_calc_wimg(pa, ma); 5992 pte_store(pte, pteval | cache_bits); 5993 } 5994 5995 void 5996 mmu_radix_kremove(vm_offset_t va) 5997 { 5998 pt_entry_t *pte; 5999 6000 CTR2(KTR_PMAP, "%s(%#x)", __func__, va); 6001 6002 pte = kvtopte(va); 6003 pte_clear(pte); 6004 } 6005 6006 int 6007 mmu_radix_decode_kernel_ptr(vm_offset_t addr, 6008 int *is_user, vm_offset_t *decoded) 6009 { 6010 6011 CTR2(KTR_PMAP, "%s(%#jx)", __func__, (uintmax_t)addr); 6012 *decoded = addr; 6013 *is_user = (addr < VM_MAXUSER_ADDRESS); 6014 return (0); 6015 } 6016 6017 static boolean_t 6018 mmu_radix_dev_direct_mapped(vm_paddr_t pa, vm_size_t size) 6019 { 6020 6021 CTR3(KTR_PMAP, "%s(%#x, %#x)", __func__, pa, size); 6022 return (mem_valid(pa, size)); 6023 } 6024 6025 static void 6026 mmu_radix_scan_init() 6027 { 6028 6029 CTR1(KTR_PMAP, "%s()", __func__); 6030 UNIMPLEMENTED(); 6031 } 6032 6033 static void 6034 mmu_radix_dumpsys_map(vm_paddr_t pa, size_t sz, 6035 void **va) 6036 { 6037 CTR4(KTR_PMAP, "%s(%#jx, %#zx, %p)", __func__, (uintmax_t)pa, sz, va); 6038 UNIMPLEMENTED(); 6039 } 6040 6041 vm_offset_t 6042 mmu_radix_quick_enter_page(vm_page_t m) 6043 { 6044 vm_paddr_t paddr; 6045 6046 CTR2(KTR_PMAP, "%s(%p)", __func__, m); 6047 paddr = VM_PAGE_TO_PHYS(m); 6048 return (PHYS_TO_DMAP(paddr)); 6049 } 6050 6051 void 6052 mmu_radix_quick_remove_page(vm_offset_t addr __unused) 6053 { 6054 /* no work to do here */ 6055 CTR2(KTR_PMAP, "%s(%#x)", __func__, addr); 6056 } 6057 6058 static void 6059 pmap_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva) 6060 { 6061 cpu_flush_dcache((void *)sva, eva - sva); 6062 } 6063 6064 int 6065 mmu_radix_change_attr(vm_offset_t va, vm_size_t size, 6066 vm_memattr_t mode) 6067 { 6068 int error; 6069 6070 CTR4(KTR_PMAP, "%s(%#x, %#zx, %d)", __func__, va, size, mode); 6071 PMAP_LOCK(kernel_pmap); 6072 error = pmap_change_attr_locked(va, size, mode, true); 6073 PMAP_UNLOCK(kernel_pmap); 6074 return (error); 6075 } 6076 6077 static int 6078 pmap_change_attr_locked(vm_offset_t va, vm_size_t size, int mode, bool flush) 6079 { 6080 vm_offset_t base, offset, tmpva; 6081 vm_paddr_t pa_start, pa_end, pa_end1; 6082 pml2_entry_t *l2e; 6083 pml3_entry_t *l3e; 6084 pt_entry_t *pte; 6085 int cache_bits, error; 6086 boolean_t changed; 6087 6088 PMAP_LOCK_ASSERT(kernel_pmap, MA_OWNED); 6089 base = trunc_page(va); 6090 offset = va & PAGE_MASK; 6091 size = round_page(offset + size); 6092 6093 /* 6094 * Only supported on kernel virtual addresses, including the direct 6095 * map but excluding the recursive map. 6096 */ 6097 if (base < DMAP_MIN_ADDRESS) 6098 return (EINVAL); 6099 6100 cache_bits = pmap_cache_bits(mode); 6101 changed = FALSE; 6102 6103 /* 6104 * Pages that aren't mapped aren't supported. Also break down 2MB pages 6105 * into 4KB pages if required. 6106 */ 6107 for (tmpva = base; tmpva < base + size; ) { 6108 l2e = pmap_pml2e(kernel_pmap, tmpva); 6109 if (l2e == NULL || *l2e == 0) 6110 return (EINVAL); 6111 if (be64toh(*l2e) & RPTE_LEAF) { 6112 /* 6113 * If the current 1GB page already has the required 6114 * memory type, then we need not demote this page. Just 6115 * increment tmpva to the next 1GB page frame. 6116 */ 6117 if ((be64toh(*l2e) & RPTE_ATTR_MASK) == cache_bits) { 6118 tmpva = trunc_1gpage(tmpva) + L2_PAGE_SIZE; 6119 continue; 6120 } 6121 6122 /* 6123 * If the current offset aligns with a 1GB page frame 6124 * and there is at least 1GB left within the range, then 6125 * we need not break down this page into 2MB pages. 6126 */ 6127 if ((tmpva & L2_PAGE_MASK) == 0 && 6128 tmpva + L2_PAGE_MASK < base + size) { 6129 tmpva += L2_PAGE_MASK; 6130 continue; 6131 } 6132 if (!pmap_demote_l2e(kernel_pmap, l2e, tmpva)) 6133 return (ENOMEM); 6134 } 6135 l3e = pmap_l2e_to_l3e(l2e, tmpva); 6136 KASSERT(l3e != NULL, ("no l3e entry for %#lx in %p\n", 6137 tmpva, l2e)); 6138 if (*l3e == 0) 6139 return (EINVAL); 6140 if (be64toh(*l3e) & RPTE_LEAF) { 6141 /* 6142 * If the current 2MB page already has the required 6143 * memory type, then we need not demote this page. Just 6144 * increment tmpva to the next 2MB page frame. 6145 */ 6146 if ((be64toh(*l3e) & RPTE_ATTR_MASK) == cache_bits) { 6147 tmpva = trunc_2mpage(tmpva) + L3_PAGE_SIZE; 6148 continue; 6149 } 6150 6151 /* 6152 * If the current offset aligns with a 2MB page frame 6153 * and there is at least 2MB left within the range, then 6154 * we need not break down this page into 4KB pages. 6155 */ 6156 if ((tmpva & L3_PAGE_MASK) == 0 && 6157 tmpva + L3_PAGE_MASK < base + size) { 6158 tmpva += L3_PAGE_SIZE; 6159 continue; 6160 } 6161 if (!pmap_demote_l3e(kernel_pmap, l3e, tmpva)) 6162 return (ENOMEM); 6163 } 6164 pte = pmap_l3e_to_pte(l3e, tmpva); 6165 if (*pte == 0) 6166 return (EINVAL); 6167 tmpva += PAGE_SIZE; 6168 } 6169 error = 0; 6170 6171 /* 6172 * Ok, all the pages exist, so run through them updating their 6173 * cache mode if required. 6174 */ 6175 pa_start = pa_end = 0; 6176 for (tmpva = base; tmpva < base + size; ) { 6177 l2e = pmap_pml2e(kernel_pmap, tmpva); 6178 if (be64toh(*l2e) & RPTE_LEAF) { 6179 if ((be64toh(*l2e) & RPTE_ATTR_MASK) != cache_bits) { 6180 pmap_pte_attr(l2e, cache_bits, 6181 RPTE_ATTR_MASK); 6182 changed = TRUE; 6183 } 6184 if (tmpva >= VM_MIN_KERNEL_ADDRESS && 6185 (*l2e & PG_PS_FRAME) < dmaplimit) { 6186 if (pa_start == pa_end) { 6187 /* Start physical address run. */ 6188 pa_start = be64toh(*l2e) & PG_PS_FRAME; 6189 pa_end = pa_start + L2_PAGE_SIZE; 6190 } else if (pa_end == (be64toh(*l2e) & PG_PS_FRAME)) 6191 pa_end += L2_PAGE_SIZE; 6192 else { 6193 /* Run ended, update direct map. */ 6194 error = pmap_change_attr_locked( 6195 PHYS_TO_DMAP(pa_start), 6196 pa_end - pa_start, mode, flush); 6197 if (error != 0) 6198 break; 6199 /* Start physical address run. */ 6200 pa_start = be64toh(*l2e) & PG_PS_FRAME; 6201 pa_end = pa_start + L2_PAGE_SIZE; 6202 } 6203 } 6204 tmpva = trunc_1gpage(tmpva) + L2_PAGE_SIZE; 6205 continue; 6206 } 6207 l3e = pmap_l2e_to_l3e(l2e, tmpva); 6208 if (be64toh(*l3e) & RPTE_LEAF) { 6209 if ((be64toh(*l3e) & RPTE_ATTR_MASK) != cache_bits) { 6210 pmap_pte_attr(l3e, cache_bits, 6211 RPTE_ATTR_MASK); 6212 changed = TRUE; 6213 } 6214 if (tmpva >= VM_MIN_KERNEL_ADDRESS && 6215 (be64toh(*l3e) & PG_PS_FRAME) < dmaplimit) { 6216 if (pa_start == pa_end) { 6217 /* Start physical address run. */ 6218 pa_start = be64toh(*l3e) & PG_PS_FRAME; 6219 pa_end = pa_start + L3_PAGE_SIZE; 6220 } else if (pa_end == (be64toh(*l3e) & PG_PS_FRAME)) 6221 pa_end += L3_PAGE_SIZE; 6222 else { 6223 /* Run ended, update direct map. */ 6224 error = pmap_change_attr_locked( 6225 PHYS_TO_DMAP(pa_start), 6226 pa_end - pa_start, mode, flush); 6227 if (error != 0) 6228 break; 6229 /* Start physical address run. */ 6230 pa_start = be64toh(*l3e) & PG_PS_FRAME; 6231 pa_end = pa_start + L3_PAGE_SIZE; 6232 } 6233 } 6234 tmpva = trunc_2mpage(tmpva) + L3_PAGE_SIZE; 6235 } else { 6236 pte = pmap_l3e_to_pte(l3e, tmpva); 6237 if ((be64toh(*pte) & RPTE_ATTR_MASK) != cache_bits) { 6238 pmap_pte_attr(pte, cache_bits, 6239 RPTE_ATTR_MASK); 6240 changed = TRUE; 6241 } 6242 if (tmpva >= VM_MIN_KERNEL_ADDRESS && 6243 (be64toh(*pte) & PG_FRAME) < dmaplimit) { 6244 if (pa_start == pa_end) { 6245 /* Start physical address run. */ 6246 pa_start = be64toh(*pte) & PG_FRAME; 6247 pa_end = pa_start + PAGE_SIZE; 6248 } else if (pa_end == (be64toh(*pte) & PG_FRAME)) 6249 pa_end += PAGE_SIZE; 6250 else { 6251 /* Run ended, update direct map. */ 6252 error = pmap_change_attr_locked( 6253 PHYS_TO_DMAP(pa_start), 6254 pa_end - pa_start, mode, flush); 6255 if (error != 0) 6256 break; 6257 /* Start physical address run. */ 6258 pa_start = be64toh(*pte) & PG_FRAME; 6259 pa_end = pa_start + PAGE_SIZE; 6260 } 6261 } 6262 tmpva += PAGE_SIZE; 6263 } 6264 } 6265 if (error == 0 && pa_start != pa_end && pa_start < dmaplimit) { 6266 pa_end1 = MIN(pa_end, dmaplimit); 6267 if (pa_start != pa_end1) 6268 error = pmap_change_attr_locked(PHYS_TO_DMAP(pa_start), 6269 pa_end1 - pa_start, mode, flush); 6270 } 6271 6272 /* 6273 * Flush CPU caches if required to make sure any data isn't cached that 6274 * shouldn't be, etc. 6275 */ 6276 if (changed) { 6277 pmap_invalidate_all(kernel_pmap); 6278 6279 if (flush) 6280 pmap_invalidate_cache_range(base, tmpva); 6281 } 6282 return (error); 6283 } 6284 6285 /* 6286 * Allocate physical memory for the vm_page array and map it into KVA, 6287 * attempting to back the vm_pages with domain-local memory. 6288 */ 6289 void 6290 mmu_radix_page_array_startup(long pages) 6291 { 6292 #ifdef notyet 6293 pml2_entry_t *l2e; 6294 pml3_entry_t *pde; 6295 pml3_entry_t newl3; 6296 vm_offset_t va; 6297 long pfn; 6298 int domain, i; 6299 #endif 6300 vm_paddr_t pa; 6301 vm_offset_t start, end; 6302 6303 vm_page_array_size = pages; 6304 6305 start = VM_MIN_KERNEL_ADDRESS; 6306 end = start + pages * sizeof(struct vm_page); 6307 6308 pa = vm_phys_early_alloc(0, end - start); 6309 6310 start = mmu_radix_map(&start, pa, end - start, VM_MEMATTR_DEFAULT); 6311 #ifdef notyet 6312 /* TODO: NUMA vm_page_array. Blocked out until then (copied from amd64). */ 6313 for (va = start; va < end; va += L3_PAGE_SIZE) { 6314 pfn = first_page + (va - start) / sizeof(struct vm_page); 6315 domain = vm_phys_domain(ptoa(pfn)); 6316 l2e = pmap_pml2e(kernel_pmap, va); 6317 if ((be64toh(*l2e) & PG_V) == 0) { 6318 pa = vm_phys_early_alloc(domain, PAGE_SIZE); 6319 dump_add_page(pa); 6320 pagezero(PHYS_TO_DMAP(pa)); 6321 pde_store(l2e, (pml2_entry_t)pa); 6322 } 6323 pde = pmap_l2e_to_l3e(l2e, va); 6324 if ((be64toh(*pde) & PG_V) != 0) 6325 panic("Unexpected pde %p", pde); 6326 pa = vm_phys_early_alloc(domain, L3_PAGE_SIZE); 6327 for (i = 0; i < NPDEPG; i++) 6328 dump_add_page(pa + i * PAGE_SIZE); 6329 newl3 = (pml3_entry_t)(pa | RPTE_EAA_P | RPTE_EAA_R | RPTE_EAA_W); 6330 pte_store(pde, newl3); 6331 } 6332 #endif 6333 vm_page_array = (vm_page_t)start; 6334 } 6335 6336 #ifdef DDB 6337 #include <sys/kdb.h> 6338 #include <ddb/ddb.h> 6339 6340 static void 6341 pmap_pte_walk(pml1_entry_t *l1, vm_offset_t va) 6342 { 6343 pml1_entry_t *l1e; 6344 pml2_entry_t *l2e; 6345 pml3_entry_t *l3e; 6346 pt_entry_t *pte; 6347 6348 l1e = &l1[pmap_pml1e_index(va)]; 6349 db_printf("VA %#016lx l1e %#016lx", va, be64toh(*l1e)); 6350 if ((be64toh(*l1e) & PG_V) == 0) { 6351 db_printf("\n"); 6352 return; 6353 } 6354 l2e = pmap_l1e_to_l2e(l1e, va); 6355 db_printf(" l2e %#016lx", be64toh(*l2e)); 6356 if ((be64toh(*l2e) & PG_V) == 0 || (be64toh(*l2e) & RPTE_LEAF) != 0) { 6357 db_printf("\n"); 6358 return; 6359 } 6360 l3e = pmap_l2e_to_l3e(l2e, va); 6361 db_printf(" l3e %#016lx", be64toh(*l3e)); 6362 if ((be64toh(*l3e) & PG_V) == 0 || (be64toh(*l3e) & RPTE_LEAF) != 0) { 6363 db_printf("\n"); 6364 return; 6365 } 6366 pte = pmap_l3e_to_pte(l3e, va); 6367 db_printf(" pte %#016lx\n", be64toh(*pte)); 6368 } 6369 6370 void 6371 pmap_page_print_mappings(vm_page_t m) 6372 { 6373 pmap_t pmap; 6374 pv_entry_t pv; 6375 6376 db_printf("page %p(%lx)\n", m, m->phys_addr); 6377 /* need to elide locks if running in ddb */ 6378 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) { 6379 db_printf("pv: %p ", pv); 6380 db_printf("va: %#016lx ", pv->pv_va); 6381 pmap = PV_PMAP(pv); 6382 db_printf("pmap %p ", pmap); 6383 if (pmap != NULL) { 6384 db_printf("asid: %lu\n", pmap->pm_pid); 6385 pmap_pte_walk(pmap->pm_pml1, pv->pv_va); 6386 } 6387 } 6388 } 6389 6390 DB_SHOW_COMMAND(pte, pmap_print_pte) 6391 { 6392 vm_offset_t va; 6393 pmap_t pmap; 6394 6395 if (!have_addr) { 6396 db_printf("show pte addr\n"); 6397 return; 6398 } 6399 va = (vm_offset_t)addr; 6400 6401 if (va >= DMAP_MIN_ADDRESS) 6402 pmap = kernel_pmap; 6403 else if (kdb_thread != NULL) 6404 pmap = vmspace_pmap(kdb_thread->td_proc->p_vmspace); 6405 else 6406 pmap = vmspace_pmap(curthread->td_proc->p_vmspace); 6407 6408 pmap_pte_walk(pmap->pm_pml1, va); 6409 } 6410 6411 #endif 6412