1 /*- 2 * Copyright (c) 1991 Regents of the University of California. 3 * All rights reserved. 4 * Copyright (c) 1994 John S. Dyson 5 * All rights reserved. 6 * Copyright (c) 1994 David Greenman 7 * All rights reserved. 8 * Copyright (c) 2003 Peter Wemm 9 * All rights reserved. 10 * Copyright (c) 2005-2010 Alan L. Cox <alc@cs.rice.edu> 11 * All rights reserved. 12 * Copyright (c) 2014 Andrew Turner 13 * All rights reserved. 14 * Copyright (c) 2014-2016 The FreeBSD Foundation 15 * All rights reserved. 16 * 17 * This code is derived from software contributed to Berkeley by 18 * the Systems Programming Group of the University of Utah Computer 19 * Science Department and William Jolitz of UUNET Technologies Inc. 20 * 21 * This software was developed by Andrew Turner under sponsorship from 22 * the FreeBSD Foundation. 23 * 24 * Redistribution and use in source and binary forms, with or without 25 * modification, are permitted provided that the following conditions 26 * are met: 27 * 1. Redistributions of source code must retain the above copyright 28 * notice, this list of conditions and the following disclaimer. 29 * 2. Redistributions in binary form must reproduce the above copyright 30 * notice, this list of conditions and the following disclaimer in the 31 * documentation and/or other materials provided with the distribution. 32 * 3. All advertising materials mentioning features or use of this software 33 * must display the following acknowledgement: 34 * This product includes software developed by the University of 35 * California, Berkeley and its contributors. 36 * 4. Neither the name of the University nor the names of its contributors 37 * may be used to endorse or promote products derived from this software 38 * without specific prior written permission. 39 * 40 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 41 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 42 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 43 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 44 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 45 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 46 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 47 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 48 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 49 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 50 * SUCH DAMAGE. 51 */ 52 /*- 53 * Copyright (c) 2003 Networks Associates Technology, Inc. 54 * All rights reserved. 55 * 56 * This software was developed for the FreeBSD Project by Jake Burkholder, 57 * Safeport Network Services, and Network Associates Laboratories, the 58 * Security Research Division of Network Associates, Inc. under 59 * DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"), as part of the DARPA 60 * CHATS research program. 61 * 62 * Redistribution and use in source and binary forms, with or without 63 * modification, are permitted provided that the following conditions 64 * are met: 65 * 1. Redistributions of source code must retain the above copyright 66 * notice, this list of conditions and the following disclaimer. 67 * 2. Redistributions in binary form must reproduce the above copyright 68 * notice, this list of conditions and the following disclaimer in the 69 * documentation and/or other materials provided with the distribution. 70 * 71 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 72 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 73 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 74 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 75 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 76 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 77 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 78 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 79 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 80 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 81 * SUCH DAMAGE. 82 */ 83 84 #include <sys/cdefs.h> 85 /* 86 * Manages physical address maps. 87 * 88 * Since the information managed by this module is 89 * also stored by the logical address mapping module, 90 * this module may throw away valid virtual-to-physical 91 * mappings at almost any time. However, invalidations 92 * of virtual-to-physical mappings must be done as 93 * requested. 94 * 95 * In order to cope with hardware architectures which 96 * make virtual-to-physical map invalidates expensive, 97 * this module may delay invalidate or reduced protection 98 * operations until such time as they are actually 99 * necessary. This module is given full information as 100 * to which processors are currently using which maps, 101 * and to when physical maps must be made correct. 102 */ 103 104 #include "opt_vm.h" 105 106 #include <sys/param.h> 107 #include <sys/asan.h> 108 #include <sys/bitstring.h> 109 #include <sys/bus.h> 110 #include <sys/systm.h> 111 #include <sys/kernel.h> 112 #include <sys/ktr.h> 113 #include <sys/limits.h> 114 #include <sys/lock.h> 115 #include <sys/malloc.h> 116 #include <sys/mman.h> 117 #include <sys/msan.h> 118 #include <sys/msgbuf.h> 119 #include <sys/mutex.h> 120 #include <sys/physmem.h> 121 #include <sys/proc.h> 122 #include <sys/rangeset.h> 123 #include <sys/rwlock.h> 124 #include <sys/sbuf.h> 125 #include <sys/sx.h> 126 #include <sys/vmem.h> 127 #include <sys/vmmeter.h> 128 #include <sys/sched.h> 129 #include <sys/sysctl.h> 130 #include <sys/_unrhdr.h> 131 #include <sys/smp.h> 132 133 #include <vm/vm.h> 134 #include <vm/vm_param.h> 135 #include <vm/vm_kern.h> 136 #include <vm/vm_page.h> 137 #include <vm/vm_map.h> 138 #include <vm/vm_object.h> 139 #include <vm/vm_extern.h> 140 #include <vm/vm_pageout.h> 141 #include <vm/vm_pager.h> 142 #include <vm/vm_phys.h> 143 #include <vm/vm_radix.h> 144 #include <vm/vm_reserv.h> 145 #include <vm/vm_dumpset.h> 146 #include <vm/uma.h> 147 148 #include <machine/asan.h> 149 #include <machine/cpu.h> 150 #include <machine/cpu_feat.h> 151 #include <machine/elf.h> 152 #include <machine/ifunc.h> 153 #include <machine/machdep.h> 154 #include <machine/md_var.h> 155 #include <machine/pcb.h> 156 #include <machine/rsi.h> 157 158 #ifdef NUMA 159 #define PMAP_MEMDOM MAXMEMDOM 160 #else 161 #define PMAP_MEMDOM 1 162 #endif 163 164 #define PMAP_ASSERT_STAGE1(pmap) MPASS((pmap)->pm_stage == PM_STAGE1) 165 #define PMAP_ASSERT_STAGE2(pmap) MPASS((pmap)->pm_stage == PM_STAGE2) 166 167 #define NL0PG (PAGE_SIZE/(sizeof (pd_entry_t))) 168 #define NL1PG (PAGE_SIZE/(sizeof (pd_entry_t))) 169 #define NL2PG (PAGE_SIZE/(sizeof (pd_entry_t))) 170 #define NL3PG (PAGE_SIZE/(sizeof (pt_entry_t))) 171 172 #define NUL0E L0_ENTRIES 173 #define NUL1E (NUL0E * NL1PG) 174 #define NUL2E (NUL1E * NL2PG) 175 176 #ifdef PV_STATS 177 #define PV_STAT(x) do { x ; } while (0) 178 #define __pvused 179 #else 180 #define PV_STAT(x) do { } while (0) 181 #define __pvused __unused 182 #endif 183 184 #define pmap_l0_pindex(v) (NUL2E + NUL1E + ((v) >> L0_SHIFT)) 185 #define pmap_l1_pindex(v) (NUL2E + ((v) >> L1_SHIFT)) 186 #define pmap_l2_pindex(v) ((v) >> L2_SHIFT) 187 188 #ifdef __ARM_FEATURE_BTI_DEFAULT 189 pt_entry_t __read_mostly pmap_gp_attr; 190 #define ATTR_KERN_GP pmap_gp_attr 191 #else 192 #define ATTR_KERN_GP 0 193 #endif 194 #define PMAP_SAN_PTE_BITS (ATTR_AF | ATTR_S1_XN | pmap_sh_attr | \ 195 ATTR_KERN_GP | ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK) | ATTR_S1_AP(ATTR_S1_AP_RW)) 196 197 static bool __read_mostly pmap_multiple_tlbi = false; 198 199 struct pmap_large_md_page { 200 struct rwlock pv_lock; 201 struct md_page pv_page; 202 /* Pad to a power of 2, see pmap_init_pv_table(). */ 203 int pv_pad[2]; 204 }; 205 206 __exclusive_cache_line static struct pmap_large_md_page pv_dummy_large; 207 #define pv_dummy pv_dummy_large.pv_page 208 __read_mostly static struct pmap_large_md_page *pv_table; 209 210 __read_mostly uint64_t prot_ns_shared_pa; 211 212 static struct pmap_large_md_page * 213 _pa_to_pmdp(vm_paddr_t pa) 214 { 215 struct vm_phys_seg *seg; 216 217 if ((seg = vm_phys_paddr_to_seg(pa)) != NULL) 218 return ((struct pmap_large_md_page *)seg->md_first + 219 pmap_l2_pindex(pa) - pmap_l2_pindex(seg->start)); 220 return (NULL); 221 } 222 223 static struct pmap_large_md_page * 224 pa_to_pmdp(vm_paddr_t pa) 225 { 226 struct pmap_large_md_page *pvd; 227 228 pvd = _pa_to_pmdp(pa); 229 if (pvd == NULL) 230 panic("pa 0x%jx not within vm_phys_segs", (uintmax_t)pa); 231 return (pvd); 232 } 233 234 static struct pmap_large_md_page * 235 page_to_pmdp(vm_page_t m) 236 { 237 struct vm_phys_seg *seg; 238 239 seg = &vm_phys_segs[m->segind]; 240 return ((struct pmap_large_md_page *)seg->md_first + 241 pmap_l2_pindex(VM_PAGE_TO_PHYS(m)) - pmap_l2_pindex(seg->start)); 242 } 243 244 #define pa_to_pvh(pa) (&(pa_to_pmdp(pa)->pv_page)) 245 #define page_to_pvh(m) (&(page_to_pmdp(m)->pv_page)) 246 247 #define PHYS_TO_PV_LIST_LOCK(pa) ({ \ 248 struct pmap_large_md_page *_pvd; \ 249 struct rwlock *_lock; \ 250 _pvd = _pa_to_pmdp(pa); \ 251 if (__predict_false(_pvd == NULL)) \ 252 _lock = &pv_dummy_large.pv_lock; \ 253 else \ 254 _lock = &(_pvd->pv_lock); \ 255 _lock; \ 256 }) 257 258 static struct rwlock * 259 VM_PAGE_TO_PV_LIST_LOCK(vm_page_t m) 260 { 261 if ((m->flags & PG_FICTITIOUS) == 0) 262 return (&page_to_pmdp(m)->pv_lock); 263 else 264 return (&pv_dummy_large.pv_lock); 265 } 266 267 #define CHANGE_PV_LIST_LOCK(lockp, new_lock) do { \ 268 struct rwlock **_lockp = (lockp); \ 269 struct rwlock *_new_lock = (new_lock); \ 270 \ 271 if (_new_lock != *_lockp) { \ 272 if (*_lockp != NULL) \ 273 rw_wunlock(*_lockp); \ 274 *_lockp = _new_lock; \ 275 rw_wlock(*_lockp); \ 276 } \ 277 } while (0) 278 279 #define CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa) \ 280 CHANGE_PV_LIST_LOCK(lockp, PHYS_TO_PV_LIST_LOCK(pa)) 281 282 #define CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m) \ 283 CHANGE_PV_LIST_LOCK(lockp, VM_PAGE_TO_PV_LIST_LOCK(m)) 284 285 #define RELEASE_PV_LIST_LOCK(lockp) do { \ 286 struct rwlock **_lockp = (lockp); \ 287 \ 288 if (*_lockp != NULL) { \ 289 rw_wunlock(*_lockp); \ 290 *_lockp = NULL; \ 291 } \ 292 } while (0) 293 294 #define PTE_TO_VM_PAGE(pte) PHYS_TO_VM_PAGE(PTE_TO_PHYS(pte)) 295 #define VM_PAGE_TO_PTE(m) PHYS_TO_PTE(VM_PAGE_TO_PHYS(m)) 296 297 static struct mtx cmap_lock; 298 static void *cmap1_addr; 299 static pt_entry_t *cmap1_pte; 300 301 /* 302 * The presence of this flag indicates that the mapping is writeable. 303 * If the ATTR_S1_AP_RO bit is also set, then the mapping is clean, otherwise 304 * it is dirty. This flag may only be set on managed mappings. 305 * 306 * The DBM bit is reserved on ARMv8.0 but it seems we can safely treat it 307 * as a software managed bit. 308 */ 309 #define ATTR_SW_DBM ATTR_DBM 310 311 struct pmap kernel_pmap_store; 312 313 /* Used for mapping ACPI memory before VM is initialized */ 314 #define PMAP_PREINIT_MAPPING_COUNT 32 315 #define PMAP_PREINIT_MAPPING_SIZE (PMAP_PREINIT_MAPPING_COUNT * L2_SIZE) 316 static vm_offset_t preinit_map_va; /* Start VA of pre-init mapping space */ 317 static int vm_initialized = 0; /* No need to use pre-init maps when set */ 318 319 /* 320 * Reserve a few L2 blocks starting from 'preinit_map_va' pointer. 321 * Always map entire L2 block for simplicity. 322 * VA of L2 block = preinit_map_va + i * L2_SIZE 323 */ 324 static struct pmap_preinit_mapping { 325 vm_paddr_t pa; 326 void *va; 327 vm_size_t size; 328 } pmap_preinit_mapping[PMAP_PREINIT_MAPPING_COUNT]; 329 330 vm_offset_t virtual_avail; /* VA of first avail page (after kernel bss) */ 331 vm_offset_t virtual_end; /* VA of last avail page (end of kernel AS) */ 332 vm_offset_t kernel_vm_end = 0; 333 334 /* 335 * Data for the pv entry allocation mechanism. 336 */ 337 #ifdef NUMA 338 static __inline int 339 pc_to_domain(struct pv_chunk *pc) 340 { 341 return (vm_phys_domain(DMAP_TO_PHYS(pc))); 342 } 343 #else 344 static __inline int 345 pc_to_domain(struct pv_chunk *pc __unused) 346 { 347 return (0); 348 } 349 #endif 350 351 struct pv_chunks_list { 352 struct mtx pvc_lock; 353 TAILQ_HEAD(pch, pv_chunk) pvc_list; 354 int active_reclaims; 355 } __aligned(CACHE_LINE_SIZE); 356 357 struct pv_chunks_list __exclusive_cache_line pv_chunks[PMAP_MEMDOM]; 358 359 vm_paddr_t dmap_phys_base; /* The start of the dmap region */ 360 vm_paddr_t dmap_phys_max; /* The limit of the dmap region */ 361 vm_offset_t dmap_max_addr; /* The virtual address limit of the dmap */ 362 static int dmap_attr = VM_MEMATTR_WRITE_BACK; 363 364 extern pt_entry_t pagetable_l0_ttbr1[]; 365 366 #define PHYSMAP_SIZE (2 * (VM_PHYSSEG_MAX - 1)) 367 static vm_paddr_t physmap[PHYSMAP_SIZE]; 368 static u_int physmap_idx; 369 370 static SYSCTL_NODE(_vm, OID_AUTO, pmap, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 371 "VM/pmap parameters"); 372 373 static int pmap_growkernel_panic = 0; 374 SYSCTL_INT(_vm_pmap, OID_AUTO, growkernel_panic, CTLFLAG_RDTUN, 375 &pmap_growkernel_panic, 0, 376 "panic on failure to allocate kernel page table page"); 377 378 bool pmap_lpa_enabled __read_mostly = false; 379 pt_entry_t pmap_sh_attr __read_mostly = ATTR_SH(ATTR_SH_IS); 380 381 #if PAGE_SIZE == PAGE_SIZE_4K 382 #define L1_BLOCKS_SUPPORTED 1 383 #else 384 #define L1_BLOCKS_SUPPORTED (pmap_lpa_enabled) 385 #endif 386 387 #define PMAP_ASSERT_L1_BLOCKS_SUPPORTED MPASS(L1_BLOCKS_SUPPORTED) 388 389 static bool pmap_l1_supported __read_mostly = false; 390 391 /* 392 * This ASID allocator uses a bit vector ("asid_set") to remember which ASIDs 393 * that it has currently allocated to a pmap, a cursor ("asid_next") to 394 * optimize its search for a free ASID in the bit vector, and an epoch number 395 * ("asid_epoch") to indicate when it has reclaimed all previously allocated 396 * ASIDs that are not currently active on a processor. 397 * 398 * The current epoch number is always in the range [0, INT_MAX). Negative 399 * numbers and INT_MAX are reserved for special cases that are described 400 * below. 401 */ 402 struct asid_set { 403 int asid_bits; 404 bitstr_t *asid_set; 405 int asid_set_size; 406 int asid_next; 407 int asid_epoch; 408 struct mtx asid_set_mutex; 409 }; 410 411 static struct asid_set asids; 412 static struct asid_set vmids; 413 414 static SYSCTL_NODE(_vm_pmap, OID_AUTO, asid, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 415 "ASID allocator"); 416 SYSCTL_INT(_vm_pmap_asid, OID_AUTO, bits, CTLFLAG_RD, &asids.asid_bits, 0, 417 "The number of bits in an ASID"); 418 SYSCTL_INT(_vm_pmap_asid, OID_AUTO, next, CTLFLAG_RD, &asids.asid_next, 0, 419 "The last allocated ASID plus one"); 420 SYSCTL_INT(_vm_pmap_asid, OID_AUTO, epoch, CTLFLAG_RD, &asids.asid_epoch, 0, 421 "The current epoch number"); 422 423 static SYSCTL_NODE(_vm_pmap, OID_AUTO, vmid, CTLFLAG_RD, 0, "VMID allocator"); 424 SYSCTL_INT(_vm_pmap_vmid, OID_AUTO, bits, CTLFLAG_RD, &vmids.asid_bits, 0, 425 "The number of bits in an VMID"); 426 SYSCTL_INT(_vm_pmap_vmid, OID_AUTO, next, CTLFLAG_RD, &vmids.asid_next, 0, 427 "The last allocated VMID plus one"); 428 SYSCTL_INT(_vm_pmap_vmid, OID_AUTO, epoch, CTLFLAG_RD, &vmids.asid_epoch, 0, 429 "The current epoch number"); 430 431 void (*pmap_clean_stage2_tlbi)(void); 432 void (*pmap_stage2_invalidate_range)(uint64_t, vm_offset_t, vm_offset_t, bool); 433 void (*pmap_stage2_invalidate_all)(uint64_t); 434 435 /* 436 * A pmap's cookie encodes an ASID and epoch number. Cookies for reserved 437 * ASIDs have a negative epoch number, specifically, INT_MIN. Cookies for 438 * dynamically allocated ASIDs have a non-negative epoch number. 439 * 440 * An invalid ASID is represented by -1. 441 * 442 * There are two special-case cookie values: (1) COOKIE_FROM(-1, INT_MIN), 443 * which indicates that an ASID should never be allocated to the pmap, and 444 * (2) COOKIE_FROM(-1, INT_MAX), which indicates that an ASID should be 445 * allocated when the pmap is next activated. 446 */ 447 #define COOKIE_FROM(asid, epoch) ((long)((u_int)(asid) | \ 448 ((u_long)(epoch) << 32))) 449 #define COOKIE_TO_ASID(cookie) ((int)(cookie)) 450 #define COOKIE_TO_EPOCH(cookie) ((int)((u_long)(cookie) >> 32)) 451 452 #define TLBI_VA_SHIFT 12 453 #define TLBI_VA_MASK ((1ul << 44) - 1) 454 #define TLBI_VA(addr) (((addr) >> TLBI_VA_SHIFT) & TLBI_VA_MASK) 455 456 /* 457 * The operand to a range-based TLBI instruction has the following fields: 458 * 459 * 63 48 47 46 45 44 43 39 38 37 36 0 460 * +----------+-------+--------+--------+--------+-----------------+ 461 * | ASID | TG | SCALE | NUM | TTL | BaseADDR | 462 * +----------+-------+--------+--------+--------+-----------------+ 463 * 464 * A single range-based TLBI instruction invalidates the TLB entries for the 465 * mappings within the address range 466 * 467 * [BaseADDR, BaseADDR + (NUM + 1) * 2^(5 * SCALE + 1) * PAGE_SIZE) 468 * 469 * BaseADDR is VA[48:PAGE_SHIFT], unless 52-bit addressing is enabled, i.e., 470 * pmap_lpa_enabled is true, in which case BaseADDR is VA[52:16] regardless 471 * of the page size. Consequently, when pmap_lpa_enabled is true, the start 472 * of the address range must be 64KB aligned, and any leading pages must be 473 * invalidated individually. 474 * 475 * TTL optionally specifies the translation table level at which every 476 * mapping within the address range can be found; we currently set TTL to 0, 477 * meaning that we are not providing a hint. 478 * 479 * A single instruction invalidates some number of units, where a unit is 480 * 2^(5 * SCALE + 1) pages. NUM is that number minus 1. 481 * 482 * TG specifies the translation granule size, i.e., PAGE_SIZE. 483 */ 484 #define TLBI_RANGE_VA_SHIFT() (pmap_lpa_enabled ? 16 : PAGE_SHIFT) 485 486 #define TLBI_RANGE_BADDR_MASK ((1ul << 37) - 1) 487 #define TLBI_RANGE_NUM_SHIFT 39 488 #define TLBI_RANGE_SCALE_SHIFT 44 489 #define TLBI_RANGE_TG_SHIFT 46 490 491 #define TLBI_RANGE_MAX_UNITS 32 492 #define TLBI_RANGE_MAX_SCALE 3 493 494 #define TLBI_RANGE_UNIT_SHIFT(scale) (5 * (scale) + 1) 495 #define TLBI_RANGE_UNIT(scale) (1ul << TLBI_RANGE_UNIT_SHIFT(scale)) 496 497 /* 498 * The largest scale such that a unit fits within the given number of pages, 499 * i.e., the largest scale such that TLBI_RANGE_UNIT(scale) <= pages. The 500 * given number of pages must be at least TLBI_RANGE_UNIT(0). 501 */ 502 #define TLBI_RANGE_SCALE(pages) \ 503 imin((flsl(pages) - 2) / 5, TLBI_RANGE_MAX_SCALE) 504 505 #if PAGE_SIZE == PAGE_SIZE_4K 506 #define TLBI_RANGE_TG (1ul << TLBI_RANGE_TG_SHIFT) 507 #elif PAGE_SIZE == PAGE_SIZE_16K 508 #define TLBI_RANGE_TG (2ul << TLBI_RANGE_TG_SHIFT) 509 #else 510 #error Unsupported page size 511 #endif 512 513 #define TLBI_RANGE_FIELDS(va, va_shift, num, scale) \ 514 (TLBI_RANGE_TG | ((u_long)(scale) << TLBI_RANGE_SCALE_SHIFT) | \ 515 ((u_long)(num) << TLBI_RANGE_NUM_SHIFT) | \ 516 (((va) >> (va_shift)) & TLBI_RANGE_BADDR_MASK)) 517 518 static bool __read_frequently pmap_tlbi_range_support = false; 519 520 static int __read_frequently superpages_enabled = 1; 521 SYSCTL_INT(_vm_pmap, OID_AUTO, superpages_enabled, 522 CTLFLAG_RDTUN | CTLFLAG_NOFETCH, &superpages_enabled, 0, 523 "Are large page mappings enabled?"); 524 525 /* 526 * True when Branch Target Identification should be used by userspace. This 527 * allows pmap to mark pages as guarded with ATTR_S1_GP. 528 */ 529 __read_mostly static bool pmap_bti_support = false; 530 531 /* 532 * Internal flags for pmap_enter()'s helper functions. 533 */ 534 #define PMAP_ENTER_NORECLAIM 0x1000000 /* Don't reclaim PV entries. */ 535 #define PMAP_ENTER_NOREPLACE 0x2000000 /* Don't replace mappings. */ 536 537 TAILQ_HEAD(pv_chunklist, pv_chunk); 538 539 static void free_pv_chunk(struct pv_chunk *pc); 540 static void free_pv_chunk_batch(struct pv_chunklist *batch); 541 static void free_pv_entry(pmap_t pmap, pv_entry_t pv); 542 static pv_entry_t get_pv_entry(pmap_t pmap, struct rwlock **lockp); 543 static vm_page_t reclaim_pv_chunk(pmap_t locked_pmap, struct rwlock **lockp); 544 static void pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va); 545 static pv_entry_t pmap_pvh_remove(struct md_page *pvh, pmap_t pmap, 546 vm_offset_t va); 547 548 static void pmap_abort_ptp(pmap_t pmap, vm_offset_t va, vm_page_t mpte); 549 static bool pmap_activate_int(struct thread *td, pmap_t pmap); 550 static void pmap_alloc_asid(pmap_t pmap); 551 static int pmap_change_props_locked(void *addr, vm_size_t size, 552 vm_prot_t prot, int mode, int old_mode, bool skip_unmapped); 553 static bool pmap_copy_l3c(pmap_t pmap, pt_entry_t *l3p, vm_offset_t va, 554 pt_entry_t l3e, vm_page_t ml3, struct rwlock **lockp); 555 static pt_entry_t *pmap_demote_l1(pmap_t pmap, pt_entry_t *l1, vm_offset_t va); 556 static pt_entry_t *pmap_demote_l2_locked(pmap_t pmap, pt_entry_t *l2, 557 vm_offset_t va, struct rwlock **lockp); 558 static pt_entry_t *pmap_demote_l2(pmap_t pmap, pt_entry_t *l2, vm_offset_t va); 559 static bool pmap_demote_l2c(pmap_t pmap, pt_entry_t *l2p, vm_offset_t va); 560 static bool pmap_demote_l3c(pmap_t pmap, pt_entry_t *l3p, vm_offset_t va); 561 static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, 562 vm_page_t m, vm_prot_t prot, vm_page_t mpte, struct rwlock **lockp); 563 static int pmap_enter_l2(pmap_t pmap, vm_offset_t va, pd_entry_t new_l2, 564 u_int flags, vm_page_t m, struct rwlock **lockp); 565 static int pmap_enter_l3c(pmap_t pmap, vm_offset_t va, pt_entry_t l3e, u_int flags, 566 vm_page_t m, vm_page_t *ml3p, struct rwlock **lockp); 567 static bool pmap_every_pte_zero(vm_paddr_t pa); 568 static int pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte, bool promoted, 569 bool all_l3e_AF_set); 570 static pt_entry_t pmap_load_l3c(pt_entry_t *l3p); 571 static void pmap_mask_set_l3c(pmap_t pmap, pt_entry_t *l3p, vm_offset_t va, 572 vm_offset_t *vap, vm_offset_t va_next, pt_entry_t mask, pt_entry_t nbits); 573 static bool pmap_page_is_mapped_locked(vm_page_t m); 574 static bool pmap_pv_insert_l3c(pmap_t pmap, vm_offset_t va, vm_page_t m, 575 struct rwlock **lockp); 576 static void pmap_remove_kernel_l2(pmap_t pmap, pt_entry_t *l2, vm_offset_t va); 577 static int pmap_remove_l2(pmap_t pmap, pt_entry_t *l2, vm_offset_t sva, 578 pd_entry_t l1e, bool demote_kl2e, struct spglist *free, 579 struct rwlock **lockp); 580 static int pmap_remove_l3(pmap_t pmap, pt_entry_t *l3, vm_offset_t sva, 581 pd_entry_t l2e, struct spglist *free, struct rwlock **lockp); 582 static bool pmap_remove_l3c(pmap_t pmap, pt_entry_t *l3p, vm_offset_t va, 583 vm_offset_t *vap, vm_offset_t va_next, vm_page_t ml3, struct spglist *free, 584 struct rwlock **lockp); 585 static void pmap_reset_asid_set(pmap_t pmap); 586 static bool pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, 587 vm_page_t m, struct rwlock **lockp); 588 589 static vm_page_t _pmap_alloc_l3(pmap_t pmap, vm_pindex_t ptepindex, 590 struct rwlock **lockp); 591 592 static void _pmap_unwire_l3(pmap_t pmap, vm_offset_t va, vm_page_t m, 593 struct spglist *free); 594 static int pmap_unuse_pt(pmap_t, vm_offset_t, pd_entry_t, struct spglist *); 595 static void pmap_update_entry(pmap_t pmap, pd_entry_t *pte, pd_entry_t newpte, 596 vm_offset_t va, vm_size_t size, bool final_only); 597 static __inline vm_page_t pmap_remove_pt_page(pmap_t pmap, vm_offset_t va); 598 599 static uma_zone_t pmap_bti_ranges_zone; 600 static bool pmap_bti_same(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, 601 pt_entry_t *pte); 602 static pt_entry_t pmap_pte_bti(pmap_t pmap, vm_offset_t va); 603 static void pmap_bti_on_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva); 604 static void *bti_dup_range(void *ctx, void *data); 605 static void bti_free_range(void *ctx, void *node); 606 static int pmap_bti_copy(pmap_t dst_pmap, pmap_t src_pmap); 607 static void pmap_bti_deassign_all(pmap_t pmap); 608 static void pagezero(void *); 609 610 static void pmap_set_protected(pt_entry_t old_l3); 611 static void pmap_set_unprotected(pt_entry_t new_l3); 612 613 /* 614 * These load the old table data and store the new value. 615 * They need to be atomic as the System MMU may write to the table at 616 * the same time as the CPU. 617 */ 618 #define pmap_clear(table) atomic_store_64(table, 0) 619 #define pmap_clear_bits(table, bits) atomic_clear_64(table, bits) 620 #define pmap_load(table) (*table) 621 #define pmap_load_clear(table) atomic_swap_64(table, 0) 622 #define pmap_load_store(table, entry) atomic_swap_64(table, entry) 623 #define pmap_set_bits(table, bits) atomic_set_64(table, bits) 624 #define pmap_store(table, entry) atomic_store_64(table, entry) 625 626 /********************/ 627 /* Inline functions */ 628 /********************/ 629 630 static __inline void 631 pagecopy(void *s, void *d) 632 { 633 634 memcpy(d, s, PAGE_SIZE); 635 } 636 637 static __inline pd_entry_t * 638 pmap_l0(pmap_t pmap, vm_offset_t va) 639 { 640 641 return (&pmap->pm_l0[pmap_l0_index(va)]); 642 } 643 644 static __inline pd_entry_t * 645 pmap_l0_to_l1(pd_entry_t *l0, vm_offset_t va) 646 { 647 pd_entry_t *l1; 648 649 l1 = PHYS_TO_DMAP(PTE_TO_PHYS(pmap_load(l0))); 650 return (&l1[pmap_l1_index(va)]); 651 } 652 653 static __inline pd_entry_t * 654 pmap_l1(pmap_t pmap, vm_offset_t va) 655 { 656 pd_entry_t *l0; 657 658 l0 = pmap_l0(pmap, va); 659 if ((pmap_load(l0) & ATTR_DESCR_MASK) != L0_TABLE) 660 return (NULL); 661 662 return (pmap_l0_to_l1(l0, va)); 663 } 664 665 static __inline pd_entry_t * 666 pmap_l1_to_l2(pd_entry_t *l1p, vm_offset_t va) 667 { 668 pd_entry_t l1, *l2p; 669 670 l1 = pmap_load(l1p); 671 672 KASSERT(ADDR_IS_CANONICAL(va), 673 ("%s: Address not in canonical form: %lx", __func__, va)); 674 /* 675 * The valid bit may be clear if pmap_update_entry() is concurrently 676 * modifying the entry, so for KVA only the entry type may be checked. 677 */ 678 KASSERT(ADDR_IS_KERNEL(va) || (l1 & ATTR_DESCR_VALID) != 0, 679 ("%s: L1 entry %#lx for %#lx is invalid", __func__, l1, va)); 680 KASSERT((l1 & ATTR_DESCR_TYPE_MASK) == ATTR_DESCR_TYPE_TABLE, 681 ("%s: L1 entry %#lx for %#lx is a leaf", __func__, l1, va)); 682 l2p = PHYS_TO_DMAP(PTE_TO_PHYS(l1)); 683 return (&l2p[pmap_l2_index(va)]); 684 } 685 686 static __inline pd_entry_t * 687 pmap_l2(pmap_t pmap, vm_offset_t va) 688 { 689 pd_entry_t *l1; 690 691 l1 = pmap_l1(pmap, va); 692 if ((pmap_load(l1) & ATTR_DESCR_MASK) != L1_TABLE) 693 return (NULL); 694 695 return (pmap_l1_to_l2(l1, va)); 696 } 697 698 static __inline pt_entry_t * 699 pmap_l2_to_l3(pd_entry_t *l2p, vm_offset_t va) 700 { 701 pd_entry_t l2; 702 pt_entry_t *l3p; 703 704 l2 = pmap_load(l2p); 705 706 KASSERT(ADDR_IS_CANONICAL(va), 707 ("%s: Address not in canonical form: %lx", __func__, va)); 708 /* 709 * The valid bit may be clear if pmap_update_entry() is concurrently 710 * modifying the entry, so for KVA only the entry type may be checked. 711 */ 712 KASSERT(ADDR_IS_KERNEL(va) || (l2 & ATTR_DESCR_VALID) != 0, 713 ("%s: L2 entry %#lx for %#lx is invalid", __func__, l2, va)); 714 KASSERT((l2 & ATTR_DESCR_TYPE_MASK) == ATTR_DESCR_TYPE_TABLE, 715 ("%s: L2 entry %#lx for %#lx is a leaf", __func__, l2, va)); 716 l3p = PHYS_TO_DMAP(PTE_TO_PHYS(l2)); 717 return (&l3p[pmap_l3_index(va)]); 718 } 719 720 /* 721 * Returns the lowest valid pde for a given virtual address. 722 * The next level may or may not point to a valid page or block. 723 */ 724 static __inline pd_entry_t * 725 pmap_pde(pmap_t pmap, vm_offset_t va, int *level) 726 { 727 pd_entry_t *l0, *l1, *l2, desc; 728 729 l0 = pmap_l0(pmap, va); 730 desc = pmap_load(l0) & ATTR_DESCR_MASK; 731 if (desc != L0_TABLE) { 732 *level = -1; 733 return (NULL); 734 } 735 736 l1 = pmap_l0_to_l1(l0, va); 737 desc = pmap_load(l1) & ATTR_DESCR_MASK; 738 if (desc != L1_TABLE) { 739 *level = 0; 740 return (l0); 741 } 742 743 l2 = pmap_l1_to_l2(l1, va); 744 desc = pmap_load(l2) & ATTR_DESCR_MASK; 745 if (desc != L2_TABLE) { 746 *level = 1; 747 return (l1); 748 } 749 750 *level = 2; 751 return (l2); 752 } 753 754 /* 755 * Returns the lowest valid pte block or table entry for a given virtual 756 * address. If there are no valid entries return NULL and set the level to 757 * the first invalid level. 758 */ 759 static __inline pt_entry_t * 760 pmap_pte(pmap_t pmap, vm_offset_t va, int *level) 761 { 762 pd_entry_t *l1, *l2, desc; 763 pt_entry_t *l3; 764 765 l1 = pmap_l1(pmap, va); 766 if (l1 == NULL) { 767 *level = 0; 768 return (NULL); 769 } 770 desc = pmap_load(l1) & ATTR_DESCR_MASK; 771 if (desc == L1_BLOCK) { 772 PMAP_ASSERT_L1_BLOCKS_SUPPORTED; 773 *level = 1; 774 return (l1); 775 } 776 777 if (desc != L1_TABLE) { 778 *level = 1; 779 return (NULL); 780 } 781 782 l2 = pmap_l1_to_l2(l1, va); 783 desc = pmap_load(l2) & ATTR_DESCR_MASK; 784 if (desc == L2_BLOCK) { 785 *level = 2; 786 return (l2); 787 } 788 789 if (desc != L2_TABLE) { 790 *level = 2; 791 return (NULL); 792 } 793 794 *level = 3; 795 l3 = pmap_l2_to_l3(l2, va); 796 if ((pmap_load(l3) & ATTR_DESCR_MASK) != L3_PAGE) 797 return (NULL); 798 799 return (l3); 800 } 801 802 /* 803 * If the given pmap has an L{1,2}_BLOCK or L3_PAGE entry at the specified 804 * level that maps the specified virtual address, then a pointer to that entry 805 * is returned. Otherwise, NULL is returned, unless INVARIANTS are enabled 806 * and a diagnostic message is provided, in which case this function panics. 807 */ 808 static __always_inline pt_entry_t * 809 pmap_pte_exists(pmap_t pmap, vm_offset_t va, int level, const char *diag) 810 { 811 pd_entry_t *l0p, *l1p, *l2p; 812 pt_entry_t desc, *l3p; 813 int walk_level __diagused; 814 815 KASSERT(level >= 0 && level < 4, 816 ("%s: %s passed an out-of-range level (%d)", __func__, diag, 817 level)); 818 l0p = pmap_l0(pmap, va); 819 desc = pmap_load(l0p) & ATTR_DESCR_MASK; 820 if (desc == L0_TABLE && level > 0) { 821 l1p = pmap_l0_to_l1(l0p, va); 822 desc = pmap_load(l1p) & ATTR_DESCR_MASK; 823 if (desc == L1_BLOCK && level == 1) { 824 PMAP_ASSERT_L1_BLOCKS_SUPPORTED; 825 return (l1p); 826 } 827 if (desc == L1_TABLE && level > 1) { 828 l2p = pmap_l1_to_l2(l1p, va); 829 desc = pmap_load(l2p) & ATTR_DESCR_MASK; 830 if (desc == L2_BLOCK && level == 2) 831 return (l2p); 832 else if (desc == L2_TABLE && level > 2) { 833 l3p = pmap_l2_to_l3(l2p, va); 834 desc = pmap_load(l3p) & ATTR_DESCR_MASK; 835 if (desc == L3_PAGE && level == 3) 836 return (l3p); 837 else 838 walk_level = 3; 839 } else 840 walk_level = 2; 841 } else 842 walk_level = 1; 843 } else 844 walk_level = 0; 845 KASSERT(diag == NULL, 846 ("%s: va %#lx not mapped at level %d, desc %ld at level %d", 847 diag, va, level, desc, walk_level)); 848 return (NULL); 849 } 850 851 bool 852 pmap_ps_enabled(pmap_t pmap) 853 { 854 /* 855 * Promotion requires a hypervisor call when the kernel is running 856 * in EL1. To stop this disable superpage support on non-stage 1 857 * pmaps for now. 858 */ 859 if (pmap->pm_stage != PM_STAGE1) 860 return (false); 861 862 #ifdef KMSAN 863 /* 864 * The break-before-make in pmap_update_entry() results in a situation 865 * where a CPU may call into the KMSAN runtime while the entry is 866 * invalid. If the entry is used to map the current thread structure, 867 * then the runtime will attempt to access unmapped memory. Avoid this 868 * by simply disabling superpage promotion for the kernel map. 869 */ 870 if (pmap == kernel_pmap) 871 return (false); 872 #endif 873 874 return (superpages_enabled != 0); 875 } 876 877 bool 878 pmap_vs_enabled(void) 879 { 880 /* 881 * 8 and 16 are the only values hardware can support, but allow for the 882 * possibility of artificially restricting the bits, e.g. for testing. 883 */ 884 KASSERT(vmids.asid_bits <= 16, ("VMID bits %d > 16", vmids.asid_bits)); 885 return (vmids.asid_bits > 8); 886 } 887 888 bool 889 pmap_get_tables(pmap_t pmap, vm_offset_t va, pd_entry_t **l0, pd_entry_t **l1, 890 pd_entry_t **l2, pt_entry_t **l3) 891 { 892 pd_entry_t *l0p, *l1p, *l2p; 893 894 if (pmap->pm_l0 == NULL) 895 return (false); 896 897 l0p = pmap_l0(pmap, va); 898 *l0 = l0p; 899 900 if ((pmap_load(l0p) & ATTR_DESCR_MASK) != L0_TABLE) 901 return (false); 902 903 l1p = pmap_l0_to_l1(l0p, va); 904 *l1 = l1p; 905 906 if ((pmap_load(l1p) & ATTR_DESCR_MASK) == L1_BLOCK) { 907 PMAP_ASSERT_L1_BLOCKS_SUPPORTED; 908 *l2 = NULL; 909 *l3 = NULL; 910 return (true); 911 } 912 913 if ((pmap_load(l1p) & ATTR_DESCR_MASK) != L1_TABLE) 914 return (false); 915 916 l2p = pmap_l1_to_l2(l1p, va); 917 *l2 = l2p; 918 919 if ((pmap_load(l2p) & ATTR_DESCR_MASK) == L2_BLOCK) { 920 *l3 = NULL; 921 return (true); 922 } 923 924 if ((pmap_load(l2p) & ATTR_DESCR_MASK) != L2_TABLE) 925 return (false); 926 927 *l3 = pmap_l2_to_l3(l2p, va); 928 929 return (true); 930 } 931 932 static __inline int 933 pmap_l3_valid(pt_entry_t l3) 934 { 935 936 return ((l3 & ATTR_DESCR_MASK) == L3_PAGE); 937 } 938 939 CTASSERT(L1_BLOCK == L2_BLOCK); 940 941 static pt_entry_t 942 pmap_pte_memattr(pmap_t pmap, vm_memattr_t memattr) 943 { 944 pt_entry_t val; 945 946 if (pmap->pm_stage == PM_STAGE1) { 947 val = ATTR_S1_IDX(memattr); 948 if (memattr == VM_MEMATTR_DEVICE) 949 val |= ATTR_S1_XN; 950 return (val); 951 } 952 953 val = 0; 954 955 switch (memattr) { 956 case VM_MEMATTR_DEVICE: 957 return (ATTR_S2_MEMATTR(ATTR_S2_MEMATTR_DEVICE_nGnRnE) | 958 ATTR_S2_XN(ATTR_S2_XN_ALL)); 959 case VM_MEMATTR_UNCACHEABLE: 960 return (ATTR_S2_MEMATTR(ATTR_S2_MEMATTR_NC)); 961 case VM_MEMATTR_WRITE_BACK: 962 return (ATTR_S2_MEMATTR(ATTR_S2_MEMATTR_WB)); 963 case VM_MEMATTR_WRITE_THROUGH: 964 return (ATTR_S2_MEMATTR(ATTR_S2_MEMATTR_WT)); 965 default: 966 panic("%s: invalid memory attribute %x", __func__, memattr); 967 } 968 } 969 970 static pt_entry_t 971 pmap_pte_prot(pmap_t pmap, vm_prot_t prot) 972 { 973 pt_entry_t val; 974 975 val = 0; 976 if (pmap->pm_stage == PM_STAGE1) { 977 if ((prot & VM_PROT_EXECUTE) == 0) 978 val |= ATTR_S1_XN; 979 if ((prot & VM_PROT_WRITE) == 0) 980 val |= ATTR_S1_AP(ATTR_S1_AP_RO); 981 } else { 982 if ((prot & VM_PROT_WRITE) != 0) 983 val |= ATTR_S2_S2AP(ATTR_S2_S2AP_WRITE); 984 if ((prot & VM_PROT_READ) != 0) 985 val |= ATTR_S2_S2AP(ATTR_S2_S2AP_READ); 986 if ((prot & VM_PROT_EXECUTE) == 0) 987 val |= ATTR_S2_XN(ATTR_S2_XN_ALL); 988 } 989 990 return (val); 991 } 992 993 /* 994 * Checks if the PTE is dirty. 995 */ 996 static inline int 997 pmap_pte_dirty(pmap_t pmap, pt_entry_t pte) 998 { 999 1000 KASSERT((pte & ATTR_SW_MANAGED) != 0, ("pte %#lx is unmanaged", pte)); 1001 1002 if (pmap->pm_stage == PM_STAGE1) { 1003 KASSERT((pte & (ATTR_S1_AP_RW_BIT | ATTR_SW_DBM)) != 0, 1004 ("pte %#lx is writeable and missing ATTR_SW_DBM", pte)); 1005 1006 return ((pte & (ATTR_S1_AP_RW_BIT | ATTR_SW_DBM)) == 1007 (ATTR_S1_AP(ATTR_S1_AP_RW) | ATTR_SW_DBM)); 1008 } 1009 1010 return ((pte & ATTR_S2_S2AP(ATTR_S2_S2AP_WRITE)) == 1011 ATTR_S2_S2AP(ATTR_S2_S2AP_WRITE)); 1012 } 1013 1014 static __inline void 1015 pmap_resident_count_inc(pmap_t pmap, int count) 1016 { 1017 1018 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1019 pmap->pm_stats.resident_count += count; 1020 } 1021 1022 static __inline void 1023 pmap_resident_count_dec(pmap_t pmap, int count) 1024 { 1025 1026 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1027 KASSERT(pmap->pm_stats.resident_count >= count, 1028 ("pmap %p resident count underflow %ld %d", pmap, 1029 pmap->pm_stats.resident_count, count)); 1030 pmap->pm_stats.resident_count -= count; 1031 } 1032 1033 static vm_paddr_t 1034 pmap_early_vtophys(vm_offset_t va) 1035 { 1036 vm_paddr_t pa_page; 1037 1038 pa_page = arm64_address_translate_s1e1r(va) & PAR_PA_MASK; 1039 return (pa_page | (va & PAR_LOW_MASK)); 1040 } 1041 1042 /* State of the bootstrapped DMAP page tables */ 1043 struct pmap_bootstrap_state { 1044 pt_entry_t *l1; 1045 pt_entry_t *l2; 1046 pt_entry_t *l3; 1047 vm_offset_t freemempos; 1048 vm_offset_t va; 1049 vm_paddr_t pa; 1050 pt_entry_t table_attrs; 1051 u_int l0_slot; 1052 u_int l1_slot; 1053 u_int l2_slot; 1054 bool dmap_valid; 1055 }; 1056 1057 /* The bootstrap state */ 1058 static struct pmap_bootstrap_state bs_state = { 1059 .l1 = NULL, 1060 .l2 = NULL, 1061 .l3 = NULL, 1062 .table_attrs = TATTR_PXN_TABLE, 1063 .l0_slot = L0_ENTRIES, 1064 .l1_slot = Ln_ENTRIES, 1065 .l2_slot = Ln_ENTRIES, 1066 .dmap_valid = false, 1067 }; 1068 1069 static void 1070 pmap_bootstrap_l0_table(struct pmap_bootstrap_state *state) 1071 { 1072 vm_paddr_t l1_pa; 1073 pd_entry_t l0e; 1074 u_int l0_slot; 1075 1076 /* Link the level 0 table to a level 1 table */ 1077 l0_slot = pmap_l0_index(state->va); 1078 if (l0_slot != state->l0_slot) { 1079 /* 1080 * Make sure we move from a low address to high address 1081 * before the DMAP region is ready. This ensures we never 1082 * modify an existing mapping until we can map from a 1083 * physical address to a virtual address. 1084 */ 1085 MPASS(state->l0_slot < l0_slot || 1086 state->l0_slot == L0_ENTRIES || 1087 state->dmap_valid); 1088 1089 /* Reset lower levels */ 1090 state->l2 = NULL; 1091 state->l3 = NULL; 1092 state->l1_slot = Ln_ENTRIES; 1093 state->l2_slot = Ln_ENTRIES; 1094 1095 /* Check the existing L0 entry */ 1096 state->l0_slot = l0_slot; 1097 if (state->dmap_valid) { 1098 l0e = pagetable_l0_ttbr1[l0_slot]; 1099 if ((l0e & ATTR_DESCR_VALID) != 0) { 1100 MPASS((l0e & ATTR_DESCR_MASK) == L0_TABLE); 1101 l1_pa = PTE_TO_PHYS(l0e); 1102 state->l1 = PHYS_TO_DMAP(l1_pa); 1103 return; 1104 } 1105 } 1106 1107 /* Create a new L0 table entry */ 1108 state->l1 = (pt_entry_t *)state->freemempos; 1109 memset_early(state->l1, 0, PAGE_SIZE); 1110 state->freemempos += PAGE_SIZE; 1111 1112 l1_pa = pmap_early_vtophys((vm_offset_t)state->l1); 1113 MPASS((l1_pa & Ln_TABLE_MASK) == 0); 1114 MPASS(pagetable_l0_ttbr1[l0_slot] == 0); 1115 pmap_store(&pagetable_l0_ttbr1[l0_slot], PHYS_TO_PTE(l1_pa) | 1116 TATTR_UXN_TABLE | TATTR_AP_TABLE_NO_EL0 | L0_TABLE); 1117 } 1118 KASSERT(state->l1 != NULL, ("%s: NULL l1", __func__)); 1119 } 1120 1121 static void 1122 pmap_bootstrap_l1_table(struct pmap_bootstrap_state *state) 1123 { 1124 vm_paddr_t l2_pa; 1125 pd_entry_t l1e; 1126 u_int l1_slot; 1127 1128 /* Make sure there is a valid L0 -> L1 table */ 1129 pmap_bootstrap_l0_table(state); 1130 1131 /* Link the level 1 table to a level 2 table */ 1132 l1_slot = pmap_l1_index(state->va); 1133 if (l1_slot != state->l1_slot) { 1134 /* See pmap_bootstrap_l0_table for a description */ 1135 MPASS(state->l1_slot < l1_slot || 1136 state->l1_slot == Ln_ENTRIES || 1137 state->dmap_valid); 1138 1139 /* Reset lower levels */ 1140 state->l3 = NULL; 1141 state->l2_slot = Ln_ENTRIES; 1142 1143 /* Check the existing L1 entry */ 1144 state->l1_slot = l1_slot; 1145 if (state->dmap_valid) { 1146 l1e = state->l1[l1_slot]; 1147 if ((l1e & ATTR_DESCR_VALID) != 0) { 1148 MPASS((l1e & ATTR_DESCR_MASK) == L1_TABLE); 1149 l2_pa = PTE_TO_PHYS(l1e); 1150 state->l2 = PHYS_TO_DMAP(l2_pa); 1151 return; 1152 } 1153 } 1154 1155 /* Create a new L1 table entry */ 1156 state->l2 = (pt_entry_t *)state->freemempos; 1157 memset_early(state->l2, 0, PAGE_SIZE); 1158 state->freemempos += PAGE_SIZE; 1159 1160 l2_pa = pmap_early_vtophys((vm_offset_t)state->l2); 1161 MPASS((l2_pa & Ln_TABLE_MASK) == 0); 1162 MPASS(state->l1[l1_slot] == 0); 1163 pmap_store(&state->l1[l1_slot], PHYS_TO_PTE(l2_pa) | 1164 state->table_attrs | L1_TABLE); 1165 } 1166 KASSERT(state->l2 != NULL, ("%s: NULL l2", __func__)); 1167 } 1168 1169 static void 1170 pmap_bootstrap_l2_table(struct pmap_bootstrap_state *state) 1171 { 1172 vm_paddr_t l3_pa; 1173 pd_entry_t l2e; 1174 u_int l2_slot; 1175 1176 /* Make sure there is a valid L1 -> L2 table */ 1177 pmap_bootstrap_l1_table(state); 1178 1179 /* Link the level 2 table to a level 3 table */ 1180 l2_slot = pmap_l2_index(state->va); 1181 if (l2_slot != state->l2_slot) { 1182 /* See pmap_bootstrap_l0_table for a description */ 1183 MPASS(state->l2_slot < l2_slot || 1184 state->l2_slot == Ln_ENTRIES || 1185 state->dmap_valid); 1186 1187 /* Check the existing L2 entry */ 1188 state->l2_slot = l2_slot; 1189 if (state->dmap_valid) { 1190 l2e = state->l2[l2_slot]; 1191 if ((l2e & ATTR_DESCR_VALID) != 0) { 1192 MPASS((l2e & ATTR_DESCR_MASK) == L2_TABLE); 1193 l3_pa = PTE_TO_PHYS(l2e); 1194 state->l3 = PHYS_TO_DMAP(l3_pa); 1195 return; 1196 } 1197 } 1198 1199 /* Create a new L2 table entry */ 1200 state->l3 = (pt_entry_t *)state->freemempos; 1201 memset_early(state->l3, 0, PAGE_SIZE); 1202 state->freemempos += PAGE_SIZE; 1203 1204 l3_pa = pmap_early_vtophys((vm_offset_t)state->l3); 1205 MPASS((l3_pa & Ln_TABLE_MASK) == 0); 1206 MPASS(state->l2[l2_slot] == 0); 1207 pmap_store(&state->l2[l2_slot], PHYS_TO_PTE(l3_pa) | 1208 state->table_attrs | L2_TABLE); 1209 } 1210 KASSERT(state->l3 != NULL, ("%s: NULL l3", __func__)); 1211 } 1212 1213 static void 1214 pmap_bootstrap_l2_block(struct pmap_bootstrap_state *state, int i) 1215 { 1216 pt_entry_t contig; 1217 u_int l2_slot; 1218 bool first; 1219 1220 if ((physmap[i + 1] - state->pa) < L2_SIZE) 1221 return; 1222 1223 /* Make sure there is a valid L1 table */ 1224 pmap_bootstrap_l1_table(state); 1225 1226 MPASS((state->va & L2_OFFSET) == 0); 1227 for (first = true, contig = 0; 1228 state->va < DMAP_MAX_ADDRESS && 1229 (physmap[i + 1] - state->pa) >= L2_SIZE; 1230 state->va += L2_SIZE, state->pa += L2_SIZE) { 1231 /* 1232 * Stop if we are about to walk off the end of what the 1233 * current L1 slot can address. 1234 */ 1235 if (!first && (state->pa & L1_OFFSET) == 0) 1236 break; 1237 1238 /* 1239 * If we have an aligned, contiguous chunk of L2C_ENTRIES 1240 * L2 blocks, set the contiguous bit within each PTE so that 1241 * the chunk can be cached using only one TLB entry. 1242 */ 1243 if ((state->pa & L2C_OFFSET) == 0) { 1244 if (state->va + L2C_SIZE < DMAP_MAX_ADDRESS && 1245 physmap[i + 1] - state->pa >= L2C_SIZE) { 1246 contig = ATTR_CONTIGUOUS; 1247 } else { 1248 contig = 0; 1249 } 1250 } 1251 1252 first = false; 1253 l2_slot = pmap_l2_index(state->va); 1254 MPASS((state->pa & L2_OFFSET) == 0); 1255 MPASS(state->l2[l2_slot] == 0); 1256 pmap_store(&state->l2[l2_slot], PHYS_TO_PTE(state->pa) | 1257 ATTR_AF | pmap_sh_attr | ATTR_S1_XN | ATTR_KERN_GP | 1258 ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK) | contig | L2_BLOCK); 1259 } 1260 MPASS(state->va == (state->pa - dmap_phys_base + DMAP_MIN_ADDRESS)); 1261 } 1262 1263 static void 1264 pmap_bootstrap_l3_page(struct pmap_bootstrap_state *state, int i) 1265 { 1266 pt_entry_t contig; 1267 u_int l3_slot; 1268 bool first; 1269 1270 if (physmap[i + 1] - state->pa < L3_SIZE) 1271 return; 1272 1273 /* Make sure there is a valid L2 table */ 1274 pmap_bootstrap_l2_table(state); 1275 1276 MPASS((state->va & L3_OFFSET) == 0); 1277 for (first = true, contig = 0; 1278 state->va < DMAP_MAX_ADDRESS && 1279 physmap[i + 1] - state->pa >= L3_SIZE; 1280 state->va += L3_SIZE, state->pa += L3_SIZE) { 1281 /* 1282 * Stop if we are about to walk off the end of what the 1283 * current L2 slot can address. 1284 */ 1285 if (!first && (state->pa & L2_OFFSET) == 0) 1286 break; 1287 1288 /* 1289 * If we have an aligned, contiguous chunk of L3C_ENTRIES 1290 * L3 pages, set the contiguous bit within each PTE so that 1291 * the chunk can be cached using only one TLB entry. 1292 */ 1293 if ((state->pa & L3C_OFFSET) == 0) { 1294 if (state->va + L3C_SIZE < DMAP_MAX_ADDRESS && 1295 physmap[i + 1] - state->pa >= L3C_SIZE) { 1296 contig = ATTR_CONTIGUOUS; 1297 } else { 1298 contig = 0; 1299 } 1300 } 1301 1302 first = false; 1303 l3_slot = pmap_l3_index(state->va); 1304 MPASS((state->pa & L3_OFFSET) == 0); 1305 MPASS(state->l3[l3_slot] == 0); 1306 pmap_store(&state->l3[l3_slot], PHYS_TO_PTE(state->pa) | 1307 ATTR_AF | pmap_sh_attr | ATTR_S1_XN | ATTR_KERN_GP | 1308 ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK) | contig | L3_PAGE); 1309 } 1310 MPASS(state->va == (state->pa - dmap_phys_base + DMAP_MIN_ADDRESS)); 1311 } 1312 1313 void 1314 pmap_bootstrap_dmap(vm_size_t kernlen) 1315 { 1316 vm_paddr_t start_pa, pa; 1317 uint64_t tcr; 1318 int i; 1319 1320 tcr = READ_SPECIALREG(tcr_el1); 1321 1322 /* Verify that the ASID is set through TTBR0. */ 1323 KASSERT((tcr & TCR_A1) == 0, ("pmap_bootstrap: TCR_EL1.A1 != 0")); 1324 1325 if ((tcr & TCR_DS) != 0) 1326 pmap_lpa_enabled = true; 1327 1328 pmap_l1_supported = L1_BLOCKS_SUPPORTED; 1329 1330 start_pa = pmap_early_vtophys(KERNBASE); 1331 1332 bs_state.freemempos = KERNBASE + kernlen; 1333 bs_state.freemempos = roundup2(bs_state.freemempos, PAGE_SIZE); 1334 1335 /* Fill in physmap array. */ 1336 physmap_idx = physmem_avail(physmap, nitems(physmap)); 1337 1338 dmap_phys_base = physmap[0] & ~L1_OFFSET; 1339 dmap_phys_max = 0; 1340 dmap_max_addr = 0; 1341 1342 for (i = 0; i < physmap_idx; i += 2) { 1343 bs_state.pa = physmap[i] & ~L3_OFFSET; 1344 bs_state.va = bs_state.pa - dmap_phys_base + DMAP_MIN_ADDRESS; 1345 1346 /* Create L3 mappings at the start of the region */ 1347 if ((bs_state.pa & L2_OFFSET) != 0) 1348 pmap_bootstrap_l3_page(&bs_state, i); 1349 MPASS(bs_state.pa <= physmap[i + 1]); 1350 1351 if (L1_BLOCKS_SUPPORTED) { 1352 /* Create L2 mappings at the start of the region */ 1353 if ((bs_state.pa & L1_OFFSET) != 0) 1354 pmap_bootstrap_l2_block(&bs_state, i); 1355 MPASS(bs_state.pa <= physmap[i + 1]); 1356 1357 /* Create the main L1 block mappings */ 1358 for (; bs_state.va < DMAP_MAX_ADDRESS && 1359 (physmap[i + 1] - bs_state.pa) >= L1_SIZE; 1360 bs_state.va += L1_SIZE, bs_state.pa += L1_SIZE) { 1361 /* Make sure there is a valid L1 table */ 1362 pmap_bootstrap_l0_table(&bs_state); 1363 MPASS((bs_state.pa & L1_OFFSET) == 0); 1364 pmap_store( 1365 &bs_state.l1[pmap_l1_index(bs_state.va)], 1366 PHYS_TO_PTE(bs_state.pa) | ATTR_AF | 1367 pmap_sh_attr | 1368 ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK) | 1369 ATTR_S1_XN | ATTR_KERN_GP | L1_BLOCK); 1370 } 1371 MPASS(bs_state.pa <= physmap[i + 1]); 1372 1373 /* Create L2 mappings at the end of the region */ 1374 pmap_bootstrap_l2_block(&bs_state, i); 1375 } else { 1376 while (bs_state.va < DMAP_MAX_ADDRESS && 1377 (physmap[i + 1] - bs_state.pa) >= L2_SIZE) { 1378 pmap_bootstrap_l2_block(&bs_state, i); 1379 } 1380 } 1381 MPASS(bs_state.pa <= physmap[i + 1]); 1382 1383 /* Create L3 mappings at the end of the region */ 1384 pmap_bootstrap_l3_page(&bs_state, i); 1385 MPASS(bs_state.pa == physmap[i + 1]); 1386 1387 if (bs_state.pa > dmap_phys_max) { 1388 dmap_phys_max = bs_state.pa; 1389 dmap_max_addr = bs_state.va; 1390 } 1391 } 1392 1393 pmap_s1_invalidate_all_kernel(); 1394 1395 bs_state.dmap_valid = true; 1396 1397 /* Exclude the kernel and DMAP region */ 1398 pa = pmap_early_vtophys(bs_state.freemempos); 1399 physmem_exclude_region(start_pa, pa - start_pa, EXFLAG_NOALLOC); 1400 } 1401 1402 static void 1403 pmap_bootstrap_l2(vm_offset_t va) 1404 { 1405 KASSERT((va & L1_OFFSET) == 0, ("Invalid virtual address")); 1406 1407 /* Leave bs_state.pa as it's only needed to bootstrap blocks and pages*/ 1408 bs_state.va = va; 1409 1410 for (; bs_state.va < VM_MAX_KERNEL_ADDRESS; bs_state.va += L1_SIZE) 1411 pmap_bootstrap_l1_table(&bs_state); 1412 } 1413 1414 static void 1415 pmap_bootstrap_l3(vm_offset_t va) 1416 { 1417 KASSERT((va & L2_OFFSET) == 0, ("Invalid virtual address")); 1418 1419 /* Leave bs_state.pa as it's only needed to bootstrap blocks and pages*/ 1420 bs_state.va = va; 1421 1422 for (; bs_state.va < VM_MAX_KERNEL_ADDRESS; bs_state.va += L2_SIZE) 1423 pmap_bootstrap_l2_table(&bs_state); 1424 } 1425 1426 /* 1427 * Bootstrap the system enough to run with virtual memory. 1428 */ 1429 void 1430 pmap_bootstrap(void) 1431 { 1432 vm_offset_t dpcpu, msgbufpv; 1433 vm_paddr_t start_pa, pa; 1434 size_t largest_phys_size; 1435 1436 /* Set this early so we can use the pagetable walking functions */ 1437 kernel_pmap_store.pm_l0 = pagetable_l0_ttbr1; 1438 mtx_init(&kernel_pmap->pm_mtx, "kernel pmap", NULL, MTX_DEF); 1439 kernel_pmap->pm_l0_paddr = 1440 pmap_early_vtophys((vm_offset_t)kernel_pmap_store.pm_l0); 1441 TAILQ_INIT(&kernel_pmap->pm_pvchunk); 1442 vm_radix_init(&kernel_pmap->pm_root); 1443 kernel_pmap->pm_cookie = COOKIE_FROM(-1, INT_MIN); 1444 kernel_pmap->pm_stage = PM_STAGE1; 1445 kernel_pmap->pm_levels = 4; 1446 kernel_pmap->pm_ttbr = kernel_pmap->pm_l0_paddr; 1447 kernel_pmap->pm_asid_set = &asids; 1448 1449 /* Reserve some VA space for early BIOS/ACPI mapping */ 1450 preinit_map_va = roundup2(bs_state.freemempos, L2_SIZE); 1451 1452 virtual_avail = preinit_map_va + PMAP_PREINIT_MAPPING_SIZE; 1453 virtual_avail = roundup2(virtual_avail, L1_SIZE); 1454 virtual_end = VM_MAX_KERNEL_ADDRESS - PMAP_MAPDEV_EARLY_SIZE - L2_SIZE; 1455 kernel_vm_end = virtual_avail; 1456 1457 /* 1458 * We only use PXN when we know nothing will be executed from it, e.g. 1459 * the DMAP region. 1460 */ 1461 bs_state.table_attrs &= ~TATTR_PXN_TABLE; 1462 1463 /* 1464 * Find the physical memory we could use. This needs to be after we 1465 * exclude any memory that is mapped into the DMAP region but should 1466 * not be used by the kernel, e.g. some UEFI memory types. 1467 */ 1468 physmap_idx = physmem_avail(physmap, nitems(physmap)); 1469 1470 /* 1471 * Find space for early allocations. We search for the largest 1472 * region. This is because the user may choose a large msgbuf. 1473 * This could be smarter, e.g. to allow multiple regions to be 1474 * used & switch to the next when one is full. 1475 */ 1476 largest_phys_size = 0; 1477 for (int i = 0; i < physmap_idx; i += 2) { 1478 if ((physmap[i + 1] - physmap[i]) > largest_phys_size) { 1479 largest_phys_size = physmap[i + 1] - physmap[i]; 1480 bs_state.freemempos = PHYS_TO_DMAP_ADDR(physmap[i]); 1481 } 1482 } 1483 1484 start_pa = pmap_early_vtophys(bs_state.freemempos); 1485 1486 /* 1487 * Create the l2 tables up to VM_MAX_KERNEL_ADDRESS. We assume that the 1488 * loader allocated the first and only l2 page table page used to map 1489 * the kernel, preloaded files and module metadata. 1490 */ 1491 pmap_bootstrap_l2(KERNBASE + L1_SIZE); 1492 /* And the l3 tables for the early devmap */ 1493 pmap_bootstrap_l3(VM_MAX_KERNEL_ADDRESS - (PMAP_MAPDEV_EARLY_SIZE)); 1494 1495 pmap_s1_invalidate_all_kernel(); 1496 1497 #define alloc_pages(var, np) \ 1498 (var) = bs_state.freemempos; \ 1499 bs_state.freemempos += (np * PAGE_SIZE); \ 1500 memset_early((char *)(var), 0, ((np) * PAGE_SIZE)); 1501 1502 /* Allocate dynamic per-cpu area. */ 1503 alloc_pages(dpcpu, DPCPU_SIZE / PAGE_SIZE); 1504 dpcpu_init((void *)dpcpu, 0); 1505 1506 /* Allocate memory for the msgbuf, e.g. for /sbin/dmesg */ 1507 alloc_pages(msgbufpv, round_page(msgbufsize) / PAGE_SIZE); 1508 msgbufp = (void *)msgbufpv; 1509 1510 /* Allocate space for the CPU0 CMAP */ 1511 bs_state.va = virtual_end; 1512 pmap_bootstrap_l2_table(&bs_state); 1513 pmap_store(&bs_state.l3[pmap_l3_index(bs_state.va)], 1514 PHYS_TO_PTE(pmap_early_vtophys((vm_offset_t)bs_state.l3)) | 1515 ATTR_AF | pmap_sh_attr | ATTR_S1_XN | ATTR_KERN_GP | 1516 ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK) | L3_PAGE); 1517 dsb(ishst); 1518 1519 mtx_init(&cmap_lock, "SYSMAPS", NULL, MTX_DEF); 1520 cmap1_addr = (void *)(virtual_end + L3_SIZE); 1521 cmap1_pte = &bs_state.l3[pmap_l3_index((vm_offset_t)cmap1_addr)]; 1522 1523 pa = pmap_early_vtophys(bs_state.freemempos); 1524 1525 physmem_exclude_region(start_pa, pa - start_pa, EXFLAG_NOALLOC); 1526 } 1527 1528 #if defined(KASAN) || defined(KMSAN) 1529 static void 1530 pmap_bootstrap_allocate_san_l2(vm_paddr_t start_pa, vm_paddr_t end_pa, 1531 vm_offset_t *vap, vm_offset_t eva) 1532 { 1533 vm_paddr_t pa; 1534 vm_offset_t va; 1535 pd_entry_t *l2; 1536 1537 va = *vap; 1538 pa = rounddown2(end_pa - L2_SIZE, L2_SIZE); 1539 for (; pa >= start_pa && va < eva; va += L2_SIZE, pa -= L2_SIZE) { 1540 l2 = pmap_l2(kernel_pmap, va); 1541 1542 /* 1543 * KASAN stack checking results in us having already allocated 1544 * part of our shadow map, so we can just skip those segments. 1545 */ 1546 if ((pmap_load(l2) & ATTR_DESCR_VALID) != 0) { 1547 pa += L2_SIZE; 1548 continue; 1549 } 1550 1551 bzero_early(PHYS_TO_DMAP(pa), L2_SIZE); 1552 physmem_exclude_region(pa, L2_SIZE, EXFLAG_NOALLOC); 1553 pmap_store(l2, PHYS_TO_PTE(pa) | PMAP_SAN_PTE_BITS | L2_BLOCK); 1554 } 1555 *vap = va; 1556 } 1557 1558 /* 1559 * Finish constructing the initial shadow map: 1560 * - Count how many pages from KERNBASE to virtual_avail (scaled for 1561 * shadow map) 1562 * - Map that entire range using L2 superpages. 1563 */ 1564 static void 1565 pmap_bootstrap_san1(vm_offset_t va, int scale) 1566 { 1567 vm_offset_t eva; 1568 vm_paddr_t kernstart; 1569 int i; 1570 1571 kernstart = pmap_early_vtophys(KERNBASE); 1572 1573 /* 1574 * Rebuild physmap one more time, we may have excluded more regions from 1575 * allocation since pmap_bootstrap(). 1576 */ 1577 physmap_idx = physmem_avail(physmap, nitems(physmap)); 1578 1579 eva = va + (virtual_avail - VM_MIN_KERNEL_ADDRESS) / scale; 1580 1581 /* 1582 * Find a slot in the physmap large enough for what we needed. We try to put 1583 * the shadow map as high up as we can to avoid depleting the lower 4GB in case 1584 * it's needed for, e.g., an xhci controller that can only do 32-bit DMA. 1585 */ 1586 for (i = physmap_idx - 2; i >= 0; i -= 2) { 1587 vm_paddr_t plow, phigh; 1588 1589 /* L2 mappings must be backed by memory that is L2-aligned */ 1590 plow = roundup2(physmap[i], L2_SIZE); 1591 phigh = physmap[i + 1]; 1592 if (plow >= phigh) 1593 continue; 1594 if (kernstart >= plow && kernstart < phigh) 1595 phigh = kernstart; 1596 if (phigh - plow >= L2_SIZE) { 1597 pmap_bootstrap_allocate_san_l2(plow, phigh, &va, eva); 1598 if (va >= eva) 1599 break; 1600 } 1601 } 1602 if (i < 0) 1603 panic("Could not find phys region for shadow map"); 1604 1605 /* 1606 * Done. We should now have a valid shadow address mapped for all KVA 1607 * that has been mapped so far, i.e., KERNBASE to virtual_avail. Thus, 1608 * shadow accesses by the sanitizer runtime will succeed for this range. 1609 * When the kernel virtual address range is later expanded, as will 1610 * happen in vm_mem_init(), the shadow map will be grown as well. This 1611 * is handled by pmap_san_enter(). 1612 */ 1613 } 1614 1615 void 1616 pmap_bootstrap_san(void) 1617 { 1618 #ifdef KASAN 1619 pmap_bootstrap_san1(KASAN_MIN_ADDRESS, KASAN_SHADOW_SCALE); 1620 #else 1621 static uint8_t kmsan_shad_ptp[PAGE_SIZE * 2] __aligned(PAGE_SIZE); 1622 static uint8_t kmsan_orig_ptp[PAGE_SIZE * 2] __aligned(PAGE_SIZE); 1623 pd_entry_t *l0, *l1; 1624 1625 if (virtual_avail - VM_MIN_KERNEL_ADDRESS > L1_SIZE) 1626 panic("initial kernel map is too large"); 1627 1628 l0 = pmap_l0(kernel_pmap, KMSAN_SHAD_MIN_ADDRESS); 1629 pmap_store(l0, L0_TABLE | PHYS_TO_PTE( 1630 pmap_early_vtophys((vm_offset_t)kmsan_shad_ptp))); 1631 l1 = pmap_l0_to_l1(l0, KMSAN_SHAD_MIN_ADDRESS); 1632 pmap_store(l1, L1_TABLE | PHYS_TO_PTE( 1633 pmap_early_vtophys((vm_offset_t)kmsan_shad_ptp + PAGE_SIZE))); 1634 pmap_bootstrap_san1(KMSAN_SHAD_MIN_ADDRESS, 1); 1635 1636 l0 = pmap_l0(kernel_pmap, KMSAN_ORIG_MIN_ADDRESS); 1637 pmap_store(l0, L0_TABLE | PHYS_TO_PTE( 1638 pmap_early_vtophys((vm_offset_t)kmsan_orig_ptp))); 1639 l1 = pmap_l0_to_l1(l0, KMSAN_ORIG_MIN_ADDRESS); 1640 pmap_store(l1, L1_TABLE | PHYS_TO_PTE( 1641 pmap_early_vtophys((vm_offset_t)kmsan_orig_ptp + PAGE_SIZE))); 1642 pmap_bootstrap_san1(KMSAN_ORIG_MIN_ADDRESS, 1); 1643 #endif 1644 } 1645 #endif 1646 1647 /* 1648 * Initialize a vm_page's machine-dependent fields. 1649 */ 1650 void 1651 pmap_page_init(vm_page_t m) 1652 { 1653 1654 TAILQ_INIT(&m->md.pv_list); 1655 m->md.pv_memattr = VM_MEMATTR_WRITE_BACK; 1656 m->md.pv_flags = 0; 1657 } 1658 1659 static void 1660 pmap_init_asids(struct asid_set *set, int bits) 1661 { 1662 int i; 1663 1664 set->asid_bits = bits; 1665 1666 /* 1667 * We may be too early in the overall initialization process to use 1668 * bit_alloc(). 1669 */ 1670 set->asid_set_size = 1 << set->asid_bits; 1671 set->asid_set = kmem_malloc(bitstr_size(set->asid_set_size), 1672 M_WAITOK | M_ZERO); 1673 for (i = 0; i < ASID_FIRST_AVAILABLE; i++) 1674 bit_set(set->asid_set, i); 1675 set->asid_next = ASID_FIRST_AVAILABLE; 1676 mtx_init(&set->asid_set_mutex, "asid set", NULL, MTX_SPIN); 1677 } 1678 1679 static void 1680 pmap_init_pv_table(void) 1681 { 1682 struct vm_phys_seg *seg, *next_seg; 1683 struct pmap_large_md_page *pvd; 1684 vm_size_t s; 1685 int domain, i, j, pages; 1686 1687 /* 1688 * We depend on the size being evenly divisible into a page so 1689 * that the pv_table array can be indexed directly while 1690 * safely spanning multiple pages from different domains. 1691 */ 1692 CTASSERT(PAGE_SIZE % sizeof(*pvd) == 0); 1693 1694 /* 1695 * Calculate the size of the array. 1696 */ 1697 s = 0; 1698 for (i = 0; i < vm_phys_nsegs; i++) { 1699 seg = &vm_phys_segs[i]; 1700 pages = pmap_l2_pindex(roundup2(seg->end, L2_SIZE)) - 1701 pmap_l2_pindex(seg->start); 1702 s += round_page(pages * sizeof(*pvd)); 1703 } 1704 pv_table = kva_alloc(s); 1705 if (pv_table == NULL) 1706 panic("%s: kva_alloc failed\n", __func__); 1707 1708 /* 1709 * Iterate physical segments to allocate domain-local memory for PV 1710 * list headers. 1711 */ 1712 pvd = pv_table; 1713 for (i = 0; i < vm_phys_nsegs; i++) { 1714 seg = &vm_phys_segs[i]; 1715 pages = pmap_l2_pindex(roundup2(seg->end, L2_SIZE)) - 1716 pmap_l2_pindex(seg->start); 1717 domain = seg->domain; 1718 1719 s = round_page(pages * sizeof(*pvd)); 1720 1721 for (j = 0; j < s; j += PAGE_SIZE) { 1722 vm_page_t m = vm_page_alloc_noobj_domain(domain, 1723 VM_ALLOC_ZERO); 1724 if (m == NULL) 1725 panic("failed to allocate PV table page"); 1726 pmap_qenter((char *)pvd + j, &m, 1); 1727 } 1728 1729 for (j = 0; j < s / sizeof(*pvd); j++) { 1730 rw_init_flags(&pvd->pv_lock, "pmap pv list", RW_NEW); 1731 TAILQ_INIT(&pvd->pv_page.pv_list); 1732 pvd++; 1733 } 1734 } 1735 pvd = &pv_dummy_large; 1736 memset(pvd, 0, sizeof(*pvd)); 1737 rw_init_flags(&pvd->pv_lock, "pmap pv list dummy", RW_NEW); 1738 TAILQ_INIT(&pvd->pv_page.pv_list); 1739 1740 /* 1741 * Set pointers from vm_phys_segs to pv_table. 1742 */ 1743 for (i = 0, pvd = pv_table; i < vm_phys_nsegs; i++) { 1744 seg = &vm_phys_segs[i]; 1745 seg->md_first = pvd; 1746 pvd += pmap_l2_pindex(roundup2(seg->end, L2_SIZE)) - 1747 pmap_l2_pindex(seg->start); 1748 1749 /* 1750 * If there is a following segment, and the final 1751 * superpage of this segment and the initial superpage 1752 * of the next segment are the same then adjust the 1753 * pv_table entry for that next segment down by one so 1754 * that the pv_table entries will be shared. 1755 */ 1756 if (i + 1 < vm_phys_nsegs) { 1757 next_seg = &vm_phys_segs[i + 1]; 1758 if (pmap_l2_pindex(roundup2(seg->end, L2_SIZE)) - 1 == 1759 pmap_l2_pindex(next_seg->start)) { 1760 pvd--; 1761 } 1762 } 1763 } 1764 } 1765 1766 static cpu_feat_en 1767 pmap_dbm_check(const struct cpu_feat *feat __unused, u_int midr __unused) 1768 { 1769 uint64_t id_aa64mmfr1; 1770 1771 id_aa64mmfr1 = READ_SPECIALREG(id_aa64mmfr1_el1); 1772 if (ID_AA64MMFR1_HAFDBS_VAL(id_aa64mmfr1) >= 1773 ID_AA64MMFR1_HAFDBS_AF_DBS) 1774 return (FEAT_DEFAULT_ENABLE); 1775 1776 return (FEAT_ALWAYS_DISABLE); 1777 } 1778 1779 static bool 1780 pmap_dbm_has_errata(const struct cpu_feat *feat __unused, u_int midr, 1781 u_int **errata_list, u_int *errata_count) 1782 { 1783 /* Disable on Cortex-A55 for erratum 1024718 - all revisions */ 1784 if (CPU_IMPL(midr) == CPU_IMPL_ARM && 1785 CPU_PART(midr) == CPU_PART_CORTEX_A55) { 1786 static u_int errata_id = 1024718; 1787 1788 *errata_list = &errata_id; 1789 *errata_count = 1; 1790 return (true); 1791 } 1792 1793 /* Disable on Cortex-A510 for erratum 2051678 - r0p0 to r0p2 */ 1794 if (midr_check_var_part_range(midr, CPU_IMPL_ARM, CPU_PART_CORTEX_A510, 1795 0, 0, 0, 2)) { 1796 static u_int errata_id = 2051678; 1797 1798 *errata_list = &errata_id; 1799 *errata_count = 1; 1800 return (true); 1801 } 1802 1803 return (false); 1804 } 1805 1806 static bool 1807 pmap_dbm_enable(const struct cpu_feat *feat __unused, 1808 cpu_feat_errata errata_status, u_int *errata_list __unused, 1809 u_int errata_count) 1810 { 1811 uint64_t tcr; 1812 1813 /* Skip if there is an erratum affecting DBM */ 1814 if (errata_status != ERRATA_NONE) 1815 return (false); 1816 1817 tcr = READ_SPECIALREG(tcr_el1) | TCR_HD; 1818 WRITE_SPECIALREG(tcr_el1, tcr); 1819 isb(); 1820 /* Flush the local TLB for the TCR_HD flag change */ 1821 dsb(nshst); 1822 __asm __volatile("tlbi vmalle1"); 1823 dsb(nsh); 1824 isb(); 1825 1826 return (true); 1827 } 1828 1829 CPU_FEAT(feat_hafdbs, "Hardware management of the Access flag and dirty state", 1830 pmap_dbm_check, pmap_dbm_has_errata, pmap_dbm_enable, NULL, 1831 CPU_FEAT_AFTER_DEV | CPU_FEAT_PER_CPU); 1832 1833 static cpu_feat_en 1834 pmap_multiple_tlbi_check(const struct cpu_feat *feat __unused, u_int midr) 1835 { 1836 /* 1837 * ARM C1-Premium erratum 4193780 1838 * ARM C1-Ultra erratum 4193780 1839 * ARM Cortex-A76 erratum 4193800 1840 * ARM Cortex-A76AE erratum 4193801 1841 * ARM Cortex-A77 erratum 4193798 1842 * ARM Cortex-A78 erratum 4193791 1843 * ARM Cortex-A78AE erratum 4193793 1844 * ARM Cortex-A78C erratum 4193794 1845 * ARM Cortex-A710 erratum 4193788 1846 * ARM Cortex-X1 erratum 4193791 1847 * ARM Cortex-X1C erratum 4193792 1848 * ARM Cortex-X2 erratum 4193788 1849 * ARM Cortex-X3 erratum 4193786 1850 * ARM Cortex-X4 erratum 4118414 1851 * ARM Cortex-X925 erratum 4193781 1852 * ARM Neoverse-N1 erratum 4193800 1853 * ARM Neoverse-N2 erratum 4193789 1854 * ARM Neoverse-V1 erratum 4193790 1855 * ARM Neoverse-V2 erratum 4193787 1856 * ARM Neoverse-V3 erratum 4193784 1857 * ARM Neoverse-V3AE erratum 4193784 1858 * Present in all revisions 1859 */ 1860 if (CPU_IMPL(midr) == CPU_IMPL_ARM) { 1861 switch(CPU_PART(midr)) { 1862 case CPU_PART_C1_PREMIUM: 1863 case CPU_PART_C1_ULTRA: 1864 case CPU_PART_CORTEX_A76: 1865 case CPU_PART_CORTEX_A76AE: 1866 case CPU_PART_CORTEX_A77: 1867 case CPU_PART_CORTEX_A78: 1868 case CPU_PART_CORTEX_A78AE: 1869 case CPU_PART_CORTEX_A78C: 1870 case CPU_PART_CORTEX_A710: 1871 case CPU_PART_CORTEX_X1: 1872 case CPU_PART_CORTEX_X1C: 1873 case CPU_PART_CORTEX_X2: 1874 case CPU_PART_CORTEX_X3: 1875 case CPU_PART_CORTEX_X4: 1876 case CPU_PART_CORTEX_X925: 1877 case CPU_PART_NEOVERSE_N1: 1878 case CPU_PART_NEOVERSE_N2: 1879 case CPU_PART_NEOVERSE_V1: 1880 case CPU_PART_NEOVERSE_V2: 1881 case CPU_PART_NEOVERSE_V3: 1882 case CPU_PART_NEOVERSE_V3AE: 1883 return (FEAT_DEFAULT_ENABLE); 1884 } 1885 } 1886 1887 /* 1888 * Cortex-A55 erratum 2441007 (Cat B rare) 1889 * Present in all revisions 1890 */ 1891 if (CPU_IMPL(midr) == CPU_IMPL_ARM && 1892 CPU_PART(midr) == CPU_PART_CORTEX_A55) 1893 return (FEAT_DEFAULT_DISABLE); 1894 1895 /* 1896 * Cortex-A510 erratum 2441009 (Cat B rare) 1897 * Present in r0p0 - r1p1 1898 * Fixed in r1p2 1899 */ 1900 if (midr_check_var_part_range(midr, CPU_IMPL_ARM, CPU_PART_CORTEX_A510, 1901 0, 0, 1, 1)) 1902 return (FEAT_DEFAULT_DISABLE); 1903 1904 return (FEAT_ALWAYS_DISABLE); 1905 } 1906 1907 static bool 1908 pmap_multiple_tlbi_enable(const struct cpu_feat *feat __unused, 1909 cpu_feat_errata errata_status, u_int *errata_list __unused, 1910 u_int errata_count __unused) 1911 { 1912 pmap_multiple_tlbi = true; 1913 return (true); 1914 } 1915 1916 CPU_FEAT(errata_multi_tlbi, "Multiple TLBI errata", 1917 pmap_multiple_tlbi_check, NULL, pmap_multiple_tlbi_enable, NULL, 1918 CPU_FEAT_EARLY_BOOT | CPU_FEAT_PER_CPU); 1919 1920 static cpu_feat_en 1921 pmap_tlbi_range_check(const struct cpu_feat *feat __unused, u_int midr __unused) 1922 { 1923 uint64_t reg; 1924 1925 /* 1926 * Range-based TLBI must be supported by every processor, so this 1927 * check is performed CPU_FEAT_AFTER_DEV. 1928 */ 1929 get_kernel_reg(ID_AA64ISAR0_EL1, ®); 1930 if (ID_AA64ISAR0_TLB_VAL(reg) >= ID_AA64ISAR0_TLB_TLBIOSR) 1931 return (FEAT_DEFAULT_ENABLE); 1932 1933 return (FEAT_ALWAYS_DISABLE); 1934 } 1935 1936 static bool 1937 pmap_tlbi_range_enable(const struct cpu_feat *feat __unused, 1938 cpu_feat_errata errata_status __unused, u_int *errata_list __unused, 1939 u_int errata_count __unused) 1940 { 1941 /* 1942 * pmap_lpa_enabled must be initialized before range-based TLBI can 1943 * be performed. 1944 */ 1945 MPASS((READ_SPECIALREG(tcr_el1) & TCR_DS) == 0 || pmap_lpa_enabled); 1946 pmap_tlbi_range_support = true; 1947 return (true); 1948 } 1949 1950 CPU_FEAT(feat_tlbi_range, "Range-based TLBI invalidation", 1951 pmap_tlbi_range_check, NULL, pmap_tlbi_range_enable, NULL, 1952 CPU_FEAT_AFTER_DEV | CPU_FEAT_SYSTEM); 1953 1954 /* 1955 * Initialize the pmap module. 1956 * 1957 * Called by vm_mem_init(), to initialize any structures that the pmap 1958 * system needs to map virtual memory. 1959 */ 1960 void 1961 pmap_init(void) 1962 { 1963 uint64_t mmfr1; 1964 int i, vmid_bits; 1965 1966 /* 1967 * Are large page mappings enabled? 1968 */ 1969 TUNABLE_INT_FETCH("vm.pmap.superpages_enabled", &superpages_enabled); 1970 if (superpages_enabled) { 1971 KASSERT(MAXPAGESIZES > 1 && pagesizes[1] == 0, 1972 ("pmap_init: can't assign to pagesizes[1]")); 1973 pagesizes[1] = L3C_SIZE; 1974 KASSERT(MAXPAGESIZES > 2 && pagesizes[2] == 0, 1975 ("pmap_init: can't assign to pagesizes[2]")); 1976 pagesizes[2] = L2_SIZE; 1977 if (L1_BLOCKS_SUPPORTED) { 1978 KASSERT(MAXPAGESIZES > 3 && pagesizes[3] == 0, 1979 ("pmap_init: can't assign to pagesizes[3]")); 1980 pagesizes[3] = L1_SIZE; 1981 } 1982 } 1983 1984 /* 1985 * Initialize the ASID allocator. 1986 */ 1987 pmap_init_asids(&asids, 1988 (READ_SPECIALREG(tcr_el1) & TCR_ASID_16) != 0 ? 16 : 8); 1989 1990 if (has_hyp()) { 1991 mmfr1 = READ_SPECIALREG(id_aa64mmfr1_el1); 1992 vmid_bits = 8; 1993 1994 if (ID_AA64MMFR1_VMIDBits_VAL(mmfr1) == 1995 ID_AA64MMFR1_VMIDBits_16) 1996 vmid_bits = 16; 1997 pmap_init_asids(&vmids, vmid_bits); 1998 } 1999 2000 /* 2001 * Initialize pv chunk lists. 2002 */ 2003 for (i = 0; i < PMAP_MEMDOM; i++) { 2004 mtx_init(&pv_chunks[i].pvc_lock, "pmap pv chunk list", NULL, 2005 MTX_DEF); 2006 TAILQ_INIT(&pv_chunks[i].pvc_list); 2007 } 2008 pmap_init_pv_table(); 2009 2010 vm_initialized = 1; 2011 } 2012 2013 static SYSCTL_NODE(_vm_pmap, OID_AUTO, l1, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 2014 "L1 (1GB/64GB) page mapping counters"); 2015 2016 static COUNTER_U64_DEFINE_EARLY(pmap_l1_demotions); 2017 SYSCTL_COUNTER_U64(_vm_pmap_l1, OID_AUTO, demotions, CTLFLAG_RD, 2018 &pmap_l1_demotions, "L1 (1GB/64GB) page demotions"); 2019 2020 SYSCTL_BOOL(_vm_pmap_l1, OID_AUTO, supported, CTLFLAG_RD, &pmap_l1_supported, 2021 0, "L1 blocks are supported"); 2022 2023 static SYSCTL_NODE(_vm_pmap, OID_AUTO, l2c, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 2024 "L2C (32MB/1GB) page mapping counters"); 2025 2026 static COUNTER_U64_DEFINE_EARLY(pmap_l2c_demotions); 2027 SYSCTL_COUNTER_U64(_vm_pmap_l2c, OID_AUTO, demotions, CTLFLAG_RD, 2028 &pmap_l2c_demotions, "L2C (32MB/1GB) page demotions"); 2029 2030 static SYSCTL_NODE(_vm_pmap, OID_AUTO, l2, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 2031 "2MB page mapping counters"); 2032 2033 static COUNTER_U64_DEFINE_EARLY(pmap_l2_demotions); 2034 SYSCTL_COUNTER_U64(_vm_pmap_l2, OID_AUTO, demotions, CTLFLAG_RD, 2035 &pmap_l2_demotions, "L2 (2MB/32MB) page demotions"); 2036 2037 static COUNTER_U64_DEFINE_EARLY(pmap_l2_mappings); 2038 SYSCTL_COUNTER_U64(_vm_pmap_l2, OID_AUTO, mappings, CTLFLAG_RD, 2039 &pmap_l2_mappings, "L2 (2MB/32MB) page mappings"); 2040 2041 static COUNTER_U64_DEFINE_EARLY(pmap_l2_p_failures); 2042 SYSCTL_COUNTER_U64(_vm_pmap_l2, OID_AUTO, p_failures, CTLFLAG_RD, 2043 &pmap_l2_p_failures, "L2 (2MB/32MB) page promotion failures"); 2044 2045 static COUNTER_U64_DEFINE_EARLY(pmap_l2_promotions); 2046 SYSCTL_COUNTER_U64(_vm_pmap_l2, OID_AUTO, promotions, CTLFLAG_RD, 2047 &pmap_l2_promotions, "L2 (2MB/32MB) page promotions"); 2048 2049 static SYSCTL_NODE(_vm_pmap, OID_AUTO, l3c, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 2050 "L3C (64KB/2MB) page mapping counters"); 2051 2052 static COUNTER_U64_DEFINE_EARLY(pmap_l3c_demotions); 2053 SYSCTL_COUNTER_U64(_vm_pmap_l3c, OID_AUTO, demotions, CTLFLAG_RD, 2054 &pmap_l3c_demotions, "L3C (64KB/2MB) page demotions"); 2055 2056 static COUNTER_U64_DEFINE_EARLY(pmap_l3c_mappings); 2057 SYSCTL_COUNTER_U64(_vm_pmap_l3c, OID_AUTO, mappings, CTLFLAG_RD, 2058 &pmap_l3c_mappings, "L3C (64KB/2MB) page mappings"); 2059 2060 static COUNTER_U64_DEFINE_EARLY(pmap_l3c_p_failures); 2061 SYSCTL_COUNTER_U64(_vm_pmap_l3c, OID_AUTO, p_failures, CTLFLAG_RD, 2062 &pmap_l3c_p_failures, "L3C (64KB/2MB) page promotion failures"); 2063 2064 static COUNTER_U64_DEFINE_EARLY(pmap_l3c_promotions); 2065 SYSCTL_COUNTER_U64(_vm_pmap_l3c, OID_AUTO, promotions, CTLFLAG_RD, 2066 &pmap_l3c_promotions, "L3C (64KB/2MB) page promotions"); 2067 2068 /* 2069 * If the given value for "final_only" is false, then any cached intermediate- 2070 * level entries, i.e., L{0,1,2}_TABLE entries, are invalidated in addition to 2071 * any cached final-level entry, i.e., either an L{1,2}_BLOCK or L3_PAGE entry. 2072 * Otherwise, just the cached final-level entry is invalidated. 2073 */ 2074 static __inline void 2075 pmap_s1_invalidate_kernel(uint64_t r, bool final_only) 2076 { 2077 if (final_only) 2078 __asm __volatile("tlbi vaale1is, %0" : : "r" (r)); 2079 else 2080 __asm __volatile("tlbi vaae1is, %0" : : "r" (r)); 2081 } 2082 2083 static __inline void 2084 pmap_s1_invalidate_user(uint64_t r, bool final_only) 2085 { 2086 if (final_only) 2087 __asm __volatile("tlbi vale1is, %0" : : "r" (r)); 2088 else 2089 __asm __volatile("tlbi vae1is, %0" : : "r" (r)); 2090 } 2091 2092 /* 2093 * The range-based counterparts to the above. These may only be performed when 2094 * pmap_tlbi_range_support is true. 2095 */ 2096 static __inline void 2097 pmap_s1_invalidate_range_kernel(uint64_t r, bool final_only) 2098 { 2099 if (final_only) 2100 __asm __volatile(".arch_extension tlb-rmi \n" 2101 "tlbi rvaale1is, %0 \n" 2102 ".arch_extension notlb-rmi" : : "r" (r)); 2103 else 2104 __asm __volatile(".arch_extension tlb-rmi \n" 2105 "tlbi rvaae1is, %0 \n" 2106 ".arch_extension notlb-rmi" : : "r" (r)); 2107 } 2108 2109 static __inline void 2110 pmap_s1_invalidate_range_user(uint64_t r, bool final_only) 2111 { 2112 if (final_only) 2113 __asm __volatile(".arch_extension tlb-rmi \n" 2114 "tlbi rvale1is, %0 \n" 2115 ".arch_extension notlb-rmi" : : "r" (r)); 2116 else 2117 __asm __volatile(".arch_extension tlb-rmi \n" 2118 "tlbi rvae1is, %0 \n" 2119 ".arch_extension notlb-rmi" : : "r" (r)); 2120 } 2121 2122 /* 2123 * Invalidates any cached final- and optionally intermediate-level TLB entries 2124 * for the specified virtual address in the given virtual address space. 2125 */ 2126 static __inline void 2127 pmap_s1_invalidate_page(pmap_t pmap, vm_offset_t va, bool final_only) 2128 { 2129 uint64_t r; 2130 2131 PMAP_ASSERT_STAGE1(pmap); 2132 2133 dsb(ishst); 2134 r = TLBI_VA(va); 2135 if (pmap == kernel_pmap) { 2136 pmap_s1_invalidate_kernel(r, final_only); 2137 } else { 2138 r |= ASID_TO_OPERAND(COOKIE_TO_ASID(pmap->pm_cookie)); 2139 pmap_s1_invalidate_user(r, final_only); 2140 } 2141 if (pmap_multiple_tlbi) { 2142 dsb(ish); 2143 __asm __volatile("tlbi vale1is, xzr" ::: "memory"); 2144 } 2145 dsb(ish); 2146 isb(); 2147 } 2148 2149 static __inline void 2150 pmap_s2_invalidate_page(pmap_t pmap, vm_offset_t va, bool final_only) 2151 { 2152 PMAP_ASSERT_STAGE2(pmap); 2153 MPASS(pmap_stage2_invalidate_range != NULL); 2154 pmap_stage2_invalidate_range(pmap_to_ttbr0(pmap), va, va + PAGE_SIZE, 2155 final_only); 2156 } 2157 2158 static __inline void 2159 pmap_invalidate_page(pmap_t pmap, vm_offset_t va, bool final_only) 2160 { 2161 if (pmap->pm_stage == PM_STAGE1) 2162 pmap_s1_invalidate_page(pmap, va, final_only); 2163 else 2164 pmap_s2_invalidate_page(pmap, va, final_only); 2165 } 2166 2167 /* 2168 * Invalidates the TLB entries for the mappings in the address range [sva, 2169 * eva), using range-based instructions where possible and single-page 2170 * instructions otherwise. When range-based invalidation is supported, the 2171 * address range is covered by as few TLBI instructions as possible: the 2172 * largest scale whose unit fits within the remaining address range is 2173 * selected, and up to TLBI_RANGE_MAX_UNITS units are invalidated per 2174 * instruction. An address that cannot be encoded as a BaseADDR, because 2175 * pmap_lpa_enabled is true and the address is not 64KB aligned, is detected 2176 * using va_mask and invalidated one stride at a time. 2177 */ 2178 static __always_inline void 2179 pmap_s1_invalidate_loop(vm_offset_t sva, vm_offset_t eva, vm_offset_t stride, 2180 int va_shift, vm_offset_t va_mask, uint64_t asid, bool kernel, 2181 bool final_only) 2182 { 2183 uint64_t units; 2184 vm_size_t pages; 2185 int scale, unit_shift; 2186 2187 for (vm_offset_t va = sva; va < eva;) { 2188 if (pmap_tlbi_range_support && (va & va_mask) == 0) { 2189 pages = atop(eva - va); 2190 if (pages >= TLBI_RANGE_UNIT(0)) { 2191 scale = TLBI_RANGE_SCALE(pages); 2192 unit_shift = TLBI_RANGE_UNIT_SHIFT(scale); 2193 units = ulmin(pages >> unit_shift, 2194 TLBI_RANGE_MAX_UNITS); 2195 if (kernel) 2196 pmap_s1_invalidate_range_kernel(asid | 2197 TLBI_RANGE_FIELDS(va, va_shift, 2198 units - 1, scale), final_only); 2199 else 2200 pmap_s1_invalidate_range_user(asid | 2201 TLBI_RANGE_FIELDS(va, va_shift, 2202 units - 1, scale), final_only); 2203 va += ptoa(units << unit_shift); 2204 continue; 2205 } 2206 } 2207 if (kernel) 2208 pmap_s1_invalidate_kernel(asid | TLBI_VA(va), 2209 final_only); 2210 else 2211 pmap_s1_invalidate_user(asid | TLBI_VA(va), 2212 final_only); 2213 va += stride; 2214 } 2215 } 2216 2217 /* 2218 * Use stride L{1,2}_SIZE when invalidating the TLB entries for L{1,2}_BLOCK 2219 * mappings. Otherwise, use stride L3_SIZE. 2220 */ 2221 static __inline void 2222 pmap_s1_invalidate_strided(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, 2223 vm_offset_t stride, bool final_only) 2224 { 2225 uint64_t asid; 2226 vm_offset_t va_mask; 2227 int va_shift; 2228 2229 PMAP_ASSERT_STAGE1(pmap); 2230 va_shift = TLBI_RANGE_VA_SHIFT(); 2231 /* va_mask will be 0 unless pmap_lpa_enabled is true. */ 2232 va_mask = (1ul << va_shift) - PAGE_SIZE; 2233 dsb(ishst); 2234 if (pmap == kernel_pmap) { 2235 pmap_s1_invalidate_loop(sva, eva, stride, va_shift, va_mask, 2236 0, true, final_only); 2237 } else { 2238 asid = ASID_TO_OPERAND(COOKIE_TO_ASID(pmap->pm_cookie)); 2239 pmap_s1_invalidate_loop(sva, eva, stride, va_shift, va_mask, 2240 asid, false, final_only); 2241 } 2242 if (pmap_multiple_tlbi) { 2243 dsb(ish); 2244 __asm __volatile("tlbi vale1is, xzr" ::: "memory"); 2245 } 2246 dsb(ish); 2247 isb(); 2248 } 2249 2250 /* 2251 * Invalidates any cached final- and optionally intermediate-level TLB entries 2252 * for the specified virtual address range in the given virtual address space. 2253 */ 2254 static __inline void 2255 pmap_s1_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, 2256 bool final_only) 2257 { 2258 pmap_s1_invalidate_strided(pmap, sva, eva, L3_SIZE, final_only); 2259 } 2260 2261 static __inline void 2262 pmap_s2_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, 2263 bool final_only) 2264 { 2265 PMAP_ASSERT_STAGE2(pmap); 2266 MPASS(pmap_stage2_invalidate_range != NULL); 2267 pmap_stage2_invalidate_range(pmap_to_ttbr0(pmap), sva, eva, final_only); 2268 } 2269 2270 static __inline void 2271 pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, 2272 bool final_only) 2273 { 2274 if (pmap->pm_stage == PM_STAGE1) 2275 pmap_s1_invalidate_range(pmap, sva, eva, final_only); 2276 else 2277 pmap_s2_invalidate_range(pmap, sva, eva, final_only); 2278 } 2279 2280 void 2281 pmap_s1_invalidate_all_kernel(void) 2282 { 2283 dsb(ishst); 2284 __asm __volatile("tlbi vmalle1is"); 2285 if (pmap_multiple_tlbi) { 2286 dsb(ish); 2287 __asm __volatile("tlbi vale1is, xzr" ::: "memory"); 2288 } 2289 dsb(ish); 2290 isb(); 2291 } 2292 2293 /* 2294 * Invalidates all cached intermediate- and final-level TLB entries for the 2295 * given virtual address space. 2296 */ 2297 static __inline void 2298 pmap_s1_invalidate_all(pmap_t pmap) 2299 { 2300 uint64_t r; 2301 2302 PMAP_ASSERT_STAGE1(pmap); 2303 2304 dsb(ishst); 2305 if (pmap == kernel_pmap) { 2306 __asm __volatile("tlbi vmalle1is"); 2307 } else { 2308 r = ASID_TO_OPERAND(COOKIE_TO_ASID(pmap->pm_cookie)); 2309 __asm __volatile("tlbi aside1is, %0" : : "r" (r)); 2310 } 2311 if (pmap_multiple_tlbi) { 2312 dsb(ish); 2313 __asm __volatile("tlbi vale1is, xzr" ::: "memory"); 2314 } 2315 dsb(ish); 2316 isb(); 2317 } 2318 2319 static __inline void 2320 pmap_s2_invalidate_all(pmap_t pmap) 2321 { 2322 PMAP_ASSERT_STAGE2(pmap); 2323 MPASS(pmap_stage2_invalidate_all != NULL); 2324 pmap_stage2_invalidate_all(pmap_to_ttbr0(pmap)); 2325 } 2326 2327 static __inline void 2328 pmap_invalidate_all(pmap_t pmap) 2329 { 2330 if (pmap->pm_stage == PM_STAGE1) 2331 pmap_s1_invalidate_all(pmap); 2332 else 2333 pmap_s2_invalidate_all(pmap); 2334 } 2335 2336 /* 2337 * Routine: pmap_extract 2338 * Function: 2339 * Extract the physical page address associated 2340 * with the given map/virtual_address pair. 2341 */ 2342 vm_paddr_t 2343 pmap_extract(pmap_t pmap, vm_offset_t va) 2344 { 2345 pt_entry_t *pte, tpte; 2346 vm_paddr_t pa; 2347 int lvl; 2348 2349 pa = 0; 2350 PMAP_LOCK(pmap); 2351 /* 2352 * Find the block or page map for this virtual address. pmap_pte 2353 * will return either a valid block/page entry, or NULL. 2354 */ 2355 pte = pmap_pte(pmap, va, &lvl); 2356 if (pte != NULL) { 2357 tpte = pmap_load(pte); 2358 pa = PTE_TO_PHYS(tpte); 2359 switch(lvl) { 2360 case 1: 2361 PMAP_ASSERT_L1_BLOCKS_SUPPORTED; 2362 KASSERT((tpte & ATTR_DESCR_MASK) == L1_BLOCK, 2363 ("pmap_extract: Invalid L1 pte found: %lx", 2364 tpte & ATTR_DESCR_MASK)); 2365 pa |= (va & L1_OFFSET); 2366 break; 2367 case 2: 2368 KASSERT((tpte & ATTR_DESCR_MASK) == L2_BLOCK, 2369 ("pmap_extract: Invalid L2 pte found: %lx", 2370 tpte & ATTR_DESCR_MASK)); 2371 pa |= (va & L2_OFFSET); 2372 break; 2373 case 3: 2374 KASSERT((tpte & ATTR_DESCR_MASK) == L3_PAGE, 2375 ("pmap_extract: Invalid L3 pte found: %lx", 2376 tpte & ATTR_DESCR_MASK)); 2377 pa |= (va & L3_OFFSET); 2378 break; 2379 } 2380 } 2381 PMAP_UNLOCK(pmap); 2382 return (pa); 2383 } 2384 2385 /* 2386 * Routine: pmap_extract_and_hold 2387 * Function: 2388 * Atomically extract and hold the physical page 2389 * with the given pmap and virtual address pair 2390 * if that mapping permits the given protection. 2391 */ 2392 vm_page_t 2393 pmap_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot) 2394 { 2395 pt_entry_t *pte, tpte; 2396 vm_offset_t off; 2397 vm_page_t m; 2398 int lvl; 2399 bool use; 2400 2401 m = NULL; 2402 PMAP_LOCK(pmap); 2403 pte = pmap_pte(pmap, va, &lvl); 2404 if (pte != NULL) { 2405 tpte = pmap_load(pte); 2406 2407 KASSERT(lvl > 0 && lvl <= 3, 2408 ("pmap_extract_and_hold: Invalid level %d", lvl)); 2409 /* 2410 * Check that the pte is either a L3 page, or a L1 or L2 block 2411 * entry. We can assume L1_BLOCK == L2_BLOCK. 2412 */ 2413 KASSERT((lvl == 3 && (tpte & ATTR_DESCR_MASK) == L3_PAGE) || 2414 (lvl < 3 && (tpte & ATTR_DESCR_MASK) == L1_BLOCK), 2415 ("pmap_extract_and_hold: Invalid pte at L%d: %lx", lvl, 2416 tpte & ATTR_DESCR_MASK)); 2417 2418 use = false; 2419 if ((prot & VM_PROT_WRITE) == 0) 2420 use = true; 2421 else if (pmap->pm_stage == PM_STAGE1 && 2422 (tpte & ATTR_S1_AP_RW_BIT) == ATTR_S1_AP(ATTR_S1_AP_RW)) 2423 use = true; 2424 else if (pmap->pm_stage == PM_STAGE2 && 2425 ((tpte & ATTR_S2_S2AP(ATTR_S2_S2AP_WRITE)) == 2426 ATTR_S2_S2AP(ATTR_S2_S2AP_WRITE))) 2427 use = true; 2428 2429 if (use) { 2430 switch (lvl) { 2431 case 1: 2432 off = va & L1_OFFSET; 2433 break; 2434 case 2: 2435 off = va & L2_OFFSET; 2436 break; 2437 case 3: 2438 default: 2439 off = 0; 2440 } 2441 m = PHYS_TO_VM_PAGE(PTE_TO_PHYS(tpte) | off); 2442 if (m != NULL && !vm_page_wire_mapped(m)) 2443 m = NULL; 2444 } 2445 } 2446 PMAP_UNLOCK(pmap); 2447 return (m); 2448 } 2449 2450 /* 2451 * Returns true if the entire kernel virtual address range is mapped 2452 */ 2453 static bool 2454 pmap_kmapped_range(void *va, vm_size_t size) 2455 { 2456 pt_entry_t *pte, tpte; 2457 vm_offset_t eva, sva; 2458 2459 sva = (vm_offset_t)va; 2460 KASSERT(sva >= VM_MIN_KERNEL_ADDRESS, 2461 ("%s: Invalid virtual address: %lx", __func__, sva)); 2462 MPASS(size != 0); 2463 eva = sva + size - 1; 2464 KASSERT(eva > sva, ("%s: Size too large: sva %lx, size %lx", __func__, 2465 sva, size)); 2466 2467 while (sva <= eva) { 2468 pte = pmap_l1(kernel_pmap, sva); 2469 if (pte == NULL) 2470 return (false); 2471 tpte = pmap_load(pte); 2472 if (tpte == 0) 2473 return (false); 2474 if ((tpte & ATTR_DESCR_TYPE_MASK) == ATTR_DESCR_TYPE_BLOCK) { 2475 sva = (sva & ~L1_OFFSET) + L1_SIZE; 2476 continue; 2477 } 2478 2479 pte = pmap_l1_to_l2(&tpte, sva); 2480 tpte = pmap_load(pte); 2481 if (tpte == 0) 2482 return (false); 2483 if ((tpte & ATTR_DESCR_TYPE_MASK) == ATTR_DESCR_TYPE_BLOCK) { 2484 sva = (sva & ~L2_OFFSET) + L2_SIZE; 2485 continue; 2486 } 2487 pte = pmap_l2_to_l3(&tpte, sva); 2488 tpte = pmap_load(pte); 2489 if (tpte == 0) 2490 return (false); 2491 MPASS((tpte & ATTR_DESCR_TYPE_MASK) == ATTR_DESCR_TYPE_PAGE); 2492 if ((tpte & ATTR_CONTIGUOUS) == ATTR_CONTIGUOUS) 2493 sva = (sva & ~L3C_OFFSET) + L3C_SIZE; 2494 else 2495 sva = (sva & ~L3_OFFSET) + L3_SIZE; 2496 } 2497 2498 return (true); 2499 } 2500 2501 /* 2502 * Walks the page tables to translate a kernel virtual address to a 2503 * physical address. Returns true if the kva is valid and stores the 2504 * physical address in pa if it is not NULL. 2505 * 2506 * See the comment above data_abort() for the rationale for specifying 2507 * NO_PERTHREAD_SSP here. 2508 */ 2509 bool NO_PERTHREAD_SSP 2510 pmap_klookup(vm_offset_t va, vm_paddr_t *pa) 2511 { 2512 pt_entry_t *pte, tpte; 2513 register_t intr; 2514 uint64_t par; 2515 2516 /* 2517 * Disable interrupts so we don't get interrupted between asking 2518 * for address translation, and getting the result back. 2519 */ 2520 intr = intr_disable(); 2521 par = arm64_address_translate_s1e1r(va); 2522 intr_restore(intr); 2523 2524 if (PAR_SUCCESS(par)) { 2525 if (pa != NULL) 2526 *pa = (par & PAR_PA_MASK) | (va & PAR_LOW_MASK); 2527 return (true); 2528 } 2529 2530 /* 2531 * Fall back to walking the page table. The address translation 2532 * instruction may fail when the page is in a break-before-make 2533 * sequence. As we only clear the valid bit in said sequence we 2534 * can walk the page table to find the physical address. 2535 */ 2536 2537 pte = pmap_l1(kernel_pmap, va); 2538 if (pte == NULL) 2539 return (false); 2540 2541 /* 2542 * A concurrent pmap_update_entry() will clear the entry's valid bit 2543 * but leave the rest of the entry unchanged. Therefore, we treat a 2544 * non-zero entry as being valid, and we ignore the valid bit when 2545 * determining whether the entry maps a block, page, or table. 2546 */ 2547 tpte = pmap_load(pte); 2548 if (tpte == 0) 2549 return (false); 2550 if ((tpte & ATTR_DESCR_TYPE_MASK) == ATTR_DESCR_TYPE_BLOCK) { 2551 if (pa != NULL) 2552 *pa = PTE_TO_PHYS(tpte) | (va & L1_OFFSET); 2553 return (true); 2554 } 2555 pte = pmap_l1_to_l2(&tpte, va); 2556 tpte = pmap_load(pte); 2557 if (tpte == 0) 2558 return (false); 2559 if ((tpte & ATTR_DESCR_TYPE_MASK) == ATTR_DESCR_TYPE_BLOCK) { 2560 if (pa != NULL) 2561 *pa = PTE_TO_PHYS(tpte) | (va & L2_OFFSET); 2562 return (true); 2563 } 2564 pte = pmap_l2_to_l3(&tpte, va); 2565 tpte = pmap_load(pte); 2566 if (tpte == 0) 2567 return (false); 2568 if (pa != NULL) 2569 *pa = PTE_TO_PHYS(tpte) | (va & L3_OFFSET); 2570 return (true); 2571 } 2572 2573 /* 2574 * Routine: pmap_kextract 2575 * Function: 2576 * Extract the physical page address associated with the given kernel 2577 * virtual address. 2578 */ 2579 vm_paddr_t 2580 pmap_kextract(vm_offset_t va) 2581 { 2582 vm_paddr_t pa; 2583 2584 if (va >= DMAP_MIN_ADDRESS && va < DMAP_MAX_ADDRESS) 2585 return (DMAP_TO_PHYS(va)); 2586 2587 if (pmap_klookup(va, &pa) == false) 2588 return (0); 2589 return (pa); 2590 } 2591 2592 /*************************************************** 2593 * Low level mapping routines..... 2594 ***************************************************/ 2595 2596 void 2597 pmap_kenter(vm_offset_t sva, vm_size_t size, vm_paddr_t pa, int mode) 2598 { 2599 pd_entry_t *pde; 2600 pt_entry_t attr, old_l3e, *pte; 2601 vm_offset_t va; 2602 vm_page_t mpte; 2603 int error, lvl; 2604 2605 KASSERT((pa & L3_OFFSET) == 0, 2606 ("pmap_kenter: Invalid physical address")); 2607 KASSERT((sva & L3_OFFSET) == 0, 2608 ("pmap_kenter: Invalid virtual address")); 2609 KASSERT((size & PAGE_MASK) == 0, 2610 ("pmap_kenter: Mapping is not page-sized")); 2611 2612 /* CCA - Map devices as nonsecure */ 2613 if (in_realm() && (mode == VM_MEMATTR_DEVICE || 2614 mode == VM_MEMATTR_DEVICE_NP)) 2615 pa |= prot_ns_shared_pa; 2616 2617 attr = ATTR_AF | pmap_sh_attr | ATTR_S1_AP(ATTR_S1_AP_RW) | 2618 ATTR_S1_XN | ATTR_KERN_GP | ATTR_S1_IDX(mode); 2619 old_l3e = 0; 2620 va = sva; 2621 while (size != 0) { 2622 pde = pmap_pde(kernel_pmap, va, &lvl); 2623 KASSERT(pde != NULL, 2624 ("pmap_kenter: Invalid page entry, va: 0x%lx", va)); 2625 KASSERT(lvl == 2, ("pmap_kenter: Invalid level %d", lvl)); 2626 2627 /* 2628 * If we have an aligned, contiguous chunk of L2_SIZE, try 2629 * to create an L2_BLOCK mapping. 2630 */ 2631 if ((va & L2_OFFSET) == 0 && size >= L2_SIZE && 2632 (pa & L2_OFFSET) == 0 && vm_initialized) { 2633 mpte = PTE_TO_VM_PAGE(pmap_load(pde)); 2634 KASSERT(pmap_every_pte_zero(VM_PAGE_TO_PHYS(mpte)), 2635 ("pmap_kenter: Unexpected mapping")); 2636 PMAP_LOCK(kernel_pmap); 2637 error = pmap_insert_pt_page(kernel_pmap, mpte, false, 2638 false); 2639 if (error == 0) { 2640 attr &= ~ATTR_CONTIGUOUS; 2641 2642 /* 2643 * Although the page table page "mpte" should 2644 * be devoid of mappings, the TLB might hold 2645 * intermediate entries that reference it, so 2646 * we perform a single-page invalidation. 2647 */ 2648 pmap_update_entry(kernel_pmap, pde, 2649 PHYS_TO_PTE(pa) | attr | L2_BLOCK, va, 2650 PAGE_SIZE, false); 2651 } 2652 PMAP_UNLOCK(kernel_pmap); 2653 if (error == 0) { 2654 va += L2_SIZE; 2655 pa += L2_SIZE; 2656 size -= L2_SIZE; 2657 continue; 2658 } 2659 } 2660 2661 /* 2662 * If we have an aligned, contiguous chunk of L3C_ENTRIES 2663 * L3 pages, set the contiguous bit within each PTE so that 2664 * the chunk can be cached using only one TLB entry. 2665 */ 2666 if ((va & L3C_OFFSET) == 0 && (pa & L3C_OFFSET) == 0) { 2667 if (size >= L3C_SIZE) 2668 attr |= ATTR_CONTIGUOUS; 2669 else 2670 attr &= ~ATTR_CONTIGUOUS; 2671 } 2672 2673 pte = pmap_l2_to_l3(pde, va); 2674 old_l3e |= pmap_load_store(pte, PHYS_TO_PTE(pa) | attr | 2675 L3_PAGE); 2676 2677 va += PAGE_SIZE; 2678 pa += PAGE_SIZE; 2679 size -= PAGE_SIZE; 2680 } 2681 if ((old_l3e & ATTR_DESCR_VALID) != 0) 2682 pmap_s1_invalidate_range(kernel_pmap, sva, va, true); 2683 else { 2684 /* 2685 * Because the old entries were invalid and the new mappings 2686 * are not executable, an isb is not required. 2687 */ 2688 dsb(ishst); 2689 } 2690 } 2691 2692 void 2693 pmap_kenter_device(vm_offset_t sva, vm_size_t size, vm_paddr_t pa) 2694 { 2695 2696 pmap_kenter(sva, size, pa, VM_MEMATTR_DEVICE); 2697 } 2698 2699 /* 2700 * Remove a page from the kernel pagetables. 2701 */ 2702 void 2703 pmap_kremove(vm_offset_t va) 2704 { 2705 pt_entry_t *pte; 2706 2707 pte = pmap_pte_exists(kernel_pmap, va, 3, __func__); 2708 KASSERT((pmap_load(pte) & ATTR_CONTIGUOUS) == 0, 2709 ("pmap_kremove: unexpected ATTR_CONTIGUOUS")); 2710 pmap_clear(pte); 2711 pmap_s1_invalidate_page(kernel_pmap, va, true); 2712 } 2713 2714 /* 2715 * Remove the specified range of mappings from the kernel address space. 2716 * 2717 * Should only be applied to mappings that were created by pmap_kenter() or 2718 * pmap_kenter_device(). Nothing about this function is actually specific 2719 * to device mappings. 2720 */ 2721 void 2722 pmap_kremove_device(vm_offset_t sva, vm_size_t size) 2723 { 2724 pt_entry_t *ptep, *ptep_end; 2725 vm_offset_t va; 2726 int lvl; 2727 2728 KASSERT((sva & L3_OFFSET) == 0, 2729 ("pmap_kremove_device: Invalid virtual address")); 2730 KASSERT((size & PAGE_MASK) == 0, 2731 ("pmap_kremove_device: Mapping is not page-sized")); 2732 2733 va = sva; 2734 while (size != 0) { 2735 ptep = pmap_pte(kernel_pmap, va, &lvl); 2736 KASSERT(ptep != NULL, ("Invalid page table, va: 0x%lx", va)); 2737 switch (lvl) { 2738 case 2: 2739 KASSERT((va & L2_OFFSET) == 0, 2740 ("Unaligned virtual address")); 2741 KASSERT(size >= L2_SIZE, ("Insufficient size")); 2742 2743 if (va != sva) { 2744 pmap_s1_invalidate_range(kernel_pmap, sva, va, 2745 true); 2746 } 2747 pmap_clear(ptep); 2748 pmap_s1_invalidate_page(kernel_pmap, va, true); 2749 PMAP_LOCK(kernel_pmap); 2750 pmap_remove_kernel_l2(kernel_pmap, ptep, va); 2751 PMAP_UNLOCK(kernel_pmap); 2752 2753 va += L2_SIZE; 2754 sva = va; 2755 size -= L2_SIZE; 2756 break; 2757 case 3: 2758 if ((pmap_load(ptep) & ATTR_CONTIGUOUS) != 0) { 2759 KASSERT((va & L3C_OFFSET) == 0, 2760 ("Unaligned L3C virtual address")); 2761 KASSERT(size >= L3C_SIZE, 2762 ("Insufficient L3C size")); 2763 2764 ptep_end = ptep + L3C_ENTRIES; 2765 for (; ptep < ptep_end; ptep++) 2766 pmap_clear(ptep); 2767 2768 va += L3C_SIZE; 2769 size -= L3C_SIZE; 2770 break; 2771 } 2772 pmap_clear(ptep); 2773 2774 va += PAGE_SIZE; 2775 size -= PAGE_SIZE; 2776 break; 2777 default: 2778 __assert_unreachable(); 2779 break; 2780 } 2781 } 2782 if (va != sva) 2783 pmap_s1_invalidate_range(kernel_pmap, sva, va, true); 2784 } 2785 2786 /* 2787 * Used to map a range of physical addresses into kernel 2788 * virtual address space. 2789 * 2790 * The value passed in '*virt' is a suggested virtual address for 2791 * the mapping. Architectures which can support a direct-mapped 2792 * physical to virtual region can return the appropriate address 2793 * within that region, leaving '*virt' unchanged. Other 2794 * architectures should map the pages starting at '*virt' and 2795 * update '*virt' with the first usable address after the mapped 2796 * region. 2797 */ 2798 void * 2799 pmap_map(vm_offset_t *virt, vm_paddr_t start, vm_paddr_t end, int prot) 2800 { 2801 return (PHYS_TO_DMAP(start)); 2802 } 2803 2804 /* 2805 * Add a list of wired pages to the kva 2806 * this routine is only used for temporary 2807 * kernel mappings that do not need to have 2808 * page modification or references recorded. 2809 * Note that old mappings are simply written 2810 * over. The page *must* be wired. 2811 * Note: SMP coherent. Uses a ranged shootdown IPI. 2812 */ 2813 void 2814 pmap_qenter(void *sva, vm_page_t *ma, int count) 2815 { 2816 pd_entry_t *pde; 2817 pt_entry_t attr, old_l3e, *pte; 2818 vm_offset_t va; 2819 vm_page_t m; 2820 int i, lvl; 2821 2822 old_l3e = 0; 2823 va = (vm_offset_t)sva; 2824 for (i = 0; i < count; i++) { 2825 pde = pmap_pde(kernel_pmap, va, &lvl); 2826 KASSERT(pde != NULL, 2827 ("pmap_qenter: Invalid page entry, va: 0x%lx", va)); 2828 KASSERT(lvl == 2, 2829 ("pmap_qenter: Invalid level %d", lvl)); 2830 2831 m = ma[i]; 2832 attr = ATTR_AF | pmap_sh_attr | 2833 ATTR_S1_AP(ATTR_S1_AP_RW) | ATTR_S1_XN | 2834 ATTR_KERN_GP | ATTR_S1_IDX(m->md.pv_memattr) | L3_PAGE; 2835 pte = pmap_l2_to_l3(pde, va); 2836 old_l3e |= pmap_load_store(pte, VM_PAGE_TO_PTE(m) | attr); 2837 2838 va += L3_SIZE; 2839 } 2840 if ((old_l3e & ATTR_DESCR_VALID) != 0) 2841 pmap_s1_invalidate_range(kernel_pmap, (vm_offset_t)sva, va, 2842 true); 2843 else { 2844 /* 2845 * Because the old entries were invalid and the new mappings 2846 * are not executable, an isb is not required. 2847 */ 2848 dsb(ishst); 2849 } 2850 } 2851 2852 /* 2853 * This routine tears out page mappings from the 2854 * kernel -- it is meant only for temporary mappings. 2855 */ 2856 void 2857 pmap_qremove(void *sva, int count) 2858 { 2859 pt_entry_t *pte; 2860 vm_offset_t va; 2861 2862 va = (vm_offset_t)sva; 2863 2864 KASSERT(ADDR_IS_CANONICAL(va), 2865 ("%s: Address not in canonical form: %p", __func__, sva)); 2866 KASSERT(ADDR_IS_KERNEL(va), ("usermode va %p", sva)); 2867 2868 while (count-- > 0) { 2869 pte = pmap_pte_exists(kernel_pmap, va, 3, NULL); 2870 if (pte != NULL) { 2871 pmap_clear(pte); 2872 } 2873 2874 va += PAGE_SIZE; 2875 } 2876 pmap_s1_invalidate_range(kernel_pmap, (vm_offset_t)sva, va, true); 2877 } 2878 2879 /*************************************************** 2880 * Page table page management routines..... 2881 ***************************************************/ 2882 /* 2883 * Schedule the specified unused page table page to be freed. Specifically, 2884 * add the page to the specified list of pages that will be released to the 2885 * physical memory manager after the TLB has been updated. 2886 */ 2887 static __inline void 2888 pmap_add_delayed_free_list(vm_page_t m, struct spglist *free, bool set_PG_ZERO) 2889 { 2890 2891 if (set_PG_ZERO) 2892 m->flags |= PG_ZERO; 2893 else 2894 m->flags &= ~PG_ZERO; 2895 SLIST_INSERT_HEAD(free, m, plinks.s.ss); 2896 } 2897 2898 /* 2899 * Decrements a page table page's reference count, which is used to record the 2900 * number of valid page table entries within the page. If the reference count 2901 * drops to zero, then the page table page is unmapped. Returns true if the 2902 * page table page was unmapped and false otherwise. 2903 */ 2904 static inline bool 2905 pmap_unwire_l3(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free) 2906 { 2907 2908 --m->ref_count; 2909 if (m->ref_count == 0) { 2910 _pmap_unwire_l3(pmap, va, m, free); 2911 return (true); 2912 } else 2913 return (false); 2914 } 2915 2916 static void 2917 _pmap_unwire_l3(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free) 2918 { 2919 2920 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 2921 /* 2922 * unmap the page table page 2923 */ 2924 if (m->pindex >= (NUL2E + NUL1E)) { 2925 /* l1 page */ 2926 pd_entry_t *l0; 2927 2928 l0 = pmap_l0(pmap, va); 2929 pmap_clear(l0); 2930 } else if (m->pindex >= NUL2E) { 2931 /* l2 page */ 2932 pd_entry_t *l1; 2933 2934 l1 = pmap_l1(pmap, va); 2935 pmap_clear(l1); 2936 } else { 2937 /* l3 page */ 2938 pd_entry_t *l2; 2939 2940 l2 = pmap_l2(pmap, va); 2941 pmap_clear(l2); 2942 } 2943 pmap_resident_count_dec(pmap, 1); 2944 if (m->pindex < NUL2E) { 2945 /* We just released an l3, unhold the matching l2 */ 2946 pd_entry_t *l1, tl1; 2947 vm_page_t l2pg; 2948 2949 l1 = pmap_l1(pmap, va); 2950 tl1 = pmap_load(l1); 2951 l2pg = PTE_TO_VM_PAGE(tl1); 2952 pmap_unwire_l3(pmap, va, l2pg, free); 2953 } else if (m->pindex < (NUL2E + NUL1E)) { 2954 /* We just released an l2, unhold the matching l1 */ 2955 pd_entry_t *l0, tl0; 2956 vm_page_t l1pg; 2957 2958 l0 = pmap_l0(pmap, va); 2959 tl0 = pmap_load(l0); 2960 l1pg = PTE_TO_VM_PAGE(tl0); 2961 pmap_unwire_l3(pmap, va, l1pg, free); 2962 } 2963 pmap_invalidate_page(pmap, va, false); 2964 2965 /* 2966 * Put page on a list so that it is released after 2967 * *ALL* TLB shootdown is done 2968 */ 2969 pmap_add_delayed_free_list(m, free, true); 2970 } 2971 2972 /* 2973 * After removing a page table entry, this routine is used to 2974 * conditionally free the page, and manage the reference count. 2975 */ 2976 static int 2977 pmap_unuse_pt(pmap_t pmap, vm_offset_t va, pd_entry_t ptepde, 2978 struct spglist *free) 2979 { 2980 vm_page_t mpte; 2981 2982 KASSERT(ADDR_IS_CANONICAL(va), 2983 ("%s: Address not in canonical form: %lx", __func__, va)); 2984 if (ADDR_IS_KERNEL(va)) 2985 return (0); 2986 KASSERT(ptepde != 0, ("pmap_unuse_pt: ptepde != 0")); 2987 mpte = PTE_TO_VM_PAGE(ptepde); 2988 return (pmap_unwire_l3(pmap, va, mpte, free)); 2989 } 2990 2991 /* 2992 * Release a page table page reference after a failed attempt to create a 2993 * mapping. 2994 */ 2995 static void 2996 pmap_abort_ptp(pmap_t pmap, vm_offset_t va, vm_page_t mpte) 2997 { 2998 struct spglist free; 2999 3000 SLIST_INIT(&free); 3001 if (pmap_unwire_l3(pmap, va, mpte, &free)) 3002 vm_page_free_pages_toq(&free, true); 3003 } 3004 3005 void 3006 pmap_pinit0(pmap_t pmap) 3007 { 3008 3009 PMAP_LOCK_INIT(pmap); 3010 bzero(&pmap->pm_stats, sizeof(pmap->pm_stats)); 3011 pmap->pm_l0_paddr = READ_SPECIALREG(ttbr0_el1); 3012 pmap->pm_l0 = PHYS_TO_DMAP(pmap->pm_l0_paddr); 3013 TAILQ_INIT(&pmap->pm_pvchunk); 3014 vm_radix_init(&pmap->pm_root); 3015 pmap->pm_cookie = COOKIE_FROM(ASID_RESERVED_FOR_PID_0, INT_MIN); 3016 pmap->pm_stage = PM_STAGE1; 3017 pmap->pm_levels = 4; 3018 pmap->pm_ttbr = pmap->pm_l0_paddr; 3019 pmap->pm_asid_set = &asids; 3020 pmap->pm_bti = NULL; 3021 3022 PCPU_SET(curpmap, pmap); 3023 } 3024 3025 int 3026 pmap_pinit_stage(pmap_t pmap, enum pmap_stage stage, int levels) 3027 { 3028 vm_page_t m; 3029 3030 /* 3031 * allocate the l0 page 3032 */ 3033 m = vm_page_alloc_noobj(VM_ALLOC_WAITOK | VM_ALLOC_WIRED | 3034 VM_ALLOC_ZERO); 3035 pmap->pm_l0_paddr = VM_PAGE_TO_PHYS(m); 3036 pmap->pm_l0 = PHYS_TO_DMAP(pmap->pm_l0_paddr); 3037 3038 TAILQ_INIT(&pmap->pm_pvchunk); 3039 vm_radix_init(&pmap->pm_root); 3040 bzero(&pmap->pm_stats, sizeof(pmap->pm_stats)); 3041 pmap->pm_cookie = COOKIE_FROM(-1, INT_MAX); 3042 3043 MPASS(levels == 3 || levels == 4); 3044 pmap->pm_levels = levels; 3045 pmap->pm_stage = stage; 3046 pmap->pm_bti = NULL; 3047 switch (stage) { 3048 case PM_STAGE1: 3049 pmap->pm_asid_set = &asids; 3050 if (pmap_bti_support) { 3051 pmap->pm_bti = malloc(sizeof(struct rangeset), M_DEVBUF, 3052 M_ZERO | M_WAITOK); 3053 rangeset_init(pmap->pm_bti, bti_dup_range, 3054 bti_free_range, pmap, M_NOWAIT); 3055 } 3056 break; 3057 case PM_STAGE2: 3058 pmap->pm_asid_set = &vmids; 3059 break; 3060 default: 3061 panic("%s: Invalid pmap type %d", __func__, stage); 3062 break; 3063 } 3064 3065 /* XXX Temporarily disable deferred ASID allocation. */ 3066 pmap_alloc_asid(pmap); 3067 3068 /* 3069 * Allocate the level 1 entry to use as the root. This will increase 3070 * the refcount on the level 1 page so it won't be removed until 3071 * pmap_release() is called. 3072 */ 3073 if (pmap->pm_levels == 3) { 3074 PMAP_LOCK(pmap); 3075 m = _pmap_alloc_l3(pmap, NUL2E + NUL1E, NULL); 3076 PMAP_UNLOCK(pmap); 3077 } 3078 pmap->pm_ttbr = VM_PAGE_TO_PHYS(m); 3079 3080 return (1); 3081 } 3082 3083 int 3084 pmap_pinit(pmap_t pmap) 3085 { 3086 3087 return (pmap_pinit_stage(pmap, PM_STAGE1, 4)); 3088 } 3089 3090 /* 3091 * This routine is called if the desired page table page does not exist. 3092 * 3093 * If page table page allocation fails, this routine may sleep before 3094 * returning NULL. It sleeps only if a lock pointer was given. 3095 * 3096 * Note: If a page allocation fails at page table level two or three, 3097 * one or two pages may be held during the wait, only to be released 3098 * afterwards. This conservative approach is easily argued to avoid 3099 * race conditions. 3100 */ 3101 static vm_page_t 3102 _pmap_alloc_l3(pmap_t pmap, vm_pindex_t ptepindex, struct rwlock **lockp) 3103 { 3104 vm_page_t m, l1pg, l2pg; 3105 3106 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 3107 3108 /* 3109 * Allocate a page table page. 3110 */ 3111 if ((m = vm_page_alloc_noobj(VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) { 3112 if (lockp != NULL) { 3113 RELEASE_PV_LIST_LOCK(lockp); 3114 PMAP_UNLOCK(pmap); 3115 vm_wait(NULL); 3116 PMAP_LOCK(pmap); 3117 } 3118 3119 /* 3120 * Indicate the need to retry. While waiting, the page table 3121 * page may have been allocated. 3122 */ 3123 return (NULL); 3124 } 3125 m->pindex = ptepindex; 3126 3127 /* 3128 * Because of AArch64's weak memory consistency model, we must have a 3129 * barrier here to ensure that the stores for zeroing "m", whether by 3130 * pmap_zero_page() or an earlier function, are visible before adding 3131 * "m" to the page table. Otherwise, a page table walk by another 3132 * processor's MMU could see the mapping to "m" and a stale, non-zero 3133 * PTE within "m". 3134 */ 3135 dmb(ishst); 3136 3137 /* 3138 * Map the pagetable page into the process address space, if 3139 * it isn't already there. 3140 */ 3141 3142 if (ptepindex >= (NUL2E + NUL1E)) { 3143 pd_entry_t *l0p, l0e; 3144 vm_pindex_t l0index; 3145 3146 l0index = ptepindex - (NUL2E + NUL1E); 3147 l0p = &pmap->pm_l0[l0index]; 3148 KASSERT((pmap_load(l0p) & ATTR_DESCR_VALID) == 0, 3149 ("%s: L0 entry %#lx is valid", __func__, pmap_load(l0p))); 3150 l0e = VM_PAGE_TO_PTE(m) | L0_TABLE; 3151 3152 /* 3153 * Mark all kernel memory as not accessible from userspace 3154 * and userspace memory as not executable from the kernel. 3155 * This has been done for the bootstrap L0 entries in 3156 * locore.S. 3157 */ 3158 if (pmap == kernel_pmap) 3159 l0e |= TATTR_UXN_TABLE | TATTR_AP_TABLE_NO_EL0; 3160 else 3161 l0e |= TATTR_PXN_TABLE; 3162 pmap_store(l0p, l0e); 3163 } else if (ptepindex >= NUL2E) { 3164 vm_pindex_t l0index, l1index; 3165 pd_entry_t *l0, *l1; 3166 pd_entry_t tl0; 3167 3168 l1index = ptepindex - NUL2E; 3169 l0index = l1index >> Ln_ENTRIES_SHIFT; 3170 3171 l0 = &pmap->pm_l0[l0index]; 3172 tl0 = pmap_load(l0); 3173 if (tl0 == 0) { 3174 /* recurse for allocating page dir */ 3175 if (_pmap_alloc_l3(pmap, NUL2E + NUL1E + l0index, 3176 lockp) == NULL) { 3177 vm_page_unwire_noq(m); 3178 vm_page_free_zero(m); 3179 return (NULL); 3180 } 3181 } else { 3182 l1pg = PTE_TO_VM_PAGE(tl0); 3183 l1pg->ref_count++; 3184 } 3185 3186 l1 = PHYS_TO_DMAP(PTE_TO_PHYS(pmap_load(l0))); 3187 l1 = &l1[ptepindex & Ln_ADDR_MASK]; 3188 KASSERT((pmap_load(l1) & ATTR_DESCR_VALID) == 0, 3189 ("%s: L1 entry %#lx is valid", __func__, pmap_load(l1))); 3190 pmap_store(l1, VM_PAGE_TO_PTE(m) | L1_TABLE); 3191 } else { 3192 vm_pindex_t l0index, l1index; 3193 pd_entry_t *l0, *l1, *l2; 3194 pd_entry_t tl0, tl1; 3195 3196 l1index = ptepindex >> Ln_ENTRIES_SHIFT; 3197 l0index = l1index >> Ln_ENTRIES_SHIFT; 3198 3199 l0 = &pmap->pm_l0[l0index]; 3200 tl0 = pmap_load(l0); 3201 if (tl0 == 0) { 3202 /* recurse for allocating page dir */ 3203 if (_pmap_alloc_l3(pmap, NUL2E + l1index, 3204 lockp) == NULL) { 3205 vm_page_unwire_noq(m); 3206 vm_page_free_zero(m); 3207 return (NULL); 3208 } 3209 tl0 = pmap_load(l0); 3210 l1 = PHYS_TO_DMAP(PTE_TO_PHYS(tl0)); 3211 l1 = &l1[l1index & Ln_ADDR_MASK]; 3212 } else { 3213 l1 = PHYS_TO_DMAP(PTE_TO_PHYS(tl0)); 3214 l1 = &l1[l1index & Ln_ADDR_MASK]; 3215 tl1 = pmap_load(l1); 3216 if (tl1 == 0) { 3217 /* recurse for allocating page dir */ 3218 if (_pmap_alloc_l3(pmap, NUL2E + l1index, 3219 lockp) == NULL) { 3220 vm_page_unwire_noq(m); 3221 vm_page_free_zero(m); 3222 return (NULL); 3223 } 3224 } else { 3225 l2pg = PTE_TO_VM_PAGE(tl1); 3226 l2pg->ref_count++; 3227 } 3228 } 3229 3230 l2 = PHYS_TO_DMAP(PTE_TO_PHYS(pmap_load(l1))); 3231 l2 = &l2[ptepindex & Ln_ADDR_MASK]; 3232 KASSERT((pmap_load(l2) & ATTR_DESCR_VALID) == 0, 3233 ("%s: L2 entry %#lx is valid", __func__, pmap_load(l2))); 3234 pmap_store(l2, VM_PAGE_TO_PTE(m) | L2_TABLE); 3235 } 3236 3237 pmap_resident_count_inc(pmap, 1); 3238 3239 return (m); 3240 } 3241 3242 static pd_entry_t * 3243 pmap_alloc_l2(pmap_t pmap, vm_offset_t va, vm_page_t *l2pgp, 3244 struct rwlock **lockp) 3245 { 3246 pd_entry_t *l1, *l2; 3247 vm_page_t l2pg; 3248 vm_pindex_t l2pindex; 3249 3250 KASSERT(ADDR_IS_CANONICAL(va), 3251 ("%s: Address not in canonical form: %lx", __func__, va)); 3252 3253 retry: 3254 l1 = pmap_l1(pmap, va); 3255 if (l1 != NULL && (pmap_load(l1) & ATTR_DESCR_MASK) == L1_TABLE) { 3256 l2 = pmap_l1_to_l2(l1, va); 3257 if (ADDR_IS_USER(va)) { 3258 /* Add a reference to the L2 page. */ 3259 l2pg = PTE_TO_VM_PAGE(pmap_load(l1)); 3260 l2pg->ref_count++; 3261 } else 3262 l2pg = NULL; 3263 } else if (ADDR_IS_USER(va)) { 3264 /* Allocate a L2 page. */ 3265 l2pindex = pmap_l2_pindex(va) >> Ln_ENTRIES_SHIFT; 3266 l2pg = _pmap_alloc_l3(pmap, NUL2E + l2pindex, lockp); 3267 if (l2pg == NULL) { 3268 if (lockp != NULL) 3269 goto retry; 3270 else 3271 return (NULL); 3272 } 3273 l2 = VM_PAGE_TO_DMAP(l2pg); 3274 l2 = &l2[pmap_l2_index(va)]; 3275 } else 3276 panic("pmap_alloc_l2: missing page table page for va %#lx", 3277 va); 3278 *l2pgp = l2pg; 3279 return (l2); 3280 } 3281 3282 static vm_page_t 3283 pmap_alloc_l3(pmap_t pmap, vm_offset_t va, struct rwlock **lockp) 3284 { 3285 vm_pindex_t ptepindex; 3286 pd_entry_t *pde, tpde; 3287 #ifdef INVARIANTS 3288 pt_entry_t *pte; 3289 #endif 3290 vm_page_t m; 3291 int lvl; 3292 3293 /* 3294 * Calculate pagetable page index 3295 */ 3296 ptepindex = pmap_l2_pindex(va); 3297 retry: 3298 /* 3299 * Get the page directory entry 3300 */ 3301 pde = pmap_pde(pmap, va, &lvl); 3302 3303 /* 3304 * If the page table page is mapped, we just increment the hold count, 3305 * and activate it. If we get a level 2 pde it will point to a level 3 3306 * table. 3307 */ 3308 switch (lvl) { 3309 case -1: 3310 break; 3311 case 0: 3312 #ifdef INVARIANTS 3313 pte = pmap_l0_to_l1(pde, va); 3314 KASSERT(pmap_load(pte) == 0, 3315 ("pmap_alloc_l3: TODO: l0 superpages")); 3316 #endif 3317 break; 3318 case 1: 3319 #ifdef INVARIANTS 3320 pte = pmap_l1_to_l2(pde, va); 3321 KASSERT(pmap_load(pte) == 0, 3322 ("pmap_alloc_l3: TODO: l1 superpages")); 3323 #endif 3324 break; 3325 case 2: 3326 tpde = pmap_load(pde); 3327 if (tpde != 0) { 3328 m = PTE_TO_VM_PAGE(tpde); 3329 m->ref_count++; 3330 return (m); 3331 } 3332 break; 3333 default: 3334 panic("pmap_alloc_l3: Invalid level %d", lvl); 3335 } 3336 3337 /* 3338 * Here if the pte page isn't mapped, or if it has been deallocated. 3339 */ 3340 m = _pmap_alloc_l3(pmap, ptepindex, lockp); 3341 if (m == NULL && lockp != NULL) 3342 goto retry; 3343 3344 return (m); 3345 } 3346 3347 /*************************************************** 3348 * Pmap allocation/deallocation routines. 3349 ***************************************************/ 3350 3351 /* 3352 * Release any resources held by the given physical map. 3353 * Called when a pmap initialized by pmap_pinit is being released. 3354 * Should only be called if the map contains no valid mappings. 3355 */ 3356 void 3357 pmap_release(pmap_t pmap) 3358 { 3359 bool rv __diagused; 3360 struct spglist freelist; 3361 struct asid_set *set; 3362 vm_page_t m; 3363 int asid; 3364 3365 if (pmap->pm_levels != 4) { 3366 PMAP_ASSERT_STAGE2(pmap); 3367 KASSERT(pmap->pm_stats.resident_count == 1, 3368 ("pmap_release: pmap resident count %ld != 0", 3369 pmap->pm_stats.resident_count)); 3370 KASSERT((pmap->pm_l0[0] & ATTR_DESCR_VALID) == ATTR_DESCR_VALID, 3371 ("pmap_release: Invalid l0 entry: %lx", pmap->pm_l0[0])); 3372 3373 SLIST_INIT(&freelist); 3374 m = PHYS_TO_VM_PAGE(pmap->pm_ttbr); 3375 PMAP_LOCK(pmap); 3376 rv = pmap_unwire_l3(pmap, 0, m, &freelist); 3377 PMAP_UNLOCK(pmap); 3378 MPASS(rv == true); 3379 vm_page_free_pages_toq(&freelist, true); 3380 } 3381 3382 KASSERT(pmap->pm_stats.resident_count == 0, 3383 ("pmap_release: pmap resident count %ld != 0", 3384 pmap->pm_stats.resident_count)); 3385 KASSERT(vm_radix_is_empty(&pmap->pm_root), 3386 ("pmap_release: pmap has reserved page table page(s)")); 3387 3388 set = pmap->pm_asid_set; 3389 KASSERT(set != NULL, ("%s: NULL asid set", __func__)); 3390 3391 /* 3392 * Allow the ASID to be reused. In stage 2 VMIDs we don't invalidate 3393 * the entries when removing them so rely on a later tlb invalidation. 3394 * this will happen when updating the VMID generation. Because of this 3395 * we don't reuse VMIDs within a generation. 3396 */ 3397 if (pmap->pm_stage == PM_STAGE1) { 3398 mtx_lock_spin(&set->asid_set_mutex); 3399 if (COOKIE_TO_EPOCH(pmap->pm_cookie) == set->asid_epoch) { 3400 asid = COOKIE_TO_ASID(pmap->pm_cookie); 3401 KASSERT(asid >= ASID_FIRST_AVAILABLE && 3402 asid < set->asid_set_size, 3403 ("pmap_release: pmap cookie has out-of-range asid")); 3404 bit_clear(set->asid_set, asid); 3405 } 3406 mtx_unlock_spin(&set->asid_set_mutex); 3407 3408 if (pmap->pm_bti != NULL) { 3409 rangeset_fini(pmap->pm_bti); 3410 free(pmap->pm_bti, M_DEVBUF); 3411 } 3412 } 3413 3414 m = PHYS_TO_VM_PAGE(pmap->pm_l0_paddr); 3415 vm_page_unwire_noq(m); 3416 vm_page_free_zero(m); 3417 } 3418 3419 static int 3420 kvm_size(SYSCTL_HANDLER_ARGS) 3421 { 3422 unsigned long ksize = VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS; 3423 3424 return sysctl_handle_long(oidp, &ksize, 0, req); 3425 } 3426 SYSCTL_PROC(_vm, OID_AUTO, kvm_size, CTLTYPE_LONG | CTLFLAG_RD | CTLFLAG_MPSAFE, 3427 0, 0, kvm_size, "LU", 3428 "Size of KVM"); 3429 3430 static int 3431 kvm_free(SYSCTL_HANDLER_ARGS) 3432 { 3433 unsigned long kfree = VM_MAX_KERNEL_ADDRESS - kernel_vm_end; 3434 3435 return sysctl_handle_long(oidp, &kfree, 0, req); 3436 } 3437 SYSCTL_PROC(_vm, OID_AUTO, kvm_free, CTLTYPE_LONG | CTLFLAG_RD | CTLFLAG_MPSAFE, 3438 0, 0, kvm_free, "LU", 3439 "Amount of KVM free"); 3440 3441 /* 3442 * grow the number of kernel page table entries, if needed 3443 */ 3444 static int 3445 pmap_growkernel_nopanic(vm_offset_t addr) 3446 { 3447 vm_page_t nkpg; 3448 pd_entry_t *l0, *l1, *l2; 3449 3450 mtx_assert(&kernel_map->system_mtx, MA_OWNED); 3451 3452 addr = roundup2(addr, L2_SIZE); 3453 if (addr - 1 >= vm_map_max(kernel_map)) 3454 addr = vm_map_max(kernel_map); 3455 if (kernel_vm_end < addr) { 3456 kasan_shadow_map(kernel_vm_end, addr - kernel_vm_end); 3457 kmsan_shadow_map(kernel_vm_end, addr - kernel_vm_end); 3458 } 3459 while (kernel_vm_end < addr) { 3460 l0 = pmap_l0(kernel_pmap, kernel_vm_end); 3461 KASSERT(pmap_load(l0) != 0, 3462 ("pmap_growkernel: No level 0 kernel entry")); 3463 3464 l1 = pmap_l0_to_l1(l0, kernel_vm_end); 3465 if (pmap_load(l1) == 0) { 3466 /* We need a new PDP entry */ 3467 nkpg = vm_page_alloc_noobj(VM_ALLOC_INTERRUPT | 3468 VM_ALLOC_NOFREE | VM_ALLOC_WIRED | VM_ALLOC_ZERO); 3469 if (nkpg == NULL) 3470 return (KERN_RESOURCE_SHORTAGE); 3471 nkpg->pindex = pmap_l1_pindex(kernel_vm_end); 3472 /* See the dmb() in _pmap_alloc_l3(). */ 3473 dmb(ishst); 3474 pmap_store(l1, VM_PAGE_TO_PTE(nkpg) | L1_TABLE); 3475 continue; /* try again */ 3476 } 3477 l2 = pmap_l1_to_l2(l1, kernel_vm_end); 3478 if (pmap_load(l2) != 0) { 3479 kernel_vm_end = (kernel_vm_end + L2_SIZE) & ~L2_OFFSET; 3480 if (kernel_vm_end - 1 >= vm_map_max(kernel_map)) { 3481 kernel_vm_end = vm_map_max(kernel_map); 3482 break; 3483 } 3484 continue; 3485 } 3486 3487 nkpg = vm_page_alloc_noobj(VM_ALLOC_INTERRUPT | 3488 VM_ALLOC_NOFREE | VM_ALLOC_WIRED | VM_ALLOC_ZERO); 3489 if (nkpg == NULL) 3490 return (KERN_RESOURCE_SHORTAGE); 3491 nkpg->pindex = pmap_l2_pindex(kernel_vm_end); 3492 /* See the dmb() in _pmap_alloc_l3(). */ 3493 dmb(ishst); 3494 pmap_store(l2, VM_PAGE_TO_PTE(nkpg) | L2_TABLE); 3495 3496 kernel_vm_end = (kernel_vm_end + L2_SIZE) & ~L2_OFFSET; 3497 if (kernel_vm_end - 1 >= vm_map_max(kernel_map)) { 3498 kernel_vm_end = vm_map_max(kernel_map); 3499 break; 3500 } 3501 } 3502 return (KERN_SUCCESS); 3503 } 3504 3505 int 3506 pmap_growkernel(vm_offset_t addr) 3507 { 3508 int rv; 3509 3510 rv = pmap_growkernel_nopanic(addr); 3511 if (rv != KERN_SUCCESS && pmap_growkernel_panic) 3512 panic("pmap_growkernel: no memory to grow kernel"); 3513 return (rv); 3514 } 3515 3516 /*************************************************** 3517 * page management routines. 3518 ***************************************************/ 3519 3520 static const uint64_t pc_freemask[_NPCM] = { 3521 [0 ... _NPCM - 2] = PC_FREEN, 3522 [_NPCM - 1] = PC_FREEL 3523 }; 3524 3525 #ifdef PV_STATS 3526 static int pc_chunk_count, pc_chunk_allocs, pc_chunk_frees, pc_chunk_tryfail; 3527 3528 SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_count, CTLFLAG_RD, &pc_chunk_count, 0, 3529 "Current number of pv entry chunks"); 3530 SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_allocs, CTLFLAG_RD, &pc_chunk_allocs, 0, 3531 "Current number of pv entry chunks allocated"); 3532 SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_frees, CTLFLAG_RD, &pc_chunk_frees, 0, 3533 "Current number of pv entry chunks frees"); 3534 SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_tryfail, CTLFLAG_RD, &pc_chunk_tryfail, 0, 3535 "Number of times tried to get a chunk page but failed."); 3536 3537 static long pv_entry_frees, pv_entry_allocs, pv_entry_count; 3538 static int pv_entry_spare; 3539 3540 SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_frees, CTLFLAG_RD, &pv_entry_frees, 0, 3541 "Current number of pv entry frees"); 3542 SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_allocs, CTLFLAG_RD, &pv_entry_allocs, 0, 3543 "Current number of pv entry allocs"); 3544 SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_count, CTLFLAG_RD, &pv_entry_count, 0, 3545 "Current number of pv entries"); 3546 SYSCTL_INT(_vm_pmap, OID_AUTO, pv_entry_spare, CTLFLAG_RD, &pv_entry_spare, 0, 3547 "Current number of spare pv entries"); 3548 #endif 3549 3550 /* 3551 * We are in a serious low memory condition. Resort to 3552 * drastic measures to free some pages so we can allocate 3553 * another pv entry chunk. 3554 * 3555 * Returns NULL if PV entries were reclaimed from the specified pmap. 3556 * 3557 * We do not, however, unmap 2mpages because subsequent accesses will 3558 * allocate per-page pv entries until repromotion occurs, thereby 3559 * exacerbating the shortage of free pv entries. 3560 */ 3561 static vm_page_t 3562 reclaim_pv_chunk_domain(pmap_t locked_pmap, struct rwlock **lockp, int domain) 3563 { 3564 struct pv_chunks_list *pvc; 3565 struct pv_chunk *pc, *pc_marker, *pc_marker_end; 3566 struct pv_chunk_header pc_marker_b, pc_marker_end_b; 3567 pd_entry_t *pde; 3568 pmap_t next_pmap, pmap; 3569 pt_entry_t *pte, tpte; 3570 pv_entry_t pv; 3571 vm_offset_t va; 3572 vm_page_t m, m_pc; 3573 struct spglist free; 3574 uint64_t inuse; 3575 int bit, field, freed, lvl; 3576 3577 PMAP_LOCK_ASSERT(locked_pmap, MA_OWNED); 3578 KASSERT(lockp != NULL, ("reclaim_pv_chunk: lockp is NULL")); 3579 3580 pmap = NULL; 3581 m_pc = NULL; 3582 SLIST_INIT(&free); 3583 bzero(&pc_marker_b, sizeof(pc_marker_b)); 3584 bzero(&pc_marker_end_b, sizeof(pc_marker_end_b)); 3585 pc_marker = (struct pv_chunk *)&pc_marker_b; 3586 pc_marker_end = (struct pv_chunk *)&pc_marker_end_b; 3587 3588 pvc = &pv_chunks[domain]; 3589 mtx_lock(&pvc->pvc_lock); 3590 pvc->active_reclaims++; 3591 TAILQ_INSERT_HEAD(&pvc->pvc_list, pc_marker, pc_lru); 3592 TAILQ_INSERT_TAIL(&pvc->pvc_list, pc_marker_end, pc_lru); 3593 while ((pc = TAILQ_NEXT(pc_marker, pc_lru)) != pc_marker_end && 3594 SLIST_EMPTY(&free)) { 3595 next_pmap = pc->pc_pmap; 3596 if (next_pmap == NULL) { 3597 /* 3598 * The next chunk is a marker. However, it is 3599 * not our marker, so active_reclaims must be 3600 * > 1. Consequently, the next_chunk code 3601 * will not rotate the pv_chunks list. 3602 */ 3603 goto next_chunk; 3604 } 3605 mtx_unlock(&pvc->pvc_lock); 3606 3607 /* 3608 * A pv_chunk can only be removed from the pc_lru list 3609 * when both pvc->pvc_lock is owned and the 3610 * corresponding pmap is locked. 3611 */ 3612 if (pmap != next_pmap) { 3613 if (pmap != NULL && pmap != locked_pmap) 3614 PMAP_UNLOCK(pmap); 3615 pmap = next_pmap; 3616 /* Avoid deadlock and lock recursion. */ 3617 if (pmap > locked_pmap) { 3618 RELEASE_PV_LIST_LOCK(lockp); 3619 PMAP_LOCK(pmap); 3620 mtx_lock(&pvc->pvc_lock); 3621 continue; 3622 } else if (pmap != locked_pmap) { 3623 if (PMAP_TRYLOCK(pmap)) { 3624 mtx_lock(&pvc->pvc_lock); 3625 continue; 3626 } else { 3627 pmap = NULL; /* pmap is not locked */ 3628 mtx_lock(&pvc->pvc_lock); 3629 pc = TAILQ_NEXT(pc_marker, pc_lru); 3630 if (pc == NULL || 3631 pc->pc_pmap != next_pmap) 3632 continue; 3633 goto next_chunk; 3634 } 3635 } 3636 } 3637 3638 /* 3639 * Destroy every non-wired, 4 KB page mapping in the chunk. 3640 */ 3641 freed = 0; 3642 for (field = 0; field < _NPCM; field++) { 3643 for (inuse = ~pc->pc_map[field] & pc_freemask[field]; 3644 inuse != 0; inuse &= ~(1UL << bit)) { 3645 bit = ffsl(inuse) - 1; 3646 pv = &pc->pc_pventry[field * 64 + bit]; 3647 va = pv->pv_va; 3648 pde = pmap_pde(pmap, va, &lvl); 3649 if (lvl != 2) 3650 continue; 3651 pte = pmap_l2_to_l3(pde, va); 3652 tpte = pmap_load(pte); 3653 if ((tpte & ATTR_SW_WIRED) != 0) 3654 continue; 3655 if ((tpte & ATTR_CONTIGUOUS) != 0) 3656 (void)pmap_demote_l3c(pmap, pte, va); 3657 tpte = pmap_load_clear(pte); 3658 m = PTE_TO_VM_PAGE(tpte); 3659 if (pmap_pte_dirty(pmap, tpte)) 3660 vm_page_dirty(m); 3661 if ((tpte & ATTR_AF) != 0) { 3662 pmap_s1_invalidate_page(pmap, va, true); 3663 vm_page_aflag_set(m, PGA_REFERENCED); 3664 } 3665 CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m); 3666 TAILQ_REMOVE(&m->md.pv_list, pv, pv_next); 3667 m->md.pv_gen++; 3668 if (!pmap_page_is_mapped_locked(m)) 3669 vm_page_aflag_clear(m, PGA_WRITEABLE); 3670 pc->pc_map[field] |= 1UL << bit; 3671 pmap_unuse_pt(pmap, va, pmap_load(pde), &free); 3672 freed++; 3673 } 3674 } 3675 if (freed == 0) { 3676 mtx_lock(&pvc->pvc_lock); 3677 goto next_chunk; 3678 } 3679 /* Every freed mapping is for a 4 KB page. */ 3680 pmap_resident_count_dec(pmap, freed); 3681 PV_STAT(atomic_add_long(&pv_entry_frees, freed)); 3682 PV_STAT(atomic_add_int(&pv_entry_spare, freed)); 3683 PV_STAT(atomic_subtract_long(&pv_entry_count, freed)); 3684 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 3685 if (pc_is_free(pc)) { 3686 PV_STAT(atomic_subtract_int(&pv_entry_spare, _NPCPV)); 3687 PV_STAT(atomic_subtract_int(&pc_chunk_count, 1)); 3688 PV_STAT(atomic_add_int(&pc_chunk_frees, 1)); 3689 /* Entire chunk is free; return it. */ 3690 m_pc = DMAP_TO_VM_PAGE(pc); 3691 dump_drop_page(m_pc->phys_addr); 3692 mtx_lock(&pvc->pvc_lock); 3693 TAILQ_REMOVE(&pvc->pvc_list, pc, pc_lru); 3694 break; 3695 } 3696 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list); 3697 mtx_lock(&pvc->pvc_lock); 3698 /* One freed pv entry in locked_pmap is sufficient. */ 3699 if (pmap == locked_pmap) 3700 break; 3701 3702 next_chunk: 3703 TAILQ_REMOVE(&pvc->pvc_list, pc_marker, pc_lru); 3704 TAILQ_INSERT_AFTER(&pvc->pvc_list, pc, pc_marker, pc_lru); 3705 if (pvc->active_reclaims == 1 && pmap != NULL) { 3706 /* 3707 * Rotate the pv chunks list so that we do not 3708 * scan the same pv chunks that could not be 3709 * freed (because they contained a wired 3710 * and/or superpage mapping) on every 3711 * invocation of reclaim_pv_chunk(). 3712 */ 3713 while ((pc = TAILQ_FIRST(&pvc->pvc_list)) != pc_marker){ 3714 MPASS(pc->pc_pmap != NULL); 3715 TAILQ_REMOVE(&pvc->pvc_list, pc, pc_lru); 3716 TAILQ_INSERT_TAIL(&pvc->pvc_list, pc, pc_lru); 3717 } 3718 } 3719 } 3720 TAILQ_REMOVE(&pvc->pvc_list, pc_marker, pc_lru); 3721 TAILQ_REMOVE(&pvc->pvc_list, pc_marker_end, pc_lru); 3722 pvc->active_reclaims--; 3723 mtx_unlock(&pvc->pvc_lock); 3724 if (pmap != NULL && pmap != locked_pmap) 3725 PMAP_UNLOCK(pmap); 3726 if (m_pc == NULL && !SLIST_EMPTY(&free)) { 3727 m_pc = SLIST_FIRST(&free); 3728 SLIST_REMOVE_HEAD(&free, plinks.s.ss); 3729 /* Recycle a freed page table page. */ 3730 m_pc->ref_count = 1; 3731 } 3732 vm_page_free_pages_toq(&free, true); 3733 return (m_pc); 3734 } 3735 3736 static vm_page_t 3737 reclaim_pv_chunk(pmap_t locked_pmap, struct rwlock **lockp) 3738 { 3739 vm_page_t m; 3740 int i, domain; 3741 3742 domain = PCPU_GET(domain); 3743 for (i = 0; i < vm_ndomains; i++) { 3744 m = reclaim_pv_chunk_domain(locked_pmap, lockp, domain); 3745 if (m != NULL) 3746 break; 3747 domain = (domain + 1) % vm_ndomains; 3748 } 3749 3750 return (m); 3751 } 3752 3753 /* 3754 * free the pv_entry back to the free list 3755 */ 3756 static void 3757 free_pv_entry(pmap_t pmap, pv_entry_t pv) 3758 { 3759 struct pv_chunk *pc; 3760 int idx, field, bit; 3761 3762 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 3763 PV_STAT(atomic_add_long(&pv_entry_frees, 1)); 3764 PV_STAT(atomic_add_int(&pv_entry_spare, 1)); 3765 PV_STAT(atomic_subtract_long(&pv_entry_count, 1)); 3766 pc = pv_to_chunk(pv); 3767 idx = pv - &pc->pc_pventry[0]; 3768 field = idx / 64; 3769 bit = idx % 64; 3770 pc->pc_map[field] |= 1ul << bit; 3771 if (!pc_is_free(pc)) { 3772 /* 98% of the time, pc is already at the head of the list. */ 3773 if (__predict_false(pc != TAILQ_FIRST(&pmap->pm_pvchunk))) { 3774 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 3775 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list); 3776 } 3777 return; 3778 } 3779 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 3780 free_pv_chunk(pc); 3781 } 3782 3783 static void 3784 free_pv_chunk_dequeued(struct pv_chunk *pc) 3785 { 3786 vm_page_t m; 3787 3788 PV_STAT(atomic_subtract_int(&pv_entry_spare, _NPCPV)); 3789 PV_STAT(atomic_subtract_int(&pc_chunk_count, 1)); 3790 PV_STAT(atomic_add_int(&pc_chunk_frees, 1)); 3791 /* entire chunk is free, return it */ 3792 m = DMAP_TO_VM_PAGE(pc); 3793 dump_drop_page(m->phys_addr); 3794 vm_page_unwire_noq(m); 3795 vm_page_free(m); 3796 } 3797 3798 static void 3799 free_pv_chunk(struct pv_chunk *pc) 3800 { 3801 struct pv_chunks_list *pvc; 3802 3803 pvc = &pv_chunks[pc_to_domain(pc)]; 3804 mtx_lock(&pvc->pvc_lock); 3805 TAILQ_REMOVE(&pvc->pvc_list, pc, pc_lru); 3806 mtx_unlock(&pvc->pvc_lock); 3807 free_pv_chunk_dequeued(pc); 3808 } 3809 3810 static void 3811 free_pv_chunk_batch(struct pv_chunklist *batch) 3812 { 3813 struct pv_chunks_list *pvc; 3814 struct pv_chunk *pc, *npc; 3815 int i; 3816 3817 for (i = 0; i < vm_ndomains; i++) { 3818 if (TAILQ_EMPTY(&batch[i])) 3819 continue; 3820 pvc = &pv_chunks[i]; 3821 mtx_lock(&pvc->pvc_lock); 3822 TAILQ_FOREACH(pc, &batch[i], pc_list) { 3823 TAILQ_REMOVE(&pvc->pvc_list, pc, pc_lru); 3824 } 3825 mtx_unlock(&pvc->pvc_lock); 3826 } 3827 3828 for (i = 0; i < vm_ndomains; i++) { 3829 TAILQ_FOREACH_SAFE(pc, &batch[i], pc_list, npc) { 3830 free_pv_chunk_dequeued(pc); 3831 } 3832 } 3833 } 3834 3835 /* 3836 * Returns a new PV entry, allocating a new PV chunk from the system when 3837 * needed. If this PV chunk allocation fails and a PV list lock pointer was 3838 * given, a PV chunk is reclaimed from an arbitrary pmap. Otherwise, NULL is 3839 * returned. 3840 * 3841 * The given PV list lock may be released. 3842 */ 3843 static pv_entry_t 3844 get_pv_entry(pmap_t pmap, struct rwlock **lockp) 3845 { 3846 struct pv_chunks_list *pvc; 3847 int bit, field; 3848 pv_entry_t pv; 3849 struct pv_chunk *pc; 3850 vm_page_t m; 3851 3852 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 3853 PV_STAT(atomic_add_long(&pv_entry_allocs, 1)); 3854 retry: 3855 pc = TAILQ_FIRST(&pmap->pm_pvchunk); 3856 if (pc != NULL) { 3857 for (field = 0; field < _NPCM; field++) { 3858 if (pc->pc_map[field]) { 3859 bit = ffsl(pc->pc_map[field]) - 1; 3860 break; 3861 } 3862 } 3863 if (field < _NPCM) { 3864 pv = &pc->pc_pventry[field * 64 + bit]; 3865 pc->pc_map[field] &= ~(1ul << bit); 3866 /* If this was the last item, move it to tail */ 3867 if (pc_is_full(pc)) { 3868 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 3869 TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, 3870 pc_list); 3871 } 3872 PV_STAT(atomic_add_long(&pv_entry_count, 1)); 3873 PV_STAT(atomic_subtract_int(&pv_entry_spare, 1)); 3874 return (pv); 3875 } 3876 } 3877 /* No free items, allocate another chunk */ 3878 m = vm_page_alloc_noobj(VM_ALLOC_WIRED); 3879 if (m == NULL) { 3880 if (lockp == NULL) { 3881 PV_STAT(pc_chunk_tryfail++); 3882 return (NULL); 3883 } 3884 m = reclaim_pv_chunk(pmap, lockp); 3885 if (m == NULL) 3886 goto retry; 3887 } 3888 PV_STAT(atomic_add_int(&pc_chunk_count, 1)); 3889 PV_STAT(atomic_add_int(&pc_chunk_allocs, 1)); 3890 dump_add_page(m->phys_addr); 3891 pc = PHYS_TO_DMAP(m->phys_addr); 3892 pc->pc_pmap = pmap; 3893 memcpy(pc->pc_map, pc_freemask, sizeof(pc_freemask)); 3894 pc->pc_map[0] &= ~1ul; /* preallocated bit 0 */ 3895 pvc = &pv_chunks[vm_page_domain(m)]; 3896 mtx_lock(&pvc->pvc_lock); 3897 TAILQ_INSERT_TAIL(&pvc->pvc_list, pc, pc_lru); 3898 mtx_unlock(&pvc->pvc_lock); 3899 pv = &pc->pc_pventry[0]; 3900 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list); 3901 PV_STAT(atomic_add_long(&pv_entry_count, 1)); 3902 PV_STAT(atomic_add_int(&pv_entry_spare, _NPCPV - 1)); 3903 return (pv); 3904 } 3905 3906 /* 3907 * Ensure that the number of spare PV entries in the specified pmap meets or 3908 * exceeds the given count, "needed". 3909 * 3910 * The given PV list lock may be released. 3911 */ 3912 static void 3913 reserve_pv_entries(pmap_t pmap, int needed, struct rwlock **lockp) 3914 { 3915 struct pv_chunks_list *pvc; 3916 struct pch new_tail[PMAP_MEMDOM]; 3917 struct pv_chunk *pc; 3918 vm_page_t m; 3919 int avail, free, i; 3920 bool reclaimed; 3921 3922 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 3923 KASSERT(lockp != NULL, ("reserve_pv_entries: lockp is NULL")); 3924 3925 /* 3926 * Newly allocated PV chunks must be stored in a private list until 3927 * the required number of PV chunks have been allocated. Otherwise, 3928 * reclaim_pv_chunk() could recycle one of these chunks. In 3929 * contrast, these chunks must be added to the pmap upon allocation. 3930 */ 3931 for (i = 0; i < PMAP_MEMDOM; i++) 3932 TAILQ_INIT(&new_tail[i]); 3933 retry: 3934 avail = 0; 3935 TAILQ_FOREACH(pc, &pmap->pm_pvchunk, pc_list) { 3936 bit_count((bitstr_t *)pc->pc_map, 0, 3937 sizeof(pc->pc_map) * NBBY, &free); 3938 if (free == 0) 3939 break; 3940 avail += free; 3941 if (avail >= needed) 3942 break; 3943 } 3944 for (reclaimed = false; avail < needed; avail += _NPCPV) { 3945 m = vm_page_alloc_noobj(VM_ALLOC_WIRED); 3946 if (m == NULL) { 3947 m = reclaim_pv_chunk(pmap, lockp); 3948 if (m == NULL) 3949 goto retry; 3950 reclaimed = true; 3951 } 3952 PV_STAT(atomic_add_int(&pc_chunk_count, 1)); 3953 PV_STAT(atomic_add_int(&pc_chunk_allocs, 1)); 3954 dump_add_page(m->phys_addr); 3955 pc = PHYS_TO_DMAP(m->phys_addr); 3956 pc->pc_pmap = pmap; 3957 memcpy(pc->pc_map, pc_freemask, sizeof(pc_freemask)); 3958 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list); 3959 TAILQ_INSERT_TAIL(&new_tail[vm_page_domain(m)], pc, pc_lru); 3960 PV_STAT(atomic_add_int(&pv_entry_spare, _NPCPV)); 3961 3962 /* 3963 * The reclaim might have freed a chunk from the current pmap. 3964 * If that chunk contained available entries, we need to 3965 * re-count the number of available entries. 3966 */ 3967 if (reclaimed) 3968 goto retry; 3969 } 3970 for (i = 0; i < vm_ndomains; i++) { 3971 if (TAILQ_EMPTY(&new_tail[i])) 3972 continue; 3973 pvc = &pv_chunks[i]; 3974 mtx_lock(&pvc->pvc_lock); 3975 TAILQ_CONCAT(&pvc->pvc_list, &new_tail[i], pc_lru); 3976 mtx_unlock(&pvc->pvc_lock); 3977 } 3978 } 3979 3980 /* 3981 * First find and then remove the pv entry for the specified pmap and virtual 3982 * address from the specified pv list. Returns the pv entry if found and NULL 3983 * otherwise. This operation can be performed on pv lists for either 4KB or 3984 * 2MB page mappings. 3985 */ 3986 static __inline pv_entry_t 3987 pmap_pvh_remove(struct md_page *pvh, pmap_t pmap, vm_offset_t va) 3988 { 3989 pv_entry_t pv; 3990 3991 TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) { 3992 if (pmap == PV_PMAP(pv) && va == pv->pv_va) { 3993 TAILQ_REMOVE(&pvh->pv_list, pv, pv_next); 3994 pvh->pv_gen++; 3995 break; 3996 } 3997 } 3998 return (pv); 3999 } 4000 4001 /* 4002 * After demotion from a 2MB page mapping to 512 4KB page mappings, 4003 * destroy the pv entry for the 2MB page mapping and reinstantiate the pv 4004 * entries for each of the 4KB page mappings. 4005 */ 4006 static void 4007 pmap_pv_demote_l2(pmap_t pmap, vm_offset_t va, vm_paddr_t pa, 4008 struct rwlock **lockp) 4009 { 4010 struct md_page *pvh; 4011 struct pv_chunk *pc; 4012 pv_entry_t pv; 4013 vm_offset_t va_last; 4014 vm_page_t m; 4015 int bit, field; 4016 4017 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 4018 KASSERT((va & L2_OFFSET) == 0, 4019 ("pmap_pv_demote_l2: va is not 2mpage aligned")); 4020 KASSERT((pa & L2_OFFSET) == 0, 4021 ("pmap_pv_demote_l2: pa is not 2mpage aligned")); 4022 CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa); 4023 4024 /* 4025 * Transfer the 2mpage's pv entry for this mapping to the first 4026 * page's pv list. Once this transfer begins, the pv list lock 4027 * must not be released until the last pv entry is reinstantiated. 4028 */ 4029 pvh = pa_to_pvh(pa); 4030 pv = pmap_pvh_remove(pvh, pmap, va); 4031 KASSERT(pv != NULL, ("pmap_pv_demote_l2: pv not found")); 4032 m = PHYS_TO_VM_PAGE(pa); 4033 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next); 4034 m->md.pv_gen++; 4035 /* Instantiate the remaining Ln_ENTRIES - 1 pv entries. */ 4036 PV_STAT(atomic_add_long(&pv_entry_allocs, Ln_ENTRIES - 1)); 4037 va_last = va + L2_SIZE - PAGE_SIZE; 4038 for (;;) { 4039 pc = TAILQ_FIRST(&pmap->pm_pvchunk); 4040 KASSERT(!pc_is_full(pc), ("pmap_pv_demote_l2: missing spare")); 4041 for (field = 0; field < _NPCM; field++) { 4042 while (pc->pc_map[field]) { 4043 bit = ffsl(pc->pc_map[field]) - 1; 4044 pc->pc_map[field] &= ~(1ul << bit); 4045 pv = &pc->pc_pventry[field * 64 + bit]; 4046 va += PAGE_SIZE; 4047 pv->pv_va = va; 4048 m++; 4049 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 4050 ("pmap_pv_demote_l2: page %p is not managed", m)); 4051 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next); 4052 m->md.pv_gen++; 4053 if (va == va_last) 4054 goto out; 4055 } 4056 } 4057 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 4058 TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list); 4059 } 4060 out: 4061 if (pc_is_full(pc)) { 4062 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 4063 TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list); 4064 } 4065 PV_STAT(atomic_add_long(&pv_entry_count, Ln_ENTRIES - 1)); 4066 PV_STAT(atomic_subtract_int(&pv_entry_spare, Ln_ENTRIES - 1)); 4067 } 4068 4069 /* 4070 * First find and then destroy the pv entry for the specified pmap and virtual 4071 * address. This operation can be performed on pv lists for either 4KB or 2MB 4072 * page mappings. 4073 */ 4074 static void 4075 pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va) 4076 { 4077 pv_entry_t pv; 4078 4079 pv = pmap_pvh_remove(pvh, pmap, va); 4080 KASSERT(pv != NULL, ("pmap_pvh_free: pv not found")); 4081 free_pv_entry(pmap, pv); 4082 } 4083 4084 /* 4085 * Conditionally create the PV entry for a 4KB page mapping if the required 4086 * memory can be allocated without resorting to reclamation. 4087 */ 4088 static bool 4089 pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, vm_page_t m, 4090 struct rwlock **lockp) 4091 { 4092 pv_entry_t pv; 4093 4094 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 4095 /* Pass NULL instead of the lock pointer to disable reclamation. */ 4096 if ((pv = get_pv_entry(pmap, NULL)) != NULL) { 4097 pv->pv_va = va; 4098 CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m); 4099 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next); 4100 m->md.pv_gen++; 4101 return (true); 4102 } else 4103 return (false); 4104 } 4105 4106 /* 4107 * Create the PV entry for a 2MB page mapping. Always returns true unless the 4108 * flag PMAP_ENTER_NORECLAIM is specified. If that flag is specified, returns 4109 * false if the PV entry cannot be allocated without resorting to reclamation. 4110 */ 4111 static bool 4112 pmap_pv_insert_l2(pmap_t pmap, vm_offset_t va, pd_entry_t l2e, u_int flags, 4113 struct rwlock **lockp) 4114 { 4115 struct md_page *pvh; 4116 pv_entry_t pv; 4117 vm_paddr_t pa; 4118 4119 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 4120 /* Pass NULL instead of the lock pointer to disable reclamation. */ 4121 if ((pv = get_pv_entry(pmap, (flags & PMAP_ENTER_NORECLAIM) != 0 ? 4122 NULL : lockp)) == NULL) 4123 return (false); 4124 pv->pv_va = va; 4125 pa = PTE_TO_PHYS(l2e); 4126 CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa); 4127 pvh = pa_to_pvh(pa); 4128 TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_next); 4129 pvh->pv_gen++; 4130 return (true); 4131 } 4132 4133 /* 4134 * Conditionally creates the PV entries for a L3C superpage mapping if 4135 * the required memory can be allocated without resorting to reclamation. 4136 */ 4137 static bool 4138 pmap_pv_insert_l3c(pmap_t pmap, vm_offset_t va, vm_page_t m, 4139 struct rwlock **lockp) 4140 { 4141 pv_entry_t pv; 4142 vm_offset_t tva; 4143 vm_paddr_t pa __diagused; 4144 vm_page_t mt; 4145 4146 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 4147 KASSERT((va & L3C_OFFSET) == 0, 4148 ("pmap_pv_insert_l3c: va is not aligned")); 4149 pa = VM_PAGE_TO_PHYS(m); 4150 KASSERT((pa & L3C_OFFSET) == 0, 4151 ("pmap_pv_insert_l3c: pa is not aligned")); 4152 CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m); 4153 for (mt = m, tva = va; mt < &m[L3C_ENTRIES]; mt++, tva += L3_SIZE) { 4154 /* Pass NULL instead of lockp to disable reclamation. */ 4155 pv = get_pv_entry(pmap, NULL); 4156 if (__predict_false(pv == NULL)) { 4157 while (tva > va) { 4158 mt--; 4159 tva -= L3_SIZE; 4160 pmap_pvh_free(&mt->md, pmap, tva); 4161 } 4162 return (false); 4163 } 4164 pv->pv_va = tva; 4165 TAILQ_INSERT_TAIL(&mt->md.pv_list, pv, pv_next); 4166 mt->md.pv_gen++; 4167 } 4168 return (true); 4169 } 4170 4171 static void 4172 pmap_remove_kernel_l2(pmap_t pmap, pt_entry_t *l2, vm_offset_t va) 4173 { 4174 pt_entry_t newl2, oldl2 __diagused; 4175 vm_page_t ml3; 4176 vm_paddr_t ml3pa; 4177 4178 KASSERT(!VIRT_IN_DMAP(va), ("removing direct mapping of %#lx", va)); 4179 KASSERT(pmap == kernel_pmap, ("pmap %p is not kernel_pmap", pmap)); 4180 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 4181 4182 ml3 = pmap_remove_pt_page(pmap, va); 4183 KASSERT(ml3 != NULL, ("pmap_remove_kernel_l2: missing pt page")); 4184 4185 ml3pa = VM_PAGE_TO_PHYS(ml3); 4186 newl2 = PHYS_TO_PTE(ml3pa) | L2_TABLE; 4187 4188 /* 4189 * If this page table page was unmapped by a promotion, then it 4190 * contains valid mappings. Zero it to invalidate those mappings. 4191 */ 4192 if (vm_page_any_valid(ml3)) 4193 pagezero(PHYS_TO_DMAP(ml3pa)); 4194 4195 /* 4196 * Demote the mapping. The caller must have already invalidated the 4197 * mapping (i.e., the "break" in break-before-make). 4198 */ 4199 oldl2 = pmap_load_store(l2, newl2); 4200 KASSERT(oldl2 == 0, ("%s: found existing mapping at %p: %#lx", 4201 __func__, l2, oldl2)); 4202 } 4203 4204 /* 4205 * pmap_remove_l2: Do the things to unmap a level 2 superpage. 4206 */ 4207 static int 4208 pmap_remove_l2(pmap_t pmap, pt_entry_t *l2, vm_offset_t sva, pd_entry_t l1e, 4209 bool demote_kl2e, struct spglist *free, struct rwlock **lockp) 4210 { 4211 struct md_page *pvh; 4212 pt_entry_t old_l2; 4213 vm_page_t m, ml3, mt; 4214 4215 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 4216 KASSERT((sva & L2_OFFSET) == 0, ("pmap_remove_l2: sva is not aligned")); 4217 old_l2 = pmap_load_clear(l2); 4218 KASSERT((old_l2 & ATTR_DESCR_MASK) == L2_BLOCK, 4219 ("pmap_remove_l2: L2e %lx is not a block mapping", old_l2)); 4220 4221 /* 4222 * Since a promotion must break the 4KB page mappings before making 4223 * the 2MB page mapping, a pmap_s1_invalidate_page() suffices. 4224 */ 4225 pmap_s1_invalidate_page(pmap, sva, true); 4226 4227 if (old_l2 & ATTR_SW_WIRED) 4228 pmap->pm_stats.wired_count -= L2_SIZE / PAGE_SIZE; 4229 pmap_resident_count_dec(pmap, L2_SIZE / PAGE_SIZE); 4230 if (old_l2 & ATTR_SW_MANAGED) { 4231 m = PTE_TO_VM_PAGE(old_l2); 4232 pvh = page_to_pvh(m); 4233 CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m); 4234 pmap_pvh_free(pvh, pmap, sva); 4235 for (mt = m; mt < &m[L2_SIZE / PAGE_SIZE]; mt++) { 4236 if (pmap_pte_dirty(pmap, old_l2)) 4237 vm_page_dirty(mt); 4238 if (old_l2 & ATTR_AF) 4239 vm_page_aflag_set(mt, PGA_REFERENCED); 4240 if (TAILQ_EMPTY(&mt->md.pv_list) && 4241 TAILQ_EMPTY(&pvh->pv_list)) 4242 vm_page_aflag_clear(mt, PGA_WRITEABLE); 4243 } 4244 } 4245 if (pmap != kernel_pmap) { 4246 ml3 = pmap_remove_pt_page(pmap, sva); 4247 if (ml3 != NULL) { 4248 KASSERT(vm_page_any_valid(ml3), 4249 ("pmap_remove_l2: l3 page not promoted")); 4250 pmap_resident_count_dec(pmap, 1); 4251 KASSERT(ml3->ref_count == NL3PG, 4252 ("pmap_remove_l2: l3 page ref count error")); 4253 ml3->ref_count = 0; 4254 pmap_add_delayed_free_list(ml3, free, false); 4255 } 4256 } else if (demote_kl2e) { 4257 pmap_remove_kernel_l2(pmap, l2, sva); 4258 } else { 4259 ml3 = vm_radix_lookup(&pmap->pm_root, pmap_l2_pindex(sva)); 4260 if (vm_page_any_valid(ml3)) { 4261 ml3->valid = 0; 4262 pmap_zero_page(ml3); 4263 } 4264 } 4265 return (pmap_unuse_pt(pmap, sva, l1e, free)); 4266 } 4267 4268 /* 4269 * pmap_remove_l3: do the things to unmap a page in a process 4270 */ 4271 static int 4272 pmap_remove_l3(pmap_t pmap, pt_entry_t *l3, vm_offset_t va, 4273 pd_entry_t l2e, struct spglist *free, struct rwlock **lockp) 4274 { 4275 pt_entry_t old_l3; 4276 vm_page_t m; 4277 4278 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 4279 old_l3 = pmap_load(l3); 4280 if ((old_l3 & ATTR_CONTIGUOUS) != 0) 4281 (void)pmap_demote_l3c(pmap, l3, va); 4282 old_l3 = pmap_load_clear(l3); 4283 pmap_s1_invalidate_page(pmap, va, true); 4284 if (old_l3 & ATTR_SW_WIRED) 4285 pmap->pm_stats.wired_count -= 1; 4286 pmap_resident_count_dec(pmap, 1); 4287 if (old_l3 & ATTR_SW_MANAGED) { 4288 m = PTE_TO_VM_PAGE(old_l3); 4289 if (pmap_pte_dirty(pmap, old_l3)) 4290 vm_page_dirty(m); 4291 if (old_l3 & ATTR_AF) 4292 vm_page_aflag_set(m, PGA_REFERENCED); 4293 CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m); 4294 pmap_pvh_free(&m->md, pmap, va); 4295 if (!pmap_page_is_mapped_locked(m)) 4296 vm_page_aflag_clear(m, PGA_WRITEABLE); 4297 } 4298 return (pmap_unuse_pt(pmap, va, l2e, free)); 4299 } 4300 4301 /* 4302 * Removes the specified L3C superpage mapping. Requests TLB invalidations 4303 * to be performed by the caller through the returned "*vap". Returns true 4304 * if the level 3 table "ml3" was unmapped and added to the spglist "free". 4305 * Otherwise, returns false. 4306 */ 4307 static bool 4308 pmap_remove_l3c(pmap_t pmap, pt_entry_t *l3p, vm_offset_t va, vm_offset_t *vap, 4309 vm_offset_t va_next, vm_page_t ml3, struct spglist *free, 4310 struct rwlock **lockp) 4311 { 4312 struct md_page *pvh; 4313 struct rwlock *new_lock; 4314 pt_entry_t first_l3e, l3e, *tl3p; 4315 vm_offset_t tva; 4316 vm_page_t m, mt; 4317 4318 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 4319 KASSERT(((uintptr_t)l3p & ((L3C_ENTRIES * sizeof(pt_entry_t)) - 1)) == 4320 0, ("pmap_remove_l3c: l3p is not aligned")); 4321 KASSERT((va & L3C_OFFSET) == 0, 4322 ("pmap_remove_l3c: va is not aligned")); 4323 4324 /* 4325 * Hardware accessed and dirty bit maintenance might only update a 4326 * single L3 entry, so we must combine the accessed and dirty bits 4327 * from this entire set of contiguous L3 entries. 4328 */ 4329 first_l3e = pmap_load_clear(l3p); 4330 for (tl3p = l3p + 1; tl3p < &l3p[L3C_ENTRIES]; tl3p++) { 4331 l3e = pmap_load_clear(tl3p); 4332 KASSERT((l3e & ATTR_CONTIGUOUS) != 0, 4333 ("pmap_remove_l3c: l3e is missing ATTR_CONTIGUOUS")); 4334 if ((l3e & (ATTR_SW_DBM | ATTR_S1_AP_RW_BIT)) == 4335 (ATTR_SW_DBM | ATTR_S1_AP(ATTR_S1_AP_RW))) 4336 first_l3e &= ~ATTR_S1_AP_RW_BIT; 4337 first_l3e |= l3e & ATTR_AF; 4338 } 4339 if ((first_l3e & ATTR_SW_WIRED) != 0) 4340 pmap->pm_stats.wired_count -= L3C_ENTRIES; 4341 pmap_resident_count_dec(pmap, L3C_ENTRIES); 4342 if ((first_l3e & ATTR_SW_MANAGED) != 0) { 4343 m = PTE_TO_VM_PAGE(first_l3e); 4344 new_lock = VM_PAGE_TO_PV_LIST_LOCK(m); 4345 if (new_lock != *lockp) { 4346 if (*lockp != NULL) { 4347 /* 4348 * Pending TLB invalidations must be 4349 * performed before the PV list lock is 4350 * released. Otherwise, a concurrent 4351 * pmap_remove_all() on a physical page 4352 * could return while a stale TLB entry 4353 * still provides access to that page. 4354 */ 4355 if (*vap != va_next) { 4356 pmap_invalidate_range(pmap, *vap, va, 4357 true); 4358 *vap = va_next; 4359 } 4360 rw_wunlock(*lockp); 4361 } 4362 *lockp = new_lock; 4363 rw_wlock(*lockp); 4364 } 4365 pvh = page_to_pvh(m); 4366 for (mt = m, tva = va; mt < &m[L3C_ENTRIES]; mt++, tva += 4367 L3_SIZE) { 4368 if (pmap_pte_dirty(pmap, first_l3e)) 4369 vm_page_dirty(mt); 4370 if ((first_l3e & ATTR_AF) != 0) 4371 vm_page_aflag_set(mt, PGA_REFERENCED); 4372 pmap_pvh_free(&mt->md, pmap, tva); 4373 if (TAILQ_EMPTY(&mt->md.pv_list) && 4374 TAILQ_EMPTY(&pvh->pv_list)) 4375 vm_page_aflag_clear(mt, PGA_WRITEABLE); 4376 } 4377 } 4378 if (*vap == va_next) 4379 *vap = va; 4380 if (ml3 != NULL) { 4381 ml3->ref_count -= L3C_ENTRIES; 4382 if (ml3->ref_count == 0) { 4383 _pmap_unwire_l3(pmap, va, ml3, free); 4384 return (true); 4385 } 4386 } 4387 return (false); 4388 } 4389 4390 /* 4391 * Remove the specified range of addresses from the L3 page table that is 4392 * identified by the given L2 entry. 4393 */ 4394 static void 4395 pmap_remove_l3_range(pmap_t pmap, pd_entry_t l2e, vm_offset_t sva, 4396 vm_offset_t eva, struct spglist *free, struct rwlock **lockp) 4397 { 4398 struct rwlock *new_lock; 4399 pt_entry_t *l3, old_l3; 4400 vm_offset_t va; 4401 vm_page_t l3pg, m; 4402 4403 KASSERT(ADDR_IS_CANONICAL(sva), 4404 ("%s: Start address not in canonical form: %lx", __func__, sva)); 4405 KASSERT(ADDR_IS_CANONICAL(eva) || eva == VM_MAX_USER_ADDRESS, 4406 ("%s: End address not in canonical form: %lx", __func__, eva)); 4407 4408 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 4409 KASSERT(rounddown2(sva, L2_SIZE) + L2_SIZE == roundup2(eva, L2_SIZE), 4410 ("pmap_remove_l3_range: range crosses an L3 page table boundary")); 4411 l3pg = ADDR_IS_USER(sva) ? PTE_TO_VM_PAGE(l2e) : NULL; 4412 va = eva; 4413 for (l3 = pmap_l2_to_l3(&l2e, sva); sva != eva; l3++, sva += L3_SIZE) { 4414 old_l3 = pmap_load(l3); 4415 if (!pmap_l3_valid(old_l3)) { 4416 if (va != eva) { 4417 pmap_invalidate_range(pmap, va, sva, true); 4418 va = eva; 4419 } 4420 continue; 4421 } 4422 if ((old_l3 & ATTR_CONTIGUOUS) != 0) { 4423 /* 4424 * Is this entire set of contiguous L3 entries being 4425 * removed? Handle the possibility that "eva" is zero 4426 * because of address wraparound. 4427 */ 4428 if ((sva & L3C_OFFSET) == 0 && 4429 sva + L3C_OFFSET <= eva - 1) { 4430 if (pmap_remove_l3c(pmap, l3, sva, &va, eva, 4431 l3pg, free, lockp)) { 4432 /* The L3 table was unmapped. */ 4433 sva += L3C_SIZE; 4434 break; 4435 } 4436 l3 += L3C_ENTRIES - 1; 4437 sva += L3C_SIZE - L3_SIZE; 4438 continue; 4439 } 4440 4441 (void)pmap_demote_l3c(pmap, l3, sva); 4442 } 4443 old_l3 = pmap_load_clear(l3); 4444 if ((old_l3 & ATTR_SW_WIRED) != 0) 4445 pmap->pm_stats.wired_count--; 4446 pmap_resident_count_dec(pmap, 1); 4447 /* Below will only be true in a realm environment. */ 4448 if (PTE_TO_PHYS(old_l3) & prot_ns_shared_pa) 4449 pmap_set_protected(old_l3); 4450 if ((old_l3 & ATTR_SW_MANAGED) != 0) { 4451 m = PTE_TO_VM_PAGE(old_l3); 4452 if (pmap_pte_dirty(pmap, old_l3)) 4453 vm_page_dirty(m); 4454 if ((old_l3 & ATTR_AF) != 0) 4455 vm_page_aflag_set(m, PGA_REFERENCED); 4456 new_lock = VM_PAGE_TO_PV_LIST_LOCK(m); 4457 if (new_lock != *lockp) { 4458 if (*lockp != NULL) { 4459 /* 4460 * Pending TLB invalidations must be 4461 * performed before the PV list lock is 4462 * released. Otherwise, a concurrent 4463 * pmap_remove_all() on a physical page 4464 * could return while a stale TLB entry 4465 * still provides access to that page. 4466 */ 4467 if (va != eva) { 4468 pmap_invalidate_range(pmap, va, 4469 sva, true); 4470 va = eva; 4471 } 4472 rw_wunlock(*lockp); 4473 } 4474 *lockp = new_lock; 4475 rw_wlock(*lockp); 4476 } 4477 pmap_pvh_free(&m->md, pmap, sva); 4478 if (!pmap_page_is_mapped_locked(m)) 4479 vm_page_aflag_clear(m, PGA_WRITEABLE); 4480 } 4481 if (l3pg != NULL && pmap_unwire_l3(pmap, sva, l3pg, free)) { 4482 /* 4483 * _pmap_unwire_l3() has already invalidated the TLB 4484 * entries at all levels for "sva". So, we need not 4485 * perform "sva += L3_SIZE;" here. Moreover, we need 4486 * not perform "va = sva;" if "sva" is at the start 4487 * of a new valid range consisting of a single page. 4488 */ 4489 break; 4490 } 4491 if (va == eva) 4492 va = sva; 4493 } 4494 if (va != eva) 4495 pmap_invalidate_range(pmap, va, sva, true); 4496 } 4497 4498 static void 4499 pmap_remove1(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, bool map_delete) 4500 { 4501 struct rwlock *lock; 4502 vm_offset_t va_next; 4503 pd_entry_t *l0, *l1, *l2; 4504 pt_entry_t l3_paddr; 4505 struct spglist free; 4506 4507 /* 4508 * Perform an unsynchronized read. This is, however, safe. 4509 */ 4510 if (pmap->pm_stats.resident_count == 0) 4511 return; 4512 4513 SLIST_INIT(&free); 4514 4515 PMAP_LOCK(pmap); 4516 if (map_delete) 4517 pmap_bti_on_remove(pmap, sva, eva); 4518 4519 lock = NULL; 4520 for (; sva < eva; sva = va_next) { 4521 if (pmap->pm_stats.resident_count == 0) 4522 break; 4523 4524 l0 = pmap_l0(pmap, sva); 4525 if (pmap_load(l0) == 0) { 4526 va_next = (sva + L0_SIZE) & ~L0_OFFSET; 4527 if (va_next < sva) 4528 va_next = eva; 4529 continue; 4530 } 4531 4532 va_next = (sva + L1_SIZE) & ~L1_OFFSET; 4533 if (va_next < sva) 4534 va_next = eva; 4535 l1 = pmap_l0_to_l1(l0, sva); 4536 if (pmap_load(l1) == 0) 4537 continue; 4538 if ((pmap_load(l1) & ATTR_DESCR_MASK) == L1_BLOCK) { 4539 PMAP_ASSERT_L1_BLOCKS_SUPPORTED; 4540 KASSERT(va_next <= eva, 4541 ("partial update of non-transparent 1G page " 4542 "l1 %#lx sva %#lx eva %#lx va_next %#lx", 4543 pmap_load(l1), sva, eva, va_next)); 4544 MPASS(pmap != kernel_pmap); 4545 MPASS((pmap_load(l1) & ATTR_SW_MANAGED) == 0); 4546 pmap_clear(l1); 4547 pmap_s1_invalidate_page(pmap, sva, true); 4548 pmap_resident_count_dec(pmap, L1_SIZE / PAGE_SIZE); 4549 pmap_unuse_pt(pmap, sva, pmap_load(l0), &free); 4550 continue; 4551 } 4552 4553 /* 4554 * Calculate index for next page table. 4555 */ 4556 va_next = (sva + L2_SIZE) & ~L2_OFFSET; 4557 if (va_next < sva) 4558 va_next = eva; 4559 4560 l2 = pmap_l1_to_l2(l1, sva); 4561 l3_paddr = pmap_load(l2); 4562 4563 if ((l3_paddr & ATTR_DESCR_MASK) == L2_BLOCK) { 4564 if (sva + L2_SIZE == va_next && eva >= va_next) { 4565 pmap_remove_l2(pmap, l2, sva, pmap_load(l1), 4566 true, &free, &lock); 4567 continue; 4568 } else if (pmap_demote_l2_locked(pmap, l2, sva, 4569 &lock) == NULL) 4570 continue; 4571 l3_paddr = pmap_load(l2); 4572 } 4573 4574 /* 4575 * Weed out invalid mappings. 4576 */ 4577 if ((l3_paddr & ATTR_DESCR_MASK) != L2_TABLE) 4578 continue; 4579 4580 /* 4581 * Limit our scan to either the end of the va represented 4582 * by the current page table page, or to the end of the 4583 * range being removed. 4584 */ 4585 if (va_next > eva) 4586 va_next = eva; 4587 4588 pmap_remove_l3_range(pmap, l3_paddr, sva, va_next, &free, 4589 &lock); 4590 } 4591 if (lock != NULL) 4592 rw_wunlock(lock); 4593 PMAP_UNLOCK(pmap); 4594 vm_page_free_pages_toq(&free, true); 4595 } 4596 4597 /* 4598 * Remove the given range of addresses from the specified map. 4599 * 4600 * It is assumed that the start and end are properly 4601 * rounded to the page size. 4602 */ 4603 void 4604 pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) 4605 { 4606 pmap_remove1(pmap, sva, eva, false); 4607 } 4608 4609 /* 4610 * Remove the given range of addresses as part of a logical unmap 4611 * operation. This has the effect of calling pmap_remove(), but 4612 * also clears any metadata that should persist for the lifetime 4613 * of a logical mapping. 4614 */ 4615 void 4616 pmap_map_delete(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) 4617 { 4618 pmap_remove1(pmap, sva, eva, true); 4619 } 4620 4621 /* 4622 * Routine: pmap_remove_all 4623 * Function: 4624 * Removes this physical page from 4625 * all physical maps in which it resides. 4626 * Reflects back modify bits to the pager. 4627 * 4628 * Notes: 4629 * Original versions of this routine were very 4630 * inefficient because they iteratively called 4631 * pmap_remove (slow...) 4632 */ 4633 4634 void 4635 pmap_remove_all(vm_page_t m) 4636 { 4637 struct md_page *pvh; 4638 pv_entry_t pv; 4639 pmap_t pmap; 4640 struct rwlock *lock; 4641 pd_entry_t *pde, tpde; 4642 pt_entry_t *pte, tpte; 4643 vm_offset_t va; 4644 struct spglist free; 4645 int lvl, pvh_gen, md_gen; 4646 4647 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 4648 ("pmap_remove_all: page %p is not managed", m)); 4649 SLIST_INIT(&free); 4650 lock = VM_PAGE_TO_PV_LIST_LOCK(m); 4651 pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy : page_to_pvh(m); 4652 rw_wlock(lock); 4653 retry: 4654 while ((pv = TAILQ_FIRST(&pvh->pv_list)) != NULL) { 4655 pmap = PV_PMAP(pv); 4656 if (!PMAP_TRYLOCK(pmap)) { 4657 pvh_gen = pvh->pv_gen; 4658 rw_wunlock(lock); 4659 PMAP_LOCK(pmap); 4660 rw_wlock(lock); 4661 if (pvh_gen != pvh->pv_gen) { 4662 PMAP_UNLOCK(pmap); 4663 goto retry; 4664 } 4665 } 4666 va = pv->pv_va; 4667 pte = pmap_pte_exists(pmap, va, 2, __func__); 4668 pmap_demote_l2_locked(pmap, pte, va, &lock); 4669 PMAP_UNLOCK(pmap); 4670 } 4671 while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) { 4672 pmap = PV_PMAP(pv); 4673 if (!PMAP_TRYLOCK(pmap)) { 4674 pvh_gen = pvh->pv_gen; 4675 md_gen = m->md.pv_gen; 4676 rw_wunlock(lock); 4677 PMAP_LOCK(pmap); 4678 rw_wlock(lock); 4679 if (pvh_gen != pvh->pv_gen || md_gen != m->md.pv_gen) { 4680 PMAP_UNLOCK(pmap); 4681 goto retry; 4682 } 4683 } 4684 pmap_resident_count_dec(pmap, 1); 4685 4686 pde = pmap_pde(pmap, pv->pv_va, &lvl); 4687 KASSERT(pde != NULL, 4688 ("pmap_remove_all: no page directory entry found")); 4689 KASSERT(lvl == 2, 4690 ("pmap_remove_all: invalid pde level %d", lvl)); 4691 tpde = pmap_load(pde); 4692 4693 pte = pmap_l2_to_l3(pde, pv->pv_va); 4694 tpte = pmap_load(pte); 4695 if ((tpte & ATTR_CONTIGUOUS) != 0) 4696 (void)pmap_demote_l3c(pmap, pte, pv->pv_va); 4697 tpte = pmap_load_clear(pte); 4698 if (tpte & ATTR_SW_WIRED) 4699 pmap->pm_stats.wired_count--; 4700 if ((tpte & ATTR_AF) != 0) { 4701 pmap_invalidate_page(pmap, pv->pv_va, true); 4702 vm_page_aflag_set(m, PGA_REFERENCED); 4703 } 4704 4705 /* 4706 * Update the vm_page_t clean and reference bits. 4707 */ 4708 if (pmap_pte_dirty(pmap, tpte)) 4709 vm_page_dirty(m); 4710 pmap_unuse_pt(pmap, pv->pv_va, tpde, &free); 4711 TAILQ_REMOVE(&m->md.pv_list, pv, pv_next); 4712 m->md.pv_gen++; 4713 free_pv_entry(pmap, pv); 4714 PMAP_UNLOCK(pmap); 4715 } 4716 vm_page_aflag_clear(m, PGA_WRITEABLE); 4717 rw_wunlock(lock); 4718 vm_page_free_pages_toq(&free, true); 4719 } 4720 4721 /* 4722 * Masks and sets bits in a level 2 page table entries in the specified pmap 4723 */ 4724 static void 4725 pmap_protect_l2(pmap_t pmap, pt_entry_t *l2, vm_offset_t sva, pt_entry_t mask, 4726 pt_entry_t nbits) 4727 { 4728 pd_entry_t old_l2; 4729 vm_page_t m, mt; 4730 4731 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 4732 PMAP_ASSERT_STAGE1(pmap); 4733 KASSERT((sva & L2_OFFSET) == 0, 4734 ("pmap_protect_l2: sva is not 2mpage aligned")); 4735 old_l2 = pmap_load(l2); 4736 KASSERT((old_l2 & ATTR_DESCR_MASK) == L2_BLOCK, 4737 ("pmap_protect_l2: L2e %lx is not a block mapping", old_l2)); 4738 4739 /* 4740 * Return if the L2 entry already has the desired access restrictions 4741 * in place. 4742 */ 4743 if ((old_l2 & mask) == nbits) 4744 return; 4745 4746 while (!atomic_fcmpset_64(l2, &old_l2, (old_l2 & ~mask) | nbits)) 4747 cpu_spinwait(); 4748 4749 /* 4750 * When a dirty read/write superpage mapping is write protected, 4751 * update the dirty field of each of the superpage's constituent 4KB 4752 * pages. 4753 */ 4754 if ((old_l2 & ATTR_SW_MANAGED) != 0 && 4755 (nbits & ATTR_S1_AP(ATTR_S1_AP_RO)) != 0 && 4756 pmap_pte_dirty(pmap, old_l2)) { 4757 m = PTE_TO_VM_PAGE(old_l2); 4758 for (mt = m; mt < &m[L2_SIZE / PAGE_SIZE]; mt++) 4759 vm_page_dirty(mt); 4760 } 4761 4762 /* 4763 * Since a promotion must break the 4KB page mappings before making 4764 * the 2MB page mapping, a pmap_s1_invalidate_page() suffices. 4765 */ 4766 pmap_s1_invalidate_page(pmap, sva, true); 4767 } 4768 4769 /* 4770 * Masks and sets bits in the specified L3C superpage mapping. 4771 * 4772 * Requests TLB invalidations to be performed by the caller through the 4773 * returned "*vap". 4774 */ 4775 static void 4776 pmap_mask_set_l3c(pmap_t pmap, pt_entry_t *l3p, vm_offset_t va, 4777 vm_offset_t *vap, vm_offset_t va_next, pt_entry_t mask, pt_entry_t nbits) 4778 { 4779 pt_entry_t l3e, *tl3p; 4780 vm_page_t m, mt; 4781 bool dirty; 4782 4783 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 4784 KASSERT(((uintptr_t)l3p & ((L3C_ENTRIES * sizeof(pt_entry_t)) - 1)) == 4785 0, ("pmap_mask_set_l3c: l3p is not aligned")); 4786 KASSERT((va & L3C_OFFSET) == 0, 4787 ("pmap_mask_set_l3c: va is not aligned")); 4788 dirty = false; 4789 for (tl3p = l3p; tl3p < &l3p[L3C_ENTRIES]; tl3p++) { 4790 l3e = pmap_load(tl3p); 4791 KASSERT((l3e & ATTR_CONTIGUOUS) != 0, 4792 ("pmap_mask_set_l3c: l3e is missing ATTR_CONTIGUOUS")); 4793 while (!atomic_fcmpset_64(tl3p, &l3e, (l3e & ~mask) | nbits)) 4794 cpu_spinwait(); 4795 if ((l3e & (ATTR_SW_DBM | ATTR_S1_AP_RW_BIT)) == 4796 (ATTR_SW_DBM | ATTR_S1_AP(ATTR_S1_AP_RW))) 4797 dirty = true; 4798 } 4799 4800 /* 4801 * When a dirty read/write superpage mapping is write protected, 4802 * update the dirty field of each of the superpage's constituent 4KB 4803 * pages. 4804 */ 4805 if ((l3e & ATTR_SW_MANAGED) != 0 && 4806 (nbits & ATTR_S1_AP(ATTR_S1_AP_RO)) != 0 && 4807 dirty) { 4808 m = PTE_TO_VM_PAGE(pmap_load(l3p)); 4809 for (mt = m; mt < &m[L3C_ENTRIES]; mt++) 4810 vm_page_dirty(mt); 4811 } 4812 4813 if (*vap == va_next) 4814 *vap = va; 4815 } 4816 4817 /* 4818 * Masks and sets bits in last level page table entries in the specified 4819 * pmap and range 4820 */ 4821 static void 4822 pmap_mask_set_locked(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, pt_entry_t mask, 4823 pt_entry_t nbits, bool invalidate) 4824 { 4825 vm_offset_t va, va_next; 4826 pd_entry_t *l0, *l1, *l2; 4827 pt_entry_t *l3p, l3; 4828 4829 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 4830 for (; sva < eva; sva = va_next) { 4831 l0 = pmap_l0(pmap, sva); 4832 if (pmap_load(l0) == 0) { 4833 va_next = (sva + L0_SIZE) & ~L0_OFFSET; 4834 if (va_next < sva) 4835 va_next = eva; 4836 continue; 4837 } 4838 4839 va_next = (sva + L1_SIZE) & ~L1_OFFSET; 4840 if (va_next < sva) 4841 va_next = eva; 4842 l1 = pmap_l0_to_l1(l0, sva); 4843 if (pmap_load(l1) == 0) 4844 continue; 4845 if ((pmap_load(l1) & ATTR_DESCR_MASK) == L1_BLOCK) { 4846 PMAP_ASSERT_L1_BLOCKS_SUPPORTED; 4847 KASSERT(va_next <= eva, 4848 ("partial update of non-transparent 1G page " 4849 "l1 %#lx sva %#lx eva %#lx va_next %#lx", 4850 pmap_load(l1), sva, eva, va_next)); 4851 MPASS((pmap_load(l1) & ATTR_SW_MANAGED) == 0); 4852 if ((pmap_load(l1) & mask) != nbits) { 4853 pmap_store(l1, (pmap_load(l1) & ~mask) | nbits); 4854 if (invalidate) 4855 pmap_s1_invalidate_page(pmap, sva, true); 4856 } 4857 continue; 4858 } 4859 4860 va_next = (sva + L2_SIZE) & ~L2_OFFSET; 4861 if (va_next < sva) 4862 va_next = eva; 4863 4864 l2 = pmap_l1_to_l2(l1, sva); 4865 if (pmap_load(l2) == 0) 4866 continue; 4867 4868 if ((pmap_load(l2) & ATTR_DESCR_MASK) == L2_BLOCK) { 4869 if (sva + L2_SIZE == va_next && eva >= va_next) { 4870 pmap_protect_l2(pmap, l2, sva, mask, nbits); 4871 continue; 4872 } else if ((pmap_load(l2) & mask) == nbits || 4873 pmap_demote_l2(pmap, l2, sva) == NULL) 4874 continue; 4875 } 4876 KASSERT((pmap_load(l2) & ATTR_DESCR_MASK) == L2_TABLE, 4877 ("pmap_protect: Invalid L2 entry after demotion")); 4878 4879 if (va_next > eva) 4880 va_next = eva; 4881 4882 va = va_next; 4883 for (l3p = pmap_l2_to_l3(l2, sva); sva != va_next; l3p++, 4884 sva += L3_SIZE) { 4885 l3 = pmap_load(l3p); 4886 4887 /* 4888 * Go to the next L3 entry if the current one is 4889 * invalid or already has the desired access 4890 * restrictions in place. (The latter case occurs 4891 * frequently. For example, in a "buildworld" 4892 * workload, almost 1 out of 4 L3 entries already 4893 * have the desired restrictions.) 4894 */ 4895 if (!pmap_l3_valid(l3) || (l3 & mask) == nbits) { 4896 if (va != va_next) { 4897 if (invalidate) 4898 pmap_s1_invalidate_range(pmap, 4899 va, sva, true); 4900 va = va_next; 4901 } 4902 if ((l3 & ATTR_CONTIGUOUS) != 0) { 4903 /* 4904 * Does this L3C page extend beyond 4905 * the requested range? Handle the 4906 * possibility that "va_next" is zero. 4907 */ 4908 if ((sva | L3C_OFFSET) > va_next - 1) 4909 break; 4910 4911 /* 4912 * Skip ahead to the last L3_PAGE 4913 * within this L3C page. 4914 */ 4915 l3p = (pt_entry_t *)((uintptr_t)l3p | 4916 ((L3C_ENTRIES - 1) * 4917 sizeof(pt_entry_t))); 4918 sva |= L3C_SIZE - L3_SIZE; 4919 } 4920 continue; 4921 } 4922 4923 if ((l3 & ATTR_CONTIGUOUS) != 0) { 4924 /* 4925 * Is this entire set of contiguous L3 entries 4926 * being protected? Handle the possibility 4927 * that "va_next" is zero because of address 4928 * wraparound. 4929 */ 4930 if ((sva & L3C_OFFSET) == 0 && 4931 sva + L3C_OFFSET <= va_next - 1) { 4932 pmap_mask_set_l3c(pmap, l3p, sva, &va, 4933 va_next, mask, nbits); 4934 l3p += L3C_ENTRIES - 1; 4935 sva += L3C_SIZE - L3_SIZE; 4936 continue; 4937 } 4938 4939 (void)pmap_demote_l3c(pmap, l3p, sva); 4940 4941 /* 4942 * The L3 entry's accessed bit may have changed. 4943 */ 4944 l3 = pmap_load(l3p); 4945 } 4946 while (!atomic_fcmpset_64(l3p, &l3, (l3 & ~mask) | 4947 nbits)) 4948 cpu_spinwait(); 4949 4950 /* 4951 * When a dirty read/write mapping is write protected, 4952 * update the page's dirty field. 4953 */ 4954 if ((l3 & ATTR_SW_MANAGED) != 0 && 4955 (nbits & ATTR_S1_AP(ATTR_S1_AP_RO)) != 0 && 4956 pmap_pte_dirty(pmap, l3)) 4957 vm_page_dirty(PTE_TO_VM_PAGE(l3)); 4958 4959 if (va == va_next) 4960 va = sva; 4961 } 4962 if (va != va_next && invalidate) 4963 pmap_s1_invalidate_range(pmap, va, sva, true); 4964 } 4965 } 4966 4967 static void 4968 pmap_mask_set(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, pt_entry_t mask, 4969 pt_entry_t nbits, bool invalidate) 4970 { 4971 PMAP_LOCK(pmap); 4972 pmap_mask_set_locked(pmap, sva, eva, mask, nbits, invalidate); 4973 PMAP_UNLOCK(pmap); 4974 } 4975 4976 /* 4977 * Set the physical protection on the 4978 * specified range of this map as requested. 4979 */ 4980 void 4981 pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot) 4982 { 4983 pt_entry_t mask, nbits; 4984 4985 PMAP_ASSERT_STAGE1(pmap); 4986 KASSERT((prot & ~VM_PROT_ALL) == 0, ("invalid prot %x", prot)); 4987 if (prot == VM_PROT_NONE) { 4988 pmap_remove(pmap, sva, eva); 4989 return; 4990 } 4991 4992 mask = nbits = 0; 4993 if ((prot & VM_PROT_WRITE) == 0) { 4994 mask |= ATTR_S1_AP_RW_BIT | ATTR_SW_DBM; 4995 nbits |= ATTR_S1_AP(ATTR_S1_AP_RO); 4996 } 4997 if ((prot & VM_PROT_EXECUTE) == 0) { 4998 mask |= ATTR_S1_XN; 4999 nbits |= ATTR_S1_XN; 5000 } 5001 if (pmap == kernel_pmap) { 5002 mask |= ATTR_KERN_GP; 5003 nbits |= ATTR_KERN_GP; 5004 } 5005 if (mask == 0) 5006 return; 5007 5008 pmap_mask_set(pmap, sva, eva, mask, nbits, true); 5009 } 5010 5011 void 5012 pmap_disable_promotion(vm_offset_t sva, vm_size_t size) 5013 { 5014 5015 MPASS((sva & L3_OFFSET) == 0); 5016 MPASS(((sva + size) & L3_OFFSET) == 0); 5017 5018 pmap_mask_set(kernel_pmap, sva, sva + size, ATTR_SW_NO_PROMOTE, 5019 ATTR_SW_NO_PROMOTE, false); 5020 } 5021 5022 /* 5023 * Inserts the specified page table page into the specified pmap's collection 5024 * of idle page table pages. Each of a pmap's page table pages is responsible 5025 * for mapping a distinct range of virtual addresses. The pmap's collection is 5026 * ordered by this virtual address range. 5027 * 5028 * If "promoted" is false, then the page table page "mpte" must be zero filled; 5029 * "mpte"'s valid field will be set to 0. 5030 * 5031 * If "promoted" is true and "all_l3e_AF_set" is false, then "mpte" must 5032 * contain valid mappings with identical attributes except for ATTR_AF; 5033 * "mpte"'s valid field will be set to 1. 5034 * 5035 * If "promoted" and "all_l3e_AF_set" are both true, then "mpte" must contain 5036 * valid mappings with identical attributes including ATTR_AF; "mpte"'s valid 5037 * field will be set to VM_PAGE_BITS_ALL. 5038 */ 5039 static __inline int 5040 pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte, bool promoted, 5041 bool all_l3e_AF_set) 5042 { 5043 5044 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 5045 KASSERT(promoted || !all_l3e_AF_set, 5046 ("a zero-filled PTP can't have ATTR_AF set in every PTE")); 5047 mpte->valid = promoted ? (all_l3e_AF_set ? VM_PAGE_BITS_ALL : 1) : 0; 5048 return (vm_radix_insert(&pmap->pm_root, mpte)); 5049 } 5050 5051 /* 5052 * Removes the page table page mapping the specified virtual address from the 5053 * specified pmap's collection of idle page table pages, and returns it. 5054 * Otherwise, returns NULL if there is no page table page corresponding to the 5055 * specified virtual address. 5056 */ 5057 static __inline vm_page_t 5058 pmap_remove_pt_page(pmap_t pmap, vm_offset_t va) 5059 { 5060 5061 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 5062 return (vm_radix_remove(&pmap->pm_root, pmap_l2_pindex(va))); 5063 } 5064 5065 /* 5066 * Performs a break-before-make update of a pmap entry. This is needed when 5067 * either promoting or demoting pages to ensure the TLB doesn't get into an 5068 * inconsistent state. The caller must pass false for "final_only" when 5069 * promoting, because the TLB might be caching an intermediate entry that 5070 * references the L{1,2}_TABLE that is being replaced. In contrast, when 5071 * demoting or the PTE's type isn't changing, no cached intermediate entry 5072 * needs to change, so the caller should pass true as an optimization. 5073 */ 5074 static __always_inline void 5075 pmap_update_entry(pmap_t pmap, pd_entry_t *ptep, pd_entry_t newpte, 5076 vm_offset_t va, vm_size_t size, bool final_only) 5077 { 5078 register_t intr; 5079 5080 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 5081 KASSERT((newpte & ATTR_SW_NO_PROMOTE) == 0, 5082 ("%s: Updating non-promote pte", __func__)); 5083 5084 /* 5085 * Ensure we don't get switched out with the page table in an 5086 * inconsistent state. We also need to ensure no interrupts fire 5087 * as they may make use of an address we are about to invalidate. 5088 */ 5089 intr = intr_disable(); 5090 5091 /* 5092 * Clear the old mapping's valid bit, but leave the rest of the entry 5093 * unchanged, so that a lockless, concurrent pmap_kextract() can still 5094 * lookup the physical address. 5095 */ 5096 pmap_clear_bits(ptep, ATTR_DESCR_VALID); 5097 5098 /* 5099 * We always inline pmap_update_entry() so that constant propagation 5100 * and dead code elimination will specialize the following code. 5101 */ 5102 pmap_s1_invalidate_range(pmap, va, va + size, final_only); 5103 5104 /* Create the new mapping */ 5105 pmap_store(ptep, newpte); 5106 dsb(ishst); 5107 5108 intr_restore(intr); 5109 } 5110 5111 /* 5112 * Performs a break-before-make update of an ATTR_CONTIGUOUS mapping. 5113 */ 5114 static void __nosanitizecoverage 5115 pmap_update_strided(pmap_t pmap, pd_entry_t *ptep, pd_entry_t *ptep_end, 5116 pd_entry_t newpte, vm_offset_t va, vm_offset_t stride, vm_size_t size) 5117 { 5118 pd_entry_t *lip; 5119 register_t intr; 5120 5121 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 5122 KASSERT((newpte & ATTR_SW_NO_PROMOTE) == 0, 5123 ("%s: Updating non-promote pte", __func__)); 5124 5125 /* 5126 * Ensure we don't get switched out with the page table in an 5127 * inconsistent state. We also need to ensure no interrupts fire 5128 * as they may make use of an address we are about to invalidate. 5129 */ 5130 intr = intr_disable(); 5131 5132 /* 5133 * Clear the old mapping's valid bits, but leave the rest of each 5134 * entry unchanged, so that a lockless, concurrent pmap_kextract() can 5135 * still lookup the physical address. 5136 */ 5137 for (lip = ptep; lip < ptep_end; lip++) 5138 pmap_clear_bits(lip, ATTR_DESCR_VALID); 5139 5140 /* Only final entries are changing. */ 5141 pmap_s1_invalidate_strided(pmap, va, va + size, stride, true); 5142 5143 /* Create the new mapping. */ 5144 for (lip = ptep; lip < ptep_end; lip++) { 5145 pmap_store(lip, newpte); 5146 newpte += stride; 5147 } 5148 dsb(ishst); 5149 5150 intr_restore(intr); 5151 } 5152 5153 #if VM_NRESERVLEVEL > 0 5154 /* 5155 * After promotion from 512 4KB page mappings to a single 2MB page mapping, 5156 * replace the many pv entries for the 4KB page mappings by a single pv entry 5157 * for the 2MB page mapping. 5158 */ 5159 static void 5160 pmap_pv_promote_l2(pmap_t pmap, vm_offset_t va, vm_paddr_t pa, 5161 struct rwlock **lockp) 5162 { 5163 struct md_page *pvh; 5164 pv_entry_t pv; 5165 vm_offset_t va_last; 5166 vm_page_t m; 5167 5168 KASSERT((pa & L2_OFFSET) == 0, 5169 ("pmap_pv_promote_l2: pa is not 2mpage aligned")); 5170 CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa); 5171 5172 /* 5173 * Transfer the first page's pv entry for this mapping to the 2mpage's 5174 * pv list. Aside from avoiding the cost of a call to get_pv_entry(), 5175 * a transfer avoids the possibility that get_pv_entry() calls 5176 * reclaim_pv_chunk() and that reclaim_pv_chunk() removes one of the 5177 * mappings that is being promoted. 5178 */ 5179 m = PHYS_TO_VM_PAGE(pa); 5180 va = va & ~L2_OFFSET; 5181 pv = pmap_pvh_remove(&m->md, pmap, va); 5182 KASSERT(pv != NULL, ("pmap_pv_promote_l2: pv not found")); 5183 pvh = page_to_pvh(m); 5184 TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_next); 5185 pvh->pv_gen++; 5186 /* Free the remaining NPTEPG - 1 pv entries. */ 5187 va_last = va + L2_SIZE - PAGE_SIZE; 5188 do { 5189 m++; 5190 va += PAGE_SIZE; 5191 pmap_pvh_free(&m->md, pmap, va); 5192 } while (va < va_last); 5193 } 5194 5195 /* 5196 * Tries to promote the 512, contiguous 4KB page mappings that are within a 5197 * single level 2 table entry to a single 2MB page mapping. For promotion 5198 * to occur, two conditions must be met: (1) the 4KB page mappings must map 5199 * aligned, contiguous physical memory and (2) the 4KB page mappings must have 5200 * identical characteristics. 5201 */ 5202 static bool 5203 pmap_promote_l2(pmap_t pmap, pd_entry_t *l2, vm_offset_t va, vm_page_t mpte, 5204 struct rwlock **lockp) 5205 { 5206 pt_entry_t all_l3e_AF, *firstl3, *l3, newl2, oldl3, pa; 5207 5208 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 5209 5210 /* 5211 * Currently, this function only supports promotion on stage 1 pmaps 5212 * because it tests stage 1 specific fields and performs a break- 5213 * before-make sequence that is incorrect for stage 2 pmaps. 5214 */ 5215 if (pmap->pm_stage != PM_STAGE1 || !pmap_ps_enabled(pmap)) 5216 return (false); 5217 5218 /* 5219 * Examine the first L3E in the specified PTP. Abort if this L3E is 5220 * ineligible for promotion... 5221 */ 5222 firstl3 = PHYS_TO_DMAP(PTE_TO_PHYS(pmap_load(l2))); 5223 newl2 = pmap_load(firstl3); 5224 if ((newl2 & ATTR_SW_NO_PROMOTE) != 0) 5225 return (false); 5226 /* ... is not the first physical page within an L2 block */ 5227 if ((PTE_TO_PHYS(newl2) & L2_OFFSET) != 0 || 5228 ((newl2 & ATTR_DESCR_MASK) != L3_PAGE)) { /* ... or is invalid */ 5229 counter_u64_add(pmap_l2_p_failures, 1); 5230 CTR2(KTR_PMAP, "pmap_promote_l2: failure for va %#lx" 5231 " in pmap %p", va, pmap); 5232 return (false); 5233 } 5234 5235 /* 5236 * Both here and in the below "for" loop, to allow for repromotion 5237 * after MADV_FREE, conditionally write protect a clean L3E before 5238 * possibly aborting the promotion due to other L3E attributes. Why? 5239 * Suppose that MADV_FREE is applied to a part of a superpage, the 5240 * address range [S, E). pmap_advise() will demote the superpage 5241 * mapping, destroy the 4KB page mapping at the end of [S, E), and 5242 * set AP_RO and clear AF in the L3Es for the rest of [S, E). Later, 5243 * imagine that the memory in [S, E) is recycled, but the last 4KB 5244 * page in [S, E) is not the last to be rewritten, or simply accessed. 5245 * In other words, there is still a 4KB page in [S, E), call it P, 5246 * that is writeable but AP_RO is set and AF is clear in P's L3E. 5247 * Unless we write protect P before aborting the promotion, if and 5248 * when P is finally rewritten, there won't be a page fault to trigger 5249 * repromotion. 5250 */ 5251 setl2: 5252 if ((newl2 & (ATTR_S1_AP_RW_BIT | ATTR_SW_DBM)) == 5253 (ATTR_S1_AP(ATTR_S1_AP_RO) | ATTR_SW_DBM)) { 5254 /* 5255 * When the mapping is clean, i.e., ATTR_S1_AP_RO is set, 5256 * ATTR_SW_DBM can be cleared without a TLB invalidation. 5257 */ 5258 if (!atomic_fcmpset_64(firstl3, &newl2, newl2 & ~ATTR_SW_DBM)) 5259 goto setl2; 5260 newl2 &= ~ATTR_SW_DBM; 5261 CTR2(KTR_PMAP, "pmap_promote_l2: protect for va %#lx" 5262 " in pmap %p", va & ~L2_OFFSET, pmap); 5263 } 5264 5265 /* 5266 * Examine each of the other L3Es in the specified PTP. Abort if this 5267 * L3E maps an unexpected 4KB physical page or does not have identical 5268 * characteristics to the first L3E. If ATTR_AF is not set in every 5269 * PTE, then request that the PTP be refilled on demotion. 5270 */ 5271 all_l3e_AF = newl2 & ATTR_AF; 5272 pa = (PTE_TO_PHYS(newl2) | (newl2 & ATTR_DESCR_MASK)) 5273 + L2_SIZE - PAGE_SIZE; 5274 for (l3 = firstl3 + NL3PG - 1; l3 > firstl3; l3--) { 5275 oldl3 = pmap_load(l3); 5276 if ((PTE_TO_PHYS(oldl3) | (oldl3 & ATTR_DESCR_MASK)) != pa) { 5277 counter_u64_add(pmap_l2_p_failures, 1); 5278 CTR2(KTR_PMAP, "pmap_promote_l2: failure for va %#lx" 5279 " in pmap %p", va, pmap); 5280 return (false); 5281 } 5282 setl3: 5283 if ((oldl3 & (ATTR_S1_AP_RW_BIT | ATTR_SW_DBM)) == 5284 (ATTR_S1_AP(ATTR_S1_AP_RO) | ATTR_SW_DBM)) { 5285 /* 5286 * When the mapping is clean, i.e., ATTR_S1_AP_RO is 5287 * set, ATTR_SW_DBM can be cleared without a TLB 5288 * invalidation. 5289 */ 5290 if (!atomic_fcmpset_64(l3, &oldl3, oldl3 & 5291 ~ATTR_SW_DBM)) 5292 goto setl3; 5293 oldl3 &= ~ATTR_SW_DBM; 5294 } 5295 if ((oldl3 & ATTR_PROMOTE) != (newl2 & ATTR_PROMOTE)) { 5296 counter_u64_add(pmap_l2_p_failures, 1); 5297 CTR2(KTR_PMAP, "pmap_promote_l2: failure for va %#lx" 5298 " in pmap %p", va, pmap); 5299 return (false); 5300 } 5301 all_l3e_AF &= oldl3; 5302 pa -= PAGE_SIZE; 5303 } 5304 5305 /* 5306 * Unless all PTEs have ATTR_AF set, clear it from the superpage 5307 * mapping, so that promotions triggered by speculative mappings, 5308 * such as pmap_enter_quick(), don't automatically mark the 5309 * underlying pages as referenced. 5310 */ 5311 newl2 &= ~(ATTR_CONTIGUOUS | ATTR_AF | ATTR_DESCR_MASK) | all_l3e_AF; 5312 5313 /* 5314 * Save the page table page in its current state until the L2 5315 * mapping the superpage is demoted by pmap_demote_l2() or 5316 * destroyed by pmap_remove_l3(). 5317 */ 5318 if (mpte == NULL) 5319 mpte = PTE_TO_VM_PAGE(pmap_load(l2)); 5320 KASSERT(mpte >= vm_page_array && 5321 mpte < &vm_page_array[vm_page_array_size], 5322 ("pmap_promote_l2: page table page is out of range")); 5323 KASSERT(mpte->pindex == pmap_l2_pindex(va), 5324 ("pmap_promote_l2: page table page's pindex is wrong")); 5325 if (pmap_insert_pt_page(pmap, mpte, true, all_l3e_AF != 0)) { 5326 counter_u64_add(pmap_l2_p_failures, 1); 5327 CTR2(KTR_PMAP, 5328 "pmap_promote_l2: failure for va %#lx in pmap %p", va, 5329 pmap); 5330 return (false); 5331 } 5332 5333 if ((newl2 & ATTR_SW_MANAGED) != 0) 5334 pmap_pv_promote_l2(pmap, va, PTE_TO_PHYS(newl2), lockp); 5335 5336 pmap_update_entry(pmap, l2, newl2 | L2_BLOCK, va & ~L2_OFFSET, L2_SIZE, 5337 false); 5338 5339 counter_u64_add(pmap_l2_promotions, 1); 5340 CTR2(KTR_PMAP, "pmap_promote_l2: success for va %#lx in pmap %p", va, 5341 pmap); 5342 return (true); 5343 } 5344 5345 /* 5346 * Tries to promote an aligned, contiguous set of base page mappings to a 5347 * single L3C page mapping. For promotion to occur, two conditions must be 5348 * met: (1) the base page mappings must map aligned, contiguous physical 5349 * memory and (2) the base page mappings must have identical characteristics 5350 * except for the accessed flag. 5351 */ 5352 static bool 5353 pmap_promote_l3c(pmap_t pmap, pd_entry_t *l3p, vm_offset_t va) 5354 { 5355 pd_entry_t all_l3e_AF, firstl3c, *l3, oldl3, pa; 5356 5357 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 5358 5359 /* 5360 * Currently, this function only supports promotion on stage 1 pmaps 5361 * because it tests stage 1 specific fields and performs a break- 5362 * before-make sequence that is incorrect for stage 2 pmaps. 5363 */ 5364 if (pmap->pm_stage != PM_STAGE1 || !pmap_ps_enabled(pmap)) 5365 return (false); 5366 5367 /* 5368 * Compute the address of the first L3 entry in the superpage 5369 * candidate. 5370 */ 5371 l3p = (pt_entry_t *)((uintptr_t)l3p & ~((L3C_ENTRIES * 5372 sizeof(pt_entry_t)) - 1)); 5373 5374 firstl3c = pmap_load(l3p); 5375 5376 /* 5377 * Examine the first L3 entry. Abort if this L3E is ineligible for 5378 * promotion... 5379 */ 5380 if ((firstl3c & ATTR_SW_NO_PROMOTE) != 0) 5381 return (false); 5382 /* ...is not properly aligned... */ 5383 if ((PTE_TO_PHYS(firstl3c) & L3C_OFFSET) != 0 || 5384 (firstl3c & ATTR_DESCR_MASK) != L3_PAGE) { /* ...or is invalid. */ 5385 counter_u64_add(pmap_l3c_p_failures, 1); 5386 CTR2(KTR_PMAP, "pmap_promote_l3c: failure for va %#lx" 5387 " in pmap %p", va, pmap); 5388 return (false); 5389 } 5390 5391 /* 5392 * If the first L3 entry is a clean read-write mapping, convert it 5393 * to a read-only mapping. See pmap_promote_l2() for the rationale. 5394 */ 5395 set_first: 5396 if ((firstl3c & (ATTR_S1_AP_RW_BIT | ATTR_SW_DBM)) == 5397 (ATTR_S1_AP(ATTR_S1_AP_RO) | ATTR_SW_DBM)) { 5398 /* 5399 * When the mapping is clean, i.e., ATTR_S1_AP_RO is set, 5400 * ATTR_SW_DBM can be cleared without a TLB invalidation. 5401 */ 5402 if (!atomic_fcmpset_64(l3p, &firstl3c, firstl3c & ~ATTR_SW_DBM)) 5403 goto set_first; 5404 firstl3c &= ~ATTR_SW_DBM; 5405 CTR2(KTR_PMAP, "pmap_promote_l3c: protect for va %#lx" 5406 " in pmap %p", va & ~L3C_OFFSET, pmap); 5407 } 5408 5409 /* 5410 * Check that the rest of the L3 entries are compatible with the first, 5411 * and convert clean read-write mappings to read-only mappings. 5412 */ 5413 all_l3e_AF = firstl3c & ATTR_AF; 5414 pa = (PTE_TO_PHYS(firstl3c) | (firstl3c & ATTR_DESCR_MASK)) + 5415 L3C_SIZE - PAGE_SIZE; 5416 for (l3 = l3p + L3C_ENTRIES - 1; l3 > l3p; l3--) { 5417 oldl3 = pmap_load(l3); 5418 if ((PTE_TO_PHYS(oldl3) | (oldl3 & ATTR_DESCR_MASK)) != pa) { 5419 counter_u64_add(pmap_l3c_p_failures, 1); 5420 CTR2(KTR_PMAP, "pmap_promote_l3c: failure for va %#lx" 5421 " in pmap %p", va, pmap); 5422 return (false); 5423 } 5424 set_l3: 5425 if ((oldl3 & (ATTR_S1_AP_RW_BIT | ATTR_SW_DBM)) == 5426 (ATTR_S1_AP(ATTR_S1_AP_RO) | ATTR_SW_DBM)) { 5427 /* 5428 * When the mapping is clean, i.e., ATTR_S1_AP_RO is 5429 * set, ATTR_SW_DBM can be cleared without a TLB 5430 * invalidation. 5431 */ 5432 if (!atomic_fcmpset_64(l3, &oldl3, oldl3 & 5433 ~ATTR_SW_DBM)) 5434 goto set_l3; 5435 oldl3 &= ~ATTR_SW_DBM; 5436 CTR2(KTR_PMAP, "pmap_promote_l3c: protect for va %#lx" 5437 " in pmap %p", (oldl3 & ~ATTR_MASK & L3C_OFFSET) | 5438 (va & ~L3C_OFFSET), pmap); 5439 } 5440 if ((oldl3 & ATTR_PROMOTE) != (firstl3c & ATTR_PROMOTE)) { 5441 counter_u64_add(pmap_l3c_p_failures, 1); 5442 CTR2(KTR_PMAP, "pmap_promote_l3c: failure for va %#lx" 5443 " in pmap %p", va, pmap); 5444 return (false); 5445 } 5446 all_l3e_AF &= oldl3; 5447 pa -= PAGE_SIZE; 5448 } 5449 5450 /* 5451 * Unless all PTEs have ATTR_AF set, clear it from the superpage 5452 * mapping, so that promotions triggered by speculative mappings, 5453 * such as pmap_enter_quick(), don't automatically mark the 5454 * underlying pages as referenced. 5455 */ 5456 firstl3c &= ~ATTR_AF | all_l3e_AF; 5457 5458 /* 5459 * Remake the mappings with the contiguous bit set. 5460 */ 5461 pmap_update_strided(pmap, l3p, l3p + L3C_ENTRIES, firstl3c | 5462 ATTR_CONTIGUOUS, va & ~L3C_OFFSET, L3_SIZE, L3C_SIZE); 5463 5464 counter_u64_add(pmap_l3c_promotions, 1); 5465 CTR2(KTR_PMAP, "pmap_promote_l3c: success for va %#lx in pmap %p", va, 5466 pmap); 5467 return (true); 5468 } 5469 #endif /* VM_NRESERVLEVEL > 0 */ 5470 5471 static int 5472 pmap_enter_largepage(pmap_t pmap, vm_offset_t va, pt_entry_t pte, int flags, 5473 int psind) 5474 { 5475 pd_entry_t *l0p, *l1p, *l2p, *l3p, newpte, origpte, *tl3p; 5476 vm_page_t mp; 5477 5478 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 5479 KASSERT(psind > 0 && psind < MAXPAGESIZES, 5480 ("psind %d unexpected", psind)); 5481 KASSERT((PTE_TO_PHYS(pte) & (pagesizes[psind] - 1)) == 0, 5482 ("unaligned phys address %#lx pte %#lx psind %d", 5483 PTE_TO_PHYS(pte), pte, psind)); 5484 5485 restart: 5486 newpte = pte; 5487 if (!pmap_bti_same(pmap, va, va + pagesizes[psind], &newpte)) 5488 return (KERN_PROTECTION_FAILURE); 5489 if (psind == 3) { 5490 PMAP_ASSERT_L1_BLOCKS_SUPPORTED; 5491 5492 KASSERT(pagesizes[psind] == L1_SIZE, 5493 ("pagesizes[%d] != L1_SIZE", psind)); 5494 l0p = pmap_l0(pmap, va); 5495 if ((pmap_load(l0p) & ATTR_DESCR_VALID) == 0) { 5496 mp = _pmap_alloc_l3(pmap, pmap_l0_pindex(va), NULL); 5497 if (mp == NULL) { 5498 if ((flags & PMAP_ENTER_NOSLEEP) != 0) 5499 return (KERN_RESOURCE_SHORTAGE); 5500 PMAP_UNLOCK(pmap); 5501 vm_wait(NULL); 5502 PMAP_LOCK(pmap); 5503 goto restart; 5504 } 5505 l1p = pmap_l0_to_l1(l0p, va); 5506 KASSERT(l1p != NULL, ("va %#lx lost l1 entry", va)); 5507 origpte = pmap_load(l1p); 5508 } else { 5509 l1p = pmap_l0_to_l1(l0p, va); 5510 KASSERT(l1p != NULL, ("va %#lx lost l1 entry", va)); 5511 origpte = pmap_load(l1p); 5512 if ((origpte & ATTR_DESCR_VALID) == 0) { 5513 mp = PTE_TO_VM_PAGE(pmap_load(l0p)); 5514 mp->ref_count++; 5515 } 5516 } 5517 KASSERT((PTE_TO_PHYS(origpte) == PTE_TO_PHYS(newpte) && 5518 (origpte & ATTR_DESCR_MASK) == L1_BLOCK) || 5519 (origpte & ATTR_DESCR_VALID) == 0, 5520 ("va %#lx changing 1G phys page l1 %#lx newpte %#lx", 5521 va, origpte, newpte)); 5522 pmap_store(l1p, newpte); 5523 } else if (psind == 2) { 5524 KASSERT(pagesizes[psind] == L2_SIZE, 5525 ("pagesizes[%d] != L2_SIZE", psind)); 5526 l2p = pmap_l2(pmap, va); 5527 if (l2p == NULL) { 5528 mp = _pmap_alloc_l3(pmap, pmap_l1_pindex(va), NULL); 5529 if (mp == NULL) { 5530 if ((flags & PMAP_ENTER_NOSLEEP) != 0) 5531 return (KERN_RESOURCE_SHORTAGE); 5532 PMAP_UNLOCK(pmap); 5533 vm_wait(NULL); 5534 PMAP_LOCK(pmap); 5535 goto restart; 5536 } 5537 l2p = VM_PAGE_TO_DMAP(mp); 5538 l2p = &l2p[pmap_l2_index(va)]; 5539 origpte = pmap_load(l2p); 5540 } else { 5541 l1p = pmap_l1(pmap, va); 5542 origpte = pmap_load(l2p); 5543 if ((origpte & ATTR_DESCR_VALID) == 0) { 5544 mp = PTE_TO_VM_PAGE(pmap_load(l1p)); 5545 mp->ref_count++; 5546 } 5547 } 5548 KASSERT((origpte & ATTR_DESCR_VALID) == 0 || 5549 ((origpte & ATTR_DESCR_MASK) == L2_BLOCK && 5550 PTE_TO_PHYS(origpte) == PTE_TO_PHYS(newpte)), 5551 ("va %#lx changing 2M phys page l2 %#lx newpte %#lx", 5552 va, origpte, newpte)); 5553 pmap_store(l2p, newpte); 5554 } else /* (psind == 1) */ { 5555 KASSERT(pagesizes[psind] == L3C_SIZE, 5556 ("pagesizes[%d] != L3C_SIZE", psind)); 5557 l2p = pmap_l2(pmap, va); 5558 if (l2p == NULL || (pmap_load(l2p) & ATTR_DESCR_VALID) == 0) { 5559 mp = _pmap_alloc_l3(pmap, pmap_l2_pindex(va), NULL); 5560 if (mp == NULL) { 5561 if ((flags & PMAP_ENTER_NOSLEEP) != 0) 5562 return (KERN_RESOURCE_SHORTAGE); 5563 PMAP_UNLOCK(pmap); 5564 vm_wait(NULL); 5565 PMAP_LOCK(pmap); 5566 goto restart; 5567 } 5568 mp->ref_count += L3C_ENTRIES - 1; 5569 l3p = VM_PAGE_TO_DMAP(mp); 5570 l3p = &l3p[pmap_l3_index(va)]; 5571 } else { 5572 l3p = pmap_l2_to_l3(l2p, va); 5573 if ((pmap_load(l3p) & ATTR_DESCR_VALID) == 0) { 5574 mp = PTE_TO_VM_PAGE(pmap_load(l2p)); 5575 mp->ref_count += L3C_ENTRIES; 5576 } 5577 } 5578 for (tl3p = l3p; tl3p < &l3p[L3C_ENTRIES]; tl3p++) { 5579 origpte = pmap_load(tl3p); 5580 KASSERT((origpte & ATTR_DESCR_VALID) == 0 || 5581 ((origpte & ATTR_CONTIGUOUS) != 0 && 5582 PTE_TO_PHYS(origpte) == PTE_TO_PHYS(newpte)), 5583 ("va %#lx changing 64K phys page l3 %#lx newpte %#lx", 5584 va, origpte, newpte)); 5585 pmap_store(tl3p, newpte); 5586 newpte += L3_SIZE; 5587 } 5588 } 5589 dsb(ishst); 5590 5591 if ((origpte & ATTR_DESCR_VALID) == 0) 5592 pmap_resident_count_inc(pmap, pagesizes[psind] / PAGE_SIZE); 5593 if ((newpte & ATTR_SW_WIRED) != 0 && (origpte & ATTR_SW_WIRED) == 0) 5594 pmap->pm_stats.wired_count += pagesizes[psind] / PAGE_SIZE; 5595 else if ((newpte & ATTR_SW_WIRED) == 0 && 5596 (origpte & ATTR_SW_WIRED) != 0) 5597 pmap->pm_stats.wired_count -= pagesizes[psind] / PAGE_SIZE; 5598 5599 return (KERN_SUCCESS); 5600 } 5601 5602 static void 5603 pmap_set_unprotected(pt_entry_t new_l3) 5604 { 5605 vm_paddr_t pa; 5606 5607 pa = PTE_TO_PHYS(new_l3) & ~prot_ns_shared_pa; 5608 5609 rsi_set_addr_range_state(pa, pa + L3_SIZE, RSI_RIPAS_EMPTY, 5610 RSI_CHANGE_DESTROYED, NULL); 5611 } 5612 5613 static void 5614 pmap_set_protected(pt_entry_t old_l3) 5615 { 5616 vm_paddr_t pa; 5617 5618 pa = PTE_TO_PHYS(old_l3) & ~prot_ns_shared_pa; 5619 5620 rsi_set_addr_range_state(pa, pa + L3_SIZE, RSI_RIPAS_RAM, 5621 RSI_CHANGE_DESTROYED, NULL); 5622 } 5623 5624 /* 5625 * Insert the given physical page (p) at 5626 * the specified virtual address (v) in the 5627 * target physical map with the protection requested. 5628 * 5629 * If specified, the page will be wired down, meaning 5630 * that the related pte can not be reclaimed. 5631 * 5632 * NB: This is the only routine which MAY NOT lazy-evaluate 5633 * or lose information. That is, this routine must actually 5634 * insert this page into the given map NOW. 5635 */ 5636 int 5637 pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, 5638 u_int flags, int8_t psind) 5639 { 5640 struct rwlock *lock; 5641 pd_entry_t *pde; 5642 pt_entry_t new_l3, orig_l3; 5643 pt_entry_t *l2, *l3; 5644 pv_entry_t pv; 5645 vm_paddr_t opa, pa; 5646 vm_page_t mpte, om; 5647 bool nosleep; 5648 int full_lvl, lvl, rv; 5649 5650 KASSERT(ADDR_IS_CANONICAL(va), 5651 ("%s: Address not in canonical form: %lx", __func__, va)); 5652 5653 va = trunc_page(va); 5654 if ((m->oflags & VPO_UNMANAGED) == 0) 5655 VM_PAGE_OBJECT_BUSY_ASSERT(m); 5656 pa = VM_PAGE_TO_PHYS(m); 5657 if (in_realm() && (flags & PMAP_ENTER_UNPROTECTED) != 0) 5658 pa |= prot_ns_shared_pa; 5659 new_l3 = (pt_entry_t)(PHYS_TO_PTE(pa) | ATTR_AF | pmap_sh_attr | 5660 L3_PAGE); 5661 new_l3 |= pmap_pte_memattr(pmap, m->md.pv_memattr); 5662 new_l3 |= pmap_pte_prot(pmap, prot); 5663 if ((flags & PMAP_ENTER_WIRED) != 0) 5664 new_l3 |= ATTR_SW_WIRED; 5665 if (pmap->pm_stage == PM_STAGE1) { 5666 if (ADDR_IS_USER(va)) 5667 new_l3 |= ATTR_S1_AP(ATTR_S1_AP_USER) | ATTR_S1_PXN; 5668 else 5669 new_l3 |= ATTR_S1_UXN; 5670 if (pmap != kernel_pmap) 5671 new_l3 |= ATTR_S1_nG; 5672 } else { 5673 /* 5674 * Clear the access flag on executable mappings, this will be 5675 * set later when the page is accessed. The fault handler is 5676 * required to invalidate the I-cache. 5677 * 5678 * TODO: Switch to the valid flag to allow hardware management 5679 * of the access flag. Much of the pmap code assumes the 5680 * valid flag is set and fails to destroy the old page tables 5681 * correctly if it is clear. 5682 */ 5683 if (prot & VM_PROT_EXECUTE) 5684 new_l3 &= ~ATTR_AF; 5685 } 5686 if ((m->oflags & VPO_UNMANAGED) == 0) { 5687 new_l3 |= ATTR_SW_MANAGED; 5688 if ((prot & VM_PROT_WRITE) != 0) { 5689 new_l3 |= ATTR_SW_DBM; 5690 if ((flags & VM_PROT_WRITE) == 0) { 5691 if (pmap->pm_stage == PM_STAGE1) 5692 new_l3 |= ATTR_S1_AP(ATTR_S1_AP_RO); 5693 else 5694 new_l3 &= 5695 ~ATTR_S2_S2AP(ATTR_S2_S2AP_WRITE); 5696 } 5697 } 5698 } 5699 5700 CTR2(KTR_PMAP, "pmap_enter: %.16lx -> %.16lx", va, pa); 5701 5702 lock = NULL; 5703 PMAP_LOCK(pmap); 5704 if ((flags & PMAP_ENTER_LARGEPAGE) != 0) { 5705 KASSERT((m->oflags & VPO_UNMANAGED) != 0, 5706 ("managed largepage va %#lx flags %#x", va, flags)); 5707 if (psind == 3) { 5708 PMAP_ASSERT_L1_BLOCKS_SUPPORTED; 5709 new_l3 &= ~L3_PAGE; 5710 new_l3 |= L1_BLOCK; 5711 } else if (psind == 2) { 5712 new_l3 &= ~L3_PAGE; 5713 new_l3 |= L2_BLOCK; 5714 } else /* (psind == 1) */ 5715 new_l3 |= ATTR_CONTIGUOUS; 5716 rv = pmap_enter_largepage(pmap, va, new_l3, flags, psind); 5717 goto out; 5718 } 5719 if (psind == 2) { 5720 /* Assert the required virtual and physical alignment. */ 5721 KASSERT((va & L2_OFFSET) == 0, ("pmap_enter: va unaligned")); 5722 KASSERT(m->psind > 1, ("pmap_enter: m->psind < psind")); 5723 rv = pmap_enter_l2(pmap, va, (new_l3 & ~L3_PAGE) | L2_BLOCK, 5724 flags, m, &lock); 5725 goto out; 5726 } 5727 mpte = NULL; 5728 if (psind == 1) { 5729 KASSERT((va & L3C_OFFSET) == 0, ("pmap_enter: va unaligned")); 5730 KASSERT(m->psind > 0, ("pmap_enter: m->psind < psind")); 5731 rv = pmap_enter_l3c(pmap, va, new_l3 | ATTR_CONTIGUOUS, flags, 5732 m, &mpte, &lock); 5733 #if VM_NRESERVLEVEL > 0 5734 /* 5735 * Attempt L2 promotion, if both the PTP and a level 1 5736 * reservation are fully populated. 5737 */ 5738 if (rv == KERN_SUCCESS && 5739 (mpte == NULL || mpte->ref_count == NL3PG) && 5740 (m->flags & PG_FICTITIOUS) == 0 && 5741 vm_reserv_level_iffullpop(m) == 1) { 5742 pde = pmap_l2(pmap, va); 5743 (void)pmap_promote_l2(pmap, pde, va, mpte, &lock); 5744 } 5745 #endif 5746 goto out; 5747 } 5748 5749 /* 5750 * In the case that a page table page is not 5751 * resident, we are creating it here. 5752 */ 5753 retry: 5754 pde = pmap_pde(pmap, va, &lvl); 5755 if (pde != NULL && lvl == 2) { 5756 l3 = pmap_l2_to_l3(pde, va); 5757 if (ADDR_IS_USER(va) && mpte == NULL) { 5758 mpte = PTE_TO_VM_PAGE(pmap_load(pde)); 5759 mpte->ref_count++; 5760 } 5761 goto havel3; 5762 } else if (pde != NULL && lvl == 1) { 5763 l2 = pmap_l1_to_l2(pde, va); 5764 if ((pmap_load(l2) & ATTR_DESCR_MASK) == L2_BLOCK && 5765 (l3 = pmap_demote_l2_locked(pmap, l2, va, &lock)) != NULL) { 5766 l3 = &l3[pmap_l3_index(va)]; 5767 if (ADDR_IS_USER(va)) { 5768 mpte = PTE_TO_VM_PAGE(pmap_load(l2)); 5769 mpte->ref_count++; 5770 } 5771 goto havel3; 5772 } 5773 /* We need to allocate an L3 table. */ 5774 } 5775 if (ADDR_IS_USER(va)) { 5776 nosleep = (flags & PMAP_ENTER_NOSLEEP) != 0; 5777 5778 /* 5779 * We use _pmap_alloc_l3() instead of pmap_alloc_l3() in order 5780 * to handle the possibility that a superpage mapping for "va" 5781 * was created while we slept. 5782 */ 5783 mpte = _pmap_alloc_l3(pmap, pmap_l2_pindex(va), 5784 nosleep ? NULL : &lock); 5785 if (mpte == NULL && nosleep) { 5786 CTR0(KTR_PMAP, "pmap_enter: mpte == NULL"); 5787 rv = KERN_RESOURCE_SHORTAGE; 5788 goto out; 5789 } 5790 goto retry; 5791 } else 5792 panic("pmap_enter: missing L3 table for kernel va %#lx", va); 5793 5794 havel3: 5795 orig_l3 = pmap_load(l3); 5796 opa = PTE_TO_PHYS(orig_l3); 5797 pv = NULL; 5798 new_l3 |= pmap_pte_bti(pmap, va); 5799 5800 /* 5801 * Is the specified virtual address already mapped? 5802 */ 5803 if (pmap_l3_valid(orig_l3)) { 5804 /* 5805 * Wiring change, just update stats. We don't worry about 5806 * wiring PT pages as they remain resident as long as there 5807 * are valid mappings in them. Hence, if a user page is wired, 5808 * the PT page will be also. 5809 */ 5810 if ((flags & PMAP_ENTER_WIRED) != 0 && 5811 (orig_l3 & ATTR_SW_WIRED) == 0) 5812 pmap->pm_stats.wired_count++; 5813 else if ((flags & PMAP_ENTER_WIRED) == 0 && 5814 (orig_l3 & ATTR_SW_WIRED) != 0) 5815 pmap->pm_stats.wired_count--; 5816 5817 /* 5818 * Remove the extra PT page reference. 5819 */ 5820 if (mpte != NULL) { 5821 mpte->ref_count--; 5822 KASSERT(mpte->ref_count > 0, 5823 ("pmap_enter: missing reference to page table page," 5824 " va: 0x%lx", va)); 5825 } 5826 5827 /* 5828 * Has the physical page changed? 5829 */ 5830 if (opa == pa) { 5831 /* 5832 * No, might be a protection or wiring change. 5833 */ 5834 if ((orig_l3 & ATTR_SW_MANAGED) != 0 && 5835 (new_l3 & ATTR_SW_DBM) != 0) 5836 vm_page_aflag_set(m, PGA_WRITEABLE); 5837 goto validate; 5838 } 5839 5840 /* 5841 * The physical page has changed. Temporarily invalidate 5842 * the mapping. 5843 */ 5844 if ((orig_l3 & ATTR_CONTIGUOUS) != 0) 5845 (void)pmap_demote_l3c(pmap, l3, va); 5846 orig_l3 = pmap_load_clear(l3); 5847 KASSERT(PTE_TO_PHYS(orig_l3) == opa, 5848 ("pmap_enter: unexpected pa update for %#lx", va)); 5849 if ((orig_l3 & ATTR_SW_MANAGED) != 0) { 5850 om = PHYS_TO_VM_PAGE(opa); 5851 5852 /* 5853 * The pmap lock is sufficient to synchronize with 5854 * concurrent calls to pmap_page_test_mappings() and 5855 * pmap_ts_referenced(). 5856 */ 5857 if (pmap_pte_dirty(pmap, orig_l3)) 5858 vm_page_dirty(om); 5859 if ((orig_l3 & ATTR_AF) != 0) { 5860 pmap_invalidate_page(pmap, va, true); 5861 vm_page_aflag_set(om, PGA_REFERENCED); 5862 } 5863 CHANGE_PV_LIST_LOCK_TO_VM_PAGE(&lock, om); 5864 pv = pmap_pvh_remove(&om->md, pmap, va); 5865 if ((m->oflags & VPO_UNMANAGED) != 0) 5866 free_pv_entry(pmap, pv); 5867 5868 /* 5869 * The old page is likely COW, so check "writeable" 5870 * first. 5871 */ 5872 if ((om->a.flags & PGA_WRITEABLE) != 0 && 5873 !pmap_page_is_mapped_locked(om)) 5874 vm_page_aflag_clear(om, PGA_WRITEABLE); 5875 } else { 5876 KASSERT((orig_l3 & ATTR_AF) != 0, 5877 ("pmap_enter: unmanaged mapping lacks ATTR_AF")); 5878 pmap_invalidate_page(pmap, va, true); 5879 } 5880 orig_l3 = 0; 5881 } else { 5882 /* 5883 * Increment the counters. 5884 */ 5885 if ((new_l3 & ATTR_SW_WIRED) != 0) 5886 pmap->pm_stats.wired_count++; 5887 pmap_resident_count_inc(pmap, 1); 5888 } 5889 /* 5890 * Enter on the PV list if part of our managed memory. 5891 */ 5892 if ((m->oflags & VPO_UNMANAGED) == 0) { 5893 if (pv == NULL) { 5894 pv = get_pv_entry(pmap, &lock); 5895 pv->pv_va = va; 5896 } 5897 CHANGE_PV_LIST_LOCK_TO_VM_PAGE(&lock, m); 5898 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next); 5899 m->md.pv_gen++; 5900 if ((new_l3 & ATTR_SW_DBM) != 0) 5901 vm_page_aflag_set(m, PGA_WRITEABLE); 5902 } 5903 5904 validate: 5905 if (pmap->pm_stage == PM_STAGE1) { 5906 /* 5907 * Sync icache if exec permission and attribute 5908 * VM_MEMATTR_WRITE_BACK is set. Do it now, before the mapping 5909 * is stored and made valid for hardware table walk. If done 5910 * later, then other can access this page before caches are 5911 * properly synced. Don't do it for kernel memory which is 5912 * mapped with exec permission even if the memory isn't going 5913 * to hold executable code. The only time when icache sync is 5914 * needed is after kernel module is loaded and the relocation 5915 * info is processed. And it's done in elf_cpu_load_file(). 5916 */ 5917 if ((prot & VM_PROT_EXECUTE) && pmap != kernel_pmap && 5918 m->md.pv_memattr == VM_MEMATTR_WRITE_BACK && 5919 (opa != pa || (orig_l3 & ATTR_S1_UXN) != 0)) { 5920 PMAP_ASSERT_STAGE1(pmap); 5921 cpu_icache_sync_range(PHYS_TO_DMAP(pa), PAGE_SIZE); 5922 } 5923 } else { 5924 cpu_dcache_wb_range(PHYS_TO_DMAP(pa), PAGE_SIZE); 5925 } 5926 5927 /* 5928 * Update the L3 entry 5929 */ 5930 if (pmap_l3_valid(orig_l3)) { 5931 KASSERT(opa == pa, ("pmap_enter: invalid update")); 5932 if ((orig_l3 & ~ATTR_AF) != (new_l3 & ~ATTR_AF)) { 5933 /* same PA, different attributes */ 5934 if ((orig_l3 & ATTR_CONTIGUOUS) != 0) 5935 (void)pmap_demote_l3c(pmap, l3, va); 5936 orig_l3 = pmap_load_store(l3, new_l3); 5937 pmap_invalidate_page(pmap, va, true); 5938 if ((orig_l3 & ATTR_SW_MANAGED) != 0 && 5939 pmap_pte_dirty(pmap, orig_l3)) 5940 vm_page_dirty(m); 5941 } else { 5942 /* 5943 * orig_l3 == new_l3 5944 * This can happens if multiple threads simultaneously 5945 * access not yet mapped page. This bad for performance 5946 * since this can cause full demotion-NOP-promotion 5947 * cycle. 5948 * Another possible reasons are: 5949 * - VM and pmap memory layout are diverged 5950 * - tlb flush is missing somewhere and CPU doesn't see 5951 * actual mapping. 5952 */ 5953 CTR4(KTR_PMAP, "%s: already mapped page - " 5954 "pmap %p va 0x%#lx pte 0x%lx", 5955 __func__, pmap, va, new_l3); 5956 } 5957 } else { 5958 /* New mapping */ 5959 pmap_store(l3, new_l3); 5960 dsb(ishst); 5961 } 5962 5963 #if VM_NRESERVLEVEL > 0 5964 /* 5965 * First, attempt L3C promotion, if the virtual and physical addresses 5966 * are aligned with each other and an underlying reservation has the 5967 * neighboring L3 pages allocated. The first condition is simply an 5968 * optimization that recognizes some eventual promotion failures early 5969 * at a lower run-time cost. Then, if both a level 1 reservation and 5970 * the PTP are fully populated, attempt L2 promotion. 5971 */ 5972 if ((va & L3C_OFFSET) == (pa & L3C_OFFSET) && 5973 (m->flags & PG_FICTITIOUS) == 0 && 5974 (full_lvl = vm_reserv_level_iffullpop(m)) >= 0 && 5975 pmap_promote_l3c(pmap, l3, va) && 5976 full_lvl == 1 && (mpte == NULL || mpte->ref_count == NL3PG)) 5977 (void)pmap_promote_l2(pmap, pde, va, mpte, &lock); 5978 #endif 5979 5980 rv = KERN_SUCCESS; 5981 5982 if (in_realm() && (flags & PMAP_ENTER_UNPROTECTED) != 0) 5983 pmap_set_unprotected(new_l3); 5984 5985 out: 5986 if (lock != NULL) 5987 rw_wunlock(lock); 5988 PMAP_UNLOCK(pmap); 5989 return (rv); 5990 } 5991 5992 /* 5993 * Tries to create a read- and/or execute-only L2 page mapping. Returns 5994 * KERN_SUCCESS if the mapping was created. Otherwise, returns an error 5995 * value. See pmap_enter_l2() for the possible error values when "no sleep", 5996 * "no replace", and "no reclaim" are specified. 5997 */ 5998 static int 5999 pmap_enter_l2_rx(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, 6000 struct rwlock **lockp) 6001 { 6002 pd_entry_t new_l2; 6003 6004 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 6005 PMAP_ASSERT_STAGE1(pmap); 6006 KASSERT(ADDR_IS_CANONICAL(va), 6007 ("%s: Address not in canonical form: %lx", __func__, va)); 6008 6009 new_l2 = (pd_entry_t)(VM_PAGE_TO_PTE(m) | pmap_sh_attr | 6010 ATTR_S1_IDX(m->md.pv_memattr) | ATTR_S1_AP(ATTR_S1_AP_RO) | 6011 L2_BLOCK); 6012 if ((m->oflags & VPO_UNMANAGED) == 0) 6013 new_l2 |= ATTR_SW_MANAGED; 6014 else 6015 new_l2 |= ATTR_AF; 6016 if ((prot & VM_PROT_EXECUTE) == 0 || 6017 m->md.pv_memattr == VM_MEMATTR_DEVICE) 6018 new_l2 |= ATTR_S1_XN; 6019 if (ADDR_IS_USER(va)) 6020 new_l2 |= ATTR_S1_AP(ATTR_S1_AP_USER) | ATTR_S1_PXN; 6021 else 6022 new_l2 |= ATTR_S1_UXN; 6023 if (pmap != kernel_pmap) 6024 new_l2 |= ATTR_S1_nG; 6025 return (pmap_enter_l2(pmap, va, new_l2, PMAP_ENTER_NOSLEEP | 6026 PMAP_ENTER_NOREPLACE | PMAP_ENTER_NORECLAIM, m, lockp)); 6027 } 6028 6029 /* 6030 * Returns true if every page table entry in the specified page table is 6031 * zero. 6032 */ 6033 static bool 6034 pmap_every_pte_zero(vm_paddr_t pa) 6035 { 6036 pt_entry_t *pt_end, *pte; 6037 6038 KASSERT((pa & PAGE_MASK) == 0, ("pa is misaligned")); 6039 pte = PHYS_TO_DMAP(pa); 6040 for (pt_end = pte + Ln_ENTRIES; pte < pt_end; pte++) { 6041 if (*pte != 0) 6042 return (false); 6043 } 6044 return (true); 6045 } 6046 6047 /* 6048 * Tries to create the specified L2 page mapping. Returns KERN_SUCCESS if 6049 * the mapping was created, and one of KERN_FAILURE, KERN_NO_SPACE, or 6050 * KERN_RESOURCE_SHORTAGE otherwise. Returns KERN_FAILURE if 6051 * PMAP_ENTER_NOREPLACE was specified and a base page mapping already exists 6052 * within the L2 virtual address range starting at the specified virtual 6053 * address. Returns KERN_NO_SPACE if PMAP_ENTER_NOREPLACE was specified and a 6054 * L2 page mapping already exists at the specified virtual address. Returns 6055 * KERN_RESOURCE_SHORTAGE if either (1) PMAP_ENTER_NOSLEEP was specified and a 6056 * page table page allocation failed or (2) PMAP_ENTER_NORECLAIM was specified 6057 * and a PV entry allocation failed. 6058 */ 6059 static int 6060 pmap_enter_l2(pmap_t pmap, vm_offset_t va, pd_entry_t new_l2, u_int flags, 6061 vm_page_t m, struct rwlock **lockp) 6062 { 6063 struct spglist free; 6064 pd_entry_t *l2, old_l2; 6065 vm_page_t l2pg, mt; 6066 vm_page_t uwptpg; 6067 6068 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 6069 KASSERT(ADDR_IS_CANONICAL(va), 6070 ("%s: Address not in canonical form: %lx", __func__, va)); 6071 KASSERT((flags & (PMAP_ENTER_NOREPLACE | PMAP_ENTER_NORECLAIM)) != 6072 PMAP_ENTER_NORECLAIM, 6073 ("pmap_enter_l2: flags is missing PMAP_ENTER_NOREPLACE")); 6074 6075 if ((l2 = pmap_alloc_l2(pmap, va, &l2pg, (flags & 6076 PMAP_ENTER_NOSLEEP) != 0 ? NULL : lockp)) == NULL) { 6077 CTR2(KTR_PMAP, "pmap_enter_l2: failure for va %#lx in pmap %p", 6078 va, pmap); 6079 return (KERN_RESOURCE_SHORTAGE); 6080 } 6081 6082 /* 6083 * If bti is not the same for the whole l2 range, return failure 6084 * and let vm_fault() cope. Check after l2 allocation, since 6085 * it could sleep. 6086 */ 6087 if (!pmap_bti_same(pmap, va, va + L2_SIZE, &new_l2)) { 6088 KASSERT(l2pg != NULL, ("pmap_enter_l2: missing L2 PTP")); 6089 pmap_abort_ptp(pmap, va, l2pg); 6090 return (KERN_PROTECTION_FAILURE); 6091 } 6092 6093 /* 6094 * If there are existing mappings, either abort or remove them. 6095 */ 6096 if ((old_l2 = pmap_load(l2)) != 0) { 6097 KASSERT(l2pg == NULL || l2pg->ref_count > 1, 6098 ("pmap_enter_l2: l2pg's ref count is too low")); 6099 if ((flags & PMAP_ENTER_NOREPLACE) != 0) { 6100 if ((old_l2 & ATTR_DESCR_MASK) == L2_BLOCK) { 6101 if (l2pg != NULL) 6102 l2pg->ref_count--; 6103 CTR2(KTR_PMAP, 6104 "pmap_enter_l2: no space for va %#lx" 6105 " in pmap %p", va, pmap); 6106 return (KERN_NO_SPACE); 6107 } else if (ADDR_IS_USER(va) || 6108 !pmap_every_pte_zero(PTE_TO_PHYS(old_l2))) { 6109 if (l2pg != NULL) 6110 l2pg->ref_count--; 6111 CTR2(KTR_PMAP, 6112 "pmap_enter_l2: failure for va %#lx" 6113 " in pmap %p", va, pmap); 6114 return (KERN_FAILURE); 6115 } 6116 } 6117 SLIST_INIT(&free); 6118 if ((old_l2 & ATTR_DESCR_MASK) == L2_BLOCK) { 6119 (void)pmap_remove_l2(pmap, l2, va, 6120 pmap_load(pmap_l1(pmap, va)), false, &free, lockp); 6121 } else { 6122 if (ADDR_IS_KERNEL(va)) { 6123 /* 6124 * Try to save the ptp in the trie 6125 * before any changes to mappings are 6126 * made. Abort on failure. 6127 */ 6128 mt = PTE_TO_VM_PAGE(old_l2); 6129 if (pmap_insert_pt_page(pmap, mt, false, 6130 false)) { 6131 CTR1(KTR_PMAP, 6132 "pmap_enter_l2: cannot ins kern ptp va %#lx", 6133 va); 6134 return (KERN_RESOURCE_SHORTAGE); 6135 } 6136 /* 6137 * Both pmap_remove_l2() and 6138 * pmap_remove_l3_range() will zero fill 6139 * the L3 kernel page table page. 6140 */ 6141 } 6142 pmap_remove_l3_range(pmap, old_l2, va, va + L2_SIZE, 6143 &free, lockp); 6144 if (ADDR_IS_KERNEL(va)) { 6145 /* 6146 * The TLB could have an intermediate 6147 * entry for the L3 kernel page table 6148 * page, so request an invalidation at 6149 * all levels after clearing the 6150 * L2_TABLE entry. 6151 */ 6152 pmap_clear(l2); 6153 pmap_s1_invalidate_page(pmap, va, false); 6154 } 6155 } 6156 KASSERT(pmap_load(l2) == 0, 6157 ("pmap_enter_l2: non-zero L2 entry %p", l2)); 6158 if (ADDR_IS_USER(va)) { 6159 vm_page_free_pages_toq(&free, true); 6160 } else { 6161 KASSERT(SLIST_EMPTY(&free), 6162 ("pmap_enter_l2: freed kernel page table page")); 6163 } 6164 } 6165 6166 /* 6167 * Allocate leaf ptpage for wired userspace pages. 6168 */ 6169 uwptpg = NULL; 6170 if ((new_l2 & ATTR_SW_WIRED) != 0 && pmap != kernel_pmap) { 6171 uwptpg = vm_page_alloc_noobj(VM_ALLOC_WIRED); 6172 if (uwptpg == NULL) { 6173 pmap_abort_ptp(pmap, va, l2pg); 6174 return (KERN_RESOURCE_SHORTAGE); 6175 } 6176 uwptpg->pindex = pmap_l2_pindex(va); 6177 if (pmap_insert_pt_page(pmap, uwptpg, true, false)) { 6178 vm_page_unwire_noq(uwptpg); 6179 vm_page_free(uwptpg); 6180 pmap_abort_ptp(pmap, va, l2pg); 6181 return (KERN_RESOURCE_SHORTAGE); 6182 } 6183 pmap_resident_count_inc(pmap, 1); 6184 uwptpg->ref_count = NL3PG; 6185 } 6186 if ((new_l2 & ATTR_SW_MANAGED) != 0) { 6187 /* 6188 * Abort this mapping if its PV entry could not be created. 6189 */ 6190 if (!pmap_pv_insert_l2(pmap, va, new_l2, flags, lockp)) { 6191 if (l2pg != NULL) 6192 pmap_abort_ptp(pmap, va, l2pg); 6193 else { 6194 KASSERT(ADDR_IS_KERNEL(va) && 6195 (pmap_load(l2) & ATTR_DESCR_MASK) == 6196 L2_TABLE, 6197 ("pmap_enter_l2: invalid kernel L2E")); 6198 mt = pmap_remove_pt_page(pmap, va); 6199 KASSERT(mt != NULL, 6200 ("pmap_enter_l2: missing kernel PTP")); 6201 } 6202 if (uwptpg != NULL) { 6203 mt = pmap_remove_pt_page(pmap, va); 6204 KASSERT(mt == uwptpg, 6205 ("removed pt page %p, expected %p", mt, 6206 uwptpg)); 6207 pmap_resident_count_dec(pmap, 1); 6208 uwptpg->ref_count = 1; 6209 vm_page_unwire_noq(uwptpg); 6210 vm_page_free(uwptpg); 6211 } 6212 CTR2(KTR_PMAP, 6213 "pmap_enter_l2: failure for va %#lx in pmap %p", 6214 va, pmap); 6215 return (KERN_RESOURCE_SHORTAGE); 6216 } 6217 if ((new_l2 & ATTR_SW_DBM) != 0) 6218 for (mt = m; mt < &m[L2_SIZE / PAGE_SIZE]; mt++) 6219 vm_page_aflag_set(mt, PGA_WRITEABLE); 6220 } 6221 6222 /* 6223 * Increment counters. 6224 */ 6225 if ((new_l2 & ATTR_SW_WIRED) != 0) 6226 pmap->pm_stats.wired_count += L2_SIZE / PAGE_SIZE; 6227 pmap->pm_stats.resident_count += L2_SIZE / PAGE_SIZE; 6228 6229 /* 6230 * Conditionally sync the icache. See pmap_enter() for details. 6231 */ 6232 if ((new_l2 & ATTR_S1_UXN) == 0 && (PTE_TO_PHYS(new_l2) != 6233 PTE_TO_PHYS(old_l2) || (old_l2 & ATTR_S1_UXN) != 0) && 6234 pmap != kernel_pmap && m->md.pv_memattr == VM_MEMATTR_WRITE_BACK) { 6235 cpu_icache_sync_range(PHYS_TO_DMAP(PTE_TO_PHYS(new_l2)), 6236 L2_SIZE); 6237 } 6238 6239 /* 6240 * Map the superpage. 6241 */ 6242 pmap_store(l2, new_l2); 6243 dsb(ishst); 6244 6245 counter_u64_add(pmap_l2_mappings, 1); 6246 CTR2(KTR_PMAP, "pmap_enter_l2: success for va %#lx in pmap %p", 6247 va, pmap); 6248 6249 return (KERN_SUCCESS); 6250 } 6251 6252 /* 6253 * Tries to create a read- and/or execute-only L3C page mapping. Returns 6254 * KERN_SUCCESS if the mapping was created. Otherwise, returns an error 6255 * value. 6256 */ 6257 static int 6258 pmap_enter_l3c_rx(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_page_t *ml3p, 6259 vm_prot_t prot, struct rwlock **lockp) 6260 { 6261 pt_entry_t l3e; 6262 6263 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 6264 PMAP_ASSERT_STAGE1(pmap); 6265 KASSERT(ADDR_IS_CANONICAL(va), 6266 ("%s: Address not in canonical form: %lx", __func__, va)); 6267 6268 l3e = VM_PAGE_TO_PTE(m) | pmap_sh_attr | 6269 ATTR_S1_IDX(m->md.pv_memattr) | ATTR_S1_AP(ATTR_S1_AP_RO) | 6270 ATTR_CONTIGUOUS | L3_PAGE; 6271 if ((m->oflags & VPO_UNMANAGED) == 0) 6272 l3e |= ATTR_SW_MANAGED; 6273 else 6274 l3e |= ATTR_AF; 6275 if ((prot & VM_PROT_EXECUTE) == 0 || 6276 m->md.pv_memattr == VM_MEMATTR_DEVICE) 6277 l3e |= ATTR_S1_XN; 6278 if (ADDR_IS_USER(va)) 6279 l3e |= ATTR_S1_AP(ATTR_S1_AP_USER) | ATTR_S1_PXN; 6280 else 6281 l3e |= ATTR_S1_UXN; 6282 if (pmap != kernel_pmap) 6283 l3e |= ATTR_S1_nG; 6284 return (pmap_enter_l3c(pmap, va, l3e, PMAP_ENTER_NOSLEEP | 6285 PMAP_ENTER_NOREPLACE | PMAP_ENTER_NORECLAIM, m, ml3p, lockp)); 6286 } 6287 6288 static int 6289 pmap_enter_l3c(pmap_t pmap, vm_offset_t va, pt_entry_t l3e, u_int flags, 6290 vm_page_t m, vm_page_t *ml3p, struct rwlock **lockp) 6291 { 6292 pd_entry_t *l2p, *pde; 6293 pt_entry_t *l3p, *tl3p; 6294 vm_page_t mt; 6295 vm_paddr_t pa; 6296 vm_pindex_t l2pindex; 6297 int lvl; 6298 6299 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 6300 KASSERT((va & L3C_OFFSET) == 0, 6301 ("pmap_enter_l3c: va is not aligned")); 6302 KASSERT(!VA_IS_CLEANMAP(va) || (l3e & ATTR_SW_MANAGED) == 0, 6303 ("pmap_enter_l3c: managed mapping within the clean submap")); 6304 KASSERT((l3e & ATTR_CONTIGUOUS) != 0, 6305 ("pmap_enter_l3c: l3e is missing ATTR_CONTIGUOUS")); 6306 6307 /* 6308 * If the L3 PTP is not resident, we attempt to create it here. 6309 */ 6310 if (ADDR_IS_USER(va)) { 6311 /* 6312 * Were we given the correct L3 PTP? If so, we can simply 6313 * increment its ref count. 6314 */ 6315 l2pindex = pmap_l2_pindex(va); 6316 if (*ml3p != NULL && (*ml3p)->pindex == l2pindex) { 6317 (*ml3p)->ref_count += L3C_ENTRIES; 6318 } else { 6319 retry: 6320 /* 6321 * Get the L2 entry. 6322 */ 6323 pde = pmap_pde(pmap, va, &lvl); 6324 6325 /* 6326 * If the L2 entry is a superpage, we either abort or 6327 * demote depending on the given flags. 6328 */ 6329 if (lvl == 1) { 6330 l2p = pmap_l1_to_l2(pde, va); 6331 if ((pmap_load(l2p) & ATTR_DESCR_MASK) == 6332 L2_BLOCK) { 6333 if ((flags & PMAP_ENTER_NOREPLACE) != 0) 6334 return (KERN_FAILURE); 6335 l3p = pmap_demote_l2_locked(pmap, l2p, 6336 va, lockp); 6337 if (l3p != NULL) { 6338 *ml3p = PTE_TO_VM_PAGE( 6339 pmap_load(l2p)); 6340 (*ml3p)->ref_count += 6341 L3C_ENTRIES; 6342 goto have_l3p; 6343 } 6344 } 6345 /* We need to allocate an L3 PTP. */ 6346 } 6347 6348 /* 6349 * If the L3 PTP is mapped, we just increment its ref 6350 * count. Otherwise, we attempt to allocate it. 6351 */ 6352 if (lvl == 2 && pmap_load(pde) != 0) { 6353 *ml3p = PTE_TO_VM_PAGE(pmap_load(pde)); 6354 (*ml3p)->ref_count += L3C_ENTRIES; 6355 } else { 6356 *ml3p = _pmap_alloc_l3(pmap, l2pindex, (flags & 6357 PMAP_ENTER_NOSLEEP) != 0 ? NULL : lockp); 6358 if (*ml3p == NULL) { 6359 if ((flags & PMAP_ENTER_NOSLEEP) != 0) 6360 return (KERN_FAILURE); 6361 6362 /* 6363 * The page table may have changed 6364 * while we slept. 6365 */ 6366 goto retry; 6367 } 6368 (*ml3p)->ref_count += L3C_ENTRIES - 1; 6369 } 6370 } 6371 l3p = VM_PAGE_TO_DMAP(*ml3p); 6372 } else { 6373 *ml3p = NULL; 6374 6375 /* 6376 * If the L2 entry is a superpage, we either abort or demote 6377 * depending on the given flags. 6378 */ 6379 pde = pmap_pde(kernel_pmap, va, &lvl); 6380 if (lvl == 1) { 6381 l2p = pmap_l1_to_l2(pde, va); 6382 KASSERT((pmap_load(l2p) & ATTR_DESCR_MASK) == L2_BLOCK, 6383 ("pmap_enter_l3c: missing L2 block")); 6384 if ((flags & PMAP_ENTER_NOREPLACE) != 0) 6385 return (KERN_FAILURE); 6386 l3p = pmap_demote_l2_locked(pmap, l2p, va, lockp); 6387 } else { 6388 KASSERT(lvl == 2, 6389 ("pmap_enter_l3c: Invalid level %d", lvl)); 6390 l3p = PHYS_TO_DMAP(PTE_TO_PHYS(pmap_load(pde))); 6391 } 6392 } 6393 have_l3p: 6394 l3p = &l3p[pmap_l3_index(va)]; 6395 6396 /* 6397 * If bti is not the same for the whole L3C range, return failure 6398 * and let vm_fault() cope. Check after L3 allocation, since 6399 * it could sleep. 6400 */ 6401 if (!pmap_bti_same(pmap, va, va + L3C_SIZE, &l3e)) { 6402 KASSERT(*ml3p != NULL, ("pmap_enter_l3c: missing L3 PTP")); 6403 (*ml3p)->ref_count -= L3C_ENTRIES - 1; 6404 pmap_abort_ptp(pmap, va, *ml3p); 6405 *ml3p = NULL; 6406 return (KERN_PROTECTION_FAILURE); 6407 } 6408 6409 /* 6410 * If there are existing mappings, either abort or remove them. 6411 */ 6412 if ((flags & PMAP_ENTER_NOREPLACE) != 0) { 6413 for (tl3p = l3p; tl3p < &l3p[L3C_ENTRIES]; tl3p++) { 6414 if (pmap_load(tl3p) != 0) { 6415 if (*ml3p != NULL) 6416 (*ml3p)->ref_count -= L3C_ENTRIES; 6417 return (KERN_FAILURE); 6418 } 6419 } 6420 } else { 6421 /* 6422 * Because we increment the L3 page's reference count above, 6423 * it is guaranteed not to be freed here and we can pass NULL 6424 * instead of a valid free list. 6425 */ 6426 pmap_remove_l3_range(pmap, pmap_load(pmap_l2(pmap, va)), va, 6427 va + L3C_SIZE, NULL, lockp); 6428 } 6429 6430 /* 6431 * Enter on the PV list if part of our managed memory. 6432 */ 6433 if ((l3e & ATTR_SW_MANAGED) != 0) { 6434 if (!pmap_pv_insert_l3c(pmap, va, m, lockp)) { 6435 if (*ml3p != NULL) { 6436 (*ml3p)->ref_count -= L3C_ENTRIES - 1; 6437 pmap_abort_ptp(pmap, va, *ml3p); 6438 *ml3p = NULL; 6439 } 6440 return (KERN_RESOURCE_SHORTAGE); 6441 } 6442 if ((l3e & ATTR_SW_DBM) != 0) 6443 for (mt = m; mt < &m[L3C_ENTRIES]; mt++) 6444 vm_page_aflag_set(mt, PGA_WRITEABLE); 6445 } 6446 6447 /* 6448 * Increment counters. 6449 */ 6450 if ((l3e & ATTR_SW_WIRED) != 0) 6451 pmap->pm_stats.wired_count += L3C_ENTRIES; 6452 pmap_resident_count_inc(pmap, L3C_ENTRIES); 6453 6454 pa = VM_PAGE_TO_PHYS(m); 6455 KASSERT((pa & L3C_OFFSET) == 0, ("pmap_enter_l3c: pa is not aligned")); 6456 6457 /* 6458 * Sync the icache before the mapping is stored. 6459 */ 6460 if ((l3e & ATTR_S1_UXN) == 0 && pmap != kernel_pmap && 6461 m->md.pv_memattr == VM_MEMATTR_WRITE_BACK) 6462 cpu_icache_sync_range(PHYS_TO_DMAP(pa), L3C_SIZE); 6463 6464 /* 6465 * Map the superpage. 6466 */ 6467 for (tl3p = l3p; tl3p < &l3p[L3C_ENTRIES]; tl3p++) { 6468 pmap_store(tl3p, l3e); 6469 l3e += L3_SIZE; 6470 } 6471 dsb(ishst); 6472 6473 counter_u64_add(pmap_l3c_mappings, 1); 6474 CTR2(KTR_PMAP, "pmap_enter_l3c: success for va %#lx in pmap %p", 6475 va, pmap); 6476 return (KERN_SUCCESS); 6477 } 6478 6479 /* 6480 * Maps a sequence of resident pages belonging to the same object. 6481 * The sequence begins with the given page m_start. This page is 6482 * mapped at the given virtual address start. Each subsequent page is 6483 * mapped at a virtual address that is offset from start by the same 6484 * amount as the page is offset from m_start within the object. The 6485 * last page in the sequence is the page with the largest offset from 6486 * m_start that can be mapped at a virtual address less than the given 6487 * virtual address end. Not every virtual page between start and end 6488 * is mapped; only those for which a resident page exists with the 6489 * corresponding offset from m_start are mapped. 6490 */ 6491 void 6492 pmap_enter_object(pmap_t pmap, vm_offset_t start, vm_offset_t end, 6493 vm_page_t m_start, vm_prot_t prot) 6494 { 6495 struct pctrie_iter pages; 6496 struct rwlock *lock; 6497 vm_offset_t va; 6498 vm_page_t m, mpte; 6499 int rv; 6500 6501 VM_OBJECT_ASSERT_LOCKED(m_start->object); 6502 6503 mpte = NULL; 6504 vm_page_iter_limit_init(&pages, m_start->object, 6505 m_start->pindex + atop(end - start)); 6506 m = vm_radix_iter_lookup(&pages, m_start->pindex); 6507 lock = NULL; 6508 PMAP_LOCK(pmap); 6509 while (m != NULL) { 6510 va = start + ptoa(m->pindex - m_start->pindex); 6511 if ((va & L2_OFFSET) == 0 && va + L2_SIZE <= end && 6512 m->psind == 2 && pmap_ps_enabled(pmap) && 6513 ((rv = pmap_enter_l2_rx(pmap, va, m, prot, &lock)) == 6514 KERN_SUCCESS || rv == KERN_NO_SPACE)) { 6515 m = vm_radix_iter_jump(&pages, L2_SIZE / PAGE_SIZE); 6516 } else if ((va & L3C_OFFSET) == 0 && va + L3C_SIZE <= end && 6517 m->psind >= 1 && pmap_ps_enabled(pmap) && 6518 ((rv = pmap_enter_l3c_rx(pmap, va, m, &mpte, prot, 6519 &lock)) == KERN_SUCCESS || rv == KERN_NO_SPACE)) { 6520 m = vm_radix_iter_jump(&pages, L3C_ENTRIES); 6521 } else { 6522 /* 6523 * In general, if a superpage mapping were possible, 6524 * it would have been created above. That said, if 6525 * start and end are not superpage aligned, then 6526 * promotion might be possible at the ends of [start, 6527 * end). However, in practice, those promotion 6528 * attempts are so unlikely to succeed that they are 6529 * not worth trying. 6530 */ 6531 mpte = pmap_enter_quick_locked(pmap, va, m, prot | 6532 VM_PROT_NO_PROMOTE, mpte, &lock); 6533 m = vm_radix_iter_step(&pages); 6534 } 6535 } 6536 if (lock != NULL) 6537 rw_wunlock(lock); 6538 PMAP_UNLOCK(pmap); 6539 } 6540 6541 /* 6542 * this code makes some *MAJOR* assumptions: 6543 * 1. Current pmap & pmap exists. 6544 * 2. Not wired. 6545 * 3. Read access. 6546 * 4. No page table pages. 6547 * but is *MUCH* faster than pmap_enter... 6548 */ 6549 6550 void 6551 pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot) 6552 { 6553 struct rwlock *lock; 6554 6555 lock = NULL; 6556 PMAP_LOCK(pmap); 6557 (void)pmap_enter_quick_locked(pmap, va, m, prot, NULL, &lock); 6558 if (lock != NULL) 6559 rw_wunlock(lock); 6560 PMAP_UNLOCK(pmap); 6561 } 6562 6563 static vm_page_t 6564 pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, 6565 vm_prot_t prot, vm_page_t mpte, struct rwlock **lockp) 6566 { 6567 pt_entry_t *l1, *l2, *l3, l3_val; 6568 vm_paddr_t pa; 6569 int full_lvl, lvl; 6570 6571 KASSERT(!VA_IS_CLEANMAP(va) || 6572 (m->oflags & VPO_UNMANAGED) != 0, 6573 ("pmap_enter_quick_locked: managed mapping within the clean submap")); 6574 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 6575 PMAP_ASSERT_STAGE1(pmap); 6576 KASSERT(ADDR_IS_CANONICAL(va), 6577 ("%s: Address not in canonical form: %lx", __func__, va)); 6578 l2 = NULL; 6579 6580 CTR2(KTR_PMAP, "pmap_enter_quick_locked: %p %lx", pmap, va); 6581 /* 6582 * In the case that a page table page is not 6583 * resident, we are creating it here. 6584 */ 6585 if (ADDR_IS_USER(va)) { 6586 vm_pindex_t l2pindex; 6587 6588 /* 6589 * Calculate pagetable page index 6590 */ 6591 l2pindex = pmap_l2_pindex(va); 6592 if (mpte && (mpte->pindex == l2pindex)) { 6593 mpte->ref_count++; 6594 } else { 6595 /* 6596 * If the page table page is mapped, we just increment 6597 * the hold count, and activate it. Otherwise, we 6598 * attempt to allocate a page table page, passing NULL 6599 * instead of the PV list lock pointer because we don't 6600 * intend to sleep. If this attempt fails, we don't 6601 * retry. Instead, we give up. 6602 */ 6603 l1 = pmap_l1(pmap, va); 6604 if (l1 != NULL && pmap_load(l1) != 0) { 6605 if ((pmap_load(l1) & ATTR_DESCR_MASK) == 6606 L1_BLOCK) 6607 return (NULL); 6608 l2 = pmap_l1_to_l2(l1, va); 6609 if (pmap_load(l2) != 0) { 6610 if ((pmap_load(l2) & ATTR_DESCR_MASK) == 6611 L2_BLOCK) 6612 return (NULL); 6613 mpte = PTE_TO_VM_PAGE(pmap_load(l2)); 6614 mpte->ref_count++; 6615 } else { 6616 mpte = _pmap_alloc_l3(pmap, l2pindex, 6617 NULL); 6618 if (mpte == NULL) 6619 return (mpte); 6620 } 6621 } else { 6622 mpte = _pmap_alloc_l3(pmap, l2pindex, NULL); 6623 if (mpte == NULL) 6624 return (mpte); 6625 } 6626 } 6627 l3 = VM_PAGE_TO_DMAP(mpte); 6628 l3 = &l3[pmap_l3_index(va)]; 6629 } else { 6630 mpte = NULL; 6631 l2 = pmap_pde(kernel_pmap, va, &lvl); 6632 KASSERT(l2 != NULL, 6633 ("pmap_enter_quick_locked: Invalid page entry, va: 0x%lx", 6634 va)); 6635 KASSERT(lvl == 2, 6636 ("pmap_enter_quick_locked: Invalid level %d", lvl)); 6637 l3 = pmap_l2_to_l3(l2, va); 6638 } 6639 6640 /* 6641 * Abort if a mapping already exists. 6642 */ 6643 if (pmap_load(l3) != 0) { 6644 if (mpte != NULL) 6645 mpte->ref_count--; 6646 return (NULL); 6647 } 6648 6649 /* 6650 * Enter on the PV list if part of our managed memory. 6651 */ 6652 if ((m->oflags & VPO_UNMANAGED) == 0 && 6653 !pmap_try_insert_pv_entry(pmap, va, m, lockp)) { 6654 if (mpte != NULL) 6655 pmap_abort_ptp(pmap, va, mpte); 6656 return (NULL); 6657 } 6658 6659 /* 6660 * Increment counters 6661 */ 6662 pmap_resident_count_inc(pmap, 1); 6663 6664 pa = VM_PAGE_TO_PHYS(m); 6665 l3_val = PHYS_TO_PTE(pa) | pmap_sh_attr | 6666 ATTR_S1_IDX(m->md.pv_memattr) | ATTR_S1_AP(ATTR_S1_AP_RO) | L3_PAGE; 6667 l3_val |= pmap_pte_bti(pmap, va); 6668 if ((prot & VM_PROT_EXECUTE) == 0 || 6669 m->md.pv_memattr == VM_MEMATTR_DEVICE) 6670 l3_val |= ATTR_S1_XN; 6671 if (ADDR_IS_USER(va)) 6672 l3_val |= ATTR_S1_AP(ATTR_S1_AP_USER) | ATTR_S1_PXN; 6673 else 6674 l3_val |= ATTR_S1_UXN; 6675 if (pmap != kernel_pmap) 6676 l3_val |= ATTR_S1_nG; 6677 6678 /* 6679 * Now validate mapping with RO protection 6680 */ 6681 if ((m->oflags & VPO_UNMANAGED) == 0) 6682 l3_val |= ATTR_SW_MANAGED; 6683 else 6684 l3_val |= ATTR_AF; 6685 6686 /* Sync icache before the mapping is stored to PTE */ 6687 if ((prot & VM_PROT_EXECUTE) && pmap != kernel_pmap && 6688 m->md.pv_memattr == VM_MEMATTR_WRITE_BACK) 6689 cpu_icache_sync_range(PHYS_TO_DMAP(pa), PAGE_SIZE); 6690 6691 pmap_store(l3, l3_val); 6692 dsb(ishst); 6693 6694 #if VM_NRESERVLEVEL > 0 6695 /* 6696 * First, attempt L3C promotion, if the virtual and physical addresses 6697 * are aligned with each other and an underlying reservation has the 6698 * neighboring L3 pages allocated. The first condition is simply an 6699 * optimization that recognizes some eventual promotion failures early 6700 * at a lower run-time cost. Then, attempt L2 promotion, if both a 6701 * level 1 reservation and the PTP are fully populated. 6702 */ 6703 if ((prot & VM_PROT_NO_PROMOTE) == 0 && 6704 (va & L3C_OFFSET) == (pa & L3C_OFFSET) && 6705 (m->flags & PG_FICTITIOUS) == 0 && 6706 (full_lvl = vm_reserv_level_iffullpop(m)) >= 0 && 6707 pmap_promote_l3c(pmap, l3, va) && 6708 full_lvl == 1 && (mpte == NULL || mpte->ref_count == NL3PG)) { 6709 if (l2 == NULL) 6710 l2 = pmap_l2(pmap, va); 6711 6712 /* 6713 * If promotion succeeds, then the next call to this function 6714 * should not be given the unmapped PTP as a hint. 6715 */ 6716 if (pmap_promote_l2(pmap, l2, va, mpte, lockp)) 6717 mpte = NULL; 6718 } 6719 #endif 6720 6721 return (mpte); 6722 } 6723 6724 /* 6725 * This code maps large physical mmap regions into the 6726 * processor address space. Note that some shortcuts 6727 * are taken, but the code works. 6728 */ 6729 void 6730 pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, vm_object_t object, 6731 vm_pindex_t pindex, vm_size_t size) 6732 { 6733 6734 VM_OBJECT_ASSERT_WLOCKED(object); 6735 KASSERT(object->type == OBJT_DEVICE || object->type == OBJT_SG, 6736 ("pmap_object_init_pt: non-device object")); 6737 } 6738 6739 /* 6740 * Clear the wired attribute from the mappings for the specified range of 6741 * addresses in the given pmap. Every valid mapping within that range 6742 * must have the wired attribute set. In contrast, invalid mappings 6743 * cannot have the wired attribute set, so they are ignored. 6744 * 6745 * The wired attribute of the page table entry is not a hardware feature, 6746 * so there is no need to invalidate any TLB entries. 6747 */ 6748 void 6749 pmap_unwire(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) 6750 { 6751 vm_offset_t va_next; 6752 pd_entry_t *l0, *l1, *l2; 6753 pt_entry_t *l3; 6754 bool partial_l3c; 6755 6756 PMAP_LOCK(pmap); 6757 for (; sva < eva; sva = va_next) { 6758 l0 = pmap_l0(pmap, sva); 6759 if (pmap_load(l0) == 0) { 6760 va_next = (sva + L0_SIZE) & ~L0_OFFSET; 6761 if (va_next < sva) 6762 va_next = eva; 6763 continue; 6764 } 6765 6766 l1 = pmap_l0_to_l1(l0, sva); 6767 va_next = (sva + L1_SIZE) & ~L1_OFFSET; 6768 if (va_next < sva) 6769 va_next = eva; 6770 if (pmap_load(l1) == 0) 6771 continue; 6772 6773 if ((pmap_load(l1) & ATTR_DESCR_MASK) == L1_BLOCK) { 6774 PMAP_ASSERT_L1_BLOCKS_SUPPORTED; 6775 KASSERT(va_next <= eva, 6776 ("partial update of non-transparent 1G page " 6777 "l1 %#lx sva %#lx eva %#lx va_next %#lx", 6778 pmap_load(l1), sva, eva, va_next)); 6779 MPASS(pmap != kernel_pmap); 6780 MPASS((pmap_load(l1) & (ATTR_SW_MANAGED | 6781 ATTR_SW_WIRED)) == ATTR_SW_WIRED); 6782 pmap_clear_bits(l1, ATTR_SW_WIRED); 6783 pmap->pm_stats.wired_count -= L1_SIZE / PAGE_SIZE; 6784 continue; 6785 } 6786 6787 va_next = (sva + L2_SIZE) & ~L2_OFFSET; 6788 if (va_next < sva) 6789 va_next = eva; 6790 6791 l2 = pmap_l1_to_l2(l1, sva); 6792 if (pmap_load(l2) == 0) 6793 continue; 6794 6795 if ((pmap_load(l2) & ATTR_DESCR_MASK) == L2_BLOCK) { 6796 if ((pmap_load(l2) & ATTR_SW_WIRED) == 0) 6797 panic("pmap_unwire: l2 %#jx is missing " 6798 "ATTR_SW_WIRED", (uintmax_t)pmap_load(l2)); 6799 6800 /* 6801 * Are we unwiring the entire large page? If not, 6802 * demote the mapping and fall through. 6803 */ 6804 if (sva + L2_SIZE == va_next && eva >= va_next) { 6805 pmap_clear_bits(l2, ATTR_SW_WIRED); 6806 pmap->pm_stats.wired_count -= L2_SIZE / 6807 PAGE_SIZE; 6808 continue; 6809 } else if (pmap_demote_l2(pmap, l2, sva) == NULL) 6810 panic("pmap_unwire: demotion failed"); 6811 } 6812 KASSERT((pmap_load(l2) & ATTR_DESCR_MASK) == L2_TABLE, 6813 ("pmap_unwire: Invalid l2 entry after demotion")); 6814 6815 if (va_next > eva) 6816 va_next = eva; 6817 for (partial_l3c = true, l3 = pmap_l2_to_l3(l2, sva); 6818 sva != va_next; l3++, sva += L3_SIZE) { 6819 if (pmap_load(l3) == 0) 6820 continue; 6821 if ((pmap_load(l3) & ATTR_CONTIGUOUS) != 0) { 6822 /* 6823 * Avoid demotion for whole-page unwiring. 6824 */ 6825 if ((sva & L3C_OFFSET) == 0) { 6826 /* 6827 * Handle the possibility that 6828 * "va_next" is zero because of 6829 * address wraparound. 6830 */ 6831 partial_l3c = sva + L3C_OFFSET > 6832 va_next - 1; 6833 } 6834 if (partial_l3c) 6835 (void)pmap_demote_l3c(pmap, l3, sva); 6836 } 6837 if ((pmap_load(l3) & ATTR_SW_WIRED) == 0) 6838 panic("pmap_unwire: l3 %#jx is missing " 6839 "ATTR_SW_WIRED", (uintmax_t)pmap_load(l3)); 6840 6841 /* 6842 * ATTR_SW_WIRED must be cleared atomically. Although 6843 * the pmap lock synchronizes access to ATTR_SW_WIRED, 6844 * the System MMU may write to the entry concurrently. 6845 */ 6846 pmap_clear_bits(l3, ATTR_SW_WIRED); 6847 pmap->pm_stats.wired_count--; 6848 } 6849 } 6850 PMAP_UNLOCK(pmap); 6851 } 6852 6853 /* 6854 * This function requires that the caller has already added one to ml3's 6855 * ref_count in anticipation of creating a 4KB page mapping. 6856 */ 6857 static bool 6858 pmap_copy_l3c(pmap_t pmap, pt_entry_t *l3p, vm_offset_t va, pt_entry_t l3e, 6859 vm_page_t ml3, struct rwlock **lockp) 6860 { 6861 pt_entry_t *tl3p; 6862 6863 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 6864 KASSERT((va & L3C_OFFSET) == 0, 6865 ("pmap_copy_l3c: va is not aligned")); 6866 KASSERT((l3e & ATTR_SW_MANAGED) != 0, 6867 ("pmap_copy_l3c: l3e is not managed")); 6868 6869 /* 6870 * Abort if a mapping already exists. 6871 */ 6872 for (tl3p = l3p; tl3p < &l3p[L3C_ENTRIES]; tl3p++) 6873 if (pmap_load(tl3p) != 0) { 6874 if (ml3 != NULL) 6875 ml3->ref_count--; 6876 return (false); 6877 } 6878 6879 if (!pmap_pv_insert_l3c(pmap, va, PTE_TO_VM_PAGE(l3e), lockp)) { 6880 if (ml3 != NULL) 6881 pmap_abort_ptp(pmap, va, ml3); 6882 return (false); 6883 } 6884 ml3->ref_count += L3C_ENTRIES - 1; 6885 6886 /* 6887 * Clear the wired and accessed bits. However, leave the dirty bit 6888 * unchanged because read/write superpage mappings are required to be 6889 * dirty. 6890 */ 6891 l3e &= ~(ATTR_SW_WIRED | ATTR_AF); 6892 6893 for (tl3p = l3p; tl3p < &l3p[L3C_ENTRIES]; tl3p++) { 6894 pmap_store(tl3p, l3e); 6895 l3e += L3_SIZE; 6896 } 6897 pmap_resident_count_inc(pmap, L3C_ENTRIES); 6898 counter_u64_add(pmap_l3c_mappings, 1); 6899 CTR2(KTR_PMAP, "pmap_copy_l3c: success for va %#lx in pmap %p", 6900 va, pmap); 6901 return (true); 6902 } 6903 6904 /* 6905 * Copy the range specified by src_addr/len 6906 * from the source map to the range dst_addr/len 6907 * in the destination map. 6908 * 6909 * This routine is only advisory and need not do anything. 6910 * 6911 * Because the executable mappings created by this routine are copied, 6912 * it should not have to flush the instruction cache. 6913 */ 6914 void 6915 pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, vm_size_t len, 6916 vm_offset_t src_addr) 6917 { 6918 struct rwlock *lock; 6919 pd_entry_t *l0, *l1, *l2, srcptepaddr; 6920 pt_entry_t *dst_pte, mask, nbits, ptetemp, *src_pte; 6921 vm_offset_t addr, end_addr, va_next; 6922 vm_page_t dst_m, dstmpte, srcmpte; 6923 6924 PMAP_ASSERT_STAGE1(dst_pmap); 6925 PMAP_ASSERT_STAGE1(src_pmap); 6926 6927 if (dst_addr != src_addr) 6928 return; 6929 end_addr = src_addr + len; 6930 lock = NULL; 6931 if (dst_pmap < src_pmap) { 6932 PMAP_LOCK(dst_pmap); 6933 PMAP_LOCK(src_pmap); 6934 } else { 6935 PMAP_LOCK(src_pmap); 6936 PMAP_LOCK(dst_pmap); 6937 } 6938 for (addr = src_addr; addr < end_addr; addr = va_next) { 6939 l0 = pmap_l0(src_pmap, addr); 6940 if (pmap_load(l0) == 0) { 6941 va_next = (addr + L0_SIZE) & ~L0_OFFSET; 6942 if (va_next < addr) 6943 va_next = end_addr; 6944 continue; 6945 } 6946 6947 va_next = (addr + L1_SIZE) & ~L1_OFFSET; 6948 if (va_next < addr) 6949 va_next = end_addr; 6950 l1 = pmap_l0_to_l1(l0, addr); 6951 if (pmap_load(l1) == 0) 6952 continue; 6953 if ((pmap_load(l1) & ATTR_DESCR_MASK) == L1_BLOCK) { 6954 PMAP_ASSERT_L1_BLOCKS_SUPPORTED; 6955 KASSERT(va_next <= end_addr, 6956 ("partial update of non-transparent 1G page " 6957 "l1 %#lx addr %#lx end_addr %#lx va_next %#lx", 6958 pmap_load(l1), addr, end_addr, va_next)); 6959 srcptepaddr = pmap_load(l1); 6960 l1 = pmap_l1(dst_pmap, addr); 6961 if (l1 == NULL) { 6962 if (_pmap_alloc_l3(dst_pmap, 6963 pmap_l0_pindex(addr), NULL) == NULL) 6964 break; 6965 l1 = pmap_l1(dst_pmap, addr); 6966 } else { 6967 l0 = pmap_l0(dst_pmap, addr); 6968 dst_m = PTE_TO_VM_PAGE(pmap_load(l0)); 6969 dst_m->ref_count++; 6970 } 6971 KASSERT(pmap_load(l1) == 0, 6972 ("1G mapping present in dst pmap " 6973 "l1 %#lx addr %#lx end_addr %#lx va_next %#lx", 6974 pmap_load(l1), addr, end_addr, va_next)); 6975 pmap_store(l1, srcptepaddr & ~ATTR_SW_WIRED); 6976 pmap_resident_count_inc(dst_pmap, L1_SIZE / PAGE_SIZE); 6977 continue; 6978 } 6979 6980 va_next = (addr + L2_SIZE) & ~L2_OFFSET; 6981 if (va_next < addr) 6982 va_next = end_addr; 6983 l2 = pmap_l1_to_l2(l1, addr); 6984 srcptepaddr = pmap_load(l2); 6985 if (srcptepaddr == 0) 6986 continue; 6987 if ((srcptepaddr & ATTR_DESCR_MASK) == L2_BLOCK) { 6988 /* 6989 * We can only virtual copy whole superpages. 6990 */ 6991 if ((addr & L2_OFFSET) != 0 || 6992 addr + L2_SIZE > end_addr) 6993 continue; 6994 l2 = pmap_alloc_l2(dst_pmap, addr, &dst_m, NULL); 6995 if (l2 == NULL) 6996 break; 6997 if (pmap_load(l2) == 0 && 6998 ((srcptepaddr & ATTR_SW_MANAGED) == 0 || 6999 pmap_pv_insert_l2(dst_pmap, addr, srcptepaddr, 7000 PMAP_ENTER_NORECLAIM, &lock))) { 7001 /* 7002 * We leave the dirty bit unchanged because 7003 * managed read/write superpage mappings are 7004 * required to be dirty. However, managed 7005 * superpage mappings are not required to 7006 * have their accessed bit set, so we clear 7007 * it because we don't know if this mapping 7008 * will be used. 7009 */ 7010 srcptepaddr &= ~ATTR_SW_WIRED; 7011 if ((srcptepaddr & ATTR_SW_MANAGED) != 0) 7012 srcptepaddr &= ~ATTR_AF; 7013 pmap_store(l2, srcptepaddr); 7014 pmap_resident_count_inc(dst_pmap, L2_SIZE / 7015 PAGE_SIZE); 7016 counter_u64_add(pmap_l2_mappings, 1); 7017 } else 7018 pmap_abort_ptp(dst_pmap, addr, dst_m); 7019 continue; 7020 } 7021 KASSERT((srcptepaddr & ATTR_DESCR_MASK) == L2_TABLE, 7022 ("pmap_copy: invalid L2 entry")); 7023 srcmpte = PTE_TO_VM_PAGE(srcptepaddr); 7024 KASSERT(srcmpte->ref_count > 0, 7025 ("pmap_copy: source page table page is unused")); 7026 if (va_next > end_addr) 7027 va_next = end_addr; 7028 src_pte = PHYS_TO_DMAP(PTE_TO_PHYS(srcptepaddr)); 7029 src_pte = &src_pte[pmap_l3_index(addr)]; 7030 dstmpte = NULL; 7031 for (; addr < va_next; addr += PAGE_SIZE, src_pte++) { 7032 ptetemp = pmap_load(src_pte); 7033 7034 /* 7035 * We only virtual copy managed pages. 7036 */ 7037 if ((ptetemp & ATTR_SW_MANAGED) == 0) 7038 continue; 7039 7040 if (dstmpte != NULL) { 7041 KASSERT(dstmpte->pindex == pmap_l2_pindex(addr), 7042 ("dstmpte pindex/addr mismatch")); 7043 dstmpte->ref_count++; 7044 } else if ((dstmpte = pmap_alloc_l3(dst_pmap, addr, 7045 NULL)) == NULL) 7046 goto out; 7047 dst_pte = VM_PAGE_TO_DMAP(dstmpte); 7048 dst_pte = &dst_pte[pmap_l3_index(addr)]; 7049 if ((ptetemp & ATTR_CONTIGUOUS) != 0 && (addr & 7050 L3C_OFFSET) == 0 && addr + L3C_OFFSET <= 7051 va_next - 1) { 7052 if (!pmap_copy_l3c(dst_pmap, dst_pte, addr, 7053 ptetemp, dstmpte, &lock)) 7054 goto out; 7055 addr += L3C_SIZE - PAGE_SIZE; 7056 src_pte += L3C_ENTRIES - 1; 7057 } else if (pmap_load(dst_pte) == 0 && 7058 pmap_try_insert_pv_entry(dst_pmap, addr, 7059 PTE_TO_VM_PAGE(ptetemp), &lock)) { 7060 /* 7061 * Clear the wired, contiguous, modified, and 7062 * accessed bits from the destination PTE. 7063 * The contiguous bit is cleared because we 7064 * are not copying the entire L3C superpage. 7065 */ 7066 mask = ATTR_SW_WIRED | ATTR_CONTIGUOUS | 7067 ATTR_AF; 7068 nbits = 0; 7069 if ((ptetemp & ATTR_SW_DBM) != 0) 7070 nbits |= ATTR_S1_AP_RW_BIT; 7071 pmap_store(dst_pte, (ptetemp & ~mask) | nbits); 7072 pmap_resident_count_inc(dst_pmap, 1); 7073 } else { 7074 pmap_abort_ptp(dst_pmap, addr, dstmpte); 7075 goto out; 7076 } 7077 /* Have we copied all of the valid mappings? */ 7078 if (dstmpte->ref_count >= srcmpte->ref_count) 7079 break; 7080 } 7081 } 7082 out: 7083 /* 7084 * XXX This barrier may not be needed because the destination pmap is 7085 * not active. 7086 */ 7087 dsb(ishst); 7088 7089 if (lock != NULL) 7090 rw_wunlock(lock); 7091 PMAP_UNLOCK(src_pmap); 7092 PMAP_UNLOCK(dst_pmap); 7093 } 7094 7095 int 7096 pmap_vmspace_copy(pmap_t dst_pmap, pmap_t src_pmap) 7097 { 7098 int error; 7099 7100 if (dst_pmap->pm_stage != src_pmap->pm_stage) 7101 return (EINVAL); 7102 7103 if (dst_pmap->pm_stage != PM_STAGE1 || src_pmap->pm_bti == NULL) 7104 return (0); 7105 7106 for (;;) { 7107 if (dst_pmap < src_pmap) { 7108 PMAP_LOCK(dst_pmap); 7109 PMAP_LOCK(src_pmap); 7110 } else { 7111 PMAP_LOCK(src_pmap); 7112 PMAP_LOCK(dst_pmap); 7113 } 7114 error = pmap_bti_copy(dst_pmap, src_pmap); 7115 /* Clean up partial copy on failure due to no memory. */ 7116 if (error == ENOMEM) 7117 pmap_bti_deassign_all(dst_pmap); 7118 PMAP_UNLOCK(src_pmap); 7119 PMAP_UNLOCK(dst_pmap); 7120 if (error != ENOMEM) 7121 break; 7122 vm_wait(NULL); 7123 } 7124 return (error); 7125 } 7126 7127 /* 7128 * pmap_zero_page zeros the specified hardware page by mapping 7129 * the page into KVM and using bzero to clear its contents. 7130 */ 7131 void 7132 pmap_zero_page(vm_page_t m) 7133 { 7134 void *va = VM_PAGE_TO_DMAP(m); 7135 7136 pagezero(va); 7137 m->md.pv_flags &= ~PV_MTE_TAGGED; 7138 } 7139 7140 /* 7141 * pmap_zero_page_area zeros the specified hardware page by mapping 7142 * the page into KVM and using bzero to clear its contents. 7143 * 7144 * off and size may not cover an area beyond a single hardware page. 7145 */ 7146 void 7147 pmap_zero_page_area(vm_page_t m, int off, int size) 7148 { 7149 void *va = VM_PAGE_TO_DMAP(m); 7150 7151 if (off == 0 && size == PAGE_SIZE) 7152 pagezero(va); 7153 else 7154 bzero((char *)va + off, size); 7155 } 7156 7157 /* 7158 * pmap_copy_page copies the specified (machine independent) 7159 * page by mapping the page into virtual memory and using 7160 * memcpy to copy the page, one machine dependent page at a 7161 * time. 7162 */ 7163 void 7164 pmap_copy_page(vm_page_t msrc, vm_page_t mdst) 7165 { 7166 void *src = VM_PAGE_TO_DMAP(msrc); 7167 void *dst = VM_PAGE_TO_DMAP(mdst); 7168 7169 /* 7170 * On a page copy, check whether the src page is tagged. If it is, 7171 * we must copy the tags before copying the contents of the page. 7172 */ 7173 if ((msrc->md.pv_flags & PV_MTE_TAGGED) != 0) 7174 mte_copy_tags(msrc, mdst, src, dst); 7175 else 7176 mdst->md.pv_flags &= ~PV_MTE_TAGGED; 7177 7178 pagecopy(src, dst); 7179 } 7180 7181 int unmapped_buf_allowed = 1; 7182 7183 void 7184 pmap_copy_pages(vm_page_t ma[], vm_offset_t a_offset, vm_page_t mb[], 7185 vm_offset_t b_offset, int xfersize) 7186 { 7187 void *a_cp, *b_cp; 7188 vm_page_t m_a, m_b; 7189 vm_paddr_t p_a, p_b; 7190 vm_offset_t a_pg_offset, b_pg_offset; 7191 int cnt; 7192 7193 while (xfersize > 0) { 7194 KASSERT(ADDR_IS_CANONICAL(a_offset), 7195 ("%s: Address not in canonical form: %lx", __func__, a_offset)); 7196 7197 a_pg_offset = a_offset & PAGE_MASK; 7198 m_a = ma[a_offset >> PAGE_SHIFT]; 7199 p_a = m_a->phys_addr; 7200 b_pg_offset = b_offset & PAGE_MASK; 7201 m_b = mb[b_offset >> PAGE_SHIFT]; 7202 p_b = m_b->phys_addr; 7203 cnt = min(xfersize, PAGE_SIZE - a_pg_offset); 7204 cnt = min(cnt, PAGE_SIZE - b_pg_offset); 7205 if (__predict_false(!PHYS_IN_DMAP(p_a))) { 7206 panic("!DMAP a %lx", p_a); 7207 } else { 7208 a_cp = (char *)PHYS_TO_DMAP(p_a) + a_pg_offset; 7209 } 7210 if (__predict_false(!PHYS_IN_DMAP(p_b))) { 7211 panic("!DMAP b %lx", p_b); 7212 } else { 7213 b_cp = (char *)PHYS_TO_DMAP(p_b) + b_pg_offset; 7214 } 7215 memcpy(b_cp, a_cp, cnt); 7216 a_offset += cnt; 7217 b_offset += cnt; 7218 xfersize -= cnt; 7219 } 7220 } 7221 7222 void * 7223 pmap_quick_enter_page(vm_page_t m) 7224 { 7225 7226 return (VM_PAGE_TO_DMAP(m)); 7227 } 7228 7229 void 7230 pmap_quick_remove_page(void *addr) 7231 { 7232 } 7233 7234 /* 7235 * Returns true if the pmap's pv is one of the first 7236 * 16 pvs linked to from this page. This count may 7237 * be changed upwards or downwards in the future; it 7238 * is only necessary that true be returned for a small 7239 * subset of pmaps for proper page aging. 7240 */ 7241 bool 7242 pmap_page_exists_quick(pmap_t pmap, vm_page_t m) 7243 { 7244 struct md_page *pvh; 7245 struct rwlock *lock; 7246 pv_entry_t pv; 7247 int loops = 0; 7248 bool rv; 7249 7250 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 7251 ("pmap_page_exists_quick: page %p is not managed", m)); 7252 rv = false; 7253 lock = VM_PAGE_TO_PV_LIST_LOCK(m); 7254 rw_rlock(lock); 7255 TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) { 7256 if (PV_PMAP(pv) == pmap) { 7257 rv = true; 7258 break; 7259 } 7260 loops++; 7261 if (loops >= 16) 7262 break; 7263 } 7264 if (!rv && loops < 16 && (m->flags & PG_FICTITIOUS) == 0) { 7265 pvh = page_to_pvh(m); 7266 TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) { 7267 if (PV_PMAP(pv) == pmap) { 7268 rv = true; 7269 break; 7270 } 7271 loops++; 7272 if (loops >= 16) 7273 break; 7274 } 7275 } 7276 rw_runlock(lock); 7277 return (rv); 7278 } 7279 7280 /* 7281 * pmap_page_wired_mappings: 7282 * 7283 * Return the number of managed mappings to the given physical page 7284 * that are wired. 7285 */ 7286 int 7287 pmap_page_wired_mappings(vm_page_t m) 7288 { 7289 struct rwlock *lock; 7290 struct md_page *pvh; 7291 pmap_t pmap; 7292 pt_entry_t *pte; 7293 pv_entry_t pv; 7294 int count, md_gen, pvh_gen; 7295 7296 if ((m->oflags & VPO_UNMANAGED) != 0) 7297 return (0); 7298 lock = VM_PAGE_TO_PV_LIST_LOCK(m); 7299 rw_rlock(lock); 7300 restart: 7301 count = 0; 7302 TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) { 7303 pmap = PV_PMAP(pv); 7304 if (!PMAP_TRYLOCK(pmap)) { 7305 md_gen = m->md.pv_gen; 7306 rw_runlock(lock); 7307 PMAP_LOCK(pmap); 7308 rw_rlock(lock); 7309 if (md_gen != m->md.pv_gen) { 7310 PMAP_UNLOCK(pmap); 7311 goto restart; 7312 } 7313 } 7314 pte = pmap_pte_exists(pmap, pv->pv_va, 3, __func__); 7315 if ((pmap_load(pte) & ATTR_SW_WIRED) != 0) 7316 count++; 7317 PMAP_UNLOCK(pmap); 7318 } 7319 if ((m->flags & PG_FICTITIOUS) == 0) { 7320 pvh = page_to_pvh(m); 7321 TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) { 7322 pmap = PV_PMAP(pv); 7323 if (!PMAP_TRYLOCK(pmap)) { 7324 md_gen = m->md.pv_gen; 7325 pvh_gen = pvh->pv_gen; 7326 rw_runlock(lock); 7327 PMAP_LOCK(pmap); 7328 rw_rlock(lock); 7329 if (md_gen != m->md.pv_gen || 7330 pvh_gen != pvh->pv_gen) { 7331 PMAP_UNLOCK(pmap); 7332 goto restart; 7333 } 7334 } 7335 pte = pmap_pte_exists(pmap, pv->pv_va, 2, __func__); 7336 if ((pmap_load(pte) & ATTR_SW_WIRED) != 0) 7337 count++; 7338 PMAP_UNLOCK(pmap); 7339 } 7340 } 7341 rw_runlock(lock); 7342 return (count); 7343 } 7344 7345 /* 7346 * Returns true if the given page is mapped individually or as part of 7347 * a 2mpage. Otherwise, returns false. 7348 */ 7349 bool 7350 pmap_page_is_mapped(vm_page_t m) 7351 { 7352 struct rwlock *lock; 7353 bool rv; 7354 7355 if ((m->oflags & VPO_UNMANAGED) != 0) 7356 return (false); 7357 lock = VM_PAGE_TO_PV_LIST_LOCK(m); 7358 rw_rlock(lock); 7359 rv = pmap_page_is_mapped_locked(m); 7360 rw_runlock(lock); 7361 return (rv); 7362 } 7363 7364 /* 7365 * The page's PV list lock must be held. 7366 */ 7367 static __always_inline bool 7368 pmap_page_is_mapped_locked(vm_page_t m) 7369 { 7370 return (!TAILQ_EMPTY(&m->md.pv_list) || 7371 ((m->flags & PG_FICTITIOUS) == 0 && 7372 !TAILQ_EMPTY(&page_to_pvh(m)->pv_list))); 7373 } 7374 7375 /* 7376 * Destroy all managed, non-wired mappings in the given user-space 7377 * pmap. This pmap cannot be active on any processor besides the 7378 * caller. 7379 * 7380 * This function cannot be applied to the kernel pmap. Moreover, it 7381 * is not intended for general use. It is only to be used during 7382 * process termination. Consequently, it can be implemented in ways 7383 * that make it faster than pmap_remove(). First, it can more quickly 7384 * destroy mappings by iterating over the pmap's collection of PV 7385 * entries, rather than searching the page table. Second, it doesn't 7386 * have to test and clear the page table entries atomically, because 7387 * no processor is currently accessing the user address space. In 7388 * particular, a page table entry's dirty bit won't change state once 7389 * this function starts. 7390 */ 7391 void 7392 pmap_remove_pages(pmap_t pmap) 7393 { 7394 pd_entry_t *pde; 7395 pt_entry_t *pte, tpte; 7396 struct spglist free; 7397 struct pv_chunklist free_chunks[PMAP_MEMDOM]; 7398 vm_page_t m, ml3, mt; 7399 pv_entry_t pv; 7400 struct md_page *pvh; 7401 struct pv_chunk *pc, *npc; 7402 struct rwlock *lock; 7403 int64_t bit; 7404 uint64_t inuse, bitmask; 7405 int allfree, field, i, idx, lvl; 7406 int freed __pvused; 7407 vm_paddr_t pa; 7408 7409 lock = NULL; 7410 7411 for (i = 0; i < PMAP_MEMDOM; i++) 7412 TAILQ_INIT(&free_chunks[i]); 7413 SLIST_INIT(&free); 7414 PMAP_LOCK(pmap); 7415 TAILQ_FOREACH_SAFE(pc, &pmap->pm_pvchunk, pc_list, npc) { 7416 allfree = 1; 7417 freed = 0; 7418 for (field = 0; field < _NPCM; field++) { 7419 inuse = ~pc->pc_map[field] & pc_freemask[field]; 7420 while (inuse != 0) { 7421 bit = ffsl(inuse) - 1; 7422 bitmask = 1UL << bit; 7423 idx = field * 64 + bit; 7424 pv = &pc->pc_pventry[idx]; 7425 inuse &= ~bitmask; 7426 7427 pde = pmap_pde(pmap, pv->pv_va, &lvl); 7428 KASSERT(pde != NULL, 7429 ("Attempting to remove an unmapped page")); 7430 7431 switch(lvl) { 7432 case 1: 7433 pte = pmap_l1_to_l2(pde, pv->pv_va); 7434 tpte = pmap_load(pte); 7435 KASSERT((tpte & ATTR_DESCR_MASK) == 7436 L2_BLOCK, 7437 ("Attempting to remove an invalid " 7438 "block: %lx", tpte)); 7439 break; 7440 case 2: 7441 pte = pmap_l2_to_l3(pde, pv->pv_va); 7442 tpte = pmap_load(pte); 7443 KASSERT((tpte & ATTR_DESCR_MASK) == 7444 L3_PAGE, 7445 ("Attempting to remove an invalid " 7446 "page: %lx", tpte)); 7447 break; 7448 default: 7449 panic( 7450 "Invalid page directory level: %d", 7451 lvl); 7452 } 7453 7454 /* 7455 * We cannot remove wired mappings at this time. 7456 * 7457 * For L3C superpages, all of the constituent PTEs 7458 * should have the wired bit set, so we don't 7459 * check for ATTR_CONTIGUOUS here. 7460 */ 7461 if (tpte & ATTR_SW_WIRED) { 7462 allfree = 0; 7463 continue; 7464 } 7465 7466 /* Mark free */ 7467 pc->pc_map[field] |= bitmask; 7468 7469 /* 7470 * Because this pmap is not active on other 7471 * processors, the dirty bit cannot have 7472 * changed state since we last loaded pte. 7473 */ 7474 pmap_clear(pte); 7475 7476 pa = PTE_TO_PHYS(tpte); 7477 7478 m = PHYS_TO_VM_PAGE(pa); 7479 KASSERT(m->phys_addr == pa, 7480 ("vm_page_t %p phys_addr mismatch %016jx %016jx", 7481 m, (uintmax_t)m->phys_addr, 7482 (uintmax_t)tpte)); 7483 7484 KASSERT((m->flags & PG_FICTITIOUS) != 0 || 7485 m < &vm_page_array[vm_page_array_size], 7486 ("pmap_remove_pages: bad pte %#jx", 7487 (uintmax_t)tpte)); 7488 7489 /* 7490 * Update the vm_page_t clean/reference bits. 7491 * 7492 * We don't check for ATTR_CONTIGUOUS here 7493 * because writeable L3C superpages are expected 7494 * to be dirty, i.e., every constituent PTE 7495 * should be dirty. 7496 */ 7497 if (pmap_pte_dirty(pmap, tpte)) { 7498 switch (lvl) { 7499 case 1: 7500 for (mt = m; mt < &m[L2_SIZE / PAGE_SIZE]; mt++) 7501 vm_page_dirty(mt); 7502 break; 7503 case 2: 7504 vm_page_dirty(m); 7505 break; 7506 } 7507 } 7508 7509 CHANGE_PV_LIST_LOCK_TO_VM_PAGE(&lock, m); 7510 7511 switch (lvl) { 7512 case 1: 7513 pmap_resident_count_dec(pmap, 7514 L2_SIZE / PAGE_SIZE); 7515 pvh = page_to_pvh(m); 7516 TAILQ_REMOVE(&pvh->pv_list, pv,pv_next); 7517 pvh->pv_gen++; 7518 if (TAILQ_EMPTY(&pvh->pv_list)) { 7519 for (mt = m; mt < &m[L2_SIZE / PAGE_SIZE]; mt++) 7520 if ((mt->a.flags & PGA_WRITEABLE) != 0 && 7521 TAILQ_EMPTY(&mt->md.pv_list)) 7522 vm_page_aflag_clear(mt, PGA_WRITEABLE); 7523 } 7524 ml3 = pmap_remove_pt_page(pmap, 7525 pv->pv_va); 7526 if (ml3 != NULL) { 7527 KASSERT(vm_page_any_valid(ml3), 7528 ("pmap_remove_pages: l3 page not promoted")); 7529 pmap_resident_count_dec(pmap,1); 7530 KASSERT(ml3->ref_count == NL3PG, 7531 ("pmap_remove_pages: l3 page ref count error")); 7532 ml3->ref_count = 0; 7533 pmap_add_delayed_free_list(ml3, 7534 &free, false); 7535 } 7536 break; 7537 case 2: 7538 pmap_resident_count_dec(pmap, 1); 7539 TAILQ_REMOVE(&m->md.pv_list, pv, 7540 pv_next); 7541 m->md.pv_gen++; 7542 if ((m->a.flags & PGA_WRITEABLE) != 0 && 7543 !pmap_page_is_mapped_locked(m)) 7544 vm_page_aflag_clear(m, 7545 PGA_WRITEABLE); 7546 break; 7547 } 7548 pmap_unuse_pt(pmap, pv->pv_va, pmap_load(pde), 7549 &free); 7550 freed++; 7551 } 7552 } 7553 PV_STAT(atomic_add_long(&pv_entry_frees, freed)); 7554 PV_STAT(atomic_add_int(&pv_entry_spare, freed)); 7555 PV_STAT(atomic_subtract_long(&pv_entry_count, freed)); 7556 if (allfree) { 7557 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 7558 TAILQ_INSERT_TAIL(&free_chunks[pc_to_domain(pc)], pc, 7559 pc_list); 7560 } 7561 } 7562 if (lock != NULL) 7563 rw_wunlock(lock); 7564 pmap_invalidate_all(pmap); 7565 pmap_bti_deassign_all(pmap); 7566 free_pv_chunk_batch(free_chunks); 7567 PMAP_UNLOCK(pmap); 7568 vm_page_free_pages_toq(&free, true); 7569 } 7570 7571 /* 7572 * This is used to check if a page has been accessed or modified. 7573 */ 7574 static bool 7575 pmap_page_test_mappings(vm_page_t m, bool accessed, bool modified) 7576 { 7577 struct rwlock *lock; 7578 pv_entry_t pv; 7579 struct md_page *pvh; 7580 pt_entry_t l3e, mask, *pte, value; 7581 pmap_t pmap; 7582 int md_gen, pvh_gen; 7583 bool rv; 7584 7585 rv = false; 7586 lock = VM_PAGE_TO_PV_LIST_LOCK(m); 7587 rw_rlock(lock); 7588 restart: 7589 TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) { 7590 pmap = PV_PMAP(pv); 7591 PMAP_ASSERT_STAGE1(pmap); 7592 if (!PMAP_TRYLOCK(pmap)) { 7593 md_gen = m->md.pv_gen; 7594 rw_runlock(lock); 7595 PMAP_LOCK(pmap); 7596 rw_rlock(lock); 7597 if (md_gen != m->md.pv_gen) { 7598 PMAP_UNLOCK(pmap); 7599 goto restart; 7600 } 7601 } 7602 pte = pmap_pte_exists(pmap, pv->pv_va, 3, __func__); 7603 mask = 0; 7604 value = 0; 7605 if (modified) { 7606 mask |= ATTR_S1_AP_RW_BIT; 7607 value |= ATTR_S1_AP(ATTR_S1_AP_RW); 7608 } 7609 if (accessed) { 7610 mask |= ATTR_AF | ATTR_DESCR_MASK; 7611 value |= ATTR_AF | L3_PAGE; 7612 } 7613 l3e = pmap_load(pte); 7614 if ((l3e & ATTR_CONTIGUOUS) != 0) 7615 l3e = pmap_load_l3c(pte); 7616 PMAP_UNLOCK(pmap); 7617 rv = (l3e & mask) == value; 7618 if (rv) 7619 goto out; 7620 } 7621 if ((m->flags & PG_FICTITIOUS) == 0) { 7622 pvh = page_to_pvh(m); 7623 TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) { 7624 pmap = PV_PMAP(pv); 7625 PMAP_ASSERT_STAGE1(pmap); 7626 if (!PMAP_TRYLOCK(pmap)) { 7627 md_gen = m->md.pv_gen; 7628 pvh_gen = pvh->pv_gen; 7629 rw_runlock(lock); 7630 PMAP_LOCK(pmap); 7631 rw_rlock(lock); 7632 if (md_gen != m->md.pv_gen || 7633 pvh_gen != pvh->pv_gen) { 7634 PMAP_UNLOCK(pmap); 7635 goto restart; 7636 } 7637 } 7638 pte = pmap_pte_exists(pmap, pv->pv_va, 2, __func__); 7639 mask = 0; 7640 value = 0; 7641 if (modified) { 7642 mask |= ATTR_S1_AP_RW_BIT; 7643 value |= ATTR_S1_AP(ATTR_S1_AP_RW); 7644 } 7645 if (accessed) { 7646 mask |= ATTR_AF | ATTR_DESCR_MASK; 7647 value |= ATTR_AF | L2_BLOCK; 7648 } 7649 rv = (pmap_load(pte) & mask) == value; 7650 PMAP_UNLOCK(pmap); 7651 if (rv) 7652 goto out; 7653 } 7654 } 7655 out: 7656 rw_runlock(lock); 7657 return (rv); 7658 } 7659 7660 /* 7661 * pmap_is_modified: 7662 * 7663 * Return whether or not the specified physical page was modified 7664 * in any physical maps. 7665 */ 7666 bool 7667 pmap_is_modified(vm_page_t m) 7668 { 7669 7670 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 7671 ("pmap_is_modified: page %p is not managed", m)); 7672 7673 /* 7674 * If the page is not busied then this check is racy. 7675 */ 7676 if (!pmap_page_is_write_mapped(m)) 7677 return (false); 7678 return (pmap_page_test_mappings(m, false, true)); 7679 } 7680 7681 /* 7682 * pmap_is_prefaultable: 7683 * 7684 * Return whether or not the specified virtual address is eligible 7685 * for prefault. 7686 */ 7687 bool 7688 pmap_is_prefaultable(pmap_t pmap, vm_offset_t addr) 7689 { 7690 pd_entry_t *pde; 7691 pt_entry_t *pte; 7692 bool rv; 7693 int lvl; 7694 7695 /* 7696 * Return true if and only if the L3 entry for the specified virtual 7697 * address is allocated but invalid. 7698 */ 7699 rv = false; 7700 PMAP_LOCK(pmap); 7701 pde = pmap_pde(pmap, addr, &lvl); 7702 if (pde != NULL && lvl == 2) { 7703 pte = pmap_l2_to_l3(pde, addr); 7704 rv = pmap_load(pte) == 0; 7705 } 7706 PMAP_UNLOCK(pmap); 7707 return (rv); 7708 } 7709 7710 /* 7711 * pmap_is_referenced: 7712 * 7713 * Return whether or not the specified physical page was referenced 7714 * in any physical maps. 7715 */ 7716 bool 7717 pmap_is_referenced(vm_page_t m) 7718 { 7719 7720 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 7721 ("pmap_is_referenced: page %p is not managed", m)); 7722 return (pmap_page_test_mappings(m, true, false)); 7723 } 7724 7725 /* 7726 * Clear the write and modified bits in each of the given page's mappings. 7727 */ 7728 void 7729 pmap_remove_write(vm_page_t m) 7730 { 7731 struct md_page *pvh; 7732 pmap_t pmap; 7733 struct rwlock *lock; 7734 pv_entry_t next_pv, pv; 7735 pt_entry_t oldpte, *pte, set, clear, mask, val; 7736 vm_offset_t va; 7737 int md_gen, pvh_gen; 7738 7739 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 7740 ("pmap_remove_write: page %p is not managed", m)); 7741 vm_page_assert_busied(m); 7742 7743 if (!pmap_page_is_write_mapped(m)) 7744 return; 7745 lock = VM_PAGE_TO_PV_LIST_LOCK(m); 7746 pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy : page_to_pvh(m); 7747 rw_wlock(lock); 7748 retry: 7749 TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_next, next_pv) { 7750 pmap = PV_PMAP(pv); 7751 PMAP_ASSERT_STAGE1(pmap); 7752 if (!PMAP_TRYLOCK(pmap)) { 7753 pvh_gen = pvh->pv_gen; 7754 rw_wunlock(lock); 7755 PMAP_LOCK(pmap); 7756 rw_wlock(lock); 7757 if (pvh_gen != pvh->pv_gen) { 7758 PMAP_UNLOCK(pmap); 7759 goto retry; 7760 } 7761 } 7762 va = pv->pv_va; 7763 pte = pmap_pte_exists(pmap, va, 2, __func__); 7764 if ((pmap_load(pte) & ATTR_SW_DBM) != 0) 7765 (void)pmap_demote_l2_locked(pmap, pte, va, &lock); 7766 KASSERT(lock == VM_PAGE_TO_PV_LIST_LOCK(m), 7767 ("inconsistent pv lock %p %p for page %p", 7768 lock, VM_PAGE_TO_PV_LIST_LOCK(m), m)); 7769 PMAP_UNLOCK(pmap); 7770 } 7771 TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) { 7772 pmap = PV_PMAP(pv); 7773 if (!PMAP_TRYLOCK(pmap)) { 7774 pvh_gen = pvh->pv_gen; 7775 md_gen = m->md.pv_gen; 7776 rw_wunlock(lock); 7777 PMAP_LOCK(pmap); 7778 rw_wlock(lock); 7779 if (pvh_gen != pvh->pv_gen || 7780 md_gen != m->md.pv_gen) { 7781 PMAP_UNLOCK(pmap); 7782 goto retry; 7783 } 7784 } 7785 pte = pmap_pte_exists(pmap, pv->pv_va, 3, __func__); 7786 oldpte = pmap_load(pte); 7787 if ((oldpte & ATTR_SW_DBM) != 0) { 7788 if ((oldpte & ATTR_CONTIGUOUS) != 0) { 7789 (void)pmap_demote_l3c(pmap, pte, pv->pv_va); 7790 7791 /* 7792 * The L3 entry's accessed bit may have 7793 * changed. 7794 */ 7795 oldpte = pmap_load(pte); 7796 } 7797 if (pmap->pm_stage == PM_STAGE1) { 7798 set = ATTR_S1_AP_RW_BIT; 7799 clear = 0; 7800 mask = ATTR_S1_AP_RW_BIT; 7801 val = ATTR_S1_AP(ATTR_S1_AP_RW); 7802 } else { 7803 set = 0; 7804 clear = ATTR_S2_S2AP(ATTR_S2_S2AP_WRITE); 7805 mask = ATTR_S2_S2AP(ATTR_S2_S2AP_WRITE); 7806 val = ATTR_S2_S2AP(ATTR_S2_S2AP_WRITE); 7807 } 7808 clear |= ATTR_SW_DBM; 7809 while (!atomic_fcmpset_64(pte, &oldpte, 7810 (oldpte | set) & ~clear)) 7811 cpu_spinwait(); 7812 7813 if ((oldpte & mask) == val) 7814 vm_page_dirty(m); 7815 pmap_invalidate_page(pmap, pv->pv_va, true); 7816 } 7817 PMAP_UNLOCK(pmap); 7818 } 7819 rw_wunlock(lock); 7820 vm_page_aflag_clear(m, PGA_WRITEABLE); 7821 } 7822 7823 /* 7824 * pmap_ts_referenced: 7825 * 7826 * Return a count of reference bits for a page, clearing those bits. 7827 * It is not necessary for every reference bit to be cleared, but it 7828 * is necessary that 0 only be returned when there are truly no 7829 * reference bits set. 7830 * 7831 * As an optimization, update the page's dirty field if a modified bit is 7832 * found while counting reference bits. This opportunistic update can be 7833 * performed at low cost and can eliminate the need for some future calls 7834 * to pmap_is_modified(). However, since this function stops after 7835 * finding PMAP_TS_REFERENCED_MAX reference bits, it may not detect some 7836 * dirty pages. Those dirty pages will only be detected by a future call 7837 * to pmap_is_modified(). 7838 */ 7839 int 7840 pmap_ts_referenced(vm_page_t m) 7841 { 7842 struct md_page *pvh; 7843 pv_entry_t pv, pvf; 7844 pmap_t pmap; 7845 struct rwlock *lock; 7846 pt_entry_t *pte, tpte; 7847 vm_offset_t va; 7848 vm_paddr_t pa; 7849 int cleared, md_gen, not_cleared, pvh_gen; 7850 struct spglist free; 7851 7852 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 7853 ("pmap_ts_referenced: page %p is not managed", m)); 7854 SLIST_INIT(&free); 7855 cleared = 0; 7856 pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy : page_to_pvh(m); 7857 lock = VM_PAGE_TO_PV_LIST_LOCK(m); 7858 rw_wlock(lock); 7859 retry: 7860 not_cleared = 0; 7861 if ((pvf = TAILQ_FIRST(&pvh->pv_list)) == NULL) 7862 goto small_mappings; 7863 pv = pvf; 7864 do { 7865 if (pvf == NULL) 7866 pvf = pv; 7867 pmap = PV_PMAP(pv); 7868 if (!PMAP_TRYLOCK(pmap)) { 7869 pvh_gen = pvh->pv_gen; 7870 rw_wunlock(lock); 7871 PMAP_LOCK(pmap); 7872 rw_wlock(lock); 7873 if (pvh_gen != pvh->pv_gen) { 7874 PMAP_UNLOCK(pmap); 7875 goto retry; 7876 } 7877 } 7878 va = pv->pv_va; 7879 pte = pmap_pte_exists(pmap, va, 2, __func__); 7880 tpte = pmap_load(pte); 7881 if (pmap_pte_dirty(pmap, tpte)) { 7882 /* 7883 * Although "tpte" is mapping a 2MB page, because 7884 * this function is called at a 4KB page granularity, 7885 * we only update the 4KB page under test. 7886 */ 7887 vm_page_dirty(m); 7888 } 7889 if ((tpte & ATTR_AF) != 0) { 7890 pa = VM_PAGE_TO_PHYS(m); 7891 7892 /* 7893 * Since this reference bit is shared by 512 4KB pages, 7894 * it should not be cleared every time it is tested. 7895 * Apply a simple "hash" function on the physical page 7896 * number, the virtual superpage number, and the pmap 7897 * address to select one 4KB page out of the 512 on 7898 * which testing the reference bit will result in 7899 * clearing that reference bit. This function is 7900 * designed to avoid the selection of the same 4KB page 7901 * for every 2MB page mapping. 7902 * 7903 * On demotion, a mapping that hasn't been referenced 7904 * is simply destroyed. To avoid the possibility of a 7905 * subsequent page fault on a demoted wired mapping, 7906 * always leave its reference bit set. Moreover, 7907 * since the superpage is wired, the current state of 7908 * its reference bit won't affect page replacement. 7909 */ 7910 if ((((pa >> PAGE_SHIFT) ^ (va >> L2_SHIFT) ^ 7911 (uintptr_t)pmap) & (Ln_ENTRIES - 1)) == 0 && 7912 (tpte & ATTR_SW_WIRED) == 0) { 7913 pmap_clear_bits(pte, ATTR_AF); 7914 pmap_invalidate_page(pmap, va, true); 7915 cleared++; 7916 } else 7917 not_cleared++; 7918 } 7919 PMAP_UNLOCK(pmap); 7920 /* Rotate the PV list if it has more than one entry. */ 7921 if (TAILQ_NEXT(pv, pv_next) != NULL) { 7922 TAILQ_REMOVE(&pvh->pv_list, pv, pv_next); 7923 TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_next); 7924 pvh->pv_gen++; 7925 } 7926 if (cleared + not_cleared >= PMAP_TS_REFERENCED_MAX) 7927 goto out; 7928 } while ((pv = TAILQ_FIRST(&pvh->pv_list)) != pvf); 7929 small_mappings: 7930 if ((pvf = TAILQ_FIRST(&m->md.pv_list)) == NULL) 7931 goto out; 7932 pv = pvf; 7933 do { 7934 if (pvf == NULL) 7935 pvf = pv; 7936 pmap = PV_PMAP(pv); 7937 if (!PMAP_TRYLOCK(pmap)) { 7938 pvh_gen = pvh->pv_gen; 7939 md_gen = m->md.pv_gen; 7940 rw_wunlock(lock); 7941 PMAP_LOCK(pmap); 7942 rw_wlock(lock); 7943 if (pvh_gen != pvh->pv_gen || md_gen != m->md.pv_gen) { 7944 PMAP_UNLOCK(pmap); 7945 goto retry; 7946 } 7947 } 7948 pte = pmap_pte_exists(pmap, pv->pv_va, 3, __func__); 7949 tpte = pmap_load(pte); 7950 if (pmap_pte_dirty(pmap, tpte)) 7951 vm_page_dirty(m); 7952 if ((tpte & ATTR_AF) != 0) { 7953 if ((tpte & ATTR_SW_WIRED) == 0) { 7954 /* 7955 * Clear the accessed bit in this L3 entry 7956 * regardless of the contiguous bit. 7957 */ 7958 pmap_clear_bits(pte, ATTR_AF); 7959 pmap_invalidate_page(pmap, pv->pv_va, true); 7960 cleared++; 7961 } else 7962 not_cleared++; 7963 } else if ((tpte & ATTR_CONTIGUOUS) != 0 && 7964 (pmap_load_l3c(pte) & ATTR_AF) != 0) { 7965 /* 7966 * An L3C superpage mapping is regarded as accessed 7967 * until the accessed bit has been cleared in all 7968 * of its constituent entries. 7969 */ 7970 not_cleared++; 7971 } 7972 PMAP_UNLOCK(pmap); 7973 /* Rotate the PV list if it has more than one entry. */ 7974 if (TAILQ_NEXT(pv, pv_next) != NULL) { 7975 TAILQ_REMOVE(&m->md.pv_list, pv, pv_next); 7976 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next); 7977 m->md.pv_gen++; 7978 } 7979 } while ((pv = TAILQ_FIRST(&m->md.pv_list)) != pvf && cleared + 7980 not_cleared < PMAP_TS_REFERENCED_MAX); 7981 out: 7982 rw_wunlock(lock); 7983 vm_page_free_pages_toq(&free, true); 7984 return (cleared + not_cleared); 7985 } 7986 7987 /* 7988 * Apply the given advice to the specified range of addresses within the 7989 * given pmap. Depending on the advice, clear the referenced and/or 7990 * modified flags in each mapping and set the mapped page's dirty field. 7991 */ 7992 void 7993 pmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int advice) 7994 { 7995 struct rwlock *lock; 7996 vm_offset_t va, va_next, dva; 7997 vm_page_t m; 7998 pd_entry_t *l0, *l1, *l2, oldl2; 7999 pt_entry_t *l3, *dl3, oldl3; 8000 8001 PMAP_ASSERT_STAGE1(pmap); 8002 8003 if (advice != MADV_DONTNEED && advice != MADV_FREE) 8004 return; 8005 8006 PMAP_LOCK(pmap); 8007 for (; sva < eva; sva = va_next) { 8008 l0 = pmap_l0(pmap, sva); 8009 if (pmap_load(l0) == 0) { 8010 va_next = (sva + L0_SIZE) & ~L0_OFFSET; 8011 if (va_next < sva) 8012 va_next = eva; 8013 continue; 8014 } 8015 8016 va_next = (sva + L1_SIZE) & ~L1_OFFSET; 8017 if (va_next < sva) 8018 va_next = eva; 8019 l1 = pmap_l0_to_l1(l0, sva); 8020 if (pmap_load(l1) == 0) 8021 continue; 8022 if ((pmap_load(l1) & ATTR_DESCR_MASK) == L1_BLOCK) { 8023 PMAP_ASSERT_L1_BLOCKS_SUPPORTED; 8024 continue; 8025 } 8026 8027 va_next = (sva + L2_SIZE) & ~L2_OFFSET; 8028 if (va_next < sva) 8029 va_next = eva; 8030 l2 = pmap_l1_to_l2(l1, sva); 8031 oldl2 = pmap_load(l2); 8032 if (oldl2 == 0) 8033 continue; 8034 if ((oldl2 & ATTR_DESCR_MASK) == L2_BLOCK) { 8035 if ((oldl2 & ATTR_SW_MANAGED) == 0) 8036 continue; 8037 lock = NULL; 8038 if (!pmap_demote_l2_locked(pmap, l2, sva, &lock)) { 8039 if (lock != NULL) 8040 rw_wunlock(lock); 8041 8042 /* 8043 * The 2MB page mapping was destroyed. 8044 */ 8045 continue; 8046 } 8047 8048 /* 8049 * Unless the page mappings are wired, remove the 8050 * mapping to a single page so that a subsequent 8051 * access may repromote. Choosing the last page 8052 * within the address range [sva, min(va_next, eva)) 8053 * generally results in more repromotions. Since the 8054 * underlying page table page is fully populated, this 8055 * removal never frees a page table page. 8056 */ 8057 if ((oldl2 & ATTR_SW_WIRED) == 0) { 8058 va = eva; 8059 if (va > va_next) 8060 va = va_next; 8061 va -= PAGE_SIZE; 8062 KASSERT(va >= sva, 8063 ("pmap_advise: no address gap")); 8064 l3 = pmap_l2_to_l3(l2, va); 8065 KASSERT(pmap_load(l3) != 0, 8066 ("pmap_advise: invalid PTE")); 8067 pmap_remove_l3(pmap, l3, va, pmap_load(l2), 8068 NULL, &lock); 8069 } 8070 if (lock != NULL) 8071 rw_wunlock(lock); 8072 } 8073 KASSERT((pmap_load(l2) & ATTR_DESCR_MASK) == L2_TABLE, 8074 ("pmap_advise: invalid L2 entry after demotion")); 8075 if (va_next > eva) 8076 va_next = eva; 8077 va = va_next; 8078 for (l3 = pmap_l2_to_l3(l2, sva); sva != va_next; l3++, 8079 sva += L3_SIZE) { 8080 oldl3 = pmap_load(l3); 8081 if ((oldl3 & (ATTR_SW_MANAGED | ATTR_DESCR_MASK)) != 8082 (ATTR_SW_MANAGED | L3_PAGE)) 8083 goto maybe_invlrng; 8084 else if (pmap_pte_dirty(pmap, oldl3)) { 8085 if (advice == MADV_DONTNEED) { 8086 /* 8087 * Future calls to pmap_is_modified() 8088 * can be avoided by making the page 8089 * dirty now. 8090 */ 8091 m = PTE_TO_VM_PAGE(oldl3); 8092 vm_page_dirty(m); 8093 } 8094 if ((oldl3 & ATTR_CONTIGUOUS) != 0) { 8095 /* 8096 * Unconditionally demote the L3C 8097 * superpage because we do not allow 8098 * writeable, clean superpages. 8099 */ 8100 (void)pmap_demote_l3c(pmap, l3, sva); 8101 8102 /* 8103 * Destroy the final mapping before the 8104 * next L3C boundary or va_next, 8105 * whichever comes first, so that a 8106 * subsequent access may act as a 8107 * repromotion trigger. 8108 */ 8109 if ((oldl3 & ATTR_SW_WIRED) == 0) { 8110 dva = MIN((sva & ~L3C_OFFSET) + 8111 L3C_SIZE - PAGE_SIZE, 8112 va_next - PAGE_SIZE); 8113 dl3 = pmap_l2_to_l3(l2, dva); 8114 KASSERT(pmap_load(dl3) != 0, 8115 ("pmap_advise: invalid PTE")); 8116 lock = NULL; 8117 pmap_remove_l3(pmap, dl3, dva, 8118 pmap_load(l2), NULL, &lock); 8119 if (lock != NULL) 8120 rw_wunlock(lock); 8121 } 8122 8123 /* 8124 * The L3 entry's accessed bit may have 8125 * changed. 8126 */ 8127 oldl3 = pmap_load(l3); 8128 } 8129 8130 /* 8131 * Check that we did not just destroy this entry so 8132 * we avoid corrupting the page able. 8133 */ 8134 if (oldl3 != 0) { 8135 while (!atomic_fcmpset_long(l3, &oldl3, 8136 (oldl3 & ~ATTR_AF) | 8137 ATTR_S1_AP(ATTR_S1_AP_RO))) 8138 cpu_spinwait(); 8139 } 8140 } else if ((oldl3 & ATTR_AF) != 0) { 8141 /* 8142 * Clear the accessed bit in this L3 entry 8143 * regardless of the contiguous bit. 8144 */ 8145 pmap_clear_bits(l3, ATTR_AF); 8146 } else 8147 goto maybe_invlrng; 8148 if (va == va_next) 8149 va = sva; 8150 continue; 8151 maybe_invlrng: 8152 if (va != va_next) { 8153 pmap_s1_invalidate_range(pmap, va, sva, true); 8154 va = va_next; 8155 } 8156 } 8157 if (va != va_next) 8158 pmap_s1_invalidate_range(pmap, va, sva, true); 8159 } 8160 PMAP_UNLOCK(pmap); 8161 } 8162 8163 /* 8164 * Clear the modify bits on the specified physical page. 8165 */ 8166 void 8167 pmap_clear_modify(vm_page_t m) 8168 { 8169 struct md_page *pvh; 8170 struct rwlock *lock; 8171 pmap_t pmap; 8172 pv_entry_t next_pv, pv; 8173 pd_entry_t *l2, oldl2; 8174 pt_entry_t *l3, oldl3; 8175 vm_offset_t va; 8176 int md_gen, pvh_gen; 8177 8178 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 8179 ("pmap_clear_modify: page %p is not managed", m)); 8180 vm_page_assert_busied(m); 8181 8182 if (!pmap_page_is_write_mapped(m)) 8183 return; 8184 pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy : page_to_pvh(m); 8185 lock = VM_PAGE_TO_PV_LIST_LOCK(m); 8186 rw_wlock(lock); 8187 restart: 8188 TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_next, next_pv) { 8189 pmap = PV_PMAP(pv); 8190 PMAP_ASSERT_STAGE1(pmap); 8191 if (!PMAP_TRYLOCK(pmap)) { 8192 pvh_gen = pvh->pv_gen; 8193 rw_wunlock(lock); 8194 PMAP_LOCK(pmap); 8195 rw_wlock(lock); 8196 if (pvh_gen != pvh->pv_gen) { 8197 PMAP_UNLOCK(pmap); 8198 goto restart; 8199 } 8200 } 8201 va = pv->pv_va; 8202 l2 = pmap_l2(pmap, va); 8203 oldl2 = pmap_load(l2); 8204 /* If oldl2 has ATTR_SW_DBM set, then it is also dirty. */ 8205 if ((oldl2 & ATTR_SW_DBM) != 0 && 8206 pmap_demote_l2_locked(pmap, l2, va, &lock) && 8207 (oldl2 & ATTR_SW_WIRED) == 0) { 8208 /* 8209 * Write protect the mapping to a single page so that 8210 * a subsequent write access may repromote. 8211 */ 8212 va += VM_PAGE_TO_PHYS(m) - PTE_TO_PHYS(oldl2); 8213 l3 = pmap_l2_to_l3(l2, va); 8214 oldl3 = pmap_load(l3); 8215 while (!atomic_fcmpset_long(l3, &oldl3, 8216 (oldl3 & ~ATTR_SW_DBM) | ATTR_S1_AP(ATTR_S1_AP_RO))) 8217 cpu_spinwait(); 8218 vm_page_dirty(m); 8219 pmap_s1_invalidate_page(pmap, va, true); 8220 } 8221 PMAP_UNLOCK(pmap); 8222 } 8223 TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) { 8224 pmap = PV_PMAP(pv); 8225 PMAP_ASSERT_STAGE1(pmap); 8226 if (!PMAP_TRYLOCK(pmap)) { 8227 md_gen = m->md.pv_gen; 8228 pvh_gen = pvh->pv_gen; 8229 rw_wunlock(lock); 8230 PMAP_LOCK(pmap); 8231 rw_wlock(lock); 8232 if (pvh_gen != pvh->pv_gen || md_gen != m->md.pv_gen) { 8233 PMAP_UNLOCK(pmap); 8234 goto restart; 8235 } 8236 } 8237 l2 = pmap_l2(pmap, pv->pv_va); 8238 l3 = pmap_l2_to_l3(l2, pv->pv_va); 8239 oldl3 = pmap_load(l3); 8240 KASSERT((oldl3 & ATTR_CONTIGUOUS) == 0 || 8241 (oldl3 & (ATTR_SW_DBM | ATTR_S1_AP_RW_BIT)) != 8242 (ATTR_SW_DBM | ATTR_S1_AP(ATTR_S1_AP_RO)), 8243 ("writeable L3C superpage not dirty")); 8244 if ((oldl3 & (ATTR_S1_AP_RW_BIT | ATTR_SW_DBM)) == ATTR_SW_DBM) { 8245 if ((oldl3 & ATTR_CONTIGUOUS) != 0) 8246 (void)pmap_demote_l3c(pmap, l3, pv->pv_va); 8247 pmap_set_bits(l3, ATTR_S1_AP(ATTR_S1_AP_RO)); 8248 pmap_s1_invalidate_page(pmap, pv->pv_va, true); 8249 } 8250 PMAP_UNLOCK(pmap); 8251 } 8252 rw_wunlock(lock); 8253 } 8254 8255 void * 8256 pmap_mapbios(vm_paddr_t pa, vm_size_t size) 8257 { 8258 struct pmap_preinit_mapping *ppim; 8259 vm_offset_t va, offset; 8260 pd_entry_t old_l2e, *pde; 8261 pt_entry_t *l2; 8262 int i, lvl, l2_blocks, free_l2_count, start_idx; 8263 8264 /* Use the DMAP region if we can */ 8265 if (PHYS_IN_DMAP(pa) && PHYS_IN_DMAP(pa + size - 1) && 8266 pmap_kmapped_range(PHYS_TO_DMAP(pa), size)) 8267 return (PHYS_TO_DMAP(pa)); 8268 8269 if (!vm_initialized) { 8270 /* 8271 * No L3 ptables so map entire L2 blocks where start VA is: 8272 * preinit_map_va + start_idx * L2_SIZE 8273 * There may be duplicate mappings (multiple VA -> same PA) but 8274 * ARM64 dcache is always PIPT so that's acceptable. 8275 */ 8276 if (size == 0) 8277 return (NULL); 8278 8279 /* Calculate how many L2 blocks are needed for the mapping */ 8280 l2_blocks = (roundup2(pa + size, L2_SIZE) - 8281 rounddown2(pa, L2_SIZE)) >> L2_SHIFT; 8282 8283 offset = pa & L2_OFFSET; 8284 8285 if (preinit_map_va == 0) 8286 return (NULL); 8287 8288 /* Map 2MiB L2 blocks from reserved VA space */ 8289 8290 free_l2_count = 0; 8291 start_idx = -1; 8292 /* Find enough free contiguous VA space */ 8293 for (i = 0; i < PMAP_PREINIT_MAPPING_COUNT; i++) { 8294 ppim = pmap_preinit_mapping + i; 8295 if (free_l2_count > 0 && ppim->pa != 0) { 8296 /* Not enough space here */ 8297 free_l2_count = 0; 8298 start_idx = -1; 8299 continue; 8300 } 8301 8302 if (ppim->pa == 0) { 8303 /* Free L2 block */ 8304 if (start_idx == -1) 8305 start_idx = i; 8306 free_l2_count++; 8307 if (free_l2_count == l2_blocks) 8308 break; 8309 } 8310 } 8311 if (free_l2_count != l2_blocks) 8312 panic("%s: too many preinit mappings", __func__); 8313 8314 va = preinit_map_va + (start_idx * L2_SIZE); 8315 for (i = start_idx; i < start_idx + l2_blocks; i++) { 8316 /* Mark entries as allocated */ 8317 ppim = pmap_preinit_mapping + i; 8318 ppim->pa = pa; 8319 ppim->va = (char *)va + offset; 8320 ppim->size = size; 8321 } 8322 8323 /* Map L2 blocks */ 8324 pa = rounddown2(pa, L2_SIZE); 8325 old_l2e = 0; 8326 for (i = 0; i < l2_blocks; i++) { 8327 pde = pmap_pde(kernel_pmap, va, &lvl); 8328 KASSERT(pde != NULL, 8329 ("pmap_mapbios: Invalid page entry, va: 0x%lx", 8330 va)); 8331 KASSERT(lvl == 1, 8332 ("pmap_mapbios: Invalid level %d", lvl)); 8333 8334 /* Insert L2_BLOCK */ 8335 l2 = pmap_l1_to_l2(pde, va); 8336 old_l2e |= pmap_load_store(l2, 8337 PHYS_TO_PTE(pa) | ATTR_AF | pmap_sh_attr | 8338 ATTR_S1_XN | ATTR_KERN_GP | 8339 ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK) | L2_BLOCK); 8340 8341 va += L2_SIZE; 8342 pa += L2_SIZE; 8343 } 8344 if ((old_l2e & ATTR_DESCR_VALID) != 0) 8345 pmap_s1_invalidate_all_kernel(); 8346 else { 8347 /* 8348 * Because the old entries were invalid and the new 8349 * mappings are not executable, an isb is not required. 8350 */ 8351 dsb(ishst); 8352 } 8353 8354 va = preinit_map_va + (start_idx * L2_SIZE); 8355 8356 } else { 8357 /* kva_alloc may be used to map the pages */ 8358 offset = pa & PAGE_MASK; 8359 size = round_page(offset + size); 8360 8361 va = (vm_offset_t)kva_alloc(size); 8362 if (va == 0) 8363 panic("%s: Couldn't allocate KVA", __func__); 8364 8365 pde = pmap_pde(kernel_pmap, va, &lvl); 8366 KASSERT(lvl == 2, ("pmap_mapbios: Invalid level %d", lvl)); 8367 8368 /* L3 table is linked */ 8369 va = trunc_page(va); 8370 pa = trunc_page(pa); 8371 pmap_kenter(va, size, pa, memory_mapping_mode(pa)); 8372 } 8373 8374 return ((void *)(va + offset)); 8375 } 8376 8377 void 8378 pmap_unmapbios(void *p, vm_size_t size) 8379 { 8380 struct pmap_preinit_mapping *ppim; 8381 char *va; 8382 vm_offset_t offset, va_trunc; 8383 pd_entry_t *pde; 8384 pt_entry_t *l2; 8385 int error __diagused, i, lvl, l2_blocks, block; 8386 bool preinit_map; 8387 8388 va = p; 8389 if (VIRT_IN_DMAP(va)) { 8390 KASSERT(VIRT_IN_DMAP(va + size - 1), 8391 ("%s: End address not in DMAP region: %p", __func__, 8392 va + size - 1)); 8393 /* Ensure the attributes are as expected for the DMAP region */ 8394 PMAP_LOCK(kernel_pmap); 8395 error = pmap_change_props_locked(va, size, 8396 PROT_READ | PROT_WRITE, VM_MEMATTR_DEFAULT, -1, false); 8397 PMAP_UNLOCK(kernel_pmap); 8398 KASSERT(error == 0, ("%s: Failed to reset DMAP attributes: %d", 8399 __func__, error)); 8400 8401 return; 8402 } 8403 8404 l2_blocks = 8405 (roundup2(va + size, L2_SIZE) - rounddown2(va, L2_SIZE)) >> L2_SHIFT; 8406 KASSERT(l2_blocks > 0, ("pmap_unmapbios: invalid size %lx", size)); 8407 8408 /* Remove preinit mapping */ 8409 preinit_map = false; 8410 block = 0; 8411 for (i = 0; i < PMAP_PREINIT_MAPPING_COUNT; i++) { 8412 ppim = pmap_preinit_mapping + i; 8413 if (ppim->va == va) { 8414 KASSERT(ppim->size == size, 8415 ("pmap_unmapbios: size mismatch")); 8416 ppim->va = NULL; 8417 ppim->pa = 0; 8418 ppim->size = 0; 8419 preinit_map = true; 8420 offset = block * L2_SIZE; 8421 va_trunc = rounddown2((vm_offset_t)va, L2_SIZE) + 8422 offset; 8423 8424 /* Remove L2_BLOCK */ 8425 pde = pmap_pde(kernel_pmap, va_trunc, &lvl); 8426 KASSERT(pde != NULL, 8427 ("pmap_unmapbios: Invalid page entry, va: 0x%lx", 8428 va_trunc)); 8429 l2 = pmap_l1_to_l2(pde, va_trunc); 8430 pmap_clear(l2); 8431 8432 if (block == (l2_blocks - 1)) 8433 break; 8434 block++; 8435 } 8436 } 8437 if (preinit_map) { 8438 pmap_s1_invalidate_all_kernel(); 8439 return; 8440 } 8441 8442 /* Unmap the pages reserved with kva_alloc. */ 8443 if (vm_initialized) { 8444 offset = (vm_offset_t)va & PAGE_MASK; 8445 size = round_page(offset + size); 8446 va = trunc_page(va); 8447 8448 /* Unmap and invalidate the pages */ 8449 pmap_kremove_device((vm_offset_t)va, size); 8450 8451 kva_free(va, size); 8452 } 8453 } 8454 8455 /* 8456 * Sets the memory attribute for the specified page. 8457 */ 8458 void 8459 pmap_page_set_memattr(vm_page_t m, vm_memattr_t ma) 8460 { 8461 if (m->md.pv_memattr == ma) 8462 return; 8463 8464 m->md.pv_memattr = ma; 8465 8466 /* 8467 * If "m" is a normal page, update its direct mapping. This update 8468 * can be relied upon to perform any cache operations that are 8469 * required for data coherence. 8470 */ 8471 if ((m->flags & PG_FICTITIOUS) == 0 && 8472 pmap_change_attr(VM_PAGE_TO_DMAP(m), PAGE_SIZE, 8473 m->md.pv_memattr) != 0) 8474 panic("memory attribute change on the direct map failed"); 8475 } 8476 8477 /* 8478 * Changes the specified virtual address range's memory type to that given by 8479 * the parameter "mode". The specified virtual address range must be 8480 * completely contained within either the direct map or the kernel map. If 8481 * the virtual address range is contained within the kernel map, then the 8482 * memory type for each of the corresponding ranges of the direct map is also 8483 * changed. (The corresponding ranges of the direct map are those ranges that 8484 * map the same physical pages as the specified virtual address range.) These 8485 * changes to the direct map are necessary because Intel describes the 8486 * behavior of their processors as "undefined" if two or more mappings to the 8487 * same physical page have different memory types. 8488 * 8489 * Returns zero if the change completed successfully, and either EINVAL or 8490 * ENOMEM if the change failed. Specifically, EINVAL is returned if some part 8491 * of the virtual address range was not mapped, and ENOMEM is returned if 8492 * there was insufficient memory available to complete the change. In the 8493 * latter case, the memory type may have been changed on some part of the 8494 * virtual address range or the direct map. 8495 */ 8496 int 8497 pmap_change_attr(void *va, vm_size_t size, int mode) 8498 { 8499 int error; 8500 8501 PMAP_LOCK(kernel_pmap); 8502 error = pmap_change_props_locked(va, size, PROT_NONE, mode, -1, false); 8503 PMAP_UNLOCK(kernel_pmap); 8504 return (error); 8505 } 8506 8507 int 8508 pmap_change_dmap_attr(int mode) 8509 { 8510 int error; 8511 8512 KASSERT(mode == VM_MEMATTR_WRITE_BACK || 8513 mode == VM_MEMATTR_TAGGED, 8514 ("%s: mode %d must be compatible with write-back", __func__, mode)); 8515 8516 PMAP_LOCK(kernel_pmap); 8517 error = pmap_change_props_locked((void *)DMAP_MIN_ADDRESS, 8518 dmap_max_addr - DMAP_MIN_ADDRESS, PROT_NONE, mode, dmap_attr, true); 8519 if (error == 0) 8520 dmap_attr = mode; 8521 PMAP_UNLOCK(kernel_pmap); 8522 return (error); 8523 } 8524 8525 /* 8526 * Changes the specified virtual address range's protections to those 8527 * specified by "prot". Like pmap_change_attr(), protections for aliases 8528 * in the direct map are updated as well. Protections on aliasing mappings may 8529 * be a subset of the requested protections; for example, mappings in the direct 8530 * map are never executable. 8531 */ 8532 int 8533 pmap_change_prot(void *va, vm_size_t size, vm_prot_t prot) 8534 { 8535 int error; 8536 8537 /* Only supported within the kernel map. */ 8538 if ((vm_offset_t)va < VM_MIN_KERNEL_ADDRESS) 8539 return (EINVAL); 8540 8541 PMAP_LOCK(kernel_pmap); 8542 error = pmap_change_props_locked(va, size, prot, -1, -1, false); 8543 PMAP_UNLOCK(kernel_pmap); 8544 return (error); 8545 } 8546 8547 static int 8548 pmap_change_props_locked(void *addr, vm_size_t size, vm_prot_t prot, 8549 int mode, int old_mode, bool skip_unmapped) 8550 { 8551 vm_offset_t base, offset, tmpva, va; 8552 vm_size_t pte_size; 8553 vm_paddr_t pa; 8554 pt_entry_t pte, *ptep, *newpte; 8555 pt_entry_t bits, mask, old_mode_bits, old_mode_mask; 8556 char *tmpptep; 8557 int lvl, rv; 8558 8559 PMAP_LOCK_ASSERT(kernel_pmap, MA_OWNED); 8560 va = (vm_offset_t)addr; 8561 base = trunc_page(va); 8562 offset = va & PAGE_MASK; 8563 size = round_page(offset + size); 8564 8565 if (!VIRT_IN_DMAP(base) && 8566 !(base >= VM_MIN_KERNEL_ADDRESS && base < VM_MAX_KERNEL_ADDRESS)) 8567 return (EINVAL); 8568 8569 bits = old_mode_bits = 0; 8570 mask = old_mode_mask = 0; 8571 if (mode != -1) { 8572 bits = ATTR_S1_IDX(mode); 8573 mask = ATTR_S1_IDX_MASK; 8574 if (mode == VM_MEMATTR_DEVICE) { 8575 mask |= ATTR_S1_XN; 8576 bits |= ATTR_S1_XN; 8577 } 8578 } 8579 if (old_mode != -1) { 8580 old_mode_bits = ATTR_S1_IDX(old_mode); 8581 old_mode_mask = ATTR_S1_IDX_MASK; 8582 } 8583 if (prot != VM_PROT_NONE) { 8584 /* Don't mark the DMAP as executable. It never is on arm64. */ 8585 if (VIRT_IN_DMAP(base)) { 8586 prot &= ~VM_PROT_EXECUTE; 8587 /* 8588 * XXX Mark the DMAP as writable for now. We rely 8589 * on this in ddb & dtrace to insert breakpoint 8590 * instructions. 8591 */ 8592 prot |= VM_PROT_WRITE; 8593 } 8594 8595 if ((prot & VM_PROT_WRITE) == 0) { 8596 bits |= ATTR_S1_AP(ATTR_S1_AP_RO); 8597 } 8598 if ((prot & VM_PROT_EXECUTE) == 0) { 8599 bits |= ATTR_S1_PXN; 8600 } 8601 bits |= ATTR_S1_UXN; 8602 mask |= ATTR_S1_AP_MASK | ATTR_S1_XN; 8603 } 8604 8605 for (tmpva = base; tmpva < base + size; ) { 8606 ptep = pmap_pte(kernel_pmap, tmpva, &lvl); 8607 if (ptep == NULL && !skip_unmapped) { 8608 return (EINVAL); 8609 } else if ((ptep == NULL && skip_unmapped) || 8610 (pmap_load(ptep) & mask) == bits || 8611 (pmap_load(ptep) & old_mode_mask) != old_mode_bits) { 8612 /* 8613 * We already have one of the following meaning 8614 * we can skip this memory region:: 8615 * - No memory mapped at this address 8616 * - The new attributes are already set 8617 * - The expected attributes are incorrect 8618 */ 8619 switch (lvl) { 8620 default: 8621 panic("Invalid DMAP table level: %d\n", lvl); 8622 case 1: 8623 tmpva = (tmpva & ~L1_OFFSET) + L1_SIZE; 8624 break; 8625 case 2: 8626 tmpva = (tmpva & ~L2_OFFSET) + L2_SIZE; 8627 break; 8628 case 3: 8629 tmpva += PAGE_SIZE; 8630 break; 8631 } 8632 } else { 8633 /* We can't demote/promote this entry */ 8634 MPASS((pmap_load(ptep) & ATTR_SW_NO_PROMOTE) == 0); 8635 8636 /* 8637 * Find the entry and demote it if the requested change 8638 * only applies to part of the address range mapped by 8639 * the entry. 8640 */ 8641 switch (lvl) { 8642 default: 8643 panic("Invalid DMAP table level: %d\n", lvl); 8644 case 1: 8645 PMAP_ASSERT_L1_BLOCKS_SUPPORTED; 8646 if ((tmpva & L1_OFFSET) == 0 && 8647 (base + size - tmpva) >= L1_SIZE) { 8648 pte_size = L1_SIZE; 8649 break; 8650 } 8651 newpte = pmap_demote_l1(kernel_pmap, ptep, 8652 tmpva & ~L1_OFFSET); 8653 if (newpte == NULL) 8654 return (EINVAL); 8655 ptep = pmap_l1_to_l2(ptep, tmpva); 8656 /* FALLTHROUGH */ 8657 case 2: 8658 if ((pmap_load(ptep) & ATTR_CONTIGUOUS) != 0) { 8659 if ((tmpva & L2C_OFFSET) == 0 && 8660 (base + size - tmpva) >= L2C_SIZE) { 8661 pte_size = L2C_SIZE; 8662 break; 8663 } 8664 if (!pmap_demote_l2c(kernel_pmap, ptep, 8665 tmpva)) 8666 return (EINVAL); 8667 } 8668 if ((tmpva & L2_OFFSET) == 0 && 8669 (base + size - tmpva) >= L2_SIZE) { 8670 pte_size = L2_SIZE; 8671 break; 8672 } 8673 newpte = pmap_demote_l2(kernel_pmap, ptep, 8674 tmpva); 8675 if (newpte == NULL) 8676 return (EINVAL); 8677 ptep = pmap_l2_to_l3(ptep, tmpva); 8678 /* FALLTHROUGH */ 8679 case 3: 8680 if ((pmap_load(ptep) & ATTR_CONTIGUOUS) != 0) { 8681 if ((tmpva & L3C_OFFSET) == 0 && 8682 (base + size - tmpva) >= L3C_SIZE) { 8683 pte_size = L3C_SIZE; 8684 break; 8685 } 8686 if (!pmap_demote_l3c(kernel_pmap, ptep, 8687 tmpva)) 8688 return (EINVAL); 8689 } 8690 pte_size = PAGE_SIZE; 8691 break; 8692 } 8693 8694 tmpptep = 0; 8695 if (tmpva <= (vm_offset_t)ptep && 8696 tmpva + pte_size > (vm_offset_t)ptep) { 8697 vm_paddr_t pte_pa; 8698 8699 mtx_lock(&cmap_lock); 8700 tmpptep = cmap1_addr; 8701 pte_pa = DMAP_TO_PHYS((vm_offset_t)ptep); 8702 pmap_store(cmap1_pte, ATTR_AF | 8703 pmap_sh_attr | ATTR_S1_AP(ATTR_S1_AP_RW) | 8704 ATTR_S1_XN | ATTR_KERN_GP | 8705 ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK) | 8706 PHYS_TO_PTE(pte_pa &~L3_OFFSET) | L3_PAGE); 8707 dsb(ishst); 8708 ptep = (pt_entry_t *)(tmpptep + 8709 ((vm_offset_t)ptep & PAGE_MASK)); 8710 } 8711 8712 /* Update the entry */ 8713 pte = pmap_load(ptep); 8714 pte &= ~mask; 8715 pte |= bits; 8716 8717 switch (pte_size) { 8718 case L2C_SIZE: 8719 pmap_update_strided(kernel_pmap, ptep, ptep + 8720 L2C_ENTRIES, pte, tmpva, L2_SIZE, L2C_SIZE); 8721 break; 8722 case L3C_SIZE: 8723 pmap_update_strided(kernel_pmap, ptep, ptep + 8724 L3C_ENTRIES, pte, tmpva, L3_SIZE, L3C_SIZE); 8725 break; 8726 default: 8727 /* 8728 * We are updating a single block or page entry, 8729 * so regardless of pte_size pass PAGE_SIZE in 8730 * order that a single TLB invalidation is 8731 * performed. 8732 */ 8733 pmap_update_entry(kernel_pmap, ptep, pte, tmpva, 8734 PAGE_SIZE, true); 8735 break; 8736 } 8737 8738 if (tmpptep != 0) { 8739 pmap_clear(cmap1_pte); 8740 pmap_s1_invalidate_page(kernel_pmap, 8741 (vm_offset_t)tmpptep, true); 8742 mtx_unlock(&cmap_lock); 8743 } 8744 8745 pa = PTE_TO_PHYS(pte); 8746 if (!VIRT_IN_DMAP(tmpva) && PHYS_IN_DMAP(pa)) { 8747 int dmap_mode; 8748 8749 /* 8750 * When booting on HW with MTE enabled we may 8751 * need to swap to a tagged type for the DMAP 8752 * to allow tags to be set through it. 8753 */ 8754 if (mode == VM_MEMATTR_WRITE_BACK) 8755 dmap_mode = dmap_attr; 8756 else 8757 dmap_mode = mode; 8758 8759 /* 8760 * Keep the DMAP memory in sync. 8761 */ 8762 rv = pmap_change_props_locked( 8763 PHYS_TO_DMAP(pa), pte_size, 8764 prot, dmap_mode, old_mode, true); 8765 if (rv != 0) 8766 return (rv); 8767 } 8768 8769 /* 8770 * If moving to a non-cacheable entry flush 8771 * the cache. 8772 */ 8773 if (mode == VM_MEMATTR_UNCACHEABLE) 8774 cpu_dcache_wbinv_range((void *)tmpva, pte_size); 8775 tmpva += pte_size; 8776 } 8777 } 8778 8779 return (0); 8780 } 8781 8782 /* 8783 * Create an L2 table to map all addresses within an L1 mapping. 8784 */ 8785 static pt_entry_t * 8786 pmap_demote_l1(pmap_t pmap, pt_entry_t *l1, vm_offset_t va) 8787 { 8788 pt_entry_t *l2, newl2, oldl1; 8789 char *tmpl1; 8790 vm_paddr_t l2phys, phys; 8791 vm_page_t ml2; 8792 int i; 8793 8794 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 8795 oldl1 = pmap_load(l1); 8796 PMAP_ASSERT_L1_BLOCKS_SUPPORTED; 8797 KASSERT((oldl1 & ATTR_DESCR_MASK) == L1_BLOCK, 8798 ("pmap_demote_l1: Demoting a non-block entry")); 8799 KASSERT((va & L1_OFFSET) == 0, 8800 ("pmap_demote_l1: Invalid virtual address %#lx", va)); 8801 KASSERT((oldl1 & ATTR_SW_MANAGED) == 0, 8802 ("pmap_demote_l1: Level 1 table shouldn't be managed")); 8803 KASSERT((oldl1 & ATTR_SW_NO_PROMOTE) == 0, 8804 ("pmap_demote_l1: Demoting entry with no-demote flag set")); 8805 8806 tmpl1 = NULL; 8807 if (va <= (vm_offset_t)l1 && va + L1_SIZE > (vm_offset_t)l1) { 8808 tmpl1 = kva_alloc(PAGE_SIZE); 8809 if (tmpl1 == NULL) 8810 return (NULL); 8811 } 8812 8813 if ((ml2 = vm_page_alloc_noobj(VM_ALLOC_INTERRUPT | VM_ALLOC_WIRED)) == 8814 NULL) { 8815 CTR2(KTR_PMAP, "pmap_demote_l1: failure for va %#lx" 8816 " in pmap %p", va, pmap); 8817 l2 = NULL; 8818 goto fail; 8819 } 8820 8821 l2phys = VM_PAGE_TO_PHYS(ml2); 8822 l2 = PHYS_TO_DMAP(l2phys); 8823 8824 /* Address the range points at */ 8825 phys = PTE_TO_PHYS(oldl1); 8826 /* The attributed from the old l1 table to be copied */ 8827 newl2 = oldl1 & ATTR_MASK; 8828 8829 /* Create the new entries */ 8830 newl2 |= ATTR_CONTIGUOUS; 8831 for (i = 0; i < Ln_ENTRIES; i++) { 8832 l2[i] = newl2 | phys; 8833 phys += L2_SIZE; 8834 } 8835 KASSERT(l2[0] == (ATTR_CONTIGUOUS | (oldl1 & ~ATTR_DESCR_MASK) | 8836 L2_BLOCK), ("Invalid l2 page (%lx != %lx)", l2[0], 8837 ATTR_CONTIGUOUS | (oldl1 & ~ATTR_DESCR_MASK) | L2_BLOCK)); 8838 8839 if (tmpl1 != NULL) { 8840 pmap_kenter((vm_offset_t)tmpl1, PAGE_SIZE, 8841 DMAP_TO_PHYS(l1) & ~L3_OFFSET, 8842 VM_MEMATTR_WRITE_BACK); 8843 l1 = (pt_entry_t *)(tmpl1 + ((vm_offset_t)l1 & PAGE_MASK)); 8844 } 8845 8846 pmap_update_entry(pmap, l1, l2phys | L1_TABLE, va, PAGE_SIZE, true); 8847 8848 counter_u64_add(pmap_l1_demotions, 1); 8849 fail: 8850 if (tmpl1 != NULL) { 8851 pmap_kremove((vm_offset_t)tmpl1); 8852 kva_free(tmpl1, PAGE_SIZE); 8853 } 8854 8855 return (l2); 8856 } 8857 8858 static void 8859 pmap_fill_l3(pt_entry_t *firstl3, pt_entry_t newl3) 8860 { 8861 pt_entry_t *l3; 8862 8863 for (l3 = firstl3; l3 - firstl3 < Ln_ENTRIES; l3++) { 8864 *l3 = newl3; 8865 newl3 += L3_SIZE; 8866 } 8867 } 8868 8869 static void 8870 pmap_demote_l2_check(pt_entry_t *firstl3p __unused, pt_entry_t newl3e __unused) 8871 { 8872 #ifdef INVARIANTS 8873 #ifdef DIAGNOSTIC 8874 pt_entry_t *xl3p, *yl3p; 8875 8876 for (xl3p = firstl3p; xl3p < firstl3p + Ln_ENTRIES; 8877 xl3p++, newl3e += PAGE_SIZE) { 8878 if (PTE_TO_PHYS(pmap_load(xl3p)) != PTE_TO_PHYS(newl3e)) { 8879 printf("pmap_demote_l2: xl3e %zd and newl3e map " 8880 "different pages: found %#lx, expected %#lx\n", 8881 xl3p - firstl3p, pmap_load(xl3p), newl3e); 8882 printf("page table dump\n"); 8883 for (yl3p = firstl3p; yl3p < firstl3p + Ln_ENTRIES; 8884 yl3p++) { 8885 printf("%zd %#lx\n", yl3p - firstl3p, 8886 pmap_load(yl3p)); 8887 } 8888 panic("firstpte"); 8889 } 8890 } 8891 #else 8892 KASSERT(PTE_TO_PHYS(pmap_load(firstl3p)) == PTE_TO_PHYS(newl3e), 8893 ("pmap_demote_l2: firstl3 and newl3e map different physical" 8894 " addresses")); 8895 #endif 8896 #endif 8897 } 8898 8899 static void 8900 pmap_demote_l2_abort(pmap_t pmap, vm_offset_t va, pt_entry_t *l2, 8901 struct rwlock **lockp) 8902 { 8903 struct spglist free; 8904 8905 SLIST_INIT(&free); 8906 (void)pmap_remove_l2(pmap, l2, va, pmap_load(pmap_l1(pmap, va)), true, 8907 &free, lockp); 8908 vm_page_free_pages_toq(&free, true); 8909 } 8910 8911 /* 8912 * Create an L3 table to map all addresses within an L2 mapping. 8913 */ 8914 static pt_entry_t * 8915 pmap_demote_l2_locked(pmap_t pmap, pt_entry_t *l2, vm_offset_t va, 8916 struct rwlock **lockp) 8917 { 8918 pt_entry_t *l3, newl3, oldl2; 8919 char *tmpl2; 8920 vm_paddr_t l3phys; 8921 vm_page_t ml3; 8922 8923 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 8924 PMAP_ASSERT_STAGE1(pmap); 8925 KASSERT(ADDR_IS_CANONICAL(va), 8926 ("%s: Address not in canonical form: %lx", __func__, va)); 8927 8928 l3 = NULL; 8929 oldl2 = pmap_load(l2); 8930 KASSERT((oldl2 & ATTR_DESCR_MASK) == L2_BLOCK, 8931 ("pmap_demote_l2: Demoting a non-block entry")); 8932 KASSERT((oldl2 & ATTR_SW_NO_PROMOTE) == 0, 8933 ("pmap_demote_l2: Demoting entry with no-demote flag set")); 8934 va &= ~L2_OFFSET; 8935 8936 tmpl2 = NULL; 8937 if (va <= (vm_offset_t)l2 && va + L2_SIZE > (vm_offset_t)l2) { 8938 tmpl2 = kva_alloc(PAGE_SIZE); 8939 if (tmpl2 == NULL) 8940 return (NULL); 8941 } 8942 8943 /* 8944 * Invalidate the 2MB page mapping and return "failure" if the 8945 * mapping was never accessed and not wired. 8946 */ 8947 if ((oldl2 & ATTR_AF) == 0) { 8948 if ((oldl2 & ATTR_SW_WIRED) == 0) { 8949 pmap_demote_l2_abort(pmap, va, l2, lockp); 8950 CTR2(KTR_PMAP, 8951 "pmap_demote_l2: failure for va %#lx in pmap %p", 8952 va, pmap); 8953 goto fail; 8954 } 8955 ml3 = pmap_remove_pt_page(pmap, va); 8956 /* Fill the PTP with L3Es that have ATTR_AF cleared. */ 8957 ml3->valid = 0; 8958 } else if ((ml3 = pmap_remove_pt_page(pmap, va)) == NULL) { 8959 KASSERT((oldl2 & ATTR_SW_WIRED) == 0, 8960 ("pmap_demote_l2: page table page for a wired mapping" 8961 " is missing")); 8962 8963 /* 8964 * If the page table page is missing and the mapping 8965 * is for a kernel address, the mapping must belong to 8966 * either the direct map or the early kernel memory. 8967 * Page table pages are preallocated for every other 8968 * part of the kernel address space, so the direct map 8969 * region and early kernel memory are the only parts of the 8970 * kernel address space that must be handled here. 8971 */ 8972 KASSERT(ADDR_IS_USER(va) || VIRT_IN_DMAP(va) || 8973 (va >= VM_MIN_KERNEL_ADDRESS && va < kernel_vm_end), 8974 ("pmap_demote_l2: No saved mpte for va %#lx", va)); 8975 8976 /* 8977 * If the 2MB page mapping belongs to the direct map 8978 * region of the kernel's address space, then the page 8979 * allocation request specifies the highest possible 8980 * priority (VM_ALLOC_INTERRUPT). Otherwise, the 8981 * priority is normal. 8982 */ 8983 ml3 = vm_page_alloc_noobj( 8984 (VIRT_IN_DMAP(va) ? VM_ALLOC_INTERRUPT : 0) | 8985 VM_ALLOC_WIRED); 8986 8987 /* 8988 * If the allocation of the new page table page fails, 8989 * invalidate the 2MB page mapping and return "failure". 8990 */ 8991 if (ml3 == NULL) { 8992 pmap_demote_l2_abort(pmap, va, l2, lockp); 8993 CTR2(KTR_PMAP, "pmap_demote_l2: failure for va %#lx" 8994 " in pmap %p", va, pmap); 8995 goto fail; 8996 } 8997 ml3->pindex = pmap_l2_pindex(va); 8998 8999 if (ADDR_IS_USER(va)) { 9000 ml3->ref_count = NL3PG; 9001 pmap_resident_count_inc(pmap, 1); 9002 } 9003 } 9004 l3phys = VM_PAGE_TO_PHYS(ml3); 9005 l3 = PHYS_TO_DMAP(l3phys); 9006 newl3 = ATTR_CONTIGUOUS | (oldl2 & ~ATTR_DESCR_MASK) | L3_PAGE; 9007 KASSERT((oldl2 & (ATTR_S1_AP_RW_BIT | ATTR_SW_DBM)) != 9008 (ATTR_S1_AP(ATTR_S1_AP_RO) | ATTR_SW_DBM), 9009 ("pmap_demote_l2: L2 entry is writeable but not dirty")); 9010 9011 /* 9012 * If the PTP is not leftover from an earlier promotion or it does not 9013 * have ATTR_AF set in every L3E, then fill it. The new L3Es will all 9014 * have ATTR_AF set, unless this is a wired mapping with ATTR_AF clear. 9015 * 9016 * When pmap_update_entry() clears the old L2 mapping, it (indirectly) 9017 * performs a dsb(). That dsb() ensures that the stores for filling 9018 * "l3" are visible before "l3" is added to the page table. 9019 */ 9020 if (!vm_page_all_valid(ml3)) 9021 pmap_fill_l3(l3, newl3); 9022 9023 pmap_demote_l2_check(l3, newl3); 9024 9025 /* 9026 * If the mapping has changed attributes, update the L3Es. 9027 */ 9028 if ((pmap_load(l3) & ATTR_PROMOTE) != (newl3 & ATTR_PROMOTE)) 9029 pmap_fill_l3(l3, newl3); 9030 9031 /* 9032 * Map the temporary page so we don't lose access to the l2 table. 9033 */ 9034 if (tmpl2 != NULL) { 9035 pmap_kenter((vm_offset_t)tmpl2, PAGE_SIZE, 9036 DMAP_TO_PHYS(l2) & ~L3_OFFSET, 9037 VM_MEMATTR_WRITE_BACK); 9038 l2 = (pt_entry_t *)(tmpl2 + ((vm_offset_t)l2 & PAGE_MASK)); 9039 } 9040 9041 /* 9042 * The spare PV entries must be reserved prior to demoting the 9043 * mapping, that is, prior to changing the PDE. Otherwise, the state 9044 * of the L2 and the PV lists will be inconsistent, which can result 9045 * in reclaim_pv_chunk() attempting to remove a PV entry from the 9046 * wrong PV list and pmap_pv_demote_l2() failing to find the expected 9047 * PV entry for the 2MB page mapping that is being demoted. 9048 */ 9049 if ((oldl2 & ATTR_SW_MANAGED) != 0) 9050 reserve_pv_entries(pmap, Ln_ENTRIES - 1, lockp); 9051 9052 /* 9053 * Pass PAGE_SIZE so that a single TLB invalidation is performed on 9054 * the 2MB page mapping. 9055 */ 9056 pmap_update_entry(pmap, l2, l3phys | L2_TABLE, va, PAGE_SIZE, true); 9057 9058 /* 9059 * Demote the PV entry. 9060 */ 9061 if ((oldl2 & ATTR_SW_MANAGED) != 0) 9062 pmap_pv_demote_l2(pmap, va, PTE_TO_PHYS(oldl2), lockp); 9063 9064 counter_u64_add(pmap_l2_demotions, 1); 9065 CTR3(KTR_PMAP, "pmap_demote_l2: success for va %#lx" 9066 " in pmap %p %lx", va, pmap, l3[0]); 9067 9068 fail: 9069 if (tmpl2 != NULL) { 9070 pmap_kremove((vm_offset_t)tmpl2); 9071 kva_free(tmpl2, PAGE_SIZE); 9072 } 9073 9074 return (l3); 9075 9076 } 9077 9078 static pt_entry_t * 9079 pmap_demote_l2(pmap_t pmap, pt_entry_t *l2, vm_offset_t va) 9080 { 9081 struct rwlock *lock; 9082 pt_entry_t *l3; 9083 9084 lock = NULL; 9085 l3 = pmap_demote_l2_locked(pmap, l2, va, &lock); 9086 if (lock != NULL) 9087 rw_wunlock(lock); 9088 return (l3); 9089 } 9090 9091 /* 9092 * Demote an L2C superpage mapping to L2C_ENTRIES L2 block mappings. 9093 */ 9094 static bool 9095 pmap_demote_l2c(pmap_t pmap, pt_entry_t *l2p, vm_offset_t va) 9096 { 9097 pd_entry_t *l2c_end, *l2c_start, l2e, mask, nbits, *tl2p; 9098 char *tmpl3; 9099 register_t intr; 9100 9101 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 9102 PMAP_ASSERT_STAGE1(pmap); 9103 l2c_start = (pd_entry_t *)((uintptr_t)l2p & ~((L2C_ENTRIES * 9104 sizeof(pd_entry_t)) - 1)); 9105 l2c_end = l2c_start + L2C_ENTRIES; 9106 tmpl3 = NULL; 9107 if ((va & ~L2C_OFFSET) < (vm_offset_t)l2c_end && 9108 (vm_offset_t)l2c_start < (va & ~L2C_OFFSET) + L2C_SIZE) { 9109 tmpl3 = kva_alloc(PAGE_SIZE); 9110 if (tmpl3 == NULL) 9111 return (false); 9112 pmap_kenter((vm_offset_t)tmpl3, PAGE_SIZE, 9113 DMAP_TO_PHYS(l2c_start) & ~L3_OFFSET, 9114 VM_MEMATTR_WRITE_BACK); 9115 l2c_start = (pd_entry_t *)(tmpl3 + 9116 ((vm_offset_t)l2c_start & PAGE_MASK)); 9117 l2c_end = (pd_entry_t *)(tmpl3 + 9118 ((vm_offset_t)l2c_end & PAGE_MASK)); 9119 } 9120 mask = 0; 9121 nbits = ATTR_DESCR_VALID; 9122 intr = intr_disable(); 9123 9124 /* 9125 * Break the mappings. 9126 */ 9127 for (tl2p = l2c_start; tl2p < l2c_end; tl2p++) { 9128 /* 9129 * Clear the mapping's contiguous and valid bits, but leave 9130 * the rest of the entry unchanged, so that a lockless, 9131 * concurrent pmap_kextract() can still lookup the physical 9132 * address. 9133 */ 9134 l2e = pmap_load(tl2p); 9135 KASSERT((l2e & ATTR_CONTIGUOUS) != 0, 9136 ("pmap_demote_l2c: missing ATTR_CONTIGUOUS")); 9137 KASSERT((l2e & (ATTR_SW_DBM | ATTR_S1_AP_RW_BIT)) != 9138 (ATTR_SW_DBM | ATTR_S1_AP(ATTR_S1_AP_RO)), 9139 ("pmap_demote_l2c: missing ATTR_S1_AP_RW")); 9140 while (!atomic_fcmpset_64(tl2p, &l2e, l2e & ~(ATTR_CONTIGUOUS | 9141 ATTR_DESCR_VALID))) 9142 cpu_spinwait(); 9143 9144 /* 9145 * Hardware accessed and dirty bit maintenance might only 9146 * update a single L2 entry, so we must combine the accessed 9147 * and dirty bits from this entire set of contiguous L2 9148 * entries. 9149 */ 9150 if ((l2e & (ATTR_S1_AP_RW_BIT | ATTR_SW_DBM)) == 9151 (ATTR_S1_AP(ATTR_S1_AP_RW) | ATTR_SW_DBM)) 9152 mask = ATTR_S1_AP_RW_BIT; 9153 nbits |= l2e & ATTR_AF; 9154 } 9155 if ((nbits & ATTR_AF) != 0) { 9156 pmap_s1_invalidate_strided(pmap, va & ~L2C_OFFSET, (va + 9157 L2C_SIZE) & ~L2C_OFFSET, L2_SIZE, true); 9158 } 9159 9160 /* 9161 * Remake the mappings, updating the accessed and dirty bits. 9162 */ 9163 l2e = (pmap_load(l2c_start) & ~mask) | nbits; 9164 for (tl2p = l2c_start; tl2p < l2c_end; tl2p++) { 9165 pmap_store(tl2p, l2e); 9166 l2e += L2_SIZE; 9167 } 9168 dsb(ishst); 9169 9170 intr_restore(intr); 9171 if (tmpl3 != NULL) { 9172 pmap_kremove((vm_offset_t)tmpl3); 9173 kva_free(tmpl3, PAGE_SIZE); 9174 } 9175 counter_u64_add(pmap_l2c_demotions, 1); 9176 CTR2(KTR_PMAP, "pmap_demote_l2c: success for va %#lx in pmap %p", 9177 va, pmap); 9178 return (true); 9179 } 9180 9181 /* 9182 * Demote a L3C superpage mapping to L3C_ENTRIES 4KB page mappings. 9183 */ 9184 static bool 9185 pmap_demote_l3c(pmap_t pmap, pt_entry_t *l3p, vm_offset_t va) 9186 { 9187 pt_entry_t *l3c_end, *l3c_start, l3e, mask, nbits, *tl3p; 9188 char *tmpl3; 9189 register_t intr; 9190 9191 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 9192 l3c_start = (pt_entry_t *)((uintptr_t)l3p & ~((L3C_ENTRIES * 9193 sizeof(pt_entry_t)) - 1)); 9194 l3c_end = l3c_start + L3C_ENTRIES; 9195 tmpl3 = NULL; 9196 if ((va & ~L3C_OFFSET) < (vm_offset_t)l3c_end && 9197 (vm_offset_t)l3c_start < (va & ~L3C_OFFSET) + L3C_SIZE) { 9198 tmpl3 = kva_alloc(PAGE_SIZE); 9199 if (tmpl3 == NULL) 9200 return (false); 9201 pmap_kenter((vm_offset_t)tmpl3, PAGE_SIZE, 9202 DMAP_TO_PHYS(l3c_start) & ~L3_OFFSET, 9203 VM_MEMATTR_WRITE_BACK); 9204 l3c_start = (pt_entry_t *)(tmpl3 + 9205 ((vm_offset_t)l3c_start & PAGE_MASK)); 9206 l3c_end = (pt_entry_t *)(tmpl3 + 9207 ((vm_offset_t)l3c_end & PAGE_MASK)); 9208 } 9209 mask = 0; 9210 nbits = ATTR_DESCR_VALID; 9211 intr = intr_disable(); 9212 9213 /* 9214 * Break the mappings. 9215 */ 9216 for (tl3p = l3c_start; tl3p < l3c_end; tl3p++) { 9217 /* 9218 * Clear the mapping's contiguous and valid bits, but leave 9219 * the rest of the entry unchanged, so that a lockless, 9220 * concurrent pmap_kextract() can still lookup the physical 9221 * address. 9222 */ 9223 l3e = pmap_load(tl3p); 9224 KASSERT((l3e & ATTR_CONTIGUOUS) != 0, 9225 ("pmap_demote_l3c: missing ATTR_CONTIGUOUS")); 9226 KASSERT((l3e & (ATTR_SW_DBM | ATTR_S1_AP_RW_BIT)) != 9227 (ATTR_SW_DBM | ATTR_S1_AP(ATTR_S1_AP_RO)), 9228 ("pmap_demote_l3c: missing ATTR_S1_AP_RW")); 9229 while (!atomic_fcmpset_64(tl3p, &l3e, l3e & ~(ATTR_CONTIGUOUS | 9230 ATTR_DESCR_VALID))) 9231 cpu_spinwait(); 9232 9233 /* 9234 * Hardware accessed and dirty bit maintenance might only 9235 * update a single L3 entry, so we must combine the accessed 9236 * and dirty bits from this entire set of contiguous L3 9237 * entries. 9238 */ 9239 if ((l3e & (ATTR_S1_AP_RW_BIT | ATTR_SW_DBM)) == 9240 (ATTR_S1_AP(ATTR_S1_AP_RW) | ATTR_SW_DBM)) 9241 mask = ATTR_S1_AP_RW_BIT; 9242 nbits |= l3e & ATTR_AF; 9243 } 9244 if ((nbits & ATTR_AF) != 0) { 9245 pmap_invalidate_range(pmap, va & ~L3C_OFFSET, (va + L3C_SIZE) & 9246 ~L3C_OFFSET, true); 9247 } 9248 9249 /* 9250 * Remake the mappings, updating the accessed and dirty bits. 9251 */ 9252 l3e = (pmap_load(l3c_start) & ~mask) | nbits; 9253 for (tl3p = l3c_start; tl3p < l3c_end; tl3p++) { 9254 pmap_store(tl3p, l3e); 9255 l3e += L3_SIZE; 9256 } 9257 dsb(ishst); 9258 9259 intr_restore(intr); 9260 if (tmpl3 != NULL) { 9261 pmap_kremove((vm_offset_t)tmpl3); 9262 kva_free(tmpl3, PAGE_SIZE); 9263 } 9264 counter_u64_add(pmap_l3c_demotions, 1); 9265 CTR2(KTR_PMAP, "pmap_demote_l3c: success for va %#lx in pmap %p", 9266 va, pmap); 9267 return (true); 9268 } 9269 9270 /* 9271 * Accumulate the accessed and dirty bits within a L3C superpage and 9272 * return the specified PTE with them applied correctly. 9273 */ 9274 static pt_entry_t 9275 pmap_load_l3c(pt_entry_t *l3p) 9276 { 9277 pt_entry_t *l3c_end, *l3c_start, l3e, mask, nbits, *tl3p; 9278 9279 l3c_start = (pt_entry_t *)((uintptr_t)l3p & ~((L3C_ENTRIES * 9280 sizeof(pt_entry_t)) - 1)); 9281 l3c_end = l3c_start + L3C_ENTRIES; 9282 mask = 0; 9283 nbits = 0; 9284 /* Iterate over each mapping in the superpage. */ 9285 for (tl3p = l3c_start; tl3p < l3c_end; tl3p++) { 9286 l3e = pmap_load(tl3p); 9287 KASSERT((l3e & ATTR_CONTIGUOUS) != 0, 9288 ("pmap_load_l3c: missing ATTR_CONTIGUOUS")); 9289 /* Update mask if the current page has its dirty bit set. */ 9290 if ((l3e & (ATTR_S1_AP_RW_BIT | ATTR_SW_DBM)) == 9291 (ATTR_S1_AP(ATTR_S1_AP_RW) | ATTR_SW_DBM)) 9292 mask = ATTR_S1_AP_RW_BIT; 9293 /* Update nbits if the accessed bit is set. */ 9294 nbits |= l3e & ATTR_AF; 9295 } 9296 return ((pmap_load(l3p) & ~mask) | nbits); 9297 } 9298 9299 /* 9300 * Perform the pmap work for mincore(2). If the page is not both referenced and 9301 * modified by this pmap, returns its physical address so that the caller can 9302 * find other mappings. 9303 */ 9304 int 9305 pmap_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *pap) 9306 { 9307 pt_entry_t *pte, tpte; 9308 vm_paddr_t mask, pa; 9309 int lvl, psind, val; 9310 bool managed; 9311 9312 PMAP_ASSERT_STAGE1(pmap); 9313 PMAP_LOCK(pmap); 9314 pte = pmap_pte(pmap, addr, &lvl); 9315 if (pte != NULL) { 9316 tpte = pmap_load(pte); 9317 9318 switch (lvl) { 9319 case 3: 9320 mask = L3_OFFSET; 9321 psind = (tpte & ATTR_CONTIGUOUS) != 0 ? 1 : 0; 9322 break; 9323 case 2: 9324 mask = L2_OFFSET; 9325 psind = 2; 9326 break; 9327 case 1: 9328 mask = L1_OFFSET; 9329 psind = 3; 9330 break; 9331 default: 9332 panic("pmap_mincore: invalid level %d", lvl); 9333 } 9334 9335 managed = (tpte & ATTR_SW_MANAGED) != 0; 9336 val = MINCORE_INCORE | MINCORE_PSIND(psind); 9337 if ((managed && pmap_pte_dirty(pmap, tpte)) || (!managed && 9338 (tpte & ATTR_S1_AP_RW_BIT) == ATTR_S1_AP(ATTR_S1_AP_RW))) 9339 val |= MINCORE_MODIFIED | MINCORE_MODIFIED_OTHER; 9340 if ((tpte & ATTR_AF) == ATTR_AF) 9341 val |= MINCORE_REFERENCED | MINCORE_REFERENCED_OTHER; 9342 9343 pa = PTE_TO_PHYS(tpte) | (addr & mask); 9344 } else { 9345 managed = false; 9346 val = 0; 9347 } 9348 9349 if ((val & (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER)) != 9350 (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER) && managed) { 9351 *pap = pa; 9352 } 9353 PMAP_UNLOCK(pmap); 9354 return (val); 9355 } 9356 9357 /* 9358 * Garbage collect every ASID that is neither active on a processor nor 9359 * reserved. 9360 */ 9361 static void 9362 pmap_reset_asid_set(pmap_t pmap) 9363 { 9364 pmap_t curpmap; 9365 int asid, cpuid, epoch; 9366 struct asid_set *set; 9367 enum pmap_stage stage; 9368 9369 set = pmap->pm_asid_set; 9370 stage = pmap->pm_stage; 9371 9372 set = pmap->pm_asid_set; 9373 KASSERT(set != NULL, ("%s: NULL asid set", __func__)); 9374 mtx_assert(&set->asid_set_mutex, MA_OWNED); 9375 9376 /* 9377 * Ensure that the store to asid_epoch is globally visible before the 9378 * loads from pc_curpmap are performed. 9379 */ 9380 epoch = set->asid_epoch + 1; 9381 if (epoch == INT_MAX) 9382 epoch = 0; 9383 set->asid_epoch = epoch; 9384 dsb(ishst); 9385 if (stage == PM_STAGE1) { 9386 __asm __volatile("tlbi vmalle1is"); 9387 } else { 9388 KASSERT(pmap_clean_stage2_tlbi != NULL, 9389 ("%s: Unset stage 2 tlb invalidation callback\n", 9390 __func__)); 9391 pmap_clean_stage2_tlbi(); 9392 } 9393 dsb(ish); 9394 bit_nclear(set->asid_set, ASID_FIRST_AVAILABLE, 9395 set->asid_set_size - 1); 9396 CPU_FOREACH(cpuid) { 9397 if (cpuid == curcpu) 9398 continue; 9399 if (stage == PM_STAGE1) { 9400 curpmap = pcpu_find(cpuid)->pc_curpmap; 9401 PMAP_ASSERT_STAGE1(pmap); 9402 } else { 9403 curpmap = pcpu_find(cpuid)->pc_curvmpmap; 9404 if (curpmap == NULL) 9405 continue; 9406 PMAP_ASSERT_STAGE2(pmap); 9407 } 9408 KASSERT(curpmap->pm_asid_set == set, ("Incorrect set")); 9409 asid = COOKIE_TO_ASID(curpmap->pm_cookie); 9410 if (asid == -1) 9411 continue; 9412 bit_set(set->asid_set, asid); 9413 curpmap->pm_cookie = COOKIE_FROM(asid, epoch); 9414 } 9415 } 9416 9417 /* 9418 * Allocate a new ASID for the specified pmap. 9419 */ 9420 static void 9421 pmap_alloc_asid(pmap_t pmap) 9422 { 9423 struct asid_set *set; 9424 int new_asid; 9425 9426 set = pmap->pm_asid_set; 9427 KASSERT(set != NULL, ("%s: NULL asid set", __func__)); 9428 9429 mtx_lock_spin(&set->asid_set_mutex); 9430 9431 /* 9432 * While this processor was waiting to acquire the asid set mutex, 9433 * pmap_reset_asid_set() running on another processor might have 9434 * updated this pmap's cookie to the current epoch. In which case, we 9435 * don't need to allocate a new ASID. 9436 */ 9437 if (COOKIE_TO_EPOCH(pmap->pm_cookie) == set->asid_epoch) 9438 goto out; 9439 9440 bit_ffc_at(set->asid_set, set->asid_next, set->asid_set_size, 9441 &new_asid); 9442 if (new_asid == -1) { 9443 bit_ffc_at(set->asid_set, ASID_FIRST_AVAILABLE, 9444 set->asid_next, &new_asid); 9445 if (new_asid == -1) { 9446 pmap_reset_asid_set(pmap); 9447 bit_ffc_at(set->asid_set, ASID_FIRST_AVAILABLE, 9448 set->asid_set_size, &new_asid); 9449 KASSERT(new_asid != -1, ("ASID allocation failure")); 9450 } 9451 } 9452 bit_set(set->asid_set, new_asid); 9453 set->asid_next = new_asid + 1; 9454 pmap->pm_cookie = COOKIE_FROM(new_asid, set->asid_epoch); 9455 out: 9456 mtx_unlock_spin(&set->asid_set_mutex); 9457 } 9458 9459 static uint64_t __read_mostly ttbr_flags; 9460 9461 /* 9462 * Compute the value that should be stored in ttbr0 to activate the specified 9463 * pmap. This value may change from time to time. 9464 */ 9465 uint64_t 9466 pmap_to_ttbr0(pmap_t pmap) 9467 { 9468 uint64_t ttbr; 9469 9470 ttbr = pmap->pm_ttbr; 9471 ttbr |= ASID_TO_OPERAND(COOKIE_TO_ASID(pmap->pm_cookie)); 9472 ttbr |= ttbr_flags; 9473 9474 return (ttbr); 9475 } 9476 9477 static void 9478 pmap_set_cnp(void *arg) 9479 { 9480 uint64_t ttbr0, ttbr1; 9481 u_int cpuid; 9482 9483 cpuid = *(u_int *)arg; 9484 if (cpuid == curcpu) { 9485 /* 9486 * Set the flags while all CPUs are handling the 9487 * smp_rendezvous so will not call pmap_to_ttbr0. Any calls 9488 * to pmap_to_ttbr0 after this will have the CnP flag set. 9489 * The dsb after invalidating the TLB will act as a barrier 9490 * to ensure all CPUs can observe this change. 9491 */ 9492 ttbr_flags |= TTBR_CnP; 9493 } 9494 9495 ttbr0 = READ_SPECIALREG(ttbr0_el1); 9496 ttbr0 |= TTBR_CnP; 9497 9498 ttbr1 = READ_SPECIALREG(ttbr1_el1); 9499 ttbr1 |= TTBR_CnP; 9500 9501 /* Update ttbr{0,1}_el1 with the CnP flag */ 9502 WRITE_SPECIALREG(ttbr0_el1, ttbr0); 9503 WRITE_SPECIALREG(ttbr1_el1, ttbr1); 9504 isb(); 9505 __asm __volatile("tlbi vmalle1is"); 9506 dsb(ish); 9507 isb(); 9508 } 9509 9510 /* 9511 * Defer enabling some features until we have read the ID registers to know 9512 * if they are supported on all CPUs. 9513 */ 9514 static void 9515 pmap_init_mp(void *dummy __unused) 9516 { 9517 uint64_t reg; 9518 9519 get_kernel_reg(ID_AA64PFR1_EL1, ®); 9520 if (ID_AA64PFR1_BT_VAL(reg) != ID_AA64PFR1_BT_NONE) { 9521 if (bootverbose) 9522 printf("Enabling BTI\n"); 9523 pmap_bti_support = true; 9524 9525 pmap_bti_ranges_zone = uma_zcreate("BTI ranges", 9526 sizeof(struct rs_el), NULL, NULL, NULL, NULL, 9527 UMA_ALIGN_PTR, 0); 9528 } 9529 } 9530 SYSINIT(pmap_init_mp, SI_SUB_CPU, SI_ORDER_ANY, pmap_init_mp, NULL); 9531 9532 /* 9533 * Defer enabling CnP until we have read the ID registers to know if it's 9534 * supported on all CPUs. 9535 */ 9536 static void 9537 pmap_init_cnp(void *dummy __unused) 9538 { 9539 uint64_t reg; 9540 u_int cpuid; 9541 9542 get_kernel_reg(ID_AA64MMFR2_EL1, ®); 9543 if (ID_AA64MMFR2_CnP_VAL(reg) != ID_AA64MMFR2_CnP_NONE) { 9544 if (bootverbose) 9545 printf("Enabling CnP\n"); 9546 cpuid = curcpu; 9547 smp_rendezvous(NULL, pmap_set_cnp, NULL, &cpuid); 9548 } 9549 9550 } 9551 SYSINIT(pmap_init_cnp, SI_SUB_SMP, SI_ORDER_ANY, pmap_init_cnp, NULL); 9552 9553 static bool 9554 pmap_activate_int(struct thread *td, pmap_t pmap) 9555 { 9556 struct asid_set *set; 9557 int epoch; 9558 9559 KASSERT(PCPU_GET(curpmap) != NULL, ("no active pmap")); 9560 KASSERT(pmap != kernel_pmap, ("kernel pmap activation")); 9561 9562 if ((pmap->pm_stage == PM_STAGE1 && pmap == PCPU_GET(curpmap)) || 9563 (pmap->pm_stage == PM_STAGE2 && pmap == PCPU_GET(curvmpmap))) { 9564 /* 9565 * Handle the possibility that the old thread was preempted 9566 * after an "ic" or "tlbi" instruction but before it performed 9567 * a "dsb" instruction. If the old thread migrates to a new 9568 * processor, its completion of a "dsb" instruction on that 9569 * new processor does not guarantee that the "ic" or "tlbi" 9570 * instructions performed on the old processor have completed. 9571 */ 9572 dsb(ish); 9573 return (false); 9574 } 9575 9576 set = pmap->pm_asid_set; 9577 KASSERT(set != NULL, ("%s: NULL asid set", __func__)); 9578 9579 /* 9580 * Ensure that the store to curpmap is globally visible before the 9581 * load from asid_epoch is performed. 9582 */ 9583 if (pmap->pm_stage == PM_STAGE1) 9584 PCPU_SET(curpmap, pmap); 9585 else 9586 PCPU_SET(curvmpmap, pmap); 9587 dsb(ish); 9588 epoch = COOKIE_TO_EPOCH(pmap->pm_cookie); 9589 if (epoch >= 0 && epoch != set->asid_epoch) 9590 pmap_alloc_asid(pmap); 9591 9592 if (pmap->pm_stage == PM_STAGE1) { 9593 uint64_t new_tcr, tcr; 9594 9595 new_tcr = td->td_proc->p_md.md_tcr; 9596 tcr = READ_SPECIALREG(tcr_el1); 9597 if ((tcr & MD_TCR_FIELDS) != new_tcr) { 9598 tcr &= ~MD_TCR_FIELDS; 9599 tcr |= new_tcr; 9600 WRITE_SPECIALREG(tcr_el1, tcr); 9601 } 9602 set_ttbr0(pmap_to_ttbr0(pmap)); 9603 if (PCPU_GET(bcast_tlbi_workaround) != 0) 9604 invalidate_local_icache(); 9605 } 9606 return (true); 9607 } 9608 9609 void 9610 pmap_activate_vm(pmap_t pmap) 9611 { 9612 9613 PMAP_ASSERT_STAGE2(pmap); 9614 9615 (void)pmap_activate_int(NULL, pmap); 9616 } 9617 9618 void 9619 pmap_activate(struct thread *td) 9620 { 9621 pmap_t pmap; 9622 9623 pmap = vmspace_pmap(td->td_proc->p_vmspace); 9624 PMAP_ASSERT_STAGE1(pmap); 9625 critical_enter(); 9626 (void)pmap_activate_int(td, pmap); 9627 critical_exit(); 9628 } 9629 9630 /* 9631 * Activate the thread we are switching to. 9632 * To simplify the assembly in cpu_throw return the new threads pcb. 9633 */ 9634 struct pcb * 9635 pmap_switch(struct thread *new) 9636 { 9637 pcpu_bp_harden bp_harden; 9638 struct pcb *pcb; 9639 uint64_t sctlr; 9640 9641 /* Store the new curthread */ 9642 PCPU_SET(curthread, new); 9643 9644 /* And the new pcb */ 9645 pcb = new->td_pcb; 9646 PCPU_SET(curpcb, pcb); 9647 9648 if ((new->td_proc->p_flag & P_KPROC) == 0) { 9649 sctlr = READ_SPECIALREG(sctlr_el1); 9650 if ((sctlr & SCTLR_USER_MASK) != new->td_md.md_sctlr) { 9651 sctlr &= ~SCTLR_USER_MASK; 9652 sctlr |= new->td_md.md_sctlr; 9653 WRITE_SPECIALREG(sctlr_el1, sctlr); 9654 isb(); 9655 } 9656 } 9657 9658 /* 9659 * TODO: We may need to flush the cache here if switching 9660 * to a user process. 9661 */ 9662 9663 if (pmap_activate_int(new, vmspace_pmap(new->td_proc->p_vmspace))) { 9664 /* 9665 * Stop userspace from training the branch predictor against 9666 * other processes. This will call into a CPU specific 9667 * function that clears the branch predictor state. 9668 */ 9669 bp_harden = PCPU_GET(bp_harden); 9670 if (bp_harden != NULL) 9671 bp_harden(); 9672 } 9673 9674 return (pcb); 9675 } 9676 9677 void 9678 pmap_sync_icache(pmap_t pmap, vm_offset_t va, vm_size_t sz) 9679 { 9680 9681 PMAP_ASSERT_STAGE1(pmap); 9682 KASSERT(ADDR_IS_CANONICAL(va), 9683 ("%s: Address not in canonical form: %lx", __func__, va)); 9684 9685 if (ADDR_IS_KERNEL(va)) { 9686 cpu_icache_sync_range((void *)va, sz); 9687 } else { 9688 u_int len, offset; 9689 vm_paddr_t pa; 9690 9691 /* Find the length of data in this page to flush */ 9692 offset = va & PAGE_MASK; 9693 len = imin(PAGE_SIZE - offset, sz); 9694 9695 while (sz != 0) { 9696 /* Extract the physical address & find it in the DMAP */ 9697 pa = pmap_extract(pmap, va); 9698 if (pa != 0) 9699 cpu_icache_sync_range(PHYS_TO_DMAP(pa), len); 9700 9701 /* Move to the next page */ 9702 sz -= len; 9703 va += len; 9704 /* Set the length for the next iteration */ 9705 len = imin(PAGE_SIZE, sz); 9706 } 9707 } 9708 } 9709 9710 static int 9711 pmap_stage2_fault(pmap_t pmap, uint64_t esr, uint64_t far) 9712 { 9713 pd_entry_t *pdep; 9714 pt_entry_t *ptep, pte; 9715 int rv, lvl, dfsc; 9716 9717 PMAP_ASSERT_STAGE2(pmap); 9718 rv = KERN_FAILURE; 9719 9720 /* Data and insn aborts use same encoding for FSC field. */ 9721 dfsc = esr & ISS_DATA_DFSC_MASK; 9722 switch (dfsc) { 9723 case ISS_DATA_DFSC_TF_L0: 9724 case ISS_DATA_DFSC_TF_L1: 9725 case ISS_DATA_DFSC_TF_L2: 9726 case ISS_DATA_DFSC_TF_L3: 9727 PMAP_LOCK(pmap); 9728 pdep = pmap_pde(pmap, far, &lvl); 9729 if (pdep == NULL || lvl != (dfsc - ISS_DATA_DFSC_TF_L1)) { 9730 PMAP_UNLOCK(pmap); 9731 break; 9732 } 9733 9734 switch (lvl) { 9735 case 0: 9736 ptep = pmap_l0_to_l1(pdep, far); 9737 break; 9738 case 1: 9739 ptep = pmap_l1_to_l2(pdep, far); 9740 break; 9741 case 2: 9742 ptep = pmap_l2_to_l3(pdep, far); 9743 break; 9744 default: 9745 panic("%s: Invalid pde level %d", __func__,lvl); 9746 } 9747 goto fault_exec; 9748 9749 case ISS_DATA_DFSC_AFF_L1: 9750 case ISS_DATA_DFSC_AFF_L2: 9751 case ISS_DATA_DFSC_AFF_L3: 9752 PMAP_LOCK(pmap); 9753 ptep = pmap_pte(pmap, far, &lvl); 9754 fault_exec: 9755 if (ptep != NULL && (pte = pmap_load(ptep)) != 0) { 9756 /* 9757 * If accessing an executable page invalidate 9758 * the I-cache so it will be valid when we 9759 * continue execution in the guest. The D-cache 9760 * is assumed to already be clean to the Point 9761 * of Coherency. 9762 */ 9763 if ((pte & ATTR_S2_XN_MASK) != 9764 ATTR_S2_XN(ATTR_S2_XN_ALL)) { 9765 invalidate_icache(); 9766 } 9767 pmap_set_bits(ptep, ATTR_AF | ATTR_DESCR_VALID); 9768 rv = KERN_SUCCESS; 9769 } 9770 PMAP_UNLOCK(pmap); 9771 break; 9772 } 9773 9774 return (rv); 9775 } 9776 9777 int 9778 pmap_fault(pmap_t pmap, uint64_t esr, uint64_t far) 9779 { 9780 pt_entry_t pte, *ptep; 9781 register_t intr; 9782 uint64_t ec, par; 9783 int lvl, rv; 9784 9785 rv = KERN_FAILURE; 9786 9787 ec = ESR_ELx_EXCEPTION(esr); 9788 switch (ec) { 9789 case EXCP_INSN_ABORT_L: 9790 case EXCP_INSN_ABORT: 9791 case EXCP_DATA_ABORT_L: 9792 case EXCP_DATA_ABORT: 9793 break; 9794 default: 9795 return (rv); 9796 } 9797 9798 if (pmap->pm_stage == PM_STAGE2) 9799 return (pmap_stage2_fault(pmap, esr, far)); 9800 9801 /* Data and insn aborts use same encoding for FSC field. */ 9802 switch (esr & ISS_DATA_DFSC_MASK) { 9803 case ISS_DATA_DFSC_AFF_L1: 9804 case ISS_DATA_DFSC_AFF_L2: 9805 case ISS_DATA_DFSC_AFF_L3: 9806 PMAP_LOCK(pmap); 9807 ptep = pmap_pte(pmap, far, &lvl); 9808 if (ptep != NULL) { 9809 pmap_set_bits(ptep, ATTR_AF); 9810 rv = KERN_SUCCESS; 9811 /* 9812 * XXXMJ as an optimization we could mark the entry 9813 * dirty if this is a write fault. 9814 */ 9815 } 9816 PMAP_UNLOCK(pmap); 9817 break; 9818 case ISS_DATA_DFSC_PF_L1: 9819 case ISS_DATA_DFSC_PF_L2: 9820 case ISS_DATA_DFSC_PF_L3: 9821 if ((ec != EXCP_DATA_ABORT_L && ec != EXCP_DATA_ABORT) || 9822 (esr & ISS_DATA_WnR) == 0) 9823 return (rv); 9824 PMAP_LOCK(pmap); 9825 ptep = pmap_pte(pmap, far, &lvl); 9826 if (ptep != NULL && 9827 ((pte = pmap_load(ptep)) & ATTR_SW_DBM) != 0) { 9828 if ((pte & ATTR_S1_AP_RW_BIT) == 9829 ATTR_S1_AP(ATTR_S1_AP_RO)) { 9830 pmap_clear_bits(ptep, ATTR_S1_AP_RW_BIT); 9831 pmap_s1_invalidate_page(pmap, far, true); 9832 } 9833 rv = KERN_SUCCESS; 9834 } 9835 PMAP_UNLOCK(pmap); 9836 break; 9837 case ISS_DATA_DFSC_TF_L0: 9838 case ISS_DATA_DFSC_TF_L1: 9839 case ISS_DATA_DFSC_TF_L2: 9840 case ISS_DATA_DFSC_TF_L3: 9841 /* 9842 * Retry the translation. A break-before-make sequence can 9843 * produce a transient fault. 9844 */ 9845 if (pmap == kernel_pmap) { 9846 /* 9847 * The translation fault may have occurred within a 9848 * critical section. Therefore, we must check the 9849 * address without acquiring the kernel pmap's lock. 9850 */ 9851 if (pmap_klookup(far, NULL)) 9852 rv = KERN_SUCCESS; 9853 } else { 9854 bool owned; 9855 9856 /* 9857 * In the EFIRT driver we lock the pmap before 9858 * calling into the runtime service. As the lock 9859 * is already owned by the current thread skip 9860 * locking it again. 9861 */ 9862 owned = PMAP_OWNED(pmap); 9863 if (!owned) 9864 PMAP_LOCK(pmap); 9865 /* Ask the MMU to check the address. */ 9866 intr = intr_disable(); 9867 par = arm64_address_translate_s1e0r(far); 9868 intr_restore(intr); 9869 if (!owned) 9870 PMAP_UNLOCK(pmap); 9871 9872 /* 9873 * If the translation was successful, then we can 9874 * return success to the trap handler. 9875 */ 9876 if (PAR_SUCCESS(par)) 9877 rv = KERN_SUCCESS; 9878 } 9879 break; 9880 } 9881 9882 return (rv); 9883 } 9884 9885 /* 9886 * Increase the starting virtual address of the given mapping if a 9887 * different alignment might result in more superpage mappings. 9888 */ 9889 void 9890 pmap_align_superpage(vm_object_t object, vm_ooffset_t offset, 9891 vm_offset_t *addr, vm_size_t size) 9892 { 9893 vm_offset_t superpage_offset; 9894 9895 if (size < L3C_SIZE) 9896 return; 9897 if (object != NULL && (object->flags & OBJ_COLORED) != 0) 9898 offset += ptoa(object->pg_color); 9899 9900 /* 9901 * Considering the object's physical alignment, is the mapping large 9902 * enough to encompass an L2 (2MB/32MB) superpage ... 9903 */ 9904 superpage_offset = offset & L2_OFFSET; 9905 if (size - ((L2_SIZE - superpage_offset) & L2_OFFSET) >= L2_SIZE) { 9906 /* 9907 * If the virtual and physical alignments differ, then 9908 * increase the virtual address so that the alignments match. 9909 */ 9910 if ((*addr & L2_OFFSET) < superpage_offset) 9911 *addr = (*addr & ~L2_OFFSET) + superpage_offset; 9912 else if ((*addr & L2_OFFSET) > superpage_offset) 9913 *addr = ((*addr + L2_OFFSET) & ~L2_OFFSET) + 9914 superpage_offset; 9915 return; 9916 } 9917 /* ... or an L3C (64KB/2MB) superpage? */ 9918 superpage_offset = offset & L3C_OFFSET; 9919 if (size - ((L3C_SIZE - superpage_offset) & L3C_OFFSET) >= L3C_SIZE) { 9920 if ((*addr & L3C_OFFSET) < superpage_offset) 9921 *addr = (*addr & ~L3C_OFFSET) + superpage_offset; 9922 else if ((*addr & L3C_OFFSET) > superpage_offset) 9923 *addr = ((*addr + L3C_OFFSET) & ~L3C_OFFSET) + 9924 superpage_offset; 9925 } 9926 } 9927 9928 /** 9929 * Get the kernel virtual address of a set of physical pages. If there are 9930 * physical addresses not covered by the DMAP perform a transient mapping 9931 * that will be removed when calling pmap_unmap_io_transient. 9932 * 9933 * \param page The pages the caller wishes to obtain the virtual 9934 * address on the kernel memory map. 9935 * \param vaddr On return contains the kernel virtual memory address 9936 * of the pages passed in the page parameter. 9937 * \param count Number of pages passed in. 9938 * \param can_fault true if the thread using the mapped pages can take 9939 * page faults, false otherwise. 9940 * 9941 * \returns true if the caller must call pmap_unmap_io_transient when 9942 * finished or false otherwise. 9943 * 9944 */ 9945 bool 9946 pmap_map_io_transient(vm_page_t page[], void *vaddr[], int count, 9947 bool can_fault) 9948 { 9949 vm_paddr_t paddr; 9950 vmem_addr_t addr; 9951 bool needs_mapping; 9952 int error __diagused, i; 9953 9954 /* 9955 * Allocate any KVA space that we need, this is done in a separate 9956 * loop to prevent calling vmem_alloc while pinned. 9957 */ 9958 needs_mapping = false; 9959 for (i = 0; i < count; i++) { 9960 paddr = VM_PAGE_TO_PHYS(page[i]); 9961 if (__predict_false(!PHYS_IN_DMAP(paddr))) { 9962 error = vmem_alloc(kernel_arena, PAGE_SIZE, 9963 M_BESTFIT | M_WAITOK, &addr); 9964 KASSERT(error == 0, ("vmem_alloc failed: %d", error)); 9965 vaddr[i] = (void *)addr; 9966 needs_mapping = true; 9967 } else { 9968 vaddr[i] = PHYS_TO_DMAP(paddr); 9969 } 9970 } 9971 9972 /* Exit early if everything is covered by the DMAP */ 9973 if (!needs_mapping) 9974 return (false); 9975 9976 if (!can_fault) 9977 sched_pin(); 9978 for (i = 0; i < count; i++) { 9979 paddr = VM_PAGE_TO_PHYS(page[i]); 9980 if (!PHYS_IN_DMAP(paddr)) { 9981 panic( 9982 "pmap_map_io_transient: TODO: Map out of DMAP data"); 9983 } 9984 } 9985 9986 return (needs_mapping); 9987 } 9988 9989 void 9990 pmap_unmap_io_transient(vm_page_t page[], void *vaddr[], int count, 9991 bool can_fault) 9992 { 9993 vm_paddr_t paddr; 9994 int i; 9995 9996 if (!can_fault) 9997 sched_unpin(); 9998 for (i = 0; i < count; i++) { 9999 paddr = VM_PAGE_TO_PHYS(page[i]); 10000 if (!PHYS_IN_DMAP(paddr)) { 10001 panic("ARM64TODO: pmap_unmap_io_transient: Unmap data"); 10002 } 10003 } 10004 } 10005 10006 bool 10007 pmap_is_valid_memattr(pmap_t pmap __unused, vm_memattr_t mode) 10008 { 10009 10010 return (mode >= 0 && mode < VM_MEMATTR_END); 10011 } 10012 10013 static void * 10014 bti_dup_range(void *ctx __unused, void *data) 10015 { 10016 struct rs_el *node, *new_node; 10017 10018 new_node = uma_zalloc(pmap_bti_ranges_zone, M_NOWAIT); 10019 if (new_node == NULL) 10020 return (NULL); 10021 node = data; 10022 memcpy(new_node, node, sizeof(*node)); 10023 return (new_node); 10024 } 10025 10026 static void 10027 bti_free_range(void *ctx __unused, void *node) 10028 { 10029 10030 uma_zfree(pmap_bti_ranges_zone, node); 10031 } 10032 10033 static int 10034 pmap_bti_assign(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) 10035 { 10036 struct rs_el *rs; 10037 int error; 10038 10039 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 10040 PMAP_ASSERT_STAGE1(pmap); 10041 MPASS(pmap->pm_bti != NULL); 10042 rs = uma_zalloc(pmap_bti_ranges_zone, M_NOWAIT); 10043 if (rs == NULL) 10044 return (ENOMEM); 10045 error = rangeset_insert(pmap->pm_bti, sva, eva, rs); 10046 if (error != 0) 10047 uma_zfree(pmap_bti_ranges_zone, rs); 10048 return (error); 10049 } 10050 10051 static void 10052 pmap_bti_deassign_all(pmap_t pmap) 10053 { 10054 10055 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 10056 if (pmap->pm_bti != NULL) 10057 rangeset_remove_all(pmap->pm_bti); 10058 } 10059 10060 /* 10061 * Returns true if the BTI setting is the same across the specified address 10062 * range, and false otherwise. When returning true, updates the referenced PTE 10063 * to reflect the BTI setting. 10064 * 10065 * Only stage 1 pmaps support BTI. The kernel pmap is always a stage 1 pmap 10066 * that has the same BTI setting implicitly across its entire address range. 10067 */ 10068 static bool 10069 pmap_bti_same(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, pt_entry_t *pte) 10070 { 10071 struct rs_el *rs; 10072 vm_offset_t va; 10073 10074 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 10075 KASSERT(ADDR_IS_CANONICAL(sva), 10076 ("%s: Start address not in canonical form: %lx", __func__, sva)); 10077 KASSERT(ADDR_IS_CANONICAL(eva), 10078 ("%s: End address not in canonical form: %lx", __func__, eva)); 10079 KASSERT((*pte & ATTR_S1_GP) == 0, 10080 ("%s: pte %lx has ATTR_S1_GP preset", __func__, *pte)); 10081 10082 if (pmap == kernel_pmap) { 10083 *pte |= ATTR_KERN_GP; 10084 return (true); 10085 } 10086 if (pmap->pm_bti == NULL) 10087 return (true); 10088 PMAP_ASSERT_STAGE1(pmap); 10089 rs = rangeset_containing(pmap->pm_bti, sva); 10090 if (rs == NULL) 10091 return (rangeset_empty(pmap->pm_bti, sva, eva)); 10092 while ((va = rs->re_end) < eva) { 10093 if ((rs = rangeset_beginning(pmap->pm_bti, va)) == NULL) 10094 return (false); 10095 } 10096 *pte |= ATTR_S1_GP; 10097 return (true); 10098 } 10099 10100 static pt_entry_t 10101 pmap_pte_bti(pmap_t pmap, vm_offset_t va) 10102 { 10103 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 10104 MPASS(ADDR_IS_CANONICAL(va)); 10105 10106 if (pmap->pm_stage != PM_STAGE1) 10107 return (0); 10108 if (pmap == kernel_pmap) 10109 return (ATTR_KERN_GP); 10110 if (pmap->pm_bti != NULL && 10111 rangeset_containing(pmap->pm_bti, va) != NULL) 10112 return (ATTR_S1_GP); 10113 return (0); 10114 } 10115 10116 static void 10117 pmap_bti_on_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) 10118 { 10119 10120 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 10121 if (pmap->pm_bti != NULL) 10122 rangeset_remove(pmap->pm_bti, sva, eva); 10123 } 10124 10125 static int 10126 pmap_bti_copy(pmap_t dst_pmap, pmap_t src_pmap) 10127 { 10128 10129 PMAP_LOCK_ASSERT(dst_pmap, MA_OWNED); 10130 PMAP_LOCK_ASSERT(src_pmap, MA_OWNED); 10131 MPASS(src_pmap->pm_stage == dst_pmap->pm_stage); 10132 MPASS(src_pmap->pm_bti != NULL); 10133 MPASS(dst_pmap->pm_bti != NULL); 10134 if (src_pmap->pm_bti->rs_data_ctx == NULL) 10135 return (0); 10136 return (rangeset_copy(dst_pmap->pm_bti, src_pmap->pm_bti)); 10137 } 10138 10139 static void 10140 pmap_bti_update_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, bool set) 10141 { 10142 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 10143 PMAP_ASSERT_STAGE1(pmap); 10144 10145 pmap_mask_set_locked(pmap, sva, eva, ATTR_S1_GP, set ? ATTR_S1_GP : 0, 10146 true); 10147 } 10148 10149 int 10150 pmap_bti_set(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) 10151 { 10152 int error; 10153 10154 if (pmap->pm_bti == NULL) 10155 return (0); 10156 if (!ADDR_IS_CANONICAL(sva) || !ADDR_IS_CANONICAL(eva)) 10157 return (EINVAL); 10158 if (pmap->pm_stage != PM_STAGE1) 10159 return (EINVAL); 10160 if (eva <= sva || ADDR_IS_KERNEL(eva)) 10161 return (EFAULT); 10162 10163 sva = trunc_page(sva); 10164 eva = round_page(eva); 10165 for (;;) { 10166 PMAP_LOCK(pmap); 10167 error = pmap_bti_assign(pmap, sva, eva); 10168 if (error == 0) 10169 pmap_bti_update_range(pmap, sva, eva, true); 10170 PMAP_UNLOCK(pmap); 10171 if (error != ENOMEM) 10172 break; 10173 vm_wait(NULL); 10174 } 10175 return (error); 10176 } 10177 10178 #if defined(KASAN) || defined(KMSAN) 10179 static pd_entry_t *pmap_san_early_l2; 10180 10181 #define SAN_BOOTSTRAP_L2_SIZE (1 * L2_SIZE) 10182 #define SAN_BOOTSTRAP_SIZE (2 * PAGE_SIZE) 10183 static vm_offset_t __nosanitizeaddress 10184 pmap_san_enter_bootstrap_alloc_l2(void) 10185 { 10186 static uint8_t bootstrap_data[SAN_BOOTSTRAP_L2_SIZE] __aligned(L2_SIZE); 10187 static size_t offset = 0; 10188 vm_offset_t addr; 10189 10190 if (offset + L2_SIZE > sizeof(bootstrap_data)) { 10191 panic("%s: out of memory for the bootstrap shadow map L2 entries", 10192 __func__); 10193 } 10194 10195 addr = (uintptr_t)&bootstrap_data[offset]; 10196 offset += L2_SIZE; 10197 return (addr); 10198 } 10199 10200 /* 10201 * SAN L1 + L2 pages, maybe L3 entries later? 10202 */ 10203 static vm_offset_t __nosanitizeaddress 10204 pmap_san_enter_bootstrap_alloc_pages(int npages) 10205 { 10206 static uint8_t bootstrap_data[SAN_BOOTSTRAP_SIZE] __aligned(PAGE_SIZE); 10207 static size_t offset = 0; 10208 vm_offset_t addr; 10209 10210 if (offset + (npages * PAGE_SIZE) > sizeof(bootstrap_data)) { 10211 panic("%s: out of memory for the bootstrap shadow map", 10212 __func__); 10213 } 10214 10215 addr = (uintptr_t)&bootstrap_data[offset]; 10216 offset += (npages * PAGE_SIZE); 10217 return (addr); 10218 } 10219 10220 static void __nosanitizeaddress 10221 pmap_san_enter_bootstrap(void) 10222 { 10223 vm_offset_t freemempos; 10224 10225 /* L1, L2 */ 10226 freemempos = pmap_san_enter_bootstrap_alloc_pages(2); 10227 bs_state.freemempos = freemempos; 10228 bs_state.va = KASAN_MIN_ADDRESS; 10229 pmap_bootstrap_l1_table(&bs_state); 10230 pmap_san_early_l2 = bs_state.l2; 10231 } 10232 10233 static vm_page_t 10234 pmap_san_enter_alloc_l3(void) 10235 { 10236 vm_page_t m; 10237 10238 m = vm_page_alloc_noobj(VM_ALLOC_INTERRUPT | VM_ALLOC_WIRED | 10239 VM_ALLOC_ZERO); 10240 if (m == NULL) 10241 panic("%s: no memory to grow shadow map", __func__); 10242 return (m); 10243 } 10244 10245 static vm_page_t 10246 pmap_san_enter_alloc_l2(void) 10247 { 10248 return (vm_page_alloc_noobj_contig(VM_ALLOC_WIRED | VM_ALLOC_ZERO, 10249 Ln_ENTRIES, 0, ~0ul, L2_SIZE, 0, VM_MEMATTR_DEFAULT)); 10250 } 10251 10252 void __nosanitizeaddress __nosanitizememory 10253 pmap_san_enter(vm_offset_t va) 10254 { 10255 pd_entry_t *l1, *l2; 10256 pt_entry_t *l3; 10257 vm_page_t m; 10258 10259 if (virtual_avail == 0) { 10260 vm_offset_t block; 10261 int slot; 10262 bool first; 10263 10264 /* Temporary shadow map prior to pmap_bootstrap(). */ 10265 first = pmap_san_early_l2 == NULL; 10266 if (first) 10267 pmap_san_enter_bootstrap(); 10268 10269 l2 = pmap_san_early_l2; 10270 slot = pmap_l2_index(va); 10271 10272 if ((pmap_load(&l2[slot]) & ATTR_DESCR_VALID) == 0) { 10273 MPASS(first); 10274 block = pmap_san_enter_bootstrap_alloc_l2(); 10275 pmap_store(&l2[slot], 10276 PHYS_TO_PTE(pmap_early_vtophys(block)) | 10277 PMAP_SAN_PTE_BITS | L2_BLOCK); 10278 dmb(ishst); 10279 } 10280 10281 return; 10282 } 10283 10284 mtx_assert(&kernel_map->system_mtx, MA_OWNED); 10285 l1 = pmap_l1(kernel_pmap, va); 10286 MPASS(l1 != NULL); 10287 if ((pmap_load(l1) & ATTR_DESCR_VALID) == 0) { 10288 m = pmap_san_enter_alloc_l3(); 10289 pmap_store(l1, VM_PAGE_TO_PTE(m) | L1_TABLE); 10290 } 10291 l2 = pmap_l1_to_l2(l1, va); 10292 if ((pmap_load(l2) & ATTR_DESCR_VALID) == 0) { 10293 m = pmap_san_enter_alloc_l2(); 10294 if (m != NULL) { 10295 pmap_store(l2, VM_PAGE_TO_PTE(m) | 10296 PMAP_SAN_PTE_BITS | L2_BLOCK); 10297 } else { 10298 m = pmap_san_enter_alloc_l3(); 10299 pmap_store(l2, VM_PAGE_TO_PTE(m) | L2_TABLE); 10300 } 10301 dmb(ishst); 10302 } 10303 if ((pmap_load(l2) & ATTR_DESCR_MASK) == L2_BLOCK) 10304 return; 10305 l3 = pmap_l2_to_l3(l2, va); 10306 if ((pmap_load(l3) & ATTR_DESCR_VALID) != 0) 10307 return; 10308 m = pmap_san_enter_alloc_l3(); 10309 pmap_store(l3, VM_PAGE_TO_PTE(m) | PMAP_SAN_PTE_BITS | L3_PAGE); 10310 dmb(ishst); 10311 } 10312 #endif /* KASAN || KMSAN */ 10313 10314 /* 10315 * Track a range of the kernel's virtual address space that is contiguous 10316 * in various mapping attributes. 10317 */ 10318 struct pmap_kernel_map_range { 10319 vm_offset_t sva; 10320 pt_entry_t attrs; 10321 int l3pages; 10322 int l3contig; 10323 int l2blocks; 10324 int l2contig; 10325 int l1blocks; 10326 }; 10327 10328 static void 10329 sysctl_kmaps_dump(struct sbuf *sb, struct pmap_kernel_map_range *range, 10330 vm_offset_t eva) 10331 { 10332 const char *mode; 10333 int index; 10334 10335 if (eva <= range->sva) 10336 return; 10337 10338 index = range->attrs & ATTR_S1_IDX_MASK; 10339 switch (index) { 10340 case ATTR_S1_IDX(VM_MEMATTR_DEVICE_NP): 10341 mode = "DEV-NP"; 10342 break; 10343 case ATTR_S1_IDX(VM_MEMATTR_DEVICE): 10344 mode = "DEV"; 10345 break; 10346 case ATTR_S1_IDX(VM_MEMATTR_UNCACHEABLE): 10347 mode = "UC"; 10348 break; 10349 case ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK): 10350 mode = "WB"; 10351 break; 10352 case ATTR_S1_IDX(VM_MEMATTR_WRITE_THROUGH): 10353 mode = "WT"; 10354 break; 10355 case ATTR_S1_IDX(VM_MEMATTR_TAGGED): 10356 mode = "TAGGED"; 10357 break; 10358 default: 10359 printf( 10360 "%s: unknown memory type %x for range 0x%016lx-0x%016lx\n", 10361 __func__, index, range->sva, eva); 10362 mode = "??"; 10363 break; 10364 } 10365 10366 sbuf_printf(sb, "0x%016lx-0x%016lx r%c%c%c%c%c %6s %d %d %d %d %d\n", 10367 range->sva, eva, 10368 (range->attrs & ATTR_S1_AP_RW_BIT) == ATTR_S1_AP_RW ? 'w' : '-', 10369 (range->attrs & ATTR_S1_PXN) != 0 ? '-' : 'x', 10370 (range->attrs & ATTR_S1_UXN) != 0 ? '-' : 'X', 10371 (range->attrs & ATTR_S1_AP(ATTR_S1_AP_USER)) != 0 ? 'u' : 's', 10372 (range->attrs & ATTR_S1_GP) != 0 ? 'g' : '-', 10373 mode, range->l1blocks, range->l2contig, range->l2blocks, 10374 range->l3contig, range->l3pages); 10375 10376 /* Reset to sentinel value. */ 10377 range->sva = 0xfffffffffffffffful; 10378 } 10379 10380 /* 10381 * Determine whether the attributes specified by a page table entry match those 10382 * being tracked by the current range. 10383 */ 10384 static bool 10385 sysctl_kmaps_match(struct pmap_kernel_map_range *range, pt_entry_t attrs) 10386 { 10387 10388 return (range->attrs == attrs); 10389 } 10390 10391 static void 10392 sysctl_kmaps_reinit(struct pmap_kernel_map_range *range, vm_offset_t va, 10393 pt_entry_t attrs) 10394 { 10395 10396 memset(range, 0, sizeof(*range)); 10397 range->sva = va; 10398 range->attrs = attrs; 10399 } 10400 10401 /* Get the block/page attributes that correspond to the table attributes */ 10402 static pt_entry_t 10403 sysctl_kmaps_table_attrs(pd_entry_t table) 10404 { 10405 pt_entry_t attrs; 10406 10407 attrs = 0; 10408 if ((table & TATTR_UXN_TABLE) != 0) 10409 attrs |= ATTR_S1_UXN; 10410 if ((table & TATTR_PXN_TABLE) != 0) 10411 attrs |= ATTR_S1_PXN; 10412 if ((table & TATTR_AP_TABLE_RO) != 0) 10413 attrs |= ATTR_S1_AP(ATTR_S1_AP_RO); 10414 10415 return (attrs); 10416 } 10417 10418 /* Read the block/page attributes we care about */ 10419 static pt_entry_t 10420 sysctl_kmaps_block_attrs(pt_entry_t block) 10421 { 10422 return (block & (ATTR_S1_AP_MASK | ATTR_S1_XN | ATTR_S1_IDX_MASK | 10423 ATTR_S1_GP)); 10424 } 10425 10426 /* 10427 * Given a leaf PTE, derive the mapping's attributes. If they do not match 10428 * those of the current run, dump the address range and its attributes, and 10429 * begin a new run. 10430 */ 10431 static void 10432 sysctl_kmaps_check(struct sbuf *sb, struct pmap_kernel_map_range *range, 10433 vm_offset_t va, pd_entry_t l0e, pd_entry_t l1e, pd_entry_t l2e, 10434 pt_entry_t l3e) 10435 { 10436 pt_entry_t attrs; 10437 10438 attrs = sysctl_kmaps_table_attrs(l0e); 10439 10440 if ((l1e & ATTR_DESCR_TYPE_MASK) == ATTR_DESCR_TYPE_BLOCK) { 10441 attrs |= sysctl_kmaps_block_attrs(l1e); 10442 goto done; 10443 } 10444 attrs |= sysctl_kmaps_table_attrs(l1e); 10445 10446 if ((l2e & ATTR_DESCR_TYPE_MASK) == ATTR_DESCR_TYPE_BLOCK) { 10447 attrs |= sysctl_kmaps_block_attrs(l2e); 10448 goto done; 10449 } 10450 attrs |= sysctl_kmaps_table_attrs(l2e); 10451 attrs |= sysctl_kmaps_block_attrs(l3e); 10452 10453 done: 10454 if (range->sva > va || !sysctl_kmaps_match(range, attrs)) { 10455 sysctl_kmaps_dump(sb, range, va); 10456 sysctl_kmaps_reinit(range, va, attrs); 10457 } 10458 } 10459 10460 static int 10461 sysctl_kmaps(SYSCTL_HANDLER_ARGS) 10462 { 10463 struct pmap_kernel_map_range range; 10464 struct sbuf sbuf, *sb; 10465 pd_entry_t l0e, *l1, l1e, *l2, l2e; 10466 pt_entry_t *l3, l3e; 10467 vm_offset_t sva; 10468 vm_paddr_t pa; 10469 int error, i, j, k, l; 10470 10471 error = sysctl_wire_old_buffer(req, 0); 10472 if (error != 0) 10473 return (error); 10474 sb = &sbuf; 10475 sbuf_new_for_sysctl(sb, NULL, PAGE_SIZE, req); 10476 10477 /* Sentinel value. */ 10478 range.sva = 0xfffffffffffffffful; 10479 10480 /* 10481 * Iterate over the kernel page tables without holding the kernel pmap 10482 * lock. Kernel page table pages are never freed, so at worst we will 10483 * observe inconsistencies in the output. 10484 */ 10485 for (sva = 0xffff000000000000ul, i = pmap_l0_index(sva); i < Ln_ENTRIES; 10486 i++) { 10487 if (i == pmap_l0_index(DMAP_MIN_ADDRESS)) 10488 sbuf_printf(sb, "\nDirect map:\n"); 10489 else if (i == pmap_l0_index(VM_MIN_KERNEL_ADDRESS)) 10490 sbuf_printf(sb, "\nKernel map:\n"); 10491 #ifdef KASAN 10492 else if (i == pmap_l0_index(KASAN_MIN_ADDRESS)) 10493 sbuf_printf(sb, "\nKASAN shadow map:\n"); 10494 #endif 10495 #ifdef KMSAN 10496 else if (i == pmap_l0_index(KMSAN_SHAD_MIN_ADDRESS)) 10497 sbuf_printf(sb, "\nKMSAN shadow map:\n"); 10498 else if (i == pmap_l0_index(KMSAN_ORIG_MIN_ADDRESS)) 10499 sbuf_printf(sb, "\nKMSAN origin map:\n"); 10500 #endif 10501 10502 l0e = kernel_pmap->pm_l0[i]; 10503 if ((l0e & ATTR_DESCR_VALID) == 0) { 10504 sysctl_kmaps_dump(sb, &range, sva); 10505 sva += L0_SIZE; 10506 continue; 10507 } 10508 pa = PTE_TO_PHYS(l0e); 10509 l1 = PHYS_TO_DMAP(pa); 10510 10511 for (j = pmap_l1_index(sva); j < Ln_ENTRIES; j++) { 10512 l1e = l1[j]; 10513 if ((l1e & ATTR_DESCR_VALID) == 0) { 10514 sysctl_kmaps_dump(sb, &range, sva); 10515 sva += L1_SIZE; 10516 continue; 10517 } 10518 if ((l1e & ATTR_DESCR_MASK) == L1_BLOCK) { 10519 PMAP_ASSERT_L1_BLOCKS_SUPPORTED; 10520 sysctl_kmaps_check(sb, &range, sva, l0e, l1e, 10521 0, 0); 10522 range.l1blocks++; 10523 sva += L1_SIZE; 10524 continue; 10525 } 10526 pa = PTE_TO_PHYS(l1e); 10527 l2 = PHYS_TO_DMAP(pa); 10528 10529 for (k = pmap_l2_index(sva); k < Ln_ENTRIES; k++) { 10530 l2e = l2[k]; 10531 if ((l2e & ATTR_DESCR_VALID) == 0) { 10532 sysctl_kmaps_dump(sb, &range, sva); 10533 sva += L2_SIZE; 10534 continue; 10535 } 10536 if ((l2e & ATTR_DESCR_MASK) == L2_BLOCK) { 10537 sysctl_kmaps_check(sb, &range, sva, 10538 l0e, l1e, l2e, 0); 10539 if ((l2e & ATTR_CONTIGUOUS) != 0) 10540 range.l2contig += 10541 k % L2C_ENTRIES == 0 ? 10542 1 : 0; 10543 else 10544 range.l2blocks++; 10545 sva += L2_SIZE; 10546 continue; 10547 } 10548 pa = PTE_TO_PHYS(l2e); 10549 l3 = PHYS_TO_DMAP(pa); 10550 10551 for (l = pmap_l3_index(sva); l < Ln_ENTRIES; 10552 l++, sva += L3_SIZE) { 10553 l3e = l3[l]; 10554 if ((l3e & ATTR_DESCR_VALID) == 0) { 10555 sysctl_kmaps_dump(sb, &range, 10556 sva); 10557 continue; 10558 } 10559 sysctl_kmaps_check(sb, &range, sva, 10560 l0e, l1e, l2e, l3e); 10561 if ((l3e & ATTR_CONTIGUOUS) != 0) 10562 range.l3contig += 10563 l % L3C_ENTRIES == 0 ? 10564 1 : 0; 10565 else 10566 range.l3pages++; 10567 } 10568 } 10569 } 10570 } 10571 10572 error = sbuf_finish(sb); 10573 sbuf_delete(sb); 10574 return (error); 10575 } 10576 SYSCTL_OID(_vm_pmap, OID_AUTO, kernel_maps, 10577 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE | CTLFLAG_SKIP, 10578 NULL, 0, sysctl_kmaps, "A", 10579 "Dump kernel address layout"); 10580 10581 10582 void pagezero_simple(void *); 10583 void pagezero_cache(void *); 10584 void pagezero_mops(void *); 10585 10586 DEFINE_IFUNC(static, void, pagezero, (void *)) 10587 { 10588 uint32_t dczid_el0; 10589 10590 dczid_el0 = READ_SPECIALREG(dczid_el0); 10591 10592 if (elf_hwcap2 & HWCAP2_MOPS) 10593 return (pagezero_mops); 10594 else if ((dczid_el0 & DCZID_DZP) == 0) 10595 return (pagezero_cache); 10596 else 10597 return (pagezero_simple); 10598 } 10599