1 /*- 2 * SPDX-License-Identifier: BSD-4-Clause 3 * 4 * Copyright (c) 1991 Regents of the University of California. 5 * All rights reserved. 6 * Copyright (c) 1994 John S. Dyson 7 * All rights reserved. 8 * Copyright (c) 1994 David Greenman 9 * All rights reserved. 10 * Copyright (c) 2003 Peter Wemm 11 * All rights reserved. 12 * Copyright (c) 2005-2010 Alan L. Cox <alc@cs.rice.edu> 13 * All rights reserved. 14 * 15 * This code is derived from software contributed to Berkeley by 16 * the Systems Programming Group of the University of Utah Computer 17 * Science Department and William Jolitz of UUNET Technologies Inc. 18 * 19 * Redistribution and use in source and binary forms, with or without 20 * modification, are permitted provided that the following conditions 21 * are met: 22 * 1. Redistributions of source code must retain the above copyright 23 * notice, this list of conditions and the following disclaimer. 24 * 2. Redistributions in binary form must reproduce the above copyright 25 * notice, this list of conditions and the following disclaimer in the 26 * documentation and/or other materials provided with the distribution. 27 * 3. All advertising materials mentioning features or use of this software 28 * must display the following acknowledgement: 29 * This product includes software developed by the University of 30 * California, Berkeley and its contributors. 31 * 4. Neither the name of the University nor the names of its contributors 32 * may be used to endorse or promote products derived from this software 33 * without specific prior written permission. 34 * 35 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 36 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 37 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 38 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 39 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 40 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 41 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 42 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 43 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 44 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 45 * SUCH DAMAGE. 46 */ 47 /*- 48 * Copyright (c) 2003 Networks Associates Technology, Inc. 49 * Copyright (c) 2014-2020 The FreeBSD Foundation 50 * All rights reserved. 51 * 52 * This software was developed for the FreeBSD Project by Jake Burkholder, 53 * Safeport Network Services, and Network Associates Laboratories, the 54 * Security Research Division of Network Associates, Inc. under 55 * DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"), as part of the DARPA 56 * CHATS research program. 57 * 58 * Portions of this software were developed by 59 * Konstantin Belousov <kib@FreeBSD.org> under sponsorship from 60 * the FreeBSD Foundation. 61 * 62 * Redistribution and use in source and binary forms, with or without 63 * modification, are permitted provided that the following conditions 64 * are met: 65 * 1. Redistributions of source code must retain the above copyright 66 * notice, this list of conditions and the following disclaimer. 67 * 2. Redistributions in binary form must reproduce the above copyright 68 * notice, this list of conditions and the following disclaimer in the 69 * documentation and/or other materials provided with the distribution. 70 * 71 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 72 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 73 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 74 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 75 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 76 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 77 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 78 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 79 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 80 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 81 * SUCH DAMAGE. 82 */ 83 84 #define AMD64_NPT_AWARE 85 86 #include <sys/cdefs.h> 87 /* 88 * Manages physical address maps. 89 * 90 * Since the information managed by this module is 91 * also stored by the logical address mapping module, 92 * this module may throw away valid virtual-to-physical 93 * mappings at almost any time. However, invalidations 94 * of virtual-to-physical mappings must be done as 95 * requested. 96 * 97 * In order to cope with hardware architectures which 98 * make virtual-to-physical map invalidates expensive, 99 * this module may delay invalidate or reduced protection 100 * operations until such time as they are actually 101 * necessary. This module is given full information as 102 * to which processors are currently using which maps, 103 * and to when physical maps must be made correct. 104 */ 105 106 #include "opt_ddb.h" 107 #include "opt_kstack_pages.h" 108 #include "opt_pmap.h" 109 #include "opt_vm.h" 110 111 #include <sys/param.h> 112 #include <sys/asan.h> 113 #include <sys/bitstring.h> 114 #include <sys/bus.h> 115 #include <sys/systm.h> 116 #include <sys/counter.h> 117 #include <sys/kernel.h> 118 #include <sys/ktr.h> 119 #include <sys/lock.h> 120 #include <sys/malloc.h> 121 #include <sys/mman.h> 122 #include <sys/msan.h> 123 #include <sys/mutex.h> 124 #include <sys/proc.h> 125 #include <sys/rangeset.h> 126 #include <sys/rwlock.h> 127 #include <sys/sbuf.h> 128 #include <sys/smr.h> 129 #include <sys/sx.h> 130 #include <sys/turnstile.h> 131 #include <sys/vmem.h> 132 #include <sys/vmmeter.h> 133 #include <sys/sched.h> 134 #include <sys/sysctl.h> 135 #include <sys/smp.h> 136 #ifdef DDB 137 #include <sys/kdb.h> 138 #include <ddb/ddb.h> 139 #endif 140 141 #include <vm/vm.h> 142 #include <vm/vm_param.h> 143 #include <vm/vm_kern.h> 144 #include <vm/vm_page.h> 145 #include <vm/vm_map.h> 146 #include <vm/vm_object.h> 147 #include <vm/vm_extern.h> 148 #include <vm/vm_pageout.h> 149 #include <vm/vm_pager.h> 150 #include <vm/vm_phys.h> 151 #include <vm/vm_radix.h> 152 #include <vm/vm_reserv.h> 153 #include <vm/vm_dumpset.h> 154 #include <vm/uma.h> 155 156 #include <machine/asan.h> 157 #include <machine/intr_machdep.h> 158 #include <x86/apicvar.h> 159 #include <x86/ifunc.h> 160 #include <machine/cpu.h> 161 #include <machine/cputypes.h> 162 #include <machine/md_var.h> 163 #include <machine/msan.h> 164 #include <machine/pcb.h> 165 #include <machine/specialreg.h> 166 #include <machine/smp.h> 167 #include <machine/sysarch.h> 168 #include <machine/tss.h> 169 170 #ifdef NUMA 171 #define PMAP_MEMDOM MAXMEMDOM 172 #else 173 #define PMAP_MEMDOM 1 174 #endif 175 176 static __inline bool 177 pmap_type_guest(pmap_t pmap) 178 { 179 180 return ((pmap->pm_type == PT_EPT) || (pmap->pm_type == PT_RVI)); 181 } 182 183 static __inline bool 184 pmap_emulate_ad_bits(pmap_t pmap) 185 { 186 187 return ((pmap->pm_flags & PMAP_EMULATE_AD_BITS) != 0); 188 } 189 190 static __inline pt_entry_t 191 pmap_valid_bit(pmap_t pmap) 192 { 193 pt_entry_t mask; 194 195 switch (pmap->pm_type) { 196 case PT_X86: 197 case PT_RVI: 198 mask = X86_PG_V; 199 break; 200 case PT_EPT: 201 if (pmap_emulate_ad_bits(pmap)) 202 mask = EPT_PG_EMUL_V; 203 else 204 mask = EPT_PG_READ; 205 break; 206 default: 207 panic("pmap_valid_bit: invalid pm_type %d", pmap->pm_type); 208 } 209 210 return (mask); 211 } 212 213 static __inline pt_entry_t 214 pmap_rw_bit(pmap_t pmap) 215 { 216 pt_entry_t mask; 217 218 switch (pmap->pm_type) { 219 case PT_X86: 220 case PT_RVI: 221 mask = X86_PG_RW; 222 break; 223 case PT_EPT: 224 if (pmap_emulate_ad_bits(pmap)) 225 mask = EPT_PG_EMUL_RW; 226 else 227 mask = EPT_PG_WRITE; 228 break; 229 default: 230 panic("pmap_rw_bit: invalid pm_type %d", pmap->pm_type); 231 } 232 233 return (mask); 234 } 235 236 static pt_entry_t pg_g; 237 238 static __inline pt_entry_t 239 pmap_global_bit(pmap_t pmap) 240 { 241 pt_entry_t mask; 242 243 switch (pmap->pm_type) { 244 case PT_X86: 245 mask = pg_g; 246 break; 247 case PT_RVI: 248 case PT_EPT: 249 mask = 0; 250 break; 251 default: 252 panic("pmap_global_bit: invalid pm_type %d", pmap->pm_type); 253 } 254 255 return (mask); 256 } 257 258 static __inline pt_entry_t 259 pmap_accessed_bit(pmap_t pmap) 260 { 261 pt_entry_t mask; 262 263 switch (pmap->pm_type) { 264 case PT_X86: 265 case PT_RVI: 266 mask = X86_PG_A; 267 break; 268 case PT_EPT: 269 if (pmap_emulate_ad_bits(pmap)) 270 mask = EPT_PG_READ; 271 else 272 mask = EPT_PG_A; 273 break; 274 default: 275 panic("pmap_accessed_bit: invalid pm_type %d", pmap->pm_type); 276 } 277 278 return (mask); 279 } 280 281 static __inline pt_entry_t 282 pmap_modified_bit(pmap_t pmap) 283 { 284 pt_entry_t mask; 285 286 switch (pmap->pm_type) { 287 case PT_X86: 288 case PT_RVI: 289 mask = X86_PG_M; 290 break; 291 case PT_EPT: 292 if (pmap_emulate_ad_bits(pmap)) 293 mask = EPT_PG_WRITE; 294 else 295 mask = EPT_PG_M; 296 break; 297 default: 298 panic("pmap_modified_bit: invalid pm_type %d", pmap->pm_type); 299 } 300 301 return (mask); 302 } 303 304 static __inline pt_entry_t 305 pmap_pku_mask_bit(pmap_t pmap) 306 { 307 308 return (pmap->pm_type == PT_X86 ? X86_PG_PKU_MASK : 0); 309 } 310 311 static __inline bool 312 safe_to_clear_referenced(pmap_t pmap, pt_entry_t pte) 313 { 314 315 if (!pmap_emulate_ad_bits(pmap)) 316 return (true); 317 318 KASSERT(pmap->pm_type == PT_EPT, ("invalid pm_type %d", pmap->pm_type)); 319 320 /* 321 * XWR = 010 or 110 will cause an unconditional EPT misconfiguration 322 * so we don't let the referenced (aka EPT_PG_READ) bit to be cleared 323 * if the EPT_PG_WRITE bit is set. 324 */ 325 if ((pte & EPT_PG_WRITE) != 0) 326 return (false); 327 328 /* 329 * XWR = 100 is allowed only if the PMAP_SUPPORTS_EXEC_ONLY is set. 330 */ 331 if ((pte & EPT_PG_EXECUTE) == 0 || 332 ((pmap->pm_flags & PMAP_SUPPORTS_EXEC_ONLY) != 0)) 333 return (true); 334 else 335 return (false); 336 } 337 338 #ifdef PV_STATS 339 #define PV_STAT(x) do { x ; } while (0) 340 #else 341 #define PV_STAT(x) do { } while (0) 342 #endif 343 344 #ifdef NUMA 345 #define pa_index(pa) ({ \ 346 KASSERT((pa) <= vm_phys_segs[vm_phys_nsegs - 1].end, \ 347 ("address %lx beyond the last segment", (pa))); \ 348 (pa) >> PDRSHIFT; \ 349 }) 350 #define pa_to_pmdp(pa) (&pv_table[pa_index(pa)]) 351 #define pa_to_pvh(pa) (&(pa_to_pmdp(pa)->pv_page)) 352 #define PHYS_TO_PV_LIST_LOCK(pa) ({ \ 353 struct rwlock *_lock; \ 354 if (__predict_false((pa) > pmap_last_pa)) \ 355 _lock = &pv_dummy_large.pv_lock; \ 356 else \ 357 _lock = &(pa_to_pmdp(pa)->pv_lock); \ 358 _lock; \ 359 }) 360 #else 361 #define pa_index(pa) ((pa) >> PDRSHIFT) 362 #define pa_to_pvh(pa) (&pv_table[pa_index(pa)]) 363 364 #define NPV_LIST_LOCKS MAXCPU 365 366 #define PHYS_TO_PV_LIST_LOCK(pa) \ 367 (&pv_list_locks[pa_index(pa) % NPV_LIST_LOCKS]) 368 #endif 369 370 #define CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa) do { \ 371 struct rwlock **_lockp = (lockp); \ 372 struct rwlock *_new_lock; \ 373 \ 374 _new_lock = PHYS_TO_PV_LIST_LOCK(pa); \ 375 if (_new_lock != *_lockp) { \ 376 if (*_lockp != NULL) \ 377 rw_wunlock(*_lockp); \ 378 *_lockp = _new_lock; \ 379 rw_wlock(*_lockp); \ 380 } \ 381 } while (0) 382 383 #define CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m) \ 384 CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, VM_PAGE_TO_PHYS(m)) 385 386 #define RELEASE_PV_LIST_LOCK(lockp) do { \ 387 struct rwlock **_lockp = (lockp); \ 388 \ 389 if (*_lockp != NULL) { \ 390 rw_wunlock(*_lockp); \ 391 *_lockp = NULL; \ 392 } \ 393 } while (0) 394 395 #define VM_PAGE_TO_PV_LIST_LOCK(m) \ 396 PHYS_TO_PV_LIST_LOCK(VM_PAGE_TO_PHYS(m)) 397 398 /* 399 * Statically allocate kernel pmap memory. However, memory for 400 * pm_pcids is obtained after the dynamic allocator is operational. 401 * Initialize it with a non-canonical pointer to catch early accesses 402 * regardless of the active mapping. 403 */ 404 struct pmap kernel_pmap_store = { 405 .pm_pcidp = (void *)0xdeadbeefdeadbeef, 406 }; 407 408 vm_offset_t virtual_avail; /* VA of first avail page (after kernel bss) */ 409 vm_offset_t virtual_end; /* VA of last avail page (end of kernel AS) */ 410 411 int nkpt; 412 SYSCTL_INT(_machdep, OID_AUTO, nkpt, CTLFLAG_RD, &nkpt, 0, 413 "Number of kernel page table pages allocated on bootup"); 414 415 static int ndmpdp; 416 vm_paddr_t dmaplimit; 417 vm_offset_t kernel_vm_end = VM_MIN_KERNEL_ADDRESS_LA48; 418 pt_entry_t pg_nx; 419 420 static SYSCTL_NODE(_vm, OID_AUTO, pmap, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 421 "VM/pmap parameters"); 422 423 static int __read_frequently pg_ps_enabled = 1; 424 SYSCTL_INT(_vm_pmap, OID_AUTO, pg_ps_enabled, CTLFLAG_RDTUN | CTLFLAG_NOFETCH, 425 &pg_ps_enabled, 0, "Are large page mappings enabled?"); 426 427 int __read_frequently la57 = 0; 428 SYSCTL_INT(_vm_pmap, OID_AUTO, la57, CTLFLAG_RDTUN | CTLFLAG_NOFETCH, 429 &la57, 0, 430 "5-level paging for host is enabled"); 431 432 /* 433 * The default value is needed in order to preserve compatibility with 434 * some userspace programs that put tags into sign-extended bits. 435 */ 436 int prefer_uva_la48 = 1; 437 SYSCTL_INT(_vm_pmap, OID_AUTO, prefer_uva_la48, CTLFLAG_RDTUN, 438 &prefer_uva_la48, 0, 439 "Userspace maps are limited to LA48 unless otherwise configured"); 440 441 static bool 442 pmap_is_la57(pmap_t pmap) 443 { 444 if (pmap->pm_type == PT_X86) 445 return (la57); 446 if (pmap->pm_type == PT_RVI) { 447 /* 448 * AMD nested paging has no field to specify the nested 449 * page table walk length; the hardware derives it from 450 * the host CR4.LA57 setting (see svm.c, where nCR3 is 451 * loaded directly from pm_pmltop with no level encoding). 452 * The NPT must therefore have the same number of levels 453 * as the host page tables, otherwise guest-physical 454 * translations are walked at the wrong depth. 455 */ 456 return (la57); 457 } 458 return (false); /* Intel EPT encodes its own walk length. */ 459 } 460 461 #define PAT_INDEX_SIZE 8 462 static int pat_index[PAT_INDEX_SIZE]; /* cache mode to PAT index conversion */ 463 464 static u_int64_t KPTphys; /* phys addr of kernel level 1 */ 465 static u_int64_t KPDphys; /* phys addr of kernel level 2 */ 466 static u_int64_t KPDPphys; /* phys addr of kernel level 3 */ 467 u_int64_t KPML4phys; /* phys addr of kernel level 4 */ 468 u_int64_t KPML5phys; /* phys addr of kernel level 5, 469 if supported */ 470 471 #ifdef KASAN 472 static uint64_t KASANPDPphys; 473 #endif 474 #ifdef KMSAN 475 static uint64_t KMSANSHADPDPphys; 476 static uint64_t KMSANORIGPDPphys; 477 478 /* 479 * To support systems with large amounts of memory, it is necessary to extend 480 * the maximum size of the direct map. This could eat into the space reserved 481 * for the shadow map. 482 */ 483 _Static_assert(DMPML4I + NDMPML4E <= KMSANSHADPML4I, "direct map overflow"); 484 #endif 485 486 static pml4_entry_t *kernel_pml4; 487 static u_int64_t DMPDphys; /* phys addr of direct mapped level 2 */ 488 static u_int64_t DMPDPphys; /* phys addr of direct mapped level 3 */ 489 static u_int64_t DMPML4phys; /* ... level 4, for la57 */ 490 static int ndmpdpphys; /* number of DMPDPphys pages */ 491 492 vm_paddr_t kernphys; /* phys addr of start of bootstrap data */ 493 vm_paddr_t KERNend; /* and the end */ 494 495 struct kva_layout_s kva_layout = { 496 .kva_min = KV4ADDR(PML4PML4I, 0, 0, 0), 497 .kva_max = KV4ADDR(NPML4EPG - 1, NPDPEPG - 1, 498 NPDEPG - 1, NPTEPG - 1), 499 .dmap_low = KV4ADDR(DMPML4I, 0, 0, 0), 500 .dmap_high = KV4ADDR(DMPML4I + NDMPML4E, 0, 0, 0), 501 .lm_low = KV4ADDR(LMSPML4I, 0, 0, 0), 502 .lm_high = KV4ADDR(LMEPML4I + 1, 0, 0, 0), 503 .km_low = KV4ADDR(KPML4BASE, 0, 0, 0), 504 .km_high = KV4ADDR(KPML4BASE + NKPML4E - 1, NPDPEPG - 1, 505 NPDEPG - 1, NPTEPG - 1), 506 .rec_pt = KV4ADDR(PML4PML4I, 0, 0, 0), 507 .kasan_shadow_low = KV4ADDR(KASANPML4I, 0, 0, 0), 508 .kasan_shadow_high = KV4ADDR(KASANPML4I + NKASANPML4E, 0, 0, 0), 509 .kmsan_shadow_low = KV4ADDR(KMSANSHADPML4I, 0, 0, 0), 510 .kmsan_shadow_high = KV4ADDR(KMSANSHADPML4I + NKMSANSHADPML4E, 511 0, 0, 0), 512 .kmsan_origin_low = KV4ADDR(KMSANORIGPML4I, 0, 0, 0), 513 .kmsan_origin_high = KV4ADDR(KMSANORIGPML4I + NKMSANORIGPML4E, 514 0, 0, 0), 515 }; 516 517 struct kva_layout_s kva_layout_la57 = { 518 .kva_min = KV5ADDR(NPML5EPG / 2, 0, 0, 0, 0), /* == rec_pt */ 519 .kva_max = KV5ADDR(NPML5EPG - 1, NPML4EPG - 1, NPDPEPG - 1, 520 NPDEPG - 1, NPTEPG - 1), 521 .dmap_low = KV5ADDR(DMPML5I, 0, 0, 0, 0), 522 .dmap_high = KV5ADDR(DMPML5I + NDMPML5E, 0, 0, 0, 0), 523 .lm_low = KV5ADDR(LMSPML5I, 0, 0, 0, 0), 524 .lm_high = KV5ADDR(LMEPML5I + 1, 0, 0, 0, 0), 525 .km_low = KV4ADDR(KPML4BASE, 0, 0, 0), 526 .km_high = KV4ADDR(KPML4BASE + NKPML4E - 1, NPDPEPG - 1, 527 NPDEPG - 1, NPTEPG - 1), 528 .rec_pt = KV5ADDR(PML5PML5I, 0, 0, 0, 0), 529 .kasan_shadow_low = KV4ADDR(KASANPML4I, 0, 0, 0), 530 .kasan_shadow_high = KV4ADDR(KASANPML4I + NKASANPML4E, 0, 0, 0), 531 .kmsan_shadow_low = KV4ADDR(KMSANSHADPML4I, 0, 0, 0), 532 .kmsan_shadow_high = KV4ADDR(KMSANSHADPML4I + NKMSANSHADPML4E, 533 0, 0, 0), 534 .kmsan_origin_low = KV4ADDR(KMSANORIGPML4I, 0, 0, 0), 535 .kmsan_origin_high = KV4ADDR(KMSANORIGPML4I + NKMSANORIGPML4E, 536 0, 0, 0), 537 }; 538 539 /* 540 * pmap_mapdev support pre initialization (i.e. console) 541 */ 542 #define PMAP_PREINIT_MAPPING_COUNT 8 543 static struct pmap_preinit_mapping { 544 vm_paddr_t pa; 545 void *va; 546 vm_size_t sz; 547 int mode; 548 } pmap_preinit_mapping[PMAP_PREINIT_MAPPING_COUNT]; 549 static int pmap_initialized; 550 551 /* 552 * Data for the pv entry allocation mechanism. 553 * Updates to pv_invl_gen are protected by the pv list lock but reads are not. 554 */ 555 #ifdef NUMA 556 static __inline int 557 pc_to_domain(struct pv_chunk *pc) 558 { 559 560 return (vm_phys_domain(DMAP_TO_PHYS(pc))); 561 } 562 #else 563 static __inline int 564 pc_to_domain(struct pv_chunk *pc __unused) 565 { 566 567 return (0); 568 } 569 #endif 570 571 struct pv_chunks_list { 572 struct mtx pvc_lock; 573 TAILQ_HEAD(pch, pv_chunk) pvc_list; 574 int active_reclaims; 575 } __aligned(CACHE_LINE_SIZE); 576 577 struct pv_chunks_list __exclusive_cache_line pv_chunks[PMAP_MEMDOM]; 578 579 #ifdef NUMA 580 struct pmap_large_md_page { 581 struct rwlock pv_lock; 582 struct md_page pv_page; 583 u_long pv_invl_gen; 584 }; 585 __exclusive_cache_line static struct pmap_large_md_page pv_dummy_large; 586 #define pv_dummy pv_dummy_large.pv_page 587 __read_mostly static struct pmap_large_md_page *pv_table; 588 __read_mostly vm_paddr_t pmap_last_pa; 589 #else 590 static struct rwlock __exclusive_cache_line pv_list_locks[NPV_LIST_LOCKS]; 591 static u_long pv_invl_gen[NPV_LIST_LOCKS]; 592 static struct md_page *pv_table; 593 static struct md_page pv_dummy; 594 #endif 595 596 /* 597 * All those kernel PT submaps that BSD is so fond of 598 */ 599 pt_entry_t *CMAP1 = NULL; 600 caddr_t CADDR1 = 0; 601 static vm_offset_t qframe = 0; 602 static struct mtx qframe_mtx; 603 604 static int pmap_flags = PMAP_PDE_SUPERPAGE; /* flags for x86 pmaps */ 605 606 static vmem_t *large_vmem; 607 static u_int lm_ents; 608 #define PMAP_ADDRESS_IN_LARGEMAP(va) ((va) >= kva_layout.lm_low && \ 609 (va) < kva_layout.lm_high) 610 611 int pmap_pcid_enabled = 1; 612 SYSCTL_INT(_vm_pmap, OID_AUTO, pcid_enabled, CTLFLAG_RDTUN | CTLFLAG_NOFETCH, 613 &pmap_pcid_enabled, 0, "Is TLB Context ID enabled ?"); 614 int invpcid_works = 0; 615 SYSCTL_INT(_vm_pmap, OID_AUTO, invpcid_works, CTLFLAG_RD, &invpcid_works, 0, 616 "Is the invpcid instruction available ?"); 617 int invlpgb_works; 618 SYSCTL_INT(_vm_pmap, OID_AUTO, invlpgb_works, CTLFLAG_RD, &invlpgb_works, 0, 619 "Is the invlpgb instruction available?"); 620 int invlpgb_maxcnt; 621 int pmap_pcid_invlpg_workaround = 0; 622 SYSCTL_INT(_vm_pmap, OID_AUTO, pcid_invlpg_workaround, 623 CTLFLAG_RDTUN | CTLFLAG_NOFETCH, 624 &pmap_pcid_invlpg_workaround, 0, 625 "Enable small core PCID/INVLPG workaround"); 626 int pmap_pcid_invlpg_workaround_uena = 1; 627 628 int __read_frequently pti = 0; 629 SYSCTL_INT(_vm_pmap, OID_AUTO, pti, CTLFLAG_RDTUN | CTLFLAG_NOFETCH, 630 &pti, 0, 631 "Page Table Isolation enabled"); 632 static vm_object_t pti_obj; 633 static pml4_entry_t *pti_pml4; 634 static vm_pindex_t pti_pg_idx; 635 static bool pti_finalized; 636 637 static int pmap_growkernel_panic = 0; 638 SYSCTL_INT(_vm_pmap, OID_AUTO, growkernel_panic, CTLFLAG_RDTUN, 639 &pmap_growkernel_panic, 0, 640 "panic on failure to allocate kernel page table page"); 641 642 struct pmap_pkru_range { 643 struct rs_el pkru_rs_el; 644 u_int pkru_keyidx; 645 int pkru_flags; 646 }; 647 648 static uma_zone_t pmap_pkru_ranges_zone; 649 static bool pmap_pkru_same(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, 650 pt_entry_t *pte); 651 static pt_entry_t pmap_pkru_get(pmap_t pmap, vm_offset_t va); 652 static void pmap_pkru_on_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva); 653 static void *pkru_dup_range(void *ctx, void *data); 654 static void pkru_free_range(void *ctx, void *node); 655 static int pmap_pkru_copy(pmap_t dst_pmap, pmap_t src_pmap); 656 static int pmap_pkru_deassign(pmap_t pmap, vm_offset_t sva, vm_offset_t eva); 657 static void pmap_pkru_deassign_all(pmap_t pmap); 658 659 static COUNTER_U64_DEFINE_EARLY(pcid_save_cnt); 660 SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, pcid_save_cnt, CTLFLAG_RD, 661 &pcid_save_cnt, "Count of saved TLB context on switch"); 662 663 static LIST_HEAD(, pmap_invl_gen) pmap_invl_gen_tracker = 664 LIST_HEAD_INITIALIZER(&pmap_invl_gen_tracker); 665 static struct mtx invl_gen_mtx; 666 /* Fake lock object to satisfy turnstiles interface. */ 667 static struct lock_object invl_gen_ts = { 668 .lo_name = "invlts", 669 }; 670 static struct pmap_invl_gen pmap_invl_gen_head = { 671 .gen = 1, 672 .next = NULL, 673 }; 674 static u_long pmap_invl_gen = 1; 675 static int pmap_invl_waiters; 676 static struct callout pmap_invl_callout; 677 static bool pmap_invl_callout_inited; 678 679 #define PMAP_ASSERT_NOT_IN_DI() \ 680 KASSERT(pmap_not_in_di(), ("DI already started")) 681 682 static bool 683 pmap_di_locked(void) 684 { 685 int tun; 686 687 if ((cpu_feature2 & CPUID2_CX16) == 0) 688 return (true); 689 tun = 0; 690 TUNABLE_INT_FETCH("vm.pmap.di_locked", &tun); 691 return (tun != 0); 692 } 693 694 static int 695 sysctl_pmap_di_locked(SYSCTL_HANDLER_ARGS) 696 { 697 int locked; 698 699 locked = pmap_di_locked(); 700 return (sysctl_handle_int(oidp, &locked, 0, req)); 701 } 702 SYSCTL_PROC(_vm_pmap, OID_AUTO, di_locked, CTLTYPE_INT | CTLFLAG_RDTUN | 703 CTLFLAG_MPSAFE, 0, 0, sysctl_pmap_di_locked, "", 704 "Locked delayed invalidation"); 705 706 static bool pmap_not_in_di_l(void); 707 static bool pmap_not_in_di_u(void); 708 DEFINE_IFUNC(, bool, pmap_not_in_di, (void)) 709 { 710 711 return (pmap_di_locked() ? pmap_not_in_di_l : pmap_not_in_di_u); 712 } 713 714 static bool 715 pmap_not_in_di_l(void) 716 { 717 struct pmap_invl_gen *invl_gen; 718 719 invl_gen = &curthread->td_md.md_invl_gen; 720 return (invl_gen->gen == 0); 721 } 722 723 static void 724 pmap_thread_init_invl_gen_l(struct thread *td) 725 { 726 struct pmap_invl_gen *invl_gen; 727 728 invl_gen = &td->td_md.md_invl_gen; 729 invl_gen->gen = 0; 730 } 731 732 static void 733 pmap_delayed_invl_wait_block(u_long *m_gen, u_long *invl_gen) 734 { 735 struct turnstile *ts; 736 737 ts = turnstile_trywait(&invl_gen_ts); 738 if (*m_gen > atomic_load_long(invl_gen)) 739 turnstile_wait(ts, NULL, TS_SHARED_QUEUE); 740 else 741 turnstile_cancel(ts); 742 } 743 744 static void 745 pmap_delayed_invl_finish_unblock(u_long new_gen) 746 { 747 struct turnstile *ts; 748 749 turnstile_chain_lock(&invl_gen_ts); 750 ts = turnstile_lookup(&invl_gen_ts); 751 if (new_gen != 0) 752 pmap_invl_gen = new_gen; 753 if (ts != NULL) { 754 turnstile_broadcast(ts, TS_SHARED_QUEUE); 755 turnstile_unpend(ts); 756 } 757 turnstile_chain_unlock(&invl_gen_ts); 758 } 759 760 /* 761 * Start a new Delayed Invalidation (DI) block of code, executed by 762 * the current thread. Within a DI block, the current thread may 763 * destroy both the page table and PV list entries for a mapping and 764 * then release the corresponding PV list lock before ensuring that 765 * the mapping is flushed from the TLBs of any processors with the 766 * pmap active. 767 */ 768 static void 769 pmap_delayed_invl_start_l(void) 770 { 771 struct pmap_invl_gen *invl_gen; 772 u_long currgen; 773 774 invl_gen = &curthread->td_md.md_invl_gen; 775 PMAP_ASSERT_NOT_IN_DI(); 776 mtx_lock(&invl_gen_mtx); 777 if (LIST_EMPTY(&pmap_invl_gen_tracker)) 778 currgen = pmap_invl_gen; 779 else 780 currgen = LIST_FIRST(&pmap_invl_gen_tracker)->gen; 781 invl_gen->gen = currgen + 1; 782 LIST_INSERT_HEAD(&pmap_invl_gen_tracker, invl_gen, link); 783 mtx_unlock(&invl_gen_mtx); 784 } 785 786 /* 787 * Finish the DI block, previously started by the current thread. All 788 * required TLB flushes for the pages marked by 789 * pmap_delayed_invl_page() must be finished before this function is 790 * called. 791 * 792 * This function works by bumping the global DI generation number to 793 * the generation number of the current thread's DI, unless there is a 794 * pending DI that started earlier. In the latter case, bumping the 795 * global DI generation number would incorrectly signal that the 796 * earlier DI had finished. Instead, this function bumps the earlier 797 * DI's generation number to match the generation number of the 798 * current thread's DI. 799 */ 800 static void 801 pmap_delayed_invl_finish_l(void) 802 { 803 struct pmap_invl_gen *invl_gen, *next; 804 805 invl_gen = &curthread->td_md.md_invl_gen; 806 KASSERT(invl_gen->gen != 0, ("missed invl_start")); 807 mtx_lock(&invl_gen_mtx); 808 next = LIST_NEXT(invl_gen, link); 809 if (next == NULL) 810 pmap_delayed_invl_finish_unblock(invl_gen->gen); 811 else 812 next->gen = invl_gen->gen; 813 LIST_REMOVE(invl_gen, link); 814 mtx_unlock(&invl_gen_mtx); 815 invl_gen->gen = 0; 816 } 817 818 static bool 819 pmap_not_in_di_u(void) 820 { 821 struct pmap_invl_gen *invl_gen; 822 823 invl_gen = &curthread->td_md.md_invl_gen; 824 return (((uintptr_t)invl_gen->next & PMAP_INVL_GEN_NEXT_INVALID) != 0); 825 } 826 827 static void 828 pmap_thread_init_invl_gen_u(struct thread *td) 829 { 830 struct pmap_invl_gen *invl_gen; 831 832 invl_gen = &td->td_md.md_invl_gen; 833 invl_gen->gen = 0; 834 invl_gen->next = (void *)PMAP_INVL_GEN_NEXT_INVALID; 835 } 836 837 static bool 838 pmap_di_load_invl(struct pmap_invl_gen *ptr, struct pmap_invl_gen *out) 839 { 840 uint64_t new_high, new_low, old_high, old_low; 841 char res; 842 843 old_low = new_low = 0; 844 old_high = new_high = (uintptr_t)0; 845 846 __asm volatile("lock;cmpxchg16b\t%1" 847 : "=@cce" (res), "+m" (*ptr), "+a" (old_low), "+d" (old_high) 848 : "b"(new_low), "c" (new_high) 849 : "memory", "cc"); 850 if (res == 0) { 851 if ((old_high & PMAP_INVL_GEN_NEXT_INVALID) != 0) 852 return (false); 853 out->gen = old_low; 854 out->next = (void *)old_high; 855 } else { 856 out->gen = new_low; 857 out->next = (void *)new_high; 858 } 859 return (true); 860 } 861 862 static bool 863 pmap_di_store_invl(struct pmap_invl_gen *ptr, struct pmap_invl_gen *old_val, 864 struct pmap_invl_gen *new_val) 865 { 866 uint64_t new_high, new_low, old_high, old_low; 867 char res; 868 869 new_low = new_val->gen; 870 new_high = (uintptr_t)new_val->next; 871 old_low = old_val->gen; 872 old_high = (uintptr_t)old_val->next; 873 874 __asm volatile("lock;cmpxchg16b\t%1" 875 : "=@cce" (res), "+m" (*ptr), "+a" (old_low), "+d" (old_high) 876 : "b"(new_low), "c" (new_high) 877 : "memory", "cc"); 878 return (res); 879 } 880 881 static COUNTER_U64_DEFINE_EARLY(pv_page_count); 882 SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, pv_page_count, CTLFLAG_RD, 883 &pv_page_count, "Current number of allocated pv pages"); 884 885 static COUNTER_U64_DEFINE_EARLY(user_pt_page_count); 886 SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, user_pt_page_count, CTLFLAG_RD, 887 &user_pt_page_count, 888 "Current number of allocated page table pages for userspace"); 889 890 static COUNTER_U64_DEFINE_EARLY(kernel_pt_page_count); 891 SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, kernel_pt_page_count, CTLFLAG_RD, 892 &kernel_pt_page_count, 893 "Current number of allocated page table pages for the kernel"); 894 895 #ifdef PV_STATS 896 897 static COUNTER_U64_DEFINE_EARLY(invl_start_restart); 898 SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, invl_start_restart, 899 CTLFLAG_RD, &invl_start_restart, 900 "Number of delayed TLB invalidation request restarts"); 901 902 static COUNTER_U64_DEFINE_EARLY(invl_finish_restart); 903 SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, invl_finish_restart, CTLFLAG_RD, 904 &invl_finish_restart, 905 "Number of delayed TLB invalidation completion restarts"); 906 907 static int invl_max_qlen; 908 SYSCTL_INT(_vm_pmap, OID_AUTO, invl_max_qlen, CTLFLAG_RD, 909 &invl_max_qlen, 0, 910 "Maximum delayed TLB invalidation request queue length"); 911 #endif 912 913 #define di_delay locks_delay 914 915 static void 916 pmap_delayed_invl_start_u(void) 917 { 918 struct pmap_invl_gen *invl_gen, *p, prev, new_prev; 919 struct thread *td; 920 struct lock_delay_arg lda; 921 uintptr_t prevl; 922 u_char pri; 923 #ifdef PV_STATS 924 int i, ii; 925 #endif 926 927 td = curthread; 928 invl_gen = &td->td_md.md_invl_gen; 929 PMAP_ASSERT_NOT_IN_DI(); 930 lock_delay_arg_init(&lda, &di_delay); 931 invl_gen->saved_pri = 0; 932 pri = td->td_base_pri; 933 if (pri > PVM) { 934 thread_lock(td); 935 pri = td->td_base_pri; 936 if (pri > PVM) { 937 invl_gen->saved_pri = pri; 938 sched_prio(td, PVM); 939 } 940 thread_unlock(td); 941 } 942 again: 943 PV_STAT(i = 0); 944 for (p = &pmap_invl_gen_head;; p = prev.next) { 945 PV_STAT(i++); 946 prevl = (uintptr_t)atomic_load_ptr(&p->next); 947 if ((prevl & PMAP_INVL_GEN_NEXT_INVALID) != 0) { 948 PV_STAT(counter_u64_add(invl_start_restart, 1)); 949 lock_delay(&lda); 950 goto again; 951 } 952 if (prevl == 0) 953 break; 954 prev.next = (void *)prevl; 955 } 956 #ifdef PV_STATS 957 if ((ii = invl_max_qlen) < i) 958 atomic_cmpset_int(&invl_max_qlen, ii, i); 959 #endif 960 961 if (!pmap_di_load_invl(p, &prev) || prev.next != NULL) { 962 PV_STAT(counter_u64_add(invl_start_restart, 1)); 963 lock_delay(&lda); 964 goto again; 965 } 966 967 new_prev.gen = prev.gen; 968 new_prev.next = invl_gen; 969 invl_gen->gen = prev.gen + 1; 970 971 /* Formal fence between store to invl->gen and updating *p. */ 972 atomic_thread_fence_rel(); 973 974 /* 975 * After inserting an invl_gen element with invalid bit set, 976 * this thread blocks any other thread trying to enter the 977 * delayed invalidation block. Do not allow to remove us from 978 * the CPU, because it causes starvation for other threads. 979 */ 980 critical_enter(); 981 982 /* 983 * ABA for *p is not possible there, since p->gen can only 984 * increase. So if the *p thread finished its di, then 985 * started a new one and got inserted into the list at the 986 * same place, its gen will appear greater than the previously 987 * read gen. 988 */ 989 if (!pmap_di_store_invl(p, &prev, &new_prev)) { 990 critical_exit(); 991 PV_STAT(counter_u64_add(invl_start_restart, 1)); 992 lock_delay(&lda); 993 goto again; 994 } 995 996 /* 997 * There we clear PMAP_INVL_GEN_NEXT_INVALID in 998 * invl_gen->next, allowing other threads to iterate past us. 999 * pmap_di_store_invl() provides fence between the generation 1000 * write and the update of next. 1001 */ 1002 invl_gen->next = NULL; 1003 critical_exit(); 1004 } 1005 1006 static bool 1007 pmap_delayed_invl_finish_u_crit(struct pmap_invl_gen *invl_gen, 1008 struct pmap_invl_gen *p) 1009 { 1010 struct pmap_invl_gen prev, new_prev; 1011 u_long mygen; 1012 1013 /* 1014 * Load invl_gen->gen after setting invl_gen->next 1015 * PMAP_INVL_GEN_NEXT_INVALID. This prevents larger 1016 * generations to propagate to our invl_gen->gen. Lock prefix 1017 * in atomic_set_ptr() worked as seq_cst fence. 1018 */ 1019 mygen = atomic_load_long(&invl_gen->gen); 1020 1021 if (!pmap_di_load_invl(p, &prev) || prev.next != invl_gen) 1022 return (false); 1023 1024 KASSERT(prev.gen < mygen, 1025 ("invalid di gen sequence %lu %lu", prev.gen, mygen)); 1026 new_prev.gen = mygen; 1027 new_prev.next = (void *)((uintptr_t)invl_gen->next & 1028 ~PMAP_INVL_GEN_NEXT_INVALID); 1029 1030 /* Formal fence between load of prev and storing update to it. */ 1031 atomic_thread_fence_rel(); 1032 1033 return (pmap_di_store_invl(p, &prev, &new_prev)); 1034 } 1035 1036 static void 1037 pmap_delayed_invl_finish_u(void) 1038 { 1039 struct pmap_invl_gen *invl_gen, *p; 1040 struct thread *td; 1041 struct lock_delay_arg lda; 1042 uintptr_t prevl; 1043 1044 td = curthread; 1045 invl_gen = &td->td_md.md_invl_gen; 1046 KASSERT(invl_gen->gen != 0, ("missed invl_start: gen 0")); 1047 KASSERT(((uintptr_t)invl_gen->next & PMAP_INVL_GEN_NEXT_INVALID) == 0, 1048 ("missed invl_start: INVALID")); 1049 lock_delay_arg_init(&lda, &di_delay); 1050 1051 again: 1052 for (p = &pmap_invl_gen_head; p != NULL; p = (void *)prevl) { 1053 prevl = (uintptr_t)atomic_load_ptr(&p->next); 1054 if ((prevl & PMAP_INVL_GEN_NEXT_INVALID) != 0) { 1055 PV_STAT(counter_u64_add(invl_finish_restart, 1)); 1056 lock_delay(&lda); 1057 goto again; 1058 } 1059 if ((void *)prevl == invl_gen) 1060 break; 1061 } 1062 1063 /* 1064 * It is legitimate to not find ourself on the list if a 1065 * thread before us finished its DI and started it again. 1066 */ 1067 if (__predict_false(p == NULL)) { 1068 PV_STAT(counter_u64_add(invl_finish_restart, 1)); 1069 lock_delay(&lda); 1070 goto again; 1071 } 1072 1073 critical_enter(); 1074 atomic_set_ptr((uintptr_t *)&invl_gen->next, 1075 PMAP_INVL_GEN_NEXT_INVALID); 1076 if (!pmap_delayed_invl_finish_u_crit(invl_gen, p)) { 1077 atomic_clear_ptr((uintptr_t *)&invl_gen->next, 1078 PMAP_INVL_GEN_NEXT_INVALID); 1079 critical_exit(); 1080 PV_STAT(counter_u64_add(invl_finish_restart, 1)); 1081 lock_delay(&lda); 1082 goto again; 1083 } 1084 critical_exit(); 1085 if (atomic_load_int(&pmap_invl_waiters) > 0) 1086 pmap_delayed_invl_finish_unblock(0); 1087 if (invl_gen->saved_pri != 0) { 1088 thread_lock(td); 1089 sched_prio(td, invl_gen->saved_pri); 1090 thread_unlock(td); 1091 } 1092 } 1093 1094 #ifdef DDB 1095 DB_SHOW_COMMAND(di_queue, pmap_di_queue) 1096 { 1097 struct pmap_invl_gen *p, *pn; 1098 struct thread *td; 1099 uintptr_t nextl; 1100 bool first; 1101 1102 for (p = &pmap_invl_gen_head, first = true; p != NULL; p = pn, 1103 first = false) { 1104 nextl = (uintptr_t)atomic_load_ptr(&p->next); 1105 pn = (void *)(nextl & ~PMAP_INVL_GEN_NEXT_INVALID); 1106 td = first ? NULL : __containerof(p, struct thread, 1107 td_md.md_invl_gen); 1108 db_printf("gen %lu inv %d td %p tid %d\n", p->gen, 1109 (nextl & PMAP_INVL_GEN_NEXT_INVALID) != 0, td, 1110 td != NULL ? td->td_tid : -1); 1111 } 1112 } 1113 #endif 1114 1115 #ifdef PV_STATS 1116 static COUNTER_U64_DEFINE_EARLY(invl_wait); 1117 SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, invl_wait, 1118 CTLFLAG_RD, &invl_wait, 1119 "Number of times DI invalidation blocked pmap_remove_all/write"); 1120 1121 static COUNTER_U64_DEFINE_EARLY(invl_wait_slow); 1122 SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, invl_wait_slow, CTLFLAG_RD, 1123 &invl_wait_slow, "Number of slow invalidation waits for lockless DI"); 1124 1125 #endif 1126 1127 #ifdef NUMA 1128 static u_long * 1129 pmap_delayed_invl_genp(vm_page_t m) 1130 { 1131 vm_paddr_t pa; 1132 u_long *gen; 1133 1134 pa = VM_PAGE_TO_PHYS(m); 1135 if (__predict_false((pa) > pmap_last_pa)) 1136 gen = &pv_dummy_large.pv_invl_gen; 1137 else 1138 gen = &(pa_to_pmdp(pa)->pv_invl_gen); 1139 1140 return (gen); 1141 } 1142 #else 1143 static u_long * 1144 pmap_delayed_invl_genp(vm_page_t m) 1145 { 1146 1147 return (&pv_invl_gen[pa_index(VM_PAGE_TO_PHYS(m)) % NPV_LIST_LOCKS]); 1148 } 1149 #endif 1150 1151 static void 1152 pmap_delayed_invl_callout_func(void *arg __unused) 1153 { 1154 1155 if (atomic_load_int(&pmap_invl_waiters) == 0) 1156 return; 1157 pmap_delayed_invl_finish_unblock(0); 1158 } 1159 1160 static void 1161 pmap_delayed_invl_callout_init(void *arg __unused) 1162 { 1163 1164 if (pmap_di_locked()) 1165 return; 1166 callout_init(&pmap_invl_callout, 1); 1167 pmap_invl_callout_inited = true; 1168 } 1169 SYSINIT(pmap_di_callout, SI_SUB_CPU, SI_ORDER_LAST, 1170 pmap_delayed_invl_callout_init, NULL); 1171 1172 /* 1173 * Ensure that all currently executing DI blocks, that need to flush 1174 * TLB for the given page m, actually flushed the TLB at the time the 1175 * function returned. If the page m has an empty PV list and we call 1176 * pmap_delayed_invl_wait(), upon its return we know that no CPU has a 1177 * valid mapping for the page m in either its page table or TLB. 1178 * 1179 * This function works by blocking until the global DI generation 1180 * number catches up with the generation number associated with the 1181 * given page m and its PV list. Since this function's callers 1182 * typically own an object lock and sometimes own a page lock, it 1183 * cannot sleep. Instead, it blocks on a turnstile to relinquish the 1184 * processor. 1185 */ 1186 static void 1187 pmap_delayed_invl_wait_l(vm_page_t m) 1188 { 1189 u_long *m_gen; 1190 #ifdef PV_STATS 1191 bool accounted = false; 1192 #endif 1193 1194 m_gen = pmap_delayed_invl_genp(m); 1195 while (*m_gen > pmap_invl_gen) { 1196 #ifdef PV_STATS 1197 if (!accounted) { 1198 counter_u64_add(invl_wait, 1); 1199 accounted = true; 1200 } 1201 #endif 1202 pmap_delayed_invl_wait_block(m_gen, &pmap_invl_gen); 1203 } 1204 } 1205 1206 static void 1207 pmap_delayed_invl_wait_u(vm_page_t m) 1208 { 1209 u_long *m_gen; 1210 struct lock_delay_arg lda; 1211 bool fast; 1212 1213 fast = true; 1214 m_gen = pmap_delayed_invl_genp(m); 1215 lock_delay_arg_init(&lda, &di_delay); 1216 while (*m_gen > atomic_load_long(&pmap_invl_gen_head.gen)) { 1217 if (fast || !pmap_invl_callout_inited) { 1218 PV_STAT(counter_u64_add(invl_wait, 1)); 1219 lock_delay(&lda); 1220 fast = false; 1221 } else { 1222 /* 1223 * The page's invalidation generation number 1224 * is still below the current thread's number. 1225 * Prepare to block so that we do not waste 1226 * CPU cycles or worse, suffer livelock. 1227 * 1228 * Since it is impossible to block without 1229 * racing with pmap_delayed_invl_finish_u(), 1230 * prepare for the race by incrementing 1231 * pmap_invl_waiters and arming a 1-tick 1232 * callout which will unblock us if we lose 1233 * the race. 1234 */ 1235 atomic_add_int(&pmap_invl_waiters, 1); 1236 1237 /* 1238 * Re-check the current thread's invalidation 1239 * generation after incrementing 1240 * pmap_invl_waiters, so that there is no race 1241 * with pmap_delayed_invl_finish_u() setting 1242 * the page generation and checking 1243 * pmap_invl_waiters. The only race allowed 1244 * is for a missed unblock, which is handled 1245 * by the callout. 1246 */ 1247 if (*m_gen > 1248 atomic_load_long(&pmap_invl_gen_head.gen)) { 1249 callout_reset(&pmap_invl_callout, 1, 1250 pmap_delayed_invl_callout_func, NULL); 1251 PV_STAT(counter_u64_add(invl_wait_slow, 1)); 1252 pmap_delayed_invl_wait_block(m_gen, 1253 &pmap_invl_gen_head.gen); 1254 } 1255 atomic_add_int(&pmap_invl_waiters, -1); 1256 } 1257 } 1258 } 1259 1260 DEFINE_IFUNC(, void, pmap_thread_init_invl_gen, (struct thread *)) 1261 { 1262 1263 return (pmap_di_locked() ? pmap_thread_init_invl_gen_l : 1264 pmap_thread_init_invl_gen_u); 1265 } 1266 1267 DEFINE_IFUNC(static, void, pmap_delayed_invl_start, (void)) 1268 { 1269 1270 return (pmap_di_locked() ? pmap_delayed_invl_start_l : 1271 pmap_delayed_invl_start_u); 1272 } 1273 1274 DEFINE_IFUNC(static, void, pmap_delayed_invl_finish, (void)) 1275 { 1276 1277 return (pmap_di_locked() ? pmap_delayed_invl_finish_l : 1278 pmap_delayed_invl_finish_u); 1279 } 1280 1281 DEFINE_IFUNC(static, void, pmap_delayed_invl_wait, (vm_page_t)) 1282 { 1283 1284 return (pmap_di_locked() ? pmap_delayed_invl_wait_l : 1285 pmap_delayed_invl_wait_u); 1286 } 1287 1288 /* 1289 * Mark the page m's PV list as participating in the current thread's 1290 * DI block. Any threads concurrently using m's PV list to remove or 1291 * restrict all mappings to m will wait for the current thread's DI 1292 * block to complete before proceeding. 1293 * 1294 * The function works by setting the DI generation number for m's PV 1295 * list to at least the DI generation number of the current thread. 1296 * This forces a caller of pmap_delayed_invl_wait() to block until 1297 * current thread calls pmap_delayed_invl_finish(). 1298 */ 1299 static void 1300 pmap_delayed_invl_page(vm_page_t m) 1301 { 1302 u_long gen, *m_gen; 1303 1304 rw_assert(VM_PAGE_TO_PV_LIST_LOCK(m), RA_WLOCKED); 1305 gen = curthread->td_md.md_invl_gen.gen; 1306 if (gen == 0) 1307 return; 1308 m_gen = pmap_delayed_invl_genp(m); 1309 if (*m_gen < gen) 1310 *m_gen = gen; 1311 } 1312 1313 /* 1314 * Crashdump maps. 1315 */ 1316 static caddr_t crashdumpmap; 1317 1318 /* 1319 * Internal flags for pmap_enter()'s helper functions. 1320 */ 1321 #define PMAP_ENTER_NORECLAIM 0x1000000 /* Don't reclaim PV entries. */ 1322 #define PMAP_ENTER_NOREPLACE 0x2000000 /* Don't replace mappings. */ 1323 1324 /* 1325 * Internal flags for pmap_mapdev_internal() and 1326 * pmap_change_props_locked(). 1327 */ 1328 #define MAPDEV_FLUSHCACHE 0x00000001 /* Flush cache after mapping. */ 1329 #define MAPDEV_SETATTR 0x00000002 /* Modify existing attrs. */ 1330 #define MAPDEV_ASSERTVALID 0x00000004 /* Assert mapping validity. */ 1331 1332 TAILQ_HEAD(pv_chunklist, pv_chunk); 1333 1334 static void free_pv_chunk(struct pv_chunk *pc); 1335 static void free_pv_chunk_batch(struct pv_chunklist *batch); 1336 static void free_pv_entry(pmap_t pmap, pv_entry_t pv); 1337 static pv_entry_t get_pv_entry(pmap_t pmap, struct rwlock **lockp); 1338 static int popcnt_pc_map_pq(uint64_t *map); 1339 static vm_page_t reclaim_pv_chunk(pmap_t locked_pmap, struct rwlock **lockp); 1340 static void reserve_pv_entries(pmap_t pmap, int needed, 1341 struct rwlock **lockp); 1342 static void pmap_pv_demote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa, 1343 struct rwlock **lockp); 1344 static bool pmap_pv_insert_pde(pmap_t pmap, vm_offset_t va, pd_entry_t pde, 1345 u_int flags, struct rwlock **lockp); 1346 #if VM_NRESERVLEVEL > 0 1347 static void pmap_pv_promote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa, 1348 struct rwlock **lockp); 1349 #endif 1350 static void pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va); 1351 static pv_entry_t pmap_pvh_remove(struct md_page *pvh, pmap_t pmap, 1352 vm_offset_t va); 1353 1354 static void pmap_abort_ptp(pmap_t pmap, vm_offset_t va, vm_page_t mpte); 1355 static int pmap_change_props_locked(void *addr, vm_size_t size, 1356 vm_prot_t prot, int mode, int flags); 1357 static bool pmap_demote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va); 1358 static bool pmap_demote_pde_locked(pmap_t pmap, pd_entry_t *pde, 1359 vm_offset_t va, struct rwlock **lockp); 1360 static bool pmap_demote_pde_mpte(pmap_t pmap, pd_entry_t *pde, 1361 vm_offset_t va, struct rwlock **lockp, vm_page_t mpte); 1362 static bool pmap_demote_pdpe(pmap_t pmap, pdp_entry_t *pdpe, 1363 vm_offset_t va, vm_page_t m); 1364 static int pmap_enter_2mpage(pmap_t pmap, vm_offset_t va, vm_page_t m, 1365 vm_prot_t prot, struct rwlock **lockp); 1366 static int pmap_enter_pde(pmap_t pmap, vm_offset_t va, pd_entry_t newpde, 1367 u_int flags, vm_page_t m, struct rwlock **lockp); 1368 static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, 1369 vm_page_t m, vm_prot_t prot, vm_page_t mpte, struct rwlock **lockp); 1370 static void pmap_fill_ptp(pt_entry_t *firstpte, pt_entry_t newpte); 1371 static int pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte, bool promoted, 1372 bool allpte_PG_A_set); 1373 static void pmap_invalidate_cache_range_selfsnoop(vm_offset_t sva, 1374 vm_offset_t eva); 1375 static void pmap_invalidate_cache_range_all(vm_offset_t sva, 1376 vm_offset_t eva); 1377 static void pmap_invalidate_pde_page(pmap_t pmap, vm_offset_t va, 1378 pd_entry_t pde); 1379 static void pmap_kenter_attr(vm_offset_t va, vm_paddr_t pa, int mode); 1380 static vm_page_t pmap_large_map_getptp_unlocked(void); 1381 static vm_paddr_t pmap_large_map_kextract(vm_offset_t va); 1382 static bool pmap_page_is_mapped_locked(vm_page_t m); 1383 #if VM_NRESERVLEVEL > 0 1384 static bool pmap_promote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va, 1385 vm_page_t mpte, struct rwlock **lockp); 1386 #endif 1387 static bool pmap_protect_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t sva, 1388 vm_prot_t prot); 1389 static void pmap_pte_props(pt_entry_t *pte, u_long bits, u_long mask); 1390 static void pmap_pti_add_kva_locked(vm_offset_t sva, vm_offset_t eva, 1391 bool exec); 1392 static pdp_entry_t *pmap_pti_pdpe(vm_offset_t va); 1393 static pd_entry_t *pmap_pti_pde(vm_offset_t va); 1394 static void pmap_pti_wire_pte(void *pte); 1395 static int pmap_remove_pde(pmap_t pmap, pd_entry_t *pdq, vm_offset_t sva, 1396 bool demote_kpde, struct spglist *free, struct rwlock **lockp); 1397 static int pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t sva, 1398 pd_entry_t ptepde, struct spglist *free, struct rwlock **lockp); 1399 static vm_page_t pmap_remove_pt_page(pmap_t pmap, vm_offset_t va); 1400 static void pmap_remove_page(pmap_t pmap, vm_offset_t va, pd_entry_t *pde, 1401 struct spglist *free); 1402 static bool pmap_remove_ptes(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, 1403 pd_entry_t *pde, struct spglist *free, 1404 struct rwlock **lockp); 1405 static bool pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, 1406 vm_page_t m, struct rwlock **lockp); 1407 static void pmap_update_pde(pmap_t pmap, vm_offset_t va, pd_entry_t *pde, 1408 pd_entry_t newpde); 1409 static void pmap_update_pde_invalidate(pmap_t, vm_offset_t va, pd_entry_t pde); 1410 1411 static pd_entry_t *pmap_alloc_pde(pmap_t pmap, vm_offset_t va, vm_page_t *pdpgp, 1412 struct rwlock **lockp); 1413 static vm_page_t pmap_allocpte_alloc(pmap_t pmap, vm_pindex_t ptepindex, 1414 struct rwlock **lockp, vm_offset_t va); 1415 static vm_page_t pmap_allocpte_nosleep(pmap_t pmap, vm_pindex_t ptepindex, 1416 struct rwlock **lockp, vm_offset_t va); 1417 static vm_page_t pmap_allocpte(pmap_t pmap, vm_offset_t va, 1418 struct rwlock **lockp); 1419 1420 static void _pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, 1421 struct spglist *free); 1422 static int pmap_unuse_pt(pmap_t, vm_offset_t, pd_entry_t, struct spglist *); 1423 1424 static vm_page_t pmap_alloc_pt_page(pmap_t, vm_pindex_t, int); 1425 static void pmap_free_pt_page(pmap_t, vm_page_t, bool); 1426 1427 /********************/ 1428 /* Inline functions */ 1429 /********************/ 1430 1431 /* 1432 * Return a non-clipped indexes for a given VA, which are page table 1433 * pages indexes at the corresponding level. 1434 */ 1435 static __inline vm_pindex_t 1436 pmap_pde_pindex(vm_offset_t va) 1437 { 1438 return (va >> PDRSHIFT); 1439 } 1440 1441 static __inline vm_pindex_t 1442 pmap_pdpe_pindex(vm_offset_t va) 1443 { 1444 return (NUPDE + (va >> PDPSHIFT)); 1445 } 1446 1447 static __inline vm_pindex_t 1448 pmap_pml4e_pindex(vm_offset_t va) 1449 { 1450 return (NUPDE + NUPDPE + (va >> PML4SHIFT)); 1451 } 1452 1453 static __inline vm_pindex_t 1454 pmap_pml5e_pindex(vm_offset_t va) 1455 { 1456 return (NUPDE + NUPDPE + NUPML4E + (va >> PML5SHIFT)); 1457 } 1458 1459 static __inline pml4_entry_t * 1460 pmap_pml5e(pmap_t pmap, vm_offset_t va) 1461 { 1462 1463 MPASS(pmap_is_la57(pmap)); 1464 return (&pmap->pm_pmltop[pmap_pml5e_index(va)]); 1465 } 1466 1467 static __inline pml4_entry_t * 1468 pmap_pml5e_u(pmap_t pmap, vm_offset_t va) 1469 { 1470 1471 MPASS(pmap_is_la57(pmap)); 1472 return (&pmap->pm_pmltopu[pmap_pml5e_index(va)]); 1473 } 1474 1475 static __inline pml4_entry_t * 1476 pmap_pml5e_to_pml4e(pml5_entry_t *pml5e, vm_offset_t va) 1477 { 1478 pml4_entry_t *pml4e; 1479 1480 /* XXX MPASS(pmap_is_la57(pmap); */ 1481 pml4e = PHYS_TO_DMAP(*pml5e & PG_FRAME); 1482 return (&pml4e[pmap_pml4e_index(va)]); 1483 } 1484 1485 /* Return a pointer to the PML4 slot that corresponds to a VA */ 1486 static __inline pml4_entry_t * 1487 pmap_pml4e(pmap_t pmap, vm_offset_t va) 1488 { 1489 pml5_entry_t *pml5e; 1490 pml4_entry_t *pml4e; 1491 pt_entry_t PG_V; 1492 1493 if (pmap_is_la57(pmap)) { 1494 pml5e = pmap_pml5e(pmap, va); 1495 PG_V = pmap_valid_bit(pmap); 1496 if ((*pml5e & PG_V) == 0) 1497 return (NULL); 1498 pml4e = PHYS_TO_DMAP(*pml5e & PG_FRAME); 1499 } else { 1500 pml4e = pmap->pm_pmltop; 1501 } 1502 return (&pml4e[pmap_pml4e_index(va)]); 1503 } 1504 1505 static __inline pml4_entry_t * 1506 pmap_pml4e_u(pmap_t pmap, vm_offset_t va) 1507 { 1508 MPASS(!pmap_is_la57(pmap)); 1509 return (&pmap->pm_pmltopu[pmap_pml4e_index(va)]); 1510 } 1511 1512 /* Return a pointer to the PDP slot that corresponds to a VA */ 1513 static __inline pdp_entry_t * 1514 pmap_pml4e_to_pdpe(pml4_entry_t *pml4e, vm_offset_t va) 1515 { 1516 pdp_entry_t *pdpe; 1517 1518 pdpe = PHYS_TO_DMAP(*pml4e & PG_FRAME); 1519 return (&pdpe[pmap_pdpe_index(va)]); 1520 } 1521 1522 /* Return a pointer to the PDP slot that corresponds to a VA */ 1523 static __inline pdp_entry_t * 1524 pmap_pdpe(pmap_t pmap, vm_offset_t va) 1525 { 1526 pml4_entry_t *pml4e; 1527 pt_entry_t PG_V; 1528 1529 PG_V = pmap_valid_bit(pmap); 1530 pml4e = pmap_pml4e(pmap, va); 1531 if (pml4e == NULL || (*pml4e & PG_V) == 0) 1532 return (NULL); 1533 return (pmap_pml4e_to_pdpe(pml4e, va)); 1534 } 1535 1536 /* Return a pointer to the PD slot that corresponds to a VA */ 1537 static __inline pd_entry_t * 1538 pmap_pdpe_to_pde(pdp_entry_t *pdpe, vm_offset_t va) 1539 { 1540 pd_entry_t *pde; 1541 1542 KASSERT((*pdpe & PG_PS) == 0, 1543 ("%s: pdpe %#lx is a leaf", __func__, *pdpe)); 1544 pde = PHYS_TO_DMAP(*pdpe & PG_FRAME); 1545 return (&pde[pmap_pde_index(va)]); 1546 } 1547 1548 /* Return a pointer to the PD slot that corresponds to a VA */ 1549 static __inline pd_entry_t * 1550 pmap_pde(pmap_t pmap, vm_offset_t va) 1551 { 1552 pdp_entry_t *pdpe; 1553 pt_entry_t PG_V; 1554 1555 PG_V = pmap_valid_bit(pmap); 1556 pdpe = pmap_pdpe(pmap, va); 1557 if (pdpe == NULL || (*pdpe & PG_V) == 0) 1558 return (NULL); 1559 KASSERT((*pdpe & PG_PS) == 0, 1560 ("pmap_pde for 1G page, pmap %p va %#lx", pmap, va)); 1561 return (pmap_pdpe_to_pde(pdpe, va)); 1562 } 1563 1564 /* Return a pointer to the PT slot that corresponds to a VA */ 1565 static __inline pt_entry_t * 1566 pmap_pde_to_pte(pd_entry_t *pde, vm_offset_t va) 1567 { 1568 pt_entry_t *pte; 1569 1570 KASSERT((*pde & PG_PS) == 0, 1571 ("%s: pde %#lx is a leaf", __func__, *pde)); 1572 pte = PHYS_TO_DMAP(*pde & PG_FRAME); 1573 return (&pte[pmap_pte_index(va)]); 1574 } 1575 1576 /* Return a pointer to the PT slot that corresponds to a VA */ 1577 static __inline pt_entry_t * 1578 pmap_pte(pmap_t pmap, vm_offset_t va) 1579 { 1580 pd_entry_t *pde; 1581 pt_entry_t PG_V; 1582 1583 PG_V = pmap_valid_bit(pmap); 1584 pde = pmap_pde(pmap, va); 1585 if (pde == NULL || (*pde & PG_V) == 0) 1586 return (NULL); 1587 if ((*pde & PG_PS) != 0) /* compat with i386 pmap_pte() */ 1588 return ((pt_entry_t *)pde); 1589 return (pmap_pde_to_pte(pde, va)); 1590 } 1591 1592 static __inline void 1593 pmap_resident_count_adj(pmap_t pmap, int count) 1594 { 1595 1596 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1597 KASSERT(pmap->pm_stats.resident_count + count >= 0, 1598 ("pmap %p resident count underflow %ld %d", pmap, 1599 pmap->pm_stats.resident_count, count)); 1600 pmap->pm_stats.resident_count += count; 1601 } 1602 1603 static __inline void 1604 pmap_pt_page_count_pinit(pmap_t pmap, int count) 1605 { 1606 KASSERT(pmap->pm_stats.resident_count + count >= 0, 1607 ("pmap %p resident count underflow %ld %d", pmap, 1608 pmap->pm_stats.resident_count, count)); 1609 pmap->pm_stats.resident_count += count; 1610 } 1611 1612 static __inline void 1613 pmap_pt_page_count_adj(pmap_t pmap, int count) 1614 { 1615 if (pmap == kernel_pmap) 1616 counter_u64_add(kernel_pt_page_count, count); 1617 else { 1618 if (pmap != NULL) 1619 pmap_resident_count_adj(pmap, count); 1620 counter_u64_add(user_pt_page_count, count); 1621 } 1622 } 1623 1624 pt_entry_t vtoptem __read_mostly = ((1ul << (NPTEPGSHIFT + NPDEPGSHIFT + 1625 NPDPEPGSHIFT + NPML4EPGSHIFT)) - 1) << 3; 1626 vm_offset_t PTmap __read_mostly = (vm_offset_t)P4Tmap; 1627 1628 pt_entry_t * 1629 vtopte(vm_offset_t va) 1630 { 1631 KASSERT(va >= VM_MAXUSER_ADDRESS, ("vtopte on a uva/gpa 0x%0lx", va)); 1632 1633 return ((pt_entry_t *)(PTmap + ((va >> (PAGE_SHIFT - 3)) & vtoptem))); 1634 } 1635 1636 pd_entry_t vtopdem __read_mostly = ((1ul << (NPDEPGSHIFT + NPDPEPGSHIFT + 1637 NPML4EPGSHIFT)) - 1) << 3; 1638 vm_offset_t PDmap __read_mostly = (vm_offset_t)P4Dmap; 1639 1640 static __inline pd_entry_t * 1641 vtopde(vm_offset_t va) 1642 { 1643 KASSERT(va >= VM_MAXUSER_ADDRESS, ("vtopde on a uva/gpa 0x%0lx", va)); 1644 1645 return ((pt_entry_t *)(PDmap + ((va >> (PDRSHIFT - 3)) & vtopdem))); 1646 } 1647 1648 static u_int64_t 1649 allocpages(vm_paddr_t *firstaddr, int n) 1650 { 1651 u_int64_t ret; 1652 1653 ret = *firstaddr; 1654 bzero((void *)ret, n * PAGE_SIZE); 1655 *firstaddr += n * PAGE_SIZE; 1656 return (ret); 1657 } 1658 1659 CTASSERT(powerof2(NDMPML4E)); 1660 1661 /* number of kernel PDP slots */ 1662 #define NKPDPE(ptpgs) howmany(ptpgs, NPDEPG) 1663 1664 static void 1665 nkpt_init(vm_paddr_t addr) 1666 { 1667 int pt_pages; 1668 1669 #ifdef NKPT 1670 pt_pages = NKPT; 1671 #else 1672 pt_pages = howmany(addr - kernphys, NBPDR) + 1; /* +1 for 2M hole @0 */ 1673 pt_pages += NKPDPE(pt_pages); 1674 1675 /* 1676 * Add some slop beyond the bare minimum required for bootstrapping 1677 * the kernel. 1678 * 1679 * This is quite important when allocating KVA for kernel modules. 1680 * The modules are required to be linked in the negative 2GB of 1681 * the address space. If we run out of KVA in this region then 1682 * pmap_growkernel() will need to allocate page table pages to map 1683 * the entire 512GB of KVA space which is an unnecessary tax on 1684 * physical memory. 1685 * 1686 * Secondly, device memory mapped as part of setting up the low- 1687 * level console(s) is taken from KVA, starting at virtual_avail. 1688 * This is because cninit() is called after pmap_bootstrap() but 1689 * before vm_mem_init() and pmap_init(). 20MB for a frame buffer 1690 * is not uncommon. 1691 */ 1692 pt_pages += 32; /* 64MB additional slop. */ 1693 #endif 1694 nkpt = pt_pages; 1695 } 1696 1697 /* 1698 * Returns the proper write/execute permission for a physical page that is 1699 * part of the initial boot allocations. 1700 * 1701 * If the page has kernel text, it is marked as read-only. If the page has 1702 * kernel read-only data, it is marked as read-only/not-executable. If the 1703 * page has only read-write data, it is marked as read-write/not-executable. 1704 * If the page is below/above the kernel range, it is marked as read-write. 1705 * 1706 * This function operates on 2M pages, since we map the kernel space that 1707 * way. 1708 */ 1709 static inline pt_entry_t 1710 bootaddr_rwx(vm_paddr_t pa) 1711 { 1712 /* 1713 * The kernel is loaded at a 2MB-aligned address, and memory below that 1714 * need not be executable. The .bss section is padded to a 2MB 1715 * boundary, so memory following the kernel need not be executable 1716 * either. Preloaded kernel modules have their mapping permissions 1717 * fixed up by the linker. 1718 */ 1719 if (pa < trunc_2mpage(kernphys + btext - KERNSTART) || 1720 pa >= trunc_2mpage(kernphys + _end - KERNSTART)) 1721 return (X86_PG_RW | pg_nx); 1722 1723 /* 1724 * The linker should ensure that the read-only and read-write 1725 * portions don't share the same 2M page, so this shouldn't 1726 * impact read-only data. However, in any case, any page with 1727 * read-write data needs to be read-write. 1728 */ 1729 if (pa >= trunc_2mpage(kernphys + brwsection - KERNSTART)) 1730 return (X86_PG_RW | pg_nx); 1731 1732 /* 1733 * Mark any 2M page containing kernel text as read-only. Mark 1734 * other pages with read-only data as read-only and not executable. 1735 * (It is likely a small portion of the read-only data section will 1736 * be marked as read-only, but executable. This should be acceptable 1737 * since the read-only protection will keep the data from changing.) 1738 * Note that fixups to the .text section will still work until we 1739 * set CR0.WP. 1740 */ 1741 if (pa < round_2mpage(kernphys + etext - KERNSTART)) 1742 return (0); 1743 return (pg_nx); 1744 } 1745 1746 extern const char la57_trampoline[]; 1747 1748 static void 1749 pmap_bootstrap_la57(vm_paddr_t *firstaddr) 1750 { 1751 void (*la57_tramp)(uint64_t pml5); 1752 pml5_entry_t *pt; 1753 uint64_t cr4; 1754 1755 if ((cpu_stdext_feature2 & CPUID_STDEXT2_LA57) == 0) 1756 return; 1757 la57 = 1; 1758 TUNABLE_INT_FETCH("vm.pmap.la57", &la57); 1759 if (!la57) 1760 return; 1761 1762 KPML5phys = allocpages(firstaddr, 1); 1763 KPML4phys = rcr3() & 0xfffff000; /* pml4 from loader must be < 4G */ 1764 1765 pt = (pml5_entry_t *)KPML5phys; 1766 pt[0] = KPML4phys | X86_PG_V | X86_PG_RW | X86_PG_A | X86_PG_M; 1767 pt[NPML4EPG - 1] = KPML4phys | X86_PG_V | X86_PG_RW | X86_PG_A | 1768 X86_PG_M; 1769 1770 la57_tramp = (void (*)(uint64_t))((uintptr_t)la57_trampoline - 1771 KERNSTART + amd64_loadaddr()); 1772 printf("Calling la57 trampoline at %p, KPML5phys %#lx ...", 1773 la57_tramp, KPML5phys); 1774 if (lass_enabled) { 1775 cr4 = rcr4(); 1776 load_cr4(cr4 & ~CR4_LASS); 1777 } 1778 la57_tramp(KPML5phys); 1779 printf(" alive in la57 mode\n"); 1780 if (lass_enabled) { 1781 cr4 = rcr4(); 1782 load_cr4(cr4 | CR4_LASS); 1783 } 1784 } 1785 1786 static void 1787 create_pagetables(vm_paddr_t *firstaddr) 1788 { 1789 pd_entry_t *pd_p; 1790 pdp_entry_t *pdp_p; 1791 pml4_entry_t *p4_p, *p4d_p; 1792 pml5_entry_t *p5_p; 1793 uint64_t DMPDkernphys; 1794 vm_paddr_t pax; 1795 #ifdef KASAN 1796 pt_entry_t *pt_p; 1797 uint64_t KASANPDphys, KASANPTphys, KASANphys; 1798 vm_offset_t kasankernbase; 1799 int kasankpdpi, kasankpdi, nkasanpte; 1800 #endif 1801 int i, j, ndm1g, nkpdpe, nkdmpde, ndmpml4phys; 1802 1803 TSENTER(); 1804 /* Allocate page table pages for the direct map */ 1805 ndmpdp = howmany(ptoa(Maxmem), NBPDP); 1806 if (ndmpdp < 4) /* Minimum 4GB of dirmap */ 1807 ndmpdp = 4; 1808 ndmpdpphys = howmany(ndmpdp, NPDPEPG); 1809 if (la57) { 1810 ndmpml4phys = howmany(ndmpdpphys, NPML4EPG); 1811 if (ndmpml4phys > NDMPML5E) { 1812 printf("NDMPML5E limits system to %ld GB\n", 1813 (u_long)NDMPML5E * NBPML5 / 1024 / 1024 / 1024); 1814 Maxmem = atop(NDMPML5E * NBPML5); 1815 ndmpml4phys = NDMPML5E; 1816 ndmpdpphys = ndmpml4phys * NPML4EPG; 1817 ndmpdp = ndmpdpphys * NPDEPG; 1818 } 1819 DMPML4phys = allocpages(firstaddr, ndmpml4phys); 1820 } else { 1821 if (ndmpdpphys > NDMPML4E) { 1822 /* 1823 * Each NDMPML4E allows 512 GB, so limit to 1824 * that, and then readjust ndmpdp and 1825 * ndmpdpphys. 1826 */ 1827 printf("NDMPML4E limits system to %d GB\n", 1828 NDMPML4E * 512); 1829 Maxmem = atop(NDMPML4E * NBPML4); 1830 ndmpdpphys = NDMPML4E; 1831 ndmpdp = NDMPML4E * NPDEPG; 1832 } 1833 } 1834 DMPDPphys = allocpages(firstaddr, ndmpdpphys); 1835 ndm1g = 0; 1836 if ((amd_feature & AMDID_PAGE1GB) != 0) { 1837 /* 1838 * Calculate the number of 1G pages that will fully fit in 1839 * Maxmem. 1840 */ 1841 ndm1g = ptoa(Maxmem) >> PDPSHIFT; 1842 1843 /* 1844 * Allocate 2M pages for the kernel. These will be used in 1845 * place of the one or more 1G pages from ndm1g that maps 1846 * kernel memory into DMAP. 1847 */ 1848 nkdmpde = howmany((vm_offset_t)brwsection - KERNSTART + 1849 kernphys - rounddown2(kernphys, NBPDP), NBPDP); 1850 DMPDkernphys = allocpages(firstaddr, nkdmpde); 1851 } 1852 if (ndm1g < ndmpdp) 1853 DMPDphys = allocpages(firstaddr, ndmpdp - ndm1g); 1854 dmaplimit = (vm_paddr_t)ndmpdp << PDPSHIFT; 1855 1856 /* Allocate pages. */ 1857 if (la57) { 1858 KPML5phys = allocpages(firstaddr, 1); 1859 p5_p = (pml5_entry_t *)KPML5phys; 1860 } 1861 KPML4phys = allocpages(firstaddr, 1); 1862 p4_p = (pml4_entry_t *)KPML4phys; 1863 1864 KPDPphys = allocpages(firstaddr, NKPML4E); 1865 #ifdef KASAN 1866 KASANPDPphys = allocpages(firstaddr, NKASANPML4E); 1867 KASANPDphys = allocpages(firstaddr, 1); 1868 #endif 1869 #ifdef KMSAN 1870 /* 1871 * The KMSAN shadow maps are initially left unpopulated, since there is 1872 * no need to shadow memory above KERNBASE. 1873 */ 1874 KMSANSHADPDPphys = allocpages(firstaddr, NKMSANSHADPML4E); 1875 KMSANORIGPDPphys = allocpages(firstaddr, NKMSANORIGPML4E); 1876 #endif 1877 1878 /* 1879 * Allocate the initial number of kernel page table pages required to 1880 * bootstrap. We defer this until after all memory-size dependent 1881 * allocations are done (e.g. direct map), so that we don't have to 1882 * build in too much slop in our estimate. 1883 * 1884 * Note that when NKPML4E > 1, we have an empty page underneath 1885 * all but the KPML4I'th one, so we need NKPML4E-1 extra (zeroed) 1886 * pages. (pmap_enter requires a PD page to exist for each KPML4E.) 1887 */ 1888 nkpt_init(*firstaddr); 1889 nkpdpe = NKPDPE(nkpt); 1890 1891 KPTphys = allocpages(firstaddr, nkpt); 1892 KPDphys = allocpages(firstaddr, nkpdpe); 1893 1894 #ifdef KASAN 1895 nkasanpte = howmany(nkpt, KASAN_SHADOW_SCALE); 1896 KASANPTphys = allocpages(firstaddr, nkasanpte); 1897 KASANphys = allocpages(firstaddr, nkasanpte * NPTEPG); 1898 #endif 1899 1900 /* 1901 * Connect the zero-filled PT pages to their PD entries. This 1902 * implicitly maps the PT pages at their correct locations within 1903 * the PTmap. 1904 */ 1905 pd_p = (pd_entry_t *)KPDphys; 1906 for (i = 0; i < nkpt; i++) 1907 pd_p[i] = (KPTphys + ptoa(i)) | X86_PG_RW | X86_PG_V; 1908 1909 /* 1910 * Map from start of the kernel in physical memory (staging 1911 * area) to the end of loader preallocated memory using 2MB 1912 * pages. This replaces some of the PD entries created above. 1913 * For compatibility, identity map 2M at the start. 1914 */ 1915 pd_p[0] = X86_PG_V | PG_PS | pg_g | X86_PG_M | X86_PG_A | 1916 X86_PG_RW | pg_nx; 1917 for (i = 1, pax = kernphys; pax < KERNend; i++, pax += NBPDR) { 1918 /* Preset PG_M and PG_A because demotion expects it. */ 1919 pd_p[i] = pax | X86_PG_V | PG_PS | pg_g | X86_PG_M | 1920 X86_PG_A | bootaddr_rwx(pax); 1921 } 1922 1923 /* 1924 * Because we map the physical blocks in 2M pages, adjust firstaddr 1925 * to record the physical blocks we've actually mapped into kernel 1926 * virtual address space. 1927 */ 1928 if (*firstaddr < round_2mpage(KERNend)) 1929 *firstaddr = round_2mpage(KERNend); 1930 1931 /* And connect up the PD to the PDP (leaving room for L4 pages) */ 1932 pdp_p = (pdp_entry_t *)(KPDPphys + ptoa(KPML4I - KPML4BASE)); 1933 for (i = 0; i < nkpdpe; i++) 1934 pdp_p[i + KPDPI] = (KPDphys + ptoa(i)) | X86_PG_RW | X86_PG_V; 1935 1936 #ifdef KASAN 1937 kasankernbase = kasan_md_addr_to_shad(KERNBASE); 1938 kasankpdpi = pmap_pdpe_index(kasankernbase); 1939 kasankpdi = pmap_pde_index(kasankernbase); 1940 1941 pdp_p = (pdp_entry_t *)KASANPDPphys; 1942 pdp_p[kasankpdpi] = (KASANPDphys | X86_PG_RW | X86_PG_V | pg_nx); 1943 1944 pd_p = (pd_entry_t *)KASANPDphys; 1945 for (i = 0; i < nkasanpte; i++) 1946 pd_p[i + kasankpdi] = (KASANPTphys + ptoa(i)) | X86_PG_RW | 1947 X86_PG_V | pg_nx; 1948 1949 pt_p = (pt_entry_t *)KASANPTphys; 1950 for (i = 0; i < nkasanpte * NPTEPG; i++) 1951 pt_p[i] = (KASANphys + ptoa(i)) | X86_PG_RW | X86_PG_V | 1952 X86_PG_M | X86_PG_A | pg_nx; 1953 #endif 1954 1955 /* 1956 * Now, set up the direct map region using 2MB and/or 1GB pages. If 1957 * the end of physical memory is not aligned to a 1GB page boundary, 1958 * then the residual physical memory is mapped with 2MB pages. Later, 1959 * if pmap_mapdev{_attr}() uses the direct map for non-write-back 1960 * memory, pmap_change_attr() will demote any 2MB or 1GB page mappings 1961 * that are partially used. 1962 */ 1963 pd_p = (pd_entry_t *)DMPDphys; 1964 for (i = NPDEPG * ndm1g, j = 0; i < NPDEPG * ndmpdp; i++, j++) { 1965 pd_p[j] = (vm_paddr_t)i << PDRSHIFT; 1966 /* Preset PG_M and PG_A because demotion expects it. */ 1967 pd_p[j] |= X86_PG_RW | X86_PG_V | PG_PS | pg_g | 1968 X86_PG_M | X86_PG_A | pg_nx; 1969 } 1970 pdp_p = (pdp_entry_t *)DMPDPphys; 1971 for (i = 0; i < ndm1g; i++) { 1972 pdp_p[i] = (vm_paddr_t)i << PDPSHIFT; 1973 /* Preset PG_M and PG_A because demotion expects it. */ 1974 pdp_p[i] |= X86_PG_RW | X86_PG_V | PG_PS | pg_g | 1975 X86_PG_M | X86_PG_A | pg_nx; 1976 } 1977 for (j = 0; i < ndmpdp; i++, j++) { 1978 pdp_p[i] = DMPDphys + ptoa(j); 1979 pdp_p[i] |= X86_PG_RW | X86_PG_V | pg_nx; 1980 } 1981 1982 /* 1983 * Connect the Direct Map slots up to the PML4. 1984 * pml5 entries for DMAP are handled below in global pml5 loop. 1985 */ 1986 p4d_p = la57 ? (pml4_entry_t *)DMPML4phys : &p4_p[DMPML4I]; 1987 for (i = 0; i < ndmpdpphys; i++) { 1988 p4d_p[i] = (DMPDPphys + ptoa(i)) | X86_PG_RW | X86_PG_V | 1989 pg_nx; 1990 } 1991 1992 /* 1993 * Instead of using a 1G page for the memory containing the kernel, 1994 * use 2M pages with read-only and no-execute permissions. (If using 1G 1995 * pages, this will partially overwrite the PDPEs above.) 1996 */ 1997 if (ndm1g > 0) { 1998 pd_p = (pd_entry_t *)DMPDkernphys; 1999 for (i = 0, pax = rounddown2(kernphys, NBPDP); 2000 i < NPDEPG * nkdmpde; i++, pax += NBPDR) { 2001 pd_p[i] = pax | X86_PG_V | PG_PS | pg_g | X86_PG_M | 2002 X86_PG_A | pg_nx | bootaddr_rwx(pax); 2003 } 2004 j = rounddown2(kernphys, NBPDP) >> PDPSHIFT; 2005 for (i = 0; i < nkdmpde; i++) { 2006 pdp_p[i + j] = (DMPDkernphys + ptoa(i)) | 2007 X86_PG_RW | X86_PG_V | pg_nx; 2008 } 2009 } 2010 2011 #ifdef KASAN 2012 /* Connect the KASAN shadow map slots up to the PML4. */ 2013 for (i = 0; i < NKASANPML4E; i++) { 2014 p4_p[KASANPML4I + i] = KASANPDPphys + ptoa(i); 2015 p4_p[KASANPML4I + i] |= X86_PG_RW | X86_PG_V | pg_nx; 2016 } 2017 #endif 2018 2019 #ifdef KMSAN 2020 /* Connect the KMSAN shadow map slots up to the PML4. */ 2021 for (i = 0; i < NKMSANSHADPML4E; i++) { 2022 p4_p[KMSANSHADPML4I + i] = KMSANSHADPDPphys + ptoa(i); 2023 p4_p[KMSANSHADPML4I + i] |= X86_PG_RW | X86_PG_V | pg_nx; 2024 } 2025 2026 /* Connect the KMSAN origin map slots up to the PML4. */ 2027 for (i = 0; i < NKMSANORIGPML4E; i++) { 2028 p4_p[KMSANORIGPML4I + i] = KMSANORIGPDPphys + ptoa(i); 2029 p4_p[KMSANORIGPML4I + i] |= X86_PG_RW | X86_PG_V | pg_nx; 2030 } 2031 #endif 2032 2033 /* Connect the KVA slots up to the PML4 */ 2034 for (i = 0; i < NKPML4E; i++) { 2035 p4_p[KPML4BASE + i] = KPDPphys + ptoa(i); 2036 p4_p[KPML4BASE + i] |= X86_PG_RW | X86_PG_V; 2037 } 2038 2039 if (la57) { 2040 /* XXXKIB bootstrap KPML5phys page is lost */ 2041 for (i = 0; i < NPML5EPG; i++) { 2042 if (i == PML5PML5I) { 2043 /* 2044 * Recursively map PML5 to itself in 2045 * order to get PTmap and PDmap. 2046 */ 2047 p5_p[i] = KPML5phys | X86_PG_RW | X86_PG_A | 2048 X86_PG_M | X86_PG_V | pg_nx; 2049 } else if (i >= DMPML5I && i < DMPML5I + ndmpml4phys) { 2050 /* Connect DMAP pml4 pages to PML5. */ 2051 p5_p[i] = (DMPML4phys + ptoa(i - DMPML5I)) | 2052 X86_PG_RW | X86_PG_V | pg_nx; 2053 } else if (i == pmap_pml5e_index(UPT_MAX_ADDRESS)) { 2054 p5_p[i] = KPML4phys | X86_PG_RW | X86_PG_A | 2055 X86_PG_M | X86_PG_V; 2056 } else { 2057 p5_p[i] = 0; 2058 } 2059 } 2060 } else { 2061 /* Recursively map PML4 to itself in order to get PTmap */ 2062 p4_p[PML4PML4I] = KPML4phys; 2063 p4_p[PML4PML4I] |= X86_PG_RW | X86_PG_V | pg_nx; 2064 } 2065 TSEXIT(); 2066 } 2067 2068 /* 2069 * Bootstrap the system enough to run with virtual memory. 2070 * 2071 * On amd64 this is called after mapping has already been enabled 2072 * and just syncs the pmap module with what has already been done. 2073 * [We can't call it easily with mapping off since the kernel is not 2074 * mapped with PA == VA, hence we would have to relocate every address 2075 * from the linked base (virtual) address "KERNBASE" to the actual 2076 * (physical) address starting relative to 0] 2077 */ 2078 void 2079 pmap_bootstrap(vm_paddr_t *firstaddr) 2080 { 2081 vm_offset_t va; 2082 pt_entry_t *pte, *pcpu_pte; 2083 struct region_descriptor r_gdt; 2084 uint64_t cr4, pcpu0_phys; 2085 u_long res; 2086 int i; 2087 2088 TSENTER(); 2089 KERNend = *firstaddr; 2090 res = atop(KERNend - (vm_paddr_t)kernphys); 2091 2092 if (!pti) 2093 pg_g = X86_PG_G; 2094 2095 /* 2096 * Create an initial set of page tables to run the kernel in. 2097 */ 2098 pmap_bootstrap_la57(firstaddr); 2099 create_pagetables(firstaddr); 2100 2101 pcpu0_phys = allocpages(firstaddr, 1); 2102 2103 /* 2104 * Add a physical memory segment (vm_phys_seg) corresponding to the 2105 * preallocated kernel page table pages so that vm_page structures 2106 * representing these pages will be created. The vm_page structures 2107 * are required for promotion of the corresponding kernel virtual 2108 * addresses to superpage mappings. 2109 */ 2110 vm_phys_early_add_seg(KPTphys, KPTphys + ptoa(nkpt)); 2111 2112 /* 2113 * Account for the virtual addresses mapped by create_pagetables(). 2114 */ 2115 virtual_avail = (vm_offset_t)KERNSTART + round_2mpage(KERNend - 2116 (vm_paddr_t)kernphys); 2117 virtual_end = kva_layout.km_high; 2118 2119 /* 2120 * Enable PG_G global pages, then switch to the kernel page 2121 * table from the bootstrap page table. After the switch, it 2122 * is possible to enable SMEP and SMAP since PG_U bits are 2123 * correct now. 2124 */ 2125 cr4 = rcr4(); 2126 cr4 |= CR4_PGE; 2127 load_cr4(cr4); 2128 load_cr3(la57 ? KPML5phys : KPML4phys); 2129 if (cpu_stdext_feature & CPUID_STDEXT_SMEP) 2130 cr4 |= CR4_SMEP; 2131 if (cpu_stdext_feature & CPUID_STDEXT_SMAP) 2132 cr4 |= CR4_SMAP; 2133 load_cr4(cr4); 2134 2135 /* 2136 * Initialize the kernel pmap (which is statically allocated). 2137 * Count bootstrap data as being resident in case any of this data is 2138 * later unmapped (using pmap_remove()) and freed. 2139 * 2140 * DMAP_TO_PHYS()/PHYS_TO_DMAP() are functional only after 2141 * kva_layout is fixed. 2142 */ 2143 mtx_init(&kernel_pmap->pm_mtx, "kernel pmap", NULL, MTX_DEF); 2144 if (la57) { 2145 kva_layout = kva_layout_la57; 2146 vtoptem = ((1ul << (NPTEPGSHIFT + NPDEPGSHIFT + NPDPEPGSHIFT + 2147 NPML4EPGSHIFT + NPML5EPGSHIFT)) - 1) << 3; 2148 PTmap = (vm_offset_t)P5Tmap; 2149 vtopdem = ((1ul << (NPDEPGSHIFT + NPDPEPGSHIFT + 2150 NPML4EPGSHIFT + NPML5EPGSHIFT)) - 1) << 3; 2151 PDmap = (vm_offset_t)P5Dmap; 2152 kernel_pmap->pm_pmltop = PHYS_TO_DMAP(KPML5phys); 2153 kernel_pmap->pm_cr3 = KPML5phys; 2154 pmap_pt_page_count_adj(kernel_pmap, 1); /* top-level page */ 2155 } else { 2156 kernel_pml4 = PHYS_TO_DMAP(KPML4phys); 2157 kernel_pmap->pm_pmltop = kernel_pml4; 2158 kernel_pmap->pm_cr3 = KPML4phys; 2159 } 2160 kernel_pmap->pm_ucr3 = PMAP_NO_CR3; 2161 TAILQ_INIT(&kernel_pmap->pm_pvchunk); 2162 kernel_pmap->pm_stats.resident_count = res; 2163 vm_radix_init(&kernel_pmap->pm_root); 2164 kernel_pmap->pm_flags = pmap_flags; 2165 if ((cpu_stdext_feature2 & CPUID_STDEXT2_PKU) != 0) { 2166 rangeset_init(&kernel_pmap->pm_pkru, pkru_dup_range, 2167 pkru_free_range, kernel_pmap, M_NOWAIT); 2168 } 2169 2170 /* 2171 * The kernel pmap is always active on all CPUs. Once CPUs are 2172 * enumerated, the mask will be set equal to all_cpus. 2173 */ 2174 CPU_FILL(&kernel_pmap->pm_active); 2175 2176 /* 2177 * Initialize the TLB invalidations generation number lock. 2178 */ 2179 mtx_init(&invl_gen_mtx, "invlgn", NULL, MTX_DEF); 2180 2181 /* 2182 * Reserve some special page table entries/VA space for temporary 2183 * mapping of pages. 2184 */ 2185 #define SYSMAP(c, p, v, n) \ 2186 v = (c)va; va += ((n)*PAGE_SIZE); p = pte; pte += (n); 2187 2188 va = virtual_avail; 2189 pte = vtopte(va); 2190 2191 /* 2192 * Crashdump maps. The first page is reused as CMAP1 for the 2193 * memory test. 2194 */ 2195 SYSMAP(caddr_t, CMAP1, crashdumpmap, MAXDUMPPGS) 2196 CADDR1 = crashdumpmap; 2197 2198 SYSMAP(struct pcpu *, pcpu_pte, __pcpu, MAXCPU); 2199 virtual_avail = va; 2200 2201 /* 2202 * Map the BSP PCPU now, the rest of the PCPUs are mapped by 2203 * amd64_mp_alloc_pcpu()/start_all_aps() when we know the 2204 * number of CPUs and NUMA affinity. 2205 */ 2206 pcpu_pte[0] = pcpu0_phys | X86_PG_V | X86_PG_RW | pg_g | pg_nx | 2207 X86_PG_M | X86_PG_A; 2208 for (i = 1; i < MAXCPU; i++) 2209 pcpu_pte[i] = 0; 2210 2211 /* 2212 * Re-initialize PCPU area for BSP after switching. 2213 * Make hardware use gdt and common_tss from the new PCPU. 2214 * Also clears the usage of temporary gdt during switch to 2215 * LA57 paging. 2216 */ 2217 STAILQ_INIT(&cpuhead); 2218 wrmsr(MSR_GSBASE, (uint64_t)&__pcpu[0]); 2219 pcpu_init(&__pcpu[0], 0, sizeof(struct pcpu)); 2220 amd64_bsp_pcpu_init1(&__pcpu[0]); 2221 amd64_bsp_ist_init(&__pcpu[0]); 2222 __pcpu[0].pc_common_tss.tss_iobase = sizeof(struct amd64tss) + 2223 IOPERM_BITMAP_SIZE; 2224 memcpy(__pcpu[0].pc_gdt, temp_bsp_pcpu.pc_gdt, NGDT * 2225 sizeof(struct user_segment_descriptor)); 2226 gdt_segs[GPROC0_SEL].ssd_base = (uintptr_t)&__pcpu[0].pc_common_tss; 2227 ssdtosyssd(&gdt_segs[GPROC0_SEL], 2228 (struct system_segment_descriptor *)&__pcpu[0].pc_gdt[GPROC0_SEL]); 2229 r_gdt.rd_limit = NGDT * sizeof(struct user_segment_descriptor) - 1; 2230 r_gdt.rd_base = (long)__pcpu[0].pc_gdt; 2231 lgdt(&r_gdt); 2232 wrmsr(MSR_GSBASE, (uint64_t)&__pcpu[0]); 2233 ltr(GSEL(GPROC0_SEL, SEL_KPL)); 2234 __pcpu[0].pc_dynamic = temp_bsp_pcpu.pc_dynamic; 2235 __pcpu[0].pc_acpi_id = temp_bsp_pcpu.pc_acpi_id; 2236 2237 /* 2238 * Initialize the PAT MSR. 2239 * pmap_init_pat() clears and sets CR4_PGE, which, as a 2240 * side-effect, invalidates stale PG_G TLB entries that might 2241 * have been created in our pre-boot environment. 2242 */ 2243 pmap_init_pat(); 2244 2245 /* Initialize TLB Context Id. */ 2246 if (pmap_pcid_enabled) { 2247 kernel_pmap->pm_pcidp = (void *)(uintptr_t) 2248 offsetof(struct pcpu, pc_kpmap_store); 2249 2250 PCPU_SET(kpmap_store.pm_pcid, PMAP_PCID_KERN); 2251 PCPU_SET(kpmap_store.pm_gen, 1); 2252 2253 /* 2254 * PMAP_PCID_KERN + 1 is used for initialization of 2255 * proc0 pmap. The pmap' pcid state might be used by 2256 * EFIRT entry before first context switch, so it 2257 * needs to be valid. 2258 */ 2259 PCPU_SET(pcid_next, PMAP_PCID_KERN + 2); 2260 PCPU_SET(pcid_gen, 1); 2261 2262 /* 2263 * pcpu area for APs is zeroed during AP startup. 2264 * pc_pcid_next and pc_pcid_gen are initialized by AP 2265 * during pcpu setup. 2266 */ 2267 load_cr4(rcr4() | CR4_PCIDE); 2268 } 2269 TSEXIT(); 2270 } 2271 2272 /* 2273 * Setup the PAT MSR. 2274 */ 2275 void 2276 pmap_init_pat(void) 2277 { 2278 uint64_t pat_msr; 2279 u_long cr0, cr4; 2280 int i; 2281 2282 /* Bail if this CPU doesn't implement PAT. */ 2283 if ((cpu_feature & CPUID_PAT) == 0) 2284 panic("no PAT??"); 2285 2286 /* Set default PAT index table. */ 2287 for (i = 0; i < PAT_INDEX_SIZE; i++) 2288 pat_index[i] = -1; 2289 pat_index[PAT_WRITE_BACK] = 0; 2290 pat_index[PAT_WRITE_THROUGH] = 1; 2291 pat_index[PAT_UNCACHEABLE] = 3; 2292 pat_index[PAT_WRITE_COMBINING] = 6; 2293 pat_index[PAT_WRITE_PROTECTED] = 5; 2294 pat_index[PAT_UNCACHED] = 2; 2295 2296 /* 2297 * Initialize default PAT entries. 2298 * Leave the indices 0-3 at the default of WB, WT, UC-, and UC. 2299 * Program 5 and 6 as WP and WC. 2300 * 2301 * Leave 4 and 7 as WB and UC. Note that a recursive page table 2302 * mapping for a 2M page uses a PAT value with the bit 3 set due 2303 * to its overload with PG_PS. 2304 */ 2305 pat_msr = PAT_VALUE(0, PAT_WRITE_BACK) | 2306 PAT_VALUE(1, PAT_WRITE_THROUGH) | 2307 PAT_VALUE(2, PAT_UNCACHED) | 2308 PAT_VALUE(3, PAT_UNCACHEABLE) | 2309 PAT_VALUE(4, PAT_WRITE_BACK) | 2310 PAT_VALUE(5, PAT_WRITE_PROTECTED) | 2311 PAT_VALUE(6, PAT_WRITE_COMBINING) | 2312 PAT_VALUE(7, PAT_UNCACHEABLE); 2313 2314 /* Disable PGE. */ 2315 cr4 = rcr4(); 2316 load_cr4(cr4 & ~CR4_PGE); 2317 2318 /* Disable caches (CD = 1, NW = 0). */ 2319 cr0 = rcr0(); 2320 load_cr0((cr0 & ~CR0_NW) | CR0_CD); 2321 2322 /* Flushes caches and TLBs. */ 2323 wbinvd(); 2324 invltlb(); 2325 2326 /* Update PAT and index table. */ 2327 wrmsr(MSR_PAT, pat_msr); 2328 2329 /* Flush caches and TLBs again. */ 2330 wbinvd(); 2331 invltlb(); 2332 2333 /* Restore caches and PGE. */ 2334 load_cr0(cr0); 2335 load_cr4(cr4); 2336 } 2337 2338 vm_page_t 2339 pmap_page_alloc_below_4g(bool zeroed) 2340 { 2341 return (vm_page_alloc_noobj_contig((zeroed ? VM_ALLOC_ZERO : 0), 2342 1, 0, (1ULL << 32), PAGE_SIZE, 0, VM_MEMATTR_DEFAULT)); 2343 } 2344 2345 /* 2346 * Initialize a vm_page's machine-dependent fields. 2347 */ 2348 void 2349 pmap_page_init(vm_page_t m) 2350 { 2351 2352 TAILQ_INIT(&m->md.pv_list); 2353 m->md.pat_mode = PAT_WRITE_BACK; 2354 } 2355 2356 static int pmap_allow_2m_x_ept; 2357 SYSCTL_INT(_vm_pmap, OID_AUTO, allow_2m_x_ept, CTLFLAG_RWTUN | CTLFLAG_NOFETCH, 2358 &pmap_allow_2m_x_ept, 0, 2359 "Allow executable superpage mappings in EPT"); 2360 2361 void 2362 pmap_allow_2m_x_ept_recalculate(void) 2363 { 2364 /* 2365 * SKL002, SKL012S. Since the EPT format is only used by 2366 * Intel CPUs, the vendor check is merely a formality. 2367 */ 2368 if (!(cpu_vendor_id != CPU_VENDOR_INTEL || 2369 (cpu_ia32_arch_caps & IA32_ARCH_CAP_IF_PSCHANGE_MC_NO) != 0 || 2370 (CPUID_TO_FAMILY(cpu_id) == 0x6 && 2371 (CPUID_TO_MODEL(cpu_id) == 0x26 || /* Atoms */ 2372 CPUID_TO_MODEL(cpu_id) == 0x27 || 2373 CPUID_TO_MODEL(cpu_id) == 0x35 || 2374 CPUID_TO_MODEL(cpu_id) == 0x36 || 2375 CPUID_TO_MODEL(cpu_id) == 0x37 || 2376 CPUID_TO_MODEL(cpu_id) == 0x86 || 2377 CPUID_TO_MODEL(cpu_id) == 0x1c || 2378 CPUID_TO_MODEL(cpu_id) == 0x4a || 2379 CPUID_TO_MODEL(cpu_id) == 0x4c || 2380 CPUID_TO_MODEL(cpu_id) == 0x4d || 2381 CPUID_TO_MODEL(cpu_id) == 0x5a || 2382 CPUID_TO_MODEL(cpu_id) == 0x5c || 2383 CPUID_TO_MODEL(cpu_id) == 0x5d || 2384 CPUID_TO_MODEL(cpu_id) == 0x5f || 2385 CPUID_TO_MODEL(cpu_id) == 0x6e || 2386 CPUID_TO_MODEL(cpu_id) == 0x7a || 2387 CPUID_TO_MODEL(cpu_id) == 0x57 || /* Knights */ 2388 CPUID_TO_MODEL(cpu_id) == 0x85)))) 2389 pmap_allow_2m_x_ept = 1; 2390 #ifndef BURN_BRIDGES 2391 TUNABLE_INT_FETCH("hw.allow_2m_x_ept", &pmap_allow_2m_x_ept); 2392 #endif 2393 TUNABLE_INT_FETCH("vm.pmap.allow_2m_x_ept", &pmap_allow_2m_x_ept); 2394 } 2395 2396 static bool 2397 pmap_allow_2m_x_page(pmap_t pmap, bool executable) 2398 { 2399 2400 return (pmap->pm_type != PT_EPT || !executable || 2401 !pmap_allow_2m_x_ept); 2402 } 2403 2404 #ifdef NUMA 2405 static void 2406 pmap_init_pv_table(void) 2407 { 2408 struct pmap_large_md_page *pvd; 2409 vm_size_t s; 2410 long start, end, highest, pv_npg; 2411 int domain, i, j, pages; 2412 2413 /* 2414 * For correctness we depend on the size being evenly divisible into a 2415 * page. As a tradeoff between performance and total memory use, the 2416 * entry is 64 bytes (aka one cacheline) in size. Not being smaller 2417 * avoids false-sharing, but not being 128 bytes potentially allows for 2418 * avoidable traffic due to adjacent cacheline prefetcher. 2419 * 2420 * Assert the size so that accidental changes fail to compile. 2421 */ 2422 CTASSERT((sizeof(*pvd) == 64)); 2423 2424 /* 2425 * Calculate the size of the array. 2426 */ 2427 pmap_last_pa = vm_phys_segs[vm_phys_nsegs - 1].end; 2428 pv_npg = howmany(pmap_last_pa, NBPDR); 2429 s = (vm_size_t)pv_npg * sizeof(struct pmap_large_md_page); 2430 s = round_page(s); 2431 pv_table = kva_alloc(s); 2432 if (pv_table == NULL) 2433 panic("%s: kva_alloc failed\n", __func__); 2434 2435 /* 2436 * Iterate physical segments to allocate space for respective pages. 2437 */ 2438 highest = -1; 2439 s = 0; 2440 for (i = 0; i < vm_phys_nsegs; i++) { 2441 end = vm_phys_segs[i].end / NBPDR; 2442 domain = vm_phys_segs[i].domain; 2443 2444 if (highest >= end) 2445 continue; 2446 2447 start = highest + 1; 2448 pvd = &pv_table[start]; 2449 2450 pages = end - start + 1; 2451 s = round_page(pages * sizeof(*pvd)); 2452 highest = start + (s / sizeof(*pvd)) - 1; 2453 2454 for (j = 0; j < s; j += PAGE_SIZE) { 2455 vm_page_t m = vm_page_alloc_noobj_domain(domain, 0); 2456 if (m == NULL) 2457 panic("failed to allocate PV table page"); 2458 pmap_qenter((char *)pvd + j, &m, 1); 2459 } 2460 2461 for (j = 0; j < s / sizeof(*pvd); j++) { 2462 rw_init_flags(&pvd->pv_lock, "pmap pv list", RW_NEW); 2463 TAILQ_INIT(&pvd->pv_page.pv_list); 2464 pvd->pv_page.pv_gen = 0; 2465 pvd->pv_page.pat_mode = 0; 2466 pvd->pv_invl_gen = 0; 2467 pvd++; 2468 } 2469 } 2470 pvd = &pv_dummy_large; 2471 rw_init_flags(&pvd->pv_lock, "pmap pv list dummy", RW_NEW); 2472 TAILQ_INIT(&pvd->pv_page.pv_list); 2473 pvd->pv_page.pv_gen = 0; 2474 pvd->pv_page.pat_mode = 0; 2475 pvd->pv_invl_gen = 0; 2476 } 2477 #else 2478 static void 2479 pmap_init_pv_table(void) 2480 { 2481 vm_size_t s; 2482 long i, pv_npg; 2483 2484 /* 2485 * Initialize the pool of pv list locks. 2486 */ 2487 for (i = 0; i < NPV_LIST_LOCKS; i++) 2488 rw_init(&pv_list_locks[i], "pmap pv list"); 2489 2490 /* 2491 * Calculate the size of the pv head table for superpages. 2492 */ 2493 pv_npg = howmany(vm_phys_segs[vm_phys_nsegs - 1].end, NBPDR); 2494 2495 /* 2496 * Allocate memory for the pv head table for superpages. 2497 */ 2498 s = (vm_size_t)pv_npg * sizeof(struct md_page); 2499 s = round_page(s); 2500 pv_table = kmem_malloc(s, M_WAITOK | M_ZERO); 2501 for (i = 0; i < pv_npg; i++) 2502 TAILQ_INIT(&pv_table[i].pv_list); 2503 TAILQ_INIT(&pv_dummy.pv_list); 2504 } 2505 #endif 2506 2507 /* 2508 * Initialize the pmap module. 2509 * 2510 * Called by vm_mem_init(), to initialize any structures that the pmap 2511 * system needs to map virtual memory. 2512 */ 2513 void 2514 pmap_init(void) 2515 { 2516 struct pmap_preinit_mapping *ppim; 2517 vm_page_t m, mpte; 2518 pml4_entry_t *pml4e; 2519 unsigned long lm_max; 2520 int error, i, ret, skz63; 2521 2522 /* L1TF, reserve page @0 unconditionally */ 2523 vm_page_blacklist_add(0, bootverbose); 2524 2525 /* Detect bare-metal Skylake Server and Skylake-X. */ 2526 if (vm_guest == VM_GUEST_NO && cpu_vendor_id == CPU_VENDOR_INTEL && 2527 CPUID_TO_FAMILY(cpu_id) == 0x6 && CPUID_TO_MODEL(cpu_id) == 0x55) { 2528 /* 2529 * Skylake-X errata SKZ63. Processor May Hang When 2530 * Executing Code In an HLE Transaction Region between 2531 * 40000000H and 403FFFFFH. 2532 * 2533 * Mark the pages in the range as preallocated. It 2534 * seems to be impossible to distinguish between 2535 * Skylake Server and Skylake X. 2536 */ 2537 skz63 = 1; 2538 TUNABLE_INT_FETCH("hw.skz63_enable", &skz63); 2539 if (skz63 != 0) { 2540 if (bootverbose) 2541 printf("SKZ63: skipping 4M RAM starting " 2542 "at physical 1G\n"); 2543 for (i = 0; i < atop(0x400000); i++) { 2544 ret = vm_page_blacklist_add(0x40000000 + 2545 ptoa(i), false); 2546 if (!ret && bootverbose) 2547 printf("page at %#x already used\n", 2548 0x40000000 + ptoa(i)); 2549 } 2550 } 2551 } 2552 2553 /* IFU */ 2554 pmap_allow_2m_x_ept_recalculate(); 2555 2556 /* 2557 * Initialize the vm page array entries for the kernel pmap's 2558 * page table pages. 2559 */ 2560 PMAP_LOCK(kernel_pmap); 2561 for (i = 0; i < nkpt; i++) { 2562 mpte = PHYS_TO_VM_PAGE(KPTphys + (i << PAGE_SHIFT)); 2563 KASSERT(mpte >= vm_page_array && 2564 mpte < &vm_page_array[vm_page_array_size], 2565 ("pmap_init: page table page is out of range")); 2566 mpte->pindex = pmap_pde_pindex(KERNBASE) + i; 2567 mpte->phys_addr = KPTphys + (i << PAGE_SHIFT); 2568 mpte->ref_count = 1; 2569 2570 /* 2571 * Collect the page table pages that were replaced by a 2MB 2572 * page in create_pagetables(). They are zero filled. 2573 */ 2574 if ((i == 0 || 2575 kernphys + ((vm_paddr_t)(i - 1) << PDRSHIFT) < KERNend) && 2576 pmap_insert_pt_page(kernel_pmap, mpte, false, false)) 2577 panic("pmap_init: pmap_insert_pt_page failed"); 2578 } 2579 PMAP_UNLOCK(kernel_pmap); 2580 vm_wire_add(nkpt); 2581 2582 /* 2583 * If the kernel is running on a virtual machine, then it must assume 2584 * that MCA is enabled by the hypervisor. Moreover, the kernel must 2585 * be prepared for the hypervisor changing the vendor and family that 2586 * are reported by CPUID. Consequently, the workaround for AMD Family 2587 * 10h Erratum 383 is enabled if the processor's feature set does not 2588 * include at least one feature that is only supported by older Intel 2589 * or newer AMD processors. 2590 */ 2591 if (vm_guest != VM_GUEST_NO && (cpu_feature & CPUID_SS) == 0 && 2592 (cpu_feature2 & (CPUID2_SSSE3 | CPUID2_SSE41 | CPUID2_AESNI | 2593 CPUID2_AVX | CPUID2_XSAVE)) == 0 && (amd_feature2 & (AMDID2_XOP | 2594 AMDID2_FMA4)) == 0) 2595 workaround_erratum383 = 1; 2596 2597 /* 2598 * Are large page mappings enabled? 2599 */ 2600 TUNABLE_INT_FETCH("vm.pmap.pg_ps_enabled", &pg_ps_enabled); 2601 if (pg_ps_enabled) { 2602 KASSERT(MAXPAGESIZES > 1 && pagesizes[1] == 0, 2603 ("pmap_init: can't assign to pagesizes[1]")); 2604 pagesizes[1] = NBPDR; 2605 if ((amd_feature & AMDID_PAGE1GB) != 0) { 2606 KASSERT(MAXPAGESIZES > 2 && pagesizes[2] == 0, 2607 ("pmap_init: can't assign to pagesizes[2]")); 2608 pagesizes[2] = NBPDP; 2609 } 2610 } 2611 2612 /* 2613 * Initialize pv chunk lists. 2614 */ 2615 for (i = 0; i < PMAP_MEMDOM; i++) { 2616 mtx_init(&pv_chunks[i].pvc_lock, "pmap pv chunk list", NULL, MTX_DEF); 2617 TAILQ_INIT(&pv_chunks[i].pvc_list); 2618 } 2619 pmap_init_pv_table(); 2620 2621 pmap_initialized = 1; 2622 for (i = 0; i < PMAP_PREINIT_MAPPING_COUNT; i++) { 2623 ppim = pmap_preinit_mapping + i; 2624 if (ppim->va == NULL) 2625 continue; 2626 /* Make the direct map consistent */ 2627 if (ppim->pa < dmaplimit && ppim->pa + ppim->sz <= dmaplimit) { 2628 (void)pmap_change_attr(PHYS_TO_DMAP(ppim->pa), 2629 ppim->sz, ppim->mode); 2630 } 2631 if (!bootverbose) 2632 continue; 2633 printf("PPIM %u: PA=%#lx, VA=%p, size=%#lx, mode=%#x\n", i, 2634 ppim->pa, ppim->va, ppim->sz, ppim->mode); 2635 } 2636 2637 mtx_init(&qframe_mtx, "qfrmlk", NULL, MTX_SPIN); 2638 error = vmem_alloc(kernel_arena, PAGE_SIZE, M_BESTFIT | M_WAITOK, 2639 (vmem_addr_t *)&qframe); 2640 if (error != 0) 2641 panic("qframe allocation failed"); 2642 2643 lm_ents = 8; 2644 TUNABLE_INT_FETCH("vm.pmap.large_map_pml4_entries", &lm_ents); 2645 lm_max = (kva_layout.lm_high - kva_layout.lm_low) / NBPML4; 2646 if (lm_ents > lm_max) { 2647 printf( 2648 "pmap: shrinking large map from requested %d slots to %ld slots\n", 2649 lm_ents, lm_max); 2650 lm_ents = lm_max; 2651 } 2652 #ifdef KMSAN 2653 if (!la57 && lm_ents > KMSANORIGPML4I - LMSPML4I) { 2654 printf( 2655 "pmap: shrinking large map for KMSAN (%d slots to %ld slots)\n", 2656 lm_ents, KMSANORIGPML4I - LMSPML4I); 2657 lm_ents = KMSANORIGPML4I - LMSPML4I; 2658 } 2659 #endif 2660 if (bootverbose) 2661 printf("pmap: large map %u PML4 slots (%lu GB)\n", 2662 lm_ents, (u_long)lm_ents * (NBPML4 / 1024 / 1024 / 1024)); 2663 if (lm_ents != 0) { 2664 large_vmem = vmem_create("large", kva_layout.lm_low, 2665 (vmem_size_t)lm_ents * NBPML4, PAGE_SIZE, 0, M_WAITOK); 2666 if (large_vmem == NULL) { 2667 printf("pmap: cannot create large map\n"); 2668 lm_ents = 0; 2669 } 2670 if (la57) { 2671 for (i = 0; i < howmany((vm_offset_t)NBPML4 * 2672 lm_ents, NBPML5); i++) { 2673 m = pmap_large_map_getptp_unlocked(); 2674 kernel_pmap->pm_pmltop[LMSPML5I + i] = X86_PG_V | 2675 X86_PG_RW | X86_PG_A | X86_PG_M | 2676 pg_nx | VM_PAGE_TO_PHYS(m); 2677 } 2678 } 2679 for (i = 0; i < lm_ents; i++) { 2680 m = pmap_large_map_getptp_unlocked(); 2681 pml4e = pmap_pml4e(kernel_pmap, kva_layout.lm_low + 2682 (u_long)i * NBPML4); 2683 *pml4e = X86_PG_V | X86_PG_RW | X86_PG_A | X86_PG_M | 2684 pg_nx | VM_PAGE_TO_PHYS(m); 2685 } 2686 } 2687 } 2688 2689 SYSCTL_UINT(_vm_pmap, OID_AUTO, large_map_pml4_entries, 2690 CTLFLAG_RDTUN | CTLFLAG_NOFETCH, &lm_ents, 0, 2691 "Maximum number of PML4 entries for use by large map (tunable). " 2692 "Each entry corresponds to 512GB of address space."); 2693 2694 static SYSCTL_NODE(_vm_pmap, OID_AUTO, pde, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 2695 "2MB page mapping counters"); 2696 2697 static COUNTER_U64_DEFINE_EARLY(pmap_pde_demotions); 2698 SYSCTL_COUNTER_U64(_vm_pmap_pde, OID_AUTO, demotions, 2699 CTLFLAG_RD, &pmap_pde_demotions, "2MB page demotions"); 2700 2701 static COUNTER_U64_DEFINE_EARLY(pmap_pde_mappings); 2702 SYSCTL_COUNTER_U64(_vm_pmap_pde, OID_AUTO, mappings, CTLFLAG_RD, 2703 &pmap_pde_mappings, "2MB page mappings"); 2704 2705 static COUNTER_U64_DEFINE_EARLY(pmap_pde_p_failures); 2706 SYSCTL_COUNTER_U64(_vm_pmap_pde, OID_AUTO, p_failures, CTLFLAG_RD, 2707 &pmap_pde_p_failures, "2MB page promotion failures"); 2708 2709 static COUNTER_U64_DEFINE_EARLY(pmap_pde_promotions); 2710 SYSCTL_COUNTER_U64(_vm_pmap_pde, OID_AUTO, promotions, CTLFLAG_RD, 2711 &pmap_pde_promotions, "2MB page promotions"); 2712 2713 static SYSCTL_NODE(_vm_pmap, OID_AUTO, pdpe, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 2714 "1GB page mapping counters"); 2715 2716 static COUNTER_U64_DEFINE_EARLY(pmap_pdpe_demotions); 2717 SYSCTL_COUNTER_U64(_vm_pmap_pdpe, OID_AUTO, demotions, CTLFLAG_RD, 2718 &pmap_pdpe_demotions, "1GB page demotions"); 2719 2720 /*************************************************** 2721 * Low level helper routines..... 2722 ***************************************************/ 2723 2724 static pt_entry_t 2725 pmap_swap_pat(pmap_t pmap, pt_entry_t entry) 2726 { 2727 int x86_pat_bits = X86_PG_PTE_PAT | X86_PG_PDE_PAT; 2728 2729 switch (pmap->pm_type) { 2730 case PT_X86: 2731 case PT_RVI: 2732 /* Verify that both PAT bits are not set at the same time */ 2733 KASSERT((entry & x86_pat_bits) != x86_pat_bits, 2734 ("Invalid PAT bits in entry %#lx", entry)); 2735 2736 /* Swap the PAT bits if one of them is set */ 2737 if ((entry & x86_pat_bits) != 0) 2738 entry ^= x86_pat_bits; 2739 break; 2740 case PT_EPT: 2741 /* 2742 * Nothing to do - the memory attributes are represented 2743 * the same way for regular pages and superpages. 2744 */ 2745 break; 2746 default: 2747 panic("pmap_switch_pat_bits: bad pm_type %d", pmap->pm_type); 2748 } 2749 2750 return (entry); 2751 } 2752 2753 bool 2754 pmap_is_valid_memattr(pmap_t pmap __unused, vm_memattr_t mode) 2755 { 2756 2757 return (mode >= 0 && mode < PAT_INDEX_SIZE && 2758 pat_index[(int)mode] >= 0); 2759 } 2760 2761 /* 2762 * Determine the appropriate bits to set in a PTE or PDE for a specified 2763 * caching mode. 2764 */ 2765 int 2766 pmap_cache_bits(pmap_t pmap, int mode, bool is_pde) 2767 { 2768 int cache_bits, pat_flag, pat_idx; 2769 2770 if (!pmap_is_valid_memattr(pmap, mode)) 2771 panic("Unknown caching mode %d\n", mode); 2772 2773 switch (pmap->pm_type) { 2774 case PT_X86: 2775 case PT_RVI: 2776 /* The PAT bit is different for PTE's and PDE's. */ 2777 pat_flag = is_pde ? X86_PG_PDE_PAT : X86_PG_PTE_PAT; 2778 2779 /* Map the caching mode to a PAT index. */ 2780 pat_idx = pat_index[mode]; 2781 2782 /* Map the 3-bit index value into the PAT, PCD, and PWT bits. */ 2783 cache_bits = 0; 2784 if (pat_idx & 0x4) 2785 cache_bits |= pat_flag; 2786 if (pat_idx & 0x2) 2787 cache_bits |= PG_NC_PCD; 2788 if (pat_idx & 0x1) 2789 cache_bits |= PG_NC_PWT; 2790 break; 2791 2792 case PT_EPT: 2793 cache_bits = EPT_PG_IGNORE_PAT | EPT_PG_MEMORY_TYPE(mode); 2794 break; 2795 2796 default: 2797 panic("unsupported pmap type %d", pmap->pm_type); 2798 } 2799 2800 return (cache_bits); 2801 } 2802 2803 static int 2804 pmap_cache_mask(pmap_t pmap, bool is_pde) 2805 { 2806 int mask; 2807 2808 switch (pmap->pm_type) { 2809 case PT_X86: 2810 case PT_RVI: 2811 mask = is_pde ? X86_PG_PDE_CACHE : X86_PG_PTE_CACHE; 2812 break; 2813 case PT_EPT: 2814 mask = EPT_PG_IGNORE_PAT | EPT_PG_MEMORY_TYPE(0x7); 2815 break; 2816 default: 2817 panic("pmap_cache_mask: invalid pm_type %d", pmap->pm_type); 2818 } 2819 2820 return (mask); 2821 } 2822 2823 static int 2824 pmap_pat_index(pmap_t pmap, pt_entry_t pte, bool is_pde) 2825 { 2826 int pat_flag, pat_idx; 2827 2828 pat_idx = 0; 2829 switch (pmap->pm_type) { 2830 case PT_X86: 2831 case PT_RVI: 2832 /* The PAT bit is different for PTE's and PDE's. */ 2833 pat_flag = is_pde ? X86_PG_PDE_PAT : X86_PG_PTE_PAT; 2834 2835 if ((pte & pat_flag) != 0) 2836 pat_idx |= 0x4; 2837 if ((pte & PG_NC_PCD) != 0) 2838 pat_idx |= 0x2; 2839 if ((pte & PG_NC_PWT) != 0) 2840 pat_idx |= 0x1; 2841 break; 2842 case PT_EPT: 2843 if ((pte & EPT_PG_IGNORE_PAT) != 0) 2844 panic("EPT PTE %#lx has no PAT memory type", pte); 2845 pat_idx = (pte & EPT_PG_MEMORY_TYPE(0x7)) >> 3; 2846 break; 2847 } 2848 2849 /* See pmap_init_pat(). */ 2850 if (pat_idx == 4) 2851 pat_idx = 0; 2852 if (pat_idx == 7) 2853 pat_idx = 3; 2854 2855 return (pat_idx); 2856 } 2857 2858 bool 2859 pmap_ps_enabled(pmap_t pmap) 2860 { 2861 2862 return (pg_ps_enabled && (pmap->pm_flags & PMAP_PDE_SUPERPAGE) != 0); 2863 } 2864 2865 static void 2866 pmap_update_pde_store(pmap_t pmap, pd_entry_t *pde, pd_entry_t newpde) 2867 { 2868 2869 switch (pmap->pm_type) { 2870 case PT_X86: 2871 break; 2872 case PT_RVI: 2873 case PT_EPT: 2874 /* 2875 * XXX 2876 * This is a little bogus since the generation number is 2877 * supposed to be bumped up when a region of the address 2878 * space is invalidated in the page tables. 2879 * 2880 * In this case the old PDE entry is valid but yet we want 2881 * to make sure that any mappings using the old entry are 2882 * invalidated in the TLB. 2883 * 2884 * The reason this works as expected is because we rendezvous 2885 * "all" host cpus and force any vcpu context to exit as a 2886 * side-effect. 2887 */ 2888 atomic_add_long(&pmap->pm_eptgen, 1); 2889 break; 2890 default: 2891 panic("pmap_update_pde_store: bad pm_type %d", pmap->pm_type); 2892 } 2893 pde_store(pde, newpde); 2894 } 2895 2896 /* 2897 * After changing the page size for the specified virtual address in the page 2898 * table, flush the corresponding entries from the processor's TLB. Only the 2899 * calling processor's TLB is affected. 2900 * 2901 * The calling thread must be pinned to a processor. 2902 */ 2903 static void 2904 pmap_update_pde_invalidate(pmap_t pmap, vm_offset_t va, pd_entry_t newpde) 2905 { 2906 pt_entry_t PG_G; 2907 2908 if (pmap_type_guest(pmap)) 2909 return; 2910 2911 KASSERT(pmap->pm_type == PT_X86, 2912 ("pmap_update_pde_invalidate: invalid type %d", pmap->pm_type)); 2913 2914 PG_G = pmap_global_bit(pmap); 2915 2916 if ((newpde & PG_PS) == 0) 2917 /* Demotion: flush a specific 2MB page mapping. */ 2918 pmap_invlpg(pmap, va); 2919 else if ((newpde & PG_G) == 0) 2920 /* 2921 * Promotion: flush every 4KB page mapping from the TLB 2922 * because there are too many to flush individually. 2923 */ 2924 invltlb(); 2925 else { 2926 /* 2927 * Promotion: flush every 4KB page mapping from the TLB, 2928 * including any global (PG_G) mappings. 2929 * 2930 * This function is only used on older processors that 2931 * do not support the invpcid instruction. 2932 */ 2933 invltlb_glob(); 2934 } 2935 } 2936 2937 /* 2938 * The amd64 pmap uses different approaches to TLB invalidation 2939 * depending on the kernel configuration, available hardware features, 2940 * and known hardware errata. The kernel configuration option that 2941 * has the greatest operational impact on TLB invalidation is PTI, 2942 * which is enabled automatically on affected Intel CPUs. The most 2943 * impactful hardware features are first PCID, and then INVPCID 2944 * instruction presence. PCID usage is quite different for PTI 2945 * vs. non-PTI. 2946 * 2947 * * Kernel Page Table Isolation (PTI or KPTI) is used to mitigate 2948 * the Meltdown bug in some Intel CPUs. Under PTI, each user address 2949 * space is served by two page tables, user and kernel. The user 2950 * page table only maps user space and a kernel trampoline. The 2951 * kernel trampoline includes the entirety of the kernel text but 2952 * only the kernel data that is needed to switch from user to kernel 2953 * mode. The kernel page table maps the user and kernel address 2954 * spaces in their entirety. It is identical to the per-process 2955 * page table used in non-PTI mode. 2956 * 2957 * User page tables are only used when the CPU is in user mode. 2958 * Consequently, some TLB invalidations can be postponed until the 2959 * switch from kernel to user mode. In contrast, the user 2960 * space part of the kernel page table is used for copyout(9), so 2961 * TLB invalidations on this page table cannot be similarly postponed. 2962 * 2963 * The existence of a user mode page table for the given pmap is 2964 * indicated by a pm_ucr3 value that differs from PMAP_NO_CR3, in 2965 * which case pm_ucr3 contains the %cr3 register value for the user 2966 * mode page table's root. 2967 * 2968 * * The pm_active bitmask indicates which CPUs currently have the 2969 * pmap active. A CPU's bit is set on context switch to the pmap, and 2970 * cleared on switching off this CPU. For the kernel page table, 2971 * the pm_active field is immutable and contains all CPUs. The 2972 * kernel page table is always logically active on every processor, 2973 * but not necessarily in use by the hardware, e.g., in PTI mode. 2974 * 2975 * When requesting invalidation of virtual addresses with 2976 * pmap_invalidate_XXX() functions, the pmap sends shootdown IPIs to 2977 * all CPUs recorded as active in pm_active. Updates to and reads 2978 * from pm_active are not synchronized, and so they may race with 2979 * each other. Shootdown handlers are prepared to handle the race. 2980 * 2981 * * PCID is an optional feature of the long mode x86 MMU where TLB 2982 * entries are tagged with the 'Process ID' of the address space 2983 * they belong to. This feature provides a limited namespace for 2984 * process identifiers, 12 bits, supporting 4095 simultaneous IDs 2985 * total. 2986 * 2987 * Allocation of a PCID to a pmap is done by an algorithm described 2988 * in section 15.12, "Other TLB Consistency Algorithms", of 2989 * Vahalia's book "Unix Internals". A PCID cannot be allocated for 2990 * the whole lifetime of a pmap in pmap_pinit() due to the limited 2991 * namespace. Instead, a per-CPU, per-pmap PCID is assigned when 2992 * the CPU is about to start caching TLB entries from a pmap, 2993 * i.e., on the context switch that activates the pmap on the CPU. 2994 * 2995 * The PCID allocator maintains a per-CPU, per-pmap generation 2996 * count, pm_gen, which is incremented each time a new PCID is 2997 * allocated. On TLB invalidation, the generation counters for the 2998 * pmap are zeroed, which signals the context switch code that the 2999 * previously allocated PCID is no longer valid. Effectively, 3000 * zeroing any of these counters triggers a TLB shootdown for the 3001 * given CPU/address space, due to the allocation of a new PCID. 3002 * 3003 * Zeroing can be performed remotely. Consequently, if a pmap is 3004 * inactive on a CPU, then a TLB shootdown for that pmap and CPU can 3005 * be initiated by an ordinary memory access to reset the target 3006 * CPU's generation count within the pmap. The CPU initiating the 3007 * TLB shootdown does not need to send an IPI to the target CPU. 3008 * 3009 * * PTI + PCID. The available PCIDs are divided into two sets: PCIDs 3010 * for complete (kernel) page tables, and PCIDs for user mode page 3011 * tables. A user PCID value is obtained from the kernel PCID value 3012 * by setting the highest bit, 11, to 1 (0x800 == PMAP_PCID_USER_PT). 3013 * 3014 * User space page tables are activated on return to user mode, by 3015 * loading pm_ucr3 into %cr3. If the PCPU(ucr3_load_mask) requests 3016 * clearing bit 63 of the loaded ucr3, this effectively causes 3017 * complete invalidation of the user mode TLB entries for the 3018 * current pmap. In which case, local invalidations of individual 3019 * pages in the user page table are skipped. 3020 * 3021 * * Local invalidation, all modes. If the requested invalidation is 3022 * for a specific address or the total invalidation of a currently 3023 * active pmap, then the TLB is flushed using INVLPG for a kernel 3024 * page table, and INVPCID(INVPCID_CTXGLOB)/invltlb_glob() for a 3025 * user space page table(s). 3026 * 3027 * If the INVPCID instruction is available, it is used to flush user 3028 * entries from the kernel page table. 3029 * 3030 * When PCID is enabled, the INVLPG instruction invalidates all TLB 3031 * entries for the given page that either match the current PCID or 3032 * are global. Since TLB entries for the same page under different 3033 * PCIDs are unaffected, kernel pages which reside in all address 3034 * spaces could be problematic. We avoid the problem by creating 3035 * all kernel PTEs with the global flag (PG_G) set, when PTI is 3036 * disabled. 3037 * 3038 * * mode: PTI disabled, PCID present. The kernel reserves PCID 0 for its 3039 * address space, all other 4095 PCIDs are used for user mode spaces 3040 * as described above. A context switch allocates a new PCID if 3041 * the recorded PCID is zero or the recorded generation does not match 3042 * the CPU's generation, effectively flushing the TLB for this address space. 3043 * Total remote invalidation is performed by zeroing pm_gen for all CPUs. 3044 * local user page: INVLPG 3045 * local kernel page: INVLPG 3046 * local user total: INVPCID(CTX) 3047 * local kernel total: INVPCID(CTXGLOB) or invltlb_glob() 3048 * remote user page, inactive pmap: zero pm_gen 3049 * remote user page, active pmap: zero pm_gen + IPI:INVLPG 3050 * (Both actions are required to handle the aforementioned pm_active races.) 3051 * remote kernel page: IPI:INVLPG 3052 * remote user total, inactive pmap: zero pm_gen 3053 * remote user total, active pmap: zero pm_gen + IPI:(INVPCID(CTX) or 3054 * reload %cr3) 3055 * (See note above about pm_active races.) 3056 * remote kernel total: IPI:(INVPCID(CTXGLOB) or invltlb_glob()) 3057 * 3058 * PTI enabled, PCID present. 3059 * local user page: INVLPG for kpt, INVPCID(ADDR) or (INVLPG for ucr3) 3060 * for upt 3061 * local kernel page: INVLPG 3062 * local user total: INVPCID(CTX) or reload %cr3 for kpt, clear PCID_SAVE 3063 * on loading UCR3 into %cr3 for upt 3064 * local kernel total: INVPCID(CTXGLOB) or invltlb_glob() 3065 * remote user page, inactive pmap: zero pm_gen 3066 * remote user page, active pmap: zero pm_gen + IPI:(INVLPG for kpt, 3067 * INVPCID(ADDR) for upt) 3068 * remote kernel page: IPI:INVLPG 3069 * remote user total, inactive pmap: zero pm_gen 3070 * remote user total, active pmap: zero pm_gen + IPI:(INVPCID(CTX) for kpt, 3071 * clear PCID_SAVE on loading UCR3 into $cr3 for upt) 3072 * remote kernel total: IPI:(INVPCID(CTXGLOB) or invltlb_glob()) 3073 * 3074 * No PCID. 3075 * local user page: INVLPG 3076 * local kernel page: INVLPG 3077 * local user total: reload %cr3 3078 * local kernel total: INVPCID(CTXGLOB) or invltlb_glob() 3079 * remote user page, inactive pmap: - 3080 * remote user page, active pmap: IPI:INVLPG 3081 * remote kernel page: IPI:INVLPG 3082 * remote user total, inactive pmap: - 3083 * remote user total, active pmap: IPI:(reload %cr3) 3084 * remote kernel total: IPI:INVPCID(CTXGLOB) or invltlb_glob() 3085 * Since on return to user mode, the reload of %cr3 with ucr3 causes 3086 * TLB invalidation, no specific action is required for user page table. 3087 * 3088 * EPT. EPT pmaps do not map KVA, all mappings are userspace. 3089 * XXX TODO 3090 */ 3091 3092 /* 3093 * Interrupt the cpus that are executing in the guest context. 3094 * This will force the vcpu to exit and the cached EPT mappings 3095 * will be invalidated by the host before the next vmresume. 3096 */ 3097 static __inline void 3098 pmap_invalidate_ept(pmap_t pmap) 3099 { 3100 smr_seq_t goal; 3101 int ipinum; 3102 3103 sched_pin(); 3104 KASSERT(!CPU_ISSET(curcpu, &pmap->pm_active), 3105 ("pmap_invalidate_ept: absurd pm_active")); 3106 3107 /* 3108 * The TLB mappings associated with a vcpu context are not 3109 * flushed each time a different vcpu is chosen to execute. 3110 * 3111 * This is in contrast with a process's vtop mappings that 3112 * are flushed from the TLB on each context switch. 3113 * 3114 * Therefore we need to do more than just a TLB shootdown on 3115 * the active cpus in 'pmap->pm_active'. To do this we keep 3116 * track of the number of invalidations performed on this pmap. 3117 * 3118 * Each vcpu keeps a cache of this counter and compares it 3119 * just before a vmresume. If the counter is out-of-date an 3120 * invept will be done to flush stale mappings from the TLB. 3121 * 3122 * To ensure that all vCPU threads have observed the new counter 3123 * value before returning, we use SMR. Ordering is important here: 3124 * the VMM enters an SMR read section before loading the counter 3125 * and after updating the pm_active bit set. Thus, pm_active is 3126 * a superset of active readers, and any reader that has observed 3127 * the goal has observed the new counter value. 3128 */ 3129 atomic_add_long(&pmap->pm_eptgen, 1); 3130 3131 goal = smr_advance(pmap->pm_eptsmr); 3132 3133 /* 3134 * Force the vcpu to exit and trap back into the hypervisor. 3135 */ 3136 ipinum = pmap->pm_flags & PMAP_NESTED_IPIMASK; 3137 ipi_selected(pmap->pm_active, ipinum); 3138 sched_unpin(); 3139 3140 /* 3141 * Ensure that all active vCPUs will observe the new generation counter 3142 * value before executing any more guest instructions. 3143 */ 3144 smr_wait(pmap->pm_eptsmr, goal); 3145 } 3146 3147 static inline void 3148 pmap_invalidate_preipi_pcid(pmap_t pmap) 3149 { 3150 struct pmap_pcid *pcidp; 3151 u_int cpuid, i; 3152 3153 sched_pin(); 3154 3155 cpuid = PCPU_GET(cpuid); 3156 if (pmap != PCPU_GET(curpmap)) 3157 cpuid = 0xffffffff; /* An impossible value */ 3158 3159 CPU_FOREACH(i) { 3160 if (cpuid != i) { 3161 pcidp = zpcpu_get_cpu(pmap->pm_pcidp, i); 3162 pcidp->pm_gen = 0; 3163 } 3164 } 3165 3166 /* 3167 * The fence is between stores to pm_gen and the read of the 3168 * pm_active mask. We need to ensure that it is impossible 3169 * for us to miss the bit update in pm_active and 3170 * simultaneously observe a non-zero pm_gen in 3171 * pmap_activate_sw(), otherwise TLB update is missed. 3172 * Without the fence, IA32 allows such an outcome. Note that 3173 * pm_active is updated by a locked operation, which provides 3174 * the reciprocal fence. 3175 */ 3176 atomic_thread_fence_seq_cst(); 3177 } 3178 3179 static void 3180 pmap_invalidate_preipi_nopcid(pmap_t pmap __unused) 3181 { 3182 sched_pin(); 3183 } 3184 3185 DEFINE_IFUNC(static, void, pmap_invalidate_preipi, (pmap_t)) 3186 { 3187 return (pmap_pcid_enabled ? pmap_invalidate_preipi_pcid : 3188 pmap_invalidate_preipi_nopcid); 3189 } 3190 3191 static inline void 3192 pmap_invalidate_page_pcid_cb(pmap_t pmap, vm_offset_t va, 3193 const bool invpcid_works1) 3194 { 3195 struct invpcid_descr d; 3196 uint64_t kcr3, ucr3; 3197 uint32_t pcid; 3198 3199 /* 3200 * Because pm_pcid is recalculated on a context switch, we 3201 * must ensure there is no preemption, not just pinning. 3202 * Otherwise, we might use a stale value below. 3203 */ 3204 CRITICAL_ASSERT(curthread); 3205 3206 /* 3207 * No need to do anything with user page tables invalidation 3208 * if there is no user page table, or invalidation is deferred 3209 * until the return to userspace. ucr3_load_mask is stable 3210 * because we have preemption disabled. 3211 */ 3212 if (pmap->pm_ucr3 == PMAP_NO_CR3 || 3213 PCPU_GET(ucr3_load_mask) != PMAP_UCR3_NOMASK) 3214 return; 3215 3216 pcid = pmap_get_pcid(pmap); 3217 if (invpcid_works1) { 3218 d.pcid = pcid | PMAP_PCID_USER_PT; 3219 d.pad = 0; 3220 d.addr = va; 3221 invpcid(&d, INVPCID_ADDR); 3222 } else { 3223 kcr3 = pmap->pm_cr3 | pcid | CR3_PCID_SAVE; 3224 ucr3 = pmap->pm_ucr3 | pcid | PMAP_PCID_USER_PT | CR3_PCID_SAVE; 3225 pmap_pti_pcid_invlpg(ucr3, kcr3, va); 3226 } 3227 } 3228 3229 static void 3230 pmap_invalidate_page_pcid_invpcid_cb(pmap_t pmap, vm_offset_t va) 3231 { 3232 pmap_invalidate_page_pcid_cb(pmap, va, true); 3233 } 3234 3235 static void 3236 pmap_invalidate_page_pcid_noinvpcid_cb(pmap_t pmap, vm_offset_t va) 3237 { 3238 pmap_invalidate_page_pcid_cb(pmap, va, false); 3239 } 3240 3241 static void 3242 pmap_invalidate_page_nopcid_cb(pmap_t pmap __unused, vm_offset_t va __unused) 3243 { 3244 } 3245 3246 DEFINE_IFUNC(static, void, pmap_invalidate_page_cb, (pmap_t, vm_offset_t)) 3247 { 3248 if (pmap_pcid_enabled) 3249 return (invpcid_works ? pmap_invalidate_page_pcid_invpcid_cb : 3250 pmap_invalidate_page_pcid_noinvpcid_cb); 3251 return (pmap_invalidate_page_nopcid_cb); 3252 } 3253 3254 static void 3255 pmap_invalidate_page_curcpu_cb(pmap_t pmap, vm_offset_t va, 3256 vm_offset_t addr2 __unused) 3257 { 3258 if (pmap == kernel_pmap) { 3259 pmap_invlpg(kernel_pmap, va); 3260 } else if (pmap == PCPU_GET(curpmap)) { 3261 invlpg(va); 3262 pmap_invalidate_page_cb(pmap, va); 3263 } 3264 } 3265 3266 void 3267 pmap_invalidate_page(pmap_t pmap, vm_offset_t va) 3268 { 3269 if (pmap_type_guest(pmap)) { 3270 pmap_invalidate_ept(pmap); 3271 return; 3272 } 3273 3274 KASSERT(pmap->pm_type == PT_X86, 3275 ("pmap_invalidate_page: invalid type %d", pmap->pm_type)); 3276 3277 pmap_invalidate_preipi(pmap); 3278 smp_masked_invlpg(va, pmap, pmap_invalidate_page_curcpu_cb); 3279 } 3280 3281 /* 4k PTEs -- Chosen to exceed the total size of Broadwell L2 TLB */ 3282 #define PMAP_INVLPG_THRESHOLD (4 * 1024 * PAGE_SIZE) 3283 3284 static void 3285 pmap_invalidate_range_pcid_cb(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, 3286 const bool invpcid_works1) 3287 { 3288 struct invpcid_descr d; 3289 uint64_t kcr3, ucr3; 3290 uint32_t pcid; 3291 3292 CRITICAL_ASSERT(curthread); 3293 3294 if (pmap != PCPU_GET(curpmap) || 3295 pmap->pm_ucr3 == PMAP_NO_CR3 || 3296 PCPU_GET(ucr3_load_mask) != PMAP_UCR3_NOMASK) 3297 return; 3298 3299 pcid = pmap_get_pcid(pmap); 3300 if (invpcid_works1) { 3301 d.pcid = pcid | PMAP_PCID_USER_PT; 3302 d.pad = 0; 3303 for (d.addr = sva; d.addr < eva; d.addr += PAGE_SIZE) 3304 invpcid(&d, INVPCID_ADDR); 3305 } else { 3306 kcr3 = pmap->pm_cr3 | pcid | CR3_PCID_SAVE; 3307 ucr3 = pmap->pm_ucr3 | pcid | PMAP_PCID_USER_PT | CR3_PCID_SAVE; 3308 pmap_pti_pcid_invlrng(ucr3, kcr3, sva, eva); 3309 } 3310 } 3311 3312 static void 3313 pmap_invalidate_range_pcid_invpcid_cb(pmap_t pmap, vm_offset_t sva, 3314 vm_offset_t eva) 3315 { 3316 pmap_invalidate_range_pcid_cb(pmap, sva, eva, true); 3317 } 3318 3319 static void 3320 pmap_invalidate_range_pcid_noinvpcid_cb(pmap_t pmap, vm_offset_t sva, 3321 vm_offset_t eva) 3322 { 3323 pmap_invalidate_range_pcid_cb(pmap, sva, eva, false); 3324 } 3325 3326 static void 3327 pmap_invalidate_range_nopcid_cb(pmap_t pmap __unused, vm_offset_t sva __unused, 3328 vm_offset_t eva __unused) 3329 { 3330 } 3331 3332 DEFINE_IFUNC(static, void, pmap_invalidate_range_cb, (pmap_t, vm_offset_t, 3333 vm_offset_t)) 3334 { 3335 if (pmap_pcid_enabled) 3336 return (invpcid_works ? pmap_invalidate_range_pcid_invpcid_cb : 3337 pmap_invalidate_range_pcid_noinvpcid_cb); 3338 return (pmap_invalidate_range_nopcid_cb); 3339 } 3340 3341 static void 3342 pmap_invalidate_range_curcpu_cb(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) 3343 { 3344 vm_offset_t addr; 3345 3346 if (pmap == kernel_pmap) { 3347 if (PCPU_GET(pcid_invlpg_workaround)) { 3348 struct invpcid_descr d = { 0 }; 3349 3350 invpcid(&d, INVPCID_CTXGLOB); 3351 } else { 3352 for (addr = sva; addr < eva; addr += PAGE_SIZE) 3353 invlpg(addr); 3354 } 3355 } else if (pmap == PCPU_GET(curpmap)) { 3356 for (addr = sva; addr < eva; addr += PAGE_SIZE) 3357 invlpg(addr); 3358 pmap_invalidate_range_cb(pmap, sva, eva); 3359 } 3360 } 3361 3362 void 3363 pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) 3364 { 3365 if (eva - sva >= PMAP_INVLPG_THRESHOLD) { 3366 pmap_invalidate_all(pmap); 3367 return; 3368 } 3369 3370 if (pmap_type_guest(pmap)) { 3371 pmap_invalidate_ept(pmap); 3372 return; 3373 } 3374 3375 KASSERT(pmap->pm_type == PT_X86, 3376 ("pmap_invalidate_range: invalid type %d", pmap->pm_type)); 3377 3378 pmap_invalidate_preipi(pmap); 3379 smp_masked_invlpg_range(sva, eva, pmap, 3380 pmap_invalidate_range_curcpu_cb); 3381 } 3382 3383 static inline void 3384 pmap_invalidate_all_cb_template(pmap_t pmap, bool pmap_pcid_enabled1, 3385 bool invpcid_works1) 3386 { 3387 struct invpcid_descr d; 3388 uint64_t kcr3; 3389 uint32_t pcid; 3390 3391 if (pmap == kernel_pmap) { 3392 if (invpcid_works1) { 3393 bzero(&d, sizeof(d)); 3394 invpcid(&d, INVPCID_CTXGLOB); 3395 } else { 3396 invltlb_glob(); 3397 } 3398 } else if (pmap == PCPU_GET(curpmap)) { 3399 if (pmap_pcid_enabled1) { 3400 CRITICAL_ASSERT(curthread); 3401 3402 pcid = pmap_get_pcid(pmap); 3403 if (invpcid_works1) { 3404 d.pcid = pcid; 3405 d.pad = 0; 3406 d.addr = 0; 3407 invpcid(&d, INVPCID_CTX); 3408 } else { 3409 kcr3 = pmap->pm_cr3 | pcid; 3410 load_cr3(kcr3); 3411 } 3412 if (pmap->pm_ucr3 != PMAP_NO_CR3) 3413 PCPU_SET(ucr3_load_mask, ~CR3_PCID_SAVE); 3414 } else { 3415 invltlb(); 3416 } 3417 } 3418 } 3419 3420 static void 3421 pmap_invalidate_all_pcid_invpcid_cb(pmap_t pmap, vm_offset_t addr1 __unused, 3422 vm_offset_t addr2 __unused) 3423 { 3424 pmap_invalidate_all_cb_template(pmap, true, true); 3425 } 3426 3427 static void 3428 pmap_invalidate_all_pcid_noinvpcid_cb(pmap_t pmap, vm_offset_t addr1 __unused, 3429 vm_offset_t addr2 __unused) 3430 { 3431 pmap_invalidate_all_cb_template(pmap, true, false); 3432 } 3433 3434 static void 3435 pmap_invalidate_all_nopcid_invpcid_cb(pmap_t pmap, vm_offset_t addr1 __unused, 3436 vm_offset_t addr2 __unused) 3437 { 3438 pmap_invalidate_all_cb_template(pmap, false, true); 3439 } 3440 3441 static void 3442 pmap_invalidate_all_nopcid_noinvpcid_cb(pmap_t pmap, vm_offset_t addr1 __unused, 3443 vm_offset_t addr2 __unused) 3444 { 3445 pmap_invalidate_all_cb_template(pmap, false, false); 3446 } 3447 3448 DEFINE_IFUNC(static, void, pmap_invalidate_all_curcpu_cb, (pmap_t, vm_offset_t, 3449 vm_offset_t)) 3450 { 3451 if (pmap_pcid_enabled) 3452 return (invpcid_works ? pmap_invalidate_all_pcid_invpcid_cb : 3453 pmap_invalidate_all_pcid_noinvpcid_cb); 3454 return (invpcid_works ? pmap_invalidate_all_nopcid_invpcid_cb : 3455 pmap_invalidate_all_nopcid_noinvpcid_cb); 3456 } 3457 3458 void 3459 pmap_invalidate_all(pmap_t pmap) 3460 { 3461 if (pmap_type_guest(pmap)) { 3462 pmap_invalidate_ept(pmap); 3463 return; 3464 } 3465 3466 KASSERT(pmap->pm_type == PT_X86, 3467 ("pmap_invalidate_all: invalid type %d", pmap->pm_type)); 3468 3469 pmap_invalidate_preipi(pmap); 3470 smp_masked_invltlb(pmap, pmap_invalidate_all_curcpu_cb); 3471 } 3472 3473 static void 3474 pmap_invalidate_cache_curcpu_cb(pmap_t pmap __unused, vm_offset_t va __unused, 3475 vm_offset_t addr2 __unused) 3476 { 3477 wbinvd(); 3478 } 3479 3480 void 3481 pmap_invalidate_cache(void) 3482 { 3483 sched_pin(); 3484 smp_cache_flush(pmap_invalidate_cache_curcpu_cb); 3485 } 3486 3487 struct pde_action { 3488 cpuset_t invalidate; /* processors that invalidate their TLB */ 3489 pmap_t pmap; 3490 vm_offset_t va; 3491 pd_entry_t *pde; 3492 pd_entry_t newpde; 3493 u_int store; /* processor that updates the PDE */ 3494 }; 3495 3496 static void 3497 pmap_update_pde_action(void *arg) 3498 { 3499 struct pde_action *act = arg; 3500 3501 if (act->store == PCPU_GET(cpuid)) 3502 pmap_update_pde_store(act->pmap, act->pde, act->newpde); 3503 } 3504 3505 static void 3506 pmap_update_pde_teardown(void *arg) 3507 { 3508 struct pde_action *act = arg; 3509 3510 if (CPU_ISSET(PCPU_GET(cpuid), &act->invalidate)) 3511 pmap_update_pde_invalidate(act->pmap, act->va, act->newpde); 3512 } 3513 3514 /* 3515 * Change the page size for the specified virtual address in a way that 3516 * prevents any possibility of the TLB ever having two entries that map the 3517 * same virtual address using different page sizes. This is the recommended 3518 * workaround for Erratum 383 on AMD Family 10h processors. It prevents a 3519 * machine check exception for a TLB state that is improperly diagnosed as a 3520 * hardware error. 3521 */ 3522 static void 3523 pmap_update_pde(pmap_t pmap, vm_offset_t va, pd_entry_t *pde, pd_entry_t newpde) 3524 { 3525 struct pde_action act; 3526 cpuset_t active, other_cpus; 3527 u_int cpuid; 3528 3529 sched_pin(); 3530 cpuid = PCPU_GET(cpuid); 3531 other_cpus = all_cpus; 3532 CPU_CLR(cpuid, &other_cpus); 3533 if (pmap == kernel_pmap || pmap_type_guest(pmap)) 3534 active = all_cpus; 3535 else { 3536 active = pmap->pm_active; 3537 } 3538 if (CPU_OVERLAP(&active, &other_cpus)) { 3539 act.store = cpuid; 3540 act.invalidate = active; 3541 act.va = va; 3542 act.pmap = pmap; 3543 act.pde = pde; 3544 act.newpde = newpde; 3545 CPU_SET(cpuid, &active); 3546 smp_rendezvous_cpus(active, 3547 smp_no_rendezvous_barrier, pmap_update_pde_action, 3548 pmap_update_pde_teardown, &act); 3549 } else { 3550 pmap_update_pde_store(pmap, pde, newpde); 3551 if (CPU_ISSET(cpuid, &active)) 3552 pmap_update_pde_invalidate(pmap, va, newpde); 3553 } 3554 sched_unpin(); 3555 } 3556 3557 static void 3558 pmap_invalidate_pde_page(pmap_t pmap, vm_offset_t va, pd_entry_t pde) 3559 { 3560 3561 /* 3562 * When the PDE has PG_PROMOTED set, the 2MB page mapping was created 3563 * by a promotion that did not invalidate the 512 4KB page mappings 3564 * that might exist in the TLB. Consequently, at this point, the TLB 3565 * may hold both 4KB and 2MB page mappings for the address range [va, 3566 * va + NBPDR). Therefore, the entire range must be invalidated here. 3567 * In contrast, when PG_PROMOTED is clear, the TLB will not hold any 3568 * 4KB page mappings for the address range [va, va + NBPDR), and so a 3569 * single INVLPG suffices to invalidate the 2MB page mapping from the 3570 * TLB. 3571 */ 3572 if ((pde & PG_PROMOTED) != 0) 3573 pmap_invalidate_range(pmap, va, va + NBPDR - 1); 3574 else 3575 pmap_invalidate_page(pmap, va); 3576 } 3577 3578 DEFINE_IFUNC(, void, pmap_invalidate_cache_range, 3579 (vm_offset_t sva, vm_offset_t eva)) 3580 { 3581 3582 if ((cpu_feature & CPUID_SS) != 0) 3583 return (pmap_invalidate_cache_range_selfsnoop); 3584 if ((cpu_feature & CPUID_CLFSH) != 0) 3585 return (pmap_force_invalidate_cache_range); 3586 return (pmap_invalidate_cache_range_all); 3587 } 3588 3589 #define PMAP_CLFLUSH_THRESHOLD (2 * 1024 * 1024) 3590 3591 static void 3592 pmap_invalidate_cache_range_check_align(vm_offset_t sva, vm_offset_t eva) 3593 { 3594 3595 KASSERT((sva & PAGE_MASK) == 0, 3596 ("pmap_invalidate_cache_range: sva not page-aligned")); 3597 KASSERT((eva & PAGE_MASK) == 0, 3598 ("pmap_invalidate_cache_range: eva not page-aligned")); 3599 } 3600 3601 static void 3602 pmap_invalidate_cache_range_selfsnoop(vm_offset_t sva, vm_offset_t eva) 3603 { 3604 3605 pmap_invalidate_cache_range_check_align(sva, eva); 3606 } 3607 3608 void 3609 pmap_force_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva) 3610 { 3611 3612 sva &= ~(vm_offset_t)(cpu_clflush_line_size - 1); 3613 3614 /* 3615 * XXX: Some CPUs fault, hang, or trash the local APIC 3616 * registers if we use CLFLUSH on the local APIC range. The 3617 * local APIC is always uncached, so we don't need to flush 3618 * for that range anyway. 3619 */ 3620 if (pmap_kextract(sva) == lapic_paddr) 3621 return; 3622 3623 if ((cpu_stdext_feature & CPUID_STDEXT_CLFLUSHOPT) != 0) { 3624 /* 3625 * Do per-cache line flush. Use a locked 3626 * instruction to insure that previous stores are 3627 * included in the write-back. The processor 3628 * propagates flush to other processors in the cache 3629 * coherence domain. 3630 */ 3631 atomic_thread_fence_seq_cst(); 3632 for (; sva < eva; sva += cpu_clflush_line_size) 3633 clflushopt(sva); 3634 atomic_thread_fence_seq_cst(); 3635 } else { 3636 /* 3637 * Writes are ordered by CLFLUSH on Intel CPUs. 3638 */ 3639 if (cpu_vendor_id != CPU_VENDOR_INTEL) 3640 mfence(); 3641 for (; sva < eva; sva += cpu_clflush_line_size) 3642 clflush(sva); 3643 if (cpu_vendor_id != CPU_VENDOR_INTEL) 3644 mfence(); 3645 } 3646 } 3647 3648 static void 3649 pmap_invalidate_cache_range_all(vm_offset_t sva, vm_offset_t eva) 3650 { 3651 3652 pmap_invalidate_cache_range_check_align(sva, eva); 3653 pmap_invalidate_cache(); 3654 } 3655 3656 /* 3657 * Remove the specified set of pages from the data and instruction caches. 3658 * 3659 * In contrast to pmap_invalidate_cache_range(), this function does not 3660 * rely on the CPU's self-snoop feature, because it is intended for use 3661 * when moving pages into a different cache domain. 3662 */ 3663 void 3664 pmap_invalidate_cache_pages(vm_page_t *pages, int count) 3665 { 3666 vm_offset_t daddr, eva; 3667 int i; 3668 bool useclflushopt; 3669 3670 useclflushopt = (cpu_stdext_feature & CPUID_STDEXT_CLFLUSHOPT) != 0; 3671 if (count >= PMAP_CLFLUSH_THRESHOLD / PAGE_SIZE || 3672 ((cpu_feature & CPUID_CLFSH) == 0 && !useclflushopt)) 3673 pmap_invalidate_cache(); 3674 else { 3675 if (useclflushopt) 3676 atomic_thread_fence_seq_cst(); 3677 else if (cpu_vendor_id != CPU_VENDOR_INTEL) 3678 mfence(); 3679 for (i = 0; i < count; i++) { 3680 daddr = PHYS_TO_DMAP_ADDR(VM_PAGE_TO_PHYS(pages[i])); 3681 eva = daddr + PAGE_SIZE; 3682 for (; daddr < eva; daddr += cpu_clflush_line_size) { 3683 if (useclflushopt) 3684 clflushopt(daddr); 3685 else 3686 clflush(daddr); 3687 } 3688 } 3689 if (useclflushopt) 3690 atomic_thread_fence_seq_cst(); 3691 else if (cpu_vendor_id != CPU_VENDOR_INTEL) 3692 mfence(); 3693 } 3694 } 3695 3696 void 3697 pmap_flush_cache_range(vm_offset_t sva, vm_offset_t eva) 3698 { 3699 3700 pmap_invalidate_cache_range_check_align(sva, eva); 3701 3702 if ((cpu_stdext_feature & CPUID_STDEXT_CLWB) == 0) { 3703 pmap_force_invalidate_cache_range(sva, eva); 3704 return; 3705 } 3706 3707 /* See comment in pmap_force_invalidate_cache_range(). */ 3708 if (pmap_kextract(sva) == lapic_paddr) 3709 return; 3710 3711 atomic_thread_fence_seq_cst(); 3712 for (; sva < eva; sva += cpu_clflush_line_size) 3713 clwb(sva); 3714 atomic_thread_fence_seq_cst(); 3715 } 3716 3717 void 3718 pmap_flush_cache_phys_range(vm_paddr_t spa, vm_paddr_t epa, vm_memattr_t mattr) 3719 { 3720 pt_entry_t *pte; 3721 vm_offset_t vaddr; 3722 int error __diagused; 3723 int pte_bits; 3724 3725 KASSERT((spa & PAGE_MASK) == 0, 3726 ("pmap_flush_cache_phys_range: spa not page-aligned")); 3727 KASSERT((epa & PAGE_MASK) == 0, 3728 ("pmap_flush_cache_phys_range: epa not page-aligned")); 3729 3730 if (spa < dmaplimit) { 3731 pmap_flush_cache_range(PHYS_TO_DMAP_ADDR(spa), 3732 PHYS_TO_DMAP_ADDR(MIN(dmaplimit, epa))); 3733 if (dmaplimit >= epa) 3734 return; 3735 spa = dmaplimit; 3736 } 3737 3738 pte_bits = pmap_cache_bits(kernel_pmap, mattr, false) | X86_PG_RW | 3739 X86_PG_V; 3740 error = vmem_alloc(kernel_arena, PAGE_SIZE, M_BESTFIT | M_WAITOK, 3741 &vaddr); 3742 KASSERT(error == 0, ("vmem_alloc failed: %d", error)); 3743 pte = vtopte(vaddr); 3744 for (; spa < epa; spa += PAGE_SIZE) { 3745 sched_pin(); 3746 pte_store(pte, spa | pte_bits); 3747 pmap_invlpg(kernel_pmap, vaddr); 3748 /* XXXKIB atomic inside flush_cache_range are excessive */ 3749 pmap_flush_cache_range(vaddr, vaddr + PAGE_SIZE); 3750 sched_unpin(); 3751 } 3752 vmem_free(kernel_arena, vaddr, PAGE_SIZE); 3753 } 3754 3755 /* 3756 * Routine: pmap_extract 3757 * Function: 3758 * Extract the physical page address associated 3759 * with the given map/virtual_address pair. 3760 */ 3761 vm_paddr_t 3762 pmap_extract(pmap_t pmap, vm_offset_t va) 3763 { 3764 pdp_entry_t *pdpe; 3765 pd_entry_t *pde; 3766 pt_entry_t *pte, PG_V; 3767 vm_paddr_t pa; 3768 3769 pa = 0; 3770 PG_V = pmap_valid_bit(pmap); 3771 PMAP_LOCK(pmap); 3772 pdpe = pmap_pdpe(pmap, va); 3773 if (pdpe != NULL && (*pdpe & PG_V) != 0) { 3774 if ((*pdpe & PG_PS) != 0) 3775 pa = (*pdpe & PG_PS_FRAME) | (va & PDPMASK); 3776 else { 3777 pde = pmap_pdpe_to_pde(pdpe, va); 3778 if ((*pde & PG_V) != 0) { 3779 if ((*pde & PG_PS) != 0) { 3780 pa = (*pde & PG_PS_FRAME) | 3781 (va & PDRMASK); 3782 } else { 3783 pte = pmap_pde_to_pte(pde, va); 3784 pa = (*pte & PG_FRAME) | 3785 (va & PAGE_MASK); 3786 } 3787 } 3788 } 3789 } 3790 PMAP_UNLOCK(pmap); 3791 return (pa); 3792 } 3793 3794 /* 3795 * Routine: pmap_extract_and_hold 3796 * Function: 3797 * Atomically extract and hold the physical page 3798 * with the given pmap and virtual address pair 3799 * if that mapping permits the given protection. 3800 */ 3801 vm_page_t 3802 pmap_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot) 3803 { 3804 pdp_entry_t pdpe, *pdpep; 3805 pd_entry_t pde, *pdep; 3806 pt_entry_t pte, PG_RW, PG_V; 3807 vm_page_t m; 3808 3809 m = NULL; 3810 PG_RW = pmap_rw_bit(pmap); 3811 PG_V = pmap_valid_bit(pmap); 3812 PMAP_LOCK(pmap); 3813 3814 pdpep = pmap_pdpe(pmap, va); 3815 if (pdpep == NULL || ((pdpe = *pdpep) & PG_V) == 0) 3816 goto out; 3817 if ((pdpe & PG_PS) != 0) { 3818 if ((pdpe & PG_RW) == 0 && (prot & VM_PROT_WRITE) != 0) 3819 goto out; 3820 m = PHYS_TO_VM_PAGE((pdpe & PG_PS_FRAME) | (va & PDPMASK)); 3821 goto check_page; 3822 } 3823 3824 pdep = pmap_pdpe_to_pde(pdpep, va); 3825 if (pdep == NULL || ((pde = *pdep) & PG_V) == 0) 3826 goto out; 3827 if ((pde & PG_PS) != 0) { 3828 if ((pde & PG_RW) == 0 && (prot & VM_PROT_WRITE) != 0) 3829 goto out; 3830 m = PHYS_TO_VM_PAGE((pde & PG_PS_FRAME) | (va & PDRMASK)); 3831 goto check_page; 3832 } 3833 3834 pte = *pmap_pde_to_pte(pdep, va); 3835 if ((pte & PG_V) == 0 || 3836 ((pte & PG_RW) == 0 && (prot & VM_PROT_WRITE) != 0)) 3837 goto out; 3838 m = PHYS_TO_VM_PAGE(pte & PG_FRAME); 3839 3840 check_page: 3841 if (m != NULL && !vm_page_wire_mapped(m)) 3842 m = NULL; 3843 out: 3844 PMAP_UNLOCK(pmap); 3845 return (m); 3846 } 3847 3848 /* 3849 * Routine: pmap_kextract 3850 * Function: 3851 * Extract the physical page address associated with the given kernel 3852 * virtual address. 3853 */ 3854 vm_paddr_t 3855 pmap_kextract(vm_offset_t va) 3856 { 3857 pd_entry_t pde; 3858 vm_paddr_t pa; 3859 3860 if (va >= kva_layout.dmap_low && va < kva_layout.dmap_high) { 3861 pa = DMAP_TO_PHYS(va); 3862 } else if (PMAP_ADDRESS_IN_LARGEMAP(va)) { 3863 pa = pmap_large_map_kextract(va); 3864 } else { 3865 pde = *vtopde(va); 3866 if (pde & PG_PS) { 3867 pa = (pde & PG_PS_FRAME) | (va & PDRMASK); 3868 } else { 3869 /* 3870 * Beware of a concurrent promotion that changes the 3871 * PDE at this point! For example, vtopte() must not 3872 * be used to access the PTE because it would use the 3873 * new PDE. It is, however, safe to use the old PDE 3874 * because the page table page is preserved by the 3875 * promotion. 3876 */ 3877 pa = *pmap_pde_to_pte(&pde, va); 3878 pa = (pa & PG_FRAME) | (va & PAGE_MASK); 3879 } 3880 } 3881 return (pa); 3882 } 3883 3884 /*************************************************** 3885 * Low level mapping routines..... 3886 ***************************************************/ 3887 3888 /* 3889 * Add a wired page to the kva. 3890 * Note: not SMP coherent. 3891 */ 3892 void 3893 pmap_kenter(vm_offset_t va, vm_paddr_t pa) 3894 { 3895 pt_entry_t *pte; 3896 3897 pte = vtopte(va); 3898 pte_store(pte, pa | pg_g | pg_nx | X86_PG_A | X86_PG_M | 3899 X86_PG_RW | X86_PG_V); 3900 } 3901 3902 static __inline void 3903 pmap_kenter_attr(vm_offset_t va, vm_paddr_t pa, int mode) 3904 { 3905 pt_entry_t *pte; 3906 int cache_bits; 3907 3908 pte = vtopte(va); 3909 cache_bits = pmap_cache_bits(kernel_pmap, mode, false); 3910 pte_store(pte, pa | pg_g | pg_nx | X86_PG_A | X86_PG_M | 3911 X86_PG_RW | X86_PG_V | cache_bits); 3912 } 3913 3914 /* 3915 * Remove a page from the kernel pagetables. 3916 * Note: not SMP coherent. 3917 */ 3918 void 3919 pmap_kremove(vm_offset_t va) 3920 { 3921 pt_entry_t *pte; 3922 3923 pte = vtopte(va); 3924 pte_clear(pte); 3925 } 3926 3927 /* 3928 * Used to map a range of physical addresses into kernel 3929 * virtual address space. 3930 * 3931 * The value passed in '*virt' is a suggested virtual address for 3932 * the mapping. Architectures which can support a direct-mapped 3933 * physical to virtual region can return the appropriate address 3934 * within that region, leaving '*virt' unchanged. Other 3935 * architectures should map the pages starting at '*virt' and 3936 * update '*virt' with the first usable address after the mapped 3937 * region. 3938 */ 3939 void * 3940 pmap_map(vm_offset_t *virt, vm_paddr_t start, vm_paddr_t end, int prot) 3941 { 3942 return (PHYS_TO_DMAP(start)); 3943 } 3944 3945 /* 3946 * Add a list of wired pages to the kva 3947 * this routine is only used for temporary 3948 * kernel mappings that do not need to have 3949 * page modification or references recorded. 3950 * Note that old mappings are simply written 3951 * over. The page *must* be wired. 3952 * Note: SMP coherent. Uses a ranged shootdown IPI. 3953 */ 3954 void 3955 pmap_qenter(void *va, vm_page_t *ma, int count) 3956 { 3957 pt_entry_t *endpte, oldpte, pa, *pte; 3958 vm_offset_t sva = (vm_offset_t)va; 3959 vm_page_t m; 3960 int cache_bits; 3961 3962 oldpte = 0; 3963 pte = vtopte(sva); 3964 endpte = pte + count; 3965 while (pte < endpte) { 3966 m = *ma++; 3967 cache_bits = pmap_cache_bits(kernel_pmap, m->md.pat_mode, false); 3968 pa = VM_PAGE_TO_PHYS(m) | cache_bits; 3969 if ((*pte & (PG_FRAME | X86_PG_PTE_CACHE)) != pa) { 3970 oldpte |= *pte; 3971 pte_store(pte, pa | pg_g | pg_nx | X86_PG_A | 3972 X86_PG_M | X86_PG_RW | X86_PG_V); 3973 } 3974 pte++; 3975 } 3976 if (__predict_false((oldpte & X86_PG_V) != 0)) 3977 pmap_invalidate_range(kernel_pmap, sva, sva + count * 3978 PAGE_SIZE); 3979 } 3980 3981 /* 3982 * This routine tears out page mappings from the 3983 * kernel -- it is meant only for temporary mappings. 3984 * Note: SMP coherent. Uses a ranged shootdown IPI. 3985 */ 3986 void 3987 pmap_qremove(void *sva, int count) 3988 { 3989 vm_offset_t va; 3990 3991 va = (vm_offset_t)sva; 3992 while (count-- > 0) { 3993 /* 3994 * pmap_enter() calls within the kernel virtual 3995 * address space happen on virtual addresses from 3996 * subarenas that import superpage-sized and -aligned 3997 * address ranges. So, the virtual address that we 3998 * allocate to use with pmap_qenter() can't be close 3999 * enough to one of those pmap_enter() calls for it to 4000 * be caught up in a promotion. 4001 */ 4002 KASSERT(va >= kva_layout.km_low, ("usermode va %lx", va)); 4003 KASSERT((*vtopde(va) & X86_PG_PS) == 0, 4004 ("pmap_qremove on promoted va %#lx", va)); 4005 4006 pmap_kremove(va); 4007 va += PAGE_SIZE; 4008 } 4009 pmap_invalidate_range(kernel_pmap, (vm_offset_t)sva, va); 4010 } 4011 4012 /*************************************************** 4013 * Page table page management routines..... 4014 ***************************************************/ 4015 /* 4016 * Schedule the specified unused page table page to be freed. Specifically, 4017 * add the page to the specified list of pages that will be released to the 4018 * physical memory manager after the TLB has been updated. 4019 */ 4020 static __inline void 4021 pmap_add_delayed_free_list(vm_page_t m, struct spglist *free, bool set_PG_ZERO) 4022 { 4023 4024 if (set_PG_ZERO) 4025 m->flags |= PG_ZERO; 4026 else 4027 m->flags &= ~PG_ZERO; 4028 SLIST_INSERT_HEAD(free, m, plinks.s.ss); 4029 } 4030 4031 /* 4032 * Inserts the specified page table page into the specified pmap's collection 4033 * of idle page table pages. Each of a pmap's page table pages is responsible 4034 * for mapping a distinct range of virtual addresses. The pmap's collection is 4035 * ordered by this virtual address range. 4036 * 4037 * If "promoted" is false, then the page table page "mpte" must be zero filled; 4038 * "mpte"'s valid field will be set to 0. 4039 * 4040 * If "promoted" is true and "allpte_PG_A_set" is false, then "mpte" must 4041 * contain valid mappings with identical attributes except for PG_A; "mpte"'s 4042 * valid field will be set to 1. 4043 * 4044 * If "promoted" and "allpte_PG_A_set" are both true, then "mpte" must contain 4045 * valid mappings with identical attributes including PG_A; "mpte"'s valid 4046 * field will be set to VM_PAGE_BITS_ALL. 4047 */ 4048 static __inline int 4049 pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte, bool promoted, 4050 bool allpte_PG_A_set) 4051 { 4052 4053 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 4054 KASSERT(promoted || !allpte_PG_A_set, 4055 ("a zero-filled PTP can't have PG_A set in every PTE")); 4056 mpte->valid = promoted ? (allpte_PG_A_set ? VM_PAGE_BITS_ALL : 1) : 0; 4057 return (vm_radix_insert(&pmap->pm_root, mpte)); 4058 } 4059 4060 /* 4061 * Removes the page table page mapping the specified virtual address from the 4062 * specified pmap's collection of idle page table pages, and returns it. 4063 * Otherwise, returns NULL if there is no page table page corresponding to the 4064 * specified virtual address. 4065 */ 4066 static __inline vm_page_t 4067 pmap_remove_pt_page(pmap_t pmap, vm_offset_t va) 4068 { 4069 4070 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 4071 return (vm_radix_remove(&pmap->pm_root, pmap_pde_pindex(va))); 4072 } 4073 4074 /* 4075 * Decrements a page table page's reference count, which is used to record the 4076 * number of valid page table entries within the page. If the reference count 4077 * drops to zero, then the page table page is unmapped. Returns true if the 4078 * page table page was unmapped and false otherwise. 4079 */ 4080 static inline bool 4081 pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free) 4082 { 4083 4084 --m->ref_count; 4085 if (m->ref_count == 0) { 4086 _pmap_unwire_ptp(pmap, va, m, free); 4087 return (true); 4088 } else 4089 return (false); 4090 } 4091 4092 static void 4093 _pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free) 4094 { 4095 pml5_entry_t *pml5; 4096 pml4_entry_t *pml4; 4097 pdp_entry_t *pdp; 4098 pd_entry_t *pd; 4099 vm_page_t pdpg, pdppg, pml4pg; 4100 4101 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 4102 4103 /* 4104 * unmap the page table page 4105 */ 4106 if (m->pindex >= NUPDE + NUPDPE + NUPML4E) { 4107 /* PML4 page */ 4108 MPASS(pmap_is_la57(pmap)); 4109 pml5 = pmap_pml5e(pmap, va); 4110 *pml5 = 0; 4111 if (pmap->pm_pmltopu != NULL && va <= VM_MAXUSER_ADDRESS) { 4112 pml5 = pmap_pml5e_u(pmap, va); 4113 *pml5 = 0; 4114 } 4115 } else if (m->pindex >= NUPDE + NUPDPE) { 4116 /* PDP page */ 4117 pml4 = pmap_pml4e(pmap, va); 4118 *pml4 = 0; 4119 if (!pmap_is_la57(pmap) && pmap->pm_pmltopu != NULL && 4120 va <= VM_MAXUSER_ADDRESS) { 4121 pml4 = pmap_pml4e_u(pmap, va); 4122 *pml4 = 0; 4123 } 4124 } else if (m->pindex >= NUPDE) { 4125 /* PD page */ 4126 pdp = pmap_pdpe(pmap, va); 4127 *pdp = 0; 4128 } else { 4129 /* PTE page */ 4130 pd = pmap_pde(pmap, va); 4131 *pd = 0; 4132 } 4133 if (m->pindex < NUPDE) { 4134 /* We just released a PT, unhold the matching PD */ 4135 pdpg = PHYS_TO_VM_PAGE(*pmap_pdpe(pmap, va) & PG_FRAME); 4136 pmap_unwire_ptp(pmap, va, pdpg, free); 4137 } else if (m->pindex < NUPDE + NUPDPE) { 4138 /* We just released a PD, unhold the matching PDP */ 4139 pdppg = PHYS_TO_VM_PAGE(*pmap_pml4e(pmap, va) & PG_FRAME); 4140 pmap_unwire_ptp(pmap, va, pdppg, free); 4141 } else if (m->pindex < NUPDE + NUPDPE + NUPML4E && pmap_is_la57(pmap)) { 4142 /* We just released a PDP, unhold the matching PML4 */ 4143 pml4pg = PHYS_TO_VM_PAGE(*pmap_pml5e(pmap, va) & PG_FRAME); 4144 pmap_unwire_ptp(pmap, va, pml4pg, free); 4145 } 4146 4147 pmap_pt_page_count_adj(pmap, -1); 4148 4149 /* 4150 * Put page on a list so that it is released after 4151 * *ALL* TLB shootdown is done 4152 */ 4153 pmap_add_delayed_free_list(m, free, true); 4154 } 4155 4156 /* 4157 * After removing a page table entry, this routine is used to 4158 * conditionally free the page, and manage the reference count. 4159 */ 4160 static int 4161 pmap_unuse_pt(pmap_t pmap, vm_offset_t va, pd_entry_t ptepde, 4162 struct spglist *free) 4163 { 4164 vm_page_t mpte; 4165 4166 if (va >= VM_MAXUSER_ADDRESS) 4167 return (0); 4168 KASSERT(ptepde != 0, ("pmap_unuse_pt: ptepde != 0")); 4169 mpte = PHYS_TO_VM_PAGE(ptepde & PG_FRAME); 4170 return (pmap_unwire_ptp(pmap, va, mpte, free)); 4171 } 4172 4173 /* 4174 * Release a page table page reference after a failed attempt to create a 4175 * mapping. 4176 */ 4177 static void 4178 pmap_abort_ptp(pmap_t pmap, vm_offset_t va, vm_page_t mpte) 4179 { 4180 struct spglist free; 4181 4182 SLIST_INIT(&free); 4183 if (pmap_unwire_ptp(pmap, va, mpte, &free)) { 4184 /* 4185 * Although "va" was never mapped, paging-structure caches 4186 * could nonetheless have entries that refer to the freed 4187 * page table pages. Invalidate those entries. 4188 */ 4189 pmap_invalidate_page(pmap, va); 4190 vm_page_free_pages_toq(&free, true); 4191 } 4192 } 4193 4194 static void 4195 pmap_pinit_pcids(pmap_t pmap, uint32_t pcid, int gen) 4196 { 4197 struct pmap_pcid *pcidp; 4198 int i; 4199 4200 CPU_FOREACH(i) { 4201 pcidp = zpcpu_get_cpu(pmap->pm_pcidp, i); 4202 pcidp->pm_pcid = pcid; 4203 pcidp->pm_gen = gen; 4204 } 4205 } 4206 4207 void 4208 pmap_pinit0(pmap_t pmap) 4209 { 4210 struct proc *p; 4211 struct thread *td; 4212 4213 PMAP_LOCK_INIT(pmap); 4214 pmap->pm_pmltop = kernel_pmap->pm_pmltop; 4215 pmap->pm_pmltopu = NULL; 4216 pmap->pm_cr3 = kernel_pmap->pm_cr3; 4217 /* hack to keep pmap_pti_pcid_invalidate() alive */ 4218 pmap->pm_ucr3 = PMAP_NO_CR3; 4219 vm_radix_init(&pmap->pm_root); 4220 CPU_ZERO(&pmap->pm_active); 4221 TAILQ_INIT(&pmap->pm_pvchunk); 4222 bzero(&pmap->pm_stats, sizeof pmap->pm_stats); 4223 pmap->pm_flags = pmap_flags; 4224 pmap->pm_pcidp = uma_zalloc_pcpu(pcpu_zone_8, M_WAITOK); 4225 pmap_pinit_pcids(pmap, PMAP_PCID_KERN + 1, 1); 4226 pmap_activate_boot(pmap); 4227 td = curthread; 4228 if (pti) { 4229 p = td->td_proc; 4230 PROC_LOCK(p); 4231 p->p_md.md_flags |= P_MD_KPTI; 4232 PROC_UNLOCK(p); 4233 } 4234 pmap_thread_init_invl_gen(td); 4235 4236 if ((cpu_stdext_feature2 & CPUID_STDEXT2_PKU) != 0) { 4237 pmap_pkru_ranges_zone = uma_zcreate("pkru ranges", 4238 sizeof(struct pmap_pkru_range), NULL, NULL, NULL, NULL, 4239 UMA_ALIGN_PTR, 0); 4240 } 4241 } 4242 4243 void 4244 pmap_pinit_pml4(vm_page_t pml4pg) 4245 { 4246 pml4_entry_t *pm_pml4; 4247 int i; 4248 4249 pm_pml4 = VM_PAGE_TO_DMAP(pml4pg); 4250 4251 /* Wire in kernel global address entries. */ 4252 for (i = 0; i < NKPML4E; i++) { 4253 pm_pml4[KPML4BASE + i] = (KPDPphys + ptoa(i)) | X86_PG_RW | 4254 X86_PG_V; 4255 } 4256 #ifdef KASAN 4257 for (i = 0; i < NKASANPML4E; i++) { 4258 pm_pml4[KASANPML4I + i] = (KASANPDPphys + ptoa(i)) | X86_PG_RW | 4259 X86_PG_V | pg_nx; 4260 } 4261 #endif 4262 #ifdef KMSAN 4263 for (i = 0; i < NKMSANSHADPML4E; i++) { 4264 pm_pml4[KMSANSHADPML4I + i] = (KMSANSHADPDPphys + ptoa(i)) | 4265 X86_PG_RW | X86_PG_V | pg_nx; 4266 } 4267 for (i = 0; i < NKMSANORIGPML4E; i++) { 4268 pm_pml4[KMSANORIGPML4I + i] = (KMSANORIGPDPphys + ptoa(i)) | 4269 X86_PG_RW | X86_PG_V | pg_nx; 4270 } 4271 #endif 4272 for (i = 0; i < ndmpdpphys; i++) { 4273 pm_pml4[DMPML4I + i] = (DMPDPphys + ptoa(i)) | X86_PG_RW | 4274 X86_PG_V; 4275 } 4276 4277 /* install self-referential address mapping entry(s) */ 4278 pm_pml4[PML4PML4I] = VM_PAGE_TO_PHYS(pml4pg) | X86_PG_V | X86_PG_RW | 4279 X86_PG_A | X86_PG_M; 4280 4281 /* install large map entries if configured */ 4282 for (i = 0; i < lm_ents; i++) 4283 pm_pml4[LMSPML4I + i] = kernel_pmap->pm_pmltop[LMSPML4I + i]; 4284 } 4285 4286 void 4287 pmap_pinit_pml5(vm_page_t pml5pg) 4288 { 4289 pml5_entry_t *pm_pml5; 4290 int i; 4291 4292 pm_pml5 = VM_PAGE_TO_DMAP(pml5pg); 4293 for (i = 0; i < NPML5EPG / 2; i++) 4294 pm_pml5[i] = 0; 4295 for (; i < NPML5EPG; i++) 4296 pm_pml5[i] = kernel_pmap->pm_pmltop[i]; 4297 } 4298 4299 static void 4300 pmap_pinit_pml4_pti(vm_page_t pml4pgu) 4301 { 4302 pml4_entry_t *pm_pml4u; 4303 int i; 4304 4305 pm_pml4u = VM_PAGE_TO_DMAP(pml4pgu); 4306 for (i = 0; i < NPML4EPG; i++) 4307 pm_pml4u[i] = pti_pml4[i]; 4308 } 4309 4310 static void 4311 pmap_pinit_pml5_pti(vm_page_t pml5pgu) 4312 { 4313 pml5_entry_t *pm_pml5u; 4314 4315 pm_pml5u = VM_PAGE_TO_DMAP(pml5pgu); 4316 pagezero(pm_pml5u); 4317 4318 /* 4319 * Add pml5 entry at top of KVA pointing to existing pml4 pti 4320 * table, entering all kernel mappings needed for usermode 4321 * into level 5 table. 4322 */ 4323 pm_pml5u[pmap_pml5e_index(UPT_MAX_ADDRESS)] = 4324 pmap_kextract((vm_offset_t)pti_pml4) | 4325 X86_PG_V | X86_PG_RW | X86_PG_A | X86_PG_M; 4326 } 4327 4328 /* Allocate a page table page and do related bookkeeping */ 4329 static vm_page_t 4330 pmap_alloc_pt_page(pmap_t pmap, vm_pindex_t pindex, int flags) 4331 { 4332 vm_page_t m; 4333 4334 m = vm_page_alloc_noobj(flags); 4335 if (__predict_false(m == NULL)) 4336 return (NULL); 4337 m->pindex = pindex; 4338 pmap_pt_page_count_adj(pmap, 1); 4339 return (m); 4340 } 4341 4342 static void 4343 pmap_free_pt_page(pmap_t pmap, vm_page_t m, bool zerofilled) 4344 { 4345 /* 4346 * This function assumes the page will need to be unwired, 4347 * even though the counterpart allocation in pmap_alloc_pt_page() 4348 * doesn't enforce VM_ALLOC_WIRED. However, all current uses 4349 * of pmap_free_pt_page() require unwiring. The case in which 4350 * a PT page doesn't require unwiring because its ref_count has 4351 * naturally reached 0 is handled through _pmap_unwire_ptp(). 4352 */ 4353 vm_page_unwire_noq(m); 4354 if (zerofilled) 4355 vm_page_free_zero(m); 4356 else 4357 vm_page_free(m); 4358 4359 pmap_pt_page_count_adj(pmap, -1); 4360 } 4361 4362 _Static_assert(sizeof(struct pmap_pcid) == 8, "Fix pcpu zone for pm_pcidp"); 4363 4364 /* 4365 * Initialize a preallocated and zeroed pmap structure, 4366 * such as one in a vmspace structure. 4367 */ 4368 int 4369 pmap_pinit_type(pmap_t pmap, enum pmap_type pm_type, int flags) 4370 { 4371 vm_page_t pmltop_pg, pmltop_pgu; 4372 vm_paddr_t pmltop_phys; 4373 4374 bzero(&pmap->pm_stats, sizeof pmap->pm_stats); 4375 4376 /* 4377 * Allocate the page directory page. Pass NULL instead of a 4378 * pointer to the pmap here to avoid calling 4379 * pmap_resident_count_adj() through pmap_pt_page_count_adj(), 4380 * since that requires pmap lock. Instead do the accounting 4381 * manually. 4382 * 4383 * Note that final call to pmap_remove() optimization that 4384 * checks for zero resident_count is basically disabled by 4385 * accounting for top-level page. But the optimization was 4386 * not effective since we started using non-managed mapping of 4387 * the shared page. 4388 */ 4389 pmltop_pg = pmap_alloc_pt_page(NULL, 0, VM_ALLOC_WIRED | VM_ALLOC_ZERO | 4390 VM_ALLOC_WAITOK); 4391 pmap_pt_page_count_pinit(pmap, 1); 4392 4393 pmltop_phys = VM_PAGE_TO_PHYS(pmltop_pg); 4394 pmap->pm_pmltop = PHYS_TO_DMAP(pmltop_phys); 4395 4396 if (pmap_pcid_enabled) { 4397 if (pmap->pm_pcidp == NULL) 4398 pmap->pm_pcidp = uma_zalloc_pcpu(pcpu_zone_8, 4399 M_WAITOK); 4400 pmap_pinit_pcids(pmap, PMAP_PCID_NONE, 0); 4401 } 4402 pmap->pm_cr3 = PMAP_NO_CR3; /* initialize to an invalid value */ 4403 pmap->pm_ucr3 = PMAP_NO_CR3; 4404 pmap->pm_pmltopu = NULL; 4405 4406 pmap->pm_type = pm_type; 4407 4408 /* 4409 * Do not install the host kernel mappings in the nested page 4410 * tables. These mappings are meaningless in the guest physical 4411 * address space. 4412 * Install minimal kernel mappings in PTI case. 4413 */ 4414 switch (pm_type) { 4415 case PT_X86: 4416 pmap->pm_cr3 = pmltop_phys; 4417 if (pmap_is_la57(pmap)) 4418 pmap_pinit_pml5(pmltop_pg); 4419 else 4420 pmap_pinit_pml4(pmltop_pg); 4421 if ((curproc->p_md.md_flags & P_MD_KPTI) != 0) { 4422 /* 4423 * As with pmltop_pg, pass NULL instead of a 4424 * pointer to the pmap to ensure that the PTI 4425 * page counted explicitly. 4426 */ 4427 pmltop_pgu = pmap_alloc_pt_page(NULL, 0, 4428 VM_ALLOC_WIRED | VM_ALLOC_WAITOK); 4429 pmap_pt_page_count_pinit(pmap, 1); 4430 pmap->pm_pmltopu = PHYS_TO_DMAP( 4431 VM_PAGE_TO_PHYS(pmltop_pgu)); 4432 if (pmap_is_la57(pmap)) 4433 pmap_pinit_pml5_pti(pmltop_pgu); 4434 else 4435 pmap_pinit_pml4_pti(pmltop_pgu); 4436 pmap->pm_ucr3 = VM_PAGE_TO_PHYS(pmltop_pgu); 4437 } 4438 if ((cpu_stdext_feature2 & CPUID_STDEXT2_PKU) != 0) { 4439 rangeset_init(&pmap->pm_pkru, pkru_dup_range, 4440 pkru_free_range, pmap, M_NOWAIT); 4441 } 4442 break; 4443 case PT_EPT: 4444 case PT_RVI: 4445 pmap->pm_eptsmr = smr_create("pmap", 0, 0); 4446 break; 4447 } 4448 4449 vm_radix_init(&pmap->pm_root); 4450 CPU_ZERO(&pmap->pm_active); 4451 TAILQ_INIT(&pmap->pm_pvchunk); 4452 pmap->pm_flags = flags; 4453 pmap->pm_eptgen = 0; 4454 4455 return (1); 4456 } 4457 4458 int 4459 pmap_pinit(pmap_t pmap) 4460 { 4461 4462 return (pmap_pinit_type(pmap, PT_X86, pmap_flags)); 4463 } 4464 4465 static void 4466 pmap_allocpte_free_unref(pmap_t pmap, vm_offset_t va, pt_entry_t *pte) 4467 { 4468 vm_page_t mpg; 4469 struct spglist free; 4470 4471 mpg = PHYS_TO_VM_PAGE(*pte & PG_FRAME); 4472 if (mpg->ref_count != 0) 4473 return; 4474 SLIST_INIT(&free); 4475 _pmap_unwire_ptp(pmap, va, mpg, &free); 4476 pmap_invalidate_page(pmap, va); 4477 vm_page_free_pages_toq(&free, true); 4478 } 4479 4480 static pml4_entry_t * 4481 pmap_allocpte_getpml4(pmap_t pmap, struct rwlock **lockp, vm_offset_t va, 4482 bool addref) 4483 { 4484 vm_pindex_t pml5index; 4485 pml5_entry_t *pml5; 4486 pml4_entry_t *pml4; 4487 vm_page_t pml4pg; 4488 pt_entry_t PG_V; 4489 bool allocated; 4490 4491 if (!pmap_is_la57(pmap)) 4492 return (&pmap->pm_pmltop[pmap_pml4e_index(va)]); 4493 4494 PG_V = pmap_valid_bit(pmap); 4495 pml5index = pmap_pml5e_index(va); 4496 pml5 = &pmap->pm_pmltop[pml5index]; 4497 if ((*pml5 & PG_V) == 0) { 4498 if (pmap_allocpte_nosleep(pmap, pmap_pml5e_pindex(va), lockp, 4499 va) == NULL) 4500 return (NULL); 4501 allocated = true; 4502 } else { 4503 allocated = false; 4504 } 4505 pml4 = PHYS_TO_DMAP(*pml5 & PG_FRAME); 4506 pml4 = &pml4[pmap_pml4e_index(va)]; 4507 if ((*pml4 & PG_V) == 0) { 4508 pml4pg = PHYS_TO_VM_PAGE(*pml5 & PG_FRAME); 4509 if (allocated && !addref) 4510 pml4pg->ref_count--; 4511 else if (!allocated && addref) 4512 pml4pg->ref_count++; 4513 } 4514 return (pml4); 4515 } 4516 4517 static pdp_entry_t * 4518 pmap_allocpte_getpdp(pmap_t pmap, struct rwlock **lockp, vm_offset_t va, 4519 bool addref) 4520 { 4521 vm_page_t pdppg; 4522 pml4_entry_t *pml4; 4523 pdp_entry_t *pdp; 4524 pt_entry_t PG_V; 4525 bool allocated; 4526 4527 PG_V = pmap_valid_bit(pmap); 4528 4529 pml4 = pmap_allocpte_getpml4(pmap, lockp, va, false); 4530 if (pml4 == NULL) 4531 return (NULL); 4532 4533 if ((*pml4 & PG_V) == 0) { 4534 /* Have to allocate a new pdp, recurse */ 4535 if (pmap_allocpte_nosleep(pmap, pmap_pml4e_pindex(va), lockp, 4536 va) == NULL) { 4537 if (pmap_is_la57(pmap)) 4538 pmap_allocpte_free_unref(pmap, va, 4539 pmap_pml5e(pmap, va)); 4540 return (NULL); 4541 } 4542 allocated = true; 4543 } else { 4544 allocated = false; 4545 } 4546 pdp = PHYS_TO_DMAP(*pml4 & PG_FRAME); 4547 pdp = &pdp[pmap_pdpe_index(va)]; 4548 if ((*pdp & PG_V) == 0) { 4549 pdppg = PHYS_TO_VM_PAGE(*pml4 & PG_FRAME); 4550 if (allocated && !addref) 4551 pdppg->ref_count--; 4552 else if (!allocated && addref) 4553 pdppg->ref_count++; 4554 } 4555 return (pdp); 4556 } 4557 4558 /* 4559 * The ptepindexes, i.e. page indices, of the page table pages encountered 4560 * while translating virtual address va are defined as follows: 4561 * - for the page table page (last level), 4562 * ptepindex = pmap_pde_pindex(va) = va >> PDRSHIFT, 4563 * in other words, it is just the index of the PDE that maps the page 4564 * table page. 4565 * - for the page directory page, 4566 * ptepindex = NUPDE (number of userland PD entries) + 4567 * (pmap_pde_index(va) >> NPDEPGSHIFT) 4568 * i.e. index of PDPE is put after the last index of PDE, 4569 * - for the page directory pointer page, 4570 * ptepindex = NUPDE + NUPDPE + (pmap_pde_index(va) >> (NPDEPGSHIFT + 4571 * NPML4EPGSHIFT), 4572 * i.e. index of pml4e is put after the last index of PDPE, 4573 * - for the PML4 page (if LA57 mode is enabled), 4574 * ptepindex = NUPDE + NUPDPE + NUPML4E + (pmap_pde_index(va) >> 4575 * (NPDEPGSHIFT + NPML4EPGSHIFT + NPML5EPGSHIFT), 4576 * i.e. index of pml5e is put after the last index of PML4E. 4577 * 4578 * Define an order on the paging entries, where all entries of the 4579 * same height are put together, then heights are put from deepest to 4580 * root. Then ptexpindex is the sequential number of the 4581 * corresponding paging entry in this order. 4582 * 4583 * The values of NUPDE, NUPDPE, and NUPML4E are determined by the size of 4584 * LA57 paging structures even in LA48 paging mode. Moreover, the 4585 * ptepindexes are calculated as if the paging structures were 5-level 4586 * regardless of the actual mode of operation. 4587 * 4588 * The root page at PML4/PML5 does not participate in this indexing scheme, 4589 * since it is statically allocated by pmap_pinit() and not by pmap_allocpte(). 4590 */ 4591 static vm_page_t 4592 pmap_allocpte_nosleep(pmap_t pmap, vm_pindex_t ptepindex, struct rwlock **lockp, 4593 vm_offset_t va) 4594 { 4595 vm_pindex_t pml5index, pml4index; 4596 pml5_entry_t *pml5, *pml5u; 4597 pml4_entry_t *pml4, *pml4u; 4598 pdp_entry_t *pdp; 4599 pd_entry_t *pd; 4600 vm_page_t m, pdpg; 4601 pt_entry_t PG_A, PG_M, PG_RW, PG_V; 4602 4603 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 4604 4605 PG_A = pmap_accessed_bit(pmap); 4606 PG_M = pmap_modified_bit(pmap); 4607 PG_V = pmap_valid_bit(pmap); 4608 PG_RW = pmap_rw_bit(pmap); 4609 4610 /* 4611 * Allocate a page table page. 4612 */ 4613 m = pmap_alloc_pt_page(pmap, ptepindex, 4614 VM_ALLOC_WIRED | VM_ALLOC_ZERO); 4615 if (m == NULL) 4616 return (NULL); 4617 4618 /* 4619 * Map the pagetable page into the process address space, if 4620 * it isn't already there. 4621 */ 4622 if (ptepindex >= NUPDE + NUPDPE + NUPML4E) { 4623 MPASS(pmap_is_la57(pmap)); 4624 4625 pml5index = pmap_pml5e_index(va); 4626 pml5 = &pmap->pm_pmltop[pml5index]; 4627 KASSERT((*pml5 & PG_V) == 0, 4628 ("pmap %p va %#lx pml5 %#lx", pmap, va, *pml5)); 4629 *pml5 = VM_PAGE_TO_PHYS(m) | PG_U | PG_RW | PG_V | PG_A | PG_M; 4630 4631 if (pmap->pm_pmltopu != NULL && pml5index < NUPML5E) { 4632 MPASS(pmap->pm_ucr3 != PMAP_NO_CR3); 4633 *pml5 |= pg_nx; 4634 4635 pml5u = &pmap->pm_pmltopu[pml5index]; 4636 *pml5u = VM_PAGE_TO_PHYS(m) | PG_U | PG_RW | PG_V | 4637 PG_A | PG_M; 4638 } 4639 } else if (ptepindex >= NUPDE + NUPDPE) { 4640 pml4index = pmap_pml4e_index(va); 4641 /* Wire up a new PDPE page */ 4642 pml4 = pmap_allocpte_getpml4(pmap, lockp, va, true); 4643 if (pml4 == NULL) { 4644 pmap_free_pt_page(pmap, m, true); 4645 return (NULL); 4646 } 4647 KASSERT((*pml4 & PG_V) == 0, 4648 ("pmap %p va %#lx pml4 %#lx", pmap, va, *pml4)); 4649 *pml4 = VM_PAGE_TO_PHYS(m) | PG_U | PG_RW | PG_V | PG_A | PG_M; 4650 4651 if (!pmap_is_la57(pmap) && pmap->pm_pmltopu != NULL && 4652 pml4index < NUPML4E) { 4653 MPASS(pmap->pm_ucr3 != PMAP_NO_CR3); 4654 4655 /* 4656 * PTI: Make all user-space mappings in the 4657 * kernel-mode page table no-execute so that 4658 * we detect any programming errors that leave 4659 * the kernel-mode page table active on return 4660 * to user space. 4661 */ 4662 *pml4 |= pg_nx; 4663 4664 pml4u = &pmap->pm_pmltopu[pml4index]; 4665 *pml4u = VM_PAGE_TO_PHYS(m) | PG_U | PG_RW | PG_V | 4666 PG_A | PG_M; 4667 } 4668 } else if (ptepindex >= NUPDE) { 4669 /* Wire up a new PDE page */ 4670 pdp = pmap_allocpte_getpdp(pmap, lockp, va, true); 4671 if (pdp == NULL) { 4672 pmap_free_pt_page(pmap, m, true); 4673 return (NULL); 4674 } 4675 KASSERT((*pdp & PG_V) == 0, 4676 ("pmap %p va %#lx pdp %#lx", pmap, va, *pdp)); 4677 *pdp = VM_PAGE_TO_PHYS(m) | PG_U | PG_RW | PG_V | PG_A | PG_M; 4678 } else { 4679 /* Wire up a new PTE page */ 4680 pdp = pmap_allocpte_getpdp(pmap, lockp, va, false); 4681 if (pdp == NULL) { 4682 pmap_free_pt_page(pmap, m, true); 4683 return (NULL); 4684 } 4685 if ((*pdp & PG_V) == 0) { 4686 /* Have to allocate a new pd, recurse */ 4687 if (pmap_allocpte_nosleep(pmap, pmap_pdpe_pindex(va), 4688 lockp, va) == NULL) { 4689 pmap_allocpte_free_unref(pmap, va, 4690 pmap_pml4e(pmap, va)); 4691 pmap_free_pt_page(pmap, m, true); 4692 return (NULL); 4693 } 4694 } else { 4695 /* Add reference to the pd page */ 4696 pdpg = PHYS_TO_VM_PAGE(*pdp & PG_FRAME); 4697 pdpg->ref_count++; 4698 } 4699 pd = PHYS_TO_DMAP(*pdp & PG_FRAME); 4700 4701 /* Now we know where the page directory page is */ 4702 pd = &pd[pmap_pde_index(va)]; 4703 KASSERT((*pd & PG_V) == 0, 4704 ("pmap %p va %#lx pd %#lx", pmap, va, *pd)); 4705 *pd = VM_PAGE_TO_PHYS(m) | PG_U | PG_RW | PG_V | PG_A | PG_M; 4706 } 4707 4708 return (m); 4709 } 4710 4711 /* 4712 * This routine is called if the desired page table page does not exist. 4713 * 4714 * If page table page allocation fails, this routine may sleep before 4715 * returning NULL. It sleeps only if a lock pointer was given. Sleep 4716 * occurs right before returning to the caller. This way, we never 4717 * drop pmap lock to sleep while a page table page has ref_count == 0, 4718 * which prevents the page from being freed under us. 4719 */ 4720 static vm_page_t 4721 pmap_allocpte_alloc(pmap_t pmap, vm_pindex_t ptepindex, struct rwlock **lockp, 4722 vm_offset_t va) 4723 { 4724 vm_page_t m; 4725 4726 m = pmap_allocpte_nosleep(pmap, ptepindex, lockp, va); 4727 if (m == NULL && lockp != NULL) { 4728 RELEASE_PV_LIST_LOCK(lockp); 4729 PMAP_UNLOCK(pmap); 4730 PMAP_ASSERT_NOT_IN_DI(); 4731 vm_wait(NULL); 4732 PMAP_LOCK(pmap); 4733 } 4734 return (m); 4735 } 4736 4737 static pd_entry_t * 4738 pmap_alloc_pde(pmap_t pmap, vm_offset_t va, vm_page_t *pdpgp, 4739 struct rwlock **lockp) 4740 { 4741 pdp_entry_t *pdpe, PG_V; 4742 pd_entry_t *pde; 4743 vm_page_t pdpg; 4744 vm_pindex_t pdpindex; 4745 4746 PG_V = pmap_valid_bit(pmap); 4747 4748 retry: 4749 pdpe = pmap_pdpe(pmap, va); 4750 if (pdpe != NULL && (*pdpe & PG_V) != 0) { 4751 pde = pmap_pdpe_to_pde(pdpe, va); 4752 if (va < VM_MAXUSER_ADDRESS) { 4753 /* Add a reference to the pd page. */ 4754 pdpg = PHYS_TO_VM_PAGE(*pdpe & PG_FRAME); 4755 pdpg->ref_count++; 4756 } else 4757 pdpg = NULL; 4758 } else if (va < VM_MAXUSER_ADDRESS) { 4759 /* Allocate a pd page. */ 4760 pdpindex = pmap_pde_pindex(va) >> NPDPEPGSHIFT; 4761 pdpg = pmap_allocpte_alloc(pmap, NUPDE + pdpindex, lockp, va); 4762 if (pdpg == NULL) { 4763 if (lockp != NULL) 4764 goto retry; 4765 else 4766 return (NULL); 4767 } 4768 pde = VM_PAGE_TO_DMAP(pdpg); 4769 pde = &pde[pmap_pde_index(va)]; 4770 } else 4771 panic("pmap_alloc_pde: missing page table page for va %#lx", 4772 va); 4773 *pdpgp = pdpg; 4774 return (pde); 4775 } 4776 4777 static vm_page_t 4778 pmap_allocpte(pmap_t pmap, vm_offset_t va, struct rwlock **lockp) 4779 { 4780 vm_pindex_t ptepindex; 4781 pd_entry_t *pd, PG_V; 4782 vm_page_t m; 4783 4784 PG_V = pmap_valid_bit(pmap); 4785 4786 /* 4787 * Calculate pagetable page index 4788 */ 4789 ptepindex = pmap_pde_pindex(va); 4790 retry: 4791 /* 4792 * Get the page directory entry 4793 */ 4794 pd = pmap_pde(pmap, va); 4795 4796 /* 4797 * This supports switching from a 2MB page to a 4798 * normal 4K page. 4799 */ 4800 if (pd != NULL && (*pd & (PG_PS | PG_V)) == (PG_PS | PG_V)) { 4801 if (!pmap_demote_pde_locked(pmap, pd, va, lockp)) { 4802 /* 4803 * Invalidation of the 2MB page mapping may have caused 4804 * the deallocation of the underlying PD page. 4805 */ 4806 pd = NULL; 4807 } 4808 } 4809 4810 /* 4811 * If the page table page is mapped, we just increment the 4812 * hold count, and activate it. 4813 */ 4814 if (pd != NULL && (*pd & PG_V) != 0) { 4815 m = PHYS_TO_VM_PAGE(*pd & PG_FRAME); 4816 m->ref_count++; 4817 } else { 4818 /* 4819 * Here if the pte page isn't mapped, or if it has been 4820 * deallocated. 4821 */ 4822 m = pmap_allocpte_alloc(pmap, ptepindex, lockp, va); 4823 if (m == NULL && lockp != NULL) 4824 goto retry; 4825 } 4826 return (m); 4827 } 4828 4829 /*************************************************** 4830 * Pmap allocation/deallocation routines. 4831 ***************************************************/ 4832 4833 /* 4834 * Release any resources held by the given physical map. 4835 * Called when a pmap initialized by pmap_pinit is being released. 4836 * Should only be called if the map contains no valid mappings. 4837 */ 4838 void 4839 pmap_release(pmap_t pmap) 4840 { 4841 vm_page_t m; 4842 int i; 4843 4844 KASSERT(vm_radix_is_empty(&pmap->pm_root), 4845 ("pmap_release: pmap %p has reserved page table page(s)", 4846 pmap)); 4847 KASSERT(CPU_EMPTY(&pmap->pm_active), 4848 ("releasing active pmap %p", pmap)); 4849 4850 m = DMAP_TO_VM_PAGE(pmap->pm_pmltop); 4851 4852 if (pmap_is_la57(pmap)) { 4853 for (i = NPML5EPG / 2; i < NPML5EPG; i++) 4854 pmap->pm_pmltop[i] = 0; 4855 } else { 4856 for (i = 0; i < NKPML4E; i++) /* KVA */ 4857 pmap->pm_pmltop[KPML4BASE + i] = 0; 4858 #ifdef KASAN 4859 for (i = 0; i < NKASANPML4E; i++) /* KASAN shadow map */ 4860 pmap->pm_pmltop[KASANPML4I + i] = 0; 4861 #endif 4862 #ifdef KMSAN 4863 for (i = 0; i < NKMSANSHADPML4E; i++) /* KMSAN shadow map */ 4864 pmap->pm_pmltop[KMSANSHADPML4I + i] = 0; 4865 for (i = 0; i < NKMSANORIGPML4E; i++) /* KMSAN shadow map */ 4866 pmap->pm_pmltop[KMSANORIGPML4I + i] = 0; 4867 #endif 4868 for (i = 0; i < ndmpdpphys; i++)/* Direct Map */ 4869 pmap->pm_pmltop[DMPML4I + i] = 0; 4870 pmap->pm_pmltop[PML4PML4I] = 0; /* Recursive Mapping */ 4871 for (i = 0; i < lm_ents; i++) /* Large Map */ 4872 pmap->pm_pmltop[LMSPML4I + i] = 0; 4873 } 4874 4875 pmap_free_pt_page(NULL, m, true); 4876 pmap_pt_page_count_pinit(pmap, -1); 4877 4878 if (pmap->pm_pmltopu != NULL) { 4879 m = DMAP_TO_VM_PAGE(pmap->pm_pmltopu); 4880 pmap_free_pt_page(NULL, m, false); 4881 pmap_pt_page_count_pinit(pmap, -1); 4882 } 4883 if (pmap->pm_type == PT_X86 && 4884 (cpu_stdext_feature2 & CPUID_STDEXT2_PKU) != 0) 4885 rangeset_fini(&pmap->pm_pkru); 4886 4887 KASSERT(pmap->pm_stats.resident_count == 0, 4888 ("pmap_release: pmap %p resident count %ld != 0", 4889 pmap, pmap->pm_stats.resident_count)); 4890 } 4891 4892 static int 4893 kvm_size(SYSCTL_HANDLER_ARGS) 4894 { 4895 unsigned long ksize = kva_layout.km_high - kva_layout.km_low; 4896 4897 return sysctl_handle_long(oidp, &ksize, 0, req); 4898 } 4899 SYSCTL_PROC(_vm, OID_AUTO, kvm_size, CTLTYPE_LONG | CTLFLAG_RD | CTLFLAG_MPSAFE, 4900 0, 0, kvm_size, "LU", 4901 "Size of KVM"); 4902 4903 static int 4904 kvm_free(SYSCTL_HANDLER_ARGS) 4905 { 4906 unsigned long kfree = kva_layout.km_high - kernel_vm_end; 4907 4908 return sysctl_handle_long(oidp, &kfree, 0, req); 4909 } 4910 SYSCTL_PROC(_vm, OID_AUTO, kvm_free, CTLTYPE_LONG | CTLFLAG_RD | CTLFLAG_MPSAFE, 4911 0, 0, kvm_free, "LU", 4912 "Amount of KVM free"); 4913 4914 #ifdef KMSAN 4915 static void 4916 pmap_kmsan_shadow_map_page_array(vm_paddr_t pdppa, vm_size_t size) 4917 { 4918 pdp_entry_t *pdpe; 4919 pd_entry_t *pde; 4920 pt_entry_t *pte; 4921 vm_paddr_t dummypa, dummypd, dummypt; 4922 int i, npde, npdpg; 4923 4924 npdpg = howmany(size, NBPDP); 4925 npde = size / NBPDR; 4926 4927 dummypa = vm_phys_early_alloc(-1, PAGE_SIZE); 4928 pagezero(PHYS_TO_DMAP(dummypa)); 4929 4930 dummypt = vm_phys_early_alloc(-1, PAGE_SIZE); 4931 pagezero(PHYS_TO_DMAP(dummypt)); 4932 dummypd = vm_phys_early_alloc(-1, PAGE_SIZE * npdpg); 4933 for (i = 0; i < npdpg; i++) 4934 pagezero(PHYS_TO_DMAP(dummypd + ptoa(i))); 4935 4936 pte = PHYS_TO_DMAP(dummypt); 4937 for (i = 0; i < NPTEPG; i++) 4938 pte[i] = (pt_entry_t)(dummypa | X86_PG_V | X86_PG_RW | 4939 X86_PG_A | X86_PG_M | pg_nx); 4940 4941 pde = PHYS_TO_DMAP(dummypd); 4942 for (i = 0; i < npde; i++) 4943 pde[i] = (pd_entry_t)(dummypt | X86_PG_V | X86_PG_RW | pg_nx); 4944 4945 pdpe = PHYS_TO_DMAP(pdppa); 4946 for (i = 0; i < npdpg; i++) 4947 pdpe[i] = (pdp_entry_t)(dummypd + ptoa(i) | X86_PG_V | 4948 X86_PG_RW | pg_nx); 4949 } 4950 4951 static void 4952 pmap_kmsan_page_array_startup(vm_offset_t start, vm_offset_t end) 4953 { 4954 vm_size_t size; 4955 4956 KASSERT(start % NBPDP == 0, ("unaligned page array start address")); 4957 4958 /* 4959 * The end of the page array's KVA region is 2MB aligned, see 4960 * kmem_init(). 4961 */ 4962 size = round_2mpage(end) - start; 4963 pmap_kmsan_shadow_map_page_array(KMSANSHADPDPphys, size); 4964 pmap_kmsan_shadow_map_page_array(KMSANORIGPDPphys, size); 4965 } 4966 #endif 4967 4968 /* 4969 * Allocate physical memory for the vm_page array and map it into KVA, 4970 * attempting to back the vm_pages with domain-local memory. 4971 */ 4972 void 4973 pmap_page_array_startup(long pages) 4974 { 4975 pdp_entry_t *pdpe; 4976 pd_entry_t *pde, newpdir; 4977 vm_offset_t va, start, end; 4978 vm_paddr_t pa; 4979 long pfn; 4980 int domain, i; 4981 4982 vm_page_array_size = pages; 4983 4984 start = kva_layout.km_low; 4985 end = start + pages * sizeof(struct vm_page); 4986 for (va = start; va < end; va += NBPDR) { 4987 pfn = first_page + (va - start) / sizeof(struct vm_page); 4988 domain = vm_phys_domain(ptoa(pfn)); 4989 pdpe = pmap_pdpe(kernel_pmap, va); 4990 if ((*pdpe & X86_PG_V) == 0) { 4991 pa = vm_phys_early_alloc(domain, PAGE_SIZE); 4992 dump_add_page(pa); 4993 pagezero(PHYS_TO_DMAP(pa)); 4994 *pdpe = (pdp_entry_t)(pa | X86_PG_V | X86_PG_RW | 4995 X86_PG_A | X86_PG_M); 4996 } 4997 pde = pmap_pdpe_to_pde(pdpe, va); 4998 if ((*pde & X86_PG_V) != 0) 4999 panic("Unexpected pde"); 5000 pa = vm_phys_early_alloc(domain, NBPDR); 5001 for (i = 0; i < NPDEPG; i++) 5002 dump_add_page(pa + i * PAGE_SIZE); 5003 newpdir = (pd_entry_t)(pa | X86_PG_V | X86_PG_RW | X86_PG_A | 5004 X86_PG_M | PG_PS | pg_g | pg_nx); 5005 pde_store(pde, newpdir); 5006 } 5007 vm_page_array = (vm_page_t)start; 5008 5009 #ifdef KMSAN 5010 pmap_kmsan_page_array_startup(start, end); 5011 #endif 5012 } 5013 5014 /* 5015 * grow the number of kernel page table entries, if needed 5016 */ 5017 static int 5018 pmap_growkernel_nopanic(vm_offset_t addr) 5019 { 5020 vm_paddr_t paddr; 5021 vm_page_t nkpg; 5022 pd_entry_t *pde, newpdir; 5023 pdp_entry_t *pdpe; 5024 vm_offset_t end; 5025 int rv; 5026 5027 TSENTER(); 5028 mtx_assert(&kernel_map->system_mtx, MA_OWNED); 5029 rv = KERN_SUCCESS; 5030 5031 /* 5032 * The kernel map covers two distinct regions of KVA: that used 5033 * for dynamic kernel memory allocations, and the uppermost 2GB 5034 * of the virtual address space. The latter is used to map the 5035 * kernel and loadable kernel modules. This scheme enables the 5036 * use of a special code generation model for kernel code which 5037 * takes advantage of compact addressing modes in machine code. 5038 * 5039 * Both regions grow upwards; to avoid wasting memory, the gap 5040 * in between is unmapped. If "addr" is above "KERNBASE", the 5041 * kernel's region is grown, otherwise the kmem region is grown. 5042 * 5043 * The correctness of this action is based on the following 5044 * argument: vm_map_insert() allocates contiguous ranges of the 5045 * kernel virtual address space. It calls this function if a range 5046 * ends after "kernel_vm_end". If the kernel is mapped between 5047 * "kernel_vm_end" and "addr", then the range cannot begin at 5048 * "kernel_vm_end". In fact, its beginning address cannot be less 5049 * than the kernel. Thus, there is no immediate need to allocate 5050 * any new kernel page table pages between "kernel_vm_end" and 5051 * "KERNBASE". 5052 */ 5053 if (KERNBASE < addr) { 5054 end = KERNBASE + nkpt * NBPDR; 5055 if (end == 0) { 5056 TSEXIT(); 5057 return (rv); 5058 } 5059 } else { 5060 end = kernel_vm_end; 5061 } 5062 5063 addr = roundup2(addr, NBPDR); 5064 if (addr - 1 >= vm_map_max(kernel_map)) 5065 addr = vm_map_max(kernel_map); 5066 if (addr <= end) { 5067 /* 5068 * The grown region is already mapped, so there is 5069 * nothing to do. 5070 */ 5071 TSEXIT(); 5072 return (rv); 5073 } 5074 5075 kasan_shadow_map(end, addr - end); 5076 kmsan_shadow_map(end, addr - end); 5077 while (end < addr) { 5078 pdpe = pmap_pdpe(kernel_pmap, end); 5079 if ((*pdpe & X86_PG_V) == 0) { 5080 nkpg = pmap_alloc_pt_page(kernel_pmap, 5081 pmap_pdpe_pindex(end), VM_ALLOC_INTERRUPT | 5082 VM_ALLOC_NOFREE | VM_ALLOC_WIRED | VM_ALLOC_ZERO); 5083 if (nkpg == NULL) { 5084 rv = KERN_RESOURCE_SHORTAGE; 5085 break; 5086 } 5087 paddr = VM_PAGE_TO_PHYS(nkpg); 5088 *pdpe = (pdp_entry_t)(paddr | X86_PG_V | X86_PG_RW | 5089 X86_PG_A | X86_PG_M); 5090 continue; /* try again */ 5091 } 5092 pde = pmap_pdpe_to_pde(pdpe, end); 5093 if ((*pde & X86_PG_V) != 0) { 5094 end = (end + NBPDR) & ~PDRMASK; 5095 if (end - 1 >= vm_map_max(kernel_map)) { 5096 end = vm_map_max(kernel_map); 5097 break; 5098 } 5099 continue; 5100 } 5101 5102 nkpg = pmap_alloc_pt_page(kernel_pmap, pmap_pde_pindex(end), 5103 VM_ALLOC_INTERRUPT | VM_ALLOC_NOFREE | VM_ALLOC_WIRED | 5104 VM_ALLOC_ZERO); 5105 if (nkpg == NULL) { 5106 rv = KERN_RESOURCE_SHORTAGE; 5107 break; 5108 } 5109 5110 paddr = VM_PAGE_TO_PHYS(nkpg); 5111 newpdir = paddr | X86_PG_V | X86_PG_RW | X86_PG_A | X86_PG_M; 5112 pde_store(pde, newpdir); 5113 5114 end = (end + NBPDR) & ~PDRMASK; 5115 if (end - 1 >= vm_map_max(kernel_map)) { 5116 end = vm_map_max(kernel_map); 5117 break; 5118 } 5119 } 5120 5121 if (end <= KERNBASE) 5122 kernel_vm_end = end; 5123 else 5124 nkpt = howmany(end - KERNBASE, NBPDR); 5125 TSEXIT(); 5126 return (rv); 5127 } 5128 5129 int 5130 pmap_growkernel(vm_offset_t addr) 5131 { 5132 int rv; 5133 5134 rv = pmap_growkernel_nopanic(addr); 5135 if (rv != KERN_SUCCESS && pmap_growkernel_panic) 5136 panic("pmap_growkernel: no memory to grow kernel"); 5137 return (rv); 5138 } 5139 5140 /*************************************************** 5141 * page management routines. 5142 ***************************************************/ 5143 5144 static const uint64_t pc_freemask[_NPCM] = { 5145 [0 ... _NPCM - 2] = PC_FREEN, 5146 [_NPCM - 1] = PC_FREEL 5147 }; 5148 5149 #ifdef PV_STATS 5150 5151 static COUNTER_U64_DEFINE_EARLY(pc_chunk_count); 5152 SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, pc_chunk_count, CTLFLAG_RD, 5153 &pc_chunk_count, "Current number of pv entry cnunks"); 5154 5155 static COUNTER_U64_DEFINE_EARLY(pc_chunk_allocs); 5156 SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, pc_chunk_allocs, CTLFLAG_RD, 5157 &pc_chunk_allocs, "Total number of pv entry chunks allocated"); 5158 5159 static COUNTER_U64_DEFINE_EARLY(pc_chunk_frees); 5160 SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, pc_chunk_frees, CTLFLAG_RD, 5161 &pc_chunk_frees, "Total number of pv entry chunks freed"); 5162 5163 static COUNTER_U64_DEFINE_EARLY(pc_chunk_tryfail); 5164 SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, pc_chunk_tryfail, CTLFLAG_RD, 5165 &pc_chunk_tryfail, 5166 "Number of failed attempts to get a pv entry chunk page"); 5167 5168 static COUNTER_U64_DEFINE_EARLY(pv_entry_frees); 5169 SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, pv_entry_frees, CTLFLAG_RD, 5170 &pv_entry_frees, "Total number of pv entries freed"); 5171 5172 static COUNTER_U64_DEFINE_EARLY(pv_entry_allocs); 5173 SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, pv_entry_allocs, CTLFLAG_RD, 5174 &pv_entry_allocs, "Total number of pv entries allocated"); 5175 5176 static COUNTER_U64_DEFINE_EARLY(pv_entry_count); 5177 SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, pv_entry_count, CTLFLAG_RD, 5178 &pv_entry_count, "Current number of pv entries"); 5179 5180 static COUNTER_U64_DEFINE_EARLY(pv_entry_spare); 5181 SYSCTL_COUNTER_U64(_vm_pmap, OID_AUTO, pv_entry_spare, CTLFLAG_RD, 5182 &pv_entry_spare, "Current number of spare pv entries"); 5183 #endif 5184 5185 static void 5186 reclaim_pv_chunk_leave_pmap(pmap_t pmap, pmap_t locked_pmap, bool start_di) 5187 { 5188 5189 if (pmap == NULL) 5190 return; 5191 pmap_invalidate_all(pmap); 5192 if (pmap != locked_pmap) 5193 PMAP_UNLOCK(pmap); 5194 if (start_di) 5195 pmap_delayed_invl_finish(); 5196 } 5197 5198 /* 5199 * We are in a serious low memory condition. Resort to 5200 * drastic measures to free some pages so we can allocate 5201 * another pv entry chunk. 5202 * 5203 * Returns NULL if PV entries were reclaimed from the specified pmap. 5204 * 5205 * We do not, however, unmap 2mpages because subsequent accesses will 5206 * allocate per-page pv entries until repromotion occurs, thereby 5207 * exacerbating the shortage of free pv entries. 5208 */ 5209 static vm_page_t 5210 reclaim_pv_chunk_domain(pmap_t locked_pmap, struct rwlock **lockp, int domain) 5211 { 5212 struct pv_chunks_list *pvc; 5213 struct pv_chunk *pc, *pc_marker, *pc_marker_end; 5214 struct pv_chunk_header pc_marker_b, pc_marker_end_b; 5215 pd_entry_t *pde; 5216 pmap_t next_pmap, pmap; 5217 pt_entry_t *pte, tpte; 5218 pt_entry_t PG_G, PG_A, PG_M, PG_RW; 5219 pv_entry_t pv; 5220 vm_offset_t va; 5221 vm_page_t m, m_pc; 5222 struct spglist free; 5223 uint64_t inuse; 5224 int bit, field, freed; 5225 bool start_di, restart; 5226 5227 PMAP_LOCK_ASSERT(locked_pmap, MA_OWNED); 5228 KASSERT(lockp != NULL, ("reclaim_pv_chunk: lockp is NULL")); 5229 pmap = NULL; 5230 m_pc = NULL; 5231 PG_G = PG_A = PG_M = PG_RW = 0; 5232 SLIST_INIT(&free); 5233 bzero(&pc_marker_b, sizeof(pc_marker_b)); 5234 bzero(&pc_marker_end_b, sizeof(pc_marker_end_b)); 5235 pc_marker = (struct pv_chunk *)&pc_marker_b; 5236 pc_marker_end = (struct pv_chunk *)&pc_marker_end_b; 5237 5238 /* 5239 * A delayed invalidation block should already be active if 5240 * pmap_advise() or pmap_remove() called this function by way 5241 * of pmap_demote_pde_locked(). 5242 */ 5243 start_di = pmap_not_in_di(); 5244 5245 pvc = &pv_chunks[domain]; 5246 mtx_lock(&pvc->pvc_lock); 5247 pvc->active_reclaims++; 5248 TAILQ_INSERT_HEAD(&pvc->pvc_list, pc_marker, pc_lru); 5249 TAILQ_INSERT_TAIL(&pvc->pvc_list, pc_marker_end, pc_lru); 5250 while ((pc = TAILQ_NEXT(pc_marker, pc_lru)) != pc_marker_end && 5251 SLIST_EMPTY(&free)) { 5252 next_pmap = pc->pc_pmap; 5253 if (next_pmap == NULL) { 5254 /* 5255 * The next chunk is a marker. However, it is 5256 * not our marker, so active_reclaims must be 5257 * > 1. Consequently, the next_chunk code 5258 * will not rotate the pv_chunks list. 5259 */ 5260 goto next_chunk; 5261 } 5262 mtx_unlock(&pvc->pvc_lock); 5263 5264 /* 5265 * A pv_chunk can only be removed from the pc_lru list 5266 * when both pc_chunks_mutex is owned and the 5267 * corresponding pmap is locked. 5268 */ 5269 if (pmap != next_pmap) { 5270 restart = false; 5271 reclaim_pv_chunk_leave_pmap(pmap, locked_pmap, 5272 start_di); 5273 pmap = next_pmap; 5274 /* Avoid deadlock and lock recursion. */ 5275 if (pmap > locked_pmap) { 5276 RELEASE_PV_LIST_LOCK(lockp); 5277 PMAP_LOCK(pmap); 5278 if (start_di) 5279 pmap_delayed_invl_start(); 5280 mtx_lock(&pvc->pvc_lock); 5281 restart = true; 5282 } else if (pmap != locked_pmap) { 5283 if (PMAP_TRYLOCK(pmap)) { 5284 if (start_di) 5285 pmap_delayed_invl_start(); 5286 mtx_lock(&pvc->pvc_lock); 5287 restart = true; 5288 } else { 5289 pmap = NULL; /* pmap is not locked */ 5290 mtx_lock(&pvc->pvc_lock); 5291 pc = TAILQ_NEXT(pc_marker, pc_lru); 5292 if (pc == NULL || 5293 pc->pc_pmap != next_pmap) 5294 continue; 5295 goto next_chunk; 5296 } 5297 } else if (start_di) 5298 pmap_delayed_invl_start(); 5299 PG_G = pmap_global_bit(pmap); 5300 PG_A = pmap_accessed_bit(pmap); 5301 PG_M = pmap_modified_bit(pmap); 5302 PG_RW = pmap_rw_bit(pmap); 5303 if (restart) 5304 continue; 5305 } 5306 5307 /* 5308 * Destroy every non-wired, 4 KB page mapping in the chunk. 5309 */ 5310 freed = 0; 5311 for (field = 0; field < _NPCM; field++) { 5312 for (inuse = ~pc->pc_map[field] & pc_freemask[field]; 5313 inuse != 0; inuse &= ~(1UL << bit)) { 5314 bit = bsfq(inuse); 5315 pv = &pc->pc_pventry[field * 64 + bit]; 5316 va = pv->pv_va; 5317 pde = pmap_pde(pmap, va); 5318 if ((*pde & PG_PS) != 0) 5319 continue; 5320 pte = pmap_pde_to_pte(pde, va); 5321 if ((*pte & PG_W) != 0) 5322 continue; 5323 tpte = pte_load_clear(pte); 5324 if ((tpte & PG_G) != 0) 5325 pmap_invalidate_page(pmap, va); 5326 m = PHYS_TO_VM_PAGE(tpte & PG_FRAME); 5327 if ((tpte & (PG_M | PG_RW)) == (PG_M | PG_RW)) 5328 vm_page_dirty(m); 5329 if ((tpte & PG_A) != 0) 5330 vm_page_aflag_set(m, PGA_REFERENCED); 5331 CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m); 5332 TAILQ_REMOVE(&m->md.pv_list, pv, pv_next); 5333 m->md.pv_gen++; 5334 if (!pmap_page_is_mapped_locked(m)) 5335 vm_page_aflag_clear(m, PGA_WRITEABLE); 5336 pmap_delayed_invl_page(m); 5337 pc->pc_map[field] |= 1UL << bit; 5338 pmap_unuse_pt(pmap, va, *pde, &free); 5339 freed++; 5340 } 5341 } 5342 if (freed == 0) { 5343 mtx_lock(&pvc->pvc_lock); 5344 goto next_chunk; 5345 } 5346 /* Every freed mapping is for a 4 KB page. */ 5347 pmap_resident_count_adj(pmap, -freed); 5348 PV_STAT(counter_u64_add(pv_entry_frees, freed)); 5349 PV_STAT(counter_u64_add(pv_entry_spare, freed)); 5350 PV_STAT(counter_u64_add(pv_entry_count, -freed)); 5351 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 5352 if (pc_is_free(pc)) { 5353 PV_STAT(counter_u64_add(pv_entry_spare, -_NPCPV)); 5354 PV_STAT(counter_u64_add(pc_chunk_count, -1)); 5355 PV_STAT(counter_u64_add(pc_chunk_frees, 1)); 5356 /* Entire chunk is free; return it. */ 5357 m_pc = DMAP_TO_VM_PAGE(pc); 5358 dump_drop_page(m_pc->phys_addr); 5359 mtx_lock(&pvc->pvc_lock); 5360 TAILQ_REMOVE(&pvc->pvc_list, pc, pc_lru); 5361 break; 5362 } 5363 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list); 5364 mtx_lock(&pvc->pvc_lock); 5365 /* One freed pv entry in locked_pmap is sufficient. */ 5366 if (pmap == locked_pmap) 5367 break; 5368 next_chunk: 5369 TAILQ_REMOVE(&pvc->pvc_list, pc_marker, pc_lru); 5370 TAILQ_INSERT_AFTER(&pvc->pvc_list, pc, pc_marker, pc_lru); 5371 if (pvc->active_reclaims == 1 && pmap != NULL) { 5372 /* 5373 * Rotate the pv chunks list so that we do not 5374 * scan the same pv chunks that could not be 5375 * freed (because they contained a wired 5376 * and/or superpage mapping) on every 5377 * invocation of reclaim_pv_chunk(). 5378 */ 5379 while ((pc = TAILQ_FIRST(&pvc->pvc_list)) != pc_marker) { 5380 MPASS(pc->pc_pmap != NULL); 5381 TAILQ_REMOVE(&pvc->pvc_list, pc, pc_lru); 5382 TAILQ_INSERT_TAIL(&pvc->pvc_list, pc, pc_lru); 5383 } 5384 } 5385 } 5386 TAILQ_REMOVE(&pvc->pvc_list, pc_marker, pc_lru); 5387 TAILQ_REMOVE(&pvc->pvc_list, pc_marker_end, pc_lru); 5388 pvc->active_reclaims--; 5389 mtx_unlock(&pvc->pvc_lock); 5390 reclaim_pv_chunk_leave_pmap(pmap, locked_pmap, start_di); 5391 if (m_pc == NULL && !SLIST_EMPTY(&free)) { 5392 m_pc = SLIST_FIRST(&free); 5393 SLIST_REMOVE_HEAD(&free, plinks.s.ss); 5394 /* Recycle a freed page table page. */ 5395 m_pc->ref_count = 1; 5396 } 5397 vm_page_free_pages_toq(&free, true); 5398 return (m_pc); 5399 } 5400 5401 static vm_page_t 5402 reclaim_pv_chunk(pmap_t locked_pmap, struct rwlock **lockp) 5403 { 5404 vm_page_t m; 5405 int i, domain; 5406 5407 domain = PCPU_GET(domain); 5408 for (i = 0; i < vm_ndomains; i++) { 5409 m = reclaim_pv_chunk_domain(locked_pmap, lockp, domain); 5410 if (m != NULL) 5411 break; 5412 domain = (domain + 1) % vm_ndomains; 5413 } 5414 5415 return (m); 5416 } 5417 5418 /* 5419 * free the pv_entry back to the free list 5420 */ 5421 static void 5422 free_pv_entry(pmap_t pmap, pv_entry_t pv) 5423 { 5424 struct pv_chunk *pc; 5425 int idx, field, bit; 5426 5427 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 5428 PV_STAT(counter_u64_add(pv_entry_frees, 1)); 5429 PV_STAT(counter_u64_add(pv_entry_spare, 1)); 5430 PV_STAT(counter_u64_add(pv_entry_count, -1)); 5431 pc = pv_to_chunk(pv); 5432 idx = pv - &pc->pc_pventry[0]; 5433 field = idx / 64; 5434 bit = idx % 64; 5435 pc->pc_map[field] |= 1ul << bit; 5436 if (!pc_is_free(pc)) { 5437 /* 98% of the time, pc is already at the head of the list. */ 5438 if (__predict_false(pc != TAILQ_FIRST(&pmap->pm_pvchunk))) { 5439 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 5440 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list); 5441 } 5442 return; 5443 } 5444 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 5445 free_pv_chunk(pc); 5446 } 5447 5448 static void 5449 free_pv_chunk_dequeued(struct pv_chunk *pc) 5450 { 5451 vm_page_t m; 5452 5453 PV_STAT(counter_u64_add(pv_entry_spare, -_NPCPV)); 5454 PV_STAT(counter_u64_add(pc_chunk_count, -1)); 5455 PV_STAT(counter_u64_add(pc_chunk_frees, 1)); 5456 counter_u64_add(pv_page_count, -1); 5457 /* entire chunk is free, return it */ 5458 m = DMAP_TO_VM_PAGE(pc); 5459 dump_drop_page(m->phys_addr); 5460 vm_page_unwire_noq(m); 5461 vm_page_free(m); 5462 } 5463 5464 static void 5465 free_pv_chunk(struct pv_chunk *pc) 5466 { 5467 struct pv_chunks_list *pvc; 5468 5469 pvc = &pv_chunks[pc_to_domain(pc)]; 5470 mtx_lock(&pvc->pvc_lock); 5471 TAILQ_REMOVE(&pvc->pvc_list, pc, pc_lru); 5472 mtx_unlock(&pvc->pvc_lock); 5473 free_pv_chunk_dequeued(pc); 5474 } 5475 5476 static void 5477 free_pv_chunk_batch(struct pv_chunklist *batch) 5478 { 5479 struct pv_chunks_list *pvc; 5480 struct pv_chunk *pc, *npc; 5481 int i; 5482 5483 for (i = 0; i < vm_ndomains; i++) { 5484 if (TAILQ_EMPTY(&batch[i])) 5485 continue; 5486 pvc = &pv_chunks[i]; 5487 mtx_lock(&pvc->pvc_lock); 5488 TAILQ_FOREACH(pc, &batch[i], pc_list) { 5489 TAILQ_REMOVE(&pvc->pvc_list, pc, pc_lru); 5490 } 5491 mtx_unlock(&pvc->pvc_lock); 5492 } 5493 5494 for (i = 0; i < vm_ndomains; i++) { 5495 TAILQ_FOREACH_SAFE(pc, &batch[i], pc_list, npc) { 5496 free_pv_chunk_dequeued(pc); 5497 } 5498 } 5499 } 5500 5501 /* 5502 * Returns a new PV entry, allocating a new PV chunk from the system when 5503 * needed. If this PV chunk allocation fails and a PV list lock pointer was 5504 * given, a PV chunk is reclaimed from an arbitrary pmap. Otherwise, NULL is 5505 * returned. 5506 * 5507 * The given PV list lock may be released. 5508 */ 5509 static pv_entry_t 5510 get_pv_entry(pmap_t pmap, struct rwlock **lockp) 5511 { 5512 struct pv_chunks_list *pvc; 5513 int bit, field; 5514 pv_entry_t pv; 5515 struct pv_chunk *pc; 5516 vm_page_t m; 5517 5518 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 5519 PV_STAT(counter_u64_add(pv_entry_allocs, 1)); 5520 retry: 5521 pc = TAILQ_FIRST(&pmap->pm_pvchunk); 5522 if (pc != NULL) { 5523 for (field = 0; field < _NPCM; field++) { 5524 if (pc->pc_map[field]) { 5525 bit = bsfq(pc->pc_map[field]); 5526 break; 5527 } 5528 } 5529 if (field < _NPCM) { 5530 pv = &pc->pc_pventry[field * 64 + bit]; 5531 pc->pc_map[field] &= ~(1ul << bit); 5532 /* If this was the last item, move it to tail */ 5533 if (pc_is_full(pc)) { 5534 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 5535 TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, 5536 pc_list); 5537 } 5538 PV_STAT(counter_u64_add(pv_entry_count, 1)); 5539 PV_STAT(counter_u64_add(pv_entry_spare, -1)); 5540 return (pv); 5541 } 5542 } 5543 /* No free items, allocate another chunk */ 5544 m = vm_page_alloc_noobj(VM_ALLOC_WIRED); 5545 if (m == NULL) { 5546 if (lockp == NULL) { 5547 PV_STAT(counter_u64_add(pc_chunk_tryfail, 1)); 5548 return (NULL); 5549 } 5550 m = reclaim_pv_chunk(pmap, lockp); 5551 if (m == NULL) 5552 goto retry; 5553 } else 5554 counter_u64_add(pv_page_count, 1); 5555 PV_STAT(counter_u64_add(pc_chunk_count, 1)); 5556 PV_STAT(counter_u64_add(pc_chunk_allocs, 1)); 5557 dump_add_page(m->phys_addr); 5558 pc = PHYS_TO_DMAP(m->phys_addr); 5559 pc->pc_pmap = pmap; 5560 pc->pc_map[0] = PC_FREEN & ~1ul; /* preallocated bit 0 */ 5561 pc->pc_map[1] = PC_FREEN; 5562 pc->pc_map[2] = PC_FREEL; 5563 pvc = &pv_chunks[vm_page_domain(m)]; 5564 mtx_lock(&pvc->pvc_lock); 5565 TAILQ_INSERT_TAIL(&pvc->pvc_list, pc, pc_lru); 5566 mtx_unlock(&pvc->pvc_lock); 5567 pv = &pc->pc_pventry[0]; 5568 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list); 5569 PV_STAT(counter_u64_add(pv_entry_count, 1)); 5570 PV_STAT(counter_u64_add(pv_entry_spare, _NPCPV - 1)); 5571 return (pv); 5572 } 5573 5574 /* 5575 * Returns the number of one bits within the given PV chunk map. 5576 * 5577 * The erratas for Intel processors state that "POPCNT Instruction May 5578 * Take Longer to Execute Than Expected". It is believed that the 5579 * issue is the spurious dependency on the destination register. 5580 * Provide a hint to the register rename logic that the destination 5581 * value is overwritten, by clearing it, as suggested in the 5582 * optimization manual. It should be cheap for unaffected processors 5583 * as well. 5584 * 5585 * Reference numbers for erratas are 5586 * 4th Gen Core: HSD146 5587 * 5th Gen Core: BDM85 5588 * 6th Gen Core: SKL029 5589 */ 5590 static int 5591 popcnt_pc_map_pq(uint64_t *map) 5592 { 5593 u_long result, tmp; 5594 5595 __asm __volatile("xorl %k0,%k0;popcntq %2,%0;" 5596 "xorl %k1,%k1;popcntq %3,%1;addl %k1,%k0;" 5597 "xorl %k1,%k1;popcntq %4,%1;addl %k1,%k0" 5598 : "=&r" (result), "=&r" (tmp) 5599 : "m" (map[0]), "m" (map[1]), "m" (map[2])); 5600 return (result); 5601 } 5602 5603 /* 5604 * Ensure that the number of spare PV entries in the specified pmap meets or 5605 * exceeds the given count, "needed". 5606 * 5607 * The given PV list lock may be released. 5608 */ 5609 static void 5610 reserve_pv_entries(pmap_t pmap, int needed, struct rwlock **lockp) 5611 { 5612 struct pv_chunks_list *pvc; 5613 struct pch new_tail[PMAP_MEMDOM]; 5614 struct pv_chunk *pc; 5615 vm_page_t m; 5616 int avail, free, i; 5617 bool reclaimed; 5618 5619 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 5620 KASSERT(lockp != NULL, ("reserve_pv_entries: lockp is NULL")); 5621 5622 /* 5623 * Newly allocated PV chunks must be stored in a private list until 5624 * the required number of PV chunks have been allocated. Otherwise, 5625 * reclaim_pv_chunk() could recycle one of these chunks. In 5626 * contrast, these chunks must be added to the pmap upon allocation. 5627 */ 5628 for (i = 0; i < PMAP_MEMDOM; i++) 5629 TAILQ_INIT(&new_tail[i]); 5630 retry: 5631 avail = 0; 5632 TAILQ_FOREACH(pc, &pmap->pm_pvchunk, pc_list) { 5633 #ifndef __POPCNT__ 5634 if ((cpu_feature2 & CPUID2_POPCNT) == 0) 5635 bit_count((bitstr_t *)pc->pc_map, 0, 5636 sizeof(pc->pc_map) * NBBY, &free); 5637 else 5638 #endif 5639 free = popcnt_pc_map_pq(pc->pc_map); 5640 if (free == 0) 5641 break; 5642 avail += free; 5643 if (avail >= needed) 5644 break; 5645 } 5646 for (reclaimed = false; avail < needed; avail += _NPCPV) { 5647 m = vm_page_alloc_noobj(VM_ALLOC_WIRED); 5648 if (m == NULL) { 5649 m = reclaim_pv_chunk(pmap, lockp); 5650 if (m == NULL) 5651 goto retry; 5652 reclaimed = true; 5653 } else 5654 counter_u64_add(pv_page_count, 1); 5655 PV_STAT(counter_u64_add(pc_chunk_count, 1)); 5656 PV_STAT(counter_u64_add(pc_chunk_allocs, 1)); 5657 dump_add_page(m->phys_addr); 5658 pc = PHYS_TO_DMAP(m->phys_addr); 5659 pc->pc_pmap = pmap; 5660 pc->pc_map[0] = PC_FREEN; 5661 pc->pc_map[1] = PC_FREEN; 5662 pc->pc_map[2] = PC_FREEL; 5663 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list); 5664 TAILQ_INSERT_TAIL(&new_tail[vm_page_domain(m)], pc, pc_lru); 5665 PV_STAT(counter_u64_add(pv_entry_spare, _NPCPV)); 5666 5667 /* 5668 * The reclaim might have freed a chunk from the current pmap. 5669 * If that chunk contained available entries, we need to 5670 * re-count the number of available entries. 5671 */ 5672 if (reclaimed) 5673 goto retry; 5674 } 5675 for (i = 0; i < vm_ndomains; i++) { 5676 if (TAILQ_EMPTY(&new_tail[i])) 5677 continue; 5678 pvc = &pv_chunks[i]; 5679 mtx_lock(&pvc->pvc_lock); 5680 TAILQ_CONCAT(&pvc->pvc_list, &new_tail[i], pc_lru); 5681 mtx_unlock(&pvc->pvc_lock); 5682 } 5683 } 5684 5685 /* 5686 * First find and then remove the pv entry for the specified pmap and virtual 5687 * address from the specified pv list. Returns the pv entry if found and NULL 5688 * otherwise. This operation can be performed on pv lists for either 4KB or 5689 * 2MB page mappings. 5690 */ 5691 static __inline pv_entry_t 5692 pmap_pvh_remove(struct md_page *pvh, pmap_t pmap, vm_offset_t va) 5693 { 5694 pv_entry_t pv; 5695 5696 TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) { 5697 if (pmap == PV_PMAP(pv) && va == pv->pv_va) { 5698 TAILQ_REMOVE(&pvh->pv_list, pv, pv_next); 5699 pvh->pv_gen++; 5700 break; 5701 } 5702 } 5703 return (pv); 5704 } 5705 5706 /* 5707 * After demotion from a 2MB page mapping to 512 4KB page mappings, 5708 * destroy the pv entry for the 2MB page mapping and reinstantiate the pv 5709 * entries for each of the 4KB page mappings. 5710 */ 5711 static void 5712 pmap_pv_demote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa, 5713 struct rwlock **lockp) 5714 { 5715 struct md_page *pvh; 5716 struct pv_chunk *pc; 5717 pv_entry_t pv; 5718 vm_offset_t va_last; 5719 vm_page_t m; 5720 int bit, field; 5721 5722 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 5723 KASSERT((pa & PDRMASK) == 0, 5724 ("pmap_pv_demote_pde: pa is not 2mpage aligned")); 5725 CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa); 5726 5727 /* 5728 * Transfer the 2mpage's pv entry for this mapping to the first 5729 * page's pv list. Once this transfer begins, the pv list lock 5730 * must not be released until the last pv entry is reinstantiated. 5731 */ 5732 pvh = pa_to_pvh(pa); 5733 va = trunc_2mpage(va); 5734 pv = pmap_pvh_remove(pvh, pmap, va); 5735 KASSERT(pv != NULL, ("pmap_pv_demote_pde: pv not found")); 5736 m = PHYS_TO_VM_PAGE(pa); 5737 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next); 5738 m->md.pv_gen++; 5739 /* Instantiate the remaining NPTEPG - 1 pv entries. */ 5740 PV_STAT(counter_u64_add(pv_entry_allocs, NPTEPG - 1)); 5741 va_last = va + NBPDR - PAGE_SIZE; 5742 for (;;) { 5743 pc = TAILQ_FIRST(&pmap->pm_pvchunk); 5744 KASSERT(!pc_is_full(pc), ("pmap_pv_demote_pde: missing spare")); 5745 for (field = 0; field < _NPCM; field++) { 5746 while (pc->pc_map[field]) { 5747 bit = bsfq(pc->pc_map[field]); 5748 pc->pc_map[field] &= ~(1ul << bit); 5749 pv = &pc->pc_pventry[field * 64 + bit]; 5750 va += PAGE_SIZE; 5751 pv->pv_va = va; 5752 m++; 5753 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 5754 ("pmap_pv_demote_pde: page %p is not managed", m)); 5755 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next); 5756 m->md.pv_gen++; 5757 if (va == va_last) 5758 goto out; 5759 } 5760 } 5761 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 5762 TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list); 5763 } 5764 out: 5765 if (pc_is_full(pc)) { 5766 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 5767 TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list); 5768 } 5769 PV_STAT(counter_u64_add(pv_entry_count, NPTEPG - 1)); 5770 PV_STAT(counter_u64_add(pv_entry_spare, -(NPTEPG - 1))); 5771 } 5772 5773 #if VM_NRESERVLEVEL > 0 5774 /* 5775 * After promotion from 512 4KB page mappings to a single 2MB page mapping, 5776 * replace the many pv entries for the 4KB page mappings by a single pv entry 5777 * for the 2MB page mapping. 5778 */ 5779 static void 5780 pmap_pv_promote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa, 5781 struct rwlock **lockp) 5782 { 5783 struct md_page *pvh; 5784 pv_entry_t pv; 5785 vm_offset_t va_last; 5786 vm_page_t m; 5787 5788 KASSERT((pa & PDRMASK) == 0, 5789 ("pmap_pv_promote_pde: pa is not 2mpage aligned")); 5790 CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa); 5791 5792 /* 5793 * Transfer the first page's pv entry for this mapping to the 2mpage's 5794 * pv list. Aside from avoiding the cost of a call to get_pv_entry(), 5795 * a transfer avoids the possibility that get_pv_entry() calls 5796 * reclaim_pv_chunk() and that reclaim_pv_chunk() removes one of the 5797 * mappings that is being promoted. 5798 */ 5799 m = PHYS_TO_VM_PAGE(pa); 5800 va = trunc_2mpage(va); 5801 pv = pmap_pvh_remove(&m->md, pmap, va); 5802 KASSERT(pv != NULL, ("pmap_pv_promote_pde: pv not found")); 5803 pvh = pa_to_pvh(pa); 5804 TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_next); 5805 pvh->pv_gen++; 5806 /* Free the remaining NPTEPG - 1 pv entries. */ 5807 va_last = va + NBPDR - PAGE_SIZE; 5808 do { 5809 m++; 5810 va += PAGE_SIZE; 5811 pmap_pvh_free(&m->md, pmap, va); 5812 } while (va < va_last); 5813 } 5814 #endif /* VM_NRESERVLEVEL > 0 */ 5815 5816 /* 5817 * First find and then destroy the pv entry for the specified pmap and virtual 5818 * address. This operation can be performed on pv lists for either 4KB or 2MB 5819 * page mappings. 5820 */ 5821 static void 5822 pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va) 5823 { 5824 pv_entry_t pv; 5825 5826 pv = pmap_pvh_remove(pvh, pmap, va); 5827 KASSERT(pv != NULL, ("pmap_pvh_free: pv not found")); 5828 free_pv_entry(pmap, pv); 5829 } 5830 5831 /* 5832 * Conditionally create the PV entry for a 4KB page mapping if the required 5833 * memory can be allocated without resorting to reclamation. 5834 */ 5835 static bool 5836 pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, vm_page_t m, 5837 struct rwlock **lockp) 5838 { 5839 pv_entry_t pv; 5840 5841 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 5842 /* Pass NULL instead of the lock pointer to disable reclamation. */ 5843 if ((pv = get_pv_entry(pmap, NULL)) != NULL) { 5844 pv->pv_va = va; 5845 CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m); 5846 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next); 5847 m->md.pv_gen++; 5848 return (true); 5849 } else 5850 return (false); 5851 } 5852 5853 /* 5854 * Create the PV entry for a 2MB page mapping. Always returns true unless the 5855 * flag PMAP_ENTER_NORECLAIM is specified. If that flag is specified, returns 5856 * false if the PV entry cannot be allocated without resorting to reclamation. 5857 */ 5858 static bool 5859 pmap_pv_insert_pde(pmap_t pmap, vm_offset_t va, pd_entry_t pde, u_int flags, 5860 struct rwlock **lockp) 5861 { 5862 struct md_page *pvh; 5863 pv_entry_t pv; 5864 vm_paddr_t pa; 5865 5866 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 5867 /* Pass NULL instead of the lock pointer to disable reclamation. */ 5868 if ((pv = get_pv_entry(pmap, (flags & PMAP_ENTER_NORECLAIM) != 0 ? 5869 NULL : lockp)) == NULL) 5870 return (false); 5871 pv->pv_va = va; 5872 pa = pde & PG_PS_FRAME; 5873 CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa); 5874 pvh = pa_to_pvh(pa); 5875 TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_next); 5876 pvh->pv_gen++; 5877 return (true); 5878 } 5879 5880 /* 5881 * Fills a page table page with mappings to consecutive physical pages. 5882 */ 5883 static void 5884 pmap_fill_ptp(pt_entry_t *firstpte, pt_entry_t newpte) 5885 { 5886 pt_entry_t *pte; 5887 5888 for (pte = firstpte; pte < firstpte + NPTEPG; pte++) { 5889 *pte = newpte; 5890 newpte += PAGE_SIZE; 5891 } 5892 } 5893 5894 /* 5895 * Tries to demote a 2MB page mapping. If demotion fails, the 2MB page 5896 * mapping is invalidated. 5897 */ 5898 static bool 5899 pmap_demote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va) 5900 { 5901 struct rwlock *lock; 5902 bool rv; 5903 5904 lock = NULL; 5905 rv = pmap_demote_pde_locked(pmap, pde, va, &lock); 5906 if (lock != NULL) 5907 rw_wunlock(lock); 5908 return (rv); 5909 } 5910 5911 static void 5912 pmap_demote_pde_check(pt_entry_t *firstpte __unused, pt_entry_t newpte __unused) 5913 { 5914 #ifdef INVARIANTS 5915 #ifdef DIAGNOSTIC 5916 pt_entry_t *xpte, *ypte; 5917 5918 for (xpte = firstpte; xpte < firstpte + NPTEPG; 5919 xpte++, newpte += PAGE_SIZE) { 5920 if ((*xpte & PG_FRAME) != (newpte & PG_FRAME)) { 5921 printf("pmap_demote_pde: xpte %zd and newpte map " 5922 "different pages: found %#lx, expected %#lx\n", 5923 xpte - firstpte, *xpte, newpte); 5924 printf("page table dump\n"); 5925 for (ypte = firstpte; ypte < firstpte + NPTEPG; ypte++) 5926 printf("%zd %#lx\n", ypte - firstpte, *ypte); 5927 panic("firstpte"); 5928 } 5929 } 5930 #else 5931 KASSERT((*firstpte & PG_FRAME) == (newpte & PG_FRAME), 5932 ("pmap_demote_pde: firstpte and newpte map different physical" 5933 " addresses")); 5934 #endif 5935 #endif 5936 } 5937 5938 static void 5939 pmap_demote_pde_abort(pmap_t pmap, vm_offset_t va, pd_entry_t *pde, 5940 pd_entry_t oldpde, struct rwlock **lockp) 5941 { 5942 struct spglist free; 5943 vm_offset_t sva; 5944 5945 SLIST_INIT(&free); 5946 sva = trunc_2mpage(va); 5947 pmap_remove_pde(pmap, pde, sva, true, &free, lockp); 5948 if ((oldpde & pmap_global_bit(pmap)) == 0) 5949 pmap_invalidate_pde_page(pmap, sva, oldpde); 5950 vm_page_free_pages_toq(&free, true); 5951 CTR2(KTR_PMAP, "pmap_demote_pde: failure for va %#lx in pmap %p", 5952 va, pmap); 5953 } 5954 5955 static bool 5956 pmap_demote_pde_locked(pmap_t pmap, pd_entry_t *pde, vm_offset_t va, 5957 struct rwlock **lockp) 5958 { 5959 return (pmap_demote_pde_mpte(pmap, pde, va, lockp, NULL)); 5960 } 5961 5962 static bool 5963 pmap_demote_pde_mpte(pmap_t pmap, pd_entry_t *pde, vm_offset_t va, 5964 struct rwlock **lockp, vm_page_t mpte) 5965 { 5966 pd_entry_t newpde, oldpde; 5967 pt_entry_t *firstpte, newpte; 5968 pt_entry_t PG_A, PG_G, PG_M, PG_PKU_MASK, PG_RW, PG_V; 5969 vm_paddr_t mptepa; 5970 int PG_PTE_CACHE; 5971 bool in_kernel; 5972 5973 PG_A = pmap_accessed_bit(pmap); 5974 PG_G = pmap_global_bit(pmap); 5975 PG_M = pmap_modified_bit(pmap); 5976 PG_RW = pmap_rw_bit(pmap); 5977 PG_V = pmap_valid_bit(pmap); 5978 PG_PTE_CACHE = pmap_cache_mask(pmap, false); 5979 PG_PKU_MASK = pmap_pku_mask_bit(pmap); 5980 5981 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 5982 oldpde = *pde; 5983 KASSERT((oldpde & (PG_PS | PG_V)) == (PG_PS | PG_V), 5984 ("pmap_demote_pde: oldpde is missing PG_PS and/or PG_V")); 5985 KASSERT((oldpde & PG_MANAGED) == 0 || lockp != NULL, 5986 ("pmap_demote_pde: lockp for a managed mapping is NULL")); 5987 in_kernel = va >= VM_MAXUSER_ADDRESS; 5988 if (mpte == NULL) { 5989 /* 5990 * Invalidate the 2MB page mapping and return "failure" if the 5991 * mapping was never accessed and not wired. 5992 */ 5993 if ((oldpde & PG_A) == 0) { 5994 if ((oldpde & PG_W) == 0) { 5995 pmap_demote_pde_abort(pmap, va, pde, oldpde, 5996 lockp); 5997 return (false); 5998 } 5999 mpte = pmap_remove_pt_page(pmap, va); 6000 /* Fill the PTP with PTEs that have PG_A cleared. */ 6001 mpte->valid = 0; 6002 } else if ((mpte = pmap_remove_pt_page(pmap, va)) == NULL) { 6003 KASSERT((oldpde & PG_W) == 0, 6004 ("pmap_demote_pde: page table page for a wired mapping is missing")); 6005 6006 /* 6007 * If the page table page is missing and the mapping 6008 * is for a kernel address, the mapping must belong to 6009 * the direct map. Page table pages are preallocated 6010 * for every other part of the kernel address space, 6011 * so the direct map region is the only part of the 6012 * kernel address space that must be handled here. 6013 */ 6014 KASSERT(!in_kernel || (va >= kva_layout.dmap_low && 6015 va < kva_layout.dmap_high), 6016 ("pmap_demote_pde: No saved mpte for va %#lx", va)); 6017 6018 /* 6019 * If the 2MB page mapping belongs to the direct map 6020 * region of the kernel's address space, then the page 6021 * allocation request specifies the highest possible 6022 * priority (VM_ALLOC_INTERRUPT). Otherwise, the 6023 * priority is normal. 6024 */ 6025 mpte = pmap_alloc_pt_page(pmap, pmap_pde_pindex(va), 6026 (in_kernel ? VM_ALLOC_INTERRUPT : 0) | 6027 VM_ALLOC_WIRED); 6028 6029 /* 6030 * If the allocation of the new page table page fails, 6031 * invalidate the 2MB page mapping and return "failure". 6032 */ 6033 if (mpte == NULL) { 6034 pmap_demote_pde_abort(pmap, va, pde, oldpde, 6035 lockp); 6036 return (false); 6037 } 6038 6039 if (!in_kernel) 6040 mpte->ref_count = NPTEPG; 6041 } 6042 } 6043 mptepa = VM_PAGE_TO_PHYS(mpte); 6044 firstpte = PHYS_TO_DMAP(mptepa); 6045 newpde = mptepa | PG_M | PG_A | (oldpde & PG_U) | PG_RW | PG_V; 6046 KASSERT((oldpde & (PG_M | PG_RW)) != PG_RW, 6047 ("pmap_demote_pde: oldpde is missing PG_M")); 6048 newpte = oldpde & ~PG_PS; 6049 newpte = pmap_swap_pat(pmap, newpte); 6050 6051 /* 6052 * If the PTP is not leftover from an earlier promotion or it does not 6053 * have PG_A set in every PTE, then fill it. The new PTEs will all 6054 * have PG_A set, unless this is a wired mapping with PG_A clear. 6055 */ 6056 if (!vm_page_all_valid(mpte)) 6057 pmap_fill_ptp(firstpte, newpte); 6058 6059 pmap_demote_pde_check(firstpte, newpte); 6060 6061 /* 6062 * If the mapping has changed attributes, update the PTEs. 6063 */ 6064 if ((*firstpte & PG_PTE_PROMOTE) != (newpte & PG_PTE_PROMOTE)) 6065 pmap_fill_ptp(firstpte, newpte); 6066 6067 /* 6068 * The spare PV entries must be reserved prior to demoting the 6069 * mapping, that is, prior to changing the PDE. Otherwise, the state 6070 * of the PDE and the PV lists will be inconsistent, which can result 6071 * in reclaim_pv_chunk() attempting to remove a PV entry from the 6072 * wrong PV list and pmap_pv_demote_pde() failing to find the expected 6073 * PV entry for the 2MB page mapping that is being demoted. 6074 */ 6075 if ((oldpde & PG_MANAGED) != 0) 6076 reserve_pv_entries(pmap, NPTEPG - 1, lockp); 6077 6078 /* 6079 * Demote the mapping. This pmap is locked. The old PDE has 6080 * PG_A set. If the old PDE has PG_RW set, it also has PG_M 6081 * set. Thus, there is no danger of a race with another 6082 * processor changing the setting of PG_A and/or PG_M between 6083 * the read above and the store below. 6084 */ 6085 if (workaround_erratum383) 6086 pmap_update_pde(pmap, va, pde, newpde); 6087 else 6088 pde_store(pde, newpde); 6089 6090 /* 6091 * Invalidate a stale recursive mapping of the page table page. 6092 */ 6093 if (in_kernel) 6094 pmap_invalidate_page(pmap, (vm_offset_t)vtopte(va)); 6095 6096 /* 6097 * Demote the PV entry. 6098 */ 6099 if ((oldpde & PG_MANAGED) != 0) 6100 pmap_pv_demote_pde(pmap, va, oldpde & PG_PS_FRAME, lockp); 6101 6102 counter_u64_add(pmap_pde_demotions, 1); 6103 CTR2(KTR_PMAP, "pmap_demote_pde: success for va %#lx in pmap %p", 6104 va, pmap); 6105 return (true); 6106 } 6107 6108 /* 6109 * pmap_remove_kernel_pde: Remove a kernel superpage mapping. 6110 */ 6111 static void 6112 pmap_remove_kernel_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va) 6113 { 6114 pd_entry_t newpde; 6115 vm_paddr_t mptepa; 6116 vm_page_t mpte; 6117 6118 KASSERT(pmap == kernel_pmap, ("pmap %p is not kernel_pmap", pmap)); 6119 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 6120 mpte = pmap_remove_pt_page(pmap, va); 6121 KASSERT(mpte != NULL, ("pmap_remove_kernel_pde: missing pt page")); 6122 6123 mptepa = VM_PAGE_TO_PHYS(mpte); 6124 newpde = mptepa | X86_PG_M | X86_PG_A | X86_PG_RW | X86_PG_V; 6125 6126 /* 6127 * If this page table page was unmapped by a promotion, then it 6128 * contains valid mappings. Zero it to invalidate those mappings. 6129 */ 6130 if (vm_page_any_valid(mpte)) 6131 pagezero(PHYS_TO_DMAP(mptepa)); 6132 6133 /* 6134 * Demote the mapping. 6135 */ 6136 if (workaround_erratum383) 6137 pmap_update_pde(pmap, va, pde, newpde); 6138 else 6139 pde_store(pde, newpde); 6140 6141 /* 6142 * Invalidate a stale recursive mapping of the page table page. 6143 */ 6144 pmap_invalidate_page(pmap, (vm_offset_t)vtopte(va)); 6145 } 6146 6147 /* 6148 * pmap_remove_pde: do the things to unmap a superpage in a process 6149 */ 6150 static int 6151 pmap_remove_pde(pmap_t pmap, pd_entry_t *pdq, vm_offset_t sva, bool demote_kpde, 6152 struct spglist *free, struct rwlock **lockp) 6153 { 6154 struct md_page *pvh; 6155 pd_entry_t oldpde; 6156 vm_offset_t eva, va; 6157 vm_page_t m, mpte; 6158 pt_entry_t PG_G, PG_A, PG_M, PG_RW; 6159 6160 PG_G = pmap_global_bit(pmap); 6161 PG_A = pmap_accessed_bit(pmap); 6162 PG_M = pmap_modified_bit(pmap); 6163 PG_RW = pmap_rw_bit(pmap); 6164 6165 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 6166 KASSERT((sva & PDRMASK) == 0, 6167 ("pmap_remove_pde: sva is not 2mpage aligned")); 6168 oldpde = pte_load_clear(pdq); 6169 if (oldpde & PG_W) 6170 pmap->pm_stats.wired_count -= NBPDR / PAGE_SIZE; 6171 if ((oldpde & PG_G) != 0) 6172 pmap_invalidate_pde_page(kernel_pmap, sva, oldpde); 6173 pmap_resident_count_adj(pmap, -NBPDR / PAGE_SIZE); 6174 if (oldpde & PG_MANAGED) { 6175 CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, oldpde & PG_PS_FRAME); 6176 pvh = pa_to_pvh(oldpde & PG_PS_FRAME); 6177 pmap_pvh_free(pvh, pmap, sva); 6178 eva = sva + NBPDR; 6179 for (va = sva, m = PHYS_TO_VM_PAGE(oldpde & PG_PS_FRAME); 6180 va < eva; va += PAGE_SIZE, m++) { 6181 if ((oldpde & (PG_M | PG_RW)) == (PG_M | PG_RW)) 6182 vm_page_dirty(m); 6183 if (oldpde & PG_A) 6184 vm_page_aflag_set(m, PGA_REFERENCED); 6185 if (TAILQ_EMPTY(&m->md.pv_list) && 6186 TAILQ_EMPTY(&pvh->pv_list)) 6187 vm_page_aflag_clear(m, PGA_WRITEABLE); 6188 pmap_delayed_invl_page(m); 6189 } 6190 } 6191 if (pmap != kernel_pmap) { 6192 mpte = pmap_remove_pt_page(pmap, sva); 6193 if (mpte != NULL) { 6194 KASSERT(vm_page_any_valid(mpte), 6195 ("pmap_remove_pde: pte page not promoted")); 6196 pmap_pt_page_count_adj(pmap, -1); 6197 KASSERT(mpte->ref_count == NPTEPG, 6198 ("pmap_remove_pde: pte page ref count error")); 6199 mpte->ref_count = 0; 6200 pmap_add_delayed_free_list(mpte, free, false); 6201 } 6202 } else if (demote_kpde) { 6203 pmap_remove_kernel_pde(pmap, pdq, sva); 6204 } else { 6205 mpte = vm_radix_lookup(&pmap->pm_root, pmap_pde_pindex(sva)); 6206 if (vm_page_any_valid(mpte)) { 6207 mpte->valid = 0; 6208 pmap_zero_page(mpte); 6209 } 6210 } 6211 return (pmap_unuse_pt(pmap, sva, *pmap_pdpe(pmap, sva), free)); 6212 } 6213 6214 /* 6215 * pmap_remove_pte: do the things to unmap a page in a process 6216 */ 6217 static int 6218 pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t va, 6219 pd_entry_t ptepde, struct spglist *free, struct rwlock **lockp) 6220 { 6221 pt_entry_t oldpte, PG_A, PG_M, PG_RW; 6222 vm_page_t m; 6223 6224 PG_A = pmap_accessed_bit(pmap); 6225 PG_M = pmap_modified_bit(pmap); 6226 PG_RW = pmap_rw_bit(pmap); 6227 6228 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 6229 oldpte = pte_load_clear(ptq); 6230 if (oldpte & PG_W) 6231 pmap->pm_stats.wired_count -= 1; 6232 pmap_resident_count_adj(pmap, -1); 6233 if (oldpte & PG_MANAGED) { 6234 m = PHYS_TO_VM_PAGE(oldpte & PG_FRAME); 6235 if ((oldpte & (PG_M | PG_RW)) == (PG_M | PG_RW)) 6236 vm_page_dirty(m); 6237 if (oldpte & PG_A) 6238 vm_page_aflag_set(m, PGA_REFERENCED); 6239 CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m); 6240 pmap_pvh_free(&m->md, pmap, va); 6241 if (!pmap_page_is_mapped_locked(m)) 6242 vm_page_aflag_clear(m, PGA_WRITEABLE); 6243 pmap_delayed_invl_page(m); 6244 } 6245 return (pmap_unuse_pt(pmap, va, ptepde, free)); 6246 } 6247 6248 /* 6249 * Remove a single page from a process address space 6250 */ 6251 static void 6252 pmap_remove_page(pmap_t pmap, vm_offset_t va, pd_entry_t *pde, 6253 struct spglist *free) 6254 { 6255 struct rwlock *lock; 6256 pt_entry_t *pte, PG_V; 6257 6258 PG_V = pmap_valid_bit(pmap); 6259 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 6260 if ((*pde & PG_V) == 0) 6261 return; 6262 pte = pmap_pde_to_pte(pde, va); 6263 if ((*pte & PG_V) == 0) 6264 return; 6265 lock = NULL; 6266 pmap_remove_pte(pmap, pte, va, *pde, free, &lock); 6267 if (lock != NULL) 6268 rw_wunlock(lock); 6269 pmap_invalidate_page(pmap, va); 6270 } 6271 6272 /* 6273 * Removes the specified range of addresses from the page table page. 6274 */ 6275 static bool 6276 pmap_remove_ptes(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, 6277 pd_entry_t *pde, struct spglist *free, struct rwlock **lockp) 6278 { 6279 pt_entry_t PG_G, *pte; 6280 vm_offset_t va; 6281 bool anyvalid; 6282 6283 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 6284 PG_G = pmap_global_bit(pmap); 6285 anyvalid = false; 6286 va = eva; 6287 for (pte = pmap_pde_to_pte(pde, sva); sva != eva; pte++, 6288 sva += PAGE_SIZE) { 6289 if (*pte == 0) { 6290 if (va != eva) { 6291 pmap_invalidate_range(pmap, va, sva); 6292 va = eva; 6293 } 6294 continue; 6295 } 6296 if ((*pte & PG_G) == 0) 6297 anyvalid = true; 6298 else if (va == eva) 6299 va = sva; 6300 if (pmap_remove_pte(pmap, pte, sva, *pde, free, lockp)) { 6301 sva += PAGE_SIZE; 6302 break; 6303 } 6304 } 6305 if (va != eva) 6306 pmap_invalidate_range(pmap, va, sva); 6307 return (anyvalid); 6308 } 6309 6310 static void 6311 pmap_remove1(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, bool map_delete) 6312 { 6313 struct rwlock *lock; 6314 vm_page_t mt; 6315 vm_offset_t va_next; 6316 pml5_entry_t *pml5e; 6317 pml4_entry_t *pml4e; 6318 pdp_entry_t *pdpe; 6319 pd_entry_t ptpaddr, *pde; 6320 pt_entry_t PG_G, PG_V; 6321 struct spglist free; 6322 int anyvalid; 6323 6324 PG_G = pmap_global_bit(pmap); 6325 PG_V = pmap_valid_bit(pmap); 6326 6327 /* 6328 * If there are no resident pages besides the top level page 6329 * table page(s), there is nothing to do. Kernel pmap always 6330 * accounts whole preloaded area as resident, which makes its 6331 * resident count > 2. 6332 * Perform an unsynchronized read. This is, however, safe. 6333 */ 6334 if (pmap->pm_stats.resident_count <= 1 + (pmap->pm_pmltopu != NULL ? 6335 1 : 0)) 6336 return; 6337 6338 anyvalid = 0; 6339 SLIST_INIT(&free); 6340 6341 pmap_delayed_invl_start(); 6342 PMAP_LOCK(pmap); 6343 if (map_delete) 6344 pmap_pkru_on_remove(pmap, sva, eva); 6345 6346 /* 6347 * special handling of removing one page. a very 6348 * common operation and easy to short circuit some 6349 * code. 6350 */ 6351 if (sva + PAGE_SIZE == eva) { 6352 pde = pmap_pde(pmap, sva); 6353 if (pde && (*pde & PG_PS) == 0) { 6354 pmap_remove_page(pmap, sva, pde, &free); 6355 goto out; 6356 } 6357 } 6358 6359 lock = NULL; 6360 for (; sva < eva; sva = va_next) { 6361 if (pmap->pm_stats.resident_count == 0) 6362 break; 6363 6364 if (pmap_is_la57(pmap)) { 6365 pml5e = pmap_pml5e(pmap, sva); 6366 if ((*pml5e & PG_V) == 0) { 6367 va_next = (sva + NBPML5) & ~PML5MASK; 6368 if (va_next < sva) 6369 va_next = eva; 6370 continue; 6371 } 6372 pml4e = pmap_pml5e_to_pml4e(pml5e, sva); 6373 } else { 6374 pml4e = pmap_pml4e(pmap, sva); 6375 } 6376 if ((*pml4e & PG_V) == 0) { 6377 va_next = (sva + NBPML4) & ~PML4MASK; 6378 if (va_next < sva) 6379 va_next = eva; 6380 continue; 6381 } 6382 6383 va_next = (sva + NBPDP) & ~PDPMASK; 6384 if (va_next < sva) 6385 va_next = eva; 6386 pdpe = pmap_pml4e_to_pdpe(pml4e, sva); 6387 if ((*pdpe & PG_V) == 0) 6388 continue; 6389 if ((*pdpe & PG_PS) != 0) { 6390 KASSERT(va_next <= eva, 6391 ("partial update of non-transparent 1G mapping " 6392 "pdpe %#lx sva %#lx eva %#lx va_next %#lx", 6393 *pdpe, sva, eva, va_next)); 6394 MPASS(pmap != kernel_pmap); /* XXXKIB */ 6395 MPASS((*pdpe & (PG_MANAGED | PG_G)) == 0); 6396 anyvalid = 1; 6397 *pdpe = 0; 6398 pmap_resident_count_adj(pmap, -NBPDP / PAGE_SIZE); 6399 mt = PHYS_TO_VM_PAGE(*pmap_pml4e(pmap, sva) & PG_FRAME); 6400 pmap_unwire_ptp(pmap, sva, mt, &free); 6401 continue; 6402 } 6403 6404 /* 6405 * Calculate index for next page table. 6406 */ 6407 va_next = (sva + NBPDR) & ~PDRMASK; 6408 if (va_next < sva) 6409 va_next = eva; 6410 6411 pde = pmap_pdpe_to_pde(pdpe, sva); 6412 ptpaddr = *pde; 6413 6414 /* 6415 * Weed out invalid mappings. 6416 */ 6417 if (ptpaddr == 0) 6418 continue; 6419 6420 /* 6421 * Check for large page. 6422 */ 6423 if ((ptpaddr & PG_PS) != 0) { 6424 /* 6425 * Are we removing the entire large page? If not, 6426 * demote the mapping and fall through. 6427 */ 6428 if (sva + NBPDR == va_next && eva >= va_next) { 6429 /* 6430 * The TLB entry for a PG_G mapping is 6431 * invalidated by pmap_remove_pde(). 6432 */ 6433 if ((ptpaddr & PG_G) == 0) 6434 anyvalid = 1; 6435 pmap_remove_pde(pmap, pde, sva, true, &free, 6436 &lock); 6437 continue; 6438 } else if (!pmap_demote_pde_locked(pmap, pde, sva, 6439 &lock)) { 6440 /* The large page mapping was destroyed. */ 6441 continue; 6442 } else 6443 ptpaddr = *pde; 6444 } 6445 6446 /* 6447 * Limit our scan to either the end of the va represented 6448 * by the current page table page, or to the end of the 6449 * range being removed. 6450 */ 6451 if (va_next > eva) 6452 va_next = eva; 6453 6454 if (pmap_remove_ptes(pmap, sva, va_next, pde, &free, &lock)) 6455 anyvalid = 1; 6456 } 6457 if (lock != NULL) 6458 rw_wunlock(lock); 6459 out: 6460 if (anyvalid) 6461 pmap_invalidate_all(pmap); 6462 PMAP_UNLOCK(pmap); 6463 pmap_delayed_invl_finish(); 6464 vm_page_free_pages_toq(&free, true); 6465 } 6466 6467 /* 6468 * Remove the given range of addresses from the specified map. 6469 * 6470 * It is assumed that the start and end are properly 6471 * rounded to the page size. 6472 */ 6473 void 6474 pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) 6475 { 6476 pmap_remove1(pmap, sva, eva, false); 6477 } 6478 6479 /* 6480 * Remove the given range of addresses as part of a logical unmap 6481 * operation. This has the effect of calling pmap_remove(), but 6482 * also clears any metadata that should persist for the lifetime 6483 * of a logical mapping. 6484 */ 6485 void 6486 pmap_map_delete(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) 6487 { 6488 pmap_remove1(pmap, sva, eva, true); 6489 } 6490 6491 /* 6492 * Routine: pmap_remove_all 6493 * Function: 6494 * Removes this physical page from 6495 * all physical maps in which it resides. 6496 * Reflects back modify bits to the pager. 6497 * 6498 * Notes: 6499 * Original versions of this routine were very 6500 * inefficient because they iteratively called 6501 * pmap_remove (slow...) 6502 */ 6503 6504 void 6505 pmap_remove_all(vm_page_t m) 6506 { 6507 struct md_page *pvh; 6508 pv_entry_t pv; 6509 pmap_t pmap; 6510 struct rwlock *lock; 6511 pt_entry_t *pte, tpte, PG_A, PG_M, PG_RW; 6512 pd_entry_t *pde; 6513 vm_offset_t va; 6514 struct spglist free; 6515 int pvh_gen, md_gen; 6516 6517 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 6518 ("pmap_remove_all: page %p is not managed", m)); 6519 SLIST_INIT(&free); 6520 lock = VM_PAGE_TO_PV_LIST_LOCK(m); 6521 pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy : 6522 pa_to_pvh(VM_PAGE_TO_PHYS(m)); 6523 rw_wlock(lock); 6524 retry: 6525 while ((pv = TAILQ_FIRST(&pvh->pv_list)) != NULL) { 6526 pmap = PV_PMAP(pv); 6527 if (!PMAP_TRYLOCK(pmap)) { 6528 pvh_gen = pvh->pv_gen; 6529 rw_wunlock(lock); 6530 PMAP_LOCK(pmap); 6531 rw_wlock(lock); 6532 if (pvh_gen != pvh->pv_gen) { 6533 PMAP_UNLOCK(pmap); 6534 goto retry; 6535 } 6536 } 6537 va = pv->pv_va; 6538 pde = pmap_pde(pmap, va); 6539 (void)pmap_demote_pde_locked(pmap, pde, va, &lock); 6540 PMAP_UNLOCK(pmap); 6541 } 6542 while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) { 6543 pmap = PV_PMAP(pv); 6544 if (!PMAP_TRYLOCK(pmap)) { 6545 pvh_gen = pvh->pv_gen; 6546 md_gen = m->md.pv_gen; 6547 rw_wunlock(lock); 6548 PMAP_LOCK(pmap); 6549 rw_wlock(lock); 6550 if (pvh_gen != pvh->pv_gen || md_gen != m->md.pv_gen) { 6551 PMAP_UNLOCK(pmap); 6552 goto retry; 6553 } 6554 } 6555 PG_A = pmap_accessed_bit(pmap); 6556 PG_M = pmap_modified_bit(pmap); 6557 PG_RW = pmap_rw_bit(pmap); 6558 pmap_resident_count_adj(pmap, -1); 6559 pde = pmap_pde(pmap, pv->pv_va); 6560 KASSERT((*pde & PG_PS) == 0, ("pmap_remove_all: found" 6561 " a 2mpage in page %p's pv list", m)); 6562 pte = pmap_pde_to_pte(pde, pv->pv_va); 6563 tpte = pte_load_clear(pte); 6564 if (tpte & PG_W) 6565 pmap->pm_stats.wired_count--; 6566 if (tpte & PG_A) 6567 vm_page_aflag_set(m, PGA_REFERENCED); 6568 6569 /* 6570 * Update the vm_page_t clean and reference bits. 6571 */ 6572 if ((tpte & (PG_M | PG_RW)) == (PG_M | PG_RW)) 6573 vm_page_dirty(m); 6574 pmap_unuse_pt(pmap, pv->pv_va, *pde, &free); 6575 pmap_invalidate_page(pmap, pv->pv_va); 6576 TAILQ_REMOVE(&m->md.pv_list, pv, pv_next); 6577 m->md.pv_gen++; 6578 free_pv_entry(pmap, pv); 6579 PMAP_UNLOCK(pmap); 6580 } 6581 vm_page_aflag_clear(m, PGA_WRITEABLE); 6582 rw_wunlock(lock); 6583 pmap_delayed_invl_wait(m); 6584 vm_page_free_pages_toq(&free, true); 6585 } 6586 6587 /* 6588 * pmap_protect_pde: do the things to protect a 2mpage in a process 6589 */ 6590 static bool 6591 pmap_protect_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t sva, vm_prot_t prot) 6592 { 6593 pd_entry_t newpde, oldpde; 6594 vm_page_t m, mt; 6595 bool anychanged; 6596 pt_entry_t PG_G, PG_M, PG_RW; 6597 6598 PG_G = pmap_global_bit(pmap); 6599 PG_M = pmap_modified_bit(pmap); 6600 PG_RW = pmap_rw_bit(pmap); 6601 6602 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 6603 KASSERT((sva & PDRMASK) == 0, 6604 ("pmap_protect_pde: sva is not 2mpage aligned")); 6605 anychanged = false; 6606 retry: 6607 oldpde = newpde = *pde; 6608 if ((prot & VM_PROT_WRITE) == 0) { 6609 if ((oldpde & (PG_MANAGED | PG_M | PG_RW)) == 6610 (PG_MANAGED | PG_M | PG_RW)) { 6611 m = PHYS_TO_VM_PAGE(oldpde & PG_PS_FRAME); 6612 for (mt = m; mt < &m[NBPDR / PAGE_SIZE]; mt++) 6613 vm_page_dirty(mt); 6614 } 6615 newpde &= ~(PG_RW | PG_M); 6616 } 6617 if ((prot & VM_PROT_EXECUTE) == 0) 6618 newpde |= pg_nx; 6619 if (newpde != oldpde) { 6620 /* 6621 * As an optimization to future operations on this PDE, clear 6622 * PG_PROMOTED. The impending invalidation will remove any 6623 * lingering 4KB page mappings from the TLB. 6624 */ 6625 if (!atomic_cmpset_long(pde, oldpde, newpde & ~PG_PROMOTED)) 6626 goto retry; 6627 if ((oldpde & PG_G) != 0) 6628 pmap_invalidate_pde_page(kernel_pmap, sva, oldpde); 6629 else 6630 anychanged = true; 6631 } 6632 return (anychanged); 6633 } 6634 6635 /* 6636 * Set the physical protection on the 6637 * specified range of this map as requested. 6638 */ 6639 void 6640 pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot) 6641 { 6642 vm_page_t m; 6643 vm_offset_t va_next; 6644 pml4_entry_t *pml4e; 6645 pdp_entry_t *pdpe; 6646 pd_entry_t ptpaddr, *pde; 6647 pt_entry_t *pte, PG_G, PG_M, PG_RW, PG_V; 6648 pt_entry_t obits, pbits; 6649 bool anychanged; 6650 6651 KASSERT((prot & ~VM_PROT_ALL) == 0, ("invalid prot %x", prot)); 6652 if (prot == VM_PROT_NONE) { 6653 pmap_remove(pmap, sva, eva); 6654 return; 6655 } 6656 6657 if ((prot & (VM_PROT_WRITE|VM_PROT_EXECUTE)) == 6658 (VM_PROT_WRITE|VM_PROT_EXECUTE)) 6659 return; 6660 6661 PG_G = pmap_global_bit(pmap); 6662 PG_M = pmap_modified_bit(pmap); 6663 PG_V = pmap_valid_bit(pmap); 6664 PG_RW = pmap_rw_bit(pmap); 6665 anychanged = false; 6666 6667 /* 6668 * Although this function delays and batches the invalidation 6669 * of stale TLB entries, it does not need to call 6670 * pmap_delayed_invl_start() and 6671 * pmap_delayed_invl_finish(), because it does not 6672 * ordinarily destroy mappings. Stale TLB entries from 6673 * protection-only changes need only be invalidated before the 6674 * pmap lock is released, because protection-only changes do 6675 * not destroy PV entries. Even operations that iterate over 6676 * a physical page's PV list of mappings, like 6677 * pmap_remove_write(), acquire the pmap lock for each 6678 * mapping. Consequently, for protection-only changes, the 6679 * pmap lock suffices to synchronize both page table and TLB 6680 * updates. 6681 * 6682 * This function only destroys a mapping if pmap_demote_pde() 6683 * fails. In that case, stale TLB entries are immediately 6684 * invalidated. 6685 */ 6686 6687 PMAP_LOCK(pmap); 6688 for (; sva < eva; sva = va_next) { 6689 pml4e = pmap_pml4e(pmap, sva); 6690 if (pml4e == NULL || (*pml4e & PG_V) == 0) { 6691 va_next = (sva + NBPML4) & ~PML4MASK; 6692 if (va_next < sva) 6693 va_next = eva; 6694 continue; 6695 } 6696 6697 va_next = (sva + NBPDP) & ~PDPMASK; 6698 if (va_next < sva) 6699 va_next = eva; 6700 pdpe = pmap_pml4e_to_pdpe(pml4e, sva); 6701 if ((*pdpe & PG_V) == 0) 6702 continue; 6703 if ((*pdpe & PG_PS) != 0) { 6704 KASSERT(va_next <= eva, 6705 ("partial update of non-transparent 1G mapping " 6706 "pdpe %#lx sva %#lx eva %#lx va_next %#lx", 6707 *pdpe, sva, eva, va_next)); 6708 retry_pdpe: 6709 obits = pbits = *pdpe; 6710 MPASS((pbits & (PG_MANAGED | PG_G)) == 0); 6711 MPASS(pmap != kernel_pmap); /* XXXKIB */ 6712 if ((prot & VM_PROT_WRITE) == 0) 6713 pbits &= ~(PG_RW | PG_M); 6714 if ((prot & VM_PROT_EXECUTE) == 0) 6715 pbits |= pg_nx; 6716 6717 if (pbits != obits) { 6718 if (!atomic_cmpset_long(pdpe, obits, pbits)) 6719 /* PG_PS cannot be cleared under us, */ 6720 goto retry_pdpe; 6721 anychanged = true; 6722 } 6723 continue; 6724 } 6725 6726 va_next = (sva + NBPDR) & ~PDRMASK; 6727 if (va_next < sva) 6728 va_next = eva; 6729 6730 pde = pmap_pdpe_to_pde(pdpe, sva); 6731 ptpaddr = *pde; 6732 6733 /* 6734 * Weed out invalid mappings. 6735 */ 6736 if (ptpaddr == 0) 6737 continue; 6738 6739 /* 6740 * Check for large page. 6741 */ 6742 if ((ptpaddr & PG_PS) != 0) { 6743 /* 6744 * Are we protecting the entire large page? 6745 */ 6746 if (sva + NBPDR == va_next && eva >= va_next) { 6747 /* 6748 * The TLB entry for a PG_G mapping is 6749 * invalidated by pmap_protect_pde(). 6750 */ 6751 if (pmap_protect_pde(pmap, pde, sva, prot)) 6752 anychanged = true; 6753 continue; 6754 } 6755 6756 /* 6757 * Does the large page mapping need to change? If so, 6758 * demote it and fall through. 6759 */ 6760 pbits = ptpaddr; 6761 if ((prot & VM_PROT_WRITE) == 0) 6762 pbits &= ~(PG_RW | PG_M); 6763 if ((prot & VM_PROT_EXECUTE) == 0) 6764 pbits |= pg_nx; 6765 if (ptpaddr == pbits || !pmap_demote_pde(pmap, pde, 6766 sva)) { 6767 /* 6768 * Either the large page mapping doesn't need 6769 * to change, or it was destroyed during 6770 * demotion. 6771 */ 6772 continue; 6773 } 6774 } 6775 6776 if (va_next > eva) 6777 va_next = eva; 6778 6779 for (pte = pmap_pde_to_pte(pde, sva); sva != va_next; pte++, 6780 sva += PAGE_SIZE) { 6781 retry: 6782 obits = pbits = *pte; 6783 if ((pbits & PG_V) == 0) 6784 continue; 6785 6786 if ((prot & VM_PROT_WRITE) == 0) { 6787 if ((pbits & (PG_MANAGED | PG_M | PG_RW)) == 6788 (PG_MANAGED | PG_M | PG_RW)) { 6789 m = PHYS_TO_VM_PAGE(pbits & PG_FRAME); 6790 vm_page_dirty(m); 6791 } 6792 pbits &= ~(PG_RW | PG_M); 6793 } 6794 if ((prot & VM_PROT_EXECUTE) == 0) 6795 pbits |= pg_nx; 6796 6797 if (pbits != obits) { 6798 if (!atomic_cmpset_long(pte, obits, pbits)) 6799 goto retry; 6800 if (obits & PG_G) 6801 pmap_invalidate_page(pmap, sva); 6802 else 6803 anychanged = true; 6804 } 6805 } 6806 } 6807 if (anychanged) 6808 pmap_invalidate_all(pmap); 6809 PMAP_UNLOCK(pmap); 6810 } 6811 6812 static bool 6813 pmap_pde_ept_executable(pmap_t pmap, pd_entry_t pde) 6814 { 6815 6816 if (pmap->pm_type != PT_EPT) 6817 return (false); 6818 return ((pde & EPT_PG_EXECUTE) != 0); 6819 } 6820 6821 #if VM_NRESERVLEVEL > 0 6822 /* 6823 * Tries to promote the 512, contiguous 4KB page mappings that are within a 6824 * single page table page (PTP) to a single 2MB page mapping. For promotion 6825 * to occur, two conditions must be met: (1) the 4KB page mappings must map 6826 * aligned, contiguous physical memory and (2) the 4KB page mappings must have 6827 * identical characteristics. 6828 */ 6829 static bool 6830 pmap_promote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va, vm_page_t mpte, 6831 struct rwlock **lockp) 6832 { 6833 pd_entry_t newpde; 6834 pt_entry_t *firstpte, oldpte, pa, *pte; 6835 pt_entry_t allpte_PG_A, PG_A, PG_G, PG_M, PG_PKU_MASK, PG_RW, PG_V; 6836 int PG_PTE_CACHE; 6837 6838 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 6839 if (!pmap_ps_enabled(pmap)) 6840 return (false); 6841 6842 PG_A = pmap_accessed_bit(pmap); 6843 PG_G = pmap_global_bit(pmap); 6844 PG_M = pmap_modified_bit(pmap); 6845 PG_V = pmap_valid_bit(pmap); 6846 PG_RW = pmap_rw_bit(pmap); 6847 PG_PKU_MASK = pmap_pku_mask_bit(pmap); 6848 PG_PTE_CACHE = pmap_cache_mask(pmap, false); 6849 6850 /* 6851 * Examine the first PTE in the specified PTP. Abort if this PTE is 6852 * ineligible for promotion due to hardware errata, invalid, or does 6853 * not map the first 4KB physical page within a 2MB page. 6854 */ 6855 firstpte = PHYS_TO_DMAP(*pde & PG_FRAME); 6856 newpde = *firstpte; 6857 if (!pmap_allow_2m_x_page(pmap, pmap_pde_ept_executable(pmap, newpde))) 6858 return (false); 6859 if ((newpde & ((PG_FRAME & PDRMASK) | PG_V)) != PG_V) { 6860 counter_u64_add(pmap_pde_p_failures, 1); 6861 CTR2(KTR_PMAP, "pmap_promote_pde: failure for va %#lx" 6862 " in pmap %p", va, pmap); 6863 return (false); 6864 } 6865 6866 /* 6867 * Both here and in the below "for" loop, to allow for repromotion 6868 * after MADV_FREE, conditionally write protect a clean PTE before 6869 * possibly aborting the promotion due to other PTE attributes. Why? 6870 * Suppose that MADV_FREE is applied to a part of a superpage, the 6871 * address range [S, E). pmap_advise() will demote the superpage 6872 * mapping, destroy the 4KB page mapping at the end of [S, E), and 6873 * clear PG_M and PG_A in the PTEs for the rest of [S, E). Later, 6874 * imagine that the memory in [S, E) is recycled, but the last 4KB 6875 * page in [S, E) is not the last to be rewritten, or simply accessed. 6876 * In other words, there is still a 4KB page in [S, E), call it P, 6877 * that is writeable but PG_M and PG_A are clear in P's PTE. Unless 6878 * we write protect P before aborting the promotion, if and when P is 6879 * finally rewritten, there won't be a page fault to trigger 6880 * repromotion. 6881 */ 6882 setpde: 6883 if ((newpde & (PG_M | PG_RW)) == PG_RW) { 6884 /* 6885 * When PG_M is already clear, PG_RW can be cleared without 6886 * a TLB invalidation. 6887 */ 6888 if (!atomic_fcmpset_long(firstpte, &newpde, newpde & ~PG_RW)) 6889 goto setpde; 6890 newpde &= ~PG_RW; 6891 CTR2(KTR_PMAP, "pmap_promote_pde: protect for va %#lx" 6892 " in pmap %p", va & ~PDRMASK, pmap); 6893 } 6894 6895 /* 6896 * Examine each of the other PTEs in the specified PTP. Abort if this 6897 * PTE maps an unexpected 4KB physical page or does not have identical 6898 * characteristics to the first PTE. 6899 */ 6900 allpte_PG_A = newpde & PG_A; 6901 pa = (newpde & (PG_PS_FRAME | PG_V)) + NBPDR - PAGE_SIZE; 6902 for (pte = firstpte + NPTEPG - 1; pte > firstpte; pte--) { 6903 oldpte = *pte; 6904 if ((oldpte & (PG_FRAME | PG_V)) != pa) { 6905 counter_u64_add(pmap_pde_p_failures, 1); 6906 CTR2(KTR_PMAP, "pmap_promote_pde: failure for va %#lx" 6907 " in pmap %p", va, pmap); 6908 return (false); 6909 } 6910 setpte: 6911 if ((oldpte & (PG_M | PG_RW)) == PG_RW) { 6912 /* 6913 * When PG_M is already clear, PG_RW can be cleared 6914 * without a TLB invalidation. 6915 */ 6916 if (!atomic_fcmpset_long(pte, &oldpte, oldpte & ~PG_RW)) 6917 goto setpte; 6918 oldpte &= ~PG_RW; 6919 CTR2(KTR_PMAP, "pmap_promote_pde: protect for va %#lx" 6920 " in pmap %p", (oldpte & PG_FRAME & PDRMASK) | 6921 (va & ~PDRMASK), pmap); 6922 } 6923 if ((oldpte & PG_PTE_PROMOTE) != (newpde & PG_PTE_PROMOTE)) { 6924 counter_u64_add(pmap_pde_p_failures, 1); 6925 CTR2(KTR_PMAP, "pmap_promote_pde: failure for va %#lx" 6926 " in pmap %p", va, pmap); 6927 return (false); 6928 } 6929 allpte_PG_A &= oldpte; 6930 pa -= PAGE_SIZE; 6931 } 6932 6933 /* 6934 * Unless all PTEs have PG_A set, clear it from the superpage mapping, 6935 * so that promotions triggered by speculative mappings, such as 6936 * pmap_enter_quick(), don't automatically mark the underlying pages 6937 * as referenced. 6938 */ 6939 newpde &= ~PG_A | allpte_PG_A; 6940 6941 /* 6942 * EPT PTEs with PG_M set and PG_A clear are not supported by early 6943 * MMUs supporting EPT. 6944 */ 6945 KASSERT((newpde & PG_A) != 0 || safe_to_clear_referenced(pmap, newpde), 6946 ("unsupported EPT PTE")); 6947 6948 /* 6949 * Save the PTP in its current state until the PDE mapping the 6950 * superpage is demoted by pmap_demote_pde() or destroyed by 6951 * pmap_remove_pde(). If PG_A is not set in every PTE, then request 6952 * that the PTP be refilled on demotion. 6953 */ 6954 if (mpte == NULL) 6955 mpte = PHYS_TO_VM_PAGE(*pde & PG_FRAME); 6956 KASSERT(mpte >= vm_page_array && 6957 mpte < &vm_page_array[vm_page_array_size], 6958 ("pmap_promote_pde: page table page is out of range")); 6959 KASSERT(mpte->pindex == pmap_pde_pindex(va), 6960 ("pmap_promote_pde: page table page's pindex is wrong " 6961 "mpte %p pidx %#lx va %#lx va pde pidx %#lx", 6962 mpte, mpte->pindex, va, pmap_pde_pindex(va))); 6963 if (pmap_insert_pt_page(pmap, mpte, true, allpte_PG_A != 0)) { 6964 counter_u64_add(pmap_pde_p_failures, 1); 6965 CTR2(KTR_PMAP, 6966 "pmap_promote_pde: failure for va %#lx in pmap %p", va, 6967 pmap); 6968 return (false); 6969 } 6970 6971 /* 6972 * Promote the pv entries. 6973 */ 6974 if ((newpde & PG_MANAGED) != 0) 6975 pmap_pv_promote_pde(pmap, va, newpde & PG_PS_FRAME, lockp); 6976 6977 /* 6978 * Propagate the PAT index to its proper position. 6979 */ 6980 newpde = pmap_swap_pat(pmap, newpde); 6981 6982 /* 6983 * Map the superpage. 6984 */ 6985 if (workaround_erratum383) 6986 pmap_update_pde(pmap, va, pde, PG_PS | newpde); 6987 else 6988 pde_store(pde, PG_PROMOTED | PG_PS | newpde); 6989 6990 counter_u64_add(pmap_pde_promotions, 1); 6991 CTR2(KTR_PMAP, "pmap_promote_pde: success for va %#lx" 6992 " in pmap %p", va, pmap); 6993 return (true); 6994 } 6995 #endif /* VM_NRESERVLEVEL > 0 */ 6996 6997 static int 6998 pmap_enter_largepage(pmap_t pmap, vm_offset_t va, pt_entry_t newpte, int flags, 6999 int psind) 7000 { 7001 vm_page_t mp; 7002 pt_entry_t origpte, *pml4e, *pdpe, *pde, pten, PG_V; 7003 7004 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 7005 KASSERT(psind > 0 && psind < MAXPAGESIZES && pagesizes[psind] != 0, 7006 ("psind %d unexpected", psind)); 7007 KASSERT(((newpte & PG_FRAME) & (pagesizes[psind] - 1)) == 0, 7008 ("unaligned phys address %#lx newpte %#lx psind %d", 7009 newpte & PG_FRAME, newpte, psind)); 7010 KASSERT((va & (pagesizes[psind] - 1)) == 0, 7011 ("unaligned va %#lx psind %d", va, psind)); 7012 KASSERT(va < VM_MAXUSER_ADDRESS, 7013 ("kernel mode non-transparent superpage")); /* XXXKIB */ 7014 KASSERT(va + pagesizes[psind] < VM_MAXUSER_ADDRESS, 7015 ("overflowing user map va %#lx psind %d", va, psind)); /* XXXKIB */ 7016 7017 PG_V = pmap_valid_bit(pmap); 7018 7019 restart: 7020 pten = newpte; 7021 if (!pmap_pkru_same(pmap, va, va + pagesizes[psind], &pten)) 7022 return (KERN_PROTECTION_FAILURE); 7023 7024 if (psind == 2) { /* 1G */ 7025 pml4e = pmap_pml4e(pmap, va); 7026 if (pml4e == NULL || (*pml4e & PG_V) == 0) { 7027 mp = pmap_allocpte_alloc(pmap, pmap_pml4e_pindex(va), 7028 NULL, va); 7029 if (mp == NULL) 7030 goto allocf; 7031 pdpe = VM_PAGE_TO_DMAP(mp); 7032 pdpe = &pdpe[pmap_pdpe_index(va)]; 7033 origpte = *pdpe; 7034 MPASS(origpte == 0); 7035 } else { 7036 pdpe = pmap_pml4e_to_pdpe(pml4e, va); 7037 KASSERT(pdpe != NULL, ("va %#lx lost pdpe", va)); 7038 origpte = *pdpe; 7039 if ((origpte & PG_V) == 0) { 7040 mp = PHYS_TO_VM_PAGE(*pml4e & PG_FRAME); 7041 mp->ref_count++; 7042 } 7043 } 7044 *pdpe = pten; 7045 } else /* (psind == 1) */ { /* 2M */ 7046 pde = pmap_pde(pmap, va); 7047 if (pde == NULL) { 7048 mp = pmap_allocpte_alloc(pmap, pmap_pdpe_pindex(va), 7049 NULL, va); 7050 if (mp == NULL) 7051 goto allocf; 7052 pde = VM_PAGE_TO_DMAP(mp); 7053 pde = &pde[pmap_pde_index(va)]; 7054 origpte = *pde; 7055 MPASS(origpte == 0); 7056 } else { 7057 origpte = *pde; 7058 if ((origpte & PG_V) == 0) { 7059 pdpe = pmap_pdpe(pmap, va); 7060 MPASS(pdpe != NULL && (*pdpe & PG_V) != 0); 7061 mp = PHYS_TO_VM_PAGE(*pdpe & PG_FRAME); 7062 mp->ref_count++; 7063 } 7064 } 7065 *pde = pten; 7066 } 7067 KASSERT((origpte & PG_V) == 0 || ((origpte & PG_PS) != 0 && 7068 (origpte & PG_PS_FRAME) == (pten & PG_PS_FRAME)), 7069 ("va %#lx changing %s phys page origpte %#lx pten %#lx", 7070 va, psind == 2 ? "1G" : "2M", origpte, pten)); 7071 if ((pten & PG_W) != 0 && (origpte & PG_W) == 0) 7072 pmap->pm_stats.wired_count += pagesizes[psind] / PAGE_SIZE; 7073 else if ((pten & PG_W) == 0 && (origpte & PG_W) != 0) 7074 pmap->pm_stats.wired_count -= pagesizes[psind] / PAGE_SIZE; 7075 if ((origpte & PG_V) == 0) 7076 pmap_resident_count_adj(pmap, pagesizes[psind] / PAGE_SIZE); 7077 7078 return (KERN_SUCCESS); 7079 7080 allocf: 7081 if ((flags & PMAP_ENTER_NOSLEEP) != 0) 7082 return (KERN_RESOURCE_SHORTAGE); 7083 PMAP_UNLOCK(pmap); 7084 vm_wait(NULL); 7085 PMAP_LOCK(pmap); 7086 goto restart; 7087 } 7088 7089 /* 7090 * Insert the given physical page (p) at 7091 * the specified virtual address (v) in the 7092 * target physical map with the protection requested. 7093 * 7094 * If specified, the page will be wired down, meaning 7095 * that the related pte can not be reclaimed. 7096 * 7097 * NB: This is the only routine which MAY NOT lazy-evaluate 7098 * or lose information. That is, this routine must actually 7099 * insert this page into the given map NOW. 7100 * 7101 * When destroying both a page table and PV entry, this function 7102 * performs the TLB invalidation before releasing the PV list 7103 * lock, so we do not need pmap_delayed_invl_page() calls here. 7104 */ 7105 int 7106 pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, 7107 u_int flags, int8_t psind) 7108 { 7109 struct rwlock *lock; 7110 pd_entry_t *pde; 7111 pt_entry_t *pte, PG_G, PG_A, PG_M, PG_RW, PG_V; 7112 pt_entry_t newpte, origpte; 7113 pv_entry_t pv; 7114 vm_paddr_t opa, pa; 7115 vm_page_t mpte, om; 7116 int rv; 7117 bool nosleep; 7118 7119 PG_A = pmap_accessed_bit(pmap); 7120 PG_G = pmap_global_bit(pmap); 7121 PG_M = pmap_modified_bit(pmap); 7122 PG_V = pmap_valid_bit(pmap); 7123 PG_RW = pmap_rw_bit(pmap); 7124 7125 va = trunc_page(va); 7126 KASSERT(va <= kva_layout.km_high, ("pmap_enter: toobig")); 7127 KASSERT(va < UPT_MIN_ADDRESS || va >= UPT_MAX_ADDRESS, 7128 ("pmap_enter: invalid to pmap_enter page table pages (va: 0x%lx)", 7129 va)); 7130 KASSERT((m->oflags & VPO_UNMANAGED) != 0 || !VA_IS_CLEANMAP(va), 7131 ("pmap_enter: managed mapping within the clean submap")); 7132 if ((m->oflags & VPO_UNMANAGED) == 0) 7133 VM_PAGE_OBJECT_BUSY_ASSERT(m); 7134 KASSERT((flags & PMAP_ENTER_RESERVED) == 0, 7135 ("pmap_enter: flags %u has reserved bits set", flags)); 7136 pa = VM_PAGE_TO_PHYS(m); 7137 newpte = (pt_entry_t)(pa | PG_A | PG_V); 7138 if ((flags & VM_PROT_WRITE) != 0) 7139 newpte |= PG_M; 7140 if ((prot & VM_PROT_WRITE) != 0) 7141 newpte |= PG_RW; 7142 KASSERT((newpte & (PG_M | PG_RW)) != PG_M, 7143 ("pmap_enter: flags includes VM_PROT_WRITE but prot doesn't")); 7144 if ((prot & VM_PROT_EXECUTE) == 0) 7145 newpte |= pg_nx; 7146 if ((flags & PMAP_ENTER_WIRED) != 0) 7147 newpte |= PG_W; 7148 if (va < VM_MAXUSER_ADDRESS) 7149 newpte |= PG_U; 7150 if (pmap == kernel_pmap) 7151 newpte |= PG_G; 7152 newpte |= pmap_cache_bits(pmap, m->md.pat_mode, psind > 0); 7153 7154 /* 7155 * Set modified bit gratuitously for writeable mappings if 7156 * the page is unmanaged. We do not want to take a fault 7157 * to do the dirty bit accounting for these mappings. 7158 */ 7159 if ((m->oflags & VPO_UNMANAGED) != 0) { 7160 if ((newpte & PG_RW) != 0) 7161 newpte |= PG_M; 7162 } else 7163 newpte |= PG_MANAGED; 7164 7165 lock = NULL; 7166 PMAP_LOCK(pmap); 7167 if ((flags & PMAP_ENTER_LARGEPAGE) != 0) { 7168 KASSERT((m->oflags & VPO_UNMANAGED) != 0, 7169 ("managed largepage va %#lx flags %#x", va, flags)); 7170 rv = pmap_enter_largepage(pmap, va, newpte | PG_PS, flags, 7171 psind); 7172 goto out; 7173 } 7174 if (psind == 1) { 7175 /* Assert the required virtual and physical alignment. */ 7176 KASSERT((va & PDRMASK) == 0, ("pmap_enter: va unaligned")); 7177 KASSERT(m->psind > 0, ("pmap_enter: m->psind < psind")); 7178 rv = pmap_enter_pde(pmap, va, newpte | PG_PS, flags, m, &lock); 7179 goto out; 7180 } 7181 mpte = NULL; 7182 7183 /* 7184 * In the case that a page table page is not 7185 * resident, we are creating it here. 7186 */ 7187 retry: 7188 pde = pmap_pde(pmap, va); 7189 if (pde != NULL && (*pde & PG_V) != 0 && ((*pde & PG_PS) == 0 || 7190 pmap_demote_pde_locked(pmap, pde, va, &lock))) { 7191 pte = pmap_pde_to_pte(pde, va); 7192 if (va < VM_MAXUSER_ADDRESS && mpte == NULL) { 7193 mpte = PHYS_TO_VM_PAGE(*pde & PG_FRAME); 7194 mpte->ref_count++; 7195 } 7196 } else if (va < VM_MAXUSER_ADDRESS) { 7197 /* 7198 * Here if the pte page isn't mapped, or if it has been 7199 * deallocated. 7200 */ 7201 nosleep = (flags & PMAP_ENTER_NOSLEEP) != 0; 7202 mpte = pmap_allocpte_alloc(pmap, pmap_pde_pindex(va), 7203 nosleep ? NULL : &lock, va); 7204 if (mpte == NULL && nosleep) { 7205 rv = KERN_RESOURCE_SHORTAGE; 7206 goto out; 7207 } 7208 goto retry; 7209 } else 7210 panic("pmap_enter: invalid page directory va=%#lx", va); 7211 7212 origpte = *pte; 7213 pv = NULL; 7214 if (va < VM_MAXUSER_ADDRESS && pmap->pm_type == PT_X86) 7215 newpte |= pmap_pkru_get(pmap, va); 7216 7217 /* 7218 * Is the specified virtual address already mapped? 7219 */ 7220 if ((origpte & PG_V) != 0) { 7221 /* 7222 * Wiring change, just update stats. We don't worry about 7223 * wiring PT pages as they remain resident as long as there 7224 * are valid mappings in them. Hence, if a user page is wired, 7225 * the PT page will be also. 7226 */ 7227 if ((newpte & PG_W) != 0 && (origpte & PG_W) == 0) 7228 pmap->pm_stats.wired_count++; 7229 else if ((newpte & PG_W) == 0 && (origpte & PG_W) != 0) 7230 pmap->pm_stats.wired_count--; 7231 7232 /* 7233 * Remove the extra PT page reference. 7234 */ 7235 if (mpte != NULL) { 7236 mpte->ref_count--; 7237 KASSERT(mpte->ref_count > 0, 7238 ("pmap_enter: missing reference to page table page," 7239 " va: 0x%lx", va)); 7240 } 7241 7242 /* 7243 * Has the physical page changed? 7244 */ 7245 opa = origpte & PG_FRAME; 7246 if (opa == pa) { 7247 /* 7248 * No, might be a protection or wiring change. 7249 */ 7250 if ((origpte & PG_MANAGED) != 0 && 7251 (newpte & PG_RW) != 0) 7252 vm_page_aflag_set(m, PGA_WRITEABLE); 7253 if (((origpte ^ newpte) & ~(PG_M | PG_A)) == 0) 7254 goto unchanged; 7255 goto validate; 7256 } 7257 7258 /* 7259 * The physical page has changed. Temporarily invalidate 7260 * the mapping. This ensures that all threads sharing the 7261 * pmap keep a consistent view of the mapping, which is 7262 * necessary for the correct handling of COW faults. It 7263 * also permits reuse of the old mapping's PV entry, 7264 * avoiding an allocation. 7265 * 7266 * For consistency, handle unmanaged mappings the same way. 7267 */ 7268 origpte = pte_load_clear(pte); 7269 KASSERT((origpte & PG_FRAME) == opa, 7270 ("pmap_enter: unexpected pa update for %#lx", va)); 7271 if ((origpte & PG_MANAGED) != 0) { 7272 om = PHYS_TO_VM_PAGE(opa); 7273 7274 /* 7275 * The pmap lock is sufficient to synchronize with 7276 * concurrent calls to pmap_page_test_mappings() and 7277 * pmap_ts_referenced(). 7278 */ 7279 if ((origpte & (PG_M | PG_RW)) == (PG_M | PG_RW)) 7280 vm_page_dirty(om); 7281 if ((origpte & PG_A) != 0) { 7282 pmap_invalidate_page(pmap, va); 7283 vm_page_aflag_set(om, PGA_REFERENCED); 7284 } 7285 CHANGE_PV_LIST_LOCK_TO_PHYS(&lock, opa); 7286 pv = pmap_pvh_remove(&om->md, pmap, va); 7287 KASSERT(pv != NULL, 7288 ("pmap_enter: no PV entry for %#lx", va)); 7289 if ((newpte & PG_MANAGED) == 0) 7290 free_pv_entry(pmap, pv); 7291 7292 /* 7293 * The old page is likely COW, so check "writeable" 7294 * first. 7295 */ 7296 if ((om->a.flags & PGA_WRITEABLE) != 0 && 7297 !pmap_page_is_mapped_locked(om)) 7298 vm_page_aflag_clear(om, PGA_WRITEABLE); 7299 } else { 7300 /* 7301 * Since this mapping is unmanaged, assume that PG_A 7302 * is set. 7303 */ 7304 pmap_invalidate_page(pmap, va); 7305 } 7306 origpte = 0; 7307 } else { 7308 /* 7309 * Increment the counters. 7310 */ 7311 if ((newpte & PG_W) != 0) 7312 pmap->pm_stats.wired_count++; 7313 pmap_resident_count_adj(pmap, 1); 7314 } 7315 7316 /* 7317 * Enter on the PV list if part of our managed memory. 7318 */ 7319 if ((newpte & PG_MANAGED) != 0) { 7320 if (pv == NULL) { 7321 pv = get_pv_entry(pmap, &lock); 7322 pv->pv_va = va; 7323 } 7324 CHANGE_PV_LIST_LOCK_TO_PHYS(&lock, pa); 7325 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next); 7326 m->md.pv_gen++; 7327 if ((newpte & PG_RW) != 0) 7328 vm_page_aflag_set(m, PGA_WRITEABLE); 7329 } 7330 7331 /* 7332 * Update the PTE. 7333 */ 7334 if ((origpte & PG_V) != 0) { 7335 validate: 7336 origpte = pte_load_store(pte, newpte); 7337 KASSERT((origpte & PG_FRAME) == pa, 7338 ("pmap_enter: unexpected pa update for %#lx", va)); 7339 if ((newpte & PG_M) == 0 && (origpte & (PG_M | PG_RW)) == 7340 (PG_M | PG_RW)) { 7341 if ((origpte & PG_MANAGED) != 0) 7342 vm_page_dirty(m); 7343 7344 /* 7345 * Although the PTE may still have PG_RW set, TLB 7346 * invalidation may nonetheless be required because 7347 * the PTE no longer has PG_M set. 7348 */ 7349 } else if ((origpte & PG_NX) != 0 || (newpte & PG_NX) == 0) { 7350 /* 7351 * This PTE change does not require TLB invalidation. 7352 */ 7353 goto unchanged; 7354 } 7355 if ((origpte & PG_A) != 0) 7356 pmap_invalidate_page(pmap, va); 7357 } else 7358 pte_store(pte, newpte); 7359 7360 unchanged: 7361 7362 #if VM_NRESERVLEVEL > 0 7363 /* 7364 * If both the page table page and the reservation are fully 7365 * populated, then attempt promotion. 7366 */ 7367 if ((mpte == NULL || mpte->ref_count == NPTEPG) && 7368 (m->flags & PG_FICTITIOUS) == 0 && 7369 vm_reserv_level_iffullpop(m) == 0) 7370 (void)pmap_promote_pde(pmap, pde, va, mpte, &lock); 7371 #endif 7372 7373 rv = KERN_SUCCESS; 7374 out: 7375 if (lock != NULL) 7376 rw_wunlock(lock); 7377 PMAP_UNLOCK(pmap); 7378 return (rv); 7379 } 7380 7381 /* 7382 * Tries to create a read- and/or execute-only 2MB page mapping. Returns 7383 * KERN_SUCCESS if the mapping was created. Otherwise, returns an error 7384 * value. See pmap_enter_pde() for the possible error values when "no sleep", 7385 * "no replace", and "no reclaim" are specified. 7386 */ 7387 static int 7388 pmap_enter_2mpage(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, 7389 struct rwlock **lockp) 7390 { 7391 pd_entry_t newpde; 7392 pt_entry_t PG_V; 7393 7394 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 7395 PG_V = pmap_valid_bit(pmap); 7396 newpde = VM_PAGE_TO_PHYS(m) | 7397 pmap_cache_bits(pmap, m->md.pat_mode, true) | PG_PS | PG_V; 7398 if ((m->oflags & VPO_UNMANAGED) == 0) 7399 newpde |= PG_MANAGED; 7400 if ((prot & VM_PROT_EXECUTE) == 0) 7401 newpde |= pg_nx; 7402 if (va < VM_MAXUSER_ADDRESS) 7403 newpde |= PG_U; 7404 return (pmap_enter_pde(pmap, va, newpde, PMAP_ENTER_NOSLEEP | 7405 PMAP_ENTER_NOREPLACE | PMAP_ENTER_NORECLAIM, NULL, lockp)); 7406 } 7407 7408 /* 7409 * Returns true if every page table entry in the specified page table page is 7410 * zero. 7411 */ 7412 static bool 7413 pmap_every_pte_zero(vm_paddr_t pa) 7414 { 7415 pt_entry_t *pt_end, *pte; 7416 7417 KASSERT((pa & PAGE_MASK) == 0, ("pa is misaligned")); 7418 pte = PHYS_TO_DMAP(pa); 7419 for (pt_end = pte + NPTEPG; pte < pt_end; pte++) { 7420 if (*pte != 0) 7421 return (false); 7422 } 7423 return (true); 7424 } 7425 7426 /* 7427 * Tries to create the specified 2MB page mapping. Returns KERN_SUCCESS if 7428 * the mapping was created, and one of KERN_FAILURE, KERN_NO_SPACE, 7429 * KERN_PROTECTION_FAILURE, or KERN_RESOURCE_SHORTAGE otherwise. Returns 7430 * KERN_FAILURE if either (1) PMAP_ENTER_NOREPLACE was specified and a 4KB 7431 * page mapping already exists within the 2MB virtual address range starting 7432 * at the specified virtual address or (2) the requested 2MB page mapping is 7433 * not supported due to hardware errata. Returns KERN_NO_SPACE if 7434 * PMAP_ENTER_NOREPLACE was specified and a 2MB page mapping already exists at 7435 * the specified virtual address. Returns KERN_PROTECTION_FAILURE if the PKRU 7436 * settings are not the same across the 2MB virtual address range starting at 7437 * the specified virtual address. Returns KERN_RESOURCE_SHORTAGE if either 7438 * (1) PMAP_ENTER_NOSLEEP was specified and a page table page allocation 7439 * failed or (2) PMAP_ENTER_NORECLAIM was specified and a PV entry allocation 7440 * failed. 7441 * 7442 * The parameter "m" is only used when creating a managed, writeable mapping. 7443 */ 7444 static int 7445 pmap_enter_pde(pmap_t pmap, vm_offset_t va, pd_entry_t newpde, u_int flags, 7446 vm_page_t m, struct rwlock **lockp) 7447 { 7448 struct spglist free; 7449 pd_entry_t oldpde, *pde; 7450 pt_entry_t PG_G, PG_RW, PG_V; 7451 vm_page_t mt, pdpg; 7452 vm_page_t uwptpg; 7453 7454 PG_G = pmap_global_bit(pmap); 7455 PG_RW = pmap_rw_bit(pmap); 7456 KASSERT((newpde & (pmap_modified_bit(pmap) | PG_RW)) != PG_RW, 7457 ("pmap_enter_pde: newpde is missing PG_M")); 7458 KASSERT((flags & (PMAP_ENTER_NOREPLACE | PMAP_ENTER_NORECLAIM)) != 7459 PMAP_ENTER_NORECLAIM, 7460 ("pmap_enter_pde: flags is missing PMAP_ENTER_NOREPLACE")); 7461 PG_V = pmap_valid_bit(pmap); 7462 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 7463 7464 if (!pmap_allow_2m_x_page(pmap, pmap_pde_ept_executable(pmap, 7465 newpde))) { 7466 CTR2(KTR_PMAP, "pmap_enter_pde: 2m x blocked for va %#lx" 7467 " in pmap %p", va, pmap); 7468 return (KERN_FAILURE); 7469 } 7470 if ((pde = pmap_alloc_pde(pmap, va, &pdpg, (flags & 7471 PMAP_ENTER_NOSLEEP) != 0 ? NULL : lockp)) == NULL) { 7472 CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx" 7473 " in pmap %p", va, pmap); 7474 return (KERN_RESOURCE_SHORTAGE); 7475 } 7476 7477 /* 7478 * If pkru is not same for the whole pde range, return failure 7479 * and let vm_fault() cope. Check after pde allocation, since 7480 * it could sleep. 7481 */ 7482 if (!pmap_pkru_same(pmap, va, va + NBPDR, &newpde)) { 7483 pmap_abort_ptp(pmap, va, pdpg); 7484 return (KERN_PROTECTION_FAILURE); 7485 } 7486 7487 /* 7488 * If there are existing mappings, either abort or remove them. 7489 */ 7490 oldpde = *pde; 7491 if ((oldpde & PG_V) != 0) { 7492 KASSERT(pdpg == NULL || pdpg->ref_count > 1, 7493 ("pmap_enter_pde: pdpg's reference count is too low")); 7494 if ((flags & PMAP_ENTER_NOREPLACE) != 0) { 7495 if ((oldpde & PG_PS) != 0) { 7496 if (pdpg != NULL) 7497 pdpg->ref_count--; 7498 CTR2(KTR_PMAP, 7499 "pmap_enter_pde: no space for va %#lx" 7500 " in pmap %p", va, pmap); 7501 return (KERN_NO_SPACE); 7502 } else if (va < VM_MAXUSER_ADDRESS || 7503 !pmap_every_pte_zero(oldpde & PG_FRAME)) { 7504 if (pdpg != NULL) 7505 pdpg->ref_count--; 7506 CTR2(KTR_PMAP, 7507 "pmap_enter_pde: failure for va %#lx" 7508 " in pmap %p", va, pmap); 7509 return (KERN_FAILURE); 7510 } 7511 } 7512 /* Break the existing mapping(s). */ 7513 SLIST_INIT(&free); 7514 if ((oldpde & PG_PS) != 0) { 7515 /* 7516 * The reference to the PD page that was acquired by 7517 * pmap_alloc_pde() ensures that it won't be freed. 7518 * However, if the PDE resulted from a promotion, and 7519 * the mapping is not from kernel_pmap, then 7520 * a reserved PT page could be freed. 7521 */ 7522 (void)pmap_remove_pde(pmap, pde, va, false, &free, 7523 lockp); 7524 if ((oldpde & PG_G) == 0) 7525 pmap_invalidate_pde_page(pmap, va, oldpde); 7526 } else { 7527 if (va >= VM_MAXUSER_ADDRESS) { 7528 /* 7529 * Try to save the ptp in the trie 7530 * before any changes to mappings are 7531 * made. Abort on failure. 7532 */ 7533 mt = PHYS_TO_VM_PAGE(oldpde & PG_FRAME); 7534 if (pmap_insert_pt_page(pmap, mt, false, 7535 false)) { 7536 CTR1(KTR_PMAP, 7537 "pmap_enter_pde: cannot ins kern ptp va %#lx", 7538 va); 7539 return (KERN_RESOURCE_SHORTAGE); 7540 } 7541 /* 7542 * Both pmap_remove_pde() and 7543 * pmap_remove_ptes() will zero-fill 7544 * the kernel page table page. 7545 */ 7546 } 7547 pmap_delayed_invl_start(); 7548 if (pmap_remove_ptes(pmap, va, va + NBPDR, pde, &free, 7549 lockp)) 7550 pmap_invalidate_all(pmap); 7551 pmap_delayed_invl_finish(); 7552 } 7553 if (va < VM_MAXUSER_ADDRESS) { 7554 vm_page_free_pages_toq(&free, true); 7555 KASSERT(*pde == 0, ("pmap_enter_pde: non-zero pde %p", 7556 pde)); 7557 } else { 7558 KASSERT(SLIST_EMPTY(&free), 7559 ("pmap_enter_pde: freed kernel page table page")); 7560 } 7561 } 7562 7563 /* 7564 * Allocate leaf ptpage for wired userspace pages. 7565 */ 7566 uwptpg = NULL; 7567 if ((newpde & PG_W) != 0 && pmap != kernel_pmap) { 7568 uwptpg = pmap_alloc_pt_page(pmap, pmap_pde_pindex(va), 7569 VM_ALLOC_WIRED); 7570 if (uwptpg == NULL) { 7571 pmap_abort_ptp(pmap, va, pdpg); 7572 return (KERN_RESOURCE_SHORTAGE); 7573 } 7574 if (pmap_insert_pt_page(pmap, uwptpg, true, false)) { 7575 pmap_free_pt_page(pmap, uwptpg, false); 7576 pmap_abort_ptp(pmap, va, pdpg); 7577 return (KERN_RESOURCE_SHORTAGE); 7578 } 7579 7580 uwptpg->ref_count = NPTEPG; 7581 } 7582 if ((newpde & PG_MANAGED) != 0) { 7583 /* 7584 * Abort this mapping if its PV entry could not be created. 7585 */ 7586 if (!pmap_pv_insert_pde(pmap, va, newpde, flags, lockp)) { 7587 if (pdpg != NULL) 7588 pmap_abort_ptp(pmap, va, pdpg); 7589 else { 7590 KASSERT(va >= VM_MAXUSER_ADDRESS && 7591 (*pde & (PG_PS | PG_V)) == PG_V, 7592 ("pmap_enter_pde: invalid kernel PDE")); 7593 mt = pmap_remove_pt_page(pmap, va); 7594 KASSERT(mt != NULL, 7595 ("pmap_enter_pde: missing kernel PTP")); 7596 } 7597 if (uwptpg != NULL) { 7598 mt = pmap_remove_pt_page(pmap, va); 7599 KASSERT(mt == uwptpg, 7600 ("removed pt page %p, expected %p", mt, 7601 uwptpg)); 7602 uwptpg->ref_count = 1; 7603 pmap_free_pt_page(pmap, uwptpg, false); 7604 } 7605 CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx" 7606 " in pmap %p", va, pmap); 7607 return (KERN_RESOURCE_SHORTAGE); 7608 } 7609 if ((newpde & PG_RW) != 0) { 7610 for (mt = m; mt < &m[NBPDR / PAGE_SIZE]; mt++) 7611 vm_page_aflag_set(mt, PGA_WRITEABLE); 7612 } 7613 } 7614 7615 /* 7616 * Increment counters. 7617 */ 7618 if ((newpde & PG_W) != 0) 7619 pmap->pm_stats.wired_count += NBPDR / PAGE_SIZE; 7620 pmap_resident_count_adj(pmap, NBPDR / PAGE_SIZE); 7621 7622 /* 7623 * Map the superpage. (This is not a promoted mapping; there will not 7624 * be any lingering 4KB page mappings in the TLB.) 7625 */ 7626 pde_store(pde, newpde); 7627 7628 counter_u64_add(pmap_pde_mappings, 1); 7629 CTR2(KTR_PMAP, "pmap_enter_pde: success for va %#lx in pmap %p", 7630 va, pmap); 7631 return (KERN_SUCCESS); 7632 } 7633 7634 /* 7635 * Maps a sequence of resident pages belonging to the same object. 7636 * The sequence begins with the given page m_start. This page is 7637 * mapped at the given virtual address start. Each subsequent page is 7638 * mapped at a virtual address that is offset from start by the same 7639 * amount as the page is offset from m_start within the object. The 7640 * last page in the sequence is the page with the largest offset from 7641 * m_start that can be mapped at a virtual address less than the given 7642 * virtual address end. Not every virtual page between start and end 7643 * is mapped; only those for which a resident page exists with the 7644 * corresponding offset from m_start are mapped. 7645 */ 7646 void 7647 pmap_enter_object(pmap_t pmap, vm_offset_t start, vm_offset_t end, 7648 vm_page_t m_start, vm_prot_t prot) 7649 { 7650 struct pctrie_iter pages; 7651 struct rwlock *lock; 7652 vm_offset_t va; 7653 vm_page_t m, mpte; 7654 int rv; 7655 7656 VM_OBJECT_ASSERT_LOCKED(m_start->object); 7657 7658 mpte = NULL; 7659 vm_page_iter_limit_init(&pages, m_start->object, 7660 m_start->pindex + atop(end - start)); 7661 m = vm_radix_iter_lookup(&pages, m_start->pindex); 7662 lock = NULL; 7663 PMAP_LOCK(pmap); 7664 while (m != NULL) { 7665 va = start + ptoa(m->pindex - m_start->pindex); 7666 if ((va & PDRMASK) == 0 && va + NBPDR <= end && 7667 m->psind == 1 && pmap_ps_enabled(pmap) && 7668 ((rv = pmap_enter_2mpage(pmap, va, m, prot, &lock)) == 7669 KERN_SUCCESS || rv == KERN_NO_SPACE)) 7670 m = vm_radix_iter_jump(&pages, NBPDR / PAGE_SIZE); 7671 else { 7672 mpte = pmap_enter_quick_locked(pmap, va, m, prot, 7673 mpte, &lock); 7674 m = vm_radix_iter_step(&pages); 7675 } 7676 } 7677 if (lock != NULL) 7678 rw_wunlock(lock); 7679 PMAP_UNLOCK(pmap); 7680 } 7681 7682 /* 7683 * this code makes some *MAJOR* assumptions: 7684 * 1. Current pmap & pmap exists. 7685 * 2. Not wired. 7686 * 3. Read access. 7687 * 4. No page table pages. 7688 * but is *MUCH* faster than pmap_enter... 7689 */ 7690 7691 void 7692 pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot) 7693 { 7694 struct rwlock *lock; 7695 7696 lock = NULL; 7697 PMAP_LOCK(pmap); 7698 (void)pmap_enter_quick_locked(pmap, va, m, prot, NULL, &lock); 7699 if (lock != NULL) 7700 rw_wunlock(lock); 7701 PMAP_UNLOCK(pmap); 7702 } 7703 7704 static vm_page_t 7705 pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, 7706 vm_prot_t prot, vm_page_t mpte, struct rwlock **lockp) 7707 { 7708 pd_entry_t *pde; 7709 pt_entry_t newpte, *pte, PG_V; 7710 7711 KASSERT(!VA_IS_CLEANMAP(va) || 7712 (m->oflags & VPO_UNMANAGED) != 0, 7713 ("pmap_enter_quick_locked: managed mapping within the clean submap")); 7714 PG_V = pmap_valid_bit(pmap); 7715 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 7716 pde = NULL; 7717 7718 /* 7719 * In the case that a page table page is not 7720 * resident, we are creating it here. 7721 */ 7722 if (va < VM_MAXUSER_ADDRESS) { 7723 pdp_entry_t *pdpe; 7724 vm_pindex_t ptepindex; 7725 7726 /* 7727 * Calculate pagetable page index 7728 */ 7729 ptepindex = pmap_pde_pindex(va); 7730 if (mpte && (mpte->pindex == ptepindex)) { 7731 mpte->ref_count++; 7732 } else { 7733 /* 7734 * If the page table page is mapped, we just increment 7735 * the hold count, and activate it. Otherwise, we 7736 * attempt to allocate a page table page, passing NULL 7737 * instead of the PV list lock pointer because we don't 7738 * intend to sleep. If this attempt fails, we don't 7739 * retry. Instead, we give up. 7740 */ 7741 pdpe = pmap_pdpe(pmap, va); 7742 if (pdpe != NULL && (*pdpe & PG_V) != 0) { 7743 if ((*pdpe & PG_PS) != 0) 7744 return (NULL); 7745 pde = pmap_pdpe_to_pde(pdpe, va); 7746 if ((*pde & PG_V) != 0) { 7747 if ((*pde & PG_PS) != 0) 7748 return (NULL); 7749 mpte = PHYS_TO_VM_PAGE(*pde & PG_FRAME); 7750 mpte->ref_count++; 7751 } else { 7752 mpte = pmap_allocpte_alloc(pmap, 7753 ptepindex, NULL, va); 7754 if (mpte == NULL) 7755 return (NULL); 7756 } 7757 } else { 7758 mpte = pmap_allocpte_alloc(pmap, ptepindex, 7759 NULL, va); 7760 if (mpte == NULL) 7761 return (NULL); 7762 } 7763 } 7764 pte = VM_PAGE_TO_DMAP(mpte); 7765 pte = &pte[pmap_pte_index(va)]; 7766 } else { 7767 mpte = NULL; 7768 pte = vtopte(va); 7769 } 7770 if (*pte) { 7771 if (mpte != NULL) 7772 mpte->ref_count--; 7773 return (NULL); 7774 } 7775 7776 /* 7777 * Enter on the PV list if part of our managed memory. 7778 */ 7779 if ((m->oflags & VPO_UNMANAGED) == 0 && 7780 !pmap_try_insert_pv_entry(pmap, va, m, lockp)) { 7781 if (mpte != NULL) 7782 pmap_abort_ptp(pmap, va, mpte); 7783 return (NULL); 7784 } 7785 7786 /* 7787 * Increment counters 7788 */ 7789 pmap_resident_count_adj(pmap, 1); 7790 7791 newpte = VM_PAGE_TO_PHYS(m) | PG_V | 7792 pmap_cache_bits(pmap, m->md.pat_mode, false); 7793 if ((m->oflags & VPO_UNMANAGED) == 0) 7794 newpte |= PG_MANAGED; 7795 if ((prot & VM_PROT_EXECUTE) == 0) 7796 newpte |= pg_nx; 7797 if (va < VM_MAXUSER_ADDRESS) 7798 newpte |= PG_U | pmap_pkru_get(pmap, va); 7799 pte_store(pte, newpte); 7800 7801 #if VM_NRESERVLEVEL > 0 7802 /* 7803 * If both the PTP and the reservation are fully populated, then 7804 * attempt promotion. 7805 */ 7806 if ((prot & VM_PROT_NO_PROMOTE) == 0 && 7807 (mpte == NULL || mpte->ref_count == NPTEPG) && 7808 (m->flags & PG_FICTITIOUS) == 0 && 7809 vm_reserv_level_iffullpop(m) == 0) { 7810 if (pde == NULL) 7811 pde = pmap_pde(pmap, va); 7812 7813 /* 7814 * If promotion succeeds, then the next call to this function 7815 * should not be given the unmapped PTP as a hint. 7816 */ 7817 if (pmap_promote_pde(pmap, pde, va, mpte, lockp)) 7818 mpte = NULL; 7819 } 7820 #endif 7821 7822 return (mpte); 7823 } 7824 7825 /* 7826 * Make a temporary mapping for a physical address. This is only intended 7827 * to be used for panic dumps. 7828 */ 7829 void * 7830 pmap_kenter_temporary(vm_paddr_t pa, int i) 7831 { 7832 vm_offset_t va; 7833 7834 va = (vm_offset_t)crashdumpmap + (i * PAGE_SIZE); 7835 pmap_kenter(va, pa); 7836 pmap_invlpg(kernel_pmap, va); 7837 return ((void *)crashdumpmap); 7838 } 7839 7840 /* 7841 * This code maps large physical mmap regions into the 7842 * processor address space. Note that some shortcuts 7843 * are taken, but the code works. 7844 */ 7845 void 7846 pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, vm_object_t object, 7847 vm_pindex_t pindex, vm_size_t size) 7848 { 7849 struct pctrie_iter pages; 7850 pd_entry_t *pde; 7851 pt_entry_t PG_A, PG_M, PG_RW, PG_V; 7852 vm_paddr_t pa, ptepa; 7853 vm_page_t p, pdpg; 7854 int pat_mode; 7855 7856 PG_A = pmap_accessed_bit(pmap); 7857 PG_M = pmap_modified_bit(pmap); 7858 PG_V = pmap_valid_bit(pmap); 7859 PG_RW = pmap_rw_bit(pmap); 7860 7861 VM_OBJECT_ASSERT_WLOCKED(object); 7862 KASSERT(object->type == OBJT_DEVICE || object->type == OBJT_SG, 7863 ("pmap_object_init_pt: non-device object")); 7864 if ((addr & (NBPDR - 1)) == 0 && (size & (NBPDR - 1)) == 0) { 7865 if (!pmap_ps_enabled(pmap)) 7866 return; 7867 if (!vm_object_populate(object, pindex, pindex + atop(size))) 7868 return; 7869 vm_page_iter_init(&pages, object); 7870 p = vm_radix_iter_lookup(&pages, pindex); 7871 KASSERT(vm_page_all_valid(p), 7872 ("pmap_object_init_pt: invalid page %p", p)); 7873 pat_mode = p->md.pat_mode; 7874 7875 /* 7876 * Abort the mapping if the first page is not physically 7877 * aligned to a 2MB page boundary. 7878 */ 7879 ptepa = VM_PAGE_TO_PHYS(p); 7880 if (ptepa & (NBPDR - 1)) 7881 return; 7882 7883 /* 7884 * Skip the first page. Abort the mapping if the rest of 7885 * the pages are not physically contiguous or have differing 7886 * memory attributes. 7887 */ 7888 for (pa = ptepa + PAGE_SIZE; pa < ptepa + size; 7889 pa += PAGE_SIZE) { 7890 p = vm_radix_iter_next(&pages); 7891 KASSERT(vm_page_all_valid(p), 7892 ("pmap_object_init_pt: invalid page %p", p)); 7893 if (pa != VM_PAGE_TO_PHYS(p) || 7894 pat_mode != p->md.pat_mode) 7895 return; 7896 } 7897 7898 /* 7899 * Map using 2MB pages. Since "ptepa" is 2M aligned and 7900 * "size" is a multiple of 2M, adding the PAT setting to "pa" 7901 * will not affect the termination of this loop. 7902 */ 7903 PMAP_LOCK(pmap); 7904 for (pa = ptepa | pmap_cache_bits(pmap, pat_mode, true); 7905 pa < ptepa + size; pa += NBPDR) { 7906 pde = pmap_alloc_pde(pmap, addr, &pdpg, NULL); 7907 if (pde == NULL) { 7908 /* 7909 * The creation of mappings below is only an 7910 * optimization. If a page directory page 7911 * cannot be allocated without blocking, 7912 * continue on to the next mapping rather than 7913 * blocking. 7914 */ 7915 addr += NBPDR; 7916 continue; 7917 } 7918 if ((*pde & PG_V) == 0) { 7919 pde_store(pde, pa | PG_PS | PG_M | PG_A | 7920 PG_U | PG_RW | PG_V); 7921 pmap_resident_count_adj(pmap, NBPDR / PAGE_SIZE); 7922 counter_u64_add(pmap_pde_mappings, 1); 7923 } else { 7924 /* Continue on if the PDE is already valid. */ 7925 pdpg->ref_count--; 7926 KASSERT(pdpg->ref_count > 0, 7927 ("pmap_object_init_pt: missing reference " 7928 "to page directory page, va: 0x%lx", addr)); 7929 } 7930 addr += NBPDR; 7931 } 7932 PMAP_UNLOCK(pmap); 7933 } 7934 } 7935 7936 /* 7937 * Clear the wired attribute from the mappings for the specified range of 7938 * addresses in the given pmap. Every valid mapping within that range 7939 * must have the wired attribute set. In contrast, invalid mappings 7940 * cannot have the wired attribute set, so they are ignored. 7941 * 7942 * The wired attribute of the page table entry is not a hardware 7943 * feature, so there is no need to invalidate any TLB entries. 7944 * Since pmap_demote_pde() for the wired entry must never fail, 7945 * pmap_delayed_invl_start()/finish() calls around the 7946 * function are not needed. 7947 */ 7948 void 7949 pmap_unwire(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) 7950 { 7951 vm_offset_t va_next; 7952 pml4_entry_t *pml4e; 7953 pdp_entry_t *pdpe; 7954 pd_entry_t *pde; 7955 pt_entry_t *pte, PG_V, PG_G __diagused; 7956 7957 PG_V = pmap_valid_bit(pmap); 7958 PG_G = pmap_global_bit(pmap); 7959 PMAP_LOCK(pmap); 7960 for (; sva < eva; sva = va_next) { 7961 pml4e = pmap_pml4e(pmap, sva); 7962 if (pml4e == NULL || (*pml4e & PG_V) == 0) { 7963 va_next = (sva + NBPML4) & ~PML4MASK; 7964 if (va_next < sva) 7965 va_next = eva; 7966 continue; 7967 } 7968 7969 va_next = (sva + NBPDP) & ~PDPMASK; 7970 if (va_next < sva) 7971 va_next = eva; 7972 pdpe = pmap_pml4e_to_pdpe(pml4e, sva); 7973 if ((*pdpe & PG_V) == 0) 7974 continue; 7975 if ((*pdpe & PG_PS) != 0) { 7976 KASSERT(va_next <= eva, 7977 ("partial update of non-transparent 1G mapping " 7978 "pdpe %#lx sva %#lx eva %#lx va_next %#lx", 7979 *pdpe, sva, eva, va_next)); 7980 MPASS(pmap != kernel_pmap); /* XXXKIB */ 7981 MPASS((*pdpe & (PG_MANAGED | PG_G)) == 0); 7982 atomic_clear_long(pdpe, PG_W); 7983 pmap->pm_stats.wired_count -= NBPDP / PAGE_SIZE; 7984 continue; 7985 } 7986 7987 va_next = (sva + NBPDR) & ~PDRMASK; 7988 if (va_next < sva) 7989 va_next = eva; 7990 pde = pmap_pdpe_to_pde(pdpe, sva); 7991 if ((*pde & PG_V) == 0) 7992 continue; 7993 if ((*pde & PG_PS) != 0) { 7994 if ((*pde & PG_W) == 0) 7995 panic("pmap_unwire: pde %#jx is missing PG_W", 7996 (uintmax_t)*pde); 7997 7998 /* 7999 * Are we unwiring the entire large page? If not, 8000 * demote the mapping and fall through. 8001 */ 8002 if (sva + NBPDR == va_next && eva >= va_next) { 8003 atomic_clear_long(pde, PG_W); 8004 pmap->pm_stats.wired_count -= NBPDR / 8005 PAGE_SIZE; 8006 continue; 8007 } else if (!pmap_demote_pde(pmap, pde, sva)) 8008 panic("pmap_unwire: demotion failed"); 8009 } 8010 if (va_next > eva) 8011 va_next = eva; 8012 for (pte = pmap_pde_to_pte(pde, sva); sva != va_next; pte++, 8013 sva += PAGE_SIZE) { 8014 if ((*pte & PG_V) == 0) 8015 continue; 8016 if ((*pte & PG_W) == 0) 8017 panic("pmap_unwire: pte %#jx is missing PG_W", 8018 (uintmax_t)*pte); 8019 8020 /* 8021 * PG_W must be cleared atomically. Although the pmap 8022 * lock synchronizes access to PG_W, another processor 8023 * could be setting PG_M and/or PG_A concurrently. 8024 */ 8025 atomic_clear_long(pte, PG_W); 8026 pmap->pm_stats.wired_count--; 8027 } 8028 } 8029 PMAP_UNLOCK(pmap); 8030 } 8031 8032 /* 8033 * Copy the range specified by src_addr/len 8034 * from the source map to the range dst_addr/len 8035 * in the destination map. 8036 * 8037 * This routine is only advisory and need not do anything. 8038 */ 8039 void 8040 pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, vm_size_t len, 8041 vm_offset_t src_addr) 8042 { 8043 struct rwlock *lock; 8044 pml4_entry_t *pml4e; 8045 pdp_entry_t *pdpe; 8046 pd_entry_t *pde, srcptepaddr; 8047 pt_entry_t *dst_pte, PG_A, PG_M, PG_V, ptetemp, *src_pte; 8048 vm_offset_t addr, end_addr, va_next; 8049 vm_page_t dst_pdpg, dstmpte, srcmpte; 8050 8051 if (dst_addr != src_addr) 8052 return; 8053 8054 if (dst_pmap->pm_type != src_pmap->pm_type) 8055 return; 8056 8057 /* 8058 * EPT page table entries that require emulation of A/D bits are 8059 * sensitive to clearing the PG_A bit (aka EPT_PG_READ). Although 8060 * we clear PG_M (aka EPT_PG_WRITE) concomitantly, the PG_U bit 8061 * (aka EPT_PG_EXECUTE) could still be set. Since some EPT 8062 * implementations flag an EPT misconfiguration for exec-only 8063 * mappings we skip this function entirely for emulated pmaps. 8064 */ 8065 if (pmap_emulate_ad_bits(dst_pmap)) 8066 return; 8067 8068 end_addr = src_addr + len; 8069 lock = NULL; 8070 if (dst_pmap < src_pmap) { 8071 PMAP_LOCK(dst_pmap); 8072 PMAP_LOCK(src_pmap); 8073 } else { 8074 PMAP_LOCK(src_pmap); 8075 PMAP_LOCK(dst_pmap); 8076 } 8077 8078 PG_A = pmap_accessed_bit(dst_pmap); 8079 PG_M = pmap_modified_bit(dst_pmap); 8080 PG_V = pmap_valid_bit(dst_pmap); 8081 8082 for (addr = src_addr; addr < end_addr; addr = va_next) { 8083 KASSERT(addr < UPT_MIN_ADDRESS, 8084 ("pmap_copy: invalid to pmap_copy page tables")); 8085 8086 pml4e = pmap_pml4e(src_pmap, addr); 8087 if (pml4e == NULL || (*pml4e & PG_V) == 0) { 8088 va_next = (addr + NBPML4) & ~PML4MASK; 8089 if (va_next < addr) 8090 va_next = end_addr; 8091 continue; 8092 } 8093 8094 va_next = (addr + NBPDP) & ~PDPMASK; 8095 if (va_next < addr) 8096 va_next = end_addr; 8097 pdpe = pmap_pml4e_to_pdpe(pml4e, addr); 8098 if ((*pdpe & PG_V) == 0) 8099 continue; 8100 if ((*pdpe & PG_PS) != 0) { 8101 KASSERT(va_next <= end_addr, 8102 ("partial update of non-transparent 1G mapping " 8103 "pdpe %#lx sva %#lx eva %#lx va_next %#lx", 8104 *pdpe, addr, end_addr, va_next)); 8105 MPASS((addr & PDPMASK) == 0); 8106 MPASS((*pdpe & PG_MANAGED) == 0); 8107 srcptepaddr = *pdpe; 8108 pdpe = pmap_pdpe(dst_pmap, addr); 8109 if (pdpe == NULL) { 8110 if (pmap_allocpte_alloc(dst_pmap, 8111 pmap_pml4e_pindex(addr), NULL, addr) == 8112 NULL) 8113 break; 8114 pdpe = pmap_pdpe(dst_pmap, addr); 8115 } else { 8116 pml4e = pmap_pml4e(dst_pmap, addr); 8117 dst_pdpg = PHYS_TO_VM_PAGE(*pml4e & PG_FRAME); 8118 dst_pdpg->ref_count++; 8119 } 8120 KASSERT(*pdpe == 0, 8121 ("1G mapping present in dst pmap " 8122 "pdpe %#lx sva %#lx eva %#lx va_next %#lx", 8123 *pdpe, addr, end_addr, va_next)); 8124 *pdpe = srcptepaddr & ~PG_W; 8125 pmap_resident_count_adj(dst_pmap, NBPDP / PAGE_SIZE); 8126 continue; 8127 } 8128 8129 va_next = (addr + NBPDR) & ~PDRMASK; 8130 if (va_next < addr) 8131 va_next = end_addr; 8132 8133 pde = pmap_pdpe_to_pde(pdpe, addr); 8134 srcptepaddr = *pde; 8135 if (srcptepaddr == 0) 8136 continue; 8137 8138 if (srcptepaddr & PG_PS) { 8139 /* 8140 * We can only virtual copy whole superpages. 8141 */ 8142 if ((addr & PDRMASK) != 0 || addr + NBPDR > end_addr) 8143 continue; 8144 pde = pmap_alloc_pde(dst_pmap, addr, &dst_pdpg, NULL); 8145 if (pde == NULL) 8146 break; 8147 if (*pde == 0 && ((srcptepaddr & PG_MANAGED) == 0 || 8148 pmap_pv_insert_pde(dst_pmap, addr, srcptepaddr, 8149 PMAP_ENTER_NORECLAIM, &lock))) { 8150 /* 8151 * We leave the dirty bit unchanged because 8152 * managed read/write superpage mappings are 8153 * required to be dirty. However, managed 8154 * superpage mappings are not required to 8155 * have their accessed bit set, so we clear 8156 * it because we don't know if this mapping 8157 * will be used. 8158 */ 8159 srcptepaddr &= ~PG_W; 8160 if ((srcptepaddr & PG_MANAGED) != 0) 8161 srcptepaddr &= ~PG_A; 8162 *pde = srcptepaddr; 8163 pmap_resident_count_adj(dst_pmap, NBPDR / 8164 PAGE_SIZE); 8165 counter_u64_add(pmap_pde_mappings, 1); 8166 } else 8167 pmap_abort_ptp(dst_pmap, addr, dst_pdpg); 8168 continue; 8169 } 8170 8171 srcptepaddr &= PG_FRAME; 8172 srcmpte = PHYS_TO_VM_PAGE(srcptepaddr); 8173 KASSERT(srcmpte->ref_count > 0, 8174 ("pmap_copy: source page table page is unused")); 8175 8176 if (va_next > end_addr) 8177 va_next = end_addr; 8178 8179 src_pte = PHYS_TO_DMAP(srcptepaddr); 8180 src_pte = &src_pte[pmap_pte_index(addr)]; 8181 dstmpte = NULL; 8182 for (; addr < va_next; addr += PAGE_SIZE, src_pte++) { 8183 ptetemp = *src_pte; 8184 8185 /* 8186 * We only virtual copy managed pages. 8187 */ 8188 if ((ptetemp & PG_MANAGED) == 0) 8189 continue; 8190 8191 if (dstmpte != NULL) { 8192 KASSERT(dstmpte->pindex == 8193 pmap_pde_pindex(addr), 8194 ("dstmpte pindex/addr mismatch")); 8195 dstmpte->ref_count++; 8196 } else if ((dstmpte = pmap_allocpte(dst_pmap, addr, 8197 NULL)) == NULL) 8198 goto out; 8199 dst_pte = VM_PAGE_TO_DMAP(dstmpte); 8200 dst_pte = &dst_pte[pmap_pte_index(addr)]; 8201 if (*dst_pte == 0 && 8202 pmap_try_insert_pv_entry(dst_pmap, addr, 8203 PHYS_TO_VM_PAGE(ptetemp & PG_FRAME), &lock)) { 8204 /* 8205 * Clear the wired, modified, and accessed 8206 * (referenced) bits during the copy. 8207 */ 8208 *dst_pte = ptetemp & ~(PG_W | PG_M | PG_A); 8209 pmap_resident_count_adj(dst_pmap, 1); 8210 } else { 8211 pmap_abort_ptp(dst_pmap, addr, dstmpte); 8212 goto out; 8213 } 8214 /* Have we copied all of the valid mappings? */ 8215 if (dstmpte->ref_count >= srcmpte->ref_count) 8216 break; 8217 } 8218 } 8219 out: 8220 if (lock != NULL) 8221 rw_wunlock(lock); 8222 PMAP_UNLOCK(src_pmap); 8223 PMAP_UNLOCK(dst_pmap); 8224 } 8225 8226 int 8227 pmap_vmspace_copy(pmap_t dst_pmap, pmap_t src_pmap) 8228 { 8229 int error; 8230 8231 if (dst_pmap->pm_type != src_pmap->pm_type || 8232 dst_pmap->pm_type != PT_X86 || 8233 (cpu_stdext_feature2 & CPUID_STDEXT2_PKU) == 0) 8234 return (0); 8235 for (;;) { 8236 if (dst_pmap < src_pmap) { 8237 PMAP_LOCK(dst_pmap); 8238 PMAP_LOCK(src_pmap); 8239 } else { 8240 PMAP_LOCK(src_pmap); 8241 PMAP_LOCK(dst_pmap); 8242 } 8243 error = pmap_pkru_copy(dst_pmap, src_pmap); 8244 /* Clean up partial copy on failure due to no memory. */ 8245 if (error == ENOMEM) 8246 pmap_pkru_deassign_all(dst_pmap); 8247 PMAP_UNLOCK(src_pmap); 8248 PMAP_UNLOCK(dst_pmap); 8249 if (error != ENOMEM) 8250 break; 8251 vm_wait(NULL); 8252 } 8253 return (error); 8254 } 8255 8256 /* 8257 * Zero the specified hardware page. 8258 */ 8259 void 8260 pmap_zero_page(vm_page_t m) 8261 { 8262 void *va; 8263 8264 #ifdef TSLOG_PAGEZERO 8265 TSENTER(); 8266 #endif 8267 va = VM_PAGE_TO_DMAP(m); 8268 pagezero(va); 8269 #ifdef TSLOG_PAGEZERO 8270 TSEXIT(); 8271 #endif 8272 } 8273 8274 /* 8275 * Zero an area within a single hardware page. off and size must not 8276 * cover an area beyond a single hardware page. 8277 */ 8278 void 8279 pmap_zero_page_area(vm_page_t m, int off, int size) 8280 { 8281 void *va = VM_PAGE_TO_DMAP(m); 8282 8283 if (off == 0 && size == PAGE_SIZE) 8284 pagezero(va); 8285 else 8286 bzero((char *)va + off, size); 8287 } 8288 8289 /* 8290 * Copy 1 specified hardware page to another. 8291 */ 8292 void 8293 pmap_copy_page(vm_page_t msrc, vm_page_t mdst) 8294 { 8295 void *src = VM_PAGE_TO_DMAP(msrc); 8296 void *dst = VM_PAGE_TO_DMAP(mdst); 8297 8298 pagecopy(src, dst); 8299 } 8300 8301 int unmapped_buf_allowed = 1; 8302 8303 void 8304 pmap_copy_pages(vm_page_t ma[], vm_offset_t a_offset, vm_page_t mb[], 8305 vm_offset_t b_offset, int xfersize) 8306 { 8307 void *a_cp, *b_cp; 8308 vm_page_t pages[2]; 8309 void *vaddr[2]; 8310 vm_offset_t a_pg_offset, b_pg_offset; 8311 int cnt; 8312 bool mapped; 8313 8314 while (xfersize > 0) { 8315 a_pg_offset = a_offset & PAGE_MASK; 8316 pages[0] = ma[a_offset >> PAGE_SHIFT]; 8317 b_pg_offset = b_offset & PAGE_MASK; 8318 pages[1] = mb[b_offset >> PAGE_SHIFT]; 8319 cnt = min(xfersize, PAGE_SIZE - a_pg_offset); 8320 cnt = min(cnt, PAGE_SIZE - b_pg_offset); 8321 mapped = pmap_map_io_transient(pages, vaddr, 2, false); 8322 a_cp = (char *)vaddr[0] + a_pg_offset; 8323 b_cp = (char *)vaddr[1] + b_pg_offset; 8324 memcpy(b_cp, a_cp, cnt); 8325 if (__predict_false(mapped)) 8326 pmap_unmap_io_transient(pages, vaddr, 2, false); 8327 a_offset += cnt; 8328 b_offset += cnt; 8329 xfersize -= cnt; 8330 } 8331 } 8332 8333 /* 8334 * Returns true if the pmap's pv is one of the first 8335 * 16 pvs linked to from this page. This count may 8336 * be changed upwards or downwards in the future; it 8337 * is only necessary that true be returned for a small 8338 * subset of pmaps for proper page aging. 8339 */ 8340 bool 8341 pmap_page_exists_quick(pmap_t pmap, vm_page_t m) 8342 { 8343 struct md_page *pvh; 8344 struct rwlock *lock; 8345 pv_entry_t pv; 8346 int loops = 0; 8347 bool rv; 8348 8349 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 8350 ("pmap_page_exists_quick: page %p is not managed", m)); 8351 rv = false; 8352 lock = VM_PAGE_TO_PV_LIST_LOCK(m); 8353 rw_rlock(lock); 8354 TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) { 8355 if (PV_PMAP(pv) == pmap) { 8356 rv = true; 8357 break; 8358 } 8359 loops++; 8360 if (loops >= 16) 8361 break; 8362 } 8363 if (!rv && loops < 16 && (m->flags & PG_FICTITIOUS) == 0) { 8364 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); 8365 TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) { 8366 if (PV_PMAP(pv) == pmap) { 8367 rv = true; 8368 break; 8369 } 8370 loops++; 8371 if (loops >= 16) 8372 break; 8373 } 8374 } 8375 rw_runlock(lock); 8376 return (rv); 8377 } 8378 8379 /* 8380 * pmap_page_wired_mappings: 8381 * 8382 * Return the number of managed mappings to the given physical page 8383 * that are wired. 8384 */ 8385 int 8386 pmap_page_wired_mappings(vm_page_t m) 8387 { 8388 struct rwlock *lock; 8389 struct md_page *pvh; 8390 pmap_t pmap; 8391 pt_entry_t *pte; 8392 pv_entry_t pv; 8393 int count, md_gen, pvh_gen; 8394 8395 if ((m->oflags & VPO_UNMANAGED) != 0) 8396 return (0); 8397 lock = VM_PAGE_TO_PV_LIST_LOCK(m); 8398 rw_rlock(lock); 8399 restart: 8400 count = 0; 8401 TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) { 8402 pmap = PV_PMAP(pv); 8403 if (!PMAP_TRYLOCK(pmap)) { 8404 md_gen = m->md.pv_gen; 8405 rw_runlock(lock); 8406 PMAP_LOCK(pmap); 8407 rw_rlock(lock); 8408 if (md_gen != m->md.pv_gen) { 8409 PMAP_UNLOCK(pmap); 8410 goto restart; 8411 } 8412 } 8413 pte = pmap_pte(pmap, pv->pv_va); 8414 if ((*pte & PG_W) != 0) 8415 count++; 8416 PMAP_UNLOCK(pmap); 8417 } 8418 if ((m->flags & PG_FICTITIOUS) == 0) { 8419 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); 8420 TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) { 8421 pmap = PV_PMAP(pv); 8422 if (!PMAP_TRYLOCK(pmap)) { 8423 md_gen = m->md.pv_gen; 8424 pvh_gen = pvh->pv_gen; 8425 rw_runlock(lock); 8426 PMAP_LOCK(pmap); 8427 rw_rlock(lock); 8428 if (md_gen != m->md.pv_gen || 8429 pvh_gen != pvh->pv_gen) { 8430 PMAP_UNLOCK(pmap); 8431 goto restart; 8432 } 8433 } 8434 pte = pmap_pde(pmap, pv->pv_va); 8435 if ((*pte & PG_W) != 0) 8436 count++; 8437 PMAP_UNLOCK(pmap); 8438 } 8439 } 8440 rw_runlock(lock); 8441 return (count); 8442 } 8443 8444 /* 8445 * Returns true if the given page is mapped individually or as part of 8446 * a 2mpage. Otherwise, returns false. 8447 */ 8448 bool 8449 pmap_page_is_mapped(vm_page_t m) 8450 { 8451 struct rwlock *lock; 8452 bool rv; 8453 8454 if ((m->oflags & VPO_UNMANAGED) != 0) 8455 return (false); 8456 lock = VM_PAGE_TO_PV_LIST_LOCK(m); 8457 rw_rlock(lock); 8458 rv = pmap_page_is_mapped_locked(m); 8459 rw_runlock(lock); 8460 return (rv); 8461 } 8462 8463 /* 8464 * The page's PV list lock must be held. 8465 */ 8466 static __always_inline bool 8467 pmap_page_is_mapped_locked(vm_page_t m) 8468 { 8469 return (!TAILQ_EMPTY(&m->md.pv_list) || 8470 ((m->flags & PG_FICTITIOUS) == 0 && 8471 !TAILQ_EMPTY(&pa_to_pvh(VM_PAGE_TO_PHYS(m))->pv_list))); 8472 } 8473 8474 /* 8475 * Destroy all managed, non-wired mappings in the given user-space 8476 * pmap. This pmap cannot be active on any processor besides the 8477 * caller. 8478 * 8479 * This function cannot be applied to the kernel pmap. Moreover, it 8480 * is not intended for general use. It is only to be used during 8481 * process termination. Consequently, it can be implemented in ways 8482 * that make it faster than pmap_remove(). First, it can more quickly 8483 * destroy mappings by iterating over the pmap's collection of PV 8484 * entries, rather than searching the page table. Second, it doesn't 8485 * have to test and clear the page table entries atomically, because 8486 * no processor is currently accessing the user address space. In 8487 * particular, a page table entry's dirty bit won't change state once 8488 * this function starts. 8489 * 8490 * Although this function destroys all of the pmap's managed, 8491 * non-wired mappings, it can delay and batch the invalidation of TLB 8492 * entries without calling pmap_delayed_invl_start() and 8493 * pmap_delayed_invl_finish(). Because the pmap is not active on 8494 * any other processor, none of these TLB entries will ever be used 8495 * before their eventual invalidation. Consequently, there is no need 8496 * for either pmap_remove_all() or pmap_remove_write() to wait for 8497 * that eventual TLB invalidation. 8498 */ 8499 void 8500 pmap_remove_pages(pmap_t pmap) 8501 { 8502 pd_entry_t ptepde; 8503 pt_entry_t *pte, tpte; 8504 pt_entry_t PG_M, PG_RW, PG_V; 8505 struct spglist free; 8506 struct pv_chunklist free_chunks[PMAP_MEMDOM]; 8507 vm_page_t m, mpte, mt; 8508 pv_entry_t pv; 8509 struct md_page *pvh; 8510 struct pv_chunk *pc, *npc; 8511 struct rwlock *lock; 8512 int64_t bit; 8513 uint64_t inuse, bitmask; 8514 int allfree, field, i, idx; 8515 #ifdef PV_STATS 8516 int freed; 8517 #endif 8518 bool superpage; 8519 vm_paddr_t pa; 8520 8521 /* 8522 * Assert that the given pmap is only active on the current 8523 * CPU. Unfortunately, we cannot block another CPU from 8524 * activating the pmap while this function is executing. 8525 */ 8526 KASSERT(pmap == PCPU_GET(curpmap), ("non-current pmap %p", pmap)); 8527 #ifdef INVARIANTS 8528 { 8529 cpuset_t other_cpus; 8530 8531 other_cpus = all_cpus; 8532 critical_enter(); 8533 CPU_CLR(PCPU_GET(cpuid), &other_cpus); 8534 CPU_AND(&other_cpus, &other_cpus, &pmap->pm_active); 8535 critical_exit(); 8536 KASSERT(CPU_EMPTY(&other_cpus), ("pmap active %p", pmap)); 8537 } 8538 #endif 8539 8540 lock = NULL; 8541 PG_M = pmap_modified_bit(pmap); 8542 PG_V = pmap_valid_bit(pmap); 8543 PG_RW = pmap_rw_bit(pmap); 8544 8545 for (i = 0; i < PMAP_MEMDOM; i++) 8546 TAILQ_INIT(&free_chunks[i]); 8547 SLIST_INIT(&free); 8548 PMAP_LOCK(pmap); 8549 TAILQ_FOREACH_SAFE(pc, &pmap->pm_pvchunk, pc_list, npc) { 8550 allfree = 1; 8551 #ifdef PV_STATS 8552 freed = 0; 8553 #endif 8554 for (field = 0; field < _NPCM; field++) { 8555 inuse = ~pc->pc_map[field] & pc_freemask[field]; 8556 while (inuse != 0) { 8557 bit = bsfq(inuse); 8558 bitmask = 1UL << bit; 8559 idx = field * 64 + bit; 8560 pv = &pc->pc_pventry[idx]; 8561 inuse &= ~bitmask; 8562 8563 pte = pmap_pdpe(pmap, pv->pv_va); 8564 ptepde = *pte; 8565 pte = pmap_pdpe_to_pde(pte, pv->pv_va); 8566 tpte = *pte; 8567 if ((tpte & (PG_PS | PG_V)) == PG_V) { 8568 superpage = false; 8569 ptepde = tpte; 8570 pte = PHYS_TO_DMAP(tpte & PG_FRAME); 8571 pte = &pte[pmap_pte_index(pv->pv_va)]; 8572 tpte = *pte; 8573 } else { 8574 /* 8575 * Keep track whether 'tpte' is a 8576 * superpage explicitly instead of 8577 * relying on PG_PS being set. 8578 * 8579 * This is because PG_PS is numerically 8580 * identical to PG_PTE_PAT and thus a 8581 * regular page could be mistaken for 8582 * a superpage. 8583 */ 8584 superpage = true; 8585 } 8586 8587 if ((tpte & PG_V) == 0) { 8588 panic("bad pte va %lx pte %lx", 8589 pv->pv_va, tpte); 8590 } 8591 8592 /* 8593 * We cannot remove wired pages from a process' mapping at this time 8594 */ 8595 if (tpte & PG_W) { 8596 allfree = 0; 8597 continue; 8598 } 8599 8600 /* Mark free */ 8601 pc->pc_map[field] |= bitmask; 8602 8603 /* 8604 * Because this pmap is not active on other 8605 * processors, the dirty bit cannot have 8606 * changed state since we last loaded pte. 8607 */ 8608 pte_clear(pte); 8609 8610 if (superpage) 8611 pa = tpte & PG_PS_FRAME; 8612 else 8613 pa = tpte & PG_FRAME; 8614 8615 m = PHYS_TO_VM_PAGE(pa); 8616 KASSERT(m->phys_addr == pa, 8617 ("vm_page_t %p phys_addr mismatch %016jx %016jx", 8618 m, (uintmax_t)m->phys_addr, 8619 (uintmax_t)tpte)); 8620 8621 KASSERT((m->flags & PG_FICTITIOUS) != 0 || 8622 m < &vm_page_array[vm_page_array_size], 8623 ("pmap_remove_pages: bad tpte %#jx", 8624 (uintmax_t)tpte)); 8625 8626 /* 8627 * Update the vm_page_t clean/reference bits. 8628 */ 8629 if ((tpte & (PG_M | PG_RW)) == (PG_M | PG_RW)) { 8630 if (superpage) { 8631 for (mt = m; mt < &m[NBPDR / PAGE_SIZE]; mt++) 8632 vm_page_dirty(mt); 8633 } else 8634 vm_page_dirty(m); 8635 } 8636 8637 CHANGE_PV_LIST_LOCK_TO_VM_PAGE(&lock, m); 8638 8639 if (superpage) { 8640 pmap_resident_count_adj(pmap, -NBPDR / PAGE_SIZE); 8641 pvh = pa_to_pvh(tpte & PG_PS_FRAME); 8642 TAILQ_REMOVE(&pvh->pv_list, pv, pv_next); 8643 pvh->pv_gen++; 8644 if (TAILQ_EMPTY(&pvh->pv_list)) { 8645 for (mt = m; mt < &m[NBPDR / PAGE_SIZE]; mt++) 8646 if ((mt->a.flags & PGA_WRITEABLE) != 0 && 8647 TAILQ_EMPTY(&mt->md.pv_list)) 8648 vm_page_aflag_clear(mt, PGA_WRITEABLE); 8649 } 8650 mpte = pmap_remove_pt_page(pmap, pv->pv_va); 8651 if (mpte != NULL) { 8652 KASSERT(vm_page_any_valid(mpte), 8653 ("pmap_remove_pages: pte page not promoted")); 8654 pmap_pt_page_count_adj(pmap, -1); 8655 KASSERT(mpte->ref_count == NPTEPG, 8656 ("pmap_remove_pages: pte page reference count error")); 8657 mpte->ref_count = 0; 8658 pmap_add_delayed_free_list(mpte, &free, false); 8659 } 8660 } else { 8661 pmap_resident_count_adj(pmap, -1); 8662 TAILQ_REMOVE(&m->md.pv_list, pv, pv_next); 8663 m->md.pv_gen++; 8664 if ((m->a.flags & PGA_WRITEABLE) != 0 && 8665 !pmap_page_is_mapped_locked(m)) 8666 vm_page_aflag_clear(m, PGA_WRITEABLE); 8667 } 8668 pmap_unuse_pt(pmap, pv->pv_va, ptepde, &free); 8669 #ifdef PV_STATS 8670 freed++; 8671 #endif 8672 } 8673 } 8674 PV_STAT(counter_u64_add(pv_entry_frees, freed)); 8675 PV_STAT(counter_u64_add(pv_entry_spare, freed)); 8676 PV_STAT(counter_u64_add(pv_entry_count, -freed)); 8677 if (allfree) { 8678 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 8679 TAILQ_INSERT_TAIL(&free_chunks[pc_to_domain(pc)], pc, pc_list); 8680 } 8681 } 8682 if (lock != NULL) 8683 rw_wunlock(lock); 8684 pmap_invalidate_all(pmap); 8685 pmap_pkru_deassign_all(pmap); 8686 free_pv_chunk_batch((struct pv_chunklist *)&free_chunks); 8687 PMAP_UNLOCK(pmap); 8688 vm_page_free_pages_toq(&free, true); 8689 } 8690 8691 static bool 8692 pmap_page_test_mappings(vm_page_t m, bool accessed, bool modified) 8693 { 8694 struct rwlock *lock; 8695 pv_entry_t pv; 8696 struct md_page *pvh; 8697 pt_entry_t *pte, mask; 8698 pt_entry_t PG_A, PG_M, PG_RW, PG_V; 8699 pmap_t pmap; 8700 int md_gen, pvh_gen; 8701 bool rv; 8702 8703 rv = false; 8704 lock = VM_PAGE_TO_PV_LIST_LOCK(m); 8705 rw_rlock(lock); 8706 restart: 8707 TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) { 8708 pmap = PV_PMAP(pv); 8709 if (!PMAP_TRYLOCK(pmap)) { 8710 md_gen = m->md.pv_gen; 8711 rw_runlock(lock); 8712 PMAP_LOCK(pmap); 8713 rw_rlock(lock); 8714 if (md_gen != m->md.pv_gen) { 8715 PMAP_UNLOCK(pmap); 8716 goto restart; 8717 } 8718 } 8719 pte = pmap_pte(pmap, pv->pv_va); 8720 mask = 0; 8721 if (modified) { 8722 PG_M = pmap_modified_bit(pmap); 8723 PG_RW = pmap_rw_bit(pmap); 8724 mask |= PG_RW | PG_M; 8725 } 8726 if (accessed) { 8727 PG_A = pmap_accessed_bit(pmap); 8728 PG_V = pmap_valid_bit(pmap); 8729 mask |= PG_V | PG_A; 8730 } 8731 rv = (*pte & mask) == mask; 8732 PMAP_UNLOCK(pmap); 8733 if (rv) 8734 goto out; 8735 } 8736 if ((m->flags & PG_FICTITIOUS) == 0) { 8737 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); 8738 TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) { 8739 pmap = PV_PMAP(pv); 8740 if (!PMAP_TRYLOCK(pmap)) { 8741 md_gen = m->md.pv_gen; 8742 pvh_gen = pvh->pv_gen; 8743 rw_runlock(lock); 8744 PMAP_LOCK(pmap); 8745 rw_rlock(lock); 8746 if (md_gen != m->md.pv_gen || 8747 pvh_gen != pvh->pv_gen) { 8748 PMAP_UNLOCK(pmap); 8749 goto restart; 8750 } 8751 } 8752 pte = pmap_pde(pmap, pv->pv_va); 8753 mask = 0; 8754 if (modified) { 8755 PG_M = pmap_modified_bit(pmap); 8756 PG_RW = pmap_rw_bit(pmap); 8757 mask |= PG_RW | PG_M; 8758 } 8759 if (accessed) { 8760 PG_A = pmap_accessed_bit(pmap); 8761 PG_V = pmap_valid_bit(pmap); 8762 mask |= PG_V | PG_A; 8763 } 8764 rv = (*pte & mask) == mask; 8765 PMAP_UNLOCK(pmap); 8766 if (rv) 8767 goto out; 8768 } 8769 } 8770 out: 8771 rw_runlock(lock); 8772 return (rv); 8773 } 8774 8775 /* 8776 * pmap_is_modified: 8777 * 8778 * Return whether or not the specified physical page was modified 8779 * in any physical maps. 8780 */ 8781 bool 8782 pmap_is_modified(vm_page_t m) 8783 { 8784 8785 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 8786 ("pmap_is_modified: page %p is not managed", m)); 8787 8788 /* 8789 * If the page is not busied then this check is racy. 8790 */ 8791 if (!pmap_page_is_write_mapped(m)) 8792 return (false); 8793 return (pmap_page_test_mappings(m, false, true)); 8794 } 8795 8796 /* 8797 * pmap_is_prefaultable: 8798 * 8799 * Return whether or not the specified virtual address is eligible 8800 * for prefault. 8801 */ 8802 bool 8803 pmap_is_prefaultable(pmap_t pmap, vm_offset_t addr) 8804 { 8805 pd_entry_t *pde; 8806 pt_entry_t *pte, PG_V; 8807 bool rv; 8808 8809 PG_V = pmap_valid_bit(pmap); 8810 8811 /* 8812 * Return true if and only if the PTE for the specified virtual 8813 * address is allocated but invalid. 8814 */ 8815 rv = false; 8816 PMAP_LOCK(pmap); 8817 pde = pmap_pde(pmap, addr); 8818 if (pde != NULL && (*pde & (PG_PS | PG_V)) == PG_V) { 8819 pte = pmap_pde_to_pte(pde, addr); 8820 rv = (*pte & PG_V) == 0; 8821 } 8822 PMAP_UNLOCK(pmap); 8823 return (rv); 8824 } 8825 8826 /* 8827 * pmap_is_referenced: 8828 * 8829 * Return whether or not the specified physical page was referenced 8830 * in any physical maps. 8831 */ 8832 bool 8833 pmap_is_referenced(vm_page_t m) 8834 { 8835 8836 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 8837 ("pmap_is_referenced: page %p is not managed", m)); 8838 return (pmap_page_test_mappings(m, true, false)); 8839 } 8840 8841 /* 8842 * Clear the write and modified bits in each of the given page's mappings. 8843 */ 8844 void 8845 pmap_remove_write(vm_page_t m) 8846 { 8847 struct md_page *pvh; 8848 pmap_t pmap; 8849 struct rwlock *lock; 8850 pv_entry_t next_pv, pv; 8851 pd_entry_t *pde; 8852 pt_entry_t oldpte, *pte, PG_M, PG_RW; 8853 vm_offset_t va; 8854 int pvh_gen, md_gen; 8855 8856 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 8857 ("pmap_remove_write: page %p is not managed", m)); 8858 8859 vm_page_assert_busied(m); 8860 if (!pmap_page_is_write_mapped(m)) 8861 return; 8862 8863 lock = VM_PAGE_TO_PV_LIST_LOCK(m); 8864 pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy : 8865 pa_to_pvh(VM_PAGE_TO_PHYS(m)); 8866 rw_wlock(lock); 8867 retry: 8868 TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_next, next_pv) { 8869 pmap = PV_PMAP(pv); 8870 if (!PMAP_TRYLOCK(pmap)) { 8871 pvh_gen = pvh->pv_gen; 8872 rw_wunlock(lock); 8873 PMAP_LOCK(pmap); 8874 rw_wlock(lock); 8875 if (pvh_gen != pvh->pv_gen) { 8876 PMAP_UNLOCK(pmap); 8877 goto retry; 8878 } 8879 } 8880 PG_RW = pmap_rw_bit(pmap); 8881 va = pv->pv_va; 8882 pde = pmap_pde(pmap, va); 8883 if ((*pde & PG_RW) != 0) 8884 (void)pmap_demote_pde_locked(pmap, pde, va, &lock); 8885 KASSERT(lock == VM_PAGE_TO_PV_LIST_LOCK(m), 8886 ("inconsistent pv lock %p %p for page %p", 8887 lock, VM_PAGE_TO_PV_LIST_LOCK(m), m)); 8888 PMAP_UNLOCK(pmap); 8889 } 8890 TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) { 8891 pmap = PV_PMAP(pv); 8892 if (!PMAP_TRYLOCK(pmap)) { 8893 pvh_gen = pvh->pv_gen; 8894 md_gen = m->md.pv_gen; 8895 rw_wunlock(lock); 8896 PMAP_LOCK(pmap); 8897 rw_wlock(lock); 8898 if (pvh_gen != pvh->pv_gen || 8899 md_gen != m->md.pv_gen) { 8900 PMAP_UNLOCK(pmap); 8901 goto retry; 8902 } 8903 } 8904 PG_M = pmap_modified_bit(pmap); 8905 PG_RW = pmap_rw_bit(pmap); 8906 pde = pmap_pde(pmap, pv->pv_va); 8907 KASSERT((*pde & PG_PS) == 0, 8908 ("pmap_remove_write: found a 2mpage in page %p's pv list", 8909 m)); 8910 pte = pmap_pde_to_pte(pde, pv->pv_va); 8911 oldpte = *pte; 8912 if (oldpte & PG_RW) { 8913 while (!atomic_fcmpset_long(pte, &oldpte, oldpte & 8914 ~(PG_RW | PG_M))) 8915 cpu_spinwait(); 8916 if ((oldpte & PG_M) != 0) 8917 vm_page_dirty(m); 8918 pmap_invalidate_page(pmap, pv->pv_va); 8919 } 8920 PMAP_UNLOCK(pmap); 8921 } 8922 rw_wunlock(lock); 8923 vm_page_aflag_clear(m, PGA_WRITEABLE); 8924 pmap_delayed_invl_wait(m); 8925 } 8926 8927 /* 8928 * pmap_ts_referenced: 8929 * 8930 * Return a count of reference bits for a page, clearing those bits. 8931 * It is not necessary for every reference bit to be cleared, but it 8932 * is necessary that 0 only be returned when there are truly no 8933 * reference bits set. 8934 * 8935 * As an optimization, update the page's dirty field if a modified bit is 8936 * found while counting reference bits. This opportunistic update can be 8937 * performed at low cost and can eliminate the need for some future calls 8938 * to pmap_is_modified(). However, since this function stops after 8939 * finding PMAP_TS_REFERENCED_MAX reference bits, it may not detect some 8940 * dirty pages. Those dirty pages will only be detected by a future call 8941 * to pmap_is_modified(). 8942 * 8943 * A DI block is not needed within this function, because 8944 * invalidations are performed before the PV list lock is 8945 * released. 8946 */ 8947 int 8948 pmap_ts_referenced(vm_page_t m) 8949 { 8950 struct md_page *pvh; 8951 pv_entry_t pv, pvf; 8952 pmap_t pmap; 8953 struct rwlock *lock; 8954 pd_entry_t oldpde, *pde; 8955 pt_entry_t *pte, PG_A, PG_M, PG_RW; 8956 vm_offset_t va; 8957 vm_paddr_t pa; 8958 int cleared, md_gen, not_cleared, pvh_gen; 8959 struct spglist free; 8960 bool demoted; 8961 8962 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 8963 ("pmap_ts_referenced: page %p is not managed", m)); 8964 SLIST_INIT(&free); 8965 cleared = 0; 8966 pa = VM_PAGE_TO_PHYS(m); 8967 lock = PHYS_TO_PV_LIST_LOCK(pa); 8968 pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy : pa_to_pvh(pa); 8969 rw_wlock(lock); 8970 retry: 8971 not_cleared = 0; 8972 if ((pvf = TAILQ_FIRST(&pvh->pv_list)) == NULL) 8973 goto small_mappings; 8974 pv = pvf; 8975 do { 8976 if (pvf == NULL) 8977 pvf = pv; 8978 pmap = PV_PMAP(pv); 8979 if (!PMAP_TRYLOCK(pmap)) { 8980 pvh_gen = pvh->pv_gen; 8981 rw_wunlock(lock); 8982 PMAP_LOCK(pmap); 8983 rw_wlock(lock); 8984 if (pvh_gen != pvh->pv_gen) { 8985 PMAP_UNLOCK(pmap); 8986 goto retry; 8987 } 8988 } 8989 PG_A = pmap_accessed_bit(pmap); 8990 PG_M = pmap_modified_bit(pmap); 8991 PG_RW = pmap_rw_bit(pmap); 8992 va = pv->pv_va; 8993 pde = pmap_pde(pmap, pv->pv_va); 8994 oldpde = *pde; 8995 if ((oldpde & (PG_M | PG_RW)) == (PG_M | PG_RW)) { 8996 /* 8997 * Although "oldpde" is mapping a 2MB page, because 8998 * this function is called at a 4KB page granularity, 8999 * we only update the 4KB page under test. 9000 */ 9001 vm_page_dirty(m); 9002 } 9003 if ((oldpde & PG_A) != 0) { 9004 /* 9005 * Since this reference bit is shared by 512 4KB 9006 * pages, it should not be cleared every time it is 9007 * tested. Apply a simple "hash" function on the 9008 * physical page number, the virtual superpage number, 9009 * and the pmap address to select one 4KB page out of 9010 * the 512 on which testing the reference bit will 9011 * result in clearing that reference bit. This 9012 * function is designed to avoid the selection of the 9013 * same 4KB page for every 2MB page mapping. 9014 * 9015 * On demotion, a mapping that hasn't been referenced 9016 * is simply destroyed. To avoid the possibility of a 9017 * subsequent page fault on a demoted wired mapping, 9018 * always leave its reference bit set. Moreover, 9019 * since the superpage is wired, the current state of 9020 * its reference bit won't affect page replacement. 9021 */ 9022 if ((((pa >> PAGE_SHIFT) ^ (pv->pv_va >> PDRSHIFT) ^ 9023 (uintptr_t)pmap) & (NPTEPG - 1)) == 0 && 9024 (oldpde & PG_W) == 0) { 9025 if (safe_to_clear_referenced(pmap, oldpde)) { 9026 atomic_clear_long(pde, PG_A); 9027 pmap_invalidate_page(pmap, pv->pv_va); 9028 demoted = false; 9029 } else if (pmap_demote_pde_locked(pmap, pde, 9030 pv->pv_va, &lock)) { 9031 /* 9032 * Remove the mapping to a single page 9033 * so that a subsequent access may 9034 * repromote. Since the underlying 9035 * page table page is fully populated, 9036 * this removal never frees a page 9037 * table page. 9038 */ 9039 demoted = true; 9040 va += VM_PAGE_TO_PHYS(m) - (oldpde & 9041 PG_PS_FRAME); 9042 pte = pmap_pde_to_pte(pde, va); 9043 pmap_remove_pte(pmap, pte, va, *pde, 9044 NULL, &lock); 9045 pmap_invalidate_page(pmap, va); 9046 } else 9047 demoted = true; 9048 9049 if (demoted) { 9050 /* 9051 * The superpage mapping was removed 9052 * entirely and therefore 'pv' is no 9053 * longer valid. 9054 */ 9055 if (pvf == pv) 9056 pvf = NULL; 9057 pv = NULL; 9058 } 9059 cleared++; 9060 KASSERT(lock == VM_PAGE_TO_PV_LIST_LOCK(m), 9061 ("inconsistent pv lock %p %p for page %p", 9062 lock, VM_PAGE_TO_PV_LIST_LOCK(m), m)); 9063 } else 9064 not_cleared++; 9065 } 9066 PMAP_UNLOCK(pmap); 9067 /* Rotate the PV list if it has more than one entry. */ 9068 if (pv != NULL && TAILQ_NEXT(pv, pv_next) != NULL) { 9069 TAILQ_REMOVE(&pvh->pv_list, pv, pv_next); 9070 TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_next); 9071 pvh->pv_gen++; 9072 } 9073 if (cleared + not_cleared >= PMAP_TS_REFERENCED_MAX) 9074 goto out; 9075 } while ((pv = TAILQ_FIRST(&pvh->pv_list)) != pvf); 9076 small_mappings: 9077 if ((pvf = TAILQ_FIRST(&m->md.pv_list)) == NULL) 9078 goto out; 9079 pv = pvf; 9080 do { 9081 if (pvf == NULL) 9082 pvf = pv; 9083 pmap = PV_PMAP(pv); 9084 if (!PMAP_TRYLOCK(pmap)) { 9085 pvh_gen = pvh->pv_gen; 9086 md_gen = m->md.pv_gen; 9087 rw_wunlock(lock); 9088 PMAP_LOCK(pmap); 9089 rw_wlock(lock); 9090 if (pvh_gen != pvh->pv_gen || md_gen != m->md.pv_gen) { 9091 PMAP_UNLOCK(pmap); 9092 goto retry; 9093 } 9094 } 9095 PG_A = pmap_accessed_bit(pmap); 9096 PG_M = pmap_modified_bit(pmap); 9097 PG_RW = pmap_rw_bit(pmap); 9098 pde = pmap_pde(pmap, pv->pv_va); 9099 KASSERT((*pde & PG_PS) == 0, 9100 ("pmap_ts_referenced: found a 2mpage in page %p's pv list", 9101 m)); 9102 pte = pmap_pde_to_pte(pde, pv->pv_va); 9103 if ((*pte & (PG_M | PG_RW)) == (PG_M | PG_RW)) 9104 vm_page_dirty(m); 9105 if ((*pte & PG_A) != 0) { 9106 if (safe_to_clear_referenced(pmap, *pte)) { 9107 atomic_clear_long(pte, PG_A); 9108 pmap_invalidate_page(pmap, pv->pv_va); 9109 cleared++; 9110 } else if ((*pte & PG_W) == 0) { 9111 /* 9112 * Wired pages cannot be paged out so 9113 * doing accessed bit emulation for 9114 * them is wasted effort. We do the 9115 * hard work for unwired pages only. 9116 */ 9117 pmap_remove_pte(pmap, pte, pv->pv_va, 9118 *pde, &free, &lock); 9119 pmap_invalidate_page(pmap, pv->pv_va); 9120 cleared++; 9121 if (pvf == pv) 9122 pvf = NULL; 9123 pv = NULL; 9124 KASSERT(lock == VM_PAGE_TO_PV_LIST_LOCK(m), 9125 ("inconsistent pv lock %p %p for page %p", 9126 lock, VM_PAGE_TO_PV_LIST_LOCK(m), m)); 9127 } else 9128 not_cleared++; 9129 } 9130 PMAP_UNLOCK(pmap); 9131 /* Rotate the PV list if it has more than one entry. */ 9132 if (pv != NULL && TAILQ_NEXT(pv, pv_next) != NULL) { 9133 TAILQ_REMOVE(&m->md.pv_list, pv, pv_next); 9134 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next); 9135 m->md.pv_gen++; 9136 } 9137 } while ((pv = TAILQ_FIRST(&m->md.pv_list)) != pvf && cleared + 9138 not_cleared < PMAP_TS_REFERENCED_MAX); 9139 out: 9140 rw_wunlock(lock); 9141 vm_page_free_pages_toq(&free, true); 9142 return (cleared + not_cleared); 9143 } 9144 9145 /* 9146 * Apply the given advice to the specified range of addresses within the 9147 * given pmap. Depending on the advice, clear the referenced and/or 9148 * modified flags in each mapping and set the mapped page's dirty field. 9149 */ 9150 void 9151 pmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int advice) 9152 { 9153 struct rwlock *lock; 9154 pml4_entry_t *pml4e; 9155 pdp_entry_t *pdpe; 9156 pd_entry_t oldpde, *pde; 9157 pt_entry_t *pte, PG_A, PG_G, PG_M, PG_RW, PG_V; 9158 vm_offset_t va, va_next; 9159 vm_page_t m; 9160 bool anychanged; 9161 9162 if (advice != MADV_DONTNEED && advice != MADV_FREE) 9163 return; 9164 9165 /* 9166 * A/D bit emulation requires an alternate code path when clearing 9167 * the modified and accessed bits below. Since this function is 9168 * advisory in nature we skip it entirely for pmaps that require 9169 * A/D bit emulation. 9170 */ 9171 if (pmap_emulate_ad_bits(pmap)) 9172 return; 9173 9174 PG_A = pmap_accessed_bit(pmap); 9175 PG_G = pmap_global_bit(pmap); 9176 PG_M = pmap_modified_bit(pmap); 9177 PG_V = pmap_valid_bit(pmap); 9178 PG_RW = pmap_rw_bit(pmap); 9179 anychanged = false; 9180 pmap_delayed_invl_start(); 9181 PMAP_LOCK(pmap); 9182 for (; sva < eva; sva = va_next) { 9183 pml4e = pmap_pml4e(pmap, sva); 9184 if (pml4e == NULL || (*pml4e & PG_V) == 0) { 9185 va_next = (sva + NBPML4) & ~PML4MASK; 9186 if (va_next < sva) 9187 va_next = eva; 9188 continue; 9189 } 9190 9191 va_next = (sva + NBPDP) & ~PDPMASK; 9192 if (va_next < sva) 9193 va_next = eva; 9194 pdpe = pmap_pml4e_to_pdpe(pml4e, sva); 9195 if ((*pdpe & PG_V) == 0) 9196 continue; 9197 if ((*pdpe & PG_PS) != 0) 9198 continue; 9199 9200 va_next = (sva + NBPDR) & ~PDRMASK; 9201 if (va_next < sva) 9202 va_next = eva; 9203 pde = pmap_pdpe_to_pde(pdpe, sva); 9204 oldpde = *pde; 9205 if ((oldpde & PG_V) == 0) 9206 continue; 9207 else if ((oldpde & PG_PS) != 0) { 9208 if ((oldpde & PG_MANAGED) == 0) 9209 continue; 9210 lock = NULL; 9211 if (!pmap_demote_pde_locked(pmap, pde, sva, &lock)) { 9212 if (lock != NULL) 9213 rw_wunlock(lock); 9214 9215 /* 9216 * The large page mapping was destroyed. 9217 */ 9218 continue; 9219 } 9220 9221 /* 9222 * Unless the page mappings are wired, remove the 9223 * mapping to a single page so that a subsequent 9224 * access may repromote. Choosing the last page 9225 * within the address range [sva, min(va_next, eva)) 9226 * generally results in more repromotions. Since the 9227 * underlying page table page is fully populated, this 9228 * removal never frees a page table page. 9229 */ 9230 if ((oldpde & PG_W) == 0) { 9231 va = eva; 9232 if (va > va_next) 9233 va = va_next; 9234 va -= PAGE_SIZE; 9235 KASSERT(va >= sva, 9236 ("pmap_advise: no address gap")); 9237 pte = pmap_pde_to_pte(pde, va); 9238 KASSERT((*pte & PG_V) != 0, 9239 ("pmap_advise: invalid PTE")); 9240 pmap_remove_pte(pmap, pte, va, *pde, NULL, 9241 &lock); 9242 anychanged = true; 9243 } 9244 if (lock != NULL) 9245 rw_wunlock(lock); 9246 } 9247 if (va_next > eva) 9248 va_next = eva; 9249 va = va_next; 9250 for (pte = pmap_pde_to_pte(pde, sva); sva != va_next; pte++, 9251 sva += PAGE_SIZE) { 9252 if ((*pte & (PG_MANAGED | PG_V)) != (PG_MANAGED | PG_V)) 9253 goto maybe_invlrng; 9254 else if ((*pte & (PG_M | PG_RW)) == (PG_M | PG_RW)) { 9255 if (advice == MADV_DONTNEED) { 9256 /* 9257 * Future calls to pmap_is_modified() 9258 * can be avoided by making the page 9259 * dirty now. 9260 */ 9261 m = PHYS_TO_VM_PAGE(*pte & PG_FRAME); 9262 vm_page_dirty(m); 9263 } 9264 atomic_clear_long(pte, PG_M | PG_A); 9265 } else if ((*pte & PG_A) != 0) 9266 atomic_clear_long(pte, PG_A); 9267 else 9268 goto maybe_invlrng; 9269 9270 if ((*pte & PG_G) != 0) { 9271 if (va == va_next) 9272 va = sva; 9273 } else 9274 anychanged = true; 9275 continue; 9276 maybe_invlrng: 9277 if (va != va_next) { 9278 pmap_invalidate_range(pmap, va, sva); 9279 va = va_next; 9280 } 9281 } 9282 if (va != va_next) 9283 pmap_invalidate_range(pmap, va, sva); 9284 } 9285 if (anychanged) 9286 pmap_invalidate_all(pmap); 9287 PMAP_UNLOCK(pmap); 9288 pmap_delayed_invl_finish(); 9289 } 9290 9291 /* 9292 * Clear the modify bits on the specified physical page. 9293 */ 9294 void 9295 pmap_clear_modify(vm_page_t m) 9296 { 9297 struct md_page *pvh; 9298 pmap_t pmap; 9299 pv_entry_t next_pv, pv; 9300 pd_entry_t oldpde, *pde; 9301 pt_entry_t *pte, PG_M, PG_RW; 9302 struct rwlock *lock; 9303 vm_offset_t va; 9304 int md_gen, pvh_gen; 9305 9306 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 9307 ("pmap_clear_modify: page %p is not managed", m)); 9308 vm_page_assert_busied(m); 9309 9310 if (!pmap_page_is_write_mapped(m)) 9311 return; 9312 pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy : 9313 pa_to_pvh(VM_PAGE_TO_PHYS(m)); 9314 lock = VM_PAGE_TO_PV_LIST_LOCK(m); 9315 rw_wlock(lock); 9316 restart: 9317 TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_next, next_pv) { 9318 pmap = PV_PMAP(pv); 9319 if (!PMAP_TRYLOCK(pmap)) { 9320 pvh_gen = pvh->pv_gen; 9321 rw_wunlock(lock); 9322 PMAP_LOCK(pmap); 9323 rw_wlock(lock); 9324 if (pvh_gen != pvh->pv_gen) { 9325 PMAP_UNLOCK(pmap); 9326 goto restart; 9327 } 9328 } 9329 PG_M = pmap_modified_bit(pmap); 9330 PG_RW = pmap_rw_bit(pmap); 9331 va = pv->pv_va; 9332 pde = pmap_pde(pmap, va); 9333 oldpde = *pde; 9334 /* If oldpde has PG_RW set, then it also has PG_M set. */ 9335 if ((oldpde & PG_RW) != 0 && 9336 pmap_demote_pde_locked(pmap, pde, va, &lock) && 9337 (oldpde & PG_W) == 0) { 9338 /* 9339 * Write protect the mapping to a single page so that 9340 * a subsequent write access may repromote. 9341 */ 9342 va += VM_PAGE_TO_PHYS(m) - (oldpde & PG_PS_FRAME); 9343 pte = pmap_pde_to_pte(pde, va); 9344 atomic_clear_long(pte, PG_M | PG_RW); 9345 vm_page_dirty(m); 9346 pmap_invalidate_page(pmap, va); 9347 } 9348 PMAP_UNLOCK(pmap); 9349 } 9350 TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) { 9351 pmap = PV_PMAP(pv); 9352 if (!PMAP_TRYLOCK(pmap)) { 9353 md_gen = m->md.pv_gen; 9354 pvh_gen = pvh->pv_gen; 9355 rw_wunlock(lock); 9356 PMAP_LOCK(pmap); 9357 rw_wlock(lock); 9358 if (pvh_gen != pvh->pv_gen || md_gen != m->md.pv_gen) { 9359 PMAP_UNLOCK(pmap); 9360 goto restart; 9361 } 9362 } 9363 PG_M = pmap_modified_bit(pmap); 9364 PG_RW = pmap_rw_bit(pmap); 9365 pde = pmap_pde(pmap, pv->pv_va); 9366 KASSERT((*pde & PG_PS) == 0, ("pmap_clear_modify: found" 9367 " a 2mpage in page %p's pv list", m)); 9368 pte = pmap_pde_to_pte(pde, pv->pv_va); 9369 if ((*pte & (PG_M | PG_RW)) == (PG_M | PG_RW)) { 9370 atomic_clear_long(pte, PG_M); 9371 pmap_invalidate_page(pmap, pv->pv_va); 9372 } 9373 PMAP_UNLOCK(pmap); 9374 } 9375 rw_wunlock(lock); 9376 } 9377 9378 /* 9379 * Miscellaneous support routines follow 9380 */ 9381 9382 /* Adjust the properties for a leaf page table entry. */ 9383 static __inline void 9384 pmap_pte_props(pt_entry_t *pte, u_long bits, u_long mask) 9385 { 9386 u_long opte, npte; 9387 9388 opte = *(u_long *)pte; 9389 do { 9390 npte = opte & ~mask; 9391 npte |= bits; 9392 } while (npte != opte && !atomic_fcmpset_long((u_long *)pte, &opte, 9393 npte)); 9394 } 9395 9396 /* 9397 * Map a set of physical memory pages into the kernel virtual 9398 * address space. Return a pointer to where it is mapped. This 9399 * routine is intended to be used for mapping device memory, 9400 * NOT real memory. 9401 */ 9402 static void * 9403 pmap_mapdev_internal(vm_paddr_t pa, vm_size_t size, int mode, int flags) 9404 { 9405 struct pmap_preinit_mapping *ppim; 9406 char *va; 9407 vm_offset_t offset; 9408 vm_size_t tmpsize; 9409 int i; 9410 9411 offset = pa & PAGE_MASK; 9412 size = round_page(offset + size); 9413 pa = trunc_page(pa); 9414 9415 if (!pmap_initialized) { 9416 va = NULL; 9417 for (i = 0; i < PMAP_PREINIT_MAPPING_COUNT; i++) { 9418 ppim = pmap_preinit_mapping + i; 9419 if (ppim->va == NULL) { 9420 ppim->pa = pa; 9421 ppim->sz = size; 9422 ppim->mode = mode; 9423 ppim->va = (void *)virtual_avail; 9424 virtual_avail += size; 9425 va = ppim->va; 9426 break; 9427 } 9428 } 9429 if (va == NULL) 9430 panic("%s: too many preinit mappings", __func__); 9431 } else { 9432 /* 9433 * If we have a preinit mapping, reuse it. 9434 */ 9435 for (i = 0; i < PMAP_PREINIT_MAPPING_COUNT; i++) { 9436 ppim = pmap_preinit_mapping + i; 9437 if (ppim->pa == pa && ppim->sz == size && 9438 (ppim->mode == mode || 9439 (flags & MAPDEV_SETATTR) == 0)) 9440 return ((char *)ppim->va + offset); 9441 } 9442 /* 9443 * If the specified range of physical addresses fits within 9444 * the direct map window, use the direct map. 9445 */ 9446 if (pa < dmaplimit && pa + size <= dmaplimit) { 9447 va = PHYS_TO_DMAP(pa); 9448 if ((flags & MAPDEV_SETATTR) != 0) { 9449 PMAP_LOCK(kernel_pmap); 9450 i = pmap_change_props_locked(va, size, 9451 PROT_NONE, mode, flags); 9452 PMAP_UNLOCK(kernel_pmap); 9453 } else 9454 i = 0; 9455 if (!i) 9456 return (va + offset); 9457 } 9458 va = kva_alloc(size); 9459 if (va == NULL) 9460 panic("%s: Couldn't allocate KVA", __func__); 9461 } 9462 for (tmpsize = 0; tmpsize < size; tmpsize += PAGE_SIZE) 9463 pmap_kenter_attr((vm_offset_t)va + tmpsize, pa + tmpsize, mode); 9464 pmap_invalidate_range(kernel_pmap, (vm_offset_t)va, 9465 (vm_offset_t)va + tmpsize); 9466 if ((flags & MAPDEV_FLUSHCACHE) != 0) 9467 pmap_invalidate_cache_range((vm_offset_t)va, 9468 (vm_offset_t)va + tmpsize); 9469 return (va + offset); 9470 } 9471 9472 void * 9473 pmap_mapdev_attr(vm_paddr_t pa, vm_size_t size, int mode) 9474 { 9475 9476 return (pmap_mapdev_internal(pa, size, mode, MAPDEV_FLUSHCACHE | 9477 MAPDEV_SETATTR)); 9478 } 9479 9480 void * 9481 pmap_mapdev(vm_paddr_t pa, vm_size_t size) 9482 { 9483 9484 return (pmap_mapdev_attr(pa, size, PAT_UNCACHEABLE)); 9485 } 9486 9487 void * 9488 pmap_mapdev_pciecfg(vm_paddr_t pa, vm_size_t size) 9489 { 9490 9491 return (pmap_mapdev_internal(pa, size, PAT_UNCACHEABLE, 9492 MAPDEV_SETATTR)); 9493 } 9494 9495 void * 9496 pmap_mapbios(vm_paddr_t pa, vm_size_t size) 9497 { 9498 9499 return (pmap_mapdev_internal(pa, size, PAT_WRITE_BACK, 9500 MAPDEV_FLUSHCACHE)); 9501 } 9502 9503 void 9504 pmap_unmapdev(void *p, vm_size_t size) 9505 { 9506 struct pmap_preinit_mapping *ppim; 9507 char *va; 9508 vm_offset_t offset; 9509 int i; 9510 9511 va = p; 9512 9513 /* If we gave a direct map region in pmap_mapdev, do nothing */ 9514 if ((vm_offset_t)va >= kva_layout.dmap_low && 9515 (vm_offset_t)va < kva_layout.dmap_high) 9516 return; 9517 offset = (vm_offset_t)va & PAGE_MASK; 9518 size = round_page(offset + size); 9519 va = trunc_page(va); 9520 for (i = 0; i < PMAP_PREINIT_MAPPING_COUNT; i++) { 9521 ppim = pmap_preinit_mapping + i; 9522 if (ppim->va == va && ppim->sz == size) { 9523 if (pmap_initialized) 9524 return; 9525 ppim->pa = 0; 9526 ppim->va = NULL; 9527 ppim->sz = 0; 9528 ppim->mode = 0; 9529 if (va + size == (void *)virtual_avail) 9530 virtual_avail = (vm_offset_t)va; 9531 return; 9532 } 9533 } 9534 if (pmap_initialized) { 9535 pmap_qremove(va, atop(size)); 9536 kva_free(va, size); 9537 } 9538 } 9539 9540 /* 9541 * Tries to demote a 1GB page mapping. 9542 */ 9543 static bool 9544 pmap_demote_pdpe(pmap_t pmap, pdp_entry_t *pdpe, vm_offset_t va, vm_page_t m) 9545 { 9546 pdp_entry_t newpdpe, oldpdpe; 9547 pd_entry_t *firstpde, newpde, *pde; 9548 pt_entry_t PG_A, PG_M, PG_RW, PG_V; 9549 vm_paddr_t pdpgpa; 9550 vm_page_t pdpg; 9551 9552 PG_A = pmap_accessed_bit(pmap); 9553 PG_M = pmap_modified_bit(pmap); 9554 PG_V = pmap_valid_bit(pmap); 9555 PG_RW = pmap_rw_bit(pmap); 9556 9557 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 9558 oldpdpe = *pdpe; 9559 KASSERT((oldpdpe & (PG_PS | PG_V)) == (PG_PS | PG_V), 9560 ("pmap_demote_pdpe: oldpdpe is missing PG_PS and/or PG_V")); 9561 if (m == NULL) { 9562 pdpg = pmap_alloc_pt_page(pmap, va >> PDPSHIFT, 9563 VM_ALLOC_WIRED); 9564 if (pdpg == NULL) { 9565 CTR2(KTR_PMAP, 9566 "pmap_demote_pdpe: failure for va %#lx in pmap %p", 9567 va, pmap); 9568 return (false); 9569 } 9570 } else { 9571 pdpg = m; 9572 pdpg->pindex = va >> PDPSHIFT; 9573 pmap_pt_page_count_adj(pmap, 1); 9574 } 9575 pdpgpa = VM_PAGE_TO_PHYS(pdpg); 9576 firstpde = PHYS_TO_DMAP(pdpgpa); 9577 newpdpe = pdpgpa | PG_M | PG_A | (oldpdpe & PG_U) | PG_RW | PG_V; 9578 KASSERT((oldpdpe & PG_A) != 0, 9579 ("pmap_demote_pdpe: oldpdpe is missing PG_A")); 9580 KASSERT((oldpdpe & (PG_M | PG_RW)) != PG_RW, 9581 ("pmap_demote_pdpe: oldpdpe is missing PG_M")); 9582 newpde = oldpdpe; 9583 9584 /* 9585 * Initialize the page directory page. 9586 */ 9587 for (pde = firstpde; pde < firstpde + NPDEPG; pde++) { 9588 *pde = newpde; 9589 newpde += NBPDR; 9590 } 9591 9592 /* 9593 * Demote the mapping. 9594 */ 9595 *pdpe = newpdpe; 9596 9597 /* 9598 * Invalidate a stale recursive mapping of the page directory page. 9599 */ 9600 pmap_invalidate_page(pmap, (vm_offset_t)vtopde(va)); 9601 9602 counter_u64_add(pmap_pdpe_demotions, 1); 9603 CTR2(KTR_PMAP, "pmap_demote_pdpe: success for va %#lx" 9604 " in pmap %p", va, pmap); 9605 return (true); 9606 } 9607 9608 /* 9609 * Sets the memory attribute for the specified page. 9610 */ 9611 void 9612 pmap_page_set_memattr(vm_page_t m, vm_memattr_t ma) 9613 { 9614 if (m->md.pat_mode == ma) 9615 return; 9616 9617 m->md.pat_mode = ma; 9618 9619 /* 9620 * If "m" is a normal page, update its direct mapping. This update 9621 * can be relied upon to perform any cache operations that are 9622 * required for data coherence. 9623 */ 9624 if ((m->flags & PG_FICTITIOUS) == 0 && 9625 pmap_change_attr(VM_PAGE_TO_DMAP(m), PAGE_SIZE, m->md.pat_mode)) 9626 panic("memory attribute change on the direct map failed"); 9627 } 9628 9629 void 9630 pmap_page_set_memattr_noflush(vm_page_t m, vm_memattr_t ma) 9631 { 9632 int error; 9633 9634 if (m->md.pat_mode == ma) 9635 return; 9636 9637 m->md.pat_mode = ma; 9638 9639 if ((m->flags & PG_FICTITIOUS) != 0) 9640 return; 9641 PMAP_LOCK(kernel_pmap); 9642 error = pmap_change_props_locked(VM_PAGE_TO_DMAP(m), PAGE_SIZE, 9643 PROT_NONE, m->md.pat_mode, 0); 9644 PMAP_UNLOCK(kernel_pmap); 9645 if (error != 0) 9646 panic("memory attribute change on the direct map failed"); 9647 } 9648 9649 /* 9650 * Changes the specified virtual address range's memory type to that given by 9651 * the parameter "mode". The specified virtual address range must be 9652 * completely contained within either the direct map or the kernel map. If 9653 * the virtual address range is contained within the kernel map, then the 9654 * memory type for each of the corresponding ranges of the direct map is also 9655 * changed. (The corresponding ranges of the direct map are those ranges that 9656 * map the same physical pages as the specified virtual address range.) These 9657 * changes to the direct map are necessary because Intel describes the 9658 * behavior of their processors as "undefined" if two or more mappings to the 9659 * same physical page have different memory types. 9660 * 9661 * Returns zero if the change completed successfully, and either EINVAL or 9662 * ENOMEM if the change failed. Specifically, EINVAL is returned if some part 9663 * of the virtual address range was not mapped, and ENOMEM is returned if 9664 * there was insufficient memory available to complete the change. In the 9665 * latter case, the memory type may have been changed on some part of the 9666 * virtual address range or the direct map. 9667 */ 9668 int 9669 pmap_change_attr(void *va, vm_size_t size, int mode) 9670 { 9671 int error; 9672 9673 PMAP_LOCK(kernel_pmap); 9674 error = pmap_change_props_locked(va, size, PROT_NONE, mode, 9675 MAPDEV_FLUSHCACHE); 9676 PMAP_UNLOCK(kernel_pmap); 9677 return (error); 9678 } 9679 9680 /* 9681 * Changes the specified virtual address range's protections to those 9682 * specified by "prot". Like pmap_change_attr(), protections for aliases 9683 * in the direct map are updated as well. Protections on aliasing mappings may 9684 * be a subset of the requested protections; for example, mappings in the direct 9685 * map are never executable. 9686 */ 9687 int 9688 pmap_change_prot(void *va, vm_size_t size, vm_prot_t prot) 9689 { 9690 int error; 9691 9692 /* Only supported within the kernel map. */ 9693 if ((vm_offset_t)va < kva_layout.km_low) 9694 return (EINVAL); 9695 9696 PMAP_LOCK(kernel_pmap); 9697 error = pmap_change_props_locked(va, size, prot, -1, 9698 MAPDEV_ASSERTVALID); 9699 PMAP_UNLOCK(kernel_pmap); 9700 return (error); 9701 } 9702 9703 static int 9704 pmap_change_props_locked(void *addr, vm_size_t size, vm_prot_t prot, 9705 int mode, int flags) 9706 { 9707 vm_offset_t base, offset, tmpva, va; 9708 vm_paddr_t pa_start, pa_end, pa_end1; 9709 pdp_entry_t *pdpe; 9710 pd_entry_t *pde, pde_bits, pde_mask; 9711 pt_entry_t *pte, pte_bits, pte_mask; 9712 int error; 9713 bool changed; 9714 9715 va = (vm_offset_t)addr; 9716 PMAP_LOCK_ASSERT(kernel_pmap, MA_OWNED); 9717 base = trunc_page(va); 9718 offset = va & PAGE_MASK; 9719 size = round_page(offset + size); 9720 9721 /* 9722 * Only supported on kernel virtual addresses, including the direct 9723 * map but excluding the recursive map. 9724 */ 9725 if (base < kva_layout.dmap_low) 9726 return (EINVAL); 9727 9728 /* 9729 * Construct our flag sets and masks. "bits" is the subset of 9730 * "mask" that will be set in each modified PTE. 9731 * 9732 * Mappings in the direct map are never allowed to be executable. 9733 */ 9734 pde_bits = pte_bits = 0; 9735 pde_mask = pte_mask = 0; 9736 if (mode != -1) { 9737 pde_bits |= pmap_cache_bits(kernel_pmap, mode, true); 9738 pde_mask |= X86_PG_PDE_CACHE; 9739 pte_bits |= pmap_cache_bits(kernel_pmap, mode, false); 9740 pte_mask |= X86_PG_PTE_CACHE; 9741 } 9742 if (prot != VM_PROT_NONE) { 9743 if ((prot & VM_PROT_WRITE) != 0) { 9744 pde_bits |= X86_PG_RW; 9745 pte_bits |= X86_PG_RW; 9746 } 9747 if ((prot & VM_PROT_EXECUTE) == 0 || 9748 va < kva_layout.km_low) { 9749 pde_bits |= pg_nx; 9750 pte_bits |= pg_nx; 9751 } 9752 pde_mask |= X86_PG_RW | pg_nx; 9753 pte_mask |= X86_PG_RW | pg_nx; 9754 } 9755 9756 /* 9757 * Pages that aren't mapped aren't supported. Also break down 2MB pages 9758 * into 4KB pages if required. 9759 */ 9760 for (tmpva = base; tmpva < base + size; ) { 9761 pdpe = pmap_pdpe(kernel_pmap, tmpva); 9762 if (pdpe == NULL || *pdpe == 0) { 9763 KASSERT((flags & MAPDEV_ASSERTVALID) == 0, 9764 ("%s: addr %#lx is not mapped", __func__, tmpva)); 9765 return (EINVAL); 9766 } 9767 if (*pdpe & PG_PS) { 9768 /* 9769 * If the current 1GB page already has the required 9770 * properties, then we need not demote this page. Just 9771 * increment tmpva to the next 1GB page frame. 9772 */ 9773 if ((*pdpe & pde_mask) == pde_bits) { 9774 tmpva = trunc_1gpage(tmpva) + NBPDP; 9775 continue; 9776 } 9777 9778 /* 9779 * If the current offset aligns with a 1GB page frame 9780 * and there is at least 1GB left within the range, then 9781 * we need not break down this page into 2MB pages. 9782 */ 9783 if ((tmpva & PDPMASK) == 0 && 9784 tmpva + PDPMASK < base + size) { 9785 tmpva += NBPDP; 9786 continue; 9787 } 9788 if (!pmap_demote_pdpe(kernel_pmap, pdpe, tmpva, NULL)) 9789 return (ENOMEM); 9790 } 9791 pde = pmap_pdpe_to_pde(pdpe, tmpva); 9792 if (*pde == 0) { 9793 KASSERT((flags & MAPDEV_ASSERTVALID) == 0, 9794 ("%s: addr %#lx is not mapped", __func__, tmpva)); 9795 return (EINVAL); 9796 } 9797 if (*pde & PG_PS) { 9798 /* 9799 * If the current 2MB page already has the required 9800 * properties, then we need not demote this page. Just 9801 * increment tmpva to the next 2MB page frame. 9802 */ 9803 if ((*pde & pde_mask) == pde_bits) { 9804 tmpva = trunc_2mpage(tmpva) + NBPDR; 9805 continue; 9806 } 9807 9808 /* 9809 * If the current offset aligns with a 2MB page frame 9810 * and there is at least 2MB left within the range, then 9811 * we need not break down this page into 4KB pages. 9812 */ 9813 if ((tmpva & PDRMASK) == 0 && 9814 tmpva + PDRMASK < base + size) { 9815 tmpva += NBPDR; 9816 continue; 9817 } 9818 if (!pmap_demote_pde(kernel_pmap, pde, tmpva)) 9819 return (ENOMEM); 9820 } 9821 pte = pmap_pde_to_pte(pde, tmpva); 9822 if (*pte == 0) { 9823 KASSERT((flags & MAPDEV_ASSERTVALID) == 0, 9824 ("%s: addr %#lx is not mapped", __func__, tmpva)); 9825 return (EINVAL); 9826 } 9827 tmpva += PAGE_SIZE; 9828 } 9829 error = 0; 9830 9831 /* 9832 * Ok, all the pages exist, so run through them updating their 9833 * properties if required. 9834 */ 9835 changed = false; 9836 pa_start = pa_end = 0; 9837 for (tmpva = base; tmpva < base + size; ) { 9838 pdpe = pmap_pdpe(kernel_pmap, tmpva); 9839 if (*pdpe & PG_PS) { 9840 if ((*pdpe & pde_mask) != pde_bits) { 9841 pmap_pte_props(pdpe, pde_bits, pde_mask); 9842 changed = true; 9843 } 9844 if (tmpva >= kva_layout.km_low && 9845 (*pdpe & PG_PS_FRAME) < dmaplimit) { 9846 if (pa_start == pa_end) { 9847 /* Start physical address run. */ 9848 pa_start = *pdpe & PG_PS_FRAME; 9849 pa_end = pa_start + NBPDP; 9850 } else if (pa_end == (*pdpe & PG_PS_FRAME)) 9851 pa_end += NBPDP; 9852 else { 9853 /* Run ended, update direct map. */ 9854 error = pmap_change_props_locked( 9855 PHYS_TO_DMAP(pa_start), 9856 pa_end - pa_start, prot, mode, 9857 flags); 9858 if (error != 0) 9859 break; 9860 /* Start physical address run. */ 9861 pa_start = *pdpe & PG_PS_FRAME; 9862 pa_end = pa_start + NBPDP; 9863 } 9864 } 9865 tmpva = trunc_1gpage(tmpva) + NBPDP; 9866 continue; 9867 } 9868 pde = pmap_pdpe_to_pde(pdpe, tmpva); 9869 if (*pde & PG_PS) { 9870 if ((*pde & pde_mask) != pde_bits) { 9871 pmap_pte_props(pde, pde_bits, pde_mask); 9872 changed = true; 9873 } 9874 if (tmpva >= kva_layout.km_low && 9875 (*pde & PG_PS_FRAME) < dmaplimit) { 9876 if (pa_start == pa_end) { 9877 /* Start physical address run. */ 9878 pa_start = *pde & PG_PS_FRAME; 9879 pa_end = pa_start + NBPDR; 9880 } else if (pa_end == (*pde & PG_PS_FRAME)) 9881 pa_end += NBPDR; 9882 else { 9883 /* Run ended, update direct map. */ 9884 error = pmap_change_props_locked( 9885 PHYS_TO_DMAP(pa_start), 9886 pa_end - pa_start, prot, mode, 9887 flags); 9888 if (error != 0) 9889 break; 9890 /* Start physical address run. */ 9891 pa_start = *pde & PG_PS_FRAME; 9892 pa_end = pa_start + NBPDR; 9893 } 9894 } 9895 tmpva = trunc_2mpage(tmpva) + NBPDR; 9896 } else { 9897 pte = pmap_pde_to_pte(pde, tmpva); 9898 if ((*pte & pte_mask) != pte_bits) { 9899 pmap_pte_props(pte, pte_bits, pte_mask); 9900 changed = true; 9901 } 9902 if (tmpva >= kva_layout.km_low && 9903 (*pte & PG_FRAME) < dmaplimit) { 9904 if (pa_start == pa_end) { 9905 /* Start physical address run. */ 9906 pa_start = *pte & PG_FRAME; 9907 pa_end = pa_start + PAGE_SIZE; 9908 } else if (pa_end == (*pte & PG_FRAME)) 9909 pa_end += PAGE_SIZE; 9910 else { 9911 /* Run ended, update direct map. */ 9912 error = pmap_change_props_locked( 9913 PHYS_TO_DMAP(pa_start), 9914 pa_end - pa_start, prot, mode, 9915 flags); 9916 if (error != 0) 9917 break; 9918 /* Start physical address run. */ 9919 pa_start = *pte & PG_FRAME; 9920 pa_end = pa_start + PAGE_SIZE; 9921 } 9922 } 9923 tmpva += PAGE_SIZE; 9924 } 9925 } 9926 if (error == 0 && pa_start != pa_end && pa_start < dmaplimit) { 9927 pa_end1 = MIN(pa_end, dmaplimit); 9928 if (pa_start != pa_end1) 9929 error = pmap_change_props_locked(PHYS_TO_DMAP(pa_start), 9930 pa_end1 - pa_start, prot, mode, flags); 9931 } 9932 9933 /* 9934 * Flush CPU caches if required to make sure any data isn't cached that 9935 * shouldn't be, etc. 9936 */ 9937 if (changed) { 9938 pmap_invalidate_range(kernel_pmap, base, tmpva); 9939 if ((flags & MAPDEV_FLUSHCACHE) != 0) 9940 pmap_invalidate_cache_range(base, tmpva); 9941 } 9942 return (error); 9943 } 9944 9945 /* 9946 * Demotes any mapping within the direct map region that covers more 9947 * than the specified range of physical addresses. This range's size 9948 * must be a power of two and its starting address must be a multiple 9949 * of its size, which means that any pdp from the mapping is fully 9950 * covered by the range if len > NBPDP. Since the demotion does not 9951 * change any attributes of the mapping, a TLB invalidation is not 9952 * mandatory. The caller may, however, request a TLB invalidation. 9953 */ 9954 void 9955 pmap_demote_DMAP(vm_paddr_t base, vm_size_t len, bool invalidate) 9956 { 9957 pdp_entry_t *pdpe; 9958 pd_entry_t *pde; 9959 vm_offset_t va; 9960 vm_page_t m, mpte; 9961 bool changed, rv __diagused; 9962 9963 if (len == 0) 9964 return; 9965 KASSERT(powerof2(len), ("pmap_demote_DMAP: len is not a power of 2")); 9966 KASSERT((base & (len - 1)) == 0, 9967 ("pmap_demote_DMAP: base is not a multiple of len")); 9968 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, "pmap_demote_DMAP"); 9969 9970 if (len < NBPDP && base < dmaplimit) { 9971 va = PHYS_TO_DMAP_ADDR(base); 9972 changed = false; 9973 9974 /* 9975 * Assume that it is fine to sleep there. 9976 * The only existing caller of pmap_demote_DMAP() is the 9977 * x86_mr_split_dmap() function. 9978 */ 9979 m = vm_page_alloc_noobj(VM_ALLOC_WIRED | VM_ALLOC_WAITOK); 9980 if (len < NBPDR) { 9981 mpte = vm_page_alloc_noobj(VM_ALLOC_WIRED | 9982 VM_ALLOC_WAITOK); 9983 } else 9984 mpte = NULL; 9985 9986 PMAP_LOCK(kernel_pmap); 9987 pdpe = pmap_pdpe(kernel_pmap, va); 9988 if ((*pdpe & X86_PG_V) == 0) 9989 panic("pmap_demote_DMAP: invalid PDPE"); 9990 if ((*pdpe & PG_PS) != 0) { 9991 rv = pmap_demote_pdpe(kernel_pmap, pdpe, va, m); 9992 KASSERT(rv, ("pmap_demote_DMAP: PDPE failed")); 9993 changed = true; 9994 m = NULL; 9995 } 9996 if (len < NBPDR) { 9997 pde = pmap_pdpe_to_pde(pdpe, va); 9998 if ((*pde & X86_PG_V) == 0) 9999 panic("pmap_demote_DMAP: invalid PDE"); 10000 if ((*pde & PG_PS) != 0) { 10001 mpte->pindex = pmap_pde_pindex(va); 10002 pmap_pt_page_count_adj(kernel_pmap, 1); 10003 rv = pmap_demote_pde_mpte(kernel_pmap, pde, va, 10004 NULL, mpte); 10005 KASSERT(rv, ("pmap_demote_DMAP: PDE failed")); 10006 changed = true; 10007 mpte = NULL; 10008 } 10009 } 10010 if (changed && invalidate) 10011 pmap_invalidate_page(kernel_pmap, va); 10012 PMAP_UNLOCK(kernel_pmap); 10013 if (m != NULL) { 10014 vm_page_unwire_noq(m); 10015 vm_page_free(m); 10016 } 10017 if (mpte != NULL) { 10018 vm_page_unwire_noq(mpte); 10019 vm_page_free(mpte); 10020 } 10021 } 10022 } 10023 10024 /* 10025 * Perform the pmap work for mincore(2). If the page is not both referenced and 10026 * modified by this pmap, returns its physical address so that the caller can 10027 * find other mappings. 10028 */ 10029 int 10030 pmap_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *pap) 10031 { 10032 pdp_entry_t *pdpe; 10033 pd_entry_t *pdep; 10034 pt_entry_t pte, PG_A, PG_M, PG_RW, PG_V; 10035 vm_paddr_t pa; 10036 int val; 10037 10038 PG_A = pmap_accessed_bit(pmap); 10039 PG_M = pmap_modified_bit(pmap); 10040 PG_V = pmap_valid_bit(pmap); 10041 PG_RW = pmap_rw_bit(pmap); 10042 10043 PMAP_LOCK(pmap); 10044 pte = 0; 10045 pa = 0; 10046 val = 0; 10047 pdpe = pmap_pdpe(pmap, addr); 10048 if (pdpe == NULL) 10049 goto out; 10050 if ((*pdpe & PG_V) != 0) { 10051 if ((*pdpe & PG_PS) != 0) { 10052 pte = *pdpe; 10053 pa = ((pte & PG_PS_PDP_FRAME) | (addr & PDPMASK)) & 10054 PG_FRAME; 10055 val = MINCORE_PSIND(2); 10056 } else { 10057 pdep = pmap_pde(pmap, addr); 10058 if (pdep != NULL && (*pdep & PG_V) != 0) { 10059 if ((*pdep & PG_PS) != 0) { 10060 pte = *pdep; 10061 /* Compute the physical address of the 4KB page. */ 10062 pa = ((pte & PG_PS_FRAME) | (addr & 10063 PDRMASK)) & PG_FRAME; 10064 val = MINCORE_PSIND(1); 10065 } else { 10066 pte = *pmap_pde_to_pte(pdep, addr); 10067 pa = pte & PG_FRAME; 10068 val = 0; 10069 } 10070 } 10071 } 10072 } 10073 if ((pte & PG_V) != 0) { 10074 val |= MINCORE_INCORE; 10075 if ((pte & (PG_M | PG_RW)) == (PG_M | PG_RW)) 10076 val |= MINCORE_MODIFIED | MINCORE_MODIFIED_OTHER; 10077 if ((pte & PG_A) != 0) 10078 val |= MINCORE_REFERENCED | MINCORE_REFERENCED_OTHER; 10079 } 10080 if ((val & (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER)) != 10081 (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER) && 10082 (pte & (PG_MANAGED | PG_V)) == (PG_MANAGED | PG_V)) { 10083 *pap = pa; 10084 } 10085 out: 10086 PMAP_UNLOCK(pmap); 10087 return (val); 10088 } 10089 10090 static uint64_t 10091 pmap_pcid_alloc(pmap_t pmap, struct pmap_pcid *pcidp) 10092 { 10093 uint32_t gen, new_gen, pcid_next; 10094 10095 CRITICAL_ASSERT(curthread); 10096 gen = PCPU_GET(pcid_gen); 10097 if (pcidp->pm_pcid == PMAP_PCID_KERN) 10098 return (pti ? 0 : CR3_PCID_SAVE); 10099 if (pcidp->pm_gen == gen) 10100 return (CR3_PCID_SAVE); 10101 pcid_next = PCPU_GET(pcid_next); 10102 KASSERT((!pti && pcid_next <= PMAP_PCID_OVERMAX) || 10103 (pti && pcid_next <= PMAP_PCID_OVERMAX_KERN), 10104 ("cpu %d pcid_next %#x", PCPU_GET(cpuid), pcid_next)); 10105 if ((!pti && pcid_next == PMAP_PCID_OVERMAX) || 10106 (pti && pcid_next == PMAP_PCID_OVERMAX_KERN)) { 10107 new_gen = gen + 1; 10108 if (new_gen == 0) 10109 new_gen = 1; 10110 PCPU_SET(pcid_gen, new_gen); 10111 pcid_next = PMAP_PCID_KERN + 1; 10112 } else { 10113 new_gen = gen; 10114 } 10115 pcidp->pm_pcid = pcid_next; 10116 pcidp->pm_gen = new_gen; 10117 PCPU_SET(pcid_next, pcid_next + 1); 10118 return (0); 10119 } 10120 10121 static uint64_t 10122 pmap_pcid_alloc_checked(pmap_t pmap, struct pmap_pcid *pcidp) 10123 { 10124 uint64_t cached; 10125 10126 cached = pmap_pcid_alloc(pmap, pcidp); 10127 KASSERT(pcidp->pm_pcid < PMAP_PCID_OVERMAX, 10128 ("pmap %p cpu %d pcid %#x", pmap, PCPU_GET(cpuid), pcidp->pm_pcid)); 10129 KASSERT(pcidp->pm_pcid != PMAP_PCID_KERN || pmap == kernel_pmap, 10130 ("non-kernel pmap pmap %p cpu %d pcid %#x", 10131 pmap, PCPU_GET(cpuid), pcidp->pm_pcid)); 10132 return (cached); 10133 } 10134 10135 static void 10136 pmap_activate_sw_pti_post(struct thread *td, pmap_t pmap) 10137 { 10138 10139 PCPU_GET(tssp)->tss_rsp0 = pmap->pm_ucr3 != PMAP_NO_CR3 ? 10140 PCPU_GET(pti_rsp0) : (uintptr_t)td->td_md.md_stack_base; 10141 } 10142 10143 static void 10144 pmap_activate_sw_pcid_pti(struct thread *td, pmap_t pmap, u_int cpuid) 10145 { 10146 pmap_t old_pmap; 10147 struct pmap_pcid *pcidp, *old_pcidp; 10148 uint64_t cached, cr3, kcr3, ucr3; 10149 10150 KASSERT((read_rflags() & PSL_I) == 0, 10151 ("PCID needs interrupts disabled in pmap_activate_sw()")); 10152 10153 /* See the comment in pmap_invalidate_page_pcid(). */ 10154 if (PCPU_GET(ucr3_load_mask) != PMAP_UCR3_NOMASK) { 10155 PCPU_SET(ucr3_load_mask, PMAP_UCR3_NOMASK); 10156 old_pmap = PCPU_GET(curpmap); 10157 MPASS(old_pmap->pm_ucr3 != PMAP_NO_CR3); 10158 old_pcidp = zpcpu_get_cpu(old_pmap->pm_pcidp, cpuid); 10159 old_pcidp->pm_gen = 0; 10160 } 10161 10162 pcidp = zpcpu_get_cpu(pmap->pm_pcidp, cpuid); 10163 cached = pmap_pcid_alloc_checked(pmap, pcidp); 10164 cr3 = rcr3(); 10165 if ((cr3 & ~CR3_PCID_MASK) != pmap->pm_cr3) 10166 load_cr3(pmap->pm_cr3 | pcidp->pm_pcid); 10167 PCPU_SET(curpmap, pmap); 10168 kcr3 = pmap->pm_cr3 | pcidp->pm_pcid; 10169 ucr3 = pmap->pm_ucr3 | pcidp->pm_pcid | PMAP_PCID_USER_PT; 10170 10171 if (!cached && pmap->pm_ucr3 != PMAP_NO_CR3) 10172 PCPU_SET(ucr3_load_mask, ~CR3_PCID_SAVE); 10173 10174 PCPU_SET(kcr3, kcr3 | CR3_PCID_SAVE); 10175 PCPU_SET(ucr3, ucr3 | CR3_PCID_SAVE); 10176 if (cached) 10177 counter_u64_add(pcid_save_cnt, 1); 10178 10179 pmap_activate_sw_pti_post(td, pmap); 10180 } 10181 10182 static void 10183 pmap_activate_sw_pcid_nopti(struct thread *td __unused, pmap_t pmap, 10184 u_int cpuid) 10185 { 10186 struct pmap_pcid *pcidp; 10187 uint64_t cached, cr3; 10188 10189 KASSERT((read_rflags() & PSL_I) == 0, 10190 ("PCID needs interrupts disabled in pmap_activate_sw()")); 10191 10192 pcidp = zpcpu_get_cpu(pmap->pm_pcidp, cpuid); 10193 cached = pmap_pcid_alloc_checked(pmap, pcidp); 10194 cr3 = rcr3(); 10195 if (!cached || (cr3 & ~CR3_PCID_MASK) != pmap->pm_cr3) 10196 load_cr3(pmap->pm_cr3 | pcidp->pm_pcid | cached); 10197 PCPU_SET(curpmap, pmap); 10198 if (cached) 10199 counter_u64_add(pcid_save_cnt, 1); 10200 } 10201 10202 static void 10203 pmap_activate_sw_nopcid_nopti(struct thread *td __unused, pmap_t pmap, 10204 u_int cpuid __unused) 10205 { 10206 10207 load_cr3(pmap->pm_cr3); 10208 PCPU_SET(curpmap, pmap); 10209 } 10210 10211 static void 10212 pmap_activate_sw_nopcid_pti(struct thread *td, pmap_t pmap, 10213 u_int cpuid __unused) 10214 { 10215 10216 pmap_activate_sw_nopcid_nopti(td, pmap, cpuid); 10217 PCPU_SET(kcr3, pmap->pm_cr3); 10218 PCPU_SET(ucr3, pmap->pm_ucr3); 10219 pmap_activate_sw_pti_post(td, pmap); 10220 } 10221 10222 DEFINE_IFUNC(static, void, pmap_activate_sw_mode, (struct thread *, pmap_t, 10223 u_int)) 10224 { 10225 10226 if (pmap_pcid_enabled && pti) 10227 return (pmap_activate_sw_pcid_pti); 10228 else if (pmap_pcid_enabled && !pti) 10229 return (pmap_activate_sw_pcid_nopti); 10230 else if (!pmap_pcid_enabled && pti) 10231 return (pmap_activate_sw_nopcid_pti); 10232 else /* if (!pmap_pcid_enabled && !pti) */ 10233 return (pmap_activate_sw_nopcid_nopti); 10234 } 10235 10236 void 10237 pmap_activate_sw(struct thread *td) 10238 { 10239 struct thread *oldtd; 10240 pmap_t oldpmap, pmap; 10241 u_int cpuid; 10242 bool oldtd_sl, td_sl; 10243 10244 oldtd = curthread; 10245 if (ia32_splitlock && oldtd != td) { 10246 oldtd_sl = (atomic_load_int(&oldtd->td_md.md_td_flags) & 10247 TDF_MD_SPLITLOCK_AC) != 0; 10248 td_sl = (atomic_load_int(&td->td_md.md_td_flags) & 10249 TDF_MD_SPLITLOCK_AC) != 0; 10250 if (oldtd_sl && !td_sl) 10251 disable_splitlock_ac(); 10252 else if (!oldtd_sl && td_sl) 10253 enable_splitlock_ac(); 10254 } 10255 10256 oldpmap = PCPU_GET(curpmap); 10257 pmap = vmspace_pmap(td->td_proc->p_vmspace); 10258 if (oldpmap == pmap) { 10259 if (cpu_vendor_id != CPU_VENDOR_INTEL) 10260 mfence(); 10261 return; 10262 } 10263 cpuid = PCPU_GET(cpuid); 10264 CPU_SET_ATOMIC(cpuid, &pmap->pm_active); 10265 pmap_activate_sw_mode(td, pmap, cpuid); 10266 CPU_CLR_ATOMIC(cpuid, &oldpmap->pm_active); 10267 } 10268 10269 void 10270 pmap_activate(struct thread *td) 10271 { 10272 /* 10273 * invltlb_{invpcid,}_pcid_handler() is used to handle an 10274 * invalidate_all IPI, which checks for curpmap == 10275 * smp_tlb_pmap. The below sequence of operations has a 10276 * window where %CR3 is loaded with the new pmap's PML4 10277 * address, but the curpmap value has not yet been updated. 10278 * This causes the invltlb IPI handler, which is called 10279 * between the updates, to execute as a NOP, which leaves 10280 * stale TLB entries. 10281 * 10282 * Note that the most common use of pmap_activate_sw(), from 10283 * a context switch, is immune to this race, because 10284 * interrupts are disabled (while the thread lock is owned), 10285 * so the IPI is delayed until after curpmap is updated. Protect 10286 * other callers in a similar way, by disabling interrupts 10287 * around the %cr3 register reload and curpmap assignment. 10288 */ 10289 spinlock_enter(); 10290 pmap_activate_sw(td); 10291 spinlock_exit(); 10292 } 10293 10294 void 10295 pmap_activate_boot(pmap_t pmap) 10296 { 10297 uint64_t kcr3; 10298 u_int cpuid; 10299 10300 /* 10301 * kernel_pmap must be never deactivated, and we ensure that 10302 * by never activating it at all. 10303 */ 10304 MPASS(pmap != kernel_pmap); 10305 10306 cpuid = PCPU_GET(cpuid); 10307 CPU_SET_ATOMIC(cpuid, &pmap->pm_active); 10308 PCPU_SET(curpmap, pmap); 10309 if (pti) { 10310 kcr3 = pmap->pm_cr3; 10311 if (pmap_pcid_enabled) 10312 kcr3 |= pmap_get_pcid(pmap) | CR3_PCID_SAVE; 10313 } else { 10314 kcr3 = PMAP_NO_CR3; 10315 } 10316 PCPU_SET(kcr3, kcr3); 10317 PCPU_SET(ucr3, PMAP_NO_CR3); 10318 } 10319 10320 void 10321 pmap_active_cpus(pmap_t pmap, cpuset_t *res) 10322 { 10323 *res = pmap->pm_active; 10324 } 10325 10326 void 10327 pmap_sync_icache(pmap_t pm, vm_offset_t va, vm_size_t sz) 10328 { 10329 } 10330 10331 /* 10332 * Increase the starting virtual address of the given mapping if a 10333 * different alignment might result in more superpage mappings. 10334 */ 10335 void 10336 pmap_align_superpage(vm_object_t object, vm_ooffset_t offset, 10337 vm_offset_t *addr, vm_size_t size) 10338 { 10339 vm_offset_t superpage_offset; 10340 10341 if (size < NBPDR) 10342 return; 10343 if (object != NULL && (object->flags & OBJ_COLORED) != 0) 10344 offset += ptoa(object->pg_color); 10345 superpage_offset = offset & PDRMASK; 10346 if (size - ((NBPDR - superpage_offset) & PDRMASK) < NBPDR || 10347 (*addr & PDRMASK) == superpage_offset) 10348 return; 10349 if ((*addr & PDRMASK) < superpage_offset) 10350 *addr = (*addr & ~PDRMASK) + superpage_offset; 10351 else 10352 *addr = ((*addr + PDRMASK) & ~PDRMASK) + superpage_offset; 10353 } 10354 10355 #ifdef INVARIANTS 10356 static unsigned long num_dirty_emulations; 10357 SYSCTL_ULONG(_vm_pmap, OID_AUTO, num_dirty_emulations, CTLFLAG_RW, 10358 &num_dirty_emulations, 0, NULL); 10359 10360 static unsigned long num_accessed_emulations; 10361 SYSCTL_ULONG(_vm_pmap, OID_AUTO, num_accessed_emulations, CTLFLAG_RW, 10362 &num_accessed_emulations, 0, NULL); 10363 10364 static unsigned long num_superpage_accessed_emulations; 10365 SYSCTL_ULONG(_vm_pmap, OID_AUTO, num_superpage_accessed_emulations, CTLFLAG_RW, 10366 &num_superpage_accessed_emulations, 0, NULL); 10367 10368 static unsigned long ad_emulation_superpage_promotions; 10369 SYSCTL_ULONG(_vm_pmap, OID_AUTO, ad_emulation_superpage_promotions, CTLFLAG_RW, 10370 &ad_emulation_superpage_promotions, 0, NULL); 10371 #endif /* INVARIANTS */ 10372 10373 int 10374 pmap_emulate_accessed_dirty(pmap_t pmap, vm_offset_t va, int ftype) 10375 { 10376 int rv; 10377 struct rwlock *lock; 10378 #if VM_NRESERVLEVEL > 0 10379 vm_page_t m, mpte; 10380 #endif 10381 pd_entry_t *pde; 10382 pt_entry_t *pte, PG_A, PG_M, PG_RW, PG_V; 10383 10384 KASSERT(ftype == VM_PROT_READ || ftype == VM_PROT_WRITE, 10385 ("pmap_emulate_accessed_dirty: invalid fault type %d", ftype)); 10386 10387 if (!pmap_emulate_ad_bits(pmap)) 10388 return (-1); 10389 10390 PG_A = pmap_accessed_bit(pmap); 10391 PG_M = pmap_modified_bit(pmap); 10392 PG_V = pmap_valid_bit(pmap); 10393 PG_RW = pmap_rw_bit(pmap); 10394 10395 rv = -1; 10396 lock = NULL; 10397 PMAP_LOCK(pmap); 10398 10399 pde = pmap_pde(pmap, va); 10400 if (pde == NULL || (*pde & PG_V) == 0) 10401 goto done; 10402 10403 if ((*pde & PG_PS) != 0) { 10404 if (ftype == VM_PROT_READ) { 10405 #ifdef INVARIANTS 10406 atomic_add_long(&num_superpage_accessed_emulations, 1); 10407 #endif 10408 *pde |= PG_A; 10409 rv = 0; 10410 } 10411 goto done; 10412 } 10413 10414 pte = pmap_pde_to_pte(pde, va); 10415 if ((*pte & PG_V) == 0) 10416 goto done; 10417 10418 if (ftype == VM_PROT_WRITE) { 10419 if ((*pte & PG_RW) == 0) 10420 goto done; 10421 /* 10422 * Set the modified and accessed bits simultaneously. 10423 * 10424 * Intel EPT PTEs that do software emulation of A/D bits map 10425 * PG_A and PG_M to EPT_PG_READ and EPT_PG_WRITE respectively. 10426 * An EPT misconfiguration is triggered if the PTE is writable 10427 * but not readable (WR=10). This is avoided by setting PG_A 10428 * and PG_M simultaneously. 10429 */ 10430 *pte |= PG_M | PG_A; 10431 } else { 10432 *pte |= PG_A; 10433 } 10434 10435 #if VM_NRESERVLEVEL > 0 10436 /* try to promote the mapping */ 10437 if (va < VM_MAXUSER_ADDRESS) 10438 mpte = PHYS_TO_VM_PAGE(*pde & PG_FRAME); 10439 else 10440 mpte = NULL; 10441 10442 m = PHYS_TO_VM_PAGE(*pte & PG_FRAME); 10443 10444 if ((mpte == NULL || mpte->ref_count == NPTEPG) && 10445 (m->flags & PG_FICTITIOUS) == 0 && 10446 vm_reserv_level_iffullpop(m) == 0 && 10447 pmap_promote_pde(pmap, pde, va, mpte, &lock)) { 10448 #ifdef INVARIANTS 10449 atomic_add_long(&ad_emulation_superpage_promotions, 1); 10450 #endif 10451 } 10452 #endif 10453 10454 #ifdef INVARIANTS 10455 if (ftype == VM_PROT_WRITE) 10456 atomic_add_long(&num_dirty_emulations, 1); 10457 else 10458 atomic_add_long(&num_accessed_emulations, 1); 10459 #endif 10460 rv = 0; /* success */ 10461 done: 10462 if (lock != NULL) 10463 rw_wunlock(lock); 10464 PMAP_UNLOCK(pmap); 10465 return (rv); 10466 } 10467 10468 void 10469 pmap_get_mapping(pmap_t pmap, vm_offset_t va, uint64_t *ptr, int *num) 10470 { 10471 pml4_entry_t *pml4; 10472 pdp_entry_t *pdp; 10473 pd_entry_t *pde; 10474 pt_entry_t *pte, PG_V; 10475 int idx; 10476 10477 idx = 0; 10478 PG_V = pmap_valid_bit(pmap); 10479 PMAP_LOCK(pmap); 10480 10481 pml4 = pmap_pml4e(pmap, va); 10482 if (pml4 == NULL) 10483 goto done; 10484 ptr[idx++] = *pml4; 10485 if ((*pml4 & PG_V) == 0) 10486 goto done; 10487 10488 pdp = pmap_pml4e_to_pdpe(pml4, va); 10489 ptr[idx++] = *pdp; 10490 if ((*pdp & PG_V) == 0 || (*pdp & PG_PS) != 0) 10491 goto done; 10492 10493 pde = pmap_pdpe_to_pde(pdp, va); 10494 ptr[idx++] = *pde; 10495 if ((*pde & PG_V) == 0 || (*pde & PG_PS) != 0) 10496 goto done; 10497 10498 pte = pmap_pde_to_pte(pde, va); 10499 ptr[idx++] = *pte; 10500 10501 done: 10502 PMAP_UNLOCK(pmap); 10503 *num = idx; 10504 } 10505 10506 /** 10507 * Get the kernel virtual address of a set of physical pages. If there are 10508 * physical addresses not covered by the DMAP perform a transient mapping 10509 * that will be removed when calling pmap_unmap_io_transient. 10510 * 10511 * \param page The pages the caller wishes to obtain the virtual 10512 * address on the kernel memory map. 10513 * \param vaddr On return contains the kernel virtual memory address 10514 * of the pages passed in the page parameter. 10515 * \param count Number of pages passed in. 10516 * \param can_fault true if the thread using the mapped pages can take 10517 * page faults, false otherwise. 10518 * 10519 * \returns true if the caller must call pmap_unmap_io_transient when 10520 * finished or false otherwise. 10521 * 10522 */ 10523 bool 10524 pmap_map_io_transient(vm_page_t page[], void *vaddr[], int count, 10525 bool can_fault) 10526 { 10527 vm_paddr_t paddr; 10528 vmem_addr_t addr; 10529 bool needs_mapping; 10530 int error __unused, i; 10531 10532 /* 10533 * Allocate any KVA space that we need, this is done in a separate 10534 * loop to prevent calling vmem_alloc while pinned. 10535 */ 10536 needs_mapping = false; 10537 for (i = 0; i < count; i++) { 10538 paddr = VM_PAGE_TO_PHYS(page[i]); 10539 if (__predict_false(paddr >= dmaplimit)) { 10540 error = vmem_alloc(kernel_arena, PAGE_SIZE, 10541 M_BESTFIT | M_WAITOK, &addr); 10542 KASSERT(error == 0, ("vmem_alloc failed: %d", error)); 10543 vaddr[i] = (void *)addr; 10544 needs_mapping = true; 10545 } else { 10546 vaddr[i] = PHYS_TO_DMAP(paddr); 10547 } 10548 } 10549 10550 /* Exit early if everything is covered by the DMAP */ 10551 if (!needs_mapping) 10552 return (false); 10553 10554 /* 10555 * NB: The sequence of updating a page table followed by accesses 10556 * to the corresponding pages used in the !DMAP case is subject to 10557 * the situation described in the "AMD64 Architecture Programmer's 10558 * Manual Volume 2: System Programming" rev. 3.23, "7.3.1 Special 10559 * Coherency Considerations". Therefore, issuing the INVLPG right 10560 * after modifying the PTE bits is crucial. 10561 */ 10562 if (!can_fault) 10563 sched_pin(); 10564 for (i = 0; i < count; i++) { 10565 paddr = VM_PAGE_TO_PHYS(page[i]); 10566 if (paddr >= dmaplimit) { 10567 if (can_fault) { 10568 /* 10569 * Slow path, since we can get page faults 10570 * while mappings are active don't pin the 10571 * thread to the CPU and instead add a global 10572 * mapping visible to all CPUs. 10573 */ 10574 pmap_qenter(vaddr[i], &page[i], 1); 10575 } else { 10576 pmap_kenter_attr((vm_offset_t)vaddr[i], paddr, 10577 page[i]->md.pat_mode); 10578 pmap_invlpg(kernel_pmap, (vm_offset_t)vaddr[i]); 10579 } 10580 } 10581 } 10582 10583 return (needs_mapping); 10584 } 10585 10586 void 10587 pmap_unmap_io_transient(vm_page_t page[], void *vaddr[], int count, 10588 bool can_fault) 10589 { 10590 vm_paddr_t paddr; 10591 int i; 10592 10593 if (!can_fault) 10594 sched_unpin(); 10595 for (i = 0; i < count; i++) { 10596 paddr = VM_PAGE_TO_PHYS(page[i]); 10597 if (paddr >= dmaplimit) { 10598 if (can_fault) 10599 pmap_qremove(vaddr[i], 1); 10600 vmem_free(kernel_arena, (vm_offset_t)vaddr[i], 10601 PAGE_SIZE); 10602 } 10603 } 10604 } 10605 10606 void * 10607 pmap_quick_enter_page(vm_page_t m) 10608 { 10609 vm_paddr_t paddr; 10610 10611 paddr = VM_PAGE_TO_PHYS(m); 10612 if (paddr < dmaplimit) 10613 return (PHYS_TO_DMAP(paddr)); 10614 mtx_lock_spin(&qframe_mtx); 10615 KASSERT(*vtopte(qframe) == 0, ("qframe busy")); 10616 10617 /* 10618 * Since qframe is exclusively mapped by us, and we do not set 10619 * PG_G, we can use INVLPG here. 10620 */ 10621 invlpg(qframe); 10622 10623 pte_store(vtopte(qframe), paddr | X86_PG_RW | X86_PG_V | X86_PG_A | 10624 X86_PG_M | pmap_cache_bits(kernel_pmap, m->md.pat_mode, false)); 10625 return ((void *)qframe); 10626 } 10627 10628 void 10629 pmap_quick_remove_page(void *addr) 10630 { 10631 10632 if ((vm_offset_t)addr != qframe) 10633 return; 10634 pte_store(vtopte(qframe), 0); 10635 mtx_unlock_spin(&qframe_mtx); 10636 } 10637 10638 /* 10639 * Pdp pages from the large map are managed differently from either 10640 * kernel or user page table pages. They are permanently allocated at 10641 * initialization time, and their reference count is permanently set to 10642 * zero. The pml4 entries pointing to those pages are copied into 10643 * each allocated pmap. 10644 * 10645 * In contrast, pd and pt pages are managed like user page table 10646 * pages. They are dynamically allocated, and their reference count 10647 * represents the number of valid entries within the page. 10648 */ 10649 static vm_page_t 10650 pmap_large_map_getptp_unlocked(void) 10651 { 10652 return (pmap_alloc_pt_page(kernel_pmap, 0, VM_ALLOC_ZERO)); 10653 } 10654 10655 static vm_page_t 10656 pmap_large_map_getptp(void) 10657 { 10658 vm_page_t m; 10659 10660 PMAP_LOCK_ASSERT(kernel_pmap, MA_OWNED); 10661 m = pmap_large_map_getptp_unlocked(); 10662 if (m == NULL) { 10663 PMAP_UNLOCK(kernel_pmap); 10664 vm_wait(NULL); 10665 PMAP_LOCK(kernel_pmap); 10666 /* Callers retry. */ 10667 } 10668 return (m); 10669 } 10670 10671 static pdp_entry_t * 10672 pmap_large_map_pdpe(vm_offset_t va) 10673 { 10674 pml4_entry_t *pml4; 10675 vm_pindex_t pml4_idx; 10676 vm_paddr_t mphys; 10677 10678 KASSERT(va >= kva_layout.lm_low && va < kva_layout.lm_low + 10679 (vm_offset_t)NBPML4 * lm_ents, ("va %#lx not in large map", va)); 10680 if (la57) { 10681 pml4 = pmap_pml4e(kernel_pmap, va); 10682 mphys = *pml4 & PG_FRAME; 10683 } else { 10684 pml4_idx = pmap_pml4e_index(va); 10685 10686 KASSERT(LMSPML4I <= pml4_idx && pml4_idx < LMSPML4I + lm_ents, 10687 ("pmap_large_map_pdpe: va %#jx out of range idx %#jx " 10688 "LMSPML4I %#jx lm_ents %d", 10689 (uintmax_t)va, (uintmax_t)pml4_idx, LMSPML4I, lm_ents)); 10690 KASSERT((kernel_pml4[pml4_idx] & X86_PG_V) != 0, 10691 ("pmap_large_map_pdpe: invalid pml4 for va %#jx idx %#jx " 10692 "LMSPML4I %#jx lm_ents %d", 10693 (uintmax_t)va, (uintmax_t)pml4_idx, LMSPML4I, lm_ents)); 10694 mphys = kernel_pml4[pml4_idx] & PG_FRAME; 10695 } 10696 return ((pdp_entry_t *)PHYS_TO_DMAP(mphys) + pmap_pdpe_index(va)); 10697 } 10698 10699 static pd_entry_t * 10700 pmap_large_map_pde(vm_offset_t va) 10701 { 10702 pdp_entry_t *pdpe; 10703 vm_page_t m; 10704 vm_paddr_t mphys; 10705 10706 retry: 10707 pdpe = pmap_large_map_pdpe(va); 10708 if (*pdpe == 0) { 10709 m = pmap_large_map_getptp(); 10710 if (m == NULL) 10711 goto retry; 10712 mphys = VM_PAGE_TO_PHYS(m); 10713 *pdpe = mphys | X86_PG_A | X86_PG_RW | X86_PG_V | pg_nx; 10714 } else { 10715 MPASS((*pdpe & X86_PG_PS) == 0); 10716 mphys = *pdpe & PG_FRAME; 10717 } 10718 return ((pd_entry_t *)PHYS_TO_DMAP(mphys) + pmap_pde_index(va)); 10719 } 10720 10721 static pt_entry_t * 10722 pmap_large_map_pte(vm_offset_t va) 10723 { 10724 pd_entry_t *pde; 10725 vm_page_t m; 10726 vm_paddr_t mphys; 10727 10728 retry: 10729 pde = pmap_large_map_pde(va); 10730 if (*pde == 0) { 10731 m = pmap_large_map_getptp(); 10732 if (m == NULL) 10733 goto retry; 10734 mphys = VM_PAGE_TO_PHYS(m); 10735 *pde = mphys | X86_PG_A | X86_PG_RW | X86_PG_V | pg_nx; 10736 DMAP_TO_VM_PAGE(pde)->ref_count++; 10737 } else { 10738 MPASS((*pde & X86_PG_PS) == 0); 10739 mphys = *pde & PG_FRAME; 10740 } 10741 return ((pt_entry_t *)PHYS_TO_DMAP(mphys) + pmap_pte_index(va)); 10742 } 10743 10744 static vm_paddr_t 10745 pmap_large_map_kextract(vm_offset_t va) 10746 { 10747 pdp_entry_t *pdpe, pdp; 10748 pd_entry_t *pde, pd; 10749 pt_entry_t *pte, pt; 10750 10751 KASSERT(PMAP_ADDRESS_IN_LARGEMAP(va), 10752 ("not largemap range %#lx", (u_long)va)); 10753 pdpe = pmap_large_map_pdpe(va); 10754 pdp = *pdpe; 10755 KASSERT((pdp & X86_PG_V) != 0, 10756 ("invalid pdp va %#lx pdpe %#lx pdp %#lx", va, 10757 (u_long)pdpe, pdp)); 10758 if ((pdp & X86_PG_PS) != 0) { 10759 KASSERT((amd_feature & AMDID_PAGE1GB) != 0, 10760 ("no 1G pages, va %#lx pdpe %#lx pdp %#lx", va, 10761 (u_long)pdpe, pdp)); 10762 return ((pdp & PG_PS_PDP_FRAME) | (va & PDPMASK)); 10763 } 10764 pde = pmap_pdpe_to_pde(pdpe, va); 10765 pd = *pde; 10766 KASSERT((pd & X86_PG_V) != 0, 10767 ("invalid pd va %#lx pde %#lx pd %#lx", va, (u_long)pde, pd)); 10768 if ((pd & X86_PG_PS) != 0) 10769 return ((pd & PG_PS_FRAME) | (va & PDRMASK)); 10770 pte = pmap_pde_to_pte(pde, va); 10771 pt = *pte; 10772 KASSERT((pt & X86_PG_V) != 0, 10773 ("invalid pte va %#lx pte %#lx pt %#lx", va, (u_long)pte, pt)); 10774 return ((pt & PG_FRAME) | (va & PAGE_MASK)); 10775 } 10776 10777 static int 10778 pmap_large_map_getva(vm_size_t len, vm_offset_t align, vm_offset_t phase, 10779 vmem_addr_t *vmem_res) 10780 { 10781 10782 /* 10783 * Large mappings are all but static. Consequently, there 10784 * is no point in waiting for an earlier allocation to be 10785 * freed. 10786 */ 10787 return (vmem_xalloc(large_vmem, len, align, phase, 0, VMEM_ADDR_MIN, 10788 VMEM_ADDR_MAX, M_NOWAIT | M_BESTFIT, vmem_res)); 10789 } 10790 10791 int 10792 pmap_large_map(vm_paddr_t spa, vm_size_t len, void **addr, 10793 vm_memattr_t mattr) 10794 { 10795 pdp_entry_t *pdpe; 10796 pd_entry_t *pde; 10797 pt_entry_t *pte; 10798 vm_offset_t va, inc; 10799 vmem_addr_t vmem_res; 10800 vm_paddr_t pa; 10801 int error; 10802 10803 if (len == 0 || spa + len < spa) 10804 return (EINVAL); 10805 10806 /* See if DMAP can serve. */ 10807 if (spa + len <= dmaplimit) { 10808 *addr = PHYS_TO_DMAP(spa); 10809 return (pmap_change_attr(*addr, len, mattr)); 10810 } 10811 10812 /* 10813 * No, allocate KVA. Fit the address with best possible 10814 * alignment for superpages. Fall back to worse align if 10815 * failed. 10816 */ 10817 error = ENOMEM; 10818 if ((amd_feature & AMDID_PAGE1GB) != 0 && rounddown2(spa + len, 10819 NBPDP) >= roundup2(spa, NBPDP) + NBPDP) 10820 error = pmap_large_map_getva(len, NBPDP, spa & PDPMASK, 10821 &vmem_res); 10822 if (error != 0 && rounddown2(spa + len, NBPDR) >= roundup2(spa, 10823 NBPDR) + NBPDR) 10824 error = pmap_large_map_getva(len, NBPDR, spa & PDRMASK, 10825 &vmem_res); 10826 if (error != 0) 10827 error = pmap_large_map_getva(len, PAGE_SIZE, 0, &vmem_res); 10828 if (error != 0) 10829 return (error); 10830 10831 /* 10832 * Fill pagetable. PG_M is not pre-set, we scan modified bits 10833 * in the pagetable to minimize flushing. No need to 10834 * invalidate TLB, since we only update invalid entries. 10835 */ 10836 PMAP_LOCK(kernel_pmap); 10837 for (pa = spa, va = vmem_res; len > 0; pa += inc, va += inc, 10838 len -= inc) { 10839 if ((amd_feature & AMDID_PAGE1GB) != 0 && len >= NBPDP && 10840 (pa & PDPMASK) == 0 && (va & PDPMASK) == 0) { 10841 pdpe = pmap_large_map_pdpe(va); 10842 MPASS(*pdpe == 0); 10843 *pdpe = pa | pg_g | X86_PG_PS | X86_PG_RW | 10844 X86_PG_V | X86_PG_A | pg_nx | 10845 pmap_cache_bits(kernel_pmap, mattr, true); 10846 inc = NBPDP; 10847 } else if (len >= NBPDR && (pa & PDRMASK) == 0 && 10848 (va & PDRMASK) == 0) { 10849 pde = pmap_large_map_pde(va); 10850 MPASS(*pde == 0); 10851 *pde = pa | pg_g | X86_PG_PS | X86_PG_RW | 10852 X86_PG_V | X86_PG_A | pg_nx | 10853 pmap_cache_bits(kernel_pmap, mattr, true); 10854 DMAP_TO_VM_PAGE(pde)->ref_count++; 10855 inc = NBPDR; 10856 } else { 10857 pte = pmap_large_map_pte(va); 10858 MPASS(*pte == 0); 10859 *pte = pa | pg_g | X86_PG_RW | X86_PG_V | 10860 X86_PG_A | pg_nx | pmap_cache_bits(kernel_pmap, 10861 mattr, false); 10862 DMAP_TO_VM_PAGE(pte)->ref_count++; 10863 inc = PAGE_SIZE; 10864 } 10865 } 10866 PMAP_UNLOCK(kernel_pmap); 10867 MPASS(len == 0); 10868 10869 *addr = (void *)vmem_res; 10870 return (0); 10871 } 10872 10873 void 10874 pmap_large_unmap(void *svaa, vm_size_t len) 10875 { 10876 vm_offset_t sva, va; 10877 vm_size_t inc; 10878 pdp_entry_t *pdpe, pdp; 10879 pd_entry_t *pde, pd; 10880 pt_entry_t *pte; 10881 vm_page_t m; 10882 struct spglist spgf; 10883 10884 sva = (vm_offset_t)svaa; 10885 if (len == 0 || sva + len < sva || (sva >= kva_layout.dmap_low && 10886 sva + len < kva_layout.dmap_high)) 10887 return; 10888 10889 SLIST_INIT(&spgf); 10890 KASSERT(PMAP_ADDRESS_IN_LARGEMAP(sva) && 10891 PMAP_ADDRESS_IN_LARGEMAP(sva + len - 1), 10892 ("not largemap range %#lx %#lx", (u_long)svaa, (u_long)svaa + len)); 10893 PMAP_LOCK(kernel_pmap); 10894 for (va = sva; va < sva + len; va += inc) { 10895 pdpe = pmap_large_map_pdpe(va); 10896 pdp = *pdpe; 10897 KASSERT((pdp & X86_PG_V) != 0, 10898 ("invalid pdp va %#lx pdpe %#lx pdp %#lx", va, 10899 (u_long)pdpe, pdp)); 10900 if ((pdp & X86_PG_PS) != 0) { 10901 KASSERT((amd_feature & AMDID_PAGE1GB) != 0, 10902 ("no 1G pages, va %#lx pdpe %#lx pdp %#lx", va, 10903 (u_long)pdpe, pdp)); 10904 KASSERT((va & PDPMASK) == 0, 10905 ("PDPMASK bit set, va %#lx pdpe %#lx pdp %#lx", va, 10906 (u_long)pdpe, pdp)); 10907 KASSERT(va + NBPDP <= sva + len, 10908 ("unmap covers partial 1GB page, sva %#lx va %#lx " 10909 "pdpe %#lx pdp %#lx len %#lx", sva, va, 10910 (u_long)pdpe, pdp, len)); 10911 *pdpe = 0; 10912 inc = NBPDP; 10913 continue; 10914 } 10915 pde = pmap_pdpe_to_pde(pdpe, va); 10916 pd = *pde; 10917 KASSERT((pd & X86_PG_V) != 0, 10918 ("invalid pd va %#lx pde %#lx pd %#lx", va, 10919 (u_long)pde, pd)); 10920 if ((pd & X86_PG_PS) != 0) { 10921 KASSERT((va & PDRMASK) == 0, 10922 ("PDRMASK bit set, va %#lx pde %#lx pd %#lx", va, 10923 (u_long)pde, pd)); 10924 KASSERT(va + NBPDR <= sva + len, 10925 ("unmap covers partial 2MB page, sva %#lx va %#lx " 10926 "pde %#lx pd %#lx len %#lx", sva, va, (u_long)pde, 10927 pd, len)); 10928 pde_store(pde, 0); 10929 inc = NBPDR; 10930 m = DMAP_TO_VM_PAGE(pde); 10931 m->ref_count--; 10932 if (m->ref_count == 0) { 10933 *pdpe = 0; 10934 SLIST_INSERT_HEAD(&spgf, m, plinks.s.ss); 10935 } 10936 continue; 10937 } 10938 pte = pmap_pde_to_pte(pde, va); 10939 KASSERT((*pte & X86_PG_V) != 0, 10940 ("invalid pte va %#lx pte %#lx pt %#lx", va, 10941 (u_long)pte, *pte)); 10942 pte_clear(pte); 10943 inc = PAGE_SIZE; 10944 m = DMAP_TO_VM_PAGE(pte); 10945 m->ref_count--; 10946 if (m->ref_count == 0) { 10947 *pde = 0; 10948 SLIST_INSERT_HEAD(&spgf, m, plinks.s.ss); 10949 m = DMAP_TO_VM_PAGE(pde); 10950 m->ref_count--; 10951 if (m->ref_count == 0) { 10952 *pdpe = 0; 10953 SLIST_INSERT_HEAD(&spgf, m, plinks.s.ss); 10954 } 10955 } 10956 } 10957 pmap_invalidate_range(kernel_pmap, sva, sva + len); 10958 PMAP_UNLOCK(kernel_pmap); 10959 vm_page_free_pages_toq(&spgf, false); 10960 vmem_free(large_vmem, sva, len); 10961 } 10962 10963 static void 10964 pmap_large_map_wb_fence_mfence(void) 10965 { 10966 10967 mfence(); 10968 } 10969 10970 static void 10971 pmap_large_map_wb_fence_atomic(void) 10972 { 10973 10974 atomic_thread_fence_seq_cst(); 10975 } 10976 10977 static void 10978 pmap_large_map_wb_fence_nop(void) 10979 { 10980 } 10981 10982 DEFINE_IFUNC(static, void, pmap_large_map_wb_fence, (void)) 10983 { 10984 10985 if (cpu_vendor_id != CPU_VENDOR_INTEL) 10986 return (pmap_large_map_wb_fence_mfence); 10987 else if ((cpu_stdext_feature & (CPUID_STDEXT_CLWB | 10988 CPUID_STDEXT_CLFLUSHOPT)) == 0) 10989 return (pmap_large_map_wb_fence_atomic); 10990 else 10991 /* clflush is strongly enough ordered */ 10992 return (pmap_large_map_wb_fence_nop); 10993 } 10994 10995 static void 10996 pmap_large_map_flush_range_clwb(vm_offset_t va, vm_size_t len) 10997 { 10998 10999 for (; len > 0; len -= cpu_clflush_line_size, 11000 va += cpu_clflush_line_size) 11001 clwb(va); 11002 } 11003 11004 static void 11005 pmap_large_map_flush_range_clflushopt(vm_offset_t va, vm_size_t len) 11006 { 11007 11008 for (; len > 0; len -= cpu_clflush_line_size, 11009 va += cpu_clflush_line_size) 11010 clflushopt(va); 11011 } 11012 11013 static void 11014 pmap_large_map_flush_range_clflush(vm_offset_t va, vm_size_t len) 11015 { 11016 11017 for (; len > 0; len -= cpu_clflush_line_size, 11018 va += cpu_clflush_line_size) 11019 clflush(va); 11020 } 11021 11022 static void 11023 pmap_large_map_flush_range_nop(vm_offset_t sva __unused, vm_size_t len __unused) 11024 { 11025 } 11026 11027 DEFINE_IFUNC(static, void, pmap_large_map_flush_range, (vm_offset_t, vm_size_t)) 11028 { 11029 11030 if ((cpu_stdext_feature & CPUID_STDEXT_CLWB) != 0) 11031 return (pmap_large_map_flush_range_clwb); 11032 else if ((cpu_stdext_feature & CPUID_STDEXT_CLFLUSHOPT) != 0) 11033 return (pmap_large_map_flush_range_clflushopt); 11034 else if ((cpu_feature & CPUID_CLFSH) != 0) 11035 return (pmap_large_map_flush_range_clflush); 11036 else 11037 return (pmap_large_map_flush_range_nop); 11038 } 11039 11040 static void 11041 pmap_large_map_wb_large(vm_offset_t sva, vm_offset_t eva) 11042 { 11043 volatile u_long *pe; 11044 u_long p; 11045 vm_offset_t va; 11046 vm_size_t inc; 11047 bool seen_other; 11048 11049 for (va = sva; va < eva; va += inc) { 11050 inc = 0; 11051 if ((amd_feature & AMDID_PAGE1GB) != 0) { 11052 pe = (volatile u_long *)pmap_large_map_pdpe(va); 11053 p = *pe; 11054 if ((p & X86_PG_PS) != 0) 11055 inc = NBPDP; 11056 } 11057 if (inc == 0) { 11058 pe = (volatile u_long *)pmap_large_map_pde(va); 11059 p = *pe; 11060 if ((p & X86_PG_PS) != 0) 11061 inc = NBPDR; 11062 } 11063 if (inc == 0) { 11064 pe = (volatile u_long *)pmap_large_map_pte(va); 11065 p = *pe; 11066 inc = PAGE_SIZE; 11067 } 11068 seen_other = false; 11069 for (;;) { 11070 if ((p & X86_PG_AVAIL1) != 0) { 11071 /* 11072 * Spin-wait for the end of a parallel 11073 * write-back. 11074 */ 11075 cpu_spinwait(); 11076 p = *pe; 11077 11078 /* 11079 * If we saw other write-back 11080 * occurring, we cannot rely on PG_M to 11081 * indicate state of the cache. The 11082 * PG_M bit is cleared before the 11083 * flush to avoid ignoring new writes, 11084 * and writes which are relevant for 11085 * us might happen after. 11086 */ 11087 seen_other = true; 11088 continue; 11089 } 11090 11091 if ((p & X86_PG_M) != 0 || seen_other) { 11092 if (!atomic_fcmpset_long(pe, &p, 11093 (p & ~X86_PG_M) | X86_PG_AVAIL1)) 11094 /* 11095 * If we saw PG_M without 11096 * PG_AVAIL1, and then on the 11097 * next attempt we do not 11098 * observe either PG_M or 11099 * PG_AVAIL1, the other 11100 * write-back started after us 11101 * and finished before us. We 11102 * can rely on it doing our 11103 * work. 11104 */ 11105 continue; 11106 pmap_large_map_flush_range(va, inc); 11107 atomic_clear_long(pe, X86_PG_AVAIL1); 11108 } 11109 break; 11110 } 11111 maybe_yield(); 11112 } 11113 } 11114 11115 /* 11116 * Write-back cache lines for the given address range. 11117 * 11118 * Must be called only on the range or sub-range returned from 11119 * pmap_large_map(). Must not be called on the coalesced ranges. 11120 * 11121 * Does nothing on CPUs without CLWB, CLFLUSHOPT, or CLFLUSH 11122 * instructions support. 11123 */ 11124 void 11125 pmap_large_map_wb(void *svap, vm_size_t len) 11126 { 11127 vm_offset_t eva, sva; 11128 11129 sva = (vm_offset_t)svap; 11130 eva = sva + len; 11131 pmap_large_map_wb_fence(); 11132 if (sva >= kva_layout.dmap_low && eva < kva_layout.dmap_high) { 11133 pmap_large_map_flush_range(sva, len); 11134 } else { 11135 KASSERT(sva >= kva_layout.lm_low && eva < kva_layout.lm_high, 11136 ("pmap_large_map_wb: not largemap %#lx %#lx", sva, len)); 11137 pmap_large_map_wb_large(sva, eva); 11138 } 11139 pmap_large_map_wb_fence(); 11140 } 11141 11142 static vm_page_t 11143 pmap_pti_alloc_page(void) 11144 { 11145 vm_page_t m; 11146 11147 VM_OBJECT_ASSERT_WLOCKED(pti_obj); 11148 m = vm_page_grab(pti_obj, pti_pg_idx++, VM_ALLOC_WIRED | VM_ALLOC_ZERO); 11149 return (m); 11150 } 11151 11152 static bool 11153 pmap_pti_free_page(vm_page_t m) 11154 { 11155 if (!vm_page_unwire_noq(m)) 11156 return (false); 11157 vm_page_xbusy_claim(m); 11158 vm_page_free_zero(m); 11159 return (true); 11160 } 11161 11162 static void 11163 pmap_pti_init(void) 11164 { 11165 vm_page_t pml4_pg; 11166 pdp_entry_t *pdpe; 11167 vm_offset_t va; 11168 int i; 11169 11170 if (!pti) 11171 return; 11172 pti_obj = vm_pager_allocate(OBJT_PHYS, NULL, 0, VM_PROT_ALL, 0, NULL); 11173 VM_OBJECT_WLOCK(pti_obj); 11174 pml4_pg = pmap_pti_alloc_page(); 11175 pti_pml4 = VM_PAGE_TO_DMAP(pml4_pg); 11176 for (va = kva_layout.km_low; va <= kva_layout.km_high && 11177 va >= kva_layout.km_low && va > NBPML4; va += NBPML4) { 11178 pdpe = pmap_pti_pdpe(va); 11179 pmap_pti_wire_pte(pdpe); 11180 } 11181 pmap_pti_add_kva_locked((vm_offset_t)&__pcpu[0], 11182 (vm_offset_t)&__pcpu[0] + sizeof(__pcpu[0]) * MAXCPU, false); 11183 pmap_pti_add_kva_locked((vm_offset_t)idt, (vm_offset_t)idt + 11184 sizeof(struct gate_descriptor) * NIDT, false); 11185 CPU_FOREACH(i) { 11186 /* Doublefault stack IST 1 */ 11187 va = __pcpu[i].pc_common_tss.tss_ist1 + sizeof(struct nmi_pcpu); 11188 pmap_pti_add_kva_locked(va - DBLFAULT_STACK_SIZE, va, false); 11189 /* NMI stack IST 2 */ 11190 va = __pcpu[i].pc_common_tss.tss_ist2 + sizeof(struct nmi_pcpu); 11191 pmap_pti_add_kva_locked(va - NMI_STACK_SIZE, va, false); 11192 /* MC# stack IST 3 */ 11193 va = __pcpu[i].pc_common_tss.tss_ist3 + 11194 sizeof(struct nmi_pcpu); 11195 pmap_pti_add_kva_locked(va - MCE_STACK_SIZE, va, false); 11196 /* DB# stack IST 4 */ 11197 va = __pcpu[i].pc_common_tss.tss_ist4 + sizeof(struct nmi_pcpu); 11198 pmap_pti_add_kva_locked(va - DBG_STACK_SIZE, va, false); 11199 } 11200 pmap_pti_add_kva_locked((vm_offset_t)KERNSTART, (vm_offset_t)etext, 11201 true); 11202 pti_finalized = true; 11203 VM_OBJECT_WUNLOCK(pti_obj); 11204 } 11205 11206 static void 11207 pmap_cpu_init(void *arg __unused) 11208 { 11209 CPU_COPY(&all_cpus, &kernel_pmap->pm_active); 11210 pmap_pti_init(); 11211 } 11212 SYSINIT(pmap_cpu, SI_SUB_CPU, SI_ORDER_LAST, pmap_cpu_init, NULL); 11213 11214 static pdp_entry_t * 11215 pmap_pti_pdpe(vm_offset_t va) 11216 { 11217 pml4_entry_t *pml4e; 11218 pdp_entry_t *pdpe; 11219 vm_page_t m; 11220 vm_pindex_t pml4_idx; 11221 vm_paddr_t mphys; 11222 11223 VM_OBJECT_ASSERT_WLOCKED(pti_obj); 11224 11225 pml4_idx = pmap_pml4e_index(va); 11226 pml4e = &pti_pml4[pml4_idx]; 11227 m = NULL; 11228 if (*pml4e == 0) { 11229 if (pti_finalized) 11230 panic("pml4 alloc after finalization\n"); 11231 m = pmap_pti_alloc_page(); 11232 if (*pml4e != 0) { 11233 pmap_pti_free_page(m); 11234 mphys = *pml4e & ~PAGE_MASK; 11235 } else { 11236 mphys = VM_PAGE_TO_PHYS(m); 11237 *pml4e = mphys | X86_PG_RW | X86_PG_V; 11238 } 11239 } else { 11240 mphys = *pml4e & ~PAGE_MASK; 11241 } 11242 pdpe = (pdp_entry_t *)PHYS_TO_DMAP(mphys) + pmap_pdpe_index(va); 11243 return (pdpe); 11244 } 11245 11246 static void 11247 pmap_pti_wire_pte(void *pte) 11248 { 11249 vm_page_t m; 11250 11251 VM_OBJECT_ASSERT_WLOCKED(pti_obj); 11252 m = DMAP_TO_VM_PAGE(pte); 11253 m->ref_count++; 11254 } 11255 11256 static void 11257 pmap_pti_unwire_pde(void *pde, bool only_ref) 11258 { 11259 vm_page_t m; 11260 11261 VM_OBJECT_ASSERT_WLOCKED(pti_obj); 11262 m = DMAP_TO_VM_PAGE(pde); 11263 MPASS(only_ref || m->ref_count > 1); 11264 pmap_pti_free_page(m); 11265 } 11266 11267 static void 11268 pmap_pti_unwire_pte(void *pte, vm_offset_t va) 11269 { 11270 vm_page_t m; 11271 pd_entry_t *pde; 11272 11273 VM_OBJECT_ASSERT_WLOCKED(pti_obj); 11274 m = DMAP_TO_VM_PAGE(pte); 11275 if (pmap_pti_free_page(m)) { 11276 pde = pmap_pti_pde(va); 11277 MPASS((*pde & (X86_PG_PS | X86_PG_V)) == X86_PG_V); 11278 *pde = 0; 11279 pmap_pti_unwire_pde(pde, false); 11280 } 11281 } 11282 11283 static pd_entry_t * 11284 pmap_pti_pde(vm_offset_t va) 11285 { 11286 pdp_entry_t *pdpe; 11287 pd_entry_t *pde; 11288 vm_page_t m; 11289 vm_pindex_t pd_idx; 11290 vm_paddr_t mphys; 11291 11292 VM_OBJECT_ASSERT_WLOCKED(pti_obj); 11293 11294 pdpe = pmap_pti_pdpe(va); 11295 if (*pdpe == 0) { 11296 m = pmap_pti_alloc_page(); 11297 if (*pdpe != 0) { 11298 pmap_pti_free_page(m); 11299 MPASS((*pdpe & X86_PG_PS) == 0); 11300 mphys = *pdpe & ~PAGE_MASK; 11301 } else { 11302 mphys = VM_PAGE_TO_PHYS(m); 11303 *pdpe = mphys | X86_PG_RW | X86_PG_V; 11304 } 11305 } else { 11306 MPASS((*pdpe & X86_PG_PS) == 0); 11307 mphys = *pdpe & ~PAGE_MASK; 11308 } 11309 11310 pde = PHYS_TO_DMAP(mphys); 11311 pd_idx = pmap_pde_index(va); 11312 pde += pd_idx; 11313 return (pde); 11314 } 11315 11316 static pt_entry_t * 11317 pmap_pti_pte(vm_offset_t va, bool *unwire_pde) 11318 { 11319 pd_entry_t *pde; 11320 pt_entry_t *pte; 11321 vm_page_t m; 11322 vm_paddr_t mphys; 11323 11324 VM_OBJECT_ASSERT_WLOCKED(pti_obj); 11325 11326 pde = pmap_pti_pde(va); 11327 if (unwire_pde != NULL) { 11328 *unwire_pde = true; 11329 pmap_pti_wire_pte(pde); 11330 } 11331 if (*pde == 0) { 11332 m = pmap_pti_alloc_page(); 11333 if (*pde != 0) { 11334 pmap_pti_free_page(m); 11335 MPASS((*pde & X86_PG_PS) == 0); 11336 mphys = *pde & ~(PAGE_MASK | pg_nx); 11337 } else { 11338 mphys = VM_PAGE_TO_PHYS(m); 11339 *pde = mphys | X86_PG_RW | X86_PG_V; 11340 if (unwire_pde != NULL) 11341 *unwire_pde = false; 11342 } 11343 } else { 11344 MPASS((*pde & X86_PG_PS) == 0); 11345 mphys = *pde & ~(PAGE_MASK | pg_nx); 11346 } 11347 11348 pte = PHYS_TO_DMAP(mphys); 11349 pte += pmap_pte_index(va); 11350 11351 return (pte); 11352 } 11353 11354 static void 11355 pmap_pti_add_kva_locked(vm_offset_t sva, vm_offset_t eva, bool exec) 11356 { 11357 vm_paddr_t pa; 11358 pd_entry_t *pde; 11359 pt_entry_t *pte, ptev; 11360 bool unwire_pde; 11361 11362 VM_OBJECT_ASSERT_WLOCKED(pti_obj); 11363 11364 sva = trunc_page(sva); 11365 MPASS(sva > VM_MAXUSER_ADDRESS); 11366 eva = round_page(eva); 11367 MPASS(sva < eva); 11368 for (; sva < eva; sva += PAGE_SIZE) { 11369 pte = pmap_pti_pte(sva, &unwire_pde); 11370 pa = pmap_kextract(sva); 11371 ptev = pa | X86_PG_RW | X86_PG_V | X86_PG_A | X86_PG_G | 11372 (exec ? 0 : pg_nx) | pmap_cache_bits(kernel_pmap, 11373 VM_MEMATTR_DEFAULT, false); 11374 if (*pte == 0) { 11375 pte_store(pte, ptev); 11376 pmap_pti_wire_pte(pte); 11377 } else { 11378 KASSERT(!pti_finalized, 11379 ("pti overlap after fin %#lx %#lx %#lx", 11380 sva, *pte, ptev)); 11381 KASSERT(*pte == ptev, 11382 ("pti non-identical pte after fin %#lx %#lx %#lx", 11383 sva, *pte, ptev)); 11384 } 11385 if (unwire_pde) { 11386 pde = pmap_pti_pde(sva); 11387 pmap_pti_unwire_pde(pde, true); 11388 } 11389 } 11390 } 11391 11392 void 11393 pmap_pti_add_kva(vm_offset_t sva, vm_offset_t eva, bool exec) 11394 { 11395 11396 if (!pti) 11397 return; 11398 VM_OBJECT_WLOCK(pti_obj); 11399 pmap_pti_add_kva_locked(sva, eva, exec); 11400 VM_OBJECT_WUNLOCK(pti_obj); 11401 } 11402 11403 void 11404 pmap_pti_remove_kva(vm_offset_t sva, vm_offset_t eva) 11405 { 11406 pt_entry_t *pte; 11407 vm_offset_t va; 11408 11409 if (!pti) 11410 return; 11411 sva = rounddown2(sva, PAGE_SIZE); 11412 MPASS(sva > VM_MAXUSER_ADDRESS); 11413 eva = roundup2(eva, PAGE_SIZE); 11414 MPASS(sva < eva); 11415 VM_OBJECT_WLOCK(pti_obj); 11416 for (va = sva; va < eva; va += PAGE_SIZE) { 11417 pte = pmap_pti_pte(va, NULL); 11418 KASSERT((*pte & X86_PG_V) != 0, 11419 ("invalid pte va %#lx pte %#lx pt %#lx", va, 11420 (u_long)pte, *pte)); 11421 pte_clear(pte); 11422 pmap_pti_unwire_pte(pte, va); 11423 } 11424 pmap_invalidate_range(kernel_pmap, sva, eva); 11425 VM_OBJECT_WUNLOCK(pti_obj); 11426 } 11427 11428 static void * 11429 pkru_dup_range(void *ctx __unused, void *data) 11430 { 11431 struct pmap_pkru_range *node, *new_node; 11432 11433 new_node = uma_zalloc(pmap_pkru_ranges_zone, M_NOWAIT); 11434 if (new_node == NULL) 11435 return (NULL); 11436 node = data; 11437 memcpy(new_node, node, sizeof(*node)); 11438 return (new_node); 11439 } 11440 11441 static void 11442 pkru_free_range(void *ctx __unused, void *node) 11443 { 11444 11445 uma_zfree(pmap_pkru_ranges_zone, node); 11446 } 11447 11448 static int 11449 pmap_pkru_assign(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, u_int keyidx, 11450 int flags) 11451 { 11452 struct pmap_pkru_range *ppr; 11453 int error; 11454 11455 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 11456 MPASS(pmap->pm_type == PT_X86); 11457 MPASS((cpu_stdext_feature2 & CPUID_STDEXT2_PKU) != 0); 11458 if ((flags & AMD64_PKRU_EXCL) != 0 && 11459 !rangeset_check_empty(&pmap->pm_pkru, sva, eva)) 11460 return (EBUSY); 11461 ppr = uma_zalloc(pmap_pkru_ranges_zone, M_NOWAIT); 11462 if (ppr == NULL) 11463 return (ENOMEM); 11464 ppr->pkru_keyidx = keyidx; 11465 ppr->pkru_flags = flags & AMD64_PKRU_PERSIST; 11466 error = rangeset_insert(&pmap->pm_pkru, sva, eva, ppr); 11467 if (error != 0) 11468 uma_zfree(pmap_pkru_ranges_zone, ppr); 11469 return (error); 11470 } 11471 11472 static int 11473 pmap_pkru_deassign(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) 11474 { 11475 11476 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 11477 MPASS(pmap->pm_type == PT_X86); 11478 MPASS((cpu_stdext_feature2 & CPUID_STDEXT2_PKU) != 0); 11479 return (rangeset_remove(&pmap->pm_pkru, sva, eva)); 11480 } 11481 11482 static void 11483 pmap_pkru_deassign_all(pmap_t pmap) 11484 { 11485 11486 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 11487 if (pmap->pm_type == PT_X86 && 11488 (cpu_stdext_feature2 & CPUID_STDEXT2_PKU) != 0) 11489 rangeset_remove_all(&pmap->pm_pkru); 11490 } 11491 11492 /* 11493 * Returns true if the PKU setting is the same across the specified address 11494 * range, and false otherwise. When returning true, updates the referenced PTE 11495 * to reflect the PKU setting. 11496 */ 11497 static bool 11498 pmap_pkru_same(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, pt_entry_t *pte) 11499 { 11500 struct pmap_pkru_range *ppr; 11501 vm_offset_t va; 11502 u_int keyidx; 11503 11504 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 11505 KASSERT(pmap->pm_type != PT_X86 || (*pte & X86_PG_PKU_MASK) == 0, 11506 ("pte %p has unexpected PKU %ld", pte, *pte & X86_PG_PKU_MASK)); 11507 if (pmap->pm_type != PT_X86 || 11508 (cpu_stdext_feature2 & CPUID_STDEXT2_PKU) == 0 || 11509 sva >= VM_MAXUSER_ADDRESS) 11510 return (true); 11511 MPASS(eva <= VM_MAXUSER_ADDRESS); 11512 ppr = rangeset_containing(&pmap->pm_pkru, sva); 11513 if (ppr == NULL) 11514 return (rangeset_empty(&pmap->pm_pkru, sva, eva)); 11515 keyidx = ppr->pkru_keyidx; 11516 while ((va = ppr->pkru_rs_el.re_end) < eva) { 11517 if ((ppr = rangeset_beginning(&pmap->pm_pkru, va)) == NULL || 11518 keyidx != ppr->pkru_keyidx) 11519 return (false); 11520 } 11521 *pte |= X86_PG_PKU(keyidx); 11522 return (true); 11523 } 11524 11525 static pt_entry_t 11526 pmap_pkru_get(pmap_t pmap, vm_offset_t va) 11527 { 11528 struct pmap_pkru_range *ppr; 11529 11530 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 11531 if (pmap->pm_type != PT_X86 || 11532 (cpu_stdext_feature2 & CPUID_STDEXT2_PKU) == 0 || 11533 va >= VM_MAXUSER_ADDRESS) 11534 return (0); 11535 ppr = rangeset_containing(&pmap->pm_pkru, va); 11536 if (ppr != NULL) 11537 return (X86_PG_PKU(ppr->pkru_keyidx)); 11538 return (0); 11539 } 11540 11541 static bool 11542 pred_pkru_on_remove(void *ctx __unused, void *r) 11543 { 11544 struct pmap_pkru_range *ppr; 11545 11546 ppr = r; 11547 return ((ppr->pkru_flags & AMD64_PKRU_PERSIST) == 0); 11548 } 11549 11550 static void 11551 pmap_pkru_on_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) 11552 { 11553 11554 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 11555 if (pmap->pm_type == PT_X86 && 11556 (cpu_stdext_feature2 & CPUID_STDEXT2_PKU) != 0) { 11557 rangeset_remove_pred(&pmap->pm_pkru, sva, eva, 11558 pred_pkru_on_remove); 11559 } 11560 } 11561 11562 static int 11563 pmap_pkru_copy(pmap_t dst_pmap, pmap_t src_pmap) 11564 { 11565 11566 PMAP_LOCK_ASSERT(dst_pmap, MA_OWNED); 11567 PMAP_LOCK_ASSERT(src_pmap, MA_OWNED); 11568 MPASS(dst_pmap->pm_type == PT_X86); 11569 MPASS(src_pmap->pm_type == PT_X86); 11570 MPASS((cpu_stdext_feature2 & CPUID_STDEXT2_PKU) != 0); 11571 if (src_pmap->pm_pkru.rs_data_ctx == NULL) 11572 return (0); 11573 return (rangeset_copy(&dst_pmap->pm_pkru, &src_pmap->pm_pkru)); 11574 } 11575 11576 static void 11577 pmap_pkru_update_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, 11578 u_int keyidx) 11579 { 11580 pml4_entry_t *pml4e; 11581 pdp_entry_t newpdpe, *pdpe; 11582 pd_entry_t newpde, ptpaddr, *pde; 11583 pt_entry_t newpte, *ptep, pte; 11584 vm_offset_t va, va_next; 11585 bool changed; 11586 11587 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 11588 MPASS(pmap->pm_type == PT_X86); 11589 MPASS(keyidx <= PMAP_MAX_PKRU_IDX); 11590 11591 for (changed = false, va = sva; va < eva; va = va_next) { 11592 pml4e = pmap_pml4e(pmap, va); 11593 if (pml4e == NULL || (*pml4e & X86_PG_V) == 0) { 11594 va_next = (va + NBPML4) & ~PML4MASK; 11595 if (va_next < va) 11596 va_next = eva; 11597 continue; 11598 } 11599 11600 pdpe = pmap_pml4e_to_pdpe(pml4e, va); 11601 if ((*pdpe & X86_PG_V) == 0) { 11602 va_next = (va + NBPDP) & ~PDPMASK; 11603 if (va_next < va) 11604 va_next = eva; 11605 continue; 11606 } 11607 if ((*pdpe & PG_PS) != 0) { 11608 va_next = (va + NBPDP) & ~PDPMASK; 11609 if (va_next < va) 11610 va_next = eva; 11611 KASSERT(va_next <= eva, 11612 ("partial update of non-transparent 1G mapping " 11613 "pdpe %#lx va %#lx eva %#lx va_next %#lx", 11614 *pdpe, va, eva, va_next)); 11615 newpdpe = (*pdpe & ~X86_PG_PKU_MASK) | 11616 X86_PG_PKU(keyidx); 11617 if (newpdpe != *pdpe) { 11618 *pdpe = newpdpe; 11619 changed = true; 11620 } 11621 continue; 11622 } 11623 11624 va_next = (va + NBPDR) & ~PDRMASK; 11625 if (va_next < va) 11626 va_next = eva; 11627 11628 pde = pmap_pdpe_to_pde(pdpe, va); 11629 ptpaddr = *pde; 11630 if (ptpaddr == 0) 11631 continue; 11632 11633 MPASS((ptpaddr & X86_PG_V) != 0); 11634 if ((ptpaddr & PG_PS) != 0) { 11635 if (va + NBPDR == va_next && eva >= va_next) { 11636 newpde = (ptpaddr & ~X86_PG_PKU_MASK) | 11637 X86_PG_PKU(keyidx); 11638 if (newpde != ptpaddr) { 11639 *pde = newpde; 11640 changed = true; 11641 } 11642 continue; 11643 } else if (!pmap_demote_pde(pmap, pde, va)) { 11644 continue; 11645 } 11646 } 11647 11648 if (va_next > eva) 11649 va_next = eva; 11650 11651 for (ptep = pmap_pde_to_pte(pde, va); va != va_next; 11652 ptep++, va += PAGE_SIZE) { 11653 pte = *ptep; 11654 if ((pte & X86_PG_V) == 0) 11655 continue; 11656 newpte = (pte & ~X86_PG_PKU_MASK) | X86_PG_PKU(keyidx); 11657 if (newpte != pte) { 11658 *ptep = newpte; 11659 changed = true; 11660 } 11661 } 11662 } 11663 if (changed) 11664 pmap_invalidate_range(pmap, sva, eva); 11665 } 11666 11667 static int 11668 pmap_pkru_check_uargs(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, 11669 u_int keyidx, int flags) 11670 { 11671 11672 if (pmap->pm_type != PT_X86 || keyidx > PMAP_MAX_PKRU_IDX || 11673 (flags & ~(AMD64_PKRU_PERSIST | AMD64_PKRU_EXCL)) != 0) 11674 return (EINVAL); 11675 if ((cpu_stdext_feature2 & CPUID_STDEXT2_PKU) == 0) 11676 return (ENOTSUP); 11677 return (0); 11678 } 11679 11680 int 11681 pmap_pkru_set(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, u_int keyidx, 11682 int flags) 11683 { 11684 int error; 11685 11686 sva = trunc_page(sva); 11687 eva = round_page(eva); 11688 error = pmap_pkru_check_uargs(pmap, sva, eva, keyidx, flags); 11689 if (error != 0) 11690 return (error); 11691 for (;;) { 11692 PMAP_LOCK(pmap); 11693 error = pmap_pkru_assign(pmap, sva, eva, keyidx, flags); 11694 if (error == 0) 11695 pmap_pkru_update_range(pmap, sva, eva, keyidx); 11696 PMAP_UNLOCK(pmap); 11697 if (error != ENOMEM) 11698 break; 11699 vm_wait(NULL); 11700 } 11701 return (error); 11702 } 11703 11704 int 11705 pmap_pkru_clear(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) 11706 { 11707 int error; 11708 11709 sva = trunc_page(sva); 11710 eva = round_page(eva); 11711 error = pmap_pkru_check_uargs(pmap, sva, eva, 0, 0); 11712 if (error != 0) 11713 return (error); 11714 for (;;) { 11715 PMAP_LOCK(pmap); 11716 error = pmap_pkru_deassign(pmap, sva, eva); 11717 if (error == 0) 11718 pmap_pkru_update_range(pmap, sva, eva, 0); 11719 PMAP_UNLOCK(pmap); 11720 if (error != ENOMEM) 11721 break; 11722 vm_wait(NULL); 11723 } 11724 return (error); 11725 } 11726 11727 #if defined(KASAN) || defined(KMSAN) 11728 11729 /* 11730 * Reserve enough memory to: 11731 * 1) allocate PDP pages for the shadow map(s), 11732 * 2) shadow the boot stack of KSTACK_PAGES pages, 11733 * 3) assuming that the kernel stack does not cross a 1GB boundary, 11734 * so we need one or two PD pages, one or two PT pages, and KSTACK_PAGES shadow 11735 * pages per shadow map. 11736 */ 11737 #ifdef KASAN 11738 #define SAN_EARLY_PAGES \ 11739 (NKASANPML4E + 2 + 2 + howmany(KSTACK_PAGES, KASAN_SHADOW_SCALE)) 11740 #else 11741 #define SAN_EARLY_PAGES \ 11742 (NKMSANSHADPML4E + NKMSANORIGPML4E + 2 * (2 + 2 + KSTACK_PAGES)) 11743 #endif 11744 11745 static uint64_t __nosanitizeaddress __nosanitizememory 11746 pmap_san_enter_early_alloc_4k(uint64_t pabase) 11747 { 11748 static uint8_t data[PAGE_SIZE * SAN_EARLY_PAGES] __aligned(PAGE_SIZE); 11749 static size_t offset = 0; 11750 uint64_t pa; 11751 11752 if (offset == sizeof(data)) { 11753 panic("%s: ran out of memory for the bootstrap shadow map", 11754 __func__); 11755 } 11756 11757 pa = pabase + ((vm_offset_t)&data[offset] - KERNSTART); 11758 offset += PAGE_SIZE; 11759 return (pa); 11760 } 11761 11762 /* 11763 * Map a shadow page, before the kernel has bootstrapped its page tables. This 11764 * is currently only used to shadow the temporary boot stack set up by locore. 11765 */ 11766 static void __nosanitizeaddress __nosanitizememory 11767 pmap_san_enter_early(vm_offset_t va) 11768 { 11769 static bool first = true; 11770 pml4_entry_t *pml4e; 11771 pdp_entry_t *pdpe; 11772 pd_entry_t *pde; 11773 pt_entry_t *pte; 11774 uint64_t cr3, pa, base; 11775 int i; 11776 11777 base = amd64_loadaddr(); 11778 cr3 = rcr3(); 11779 11780 if (first) { 11781 /* 11782 * If this the first call, we need to allocate new PML4Es for 11783 * the bootstrap shadow map(s). We don't know how the PML4 page 11784 * was initialized by the boot loader, so we can't simply test 11785 * whether the shadow map's PML4Es are zero. 11786 */ 11787 first = false; 11788 #ifdef KASAN 11789 for (i = 0; i < NKASANPML4E; i++) { 11790 pa = pmap_san_enter_early_alloc_4k(base); 11791 11792 pml4e = (pml4_entry_t *)cr3 + 11793 pmap_pml4e_index(KASAN_MIN_ADDRESS + i * NBPML4); 11794 *pml4e = (pml4_entry_t)(pa | X86_PG_RW | X86_PG_V); 11795 } 11796 #else 11797 for (i = 0; i < NKMSANORIGPML4E; i++) { 11798 pa = pmap_san_enter_early_alloc_4k(base); 11799 11800 pml4e = (pml4_entry_t *)cr3 + 11801 pmap_pml4e_index(KMSAN_ORIG_MIN_ADDRESS + 11802 i * NBPML4); 11803 *pml4e = (pml4_entry_t)(pa | X86_PG_RW | X86_PG_V); 11804 } 11805 for (i = 0; i < NKMSANSHADPML4E; i++) { 11806 pa = pmap_san_enter_early_alloc_4k(base); 11807 11808 pml4e = (pml4_entry_t *)cr3 + 11809 pmap_pml4e_index(KMSAN_SHAD_MIN_ADDRESS + 11810 i * NBPML4); 11811 *pml4e = (pml4_entry_t)(pa | X86_PG_RW | X86_PG_V); 11812 } 11813 #endif 11814 } 11815 pml4e = (pml4_entry_t *)cr3 + pmap_pml4e_index(va); 11816 pdpe = (pdp_entry_t *)(*pml4e & PG_FRAME) + pmap_pdpe_index(va); 11817 if (*pdpe == 0) { 11818 pa = pmap_san_enter_early_alloc_4k(base); 11819 *pdpe = (pdp_entry_t)(pa | X86_PG_RW | X86_PG_V); 11820 } 11821 pde = (pd_entry_t *)(*pdpe & PG_FRAME) + pmap_pde_index(va); 11822 if (*pde == 0) { 11823 pa = pmap_san_enter_early_alloc_4k(base); 11824 *pde = (pd_entry_t)(pa | X86_PG_RW | X86_PG_V); 11825 } 11826 pte = (pt_entry_t *)(*pde & PG_FRAME) + pmap_pte_index(va); 11827 if (*pte != 0) 11828 panic("%s: PTE for %#lx is already initialized", __func__, va); 11829 pa = pmap_san_enter_early_alloc_4k(base); 11830 *pte = (pt_entry_t)(pa | X86_PG_A | X86_PG_M | X86_PG_RW | X86_PG_V); 11831 } 11832 11833 static vm_page_t 11834 pmap_san_enter_alloc_4k(void) 11835 { 11836 vm_page_t m; 11837 11838 m = vm_page_alloc_noobj(VM_ALLOC_INTERRUPT | VM_ALLOC_WIRED | 11839 VM_ALLOC_ZERO); 11840 if (m == NULL) 11841 panic("%s: no memory to grow shadow map", __func__); 11842 return (m); 11843 } 11844 11845 static vm_page_t 11846 pmap_san_enter_alloc_2m(void) 11847 { 11848 return (vm_page_alloc_noobj_contig(VM_ALLOC_WIRED | VM_ALLOC_ZERO, 11849 NPTEPG, 0, ~0ul, NBPDR, 0, VM_MEMATTR_DEFAULT)); 11850 } 11851 11852 /* 11853 * Grow a shadow map by at least one 4KB page at the specified address. Use 2MB 11854 * pages when possible. 11855 */ 11856 void __nosanitizeaddress __nosanitizememory 11857 pmap_san_enter(vm_offset_t va) 11858 { 11859 pdp_entry_t *pdpe; 11860 pd_entry_t *pde; 11861 pt_entry_t *pte; 11862 vm_page_t m; 11863 11864 if (kernphys == 0) { 11865 /* 11866 * We're creating a temporary shadow map for the boot stack. 11867 */ 11868 pmap_san_enter_early(va); 11869 return; 11870 } 11871 11872 mtx_assert(&kernel_map->system_mtx, MA_OWNED); 11873 11874 pdpe = pmap_pdpe(kernel_pmap, va); 11875 if ((*pdpe & X86_PG_V) == 0) { 11876 m = pmap_san_enter_alloc_4k(); 11877 *pdpe = (pdp_entry_t)(VM_PAGE_TO_PHYS(m) | X86_PG_RW | 11878 X86_PG_V | pg_nx); 11879 } 11880 pde = pmap_pdpe_to_pde(pdpe, va); 11881 if ((*pde & X86_PG_V) == 0) { 11882 m = pmap_san_enter_alloc_2m(); 11883 if (m != NULL) { 11884 *pde = (pd_entry_t)(VM_PAGE_TO_PHYS(m) | X86_PG_RW | 11885 X86_PG_PS | X86_PG_V | X86_PG_A | X86_PG_M | pg_nx); 11886 } else { 11887 m = pmap_san_enter_alloc_4k(); 11888 *pde = (pd_entry_t)(VM_PAGE_TO_PHYS(m) | X86_PG_RW | 11889 X86_PG_V | pg_nx); 11890 } 11891 } 11892 if ((*pde & X86_PG_PS) != 0) 11893 return; 11894 pte = pmap_pde_to_pte(pde, va); 11895 if ((*pte & X86_PG_V) != 0) 11896 return; 11897 m = pmap_san_enter_alloc_4k(); 11898 *pte = (pt_entry_t)(VM_PAGE_TO_PHYS(m) | X86_PG_RW | X86_PG_V | 11899 X86_PG_M | X86_PG_A | pg_nx); 11900 } 11901 #endif 11902 11903 /* 11904 * Track a range of the kernel's virtual address space that is contiguous 11905 * in various mapping attributes. 11906 */ 11907 struct pmap_kernel_map_range { 11908 vm_offset_t sva; 11909 pt_entry_t attrs; 11910 int ptes; 11911 int pdes; 11912 int pdpes; 11913 }; 11914 11915 static void 11916 sysctl_kmaps_dump(struct sbuf *sb, struct pmap_kernel_map_range *range, 11917 vm_offset_t eva) 11918 { 11919 const char *mode; 11920 int i, pat_idx; 11921 11922 if (eva <= range->sva) 11923 return; 11924 11925 pat_idx = pmap_pat_index(kernel_pmap, range->attrs, true); 11926 for (i = 0; i < PAT_INDEX_SIZE; i++) 11927 if (pat_index[i] == pat_idx) 11928 break; 11929 11930 switch (i) { 11931 case PAT_WRITE_BACK: 11932 mode = "WB"; 11933 break; 11934 case PAT_WRITE_THROUGH: 11935 mode = "WT"; 11936 break; 11937 case PAT_UNCACHEABLE: 11938 mode = "UC"; 11939 break; 11940 case PAT_UNCACHED: 11941 mode = "U-"; 11942 break; 11943 case PAT_WRITE_PROTECTED: 11944 mode = "WP"; 11945 break; 11946 case PAT_WRITE_COMBINING: 11947 mode = "WC"; 11948 break; 11949 default: 11950 printf("%s: unknown PAT mode %#x for range 0x%016lx-0x%016lx\n", 11951 __func__, pat_idx, range->sva, eva); 11952 mode = "??"; 11953 break; 11954 } 11955 11956 sbuf_printf(sb, "0x%016lx-0x%016lx r%c%c%c%c %s %d %d %d\n", 11957 range->sva, eva, 11958 (range->attrs & X86_PG_RW) != 0 ? 'w' : '-', 11959 (range->attrs & pg_nx) != 0 ? '-' : 'x', 11960 (range->attrs & X86_PG_U) != 0 ? 'u' : 's', 11961 (range->attrs & X86_PG_G) != 0 ? 'g' : '-', 11962 mode, range->pdpes, range->pdes, range->ptes); 11963 11964 /* Reset to sentinel value. */ 11965 range->sva = kva_layout.kva_max; 11966 } 11967 11968 /* 11969 * Determine whether the attributes specified by a page table entry match those 11970 * being tracked by the current range. This is not quite as simple as a direct 11971 * flag comparison since some PAT modes have multiple representations. 11972 */ 11973 static bool 11974 sysctl_kmaps_match(struct pmap_kernel_map_range *range, pt_entry_t attrs) 11975 { 11976 pt_entry_t diff, mask; 11977 11978 mask = X86_PG_G | X86_PG_RW | X86_PG_U | X86_PG_PDE_CACHE | pg_nx; 11979 diff = (range->attrs ^ attrs) & mask; 11980 if (diff == 0) 11981 return (true); 11982 if ((diff & ~X86_PG_PDE_PAT) == 0 && 11983 pmap_pat_index(kernel_pmap, range->attrs, true) == 11984 pmap_pat_index(kernel_pmap, attrs, true)) 11985 return (true); 11986 return (false); 11987 } 11988 11989 static void 11990 sysctl_kmaps_reinit(struct pmap_kernel_map_range *range, vm_offset_t va, 11991 pt_entry_t attrs) 11992 { 11993 11994 memset(range, 0, sizeof(*range)); 11995 range->sva = va; 11996 range->attrs = attrs; 11997 } 11998 11999 /* 12000 * Given a leaf PTE, derive the mapping's attributes. If they do not match 12001 * those of the current run, dump the address range and its attributes, and 12002 * begin a new run. 12003 */ 12004 static void 12005 sysctl_kmaps_check(struct sbuf *sb, struct pmap_kernel_map_range *range, 12006 vm_offset_t va, pml5_entry_t pml5e, pml4_entry_t pml4e, pdp_entry_t pdpe, 12007 pd_entry_t pde, pt_entry_t pte) 12008 { 12009 pt_entry_t attrs; 12010 12011 if (la57) { 12012 attrs = pml5e & (X86_PG_RW | X86_PG_U | pg_nx); 12013 attrs |= pml4e & pg_nx; 12014 attrs &= pg_nx | (pml4e & (X86_PG_RW | X86_PG_U)); 12015 } else { 12016 attrs = pml4e & (X86_PG_RW | X86_PG_U | pg_nx); 12017 } 12018 12019 attrs |= pdpe & pg_nx; 12020 attrs &= pg_nx | (pdpe & (X86_PG_RW | X86_PG_U)); 12021 if ((pdpe & PG_PS) != 0) { 12022 attrs |= pdpe & (X86_PG_G | X86_PG_PDE_CACHE); 12023 } else if (pde != 0) { 12024 attrs |= pde & pg_nx; 12025 attrs &= pg_nx | (pde & (X86_PG_RW | X86_PG_U)); 12026 } 12027 if ((pde & PG_PS) != 0) { 12028 attrs |= pde & (X86_PG_G | X86_PG_PDE_CACHE); 12029 } else if (pte != 0) { 12030 attrs |= pte & pg_nx; 12031 attrs &= pg_nx | (pte & (X86_PG_RW | X86_PG_U)); 12032 attrs |= pte & (X86_PG_G | X86_PG_PTE_CACHE); 12033 12034 /* Canonicalize by always using the PDE PAT bit. */ 12035 if ((attrs & X86_PG_PTE_PAT) != 0) 12036 attrs ^= X86_PG_PDE_PAT | X86_PG_PTE_PAT; 12037 } 12038 12039 if (range->sva > va || !sysctl_kmaps_match(range, attrs)) { 12040 sysctl_kmaps_dump(sb, range, va); 12041 sysctl_kmaps_reinit(range, va, attrs); 12042 } 12043 } 12044 12045 static int 12046 sysctl_kmaps(SYSCTL_HANDLER_ARGS) 12047 { 12048 struct pmap_kernel_map_range range; 12049 struct sbuf sbuf, *sb; 12050 pml5_entry_t pml5e; 12051 pml4_entry_t pml4e; 12052 pdp_entry_t *pdp, pdpe; 12053 pd_entry_t *pd, pde; 12054 pt_entry_t *pt, pte; 12055 vm_offset_t sva; 12056 vm_paddr_t pa; 12057 int error, j, k, l; 12058 bool first; 12059 12060 error = sysctl_wire_old_buffer(req, 0); 12061 if (error != 0) 12062 return (error); 12063 sb = &sbuf; 12064 sbuf_new_for_sysctl(sb, NULL, PAGE_SIZE, req); 12065 12066 /* Sentinel value. */ 12067 range.sva = kva_layout.kva_max; 12068 pml5e = 0; /* no UB for la48 */ 12069 12070 /* 12071 * Iterate over the kernel page tables without holding the kernel pmap 12072 * lock. Outside of the large map, kernel page table pages are never 12073 * freed, so at worst we will observe inconsistencies in the output. 12074 * Within the large map, ensure that PDP and PD page addresses are 12075 * valid before descending. 12076 */ 12077 for (first = true, sva = 0; sva != 0 || first; first = false) { 12078 if (sva == kva_layout.rec_pt) 12079 sbuf_printf(sb, "\nRecursive map:\n"); 12080 else if (sva == kva_layout.dmap_low) 12081 sbuf_printf(sb, "\nDirect map:\n"); 12082 #ifdef KASAN 12083 else if (sva == kva_layout.kasan_shadow_low) 12084 sbuf_printf(sb, "\nKASAN shadow map:\n"); 12085 #endif 12086 #ifdef KMSAN 12087 else if (sva == kva_layout.kmsan_shadow_low) 12088 sbuf_printf(sb, "\nKMSAN shadow map:\n"); 12089 else if (sva == kva_layout.kmsan_origin_low) 12090 sbuf_printf(sb, "\nKMSAN origin map:\n"); 12091 #endif 12092 else if (sva == kva_layout.km_low) 12093 sbuf_printf(sb, "\nKernel map:\n"); 12094 else if (sva == kva_layout.lm_low) 12095 sbuf_printf(sb, "\nLarge map:\n"); 12096 12097 /* Convert to canonical form. */ 12098 if (la57) { 12099 if (sva == 1ul << 56) { 12100 sva |= -1ul << 57; 12101 continue; 12102 } 12103 } else { 12104 if (sva == 1ul << 47) { 12105 sva |= -1ul << 48; 12106 continue; 12107 } 12108 } 12109 12110 restart: 12111 if (la57) { 12112 pml5e = *pmap_pml5e(kernel_pmap, sva); 12113 if ((pml5e & X86_PG_V) == 0) { 12114 sva = rounddown2(sva, NBPML5); 12115 sysctl_kmaps_dump(sb, &range, sva); 12116 sva += NBPML5; 12117 continue; 12118 } 12119 } 12120 pml4e = *pmap_pml4e(kernel_pmap, sva); 12121 if ((pml4e & X86_PG_V) == 0) { 12122 sva = rounddown2(sva, NBPML4); 12123 sysctl_kmaps_dump(sb, &range, sva); 12124 sva += NBPML4; 12125 continue; 12126 } 12127 pa = pml4e & PG_FRAME; 12128 pdp = PHYS_TO_DMAP(pa); 12129 12130 for (j = pmap_pdpe_index(sva); j < NPDPEPG; j++) { 12131 pdpe = pdp[j]; 12132 if ((pdpe & X86_PG_V) == 0) { 12133 sva = rounddown2(sva, NBPDP); 12134 sysctl_kmaps_dump(sb, &range, sva); 12135 sva += NBPDP; 12136 continue; 12137 } 12138 pa = pdpe & PG_FRAME; 12139 if ((pdpe & PG_PS) != 0) { 12140 sva = rounddown2(sva, NBPDP); 12141 sysctl_kmaps_check(sb, &range, sva, pml5e, 12142 pml4e, pdpe, 0, 0); 12143 range.pdpes++; 12144 sva += NBPDP; 12145 continue; 12146 } 12147 if (PMAP_ADDRESS_IN_LARGEMAP(sva) && 12148 vm_phys_paddr_to_vm_page(pa) == NULL) { 12149 /* 12150 * Page table pages for the large map may be 12151 * freed. Validate the next-level address 12152 * before descending. 12153 */ 12154 sva += NBPDP; 12155 goto restart; 12156 } 12157 pd = PHYS_TO_DMAP(pa); 12158 12159 for (k = pmap_pde_index(sva); k < NPDEPG; k++) { 12160 pde = pd[k]; 12161 if ((pde & X86_PG_V) == 0) { 12162 sva = rounddown2(sva, NBPDR); 12163 sysctl_kmaps_dump(sb, &range, sva); 12164 sva += NBPDR; 12165 continue; 12166 } 12167 pa = pde & PG_FRAME; 12168 if ((pde & PG_PS) != 0) { 12169 sva = rounddown2(sva, NBPDR); 12170 sysctl_kmaps_check(sb, &range, sva, 12171 pml5e, pml4e, pdpe, pde, 0); 12172 range.pdes++; 12173 sva += NBPDR; 12174 continue; 12175 } 12176 if (PMAP_ADDRESS_IN_LARGEMAP(sva) && 12177 vm_phys_paddr_to_vm_page(pa) == NULL) { 12178 /* 12179 * Page table pages for the large map 12180 * may be freed. Validate the 12181 * next-level address before descending. 12182 */ 12183 sva += NBPDR; 12184 goto restart; 12185 } 12186 pt = PHYS_TO_DMAP(pa); 12187 12188 for (l = pmap_pte_index(sva); l < NPTEPG; l++, 12189 sva += PAGE_SIZE) { 12190 pte = pt[l]; 12191 if ((pte & X86_PG_V) == 0) { 12192 sysctl_kmaps_dump(sb, &range, 12193 sva); 12194 continue; 12195 } 12196 sysctl_kmaps_check(sb, &range, sva, 12197 pml5e, pml4e, pdpe, pde, pte); 12198 range.ptes++; 12199 } 12200 } 12201 } 12202 } 12203 12204 error = sbuf_finish(sb); 12205 sbuf_delete(sb); 12206 return (error); 12207 } 12208 SYSCTL_OID(_vm_pmap, OID_AUTO, kernel_maps, 12209 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE | CTLFLAG_SKIP, 12210 NULL, 0, sysctl_kmaps, "A", 12211 "Dump kernel address layout"); 12212 12213 #ifdef DDB 12214 DB_SHOW_COMMAND(pte, pmap_print_pte) 12215 { 12216 pmap_t pmap; 12217 pml5_entry_t *pml5; 12218 pml4_entry_t *pml4; 12219 pdp_entry_t *pdp; 12220 pd_entry_t *pde; 12221 pt_entry_t *pte, PG_V; 12222 vm_offset_t va; 12223 12224 if (!have_addr) { 12225 db_printf("show pte addr\n"); 12226 return; 12227 } 12228 va = (vm_offset_t)addr; 12229 12230 if (kdb_thread != NULL) 12231 pmap = vmspace_pmap(kdb_thread->td_proc->p_vmspace); 12232 else 12233 pmap = PCPU_GET(curpmap); 12234 12235 PG_V = pmap_valid_bit(pmap); 12236 db_printf("VA 0x%016lx", va); 12237 12238 if (pmap_is_la57(pmap)) { 12239 pml5 = pmap_pml5e(pmap, va); 12240 db_printf(" pml5e@0x%016lx 0x%016lx", (uint64_t)pml5, *pml5); 12241 if ((*pml5 & PG_V) == 0) { 12242 db_printf("\n"); 12243 return; 12244 } 12245 pml4 = pmap_pml5e_to_pml4e(pml5, va); 12246 } else { 12247 pml4 = pmap_pml4e(pmap, va); 12248 } 12249 db_printf(" pml4e@0x%016lx 0x%016lx", (uint64_t)pml4, *pml4); 12250 if ((*pml4 & PG_V) == 0) { 12251 db_printf("\n"); 12252 return; 12253 } 12254 pdp = pmap_pml4e_to_pdpe(pml4, va); 12255 db_printf(" pdpe@0x%016lx 0x%016lx", (uint64_t)pdp, *pdp); 12256 if ((*pdp & PG_V) == 0 || (*pdp & PG_PS) != 0) { 12257 db_printf("\n"); 12258 return; 12259 } 12260 pde = pmap_pdpe_to_pde(pdp, va); 12261 db_printf(" pde@0x%016lx 0x%016lx", (uint64_t)pde, *pde); 12262 if ((*pde & PG_V) == 0 || (*pde & PG_PS) != 0) { 12263 db_printf("\n"); 12264 return; 12265 } 12266 pte = pmap_pde_to_pte(pde, va); 12267 db_printf(" pte@0x%016lx 0x%016lx\n", (uint64_t)pte, *pte); 12268 } 12269 12270 DB_SHOW_COMMAND(phys2dmap, pmap_phys2dmap) 12271 { 12272 vm_paddr_t a; 12273 12274 if (have_addr) { 12275 a = (vm_paddr_t)addr; 12276 db_printf("%p\n", PHYS_TO_DMAP(a)); 12277 } else { 12278 db_printf("show phys2dmap addr\n"); 12279 } 12280 } 12281 12282 static void 12283 ptpages_show_page(int level, int idx, vm_page_t pg) 12284 { 12285 db_printf("l %d i %d pg %p phys %#lx ref %x\n", 12286 level, idx, pg, VM_PAGE_TO_PHYS(pg), pg->ref_count); 12287 } 12288 12289 static void 12290 ptpages_show_complain(int level, int idx, uint64_t pte) 12291 { 12292 db_printf("l %d i %d pte %#lx\n", level, idx, pte); 12293 } 12294 12295 static void 12296 ptpages_show_pml4(vm_page_t pg4, int num_entries, uint64_t PG_V) 12297 { 12298 vm_page_t pg3, pg2, pg1; 12299 pml4_entry_t *pml4; 12300 pdp_entry_t *pdp; 12301 pd_entry_t *pd; 12302 int i4, i3, i2; 12303 12304 pml4 = VM_PAGE_TO_DMAP(pg4); 12305 for (i4 = 0; i4 < num_entries; i4++) { 12306 if ((pml4[i4] & PG_V) == 0) 12307 continue; 12308 pg3 = PHYS_TO_VM_PAGE(pml4[i4] & PG_FRAME); 12309 if (pg3 == NULL) { 12310 ptpages_show_complain(3, i4, pml4[i4]); 12311 continue; 12312 } 12313 ptpages_show_page(3, i4, pg3); 12314 pdp = VM_PAGE_TO_DMAP(pg3); 12315 for (i3 = 0; i3 < NPDPEPG; i3++) { 12316 if ((pdp[i3] & PG_V) == 0) 12317 continue; 12318 pg2 = PHYS_TO_VM_PAGE(pdp[i3] & PG_FRAME); 12319 if (pg3 == NULL) { 12320 ptpages_show_complain(2, i3, pdp[i3]); 12321 continue; 12322 } 12323 ptpages_show_page(2, i3, pg2); 12324 pd = VM_PAGE_TO_DMAP(pg2); 12325 for (i2 = 0; i2 < NPDEPG; i2++) { 12326 if ((pd[i2] & PG_V) == 0) 12327 continue; 12328 pg1 = PHYS_TO_VM_PAGE(pd[i2] & PG_FRAME); 12329 if (pg1 == NULL) { 12330 ptpages_show_complain(1, i2, pd[i2]); 12331 continue; 12332 } 12333 ptpages_show_page(1, i2, pg1); 12334 } 12335 } 12336 } 12337 } 12338 12339 DB_SHOW_COMMAND(ptpages, pmap_ptpages) 12340 { 12341 pmap_t pmap; 12342 vm_page_t pg; 12343 pml5_entry_t *pml5; 12344 uint64_t PG_V; 12345 int i5; 12346 12347 if (have_addr) 12348 pmap = (pmap_t)addr; 12349 else 12350 pmap = PCPU_GET(curpmap); 12351 12352 PG_V = pmap_valid_bit(pmap); 12353 12354 if (pmap_is_la57(pmap)) { 12355 pml5 = pmap->pm_pmltop; 12356 for (i5 = 0; i5 < NUPML5E; i5++) { 12357 if ((pml5[i5] & PG_V) == 0) 12358 continue; 12359 pg = PHYS_TO_VM_PAGE(pml5[i5] & PG_FRAME); 12360 if (pg == NULL) { 12361 ptpages_show_complain(4, i5, pml5[i5]); 12362 continue; 12363 } 12364 ptpages_show_page(4, i5, pg); 12365 ptpages_show_pml4(pg, NPML4EPG, PG_V); 12366 } 12367 } else { 12368 ptpages_show_pml4(DMAP_TO_VM_PAGE(pmap->pm_pmltop), NUP4ML4E, 12369 PG_V); 12370 } 12371 } 12372 #endif 12373