1 /*- 2 * SPDX-License-Identifier: BSD-4-Clause 3 * 4 * Copyright (c) 1991 Regents of the University of California. 5 * All rights reserved. 6 * Copyright (c) 1994 John S. Dyson 7 * All rights reserved. 8 * Copyright (c) 1994 David Greenman 9 * All rights reserved. 10 * Copyright (c) 2005-2010 Alan L. Cox <alc@cs.rice.edu> 11 * All rights reserved. 12 * 13 * This code is derived from software contributed to Berkeley by 14 * the Systems Programming Group of the University of Utah Computer 15 * Science Department and William Jolitz of UUNET Technologies Inc. 16 * 17 * Redistribution and use in source and binary forms, with or without 18 * modification, are permitted provided that the following conditions 19 * are met: 20 * 1. Redistributions of source code must retain the above copyright 21 * notice, this list of conditions and the following disclaimer. 22 * 2. Redistributions in binary form must reproduce the above copyright 23 * notice, this list of conditions and the following disclaimer in the 24 * documentation and/or other materials provided with the distribution. 25 * 3. All advertising materials mentioning features or use of this software 26 * must display the following acknowledgement: 27 * This product includes software developed by the University of 28 * California, Berkeley and its contributors. 29 * 4. Neither the name of the University nor the names of its contributors 30 * may be used to endorse or promote products derived from this software 31 * without specific prior written permission. 32 * 33 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 34 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 35 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 36 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 37 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 38 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 39 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 40 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 41 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 42 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 43 * SUCH DAMAGE. 44 * 45 * from: @(#)pmap.c 7.7 (Berkeley) 5/12/91 46 */ 47 /*- 48 * Copyright (c) 2003 Networks Associates Technology, Inc. 49 * All rights reserved. 50 * Copyright (c) 2018 The FreeBSD Foundation 51 * All rights reserved. 52 * 53 * This software was developed for the FreeBSD Project by Jake Burkholder, 54 * Safeport Network Services, and Network Associates Laboratories, the 55 * Security Research Division of Network Associates, Inc. under 56 * DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"), as part of the DARPA 57 * CHATS research program. 58 * 59 * Portions of this software were developed by 60 * Konstantin Belousov <kib@FreeBSD.org> under sponsorship from 61 * the FreeBSD Foundation. 62 * 63 * Redistribution and use in source and binary forms, with or without 64 * modification, are permitted provided that the following conditions 65 * are met: 66 * 1. Redistributions of source code must retain the above copyright 67 * notice, this list of conditions and the following disclaimer. 68 * 2. Redistributions in binary form must reproduce the above copyright 69 * notice, this list of conditions and the following disclaimer in the 70 * documentation and/or other materials provided with the distribution. 71 * 72 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 73 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 74 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 75 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 76 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 77 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 78 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 79 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 80 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 81 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 82 * SUCH DAMAGE. 83 */ 84 85 #include <sys/cdefs.h> 86 __FBSDID("$FreeBSD$"); 87 88 #include "opt_apic.h" 89 #include "opt_cpu.h" 90 #include "opt_pmap.h" 91 #include "opt_smp.h" 92 #include "opt_vm.h" 93 94 #include <sys/param.h> 95 #include <sys/systm.h> 96 #include <sys/kernel.h> 97 #include <sys/vmmeter.h> 98 #include <sys/sysctl.h> 99 #include <machine/bootinfo.h> 100 #include <machine/cpu.h> 101 #include <machine/cputypes.h> 102 #include <machine/md_var.h> 103 #ifdef DEV_APIC 104 #include <sys/bus.h> 105 #include <machine/intr_machdep.h> 106 #include <x86/apicvar.h> 107 #endif 108 #include <x86/ifunc.h> 109 110 static SYSCTL_NODE(_vm, OID_AUTO, pmap, CTLFLAG_RD, 0, "VM/pmap parameters"); 111 112 #include <machine/vmparam.h> 113 #include <vm/vm.h> 114 #include <vm/vm_page.h> 115 #include <vm/pmap.h> 116 #include <machine/pmap_base.h> 117 118 vm_offset_t virtual_avail; /* VA of first avail page (after kernel bss) */ 119 vm_offset_t virtual_end; /* VA of last avail page (end of kernel AS) */ 120 121 int unmapped_buf_allowed = 1; 122 123 int pti; 124 125 u_long physfree; /* phys addr of next free page */ 126 u_long vm86phystk; /* PA of vm86/bios stack */ 127 u_long vm86paddr; /* address of vm86 region */ 128 int vm86pa; /* phys addr of vm86 region */ 129 u_long KERNend; /* phys addr end of kernel (just after bss) */ 130 u_long KPTphys; /* phys addr of kernel page tables */ 131 caddr_t ptvmmap = 0; 132 vm_offset_t kernel_vm_end; 133 134 int i386_pmap_VM_NFREEORDER; 135 int i386_pmap_VM_LEVEL_0_ORDER; 136 int i386_pmap_PDRSHIFT; 137 138 int pat_works = 1; 139 SYSCTL_INT(_vm_pmap, OID_AUTO, pat_works, CTLFLAG_RD, 140 &pat_works, 0, 141 "Is page attribute table fully functional?"); 142 143 int pg_ps_enabled = 1; 144 SYSCTL_INT(_vm_pmap, OID_AUTO, pg_ps_enabled, CTLFLAG_RDTUN | CTLFLAG_NOFETCH, 145 &pg_ps_enabled, 0, 146 "Are large page mappings enabled?"); 147 148 int pv_entry_max = 0; 149 SYSCTL_INT(_vm_pmap, OID_AUTO, pv_entry_max, CTLFLAG_RD, 150 &pv_entry_max, 0, 151 "Max number of PV entries"); 152 153 int pv_entry_count = 0; 154 SYSCTL_INT(_vm_pmap, OID_AUTO, pv_entry_count, CTLFLAG_RD, 155 &pv_entry_count, 0, 156 "Current number of pv entries"); 157 158 #ifndef PMAP_SHPGPERPROC 159 #define PMAP_SHPGPERPROC 200 160 #endif 161 162 int shpgperproc = PMAP_SHPGPERPROC; 163 SYSCTL_INT(_vm_pmap, OID_AUTO, shpgperproc, CTLFLAG_RD, 164 &shpgperproc, 0, 165 "Page share factor per proc"); 166 167 static SYSCTL_NODE(_vm_pmap, OID_AUTO, pde, CTLFLAG_RD, 0, 168 "2/4MB page mapping counters"); 169 170 u_long pmap_pde_demotions; 171 SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, demotions, CTLFLAG_RD, 172 &pmap_pde_demotions, 0, 173 "2/4MB page demotions"); 174 175 u_long pmap_pde_mappings; 176 SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, mappings, CTLFLAG_RD, 177 &pmap_pde_mappings, 0, 178 "2/4MB page mappings"); 179 180 u_long pmap_pde_p_failures; 181 SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, p_failures, CTLFLAG_RD, 182 &pmap_pde_p_failures, 0, 183 "2/4MB page promotion failures"); 184 185 u_long pmap_pde_promotions; 186 SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, promotions, CTLFLAG_RD, 187 &pmap_pde_promotions, 0, 188 "2/4MB page promotions"); 189 190 #ifdef SMP 191 int PMAP1changedcpu; 192 SYSCTL_INT(_debug, OID_AUTO, PMAP1changedcpu, CTLFLAG_RD, 193 &PMAP1changedcpu, 0, 194 "Number of times pmap_pte_quick changed CPU with same PMAP1"); 195 #endif 196 197 int PMAP1changed; 198 SYSCTL_INT(_debug, OID_AUTO, PMAP1changed, CTLFLAG_RD, 199 &PMAP1changed, 0, 200 "Number of times pmap_pte_quick changed PMAP1"); 201 int PMAP1unchanged; 202 SYSCTL_INT(_debug, OID_AUTO, PMAP1unchanged, CTLFLAG_RD, 203 &PMAP1unchanged, 0, 204 "Number of times pmap_pte_quick didn't change PMAP1"); 205 206 static int 207 kvm_size(SYSCTL_HANDLER_ARGS) 208 { 209 unsigned long ksize; 210 211 ksize = VM_MAX_KERNEL_ADDRESS - KERNBASE; 212 return (sysctl_handle_long(oidp, &ksize, 0, req)); 213 } 214 SYSCTL_PROC(_vm, OID_AUTO, kvm_size, CTLTYPE_LONG | CTLFLAG_RD | CTLFLAG_MPSAFE, 215 0, 0, kvm_size, "IU", 216 "Size of KVM"); 217 218 static int 219 kvm_free(SYSCTL_HANDLER_ARGS) 220 { 221 unsigned long kfree; 222 223 kfree = VM_MAX_KERNEL_ADDRESS - kernel_vm_end; 224 return (sysctl_handle_long(oidp, &kfree, 0, req)); 225 } 226 SYSCTL_PROC(_vm, OID_AUTO, kvm_free, CTLTYPE_LONG | CTLFLAG_RD | CTLFLAG_MPSAFE, 227 0, 0, kvm_free, "IU", 228 "Amount of KVM free"); 229 230 #ifdef PV_STATS 231 int pc_chunk_count, pc_chunk_allocs, pc_chunk_frees, pc_chunk_tryfail; 232 long pv_entry_frees, pv_entry_allocs; 233 int pv_entry_spare; 234 235 SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_count, CTLFLAG_RD, 236 &pc_chunk_count, 0, 237 "Current number of pv entry chunks"); 238 SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_allocs, CTLFLAG_RD, 239 &pc_chunk_allocs, 0, 240 "Current number of pv entry chunks allocated"); 241 SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_frees, CTLFLAG_RD, 242 &pc_chunk_frees, 0, 243 "Current number of pv entry chunks frees"); 244 SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_tryfail, CTLFLAG_RD, 245 &pc_chunk_tryfail, 0, 246 "Number of times tried to get a chunk page but failed."); 247 SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_frees, CTLFLAG_RD, 248 &pv_entry_frees, 0, 249 "Current number of pv entry frees"); 250 SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_allocs, CTLFLAG_RD, 251 &pv_entry_allocs, 0, 252 "Current number of pv entry allocs"); 253 SYSCTL_INT(_vm_pmap, OID_AUTO, pv_entry_spare, CTLFLAG_RD, 254 &pv_entry_spare, 0, 255 "Current number of spare pv entries"); 256 #endif 257 258 struct pmap kernel_pmap_store; 259 static struct pmap_methods *pmap_methods_ptr; 260 261 static int 262 sysctl_kmaps(SYSCTL_HANDLER_ARGS) 263 { 264 return (pmap_methods_ptr->pm_sysctl_kmaps(oidp, arg1, arg2, req)); 265 } 266 SYSCTL_OID(_vm_pmap, OID_AUTO, kernel_maps, 267 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, 268 NULL, 0, sysctl_kmaps, "A", 269 "Dump kernel address layout"); 270 271 272 /* 273 * Initialize a vm_page's machine-dependent fields. 274 */ 275 void 276 pmap_page_init(vm_page_t m) 277 { 278 279 TAILQ_INIT(&m->md.pv_list); 280 m->md.pat_mode = PAT_WRITE_BACK; 281 } 282 283 void 284 invltlb_glob(void) 285 { 286 287 invltlb(); 288 } 289 290 static void pmap_invalidate_cache_range_selfsnoop(vm_offset_t sva, 291 vm_offset_t eva); 292 static void pmap_invalidate_cache_range_all(vm_offset_t sva, 293 vm_offset_t eva); 294 295 void 296 pmap_flush_page(vm_page_t m) 297 { 298 299 pmap_methods_ptr->pm_flush_page(m); 300 } 301 302 DEFINE_IFUNC(, void, pmap_invalidate_cache_range, (vm_offset_t, vm_offset_t)) 303 { 304 305 if ((cpu_feature & CPUID_SS) != 0) 306 return (pmap_invalidate_cache_range_selfsnoop); 307 if ((cpu_feature & CPUID_CLFSH) != 0) 308 return (pmap_force_invalidate_cache_range); 309 return (pmap_invalidate_cache_range_all); 310 } 311 312 #define PMAP_CLFLUSH_THRESHOLD (2 * 1024 * 1024) 313 314 static void 315 pmap_invalidate_cache_range_check_align(vm_offset_t sva, vm_offset_t eva) 316 { 317 318 KASSERT((sva & PAGE_MASK) == 0, 319 ("pmap_invalidate_cache_range: sva not page-aligned")); 320 KASSERT((eva & PAGE_MASK) == 0, 321 ("pmap_invalidate_cache_range: eva not page-aligned")); 322 } 323 324 static void 325 pmap_invalidate_cache_range_selfsnoop(vm_offset_t sva, vm_offset_t eva) 326 { 327 328 pmap_invalidate_cache_range_check_align(sva, eva); 329 } 330 331 void 332 pmap_force_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva) 333 { 334 335 sva &= ~(vm_offset_t)(cpu_clflush_line_size - 1); 336 if (eva - sva >= PMAP_CLFLUSH_THRESHOLD) { 337 /* 338 * The supplied range is bigger than 2MB. 339 * Globally invalidate cache. 340 */ 341 pmap_invalidate_cache(); 342 return; 343 } 344 345 #ifdef DEV_APIC 346 /* 347 * XXX: Some CPUs fault, hang, or trash the local APIC 348 * registers if we use CLFLUSH on the local APIC 349 * range. The local APIC is always uncached, so we 350 * don't need to flush for that range anyway. 351 */ 352 if (pmap_kextract(sva) == lapic_paddr) 353 return; 354 #endif 355 356 if ((cpu_stdext_feature & CPUID_STDEXT_CLFLUSHOPT) != 0) { 357 /* 358 * Do per-cache line flush. Use the sfence 359 * instruction to insure that previous stores are 360 * included in the write-back. The processor 361 * propagates flush to other processors in the cache 362 * coherence domain. 363 */ 364 sfence(); 365 for (; sva < eva; sva += cpu_clflush_line_size) 366 clflushopt(sva); 367 sfence(); 368 } else { 369 /* 370 * Writes are ordered by CLFLUSH on Intel CPUs. 371 */ 372 if (cpu_vendor_id != CPU_VENDOR_INTEL) 373 mfence(); 374 for (; sva < eva; sva += cpu_clflush_line_size) 375 clflush(sva); 376 if (cpu_vendor_id != CPU_VENDOR_INTEL) 377 mfence(); 378 } 379 } 380 381 static void 382 pmap_invalidate_cache_range_all(vm_offset_t sva, vm_offset_t eva) 383 { 384 385 pmap_invalidate_cache_range_check_align(sva, eva); 386 pmap_invalidate_cache(); 387 } 388 389 void 390 pmap_invalidate_cache_pages(vm_page_t *pages, int count) 391 { 392 int i; 393 394 if (count >= PMAP_CLFLUSH_THRESHOLD / PAGE_SIZE || 395 (cpu_feature & CPUID_CLFSH) == 0) { 396 pmap_invalidate_cache(); 397 } else { 398 for (i = 0; i < count; i++) 399 pmap_flush_page(pages[i]); 400 } 401 } 402 403 void 404 pmap_ksetrw(vm_offset_t va) 405 { 406 407 pmap_methods_ptr->pm_ksetrw(va); 408 } 409 410 void 411 pmap_remap_lower(bool enable) 412 { 413 414 pmap_methods_ptr->pm_remap_lower(enable); 415 } 416 417 void 418 pmap_remap_lowptdi(bool enable) 419 { 420 421 pmap_methods_ptr->pm_remap_lowptdi(enable); 422 } 423 424 void 425 pmap_align_superpage(vm_object_t object, vm_ooffset_t offset, 426 vm_offset_t *addr, vm_size_t size) 427 { 428 429 return (pmap_methods_ptr->pm_align_superpage(object, offset, 430 addr, size)); 431 } 432 433 vm_offset_t 434 pmap_quick_enter_page(vm_page_t m) 435 { 436 437 return (pmap_methods_ptr->pm_quick_enter_page(m)); 438 } 439 440 void 441 pmap_quick_remove_page(vm_offset_t addr) 442 { 443 444 return (pmap_methods_ptr->pm_quick_remove_page(addr)); 445 } 446 447 void * 448 pmap_trm_alloc(size_t size, int flags) 449 { 450 451 return (pmap_methods_ptr->pm_trm_alloc(size, flags)); 452 } 453 454 void 455 pmap_trm_free(void *addr, size_t size) 456 { 457 458 pmap_methods_ptr->pm_trm_free(addr, size); 459 } 460 461 void 462 pmap_sync_icache(pmap_t pm, vm_offset_t va, vm_size_t sz) 463 { 464 } 465 466 vm_offset_t 467 pmap_get_map_low(void) 468 { 469 470 return (pmap_methods_ptr->pm_get_map_low()); 471 } 472 473 vm_offset_t 474 pmap_get_vm_maxuser_address(void) 475 { 476 477 return (pmap_methods_ptr->pm_get_vm_maxuser_address()); 478 } 479 480 vm_paddr_t 481 pmap_kextract(vm_offset_t va) 482 { 483 484 return (pmap_methods_ptr->pm_kextract(va)); 485 } 486 487 vm_paddr_t 488 pmap_pg_frame(vm_paddr_t pa) 489 { 490 491 return (pmap_methods_ptr->pm_pg_frame(pa)); 492 } 493 494 void 495 pmap_sf_buf_map(struct sf_buf *sf) 496 { 497 498 pmap_methods_ptr->pm_sf_buf_map(sf); 499 } 500 501 void 502 pmap_cp_slow0_map(vm_offset_t kaddr, int plen, vm_page_t *ma) 503 { 504 505 pmap_methods_ptr->pm_cp_slow0_map(kaddr, plen, ma); 506 } 507 508 u_int 509 pmap_get_kcr3(void) 510 { 511 512 return (pmap_methods_ptr->pm_get_kcr3()); 513 } 514 515 u_int 516 pmap_get_cr3(pmap_t pmap) 517 { 518 519 return (pmap_methods_ptr->pm_get_cr3(pmap)); 520 } 521 522 caddr_t 523 pmap_cmap3(vm_paddr_t pa, u_int pte_flags) 524 { 525 526 return (pmap_methods_ptr->pm_cmap3(pa, pte_flags)); 527 } 528 529 void 530 pmap_basemem_setup(u_int basemem) 531 { 532 533 pmap_methods_ptr->pm_basemem_setup(basemem); 534 } 535 536 void 537 pmap_set_nx(void) 538 { 539 540 pmap_methods_ptr->pm_set_nx(); 541 } 542 543 void * 544 pmap_bios16_enter(void) 545 { 546 547 return (pmap_methods_ptr->pm_bios16_enter()); 548 } 549 550 void 551 pmap_bios16_leave(void *handle) 552 { 553 554 pmap_methods_ptr->pm_bios16_leave(handle); 555 } 556 557 void 558 pmap_bootstrap(vm_paddr_t firstaddr) 559 { 560 561 pmap_methods_ptr->pm_bootstrap(firstaddr); 562 } 563 564 boolean_t 565 pmap_is_valid_memattr(pmap_t pmap, vm_memattr_t mode) 566 { 567 568 return (pmap_methods_ptr->pm_is_valid_memattr(pmap, mode)); 569 } 570 571 int 572 pmap_cache_bits(pmap_t pmap, int mode, boolean_t is_pde) 573 { 574 575 return (pmap_methods_ptr->pm_cache_bits(pmap, mode, is_pde)); 576 } 577 578 bool 579 pmap_ps_enabled(pmap_t pmap) 580 { 581 582 return (pmap_methods_ptr->pm_ps_enabled(pmap)); 583 } 584 585 void 586 pmap_pinit0(pmap_t pmap) 587 { 588 589 pmap_methods_ptr->pm_pinit0(pmap); 590 } 591 592 int 593 pmap_pinit(pmap_t pmap) 594 { 595 596 return (pmap_methods_ptr->pm_pinit(pmap)); 597 } 598 599 void 600 pmap_activate(struct thread *td) 601 { 602 603 pmap_methods_ptr->pm_activate(td); 604 } 605 606 void 607 pmap_activate_boot(pmap_t pmap) 608 { 609 610 pmap_methods_ptr->pm_activate_boot(pmap); 611 } 612 613 void 614 pmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int advice) 615 { 616 617 pmap_methods_ptr->pm_advise(pmap, sva, eva, advice); 618 } 619 620 void 621 pmap_clear_modify(vm_page_t m) 622 { 623 624 pmap_methods_ptr->pm_clear_modify(m); 625 } 626 627 int 628 pmap_change_attr(vm_offset_t va, vm_size_t size, int mode) 629 { 630 631 return (pmap_methods_ptr->pm_change_attr(va, size, mode)); 632 } 633 634 int 635 pmap_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *pap) 636 { 637 638 return (pmap_methods_ptr->pm_mincore(pmap, addr, pap)); 639 } 640 641 void 642 pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, vm_size_t len, 643 vm_offset_t src_addr) 644 { 645 646 pmap_methods_ptr->pm_copy(dst_pmap, src_pmap, dst_addr, len, src_addr); 647 } 648 649 void 650 pmap_copy_page(vm_page_t src, vm_page_t dst) 651 { 652 653 pmap_methods_ptr->pm_copy_page(src, dst); 654 } 655 656 void 657 pmap_copy_pages(vm_page_t ma[], vm_offset_t a_offset, vm_page_t mb[], 658 vm_offset_t b_offset, int xfersize) 659 { 660 661 pmap_methods_ptr->pm_copy_pages(ma, a_offset, mb, b_offset, xfersize); 662 } 663 664 void 665 pmap_zero_page(vm_page_t m) 666 { 667 668 pmap_methods_ptr->pm_zero_page(m); 669 } 670 671 void 672 pmap_zero_page_area(vm_page_t m, int off, int size) 673 { 674 675 pmap_methods_ptr->pm_zero_page_area(m, off, size); 676 } 677 678 int 679 pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, 680 u_int flags, int8_t psind) 681 { 682 683 return (pmap_methods_ptr->pm_enter(pmap, va, m, prot, flags, psind)); 684 } 685 686 void 687 pmap_enter_object(pmap_t pmap, vm_offset_t start, vm_offset_t end, 688 vm_page_t m_start, vm_prot_t prot) 689 { 690 691 pmap_methods_ptr->pm_enter_object(pmap, start, end, m_start, prot); 692 } 693 694 void 695 pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot) 696 { 697 698 pmap_methods_ptr->pm_enter_quick(pmap, va, m, prot); 699 } 700 701 void * 702 pmap_kenter_temporary(vm_paddr_t pa, int i) 703 { 704 705 return (pmap_methods_ptr->pm_kenter_temporary(pa, i)); 706 } 707 708 void 709 pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, vm_object_t object, 710 vm_pindex_t pindex, vm_size_t size) 711 { 712 713 pmap_methods_ptr->pm_object_init_pt(pmap, addr, object, pindex, size); 714 } 715 716 void 717 pmap_unwire(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) 718 { 719 720 pmap_methods_ptr->pm_unwire(pmap, sva, eva); 721 } 722 723 boolean_t 724 pmap_page_exists_quick(pmap_t pmap, vm_page_t m) 725 { 726 727 return (pmap_methods_ptr->pm_page_exists_quick(pmap, m)); 728 } 729 730 int 731 pmap_page_wired_mappings(vm_page_t m) 732 { 733 734 return (pmap_methods_ptr->pm_page_wired_mappings(m)); 735 } 736 737 boolean_t 738 pmap_page_is_mapped(vm_page_t m) 739 { 740 741 return (pmap_methods_ptr->pm_page_is_mapped(m)); 742 } 743 744 void 745 pmap_remove_pages(pmap_t pmap) 746 { 747 748 pmap_methods_ptr->pm_remove_pages(pmap); 749 } 750 751 boolean_t 752 pmap_is_modified(vm_page_t m) 753 { 754 755 return (pmap_methods_ptr->pm_is_modified(m)); 756 } 757 758 boolean_t 759 pmap_is_prefaultable(pmap_t pmap, vm_offset_t addr) 760 { 761 762 return (pmap_methods_ptr->pm_is_prefaultable(pmap, addr)); 763 } 764 765 boolean_t 766 pmap_is_referenced(vm_page_t m) 767 { 768 769 return (pmap_methods_ptr->pm_is_referenced(m)); 770 } 771 772 void 773 pmap_remove_write(vm_page_t m) 774 { 775 776 pmap_methods_ptr->pm_remove_write(m); 777 } 778 779 int 780 pmap_ts_referenced(vm_page_t m) 781 { 782 783 return (pmap_methods_ptr->pm_ts_referenced(m)); 784 } 785 786 void * 787 pmap_mapdev_attr(vm_paddr_t pa, vm_size_t size, int mode) 788 { 789 790 return (pmap_methods_ptr->pm_mapdev_attr(pa, size, mode, 791 MAPDEV_SETATTR)); 792 } 793 794 void * 795 pmap_mapdev(vm_paddr_t pa, vm_size_t size) 796 { 797 798 return (pmap_methods_ptr->pm_mapdev_attr(pa, size, PAT_UNCACHEABLE, 799 MAPDEV_SETATTR)); 800 } 801 802 void * 803 pmap_mapbios(vm_paddr_t pa, vm_size_t size) 804 { 805 806 return (pmap_methods_ptr->pm_mapdev_attr(pa, size, PAT_WRITE_BACK, 0)); 807 } 808 809 void 810 pmap_unmapdev(vm_offset_t va, vm_size_t size) 811 { 812 813 pmap_methods_ptr->pm_unmapdev(va, size); 814 } 815 816 void 817 pmap_page_set_memattr(vm_page_t m, vm_memattr_t ma) 818 { 819 820 pmap_methods_ptr->pm_page_set_memattr(m, ma); 821 } 822 823 vm_paddr_t 824 pmap_extract(pmap_t pmap, vm_offset_t va) 825 { 826 827 return (pmap_methods_ptr->pm_extract(pmap, va)); 828 } 829 830 vm_page_t 831 pmap_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot) 832 { 833 834 return (pmap_methods_ptr->pm_extract_and_hold(pmap, va, prot)); 835 } 836 837 vm_offset_t 838 pmap_map(vm_offset_t *virt, vm_paddr_t start, vm_paddr_t end, int prot) 839 { 840 841 return (pmap_methods_ptr->pm_map(virt, start, end, prot)); 842 } 843 844 void 845 pmap_qenter(vm_offset_t sva, vm_page_t *ma, int count) 846 { 847 848 pmap_methods_ptr->pm_qenter(sva, ma, count); 849 } 850 851 void 852 pmap_qremove(vm_offset_t sva, int count) 853 { 854 855 pmap_methods_ptr->pm_qremove(sva, count); 856 } 857 858 void 859 pmap_release(pmap_t pmap) 860 { 861 862 pmap_methods_ptr->pm_release(pmap); 863 } 864 865 void 866 pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) 867 { 868 869 pmap_methods_ptr->pm_remove(pmap, sva, eva); 870 } 871 872 void 873 pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot) 874 { 875 876 pmap_methods_ptr->pm_protect(pmap, sva, eva, prot); 877 } 878 879 void 880 pmap_remove_all(vm_page_t m) 881 { 882 883 pmap_methods_ptr->pm_remove_all(m); 884 } 885 886 void 887 pmap_init(void) 888 { 889 890 pmap_methods_ptr->pm_init(); 891 } 892 893 void 894 pmap_init_pat(void) 895 { 896 897 pmap_methods_ptr->pm_init_pat(); 898 } 899 900 void 901 pmap_growkernel(vm_offset_t addr) 902 { 903 904 pmap_methods_ptr->pm_growkernel(addr); 905 } 906 907 void 908 pmap_invalidate_page(pmap_t pmap, vm_offset_t va) 909 { 910 911 pmap_methods_ptr->pm_invalidate_page(pmap, va); 912 } 913 914 void 915 pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) 916 { 917 918 pmap_methods_ptr->pm_invalidate_range(pmap, sva, eva); 919 } 920 921 void 922 pmap_invalidate_all(pmap_t pmap) 923 { 924 925 pmap_methods_ptr->pm_invalidate_all(pmap); 926 } 927 928 void 929 pmap_invalidate_cache(void) 930 { 931 932 pmap_methods_ptr->pm_invalidate_cache(); 933 } 934 935 void 936 pmap_kenter(vm_offset_t va, vm_paddr_t pa) 937 { 938 939 pmap_methods_ptr->pm_kenter(va, pa); 940 } 941 942 void 943 pmap_kremove(vm_offset_t va) 944 { 945 946 pmap_methods_ptr->pm_kremove(va); 947 } 948 949 extern struct pmap_methods pmap_pae_methods, pmap_nopae_methods; 950 int pae_mode; 951 SYSCTL_INT(_vm_pmap, OID_AUTO, pae_mode, CTLFLAG_RDTUN | CTLFLAG_NOFETCH, 952 &pae_mode, 0, 953 "PAE"); 954 955 void 956 pmap_cold(void) 957 { 958 959 init_static_kenv((char *)bootinfo.bi_envp, 0); 960 pae_mode = (cpu_feature & CPUID_PAE) != 0; 961 if (pae_mode) 962 TUNABLE_INT_FETCH("vm.pmap.pae_mode", &pae_mode); 963 if (pae_mode) { 964 pmap_methods_ptr = &pmap_pae_methods; 965 pmap_pae_cold(); 966 } else { 967 pmap_methods_ptr = &pmap_nopae_methods; 968 pmap_nopae_cold(); 969 } 970 } 971