1 /*- 2 * SPDX-License-Identifier: BSD-4-Clause 3 * 4 * Copyright (c) 1991 Regents of the University of California. 5 * All rights reserved. 6 * Copyright (c) 1994 John S. Dyson 7 * All rights reserved. 8 * Copyright (c) 1994 David Greenman 9 * All rights reserved. 10 * Copyright (c) 2005-2010 Alan L. Cox <alc@cs.rice.edu> 11 * All rights reserved. 12 * 13 * This code is derived from software contributed to Berkeley by 14 * the Systems Programming Group of the University of Utah Computer 15 * Science Department and William Jolitz of UUNET Technologies Inc. 16 * 17 * Redistribution and use in source and binary forms, with or without 18 * modification, are permitted provided that the following conditions 19 * are met: 20 * 1. Redistributions of source code must retain the above copyright 21 * notice, this list of conditions and the following disclaimer. 22 * 2. Redistributions in binary form must reproduce the above copyright 23 * notice, this list of conditions and the following disclaimer in the 24 * documentation and/or other materials provided with the distribution. 25 * 3. All advertising materials mentioning features or use of this software 26 * must display the following acknowledgement: 27 * This product includes software developed by the University of 28 * California, Berkeley and its contributors. 29 * 4. Neither the name of the University nor the names of its contributors 30 * may be used to endorse or promote products derived from this software 31 * without specific prior written permission. 32 * 33 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 34 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 35 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 36 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 37 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 38 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 39 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 40 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 41 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 42 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 43 * SUCH DAMAGE. 44 */ 45 /*- 46 * Copyright (c) 2003 Networks Associates Technology, Inc. 47 * All rights reserved. 48 * Copyright (c) 2018 The FreeBSD Foundation 49 * All rights reserved. 50 * 51 * This software was developed for the FreeBSD Project by Jake Burkholder, 52 * Safeport Network Services, and Network Associates Laboratories, the 53 * Security Research Division of Network Associates, Inc. under 54 * DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"), as part of the DARPA 55 * CHATS research program. 56 * 57 * Portions of this software were developed by 58 * Konstantin Belousov <kib@FreeBSD.org> under sponsorship from 59 * the FreeBSD Foundation. 60 * 61 * Redistribution and use in source and binary forms, with or without 62 * modification, are permitted provided that the following conditions 63 * are met: 64 * 1. Redistributions of source code must retain the above copyright 65 * notice, this list of conditions and the following disclaimer. 66 * 2. Redistributions in binary form must reproduce the above copyright 67 * notice, this list of conditions and the following disclaimer in the 68 * documentation and/or other materials provided with the distribution. 69 * 70 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 71 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 72 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 73 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 74 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 75 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 76 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 77 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 78 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 79 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 80 * SUCH DAMAGE. 81 */ 82 83 #include <sys/cdefs.h> 84 #include "opt_apic.h" 85 #include "opt_cpu.h" 86 #include "opt_pmap.h" 87 #include "opt_smp.h" 88 #include "opt_vm.h" 89 90 #include <sys/param.h> 91 #include <sys/systm.h> 92 #include <sys/kernel.h> 93 #include <sys/vmmeter.h> 94 #include <sys/sysctl.h> 95 #include <machine/bootinfo.h> 96 #include <machine/cpu.h> 97 #include <machine/cputypes.h> 98 #include <machine/md_var.h> 99 #ifdef DEV_APIC 100 #include <sys/bus.h> 101 #include <machine/intr_machdep.h> 102 #include <x86/apicvar.h> 103 #endif 104 #include <x86/ifunc.h> 105 106 static SYSCTL_NODE(_vm, OID_AUTO, pmap, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 107 "VM/pmap parameters"); 108 109 #include <machine/vmparam.h> 110 #include <vm/vm.h> 111 #include <vm/vm_page.h> 112 #include <vm/pmap.h> 113 #include <machine/pmap_base.h> 114 115 vm_offset_t virtual_avail; /* VA of first avail page (after kernel bss) */ 116 vm_offset_t virtual_end; /* VA of last avail page (end of kernel AS) */ 117 118 int unmapped_buf_allowed = 1; 119 120 int pti; 121 122 u_long physfree; /* phys addr of next free page */ 123 u_long vm86phystk; /* PA of vm86/bios stack */ 124 u_long vm86paddr; /* address of vm86 region */ 125 int vm86pa; /* phys addr of vm86 region */ 126 u_long KERNend; /* phys addr end of kernel (just after bss) */ 127 u_long KPTphys; /* phys addr of kernel page tables */ 128 caddr_t ptvmmap = 0; 129 vm_offset_t kernel_vm_end; 130 131 int i386_pmap_VM_NFREEORDER; 132 int i386_pmap_VM_LEVEL_0_ORDER; 133 int i386_pmap_PDRSHIFT; 134 135 int pat_works = 1; 136 SYSCTL_INT(_vm_pmap, OID_AUTO, pat_works, CTLFLAG_RD, 137 &pat_works, 0, 138 "Is page attribute table fully functional?"); 139 140 int pg_ps_enabled = 1; 141 SYSCTL_INT(_vm_pmap, OID_AUTO, pg_ps_enabled, CTLFLAG_RDTUN | CTLFLAG_NOFETCH, 142 &pg_ps_enabled, 0, 143 "Are large page mappings enabled?"); 144 145 int pv_entry_max = 0; 146 SYSCTL_INT(_vm_pmap, OID_AUTO, pv_entry_max, CTLFLAG_RD, 147 &pv_entry_max, 0, 148 "Max number of PV entries"); 149 150 int pv_entry_count = 0; 151 SYSCTL_INT(_vm_pmap, OID_AUTO, pv_entry_count, CTLFLAG_RD, 152 &pv_entry_count, 0, 153 "Current number of pv entries"); 154 155 #ifndef PMAP_SHPGPERPROC 156 #define PMAP_SHPGPERPROC 200 157 #endif 158 159 int shpgperproc = PMAP_SHPGPERPROC; 160 SYSCTL_INT(_vm_pmap, OID_AUTO, shpgperproc, CTLFLAG_RD, 161 &shpgperproc, 0, 162 "Page share factor per proc"); 163 164 static SYSCTL_NODE(_vm_pmap, OID_AUTO, pde, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 165 "2/4MB page mapping counters"); 166 167 u_long pmap_pde_demotions; 168 SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, demotions, CTLFLAG_RD, 169 &pmap_pde_demotions, 0, 170 "2/4MB page demotions"); 171 172 u_long pmap_pde_mappings; 173 SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, mappings, CTLFLAG_RD, 174 &pmap_pde_mappings, 0, 175 "2/4MB page mappings"); 176 177 u_long pmap_pde_p_failures; 178 SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, p_failures, CTLFLAG_RD, 179 &pmap_pde_p_failures, 0, 180 "2/4MB page promotion failures"); 181 182 u_long pmap_pde_promotions; 183 SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, promotions, CTLFLAG_RD, 184 &pmap_pde_promotions, 0, 185 "2/4MB page promotions"); 186 187 #ifdef SMP 188 int PMAP1changedcpu; 189 SYSCTL_INT(_debug, OID_AUTO, PMAP1changedcpu, CTLFLAG_RD, 190 &PMAP1changedcpu, 0, 191 "Number of times pmap_pte_quick changed CPU with same PMAP1"); 192 #endif 193 194 int PMAP1changed; 195 SYSCTL_INT(_debug, OID_AUTO, PMAP1changed, CTLFLAG_RD, 196 &PMAP1changed, 0, 197 "Number of times pmap_pte_quick changed PMAP1"); 198 int PMAP1unchanged; 199 SYSCTL_INT(_debug, OID_AUTO, PMAP1unchanged, CTLFLAG_RD, 200 &PMAP1unchanged, 0, 201 "Number of times pmap_pte_quick didn't change PMAP1"); 202 203 static int 204 kvm_size(SYSCTL_HANDLER_ARGS) 205 { 206 unsigned long ksize; 207 208 ksize = VM_MAX_KERNEL_ADDRESS - KERNBASE; 209 return (sysctl_handle_long(oidp, &ksize, 0, req)); 210 } 211 SYSCTL_PROC(_vm, OID_AUTO, kvm_size, CTLTYPE_LONG | CTLFLAG_RD | CTLFLAG_MPSAFE, 212 0, 0, kvm_size, "IU", 213 "Size of KVM"); 214 215 static int 216 kvm_free(SYSCTL_HANDLER_ARGS) 217 { 218 unsigned long kfree; 219 220 kfree = VM_MAX_KERNEL_ADDRESS - kernel_vm_end; 221 return (sysctl_handle_long(oidp, &kfree, 0, req)); 222 } 223 SYSCTL_PROC(_vm, OID_AUTO, kvm_free, CTLTYPE_LONG | CTLFLAG_RD | CTLFLAG_MPSAFE, 224 0, 0, kvm_free, "IU", 225 "Amount of KVM free"); 226 227 #ifdef PV_STATS 228 int pc_chunk_count, pc_chunk_allocs, pc_chunk_frees, pc_chunk_tryfail; 229 long pv_entry_frees, pv_entry_allocs; 230 int pv_entry_spare; 231 232 SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_count, CTLFLAG_RD, 233 &pc_chunk_count, 0, 234 "Current number of pv entry chunks"); 235 SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_allocs, CTLFLAG_RD, 236 &pc_chunk_allocs, 0, 237 "Current number of pv entry chunks allocated"); 238 SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_frees, CTLFLAG_RD, 239 &pc_chunk_frees, 0, 240 "Current number of pv entry chunks frees"); 241 SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_tryfail, CTLFLAG_RD, 242 &pc_chunk_tryfail, 0, 243 "Number of times tried to get a chunk page but failed."); 244 SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_frees, CTLFLAG_RD, 245 &pv_entry_frees, 0, 246 "Current number of pv entry frees"); 247 SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_allocs, CTLFLAG_RD, 248 &pv_entry_allocs, 0, 249 "Current number of pv entry allocs"); 250 SYSCTL_INT(_vm_pmap, OID_AUTO, pv_entry_spare, CTLFLAG_RD, 251 &pv_entry_spare, 0, 252 "Current number of spare pv entries"); 253 #endif 254 255 struct pmap kernel_pmap_store; 256 static struct pmap_methods *pmap_methods_ptr; 257 258 static int 259 sysctl_kmaps(SYSCTL_HANDLER_ARGS) 260 { 261 return (pmap_methods_ptr->pm_sysctl_kmaps(oidp, arg1, arg2, req)); 262 } 263 SYSCTL_OID(_vm_pmap, OID_AUTO, kernel_maps, 264 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE | CTLFLAG_SKIP, 265 NULL, 0, sysctl_kmaps, "A", 266 "Dump kernel address layout"); 267 268 /* 269 * Initialize a vm_page's machine-dependent fields. 270 */ 271 void 272 pmap_page_init(vm_page_t m) 273 { 274 275 TAILQ_INIT(&m->md.pv_list); 276 m->md.pat_mode = PAT_WRITE_BACK; 277 } 278 279 void 280 invltlb_glob(void) 281 { 282 283 invltlb(); 284 } 285 286 static void pmap_invalidate_cache_range_selfsnoop(vm_offset_t sva, 287 vm_offset_t eva); 288 static void pmap_invalidate_cache_range_all(vm_offset_t sva, 289 vm_offset_t eva); 290 291 void 292 pmap_flush_page(vm_page_t m) 293 { 294 295 pmap_methods_ptr->pm_flush_page(m); 296 } 297 298 DEFINE_IFUNC(, void, pmap_invalidate_cache_range, (vm_offset_t, vm_offset_t)) 299 { 300 301 if ((cpu_feature & CPUID_SS) != 0) 302 return (pmap_invalidate_cache_range_selfsnoop); 303 if ((cpu_feature & CPUID_CLFSH) != 0) 304 return (pmap_force_invalidate_cache_range); 305 return (pmap_invalidate_cache_range_all); 306 } 307 308 #define PMAP_CLFLUSH_THRESHOLD (2 * 1024 * 1024) 309 310 static void 311 pmap_invalidate_cache_range_check_align(vm_offset_t sva, vm_offset_t eva) 312 { 313 314 KASSERT((sva & PAGE_MASK) == 0, 315 ("pmap_invalidate_cache_range: sva not page-aligned")); 316 KASSERT((eva & PAGE_MASK) == 0, 317 ("pmap_invalidate_cache_range: eva not page-aligned")); 318 } 319 320 static void 321 pmap_invalidate_cache_range_selfsnoop(vm_offset_t sva, vm_offset_t eva) 322 { 323 324 pmap_invalidate_cache_range_check_align(sva, eva); 325 } 326 327 void 328 pmap_force_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva) 329 { 330 331 sva &= ~(vm_offset_t)(cpu_clflush_line_size - 1); 332 if (eva - sva >= PMAP_CLFLUSH_THRESHOLD) { 333 /* 334 * The supplied range is bigger than 2MB. 335 * Globally invalidate cache. 336 */ 337 pmap_invalidate_cache(); 338 return; 339 } 340 341 #ifdef DEV_APIC 342 /* 343 * XXX: Some CPUs fault, hang, or trash the local APIC 344 * registers if we use CLFLUSH on the local APIC 345 * range. The local APIC is always uncached, so we 346 * don't need to flush for that range anyway. 347 */ 348 if (pmap_kextract(sva) == lapic_paddr) 349 return; 350 #endif 351 352 if ((cpu_stdext_feature & CPUID_STDEXT_CLFLUSHOPT) != 0) { 353 /* 354 * Do per-cache line flush. Use the sfence 355 * instruction to insure that previous stores are 356 * included in the write-back. The processor 357 * propagates flush to other processors in the cache 358 * coherence domain. 359 */ 360 sfence(); 361 for (; sva < eva; sva += cpu_clflush_line_size) 362 clflushopt(sva); 363 sfence(); 364 } else { 365 /* 366 * Writes are ordered by CLFLUSH on Intel CPUs. 367 */ 368 if (cpu_vendor_id != CPU_VENDOR_INTEL) 369 mfence(); 370 for (; sva < eva; sva += cpu_clflush_line_size) 371 clflush(sva); 372 if (cpu_vendor_id != CPU_VENDOR_INTEL) 373 mfence(); 374 } 375 } 376 377 static void 378 pmap_invalidate_cache_range_all(vm_offset_t sva, vm_offset_t eva) 379 { 380 381 pmap_invalidate_cache_range_check_align(sva, eva); 382 pmap_invalidate_cache(); 383 } 384 385 void 386 pmap_invalidate_cache_pages(vm_page_t *pages, int count) 387 { 388 int i; 389 390 if (count >= PMAP_CLFLUSH_THRESHOLD / PAGE_SIZE || 391 (cpu_feature & CPUID_CLFSH) == 0) { 392 pmap_invalidate_cache(); 393 } else { 394 for (i = 0; i < count; i++) 395 pmap_flush_page(pages[i]); 396 } 397 } 398 399 void 400 pmap_ksetrw(vm_offset_t va) 401 { 402 403 pmap_methods_ptr->pm_ksetrw(va); 404 } 405 406 void 407 pmap_remap_lower(bool enable) 408 { 409 410 pmap_methods_ptr->pm_remap_lower(enable); 411 } 412 413 void 414 pmap_remap_lowptdi(bool enable) 415 { 416 417 pmap_methods_ptr->pm_remap_lowptdi(enable); 418 } 419 420 void 421 pmap_align_superpage(vm_object_t object, vm_ooffset_t offset, 422 vm_offset_t *addr, vm_size_t size) 423 { 424 425 return (pmap_methods_ptr->pm_align_superpage(object, offset, 426 addr, size)); 427 } 428 429 vm_offset_t 430 pmap_quick_enter_page(vm_page_t m) 431 { 432 433 return (pmap_methods_ptr->pm_quick_enter_page(m)); 434 } 435 436 void 437 pmap_quick_remove_page(vm_offset_t addr) 438 { 439 440 return (pmap_methods_ptr->pm_quick_remove_page(addr)); 441 } 442 443 void * 444 pmap_trm_alloc(size_t size, int flags) 445 { 446 447 return (pmap_methods_ptr->pm_trm_alloc(size, flags)); 448 } 449 450 void 451 pmap_trm_free(void *addr, size_t size) 452 { 453 454 pmap_methods_ptr->pm_trm_free(addr, size); 455 } 456 457 void 458 pmap_sync_icache(pmap_t pm, vm_offset_t va, vm_size_t sz) 459 { 460 } 461 462 vm_offset_t 463 pmap_get_map_low(void) 464 { 465 466 return (pmap_methods_ptr->pm_get_map_low()); 467 } 468 469 vm_offset_t 470 pmap_get_vm_maxuser_address(void) 471 { 472 473 return (pmap_methods_ptr->pm_get_vm_maxuser_address()); 474 } 475 476 vm_paddr_t 477 pmap_kextract(vm_offset_t va) 478 { 479 480 return (pmap_methods_ptr->pm_kextract(va)); 481 } 482 483 vm_paddr_t 484 pmap_pg_frame(vm_paddr_t pa) 485 { 486 487 return (pmap_methods_ptr->pm_pg_frame(pa)); 488 } 489 490 void 491 pmap_sf_buf_map(struct sf_buf *sf) 492 { 493 494 pmap_methods_ptr->pm_sf_buf_map(sf); 495 } 496 497 void 498 pmap_cp_slow0_map(vm_offset_t kaddr, int plen, vm_page_t *ma) 499 { 500 501 pmap_methods_ptr->pm_cp_slow0_map(kaddr, plen, ma); 502 } 503 504 u_int 505 pmap_get_kcr3(void) 506 { 507 508 return (pmap_methods_ptr->pm_get_kcr3()); 509 } 510 511 u_int 512 pmap_get_cr3(pmap_t pmap) 513 { 514 515 return (pmap_methods_ptr->pm_get_cr3(pmap)); 516 } 517 518 caddr_t 519 pmap_cmap3(vm_paddr_t pa, u_int pte_flags) 520 { 521 522 return (pmap_methods_ptr->pm_cmap3(pa, pte_flags)); 523 } 524 525 void 526 pmap_basemem_setup(u_int basemem) 527 { 528 529 pmap_methods_ptr->pm_basemem_setup(basemem); 530 } 531 532 void 533 pmap_set_nx(void) 534 { 535 536 pmap_methods_ptr->pm_set_nx(); 537 } 538 539 void * 540 pmap_bios16_enter(void) 541 { 542 543 return (pmap_methods_ptr->pm_bios16_enter()); 544 } 545 546 void 547 pmap_bios16_leave(void *handle) 548 { 549 550 pmap_methods_ptr->pm_bios16_leave(handle); 551 } 552 553 void 554 pmap_bootstrap(vm_paddr_t firstaddr) 555 { 556 557 pmap_methods_ptr->pm_bootstrap(firstaddr); 558 } 559 560 boolean_t 561 pmap_is_valid_memattr(pmap_t pmap, vm_memattr_t mode) 562 { 563 564 return (pmap_methods_ptr->pm_is_valid_memattr(pmap, mode)); 565 } 566 567 int 568 pmap_cache_bits(pmap_t pmap, int mode, boolean_t is_pde) 569 { 570 571 return (pmap_methods_ptr->pm_cache_bits(pmap, mode, is_pde)); 572 } 573 574 bool 575 pmap_ps_enabled(pmap_t pmap) 576 { 577 578 return (pmap_methods_ptr->pm_ps_enabled(pmap)); 579 } 580 581 void 582 pmap_pinit0(pmap_t pmap) 583 { 584 585 pmap_methods_ptr->pm_pinit0(pmap); 586 } 587 588 int 589 pmap_pinit(pmap_t pmap) 590 { 591 592 return (pmap_methods_ptr->pm_pinit(pmap)); 593 } 594 595 void 596 pmap_activate(struct thread *td) 597 { 598 599 pmap_methods_ptr->pm_activate(td); 600 } 601 602 void 603 pmap_activate_boot(pmap_t pmap) 604 { 605 606 pmap_methods_ptr->pm_activate_boot(pmap); 607 } 608 609 void 610 pmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int advice) 611 { 612 613 pmap_methods_ptr->pm_advise(pmap, sva, eva, advice); 614 } 615 616 void 617 pmap_clear_modify(vm_page_t m) 618 { 619 620 pmap_methods_ptr->pm_clear_modify(m); 621 } 622 623 int 624 pmap_change_attr(vm_offset_t va, vm_size_t size, int mode) 625 { 626 627 return (pmap_methods_ptr->pm_change_attr(va, size, mode)); 628 } 629 630 int 631 pmap_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *pap) 632 { 633 634 return (pmap_methods_ptr->pm_mincore(pmap, addr, pap)); 635 } 636 637 void 638 pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, vm_size_t len, 639 vm_offset_t src_addr) 640 { 641 642 pmap_methods_ptr->pm_copy(dst_pmap, src_pmap, dst_addr, len, src_addr); 643 } 644 645 void 646 pmap_copy_page(vm_page_t src, vm_page_t dst) 647 { 648 649 pmap_methods_ptr->pm_copy_page(src, dst); 650 } 651 652 void 653 pmap_copy_pages(vm_page_t ma[], vm_offset_t a_offset, vm_page_t mb[], 654 vm_offset_t b_offset, int xfersize) 655 { 656 657 pmap_methods_ptr->pm_copy_pages(ma, a_offset, mb, b_offset, xfersize); 658 } 659 660 void 661 pmap_zero_page(vm_page_t m) 662 { 663 664 pmap_methods_ptr->pm_zero_page(m); 665 } 666 667 void 668 pmap_zero_page_area(vm_page_t m, int off, int size) 669 { 670 671 pmap_methods_ptr->pm_zero_page_area(m, off, size); 672 } 673 674 int 675 pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, 676 u_int flags, int8_t psind) 677 { 678 679 return (pmap_methods_ptr->pm_enter(pmap, va, m, prot, flags, psind)); 680 } 681 682 void 683 pmap_enter_object(pmap_t pmap, vm_offset_t start, vm_offset_t end, 684 vm_page_t m_start, vm_prot_t prot) 685 { 686 687 pmap_methods_ptr->pm_enter_object(pmap, start, end, m_start, prot); 688 } 689 690 void 691 pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot) 692 { 693 694 pmap_methods_ptr->pm_enter_quick(pmap, va, m, prot); 695 } 696 697 void * 698 pmap_kenter_temporary(vm_paddr_t pa, int i) 699 { 700 701 return (pmap_methods_ptr->pm_kenter_temporary(pa, i)); 702 } 703 704 void 705 pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, vm_object_t object, 706 vm_pindex_t pindex, vm_size_t size) 707 { 708 709 pmap_methods_ptr->pm_object_init_pt(pmap, addr, object, pindex, size); 710 } 711 712 void 713 pmap_unwire(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) 714 { 715 716 pmap_methods_ptr->pm_unwire(pmap, sva, eva); 717 } 718 719 boolean_t 720 pmap_page_exists_quick(pmap_t pmap, vm_page_t m) 721 { 722 723 return (pmap_methods_ptr->pm_page_exists_quick(pmap, m)); 724 } 725 726 int 727 pmap_page_wired_mappings(vm_page_t m) 728 { 729 730 return (pmap_methods_ptr->pm_page_wired_mappings(m)); 731 } 732 733 boolean_t 734 pmap_page_is_mapped(vm_page_t m) 735 { 736 737 return (pmap_methods_ptr->pm_page_is_mapped(m)); 738 } 739 740 void 741 pmap_remove_pages(pmap_t pmap) 742 { 743 744 pmap_methods_ptr->pm_remove_pages(pmap); 745 } 746 747 boolean_t 748 pmap_is_modified(vm_page_t m) 749 { 750 751 return (pmap_methods_ptr->pm_is_modified(m)); 752 } 753 754 boolean_t 755 pmap_is_prefaultable(pmap_t pmap, vm_offset_t addr) 756 { 757 758 return (pmap_methods_ptr->pm_is_prefaultable(pmap, addr)); 759 } 760 761 boolean_t 762 pmap_is_referenced(vm_page_t m) 763 { 764 765 return (pmap_methods_ptr->pm_is_referenced(m)); 766 } 767 768 void 769 pmap_remove_write(vm_page_t m) 770 { 771 772 pmap_methods_ptr->pm_remove_write(m); 773 } 774 775 int 776 pmap_ts_referenced(vm_page_t m) 777 { 778 779 return (pmap_methods_ptr->pm_ts_referenced(m)); 780 } 781 782 void * 783 pmap_mapdev_attr(vm_paddr_t pa, vm_size_t size, int mode) 784 { 785 786 return (pmap_methods_ptr->pm_mapdev_attr(pa, size, mode, 787 MAPDEV_SETATTR)); 788 } 789 790 void * 791 pmap_mapdev(vm_paddr_t pa, vm_size_t size) 792 { 793 794 return (pmap_methods_ptr->pm_mapdev_attr(pa, size, PAT_UNCACHEABLE, 795 MAPDEV_SETATTR)); 796 } 797 798 void * 799 pmap_mapbios(vm_paddr_t pa, vm_size_t size) 800 { 801 802 return (pmap_methods_ptr->pm_mapdev_attr(pa, size, PAT_WRITE_BACK, 0)); 803 } 804 805 void 806 pmap_unmapdev(void *p, vm_size_t size) 807 { 808 809 pmap_methods_ptr->pm_unmapdev(p, size); 810 } 811 812 void 813 pmap_page_set_memattr(vm_page_t m, vm_memattr_t ma) 814 { 815 816 pmap_methods_ptr->pm_page_set_memattr(m, ma); 817 } 818 819 vm_paddr_t 820 pmap_extract(pmap_t pmap, vm_offset_t va) 821 { 822 823 return (pmap_methods_ptr->pm_extract(pmap, va)); 824 } 825 826 vm_page_t 827 pmap_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot) 828 { 829 830 return (pmap_methods_ptr->pm_extract_and_hold(pmap, va, prot)); 831 } 832 833 vm_offset_t 834 pmap_map(vm_offset_t *virt, vm_paddr_t start, vm_paddr_t end, int prot) 835 { 836 837 return (pmap_methods_ptr->pm_map(virt, start, end, prot)); 838 } 839 840 void 841 pmap_qenter(vm_offset_t sva, vm_page_t *ma, int count) 842 { 843 844 pmap_methods_ptr->pm_qenter(sva, ma, count); 845 } 846 847 void 848 pmap_qremove(vm_offset_t sva, int count) 849 { 850 851 pmap_methods_ptr->pm_qremove(sva, count); 852 } 853 854 void 855 pmap_release(pmap_t pmap) 856 { 857 858 pmap_methods_ptr->pm_release(pmap); 859 } 860 861 void 862 pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) 863 { 864 865 pmap_methods_ptr->pm_remove(pmap, sva, eva); 866 } 867 868 void 869 pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot) 870 { 871 872 pmap_methods_ptr->pm_protect(pmap, sva, eva, prot); 873 } 874 875 void 876 pmap_remove_all(vm_page_t m) 877 { 878 879 pmap_methods_ptr->pm_remove_all(m); 880 } 881 882 void 883 pmap_init(void) 884 { 885 886 pmap_methods_ptr->pm_init(); 887 } 888 889 void 890 pmap_init_pat(void) 891 { 892 893 pmap_methods_ptr->pm_init_pat(); 894 } 895 896 void 897 pmap_growkernel(vm_offset_t addr) 898 { 899 900 pmap_methods_ptr->pm_growkernel(addr); 901 } 902 903 void 904 pmap_invalidate_page(pmap_t pmap, vm_offset_t va) 905 { 906 907 pmap_methods_ptr->pm_invalidate_page(pmap, va); 908 } 909 910 void 911 pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) 912 { 913 914 pmap_methods_ptr->pm_invalidate_range(pmap, sva, eva); 915 } 916 917 void 918 pmap_invalidate_all(pmap_t pmap) 919 { 920 921 pmap_methods_ptr->pm_invalidate_all(pmap); 922 } 923 924 void 925 pmap_invalidate_cache(void) 926 { 927 928 pmap_methods_ptr->pm_invalidate_cache(); 929 } 930 931 void 932 pmap_kenter(vm_offset_t va, vm_paddr_t pa) 933 { 934 935 pmap_methods_ptr->pm_kenter(va, pa); 936 } 937 938 void 939 pmap_kremove(vm_offset_t va) 940 { 941 942 pmap_methods_ptr->pm_kremove(va); 943 } 944 945 void 946 pmap_active_cpus(pmap_t pmap, cpuset_t *res) 947 { 948 *res = pmap->pm_active; 949 } 950 951 extern struct pmap_methods pmap_pae_methods, pmap_nopae_methods; 952 int pae_mode; 953 SYSCTL_INT(_vm_pmap, OID_AUTO, pae_mode, CTLFLAG_RDTUN | CTLFLAG_NOFETCH, 954 &pae_mode, 0, 955 "PAE"); 956 957 void 958 pmap_cold(void) 959 { 960 961 init_static_kenv((char *)bootinfo.bi_envp, 0); 962 pae_mode = (cpu_feature & CPUID_PAE) != 0; 963 if (pae_mode) 964 TUNABLE_INT_FETCH("vm.pmap.pae_mode", &pae_mode); 965 if (pae_mode) { 966 pmap_methods_ptr = &pmap_pae_methods; 967 pmap_pae_cold(); 968 } else { 969 pmap_methods_ptr = &pmap_nopae_methods; 970 pmap_nopae_cold(); 971 } 972 } 973