xref: /freebsd/sys/i386/i386/pmap_base.c (revision 13ea0450a9c8742119d36f3bf8f47accdce46e54)
1 /*-
2  * SPDX-License-Identifier: BSD-4-Clause
3  *
4  * Copyright (c) 1991 Regents of the University of California.
5  * All rights reserved.
6  * Copyright (c) 1994 John S. Dyson
7  * All rights reserved.
8  * Copyright (c) 1994 David Greenman
9  * All rights reserved.
10  * Copyright (c) 2005-2010 Alan L. Cox <alc@cs.rice.edu>
11  * All rights reserved.
12  *
13  * This code is derived from software contributed to Berkeley by
14  * the Systems Programming Group of the University of Utah Computer
15  * Science Department and William Jolitz of UUNET Technologies Inc.
16  *
17  * Redistribution and use in source and binary forms, with or without
18  * modification, are permitted provided that the following conditions
19  * are met:
20  * 1. Redistributions of source code must retain the above copyright
21  *    notice, this list of conditions and the following disclaimer.
22  * 2. Redistributions in binary form must reproduce the above copyright
23  *    notice, this list of conditions and the following disclaimer in the
24  *    documentation and/or other materials provided with the distribution.
25  * 3. All advertising materials mentioning features or use of this software
26  *    must display the following acknowledgement:
27  *	This product includes software developed by the University of
28  *	California, Berkeley and its contributors.
29  * 4. Neither the name of the University nor the names of its contributors
30  *    may be used to endorse or promote products derived from this software
31  *    without specific prior written permission.
32  *
33  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
34  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
35  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
36  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
37  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
38  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
39  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
40  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
41  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
42  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
43  * SUCH DAMAGE.
44  *
45  *	from:	@(#)pmap.c	7.7 (Berkeley)	5/12/91
46  */
47 /*-
48  * Copyright (c) 2003 Networks Associates Technology, Inc.
49  * All rights reserved.
50  * Copyright (c) 2018 The FreeBSD Foundation
51  * All rights reserved.
52  *
53  * This software was developed for the FreeBSD Project by Jake Burkholder,
54  * Safeport Network Services, and Network Associates Laboratories, the
55  * Security Research Division of Network Associates, Inc. under
56  * DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"), as part of the DARPA
57  * CHATS research program.
58  *
59  * Portions of this software were developed by
60  * Konstantin Belousov <kib@FreeBSD.org> under sponsorship from
61  * the FreeBSD Foundation.
62  *
63  * Redistribution and use in source and binary forms, with or without
64  * modification, are permitted provided that the following conditions
65  * are met:
66  * 1. Redistributions of source code must retain the above copyright
67  *    notice, this list of conditions and the following disclaimer.
68  * 2. Redistributions in binary form must reproduce the above copyright
69  *    notice, this list of conditions and the following disclaimer in the
70  *    documentation and/or other materials provided with the distribution.
71  *
72  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
73  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
74  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
75  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
76  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
77  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
78  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
79  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
80  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
81  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
82  * SUCH DAMAGE.
83  */
84 
85 #include <sys/cdefs.h>
86 __FBSDID("$FreeBSD$");
87 
88 #include "opt_apic.h"
89 #include "opt_cpu.h"
90 #include "opt_pmap.h"
91 #include "opt_smp.h"
92 #include "opt_vm.h"
93 
94 #include <sys/param.h>
95 #include <sys/systm.h>
96 #include <sys/kernel.h>
97 #include <sys/vmmeter.h>
98 #include <sys/sysctl.h>
99 #include <machine/bootinfo.h>
100 #include <machine/cpu.h>
101 #include <machine/cputypes.h>
102 #include <machine/md_var.h>
103 #ifdef DEV_APIC
104 #include <sys/bus.h>
105 #include <machine/intr_machdep.h>
106 #include <x86/apicvar.h>
107 #endif
108 #include <x86/ifunc.h>
109 
110 static SYSCTL_NODE(_vm, OID_AUTO, pmap, CTLFLAG_RD, 0, "VM/pmap parameters");
111 
112 #include <machine/vmparam.h>
113 #include <vm/vm.h>
114 #include <vm/vm_page.h>
115 #include <vm/pmap.h>
116 #include <machine/pmap_base.h>
117 
118 vm_offset_t virtual_avail;	/* VA of first avail page (after kernel bss) */
119 vm_offset_t virtual_end;	/* VA of last avail page (end of kernel AS) */
120 
121 int unmapped_buf_allowed = 1;
122 
123 int pti;
124 
125 u_long physfree;	/* phys addr of next free page */
126 u_long vm86phystk;	/* PA of vm86/bios stack */
127 u_long vm86paddr;	/* address of vm86 region */
128 int vm86pa;		/* phys addr of vm86 region */
129 u_long KERNend;		/* phys addr end of kernel (just after bss) */
130 u_long KPTphys;		/* phys addr of kernel page tables */
131 caddr_t ptvmmap = 0;
132 vm_offset_t kernel_vm_end;
133 
134 int i386_pmap_VM_NFREEORDER;
135 int i386_pmap_VM_LEVEL_0_ORDER;
136 int i386_pmap_PDRSHIFT;
137 
138 int pat_works = 1;
139 SYSCTL_INT(_vm_pmap, OID_AUTO, pat_works, CTLFLAG_RD,
140     &pat_works, 0,
141     "Is page attribute table fully functional?");
142 
143 int pg_ps_enabled = 1;
144 SYSCTL_INT(_vm_pmap, OID_AUTO, pg_ps_enabled, CTLFLAG_RDTUN | CTLFLAG_NOFETCH,
145     &pg_ps_enabled, 0,
146     "Are large page mappings enabled?");
147 
148 int pv_entry_max = 0;
149 SYSCTL_INT(_vm_pmap, OID_AUTO, pv_entry_max, CTLFLAG_RD,
150     &pv_entry_max, 0,
151     "Max number of PV entries");
152 
153 int pv_entry_count = 0;
154 SYSCTL_INT(_vm_pmap, OID_AUTO, pv_entry_count, CTLFLAG_RD,
155     &pv_entry_count, 0,
156     "Current number of pv entries");
157 
158 #ifndef PMAP_SHPGPERPROC
159 #define PMAP_SHPGPERPROC 200
160 #endif
161 
162 int shpgperproc = PMAP_SHPGPERPROC;
163 SYSCTL_INT(_vm_pmap, OID_AUTO, shpgperproc, CTLFLAG_RD,
164     &shpgperproc, 0,
165     "Page share factor per proc");
166 
167 static SYSCTL_NODE(_vm_pmap, OID_AUTO, pde, CTLFLAG_RD, 0,
168     "2/4MB page mapping counters");
169 
170 u_long pmap_pde_demotions;
171 SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, demotions, CTLFLAG_RD,
172     &pmap_pde_demotions, 0,
173     "2/4MB page demotions");
174 
175 u_long pmap_pde_mappings;
176 SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, mappings, CTLFLAG_RD,
177     &pmap_pde_mappings, 0,
178     "2/4MB page mappings");
179 
180 u_long pmap_pde_p_failures;
181 SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, p_failures, CTLFLAG_RD,
182     &pmap_pde_p_failures, 0,
183     "2/4MB page promotion failures");
184 
185 u_long pmap_pde_promotions;
186 SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, promotions, CTLFLAG_RD,
187     &pmap_pde_promotions, 0,
188     "2/4MB page promotions");
189 
190 #ifdef SMP
191 int PMAP1changedcpu;
192 SYSCTL_INT(_debug, OID_AUTO, PMAP1changedcpu, CTLFLAG_RD,
193     &PMAP1changedcpu, 0,
194     "Number of times pmap_pte_quick changed CPU with same PMAP1");
195 #endif
196 
197 int PMAP1changed;
198 SYSCTL_INT(_debug, OID_AUTO, PMAP1changed, CTLFLAG_RD,
199     &PMAP1changed, 0,
200     "Number of times pmap_pte_quick changed PMAP1");
201 int PMAP1unchanged;
202 SYSCTL_INT(_debug, OID_AUTO, PMAP1unchanged, CTLFLAG_RD,
203     &PMAP1unchanged, 0,
204     "Number of times pmap_pte_quick didn't change PMAP1");
205 
206 static int
207 kvm_size(SYSCTL_HANDLER_ARGS)
208 {
209 	unsigned long ksize;
210 
211 	ksize = VM_MAX_KERNEL_ADDRESS - KERNBASE;
212 	return (sysctl_handle_long(oidp, &ksize, 0, req));
213 }
214 SYSCTL_PROC(_vm, OID_AUTO, kvm_size, CTLTYPE_LONG | CTLFLAG_RD | CTLFLAG_MPSAFE,
215     0, 0, kvm_size, "IU",
216     "Size of KVM");
217 
218 static int
219 kvm_free(SYSCTL_HANDLER_ARGS)
220 {
221 	unsigned long kfree;
222 
223 	kfree = VM_MAX_KERNEL_ADDRESS - kernel_vm_end;
224 	return (sysctl_handle_long(oidp, &kfree, 0, req));
225 }
226 SYSCTL_PROC(_vm, OID_AUTO, kvm_free, CTLTYPE_LONG | CTLFLAG_RD | CTLFLAG_MPSAFE,
227     0, 0, kvm_free, "IU",
228     "Amount of KVM free");
229 
230 #ifdef PV_STATS
231 int pc_chunk_count, pc_chunk_allocs, pc_chunk_frees, pc_chunk_tryfail;
232 long pv_entry_frees, pv_entry_allocs;
233 int pv_entry_spare;
234 
235 SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_count, CTLFLAG_RD,
236     &pc_chunk_count, 0,
237     "Current number of pv entry chunks");
238 SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_allocs, CTLFLAG_RD,
239     &pc_chunk_allocs, 0,
240     "Current number of pv entry chunks allocated");
241 SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_frees, CTLFLAG_RD,
242     &pc_chunk_frees, 0,
243     "Current number of pv entry chunks frees");
244 SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_tryfail, CTLFLAG_RD,
245     &pc_chunk_tryfail, 0,
246     "Number of times tried to get a chunk page but failed.");
247 SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_frees, CTLFLAG_RD,
248     &pv_entry_frees, 0,
249     "Current number of pv entry frees");
250 SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_allocs, CTLFLAG_RD,
251     &pv_entry_allocs, 0,
252     "Current number of pv entry allocs");
253 SYSCTL_INT(_vm_pmap, OID_AUTO, pv_entry_spare, CTLFLAG_RD,
254     &pv_entry_spare, 0,
255     "Current number of spare pv entries");
256 #endif
257 
258 struct pmap kernel_pmap_store;
259 static struct pmap_methods *pmap_methods_ptr;
260 
261 /*
262  * Initialize a vm_page's machine-dependent fields.
263  */
264 void
265 pmap_page_init(vm_page_t m)
266 {
267 
268 	TAILQ_INIT(&m->md.pv_list);
269 	m->md.pat_mode = PAT_WRITE_BACK;
270 }
271 
272 void
273 invltlb_glob(void)
274 {
275 
276 	invltlb();
277 }
278 
279 static void pmap_invalidate_cache_range_selfsnoop(vm_offset_t sva,
280     vm_offset_t eva);
281 static void pmap_invalidate_cache_range_all(vm_offset_t sva,
282     vm_offset_t eva);
283 
284 void
285 pmap_flush_page(vm_page_t m)
286 {
287 
288 	pmap_methods_ptr->pm_flush_page(m);
289 }
290 
291 DEFINE_IFUNC(, void, pmap_invalidate_cache_range, (vm_offset_t, vm_offset_t),
292     static)
293 {
294 
295 	if ((cpu_feature & CPUID_SS) != 0)
296 		return (pmap_invalidate_cache_range_selfsnoop);
297 	if ((cpu_feature & CPUID_CLFSH) != 0)
298 		return (pmap_force_invalidate_cache_range);
299 	return (pmap_invalidate_cache_range_all);
300 }
301 
302 #define	PMAP_CLFLUSH_THRESHOLD	(2 * 1024 * 1024)
303 
304 static void
305 pmap_invalidate_cache_range_check_align(vm_offset_t sva, vm_offset_t eva)
306 {
307 
308 	KASSERT((sva & PAGE_MASK) == 0,
309 	    ("pmap_invalidate_cache_range: sva not page-aligned"));
310 	KASSERT((eva & PAGE_MASK) == 0,
311 	    ("pmap_invalidate_cache_range: eva not page-aligned"));
312 }
313 
314 static void
315 pmap_invalidate_cache_range_selfsnoop(vm_offset_t sva, vm_offset_t eva)
316 {
317 
318 	pmap_invalidate_cache_range_check_align(sva, eva);
319 }
320 
321 void
322 pmap_force_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva)
323 {
324 
325 	sva &= ~(vm_offset_t)(cpu_clflush_line_size - 1);
326 	if (eva - sva >= PMAP_CLFLUSH_THRESHOLD) {
327 		/*
328 		 * The supplied range is bigger than 2MB.
329 		 * Globally invalidate cache.
330 		 */
331 		pmap_invalidate_cache();
332 		return;
333 	}
334 
335 #ifdef DEV_APIC
336 	/*
337 	 * XXX: Some CPUs fault, hang, or trash the local APIC
338 	 * registers if we use CLFLUSH on the local APIC
339 	 * range.  The local APIC is always uncached, so we
340 	 * don't need to flush for that range anyway.
341 	 */
342 	if (pmap_kextract(sva) == lapic_paddr)
343 		return;
344 #endif
345 
346 	if ((cpu_stdext_feature & CPUID_STDEXT_CLFLUSHOPT) != 0) {
347 		/*
348 		 * Do per-cache line flush.  Use the sfence
349 		 * instruction to insure that previous stores are
350 		 * included in the write-back.  The processor
351 		 * propagates flush to other processors in the cache
352 		 * coherence domain.
353 		 */
354 		sfence();
355 		for (; sva < eva; sva += cpu_clflush_line_size)
356 			clflushopt(sva);
357 		sfence();
358 	} else {
359 		/*
360 		 * Writes are ordered by CLFLUSH on Intel CPUs.
361 		 */
362 		if (cpu_vendor_id != CPU_VENDOR_INTEL)
363 			mfence();
364 		for (; sva < eva; sva += cpu_clflush_line_size)
365 			clflush(sva);
366 		if (cpu_vendor_id != CPU_VENDOR_INTEL)
367 			mfence();
368 	}
369 }
370 
371 static void
372 pmap_invalidate_cache_range_all(vm_offset_t sva, vm_offset_t eva)
373 {
374 
375 	pmap_invalidate_cache_range_check_align(sva, eva);
376 	pmap_invalidate_cache();
377 }
378 
379 void
380 pmap_invalidate_cache_pages(vm_page_t *pages, int count)
381 {
382 	int i;
383 
384 	if (count >= PMAP_CLFLUSH_THRESHOLD / PAGE_SIZE ||
385 	    (cpu_feature & CPUID_CLFSH) == 0) {
386 		pmap_invalidate_cache();
387 	} else {
388 		for (i = 0; i < count; i++)
389 			pmap_flush_page(pages[i]);
390 	}
391 }
392 
393 void
394 pmap_ksetrw(vm_offset_t va)
395 {
396 
397 	pmap_methods_ptr->pm_ksetrw(va);
398 }
399 
400 void
401 pmap_remap_lower(bool enable)
402 {
403 
404 	pmap_methods_ptr->pm_remap_lower(enable);
405 }
406 
407 void
408 pmap_remap_lowptdi(bool enable)
409 {
410 
411 	pmap_methods_ptr->pm_remap_lowptdi(enable);
412 }
413 
414 void
415 pmap_align_superpage(vm_object_t object, vm_ooffset_t offset,
416     vm_offset_t *addr, vm_size_t size)
417 {
418 
419 	return (pmap_methods_ptr->pm_align_superpage(object, offset,
420 	    addr, size));
421 }
422 
423 vm_offset_t
424 pmap_quick_enter_page(vm_page_t m)
425 {
426 
427 	return (pmap_methods_ptr->pm_quick_enter_page(m));
428 }
429 
430 void
431 pmap_quick_remove_page(vm_offset_t addr)
432 {
433 
434 	return (pmap_methods_ptr->pm_quick_remove_page(addr));
435 }
436 
437 void *
438 pmap_trm_alloc(size_t size, int flags)
439 {
440 
441 	return (pmap_methods_ptr->pm_trm_alloc(size, flags));
442 }
443 
444 void
445 pmap_trm_free(void *addr, size_t size)
446 {
447 
448 	pmap_methods_ptr->pm_trm_free(addr, size);
449 }
450 
451 void
452 pmap_sync_icache(pmap_t pm, vm_offset_t va, vm_size_t sz)
453 {
454 }
455 
456 vm_offset_t
457 pmap_get_map_low(void)
458 {
459 
460 	return (pmap_methods_ptr->pm_get_map_low());
461 }
462 
463 vm_offset_t
464 pmap_get_vm_maxuser_address(void)
465 {
466 
467 	return (pmap_methods_ptr->pm_get_vm_maxuser_address());
468 }
469 
470 vm_paddr_t
471 pmap_kextract(vm_offset_t va)
472 {
473 
474 	return (pmap_methods_ptr->pm_kextract(va));
475 }
476 
477 vm_paddr_t
478 pmap_pg_frame(vm_paddr_t pa)
479 {
480 
481 	return (pmap_methods_ptr->pm_pg_frame(pa));
482 }
483 
484 void
485 pmap_sf_buf_map(struct sf_buf *sf)
486 {
487 
488 	pmap_methods_ptr->pm_sf_buf_map(sf);
489 }
490 
491 void
492 pmap_cp_slow0_map(vm_offset_t kaddr, int plen, vm_page_t *ma)
493 {
494 
495 	pmap_methods_ptr->pm_cp_slow0_map(kaddr, plen, ma);
496 }
497 
498 u_int
499 pmap_get_kcr3(void)
500 {
501 
502 	return (pmap_methods_ptr->pm_get_kcr3());
503 }
504 
505 u_int
506 pmap_get_cr3(pmap_t pmap)
507 {
508 
509 	return (pmap_methods_ptr->pm_get_cr3(pmap));
510 }
511 
512 caddr_t
513 pmap_cmap3(vm_paddr_t pa, u_int pte_flags)
514 {
515 
516 	return (pmap_methods_ptr->pm_cmap3(pa, pte_flags));
517 }
518 
519 void
520 pmap_basemem_setup(u_int basemem)
521 {
522 
523 	pmap_methods_ptr->pm_basemem_setup(basemem);
524 }
525 
526 void
527 pmap_set_nx(void)
528 {
529 
530 	pmap_methods_ptr->pm_set_nx();
531 }
532 
533 void *
534 pmap_bios16_enter(void)
535 {
536 
537 	return (pmap_methods_ptr->pm_bios16_enter());
538 }
539 
540 void
541 pmap_bios16_leave(void *handle)
542 {
543 
544 	pmap_methods_ptr->pm_bios16_leave(handle);
545 }
546 
547 void
548 pmap_bootstrap(vm_paddr_t firstaddr)
549 {
550 
551 	pmap_methods_ptr->pm_bootstrap(firstaddr);
552 }
553 
554 boolean_t
555 pmap_is_valid_memattr(pmap_t pmap, vm_memattr_t mode)
556 {
557 
558 	return (pmap_methods_ptr->pm_is_valid_memattr(pmap, mode));
559 }
560 
561 int
562 pmap_cache_bits(pmap_t pmap, int mode, boolean_t is_pde)
563 {
564 
565 	return (pmap_methods_ptr->pm_cache_bits(pmap, mode, is_pde));
566 }
567 
568 bool
569 pmap_ps_enabled(pmap_t pmap)
570 {
571 
572 	return (pmap_methods_ptr->pm_ps_enabled(pmap));
573 }
574 
575 void
576 pmap_pinit0(pmap_t pmap)
577 {
578 
579 	pmap_methods_ptr->pm_pinit0(pmap);
580 }
581 
582 int
583 pmap_pinit(pmap_t pmap)
584 {
585 
586 	return (pmap_methods_ptr->pm_pinit(pmap));
587 }
588 
589 void
590 pmap_activate(struct thread *td)
591 {
592 
593 	pmap_methods_ptr->pm_activate(td);
594 }
595 
596 void
597 pmap_activate_boot(pmap_t pmap)
598 {
599 
600 	pmap_methods_ptr->pm_activate_boot(pmap);
601 }
602 
603 void
604 pmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int advice)
605 {
606 
607 	pmap_methods_ptr->pm_advise(pmap, sva, eva, advice);
608 }
609 
610 void
611 pmap_clear_modify(vm_page_t m)
612 {
613 
614 	pmap_methods_ptr->pm_clear_modify(m);
615 }
616 
617 int
618 pmap_change_attr(vm_offset_t va, vm_size_t size, int mode)
619 {
620 
621 	return (pmap_methods_ptr->pm_change_attr(va, size, mode));
622 }
623 
624 int
625 pmap_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *locked_pa)
626 {
627 
628 	return (pmap_methods_ptr->pm_mincore(pmap, addr, locked_pa));
629 }
630 
631 void
632 pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, vm_size_t len,
633     vm_offset_t src_addr)
634 {
635 
636 	pmap_methods_ptr->pm_copy(dst_pmap, src_pmap, dst_addr, len, src_addr);
637 }
638 
639 void
640 pmap_copy_page(vm_page_t src, vm_page_t dst)
641 {
642 
643 	pmap_methods_ptr->pm_copy_page(src, dst);
644 }
645 
646 void
647 pmap_copy_pages(vm_page_t ma[], vm_offset_t a_offset, vm_page_t mb[],
648     vm_offset_t b_offset, int xfersize)
649 {
650 
651 	pmap_methods_ptr->pm_copy_pages(ma, a_offset, mb, b_offset, xfersize);
652 }
653 
654 void
655 pmap_zero_page(vm_page_t m)
656 {
657 
658 	pmap_methods_ptr->pm_zero_page(m);
659 }
660 
661 void
662 pmap_zero_page_area(vm_page_t m, int off, int size)
663 {
664 
665 	pmap_methods_ptr->pm_zero_page_area(m, off, size);
666 }
667 
668 int
669 pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,
670     u_int flags, int8_t psind)
671 {
672 
673 	return (pmap_methods_ptr->pm_enter(pmap, va, m, prot, flags, psind));
674 }
675 
676 void
677 pmap_enter_object(pmap_t pmap, vm_offset_t start, vm_offset_t end,
678     vm_page_t m_start, vm_prot_t prot)
679 {
680 
681 	pmap_methods_ptr->pm_enter_object(pmap, start, end, m_start, prot);
682 }
683 
684 void
685 pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot)
686 {
687 
688 	pmap_methods_ptr->pm_enter_quick(pmap, va, m, prot);
689 }
690 
691 void *
692 pmap_kenter_temporary(vm_paddr_t pa, int i)
693 {
694 
695 	return (pmap_methods_ptr->pm_kenter_temporary(pa, i));
696 }
697 
698 void
699 pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, vm_object_t object,
700     vm_pindex_t pindex, vm_size_t size)
701 {
702 
703 	pmap_methods_ptr->pm_object_init_pt(pmap, addr, object, pindex, size);
704 }
705 
706 void
707 pmap_unwire(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
708 {
709 
710 	pmap_methods_ptr->pm_unwire(pmap, sva, eva);
711 }
712 
713 boolean_t
714 pmap_page_exists_quick(pmap_t pmap, vm_page_t m)
715 {
716 
717 	return (pmap_methods_ptr->pm_page_exists_quick(pmap, m));
718 }
719 
720 int
721 pmap_page_wired_mappings(vm_page_t m)
722 {
723 
724 	return (pmap_methods_ptr->pm_page_wired_mappings(m));
725 }
726 
727 boolean_t
728 pmap_page_is_mapped(vm_page_t m)
729 {
730 
731 	return (pmap_methods_ptr->pm_page_is_mapped(m));
732 }
733 
734 void
735 pmap_remove_pages(pmap_t pmap)
736 {
737 
738 	pmap_methods_ptr->pm_remove_pages(pmap);
739 }
740 
741 boolean_t
742 pmap_is_modified(vm_page_t m)
743 {
744 
745 	return (pmap_methods_ptr->pm_is_modified(m));
746 }
747 
748 boolean_t
749 pmap_is_prefaultable(pmap_t pmap, vm_offset_t addr)
750 {
751 
752 	return (pmap_methods_ptr->pm_is_prefaultable(pmap, addr));
753 }
754 
755 boolean_t
756 pmap_is_referenced(vm_page_t m)
757 {
758 
759 	return (pmap_methods_ptr->pm_is_referenced(m));
760 }
761 
762 void
763 pmap_remove_write(vm_page_t m)
764 {
765 
766 	pmap_methods_ptr->pm_remove_write(m);
767 }
768 
769 int
770 pmap_ts_referenced(vm_page_t m)
771 {
772 
773 	return (pmap_methods_ptr->pm_ts_referenced(m));
774 }
775 
776 void *
777 pmap_mapdev_attr(vm_paddr_t pa, vm_size_t size, int mode)
778 {
779 
780 	return (pmap_methods_ptr->pm_mapdev_attr(pa, size, mode));
781 }
782 
783 void *
784 pmap_mapdev(vm_paddr_t pa, vm_size_t size)
785 {
786 
787 	return (pmap_methods_ptr->pm_mapdev_attr(pa, size, PAT_UNCACHEABLE));
788 }
789 
790 void *
791 pmap_mapbios(vm_paddr_t pa, vm_size_t size)
792 {
793 
794 	return (pmap_methods_ptr->pm_mapdev_attr(pa, size, PAT_WRITE_BACK));
795 }
796 
797 void
798 pmap_unmapdev(vm_offset_t va, vm_size_t size)
799 {
800 
801 	pmap_methods_ptr->pm_unmapdev(va, size);
802 }
803 
804 void
805 pmap_page_set_memattr(vm_page_t m, vm_memattr_t ma)
806 {
807 
808 	pmap_methods_ptr->pm_page_set_memattr(m, ma);
809 }
810 
811 vm_paddr_t
812 pmap_extract(pmap_t pmap, vm_offset_t va)
813 {
814 
815 	return (pmap_methods_ptr->pm_extract(pmap, va));
816 }
817 
818 vm_page_t
819 pmap_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot)
820 {
821 
822 	return (pmap_methods_ptr->pm_extract_and_hold(pmap, va, prot));
823 }
824 
825 vm_offset_t
826 pmap_map(vm_offset_t *virt, vm_paddr_t start, vm_paddr_t end, int prot)
827 {
828 
829 	return (pmap_methods_ptr->pm_map(virt, start, end, prot));
830 }
831 
832 void
833 pmap_qenter(vm_offset_t sva, vm_page_t *ma, int count)
834 {
835 
836 	pmap_methods_ptr->pm_qenter(sva, ma, count);
837 }
838 
839 void
840 pmap_qremove(vm_offset_t sva, int count)
841 {
842 
843 	pmap_methods_ptr->pm_qremove(sva, count);
844 }
845 
846 void
847 pmap_release(pmap_t pmap)
848 {
849 
850 	pmap_methods_ptr->pm_release(pmap);
851 }
852 
853 void
854 pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
855 {
856 
857 	pmap_methods_ptr->pm_remove(pmap, sva, eva);
858 }
859 
860 void
861 pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot)
862 {
863 
864 	pmap_methods_ptr->pm_protect(pmap, sva, eva, prot);
865 }
866 
867 void
868 pmap_remove_all(vm_page_t m)
869 {
870 
871 	pmap_methods_ptr->pm_remove_all(m);
872 }
873 
874 void
875 pmap_init(void)
876 {
877 
878 	pmap_methods_ptr->pm_init();
879 }
880 
881 void
882 pmap_init_pat(void)
883 {
884 
885 	pmap_methods_ptr->pm_init_pat();
886 }
887 
888 void
889 pmap_growkernel(vm_offset_t addr)
890 {
891 
892 	pmap_methods_ptr->pm_growkernel(addr);
893 }
894 
895 void
896 pmap_invalidate_page(pmap_t pmap, vm_offset_t va)
897 {
898 
899 	pmap_methods_ptr->pm_invalidate_page(pmap, va);
900 }
901 
902 void
903 pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
904 {
905 
906 	pmap_methods_ptr->pm_invalidate_range(pmap, sva, eva);
907 }
908 
909 void
910 pmap_invalidate_all(pmap_t pmap)
911 {
912 
913 	pmap_methods_ptr->pm_invalidate_all(pmap);
914 }
915 
916 void
917 pmap_invalidate_cache(void)
918 {
919 
920 	pmap_methods_ptr->pm_invalidate_cache();
921 }
922 
923 void
924 pmap_kenter(vm_offset_t va, vm_paddr_t pa)
925 {
926 
927 	pmap_methods_ptr->pm_kenter(va, pa);
928 }
929 
930 void
931 pmap_kremove(vm_offset_t va)
932 {
933 
934 	pmap_methods_ptr->pm_kremove(va);
935 }
936 
937 extern struct pmap_methods pmap_pae_methods, pmap_nopae_methods;
938 int pae_mode;
939 SYSCTL_INT(_vm_pmap, OID_AUTO, pae_mode, CTLFLAG_RDTUN | CTLFLAG_NOFETCH,
940     &pae_mode, 0,
941     "PAE");
942 
943 void
944 pmap_cold(void)
945 {
946 
947 	init_static_kenv((char *)bootinfo.bi_envp, 0);
948 	pae_mode = (cpu_feature & CPUID_PAE) != 0;
949 	if (pae_mode)
950 		TUNABLE_INT_FETCH("vm.pmap.pae_mode", &pae_mode);
951 	if (pae_mode) {
952 		pmap_methods_ptr = &pmap_pae_methods;
953 		pmap_pae_cold();
954 	} else {
955 		pmap_methods_ptr = &pmap_nopae_methods;
956 		pmap_nopae_cold();
957 	}
958 }
959