1 /*-
2 * SPDX-License-Identifier: BSD-4-Clause
3 *
4 * Copyright (c) 1991 Regents of the University of California.
5 * All rights reserved.
6 * Copyright (c) 1994 John S. Dyson
7 * All rights reserved.
8 * Copyright (c) 1994 David Greenman
9 * All rights reserved.
10 * Copyright (c) 2005-2010 Alan L. Cox <alc@cs.rice.edu>
11 * All rights reserved.
12 *
13 * This code is derived from software contributed to Berkeley by
14 * the Systems Programming Group of the University of Utah Computer
15 * Science Department and William Jolitz of UUNET Technologies Inc.
16 *
17 * Redistribution and use in source and binary forms, with or without
18 * modification, are permitted provided that the following conditions
19 * are met:
20 * 1. Redistributions of source code must retain the above copyright
21 * notice, this list of conditions and the following disclaimer.
22 * 2. Redistributions in binary form must reproduce the above copyright
23 * notice, this list of conditions and the following disclaimer in the
24 * documentation and/or other materials provided with the distribution.
25 * 3. All advertising materials mentioning features or use of this software
26 * must display the following acknowledgement:
27 * This product includes software developed by the University of
28 * California, Berkeley and its contributors.
29 * 4. Neither the name of the University nor the names of its contributors
30 * may be used to endorse or promote products derived from this software
31 * without specific prior written permission.
32 *
33 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
34 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
35 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
36 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
37 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
38 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
39 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
40 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
41 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
42 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
43 * SUCH DAMAGE.
44 */
45 /*-
46 * Copyright (c) 2003 Networks Associates Technology, Inc.
47 * All rights reserved.
48 * Copyright (c) 2018 The FreeBSD Foundation
49 * All rights reserved.
50 *
51 * This software was developed for the FreeBSD Project by Jake Burkholder,
52 * Safeport Network Services, and Network Associates Laboratories, the
53 * Security Research Division of Network Associates, Inc. under
54 * DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"), as part of the DARPA
55 * CHATS research program.
56 *
57 * Portions of this software were developed by
58 * Konstantin Belousov <kib@FreeBSD.org> under sponsorship from
59 * the FreeBSD Foundation.
60 *
61 * Redistribution and use in source and binary forms, with or without
62 * modification, are permitted provided that the following conditions
63 * are met:
64 * 1. Redistributions of source code must retain the above copyright
65 * notice, this list of conditions and the following disclaimer.
66 * 2. Redistributions in binary form must reproduce the above copyright
67 * notice, this list of conditions and the following disclaimer in the
68 * documentation and/or other materials provided with the distribution.
69 *
70 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
71 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
72 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
73 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
74 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
75 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
76 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
77 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
78 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
79 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
80 * SUCH DAMAGE.
81 */
82
83 #include <sys/cdefs.h>
84 /*
85 * Manages physical address maps.
86 *
87 * Since the information managed by this module is
88 * also stored by the logical address mapping module,
89 * this module may throw away valid virtual-to-physical
90 * mappings at almost any time. However, invalidations
91 * of virtual-to-physical mappings must be done as
92 * requested.
93 *
94 * In order to cope with hardware architectures which
95 * make virtual-to-physical map invalidates expensive,
96 * this module may delay invalidate or reduced protection
97 * operations until such time as they are actually
98 * necessary. This module is given full information as
99 * to which processors are currently using which maps,
100 * and to when physical maps must be made correct.
101 */
102
103 #include "opt_apic.h"
104 #include "opt_cpu.h"
105 #include "opt_pmap.h"
106 #include "opt_smp.h"
107 #include "opt_vm.h"
108
109 #include <sys/param.h>
110 #include <sys/systm.h>
111 #include <sys/kernel.h>
112 #include <sys/ktr.h>
113 #include <sys/lock.h>
114 #include <sys/malloc.h>
115 #include <sys/mman.h>
116 #include <sys/msgbuf.h>
117 #include <sys/mutex.h>
118 #include <sys/proc.h>
119 #include <sys/rwlock.h>
120 #include <sys/sbuf.h>
121 #include <sys/sf_buf.h>
122 #include <sys/sx.h>
123 #include <sys/vmmeter.h>
124 #include <sys/sched.h>
125 #include <sys/sysctl.h>
126 #include <sys/smp.h>
127 #include <sys/vmem.h>
128
129 #include <vm/vm.h>
130 #include <vm/vm_param.h>
131 #include <vm/vm_kern.h>
132 #include <vm/vm_page.h>
133 #include <vm/vm_map.h>
134 #include <vm/vm_object.h>
135 #include <vm/vm_extern.h>
136 #include <vm/vm_pageout.h>
137 #include <vm/vm_pager.h>
138 #include <vm/vm_phys.h>
139 #include <vm/vm_radix.h>
140 #include <vm/vm_reserv.h>
141 #include <vm/uma.h>
142
143 #ifdef DEV_APIC
144 #include <sys/bus.h>
145 #include <machine/intr_machdep.h>
146 #include <x86/apicvar.h>
147 #endif
148 #include <x86/ifunc.h>
149 #include <machine/bootinfo.h>
150 #include <machine/cpu.h>
151 #include <machine/cputypes.h>
152 #include <machine/md_var.h>
153 #include <machine/pcb.h>
154 #include <machine/specialreg.h>
155 #ifdef SMP
156 #include <machine/smp.h>
157 #endif
158 #include <machine/pmap_base.h>
159
160 #ifdef PV_STATS
161 #define PV_STAT(x) do { x ; } while (0)
162 #else
163 #define PV_STAT(x) do { } while (0)
164 #endif
165
166 #define pa_index(pa) ((pa) >> PDRSHIFT)
167 #define pa_to_pvh(pa) (&pv_table[pa_index(pa)])
168
169 /*
170 * PTmap is recursive pagemap at top of virtual address space.
171 * Within PTmap, the page directory can be found (third indirection).
172 */
173 #define PTmap ((pt_entry_t *)(PTDPTDI << PDRSHIFT))
174 #define PTD ((pd_entry_t *)((PTDPTDI << PDRSHIFT) + (PTDPTDI * PAGE_SIZE)))
175 #define PTDpde ((pd_entry_t *)((PTDPTDI << PDRSHIFT) + (PTDPTDI * PAGE_SIZE) + \
176 (PTDPTDI * PDESIZE)))
177
178 /*
179 * Translate a virtual address to the kernel virtual address of its page table
180 * entry (PTE). This can be used recursively. If the address of a PTE as
181 * previously returned by this macro is itself given as the argument, then the
182 * address of the page directory entry (PDE) that maps the PTE will be
183 * returned.
184 *
185 * This macro may be used before pmap_bootstrap() is called.
186 */
187 #define vtopte(va) (PTmap + i386_btop(va))
188
189 /*
190 * Get PDEs and PTEs for user/kernel address space
191 */
192 #define pmap_pde(m, v) (&((m)->pm_pdir[(vm_offset_t)(v) >> PDRSHIFT]))
193 #define pdir_pde(m, v) (m[(vm_offset_t)(v) >> PDRSHIFT])
194
195 #define pmap_pde_v(pte) ((*(int *)pte & PG_V) != 0)
196 #define pmap_pte_w(pte) ((*(int *)pte & PG_W) != 0)
197 #define pmap_pte_m(pte) ((*(int *)pte & PG_M) != 0)
198 #define pmap_pte_u(pte) ((*(int *)pte & PG_A) != 0)
199 #define pmap_pte_v(pte) ((*(int *)pte & PG_V) != 0)
200
201 #define pmap_pte_set_w(pte, v) ((v) ? atomic_set_int((u_int *)(pte), PG_W) : \
202 atomic_clear_int((u_int *)(pte), PG_W))
203 #define pmap_pte_set_prot(pte, v) ((*(int *)pte &= ~PG_PROT), (*(int *)pte |= (v)))
204
205 static int pgeflag = 0; /* PG_G or-in */
206 static int pseflag = 0; /* PG_PS or-in */
207
208 static int nkpt = NKPT;
209
210 #ifdef PMAP_PAE_COMP
211 pt_entry_t pg_nx;
212 static uma_zone_t pdptzone;
213 #else
214 #define pg_nx 0
215 #endif
216
217 _Static_assert(VM_MAXUSER_ADDRESS == VADDR(TRPTDI, 0), "VM_MAXUSER_ADDRESS");
218 _Static_assert(VM_MAX_KERNEL_ADDRESS <= VADDR(PTDPTDI, 0),
219 "VM_MAX_KERNEL_ADDRESS");
220 _Static_assert(PMAP_MAP_LOW == VADDR(LOWPTDI, 0), "PMAP_MAP_LOW");
221 _Static_assert(KERNLOAD == (KERNPTDI << PDRSHIFT), "KERNLOAD");
222
223 extern int pat_works;
224 extern int pg_ps_enabled;
225
226 extern int elf32_nxstack;
227
228 #define PAT_INDEX_SIZE 8
229 static int pat_index[PAT_INDEX_SIZE]; /* cache mode to PAT index conversion */
230
231 /*
232 * pmap_mapdev support pre initialization (i.e. console)
233 */
234 #define PMAP_PREINIT_MAPPING_COUNT 8
235 static struct pmap_preinit_mapping {
236 vm_paddr_t pa;
237 vm_offset_t va;
238 vm_size_t sz;
239 int mode;
240 } pmap_preinit_mapping[PMAP_PREINIT_MAPPING_COUNT];
241 static int pmap_initialized;
242
243 static struct rwlock_padalign pvh_global_lock;
244
245 /*
246 * Data for the pv entry allocation mechanism
247 */
248 static TAILQ_HEAD(pch, pv_chunk) pv_chunks = TAILQ_HEAD_INITIALIZER(pv_chunks);
249 extern int pv_entry_max, pv_entry_count;
250 static int pv_entry_high_water = 0;
251 static struct md_page *pv_table;
252 extern int shpgperproc;
253
254 static struct pv_chunk *pv_chunkbase; /* KVA block for pv_chunks */
255 static int pv_maxchunks; /* How many chunks we have KVA for */
256 static vm_offset_t pv_vafree; /* freelist stored in the PTE */
257
258 /*
259 * All those kernel PT submaps that BSD is so fond of
260 */
261 static pt_entry_t *CMAP3;
262 static pd_entry_t *KPTD;
263 static caddr_t CADDR3;
264
265 /*
266 * Crashdump maps.
267 */
268 static caddr_t crashdumpmap;
269
270 static pt_entry_t *PMAP1 = NULL, *PMAP2, *PMAP3;
271 static pt_entry_t *PADDR1 = NULL, *PADDR2, *PADDR3;
272 #ifdef SMP
273 static int PMAP1cpu, PMAP3cpu;
274 extern int PMAP1changedcpu;
275 #endif
276 extern int PMAP1changed;
277 extern int PMAP1unchanged;
278 static struct mtx PMAP2mutex;
279
280 /*
281 * Internal flags for pmap_enter()'s helper functions.
282 */
283 #define PMAP_ENTER_NORECLAIM 0x1000000 /* Don't reclaim PV entries. */
284 #define PMAP_ENTER_NOREPLACE 0x2000000 /* Don't replace mappings. */
285
286 static void free_pv_chunk(struct pv_chunk *pc);
287 static void free_pv_entry(pmap_t pmap, pv_entry_t pv);
288 static pv_entry_t get_pv_entry(pmap_t pmap, bool try);
289 static void pmap_pv_demote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa);
290 static bool pmap_pv_insert_pde(pmap_t pmap, vm_offset_t va, pd_entry_t pde,
291 u_int flags);
292 #if VM_NRESERVLEVEL > 0
293 static void pmap_pv_promote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa);
294 #endif
295 static void pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va);
296 static pv_entry_t pmap_pvh_remove(struct md_page *pvh, pmap_t pmap,
297 vm_offset_t va);
298 static int pmap_pvh_wired_mappings(struct md_page *pvh, int count);
299
300 static void pmap_abort_ptp(pmap_t pmap, vm_offset_t va, vm_page_t mpte);
301 static bool pmap_demote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va);
302 static int pmap_enter_4mpage(pmap_t pmap, vm_offset_t va, vm_page_t m,
303 vm_prot_t prot);
304 static int pmap_enter_pde(pmap_t pmap, vm_offset_t va, pd_entry_t newpde,
305 u_int flags, vm_page_t m);
306 static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va,
307 vm_page_t m, vm_prot_t prot, vm_page_t mpte);
308 static int pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte, bool promoted,
309 bool allpte_PG_A_set);
310 static void pmap_invalidate_pde_page(pmap_t pmap, vm_offset_t va,
311 pd_entry_t pde);
312 static void pmap_fill_ptp(pt_entry_t *firstpte, pt_entry_t newpte);
313 static bool pmap_is_modified_pvh(struct md_page *pvh);
314 static bool pmap_is_referenced_pvh(struct md_page *pvh);
315 static void pmap_kenter_attr(vm_offset_t va, vm_paddr_t pa, int mode);
316 static void pmap_kenter_pde(vm_offset_t va, pd_entry_t newpde);
317 static void pmap_pde_attr(pd_entry_t *pde, int cache_bits);
318 #if VM_NRESERVLEVEL > 0
319 static bool pmap_promote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va,
320 vm_page_t mpte);
321 #endif
322 static bool pmap_protect_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t sva,
323 vm_prot_t prot);
324 static void pmap_pte_attr(pt_entry_t *pte, int cache_bits);
325 static void pmap_remove_pde(pmap_t pmap, pd_entry_t *pdq, vm_offset_t sva,
326 struct spglist *free);
327 static int pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t sva,
328 struct spglist *free);
329 static vm_page_t pmap_remove_pt_page(pmap_t pmap, vm_offset_t va);
330 static void pmap_remove_page(pmap_t pmap, vm_offset_t va, struct spglist *free);
331 static bool pmap_remove_ptes(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,
332 struct spglist *free);
333 static void pmap_remove_entry(pmap_t pmap, vm_page_t m, vm_offset_t va);
334 static void pmap_insert_entry(pmap_t pmap, vm_offset_t va, vm_page_t m);
335 static bool pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va,
336 vm_page_t m);
337 static void pmap_update_pde(pmap_t pmap, vm_offset_t va, pd_entry_t *pde,
338 pd_entry_t newpde);
339 static void pmap_update_pde_invalidate(vm_offset_t va, pd_entry_t newpde);
340
341 static vm_page_t pmap_allocpte(pmap_t pmap, vm_offset_t va, u_int flags);
342
343 static vm_page_t _pmap_allocpte(pmap_t pmap, u_int ptepindex, u_int flags);
344 static void _pmap_unwire_ptp(pmap_t pmap, vm_page_t m, struct spglist *free);
345 static pt_entry_t *pmap_pte_quick(pmap_t pmap, vm_offset_t va);
346 static void pmap_pte_release(pt_entry_t *pte);
347 static int pmap_unuse_pt(pmap_t, vm_offset_t, struct spglist *);
348 #ifdef PMAP_PAE_COMP
349 static void *pmap_pdpt_allocf(uma_zone_t zone, vm_size_t bytes, int domain,
350 uint8_t *flags, int wait);
351 #endif
352 static void pmap_init_trm(void);
353 static void pmap_invalidate_all_int(pmap_t pmap);
354
355 static __inline void pagezero(void *page);
356
357 CTASSERT(1 << PDESHIFT == sizeof(pd_entry_t));
358 CTASSERT(1 << PTESHIFT == sizeof(pt_entry_t));
359
360 extern char _end[];
361 extern u_long physfree; /* phys addr of next free page */
362 extern u_long vm86phystk;/* PA of vm86/bios stack */
363 extern u_long vm86paddr;/* address of vm86 region */
364 extern int vm86pa; /* phys addr of vm86 region */
365 extern u_long KERNend; /* phys addr end of kernel (just after bss) */
366 #ifdef PMAP_PAE_COMP
367 pd_entry_t *IdlePTD_pae; /* phys addr of kernel PTD */
368 pdpt_entry_t *IdlePDPT; /* phys addr of kernel PDPT */
369 pt_entry_t *KPTmap_pae; /* address of kernel page tables */
370 #define IdlePTD IdlePTD_pae
371 #define KPTmap KPTmap_pae
372 #else
373 pd_entry_t *IdlePTD_nopae;
374 pt_entry_t *KPTmap_nopae;
375 #define IdlePTD IdlePTD_nopae
376 #define KPTmap KPTmap_nopae
377 #endif
378 extern u_long KPTphys; /* phys addr of kernel page tables */
379 extern u_long tramp_idleptd;
380
381 static u_long
allocpages(u_int cnt,u_long * physfree)382 allocpages(u_int cnt, u_long *physfree)
383 {
384 u_long res;
385
386 res = *physfree;
387 *physfree += PAGE_SIZE * cnt;
388 bzero((void *)res, PAGE_SIZE * cnt);
389 return (res);
390 }
391
392 static void
pmap_cold_map(u_long pa,u_long va,u_long cnt)393 pmap_cold_map(u_long pa, u_long va, u_long cnt)
394 {
395 pt_entry_t *pt;
396
397 for (pt = (pt_entry_t *)KPTphys + atop(va); cnt > 0;
398 cnt--, pt++, va += PAGE_SIZE, pa += PAGE_SIZE)
399 *pt = pa | PG_V | PG_RW | PG_A | PG_M;
400 }
401
402 static void
pmap_cold_mapident(u_long pa,u_long cnt)403 pmap_cold_mapident(u_long pa, u_long cnt)
404 {
405
406 pmap_cold_map(pa, pa, cnt);
407 }
408
409 _Static_assert(LOWPTDI * 2 * NBPDR == KERNBASE,
410 "Broken double-map of zero PTD");
411
412 static void
__CONCAT(PMTYPE,remap_lower)413 __CONCAT(PMTYPE, remap_lower)(bool enable)
414 {
415 int i;
416
417 for (i = 0; i < LOWPTDI; i++)
418 IdlePTD[i] = enable ? IdlePTD[LOWPTDI + i] : 0;
419 load_cr3(rcr3()); /* invalidate TLB */
420 }
421
422 /*
423 * Called from locore.s before paging is enabled. Sets up the first
424 * kernel page table. Since kernel is mapped with PA == VA, this code
425 * does not require relocations.
426 */
427 void
__CONCAT(PMTYPE,cold)428 __CONCAT(PMTYPE, cold)(void)
429 {
430 pt_entry_t *pt;
431 u_long a;
432 u_int cr3, ncr4;
433
434 physfree = (u_long)&_end;
435 if (bootinfo.bi_esymtab != 0)
436 physfree = bootinfo.bi_esymtab;
437 if (bootinfo.bi_kernend != 0)
438 physfree = bootinfo.bi_kernend;
439 physfree = roundup2(physfree, NBPDR);
440 KERNend = physfree;
441
442 /* Allocate Kernel Page Tables */
443 KPTphys = allocpages(NKPT, &physfree);
444 KPTmap = (pt_entry_t *)KPTphys;
445
446 /* Allocate Page Table Directory */
447 #ifdef PMAP_PAE_COMP
448 /* XXX only need 32 bytes (easier for now) */
449 IdlePDPT = (pdpt_entry_t *)allocpages(1, &physfree);
450 #endif
451 IdlePTD = (pd_entry_t *)allocpages(NPGPTD, &physfree);
452
453 /*
454 * Allocate KSTACK. Leave a guard page between IdlePTD and
455 * proc0kstack, to control stack overflow for thread0 and
456 * prevent corruption of the page table. We leak the guard
457 * physical memory due to 1:1 mappings.
458 */
459 allocpages(1, &physfree);
460 proc0kstack = allocpages(TD0_KSTACK_PAGES, &physfree);
461
462 /* vm86/bios stack */
463 vm86phystk = allocpages(1, &physfree);
464
465 /* pgtable + ext + IOPAGES */
466 vm86paddr = vm86pa = allocpages(3, &physfree);
467
468 /* Install page tables into PTD. Page table page 1 is wasted. */
469 for (a = 0; a < NKPT; a++)
470 IdlePTD[a] = (KPTphys + ptoa(a)) | PG_V | PG_RW | PG_A | PG_M;
471
472 #ifdef PMAP_PAE_COMP
473 /* PAE install PTD pointers into PDPT */
474 for (a = 0; a < NPGPTD; a++)
475 IdlePDPT[a] = ((u_int)IdlePTD + ptoa(a)) | PG_V;
476 #endif
477
478 /*
479 * Install recursive mapping for kernel page tables into
480 * itself.
481 */
482 for (a = 0; a < NPGPTD; a++)
483 IdlePTD[PTDPTDI + a] = ((u_int)IdlePTD + ptoa(a)) | PG_V |
484 PG_RW;
485
486 /*
487 * Initialize page table pages mapping physical address zero
488 * through the (physical) end of the kernel. Many of these
489 * pages must be reserved, and we reserve them all and map
490 * them linearly for convenience. We do this even if we've
491 * enabled PSE above; we'll just switch the corresponding
492 * kernel PDEs before we turn on paging.
493 *
494 * This and all other page table entries allow read and write
495 * access for various reasons. Kernel mappings never have any
496 * access restrictions.
497 */
498 pmap_cold_mapident(0, atop(NBPDR) * LOWPTDI);
499 pmap_cold_map(0, NBPDR * LOWPTDI, atop(NBPDR) * LOWPTDI);
500 pmap_cold_mapident(KERNBASE, atop(KERNend - KERNBASE));
501
502 /* Map page table directory */
503 #ifdef PMAP_PAE_COMP
504 pmap_cold_mapident((u_long)IdlePDPT, 1);
505 #endif
506 pmap_cold_mapident((u_long)IdlePTD, NPGPTD);
507
508 /* Map early KPTmap. It is really pmap_cold_mapident. */
509 pmap_cold_map(KPTphys, (u_long)KPTmap, NKPT);
510
511 /* Map proc0kstack */
512 pmap_cold_mapident(proc0kstack, TD0_KSTACK_PAGES);
513 /* ISA hole already mapped */
514
515 pmap_cold_mapident(vm86phystk, 1);
516 pmap_cold_mapident(vm86pa, 3);
517
518 /* Map page 0 into the vm86 page table */
519 *(pt_entry_t *)vm86pa = 0 | PG_RW | PG_U | PG_A | PG_M | PG_V;
520
521 /* ...likewise for the ISA hole for vm86 */
522 for (pt = (pt_entry_t *)vm86pa + atop(ISA_HOLE_START), a = 0;
523 a < atop(ISA_HOLE_LENGTH); a++, pt++)
524 *pt = (ISA_HOLE_START + ptoa(a)) | PG_RW | PG_U | PG_A |
525 PG_M | PG_V;
526
527 /* Enable PSE, PGE, VME, and PAE if configured. */
528 ncr4 = 0;
529 if ((cpu_feature & CPUID_PSE) != 0) {
530 ncr4 |= CR4_PSE;
531 pseflag = PG_PS;
532 /*
533 * Superpage mapping of the kernel text. Existing 4k
534 * page table pages are wasted.
535 */
536 for (a = KERNBASE; a < KERNend; a += NBPDR)
537 IdlePTD[a >> PDRSHIFT] = a | PG_PS | PG_A | PG_M |
538 PG_RW | PG_V;
539 }
540 if ((cpu_feature & CPUID_PGE) != 0) {
541 ncr4 |= CR4_PGE;
542 pgeflag = PG_G;
543 }
544 ncr4 |= (cpu_feature & CPUID_VME) != 0 ? CR4_VME : 0;
545 #ifdef PMAP_PAE_COMP
546 ncr4 |= CR4_PAE;
547 #endif
548 if (ncr4 != 0)
549 load_cr4(rcr4() | ncr4);
550
551 /* Now enable paging */
552 #ifdef PMAP_PAE_COMP
553 cr3 = (u_int)IdlePDPT;
554 if ((cpu_feature & CPUID_PAT) == 0)
555 wbinvd();
556 #else
557 cr3 = (u_int)IdlePTD;
558 #endif
559 tramp_idleptd = cr3;
560 load_cr3(cr3);
561 load_cr0(rcr0() | CR0_PG);
562
563 /*
564 * Now running relocated at KERNBASE where the system is
565 * linked to run.
566 */
567
568 /*
569 * Remove the lowest part of the double mapping of low memory
570 * to get some null pointer checks.
571 */
572 __CONCAT(PMTYPE, remap_lower)(false);
573
574 kernel_vm_end = /* 0 + */ NKPT * NBPDR;
575 #ifdef PMAP_PAE_COMP
576 i386_pmap_VM_NFREEORDER = VM_NFREEORDER_PAE;
577 i386_pmap_VM_LEVEL_0_ORDER = VM_LEVEL_0_ORDER_PAE;
578 i386_pmap_PDRSHIFT = PDRSHIFT_PAE;
579 #else
580 i386_pmap_VM_NFREEORDER = VM_NFREEORDER_NOPAE;
581 i386_pmap_VM_LEVEL_0_ORDER = VM_LEVEL_0_ORDER_NOPAE;
582 i386_pmap_PDRSHIFT = PDRSHIFT_NOPAE;
583 #endif
584 }
585
586 static void
__CONCAT(PMTYPE,set_nx)587 __CONCAT(PMTYPE, set_nx)(void)
588 {
589
590 #ifdef PMAP_PAE_COMP
591 if ((amd_feature & AMDID_NX) == 0)
592 return;
593 pg_nx = PG_NX;
594 elf32_nxstack = 1;
595 /* EFER.EFER_NXE is set in initializecpu(). */
596 #endif
597 }
598
599 /*
600 * Bootstrap the system enough to run with virtual memory.
601 *
602 * On the i386 this is called after pmap_cold() created initial
603 * kernel page table and enabled paging, and just syncs the pmap
604 * module with what has already been done.
605 */
606 static void
__CONCAT(PMTYPE,bootstrap)607 __CONCAT(PMTYPE, bootstrap)(vm_paddr_t firstaddr)
608 {
609 vm_offset_t va;
610 pt_entry_t *pte, *unused __unused;
611 struct pcpu *pc;
612 u_long res;
613 int i;
614
615 res = atop(firstaddr - (vm_paddr_t)KERNLOAD);
616
617 /*
618 * Initialize the first available kernel virtual address.
619 * However, using "firstaddr" may waste a few pages of the
620 * kernel virtual address space, because pmap_cold() may not
621 * have mapped every physical page that it allocated.
622 * Preferably, pmap_cold() would provide a first unused
623 * virtual address in addition to "firstaddr".
624 */
625 virtual_avail = (vm_offset_t)firstaddr;
626 virtual_end = VM_MAX_KERNEL_ADDRESS;
627
628 /*
629 * Initialize the kernel pmap (which is statically allocated).
630 * Count bootstrap data as being resident in case any of this data is
631 * later unmapped (using pmap_remove()) and freed.
632 */
633 mtx_init(&kernel_pmap->pm_mtx, "kernel pmap", NULL, MTX_DEF);
634 kernel_pmap->pm_pdir = IdlePTD;
635 #ifdef PMAP_PAE_COMP
636 kernel_pmap->pm_pdpt = IdlePDPT;
637 #endif
638 CPU_FILL(&kernel_pmap->pm_active); /* don't allow deactivation */
639 kernel_pmap->pm_stats.resident_count = res;
640 TAILQ_INIT(&kernel_pmap->pm_pvchunk);
641 vm_radix_init(&kernel_pmap->pm_root);
642
643 /*
644 * Initialize the global pv list lock.
645 */
646 rw_init(&pvh_global_lock, "pmap pv global");
647
648 /*
649 * Reserve some special page table entries/VA space for temporary
650 * mapping of pages.
651 */
652 #define SYSMAP(c, p, v, n) \
653 v = (c)va; va += ((n)*PAGE_SIZE); p = pte; pte += (n);
654
655 va = virtual_avail;
656 pte = vtopte(va);
657
658 /*
659 * Initialize temporary map objects on the current CPU for use
660 * during early boot.
661 * CMAP1/CMAP2 are used for zeroing and copying pages.
662 * CMAP3 is used for the boot-time memory test.
663 */
664 pc = get_pcpu();
665 mtx_init(&pc->pc_cmap_lock, "SYSMAPS", NULL, MTX_DEF);
666 SYSMAP(caddr_t, pc->pc_cmap_pte1, pc->pc_cmap_addr1, 1)
667 SYSMAP(caddr_t, pc->pc_cmap_pte2, pc->pc_cmap_addr2, 1)
668 SYSMAP(caddr_t, pte, pc->pc_qmap_addr, 1)
669
670 SYSMAP(caddr_t, CMAP3, CADDR3, 1);
671
672 /*
673 * Crashdump maps.
674 */
675 SYSMAP(caddr_t, unused, crashdumpmap, MAXDUMPPGS)
676
677 /*
678 * ptvmmap is used for reading arbitrary physical pages via /dev/mem.
679 */
680 SYSMAP(caddr_t, unused, ptvmmap, 1)
681
682 /*
683 * msgbufp is used to map the system message buffer.
684 */
685 SYSMAP(struct msgbuf *, unused, msgbufp, atop(round_page(msgbufsize)))
686
687 /*
688 * KPTmap is used by pmap_kextract().
689 *
690 * KPTmap is first initialized by pmap_cold(). However, that initial
691 * KPTmap can only support NKPT page table pages. Here, a larger
692 * KPTmap is created that can support KVA_PAGES page table pages.
693 */
694 SYSMAP(pt_entry_t *, KPTD, KPTmap, KVA_PAGES)
695
696 for (i = 0; i < NKPT; i++)
697 KPTD[i] = (KPTphys + ptoa(i)) | PG_RW | PG_V;
698
699 /*
700 * PADDR1 and PADDR2 are used by pmap_pte_quick() and pmap_pte(),
701 * respectively.
702 */
703 SYSMAP(pt_entry_t *, PMAP1, PADDR1, 1)
704 SYSMAP(pt_entry_t *, PMAP2, PADDR2, 1)
705 SYSMAP(pt_entry_t *, PMAP3, PADDR3, 1)
706
707 mtx_init(&PMAP2mutex, "PMAP2", NULL, MTX_DEF);
708
709 virtual_avail = va;
710
711 /*
712 * Initialize the PAT MSR if present.
713 * pmap_init_pat() clears and sets CR4_PGE, which, as a
714 * side-effect, invalidates stale PG_G TLB entries that might
715 * have been created in our pre-boot environment. We assume
716 * that PAT support implies PGE and in reverse, PGE presence
717 * comes with PAT. Both features were added for Pentium Pro.
718 */
719 pmap_init_pat();
720 }
721
722 static void
pmap_init_reserved_pages(void * dummy __unused)723 pmap_init_reserved_pages(void *dummy __unused)
724 {
725 struct pcpu *pc;
726 char *pages;
727 int i;
728
729 #ifdef PMAP_PAE_COMP
730 if (!pae_mode)
731 return;
732 #else
733 if (pae_mode)
734 return;
735 #endif
736 CPU_FOREACH(i) {
737 pc = pcpu_find(i);
738 mtx_init(&pc->pc_copyout_mlock, "cpmlk", NULL, MTX_DEF |
739 MTX_NEW);
740 pc->pc_copyout_maddr = kva_alloc(ptoa(2));
741 if (pc->pc_copyout_maddr == NULL)
742 panic("unable to allocate non-sleepable copyout KVA");
743 sx_init(&pc->pc_copyout_slock, "cpslk");
744 pc->pc_copyout_saddr = kva_alloc(ptoa(2));
745 if (pc->pc_copyout_saddr == NULL)
746 panic("unable to allocate sleepable copyout KVA");
747 pc->pc_pmap_eh_va = (vm_offset_t)kva_alloc(ptoa(1));
748 if (pc->pc_pmap_eh_va == 0)
749 panic("unable to allocate pmap_extract_and_hold KVA");
750 pc->pc_pmap_eh_ptep = (char *)vtopte(pc->pc_pmap_eh_va);
751
752 /*
753 * Skip if the mappings have already been initialized,
754 * i.e. this is the BSP.
755 */
756 if (pc->pc_cmap_addr1 != 0)
757 continue;
758
759 mtx_init(&pc->pc_cmap_lock, "SYSMAPS", NULL, MTX_DEF);
760 pages = kva_alloc(PAGE_SIZE * 3);
761 if (pages == NULL)
762 panic("unable to allocate CMAP KVA");
763 pc->pc_cmap_pte1 = vtopte((vm_offset_t)pages);
764 pc->pc_cmap_pte2 = vtopte((vm_offset_t)pages + PAGE_SIZE);
765 pc->pc_cmap_addr1 = pages;
766 pc->pc_cmap_addr2 = pages + PAGE_SIZE;
767 pc->pc_qmap_addr = pages + ptoa(2);
768 }
769 }
770
771 SYSINIT(rpages_init, SI_SUB_CPU, SI_ORDER_ANY, pmap_init_reserved_pages, NULL);
772
773 /*
774 * Setup the PAT MSR.
775 */
776 static void
__CONCAT(PMTYPE,init_pat)777 __CONCAT(PMTYPE, init_pat)(void)
778 {
779 int pat_table[PAT_INDEX_SIZE];
780 uint64_t pat_msr;
781 u_long cr0, cr4;
782 int i;
783
784 /* Set default PAT index table. */
785 for (i = 0; i < PAT_INDEX_SIZE; i++)
786 pat_table[i] = -1;
787 pat_table[PAT_WRITE_BACK] = 0;
788 pat_table[PAT_WRITE_THROUGH] = 1;
789 pat_table[PAT_UNCACHEABLE] = 3;
790 pat_table[PAT_WRITE_COMBINING] = 3;
791 pat_table[PAT_WRITE_PROTECTED] = 3;
792 pat_table[PAT_UNCACHED] = 3;
793
794 /*
795 * Bail if this CPU doesn't implement PAT.
796 * We assume that PAT support implies PGE.
797 */
798 if ((cpu_feature & CPUID_PAT) == 0) {
799 for (i = 0; i < PAT_INDEX_SIZE; i++)
800 pat_index[i] = pat_table[i];
801 pat_works = 0;
802 return;
803 }
804
805 /*
806 * Due to some Intel errata, we can only safely use the lower 4
807 * PAT entries.
808 *
809 * Intel Pentium III Processor Specification Update
810 * Errata E.27 (Upper Four PAT Entries Not Usable With Mode B
811 * or Mode C Paging)
812 *
813 * Intel Pentium IV Processor Specification Update
814 * Errata N46 (PAT Index MSB May Be Calculated Incorrectly)
815 */
816 if (cpu_vendor_id == CPU_VENDOR_INTEL &&
817 !(CPUID_TO_FAMILY(cpu_id) == 6 && CPUID_TO_MODEL(cpu_id) >= 0xe))
818 pat_works = 0;
819
820 /* Initialize default PAT entries. */
821 pat_msr = PAT_VALUE(0, PAT_WRITE_BACK) |
822 PAT_VALUE(1, PAT_WRITE_THROUGH) |
823 PAT_VALUE(2, PAT_UNCACHED) |
824 PAT_VALUE(3, PAT_UNCACHEABLE) |
825 PAT_VALUE(4, PAT_WRITE_BACK) |
826 PAT_VALUE(5, PAT_WRITE_THROUGH) |
827 PAT_VALUE(6, PAT_UNCACHED) |
828 PAT_VALUE(7, PAT_UNCACHEABLE);
829
830 if (pat_works) {
831 /*
832 * Leave the indices 0-3 at the default of WB, WT, UC-, and UC.
833 * Program 5 and 6 as WP and WC.
834 * Leave 4 and 7 as WB and UC.
835 */
836 pat_msr &= ~(PAT_MASK(5) | PAT_MASK(6));
837 pat_msr |= PAT_VALUE(5, PAT_WRITE_PROTECTED) |
838 PAT_VALUE(6, PAT_WRITE_COMBINING);
839 pat_table[PAT_UNCACHED] = 2;
840 pat_table[PAT_WRITE_PROTECTED] = 5;
841 pat_table[PAT_WRITE_COMBINING] = 6;
842 } else {
843 /*
844 * Just replace PAT Index 2 with WC instead of UC-.
845 */
846 pat_msr &= ~PAT_MASK(2);
847 pat_msr |= PAT_VALUE(2, PAT_WRITE_COMBINING);
848 pat_table[PAT_WRITE_COMBINING] = 2;
849 }
850
851 /* Disable PGE. */
852 cr4 = rcr4();
853 load_cr4(cr4 & ~CR4_PGE);
854
855 /* Disable caches (CD = 1, NW = 0). */
856 cr0 = rcr0();
857 load_cr0((cr0 & ~CR0_NW) | CR0_CD);
858
859 /* Flushes caches and TLBs. */
860 wbinvd();
861 invltlb();
862
863 /* Update PAT and index table. */
864 wrmsr(MSR_PAT, pat_msr);
865 for (i = 0; i < PAT_INDEX_SIZE; i++)
866 pat_index[i] = pat_table[i];
867
868 /* Flush caches and TLBs again. */
869 wbinvd();
870 invltlb();
871
872 /* Restore caches and PGE. */
873 load_cr0(cr0);
874 load_cr4(cr4);
875 }
876
877 #ifdef PMAP_PAE_COMP
878 static void *
pmap_pdpt_allocf(uma_zone_t zone,vm_size_t bytes,int domain,uint8_t * sflagsp,int flags)879 pmap_pdpt_allocf(uma_zone_t zone, vm_size_t bytes, int domain, uint8_t *sflagsp,
880 int flags)
881 {
882
883 /* Inform UMA that this allocator uses kernel_map/object. */
884 *sflagsp = UMA_SLAB_KERNEL;
885 /* contig allocations cannot be NEVERFREED */
886 flags &= ~M_NEVERFREED;
887 return ((void *)kmem_alloc_contig_domainset(DOMAINSET_FIXED(domain),
888 bytes, flags, 0x0ULL, 0xffffffffULL, 1, 0, VM_MEMATTR_DEFAULT));
889 }
890 #endif
891
892 /*
893 * Abuse the pte nodes for unmapped kva to thread a kva freelist through.
894 * Requirements:
895 * - Must deal with pages in order to ensure that none of the PG_* bits
896 * are ever set, PG_V in particular.
897 * - Assumes we can write to ptes without pte_store() atomic ops, even
898 * on PAE systems. This should be ok.
899 * - Assumes nothing will ever test these addresses for 0 to indicate
900 * no mapping instead of correctly checking PG_V.
901 * - Assumes a vm_offset_t will fit in a pte (true for i386).
902 * Because PG_V is never set, there can be no mappings to invalidate.
903 */
904 static vm_offset_t
pmap_ptelist_alloc(vm_offset_t * head)905 pmap_ptelist_alloc(vm_offset_t *head)
906 {
907 pt_entry_t *pte;
908 vm_offset_t va;
909
910 va = *head;
911 if (va == 0)
912 panic("pmap_ptelist_alloc: exhausted ptelist KVA");
913 pte = vtopte(va);
914 *head = *pte;
915 if (*head & PG_V)
916 panic("pmap_ptelist_alloc: va with PG_V set!");
917 *pte = 0;
918 return (va);
919 }
920
921 static void
pmap_ptelist_free(vm_offset_t * head,vm_offset_t va)922 pmap_ptelist_free(vm_offset_t *head, vm_offset_t va)
923 {
924 pt_entry_t *pte;
925
926 if (va & PG_V)
927 panic("pmap_ptelist_free: freeing va with PG_V set!");
928 pte = vtopte(va);
929 *pte = *head; /* virtual! PG_V is 0 though */
930 *head = va;
931 }
932
933 static void
pmap_ptelist_init(vm_offset_t * head,void * base,int npages)934 pmap_ptelist_init(vm_offset_t *head, void *base, int npages)
935 {
936 int i;
937 vm_offset_t va;
938
939 *head = 0;
940 for (i = npages - 1; i >= 0; i--) {
941 va = (vm_offset_t)base + i * PAGE_SIZE;
942 pmap_ptelist_free(head, va);
943 }
944 }
945
946 /*
947 * Initialize the pmap module.
948 *
949 * Called by vm_mem_init(), to initialize any structures that the pmap
950 * system needs to map virtual memory.
951 */
952 static void
__CONCAT(PMTYPE,init)953 __CONCAT(PMTYPE, init)(void)
954 {
955 struct pmap_preinit_mapping *ppim;
956 vm_page_t mpte;
957 vm_size_t s;
958 int i, pv_npg;
959
960 /*
961 * Initialize the vm page array entries for the kernel pmap's
962 * page table pages.
963 */
964 PMAP_LOCK(kernel_pmap);
965 for (i = 0; i < NKPT; i++) {
966 mpte = PHYS_TO_VM_PAGE(KPTphys + ptoa(i));
967 KASSERT(mpte >= vm_page_array &&
968 mpte < &vm_page_array[vm_page_array_size],
969 ("pmap_init: page table page is out of range"));
970 mpte->pindex = i + KPTDI;
971 mpte->phys_addr = KPTphys + ptoa(i);
972 mpte->ref_count = 1;
973
974 /*
975 * Collect the page table pages that were replaced by a 2/4MB
976 * page. They are filled with equivalent 4KB page mappings.
977 */
978 if (pseflag != 0 &&
979 KERNBASE <= i << PDRSHIFT && i << PDRSHIFT < KERNend &&
980 pmap_insert_pt_page(kernel_pmap, mpte, true, true))
981 panic("pmap_init: pmap_insert_pt_page failed");
982 }
983 PMAP_UNLOCK(kernel_pmap);
984 vm_wire_add(NKPT);
985
986 /*
987 * Initialize the address space (zone) for the pv entries. Set a
988 * high water mark so that the system can recover from excessive
989 * numbers of pv entries.
990 */
991 TUNABLE_INT_FETCH("vm.pmap.shpgperproc", &shpgperproc);
992 pv_entry_max = shpgperproc * maxproc + vm_cnt.v_page_count;
993 TUNABLE_INT_FETCH("vm.pmap.pv_entry_max", &pv_entry_max);
994 pv_entry_max = roundup(pv_entry_max, _NPCPV);
995 pv_entry_high_water = 9 * (pv_entry_max / 10);
996
997 /*
998 * If the kernel is running on a virtual machine, then it must assume
999 * that MCA is enabled by the hypervisor. Moreover, the kernel must
1000 * be prepared for the hypervisor changing the vendor and family that
1001 * are reported by CPUID. Consequently, the workaround for AMD Family
1002 * 10h Erratum 383 is enabled if the processor's feature set does not
1003 * include at least one feature that is only supported by older Intel
1004 * or newer AMD processors.
1005 */
1006 if (vm_guest != VM_GUEST_NO && (cpu_feature & CPUID_SS) == 0 &&
1007 (cpu_feature2 & (CPUID2_SSSE3 | CPUID2_SSE41 | CPUID2_AESNI |
1008 CPUID2_AVX | CPUID2_XSAVE)) == 0 && (amd_feature2 & (AMDID2_XOP |
1009 AMDID2_FMA4)) == 0)
1010 workaround_erratum383 = 1;
1011
1012 /*
1013 * Are large page mappings supported and enabled?
1014 */
1015 TUNABLE_INT_FETCH("vm.pmap.pg_ps_enabled", &pg_ps_enabled);
1016 if (pseflag == 0)
1017 pg_ps_enabled = 0;
1018 else if (pg_ps_enabled) {
1019 KASSERT(MAXPAGESIZES > 1 && pagesizes[1] == 0,
1020 ("pmap_init: can't assign to pagesizes[1]"));
1021 pagesizes[1] = NBPDR;
1022 }
1023
1024 /*
1025 * Calculate the size of the pv head table for superpages.
1026 * Handle the possibility that "vm_phys_segs[...].end" is zero.
1027 */
1028 pv_npg = trunc_4mpage(vm_phys_segs[vm_phys_nsegs - 1].end -
1029 PAGE_SIZE) / NBPDR + 1;
1030
1031 /*
1032 * Allocate memory for the pv head table for superpages.
1033 */
1034 s = (vm_size_t)(pv_npg * sizeof(struct md_page));
1035 s = round_page(s);
1036 pv_table = kmem_malloc(s, M_WAITOK | M_ZERO);
1037 for (i = 0; i < pv_npg; i++)
1038 TAILQ_INIT(&pv_table[i].pv_list);
1039
1040 pv_maxchunks = MAX(pv_entry_max / _NPCPV, maxproc);
1041 pv_chunkbase = kva_alloc(PAGE_SIZE * pv_maxchunks);
1042 if (pv_chunkbase == NULL)
1043 panic("pmap_init: not enough kvm for pv chunks");
1044 pmap_ptelist_init(&pv_vafree, pv_chunkbase, pv_maxchunks);
1045 #ifdef PMAP_PAE_COMP
1046 pdptzone = uma_zcreate("PDPT", NPGPTD * sizeof(pdpt_entry_t), NULL,
1047 NULL, NULL, NULL, (NPGPTD * sizeof(pdpt_entry_t)) - 1,
1048 UMA_ZONE_CONTIG | UMA_ZONE_VM | UMA_ZONE_NOFREE);
1049 uma_zone_set_allocf(pdptzone, pmap_pdpt_allocf);
1050 #endif
1051
1052 pmap_initialized = 1;
1053 pmap_init_trm();
1054
1055 if (!bootverbose)
1056 return;
1057 for (i = 0; i < PMAP_PREINIT_MAPPING_COUNT; i++) {
1058 ppim = pmap_preinit_mapping + i;
1059 if (ppim->va == 0)
1060 continue;
1061 printf("PPIM %u: PA=%#jx, VA=%#x, size=%#x, mode=%#x\n", i,
1062 (uintmax_t)ppim->pa, ppim->va, ppim->sz, ppim->mode);
1063 }
1064
1065 }
1066
1067 extern u_long pmap_pde_demotions;
1068 extern u_long pmap_pde_mappings;
1069 extern u_long pmap_pde_p_failures;
1070 extern u_long pmap_pde_promotions;
1071
1072 /***************************************************
1073 * Low level helper routines.....
1074 ***************************************************/
1075
1076 static bool
__CONCAT(PMTYPE,is_valid_memattr)1077 __CONCAT(PMTYPE, is_valid_memattr)(pmap_t pmap __unused, vm_memattr_t mode)
1078 {
1079
1080 return (mode >= 0 && mode < PAT_INDEX_SIZE &&
1081 pat_index[(int)mode] >= 0);
1082 }
1083
1084 /*
1085 * Determine the appropriate bits to set in a PTE or PDE for a specified
1086 * caching mode.
1087 */
1088 static int
__CONCAT(PMTYPE,cache_bits)1089 __CONCAT(PMTYPE, cache_bits)(pmap_t pmap, int mode, bool is_pde)
1090 {
1091 int cache_bits, pat_flag, pat_idx;
1092
1093 if (!pmap_is_valid_memattr(pmap, mode))
1094 panic("Unknown caching mode %d\n", mode);
1095
1096 /* The PAT bit is different for PTE's and PDE's. */
1097 pat_flag = is_pde ? PG_PDE_PAT : PG_PTE_PAT;
1098
1099 /* Map the caching mode to a PAT index. */
1100 pat_idx = pat_index[mode];
1101
1102 /* Map the 3-bit index value into the PAT, PCD, and PWT bits. */
1103 cache_bits = 0;
1104 if (pat_idx & 0x4)
1105 cache_bits |= pat_flag;
1106 if (pat_idx & 0x2)
1107 cache_bits |= PG_NC_PCD;
1108 if (pat_idx & 0x1)
1109 cache_bits |= PG_NC_PWT;
1110 return (cache_bits);
1111 }
1112
1113 static int
pmap_pat_index(pmap_t pmap,pt_entry_t pte,bool is_pde)1114 pmap_pat_index(pmap_t pmap, pt_entry_t pte, bool is_pde)
1115 {
1116 int pat_flag, pat_idx;
1117
1118 if ((cpu_feature & CPUID_PAT) == 0)
1119 return (0);
1120
1121 pat_idx = 0;
1122 /* The PAT bit is different for PTE's and PDE's. */
1123 pat_flag = is_pde ? PG_PDE_PAT : PG_PTE_PAT;
1124
1125 if ((pte & pat_flag) != 0)
1126 pat_idx |= 0x4;
1127 if ((pte & PG_NC_PCD) != 0)
1128 pat_idx |= 0x2;
1129 if ((pte & PG_NC_PWT) != 0)
1130 pat_idx |= 0x1;
1131
1132 /* See pmap_init_pat(). */
1133 if (pat_works) {
1134 if (pat_idx == 4)
1135 pat_idx = 0;
1136 if (pat_idx == 7)
1137 pat_idx = 3;
1138 } else {
1139 /* XXXKIB */
1140 }
1141
1142 return (pat_idx);
1143 }
1144
1145 static bool
__CONCAT(PMTYPE,ps_enabled)1146 __CONCAT(PMTYPE, ps_enabled)(pmap_t pmap __unused)
1147 {
1148
1149 return (pg_ps_enabled);
1150 }
1151
1152 /*
1153 * The caller is responsible for maintaining TLB consistency.
1154 */
1155 static void
pmap_kenter_pde(vm_offset_t va,pd_entry_t newpde)1156 pmap_kenter_pde(vm_offset_t va, pd_entry_t newpde)
1157 {
1158 pd_entry_t *pde;
1159
1160 pde = pmap_pde(kernel_pmap, va);
1161 pde_store(pde, newpde);
1162 }
1163
1164 /*
1165 * After changing the page size for the specified virtual address in the page
1166 * table, flush the corresponding entries from the processor's TLB. Only the
1167 * calling processor's TLB is affected.
1168 *
1169 * The calling thread must be pinned to a processor.
1170 */
1171 static void
pmap_update_pde_invalidate(vm_offset_t va,pd_entry_t newpde)1172 pmap_update_pde_invalidate(vm_offset_t va, pd_entry_t newpde)
1173 {
1174
1175 if ((newpde & PG_PS) == 0)
1176 /* Demotion: flush a specific 2MB page mapping. */
1177 invlpg(va);
1178 else /* if ((newpde & PG_G) == 0) */
1179 /*
1180 * Promotion: flush every 4KB page mapping from the TLB
1181 * because there are too many to flush individually.
1182 */
1183 invltlb();
1184 }
1185
1186 #ifdef SMP
1187
1188 static void
pmap_curcpu_cb_dummy(pmap_t pmap __unused,vm_offset_t addr1 __unused,vm_offset_t addr2 __unused)1189 pmap_curcpu_cb_dummy(pmap_t pmap __unused, vm_offset_t addr1 __unused,
1190 vm_offset_t addr2 __unused)
1191 {
1192 }
1193
1194 /*
1195 * For SMP, these functions have to use the IPI mechanism for coherence.
1196 *
1197 * N.B.: Before calling any of the following TLB invalidation functions,
1198 * the calling processor must ensure that all stores updating a non-
1199 * kernel page table are globally performed. Otherwise, another
1200 * processor could cache an old, pre-update entry without being
1201 * invalidated. This can happen one of two ways: (1) The pmap becomes
1202 * active on another processor after its pm_active field is checked by
1203 * one of the following functions but before a store updating the page
1204 * table is globally performed. (2) The pmap becomes active on another
1205 * processor before its pm_active field is checked but due to
1206 * speculative loads one of the following functions stills reads the
1207 * pmap as inactive on the other processor.
1208 *
1209 * The kernel page table is exempt because its pm_active field is
1210 * immutable. The kernel page table is always active on every
1211 * processor.
1212 */
1213 static void
pmap_invalidate_page_int(pmap_t pmap,vm_offset_t va)1214 pmap_invalidate_page_int(pmap_t pmap, vm_offset_t va)
1215 {
1216 cpuset_t *mask, other_cpus;
1217 u_int cpuid;
1218
1219 sched_pin();
1220 if (pmap == kernel_pmap) {
1221 invlpg(va);
1222 mask = &all_cpus;
1223 } else if (!CPU_CMP(&pmap->pm_active, &all_cpus)) {
1224 mask = &all_cpus;
1225 } else {
1226 cpuid = PCPU_GET(cpuid);
1227 other_cpus = all_cpus;
1228 CPU_CLR(cpuid, &other_cpus);
1229 CPU_AND(&other_cpus, &other_cpus, &pmap->pm_active);
1230 mask = &other_cpus;
1231 }
1232 smp_masked_invlpg(*mask, va, pmap, pmap_curcpu_cb_dummy);
1233 sched_unpin();
1234 }
1235
1236 /* 4k PTEs -- Chosen to exceed the total size of Broadwell L2 TLB */
1237 #define PMAP_INVLPG_THRESHOLD (4 * 1024 * PAGE_SIZE)
1238
1239 static void
pmap_invalidate_range_int(pmap_t pmap,vm_offset_t sva,vm_offset_t eva)1240 pmap_invalidate_range_int(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
1241 {
1242 cpuset_t *mask, other_cpus;
1243 vm_offset_t addr;
1244 u_int cpuid;
1245
1246 if (eva - sva >= PMAP_INVLPG_THRESHOLD) {
1247 pmap_invalidate_all_int(pmap);
1248 return;
1249 }
1250
1251 sched_pin();
1252 if (pmap == kernel_pmap) {
1253 for (addr = sva; addr < eva; addr += PAGE_SIZE)
1254 invlpg(addr);
1255 mask = &all_cpus;
1256 } else if (!CPU_CMP(&pmap->pm_active, &all_cpus)) {
1257 mask = &all_cpus;
1258 } else {
1259 cpuid = PCPU_GET(cpuid);
1260 other_cpus = all_cpus;
1261 CPU_CLR(cpuid, &other_cpus);
1262 CPU_AND(&other_cpus, &other_cpus, &pmap->pm_active);
1263 mask = &other_cpus;
1264 }
1265 smp_masked_invlpg_range(*mask, sva, eva, pmap, pmap_curcpu_cb_dummy);
1266 sched_unpin();
1267 }
1268
1269 static void
pmap_invalidate_all_int(pmap_t pmap)1270 pmap_invalidate_all_int(pmap_t pmap)
1271 {
1272 cpuset_t *mask, other_cpus;
1273 u_int cpuid;
1274
1275 sched_pin();
1276 if (pmap == kernel_pmap) {
1277 invltlb();
1278 mask = &all_cpus;
1279 } else if (!CPU_CMP(&pmap->pm_active, &all_cpus)) {
1280 mask = &all_cpus;
1281 } else {
1282 cpuid = PCPU_GET(cpuid);
1283 other_cpus = all_cpus;
1284 CPU_CLR(cpuid, &other_cpus);
1285 CPU_AND(&other_cpus, &other_cpus, &pmap->pm_active);
1286 mask = &other_cpus;
1287 }
1288 smp_masked_invltlb(*mask, pmap, pmap_curcpu_cb_dummy);
1289 sched_unpin();
1290 }
1291
1292 static void
pmap_invalidate_cache_curcpu_cb(pmap_t pmap __unused,vm_offset_t addr1 __unused,vm_offset_t addr2 __unused)1293 pmap_invalidate_cache_curcpu_cb(pmap_t pmap __unused,
1294 vm_offset_t addr1 __unused, vm_offset_t addr2 __unused)
1295 {
1296 wbinvd();
1297 }
1298
1299 static void
__CONCAT(PMTYPE,invalidate_cache)1300 __CONCAT(PMTYPE, invalidate_cache)(void)
1301 {
1302 smp_cache_flush(pmap_invalidate_cache_curcpu_cb);
1303 }
1304
1305 struct pde_action {
1306 cpuset_t invalidate; /* processors that invalidate their TLB */
1307 vm_offset_t va;
1308 pd_entry_t *pde;
1309 pd_entry_t newpde;
1310 u_int store; /* processor that updates the PDE */
1311 };
1312
1313 static void
pmap_update_pde_kernel(void * arg)1314 pmap_update_pde_kernel(void *arg)
1315 {
1316 struct pde_action *act = arg;
1317 pd_entry_t *pde;
1318
1319 if (act->store == PCPU_GET(cpuid)) {
1320 pde = pmap_pde(kernel_pmap, act->va);
1321 pde_store(pde, act->newpde);
1322 }
1323 }
1324
1325 static void
pmap_update_pde_user(void * arg)1326 pmap_update_pde_user(void *arg)
1327 {
1328 struct pde_action *act = arg;
1329
1330 if (act->store == PCPU_GET(cpuid))
1331 pde_store(act->pde, act->newpde);
1332 }
1333
1334 static void
pmap_update_pde_teardown(void * arg)1335 pmap_update_pde_teardown(void *arg)
1336 {
1337 struct pde_action *act = arg;
1338
1339 if (CPU_ISSET(PCPU_GET(cpuid), &act->invalidate))
1340 pmap_update_pde_invalidate(act->va, act->newpde);
1341 }
1342
1343 /*
1344 * Change the page size for the specified virtual address in a way that
1345 * prevents any possibility of the TLB ever having two entries that map the
1346 * same virtual address using different page sizes. This is the recommended
1347 * workaround for Erratum 383 on AMD Family 10h processors. It prevents a
1348 * machine check exception for a TLB state that is improperly diagnosed as a
1349 * hardware error.
1350 */
1351 static void
pmap_update_pde(pmap_t pmap,vm_offset_t va,pd_entry_t * pde,pd_entry_t newpde)1352 pmap_update_pde(pmap_t pmap, vm_offset_t va, pd_entry_t *pde, pd_entry_t newpde)
1353 {
1354 struct pde_action act;
1355 cpuset_t active, other_cpus;
1356 u_int cpuid;
1357
1358 sched_pin();
1359 cpuid = PCPU_GET(cpuid);
1360 other_cpus = all_cpus;
1361 CPU_CLR(cpuid, &other_cpus);
1362 if (pmap == kernel_pmap)
1363 active = all_cpus;
1364 else
1365 active = pmap->pm_active;
1366 if (CPU_OVERLAP(&active, &other_cpus)) {
1367 act.store = cpuid;
1368 act.invalidate = active;
1369 act.va = va;
1370 act.pde = pde;
1371 act.newpde = newpde;
1372 CPU_SET(cpuid, &active);
1373 smp_rendezvous_cpus(active,
1374 smp_no_rendezvous_barrier, pmap == kernel_pmap ?
1375 pmap_update_pde_kernel : pmap_update_pde_user,
1376 pmap_update_pde_teardown, &act);
1377 } else {
1378 if (pmap == kernel_pmap)
1379 pmap_kenter_pde(va, newpde);
1380 else
1381 pde_store(pde, newpde);
1382 if (CPU_ISSET(cpuid, &active))
1383 pmap_update_pde_invalidate(va, newpde);
1384 }
1385 sched_unpin();
1386 }
1387 #else /* !SMP */
1388 /*
1389 * Normal, non-SMP, 486+ invalidation functions.
1390 * We inline these within pmap.c for speed.
1391 */
1392 static void
pmap_invalidate_page_int(pmap_t pmap,vm_offset_t va)1393 pmap_invalidate_page_int(pmap_t pmap, vm_offset_t va)
1394 {
1395
1396 if (pmap == kernel_pmap)
1397 invlpg(va);
1398 }
1399
1400 static void
pmap_invalidate_range_int(pmap_t pmap,vm_offset_t sva,vm_offset_t eva)1401 pmap_invalidate_range_int(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
1402 {
1403 vm_offset_t addr;
1404
1405 if (pmap == kernel_pmap)
1406 for (addr = sva; addr < eva; addr += PAGE_SIZE)
1407 invlpg(addr);
1408 }
1409
1410 static void
pmap_invalidate_all_int(pmap_t pmap)1411 pmap_invalidate_all_int(pmap_t pmap)
1412 {
1413
1414 if (pmap == kernel_pmap)
1415 invltlb();
1416 }
1417
1418 static void
__CONCAT(PMTYPE,invalidate_cache)1419 __CONCAT(PMTYPE, invalidate_cache)(void)
1420 {
1421
1422 wbinvd();
1423 }
1424
1425 static void
pmap_update_pde(pmap_t pmap,vm_offset_t va,pd_entry_t * pde,pd_entry_t newpde)1426 pmap_update_pde(pmap_t pmap, vm_offset_t va, pd_entry_t *pde, pd_entry_t newpde)
1427 {
1428
1429 if (pmap == kernel_pmap)
1430 pmap_kenter_pde(va, newpde);
1431 else
1432 pde_store(pde, newpde);
1433 if (pmap == kernel_pmap || !CPU_EMPTY(&pmap->pm_active))
1434 pmap_update_pde_invalidate(va, newpde);
1435 }
1436 #endif /* !SMP */
1437
1438 static void
__CONCAT(PMTYPE,invalidate_page)1439 __CONCAT(PMTYPE, invalidate_page)(pmap_t pmap, vm_offset_t va)
1440 {
1441
1442 pmap_invalidate_page_int(pmap, va);
1443 }
1444
1445 static void
__CONCAT(PMTYPE,invalidate_range)1446 __CONCAT(PMTYPE, invalidate_range)(pmap_t pmap, vm_offset_t sva,
1447 vm_offset_t eva)
1448 {
1449
1450 pmap_invalidate_range_int(pmap, sva, eva);
1451 }
1452
1453 static void
__CONCAT(PMTYPE,invalidate_all)1454 __CONCAT(PMTYPE, invalidate_all)(pmap_t pmap)
1455 {
1456
1457 pmap_invalidate_all_int(pmap);
1458 }
1459
1460 static void
pmap_invalidate_pde_page(pmap_t pmap,vm_offset_t va,pd_entry_t pde)1461 pmap_invalidate_pde_page(pmap_t pmap, vm_offset_t va, pd_entry_t pde)
1462 {
1463
1464 /*
1465 * When the PDE has PG_PROMOTED set, the 2- or 4MB page mapping was
1466 * created by a promotion that did not invalidate the 512 or 1024 4KB
1467 * page mappings that might exist in the TLB. Consequently, at this
1468 * point, the TLB may hold both 4KB and 2- or 4MB page mappings for
1469 * the address range [va, va + NBPDR). Therefore, the entire range
1470 * must be invalidated here. In contrast, when PG_PROMOTED is clear,
1471 * the TLB will not hold any 4KB page mappings for the address range
1472 * [va, va + NBPDR), and so a single INVLPG suffices to invalidate the
1473 * 2- or 4MB page mapping from the TLB.
1474 */
1475 if ((pde & PG_PROMOTED) != 0)
1476 pmap_invalidate_range_int(pmap, va, va + NBPDR - 1);
1477 else
1478 pmap_invalidate_page_int(pmap, va);
1479 }
1480
1481 /*
1482 * Are we current address space or kernel?
1483 */
1484 static __inline int
pmap_is_current(pmap_t pmap)1485 pmap_is_current(pmap_t pmap)
1486 {
1487
1488 return (pmap == kernel_pmap);
1489 }
1490
1491 /*
1492 * If the given pmap is not the current or kernel pmap, the returned pte must
1493 * be released by passing it to pmap_pte_release().
1494 */
1495 static pt_entry_t *
__CONCAT(PMTYPE,pte)1496 __CONCAT(PMTYPE, pte)(pmap_t pmap, vm_offset_t va)
1497 {
1498 pd_entry_t newpf;
1499 pd_entry_t *pde;
1500
1501 pde = pmap_pde(pmap, va);
1502 if (*pde & PG_PS)
1503 return (pde);
1504 if (*pde != 0) {
1505 /* are we current address space or kernel? */
1506 if (pmap_is_current(pmap))
1507 return (vtopte(va));
1508 mtx_lock(&PMAP2mutex);
1509 newpf = *pde & PG_FRAME;
1510 if ((*PMAP2 & PG_FRAME) != newpf) {
1511 *PMAP2 = newpf | PG_RW | PG_V | PG_A | PG_M;
1512 pmap_invalidate_page_int(kernel_pmap,
1513 (vm_offset_t)PADDR2);
1514 }
1515 return (PADDR2 + (i386_btop(va) & (NPTEPG - 1)));
1516 }
1517 return (NULL);
1518 }
1519
1520 /*
1521 * Releases a pte that was obtained from pmap_pte(). Be prepared for the pte
1522 * being NULL.
1523 */
1524 static __inline void
pmap_pte_release(pt_entry_t * pte)1525 pmap_pte_release(pt_entry_t *pte)
1526 {
1527
1528 if ((pt_entry_t *)((vm_offset_t)pte & ~PAGE_MASK) == PADDR2)
1529 mtx_unlock(&PMAP2mutex);
1530 }
1531
1532 /*
1533 * NB: The sequence of updating a page table followed by accesses to the
1534 * corresponding pages is subject to the situation described in the "AMD64
1535 * Architecture Programmer's Manual Volume 2: System Programming" rev. 3.23,
1536 * "7.3.1 Special Coherency Considerations". Therefore, issuing the INVLPG
1537 * right after modifying the PTE bits is crucial.
1538 */
1539 static __inline void
invlcaddr(void * caddr)1540 invlcaddr(void *caddr)
1541 {
1542
1543 invlpg((u_int)caddr);
1544 }
1545
1546 /*
1547 * Super fast pmap_pte routine best used when scanning
1548 * the pv lists. This eliminates many coarse-grained
1549 * invltlb calls. Note that many of the pv list
1550 * scans are across different pmaps. It is very wasteful
1551 * to do an entire invltlb for checking a single mapping.
1552 *
1553 * If the given pmap is not the current pmap, pvh_global_lock
1554 * must be held and curthread pinned to a CPU.
1555 */
1556 static pt_entry_t *
pmap_pte_quick(pmap_t pmap,vm_offset_t va)1557 pmap_pte_quick(pmap_t pmap, vm_offset_t va)
1558 {
1559 pd_entry_t newpf;
1560 pd_entry_t *pde;
1561
1562 pde = pmap_pde(pmap, va);
1563 if (*pde & PG_PS)
1564 return (pde);
1565 if (*pde != 0) {
1566 /* are we current address space or kernel? */
1567 if (pmap_is_current(pmap))
1568 return (vtopte(va));
1569 rw_assert(&pvh_global_lock, RA_WLOCKED);
1570 KASSERT(curthread->td_pinned > 0, ("curthread not pinned"));
1571 newpf = *pde & PG_FRAME;
1572 if ((*PMAP1 & PG_FRAME) != newpf) {
1573 *PMAP1 = newpf | PG_RW | PG_V | PG_A | PG_M;
1574 #ifdef SMP
1575 PMAP1cpu = PCPU_GET(cpuid);
1576 #endif
1577 invlcaddr(PADDR1);
1578 PMAP1changed++;
1579 } else
1580 #ifdef SMP
1581 if (PMAP1cpu != PCPU_GET(cpuid)) {
1582 PMAP1cpu = PCPU_GET(cpuid);
1583 invlcaddr(PADDR1);
1584 PMAP1changedcpu++;
1585 } else
1586 #endif
1587 PMAP1unchanged++;
1588 return (PADDR1 + (i386_btop(va) & (NPTEPG - 1)));
1589 }
1590 return (0);
1591 }
1592
1593 static pt_entry_t *
pmap_pte_quick3(pmap_t pmap,vm_offset_t va)1594 pmap_pte_quick3(pmap_t pmap, vm_offset_t va)
1595 {
1596 pd_entry_t newpf;
1597 pd_entry_t *pde;
1598
1599 pde = pmap_pde(pmap, va);
1600 if (*pde & PG_PS)
1601 return (pde);
1602 if (*pde != 0) {
1603 rw_assert(&pvh_global_lock, RA_WLOCKED);
1604 KASSERT(curthread->td_pinned > 0, ("curthread not pinned"));
1605 newpf = *pde & PG_FRAME;
1606 if ((*PMAP3 & PG_FRAME) != newpf) {
1607 *PMAP3 = newpf | PG_RW | PG_V | PG_A | PG_M;
1608 #ifdef SMP
1609 PMAP3cpu = PCPU_GET(cpuid);
1610 #endif
1611 invlcaddr(PADDR3);
1612 PMAP1changed++;
1613 } else
1614 #ifdef SMP
1615 if (PMAP3cpu != PCPU_GET(cpuid)) {
1616 PMAP3cpu = PCPU_GET(cpuid);
1617 invlcaddr(PADDR3);
1618 PMAP1changedcpu++;
1619 } else
1620 #endif
1621 PMAP1unchanged++;
1622 return (PADDR3 + (i386_btop(va) & (NPTEPG - 1)));
1623 }
1624 return (0);
1625 }
1626
1627 static pt_entry_t
pmap_pte_ufast(pmap_t pmap,vm_offset_t va,pd_entry_t pde)1628 pmap_pte_ufast(pmap_t pmap, vm_offset_t va, pd_entry_t pde)
1629 {
1630 pt_entry_t *eh_ptep, pte, *ptep;
1631
1632 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
1633 pde &= PG_FRAME;
1634 critical_enter();
1635 eh_ptep = (pt_entry_t *)PCPU_GET(pmap_eh_ptep);
1636 if ((*eh_ptep & PG_FRAME) != pde) {
1637 *eh_ptep = pde | PG_RW | PG_V | PG_A | PG_M;
1638 invlcaddr((void *)PCPU_GET(pmap_eh_va));
1639 }
1640 ptep = (pt_entry_t *)PCPU_GET(pmap_eh_va) + (i386_btop(va) &
1641 (NPTEPG - 1));
1642 pte = *ptep;
1643 critical_exit();
1644 return (pte);
1645 }
1646
1647 /*
1648 * Extract from the kernel page table the physical address that is mapped by
1649 * the given virtual address "va".
1650 *
1651 * This function may be used before pmap_bootstrap() is called.
1652 */
1653 static vm_paddr_t
__CONCAT(PMTYPE,kextract)1654 __CONCAT(PMTYPE, kextract)(vm_offset_t va)
1655 {
1656 vm_paddr_t pa;
1657
1658 if ((pa = pte_load(&PTD[va >> PDRSHIFT])) & PG_PS) {
1659 pa = (pa & PG_PS_FRAME) | (va & PDRMASK);
1660 } else {
1661 /*
1662 * Beware of a concurrent promotion that changes the PDE at
1663 * this point! For example, vtopte() must not be used to
1664 * access the PTE because it would use the new PDE. It is,
1665 * however, safe to use the old PDE because the page table
1666 * page is preserved by the promotion.
1667 */
1668 pa = KPTmap[i386_btop(va)];
1669 pa = (pa & PG_FRAME) | (va & PAGE_MASK);
1670 }
1671 return (pa);
1672 }
1673
1674 /*
1675 * Routine: pmap_extract
1676 * Function:
1677 * Extract the physical page address associated
1678 * with the given map/virtual_address pair.
1679 */
1680 static vm_paddr_t
__CONCAT(PMTYPE,extract)1681 __CONCAT(PMTYPE, extract)(pmap_t pmap, vm_offset_t va)
1682 {
1683 vm_paddr_t rtval;
1684 pt_entry_t pte;
1685 pd_entry_t pde;
1686
1687 rtval = 0;
1688 PMAP_LOCK(pmap);
1689 pde = pmap->pm_pdir[va >> PDRSHIFT];
1690 if (pde != 0) {
1691 if ((pde & PG_PS) != 0)
1692 rtval = (pde & PG_PS_FRAME) | (va & PDRMASK);
1693 else {
1694 pte = pmap_pte_ufast(pmap, va, pde);
1695 rtval = (pte & PG_FRAME) | (va & PAGE_MASK);
1696 }
1697 }
1698 PMAP_UNLOCK(pmap);
1699 return (rtval);
1700 }
1701
1702 /*
1703 * Routine: pmap_extract_and_hold
1704 * Function:
1705 * Atomically extract and hold the physical page
1706 * with the given pmap and virtual address pair
1707 * if that mapping permits the given protection.
1708 */
1709 static vm_page_t
__CONCAT(PMTYPE,extract_and_hold)1710 __CONCAT(PMTYPE, extract_and_hold)(pmap_t pmap, vm_offset_t va, vm_prot_t prot)
1711 {
1712 pd_entry_t pde;
1713 pt_entry_t pte;
1714 vm_page_t m;
1715
1716 m = NULL;
1717 PMAP_LOCK(pmap);
1718 pde = *pmap_pde(pmap, va);
1719 if (pde != 0) {
1720 if (pde & PG_PS) {
1721 if ((pde & PG_RW) || (prot & VM_PROT_WRITE) == 0)
1722 m = PHYS_TO_VM_PAGE((pde & PG_PS_FRAME) |
1723 (va & PDRMASK));
1724 } else {
1725 pte = pmap_pte_ufast(pmap, va, pde);
1726 if (pte != 0 &&
1727 ((pte & PG_RW) || (prot & VM_PROT_WRITE) == 0))
1728 m = PHYS_TO_VM_PAGE(pte & PG_FRAME);
1729 }
1730 if (m != NULL && !vm_page_wire_mapped(m))
1731 m = NULL;
1732 }
1733 PMAP_UNLOCK(pmap);
1734 return (m);
1735 }
1736
1737 /***************************************************
1738 * Low level mapping routines.....
1739 ***************************************************/
1740
1741 /*
1742 * Add a wired page to the kva.
1743 * Note: not SMP coherent.
1744 *
1745 * This function may be used before pmap_bootstrap() is called.
1746 */
1747 static void
__CONCAT(PMTYPE,kenter)1748 __CONCAT(PMTYPE, kenter)(vm_offset_t va, vm_paddr_t pa)
1749 {
1750 pt_entry_t *pte;
1751
1752 pte = vtopte(va);
1753 pte_store(pte, pa | PG_RW | PG_V);
1754 }
1755
1756 static __inline void
pmap_kenter_attr(vm_offset_t va,vm_paddr_t pa,int mode)1757 pmap_kenter_attr(vm_offset_t va, vm_paddr_t pa, int mode)
1758 {
1759 pt_entry_t *pte;
1760
1761 pte = vtopte(va);
1762 pte_store(pte, pa | PG_RW | PG_V | pmap_cache_bits(kernel_pmap,
1763 mode, false));
1764 }
1765
1766 /*
1767 * Remove a page from the kernel pagetables.
1768 * Note: not SMP coherent.
1769 *
1770 * This function may be used before pmap_bootstrap() is called.
1771 */
1772 static void
__CONCAT(PMTYPE,kremove)1773 __CONCAT(PMTYPE, kremove)(vm_offset_t va)
1774 {
1775 pt_entry_t *pte;
1776
1777 pte = vtopte(va);
1778 pte_clear(pte);
1779 }
1780
1781 /*
1782 * Used to map a range of physical addresses into kernel
1783 * virtual address space.
1784 *
1785 * The value passed in '*virt' is a suggested virtual address for
1786 * the mapping. Architectures which can support a direct-mapped
1787 * physical to virtual region can return the appropriate address
1788 * within that region, leaving '*virt' unchanged. Other
1789 * architectures should map the pages starting at '*virt' and
1790 * update '*virt' with the first usable address after the mapped
1791 * region.
1792 */
1793 static void *
__CONCAT(PMTYPE,map)1794 __CONCAT(PMTYPE, map)(vm_offset_t *virt, vm_paddr_t start, vm_paddr_t end,
1795 int prot)
1796 {
1797 vm_offset_t va, sva;
1798 vm_paddr_t superpage_offset;
1799 pd_entry_t newpde;
1800
1801 va = *virt;
1802 /*
1803 * Does the physical address range's size and alignment permit at
1804 * least one superpage mapping to be created?
1805 */
1806 superpage_offset = start & PDRMASK;
1807 if ((end - start) - ((NBPDR - superpage_offset) & PDRMASK) >= NBPDR) {
1808 /*
1809 * Increase the starting virtual address so that its alignment
1810 * does not preclude the use of superpage mappings.
1811 */
1812 if ((va & PDRMASK) < superpage_offset)
1813 va = (va & ~PDRMASK) + superpage_offset;
1814 else if ((va & PDRMASK) > superpage_offset)
1815 va = ((va + PDRMASK) & ~PDRMASK) + superpage_offset;
1816 }
1817 sva = va;
1818 while (start < end) {
1819 if ((start & PDRMASK) == 0 && end - start >= NBPDR &&
1820 pseflag != 0) {
1821 KASSERT((va & PDRMASK) == 0,
1822 ("pmap_map: misaligned va %#x", va));
1823 newpde = start | PG_PS | PG_RW | PG_V;
1824 pmap_kenter_pde(va, newpde);
1825 va += NBPDR;
1826 start += NBPDR;
1827 } else {
1828 pmap_kenter(va, start);
1829 va += PAGE_SIZE;
1830 start += PAGE_SIZE;
1831 }
1832 }
1833 pmap_invalidate_range_int(kernel_pmap, sva, va);
1834 *virt = va;
1835 return ((void *)sva);
1836 }
1837
1838 /*
1839 * Add a list of wired pages to the kva
1840 * this routine is only used for temporary
1841 * kernel mappings that do not need to have
1842 * page modification or references recorded.
1843 * Note that old mappings are simply written
1844 * over. The page *must* be wired.
1845 * Note: SMP coherent. Uses a ranged shootdown IPI.
1846 */
1847 static void
__CONCAT(PMTYPE,qenter)1848 __CONCAT(PMTYPE, qenter)(void *va, vm_page_t *ma, int count)
1849 {
1850 vm_offset_t sva;
1851 pt_entry_t *endpte, oldpte, pa, *pte;
1852 vm_page_t m;
1853
1854 sva = (vm_offset_t)va;
1855 oldpte = 0;
1856 pte = vtopte(sva);
1857 endpte = pte + count;
1858 while (pte < endpte) {
1859 m = *ma++;
1860 pa = VM_PAGE_TO_PHYS(m) | pmap_cache_bits(kernel_pmap,
1861 m->md.pat_mode, false);
1862 if ((*pte & (PG_FRAME | PG_PTE_CACHE)) != pa) {
1863 oldpte |= *pte;
1864 pte_store(pte, pa | pg_nx | PG_RW | PG_V);
1865 }
1866 pte++;
1867 }
1868 if (__predict_false((oldpte & PG_V) != 0))
1869 pmap_invalidate_range_int(kernel_pmap, sva, sva + count *
1870 PAGE_SIZE);
1871 }
1872
1873 /*
1874 * This routine tears out page mappings from the
1875 * kernel -- it is meant only for temporary mappings.
1876 * Note: SMP coherent. Uses a ranged shootdown IPI.
1877 */
1878 static void
__CONCAT(PMTYPE,qremove)1879 __CONCAT(PMTYPE, qremove)(void *sva, int count)
1880 {
1881 vm_offset_t va;
1882
1883 va = (vm_offset_t)sva;
1884 while (count-- > 0) {
1885 pmap_kremove(va);
1886 va += PAGE_SIZE;
1887 }
1888 pmap_invalidate_range_int(kernel_pmap, (vm_offset_t)sva, va);
1889 }
1890
1891 /***************************************************
1892 * Page table page management routines.....
1893 ***************************************************/
1894 /*
1895 * Schedule the specified unused page table page to be freed. Specifically,
1896 * add the page to the specified list of pages that will be released to the
1897 * physical memory manager after the TLB has been updated.
1898 */
1899 static __inline void
pmap_add_delayed_free_list(vm_page_t m,struct spglist * free,bool set_PG_ZERO)1900 pmap_add_delayed_free_list(vm_page_t m, struct spglist *free, bool set_PG_ZERO)
1901 {
1902
1903 if (set_PG_ZERO)
1904 m->flags |= PG_ZERO;
1905 else
1906 m->flags &= ~PG_ZERO;
1907 SLIST_INSERT_HEAD(free, m, plinks.s.ss);
1908 }
1909
1910 /*
1911 * Inserts the specified page table page into the specified pmap's collection
1912 * of idle page table pages. Each of a pmap's page table pages is responsible
1913 * for mapping a distinct range of virtual addresses. The pmap's collection is
1914 * ordered by this virtual address range.
1915 *
1916 * If "promoted" is false, then the page table page "mpte" must be zero filled;
1917 * "mpte"'s valid field will be set to 0.
1918 *
1919 * If "promoted" is true and "allpte_PG_A_set" is false, then "mpte" must
1920 * contain valid mappings with identical attributes except for PG_A; "mpte"'s
1921 * valid field will be set to 1.
1922 *
1923 * If "promoted" and "allpte_PG_A_set" are both true, then "mpte" must contain
1924 * valid mappings with identical attributes including PG_A; "mpte"'s valid
1925 * field will be set to VM_PAGE_BITS_ALL.
1926 */
1927 static __inline int
pmap_insert_pt_page(pmap_t pmap,vm_page_t mpte,bool promoted,bool allpte_PG_A_set)1928 pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte, bool promoted,
1929 bool allpte_PG_A_set)
1930 {
1931
1932 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
1933 KASSERT(promoted || !allpte_PG_A_set,
1934 ("a zero-filled PTP can't have PG_A set in every PTE"));
1935 mpte->valid = promoted ? (allpte_PG_A_set ? VM_PAGE_BITS_ALL : 1) : 0;
1936 return (vm_radix_insert(&pmap->pm_root, mpte));
1937 }
1938
1939 /*
1940 * Removes the page table page mapping the specified virtual address from the
1941 * specified pmap's collection of idle page table pages, and returns it.
1942 * Otherwise, returns NULL if there is no page table page corresponding to the
1943 * specified virtual address.
1944 */
1945 static __inline vm_page_t
pmap_remove_pt_page(pmap_t pmap,vm_offset_t va)1946 pmap_remove_pt_page(pmap_t pmap, vm_offset_t va)
1947 {
1948
1949 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
1950 return (vm_radix_remove(&pmap->pm_root, va >> PDRSHIFT));
1951 }
1952
1953 /*
1954 * Decrements a page table page's reference count, which is used to record the
1955 * number of valid page table entries within the page. If the reference count
1956 * drops to zero, then the page table page is unmapped. Returns true if the
1957 * page table page was unmapped and false otherwise.
1958 */
1959 static inline bool
pmap_unwire_ptp(pmap_t pmap,vm_page_t m,struct spglist * free)1960 pmap_unwire_ptp(pmap_t pmap, vm_page_t m, struct spglist *free)
1961 {
1962
1963 --m->ref_count;
1964 if (m->ref_count == 0) {
1965 _pmap_unwire_ptp(pmap, m, free);
1966 return (true);
1967 } else
1968 return (false);
1969 }
1970
1971 static void
_pmap_unwire_ptp(pmap_t pmap,vm_page_t m,struct spglist * free)1972 _pmap_unwire_ptp(pmap_t pmap, vm_page_t m, struct spglist *free)
1973 {
1974
1975 /*
1976 * unmap the page table page
1977 */
1978 pmap->pm_pdir[m->pindex] = 0;
1979 --pmap->pm_stats.resident_count;
1980
1981 /*
1982 * There is not need to invalidate the recursive mapping since
1983 * we never instantiate such mapping for the usermode pmaps,
1984 * and never remove page table pages from the kernel pmap.
1985 * Put page on a list so that it is released since all TLB
1986 * shootdown is done.
1987 */
1988 MPASS(pmap != kernel_pmap);
1989 pmap_add_delayed_free_list(m, free, true);
1990 }
1991
1992 /*
1993 * After removing a page table entry, this routine is used to
1994 * conditionally free the page, and manage the reference count.
1995 */
1996 static int
pmap_unuse_pt(pmap_t pmap,vm_offset_t va,struct spglist * free)1997 pmap_unuse_pt(pmap_t pmap, vm_offset_t va, struct spglist *free)
1998 {
1999 pd_entry_t ptepde;
2000 vm_page_t mpte;
2001
2002 if (pmap == kernel_pmap)
2003 return (0);
2004 ptepde = *pmap_pde(pmap, va);
2005 mpte = PHYS_TO_VM_PAGE(ptepde & PG_FRAME);
2006 return (pmap_unwire_ptp(pmap, mpte, free));
2007 }
2008
2009 /*
2010 * Release a page table page reference after a failed attempt to create a
2011 * mapping.
2012 */
2013 static void
pmap_abort_ptp(pmap_t pmap,vm_offset_t va,vm_page_t mpte)2014 pmap_abort_ptp(pmap_t pmap, vm_offset_t va, vm_page_t mpte)
2015 {
2016 struct spglist free;
2017
2018 SLIST_INIT(&free);
2019 if (pmap_unwire_ptp(pmap, mpte, &free)) {
2020 /*
2021 * Although "va" was never mapped, paging-structure caches
2022 * could nonetheless have entries that refer to the freed
2023 * page table pages. Invalidate those entries.
2024 */
2025 pmap_invalidate_page_int(pmap, va);
2026 vm_page_free_pages_toq(&free, true);
2027 }
2028 }
2029
2030 /*
2031 * Initialize the pmap for proc0.
2032 */
2033 static void
__CONCAT(PMTYPE,pinit0)2034 __CONCAT(PMTYPE, pinit0)(pmap_t pmap)
2035 {
2036
2037 PMAP_LOCK_INIT(pmap);
2038 pmap->pm_pdir = IdlePTD;
2039 #ifdef PMAP_PAE_COMP
2040 pmap->pm_pdpt = IdlePDPT;
2041 #endif
2042 vm_radix_init(&pmap->pm_root);
2043 CPU_ZERO(&pmap->pm_active);
2044 TAILQ_INIT(&pmap->pm_pvchunk);
2045 bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
2046 pmap_activate_boot(pmap);
2047 }
2048
2049 /*
2050 * Initialize a preallocated and zeroed pmap structure,
2051 * such as one in a vmspace structure.
2052 */
2053 static int
__CONCAT(PMTYPE,pinit)2054 __CONCAT(PMTYPE, pinit)(pmap_t pmap)
2055 {
2056 int i;
2057
2058 /*
2059 * No need to allocate page table space yet but we do need a valid
2060 * page directory table.
2061 */
2062 if (pmap->pm_pdir == NULL) {
2063 pmap->pm_pdir = kva_alloc(NBPTD);
2064 if (pmap->pm_pdir == NULL)
2065 return (0);
2066 #ifdef PMAP_PAE_COMP
2067 pmap->pm_pdpt = uma_zalloc(pdptzone, M_WAITOK | M_ZERO);
2068 KASSERT(((vm_offset_t)pmap->pm_pdpt &
2069 ((NPGPTD * sizeof(pdpt_entry_t)) - 1)) == 0,
2070 ("pmap_pinit: pdpt misaligned"));
2071 KASSERT(pmap_kextract((vm_offset_t)pmap->pm_pdpt) < (4ULL<<30),
2072 ("pmap_pinit: pdpt above 4g"));
2073 #endif
2074 vm_radix_init(&pmap->pm_root);
2075 }
2076 KASSERT(vm_radix_is_empty(&pmap->pm_root),
2077 ("pmap_pinit: pmap has reserved page table page(s)"));
2078
2079 /*
2080 * allocate the page directory page(s)
2081 */
2082 for (i = 0; i < NPGPTD; i++) {
2083 pmap->pm_ptdpg[i] = vm_page_alloc_noobj(VM_ALLOC_WIRED |
2084 VM_ALLOC_ZERO | VM_ALLOC_WAITOK);
2085 #ifdef PMAP_PAE_COMP
2086 pmap->pm_pdpt[i] = VM_PAGE_TO_PHYS(pmap->pm_ptdpg[i]) | PG_V;
2087 #endif
2088 }
2089
2090 pmap_qenter(pmap->pm_pdir, pmap->pm_ptdpg, NPGPTD);
2091 #ifdef PMAP_PAE_COMP
2092 if ((cpu_feature & CPUID_PAT) == 0) {
2093 pmap_invalidate_cache_range(
2094 trunc_page((vm_offset_t)pmap->pm_pdpt),
2095 round_page((vm_offset_t)pmap->pm_pdpt +
2096 NPGPTD * sizeof(pdpt_entry_t)));
2097 }
2098 #endif
2099
2100 /* Install the trampoline mapping. */
2101 pmap->pm_pdir[TRPTDI] = PTD[TRPTDI];
2102
2103 CPU_ZERO(&pmap->pm_active);
2104 TAILQ_INIT(&pmap->pm_pvchunk);
2105 bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
2106
2107 return (1);
2108 }
2109
2110 /*
2111 * this routine is called if the page table page is not
2112 * mapped correctly.
2113 */
2114 static vm_page_t
_pmap_allocpte(pmap_t pmap,u_int ptepindex,u_int flags)2115 _pmap_allocpte(pmap_t pmap, u_int ptepindex, u_int flags)
2116 {
2117 vm_paddr_t ptepa;
2118 vm_page_t m;
2119
2120 /*
2121 * Allocate a page table page.
2122 */
2123 if ((m = vm_page_alloc_noobj(VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) {
2124 if ((flags & PMAP_ENTER_NOSLEEP) == 0) {
2125 PMAP_UNLOCK(pmap);
2126 rw_wunlock(&pvh_global_lock);
2127 vm_wait(NULL);
2128 rw_wlock(&pvh_global_lock);
2129 PMAP_LOCK(pmap);
2130 }
2131
2132 /*
2133 * Indicate the need to retry. While waiting, the page table
2134 * page may have been allocated.
2135 */
2136 return (NULL);
2137 }
2138 m->pindex = ptepindex;
2139
2140 /*
2141 * Map the pagetable page into the process address space, if
2142 * it isn't already there.
2143 */
2144
2145 pmap->pm_stats.resident_count++;
2146
2147 ptepa = VM_PAGE_TO_PHYS(m);
2148 KASSERT((pmap->pm_pdir[ptepindex] & PG_V) == 0,
2149 ("%s: page directory entry %#jx is valid",
2150 __func__, (uintmax_t)pmap->pm_pdir[ptepindex]));
2151 pmap->pm_pdir[ptepindex] =
2152 (pd_entry_t)(ptepa | PG_U | PG_RW | PG_V | PG_A | PG_M);
2153
2154 return (m);
2155 }
2156
2157 static vm_page_t
pmap_allocpte(pmap_t pmap,vm_offset_t va,u_int flags)2158 pmap_allocpte(pmap_t pmap, vm_offset_t va, u_int flags)
2159 {
2160 u_int ptepindex;
2161 pd_entry_t ptepa;
2162 vm_page_t m;
2163
2164 /*
2165 * Calculate pagetable page index
2166 */
2167 ptepindex = va >> PDRSHIFT;
2168 retry:
2169 /*
2170 * Get the page directory entry
2171 */
2172 ptepa = pmap->pm_pdir[ptepindex];
2173
2174 /*
2175 * This supports switching from a 4MB page to a
2176 * normal 4K page.
2177 */
2178 if (ptepa & PG_PS) {
2179 (void)pmap_demote_pde(pmap, &pmap->pm_pdir[ptepindex], va);
2180 ptepa = pmap->pm_pdir[ptepindex];
2181 }
2182
2183 /*
2184 * If the page table page is mapped, we just increment the
2185 * hold count, and activate it.
2186 */
2187 if (ptepa) {
2188 m = PHYS_TO_VM_PAGE(ptepa & PG_FRAME);
2189 m->ref_count++;
2190 } else {
2191 /*
2192 * Here if the pte page isn't mapped, or if it has
2193 * been deallocated.
2194 */
2195 m = _pmap_allocpte(pmap, ptepindex, flags);
2196 if (m == NULL && (flags & PMAP_ENTER_NOSLEEP) == 0)
2197 goto retry;
2198 }
2199 return (m);
2200 }
2201
2202 /***************************************************
2203 * Pmap allocation/deallocation routines.
2204 ***************************************************/
2205
2206 /*
2207 * Release any resources held by the given physical map.
2208 * Called when a pmap initialized by pmap_pinit is being released.
2209 * Should only be called if the map contains no valid mappings.
2210 */
2211 static void
__CONCAT(PMTYPE,release)2212 __CONCAT(PMTYPE, release)(pmap_t pmap)
2213 {
2214 vm_page_t m;
2215 int i;
2216
2217 KASSERT(pmap->pm_stats.resident_count == 0,
2218 ("pmap_release: pmap resident count %ld != 0",
2219 pmap->pm_stats.resident_count));
2220 KASSERT(vm_radix_is_empty(&pmap->pm_root),
2221 ("pmap_release: pmap has reserved page table page(s)"));
2222 KASSERT(CPU_EMPTY(&pmap->pm_active),
2223 ("releasing active pmap %p", pmap));
2224
2225 pmap_qremove(pmap->pm_pdir, NPGPTD);
2226
2227 for (i = 0; i < NPGPTD; i++) {
2228 m = pmap->pm_ptdpg[i];
2229 #ifdef PMAP_PAE_COMP
2230 KASSERT(VM_PAGE_TO_PHYS(m) == (pmap->pm_pdpt[i] & PG_FRAME),
2231 ("pmap_release: got wrong ptd page"));
2232 #endif
2233 vm_page_unwire_noq(m);
2234 vm_page_free(m);
2235 }
2236 }
2237
2238 /*
2239 * grow the number of kernel page table entries, if needed
2240 */
2241 static int
__CONCAT(PMTYPE,growkernel)2242 __CONCAT(PMTYPE, growkernel)(vm_offset_t addr)
2243 {
2244 vm_paddr_t ptppaddr;
2245 vm_page_t nkpg;
2246 pd_entry_t newpdir;
2247
2248 mtx_assert(&kernel_map->system_mtx, MA_OWNED);
2249 addr = roundup2(addr, NBPDR);
2250 if (addr - 1 >= vm_map_max(kernel_map))
2251 addr = vm_map_max(kernel_map);
2252 while (kernel_vm_end < addr) {
2253 if (pdir_pde(PTD, kernel_vm_end)) {
2254 kernel_vm_end = (kernel_vm_end + NBPDR) & ~PDRMASK;
2255 if (kernel_vm_end - 1 >= vm_map_max(kernel_map)) {
2256 kernel_vm_end = vm_map_max(kernel_map);
2257 break;
2258 }
2259 continue;
2260 }
2261
2262 nkpg = vm_page_alloc_noobj(VM_ALLOC_INTERRUPT |
2263 VM_ALLOC_NOFREE | VM_ALLOC_WIRED | VM_ALLOC_ZERO);
2264 if (nkpg == NULL)
2265 return (KERN_RESOURCE_SHORTAGE);
2266 nkpg->pindex = kernel_vm_end >> PDRSHIFT;
2267 nkpt++;
2268
2269 ptppaddr = VM_PAGE_TO_PHYS(nkpg);
2270 newpdir = (pd_entry_t) (ptppaddr | PG_V | PG_RW | PG_A | PG_M);
2271 pdir_pde(KPTD, kernel_vm_end) = newpdir;
2272
2273 pmap_kenter_pde(kernel_vm_end, newpdir);
2274 kernel_vm_end = (kernel_vm_end + NBPDR) & ~PDRMASK;
2275 if (kernel_vm_end - 1 >= vm_map_max(kernel_map)) {
2276 kernel_vm_end = vm_map_max(kernel_map);
2277 break;
2278 }
2279 }
2280
2281 return (KERN_SUCCESS);
2282 }
2283
2284 /***************************************************
2285 * page management routines.
2286 ***************************************************/
2287
2288 static const uint32_t pc_freemask[_NPCM] = {
2289 [0 ... _NPCM - 2] = PC_FREEN,
2290 [_NPCM - 1] = PC_FREEL
2291 };
2292
2293 #ifdef PV_STATS
2294 extern int pc_chunk_count, pc_chunk_allocs, pc_chunk_frees, pc_chunk_tryfail;
2295 extern long pv_entry_frees, pv_entry_allocs;
2296 extern int pv_entry_spare;
2297 #endif
2298
2299 /*
2300 * We are in a serious low memory condition. Resort to
2301 * drastic measures to free some pages so we can allocate
2302 * another pv entry chunk.
2303 */
2304 static vm_page_t
pmap_pv_reclaim(pmap_t locked_pmap)2305 pmap_pv_reclaim(pmap_t locked_pmap)
2306 {
2307 struct pch newtail;
2308 struct pv_chunk *pc;
2309 struct md_page *pvh;
2310 pd_entry_t *pde;
2311 pmap_t pmap;
2312 pt_entry_t *pte, tpte;
2313 pv_entry_t pv;
2314 vm_offset_t va;
2315 vm_page_t m, m_pc;
2316 struct spglist free;
2317 uint32_t inuse;
2318 int bit, field, freed;
2319
2320 PMAP_LOCK_ASSERT(locked_pmap, MA_OWNED);
2321 pmap = NULL;
2322 m_pc = NULL;
2323 SLIST_INIT(&free);
2324 TAILQ_INIT(&newtail);
2325 while ((pc = TAILQ_FIRST(&pv_chunks)) != NULL && (pv_vafree == 0 ||
2326 SLIST_EMPTY(&free))) {
2327 TAILQ_REMOVE(&pv_chunks, pc, pc_lru);
2328 if (pmap != pc->pc_pmap) {
2329 if (pmap != NULL) {
2330 pmap_invalidate_all_int(pmap);
2331 if (pmap != locked_pmap)
2332 PMAP_UNLOCK(pmap);
2333 }
2334 pmap = pc->pc_pmap;
2335 /* Avoid deadlock and lock recursion. */
2336 if (pmap > locked_pmap)
2337 PMAP_LOCK(pmap);
2338 else if (pmap != locked_pmap && !PMAP_TRYLOCK(pmap)) {
2339 pmap = NULL;
2340 TAILQ_INSERT_TAIL(&newtail, pc, pc_lru);
2341 continue;
2342 }
2343 }
2344
2345 /*
2346 * Destroy every non-wired, 4 KB page mapping in the chunk.
2347 */
2348 freed = 0;
2349 for (field = 0; field < _NPCM; field++) {
2350 for (inuse = ~pc->pc_map[field] & pc_freemask[field];
2351 inuse != 0; inuse &= ~(1UL << bit)) {
2352 bit = bsfl(inuse);
2353 pv = &pc->pc_pventry[field * 32 + bit];
2354 va = pv->pv_va;
2355 pde = pmap_pde(pmap, va);
2356 if ((*pde & PG_PS) != 0)
2357 continue;
2358 pte = __CONCAT(PMTYPE, pte)(pmap, va);
2359 tpte = *pte;
2360 if ((tpte & PG_W) == 0)
2361 tpte = pte_load_clear(pte);
2362 pmap_pte_release(pte);
2363 if ((tpte & PG_W) != 0)
2364 continue;
2365 KASSERT(tpte != 0,
2366 ("pmap_pv_reclaim: pmap %p va %x zero pte",
2367 pmap, va));
2368 if ((tpte & PG_G) != 0)
2369 pmap_invalidate_page_int(pmap, va);
2370 m = PHYS_TO_VM_PAGE(tpte & PG_FRAME);
2371 if ((tpte & (PG_M | PG_RW)) == (PG_M | PG_RW))
2372 vm_page_dirty(m);
2373 if ((tpte & PG_A) != 0)
2374 vm_page_aflag_set(m, PGA_REFERENCED);
2375 TAILQ_REMOVE(&m->md.pv_list, pv, pv_next);
2376 if (TAILQ_EMPTY(&m->md.pv_list) &&
2377 (m->flags & PG_FICTITIOUS) == 0) {
2378 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
2379 if (TAILQ_EMPTY(&pvh->pv_list)) {
2380 vm_page_aflag_clear(m,
2381 PGA_WRITEABLE);
2382 }
2383 }
2384 pc->pc_map[field] |= 1UL << bit;
2385 pmap_unuse_pt(pmap, va, &free);
2386 freed++;
2387 }
2388 }
2389 if (freed == 0) {
2390 TAILQ_INSERT_TAIL(&newtail, pc, pc_lru);
2391 continue;
2392 }
2393 /* Every freed mapping is for a 4 KB page. */
2394 pmap->pm_stats.resident_count -= freed;
2395 PV_STAT(pv_entry_frees += freed);
2396 PV_STAT(pv_entry_spare += freed);
2397 pv_entry_count -= freed;
2398 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
2399 for (field = 0; field < _NPCM; field++)
2400 if (pc->pc_map[field] != pc_freemask[field]) {
2401 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc,
2402 pc_list);
2403 TAILQ_INSERT_TAIL(&newtail, pc, pc_lru);
2404
2405 /*
2406 * One freed pv entry in locked_pmap is
2407 * sufficient.
2408 */
2409 if (pmap == locked_pmap)
2410 goto out;
2411 break;
2412 }
2413 if (field == _NPCM) {
2414 PV_STAT(pv_entry_spare -= _NPCPV);
2415 PV_STAT(pc_chunk_count--);
2416 PV_STAT(pc_chunk_frees++);
2417 /* Entire chunk is free; return it. */
2418 m_pc = PHYS_TO_VM_PAGE(pmap_kextract((vm_offset_t)pc));
2419 pmap_qremove(pc, 1);
2420 pmap_ptelist_free(&pv_vafree, (vm_offset_t)pc);
2421 break;
2422 }
2423 }
2424 out:
2425 TAILQ_CONCAT(&pv_chunks, &newtail, pc_lru);
2426 if (pmap != NULL) {
2427 pmap_invalidate_all_int(pmap);
2428 if (pmap != locked_pmap)
2429 PMAP_UNLOCK(pmap);
2430 }
2431 if (m_pc == NULL && pv_vafree != 0 && SLIST_EMPTY(&free)) {
2432 m_pc = SLIST_FIRST(&free);
2433 SLIST_REMOVE_HEAD(&free, plinks.s.ss);
2434 /* Recycle a freed page table page. */
2435 m_pc->ref_count = 1;
2436 }
2437 vm_page_free_pages_toq(&free, true);
2438 return (m_pc);
2439 }
2440
2441 /*
2442 * free the pv_entry back to the free list
2443 */
2444 static void
free_pv_entry(pmap_t pmap,pv_entry_t pv)2445 free_pv_entry(pmap_t pmap, pv_entry_t pv)
2446 {
2447 struct pv_chunk *pc;
2448 int idx, field, bit;
2449
2450 rw_assert(&pvh_global_lock, RA_WLOCKED);
2451 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
2452 PV_STAT(pv_entry_frees++);
2453 PV_STAT(pv_entry_spare++);
2454 pv_entry_count--;
2455 pc = pv_to_chunk(pv);
2456 idx = pv - &pc->pc_pventry[0];
2457 field = idx / 32;
2458 bit = idx % 32;
2459 pc->pc_map[field] |= 1ul << bit;
2460 for (idx = 0; idx < _NPCM; idx++)
2461 if (pc->pc_map[idx] != pc_freemask[idx]) {
2462 /*
2463 * 98% of the time, pc is already at the head of the
2464 * list. If it isn't already, move it to the head.
2465 */
2466 if (__predict_false(TAILQ_FIRST(&pmap->pm_pvchunk) !=
2467 pc)) {
2468 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
2469 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc,
2470 pc_list);
2471 }
2472 return;
2473 }
2474 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
2475 free_pv_chunk(pc);
2476 }
2477
2478 static void
free_pv_chunk(struct pv_chunk * pc)2479 free_pv_chunk(struct pv_chunk *pc)
2480 {
2481 vm_page_t m;
2482
2483 TAILQ_REMOVE(&pv_chunks, pc, pc_lru);
2484 PV_STAT(pv_entry_spare -= _NPCPV);
2485 PV_STAT(pc_chunk_count--);
2486 PV_STAT(pc_chunk_frees++);
2487 /* entire chunk is free, return it */
2488 m = PHYS_TO_VM_PAGE(pmap_kextract((vm_offset_t)pc));
2489 pmap_qremove(pc, 1);
2490 vm_page_unwire_noq(m);
2491 vm_page_free(m);
2492 pmap_ptelist_free(&pv_vafree, (vm_offset_t)pc);
2493 }
2494
2495 /*
2496 * get a new pv_entry, allocating a block from the system
2497 * when needed.
2498 */
2499 static pv_entry_t
get_pv_entry(pmap_t pmap,bool try)2500 get_pv_entry(pmap_t pmap, bool try)
2501 {
2502 static const struct timeval printinterval = { 60, 0 };
2503 static struct timeval lastprint;
2504 int bit, field;
2505 pv_entry_t pv;
2506 struct pv_chunk *pc;
2507 vm_page_t m;
2508
2509 rw_assert(&pvh_global_lock, RA_WLOCKED);
2510 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
2511 PV_STAT(pv_entry_allocs++);
2512 pv_entry_count++;
2513 if (pv_entry_count > pv_entry_high_water)
2514 if (ratecheck(&lastprint, &printinterval))
2515 printf("Approaching the limit on PV entries, consider "
2516 "increasing either the vm.pmap.shpgperproc or the "
2517 "vm.pmap.pv_entry_max tunable.\n");
2518 retry:
2519 pc = TAILQ_FIRST(&pmap->pm_pvchunk);
2520 if (pc != NULL) {
2521 for (field = 0; field < _NPCM; field++) {
2522 if (pc->pc_map[field]) {
2523 bit = bsfl(pc->pc_map[field]);
2524 break;
2525 }
2526 }
2527 if (field < _NPCM) {
2528 pv = &pc->pc_pventry[field * 32 + bit];
2529 pc->pc_map[field] &= ~(1ul << bit);
2530 /* If this was the last item, move it to tail */
2531 for (field = 0; field < _NPCM; field++)
2532 if (pc->pc_map[field] != 0) {
2533 PV_STAT(pv_entry_spare--);
2534 return (pv); /* not full, return */
2535 }
2536 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
2537 TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list);
2538 PV_STAT(pv_entry_spare--);
2539 return (pv);
2540 }
2541 }
2542 /*
2543 * Access to the ptelist "pv_vafree" is synchronized by the pvh
2544 * global lock. If "pv_vafree" is currently non-empty, it will
2545 * remain non-empty until pmap_ptelist_alloc() completes.
2546 */
2547 if (pv_vafree == 0 ||
2548 (m = vm_page_alloc_noobj(VM_ALLOC_WIRED)) == NULL) {
2549 if (try) {
2550 pv_entry_count--;
2551 PV_STAT(pc_chunk_tryfail++);
2552 return (NULL);
2553 }
2554 m = pmap_pv_reclaim(pmap);
2555 if (m == NULL)
2556 goto retry;
2557 }
2558 PV_STAT(pc_chunk_count++);
2559 PV_STAT(pc_chunk_allocs++);
2560 pc = (struct pv_chunk *)pmap_ptelist_alloc(&pv_vafree);
2561 pmap_qenter(pc, &m, 1);
2562 pc->pc_pmap = pmap;
2563 pc->pc_map[0] = pc_freemask[0] & ~1ul; /* preallocated bit 0 */
2564 for (field = 1; field < _NPCM; field++)
2565 pc->pc_map[field] = pc_freemask[field];
2566 TAILQ_INSERT_TAIL(&pv_chunks, pc, pc_lru);
2567 pv = &pc->pc_pventry[0];
2568 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);
2569 PV_STAT(pv_entry_spare += _NPCPV - 1);
2570 return (pv);
2571 }
2572
2573 static __inline pv_entry_t
pmap_pvh_remove(struct md_page * pvh,pmap_t pmap,vm_offset_t va)2574 pmap_pvh_remove(struct md_page *pvh, pmap_t pmap, vm_offset_t va)
2575 {
2576 pv_entry_t pv;
2577
2578 rw_assert(&pvh_global_lock, RA_WLOCKED);
2579 TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) {
2580 if (pmap == PV_PMAP(pv) && va == pv->pv_va) {
2581 TAILQ_REMOVE(&pvh->pv_list, pv, pv_next);
2582 break;
2583 }
2584 }
2585 return (pv);
2586 }
2587
2588 static void
pmap_pv_demote_pde(pmap_t pmap,vm_offset_t va,vm_paddr_t pa)2589 pmap_pv_demote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa)
2590 {
2591 struct md_page *pvh;
2592 pv_entry_t pv;
2593 vm_offset_t va_last;
2594 vm_page_t m;
2595
2596 rw_assert(&pvh_global_lock, RA_WLOCKED);
2597 KASSERT((pa & PDRMASK) == 0,
2598 ("pmap_pv_demote_pde: pa is not 4mpage aligned"));
2599
2600 /*
2601 * Transfer the 4mpage's pv entry for this mapping to the first
2602 * page's pv list.
2603 */
2604 pvh = pa_to_pvh(pa);
2605 va = trunc_4mpage(va);
2606 pv = pmap_pvh_remove(pvh, pmap, va);
2607 KASSERT(pv != NULL, ("pmap_pv_demote_pde: pv not found"));
2608 m = PHYS_TO_VM_PAGE(pa);
2609 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next);
2610 /* Instantiate the remaining NPTEPG - 1 pv entries. */
2611 va_last = va + NBPDR - PAGE_SIZE;
2612 do {
2613 m++;
2614 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
2615 ("pmap_pv_demote_pde: page %p is not managed", m));
2616 va += PAGE_SIZE;
2617 pmap_insert_entry(pmap, va, m);
2618 } while (va < va_last);
2619 }
2620
2621 #if VM_NRESERVLEVEL > 0
2622 static void
pmap_pv_promote_pde(pmap_t pmap,vm_offset_t va,vm_paddr_t pa)2623 pmap_pv_promote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa)
2624 {
2625 struct md_page *pvh;
2626 pv_entry_t pv;
2627 vm_offset_t va_last;
2628 vm_page_t m;
2629
2630 rw_assert(&pvh_global_lock, RA_WLOCKED);
2631 KASSERT((pa & PDRMASK) == 0,
2632 ("pmap_pv_promote_pde: pa is not 4mpage aligned"));
2633
2634 /*
2635 * Transfer the first page's pv entry for this mapping to the
2636 * 4mpage's pv list. Aside from avoiding the cost of a call
2637 * to get_pv_entry(), a transfer avoids the possibility that
2638 * get_pv_entry() calls pmap_collect() and that pmap_collect()
2639 * removes one of the mappings that is being promoted.
2640 */
2641 m = PHYS_TO_VM_PAGE(pa);
2642 va = trunc_4mpage(va);
2643 pv = pmap_pvh_remove(&m->md, pmap, va);
2644 KASSERT(pv != NULL, ("pmap_pv_promote_pde: pv not found"));
2645 pvh = pa_to_pvh(pa);
2646 TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_next);
2647 /* Free the remaining NPTEPG - 1 pv entries. */
2648 va_last = va + NBPDR - PAGE_SIZE;
2649 do {
2650 m++;
2651 va += PAGE_SIZE;
2652 pmap_pvh_free(&m->md, pmap, va);
2653 } while (va < va_last);
2654 }
2655 #endif /* VM_NRESERVLEVEL > 0 */
2656
2657 static void
pmap_pvh_free(struct md_page * pvh,pmap_t pmap,vm_offset_t va)2658 pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va)
2659 {
2660 pv_entry_t pv;
2661
2662 pv = pmap_pvh_remove(pvh, pmap, va);
2663 KASSERT(pv != NULL, ("pmap_pvh_free: pv not found"));
2664 free_pv_entry(pmap, pv);
2665 }
2666
2667 static void
pmap_remove_entry(pmap_t pmap,vm_page_t m,vm_offset_t va)2668 pmap_remove_entry(pmap_t pmap, vm_page_t m, vm_offset_t va)
2669 {
2670 struct md_page *pvh;
2671
2672 rw_assert(&pvh_global_lock, RA_WLOCKED);
2673 pmap_pvh_free(&m->md, pmap, va);
2674 if (TAILQ_EMPTY(&m->md.pv_list) && (m->flags & PG_FICTITIOUS) == 0) {
2675 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
2676 if (TAILQ_EMPTY(&pvh->pv_list))
2677 vm_page_aflag_clear(m, PGA_WRITEABLE);
2678 }
2679 }
2680
2681 /*
2682 * Create a pv entry for page at pa for
2683 * (pmap, va).
2684 */
2685 static void
pmap_insert_entry(pmap_t pmap,vm_offset_t va,vm_page_t m)2686 pmap_insert_entry(pmap_t pmap, vm_offset_t va, vm_page_t m)
2687 {
2688 pv_entry_t pv;
2689
2690 rw_assert(&pvh_global_lock, RA_WLOCKED);
2691 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
2692 pv = get_pv_entry(pmap, false);
2693 pv->pv_va = va;
2694 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next);
2695 }
2696
2697 /*
2698 * Conditionally create a pv entry.
2699 */
2700 static bool
pmap_try_insert_pv_entry(pmap_t pmap,vm_offset_t va,vm_page_t m)2701 pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, vm_page_t m)
2702 {
2703 pv_entry_t pv;
2704
2705 rw_assert(&pvh_global_lock, RA_WLOCKED);
2706 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
2707 if (pv_entry_count < pv_entry_high_water &&
2708 (pv = get_pv_entry(pmap, true)) != NULL) {
2709 pv->pv_va = va;
2710 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next);
2711 return (true);
2712 } else
2713 return (false);
2714 }
2715
2716 /*
2717 * Create the pv entries for each of the pages within a superpage.
2718 */
2719 static bool
pmap_pv_insert_pde(pmap_t pmap,vm_offset_t va,pd_entry_t pde,u_int flags)2720 pmap_pv_insert_pde(pmap_t pmap, vm_offset_t va, pd_entry_t pde, u_int flags)
2721 {
2722 struct md_page *pvh;
2723 pv_entry_t pv;
2724 bool noreclaim;
2725
2726 rw_assert(&pvh_global_lock, RA_WLOCKED);
2727 noreclaim = (flags & PMAP_ENTER_NORECLAIM) != 0;
2728 if ((noreclaim && pv_entry_count >= pv_entry_high_water) ||
2729 (pv = get_pv_entry(pmap, noreclaim)) == NULL)
2730 return (false);
2731 pv->pv_va = va;
2732 pvh = pa_to_pvh(pde & PG_PS_FRAME);
2733 TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_next);
2734 return (true);
2735 }
2736
2737 /*
2738 * Fills a page table page with mappings to consecutive physical pages.
2739 */
2740 static void
pmap_fill_ptp(pt_entry_t * firstpte,pt_entry_t newpte)2741 pmap_fill_ptp(pt_entry_t *firstpte, pt_entry_t newpte)
2742 {
2743 pt_entry_t *pte;
2744
2745 for (pte = firstpte; pte < firstpte + NPTEPG; pte++) {
2746 *pte = newpte;
2747 newpte += PAGE_SIZE;
2748 }
2749 }
2750
2751 /*
2752 * Tries to demote a 2- or 4MB page mapping. If demotion fails, the
2753 * 2- or 4MB page mapping is invalidated.
2754 */
2755 static bool
pmap_demote_pde(pmap_t pmap,pd_entry_t * pde,vm_offset_t va)2756 pmap_demote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va)
2757 {
2758 pd_entry_t newpde, oldpde;
2759 pt_entry_t *firstpte, newpte;
2760 vm_paddr_t mptepa;
2761 vm_page_t mpte;
2762 struct spglist free;
2763 vm_offset_t sva;
2764
2765 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
2766 oldpde = *pde;
2767 KASSERT((oldpde & (PG_PS | PG_V)) == (PG_PS | PG_V),
2768 ("pmap_demote_pde: oldpde is missing PG_PS and/or PG_V"));
2769 if ((oldpde & PG_A) == 0 || (mpte = pmap_remove_pt_page(pmap, va)) ==
2770 NULL) {
2771 KASSERT((oldpde & PG_W) == 0,
2772 ("pmap_demote_pde: page table page for a wired mapping"
2773 " is missing"));
2774
2775 /*
2776 * Invalidate the 2- or 4MB page mapping and return
2777 * "failure" if the mapping was never accessed or the
2778 * allocation of the new page table page fails.
2779 */
2780 if ((oldpde & PG_A) == 0 ||
2781 (mpte = vm_page_alloc_noobj(VM_ALLOC_WIRED)) == NULL) {
2782 SLIST_INIT(&free);
2783 sva = trunc_4mpage(va);
2784 pmap_remove_pde(pmap, pde, sva, &free);
2785 if ((oldpde & PG_G) == 0)
2786 pmap_invalidate_pde_page(pmap, sva, oldpde);
2787 vm_page_free_pages_toq(&free, true);
2788 CTR2(KTR_PMAP, "pmap_demote_pde: failure for va %#x"
2789 " in pmap %p", va, pmap);
2790 return (false);
2791 }
2792 mpte->pindex = va >> PDRSHIFT;
2793 if (pmap != kernel_pmap) {
2794 mpte->ref_count = NPTEPG;
2795 pmap->pm_stats.resident_count++;
2796 }
2797 }
2798 mptepa = VM_PAGE_TO_PHYS(mpte);
2799
2800 /*
2801 * If the page mapping is in the kernel's address space, then the
2802 * KPTmap can provide access to the page table page. Otherwise,
2803 * temporarily map the page table page (mpte) into the kernel's
2804 * address space at either PADDR1 or PADDR2.
2805 */
2806 if (pmap == kernel_pmap)
2807 firstpte = &KPTmap[i386_btop(trunc_4mpage(va))];
2808 else if (curthread->td_pinned > 0 && rw_wowned(&pvh_global_lock)) {
2809 if ((*PMAP1 & PG_FRAME) != mptepa) {
2810 *PMAP1 = mptepa | PG_RW | PG_V | PG_A | PG_M;
2811 #ifdef SMP
2812 PMAP1cpu = PCPU_GET(cpuid);
2813 #endif
2814 invlcaddr(PADDR1);
2815 PMAP1changed++;
2816 } else
2817 #ifdef SMP
2818 if (PMAP1cpu != PCPU_GET(cpuid)) {
2819 PMAP1cpu = PCPU_GET(cpuid);
2820 invlcaddr(PADDR1);
2821 PMAP1changedcpu++;
2822 } else
2823 #endif
2824 PMAP1unchanged++;
2825 firstpte = PADDR1;
2826 } else {
2827 mtx_lock(&PMAP2mutex);
2828 if ((*PMAP2 & PG_FRAME) != mptepa) {
2829 *PMAP2 = mptepa | PG_RW | PG_V | PG_A | PG_M;
2830 pmap_invalidate_page_int(kernel_pmap,
2831 (vm_offset_t)PADDR2);
2832 }
2833 firstpte = PADDR2;
2834 }
2835 newpde = mptepa | PG_M | PG_A | (oldpde & PG_U) | PG_RW | PG_V;
2836 KASSERT((oldpde & PG_A) != 0,
2837 ("pmap_demote_pde: oldpde is missing PG_A"));
2838 KASSERT((oldpde & (PG_M | PG_RW)) != PG_RW,
2839 ("pmap_demote_pde: oldpde is missing PG_M"));
2840 newpte = oldpde & ~PG_PS;
2841 if ((newpte & PG_PDE_PAT) != 0)
2842 newpte ^= PG_PDE_PAT | PG_PTE_PAT;
2843
2844 /*
2845 * If the PTP is not leftover from an earlier promotion or it does not
2846 * have PG_A set in every PTE, then fill it. The new PTEs will all
2847 * have PG_A set.
2848 */
2849 if (!vm_page_all_valid(mpte))
2850 pmap_fill_ptp(firstpte, newpte);
2851
2852 KASSERT((*firstpte & PG_FRAME) == (newpte & PG_FRAME),
2853 ("pmap_demote_pde: firstpte and newpte map different physical"
2854 " addresses"));
2855
2856 /*
2857 * If the mapping has changed attributes, update the PTEs.
2858 */
2859 if ((*firstpte & PG_PTE_PROMOTE) != (newpte & PG_PTE_PROMOTE))
2860 pmap_fill_ptp(firstpte, newpte);
2861
2862 /*
2863 * Demote the mapping. This pmap is locked. The old PDE has
2864 * PG_A set. If the old PDE has PG_RW set, it also has PG_M
2865 * set. Thus, there is no danger of a race with another
2866 * processor changing the setting of PG_A and/or PG_M between
2867 * the read above and the store below.
2868 */
2869 if (workaround_erratum383)
2870 pmap_update_pde(pmap, va, pde, newpde);
2871 else if (pmap == kernel_pmap)
2872 pmap_kenter_pde(va, newpde);
2873 else
2874 pde_store(pde, newpde);
2875 if (firstpte == PADDR2)
2876 mtx_unlock(&PMAP2mutex);
2877
2878 /*
2879 * Invalidate the recursive mapping of the page table page.
2880 */
2881 pmap_invalidate_page_int(pmap, (vm_offset_t)vtopte(va));
2882
2883 /*
2884 * Demote the pv entry. This depends on the earlier demotion
2885 * of the mapping. Specifically, the (re)creation of a per-
2886 * page pv entry might trigger the execution of pmap_collect(),
2887 * which might reclaim a newly (re)created per-page pv entry
2888 * and destroy the associated mapping. In order to destroy
2889 * the mapping, the PDE must have already changed from mapping
2890 * the 2mpage to referencing the page table page.
2891 */
2892 if ((oldpde & PG_MANAGED) != 0)
2893 pmap_pv_demote_pde(pmap, va, oldpde & PG_PS_FRAME);
2894
2895 pmap_pde_demotions++;
2896 CTR2(KTR_PMAP, "pmap_demote_pde: success for va %#x"
2897 " in pmap %p", va, pmap);
2898 return (true);
2899 }
2900
2901 /*
2902 * Removes a 2- or 4MB page mapping from the kernel pmap.
2903 */
2904 static void
pmap_remove_kernel_pde(pmap_t pmap,pd_entry_t * pde,vm_offset_t va)2905 pmap_remove_kernel_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va)
2906 {
2907 pd_entry_t newpde;
2908 vm_paddr_t mptepa;
2909 vm_page_t mpte;
2910
2911 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
2912 mpte = pmap_remove_pt_page(pmap, va);
2913 if (mpte == NULL)
2914 panic("pmap_remove_kernel_pde: Missing pt page.");
2915
2916 mptepa = VM_PAGE_TO_PHYS(mpte);
2917 newpde = mptepa | PG_M | PG_A | PG_RW | PG_V;
2918
2919 /*
2920 * If this page table page was unmapped by a promotion, then it
2921 * contains valid mappings. Zero it to invalidate those mappings.
2922 */
2923 if (vm_page_any_valid(mpte))
2924 pagezero((void *)&KPTmap[i386_btop(trunc_4mpage(va))]);
2925
2926 /*
2927 * Remove the mapping.
2928 */
2929 if (workaround_erratum383)
2930 pmap_update_pde(pmap, va, pde, newpde);
2931 else
2932 pmap_kenter_pde(va, newpde);
2933
2934 /*
2935 * Invalidate the recursive mapping of the page table page.
2936 */
2937 pmap_invalidate_page_int(pmap, (vm_offset_t)vtopte(va));
2938 }
2939
2940 /*
2941 * pmap_remove_pde: do the things to unmap a superpage in a process
2942 */
2943 static void
pmap_remove_pde(pmap_t pmap,pd_entry_t * pdq,vm_offset_t sva,struct spglist * free)2944 pmap_remove_pde(pmap_t pmap, pd_entry_t *pdq, vm_offset_t sva,
2945 struct spglist *free)
2946 {
2947 struct md_page *pvh;
2948 pd_entry_t oldpde;
2949 vm_offset_t eva, va;
2950 vm_page_t m, mpte;
2951
2952 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
2953 KASSERT((sva & PDRMASK) == 0,
2954 ("pmap_remove_pde: sva is not 4mpage aligned"));
2955 oldpde = pte_load_clear(pdq);
2956 if (oldpde & PG_W)
2957 pmap->pm_stats.wired_count -= NBPDR / PAGE_SIZE;
2958
2959 /*
2960 * Machines that don't support invlpg, also don't support
2961 * PG_G.
2962 */
2963 if ((oldpde & PG_G) != 0)
2964 pmap_invalidate_pde_page(kernel_pmap, sva, oldpde);
2965
2966 pmap->pm_stats.resident_count -= NBPDR / PAGE_SIZE;
2967 if (oldpde & PG_MANAGED) {
2968 pvh = pa_to_pvh(oldpde & PG_PS_FRAME);
2969 pmap_pvh_free(pvh, pmap, sva);
2970 eva = sva + NBPDR;
2971 for (va = sva, m = PHYS_TO_VM_PAGE(oldpde & PG_PS_FRAME);
2972 va < eva; va += PAGE_SIZE, m++) {
2973 if ((oldpde & (PG_M | PG_RW)) == (PG_M | PG_RW))
2974 vm_page_dirty(m);
2975 if (oldpde & PG_A)
2976 vm_page_aflag_set(m, PGA_REFERENCED);
2977 if (TAILQ_EMPTY(&m->md.pv_list) &&
2978 TAILQ_EMPTY(&pvh->pv_list))
2979 vm_page_aflag_clear(m, PGA_WRITEABLE);
2980 }
2981 }
2982 if (pmap == kernel_pmap) {
2983 pmap_remove_kernel_pde(pmap, pdq, sva);
2984 } else {
2985 mpte = pmap_remove_pt_page(pmap, sva);
2986 if (mpte != NULL) {
2987 KASSERT(vm_page_any_valid(mpte),
2988 ("pmap_remove_pde: pte page not promoted"));
2989 pmap->pm_stats.resident_count--;
2990 KASSERT(mpte->ref_count == NPTEPG,
2991 ("pmap_remove_pde: pte page ref count error"));
2992 mpte->ref_count = 0;
2993 pmap_add_delayed_free_list(mpte, free, false);
2994 }
2995 }
2996 }
2997
2998 /*
2999 * pmap_remove_pte: do the things to unmap a page in a process
3000 */
3001 static int
pmap_remove_pte(pmap_t pmap,pt_entry_t * ptq,vm_offset_t va,struct spglist * free)3002 pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t va,
3003 struct spglist *free)
3004 {
3005 pt_entry_t oldpte;
3006 vm_page_t m;
3007
3008 rw_assert(&pvh_global_lock, RA_WLOCKED);
3009 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
3010 oldpte = pte_load_clear(ptq);
3011 KASSERT(oldpte != 0,
3012 ("pmap_remove_pte: pmap %p va %x zero pte", pmap, va));
3013 if (oldpte & PG_W)
3014 pmap->pm_stats.wired_count -= 1;
3015 /*
3016 * Machines that don't support invlpg, also don't support
3017 * PG_G.
3018 */
3019 if (oldpte & PG_G)
3020 pmap_invalidate_page_int(kernel_pmap, va);
3021 pmap->pm_stats.resident_count -= 1;
3022 if (oldpte & PG_MANAGED) {
3023 m = PHYS_TO_VM_PAGE(oldpte & PG_FRAME);
3024 if ((oldpte & (PG_M | PG_RW)) == (PG_M | PG_RW))
3025 vm_page_dirty(m);
3026 if (oldpte & PG_A)
3027 vm_page_aflag_set(m, PGA_REFERENCED);
3028 pmap_remove_entry(pmap, m, va);
3029 }
3030 return (pmap_unuse_pt(pmap, va, free));
3031 }
3032
3033 /*
3034 * Remove a single page from a process address space
3035 */
3036 static void
pmap_remove_page(pmap_t pmap,vm_offset_t va,struct spglist * free)3037 pmap_remove_page(pmap_t pmap, vm_offset_t va, struct spglist *free)
3038 {
3039 pt_entry_t *pte;
3040
3041 rw_assert(&pvh_global_lock, RA_WLOCKED);
3042 KASSERT(curthread->td_pinned > 0, ("curthread not pinned"));
3043 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
3044 if ((pte = pmap_pte_quick(pmap, va)) == NULL || *pte == 0)
3045 return;
3046 pmap_remove_pte(pmap, pte, va, free);
3047 pmap_invalidate_page_int(pmap, va);
3048 }
3049
3050 /*
3051 * Removes the specified range of addresses from the page table page.
3052 */
3053 static bool
pmap_remove_ptes(pmap_t pmap,vm_offset_t sva,vm_offset_t eva,struct spglist * free)3054 pmap_remove_ptes(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,
3055 struct spglist *free)
3056 {
3057 pt_entry_t *pte;
3058 bool anyvalid;
3059
3060 rw_assert(&pvh_global_lock, RA_WLOCKED);
3061 KASSERT(curthread->td_pinned > 0, ("curthread not pinned"));
3062 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
3063 anyvalid = false;
3064 for (pte = pmap_pte_quick(pmap, sva); sva != eva; pte++,
3065 sva += PAGE_SIZE) {
3066 if (*pte == 0)
3067 continue;
3068
3069 /*
3070 * The TLB entry for a PG_G mapping is invalidated by
3071 * pmap_remove_pte().
3072 */
3073 if ((*pte & PG_G) == 0)
3074 anyvalid = true;
3075
3076 if (pmap_remove_pte(pmap, pte, sva, free))
3077 break;
3078 }
3079 return (anyvalid);
3080 }
3081
3082 /*
3083 * Remove the given range of addresses from the specified map.
3084 *
3085 * It is assumed that the start and end are properly
3086 * rounded to the page size.
3087 */
3088 static void
__CONCAT(PMTYPE,remove)3089 __CONCAT(PMTYPE, remove)(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
3090 {
3091 vm_offset_t pdnxt;
3092 pd_entry_t ptpaddr;
3093 struct spglist free;
3094 int anyvalid;
3095
3096 /*
3097 * Perform an unsynchronized read. This is, however, safe.
3098 */
3099 if (pmap->pm_stats.resident_count == 0)
3100 return;
3101
3102 anyvalid = 0;
3103 SLIST_INIT(&free);
3104
3105 rw_wlock(&pvh_global_lock);
3106 sched_pin();
3107 PMAP_LOCK(pmap);
3108
3109 /*
3110 * special handling of removing one page. a very
3111 * common operation and easy to short circuit some
3112 * code.
3113 */
3114 if ((sva + PAGE_SIZE == eva) &&
3115 ((pmap->pm_pdir[(sva >> PDRSHIFT)] & PG_PS) == 0)) {
3116 pmap_remove_page(pmap, sva, &free);
3117 goto out;
3118 }
3119
3120 for (; sva < eva; sva = pdnxt) {
3121 u_int pdirindex;
3122
3123 /*
3124 * Calculate index for next page table.
3125 */
3126 pdnxt = (sva + NBPDR) & ~PDRMASK;
3127 if (pdnxt < sva)
3128 pdnxt = eva;
3129 if (pmap->pm_stats.resident_count == 0)
3130 break;
3131
3132 pdirindex = sva >> PDRSHIFT;
3133 ptpaddr = pmap->pm_pdir[pdirindex];
3134
3135 /*
3136 * Weed out invalid mappings. Note: we assume that the page
3137 * directory table is always allocated, and in kernel virtual.
3138 */
3139 if (ptpaddr == 0)
3140 continue;
3141
3142 /*
3143 * Check for large page.
3144 */
3145 if ((ptpaddr & PG_PS) != 0) {
3146 /*
3147 * Are we removing the entire large page? If not,
3148 * demote the mapping and fall through.
3149 */
3150 if (sva + NBPDR == pdnxt && eva >= pdnxt) {
3151 /*
3152 * The TLB entry for a PG_G mapping is
3153 * invalidated by pmap_remove_pde().
3154 */
3155 if ((ptpaddr & PG_G) == 0)
3156 anyvalid = 1;
3157 pmap_remove_pde(pmap,
3158 &pmap->pm_pdir[pdirindex], sva, &free);
3159 continue;
3160 } else if (!pmap_demote_pde(pmap,
3161 &pmap->pm_pdir[pdirindex], sva)) {
3162 /* The large page mapping was destroyed. */
3163 continue;
3164 }
3165 }
3166
3167 /*
3168 * Limit our scan to either the end of the va represented
3169 * by the current page table page, or to the end of the
3170 * range being removed.
3171 */
3172 if (pdnxt > eva)
3173 pdnxt = eva;
3174
3175 if (pmap_remove_ptes(pmap, sva, pdnxt, &free))
3176 anyvalid = 1;
3177 }
3178 out:
3179 sched_unpin();
3180 if (anyvalid)
3181 pmap_invalidate_all_int(pmap);
3182 rw_wunlock(&pvh_global_lock);
3183 PMAP_UNLOCK(pmap);
3184 vm_page_free_pages_toq(&free, true);
3185 }
3186
3187 /*
3188 * Routine: pmap_remove_all
3189 * Function:
3190 * Removes this physical page from
3191 * all physical maps in which it resides.
3192 * Reflects back modify bits to the pager.
3193 *
3194 * Notes:
3195 * Original versions of this routine were very
3196 * inefficient because they iteratively called
3197 * pmap_remove (slow...)
3198 */
3199
3200 static void
__CONCAT(PMTYPE,remove_all)3201 __CONCAT(PMTYPE, remove_all)(vm_page_t m)
3202 {
3203 struct md_page *pvh;
3204 pv_entry_t pv;
3205 pmap_t pmap;
3206 pt_entry_t *pte, tpte;
3207 pd_entry_t *pde;
3208 vm_offset_t va;
3209 struct spglist free;
3210
3211 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
3212 ("pmap_remove_all: page %p is not managed", m));
3213 SLIST_INIT(&free);
3214 rw_wlock(&pvh_global_lock);
3215 sched_pin();
3216 if ((m->flags & PG_FICTITIOUS) != 0)
3217 goto small_mappings;
3218 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
3219 while ((pv = TAILQ_FIRST(&pvh->pv_list)) != NULL) {
3220 va = pv->pv_va;
3221 pmap = PV_PMAP(pv);
3222 PMAP_LOCK(pmap);
3223 pde = pmap_pde(pmap, va);
3224 (void)pmap_demote_pde(pmap, pde, va);
3225 PMAP_UNLOCK(pmap);
3226 }
3227 small_mappings:
3228 while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) {
3229 pmap = PV_PMAP(pv);
3230 PMAP_LOCK(pmap);
3231 pmap->pm_stats.resident_count--;
3232 pde = pmap_pde(pmap, pv->pv_va);
3233 KASSERT((*pde & PG_PS) == 0, ("pmap_remove_all: found"
3234 " a 4mpage in page %p's pv list", m));
3235 pte = pmap_pte_quick(pmap, pv->pv_va);
3236 tpte = pte_load_clear(pte);
3237 KASSERT(tpte != 0, ("pmap_remove_all: pmap %p va %x zero pte",
3238 pmap, pv->pv_va));
3239 if (tpte & PG_W)
3240 pmap->pm_stats.wired_count--;
3241 if (tpte & PG_A)
3242 vm_page_aflag_set(m, PGA_REFERENCED);
3243
3244 /*
3245 * Update the vm_page_t clean and reference bits.
3246 */
3247 if ((tpte & (PG_M | PG_RW)) == (PG_M | PG_RW))
3248 vm_page_dirty(m);
3249 pmap_unuse_pt(pmap, pv->pv_va, &free);
3250 pmap_invalidate_page_int(pmap, pv->pv_va);
3251 TAILQ_REMOVE(&m->md.pv_list, pv, pv_next);
3252 free_pv_entry(pmap, pv);
3253 PMAP_UNLOCK(pmap);
3254 }
3255 vm_page_aflag_clear(m, PGA_WRITEABLE);
3256 sched_unpin();
3257 rw_wunlock(&pvh_global_lock);
3258 vm_page_free_pages_toq(&free, true);
3259 }
3260
3261 /*
3262 * pmap_protect_pde: do the things to protect a 4mpage in a process
3263 */
3264 static bool
pmap_protect_pde(pmap_t pmap,pd_entry_t * pde,vm_offset_t sva,vm_prot_t prot)3265 pmap_protect_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t sva, vm_prot_t prot)
3266 {
3267 pd_entry_t newpde, oldpde;
3268 vm_page_t m, mt;
3269 bool anychanged;
3270
3271 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
3272 KASSERT((sva & PDRMASK) == 0,
3273 ("pmap_protect_pde: sva is not 4mpage aligned"));
3274 anychanged = false;
3275 retry:
3276 oldpde = newpde = *pde;
3277 if ((prot & VM_PROT_WRITE) == 0) {
3278 if ((oldpde & (PG_MANAGED | PG_M | PG_RW)) ==
3279 (PG_MANAGED | PG_M | PG_RW)) {
3280 m = PHYS_TO_VM_PAGE(oldpde & PG_PS_FRAME);
3281 for (mt = m; mt < &m[NBPDR / PAGE_SIZE]; mt++)
3282 vm_page_dirty(mt);
3283 }
3284 newpde &= ~(PG_RW | PG_M);
3285 }
3286 #ifdef PMAP_PAE_COMP
3287 if ((prot & VM_PROT_EXECUTE) == 0 && !i386_read_exec)
3288 newpde |= pg_nx;
3289 #endif
3290 if (newpde != oldpde) {
3291 /*
3292 * As an optimization to future operations on this PDE, clear
3293 * PG_PROMOTED. The impending invalidation will remove any
3294 * lingering 4KB page mappings from the TLB.
3295 */
3296 if (!pde_cmpset(pde, oldpde, newpde & ~PG_PROMOTED))
3297 goto retry;
3298 if ((oldpde & PG_G) != 0)
3299 pmap_invalidate_pde_page(kernel_pmap, sva, oldpde);
3300 else
3301 anychanged = true;
3302 }
3303 return (anychanged);
3304 }
3305
3306 /*
3307 * Set the physical protection on the
3308 * specified range of this map as requested.
3309 */
3310 static void
__CONCAT(PMTYPE,protect)3311 __CONCAT(PMTYPE, protect)(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,
3312 vm_prot_t prot)
3313 {
3314 vm_offset_t pdnxt;
3315 pd_entry_t ptpaddr;
3316 pt_entry_t *pte;
3317 bool anychanged, pv_lists_locked;
3318
3319 KASSERT((prot & ~VM_PROT_ALL) == 0, ("invalid prot %x", prot));
3320 if (prot == VM_PROT_NONE) {
3321 pmap_remove(pmap, sva, eva);
3322 return;
3323 }
3324
3325 #ifdef PMAP_PAE_COMP
3326 if ((prot & (VM_PROT_WRITE | VM_PROT_EXECUTE)) ==
3327 (VM_PROT_WRITE | VM_PROT_EXECUTE))
3328 return;
3329 #else
3330 if (prot & VM_PROT_WRITE)
3331 return;
3332 #endif
3333
3334 if (pmap_is_current(pmap))
3335 pv_lists_locked = false;
3336 else {
3337 pv_lists_locked = true;
3338 resume:
3339 rw_wlock(&pvh_global_lock);
3340 sched_pin();
3341 }
3342 anychanged = false;
3343
3344 PMAP_LOCK(pmap);
3345 for (; sva < eva; sva = pdnxt) {
3346 pt_entry_t obits, pbits;
3347 u_int pdirindex;
3348
3349 pdnxt = (sva + NBPDR) & ~PDRMASK;
3350 if (pdnxt < sva)
3351 pdnxt = eva;
3352
3353 pdirindex = sva >> PDRSHIFT;
3354 ptpaddr = pmap->pm_pdir[pdirindex];
3355
3356 /*
3357 * Weed out invalid mappings. Note: we assume that the page
3358 * directory table is always allocated, and in kernel virtual.
3359 */
3360 if (ptpaddr == 0)
3361 continue;
3362
3363 /*
3364 * Check for large page.
3365 */
3366 if ((ptpaddr & PG_PS) != 0) {
3367 /*
3368 * Are we protecting the entire large page? If not,
3369 * demote the mapping and fall through.
3370 */
3371 if (sva + NBPDR == pdnxt && eva >= pdnxt) {
3372 /*
3373 * The TLB entry for a PG_G mapping is
3374 * invalidated by pmap_protect_pde().
3375 */
3376 if (pmap_protect_pde(pmap,
3377 &pmap->pm_pdir[pdirindex], sva, prot))
3378 anychanged = true;
3379 continue;
3380 } else {
3381 if (!pv_lists_locked) {
3382 pv_lists_locked = true;
3383 if (!rw_try_wlock(&pvh_global_lock)) {
3384 if (anychanged)
3385 pmap_invalidate_all_int(
3386 pmap);
3387 PMAP_UNLOCK(pmap);
3388 goto resume;
3389 }
3390 sched_pin();
3391 }
3392 if (!pmap_demote_pde(pmap,
3393 &pmap->pm_pdir[pdirindex], sva)) {
3394 /*
3395 * The large page mapping was
3396 * destroyed.
3397 */
3398 continue;
3399 }
3400 }
3401 }
3402
3403 if (pdnxt > eva)
3404 pdnxt = eva;
3405
3406 for (pte = pmap_pte_quick(pmap, sva); sva != pdnxt; pte++,
3407 sva += PAGE_SIZE) {
3408 vm_page_t m;
3409
3410 retry:
3411 /*
3412 * Regardless of whether a pte is 32 or 64 bits in
3413 * size, PG_RW, PG_A, and PG_M are among the least
3414 * significant 32 bits.
3415 */
3416 obits = pbits = *pte;
3417 if ((pbits & PG_V) == 0)
3418 continue;
3419
3420 if ((prot & VM_PROT_WRITE) == 0) {
3421 if ((pbits & (PG_MANAGED | PG_M | PG_RW)) ==
3422 (PG_MANAGED | PG_M | PG_RW)) {
3423 m = PHYS_TO_VM_PAGE(pbits & PG_FRAME);
3424 vm_page_dirty(m);
3425 }
3426 pbits &= ~(PG_RW | PG_M);
3427 }
3428 #ifdef PMAP_PAE_COMP
3429 if ((prot & VM_PROT_EXECUTE) == 0 && !i386_read_exec)
3430 pbits |= pg_nx;
3431 #endif
3432
3433 if (pbits != obits) {
3434 #ifdef PMAP_PAE_COMP
3435 if (!atomic_cmpset_64(pte, obits, pbits))
3436 goto retry;
3437 #else
3438 if (!atomic_cmpset_int((u_int *)pte, obits,
3439 pbits))
3440 goto retry;
3441 #endif
3442 if (obits & PG_G)
3443 pmap_invalidate_page_int(pmap, sva);
3444 else
3445 anychanged = true;
3446 }
3447 }
3448 }
3449 if (anychanged)
3450 pmap_invalidate_all_int(pmap);
3451 if (pv_lists_locked) {
3452 sched_unpin();
3453 rw_wunlock(&pvh_global_lock);
3454 }
3455 PMAP_UNLOCK(pmap);
3456 }
3457
3458 #if VM_NRESERVLEVEL > 0
3459 /*
3460 * Tries to promote the 512 or 1024, contiguous 4KB page mappings that are
3461 * within a single page table page (PTP) to a single 2- or 4MB page mapping.
3462 * For promotion to occur, two conditions must be met: (1) the 4KB page
3463 * mappings must map aligned, contiguous physical memory and (2) the 4KB page
3464 * mappings must have identical characteristics.
3465 *
3466 * Managed (PG_MANAGED) mappings within the kernel address space are not
3467 * promoted. The reason is that kernel PDEs are replicated in each pmap but
3468 * pmap_clear_ptes() and pmap_ts_referenced() only read the PDE from the kernel
3469 * pmap.
3470 */
3471 static bool
pmap_promote_pde(pmap_t pmap,pd_entry_t * pde,vm_offset_t va,vm_page_t mpte)3472 pmap_promote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va, vm_page_t mpte)
3473 {
3474 pd_entry_t newpde;
3475 pt_entry_t allpte_PG_A, *firstpte, oldpte, pa, *pte;
3476 #ifdef KTR
3477 vm_offset_t oldpteva;
3478 #endif
3479
3480 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
3481 if (!pg_ps_enabled)
3482 return (false);
3483
3484 /*
3485 * Examine the first PTE in the specified PTP. Abort if this PTE is
3486 * either invalid or does not map the first 4KB physical page
3487 * within a 2- or 4MB page.
3488 */
3489 firstpte = pmap_pte_quick(pmap, trunc_4mpage(va));
3490 setpde:
3491 newpde = *firstpte;
3492 if ((newpde & ((PG_FRAME & PDRMASK) | PG_V)) != PG_V) {
3493 pmap_pde_p_failures++;
3494 CTR2(KTR_PMAP, "pmap_promote_pde: failure for va %#x"
3495 " in pmap %p", va, pmap);
3496 return (false);
3497 }
3498 if ((*firstpte & PG_MANAGED) != 0 && pmap == kernel_pmap) {
3499 pmap_pde_p_failures++;
3500 CTR2(KTR_PMAP, "pmap_promote_pde: failure for va %#x"
3501 " in pmap %p", va, pmap);
3502 return (false);
3503 }
3504
3505 /*
3506 * Both here and in the below "for" loop, to allow for repromotion
3507 * after MADV_FREE, conditionally write protect a clean PTE before
3508 * possibly aborting the promotion due to other PTE attributes. Why?
3509 * Suppose that MADV_FREE is applied to a part of a superpage, the
3510 * address range [S, E). pmap_advise() will demote the superpage
3511 * mapping, destroy the 4KB page mapping at the end of [S, E), and
3512 * clear PG_M and PG_A in the PTEs for the rest of [S, E). Later,
3513 * imagine that the memory in [S, E) is recycled, but the last 4KB
3514 * page in [S, E) is not the last to be rewritten, or simply accessed.
3515 * In other words, there is still a 4KB page in [S, E), call it P,
3516 * that is writeable but PG_M and PG_A are clear in P's PTE. Unless
3517 * we write protect P before aborting the promotion, if and when P is
3518 * finally rewritten, there won't be a page fault to trigger
3519 * repromotion.
3520 */
3521 if ((newpde & (PG_M | PG_RW)) == PG_RW) {
3522 /*
3523 * When PG_M is already clear, PG_RW can be cleared without
3524 * a TLB invalidation.
3525 */
3526 if (!atomic_cmpset_int((u_int *)firstpte, newpde, newpde &
3527 ~PG_RW))
3528 goto setpde;
3529 newpde &= ~PG_RW;
3530 CTR2(KTR_PMAP, "pmap_promote_pde: protect for va %#lx"
3531 " in pmap %p", va & ~PDRMASK, pmap);
3532 }
3533
3534 /*
3535 * Examine each of the other PTEs in the specified PTP. Abort if this
3536 * PTE maps an unexpected 4KB physical page or does not have identical
3537 * characteristics to the first PTE.
3538 */
3539 allpte_PG_A = newpde & PG_A;
3540 pa = (newpde & (PG_PS_FRAME | PG_V)) + NBPDR - PAGE_SIZE;
3541 for (pte = firstpte + NPTEPG - 1; pte > firstpte; pte--) {
3542 setpte:
3543 oldpte = *pte;
3544 if ((oldpte & (PG_FRAME | PG_V)) != pa) {
3545 pmap_pde_p_failures++;
3546 CTR2(KTR_PMAP, "pmap_promote_pde: failure for va %#x"
3547 " in pmap %p", va, pmap);
3548 return (false);
3549 }
3550 if ((oldpte & (PG_M | PG_RW)) == PG_RW) {
3551 /*
3552 * When PG_M is already clear, PG_RW can be cleared
3553 * without a TLB invalidation.
3554 */
3555 if (!atomic_cmpset_int((u_int *)pte, oldpte,
3556 oldpte & ~PG_RW))
3557 goto setpte;
3558 oldpte &= ~PG_RW;
3559 #ifdef KTR
3560 oldpteva = (oldpte & PG_FRAME & PDRMASK) |
3561 (va & ~PDRMASK);
3562 #endif
3563 CTR2(KTR_PMAP, "pmap_promote_pde: protect for va %#x"
3564 " in pmap %p", oldpteva, pmap);
3565 }
3566 if ((oldpte & PG_PTE_PROMOTE) != (newpde & PG_PTE_PROMOTE)) {
3567 pmap_pde_p_failures++;
3568 CTR2(KTR_PMAP, "pmap_promote_pde: failure for va %#x"
3569 " in pmap %p", va, pmap);
3570 return (false);
3571 }
3572 allpte_PG_A &= oldpte;
3573 pa -= PAGE_SIZE;
3574 }
3575
3576 /*
3577 * Unless all PTEs have PG_A set, clear it from the superpage mapping,
3578 * so that promotions triggered by speculative mappings, such as
3579 * pmap_enter_quick(), don't automatically mark the underlying pages
3580 * as referenced.
3581 */
3582 newpde &= ~PG_A | allpte_PG_A;
3583
3584 /*
3585 * Save the PTP in its current state until the PDE mapping the
3586 * superpage is demoted by pmap_demote_pde() or destroyed by
3587 * pmap_remove_pde(). If PG_A is not set in every PTE, then request
3588 * that the PTP be refilled on demotion.
3589 */
3590 if (mpte == NULL)
3591 mpte = PHYS_TO_VM_PAGE(*pde & PG_FRAME);
3592 KASSERT(mpte >= vm_page_array &&
3593 mpte < &vm_page_array[vm_page_array_size],
3594 ("pmap_promote_pde: page table page is out of range"));
3595 KASSERT(mpte->pindex == va >> PDRSHIFT,
3596 ("pmap_promote_pde: page table page's pindex is wrong"));
3597 if (pmap_insert_pt_page(pmap, mpte, true, allpte_PG_A != 0)) {
3598 pmap_pde_p_failures++;
3599 CTR2(KTR_PMAP,
3600 "pmap_promote_pde: failure for va %#x in pmap %p", va,
3601 pmap);
3602 return (false);
3603 }
3604
3605 /*
3606 * Promote the pv entries.
3607 */
3608 if ((newpde & PG_MANAGED) != 0)
3609 pmap_pv_promote_pde(pmap, va, newpde & PG_PS_FRAME);
3610
3611 /*
3612 * Propagate the PAT index to its proper position.
3613 */
3614 if ((newpde & PG_PTE_PAT) != 0)
3615 newpde ^= PG_PDE_PAT | PG_PTE_PAT;
3616
3617 /*
3618 * Map the superpage.
3619 */
3620 if (workaround_erratum383)
3621 pmap_update_pde(pmap, va, pde, PG_PS | newpde);
3622 else if (pmap == kernel_pmap)
3623 pmap_kenter_pde(va, PG_PROMOTED | PG_PS | newpde);
3624 else
3625 pde_store(pde, PG_PROMOTED | PG_PS | newpde);
3626
3627 pmap_pde_promotions++;
3628 CTR2(KTR_PMAP, "pmap_promote_pde: success for va %#x"
3629 " in pmap %p", va, pmap);
3630 return (true);
3631 }
3632 #endif /* VM_NRESERVLEVEL > 0 */
3633
3634 /*
3635 * Insert the given physical page (p) at
3636 * the specified virtual address (v) in the
3637 * target physical map with the protection requested.
3638 *
3639 * If specified, the page will be wired down, meaning
3640 * that the related pte can not be reclaimed.
3641 *
3642 * NB: This is the only routine which MAY NOT lazy-evaluate
3643 * or lose information. That is, this routine must actually
3644 * insert this page into the given map NOW.
3645 */
3646 static int
__CONCAT(PMTYPE,enter)3647 __CONCAT(PMTYPE, enter)(pmap_t pmap, vm_offset_t va, vm_page_t m,
3648 vm_prot_t prot, u_int flags, int8_t psind)
3649 {
3650 pd_entry_t *pde;
3651 pt_entry_t *pte;
3652 pt_entry_t newpte, origpte;
3653 pv_entry_t pv;
3654 vm_paddr_t opa, pa;
3655 vm_page_t mpte, om;
3656 int rv;
3657
3658 va = trunc_page(va);
3659 KASSERT((pmap == kernel_pmap && va < VM_MAX_KERNEL_ADDRESS) ||
3660 (pmap != kernel_pmap && va < VM_MAXUSER_ADDRESS),
3661 ("pmap_enter: toobig k%d %#x", pmap == kernel_pmap, va));
3662 KASSERT(va < PMAP_TRM_MIN_ADDRESS,
3663 ("pmap_enter: invalid to pmap_enter into trampoline (va: 0x%x)",
3664 va));
3665 KASSERT(pmap != kernel_pmap || (m->oflags & VPO_UNMANAGED) != 0 ||
3666 !VA_IS_CLEANMAP(va),
3667 ("pmap_enter: managed mapping within the clean submap"));
3668 if ((m->oflags & VPO_UNMANAGED) == 0)
3669 VM_PAGE_OBJECT_BUSY_ASSERT(m);
3670 KASSERT((flags & PMAP_ENTER_RESERVED) == 0,
3671 ("pmap_enter: flags %u has reserved bits set", flags));
3672 pa = VM_PAGE_TO_PHYS(m);
3673 newpte = (pt_entry_t)(pa | PG_A | PG_V);
3674 if ((flags & VM_PROT_WRITE) != 0)
3675 newpte |= PG_M;
3676 if ((prot & VM_PROT_WRITE) != 0)
3677 newpte |= PG_RW;
3678 KASSERT((newpte & (PG_M | PG_RW)) != PG_M,
3679 ("pmap_enter: flags includes VM_PROT_WRITE but prot doesn't"));
3680 #ifdef PMAP_PAE_COMP
3681 if ((prot & VM_PROT_EXECUTE) == 0 && !i386_read_exec)
3682 newpte |= pg_nx;
3683 #endif
3684 if ((flags & PMAP_ENTER_WIRED) != 0)
3685 newpte |= PG_W;
3686 if (pmap != kernel_pmap)
3687 newpte |= PG_U;
3688 newpte |= pmap_cache_bits(pmap, m->md.pat_mode, psind > 0);
3689 if ((m->oflags & VPO_UNMANAGED) == 0)
3690 newpte |= PG_MANAGED;
3691
3692 rw_wlock(&pvh_global_lock);
3693 PMAP_LOCK(pmap);
3694 sched_pin();
3695 if (psind == 1) {
3696 /* Assert the required virtual and physical alignment. */
3697 KASSERT((va & PDRMASK) == 0, ("pmap_enter: va unaligned"));
3698 KASSERT(m->psind > 0, ("pmap_enter: m->psind < psind"));
3699 rv = pmap_enter_pde(pmap, va, newpte | PG_PS, flags, m);
3700 goto out;
3701 }
3702
3703 pde = pmap_pde(pmap, va);
3704 if (pmap != kernel_pmap) {
3705 /*
3706 * va is for UVA.
3707 * In the case that a page table page is not resident,
3708 * we are creating it here. pmap_allocpte() handles
3709 * demotion.
3710 */
3711 mpte = pmap_allocpte(pmap, va, flags);
3712 if (mpte == NULL) {
3713 KASSERT((flags & PMAP_ENTER_NOSLEEP) != 0,
3714 ("pmap_allocpte failed with sleep allowed"));
3715 rv = KERN_RESOURCE_SHORTAGE;
3716 goto out;
3717 }
3718 } else {
3719 /*
3720 * va is for KVA, so pmap_demote_pde() will never fail
3721 * to install a page table page. PG_V is also
3722 * asserted by pmap_demote_pde().
3723 */
3724 mpte = NULL;
3725 KASSERT(pde != NULL && (*pde & PG_V) != 0,
3726 ("KVA %#x invalid pde pdir %#jx", va,
3727 (uintmax_t)pmap->pm_pdir[PTDPTDI]));
3728 if ((*pde & PG_PS) != 0)
3729 pmap_demote_pde(pmap, pde, va);
3730 }
3731 pte = pmap_pte_quick(pmap, va);
3732
3733 /*
3734 * Page Directory table entry is not valid, which should not
3735 * happen. We should have either allocated the page table
3736 * page or demoted the existing mapping above.
3737 */
3738 if (pte == NULL) {
3739 panic("pmap_enter: invalid page directory pdir=%#jx, va=%#x",
3740 (uintmax_t)pmap->pm_pdir[PTDPTDI], va);
3741 }
3742
3743 origpte = *pte;
3744 pv = NULL;
3745
3746 /*
3747 * Is the specified virtual address already mapped?
3748 */
3749 if ((origpte & PG_V) != 0) {
3750 /*
3751 * Wiring change, just update stats. We don't worry about
3752 * wiring PT pages as they remain resident as long as there
3753 * are valid mappings in them. Hence, if a user page is wired,
3754 * the PT page will be also.
3755 */
3756 if ((newpte & PG_W) != 0 && (origpte & PG_W) == 0)
3757 pmap->pm_stats.wired_count++;
3758 else if ((newpte & PG_W) == 0 && (origpte & PG_W) != 0)
3759 pmap->pm_stats.wired_count--;
3760
3761 /*
3762 * Remove the extra PT page reference.
3763 */
3764 if (mpte != NULL) {
3765 mpte->ref_count--;
3766 KASSERT(mpte->ref_count > 0,
3767 ("pmap_enter: missing reference to page table page,"
3768 " va: 0x%x", va));
3769 }
3770
3771 /*
3772 * Has the physical page changed?
3773 */
3774 opa = origpte & PG_FRAME;
3775 if (opa == pa) {
3776 /*
3777 * No, might be a protection or wiring change.
3778 */
3779 if ((origpte & PG_MANAGED) != 0 &&
3780 (newpte & PG_RW) != 0)
3781 vm_page_aflag_set(m, PGA_WRITEABLE);
3782 if (((origpte ^ newpte) & ~(PG_M | PG_A)) == 0)
3783 goto unchanged;
3784 goto validate;
3785 }
3786
3787 /*
3788 * The physical page has changed. Temporarily invalidate
3789 * the mapping. This ensures that all threads sharing the
3790 * pmap keep a consistent view of the mapping, which is
3791 * necessary for the correct handling of COW faults. It
3792 * also permits reuse of the old mapping's PV entry,
3793 * avoiding an allocation.
3794 *
3795 * For consistency, handle unmanaged mappings the same way.
3796 */
3797 origpte = pte_load_clear(pte);
3798 KASSERT((origpte & PG_FRAME) == opa,
3799 ("pmap_enter: unexpected pa update for %#x", va));
3800 if ((origpte & PG_MANAGED) != 0) {
3801 om = PHYS_TO_VM_PAGE(opa);
3802
3803 /*
3804 * The pmap lock is sufficient to synchronize with
3805 * concurrent calls to pmap_page_test_mappings() and
3806 * pmap_ts_referenced().
3807 */
3808 if ((origpte & (PG_M | PG_RW)) == (PG_M | PG_RW))
3809 vm_page_dirty(om);
3810 if ((origpte & PG_A) != 0) {
3811 pmap_invalidate_page_int(pmap, va);
3812 vm_page_aflag_set(om, PGA_REFERENCED);
3813 }
3814 pv = pmap_pvh_remove(&om->md, pmap, va);
3815 KASSERT(pv != NULL,
3816 ("pmap_enter: no PV entry for %#x", va));
3817 if ((newpte & PG_MANAGED) == 0)
3818 free_pv_entry(pmap, pv);
3819 if ((om->a.flags & PGA_WRITEABLE) != 0 &&
3820 TAILQ_EMPTY(&om->md.pv_list) &&
3821 ((om->flags & PG_FICTITIOUS) != 0 ||
3822 TAILQ_EMPTY(&pa_to_pvh(opa)->pv_list)))
3823 vm_page_aflag_clear(om, PGA_WRITEABLE);
3824 } else {
3825 /*
3826 * Since this mapping is unmanaged, assume that PG_A
3827 * is set.
3828 */
3829 pmap_invalidate_page_int(pmap, va);
3830 }
3831 origpte = 0;
3832 } else {
3833 /*
3834 * Increment the counters.
3835 */
3836 if ((newpte & PG_W) != 0)
3837 pmap->pm_stats.wired_count++;
3838 pmap->pm_stats.resident_count++;
3839 }
3840
3841 /*
3842 * Enter on the PV list if part of our managed memory.
3843 */
3844 if ((newpte & PG_MANAGED) != 0) {
3845 if (pv == NULL) {
3846 pv = get_pv_entry(pmap, false);
3847 pv->pv_va = va;
3848 }
3849 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next);
3850 if ((newpte & PG_RW) != 0)
3851 vm_page_aflag_set(m, PGA_WRITEABLE);
3852 }
3853
3854 /*
3855 * Update the PTE.
3856 */
3857 if ((origpte & PG_V) != 0) {
3858 validate:
3859 origpte = pte_load_store(pte, newpte);
3860 KASSERT((origpte & PG_FRAME) == pa,
3861 ("pmap_enter: unexpected pa update for %#x", va));
3862 if ((newpte & PG_M) == 0 && (origpte & (PG_M | PG_RW)) ==
3863 (PG_M | PG_RW)) {
3864 if ((origpte & PG_MANAGED) != 0)
3865 vm_page_dirty(m);
3866
3867 /*
3868 * Although the PTE may still have PG_RW set, TLB
3869 * invalidation may nonetheless be required because
3870 * the PTE no longer has PG_M set.
3871 */
3872 }
3873 #ifdef PMAP_PAE_COMP
3874 else if ((origpte & PG_NX) != 0 || (newpte & PG_NX) == 0) {
3875 /*
3876 * This PTE change does not require TLB invalidation.
3877 */
3878 goto unchanged;
3879 }
3880 #endif
3881 if ((origpte & PG_A) != 0)
3882 pmap_invalidate_page_int(pmap, va);
3883 } else
3884 pte_store_zero(pte, newpte);
3885
3886 unchanged:
3887
3888 #if VM_NRESERVLEVEL > 0
3889 /*
3890 * If both the page table page and the reservation are fully
3891 * populated, then attempt promotion.
3892 */
3893 if ((mpte == NULL || mpte->ref_count == NPTEPG) &&
3894 (m->flags & PG_FICTITIOUS) == 0 &&
3895 vm_reserv_level_iffullpop(m) == 0)
3896 (void)pmap_promote_pde(pmap, pde, va, mpte);
3897 #endif
3898
3899 rv = KERN_SUCCESS;
3900 out:
3901 sched_unpin();
3902 rw_wunlock(&pvh_global_lock);
3903 PMAP_UNLOCK(pmap);
3904 return (rv);
3905 }
3906
3907 /*
3908 * Tries to create a read- and/or execute-only 2 or 4 MB page mapping. Returns
3909 * KERN_SUCCESS if the mapping was created. Otherwise, returns an error
3910 * value. See pmap_enter_pde() for the possible error values when "no sleep",
3911 * "no replace", and "no reclaim" are specified.
3912 */
3913 static int
pmap_enter_4mpage(pmap_t pmap,vm_offset_t va,vm_page_t m,vm_prot_t prot)3914 pmap_enter_4mpage(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot)
3915 {
3916 pd_entry_t newpde;
3917
3918 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
3919 newpde = VM_PAGE_TO_PHYS(m) |
3920 pmap_cache_bits(pmap, m->md.pat_mode, true) | PG_PS | PG_V;
3921 if ((m->oflags & VPO_UNMANAGED) == 0)
3922 newpde |= PG_MANAGED;
3923 #ifdef PMAP_PAE_COMP
3924 if ((prot & VM_PROT_EXECUTE) == 0 && !i386_read_exec)
3925 newpde |= pg_nx;
3926 #endif
3927 if (pmap != kernel_pmap)
3928 newpde |= PG_U;
3929 return (pmap_enter_pde(pmap, va, newpde, PMAP_ENTER_NOSLEEP |
3930 PMAP_ENTER_NOREPLACE | PMAP_ENTER_NORECLAIM, NULL));
3931 }
3932
3933 /*
3934 * Returns true if every page table entry in the page table page that maps
3935 * the specified kernel virtual address is zero.
3936 */
3937 static bool
pmap_every_pte_zero(vm_offset_t va)3938 pmap_every_pte_zero(vm_offset_t va)
3939 {
3940 pt_entry_t *pt_end, *pte;
3941
3942 KASSERT((va & PDRMASK) == 0, ("va is misaligned"));
3943 pte = vtopte(va);
3944 for (pt_end = pte + NPTEPG; pte < pt_end; pte++) {
3945 if (*pte != 0)
3946 return (false);
3947 }
3948 return (true);
3949 }
3950
3951 /*
3952 * Tries to create the specified 2 or 4 MB page mapping. Returns KERN_SUCCESS
3953 * if the mapping was created, and one of KERN_FAILURE, KERN_NO_SPACE,
3954 * or KERN_RESOURCE_SHORTAGE otherwise. Returns KERN_FAILURE if
3955 * PMAP_ENTER_NOREPLACE was specified and a 4 KB page mapping already exists
3956 * within the 2 or 4 MB virtual address range starting at the specified virtual
3957 * address. Returns KERN_NO_SPACE if PMAP_ENTER_NOREPLACE was specified and a
3958 * 2 or 4 MB page mapping already exists at the specified virtual address.
3959 * Returns KERN_RESOURCE_SHORTAGE if PMAP_ENTER_NORECLAIM was specified and a
3960 * PV entry allocation failed.
3961 *
3962 * The parameter "m" is only used when creating a managed, writeable mapping.
3963 */
3964 static int
pmap_enter_pde(pmap_t pmap,vm_offset_t va,pd_entry_t newpde,u_int flags,vm_page_t m)3965 pmap_enter_pde(pmap_t pmap, vm_offset_t va, pd_entry_t newpde, u_int flags,
3966 vm_page_t m)
3967 {
3968 struct spglist free;
3969 pd_entry_t oldpde, *pde;
3970 vm_page_t mt;
3971 vm_page_t uwptpg;
3972
3973 rw_assert(&pvh_global_lock, RA_WLOCKED);
3974 KASSERT((newpde & (PG_M | PG_RW)) != PG_RW,
3975 ("pmap_enter_pde: newpde is missing PG_M"));
3976 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
3977 pde = pmap_pde(pmap, va);
3978 oldpde = *pde;
3979 if ((oldpde & PG_V) != 0) {
3980 if ((flags & PMAP_ENTER_NOREPLACE) != 0) {
3981 if ((oldpde & PG_PS) != 0) {
3982 CTR2(KTR_PMAP,
3983 "pmap_enter_pde: no space for va %#lx"
3984 " in pmap %p", va, pmap);
3985 return (KERN_NO_SPACE);
3986 } else if (pmap != kernel_pmap ||
3987 !pmap_every_pte_zero(va)) {
3988 CTR2(KTR_PMAP,
3989 "pmap_enter_pde: failure for va %#lx"
3990 " in pmap %p", va, pmap);
3991 return (KERN_FAILURE);
3992 }
3993 }
3994 /* Break the existing mapping(s). */
3995 SLIST_INIT(&free);
3996 if ((oldpde & PG_PS) != 0) {
3997 /*
3998 * If the PDE resulted from a promotion, then a
3999 * reserved PT page could be freed.
4000 */
4001 (void)pmap_remove_pde(pmap, pde, va, &free);
4002 if ((oldpde & PG_G) == 0)
4003 pmap_invalidate_pde_page(pmap, va, oldpde);
4004 } else {
4005 if (pmap_remove_ptes(pmap, va, va + NBPDR, &free))
4006 pmap_invalidate_all_int(pmap);
4007 }
4008 if (pmap != kernel_pmap) {
4009 vm_page_free_pages_toq(&free, true);
4010 KASSERT(*pde == 0, ("pmap_enter_pde: non-zero pde %p",
4011 pde));
4012 } else {
4013 KASSERT(SLIST_EMPTY(&free),
4014 ("pmap_enter_pde: freed kernel page table page"));
4015
4016 /*
4017 * Both pmap_remove_pde() and pmap_remove_ptes() will
4018 * leave the kernel page table page zero filled.
4019 */
4020 mt = PHYS_TO_VM_PAGE(*pde & PG_FRAME);
4021 if (pmap_insert_pt_page(pmap, mt, false, false))
4022 panic("pmap_enter_pde: trie insert failed");
4023 }
4024 }
4025
4026 /*
4027 * Allocate a leaf ptpage for wired userspace pages.
4028 */
4029 uwptpg = NULL;
4030 if ((newpde & PG_W) != 0 && pmap != kernel_pmap) {
4031 uwptpg = vm_page_alloc_noobj(VM_ALLOC_WIRED);
4032 if (uwptpg == NULL) {
4033 return (KERN_RESOURCE_SHORTAGE);
4034 }
4035 uwptpg->pindex = va >> PDRSHIFT;
4036 if (pmap_insert_pt_page(pmap, uwptpg, true, false)) {
4037 vm_page_unwire_noq(uwptpg);
4038 vm_page_free(uwptpg);
4039 return (KERN_RESOURCE_SHORTAGE);
4040 }
4041 pmap->pm_stats.resident_count++;
4042 uwptpg->ref_count = NPTEPG;
4043 }
4044 if ((newpde & PG_MANAGED) != 0) {
4045 /*
4046 * Abort this mapping if its PV entry could not be created.
4047 */
4048 if (!pmap_pv_insert_pde(pmap, va, newpde, flags)) {
4049 if (uwptpg != NULL) {
4050 mt = pmap_remove_pt_page(pmap, va);
4051 KASSERT(mt == uwptpg,
4052 ("removed pt page %p, expected %p", mt,
4053 uwptpg));
4054 pmap->pm_stats.resident_count--;
4055 uwptpg->ref_count = 1;
4056 vm_page_unwire_noq(uwptpg);
4057 vm_page_free(uwptpg);
4058 }
4059 CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx"
4060 " in pmap %p", va, pmap);
4061 return (KERN_RESOURCE_SHORTAGE);
4062 }
4063 if ((newpde & PG_RW) != 0) {
4064 for (mt = m; mt < &m[NBPDR / PAGE_SIZE]; mt++)
4065 vm_page_aflag_set(mt, PGA_WRITEABLE);
4066 }
4067 }
4068
4069 /*
4070 * Increment counters.
4071 */
4072 if ((newpde & PG_W) != 0)
4073 pmap->pm_stats.wired_count += NBPDR / PAGE_SIZE;
4074 pmap->pm_stats.resident_count += NBPDR / PAGE_SIZE;
4075
4076 /*
4077 * Map the superpage. (This is not a promoted mapping; there will not
4078 * be any lingering 4KB page mappings in the TLB.)
4079 */
4080 pde_store(pde, newpde);
4081
4082 pmap_pde_mappings++;
4083 CTR2(KTR_PMAP, "pmap_enter_pde: success for va %#lx in pmap %p",
4084 va, pmap);
4085 return (KERN_SUCCESS);
4086 }
4087
4088 /*
4089 * Maps a sequence of resident pages belonging to the same object.
4090 * The sequence begins with the given page m_start. This page is
4091 * mapped at the given virtual address start. Each subsequent page is
4092 * mapped at a virtual address that is offset from start by the same
4093 * amount as the page is offset from m_start within the object. The
4094 * last page in the sequence is the page with the largest offset from
4095 * m_start that can be mapped at a virtual address less than the given
4096 * virtual address end. Not every virtual page between start and end
4097 * is mapped; only those for which a resident page exists with the
4098 * corresponding offset from m_start are mapped.
4099 */
4100 static void
__CONCAT(PMTYPE,enter_object)4101 __CONCAT(PMTYPE, enter_object)(pmap_t pmap, vm_offset_t start, vm_offset_t end,
4102 vm_page_t m_start, vm_prot_t prot)
4103 {
4104 struct pctrie_iter pages;
4105 vm_offset_t va;
4106 vm_page_t m, mpte;
4107 int rv;
4108
4109 VM_OBJECT_ASSERT_LOCKED(m_start->object);
4110
4111 mpte = NULL;
4112 vm_page_iter_limit_init(&pages, m_start->object,
4113 m_start->pindex + atop(end - start));
4114 m = vm_radix_iter_lookup(&pages, m_start->pindex);
4115 rw_wlock(&pvh_global_lock);
4116 PMAP_LOCK(pmap);
4117 while (m != NULL) {
4118 va = start + ptoa(m->pindex - m_start->pindex);
4119 if ((va & PDRMASK) == 0 && va + NBPDR <= end &&
4120 m->psind == 1 && pg_ps_enabled &&
4121 ((rv = pmap_enter_4mpage(pmap, va, m, prot)) ==
4122 KERN_SUCCESS || rv == KERN_NO_SPACE)) {
4123 m = vm_radix_iter_jump(&pages, NBPDR / PAGE_SIZE);
4124 } else {
4125 mpte = pmap_enter_quick_locked(pmap, va, m, prot, mpte);
4126 m = vm_radix_iter_step(&pages);
4127 }
4128 }
4129 rw_wunlock(&pvh_global_lock);
4130 PMAP_UNLOCK(pmap);
4131 }
4132
4133 /*
4134 * this code makes some *MAJOR* assumptions:
4135 * 1. Current pmap & pmap exists.
4136 * 2. Not wired.
4137 * 3. Read access.
4138 * 4. No page table pages.
4139 * but is *MUCH* faster than pmap_enter...
4140 */
4141
4142 static void
__CONCAT(PMTYPE,enter_quick)4143 __CONCAT(PMTYPE, enter_quick)(pmap_t pmap, vm_offset_t va, vm_page_t m,
4144 vm_prot_t prot)
4145 {
4146
4147 rw_wlock(&pvh_global_lock);
4148 PMAP_LOCK(pmap);
4149 (void)pmap_enter_quick_locked(pmap, va, m, prot, NULL);
4150 rw_wunlock(&pvh_global_lock);
4151 PMAP_UNLOCK(pmap);
4152 }
4153
4154 static vm_page_t
pmap_enter_quick_locked(pmap_t pmap,vm_offset_t va,vm_page_t m,vm_prot_t prot,vm_page_t mpte)4155 pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m,
4156 vm_prot_t prot, vm_page_t mpte)
4157 {
4158 pt_entry_t newpte, *pte;
4159 pd_entry_t *pde;
4160
4161 KASSERT(pmap != kernel_pmap || !VA_IS_CLEANMAP(va) ||
4162 (m->oflags & VPO_UNMANAGED) != 0,
4163 ("pmap_enter_quick_locked: managed mapping within the clean submap"));
4164 rw_assert(&pvh_global_lock, RA_WLOCKED);
4165 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
4166 pde = NULL;
4167
4168 /*
4169 * In the case that a page table page is not
4170 * resident, we are creating it here.
4171 */
4172 if (pmap != kernel_pmap) {
4173 u_int ptepindex;
4174 pd_entry_t ptepa;
4175
4176 /*
4177 * Calculate pagetable page index
4178 */
4179 ptepindex = va >> PDRSHIFT;
4180 if (mpte && (mpte->pindex == ptepindex)) {
4181 mpte->ref_count++;
4182 } else {
4183 /*
4184 * Get the page directory entry
4185 */
4186 pde = &pmap->pm_pdir[ptepindex];
4187 ptepa = *pde;
4188
4189 /*
4190 * If the page table page is mapped, we just increment
4191 * the hold count, and activate it.
4192 */
4193 if (ptepa) {
4194 if (ptepa & PG_PS)
4195 return (NULL);
4196 mpte = PHYS_TO_VM_PAGE(ptepa & PG_FRAME);
4197 mpte->ref_count++;
4198 } else {
4199 mpte = _pmap_allocpte(pmap, ptepindex,
4200 PMAP_ENTER_NOSLEEP);
4201 if (mpte == NULL)
4202 return (mpte);
4203 }
4204 }
4205 } else {
4206 mpte = NULL;
4207 }
4208
4209 sched_pin();
4210 pte = pmap_pte_quick(pmap, va);
4211 if (*pte) {
4212 if (mpte != NULL)
4213 mpte->ref_count--;
4214 sched_unpin();
4215 return (NULL);
4216 }
4217
4218 /*
4219 * Enter on the PV list if part of our managed memory.
4220 */
4221 if ((m->oflags & VPO_UNMANAGED) == 0 &&
4222 !pmap_try_insert_pv_entry(pmap, va, m)) {
4223 if (mpte != NULL)
4224 pmap_abort_ptp(pmap, va, mpte);
4225 sched_unpin();
4226 return (NULL);
4227 }
4228
4229 /*
4230 * Increment counters
4231 */
4232 pmap->pm_stats.resident_count++;
4233
4234 newpte = VM_PAGE_TO_PHYS(m) | PG_V |
4235 pmap_cache_bits(pmap, m->md.pat_mode, false);
4236 if ((m->oflags & VPO_UNMANAGED) == 0)
4237 newpte |= PG_MANAGED;
4238 #ifdef PMAP_PAE_COMP
4239 if ((prot & VM_PROT_EXECUTE) == 0 && !i386_read_exec)
4240 newpte |= pg_nx;
4241 #endif
4242 if (pmap != kernel_pmap)
4243 newpte |= PG_U;
4244 pte_store_zero(pte, newpte);
4245
4246 #if VM_NRESERVLEVEL > 0
4247 /*
4248 * If both the PTP and the reservation are fully populated, then
4249 * attempt promotion.
4250 */
4251 if ((prot & VM_PROT_NO_PROMOTE) == 0 &&
4252 (mpte == NULL || mpte->ref_count == NPTEPG) &&
4253 (m->flags & PG_FICTITIOUS) == 0 &&
4254 vm_reserv_level_iffullpop(m) == 0) {
4255 if (pde == NULL)
4256 pde = pmap_pde(pmap, va);
4257
4258 /*
4259 * If promotion succeeds, then the next call to this function
4260 * should not be given the unmapped PTP as a hint.
4261 */
4262 if (pmap_promote_pde(pmap, pde, va, mpte))
4263 mpte = NULL;
4264 }
4265 #endif
4266
4267 sched_unpin();
4268 return (mpte);
4269 }
4270
4271 /*
4272 * Make a temporary mapping for a physical address. This is only intended
4273 * to be used for panic dumps.
4274 */
4275 static void *
__CONCAT(PMTYPE,kenter_temporary)4276 __CONCAT(PMTYPE, kenter_temporary)(vm_paddr_t pa, int i)
4277 {
4278 vm_offset_t va;
4279
4280 va = (vm_offset_t)crashdumpmap + (i * PAGE_SIZE);
4281 pmap_kenter(va, pa);
4282 invlpg(va);
4283 return ((void *)crashdumpmap);
4284 }
4285
4286 /*
4287 * This code maps large physical mmap regions into the
4288 * processor address space. Note that some shortcuts
4289 * are taken, but the code works.
4290 */
4291 static void
__CONCAT(PMTYPE,object_init_pt)4292 __CONCAT(PMTYPE, object_init_pt)(pmap_t pmap, vm_offset_t addr,
4293 vm_object_t object, vm_pindex_t pindex, vm_size_t size)
4294 {
4295 struct pctrie_iter pages;
4296 pd_entry_t *pde;
4297 vm_paddr_t pa, ptepa;
4298 vm_page_t p;
4299 int pat_mode;
4300
4301 VM_OBJECT_ASSERT_WLOCKED(object);
4302 KASSERT(object->type == OBJT_DEVICE || object->type == OBJT_SG,
4303 ("pmap_object_init_pt: non-device object"));
4304 if (pg_ps_enabled &&
4305 (addr & (NBPDR - 1)) == 0 && (size & (NBPDR - 1)) == 0) {
4306 if (!vm_object_populate(object, pindex, pindex + atop(size)))
4307 return;
4308 vm_page_iter_init(&pages, object);
4309 p = vm_radix_iter_lookup(&pages, pindex);
4310 KASSERT(vm_page_all_valid(p),
4311 ("pmap_object_init_pt: invalid page %p", p));
4312 pat_mode = p->md.pat_mode;
4313
4314 /*
4315 * Abort the mapping if the first page is not physically
4316 * aligned to a 2/4MB page boundary.
4317 */
4318 ptepa = VM_PAGE_TO_PHYS(p);
4319 if (ptepa & (NBPDR - 1))
4320 return;
4321
4322 /*
4323 * Skip the first page. Abort the mapping if the rest of
4324 * the pages are not physically contiguous or have differing
4325 * memory attributes.
4326 */
4327 for (pa = ptepa + PAGE_SIZE; pa < ptepa + size;
4328 pa += PAGE_SIZE) {
4329 p = vm_radix_iter_next(&pages);
4330 KASSERT(vm_page_all_valid(p),
4331 ("pmap_object_init_pt: invalid page %p", p));
4332 if (pa != VM_PAGE_TO_PHYS(p) ||
4333 pat_mode != p->md.pat_mode)
4334 return;
4335 }
4336
4337 /*
4338 * Map using 2/4MB pages. Since "ptepa" is 2/4M aligned and
4339 * "size" is a multiple of 2/4M, adding the PAT setting to
4340 * "pa" will not affect the termination of this loop.
4341 */
4342 PMAP_LOCK(pmap);
4343 for (pa = ptepa | pmap_cache_bits(pmap, pat_mode, true);
4344 pa < ptepa + size; pa += NBPDR) {
4345 pde = pmap_pde(pmap, addr);
4346 if (*pde == 0) {
4347 pde_store(pde, pa | PG_PS | PG_M | PG_A |
4348 PG_U | PG_RW | PG_V);
4349 pmap->pm_stats.resident_count += NBPDR /
4350 PAGE_SIZE;
4351 pmap_pde_mappings++;
4352 }
4353 /* Else continue on if the PDE is already valid. */
4354 addr += NBPDR;
4355 }
4356 PMAP_UNLOCK(pmap);
4357 }
4358 }
4359
4360 /*
4361 * Clear the wired attribute from the mappings for the specified range of
4362 * addresses in the given pmap. Every valid mapping within that range
4363 * must have the wired attribute set. In contrast, invalid mappings
4364 * cannot have the wired attribute set, so they are ignored.
4365 *
4366 * The wired attribute of the page table entry is not a hardware feature,
4367 * so there is no need to invalidate any TLB entries.
4368 */
4369 static void
__CONCAT(PMTYPE,unwire)4370 __CONCAT(PMTYPE, unwire)(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
4371 {
4372 vm_offset_t pdnxt;
4373 pd_entry_t *pde;
4374 pt_entry_t *pte;
4375 bool pv_lists_locked;
4376
4377 if (pmap_is_current(pmap))
4378 pv_lists_locked = false;
4379 else {
4380 pv_lists_locked = true;
4381 resume:
4382 rw_wlock(&pvh_global_lock);
4383 sched_pin();
4384 }
4385 PMAP_LOCK(pmap);
4386 for (; sva < eva; sva = pdnxt) {
4387 pdnxt = (sva + NBPDR) & ~PDRMASK;
4388 if (pdnxt < sva)
4389 pdnxt = eva;
4390 pde = pmap_pde(pmap, sva);
4391 if ((*pde & PG_V) == 0)
4392 continue;
4393 if ((*pde & PG_PS) != 0) {
4394 if ((*pde & PG_W) == 0)
4395 panic("pmap_unwire: pde %#jx is missing PG_W",
4396 (uintmax_t)*pde);
4397
4398 /*
4399 * Are we unwiring the entire large page? If not,
4400 * demote the mapping and fall through.
4401 */
4402 if (sva + NBPDR == pdnxt && eva >= pdnxt) {
4403 /*
4404 * Regardless of whether a pde (or pte) is 32
4405 * or 64 bits in size, PG_W is among the least
4406 * significant 32 bits.
4407 */
4408 atomic_clear_int((u_int *)pde, PG_W);
4409 pmap->pm_stats.wired_count -= NBPDR /
4410 PAGE_SIZE;
4411 continue;
4412 } else {
4413 if (!pv_lists_locked) {
4414 pv_lists_locked = true;
4415 if (!rw_try_wlock(&pvh_global_lock)) {
4416 PMAP_UNLOCK(pmap);
4417 /* Repeat sva. */
4418 goto resume;
4419 }
4420 sched_pin();
4421 }
4422 if (!pmap_demote_pde(pmap, pde, sva))
4423 panic("pmap_unwire: demotion failed");
4424 }
4425 }
4426 if (pdnxt > eva)
4427 pdnxt = eva;
4428 for (pte = pmap_pte_quick(pmap, sva); sva != pdnxt; pte++,
4429 sva += PAGE_SIZE) {
4430 if ((*pte & PG_V) == 0)
4431 continue;
4432 if ((*pte & PG_W) == 0)
4433 panic("pmap_unwire: pte %#jx is missing PG_W",
4434 (uintmax_t)*pte);
4435
4436 /*
4437 * PG_W must be cleared atomically. Although the pmap
4438 * lock synchronizes access to PG_W, another processor
4439 * could be setting PG_M and/or PG_A concurrently.
4440 *
4441 * PG_W is among the least significant 32 bits.
4442 */
4443 atomic_clear_int((u_int *)pte, PG_W);
4444 pmap->pm_stats.wired_count--;
4445 }
4446 }
4447 if (pv_lists_locked) {
4448 sched_unpin();
4449 rw_wunlock(&pvh_global_lock);
4450 }
4451 PMAP_UNLOCK(pmap);
4452 }
4453
4454 /*
4455 * Copy the range specified by src_addr/len
4456 * from the source map to the range dst_addr/len
4457 * in the destination map.
4458 *
4459 * This routine is only advisory and need not do anything. Since
4460 * current pmap is always the kernel pmap when executing in
4461 * kernel, and we do not copy from the kernel pmap to a user
4462 * pmap, this optimization is not usable in 4/4G full split i386
4463 * world.
4464 */
4465
4466 static void
__CONCAT(PMTYPE,copy)4467 __CONCAT(PMTYPE, copy)(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr,
4468 vm_size_t len, vm_offset_t src_addr)
4469 {
4470 pt_entry_t *src_pte, *dst_pte, ptetemp;
4471 pd_entry_t srcptepaddr;
4472 vm_page_t dstmpte, srcmpte;
4473 vm_offset_t addr, end_addr, pdnxt;
4474 u_int ptepindex;
4475
4476 if (dst_addr != src_addr)
4477 return;
4478
4479 end_addr = src_addr + len;
4480
4481 rw_wlock(&pvh_global_lock);
4482 if (dst_pmap < src_pmap) {
4483 PMAP_LOCK(dst_pmap);
4484 PMAP_LOCK(src_pmap);
4485 } else {
4486 PMAP_LOCK(src_pmap);
4487 PMAP_LOCK(dst_pmap);
4488 }
4489 sched_pin();
4490 for (addr = src_addr; addr < end_addr; addr = pdnxt) {
4491 KASSERT(addr < PMAP_TRM_MIN_ADDRESS,
4492 ("pmap_copy: invalid to pmap_copy the trampoline"));
4493
4494 pdnxt = (addr + NBPDR) & ~PDRMASK;
4495 if (pdnxt < addr)
4496 pdnxt = end_addr;
4497 ptepindex = addr >> PDRSHIFT;
4498
4499 srcptepaddr = src_pmap->pm_pdir[ptepindex];
4500 if (srcptepaddr == 0)
4501 continue;
4502
4503 if (srcptepaddr & PG_PS) {
4504 if ((addr & PDRMASK) != 0 || addr + NBPDR > end_addr)
4505 continue;
4506 if (dst_pmap->pm_pdir[ptepindex] == 0 &&
4507 ((srcptepaddr & PG_MANAGED) == 0 ||
4508 pmap_pv_insert_pde(dst_pmap, addr, srcptepaddr,
4509 PMAP_ENTER_NORECLAIM))) {
4510 dst_pmap->pm_pdir[ptepindex] = srcptepaddr &
4511 ~PG_W;
4512 dst_pmap->pm_stats.resident_count +=
4513 NBPDR / PAGE_SIZE;
4514 pmap_pde_mappings++;
4515 }
4516 continue;
4517 }
4518
4519 srcmpte = PHYS_TO_VM_PAGE(srcptepaddr & PG_FRAME);
4520 KASSERT(srcmpte->ref_count > 0,
4521 ("pmap_copy: source page table page is unused"));
4522
4523 if (pdnxt > end_addr)
4524 pdnxt = end_addr;
4525
4526 src_pte = pmap_pte_quick3(src_pmap, addr);
4527 while (addr < pdnxt) {
4528 ptetemp = *src_pte;
4529 /*
4530 * we only virtual copy managed pages
4531 */
4532 if ((ptetemp & PG_MANAGED) != 0) {
4533 dstmpte = pmap_allocpte(dst_pmap, addr,
4534 PMAP_ENTER_NOSLEEP);
4535 if (dstmpte == NULL)
4536 goto out;
4537 dst_pte = pmap_pte_quick(dst_pmap, addr);
4538 if (*dst_pte == 0 &&
4539 pmap_try_insert_pv_entry(dst_pmap, addr,
4540 PHYS_TO_VM_PAGE(ptetemp & PG_FRAME))) {
4541 /*
4542 * Clear the wired, modified, and
4543 * accessed (referenced) bits
4544 * during the copy.
4545 */
4546 *dst_pte = ptetemp & ~(PG_W | PG_M |
4547 PG_A);
4548 dst_pmap->pm_stats.resident_count++;
4549 } else {
4550 pmap_abort_ptp(dst_pmap, addr, dstmpte);
4551 goto out;
4552 }
4553 if (dstmpte->ref_count >= srcmpte->ref_count)
4554 break;
4555 }
4556 addr += PAGE_SIZE;
4557 src_pte++;
4558 }
4559 }
4560 out:
4561 sched_unpin();
4562 rw_wunlock(&pvh_global_lock);
4563 PMAP_UNLOCK(src_pmap);
4564 PMAP_UNLOCK(dst_pmap);
4565 }
4566
4567 /*
4568 * Zero 1 page of virtual memory mapped from a hardware page by the caller.
4569 */
4570 static __inline void
pagezero(void * page)4571 pagezero(void *page)
4572 {
4573 #if defined(I686_CPU)
4574 if (cpu_class == CPUCLASS_686) {
4575 if (cpu_feature & CPUID_SSE2)
4576 sse2_pagezero(page);
4577 else
4578 i686_pagezero(page);
4579 } else
4580 #endif
4581 bzero(page, PAGE_SIZE);
4582 }
4583
4584 /*
4585 * Zero the specified hardware page.
4586 */
4587 static void
__CONCAT(PMTYPE,zero_page)4588 __CONCAT(PMTYPE, zero_page)(vm_page_t m)
4589 {
4590 pt_entry_t *cmap_pte2;
4591 struct pcpu *pc;
4592
4593 sched_pin();
4594 pc = get_pcpu();
4595 cmap_pte2 = pc->pc_cmap_pte2;
4596 mtx_lock(&pc->pc_cmap_lock);
4597 if (*cmap_pte2)
4598 panic("pmap_zero_page: CMAP2 busy");
4599 *cmap_pte2 = PG_V | PG_RW | VM_PAGE_TO_PHYS(m) | PG_A | PG_M |
4600 pmap_cache_bits(kernel_pmap, m->md.pat_mode, false);
4601 invlcaddr(pc->pc_cmap_addr2);
4602 pagezero(pc->pc_cmap_addr2);
4603 *cmap_pte2 = 0;
4604
4605 /*
4606 * Unpin the thread before releasing the lock. Otherwise the thread
4607 * could be rescheduled while still bound to the current CPU, only
4608 * to unpin itself immediately upon resuming execution.
4609 */
4610 sched_unpin();
4611 mtx_unlock(&pc->pc_cmap_lock);
4612 }
4613
4614 /*
4615 * Zero an area within a single hardware page. off and size must not
4616 * cover an area beyond a single hardware page.
4617 */
4618 static void
__CONCAT(PMTYPE,zero_page_area)4619 __CONCAT(PMTYPE, zero_page_area)(vm_page_t m, int off, int size)
4620 {
4621 pt_entry_t *cmap_pte2;
4622 struct pcpu *pc;
4623
4624 sched_pin();
4625 pc = get_pcpu();
4626 cmap_pte2 = pc->pc_cmap_pte2;
4627 mtx_lock(&pc->pc_cmap_lock);
4628 if (*cmap_pte2)
4629 panic("pmap_zero_page_area: CMAP2 busy");
4630 *cmap_pte2 = PG_V | PG_RW | VM_PAGE_TO_PHYS(m) | PG_A | PG_M |
4631 pmap_cache_bits(kernel_pmap, m->md.pat_mode, false);
4632 invlcaddr(pc->pc_cmap_addr2);
4633 if (off == 0 && size == PAGE_SIZE)
4634 pagezero(pc->pc_cmap_addr2);
4635 else
4636 bzero(pc->pc_cmap_addr2 + off, size);
4637 *cmap_pte2 = 0;
4638 sched_unpin();
4639 mtx_unlock(&pc->pc_cmap_lock);
4640 }
4641
4642 /*
4643 * Copy 1 specified hardware page to another.
4644 */
4645 static void
__CONCAT(PMTYPE,copy_page)4646 __CONCAT(PMTYPE, copy_page)(vm_page_t src, vm_page_t dst)
4647 {
4648 pt_entry_t *cmap_pte1, *cmap_pte2;
4649 struct pcpu *pc;
4650
4651 sched_pin();
4652 pc = get_pcpu();
4653 cmap_pte1 = pc->pc_cmap_pte1;
4654 cmap_pte2 = pc->pc_cmap_pte2;
4655 mtx_lock(&pc->pc_cmap_lock);
4656 if (*cmap_pte1)
4657 panic("pmap_copy_page: CMAP1 busy");
4658 if (*cmap_pte2)
4659 panic("pmap_copy_page: CMAP2 busy");
4660 *cmap_pte1 = PG_V | VM_PAGE_TO_PHYS(src) | PG_A |
4661 pmap_cache_bits(kernel_pmap, src->md.pat_mode, false);
4662 invlcaddr(pc->pc_cmap_addr1);
4663 *cmap_pte2 = PG_V | PG_RW | VM_PAGE_TO_PHYS(dst) | PG_A | PG_M |
4664 pmap_cache_bits(kernel_pmap, dst->md.pat_mode, false);
4665 invlcaddr(pc->pc_cmap_addr2);
4666 bcopy(pc->pc_cmap_addr1, pc->pc_cmap_addr2, PAGE_SIZE);
4667 *cmap_pte1 = 0;
4668 *cmap_pte2 = 0;
4669 sched_unpin();
4670 mtx_unlock(&pc->pc_cmap_lock);
4671 }
4672
4673 static void
__CONCAT(PMTYPE,copy_pages)4674 __CONCAT(PMTYPE, copy_pages)(vm_page_t ma[], vm_offset_t a_offset,
4675 vm_page_t mb[], vm_offset_t b_offset, int xfersize)
4676 {
4677 vm_page_t a_pg, b_pg;
4678 char *a_cp, *b_cp;
4679 vm_offset_t a_pg_offset, b_pg_offset;
4680 pt_entry_t *cmap_pte1, *cmap_pte2;
4681 struct pcpu *pc;
4682 int cnt;
4683
4684 sched_pin();
4685 pc = get_pcpu();
4686 cmap_pte1 = pc->pc_cmap_pte1;
4687 cmap_pte2 = pc->pc_cmap_pte2;
4688 mtx_lock(&pc->pc_cmap_lock);
4689 if (*cmap_pte1 != 0)
4690 panic("pmap_copy_pages: CMAP1 busy");
4691 if (*cmap_pte2 != 0)
4692 panic("pmap_copy_pages: CMAP2 busy");
4693 while (xfersize > 0) {
4694 a_pg = ma[a_offset >> PAGE_SHIFT];
4695 a_pg_offset = a_offset & PAGE_MASK;
4696 cnt = min(xfersize, PAGE_SIZE - a_pg_offset);
4697 b_pg = mb[b_offset >> PAGE_SHIFT];
4698 b_pg_offset = b_offset & PAGE_MASK;
4699 cnt = min(cnt, PAGE_SIZE - b_pg_offset);
4700 *cmap_pte1 = PG_V | VM_PAGE_TO_PHYS(a_pg) | PG_A |
4701 pmap_cache_bits(kernel_pmap, a_pg->md.pat_mode, false);
4702 invlcaddr(pc->pc_cmap_addr1);
4703 *cmap_pte2 = PG_V | PG_RW | VM_PAGE_TO_PHYS(b_pg) | PG_A |
4704 PG_M | pmap_cache_bits(kernel_pmap, b_pg->md.pat_mode,
4705 false);
4706 invlcaddr(pc->pc_cmap_addr2);
4707 a_cp = pc->pc_cmap_addr1 + a_pg_offset;
4708 b_cp = pc->pc_cmap_addr2 + b_pg_offset;
4709 bcopy(a_cp, b_cp, cnt);
4710 a_offset += cnt;
4711 b_offset += cnt;
4712 xfersize -= cnt;
4713 }
4714 *cmap_pte1 = 0;
4715 *cmap_pte2 = 0;
4716 sched_unpin();
4717 mtx_unlock(&pc->pc_cmap_lock);
4718 }
4719
4720 /*
4721 * Returns true if the pmap's pv is one of the first
4722 * 16 pvs linked to from this page. This count may
4723 * be changed upwards or downwards in the future; it
4724 * is only necessary that true be returned for a small
4725 * subset of pmaps for proper page aging.
4726 */
4727 static bool
__CONCAT(PMTYPE,page_exists_quick)4728 __CONCAT(PMTYPE, page_exists_quick)(pmap_t pmap, vm_page_t m)
4729 {
4730 struct md_page *pvh;
4731 pv_entry_t pv;
4732 int loops = 0;
4733 bool rv;
4734
4735 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
4736 ("pmap_page_exists_quick: page %p is not managed", m));
4737 rv = false;
4738 rw_wlock(&pvh_global_lock);
4739 TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) {
4740 if (PV_PMAP(pv) == pmap) {
4741 rv = true;
4742 break;
4743 }
4744 loops++;
4745 if (loops >= 16)
4746 break;
4747 }
4748 if (!rv && loops < 16 && (m->flags & PG_FICTITIOUS) == 0) {
4749 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
4750 TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) {
4751 if (PV_PMAP(pv) == pmap) {
4752 rv = true;
4753 break;
4754 }
4755 loops++;
4756 if (loops >= 16)
4757 break;
4758 }
4759 }
4760 rw_wunlock(&pvh_global_lock);
4761 return (rv);
4762 }
4763
4764 /*
4765 * pmap_page_wired_mappings:
4766 *
4767 * Return the number of managed mappings to the given physical page
4768 * that are wired.
4769 */
4770 static int
__CONCAT(PMTYPE,page_wired_mappings)4771 __CONCAT(PMTYPE, page_wired_mappings)(vm_page_t m)
4772 {
4773 int count;
4774
4775 count = 0;
4776 if ((m->oflags & VPO_UNMANAGED) != 0)
4777 return (count);
4778 rw_wlock(&pvh_global_lock);
4779 count = pmap_pvh_wired_mappings(&m->md, count);
4780 if ((m->flags & PG_FICTITIOUS) == 0) {
4781 count = pmap_pvh_wired_mappings(pa_to_pvh(VM_PAGE_TO_PHYS(m)),
4782 count);
4783 }
4784 rw_wunlock(&pvh_global_lock);
4785 return (count);
4786 }
4787
4788 /*
4789 * pmap_pvh_wired_mappings:
4790 *
4791 * Return the updated number "count" of managed mappings that are wired.
4792 */
4793 static int
pmap_pvh_wired_mappings(struct md_page * pvh,int count)4794 pmap_pvh_wired_mappings(struct md_page *pvh, int count)
4795 {
4796 pmap_t pmap;
4797 pt_entry_t *pte;
4798 pv_entry_t pv;
4799
4800 rw_assert(&pvh_global_lock, RA_WLOCKED);
4801 sched_pin();
4802 TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) {
4803 pmap = PV_PMAP(pv);
4804 PMAP_LOCK(pmap);
4805 pte = pmap_pte_quick(pmap, pv->pv_va);
4806 if ((*pte & PG_W) != 0)
4807 count++;
4808 PMAP_UNLOCK(pmap);
4809 }
4810 sched_unpin();
4811 return (count);
4812 }
4813
4814 /*
4815 * Returns true if the given page is mapped individually or as part of
4816 * a 4mpage. Otherwise, returns false.
4817 */
4818 static bool
__CONCAT(PMTYPE,page_is_mapped)4819 __CONCAT(PMTYPE, page_is_mapped)(vm_page_t m)
4820 {
4821 bool rv;
4822
4823 if ((m->oflags & VPO_UNMANAGED) != 0)
4824 return (false);
4825 rw_wlock(&pvh_global_lock);
4826 rv = !TAILQ_EMPTY(&m->md.pv_list) ||
4827 ((m->flags & PG_FICTITIOUS) == 0 &&
4828 !TAILQ_EMPTY(&pa_to_pvh(VM_PAGE_TO_PHYS(m))->pv_list));
4829 rw_wunlock(&pvh_global_lock);
4830 return (rv);
4831 }
4832
4833 /*
4834 * Remove all pages from specified address space
4835 * this aids process exit speeds. Also, this code
4836 * is special cased for current process only, but
4837 * can have the more generic (and slightly slower)
4838 * mode enabled. This is much faster than pmap_remove
4839 * in the case of running down an entire address space.
4840 */
4841 static void
__CONCAT(PMTYPE,remove_pages)4842 __CONCAT(PMTYPE, remove_pages)(pmap_t pmap)
4843 {
4844 pt_entry_t *pte, tpte;
4845 vm_page_t m, mpte, mt;
4846 pv_entry_t pv;
4847 struct md_page *pvh;
4848 struct pv_chunk *pc, *npc;
4849 struct spglist free;
4850 int field, idx;
4851 int32_t bit;
4852 uint32_t inuse, bitmask;
4853 int allfree;
4854
4855 if (pmap != PCPU_GET(curpmap)) {
4856 printf("warning: pmap_remove_pages called with non-current pmap\n");
4857 return;
4858 }
4859 SLIST_INIT(&free);
4860 rw_wlock(&pvh_global_lock);
4861 PMAP_LOCK(pmap);
4862 sched_pin();
4863 TAILQ_FOREACH_SAFE(pc, &pmap->pm_pvchunk, pc_list, npc) {
4864 KASSERT(pc->pc_pmap == pmap, ("Wrong pmap %p %p", pmap,
4865 pc->pc_pmap));
4866 allfree = 1;
4867 for (field = 0; field < _NPCM; field++) {
4868 inuse = ~pc->pc_map[field] & pc_freemask[field];
4869 while (inuse != 0) {
4870 bit = bsfl(inuse);
4871 bitmask = 1UL << bit;
4872 idx = field * 32 + bit;
4873 pv = &pc->pc_pventry[idx];
4874 inuse &= ~bitmask;
4875
4876 pte = pmap_pde(pmap, pv->pv_va);
4877 tpte = *pte;
4878 if ((tpte & PG_PS) == 0) {
4879 pte = pmap_pte_quick(pmap, pv->pv_va);
4880 tpte = *pte & ~PG_PTE_PAT;
4881 }
4882
4883 if (tpte == 0) {
4884 printf(
4885 "TPTE at %p IS ZERO @ VA %08x\n",
4886 pte, pv->pv_va);
4887 panic("bad pte");
4888 }
4889
4890 /*
4891 * We cannot remove wired pages from a process' mapping at this time
4892 */
4893 if (tpte & PG_W) {
4894 allfree = 0;
4895 continue;
4896 }
4897
4898 m = PHYS_TO_VM_PAGE(tpte & PG_FRAME);
4899 KASSERT(m->phys_addr == (tpte & PG_FRAME),
4900 ("vm_page_t %p phys_addr mismatch %016jx %016jx",
4901 m, (uintmax_t)m->phys_addr,
4902 (uintmax_t)tpte));
4903
4904 KASSERT((m->flags & PG_FICTITIOUS) != 0 ||
4905 m < &vm_page_array[vm_page_array_size],
4906 ("pmap_remove_pages: bad tpte %#jx",
4907 (uintmax_t)tpte));
4908
4909 pte_clear(pte);
4910
4911 /*
4912 * Update the vm_page_t clean/reference bits.
4913 */
4914 if ((tpte & (PG_M | PG_RW)) == (PG_M | PG_RW)) {
4915 if ((tpte & PG_PS) != 0) {
4916 for (mt = m; mt < &m[NBPDR / PAGE_SIZE]; mt++)
4917 vm_page_dirty(mt);
4918 } else
4919 vm_page_dirty(m);
4920 }
4921
4922 /* Mark free */
4923 PV_STAT(pv_entry_frees++);
4924 PV_STAT(pv_entry_spare++);
4925 pv_entry_count--;
4926 pc->pc_map[field] |= bitmask;
4927 if ((tpte & PG_PS) != 0) {
4928 pmap->pm_stats.resident_count -= NBPDR / PAGE_SIZE;
4929 pvh = pa_to_pvh(tpte & PG_PS_FRAME);
4930 TAILQ_REMOVE(&pvh->pv_list, pv, pv_next);
4931 if (TAILQ_EMPTY(&pvh->pv_list)) {
4932 for (mt = m; mt < &m[NBPDR / PAGE_SIZE]; mt++)
4933 if (TAILQ_EMPTY(&mt->md.pv_list))
4934 vm_page_aflag_clear(mt, PGA_WRITEABLE);
4935 }
4936 mpte = pmap_remove_pt_page(pmap, pv->pv_va);
4937 if (mpte != NULL) {
4938 KASSERT(vm_page_any_valid(mpte),
4939 ("pmap_remove_pages: pte page not promoted"));
4940 pmap->pm_stats.resident_count--;
4941 KASSERT(mpte->ref_count == NPTEPG,
4942 ("pmap_remove_pages: pte page ref count error"));
4943 mpte->ref_count = 0;
4944 pmap_add_delayed_free_list(mpte, &free, false);
4945 }
4946 } else {
4947 pmap->pm_stats.resident_count--;
4948 TAILQ_REMOVE(&m->md.pv_list, pv, pv_next);
4949 if (TAILQ_EMPTY(&m->md.pv_list) &&
4950 (m->flags & PG_FICTITIOUS) == 0) {
4951 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
4952 if (TAILQ_EMPTY(&pvh->pv_list))
4953 vm_page_aflag_clear(m, PGA_WRITEABLE);
4954 }
4955 pmap_unuse_pt(pmap, pv->pv_va, &free);
4956 }
4957 }
4958 }
4959 if (allfree) {
4960 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
4961 free_pv_chunk(pc);
4962 }
4963 }
4964 sched_unpin();
4965 pmap_invalidate_all_int(pmap);
4966 rw_wunlock(&pvh_global_lock);
4967 PMAP_UNLOCK(pmap);
4968 vm_page_free_pages_toq(&free, true);
4969 }
4970
4971 /*
4972 * pmap_is_modified:
4973 *
4974 * Return whether or not the specified physical page was modified
4975 * in any physical maps.
4976 */
4977 static bool
__CONCAT(PMTYPE,is_modified)4978 __CONCAT(PMTYPE, is_modified)(vm_page_t m)
4979 {
4980 bool rv;
4981
4982 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
4983 ("pmap_is_modified: page %p is not managed", m));
4984
4985 /*
4986 * If the page is not busied then this check is racy.
4987 */
4988 if (!pmap_page_is_write_mapped(m))
4989 return (false);
4990 rw_wlock(&pvh_global_lock);
4991 rv = pmap_is_modified_pvh(&m->md) ||
4992 ((m->flags & PG_FICTITIOUS) == 0 &&
4993 pmap_is_modified_pvh(pa_to_pvh(VM_PAGE_TO_PHYS(m))));
4994 rw_wunlock(&pvh_global_lock);
4995 return (rv);
4996 }
4997
4998 /*
4999 * Returns true if any of the given mappings were used to modify
5000 * physical memory. Otherwise, returns false. Both page and 2mpage
5001 * mappings are supported.
5002 */
5003 static bool
pmap_is_modified_pvh(struct md_page * pvh)5004 pmap_is_modified_pvh(struct md_page *pvh)
5005 {
5006 pv_entry_t pv;
5007 pt_entry_t *pte;
5008 pmap_t pmap;
5009 bool rv;
5010
5011 rw_assert(&pvh_global_lock, RA_WLOCKED);
5012 rv = false;
5013 sched_pin();
5014 TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) {
5015 pmap = PV_PMAP(pv);
5016 PMAP_LOCK(pmap);
5017 pte = pmap_pte_quick(pmap, pv->pv_va);
5018 rv = (*pte & (PG_M | PG_RW)) == (PG_M | PG_RW);
5019 PMAP_UNLOCK(pmap);
5020 if (rv)
5021 break;
5022 }
5023 sched_unpin();
5024 return (rv);
5025 }
5026
5027 /*
5028 * pmap_is_prefaultable:
5029 *
5030 * Return whether or not the specified virtual address is elgible
5031 * for prefault.
5032 */
5033 static bool
__CONCAT(PMTYPE,is_prefaultable)5034 __CONCAT(PMTYPE, is_prefaultable)(pmap_t pmap, vm_offset_t addr)
5035 {
5036 pd_entry_t pde;
5037 bool rv;
5038
5039 rv = false;
5040 PMAP_LOCK(pmap);
5041 pde = *pmap_pde(pmap, addr);
5042 if (pde != 0 && (pde & PG_PS) == 0)
5043 rv = pmap_pte_ufast(pmap, addr, pde) == 0;
5044 PMAP_UNLOCK(pmap);
5045 return (rv);
5046 }
5047
5048 /*
5049 * pmap_is_referenced:
5050 *
5051 * Return whether or not the specified physical page was referenced
5052 * in any physical maps.
5053 */
5054 static bool
__CONCAT(PMTYPE,is_referenced)5055 __CONCAT(PMTYPE, is_referenced)(vm_page_t m)
5056 {
5057 bool rv;
5058
5059 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
5060 ("pmap_is_referenced: page %p is not managed", m));
5061 rw_wlock(&pvh_global_lock);
5062 rv = pmap_is_referenced_pvh(&m->md) ||
5063 ((m->flags & PG_FICTITIOUS) == 0 &&
5064 pmap_is_referenced_pvh(pa_to_pvh(VM_PAGE_TO_PHYS(m))));
5065 rw_wunlock(&pvh_global_lock);
5066 return (rv);
5067 }
5068
5069 /*
5070 * Returns true if any of the given mappings were referenced and false
5071 * otherwise. Both page and 4mpage mappings are supported.
5072 */
5073 static bool
pmap_is_referenced_pvh(struct md_page * pvh)5074 pmap_is_referenced_pvh(struct md_page *pvh)
5075 {
5076 pv_entry_t pv;
5077 pt_entry_t *pte;
5078 pmap_t pmap;
5079 bool rv;
5080
5081 rw_assert(&pvh_global_lock, RA_WLOCKED);
5082 rv = false;
5083 sched_pin();
5084 TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) {
5085 pmap = PV_PMAP(pv);
5086 PMAP_LOCK(pmap);
5087 pte = pmap_pte_quick(pmap, pv->pv_va);
5088 rv = (*pte & (PG_A | PG_V)) == (PG_A | PG_V);
5089 PMAP_UNLOCK(pmap);
5090 if (rv)
5091 break;
5092 }
5093 sched_unpin();
5094 return (rv);
5095 }
5096
5097 /*
5098 * Clear the write and modified bits in each of the given page's mappings.
5099 */
5100 static void
__CONCAT(PMTYPE,remove_write)5101 __CONCAT(PMTYPE, remove_write)(vm_page_t m)
5102 {
5103 struct md_page *pvh;
5104 pv_entry_t next_pv, pv;
5105 pmap_t pmap;
5106 pd_entry_t *pde;
5107 pt_entry_t oldpte, *pte;
5108 vm_offset_t va;
5109
5110 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
5111 ("pmap_remove_write: page %p is not managed", m));
5112 vm_page_assert_busied(m);
5113
5114 if (!pmap_page_is_write_mapped(m))
5115 return;
5116 rw_wlock(&pvh_global_lock);
5117 sched_pin();
5118 if ((m->flags & PG_FICTITIOUS) != 0)
5119 goto small_mappings;
5120 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
5121 TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_next, next_pv) {
5122 va = pv->pv_va;
5123 pmap = PV_PMAP(pv);
5124 PMAP_LOCK(pmap);
5125 pde = pmap_pde(pmap, va);
5126 if ((*pde & PG_RW) != 0)
5127 (void)pmap_demote_pde(pmap, pde, va);
5128 PMAP_UNLOCK(pmap);
5129 }
5130 small_mappings:
5131 TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) {
5132 pmap = PV_PMAP(pv);
5133 PMAP_LOCK(pmap);
5134 pde = pmap_pde(pmap, pv->pv_va);
5135 KASSERT((*pde & PG_PS) == 0, ("pmap_clear_write: found"
5136 " a 4mpage in page %p's pv list", m));
5137 pte = pmap_pte_quick(pmap, pv->pv_va);
5138 retry:
5139 oldpte = *pte;
5140 if ((oldpte & PG_RW) != 0) {
5141 /*
5142 * Regardless of whether a pte is 32 or 64 bits
5143 * in size, PG_RW and PG_M are among the least
5144 * significant 32 bits.
5145 */
5146 if (!atomic_cmpset_int((u_int *)pte, oldpte,
5147 oldpte & ~(PG_RW | PG_M)))
5148 goto retry;
5149 if ((oldpte & PG_M) != 0)
5150 vm_page_dirty(m);
5151 pmap_invalidate_page_int(pmap, pv->pv_va);
5152 }
5153 PMAP_UNLOCK(pmap);
5154 }
5155 vm_page_aflag_clear(m, PGA_WRITEABLE);
5156 sched_unpin();
5157 rw_wunlock(&pvh_global_lock);
5158 }
5159
5160 /*
5161 * pmap_ts_referenced:
5162 *
5163 * Return a count of reference bits for a page, clearing those bits.
5164 * It is not necessary for every reference bit to be cleared, but it
5165 * is necessary that 0 only be returned when there are truly no
5166 * reference bits set.
5167 *
5168 * As an optimization, update the page's dirty field if a modified bit is
5169 * found while counting reference bits. This opportunistic update can be
5170 * performed at low cost and can eliminate the need for some future calls
5171 * to pmap_is_modified(). However, since this function stops after
5172 * finding PMAP_TS_REFERENCED_MAX reference bits, it may not detect some
5173 * dirty pages. Those dirty pages will only be detected by a future call
5174 * to pmap_is_modified().
5175 */
5176 static int
__CONCAT(PMTYPE,ts_referenced)5177 __CONCAT(PMTYPE, ts_referenced)(vm_page_t m)
5178 {
5179 struct md_page *pvh;
5180 pv_entry_t pv, pvf;
5181 pmap_t pmap;
5182 pd_entry_t *pde;
5183 pt_entry_t *pte;
5184 vm_paddr_t pa;
5185 int rtval = 0;
5186
5187 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
5188 ("pmap_ts_referenced: page %p is not managed", m));
5189 pa = VM_PAGE_TO_PHYS(m);
5190 pvh = pa_to_pvh(pa);
5191 rw_wlock(&pvh_global_lock);
5192 sched_pin();
5193 if ((m->flags & PG_FICTITIOUS) != 0 ||
5194 (pvf = TAILQ_FIRST(&pvh->pv_list)) == NULL)
5195 goto small_mappings;
5196 pv = pvf;
5197 do {
5198 pmap = PV_PMAP(pv);
5199 PMAP_LOCK(pmap);
5200 pde = pmap_pde(pmap, pv->pv_va);
5201 if ((*pde & (PG_M | PG_RW)) == (PG_M | PG_RW)) {
5202 /*
5203 * Although "*pde" is mapping a 2/4MB page, because
5204 * this function is called at a 4KB page granularity,
5205 * we only update the 4KB page under test.
5206 */
5207 vm_page_dirty(m);
5208 }
5209 if ((*pde & PG_A) != 0) {
5210 /*
5211 * Since this reference bit is shared by either 1024
5212 * or 512 4KB pages, it should not be cleared every
5213 * time it is tested. Apply a simple "hash" function
5214 * on the physical page number, the virtual superpage
5215 * number, and the pmap address to select one 4KB page
5216 * out of the 1024 or 512 on which testing the
5217 * reference bit will result in clearing that bit.
5218 * This function is designed to avoid the selection of
5219 * the same 4KB page for every 2- or 4MB page mapping.
5220 *
5221 * On demotion, a mapping that hasn't been referenced
5222 * is simply destroyed. To avoid the possibility of a
5223 * subsequent page fault on a demoted wired mapping,
5224 * always leave its reference bit set. Moreover,
5225 * since the superpage is wired, the current state of
5226 * its reference bit won't affect page replacement.
5227 */
5228 if ((((pa >> PAGE_SHIFT) ^ (pv->pv_va >> PDRSHIFT) ^
5229 (uintptr_t)pmap) & (NPTEPG - 1)) == 0 &&
5230 (*pde & PG_W) == 0) {
5231 atomic_clear_int((u_int *)pde, PG_A);
5232 pmap_invalidate_page_int(pmap, pv->pv_va);
5233 }
5234 rtval++;
5235 }
5236 PMAP_UNLOCK(pmap);
5237 /* Rotate the PV list if it has more than one entry. */
5238 if (TAILQ_NEXT(pv, pv_next) != NULL) {
5239 TAILQ_REMOVE(&pvh->pv_list, pv, pv_next);
5240 TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_next);
5241 }
5242 if (rtval >= PMAP_TS_REFERENCED_MAX)
5243 goto out;
5244 } while ((pv = TAILQ_FIRST(&pvh->pv_list)) != pvf);
5245 small_mappings:
5246 if ((pvf = TAILQ_FIRST(&m->md.pv_list)) == NULL)
5247 goto out;
5248 pv = pvf;
5249 do {
5250 pmap = PV_PMAP(pv);
5251 PMAP_LOCK(pmap);
5252 pde = pmap_pde(pmap, pv->pv_va);
5253 KASSERT((*pde & PG_PS) == 0,
5254 ("pmap_ts_referenced: found a 4mpage in page %p's pv list",
5255 m));
5256 pte = pmap_pte_quick(pmap, pv->pv_va);
5257 if ((*pte & (PG_M | PG_RW)) == (PG_M | PG_RW))
5258 vm_page_dirty(m);
5259 if ((*pte & PG_A) != 0) {
5260 atomic_clear_int((u_int *)pte, PG_A);
5261 pmap_invalidate_page_int(pmap, pv->pv_va);
5262 rtval++;
5263 }
5264 PMAP_UNLOCK(pmap);
5265 /* Rotate the PV list if it has more than one entry. */
5266 if (TAILQ_NEXT(pv, pv_next) != NULL) {
5267 TAILQ_REMOVE(&m->md.pv_list, pv, pv_next);
5268 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next);
5269 }
5270 } while ((pv = TAILQ_FIRST(&m->md.pv_list)) != pvf && rtval <
5271 PMAP_TS_REFERENCED_MAX);
5272 out:
5273 sched_unpin();
5274 rw_wunlock(&pvh_global_lock);
5275 return (rtval);
5276 }
5277
5278 /*
5279 * Apply the given advice to the specified range of addresses within the
5280 * given pmap. Depending on the advice, clear the referenced and/or
5281 * modified flags in each mapping and set the mapped page's dirty field.
5282 */
5283 static void
__CONCAT(PMTYPE,advise)5284 __CONCAT(PMTYPE, advise)(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,
5285 int advice)
5286 {
5287 pd_entry_t oldpde, *pde;
5288 pt_entry_t *pte;
5289 vm_offset_t va, pdnxt;
5290 vm_page_t m;
5291 bool anychanged, pv_lists_locked;
5292
5293 if (advice != MADV_DONTNEED && advice != MADV_FREE)
5294 return;
5295 if (pmap_is_current(pmap))
5296 pv_lists_locked = false;
5297 else {
5298 pv_lists_locked = true;
5299 resume:
5300 rw_wlock(&pvh_global_lock);
5301 sched_pin();
5302 }
5303 anychanged = false;
5304 PMAP_LOCK(pmap);
5305 for (; sva < eva; sva = pdnxt) {
5306 pdnxt = (sva + NBPDR) & ~PDRMASK;
5307 if (pdnxt < sva)
5308 pdnxt = eva;
5309 pde = pmap_pde(pmap, sva);
5310 oldpde = *pde;
5311 if ((oldpde & PG_V) == 0)
5312 continue;
5313 else if ((oldpde & PG_PS) != 0) {
5314 if ((oldpde & PG_MANAGED) == 0)
5315 continue;
5316 if (!pv_lists_locked) {
5317 pv_lists_locked = true;
5318 if (!rw_try_wlock(&pvh_global_lock)) {
5319 if (anychanged)
5320 pmap_invalidate_all_int(pmap);
5321 PMAP_UNLOCK(pmap);
5322 goto resume;
5323 }
5324 sched_pin();
5325 }
5326 if (!pmap_demote_pde(pmap, pde, sva)) {
5327 /*
5328 * The large page mapping was destroyed.
5329 */
5330 continue;
5331 }
5332
5333 /*
5334 * Unless the page mappings are wired, remove the
5335 * mapping to a single page so that a subsequent
5336 * access may repromote. Choosing the last page
5337 * within the address range [sva, min(pdnxt, eva))
5338 * generally results in more repromotions. Since the
5339 * underlying page table page is fully populated, this
5340 * removal never frees a page table page.
5341 */
5342 if ((oldpde & PG_W) == 0) {
5343 va = eva;
5344 if (va > pdnxt)
5345 va = pdnxt;
5346 va -= PAGE_SIZE;
5347 KASSERT(va >= sva,
5348 ("pmap_advise: no address gap"));
5349 pte = pmap_pte_quick(pmap, va);
5350 KASSERT((*pte & PG_V) != 0,
5351 ("pmap_advise: invalid PTE"));
5352 pmap_remove_pte(pmap, pte, va, NULL);
5353 anychanged = true;
5354 }
5355 }
5356 if (pdnxt > eva)
5357 pdnxt = eva;
5358 va = pdnxt;
5359 for (pte = pmap_pte_quick(pmap, sva); sva != pdnxt; pte++,
5360 sva += PAGE_SIZE) {
5361 if ((*pte & (PG_MANAGED | PG_V)) != (PG_MANAGED | PG_V))
5362 goto maybe_invlrng;
5363 else if ((*pte & (PG_M | PG_RW)) == (PG_M | PG_RW)) {
5364 if (advice == MADV_DONTNEED) {
5365 /*
5366 * Future calls to pmap_is_modified()
5367 * can be avoided by making the page
5368 * dirty now.
5369 */
5370 m = PHYS_TO_VM_PAGE(*pte & PG_FRAME);
5371 vm_page_dirty(m);
5372 }
5373 atomic_clear_int((u_int *)pte, PG_M | PG_A);
5374 } else if ((*pte & PG_A) != 0)
5375 atomic_clear_int((u_int *)pte, PG_A);
5376 else
5377 goto maybe_invlrng;
5378 if ((*pte & PG_G) != 0) {
5379 if (va == pdnxt)
5380 va = sva;
5381 } else
5382 anychanged = true;
5383 continue;
5384 maybe_invlrng:
5385 if (va != pdnxt) {
5386 pmap_invalidate_range_int(pmap, va, sva);
5387 va = pdnxt;
5388 }
5389 }
5390 if (va != pdnxt)
5391 pmap_invalidate_range_int(pmap, va, sva);
5392 }
5393 if (anychanged)
5394 pmap_invalidate_all_int(pmap);
5395 if (pv_lists_locked) {
5396 sched_unpin();
5397 rw_wunlock(&pvh_global_lock);
5398 }
5399 PMAP_UNLOCK(pmap);
5400 }
5401
5402 /*
5403 * Clear the modify bits on the specified physical page.
5404 */
5405 static void
__CONCAT(PMTYPE,clear_modify)5406 __CONCAT(PMTYPE, clear_modify)(vm_page_t m)
5407 {
5408 struct md_page *pvh;
5409 pv_entry_t next_pv, pv;
5410 pmap_t pmap;
5411 pd_entry_t oldpde, *pde;
5412 pt_entry_t *pte;
5413 vm_offset_t va;
5414
5415 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
5416 ("pmap_clear_modify: page %p is not managed", m));
5417 vm_page_assert_busied(m);
5418
5419 if (!pmap_page_is_write_mapped(m))
5420 return;
5421 rw_wlock(&pvh_global_lock);
5422 sched_pin();
5423 if ((m->flags & PG_FICTITIOUS) != 0)
5424 goto small_mappings;
5425 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
5426 TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_next, next_pv) {
5427 va = pv->pv_va;
5428 pmap = PV_PMAP(pv);
5429 PMAP_LOCK(pmap);
5430 pde = pmap_pde(pmap, va);
5431 oldpde = *pde;
5432 /* If oldpde has PG_RW set, then it also has PG_M set. */
5433 if ((oldpde & PG_RW) != 0 &&
5434 pmap_demote_pde(pmap, pde, va) &&
5435 (oldpde & PG_W) == 0) {
5436 /*
5437 * Write protect the mapping to a single page so that
5438 * a subsequent write access may repromote.
5439 */
5440 va += VM_PAGE_TO_PHYS(m) - (oldpde & PG_PS_FRAME);
5441 pte = pmap_pte_quick(pmap, va);
5442 /*
5443 * Regardless of whether a pte is 32 or 64 bits
5444 * in size, PG_RW and PG_M are among the least
5445 * significant 32 bits.
5446 */
5447 atomic_clear_int((u_int *)pte, PG_M | PG_RW);
5448 vm_page_dirty(m);
5449 pmap_invalidate_page_int(pmap, va);
5450 }
5451 PMAP_UNLOCK(pmap);
5452 }
5453 small_mappings:
5454 TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) {
5455 pmap = PV_PMAP(pv);
5456 PMAP_LOCK(pmap);
5457 pde = pmap_pde(pmap, pv->pv_va);
5458 KASSERT((*pde & PG_PS) == 0, ("pmap_clear_modify: found"
5459 " a 4mpage in page %p's pv list", m));
5460 pte = pmap_pte_quick(pmap, pv->pv_va);
5461 if ((*pte & (PG_M | PG_RW)) == (PG_M | PG_RW)) {
5462 /*
5463 * Regardless of whether a pte is 32 or 64 bits
5464 * in size, PG_M is among the least significant
5465 * 32 bits.
5466 */
5467 atomic_clear_int((u_int *)pte, PG_M);
5468 pmap_invalidate_page_int(pmap, pv->pv_va);
5469 }
5470 PMAP_UNLOCK(pmap);
5471 }
5472 sched_unpin();
5473 rw_wunlock(&pvh_global_lock);
5474 }
5475
5476 /*
5477 * Miscellaneous support routines follow
5478 */
5479
5480 /* Adjust the cache mode for a 4KB page mapped via a PTE. */
5481 static __inline void
pmap_pte_attr(pt_entry_t * pte,int cache_bits)5482 pmap_pte_attr(pt_entry_t *pte, int cache_bits)
5483 {
5484 u_int opte, npte;
5485
5486 /*
5487 * The cache mode bits are all in the low 32-bits of the
5488 * PTE, so we can just spin on updating the low 32-bits.
5489 */
5490 do {
5491 opte = *(u_int *)pte;
5492 npte = opte & ~PG_PTE_CACHE;
5493 npte |= cache_bits;
5494 } while (npte != opte && !atomic_cmpset_int((u_int *)pte, opte, npte));
5495 }
5496
5497 /* Adjust the cache mode for a 2/4MB page mapped via a PDE. */
5498 static __inline void
pmap_pde_attr(pd_entry_t * pde,int cache_bits)5499 pmap_pde_attr(pd_entry_t *pde, int cache_bits)
5500 {
5501 u_int opde, npde;
5502
5503 /*
5504 * The cache mode bits are all in the low 32-bits of the
5505 * PDE, so we can just spin on updating the low 32-bits.
5506 */
5507 do {
5508 opde = *(u_int *)pde;
5509 npde = opde & ~PG_PDE_CACHE;
5510 npde |= cache_bits;
5511 } while (npde != opde && !atomic_cmpset_int((u_int *)pde, opde, npde));
5512 }
5513
5514 /*
5515 * Map a set of physical memory pages into the kernel virtual
5516 * address space. Return a pointer to where it is mapped. This
5517 * routine is intended to be used for mapping device memory,
5518 * NOT real memory.
5519 */
5520 static void *
__CONCAT(PMTYPE,mapdev_attr)5521 __CONCAT(PMTYPE, mapdev_attr)(vm_paddr_t pa, vm_size_t size, int mode,
5522 int flags)
5523 {
5524 struct pmap_preinit_mapping *ppim;
5525 vm_offset_t va, offset;
5526 vm_page_t m;
5527 vm_size_t tmpsize;
5528 int i;
5529
5530 offset = pa & PAGE_MASK;
5531 size = round_page(offset + size);
5532 pa = pa & PG_FRAME;
5533
5534 if (pa < PMAP_MAP_LOW && pa + size <= PMAP_MAP_LOW) {
5535 va = pa + PMAP_MAP_LOW;
5536 if ((flags & MAPDEV_SETATTR) == 0)
5537 return ((void *)(va + offset));
5538 } else if (!pmap_initialized) {
5539 va = 0;
5540 for (i = 0; i < PMAP_PREINIT_MAPPING_COUNT; i++) {
5541 ppim = pmap_preinit_mapping + i;
5542 if (ppim->va == 0) {
5543 ppim->pa = pa;
5544 ppim->sz = size;
5545 ppim->mode = mode;
5546 ppim->va = virtual_avail;
5547 virtual_avail += size;
5548 va = ppim->va;
5549 break;
5550 }
5551 }
5552 if (va == 0)
5553 panic("%s: too many preinit mappings", __func__);
5554 } else {
5555 /*
5556 * If we have a preinit mapping, re-use it.
5557 */
5558 for (i = 0; i < PMAP_PREINIT_MAPPING_COUNT; i++) {
5559 ppim = pmap_preinit_mapping + i;
5560 if (ppim->pa == pa && ppim->sz == size &&
5561 (ppim->mode == mode ||
5562 (flags & MAPDEV_SETATTR) == 0))
5563 return ((void *)(ppim->va + offset));
5564 }
5565 va = (vm_offset_t)kva_alloc(size);
5566 if (va == 0)
5567 panic("%s: Couldn't allocate KVA", __func__);
5568 }
5569 for (tmpsize = 0; tmpsize < size; tmpsize += PAGE_SIZE) {
5570 if ((flags & MAPDEV_SETATTR) == 0 && pmap_initialized) {
5571 m = PHYS_TO_VM_PAGE(pa);
5572 if (m != NULL && VM_PAGE_TO_PHYS(m) == pa) {
5573 pmap_kenter_attr(va + tmpsize, pa + tmpsize,
5574 m->md.pat_mode);
5575 continue;
5576 }
5577 }
5578 pmap_kenter_attr(va + tmpsize, pa + tmpsize, mode);
5579 }
5580 pmap_invalidate_range_int(kernel_pmap, va, va + tmpsize);
5581 pmap_invalidate_cache_range(va, va + size);
5582 return ((void *)(va + offset));
5583 }
5584
5585 static void
__CONCAT(PMTYPE,unmapdev)5586 __CONCAT(PMTYPE, unmapdev)(void *p, vm_size_t size)
5587 {
5588 struct pmap_preinit_mapping *ppim;
5589 vm_offset_t offset, va;
5590 int i;
5591
5592 va = (vm_offset_t)p;
5593 if (va >= PMAP_MAP_LOW && va <= KERNBASE && va + size <= KERNBASE)
5594 return;
5595 offset = va & PAGE_MASK;
5596 size = round_page(offset + size);
5597 va = trunc_page(va);
5598 for (i = 0; i < PMAP_PREINIT_MAPPING_COUNT; i++) {
5599 ppim = pmap_preinit_mapping + i;
5600 if (ppim->va == va && ppim->sz == size) {
5601 if (pmap_initialized)
5602 return;
5603 ppim->pa = 0;
5604 ppim->va = 0;
5605 ppim->sz = 0;
5606 ppim->mode = 0;
5607 if (va + size == virtual_avail)
5608 virtual_avail = va;
5609 return;
5610 }
5611 }
5612 if (pmap_initialized) {
5613 pmap_qremove((void *)va, atop(size));
5614 kva_free((void *)va, size);
5615 }
5616 }
5617
5618 /*
5619 * Sets the memory attribute for the specified page.
5620 */
5621 static void
__CONCAT(PMTYPE,page_set_memattr)5622 __CONCAT(PMTYPE, page_set_memattr)(vm_page_t m, vm_memattr_t ma)
5623 {
5624 if (m->md.pat_mode == ma)
5625 return;
5626
5627 m->md.pat_mode = ma;
5628 if ((m->flags & PG_FICTITIOUS) != 0)
5629 return;
5630
5631 /*
5632 * If "m" is a normal page, flush it from the cache.
5633 * See pmap_invalidate_cache_range().
5634 *
5635 * First, try to find an existing mapping of the page by sf
5636 * buffer. sf_buf_invalidate_cache() modifies mapping and
5637 * flushes the cache.
5638 */
5639 if (sf_buf_invalidate_cache(m))
5640 return;
5641
5642 /*
5643 * If page is not mapped by sf buffer, but CPU does not
5644 * support self snoop, map the page transient and do
5645 * invalidation. In the worst case, whole cache is flushed by
5646 * pmap_invalidate_cache_range().
5647 */
5648 if ((cpu_feature & CPUID_SS) == 0)
5649 pmap_flush_page(m);
5650 }
5651
5652 static void
__CONCAT(PMTYPE,flush_page)5653 __CONCAT(PMTYPE, flush_page)(vm_page_t m)
5654 {
5655 pt_entry_t *cmap_pte2;
5656 struct pcpu *pc;
5657 vm_offset_t sva, eva;
5658 bool useclflushopt;
5659
5660 useclflushopt = (cpu_stdext_feature & CPUID_STDEXT_CLFLUSHOPT) != 0;
5661 if (useclflushopt || (cpu_feature & CPUID_CLFSH) != 0) {
5662 sched_pin();
5663 pc = get_pcpu();
5664 cmap_pte2 = pc->pc_cmap_pte2;
5665 mtx_lock(&pc->pc_cmap_lock);
5666 if (*cmap_pte2)
5667 panic("pmap_flush_page: CMAP2 busy");
5668 *cmap_pte2 = PG_V | PG_RW | VM_PAGE_TO_PHYS(m) |
5669 PG_A | PG_M | pmap_cache_bits(kernel_pmap, m->md.pat_mode,
5670 false);
5671 invlcaddr(pc->pc_cmap_addr2);
5672 sva = (vm_offset_t)pc->pc_cmap_addr2;
5673 eva = sva + PAGE_SIZE;
5674
5675 /*
5676 * Use mfence or sfence despite the ordering implied by
5677 * mtx_{un,}lock() because clflush on non-Intel CPUs
5678 * and clflushopt are not guaranteed to be ordered by
5679 * any other instruction.
5680 */
5681 if (useclflushopt)
5682 sfence();
5683 else if (cpu_vendor_id != CPU_VENDOR_INTEL)
5684 mfence();
5685 for (; sva < eva; sva += cpu_clflush_line_size) {
5686 if (useclflushopt)
5687 clflushopt(sva);
5688 else
5689 clflush(sva);
5690 }
5691 if (useclflushopt)
5692 sfence();
5693 else if (cpu_vendor_id != CPU_VENDOR_INTEL)
5694 mfence();
5695 *cmap_pte2 = 0;
5696 sched_unpin();
5697 mtx_unlock(&pc->pc_cmap_lock);
5698 } else
5699 pmap_invalidate_cache();
5700 }
5701
5702 /*
5703 * Changes the specified virtual address range's memory type to that given by
5704 * the parameter "mode". The specified virtual address range must be
5705 * completely contained within either the kernel map.
5706 *
5707 * Returns zero if the change completed successfully, and either EINVAL or
5708 * ENOMEM if the change failed. Specifically, EINVAL is returned if some part
5709 * of the virtual address range was not mapped, and ENOMEM is returned if
5710 * there was insufficient memory available to complete the change.
5711 */
5712 static int
__CONCAT(PMTYPE,change_attr)5713 __CONCAT(PMTYPE, change_attr)(void *addr, vm_size_t size, int mode)
5714 {
5715 vm_offset_t base, offset, tmpva, va;
5716 pd_entry_t *pde;
5717 pt_entry_t *pte;
5718 int cache_bits_pte, cache_bits_pde;
5719 bool changed;
5720
5721 va = (vm_offset_t)addr;
5722 base = trunc_page(va);
5723 offset = va & PAGE_MASK;
5724 size = round_page(offset + size);
5725
5726 /*
5727 * Only supported on kernel virtual addresses above the recursive map.
5728 */
5729 if (base < VM_MIN_KERNEL_ADDRESS)
5730 return (EINVAL);
5731
5732 cache_bits_pde = pmap_cache_bits(kernel_pmap, mode, true);
5733 cache_bits_pte = pmap_cache_bits(kernel_pmap, mode, false);
5734 changed = false;
5735
5736 /*
5737 * Pages that aren't mapped aren't supported. Also break down
5738 * 2/4MB pages into 4KB pages if required.
5739 */
5740 PMAP_LOCK(kernel_pmap);
5741 for (tmpva = base; tmpva < base + size; ) {
5742 pde = pmap_pde(kernel_pmap, tmpva);
5743 if (*pde == 0) {
5744 PMAP_UNLOCK(kernel_pmap);
5745 return (EINVAL);
5746 }
5747 if (*pde & PG_PS) {
5748 /*
5749 * If the current 2/4MB page already has
5750 * the required memory type, then we need not
5751 * demote this page. Just increment tmpva to
5752 * the next 2/4MB page frame.
5753 */
5754 if ((*pde & PG_PDE_CACHE) == cache_bits_pde) {
5755 tmpva = trunc_4mpage(tmpva) + NBPDR;
5756 continue;
5757 }
5758
5759 /*
5760 * If the current offset aligns with a 2/4MB
5761 * page frame and there is at least 2/4MB left
5762 * within the range, then we need not break
5763 * down this page into 4KB pages.
5764 */
5765 if ((tmpva & PDRMASK) == 0 &&
5766 tmpva + PDRMASK < base + size) {
5767 tmpva += NBPDR;
5768 continue;
5769 }
5770 if (!pmap_demote_pde(kernel_pmap, pde, tmpva)) {
5771 PMAP_UNLOCK(kernel_pmap);
5772 return (ENOMEM);
5773 }
5774 }
5775 pte = vtopte(tmpva);
5776 if (*pte == 0) {
5777 PMAP_UNLOCK(kernel_pmap);
5778 return (EINVAL);
5779 }
5780 tmpva += PAGE_SIZE;
5781 }
5782 PMAP_UNLOCK(kernel_pmap);
5783
5784 /*
5785 * Ok, all the pages exist, so run through them updating their
5786 * cache mode if required.
5787 */
5788 for (tmpva = base; tmpva < base + size; ) {
5789 pde = pmap_pde(kernel_pmap, tmpva);
5790 if (*pde & PG_PS) {
5791 if ((*pde & PG_PDE_CACHE) != cache_bits_pde) {
5792 pmap_pde_attr(pde, cache_bits_pde);
5793 changed = true;
5794 }
5795 tmpva = trunc_4mpage(tmpva) + NBPDR;
5796 } else {
5797 pte = vtopte(tmpva);
5798 if ((*pte & PG_PTE_CACHE) != cache_bits_pte) {
5799 pmap_pte_attr(pte, cache_bits_pte);
5800 changed = true;
5801 }
5802 tmpva += PAGE_SIZE;
5803 }
5804 }
5805
5806 /*
5807 * Flush CPU caches to make sure any data isn't cached that
5808 * shouldn't be, etc.
5809 */
5810 if (changed) {
5811 pmap_invalidate_range_int(kernel_pmap, base, tmpva);
5812 pmap_invalidate_cache_range(base, tmpva);
5813 }
5814 return (0);
5815 }
5816
5817 /*
5818 * Perform the pmap work for mincore(2). If the page is not both referenced and
5819 * modified by this pmap, returns its physical address so that the caller can
5820 * find other mappings.
5821 */
5822 static int
__CONCAT(PMTYPE,mincore)5823 __CONCAT(PMTYPE, mincore)(pmap_t pmap, vm_offset_t addr, vm_paddr_t *pap)
5824 {
5825 pd_entry_t pde;
5826 pt_entry_t pte;
5827 vm_paddr_t pa;
5828 int val;
5829
5830 PMAP_LOCK(pmap);
5831 pde = *pmap_pde(pmap, addr);
5832 if (pde != 0) {
5833 if ((pde & PG_PS) != 0) {
5834 pte = pde;
5835 /* Compute the physical address of the 4KB page. */
5836 pa = ((pde & PG_PS_FRAME) | (addr & PDRMASK)) &
5837 PG_FRAME;
5838 val = MINCORE_PSIND(1);
5839 } else {
5840 pte = pmap_pte_ufast(pmap, addr, pde);
5841 pa = pte & PG_FRAME;
5842 val = 0;
5843 }
5844 } else {
5845 pte = 0;
5846 pa = 0;
5847 val = 0;
5848 }
5849 if ((pte & PG_V) != 0) {
5850 val |= MINCORE_INCORE;
5851 if ((pte & (PG_M | PG_RW)) == (PG_M | PG_RW))
5852 val |= MINCORE_MODIFIED | MINCORE_MODIFIED_OTHER;
5853 if ((pte & PG_A) != 0)
5854 val |= MINCORE_REFERENCED | MINCORE_REFERENCED_OTHER;
5855 }
5856 if ((val & (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER)) !=
5857 (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER) &&
5858 (pte & (PG_MANAGED | PG_V)) == (PG_MANAGED | PG_V)) {
5859 *pap = pa;
5860 }
5861 PMAP_UNLOCK(pmap);
5862 return (val);
5863 }
5864
5865 static void
__CONCAT(PMTYPE,activate)5866 __CONCAT(PMTYPE, activate)(struct thread *td)
5867 {
5868 pmap_t pmap, oldpmap;
5869 u_int cpuid;
5870 u_int32_t cr3;
5871
5872 critical_enter();
5873 pmap = vmspace_pmap(td->td_proc->p_vmspace);
5874 oldpmap = PCPU_GET(curpmap);
5875 cpuid = PCPU_GET(cpuid);
5876 #if defined(SMP)
5877 CPU_CLR_ATOMIC(cpuid, &oldpmap->pm_active);
5878 CPU_SET_ATOMIC(cpuid, &pmap->pm_active);
5879 #else
5880 CPU_CLR(cpuid, &oldpmap->pm_active);
5881 CPU_SET(cpuid, &pmap->pm_active);
5882 #endif
5883 #ifdef PMAP_PAE_COMP
5884 cr3 = vtophys(pmap->pm_pdpt);
5885 #else
5886 cr3 = vtophys(pmap->pm_pdir);
5887 #endif
5888 /*
5889 * pmap_activate is for the current thread on the current cpu
5890 */
5891 td->td_pcb->pcb_cr3 = cr3;
5892 PCPU_SET(curpmap, pmap);
5893 critical_exit();
5894 }
5895
5896 static void
__CONCAT(PMTYPE,activate_boot)5897 __CONCAT(PMTYPE, activate_boot)(pmap_t pmap)
5898 {
5899 u_int cpuid;
5900
5901 cpuid = PCPU_GET(cpuid);
5902 #if defined(SMP)
5903 CPU_SET_ATOMIC(cpuid, &pmap->pm_active);
5904 #else
5905 CPU_SET(cpuid, &pmap->pm_active);
5906 #endif
5907 PCPU_SET(curpmap, pmap);
5908 }
5909
5910 /*
5911 * Increase the starting virtual address of the given mapping if a
5912 * different alignment might result in more superpage mappings.
5913 */
5914 static void
__CONCAT(PMTYPE,align_superpage)5915 __CONCAT(PMTYPE, align_superpage)(vm_object_t object, vm_ooffset_t offset,
5916 vm_offset_t *addr, vm_size_t size)
5917 {
5918 vm_offset_t superpage_offset;
5919
5920 if (size < NBPDR)
5921 return;
5922 if (object != NULL && (object->flags & OBJ_COLORED) != 0)
5923 offset += ptoa(object->pg_color);
5924 superpage_offset = offset & PDRMASK;
5925 if (size - ((NBPDR - superpage_offset) & PDRMASK) < NBPDR ||
5926 (*addr & PDRMASK) == superpage_offset)
5927 return;
5928 if ((*addr & PDRMASK) < superpage_offset)
5929 *addr = (*addr & ~PDRMASK) + superpage_offset;
5930 else
5931 *addr = ((*addr + PDRMASK) & ~PDRMASK) + superpage_offset;
5932 }
5933
5934 static void *
__CONCAT(PMTYPE,quick_enter_page)5935 __CONCAT(PMTYPE, quick_enter_page)(vm_page_t m)
5936 {
5937 void *qaddr;
5938 pt_entry_t *pte;
5939
5940 critical_enter();
5941 qaddr = PCPU_GET(qmap_addr);
5942 pte = vtopte((vm_offset_t)qaddr);
5943
5944 KASSERT(*pte == 0,
5945 ("pmap_quick_enter_page: PTE busy %#jx", (uintmax_t)*pte));
5946 *pte = PG_V | PG_RW | VM_PAGE_TO_PHYS(m) | PG_A | PG_M |
5947 pmap_cache_bits(kernel_pmap, pmap_page_get_memattr(m), false);
5948 invlpg((vm_offset_t)qaddr);
5949
5950 return (qaddr);
5951 }
5952
5953 static void
__CONCAT(PMTYPE,quick_remove_page)5954 __CONCAT(PMTYPE, quick_remove_page)(void *addr)
5955 {
5956 void *qaddr;
5957 pt_entry_t *pte;
5958
5959 qaddr = PCPU_GET(qmap_addr);
5960 pte = vtopte((vm_offset_t)qaddr);
5961
5962 KASSERT(*pte != 0, ("pmap_quick_remove_page: PTE not in use"));
5963 KASSERT(addr == qaddr,
5964 ("pmap_quick_remove_page: invalid address"));
5965
5966 *pte = 0;
5967 critical_exit();
5968 }
5969
5970 static vmem_t *pmap_trm_arena;
5971 static vmem_addr_t pmap_trm_arena_last = PMAP_TRM_MIN_ADDRESS;
5972 static int trm_guard = PAGE_SIZE;
5973
5974 static int
pmap_trm_import(void * unused __unused,vmem_size_t size,int flags,vmem_addr_t * addrp)5975 pmap_trm_import(void *unused __unused, vmem_size_t size, int flags,
5976 vmem_addr_t *addrp)
5977 {
5978 vm_page_t m;
5979 vmem_addr_t af, addr, prev_addr;
5980 pt_entry_t *trm_pte;
5981
5982 prev_addr = atomic_load_int(&pmap_trm_arena_last);
5983 size = round_page(size) + trm_guard;
5984 for (;;) {
5985 if (prev_addr + size < prev_addr || prev_addr + size < size ||
5986 prev_addr + size > PMAP_TRM_MAX_ADDRESS)
5987 return (ENOMEM);
5988 addr = prev_addr + size;
5989 if (atomic_fcmpset_int(&pmap_trm_arena_last, &prev_addr, addr))
5990 break;
5991 }
5992 prev_addr += trm_guard;
5993 trm_pte = PTmap + atop(prev_addr);
5994 for (af = prev_addr; af < addr; af += PAGE_SIZE) {
5995 m = vm_page_alloc_noobj(VM_ALLOC_WIRED | VM_ALLOC_WAITOK);
5996 pte_store(&trm_pte[atop(af - prev_addr)], VM_PAGE_TO_PHYS(m) |
5997 PG_M | PG_A | PG_RW | PG_V | pgeflag |
5998 pmap_cache_bits(kernel_pmap, VM_MEMATTR_DEFAULT, false));
5999 }
6000 *addrp = prev_addr;
6001 return (0);
6002 }
6003
6004 void
pmap_init_trm(void)6005 pmap_init_trm(void)
6006 {
6007 vm_page_t pd_m;
6008
6009 TUNABLE_INT_FETCH("machdep.trm_guard", &trm_guard);
6010 if ((trm_guard & PAGE_MASK) != 0)
6011 trm_guard = 0;
6012 pmap_trm_arena = vmem_create("i386trampoline", 0, 0, 1, 0, M_WAITOK);
6013 vmem_set_import(pmap_trm_arena, pmap_trm_import, NULL, NULL, PAGE_SIZE);
6014 pd_m = vm_page_alloc_noobj(VM_ALLOC_WIRED | VM_ALLOC_WAITOK |
6015 VM_ALLOC_ZERO);
6016 PTD[TRPTDI] = VM_PAGE_TO_PHYS(pd_m) | PG_M | PG_A | PG_RW | PG_V |
6017 pmap_cache_bits(kernel_pmap, VM_MEMATTR_DEFAULT, true);
6018 }
6019
6020 static void *
__CONCAT(PMTYPE,trm_alloc)6021 __CONCAT(PMTYPE, trm_alloc)(size_t size, int flags)
6022 {
6023 vmem_addr_t res;
6024 int error;
6025
6026 MPASS((flags & ~(M_WAITOK | M_NOWAIT | M_ZERO)) == 0);
6027 error = vmem_xalloc(pmap_trm_arena, roundup2(size, 4), sizeof(int),
6028 0, 0, VMEM_ADDR_MIN, VMEM_ADDR_MAX, flags | M_FIRSTFIT, &res);
6029 if (error != 0)
6030 return (NULL);
6031 if ((flags & M_ZERO) != 0)
6032 bzero((void *)res, size);
6033 return ((void *)res);
6034 }
6035
6036 static void
__CONCAT(PMTYPE,trm_free)6037 __CONCAT(PMTYPE, trm_free)(void *addr, size_t size)
6038 {
6039
6040 vmem_free(pmap_trm_arena, (uintptr_t)addr, roundup2(size, 4));
6041 }
6042
6043 static void
__CONCAT(PMTYPE,ksetrw)6044 __CONCAT(PMTYPE, ksetrw)(vm_offset_t va)
6045 {
6046
6047 *vtopte(va) |= PG_RW;
6048 }
6049
6050 static void
__CONCAT(PMTYPE,remap_lowptdi)6051 __CONCAT(PMTYPE, remap_lowptdi)(bool enable)
6052 {
6053
6054 PTD[KPTDI] = enable ? PTD[LOWPTDI] : 0;
6055 invltlb_glob();
6056 }
6057
6058 static vm_offset_t
__CONCAT(PMTYPE,get_map_low)6059 __CONCAT(PMTYPE, get_map_low)(void)
6060 {
6061
6062 return (PMAP_MAP_LOW);
6063 }
6064
6065 static vm_offset_t
__CONCAT(PMTYPE,get_vm_maxuser_address)6066 __CONCAT(PMTYPE, get_vm_maxuser_address)(void)
6067 {
6068
6069 return (VM_MAXUSER_ADDRESS);
6070 }
6071
6072 static vm_paddr_t
__CONCAT(PMTYPE,pg_frame)6073 __CONCAT(PMTYPE, pg_frame)(vm_paddr_t pa)
6074 {
6075
6076 return (pa & PG_FRAME);
6077 }
6078
6079 static void
__CONCAT(PMTYPE,sf_buf_map)6080 __CONCAT(PMTYPE, sf_buf_map)(struct sf_buf *sf)
6081 {
6082 pt_entry_t opte, *ptep;
6083
6084 /*
6085 * Update the sf_buf's virtual-to-physical mapping, flushing the
6086 * virtual address from the TLB. Since the reference count for
6087 * the sf_buf's old mapping was zero, that mapping is not
6088 * currently in use. Consequently, there is no need to exchange
6089 * the old and new PTEs atomically, even under PAE.
6090 */
6091 ptep = vtopte(sf->kva);
6092 opte = *ptep;
6093 *ptep = VM_PAGE_TO_PHYS(sf->m) | PG_RW | PG_V |
6094 pmap_cache_bits(kernel_pmap, sf->m->md.pat_mode, false);
6095
6096 /*
6097 * Avoid unnecessary TLB invalidations: If the sf_buf's old
6098 * virtual-to-physical mapping was not used, then any processor
6099 * that has invalidated the sf_buf's virtual address from its TLB
6100 * since the last used mapping need not invalidate again.
6101 */
6102 #ifdef SMP
6103 if ((opte & (PG_V | PG_A)) == (PG_V | PG_A))
6104 CPU_ZERO(&sf->cpumask);
6105 #else
6106 if ((opte & (PG_V | PG_A)) == (PG_V | PG_A))
6107 pmap_invalidate_page_int(kernel_pmap, sf->kva);
6108 #endif
6109 }
6110
6111 static void
__CONCAT(PMTYPE,cp_slow0_map)6112 __CONCAT(PMTYPE, cp_slow0_map)(vm_offset_t kaddr, int plen, vm_page_t *ma)
6113 {
6114 pt_entry_t *pte;
6115 int i;
6116
6117 for (i = 0, pte = vtopte(kaddr); i < plen; i++, pte++) {
6118 *pte = PG_V | PG_RW | PG_A | PG_M | VM_PAGE_TO_PHYS(ma[i]) |
6119 pmap_cache_bits(kernel_pmap, pmap_page_get_memattr(ma[i]),
6120 false);
6121 invlpg(kaddr + ptoa(i));
6122 }
6123 }
6124
6125 static u_int
__CONCAT(PMTYPE,get_kcr3)6126 __CONCAT(PMTYPE, get_kcr3)(void)
6127 {
6128
6129 #ifdef PMAP_PAE_COMP
6130 return ((u_int)IdlePDPT);
6131 #else
6132 return ((u_int)IdlePTD);
6133 #endif
6134 }
6135
6136 static u_int
__CONCAT(PMTYPE,get_cr3)6137 __CONCAT(PMTYPE, get_cr3)(pmap_t pmap)
6138 {
6139
6140 #ifdef PMAP_PAE_COMP
6141 return ((u_int)vtophys(pmap->pm_pdpt));
6142 #else
6143 return ((u_int)vtophys(pmap->pm_pdir));
6144 #endif
6145 }
6146
6147 static caddr_t
__CONCAT(PMTYPE,cmap3)6148 __CONCAT(PMTYPE, cmap3)(vm_paddr_t pa, u_int pte_bits)
6149 {
6150 pt_entry_t *pte;
6151
6152 pte = CMAP3;
6153 *pte = pa | pte_bits;
6154 invltlb();
6155 return (CADDR3);
6156 }
6157
6158 static void
__CONCAT(PMTYPE,basemem_setup)6159 __CONCAT(PMTYPE, basemem_setup)(u_int basemem)
6160 {
6161 pt_entry_t *pte;
6162 int i;
6163
6164 /*
6165 * Map pages between basemem and ISA_HOLE_START, if any, r/w into
6166 * the vm86 page table so that vm86 can scribble on them using
6167 * the vm86 map too. XXX: why 2 ways for this and only 1 way for
6168 * page 0, at least as initialized here?
6169 */
6170 pte = (pt_entry_t *)vm86paddr;
6171 for (i = basemem / 4; i < 160; i++)
6172 pte[i] = (i << PAGE_SHIFT) | PG_V | PG_RW | PG_U;
6173 }
6174
6175 struct bios16_pmap_handle {
6176 pt_entry_t *pte;
6177 pd_entry_t *ptd;
6178 pt_entry_t orig_ptd;
6179 };
6180
6181 static void *
__CONCAT(PMTYPE,bios16_enter)6182 __CONCAT(PMTYPE, bios16_enter)(void)
6183 {
6184 struct bios16_pmap_handle *h;
6185
6186 /*
6187 * no page table, so create one and install it.
6188 */
6189 h = malloc(sizeof(struct bios16_pmap_handle), M_TEMP, M_WAITOK);
6190 h->pte = (pt_entry_t *)malloc(PAGE_SIZE, M_TEMP, M_WAITOK);
6191 h->ptd = IdlePTD;
6192 *h->pte = vm86phystk | PG_RW | PG_V;
6193 h->orig_ptd = *h->ptd;
6194 *h->ptd = vtophys(h->pte) | PG_RW | PG_V;
6195 pmap_invalidate_all_int(kernel_pmap); /* XXX insurance for now */
6196 return (h);
6197 }
6198
6199 static void
__CONCAT(PMTYPE,bios16_leave)6200 __CONCAT(PMTYPE, bios16_leave)(void *arg)
6201 {
6202 struct bios16_pmap_handle *h;
6203
6204 h = arg;
6205 *h->ptd = h->orig_ptd; /* remove page table */
6206 /*
6207 * XXX only needs to be invlpg(0) but that doesn't work on the 386
6208 */
6209 pmap_invalidate_all_int(kernel_pmap);
6210 free(h->pte, M_TEMP); /* ... and free it */
6211 }
6212
6213 struct pmap_kernel_map_range {
6214 vm_offset_t sva;
6215 pt_entry_t attrs;
6216 int ptes;
6217 int pdes;
6218 int pdpes;
6219 };
6220
6221 static void
sysctl_kmaps_dump(struct sbuf * sb,struct pmap_kernel_map_range * range,vm_offset_t eva)6222 sysctl_kmaps_dump(struct sbuf *sb, struct pmap_kernel_map_range *range,
6223 vm_offset_t eva)
6224 {
6225 const char *mode;
6226 int i, pat_idx;
6227
6228 if (eva <= range->sva)
6229 return;
6230
6231 pat_idx = pmap_pat_index(kernel_pmap, range->attrs, true);
6232 for (i = 0; i < PAT_INDEX_SIZE; i++)
6233 if (pat_index[i] == pat_idx)
6234 break;
6235
6236 switch (i) {
6237 case PAT_WRITE_BACK:
6238 mode = "WB";
6239 break;
6240 case PAT_WRITE_THROUGH:
6241 mode = "WT";
6242 break;
6243 case PAT_UNCACHEABLE:
6244 mode = "UC";
6245 break;
6246 case PAT_UNCACHED:
6247 mode = "U-";
6248 break;
6249 case PAT_WRITE_PROTECTED:
6250 mode = "WP";
6251 break;
6252 case PAT_WRITE_COMBINING:
6253 mode = "WC";
6254 break;
6255 default:
6256 printf("%s: unknown PAT mode %#x for range 0x%08x-0x%08x\n",
6257 __func__, pat_idx, range->sva, eva);
6258 mode = "??";
6259 break;
6260 }
6261
6262 sbuf_printf(sb, "0x%08x-0x%08x r%c%c%c%c %s %d %d %d\n",
6263 range->sva, eva,
6264 (range->attrs & PG_RW) != 0 ? 'w' : '-',
6265 (range->attrs & pg_nx) != 0 ? '-' : 'x',
6266 (range->attrs & PG_U) != 0 ? 'u' : 's',
6267 (range->attrs & PG_G) != 0 ? 'g' : '-',
6268 mode, range->pdpes, range->pdes, range->ptes);
6269
6270 /* Reset to sentinel value. */
6271 range->sva = 0xffffffff;
6272 }
6273
6274 /*
6275 * Determine whether the attributes specified by a page table entry match those
6276 * being tracked by the current range. This is not quite as simple as a direct
6277 * flag comparison since some PAT modes have multiple representations.
6278 */
6279 static bool
sysctl_kmaps_match(struct pmap_kernel_map_range * range,pt_entry_t attrs)6280 sysctl_kmaps_match(struct pmap_kernel_map_range *range, pt_entry_t attrs)
6281 {
6282 pt_entry_t diff, mask;
6283
6284 mask = pg_nx | PG_G | PG_RW | PG_U | PG_PDE_CACHE;
6285 diff = (range->attrs ^ attrs) & mask;
6286 if (diff == 0)
6287 return (true);
6288 if ((diff & ~PG_PDE_PAT) == 0 &&
6289 pmap_pat_index(kernel_pmap, range->attrs, true) ==
6290 pmap_pat_index(kernel_pmap, attrs, true))
6291 return (true);
6292 return (false);
6293 }
6294
6295 static void
sysctl_kmaps_reinit(struct pmap_kernel_map_range * range,vm_offset_t va,pt_entry_t attrs)6296 sysctl_kmaps_reinit(struct pmap_kernel_map_range *range, vm_offset_t va,
6297 pt_entry_t attrs)
6298 {
6299
6300 memset(range, 0, sizeof(*range));
6301 range->sva = va;
6302 range->attrs = attrs;
6303 }
6304
6305 /*
6306 * Given a leaf PTE, derive the mapping's attributes. If they do not match
6307 * those of the current run, dump the address range and its attributes, and
6308 * begin a new run.
6309 */
6310 static void
sysctl_kmaps_check(struct sbuf * sb,struct pmap_kernel_map_range * range,vm_offset_t va,pd_entry_t pde,pt_entry_t pte)6311 sysctl_kmaps_check(struct sbuf *sb, struct pmap_kernel_map_range *range,
6312 vm_offset_t va, pd_entry_t pde, pt_entry_t pte)
6313 {
6314 pt_entry_t attrs;
6315
6316 attrs = pde & (PG_RW | PG_U | pg_nx);
6317
6318 if ((pde & PG_PS) != 0) {
6319 attrs |= pde & (PG_G | PG_PDE_CACHE);
6320 } else if (pte != 0) {
6321 attrs |= pte & pg_nx;
6322 attrs &= pg_nx | (pte & (PG_RW | PG_U));
6323 attrs |= pte & (PG_G | PG_PTE_CACHE);
6324
6325 /* Canonicalize by always using the PDE PAT bit. */
6326 if ((attrs & PG_PTE_PAT) != 0)
6327 attrs ^= PG_PDE_PAT | PG_PTE_PAT;
6328 }
6329
6330 if (range->sva > va || !sysctl_kmaps_match(range, attrs)) {
6331 sysctl_kmaps_dump(sb, range, va);
6332 sysctl_kmaps_reinit(range, va, attrs);
6333 }
6334 }
6335
6336 static int
__CONCAT(PMTYPE,sysctl_kmaps)6337 __CONCAT(PMTYPE, sysctl_kmaps)(SYSCTL_HANDLER_ARGS)
6338 {
6339 struct pmap_kernel_map_range range;
6340 struct sbuf sbuf, *sb;
6341 pd_entry_t pde;
6342 pt_entry_t *pt, pte;
6343 vm_offset_t sva;
6344 int error;
6345 u_int i, k;
6346
6347 error = sysctl_wire_old_buffer(req, 0);
6348 if (error != 0)
6349 return (error);
6350 sb = &sbuf;
6351 sbuf_new_for_sysctl(sb, NULL, PAGE_SIZE, req);
6352
6353 /* Sentinel value. */
6354 range.sva = 0xffffffff;
6355
6356 /*
6357 * Iterate over the kernel page tables without holding the
6358 * kernel pmap lock. Kernel page table pages are never freed,
6359 * so at worst we will observe inconsistencies in the output.
6360 */
6361 for (sva = 0, i = 0; i < NPTEPG * NPGPTD * NPDEPG ;) {
6362 if (i == 0)
6363 sbuf_printf(sb, "\nLow PDE:\n");
6364 else if (i == LOWPTDI * NPTEPG)
6365 sbuf_printf(sb, "Low PDE dup:\n");
6366 else if (i == PTDPTDI * NPTEPG)
6367 sbuf_printf(sb, "Recursive map:\n");
6368 else if (i == KERNPTDI * NPTEPG)
6369 sbuf_printf(sb, "Kernel base:\n");
6370 else if (i == TRPTDI * NPTEPG)
6371 sbuf_printf(sb, "Trampoline:\n");
6372 pde = IdlePTD[sva >> PDRSHIFT];
6373 if ((pde & PG_V) == 0) {
6374 sva = rounddown2(sva, NBPDR);
6375 sysctl_kmaps_dump(sb, &range, sva);
6376 sva += NBPDR;
6377 i += NPTEPG;
6378 continue;
6379 }
6380 if ((pde & PG_PS) != 0) {
6381 sysctl_kmaps_check(sb, &range, sva, pde, 0);
6382 range.pdes++;
6383 sva += NBPDR;
6384 i += NPTEPG;
6385 continue;
6386 }
6387 for (pt = vtopte(sva), k = 0; k < NPTEPG; i++, k++, pt++,
6388 sva += PAGE_SIZE) {
6389 pte = *pt;
6390 if ((pte & PG_V) == 0) {
6391 sysctl_kmaps_dump(sb, &range, sva);
6392 continue;
6393 }
6394 sysctl_kmaps_check(sb, &range, sva, pde, pte);
6395 range.ptes++;
6396 }
6397 }
6398
6399 error = sbuf_finish(sb);
6400 sbuf_delete(sb);
6401 return (error);
6402 }
6403
6404 #define PMM(a) \
6405 .pm_##a = __CONCAT(PMTYPE, a),
6406
6407 struct pmap_methods __CONCAT(PMTYPE, methods) = {
6408 PMM(ksetrw)
6409 PMM(remap_lower)
6410 PMM(remap_lowptdi)
6411 PMM(align_superpage)
6412 PMM(quick_enter_page)
6413 PMM(quick_remove_page)
6414 PMM(trm_alloc)
6415 PMM(trm_free)
6416 PMM(get_map_low)
6417 PMM(get_vm_maxuser_address)
6418 PMM(kextract)
6419 PMM(pg_frame)
6420 PMM(sf_buf_map)
6421 PMM(cp_slow0_map)
6422 PMM(get_kcr3)
6423 PMM(get_cr3)
6424 PMM(cmap3)
6425 PMM(basemem_setup)
6426 PMM(set_nx)
6427 PMM(bios16_enter)
6428 PMM(bios16_leave)
6429 PMM(bootstrap)
6430 PMM(is_valid_memattr)
6431 PMM(cache_bits)
6432 PMM(ps_enabled)
6433 PMM(pinit0)
6434 PMM(pinit)
6435 PMM(activate)
6436 PMM(activate_boot)
6437 PMM(advise)
6438 PMM(clear_modify)
6439 PMM(change_attr)
6440 PMM(mincore)
6441 PMM(copy)
6442 PMM(copy_page)
6443 PMM(copy_pages)
6444 PMM(zero_page)
6445 PMM(zero_page_area)
6446 PMM(enter)
6447 PMM(enter_object)
6448 PMM(enter_quick)
6449 PMM(kenter_temporary)
6450 PMM(object_init_pt)
6451 PMM(unwire)
6452 PMM(page_exists_quick)
6453 PMM(page_wired_mappings)
6454 PMM(page_is_mapped)
6455 PMM(remove_pages)
6456 PMM(is_modified)
6457 PMM(is_prefaultable)
6458 PMM(is_referenced)
6459 PMM(remove_write)
6460 PMM(ts_referenced)
6461 PMM(mapdev_attr)
6462 PMM(unmapdev)
6463 PMM(page_set_memattr)
6464 PMM(extract)
6465 PMM(extract_and_hold)
6466 PMM(map)
6467 PMM(qenter)
6468 PMM(qremove)
6469 PMM(release)
6470 PMM(remove)
6471 PMM(protect)
6472 PMM(remove_all)
6473 PMM(init)
6474 PMM(init_pat)
6475 PMM(growkernel)
6476 PMM(invalidate_page)
6477 PMM(invalidate_range)
6478 PMM(invalidate_all)
6479 PMM(invalidate_cache)
6480 PMM(flush_page)
6481 PMM(kenter)
6482 PMM(kremove)
6483 PMM(sysctl_kmaps)
6484 };
6485