1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2018 Matthew Macy
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 *
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28 #include "opt_platform.h"
29
30 #include <sys/param.h>
31 #include <sys/kernel.h>
32 #include <sys/systm.h>
33 #include <sys/conf.h>
34 #include <sys/bitstring.h>
35 #include <sys/queue.h>
36 #include <sys/cpuset.h>
37 #include <sys/endian.h>
38 #include <sys/kerneldump.h>
39 #include <sys/ktr.h>
40 #include <sys/lock.h>
41 #include <sys/syslog.h>
42 #include <sys/msgbuf.h>
43 #include <sys/malloc.h>
44 #include <sys/mman.h>
45 #include <sys/mutex.h>
46 #include <sys/proc.h>
47 #include <sys/rwlock.h>
48 #include <sys/sched.h>
49 #include <sys/sysctl.h>
50 #include <sys/systm.h>
51 #include <sys/vmem.h>
52 #include <sys/vmmeter.h>
53 #include <sys/smp.h>
54
55 #include <sys/kdb.h>
56
57 #include <dev/ofw/openfirm.h>
58
59 #include <vm/vm.h>
60 #include <vm/pmap.h>
61 #include <vm/vm_param.h>
62 #include <vm/vm_kern.h>
63 #include <vm/vm_page.h>
64 #include <vm/vm_map.h>
65 #include <vm/vm_object.h>
66 #include <vm/vm_extern.h>
67 #include <vm/vm_pageout.h>
68 #include <vm/vm_phys.h>
69 #include <vm/vm_radix.h>
70 #include <vm/vm_reserv.h>
71 #include <vm/vm_dumpset.h>
72 #include <vm/uma.h>
73
74 #include <machine/_inttypes.h>
75 #include <machine/cpu.h>
76 #include <machine/platform.h>
77 #include <machine/frame.h>
78 #include <machine/md_var.h>
79 #include <machine/psl.h>
80 #include <machine/bat.h>
81 #include <machine/hid.h>
82 #include <machine/pte.h>
83 #include <machine/sr.h>
84 #include <machine/trap.h>
85 #include <machine/mmuvar.h>
86
87 /* For pseries bit. */
88 #include <powerpc/pseries/phyp-hvcall.h>
89
90 #ifdef INVARIANTS
91 #include <vm/uma_dbg.h>
92 #endif
93
94 #define PPC_BITLSHIFT(bit) (sizeof(long)*NBBY - 1 - (bit))
95 #define PPC_BIT(bit) (1UL << PPC_BITLSHIFT(bit))
96 #define PPC_BITLSHIFT_VAL(val, bit) ((val) << PPC_BITLSHIFT(bit))
97
98 #include "opt_ddb.h"
99
100 #ifdef DDB
101 static void pmap_pte_walk(pml1_entry_t *l1, vm_offset_t va);
102 #endif
103
104 #define PG_W RPTE_WIRED
105 #define PG_V RPTE_VALID
106 #define PG_MANAGED RPTE_MANAGED
107 #define PG_PROMOTED RPTE_PROMOTED
108 #define PG_M RPTE_C
109 #define PG_A RPTE_R
110 #define PG_X RPTE_EAA_X
111 #define PG_RW RPTE_EAA_W
112 #define PG_PTE_CACHE RPTE_ATTR_MASK
113
114 #define RPTE_SHIFT 9
115 #define NLS_MASK ((1UL<<5)-1)
116 #define RPTE_ENTRIES (1UL<<RPTE_SHIFT)
117 #define RPTE_MASK (RPTE_ENTRIES-1)
118
119 #define NLB_SHIFT 0
120 #define NLB_MASK (((1UL<<52)-1) << 8)
121
122 extern int nkpt;
123 extern caddr_t crashdumpmap;
124
125 #define RIC_FLUSH_TLB 0
126 #define RIC_FLUSH_PWC 1
127 #define RIC_FLUSH_ALL 2
128
129 #define POWER9_TLB_SETS_RADIX 128 /* # sets in POWER9 TLB Radix mode */
130
131 #define PPC_INST_TLBIE 0x7c000264
132 #define PPC_INST_TLBIEL 0x7c000224
133 #define PPC_INST_SLBIA 0x7c0003e4
134
135 #define ___PPC_RA(a) (((a) & 0x1f) << 16)
136 #define ___PPC_RB(b) (((b) & 0x1f) << 11)
137 #define ___PPC_RS(s) (((s) & 0x1f) << 21)
138 #define ___PPC_RT(t) ___PPC_RS(t)
139 #define ___PPC_R(r) (((r) & 0x1) << 16)
140 #define ___PPC_PRS(prs) (((prs) & 0x1) << 17)
141 #define ___PPC_RIC(ric) (((ric) & 0x3) << 18)
142
143 #define PPC_SLBIA(IH) __XSTRING(.long PPC_INST_SLBIA | \
144 ((IH & 0x7) << 21))
145 #define PPC_TLBIE_5(rb,rs,ric,prs,r) \
146 __XSTRING(.long PPC_INST_TLBIE | \
147 ___PPC_RB(rb) | ___PPC_RS(rs) | \
148 ___PPC_RIC(ric) | ___PPC_PRS(prs) | \
149 ___PPC_R(r))
150
151 #define PPC_TLBIEL(rb,rs,ric,prs,r) \
152 __XSTRING(.long PPC_INST_TLBIEL | \
153 ___PPC_RB(rb) | ___PPC_RS(rs) | \
154 ___PPC_RIC(ric) | ___PPC_PRS(prs) | \
155 ___PPC_R(r))
156
157 #define PPC_INVALIDATE_ERAT PPC_SLBIA(7)
158
159 static __inline void
ttusync(void)160 ttusync(void)
161 {
162 __asm __volatile("eieio; tlbsync; ptesync" ::: "memory");
163 }
164
165 #define TLBIEL_INVAL_SEL_MASK 0xc00 /* invalidation selector */
166 #define TLBIEL_INVAL_PAGE 0x000 /* invalidate a single page */
167 #define TLBIEL_INVAL_SET_PID 0x400 /* invalidate a set for the current PID */
168 #define TLBIEL_INVAL_SET_LPID 0x800 /* invalidate a set for current LPID */
169 #define TLBIEL_INVAL_SET 0xc00 /* invalidate a set for all LPIDs */
170
171 #define TLBIE_ACTUAL_PAGE_MASK 0xe0
172 #define TLBIE_ACTUAL_PAGE_4K 0x00
173 #define TLBIE_ACTUAL_PAGE_64K 0xa0
174 #define TLBIE_ACTUAL_PAGE_2M 0x20
175 #define TLBIE_ACTUAL_PAGE_1G 0x40
176
177 #define TLBIE_PRS_PARTITION_SCOPE 0x0
178 #define TLBIE_PRS_PROCESS_SCOPE 0x1
179
180 #define TLBIE_RIC_INVALIDATE_TLB 0x0 /* Invalidate just TLB */
181 #define TLBIE_RIC_INVALIDATE_PWC 0x1 /* Invalidate just PWC */
182 #define TLBIE_RIC_INVALIDATE_ALL 0x2 /* Invalidate TLB, PWC,
183 * cached {proc, part}tab entries
184 */
185 #define TLBIE_RIC_INVALIDATE_SEQ 0x3 /* HPT - only:
186 * Invalidate a range of translations
187 */
188
189 static __always_inline void
radix_tlbie(uint8_t ric,uint8_t prs,uint16_t is,uint32_t pid,uint32_t lpid,vm_offset_t va,uint16_t ap)190 radix_tlbie(uint8_t ric, uint8_t prs, uint16_t is, uint32_t pid, uint32_t lpid,
191 vm_offset_t va, uint16_t ap)
192 {
193 uint64_t rb, rs;
194
195 MPASS((va & PAGE_MASK) == 0);
196
197 rs = ((uint64_t)pid << 32) | lpid;
198 rb = va | is | ap;
199 __asm __volatile(PPC_TLBIE_5(%0, %1, %2, %3, 1) : :
200 "r" (rb), "r" (rs), "i" (ric), "i" (prs) : "memory");
201 }
202
203 static __inline void
radix_tlbie_fixup(uint32_t pid,vm_offset_t va,int ap)204 radix_tlbie_fixup(uint32_t pid, vm_offset_t va, int ap)
205 {
206
207 __asm __volatile("ptesync" ::: "memory");
208 radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE,
209 TLBIEL_INVAL_PAGE, 0, 0, va, ap);
210 __asm __volatile("ptesync" ::: "memory");
211 radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE,
212 TLBIEL_INVAL_PAGE, pid, 0, va, ap);
213 }
214
215 static __inline void
radix_tlbie_invlpg_user_4k(uint32_t pid,vm_offset_t va)216 radix_tlbie_invlpg_user_4k(uint32_t pid, vm_offset_t va)
217 {
218
219 radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE,
220 TLBIEL_INVAL_PAGE, pid, 0, va, TLBIE_ACTUAL_PAGE_4K);
221 radix_tlbie_fixup(pid, va, TLBIE_ACTUAL_PAGE_4K);
222 }
223
224 static __inline void
radix_tlbie_invlpg_user_2m(uint32_t pid,vm_offset_t va)225 radix_tlbie_invlpg_user_2m(uint32_t pid, vm_offset_t va)
226 {
227
228 radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE,
229 TLBIEL_INVAL_PAGE, pid, 0, va, TLBIE_ACTUAL_PAGE_2M);
230 radix_tlbie_fixup(pid, va, TLBIE_ACTUAL_PAGE_2M);
231 }
232
233 static __inline void
radix_tlbie_invlpwc_user(uint32_t pid)234 radix_tlbie_invlpwc_user(uint32_t pid)
235 {
236
237 radix_tlbie(TLBIE_RIC_INVALIDATE_PWC, TLBIE_PRS_PROCESS_SCOPE,
238 TLBIEL_INVAL_SET_PID, pid, 0, 0, 0);
239 }
240
241 static __inline void
radix_tlbie_flush_user(uint32_t pid)242 radix_tlbie_flush_user(uint32_t pid)
243 {
244
245 radix_tlbie(TLBIE_RIC_INVALIDATE_ALL, TLBIE_PRS_PROCESS_SCOPE,
246 TLBIEL_INVAL_SET_PID, pid, 0, 0, 0);
247 }
248
249 static __inline void
radix_tlbie_invlpg_kernel_4k(vm_offset_t va)250 radix_tlbie_invlpg_kernel_4k(vm_offset_t va)
251 {
252
253 radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE,
254 TLBIEL_INVAL_PAGE, 0, 0, va, TLBIE_ACTUAL_PAGE_4K);
255 radix_tlbie_fixup(0, va, TLBIE_ACTUAL_PAGE_4K);
256 }
257
258 static __inline void
radix_tlbie_invlpg_kernel_2m(vm_offset_t va)259 radix_tlbie_invlpg_kernel_2m(vm_offset_t va)
260 {
261
262 radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE,
263 TLBIEL_INVAL_PAGE, 0, 0, va, TLBIE_ACTUAL_PAGE_2M);
264 radix_tlbie_fixup(0, va, TLBIE_ACTUAL_PAGE_2M);
265 }
266
267 /* 1GB pages aren't currently supported. */
268 static __inline __unused void
radix_tlbie_invlpg_kernel_1g(vm_offset_t va)269 radix_tlbie_invlpg_kernel_1g(vm_offset_t va)
270 {
271
272 radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE,
273 TLBIEL_INVAL_PAGE, 0, 0, va, TLBIE_ACTUAL_PAGE_1G);
274 radix_tlbie_fixup(0, va, TLBIE_ACTUAL_PAGE_1G);
275 }
276
277 static __inline void
radix_tlbie_invlpwc_kernel(void)278 radix_tlbie_invlpwc_kernel(void)
279 {
280
281 radix_tlbie(TLBIE_RIC_INVALIDATE_PWC, TLBIE_PRS_PROCESS_SCOPE,
282 TLBIEL_INVAL_SET_LPID, 0, 0, 0, 0);
283 }
284
285 static __inline void
radix_tlbie_flush_kernel(void)286 radix_tlbie_flush_kernel(void)
287 {
288
289 radix_tlbie(TLBIE_RIC_INVALIDATE_ALL, TLBIE_PRS_PROCESS_SCOPE,
290 TLBIEL_INVAL_SET_LPID, 0, 0, 0, 0);
291 }
292
293 static __inline vm_pindex_t
pmap_l3e_pindex(vm_offset_t va)294 pmap_l3e_pindex(vm_offset_t va)
295 {
296 return ((va & PG_FRAME) >> L3_PAGE_SIZE_SHIFT);
297 }
298
299 static __inline vm_pindex_t
pmap_pml3e_index(vm_offset_t va)300 pmap_pml3e_index(vm_offset_t va)
301 {
302
303 return ((va >> L3_PAGE_SIZE_SHIFT) & RPTE_MASK);
304 }
305
306 static __inline vm_pindex_t
pmap_pml2e_index(vm_offset_t va)307 pmap_pml2e_index(vm_offset_t va)
308 {
309 return ((va >> L2_PAGE_SIZE_SHIFT) & RPTE_MASK);
310 }
311
312 static __inline vm_pindex_t
pmap_pml1e_index(vm_offset_t va)313 pmap_pml1e_index(vm_offset_t va)
314 {
315 return ((va & PG_FRAME) >> L1_PAGE_SIZE_SHIFT);
316 }
317
318 /* Return various clipped indexes for a given VA */
319 static __inline vm_pindex_t
pmap_pte_index(vm_offset_t va)320 pmap_pte_index(vm_offset_t va)
321 {
322
323 return ((va >> PAGE_SHIFT) & RPTE_MASK);
324 }
325
326 /* Return a pointer to the PT slot that corresponds to a VA */
327 static __inline pt_entry_t *
pmap_l3e_to_pte(pt_entry_t * l3e,vm_offset_t va)328 pmap_l3e_to_pte(pt_entry_t *l3e, vm_offset_t va)
329 {
330 pt_entry_t *pte;
331 vm_paddr_t ptepa;
332
333 ptepa = (be64toh(*l3e) & NLB_MASK);
334 pte = PHYS_TO_DMAP(ptepa);
335 return (&pte[pmap_pte_index(va)]);
336 }
337
338 /* Return a pointer to the PD slot that corresponds to a VA */
339 static __inline pt_entry_t *
pmap_l2e_to_l3e(pt_entry_t * l2e,vm_offset_t va)340 pmap_l2e_to_l3e(pt_entry_t *l2e, vm_offset_t va)
341 {
342 pt_entry_t *l3e;
343 vm_paddr_t l3pa;
344
345 l3pa = (be64toh(*l2e) & NLB_MASK);
346 l3e = PHYS_TO_DMAP(l3pa);
347 return (&l3e[pmap_pml3e_index(va)]);
348 }
349
350 /* Return a pointer to the PD slot that corresponds to a VA */
351 static __inline pt_entry_t *
pmap_l1e_to_l2e(pt_entry_t * l1e,vm_offset_t va)352 pmap_l1e_to_l2e(pt_entry_t *l1e, vm_offset_t va)
353 {
354 pt_entry_t *l2e;
355 vm_paddr_t l2pa;
356
357 l2pa = (be64toh(*l1e) & NLB_MASK);
358
359 l2e = PHYS_TO_DMAP(l2pa);
360 return (&l2e[pmap_pml2e_index(va)]);
361 }
362
363 static __inline pml1_entry_t *
pmap_pml1e(pmap_t pmap,vm_offset_t va)364 pmap_pml1e(pmap_t pmap, vm_offset_t va)
365 {
366
367 return (&pmap->pm_pml1[pmap_pml1e_index(va)]);
368 }
369
370 static pt_entry_t *
pmap_pml2e(pmap_t pmap,vm_offset_t va)371 pmap_pml2e(pmap_t pmap, vm_offset_t va)
372 {
373 pt_entry_t *l1e;
374
375 l1e = pmap_pml1e(pmap, va);
376 if (l1e == NULL || (be64toh(*l1e) & RPTE_VALID) == 0)
377 return (NULL);
378 return (pmap_l1e_to_l2e(l1e, va));
379 }
380
381 static __inline pt_entry_t *
pmap_pml3e(pmap_t pmap,vm_offset_t va)382 pmap_pml3e(pmap_t pmap, vm_offset_t va)
383 {
384 pt_entry_t *l2e;
385
386 l2e = pmap_pml2e(pmap, va);
387 if (l2e == NULL || (be64toh(*l2e) & RPTE_VALID) == 0)
388 return (NULL);
389 return (pmap_l2e_to_l3e(l2e, va));
390 }
391
392 static __inline pt_entry_t *
pmap_pte(pmap_t pmap,vm_offset_t va)393 pmap_pte(pmap_t pmap, vm_offset_t va)
394 {
395 pt_entry_t *l3e;
396
397 l3e = pmap_pml3e(pmap, va);
398 if (l3e == NULL || (be64toh(*l3e) & RPTE_VALID) == 0)
399 return (NULL);
400 return (pmap_l3e_to_pte(l3e, va));
401 }
402
403 int nkpt = 64;
404 SYSCTL_INT(_machdep, OID_AUTO, nkpt, CTLFLAG_RD, &nkpt, 0,
405 "Number of kernel page table pages allocated on bootup");
406
407 vm_paddr_t dmaplimit;
408
409 SYSCTL_DECL(_vm_pmap);
410
411 #ifdef INVARIANTS
412 #define VERBOSE_PMAP 0
413 #define VERBOSE_PROTECT 0
414 static int pmap_logging;
415 SYSCTL_INT(_vm_pmap, OID_AUTO, pmap_logging, CTLFLAG_RWTUN,
416 &pmap_logging, 0, "verbose debug logging");
417 #endif
418
419 static u_int64_t KPTphys; /* phys addr of kernel level 1 */
420
421 //static vm_paddr_t KERNend; /* phys addr of end of bootstrap data */
422
423 static vm_offset_t qframe = 0;
424 static struct mtx qframe_mtx;
425
426 void mmu_radix_activate(struct thread *);
427 void mmu_radix_advise(pmap_t, vm_offset_t, vm_offset_t, int);
428 void mmu_radix_align_superpage(vm_object_t, vm_ooffset_t, vm_offset_t *,
429 vm_size_t);
430 void mmu_radix_clear_modify(vm_page_t);
431 void mmu_radix_copy(pmap_t, pmap_t, vm_offset_t, vm_size_t, vm_offset_t);
432 int mmu_radix_decode_kernel_ptr(vm_offset_t, int *, vm_offset_t *);
433 int mmu_radix_enter(pmap_t, vm_offset_t, vm_page_t, vm_prot_t, u_int, int8_t);
434 void mmu_radix_enter_object(pmap_t, vm_offset_t, vm_offset_t, vm_page_t,
435 vm_prot_t);
436 void mmu_radix_enter_quick(pmap_t, vm_offset_t, vm_page_t, vm_prot_t);
437 vm_paddr_t mmu_radix_extract(pmap_t pmap, vm_offset_t va);
438 vm_page_t mmu_radix_extract_and_hold(pmap_t, vm_offset_t, vm_prot_t);
439 void mmu_radix_kenter(vm_offset_t, vm_paddr_t);
440 vm_paddr_t mmu_radix_kextract(vm_offset_t);
441 void mmu_radix_kremove(vm_offset_t);
442 bool mmu_radix_is_modified(vm_page_t);
443 bool mmu_radix_is_prefaultable(pmap_t, vm_offset_t);
444 bool mmu_radix_is_referenced(vm_page_t);
445 void mmu_radix_object_init_pt(pmap_t, vm_offset_t, vm_object_t,
446 vm_pindex_t, vm_size_t);
447 bool mmu_radix_page_exists_quick(pmap_t, vm_page_t);
448 void mmu_radix_page_init(vm_page_t);
449 bool mmu_radix_page_is_mapped(vm_page_t m);
450 void mmu_radix_page_set_memattr(vm_page_t, vm_memattr_t);
451 int mmu_radix_page_wired_mappings(vm_page_t);
452 int mmu_radix_pinit(pmap_t);
453 void mmu_radix_protect(pmap_t, vm_offset_t, vm_offset_t, vm_prot_t);
454 bool mmu_radix_ps_enabled(pmap_t);
455 void mmu_radix_qenter(void *, vm_page_t *, int);
456 void mmu_radix_qremove(void *, int);
457 void *mmu_radix_quick_enter_page(vm_page_t);
458 void mmu_radix_quick_remove_page(void *);
459 int mmu_radix_ts_referenced(vm_page_t);
460 void mmu_radix_release(pmap_t);
461 void mmu_radix_remove(pmap_t, vm_offset_t, vm_offset_t);
462 void mmu_radix_remove_all(vm_page_t);
463 void mmu_radix_remove_pages(pmap_t);
464 void mmu_radix_remove_write(vm_page_t);
465 void mmu_radix_sync_icache(pmap_t pm, vm_offset_t va, vm_size_t sz);
466 void mmu_radix_unwire(pmap_t, vm_offset_t, vm_offset_t);
467 void mmu_radix_zero_page(vm_page_t);
468 void mmu_radix_zero_page_area(vm_page_t, int, int);
469 int mmu_radix_change_attr(void *, vm_size_t, vm_memattr_t);
470 void mmu_radix_page_array_startup(long pages);
471
472 #include "mmu_oea64.h"
473
474 /*
475 * Kernel MMU interface
476 */
477
478 static void mmu_radix_bootstrap(vm_offset_t, vm_offset_t);
479
480 static void mmu_radix_copy_page(vm_page_t, vm_page_t);
481 static void mmu_radix_copy_pages(vm_page_t *ma, vm_offset_t a_offset,
482 vm_page_t *mb, vm_offset_t b_offset, int xfersize);
483 static int mmu_radix_growkernel(vm_offset_t);
484 static void mmu_radix_init(void);
485 static int mmu_radix_mincore(pmap_t, vm_offset_t, vm_paddr_t *);
486 static void *mmu_radix_map(vm_offset_t *, vm_paddr_t, vm_paddr_t, int);
487 static void mmu_radix_pinit0(pmap_t);
488
489 static void *mmu_radix_mapdev(vm_paddr_t, vm_size_t);
490 static void *mmu_radix_mapdev_attr(vm_paddr_t, vm_size_t, vm_memattr_t);
491 static void mmu_radix_unmapdev(void *, vm_size_t);
492 static void mmu_radix_kenter_attr(vm_offset_t, vm_paddr_t, vm_memattr_t ma);
493 static int mmu_radix_dev_direct_mapped(vm_paddr_t, vm_size_t);
494 static void mmu_radix_dumpsys_map(vm_paddr_t pa, size_t sz, void **va);
495 static void mmu_radix_scan_init(void);
496 static void mmu_radix_cpu_bootstrap(int ap);
497 static void mmu_radix_tlbie_all(void);
498
499 static struct pmap_funcs mmu_radix_methods = {
500 .bootstrap = mmu_radix_bootstrap,
501 .copy_page = mmu_radix_copy_page,
502 .copy_pages = mmu_radix_copy_pages,
503 .cpu_bootstrap = mmu_radix_cpu_bootstrap,
504 .growkernel_nopanic = mmu_radix_growkernel,
505 .init = mmu_radix_init,
506 .map = mmu_radix_map,
507 .mincore = mmu_radix_mincore,
508 .pinit = mmu_radix_pinit,
509 .pinit0 = mmu_radix_pinit0,
510
511 .mapdev = mmu_radix_mapdev,
512 .mapdev_attr = mmu_radix_mapdev_attr,
513 .unmapdev = mmu_radix_unmapdev,
514 .kenter_attr = mmu_radix_kenter_attr,
515 .dev_direct_mapped = mmu_radix_dev_direct_mapped,
516 .dumpsys_pa_init = mmu_radix_scan_init,
517 .dumpsys_map_chunk = mmu_radix_dumpsys_map,
518 .page_is_mapped = mmu_radix_page_is_mapped,
519 .ps_enabled = mmu_radix_ps_enabled,
520 .align_superpage = mmu_radix_align_superpage,
521 .object_init_pt = mmu_radix_object_init_pt,
522 .protect = mmu_radix_protect,
523 /* pmap dispatcher interface */
524 .clear_modify = mmu_radix_clear_modify,
525 .copy = mmu_radix_copy,
526 .enter = mmu_radix_enter,
527 .enter_object = mmu_radix_enter_object,
528 .enter_quick = mmu_radix_enter_quick,
529 .extract = mmu_radix_extract,
530 .extract_and_hold = mmu_radix_extract_and_hold,
531 .is_modified = mmu_radix_is_modified,
532 .is_prefaultable = mmu_radix_is_prefaultable,
533 .is_referenced = mmu_radix_is_referenced,
534 .ts_referenced = mmu_radix_ts_referenced,
535 .page_exists_quick = mmu_radix_page_exists_quick,
536 .page_init = mmu_radix_page_init,
537 .page_wired_mappings = mmu_radix_page_wired_mappings,
538 .qenter = mmu_radix_qenter,
539 .qremove = mmu_radix_qremove,
540 .release = mmu_radix_release,
541 .remove = mmu_radix_remove,
542 .remove_all = mmu_radix_remove_all,
543 .remove_write = mmu_radix_remove_write,
544 .sync_icache = mmu_radix_sync_icache,
545 .unwire = mmu_radix_unwire,
546 .zero_page = mmu_radix_zero_page,
547 .zero_page_area = mmu_radix_zero_page_area,
548 .activate = mmu_radix_activate,
549 .quick_enter_page = mmu_radix_quick_enter_page,
550 .quick_remove_page = mmu_radix_quick_remove_page,
551 .page_set_memattr = mmu_radix_page_set_memattr,
552 .page_array_startup = mmu_radix_page_array_startup,
553
554 /* Internal interfaces */
555 .kenter = mmu_radix_kenter,
556 .kextract = mmu_radix_kextract,
557 .kremove = mmu_radix_kremove,
558 .change_attr = mmu_radix_change_attr,
559 .decode_kernel_ptr = mmu_radix_decode_kernel_ptr,
560
561 .tlbie_all = mmu_radix_tlbie_all,
562 };
563
564 MMU_DEF(mmu_radix, MMU_TYPE_RADIX, mmu_radix_methods);
565
566 static bool pmap_demote_l3e_locked(pmap_t pmap, pml3_entry_t *l3e, vm_offset_t va,
567 struct rwlock **lockp);
568 static bool pmap_demote_l3e(pmap_t pmap, pml3_entry_t *pde, vm_offset_t va);
569 static int pmap_unuse_pt(pmap_t, vm_offset_t, pml3_entry_t, struct spglist *);
570 static int pmap_remove_l3e(pmap_t pmap, pml3_entry_t *pdq, vm_offset_t sva,
571 struct spglist *free, struct rwlock **lockp);
572 static int pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t sva,
573 pml3_entry_t ptepde, struct spglist *free, struct rwlock **lockp);
574 static vm_page_t pmap_remove_pt_page(pmap_t pmap, vm_offset_t va);
575 static bool pmap_remove_page(pmap_t pmap, vm_offset_t va, pml3_entry_t *pde,
576 struct spglist *free);
577 static bool pmap_remove_ptes(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,
578 pml3_entry_t *l3e, struct spglist *free, struct rwlock **lockp);
579
580 static bool pmap_pv_insert_l3e(pmap_t pmap, vm_offset_t va, pml3_entry_t l3e,
581 u_int flags, struct rwlock **lockp);
582 #if VM_NRESERVLEVEL > 0
583 static void pmap_pv_promote_l3e(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,
584 struct rwlock **lockp);
585 #endif
586 static void pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va);
587 static int pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte);
588 static vm_page_t mmu_radix_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m,
589 vm_prot_t prot, vm_page_t mpte, struct rwlock **lockp, bool *invalidate);
590
591 static bool pmap_enter_2mpage(pmap_t pmap, vm_offset_t va, vm_page_t m,
592 vm_prot_t prot, struct rwlock **lockp);
593 static int pmap_enter_l3e(pmap_t pmap, vm_offset_t va, pml3_entry_t newpde,
594 u_int flags, vm_page_t m, struct rwlock **lockp);
595
596 static vm_page_t reclaim_pv_chunk(pmap_t locked_pmap, struct rwlock **lockp);
597 static void free_pv_chunk(struct pv_chunk *pc);
598 static vm_page_t _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, struct rwlock **lockp);
599 static vm_page_t pmap_allocl3e(pmap_t pmap, vm_offset_t va,
600 struct rwlock **lockp);
601 static vm_page_t pmap_allocpte(pmap_t pmap, vm_offset_t va,
602 struct rwlock **lockp);
603 static void _pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m,
604 struct spglist *free);
605 static bool pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free);
606
607 static void pmap_invalidate_page(pmap_t pmap, vm_offset_t start);
608 static void pmap_invalidate_all(pmap_t pmap);
609 static int pmap_change_attr_locked(void *va, vm_size_t size, int mode, bool flush);
610 static void pmap_fill_ptp(pt_entry_t *firstpte, pt_entry_t newpte);
611
612 /*
613 * Internal flags for pmap_enter()'s helper functions.
614 */
615 #define PMAP_ENTER_NORECLAIM 0x1000000 /* Don't reclaim PV entries. */
616 #define PMAP_ENTER_NOREPLACE 0x2000000 /* Don't replace mappings. */
617
618 #define UNIMPLEMENTED() panic("%s not implemented", __func__)
619 #define UNTESTED() panic("%s not yet tested", __func__)
620
621 /* Number of supported PID bits */
622 static unsigned int isa3_pid_bits;
623
624 /* PID to start allocating from */
625 static unsigned int isa3_base_pid;
626
627 #define PROCTAB_SIZE_SHIFT (isa3_pid_bits + 4)
628 #define PROCTAB_ENTRIES (1ul << isa3_pid_bits)
629
630 /*
631 * Map of physical memory regions.
632 */
633 static struct mem_region *regions, *pregions;
634 static struct numa_mem_region *numa_pregions;
635 static int regions_sz, pregions_sz, numa_pregions_sz;
636 static struct pate *isa3_parttab;
637 static struct prte *isa3_proctab;
638 static vmem_t *asid_arena;
639
640 extern void bs_remap_earlyboot(void);
641
642 #define RADIX_PGD_SIZE_SHIFT 16
643 #define RADIX_PGD_SIZE (1UL << RADIX_PGD_SIZE_SHIFT)
644
645 #define RADIX_PGD_INDEX_SHIFT (RADIX_PGD_SIZE_SHIFT-3)
646 #define NL2EPG (PAGE_SIZE/sizeof(pml2_entry_t))
647 #define NL3EPG (PAGE_SIZE/sizeof(pml3_entry_t))
648
649 #define NUPML1E (RADIX_PGD_SIZE/sizeof(uint64_t)) /* number of userland PML1 pages */
650 #define NUPDPE (NUPML1E * NL2EPG)/* number of userland PDP pages */
651 #define NUPDE (NUPDPE * NL3EPG) /* number of userland PD entries */
652
653 /* POWER9 only permits a 64k partition table size. */
654 #define PARTTAB_SIZE_SHIFT 16
655 #define PARTTAB_SIZE (1UL << PARTTAB_SIZE_SHIFT)
656
657 #define PARTTAB_HR (1UL << 63) /* host uses radix */
658 #define PARTTAB_GR (1UL << 63) /* guest uses radix must match host */
659
660 /* TLB flush actions. Used as argument to tlbiel_flush() */
661 enum {
662 TLB_INVAL_SCOPE_LPID = 2, /* invalidate TLBs for current LPID */
663 TLB_INVAL_SCOPE_GLOBAL = 3, /* invalidate all TLBs */
664 };
665
666 #define NPV_LIST_LOCKS MAXCPU
667 static int pmap_initialized;
668 static vm_paddr_t proctab0pa;
669 static vm_paddr_t parttab_phys;
670 CTASSERT(sizeof(struct pv_chunk) == PAGE_SIZE);
671
672 /*
673 * Data for the pv entry allocation mechanism.
674 * Updates to pv_invl_gen are protected by the pv_list_locks[]
675 * elements, but reads are not.
676 */
677 static TAILQ_HEAD(pch, pv_chunk) pv_chunks = TAILQ_HEAD_INITIALIZER(pv_chunks);
678 static struct mtx __exclusive_cache_line pv_chunks_mutex;
679 static struct rwlock __exclusive_cache_line pv_list_locks[NPV_LIST_LOCKS];
680 static struct md_page *pv_table;
681 static struct md_page pv_dummy;
682
683 #ifdef PV_STATS
684 #define PV_STAT(x) do { x ; } while (0)
685 #else
686 #define PV_STAT(x) do { } while (0)
687 #endif
688
689 #define pa_radix_index(pa) ((pa) >> L3_PAGE_SIZE_SHIFT)
690 #define pa_to_pvh(pa) (&pv_table[pa_radix_index(pa)])
691
692 #define PHYS_TO_PV_LIST_LOCK(pa) \
693 (&pv_list_locks[pa_radix_index(pa) % NPV_LIST_LOCKS])
694
695 #define CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa) do { \
696 struct rwlock **_lockp = (lockp); \
697 struct rwlock *_new_lock; \
698 \
699 _new_lock = PHYS_TO_PV_LIST_LOCK(pa); \
700 if (_new_lock != *_lockp) { \
701 if (*_lockp != NULL) \
702 rw_wunlock(*_lockp); \
703 *_lockp = _new_lock; \
704 rw_wlock(*_lockp); \
705 } \
706 } while (0)
707
708 #define CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m) \
709 CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, VM_PAGE_TO_PHYS(m))
710
711 #define RELEASE_PV_LIST_LOCK(lockp) do { \
712 struct rwlock **_lockp = (lockp); \
713 \
714 if (*_lockp != NULL) { \
715 rw_wunlock(*_lockp); \
716 *_lockp = NULL; \
717 } \
718 } while (0)
719
720 #define VM_PAGE_TO_PV_LIST_LOCK(m) \
721 PHYS_TO_PV_LIST_LOCK(VM_PAGE_TO_PHYS(m))
722
723 /*
724 * We support 52 bits, hence:
725 * bits 52 - 31 = 21, 0b10101
726 * RTS encoding details
727 * bits 0 - 3 of rts -> bits 6 - 8 unsigned long
728 * bits 4 - 5 of rts -> bits 62 - 63 of unsigned long
729 */
730 #define RTS_SIZE ((0x2UL << 61) | (0x5UL << 5))
731
732 static int powernv_enabled = 1;
733
734 static __always_inline void
tlbiel_radix_set_isa300(uint32_t set,uint32_t is,uint32_t pid,uint32_t ric,uint32_t prs)735 tlbiel_radix_set_isa300(uint32_t set, uint32_t is,
736 uint32_t pid, uint32_t ric, uint32_t prs)
737 {
738 uint64_t rb;
739 uint64_t rs;
740
741 rb = PPC_BITLSHIFT_VAL(set, 51) | PPC_BITLSHIFT_VAL(is, 53);
742 rs = PPC_BITLSHIFT_VAL((uint64_t)pid, 31);
743
744 __asm __volatile(PPC_TLBIEL(%0, %1, %2, %3, 1)
745 : : "r"(rb), "r"(rs), "i"(ric), "i"(prs)
746 : "memory");
747 }
748
749 static void
tlbiel_flush_isa3(uint32_t num_sets,uint32_t is)750 tlbiel_flush_isa3(uint32_t num_sets, uint32_t is)
751 {
752 uint32_t set;
753
754 __asm __volatile("ptesync": : :"memory");
755
756 /*
757 * Flush the first set of the TLB, and the entire Page Walk Cache
758 * and partition table entries. Then flush the remaining sets of the
759 * TLB.
760 */
761 if (is == TLB_INVAL_SCOPE_GLOBAL) {
762 tlbiel_radix_set_isa300(0, is, 0, RIC_FLUSH_ALL, 0);
763 for (set = 1; set < num_sets; set++)
764 tlbiel_radix_set_isa300(set, is, 0, RIC_FLUSH_TLB, 0);
765 }
766
767 /* Do the same for process scoped entries. */
768 tlbiel_radix_set_isa300(0, is, 0, RIC_FLUSH_ALL, 1);
769 for (set = 1; set < num_sets; set++)
770 tlbiel_radix_set_isa300(set, is, 0, RIC_FLUSH_TLB, 1);
771
772 __asm __volatile("ptesync": : :"memory");
773 }
774
775 static void
mmu_radix_tlbiel_flush(int scope)776 mmu_radix_tlbiel_flush(int scope)
777 {
778 MPASS(scope == TLB_INVAL_SCOPE_LPID ||
779 scope == TLB_INVAL_SCOPE_GLOBAL);
780
781 tlbiel_flush_isa3(POWER9_TLB_SETS_RADIX, scope);
782 __asm __volatile(PPC_INVALIDATE_ERAT "; isync" : : :"memory");
783 }
784
785 static void
mmu_radix_tlbie_all(void)786 mmu_radix_tlbie_all(void)
787 {
788 if (powernv_enabled)
789 mmu_radix_tlbiel_flush(TLB_INVAL_SCOPE_GLOBAL);
790 else
791 mmu_radix_tlbiel_flush(TLB_INVAL_SCOPE_LPID);
792 }
793
794 static void
mmu_radix_init_amor(void)795 mmu_radix_init_amor(void)
796 {
797 /*
798 * In HV mode, we init AMOR (Authority Mask Override Register) so that
799 * the hypervisor and guest can setup IAMR (Instruction Authority Mask
800 * Register), enable key 0 and set it to 1.
801 *
802 * AMOR = 0b1100 .... 0000 (Mask for key 0 is 11)
803 */
804 mtspr(SPR_AMOR, (3ul << 62));
805 }
806
807 static void
mmu_radix_init_iamr(void)808 mmu_radix_init_iamr(void)
809 {
810 /*
811 * Radix always uses key0 of the IAMR to determine if an access is
812 * allowed. We set bit 0 (IBM bit 1) of key0, to prevent instruction
813 * fetch.
814 */
815 mtspr(SPR_IAMR, (1ul << 62));
816 }
817
818 static void
mmu_radix_pid_set(pmap_t pmap)819 mmu_radix_pid_set(pmap_t pmap)
820 {
821
822 mtspr(SPR_PID, pmap->pm_pid);
823 isync();
824 }
825
826 /* Quick sort callout for comparing physical addresses. */
827 static int
pa_cmp(const void * a,const void * b)828 pa_cmp(const void *a, const void *b)
829 {
830 const vm_paddr_t *pa = a, *pb = b;
831
832 if (*pa < *pb)
833 return (-1);
834 else if (*pa > *pb)
835 return (1);
836 else
837 return (0);
838 }
839
840 #define pte_load_store(ptep, pte) atomic_swap_long(ptep, pte)
841 #define pte_load_clear(ptep) atomic_swap_long(ptep, 0)
842 #define pte_store(ptep, pte) do { \
843 MPASS((pte) & (RPTE_EAA_R | RPTE_EAA_W | RPTE_EAA_X)); \
844 *(u_long *)(ptep) = htobe64((u_long)((pte) | PG_V | RPTE_LEAF)); \
845 } while (0)
846 /*
847 * NB: should only be used for adding directories - not for direct mappings
848 */
849 #define pde_store(ptep, pa) do { \
850 *(u_long *)(ptep) = htobe64((u_long)(pa|RPTE_VALID|RPTE_SHIFT)); \
851 } while (0)
852
853 #define pte_clear(ptep) do { \
854 *(u_long *)(ptep) = (u_long)(0); \
855 } while (0)
856
857 #define PMAP_PDE_SUPERPAGE (1 << 8) /* supports 2MB superpages */
858
859 /*
860 * Promotion to a 2MB (PDE) page mapping requires that the corresponding 4KB
861 * (PTE) page mappings have identical settings for the following fields:
862 */
863 #define PG_PTE_PROMOTE (PG_X | PG_MANAGED | PG_W | PG_PTE_CACHE | \
864 PG_M | PG_A | RPTE_EAA_MASK | PG_V)
865
866 static __inline void
pmap_resident_count_inc(pmap_t pmap,int count)867 pmap_resident_count_inc(pmap_t pmap, int count)
868 {
869
870 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
871 pmap->pm_stats.resident_count += count;
872 }
873
874 static __inline void
pmap_resident_count_dec(pmap_t pmap,int count)875 pmap_resident_count_dec(pmap_t pmap, int count)
876 {
877
878 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
879 KASSERT(pmap->pm_stats.resident_count >= count,
880 ("pmap %p resident count underflow %ld %d", pmap,
881 pmap->pm_stats.resident_count, count));
882 pmap->pm_stats.resident_count -= count;
883 }
884
885 /*
886 * dcbz establishes a zeroed cache line without fetching it from memory,
887 * but raises an alignment interrupt on caching-inhibited mappings, so it
888 * may only be used on write-back memory. The internal callers only zero
889 * freshly allocated page table pages, which are always write-back; the
890 * vm_page facing entry point guards on the page's memattr.
891 */
892 static void
pagezero(void * va)893 pagezero(void *va)
894 {
895 vm_offset_t off;
896
897 va = trunc_page(va);
898 for (off = 0; off < PAGE_SIZE; off += cacheline_size)
899 __asm __volatile("dcbz 0,%0" ::
900 "r"((char *)va + off) : "memory");
901 }
902
903 static uint64_t
allocpages(int n)904 allocpages(int n)
905 {
906 vm_paddr_t ret;
907
908 ret = moea64_bootstrap_alloc(n * PAGE_SIZE, PAGE_SIZE);
909 for (int i = 0; i < n; i++)
910 pagezero(PHYS_TO_DMAP(ret + i * PAGE_SIZE));
911 return (ret);
912 }
913
914 static pt_entry_t *
kvtopte(vm_offset_t va)915 kvtopte(vm_offset_t va)
916 {
917 pt_entry_t *l3e;
918
919 l3e = pmap_pml3e(kernel_pmap, va);
920 if (l3e == NULL || (be64toh(*l3e) & RPTE_VALID) == 0)
921 return (NULL);
922 return (pmap_l3e_to_pte(l3e, va));
923 }
924
925 void
mmu_radix_kenter(vm_offset_t va,vm_paddr_t pa)926 mmu_radix_kenter(vm_offset_t va, vm_paddr_t pa)
927 {
928 pt_entry_t *pte;
929
930 pte = kvtopte(va);
931 MPASS(pte != NULL);
932 *pte = htobe64(pa | RPTE_VALID | RPTE_LEAF | RPTE_EAA_R | \
933 RPTE_EAA_W | RPTE_EAA_P | PG_M | PG_A);
934 }
935
936 bool
mmu_radix_ps_enabled(pmap_t pmap)937 mmu_radix_ps_enabled(pmap_t pmap)
938 {
939 return (superpages_enabled && (pmap->pm_flags & PMAP_PDE_SUPERPAGE) != 0);
940 }
941
942 static pt_entry_t *
pmap_nofault_pte(pmap_t pmap,vm_offset_t va,int * is_l3e)943 pmap_nofault_pte(pmap_t pmap, vm_offset_t va, int *is_l3e)
944 {
945 pml3_entry_t *l3e;
946 pt_entry_t *pte;
947
948 va &= PG_PS_FRAME;
949 l3e = pmap_pml3e(pmap, va);
950 if (l3e == NULL || (be64toh(*l3e) & PG_V) == 0)
951 return (NULL);
952
953 if (be64toh(*l3e) & RPTE_LEAF) {
954 *is_l3e = 1;
955 return (l3e);
956 }
957 *is_l3e = 0;
958 va &= PG_FRAME;
959 pte = pmap_l3e_to_pte(l3e, va);
960 if (pte == NULL || (be64toh(*pte) & PG_V) == 0)
961 return (NULL);
962 return (pte);
963 }
964
965 int
pmap_nofault(pmap_t pmap,vm_offset_t va,vm_prot_t flags)966 pmap_nofault(pmap_t pmap, vm_offset_t va, vm_prot_t flags)
967 {
968 pt_entry_t *pte;
969 pt_entry_t startpte, origpte, newpte;
970 vm_page_t m;
971 int is_l3e;
972
973 startpte = 0;
974 retry:
975 if ((pte = pmap_nofault_pte(pmap, va, &is_l3e)) == NULL)
976 return (KERN_INVALID_ADDRESS);
977 origpte = newpte = be64toh(*pte);
978 if (startpte == 0) {
979 startpte = origpte;
980 if (((flags & VM_PROT_WRITE) && (startpte & PG_M)) ||
981 ((flags & VM_PROT_READ) && (startpte & PG_A))) {
982 pmap_invalidate_all(pmap);
983 #ifdef INVARIANTS
984 if (VERBOSE_PMAP || pmap_logging)
985 printf("%s(%p, %#lx, %#x) (%#lx) -- invalidate all\n",
986 __func__, pmap, va, flags, origpte);
987 #endif
988 return (KERN_FAILURE);
989 }
990 }
991 #ifdef INVARIANTS
992 if (VERBOSE_PMAP || pmap_logging)
993 printf("%s(%p, %#lx, %#x) (%#lx)\n", __func__, pmap, va,
994 flags, origpte);
995 #endif
996 PMAP_LOCK(pmap);
997 if ((pte = pmap_nofault_pte(pmap, va, &is_l3e)) == NULL ||
998 be64toh(*pte) != origpte) {
999 PMAP_UNLOCK(pmap);
1000 return (KERN_FAILURE);
1001 }
1002 m = PHYS_TO_VM_PAGE(newpte & PG_FRAME);
1003 MPASS(m != NULL);
1004 switch (flags) {
1005 case VM_PROT_READ:
1006 if ((newpte & (RPTE_EAA_R|RPTE_EAA_X)) == 0)
1007 goto protfail;
1008 newpte |= PG_A;
1009 vm_page_aflag_set(m, PGA_REFERENCED);
1010 break;
1011 case VM_PROT_WRITE:
1012 if ((newpte & RPTE_EAA_W) == 0)
1013 goto protfail;
1014 if (is_l3e)
1015 goto protfail;
1016 newpte |= PG_M;
1017 vm_page_dirty(m);
1018 break;
1019 case VM_PROT_EXECUTE:
1020 if ((newpte & RPTE_EAA_X) == 0)
1021 goto protfail;
1022 newpte |= PG_A;
1023 vm_page_aflag_set(m, PGA_REFERENCED);
1024 break;
1025 }
1026
1027 if (!atomic_cmpset_long(pte, htobe64(origpte), htobe64(newpte)))
1028 goto retry;
1029 ptesync();
1030 PMAP_UNLOCK(pmap);
1031 if (startpte == newpte)
1032 return (KERN_FAILURE);
1033 return (0);
1034 protfail:
1035 PMAP_UNLOCK(pmap);
1036 return (KERN_PROTECTION_FAILURE);
1037 }
1038
1039 /*
1040 * Returns true if the given page is mapped individually or as part of
1041 * a 2mpage. Otherwise, returns false.
1042 */
1043 bool
mmu_radix_page_is_mapped(vm_page_t m)1044 mmu_radix_page_is_mapped(vm_page_t m)
1045 {
1046 struct rwlock *lock;
1047 bool rv;
1048
1049 if ((m->oflags & VPO_UNMANAGED) != 0)
1050 return (false);
1051 lock = VM_PAGE_TO_PV_LIST_LOCK(m);
1052 rw_rlock(lock);
1053 rv = !TAILQ_EMPTY(&m->md.pv_list) ||
1054 ((m->flags & PG_FICTITIOUS) == 0 &&
1055 !TAILQ_EMPTY(&pa_to_pvh(VM_PAGE_TO_PHYS(m))->pv_list));
1056 rw_runlock(lock);
1057 return (rv);
1058 }
1059
1060 /*
1061 * Determine the appropriate bits to set in a PTE or PDE for a specified
1062 * caching mode.
1063 */
1064 static int
pmap_cache_bits(vm_memattr_t ma)1065 pmap_cache_bits(vm_memattr_t ma)
1066 {
1067 if (ma != VM_MEMATTR_DEFAULT) {
1068 switch (ma) {
1069 case VM_MEMATTR_UNCACHEABLE:
1070 return (RPTE_ATTR_GUARDEDIO);
1071 case VM_MEMATTR_CACHEABLE:
1072 return (RPTE_ATTR_MEM);
1073 case VM_MEMATTR_WRITE_BACK:
1074 case VM_MEMATTR_PREFETCHABLE:
1075 case VM_MEMATTR_WRITE_COMBINING:
1076 return (RPTE_ATTR_UNGUARDEDIO);
1077 }
1078 }
1079 return (0);
1080 }
1081
1082 static void
pmap_invalidate_page(pmap_t pmap,vm_offset_t start)1083 pmap_invalidate_page(pmap_t pmap, vm_offset_t start)
1084 {
1085 ptesync();
1086 if (pmap == kernel_pmap)
1087 radix_tlbie_invlpg_kernel_4k(start);
1088 else
1089 radix_tlbie_invlpg_user_4k(pmap->pm_pid, start);
1090 ttusync();
1091 }
1092
1093 static void
pmap_invalidate_page_2m(pmap_t pmap,vm_offset_t start)1094 pmap_invalidate_page_2m(pmap_t pmap, vm_offset_t start)
1095 {
1096 ptesync();
1097 if (pmap == kernel_pmap)
1098 radix_tlbie_invlpg_kernel_2m(start);
1099 else
1100 radix_tlbie_invlpg_user_2m(pmap->pm_pid, start);
1101 ttusync();
1102 }
1103
1104 static void
pmap_invalidate_pwc(pmap_t pmap)1105 pmap_invalidate_pwc(pmap_t pmap)
1106 {
1107 ptesync();
1108 if (pmap == kernel_pmap)
1109 radix_tlbie_invlpwc_kernel();
1110 else
1111 radix_tlbie_invlpwc_user(pmap->pm_pid);
1112 ttusync();
1113 }
1114
1115 static void
pmap_invalidate_range(pmap_t pmap,vm_offset_t start,vm_offset_t end)1116 pmap_invalidate_range(pmap_t pmap, vm_offset_t start, vm_offset_t end)
1117 {
1118 if (((start - end) >> PAGE_SHIFT) > 8) {
1119 pmap_invalidate_all(pmap);
1120 return;
1121 }
1122 ptesync();
1123 if (pmap == kernel_pmap) {
1124 while (start < end) {
1125 radix_tlbie_invlpg_kernel_4k(start);
1126 start += PAGE_SIZE;
1127 }
1128 } else {
1129 while (start < end) {
1130 radix_tlbie_invlpg_user_4k(pmap->pm_pid, start);
1131 start += PAGE_SIZE;
1132 }
1133 }
1134 ttusync();
1135 }
1136
1137 static void
pmap_invalidate_all(pmap_t pmap)1138 pmap_invalidate_all(pmap_t pmap)
1139 {
1140 ptesync();
1141 if (pmap == kernel_pmap)
1142 radix_tlbie_flush_kernel();
1143 else
1144 radix_tlbie_flush_user(pmap->pm_pid);
1145 ttusync();
1146 }
1147
1148 static void
pmap_invalidate_l3e_page(pmap_t pmap,vm_offset_t va,pml3_entry_t l3e)1149 pmap_invalidate_l3e_page(pmap_t pmap, vm_offset_t va, pml3_entry_t l3e)
1150 {
1151
1152 /*
1153 * When the PDE has PG_PROMOTED set, the 2MB page mapping was created
1154 * by a promotion that did not invalidate the 512 4KB page mappings
1155 * that might exist in the TLB. Consequently, at this point, the TLB
1156 * may hold both 4KB and 2MB page mappings for the address range [va,
1157 * va + L3_PAGE_SIZE). Therefore, the entire range must be invalidated here.
1158 * In contrast, when PG_PROMOTED is clear, the TLB will not hold any
1159 * 4KB page mappings for the address range [va, va + L3_PAGE_SIZE), and so a
1160 * single INVLPG suffices to invalidate the 2MB page mapping from the
1161 * TLB.
1162 */
1163 ptesync();
1164 if ((l3e & PG_PROMOTED) != 0)
1165 pmap_invalidate_range(pmap, va, va + L3_PAGE_SIZE - 1);
1166 else
1167 pmap_invalidate_page_2m(pmap, va);
1168
1169 pmap_invalidate_pwc(pmap);
1170 }
1171
1172 static __inline struct pv_chunk *
pv_to_chunk(pv_entry_t pv)1173 pv_to_chunk(pv_entry_t pv)
1174 {
1175
1176 return ((struct pv_chunk *)((uintptr_t)pv & ~(uintptr_t)PAGE_MASK));
1177 }
1178
1179 #define PV_PMAP(pv) (pv_to_chunk(pv)->pc_pmap)
1180
1181 #define PC_FREE0 0xfffffffffffffffful
1182 #define PC_FREE1 ((1ul << (_NPCPV % 64)) - 1)
1183
1184 static const uint64_t pc_freemask[_NPCM] = { PC_FREE0, PC_FREE1 };
1185
1186 /*
1187 * Ensure that the number of spare PV entries in the specified pmap meets or
1188 * exceeds the given count, "needed".
1189 *
1190 * The given PV list lock may be released.
1191 */
1192 static void
reserve_pv_entries(pmap_t pmap,int needed,struct rwlock ** lockp)1193 reserve_pv_entries(pmap_t pmap, int needed, struct rwlock **lockp)
1194 {
1195 struct pch new_tail;
1196 struct pv_chunk *pc;
1197 vm_page_t m;
1198 int avail, free;
1199 bool reclaimed;
1200
1201 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
1202 KASSERT(lockp != NULL, ("reserve_pv_entries: lockp is NULL"));
1203
1204 /*
1205 * Newly allocated PV chunks must be stored in a private list until
1206 * the required number of PV chunks have been allocated. Otherwise,
1207 * reclaim_pv_chunk() could recycle one of these chunks. In
1208 * contrast, these chunks must be added to the pmap upon allocation.
1209 */
1210 TAILQ_INIT(&new_tail);
1211 retry:
1212 avail = 0;
1213 TAILQ_FOREACH(pc, &pmap->pm_pvchunk, pc_list) {
1214 // if ((cpu_feature2 & CPUID2_POPCNT) == 0)
1215 bit_count((bitstr_t *)pc->pc_map, 0,
1216 sizeof(pc->pc_map) * NBBY, &free);
1217 #if 0
1218 free = popcnt_pc_map_pq(pc->pc_map);
1219 #endif
1220 if (free == 0)
1221 break;
1222 avail += free;
1223 if (avail >= needed)
1224 break;
1225 }
1226 for (reclaimed = false; avail < needed; avail += _NPCPV) {
1227 m = vm_page_alloc_noobj(VM_ALLOC_WIRED);
1228 if (m == NULL) {
1229 m = reclaim_pv_chunk(pmap, lockp);
1230 if (m == NULL)
1231 goto retry;
1232 reclaimed = true;
1233 }
1234 PV_STAT(atomic_add_int(&pc_chunk_count, 1));
1235 PV_STAT(atomic_add_int(&pc_chunk_allocs, 1));
1236 dump_add_page(m->phys_addr);
1237 pc = VM_PAGE_TO_DMAP(m);
1238 pc->pc_pmap = pmap;
1239 pc->pc_map[0] = PC_FREE0;
1240 pc->pc_map[1] = PC_FREE1;
1241 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);
1242 TAILQ_INSERT_TAIL(&new_tail, pc, pc_lru);
1243 PV_STAT(atomic_add_int(&pv_entry_spare, _NPCPV));
1244
1245 /*
1246 * The reclaim might have freed a chunk from the current pmap.
1247 * If that chunk contained available entries, we need to
1248 * re-count the number of available entries.
1249 */
1250 if (reclaimed)
1251 goto retry;
1252 }
1253 if (!TAILQ_EMPTY(&new_tail)) {
1254 mtx_lock(&pv_chunks_mutex);
1255 TAILQ_CONCAT(&pv_chunks, &new_tail, pc_lru);
1256 mtx_unlock(&pv_chunks_mutex);
1257 }
1258 }
1259
1260 /*
1261 * First find and then remove the pv entry for the specified pmap and virtual
1262 * address from the specified pv list. Returns the pv entry if found and NULL
1263 * otherwise. This operation can be performed on pv lists for either 4KB or
1264 * 2MB page mappings.
1265 */
1266 static __inline pv_entry_t
pmap_pvh_remove(struct md_page * pvh,pmap_t pmap,vm_offset_t va)1267 pmap_pvh_remove(struct md_page *pvh, pmap_t pmap, vm_offset_t va)
1268 {
1269 pv_entry_t pv;
1270
1271 TAILQ_FOREACH(pv, &pvh->pv_list, pv_link) {
1272 #ifdef INVARIANTS
1273 if (PV_PMAP(pv) == NULL) {
1274 printf("corrupted pv_chunk/pv %p\n", pv);
1275 printf("pv_chunk: %64D\n", pv_to_chunk(pv), ":");
1276 }
1277 MPASS(PV_PMAP(pv) != NULL);
1278 MPASS(pv->pv_va != 0);
1279 #endif
1280 if (pmap == PV_PMAP(pv) && va == pv->pv_va) {
1281 TAILQ_REMOVE(&pvh->pv_list, pv, pv_link);
1282 pvh->pv_gen++;
1283 break;
1284 }
1285 }
1286 return (pv);
1287 }
1288
1289 /*
1290 * After demotion from a 2MB page mapping to 512 4KB page mappings,
1291 * destroy the pv entry for the 2MB page mapping and reinstantiate the pv
1292 * entries for each of the 4KB page mappings.
1293 */
1294 static void
pmap_pv_demote_l3e(pmap_t pmap,vm_offset_t va,vm_paddr_t pa,struct rwlock ** lockp)1295 pmap_pv_demote_l3e(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,
1296 struct rwlock **lockp)
1297 {
1298 struct md_page *pvh;
1299 struct pv_chunk *pc;
1300 pv_entry_t pv;
1301 vm_offset_t va_last;
1302 vm_page_t m;
1303 int bit, field;
1304
1305 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
1306 KASSERT((pa & L3_PAGE_MASK) == 0,
1307 ("pmap_pv_demote_pde: pa is not 2mpage aligned"));
1308 CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa);
1309
1310 /*
1311 * Transfer the 2mpage's pv entry for this mapping to the first
1312 * page's pv list. Once this transfer begins, the pv list lock
1313 * must not be released until the last pv entry is reinstantiated.
1314 */
1315 pvh = pa_to_pvh(pa);
1316 va = trunc_2mpage(va);
1317 pv = pmap_pvh_remove(pvh, pmap, va);
1318 KASSERT(pv != NULL, ("pmap_pv_demote_pde: pv not found"));
1319 m = PHYS_TO_VM_PAGE(pa);
1320 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_link);
1321
1322 m->md.pv_gen++;
1323 /* Instantiate the remaining NPTEPG - 1 pv entries. */
1324 PV_STAT(atomic_add_long(&pv_entry_allocs, NPTEPG - 1));
1325 va_last = va + L3_PAGE_SIZE - PAGE_SIZE;
1326 for (;;) {
1327 pc = TAILQ_FIRST(&pmap->pm_pvchunk);
1328 KASSERT(pc->pc_map[0] != 0 || pc->pc_map[1] != 0
1329 , ("pmap_pv_demote_pde: missing spare"));
1330 for (field = 0; field < _NPCM; field++) {
1331 while (pc->pc_map[field]) {
1332 bit = cnttzd(pc->pc_map[field]);
1333 pc->pc_map[field] &= ~(1ul << bit);
1334 pv = &pc->pc_pventry[field * 64 + bit];
1335 va += PAGE_SIZE;
1336 pv->pv_va = va;
1337 m++;
1338 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
1339 ("pmap_pv_demote_pde: page %p is not managed", m));
1340 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_link);
1341
1342 m->md.pv_gen++;
1343 if (va == va_last)
1344 goto out;
1345 }
1346 }
1347 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
1348 TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list);
1349 }
1350 out:
1351 if (pc->pc_map[0] == 0 && pc->pc_map[1] == 0) {
1352 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
1353 TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list);
1354 }
1355 PV_STAT(atomic_add_long(&pv_entry_count, NPTEPG - 1));
1356 PV_STAT(atomic_subtract_int(&pv_entry_spare, NPTEPG - 1));
1357 }
1358
1359 static void
reclaim_pv_chunk_leave_pmap(pmap_t pmap,pmap_t locked_pmap)1360 reclaim_pv_chunk_leave_pmap(pmap_t pmap, pmap_t locked_pmap)
1361 {
1362
1363 if (pmap == NULL)
1364 return;
1365 pmap_invalidate_all(pmap);
1366 if (pmap != locked_pmap)
1367 PMAP_UNLOCK(pmap);
1368 }
1369
1370 /*
1371 * We are in a serious low memory condition. Resort to
1372 * drastic measures to free some pages so we can allocate
1373 * another pv entry chunk.
1374 *
1375 * Returns NULL if PV entries were reclaimed from the specified pmap.
1376 *
1377 * We do not, however, unmap 2mpages because subsequent accesses will
1378 * allocate per-page pv entries until repromotion occurs, thereby
1379 * exacerbating the shortage of free pv entries.
1380 */
1381 static int active_reclaims = 0;
1382 static vm_page_t
reclaim_pv_chunk(pmap_t locked_pmap,struct rwlock ** lockp)1383 reclaim_pv_chunk(pmap_t locked_pmap, struct rwlock **lockp)
1384 {
1385 struct pv_chunk *pc, *pc_marker, *pc_marker_end;
1386 struct pv_chunk_header pc_marker_b, pc_marker_end_b;
1387 struct md_page *pvh;
1388 pml3_entry_t *l3e;
1389 pmap_t next_pmap, pmap;
1390 pt_entry_t *pte, tpte;
1391 pv_entry_t pv;
1392 vm_offset_t va;
1393 vm_page_t m, m_pc;
1394 struct spglist free;
1395 uint64_t inuse;
1396 int bit, field, freed;
1397
1398 PMAP_LOCK_ASSERT(locked_pmap, MA_OWNED);
1399 KASSERT(lockp != NULL, ("reclaim_pv_chunk: lockp is NULL"));
1400 pmap = NULL;
1401 m_pc = NULL;
1402 SLIST_INIT(&free);
1403 bzero(&pc_marker_b, sizeof(pc_marker_b));
1404 bzero(&pc_marker_end_b, sizeof(pc_marker_end_b));
1405 pc_marker = (struct pv_chunk *)&pc_marker_b;
1406 pc_marker_end = (struct pv_chunk *)&pc_marker_end_b;
1407
1408 mtx_lock(&pv_chunks_mutex);
1409 active_reclaims++;
1410 TAILQ_INSERT_HEAD(&pv_chunks, pc_marker, pc_lru);
1411 TAILQ_INSERT_TAIL(&pv_chunks, pc_marker_end, pc_lru);
1412 while ((pc = TAILQ_NEXT(pc_marker, pc_lru)) != pc_marker_end &&
1413 SLIST_EMPTY(&free)) {
1414 next_pmap = pc->pc_pmap;
1415 if (next_pmap == NULL) {
1416 /*
1417 * The next chunk is a marker. However, it is
1418 * not our marker, so active_reclaims must be
1419 * > 1. Consequently, the next_chunk code
1420 * will not rotate the pv_chunks list.
1421 */
1422 goto next_chunk;
1423 }
1424 mtx_unlock(&pv_chunks_mutex);
1425
1426 /*
1427 * A pv_chunk can only be removed from the pc_lru list
1428 * when both pc_chunks_mutex is owned and the
1429 * corresponding pmap is locked.
1430 */
1431 if (pmap != next_pmap) {
1432 reclaim_pv_chunk_leave_pmap(pmap, locked_pmap);
1433 pmap = next_pmap;
1434 /* Avoid deadlock and lock recursion. */
1435 if (pmap > locked_pmap) {
1436 RELEASE_PV_LIST_LOCK(lockp);
1437 PMAP_LOCK(pmap);
1438 mtx_lock(&pv_chunks_mutex);
1439 continue;
1440 } else if (pmap != locked_pmap) {
1441 if (PMAP_TRYLOCK(pmap)) {
1442 mtx_lock(&pv_chunks_mutex);
1443 continue;
1444 } else {
1445 pmap = NULL; /* pmap is not locked */
1446 mtx_lock(&pv_chunks_mutex);
1447 pc = TAILQ_NEXT(pc_marker, pc_lru);
1448 if (pc == NULL ||
1449 pc->pc_pmap != next_pmap)
1450 continue;
1451 goto next_chunk;
1452 }
1453 }
1454 }
1455
1456 /*
1457 * Destroy every non-wired, 4 KB page mapping in the chunk.
1458 */
1459 freed = 0;
1460 for (field = 0; field < _NPCM; field++) {
1461 for (inuse = ~pc->pc_map[field] & pc_freemask[field];
1462 inuse != 0; inuse &= ~(1UL << bit)) {
1463 bit = cnttzd(inuse);
1464 pv = &pc->pc_pventry[field * 64 + bit];
1465 va = pv->pv_va;
1466 l3e = pmap_pml3e(pmap, va);
1467 if ((be64toh(*l3e) & RPTE_LEAF) != 0)
1468 continue;
1469 pte = pmap_l3e_to_pte(l3e, va);
1470 if ((be64toh(*pte) & PG_W) != 0)
1471 continue;
1472 tpte = be64toh(pte_load_clear(pte));
1473 m = PHYS_TO_VM_PAGE(tpte & PG_FRAME);
1474 if ((tpte & (PG_M | PG_RW)) == (PG_M | PG_RW))
1475 vm_page_dirty(m);
1476 if ((tpte & PG_A) != 0)
1477 vm_page_aflag_set(m, PGA_REFERENCED);
1478 CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m);
1479 TAILQ_REMOVE(&m->md.pv_list, pv, pv_link);
1480
1481 m->md.pv_gen++;
1482 if (TAILQ_EMPTY(&m->md.pv_list) &&
1483 (m->flags & PG_FICTITIOUS) == 0) {
1484 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
1485 if (TAILQ_EMPTY(&pvh->pv_list)) {
1486 vm_page_aflag_clear(m,
1487 PGA_WRITEABLE);
1488 }
1489 }
1490 pc->pc_map[field] |= 1UL << bit;
1491 pmap_unuse_pt(pmap, va, be64toh(*l3e), &free);
1492 freed++;
1493 }
1494 }
1495 if (freed == 0) {
1496 mtx_lock(&pv_chunks_mutex);
1497 goto next_chunk;
1498 }
1499 /* Every freed mapping is for a 4 KB page. */
1500 pmap_resident_count_dec(pmap, freed);
1501 PV_STAT(atomic_add_long(&pv_entry_frees, freed));
1502 PV_STAT(atomic_add_int(&pv_entry_spare, freed));
1503 PV_STAT(atomic_subtract_long(&pv_entry_count, freed));
1504 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
1505 if (pc->pc_map[0] == PC_FREE0 && pc->pc_map[1] == PC_FREE1) {
1506 PV_STAT(atomic_subtract_int(&pv_entry_spare, _NPCPV));
1507 PV_STAT(atomic_subtract_int(&pc_chunk_count, 1));
1508 PV_STAT(atomic_add_int(&pc_chunk_frees, 1));
1509 /* Entire chunk is free; return it. */
1510 m_pc = DMAP_TO_VM_PAGE(pc);
1511 dump_drop_page(m_pc->phys_addr);
1512 mtx_lock(&pv_chunks_mutex);
1513 TAILQ_REMOVE(&pv_chunks, pc, pc_lru);
1514 break;
1515 }
1516 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);
1517 mtx_lock(&pv_chunks_mutex);
1518 /* One freed pv entry in locked_pmap is sufficient. */
1519 if (pmap == locked_pmap)
1520 break;
1521 next_chunk:
1522 TAILQ_REMOVE(&pv_chunks, pc_marker, pc_lru);
1523 TAILQ_INSERT_AFTER(&pv_chunks, pc, pc_marker, pc_lru);
1524 if (active_reclaims == 1 && pmap != NULL) {
1525 /*
1526 * Rotate the pv chunks list so that we do not
1527 * scan the same pv chunks that could not be
1528 * freed (because they contained a wired
1529 * and/or superpage mapping) on every
1530 * invocation of reclaim_pv_chunk().
1531 */
1532 while ((pc = TAILQ_FIRST(&pv_chunks)) != pc_marker) {
1533 MPASS(pc->pc_pmap != NULL);
1534 TAILQ_REMOVE(&pv_chunks, pc, pc_lru);
1535 TAILQ_INSERT_TAIL(&pv_chunks, pc, pc_lru);
1536 }
1537 }
1538 }
1539 TAILQ_REMOVE(&pv_chunks, pc_marker, pc_lru);
1540 TAILQ_REMOVE(&pv_chunks, pc_marker_end, pc_lru);
1541 active_reclaims--;
1542 mtx_unlock(&pv_chunks_mutex);
1543 reclaim_pv_chunk_leave_pmap(pmap, locked_pmap);
1544 if (m_pc == NULL && !SLIST_EMPTY(&free)) {
1545 m_pc = SLIST_FIRST(&free);
1546 SLIST_REMOVE_HEAD(&free, plinks.s.ss);
1547 /* Recycle a freed page table page. */
1548 m_pc->ref_count = 1;
1549 }
1550 vm_page_free_pages_toq(&free, true);
1551 return (m_pc);
1552 }
1553
1554 /*
1555 * free the pv_entry back to the free list
1556 */
1557 static void
free_pv_entry(pmap_t pmap,pv_entry_t pv)1558 free_pv_entry(pmap_t pmap, pv_entry_t pv)
1559 {
1560 struct pv_chunk *pc;
1561 int idx, field, bit;
1562
1563 #ifdef VERBOSE_PV
1564 if (pmap != kernel_pmap)
1565 printf("%s(%p, %p)\n", __func__, pmap, pv);
1566 #endif
1567 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
1568 PV_STAT(atomic_add_long(&pv_entry_frees, 1));
1569 PV_STAT(atomic_add_int(&pv_entry_spare, 1));
1570 PV_STAT(atomic_subtract_long(&pv_entry_count, 1));
1571 pc = pv_to_chunk(pv);
1572 idx = pv - &pc->pc_pventry[0];
1573 field = idx / 64;
1574 bit = idx % 64;
1575 pc->pc_map[field] |= 1ul << bit;
1576 if (pc->pc_map[0] != PC_FREE0 || pc->pc_map[1] != PC_FREE1) {
1577 /* 98% of the time, pc is already at the head of the list. */
1578 if (__predict_false(pc != TAILQ_FIRST(&pmap->pm_pvchunk))) {
1579 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
1580 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);
1581 }
1582 return;
1583 }
1584 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
1585 free_pv_chunk(pc);
1586 }
1587
1588 static void
free_pv_chunk(struct pv_chunk * pc)1589 free_pv_chunk(struct pv_chunk *pc)
1590 {
1591 vm_page_t m;
1592
1593 mtx_lock(&pv_chunks_mutex);
1594 TAILQ_REMOVE(&pv_chunks, pc, pc_lru);
1595 mtx_unlock(&pv_chunks_mutex);
1596 PV_STAT(atomic_subtract_int(&pv_entry_spare, _NPCPV));
1597 PV_STAT(atomic_subtract_int(&pc_chunk_count, 1));
1598 PV_STAT(atomic_add_int(&pc_chunk_frees, 1));
1599 /* entire chunk is free, return it */
1600 m = DMAP_TO_VM_PAGE(pc);
1601 dump_drop_page(m->phys_addr);
1602 vm_page_unwire_noq(m);
1603 vm_page_free(m);
1604 }
1605
1606 /*
1607 * Returns a new PV entry, allocating a new PV chunk from the system when
1608 * needed. If this PV chunk allocation fails and a PV list lock pointer was
1609 * given, a PV chunk is reclaimed from an arbitrary pmap. Otherwise, NULL is
1610 * returned.
1611 *
1612 * The given PV list lock may be released.
1613 */
1614 static pv_entry_t
get_pv_entry(pmap_t pmap,struct rwlock ** lockp)1615 get_pv_entry(pmap_t pmap, struct rwlock **lockp)
1616 {
1617 int bit, field;
1618 pv_entry_t pv;
1619 struct pv_chunk *pc;
1620 vm_page_t m;
1621
1622 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
1623 PV_STAT(atomic_add_long(&pv_entry_allocs, 1));
1624 retry:
1625 pc = TAILQ_FIRST(&pmap->pm_pvchunk);
1626 if (pc != NULL) {
1627 for (field = 0; field < _NPCM; field++) {
1628 if (pc->pc_map[field]) {
1629 bit = cnttzd(pc->pc_map[field]);
1630 break;
1631 }
1632 }
1633 if (field < _NPCM) {
1634 pv = &pc->pc_pventry[field * 64 + bit];
1635 pc->pc_map[field] &= ~(1ul << bit);
1636 /* If this was the last item, move it to tail */
1637 if (pc->pc_map[0] == 0 && pc->pc_map[1] == 0) {
1638 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
1639 TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc,
1640 pc_list);
1641 }
1642 PV_STAT(atomic_add_long(&pv_entry_count, 1));
1643 PV_STAT(atomic_subtract_int(&pv_entry_spare, 1));
1644 MPASS(PV_PMAP(pv) != NULL);
1645 return (pv);
1646 }
1647 }
1648 /* No free items, allocate another chunk */
1649 m = vm_page_alloc_noobj(VM_ALLOC_WIRED);
1650 if (m == NULL) {
1651 if (lockp == NULL) {
1652 PV_STAT(pc_chunk_tryfail++);
1653 return (NULL);
1654 }
1655 m = reclaim_pv_chunk(pmap, lockp);
1656 if (m == NULL)
1657 goto retry;
1658 }
1659 PV_STAT(atomic_add_int(&pc_chunk_count, 1));
1660 PV_STAT(atomic_add_int(&pc_chunk_allocs, 1));
1661 dump_add_page(m->phys_addr);
1662 pc = VM_PAGE_TO_DMAP(m);
1663 pc->pc_pmap = pmap;
1664 pc->pc_map[0] = PC_FREE0 & ~1ul; /* preallocated bit 0 */
1665 pc->pc_map[1] = PC_FREE1;
1666 mtx_lock(&pv_chunks_mutex);
1667 TAILQ_INSERT_TAIL(&pv_chunks, pc, pc_lru);
1668 mtx_unlock(&pv_chunks_mutex);
1669 pv = &pc->pc_pventry[0];
1670 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);
1671 PV_STAT(atomic_add_long(&pv_entry_count, 1));
1672 PV_STAT(atomic_add_int(&pv_entry_spare, _NPCPV - 1));
1673 MPASS(PV_PMAP(pv) != NULL);
1674 return (pv);
1675 }
1676
1677 #if VM_NRESERVLEVEL > 0
1678 /*
1679 * After promotion from 512 4KB page mappings to a single 2MB page mapping,
1680 * replace the many pv entries for the 4KB page mappings by a single pv entry
1681 * for the 2MB page mapping.
1682 */
1683 static void
pmap_pv_promote_l3e(pmap_t pmap,vm_offset_t va,vm_paddr_t pa,struct rwlock ** lockp)1684 pmap_pv_promote_l3e(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,
1685 struct rwlock **lockp)
1686 {
1687 struct md_page *pvh;
1688 pv_entry_t pv;
1689 vm_offset_t va_last;
1690 vm_page_t m;
1691
1692 KASSERT((pa & L3_PAGE_MASK) == 0,
1693 ("pmap_pv_promote_pde: pa is not 2mpage aligned"));
1694 CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa);
1695
1696 /*
1697 * Transfer the first page's pv entry for this mapping to the 2mpage's
1698 * pv list. Aside from avoiding the cost of a call to get_pv_entry(),
1699 * a transfer avoids the possibility that get_pv_entry() calls
1700 * reclaim_pv_chunk() and that reclaim_pv_chunk() removes one of the
1701 * mappings that is being promoted.
1702 */
1703 m = PHYS_TO_VM_PAGE(pa);
1704 va = trunc_2mpage(va);
1705 pv = pmap_pvh_remove(&m->md, pmap, va);
1706 KASSERT(pv != NULL, ("pmap_pv_promote_pde: pv not found"));
1707 pvh = pa_to_pvh(pa);
1708 TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_link);
1709 pvh->pv_gen++;
1710 /* Free the remaining NPTEPG - 1 pv entries. */
1711 va_last = va + L3_PAGE_SIZE - PAGE_SIZE;
1712 do {
1713 m++;
1714 va += PAGE_SIZE;
1715 pmap_pvh_free(&m->md, pmap, va);
1716 } while (va < va_last);
1717 }
1718 #endif /* VM_NRESERVLEVEL > 0 */
1719
1720 /*
1721 * First find and then destroy the pv entry for the specified pmap and virtual
1722 * address. This operation can be performed on pv lists for either 4KB or 2MB
1723 * page mappings.
1724 */
1725 static void
pmap_pvh_free(struct md_page * pvh,pmap_t pmap,vm_offset_t va)1726 pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va)
1727 {
1728 pv_entry_t pv;
1729
1730 pv = pmap_pvh_remove(pvh, pmap, va);
1731 KASSERT(pv != NULL, ("pmap_pvh_free: pv not found"));
1732 free_pv_entry(pmap, pv);
1733 }
1734
1735 /*
1736 * Conditionally create the PV entry for a 4KB page mapping if the required
1737 * memory can be allocated without resorting to reclamation.
1738 */
1739 static bool
pmap_try_insert_pv_entry(pmap_t pmap,vm_offset_t va,vm_page_t m,struct rwlock ** lockp)1740 pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, vm_page_t m,
1741 struct rwlock **lockp)
1742 {
1743 pv_entry_t pv;
1744
1745 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
1746 /* Pass NULL instead of the lock pointer to disable reclamation. */
1747 if ((pv = get_pv_entry(pmap, NULL)) != NULL) {
1748 pv->pv_va = va;
1749 CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m);
1750 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_link);
1751 m->md.pv_gen++;
1752 return (true);
1753 } else
1754 return (false);
1755 }
1756
1757 vm_paddr_t phys_avail_debug[2 * VM_PHYSSEG_MAX];
1758 #ifdef INVARIANTS
1759 static void
validate_addr(vm_paddr_t addr,vm_size_t size)1760 validate_addr(vm_paddr_t addr, vm_size_t size)
1761 {
1762 vm_paddr_t end = addr + size;
1763 bool found = false;
1764
1765 for (int i = 0; i < 2 * phys_avail_count; i += 2) {
1766 if (addr >= phys_avail_debug[i] &&
1767 end <= phys_avail_debug[i + 1]) {
1768 found = true;
1769 break;
1770 }
1771 }
1772 KASSERT(found, ("%#lx-%#lx outside of initial phys_avail array",
1773 addr, end));
1774 }
1775 #else
validate_addr(vm_paddr_t addr,vm_size_t size)1776 static void validate_addr(vm_paddr_t addr, vm_size_t size) {}
1777 #endif
1778 #define DMAP_PAGE_BITS (RPTE_VALID | RPTE_LEAF | RPTE_EAA_MASK | PG_M | PG_A)
1779
1780 static vm_paddr_t
alloc_pt_page(void)1781 alloc_pt_page(void)
1782 {
1783 vm_paddr_t page;
1784
1785 page = allocpages(1);
1786 pagezero(PHYS_TO_DMAP(page));
1787 return (page);
1788 }
1789
1790 static void
mmu_radix_dmap_range(vm_paddr_t start,vm_paddr_t end)1791 mmu_radix_dmap_range(vm_paddr_t start, vm_paddr_t end)
1792 {
1793 pt_entry_t *pte, pteval;
1794 vm_paddr_t page;
1795
1796 if (bootverbose)
1797 printf("%s %lx -> %lx\n", __func__, start, end);
1798 while (start < end) {
1799 pteval = start | DMAP_PAGE_BITS;
1800 pte = pmap_pml1e(kernel_pmap, PHYS_TO_DMAP_ADDR(start));
1801 if ((be64toh(*pte) & RPTE_VALID) == 0) {
1802 page = alloc_pt_page();
1803 pde_store(pte, page);
1804 }
1805 pte = pmap_l1e_to_l2e(pte, PHYS_TO_DMAP_ADDR(start));
1806 if ((start & L2_PAGE_MASK) == 0 &&
1807 end - start >= L2_PAGE_SIZE) {
1808 start += L2_PAGE_SIZE;
1809 goto done;
1810 } else if ((be64toh(*pte) & RPTE_VALID) == 0) {
1811 page = alloc_pt_page();
1812 pde_store(pte, page);
1813 }
1814
1815 pte = pmap_l2e_to_l3e(pte, PHYS_TO_DMAP_ADDR(start));
1816 if ((start & L3_PAGE_MASK) == 0 &&
1817 end - start >= L3_PAGE_SIZE) {
1818 start += L3_PAGE_SIZE;
1819 goto done;
1820 } else if ((be64toh(*pte) & RPTE_VALID) == 0) {
1821 page = alloc_pt_page();
1822 pde_store(pte, page);
1823 }
1824 pte = pmap_l3e_to_pte(pte, PHYS_TO_DMAP_ADDR(start));
1825 start += PAGE_SIZE;
1826 done:
1827 pte_store(pte, pteval);
1828 }
1829 }
1830
1831 static void
mmu_radix_dmap_populate(vm_size_t hwphyssz)1832 mmu_radix_dmap_populate(vm_size_t hwphyssz)
1833 {
1834 vm_paddr_t start, end;
1835
1836 for (int i = 0; i < pregions_sz; i++) {
1837 start = pregions[i].mr_start;
1838 end = start + pregions[i].mr_size;
1839 if (hwphyssz && start >= hwphyssz)
1840 break;
1841 if (hwphyssz && hwphyssz < end)
1842 end = hwphyssz;
1843 mmu_radix_dmap_range(start, end);
1844 }
1845 }
1846
1847 static void
mmu_radix_setup_pagetables(vm_size_t hwphyssz)1848 mmu_radix_setup_pagetables(vm_size_t hwphyssz)
1849 {
1850 vm_paddr_t ptpages, pages;
1851 pt_entry_t *pte;
1852 vm_paddr_t l1phys;
1853
1854 bzero(kernel_pmap, sizeof(struct pmap));
1855 mtx_init(&kernel_pmap->pm_mtx, "kernel pmap", NULL, MTX_DEF);
1856 vm_radix_init(&kernel_pmap->pm_radix);
1857
1858 ptpages = allocpages(3);
1859 l1phys = moea64_bootstrap_alloc(RADIX_PGD_SIZE, RADIX_PGD_SIZE);
1860 validate_addr(l1phys, RADIX_PGD_SIZE);
1861 if (bootverbose)
1862 printf("l1phys=%lx\n", l1phys);
1863 MPASS((l1phys & (RADIX_PGD_SIZE-1)) == 0);
1864 for (int i = 0; i < RADIX_PGD_SIZE/PAGE_SIZE; i++)
1865 pagezero(PHYS_TO_DMAP(l1phys + i * PAGE_SIZE));
1866 kernel_pmap->pm_pml1 = PHYS_TO_DMAP(l1phys);
1867
1868 mmu_radix_dmap_populate(hwphyssz);
1869
1870 /*
1871 * Create page tables for first 128MB of KVA
1872 */
1873 pages = ptpages;
1874 pte = pmap_pml1e(kernel_pmap, VM_MIN_KERNEL_ADDRESS);
1875 *pte = htobe64(pages | RPTE_VALID | RPTE_SHIFT);
1876 pages += PAGE_SIZE;
1877 pte = pmap_l1e_to_l2e(pte, VM_MIN_KERNEL_ADDRESS);
1878 *pte = htobe64(pages | RPTE_VALID | RPTE_SHIFT);
1879 pages += PAGE_SIZE;
1880 pte = pmap_l2e_to_l3e(pte, VM_MIN_KERNEL_ADDRESS);
1881 /*
1882 * the kernel page table pages need to be preserved in
1883 * phys_avail and not overlap with previous allocations
1884 */
1885 pages = allocpages(nkpt);
1886 if (bootverbose) {
1887 printf("phys_avail after dmap populate and nkpt allocation\n");
1888 for (int j = 0; j < 2 * phys_avail_count; j+=2)
1889 printf("phys_avail[%d]=%08lx - phys_avail[%d]=%08lx\n",
1890 j, phys_avail[j], j + 1, phys_avail[j + 1]);
1891 }
1892 KPTphys = pages;
1893 for (int i = 0; i < nkpt; i++, pte++, pages += PAGE_SIZE)
1894 *pte = htobe64(pages | RPTE_VALID | RPTE_SHIFT);
1895 kernel_vm_end = VM_MIN_KERNEL_ADDRESS + nkpt * L3_PAGE_SIZE;
1896 if (bootverbose)
1897 printf("kernel_pmap pml1 %p\n", kernel_pmap->pm_pml1);
1898 /*
1899 * Add a physical memory segment (vm_phys_seg) corresponding to the
1900 * preallocated kernel page table pages so that vm_page structures
1901 * representing these pages will be created. The vm_page structures
1902 * are required for promotion of the corresponding kernel virtual
1903 * addresses to superpage mappings.
1904 */
1905 vm_phys_add_seg(KPTphys, KPTphys + ptoa(nkpt));
1906 }
1907
1908 static void
mmu_radix_early_bootstrap(vm_offset_t start,vm_offset_t end)1909 mmu_radix_early_bootstrap(vm_offset_t start, vm_offset_t end)
1910 {
1911 vm_paddr_t kpstart, kpend;
1912 vm_size_t physsz, hwphyssz;
1913 //uint64_t l2virt;
1914 int rm_pavail, proctab_size;
1915 int i, j;
1916
1917 kpstart = start & ~DMAP_BASE_ADDRESS;
1918 kpend = end & ~DMAP_BASE_ADDRESS;
1919
1920 /* Get physical memory regions from firmware */
1921 mem_regions(&pregions, &pregions_sz, ®ions, ®ions_sz);
1922 CTR0(KTR_PMAP, "mmu_radix_early_bootstrap: physical memory");
1923
1924 if (2 * VM_PHYSSEG_MAX < regions_sz)
1925 panic("mmu_radix_early_bootstrap: phys_avail too small");
1926
1927 if (bootverbose)
1928 for (int i = 0; i < regions_sz; i++)
1929 printf("regions[%d].mr_start=%lx regions[%d].mr_size=%lx\n",
1930 i, regions[i].mr_start, i, regions[i].mr_size);
1931 /*
1932 * XXX workaround a simulator bug
1933 */
1934 for (int i = 0; i < regions_sz; i++)
1935 if (regions[i].mr_start & PAGE_MASK) {
1936 regions[i].mr_start += PAGE_MASK;
1937 regions[i].mr_start &= ~PAGE_MASK;
1938 regions[i].mr_size &= ~PAGE_MASK;
1939 }
1940 if (bootverbose)
1941 for (int i = 0; i < pregions_sz; i++)
1942 printf("pregions[%d].mr_start=%lx pregions[%d].mr_size=%lx\n",
1943 i, pregions[i].mr_start, i, pregions[i].mr_size);
1944
1945 phys_avail_count = 0;
1946 physsz = 0;
1947 hwphyssz = 0;
1948 TUNABLE_ULONG_FETCH("hw.physmem", (u_long *) &hwphyssz);
1949 for (i = 0, j = 0; i < regions_sz; i++) {
1950 if (bootverbose)
1951 printf("regions[%d].mr_start=%016lx regions[%d].mr_size=%016lx\n",
1952 i, regions[i].mr_start, i, regions[i].mr_size);
1953
1954 if (regions[i].mr_size < PAGE_SIZE)
1955 continue;
1956
1957 if (hwphyssz != 0 &&
1958 (physsz + regions[i].mr_size) >= hwphyssz) {
1959 if (physsz < hwphyssz) {
1960 phys_avail[j] = regions[i].mr_start;
1961 phys_avail[j + 1] = regions[i].mr_start +
1962 (hwphyssz - physsz);
1963 physsz = hwphyssz;
1964 phys_avail_count++;
1965 dump_avail[j] = phys_avail[j];
1966 dump_avail[j + 1] = phys_avail[j + 1];
1967 }
1968 break;
1969 }
1970 phys_avail[j] = regions[i].mr_start;
1971 phys_avail[j + 1] = regions[i].mr_start + regions[i].mr_size;
1972 dump_avail[j] = phys_avail[j];
1973 dump_avail[j + 1] = phys_avail[j + 1];
1974
1975 phys_avail_count++;
1976 physsz += regions[i].mr_size;
1977 j += 2;
1978 }
1979
1980 /* Check for overlap with the kernel and exception vectors */
1981 rm_pavail = 0;
1982 for (j = 0; j < 2 * phys_avail_count; j+=2) {
1983 if (phys_avail[j] < EXC_LAST)
1984 phys_avail[j] += EXC_LAST;
1985
1986 if (phys_avail[j] >= kpstart &&
1987 phys_avail[j + 1] <= kpend) {
1988 phys_avail[j] = phys_avail[j + 1] = ~0;
1989 rm_pavail++;
1990 continue;
1991 }
1992
1993 if (kpstart >= phys_avail[j] &&
1994 kpstart < phys_avail[j + 1]) {
1995 if (kpend < phys_avail[j + 1]) {
1996 phys_avail[2 * phys_avail_count] =
1997 (kpend & ~PAGE_MASK) + PAGE_SIZE;
1998 phys_avail[2 * phys_avail_count + 1] =
1999 phys_avail[j + 1];
2000 phys_avail_count++;
2001 }
2002
2003 phys_avail[j + 1] = kpstart & ~PAGE_MASK;
2004 }
2005
2006 if (kpend >= phys_avail[j] &&
2007 kpend < phys_avail[j + 1]) {
2008 if (kpstart > phys_avail[j]) {
2009 phys_avail[2 * phys_avail_count] = phys_avail[j];
2010 phys_avail[2 * phys_avail_count + 1] =
2011 kpstart & ~PAGE_MASK;
2012 phys_avail_count++;
2013 }
2014
2015 phys_avail[j] = (kpend & ~PAGE_MASK) +
2016 PAGE_SIZE;
2017 }
2018 }
2019 qsort(phys_avail, 2 * phys_avail_count, sizeof(phys_avail[0]), pa_cmp);
2020 for (i = 0; i < 2 * phys_avail_count; i++)
2021 phys_avail_debug[i] = phys_avail[i];
2022
2023 /* Remove physical available regions marked for removal (~0) */
2024 if (rm_pavail) {
2025 phys_avail_count -= rm_pavail;
2026 for (i = 2 * phys_avail_count;
2027 i < 2*(phys_avail_count + rm_pavail); i+=2)
2028 phys_avail[i] = phys_avail[i + 1] = 0;
2029 }
2030 if (bootverbose) {
2031 printf("phys_avail ranges after filtering:\n");
2032 for (j = 0; j < 2 * phys_avail_count; j+=2)
2033 printf("phys_avail[%d]=%08lx - phys_avail[%d]=%08lx\n",
2034 j, phys_avail[j], j + 1, phys_avail[j + 1]);
2035 }
2036 physmem = btoc(physsz);
2037
2038 /* XXX assume we're running non-virtualized and
2039 * we don't support BHYVE
2040 */
2041 if (isa3_pid_bits == 0)
2042 isa3_pid_bits = 20;
2043 if (powernv_enabled) {
2044 parttab_phys =
2045 moea64_bootstrap_alloc(PARTTAB_SIZE, PARTTAB_SIZE);
2046 validate_addr(parttab_phys, PARTTAB_SIZE);
2047 for (int i = 0; i < PARTTAB_SIZE/PAGE_SIZE; i++)
2048 pagezero(PHYS_TO_DMAP(parttab_phys + i * PAGE_SIZE));
2049
2050 }
2051 proctab_size = 1UL << PROCTAB_SIZE_SHIFT;
2052 proctab0pa = moea64_bootstrap_alloc(proctab_size, proctab_size);
2053 validate_addr(proctab0pa, proctab_size);
2054 for (int i = 0; i < proctab_size/PAGE_SIZE; i++)
2055 pagezero(PHYS_TO_DMAP(proctab0pa + i * PAGE_SIZE));
2056
2057 mmu_radix_setup_pagetables(hwphyssz);
2058 }
2059
2060 static void
mmu_radix_late_bootstrap(vm_offset_t start,vm_offset_t end)2061 mmu_radix_late_bootstrap(vm_offset_t start, vm_offset_t end)
2062 {
2063 int i;
2064 vm_paddr_t pa;
2065 void *dpcpu;
2066 vm_offset_t va;
2067
2068 /*
2069 * Set up the Open Firmware pmap and add its mappings if not in real
2070 * mode.
2071 */
2072 if (bootverbose)
2073 printf("%s enter\n", __func__);
2074
2075 /*
2076 * Calculate the last available physical address, and reserve the
2077 * vm_page_array (upper bound).
2078 */
2079 Maxmem = 0;
2080 for (i = 0; phys_avail[i + 1] != 0; i += 2)
2081 Maxmem = MAX(Maxmem, powerpc_btop(phys_avail[i + 1]));
2082
2083 /*
2084 * Remap any early IO mappings (console framebuffer, etc.)
2085 */
2086 bs_remap_earlyboot();
2087
2088 /*
2089 * Allocate a kernel stack with a guard page for thread0 and map it
2090 * into the kernel page map.
2091 */
2092 pa = allocpages(kstack_pages);
2093 va = virtual_avail + KSTACK_GUARD_PAGES * PAGE_SIZE;
2094 virtual_avail = va + kstack_pages * PAGE_SIZE;
2095 CTR2(KTR_PMAP, "moea64_bootstrap: kstack0 at %#x (%#x)", pa, va);
2096 thread0.td_kstack = (char *)va;
2097 for (i = 0; i < kstack_pages; i++) {
2098 mmu_radix_kenter(va, pa);
2099 pa += PAGE_SIZE;
2100 va += PAGE_SIZE;
2101 }
2102 thread0.td_kstack_pages = kstack_pages;
2103
2104 /*
2105 * Allocate virtual address space for the message buffer.
2106 */
2107 pa = msgbuf_phys = allocpages((msgbufsize + PAGE_MASK) >> PAGE_SHIFT);
2108 msgbufp = PHYS_TO_DMAP(pa);
2109
2110 /*
2111 * Allocate virtual address space for the dynamic percpu area.
2112 */
2113 pa = allocpages(DPCPU_SIZE >> PAGE_SHIFT);
2114 dpcpu = PHYS_TO_DMAP(pa);
2115 dpcpu_init(dpcpu, curcpu);
2116
2117 crashdumpmap = (caddr_t)virtual_avail;
2118 virtual_avail += MAXDUMPPGS * PAGE_SIZE;
2119
2120 /*
2121 * Reserve some special page table entries/VA space for temporary
2122 * mapping of pages.
2123 */
2124 }
2125
2126 static void
mmu_parttab_init(void)2127 mmu_parttab_init(void)
2128 {
2129 uint64_t ptcr;
2130
2131 isa3_parttab = PHYS_TO_DMAP(parttab_phys);
2132
2133 if (bootverbose)
2134 printf("%s parttab: %p\n", __func__, isa3_parttab);
2135 ptcr = parttab_phys | (PARTTAB_SIZE_SHIFT-12);
2136 if (bootverbose)
2137 printf("setting ptcr %lx\n", ptcr);
2138 mtspr(SPR_PTCR, ptcr);
2139 }
2140
2141 static void
mmu_parttab_update(uint64_t lpid,uint64_t pagetab,uint64_t proctab)2142 mmu_parttab_update(uint64_t lpid, uint64_t pagetab, uint64_t proctab)
2143 {
2144 uint64_t prev;
2145
2146 if (bootverbose)
2147 printf("%s isa3_parttab %p lpid %lx pagetab %lx proctab %lx\n", __func__, isa3_parttab,
2148 lpid, pagetab, proctab);
2149 prev = be64toh(isa3_parttab[lpid].pagetab);
2150 isa3_parttab[lpid].pagetab = htobe64(pagetab);
2151 isa3_parttab[lpid].proctab = htobe64(proctab);
2152
2153 if (prev & PARTTAB_HR) {
2154 __asm __volatile(PPC_TLBIE_5(%0,%1,2,0,1) : :
2155 "r" (TLBIEL_INVAL_SET_LPID), "r" (lpid));
2156 __asm __volatile(PPC_TLBIE_5(%0,%1,2,1,1) : :
2157 "r" (TLBIEL_INVAL_SET_LPID), "r" (lpid));
2158 } else {
2159 __asm __volatile(PPC_TLBIE_5(%0,%1,2,0,0) : :
2160 "r" (TLBIEL_INVAL_SET_LPID), "r" (lpid));
2161 }
2162 ttusync();
2163 }
2164
2165 static void
mmu_radix_parttab_init(void)2166 mmu_radix_parttab_init(void)
2167 {
2168 uint64_t pagetab;
2169
2170 mmu_parttab_init();
2171 pagetab = RTS_SIZE | DMAP_TO_PHYS(kernel_pmap->pm_pml1) | \
2172 RADIX_PGD_INDEX_SHIFT | PARTTAB_HR;
2173 mmu_parttab_update(0, pagetab, 0);
2174 }
2175
2176 static void
mmu_radix_proctab_register(vm_paddr_t proctabpa,uint64_t table_size)2177 mmu_radix_proctab_register(vm_paddr_t proctabpa, uint64_t table_size)
2178 {
2179 uint64_t pagetab, proctab;
2180
2181 pagetab = be64toh(isa3_parttab[0].pagetab);
2182 proctab = proctabpa | table_size | PARTTAB_GR;
2183 mmu_parttab_update(0, pagetab, proctab);
2184 }
2185
2186 static void
mmu_radix_proctab_init(void)2187 mmu_radix_proctab_init(void)
2188 {
2189
2190 isa3_base_pid = 1;
2191
2192 isa3_proctab = PHYS_TO_DMAP(proctab0pa);
2193 isa3_proctab->proctab0 =
2194 htobe64(RTS_SIZE | DMAP_TO_PHYS(kernel_pmap->pm_pml1) |
2195 RADIX_PGD_INDEX_SHIFT);
2196
2197 if (powernv_enabled) {
2198 mmu_radix_proctab_register(proctab0pa, PROCTAB_SIZE_SHIFT - 12);
2199 __asm __volatile("ptesync" : : : "memory");
2200 __asm __volatile(PPC_TLBIE_5(%0,%1,2,1,1) : :
2201 "r" (TLBIEL_INVAL_SET_LPID), "r" (0));
2202 __asm __volatile("eieio; tlbsync; ptesync" : : : "memory");
2203 #ifdef PSERIES
2204 } else {
2205 int64_t rc;
2206
2207 rc = phyp_hcall(H_REGISTER_PROC_TBL,
2208 PROC_TABLE_NEW | PROC_TABLE_RADIX | PROC_TABLE_GTSE,
2209 proctab0pa, 0, PROCTAB_SIZE_SHIFT - 12);
2210 if (rc != H_SUCCESS)
2211 panic("mmu_radix_proctab_init: "
2212 "failed to register process table: rc=%jd",
2213 (intmax_t)rc);
2214 #endif
2215 }
2216
2217 if (bootverbose)
2218 printf("process table %p and kernel radix PDE: %p\n",
2219 isa3_proctab, kernel_pmap->pm_pml1);
2220 mtmsr(mfmsr() | PSL_DR );
2221 mtmsr(mfmsr() & ~PSL_DR);
2222 kernel_pmap->pm_pid = isa3_base_pid;
2223 isa3_base_pid++;
2224 }
2225
2226 void
mmu_radix_advise(pmap_t pmap,vm_offset_t sva,vm_offset_t eva,int advice)2227 mmu_radix_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,
2228 int advice)
2229 {
2230 struct rwlock *lock;
2231 pml1_entry_t *l1e;
2232 pml2_entry_t *l2e;
2233 pml3_entry_t oldl3e, *l3e;
2234 pt_entry_t *pte;
2235 vm_offset_t va, va_next;
2236 vm_page_t m;
2237 bool anychanged;
2238
2239 if (advice != MADV_DONTNEED && advice != MADV_FREE)
2240 return;
2241 anychanged = false;
2242 PMAP_LOCK(pmap);
2243 for (; sva < eva; sva = va_next) {
2244 l1e = pmap_pml1e(pmap, sva);
2245 if ((be64toh(*l1e) & PG_V) == 0) {
2246 va_next = (sva + L1_PAGE_SIZE) & ~L1_PAGE_MASK;
2247 if (va_next < sva)
2248 va_next = eva;
2249 continue;
2250 }
2251 l2e = pmap_l1e_to_l2e(l1e, sva);
2252 if ((be64toh(*l2e) & PG_V) == 0) {
2253 va_next = (sva + L2_PAGE_SIZE) & ~L2_PAGE_MASK;
2254 if (va_next < sva)
2255 va_next = eva;
2256 continue;
2257 }
2258 va_next = (sva + L3_PAGE_SIZE) & ~L3_PAGE_MASK;
2259 if (va_next < sva)
2260 va_next = eva;
2261 l3e = pmap_l2e_to_l3e(l2e, sva);
2262 oldl3e = be64toh(*l3e);
2263 if ((oldl3e & PG_V) == 0)
2264 continue;
2265 else if ((oldl3e & RPTE_LEAF) != 0) {
2266 if ((oldl3e & PG_MANAGED) == 0)
2267 continue;
2268 lock = NULL;
2269 if (!pmap_demote_l3e_locked(pmap, l3e, sva, &lock)) {
2270 if (lock != NULL)
2271 rw_wunlock(lock);
2272
2273 /*
2274 * The large page mapping was destroyed.
2275 */
2276 continue;
2277 }
2278
2279 /*
2280 * Unless the page mappings are wired, remove the
2281 * mapping to a single page so that a subsequent
2282 * access may repromote. Choosing the last page
2283 * within the address range [sva, min(va_next, eva))
2284 * generally results in more repromotions. Since the
2285 * underlying page table page is fully populated, this
2286 * removal never frees a page table page.
2287 */
2288 if ((oldl3e & PG_W) == 0) {
2289 va = eva;
2290 if (va > va_next)
2291 va = va_next;
2292 va -= PAGE_SIZE;
2293 KASSERT(va >= sva,
2294 ("mmu_radix_advise: no address gap"));
2295 pte = pmap_l3e_to_pte(l3e, va);
2296 KASSERT((be64toh(*pte) & PG_V) != 0,
2297 ("pmap_advise: invalid PTE"));
2298 pmap_remove_pte(pmap, pte, va, be64toh(*l3e), NULL,
2299 &lock);
2300 anychanged = true;
2301 }
2302 if (lock != NULL)
2303 rw_wunlock(lock);
2304 }
2305 if (va_next > eva)
2306 va_next = eva;
2307 va = va_next;
2308 for (pte = pmap_l3e_to_pte(l3e, sva); sva != va_next;
2309 pte++, sva += PAGE_SIZE) {
2310 MPASS(pte == pmap_pte(pmap, sva));
2311
2312 if ((be64toh(*pte) & (PG_MANAGED | PG_V)) != (PG_MANAGED | PG_V))
2313 goto maybe_invlrng;
2314 else if ((be64toh(*pte) & (PG_M | PG_RW)) == (PG_M | PG_RW)) {
2315 if (advice == MADV_DONTNEED) {
2316 /*
2317 * Future calls to pmap_is_modified()
2318 * can be avoided by making the page
2319 * dirty now.
2320 */
2321 m = PHYS_TO_VM_PAGE(be64toh(*pte) & PG_FRAME);
2322 vm_page_dirty(m);
2323 }
2324 atomic_clear_long(pte, htobe64(PG_M | PG_A));
2325 } else if ((be64toh(*pte) & PG_A) != 0)
2326 atomic_clear_long(pte, htobe64(PG_A));
2327 else
2328 goto maybe_invlrng;
2329 anychanged = true;
2330 continue;
2331 maybe_invlrng:
2332 if (va != va_next) {
2333 anychanged = true;
2334 va = va_next;
2335 }
2336 }
2337 if (va != va_next)
2338 anychanged = true;
2339 }
2340 if (anychanged)
2341 pmap_invalidate_all(pmap);
2342 PMAP_UNLOCK(pmap);
2343 }
2344
2345 /*
2346 * Routines used in machine-dependent code
2347 */
2348 static void
mmu_radix_bootstrap(vm_offset_t start,vm_offset_t end)2349 mmu_radix_bootstrap(vm_offset_t start, vm_offset_t end)
2350 {
2351 uint64_t lpcr;
2352
2353 if (bootverbose)
2354 printf("%s\n", __func__);
2355 hw_direct_map = 1;
2356 powernv_enabled = (mfmsr() & PSL_HV) ? 1 : 0;
2357 mmu_radix_early_bootstrap(start, end);
2358 if (bootverbose)
2359 printf("early bootstrap complete\n");
2360 if (powernv_enabled) {
2361 lpcr = mfspr(SPR_LPCR);
2362 mtspr(SPR_LPCR, lpcr | LPCR_UPRT | LPCR_HR);
2363 mmu_radix_parttab_init();
2364 mmu_radix_init_amor();
2365 if (bootverbose)
2366 printf("powernv init complete\n");
2367 }
2368 mmu_radix_init_iamr();
2369 mmu_radix_proctab_init();
2370 mmu_radix_pid_set(kernel_pmap);
2371 if (powernv_enabled)
2372 mmu_radix_tlbiel_flush(TLB_INVAL_SCOPE_GLOBAL);
2373 else
2374 mmu_radix_tlbiel_flush(TLB_INVAL_SCOPE_LPID);
2375
2376 mmu_radix_late_bootstrap(start, end);
2377 numa_mem_regions(&numa_pregions, &numa_pregions_sz);
2378 if (bootverbose)
2379 printf("%s done\n", __func__);
2380 pmap_bootstrapped = 1;
2381 dmaplimit = roundup2(powerpc_ptob(Maxmem), L2_PAGE_SIZE);
2382 PCPU_SET(flags, PCPU_GET(flags) | PC_FLAG_NOSRS);
2383 }
2384
2385 static void
mmu_radix_cpu_bootstrap(int ap)2386 mmu_radix_cpu_bootstrap(int ap)
2387 {
2388 uint64_t lpcr;
2389 uint64_t ptcr;
2390
2391 if (powernv_enabled) {
2392 lpcr = mfspr(SPR_LPCR);
2393 mtspr(SPR_LPCR, lpcr | LPCR_UPRT | LPCR_HR);
2394
2395 ptcr = parttab_phys | (PARTTAB_SIZE_SHIFT-12);
2396 mtspr(SPR_PTCR, ptcr);
2397 mmu_radix_init_amor();
2398 }
2399 mmu_radix_init_iamr();
2400 mmu_radix_pid_set(kernel_pmap);
2401 if (powernv_enabled)
2402 mmu_radix_tlbiel_flush(TLB_INVAL_SCOPE_GLOBAL);
2403 else
2404 mmu_radix_tlbiel_flush(TLB_INVAL_SCOPE_LPID);
2405 }
2406
2407 static SYSCTL_NODE(_vm_pmap, OID_AUTO, l3e, CTLFLAG_RD, 0,
2408 "2MB page mapping counters");
2409
2410 static COUNTER_U64_DEFINE_EARLY(pmap_l3e_demotions);
2411 SYSCTL_COUNTER_U64(_vm_pmap_l3e, OID_AUTO, demotions, CTLFLAG_RD,
2412 &pmap_l3e_demotions, "2MB page demotions");
2413
2414 static COUNTER_U64_DEFINE_EARLY(pmap_l3e_mappings);
2415 SYSCTL_COUNTER_U64(_vm_pmap_l3e, OID_AUTO, mappings, CTLFLAG_RD,
2416 &pmap_l3e_mappings, "2MB page mappings");
2417
2418 static COUNTER_U64_DEFINE_EARLY(pmap_l3e_p_failures);
2419 SYSCTL_COUNTER_U64(_vm_pmap_l3e, OID_AUTO, p_failures, CTLFLAG_RD,
2420 &pmap_l3e_p_failures, "2MB page promotion failures");
2421
2422 static COUNTER_U64_DEFINE_EARLY(pmap_l3e_promotions);
2423 SYSCTL_COUNTER_U64(_vm_pmap_l3e, OID_AUTO, promotions, CTLFLAG_RD,
2424 &pmap_l3e_promotions, "2MB page promotions");
2425
2426 static SYSCTL_NODE(_vm_pmap, OID_AUTO, l2e, CTLFLAG_RD, 0,
2427 "1GB page mapping counters");
2428
2429 static COUNTER_U64_DEFINE_EARLY(pmap_l2e_demotions);
2430 SYSCTL_COUNTER_U64(_vm_pmap_l2e, OID_AUTO, demotions, CTLFLAG_RD,
2431 &pmap_l2e_demotions, "1GB page demotions");
2432
2433 void
mmu_radix_clear_modify(vm_page_t m)2434 mmu_radix_clear_modify(vm_page_t m)
2435 {
2436 struct md_page *pvh;
2437 pmap_t pmap;
2438 pv_entry_t next_pv, pv;
2439 pml3_entry_t oldl3e, *l3e;
2440 pt_entry_t oldpte, *pte;
2441 struct rwlock *lock;
2442 vm_offset_t va;
2443 int md_gen, pvh_gen;
2444
2445 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
2446 ("pmap_clear_modify: page %p is not managed", m));
2447 vm_page_assert_busied(m);
2448 CTR2(KTR_PMAP, "%s(%p)", __func__, m);
2449
2450 /*
2451 * If the page is not PGA_WRITEABLE, then no PTEs can have PG_M set.
2452 * If the object containing the page is locked and the page is not
2453 * exclusive busied, then PGA_WRITEABLE cannot be concurrently set.
2454 */
2455 if ((m->a.flags & PGA_WRITEABLE) == 0)
2456 return;
2457 pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy :
2458 pa_to_pvh(VM_PAGE_TO_PHYS(m));
2459 lock = VM_PAGE_TO_PV_LIST_LOCK(m);
2460 rw_wlock(lock);
2461 restart:
2462 TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_link, next_pv) {
2463 pmap = PV_PMAP(pv);
2464 if (!PMAP_TRYLOCK(pmap)) {
2465 pvh_gen = pvh->pv_gen;
2466 rw_wunlock(lock);
2467 PMAP_LOCK(pmap);
2468 rw_wlock(lock);
2469 if (pvh_gen != pvh->pv_gen) {
2470 PMAP_UNLOCK(pmap);
2471 goto restart;
2472 }
2473 }
2474 va = pv->pv_va;
2475 l3e = pmap_pml3e(pmap, va);
2476 oldl3e = be64toh(*l3e);
2477 if ((oldl3e & PG_RW) != 0 &&
2478 pmap_demote_l3e_locked(pmap, l3e, va, &lock) &&
2479 (oldl3e & PG_W) == 0) {
2480 /*
2481 * Write protect the mapping to a
2482 * single page so that a subsequent
2483 * write access may repromote.
2484 */
2485 va += VM_PAGE_TO_PHYS(m) - (oldl3e &
2486 PG_PS_FRAME);
2487 pte = pmap_l3e_to_pte(l3e, va);
2488 oldpte = be64toh(*pte);
2489 while (!atomic_cmpset_long(pte,
2490 htobe64(oldpte),
2491 htobe64((oldpte | RPTE_EAA_R) & ~(PG_M | PG_RW))))
2492 oldpte = be64toh(*pte);
2493 vm_page_dirty(m);
2494 pmap_invalidate_page(pmap, va);
2495 }
2496 PMAP_UNLOCK(pmap);
2497 }
2498 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) {
2499 pmap = PV_PMAP(pv);
2500 if (!PMAP_TRYLOCK(pmap)) {
2501 md_gen = m->md.pv_gen;
2502 pvh_gen = pvh->pv_gen;
2503 rw_wunlock(lock);
2504 PMAP_LOCK(pmap);
2505 rw_wlock(lock);
2506 if (pvh_gen != pvh->pv_gen || md_gen != m->md.pv_gen) {
2507 PMAP_UNLOCK(pmap);
2508 goto restart;
2509 }
2510 }
2511 l3e = pmap_pml3e(pmap, pv->pv_va);
2512 KASSERT((be64toh(*l3e) & RPTE_LEAF) == 0, ("pmap_clear_modify: found"
2513 " a 2mpage in page %p's pv list", m));
2514 pte = pmap_l3e_to_pte(l3e, pv->pv_va);
2515 if ((be64toh(*pte) & (PG_M | PG_RW)) == (PG_M | PG_RW)) {
2516 atomic_clear_long(pte, htobe64(PG_M));
2517 pmap_invalidate_page(pmap, pv->pv_va);
2518 }
2519 PMAP_UNLOCK(pmap);
2520 }
2521 rw_wunlock(lock);
2522 }
2523
2524 void
mmu_radix_copy(pmap_t dst_pmap,pmap_t src_pmap,vm_offset_t dst_addr,vm_size_t len,vm_offset_t src_addr)2525 mmu_radix_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr,
2526 vm_size_t len, vm_offset_t src_addr)
2527 {
2528 struct rwlock *lock;
2529 struct spglist free;
2530 vm_offset_t addr;
2531 vm_offset_t end_addr = src_addr + len;
2532 vm_offset_t va_next;
2533 vm_page_t dst_pdpg, dstmpte, srcmpte;
2534 bool invalidate_all;
2535
2536 CTR6(KTR_PMAP,
2537 "%s(dst_pmap=%p, src_pmap=%p, dst_addr=%lx, len=%lu, src_addr=%lx)\n",
2538 __func__, dst_pmap, src_pmap, dst_addr, len, src_addr);
2539
2540 if (dst_addr != src_addr)
2541 return;
2542 lock = NULL;
2543 invalidate_all = false;
2544 if (dst_pmap < src_pmap) {
2545 PMAP_LOCK(dst_pmap);
2546 PMAP_LOCK(src_pmap);
2547 } else {
2548 PMAP_LOCK(src_pmap);
2549 PMAP_LOCK(dst_pmap);
2550 }
2551
2552 for (addr = src_addr; addr < end_addr; addr = va_next) {
2553 pml1_entry_t *l1e;
2554 pml2_entry_t *l2e;
2555 pml3_entry_t srcptepaddr, *l3e;
2556 pt_entry_t *src_pte, *dst_pte;
2557
2558 l1e = pmap_pml1e(src_pmap, addr);
2559 if ((be64toh(*l1e) & PG_V) == 0) {
2560 va_next = (addr + L1_PAGE_SIZE) & ~L1_PAGE_MASK;
2561 if (va_next < addr)
2562 va_next = end_addr;
2563 continue;
2564 }
2565
2566 l2e = pmap_l1e_to_l2e(l1e, addr);
2567 if ((be64toh(*l2e) & PG_V) == 0) {
2568 va_next = (addr + L2_PAGE_SIZE) & ~L2_PAGE_MASK;
2569 if (va_next < addr)
2570 va_next = end_addr;
2571 continue;
2572 }
2573
2574 va_next = (addr + L3_PAGE_SIZE) & ~L3_PAGE_MASK;
2575 if (va_next < addr)
2576 va_next = end_addr;
2577
2578 l3e = pmap_l2e_to_l3e(l2e, addr);
2579 srcptepaddr = be64toh(*l3e);
2580 if (srcptepaddr == 0)
2581 continue;
2582
2583 if (srcptepaddr & RPTE_LEAF) {
2584 if ((addr & L3_PAGE_MASK) != 0 ||
2585 addr + L3_PAGE_SIZE > end_addr)
2586 continue;
2587 dst_pdpg = pmap_allocl3e(dst_pmap, addr, NULL);
2588 if (dst_pdpg == NULL)
2589 break;
2590 l3e = VM_PAGE_TO_DMAP(dst_pdpg);
2591 l3e = &l3e[pmap_pml3e_index(addr)];
2592 if (be64toh(*l3e) == 0 && ((srcptepaddr & PG_MANAGED) == 0 ||
2593 pmap_pv_insert_l3e(dst_pmap, addr, srcptepaddr,
2594 PMAP_ENTER_NORECLAIM, &lock))) {
2595 *l3e = htobe64(srcptepaddr & ~PG_W);
2596 pmap_resident_count_inc(dst_pmap,
2597 L3_PAGE_SIZE / PAGE_SIZE);
2598 counter_u64_add(pmap_l3e_mappings, 1);
2599 } else
2600 dst_pdpg->ref_count--;
2601 continue;
2602 }
2603
2604 srcptepaddr &= PG_FRAME;
2605 srcmpte = PHYS_TO_VM_PAGE(srcptepaddr);
2606 KASSERT(srcmpte->ref_count > 0,
2607 ("pmap_copy: source page table page is unused"));
2608
2609 if (va_next > end_addr)
2610 va_next = end_addr;
2611
2612 src_pte = PHYS_TO_DMAP(srcptepaddr);
2613 src_pte = &src_pte[pmap_pte_index(addr)];
2614 dstmpte = NULL;
2615 while (addr < va_next) {
2616 pt_entry_t ptetemp;
2617 ptetemp = be64toh(*src_pte);
2618 /*
2619 * we only virtual copy managed pages
2620 */
2621 if ((ptetemp & PG_MANAGED) != 0) {
2622 if (dstmpte != NULL &&
2623 dstmpte->pindex == pmap_l3e_pindex(addr))
2624 dstmpte->ref_count++;
2625 else if ((dstmpte = pmap_allocpte(dst_pmap,
2626 addr, NULL)) == NULL)
2627 goto out;
2628 dst_pte = VM_PAGE_TO_DMAP(dstmpte);
2629 dst_pte = &dst_pte[pmap_pte_index(addr)];
2630 if (be64toh(*dst_pte) == 0 &&
2631 pmap_try_insert_pv_entry(dst_pmap, addr,
2632 PHYS_TO_VM_PAGE(ptetemp & PG_FRAME),
2633 &lock)) {
2634 /*
2635 * Clear the wired, modified, and
2636 * accessed (referenced) bits
2637 * during the copy.
2638 */
2639 *dst_pte = htobe64(ptetemp & ~(PG_W | PG_M |
2640 PG_A));
2641 pmap_resident_count_inc(dst_pmap, 1);
2642 } else {
2643 SLIST_INIT(&free);
2644 if (pmap_unwire_ptp(dst_pmap, addr,
2645 dstmpte, &free)) {
2646 /*
2647 * Although "addr" is not
2648 * mapped, paging-structure
2649 * caches could nonetheless
2650 * have entries that refer to
2651 * the freed page table pages.
2652 * Invalidate those entries.
2653 */
2654 invalidate_all = true;
2655 vm_page_free_pages_toq(&free,
2656 true);
2657 }
2658 goto out;
2659 }
2660 if (dstmpte->ref_count >= srcmpte->ref_count)
2661 break;
2662 }
2663 addr += PAGE_SIZE;
2664 if (__predict_false((addr & L3_PAGE_MASK) == 0))
2665 src_pte = pmap_pte(src_pmap, addr);
2666 else
2667 src_pte++;
2668 }
2669 }
2670 out:
2671 if (invalidate_all)
2672 pmap_invalidate_all(dst_pmap);
2673 if (lock != NULL)
2674 rw_wunlock(lock);
2675 PMAP_UNLOCK(src_pmap);
2676 PMAP_UNLOCK(dst_pmap);
2677 }
2678
2679 static void
mmu_radix_copy_page(vm_page_t msrc,vm_page_t mdst)2680 mmu_radix_copy_page(vm_page_t msrc, vm_page_t mdst)
2681 {
2682 void *src = VM_PAGE_TO_DMAP(msrc);
2683 void *dst = VM_PAGE_TO_DMAP(mdst);
2684
2685 CTR3(KTR_PMAP, "%s(%p, %p)", __func__, src, dst);
2686 /*
2687 * XXX slow
2688 */
2689 bcopy(src, dst, PAGE_SIZE);
2690 }
2691
2692 static void
mmu_radix_copy_pages(vm_page_t ma[],vm_offset_t a_offset,vm_page_t mb[],vm_offset_t b_offset,int xfersize)2693 mmu_radix_copy_pages(vm_page_t ma[], vm_offset_t a_offset, vm_page_t mb[],
2694 vm_offset_t b_offset, int xfersize)
2695 {
2696 void *a_cp, *b_cp;
2697 vm_offset_t a_pg_offset, b_pg_offset;
2698 int cnt;
2699
2700 CTR6(KTR_PMAP, "%s(%p, %#x, %p, %#x, %#x)", __func__, ma,
2701 a_offset, mb, b_offset, xfersize);
2702
2703 while (xfersize > 0) {
2704 a_pg_offset = a_offset & PAGE_MASK;
2705 cnt = min(xfersize, PAGE_SIZE - a_pg_offset);
2706 a_cp = (char *)VM_PAGE_TO_DMAP(ma[a_offset >> PAGE_SHIFT]) +
2707 a_pg_offset;
2708 b_pg_offset = b_offset & PAGE_MASK;
2709 cnt = min(cnt, PAGE_SIZE - b_pg_offset);
2710 b_cp = (char *)VM_PAGE_TO_DMAP(mb[b_offset >> PAGE_SHIFT]) +
2711 b_pg_offset;
2712 bcopy(a_cp, b_cp, cnt);
2713 a_offset += cnt;
2714 b_offset += cnt;
2715 xfersize -= cnt;
2716 }
2717 }
2718
2719 #if VM_NRESERVLEVEL > 0
2720 /*
2721 * Tries to promote the 512, contiguous 4KB page mappings that are within a
2722 * single page table page (PTP) to a single 2MB page mapping. For promotion
2723 * to occur, two conditions must be met: (1) the 4KB page mappings must map
2724 * aligned, contiguous physical memory and (2) the 4KB page mappings must have
2725 * identical characteristics.
2726 */
2727 static int
pmap_promote_l3e(pmap_t pmap,pml3_entry_t * pde,vm_offset_t va,struct rwlock ** lockp)2728 pmap_promote_l3e(pmap_t pmap, pml3_entry_t *pde, vm_offset_t va,
2729 struct rwlock **lockp)
2730 {
2731 pml3_entry_t newpde;
2732 pt_entry_t *firstpte, oldpte, pa, *pte;
2733 vm_page_t mpte;
2734
2735 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
2736
2737 /*
2738 * Examine the first PTE in the specified PTP. Abort if this PTE is
2739 * either invalid, unused, or does not map the first 4KB physical page
2740 * within a 2MB page.
2741 */
2742 firstpte = PHYS_TO_DMAP(be64toh(*pde) & PG_FRAME);
2743 setpde:
2744 newpde = be64toh(*firstpte);
2745 if ((newpde & ((PG_FRAME & L3_PAGE_MASK) | PG_A | PG_V)) != (PG_A | PG_V)) {
2746 CTR2(KTR_PMAP, "pmap_promote_l3e: failure for va %#lx"
2747 " in pmap %p", va, pmap);
2748 goto fail;
2749 }
2750 if ((newpde & (PG_M | PG_RW)) == PG_RW) {
2751 /*
2752 * When PG_M is already clear, PG_RW can be cleared without
2753 * a TLB invalidation.
2754 */
2755 if (!atomic_cmpset_long(firstpte, htobe64(newpde), htobe64((newpde | RPTE_EAA_R) & ~RPTE_EAA_W)))
2756 goto setpde;
2757 newpde &= ~RPTE_EAA_W;
2758 }
2759
2760 /*
2761 * Examine each of the other PTEs in the specified PTP. Abort if this
2762 * PTE maps an unexpected 4KB physical page or does not have identical
2763 * characteristics to the first PTE.
2764 */
2765 pa = (newpde & (PG_PS_FRAME | PG_A | PG_V)) + L3_PAGE_SIZE - PAGE_SIZE;
2766 for (pte = firstpte + NPTEPG - 1; pte > firstpte; pte--) {
2767 setpte:
2768 oldpte = be64toh(*pte);
2769 if ((oldpte & (PG_FRAME | PG_A | PG_V)) != pa) {
2770 CTR2(KTR_PMAP, "pmap_promote_l3e: failure for va %#lx"
2771 " in pmap %p", va, pmap);
2772 goto fail;
2773 }
2774 if ((oldpte & (PG_M | PG_RW)) == PG_RW) {
2775 /*
2776 * When PG_M is already clear, PG_RW can be cleared
2777 * without a TLB invalidation.
2778 */
2779 if (!atomic_cmpset_long(pte, htobe64(oldpte), htobe64((oldpte | RPTE_EAA_R) & ~RPTE_EAA_W)))
2780 goto setpte;
2781 oldpte &= ~RPTE_EAA_W;
2782 CTR2(KTR_PMAP, "pmap_promote_l3e: protect for va %#lx"
2783 " in pmap %p", (oldpte & PG_FRAME & L3_PAGE_MASK) |
2784 (va & ~L3_PAGE_MASK), pmap);
2785 }
2786 if ((oldpte & PG_PTE_PROMOTE) != (newpde & PG_PTE_PROMOTE)) {
2787 CTR2(KTR_PMAP, "pmap_promote_l3e: failure for va %#lx"
2788 " in pmap %p", va, pmap);
2789 goto fail;
2790 }
2791 pa -= PAGE_SIZE;
2792 }
2793
2794 /*
2795 * Save the page table page in its current state until the PDE
2796 * mapping the superpage is demoted by pmap_demote_pde() or
2797 * destroyed by pmap_remove_pde().
2798 */
2799 mpte = PHYS_TO_VM_PAGE(be64toh(*pde) & PG_FRAME);
2800 KASSERT(mpte >= vm_page_array &&
2801 mpte < &vm_page_array[vm_page_array_size],
2802 ("pmap_promote_l3e: page table page is out of range"));
2803 KASSERT(mpte->pindex == pmap_l3e_pindex(va),
2804 ("pmap_promote_l3e: page table page's pindex is wrong"));
2805 if (pmap_insert_pt_page(pmap, mpte)) {
2806 CTR2(KTR_PMAP,
2807 "pmap_promote_l3e: failure for va %#lx in pmap %p", va,
2808 pmap);
2809 goto fail;
2810 }
2811
2812 /*
2813 * Promote the pv entries.
2814 */
2815 if ((newpde & PG_MANAGED) != 0)
2816 pmap_pv_promote_l3e(pmap, va, newpde & PG_PS_FRAME, lockp);
2817
2818 pte_store(pde, PG_PROMOTED | newpde);
2819 ptesync();
2820 counter_u64_add(pmap_l3e_promotions, 1);
2821 CTR2(KTR_PMAP, "pmap_promote_l3e: success for va %#lx"
2822 " in pmap %p", va, pmap);
2823 return (0);
2824 fail:
2825 counter_u64_add(pmap_l3e_p_failures, 1);
2826 return (KERN_FAILURE);
2827 }
2828 #endif /* VM_NRESERVLEVEL > 0 */
2829
2830 int
mmu_radix_enter(pmap_t pmap,vm_offset_t va,vm_page_t m,vm_prot_t prot,u_int flags,int8_t psind)2831 mmu_radix_enter(pmap_t pmap, vm_offset_t va, vm_page_t m,
2832 vm_prot_t prot, u_int flags, int8_t psind)
2833 {
2834 struct rwlock *lock;
2835 pml3_entry_t *l3e;
2836 pt_entry_t *pte;
2837 pt_entry_t newpte, origpte;
2838 pv_entry_t pv;
2839 vm_paddr_t opa, pa;
2840 vm_page_t mpte, om;
2841 int rv, retrycount;
2842 bool nosleep, invalidate_all, invalidate_page;
2843
2844 va = trunc_page(va);
2845 retrycount = 0;
2846 invalidate_page = invalidate_all = false;
2847 CTR6(KTR_PMAP, "pmap_enter(%p, %#lx, %p, %#x, %#x, %d)", pmap, va,
2848 m, prot, flags, psind);
2849 KASSERT(va <= VM_MAX_KERNEL_ADDRESS, ("pmap_enter: toobig"));
2850 KASSERT((m->oflags & VPO_UNMANAGED) != 0 || !VA_IS_CLEANMAP(va),
2851 ("pmap_enter: managed mapping within the clean submap"));
2852 if ((m->oflags & VPO_UNMANAGED) == 0)
2853 VM_PAGE_OBJECT_BUSY_ASSERT(m);
2854
2855 KASSERT((flags & PMAP_ENTER_RESERVED) == 0,
2856 ("pmap_enter: flags %u has reserved bits set", flags));
2857 pa = VM_PAGE_TO_PHYS(m);
2858 newpte = (pt_entry_t)(pa | PG_A | PG_V | RPTE_LEAF);
2859 if ((flags & VM_PROT_WRITE) != 0)
2860 newpte |= PG_M;
2861 if ((flags & VM_PROT_READ) != 0)
2862 newpte |= PG_A;
2863 if (prot & VM_PROT_READ)
2864 newpte |= RPTE_EAA_R;
2865 if ((prot & VM_PROT_WRITE) != 0)
2866 newpte |= RPTE_EAA_W;
2867 KASSERT((newpte & (PG_M | PG_RW)) != PG_M,
2868 ("pmap_enter: flags includes VM_PROT_WRITE but prot doesn't"));
2869
2870 if (prot & VM_PROT_EXECUTE)
2871 newpte |= PG_X;
2872 if ((flags & PMAP_ENTER_WIRED) != 0)
2873 newpte |= PG_W;
2874 if (va >= DMAP_MIN_ADDRESS)
2875 newpte |= RPTE_EAA_P;
2876 newpte |= pmap_cache_bits(m->md.mdpg_cache_attrs);
2877 /*
2878 * Set modified bit gratuitously for writeable mappings if
2879 * the page is unmanaged. We do not want to take a fault
2880 * to do the dirty bit accounting for these mappings.
2881 */
2882 if ((m->oflags & VPO_UNMANAGED) != 0) {
2883 if ((newpte & PG_RW) != 0)
2884 newpte |= PG_M;
2885 } else
2886 newpte |= PG_MANAGED;
2887
2888 lock = NULL;
2889 PMAP_LOCK(pmap);
2890 if (psind == 1) {
2891 /* Assert the required virtual and physical alignment. */
2892 KASSERT((va & L3_PAGE_MASK) == 0, ("pmap_enter: va unaligned"));
2893 KASSERT(m->psind > 0, ("pmap_enter: m->psind < psind"));
2894 rv = pmap_enter_l3e(pmap, va, newpte | RPTE_LEAF, flags, m, &lock);
2895 goto out;
2896 }
2897 mpte = NULL;
2898
2899 /*
2900 * In the case that a page table page is not
2901 * resident, we are creating it here.
2902 */
2903 retry:
2904 l3e = pmap_pml3e(pmap, va);
2905 if (l3e != NULL && (be64toh(*l3e) & PG_V) != 0 && ((be64toh(*l3e) & RPTE_LEAF) == 0 ||
2906 pmap_demote_l3e_locked(pmap, l3e, va, &lock))) {
2907 pte = pmap_l3e_to_pte(l3e, va);
2908 if (va < VM_MAXUSER_ADDRESS && mpte == NULL) {
2909 mpte = PHYS_TO_VM_PAGE(be64toh(*l3e) & PG_FRAME);
2910 mpte->ref_count++;
2911 }
2912 } else if (va < VM_MAXUSER_ADDRESS) {
2913 /*
2914 * Here if the pte page isn't mapped, or if it has been
2915 * deallocated.
2916 */
2917 nosleep = (flags & PMAP_ENTER_NOSLEEP) != 0;
2918 mpte = _pmap_allocpte(pmap, pmap_l3e_pindex(va),
2919 nosleep ? NULL : &lock);
2920 if (mpte == NULL && nosleep) {
2921 rv = KERN_RESOURCE_SHORTAGE;
2922 goto out;
2923 }
2924 if (__predict_false(retrycount++ == 6))
2925 panic("too many retries");
2926 invalidate_all = true;
2927 goto retry;
2928 } else
2929 panic("pmap_enter: invalid page directory va=%#lx", va);
2930
2931 origpte = be64toh(*pte);
2932 pv = NULL;
2933
2934 /*
2935 * Is the specified virtual address already mapped?
2936 */
2937 if ((origpte & PG_V) != 0) {
2938 #ifdef INVARIANTS
2939 if (VERBOSE_PMAP || pmap_logging) {
2940 printf("cow fault pmap_enter(%p, %#lx, %p, %#x, %x, %d) --"
2941 " asid=%lu curpid=%d name=%s origpte0x%lx\n",
2942 pmap, va, m, prot, flags, psind, pmap->pm_pid,
2943 curproc->p_pid, curproc->p_comm, origpte);
2944 #ifdef DDB
2945 pmap_pte_walk(pmap->pm_pml1, va);
2946 #endif
2947 }
2948 #endif
2949 /*
2950 * Wiring change, just update stats. We don't worry about
2951 * wiring PT pages as they remain resident as long as there
2952 * are valid mappings in them. Hence, if a user page is wired,
2953 * the PT page will be also.
2954 */
2955 if ((newpte & PG_W) != 0 && (origpte & PG_W) == 0)
2956 pmap->pm_stats.wired_count++;
2957 else if ((newpte & PG_W) == 0 && (origpte & PG_W) != 0)
2958 pmap->pm_stats.wired_count--;
2959
2960 /*
2961 * Remove the extra PT page reference.
2962 */
2963 if (mpte != NULL) {
2964 mpte->ref_count--;
2965 KASSERT(mpte->ref_count > 0,
2966 ("pmap_enter: missing reference to page table page,"
2967 " va: 0x%lx", va));
2968 }
2969
2970 /*
2971 * Has the physical page changed?
2972 */
2973 opa = origpte & PG_FRAME;
2974 if (opa == pa) {
2975 /*
2976 * No, might be a protection or wiring change.
2977 */
2978 if ((origpte & PG_MANAGED) != 0 &&
2979 (newpte & PG_RW) != 0)
2980 vm_page_aflag_set(m, PGA_WRITEABLE);
2981 if (((origpte ^ newpte) & ~(PG_M | PG_A)) == 0) {
2982 if ((newpte & (PG_A|PG_M)) != (origpte & (PG_A|PG_M))) {
2983 if (!atomic_cmpset_long(pte, htobe64(origpte), htobe64(newpte)))
2984 goto retry;
2985 if ((newpte & PG_M) != (origpte & PG_M))
2986 vm_page_dirty(m);
2987 if ((newpte & PG_A) != (origpte & PG_A))
2988 vm_page_aflag_set(m, PGA_REFERENCED);
2989 ptesync();
2990 } else
2991 invalidate_all = true;
2992 if (((origpte ^ newpte) & ~(PG_M | PG_A)) == 0)
2993 goto unchanged;
2994 }
2995 goto validate;
2996 }
2997
2998 /*
2999 * The physical page has changed. Temporarily invalidate
3000 * the mapping. This ensures that all threads sharing the
3001 * pmap keep a consistent view of the mapping, which is
3002 * necessary for the correct handling of COW faults. It
3003 * also permits reuse of the old mapping's PV entry,
3004 * avoiding an allocation.
3005 *
3006 * For consistency, handle unmanaged mappings the same way.
3007 */
3008 origpte = be64toh(pte_load_clear(pte));
3009 KASSERT((origpte & PG_FRAME) == opa,
3010 ("pmap_enter: unexpected pa update for %#lx", va));
3011 if ((origpte & PG_MANAGED) != 0) {
3012 om = PHYS_TO_VM_PAGE(opa);
3013
3014 /*
3015 * The pmap lock is sufficient to synchronize with
3016 * concurrent calls to pmap_page_test_mappings() and
3017 * pmap_ts_referenced().
3018 */
3019 if ((origpte & (PG_M | PG_RW)) == (PG_M | PG_RW))
3020 vm_page_dirty(om);
3021 if ((origpte & PG_A) != 0)
3022 vm_page_aflag_set(om, PGA_REFERENCED);
3023 CHANGE_PV_LIST_LOCK_TO_PHYS(&lock, opa);
3024 pv = pmap_pvh_remove(&om->md, pmap, va);
3025 if ((newpte & PG_MANAGED) == 0)
3026 free_pv_entry(pmap, pv);
3027 #ifdef INVARIANTS
3028 else if (origpte & PG_MANAGED) {
3029 if (pv == NULL) {
3030 #ifdef DDB
3031 pmap_page_print_mappings(om);
3032 #endif
3033 MPASS(pv != NULL);
3034 }
3035 }
3036 #endif
3037 if ((om->a.flags & PGA_WRITEABLE) != 0 &&
3038 TAILQ_EMPTY(&om->md.pv_list) &&
3039 ((om->flags & PG_FICTITIOUS) != 0 ||
3040 TAILQ_EMPTY(&pa_to_pvh(opa)->pv_list)))
3041 vm_page_aflag_clear(om, PGA_WRITEABLE);
3042 }
3043 if ((origpte & PG_A) != 0)
3044 invalidate_page = true;
3045 origpte = 0;
3046 } else {
3047 if (pmap != kernel_pmap) {
3048 #ifdef INVARIANTS
3049 if (VERBOSE_PMAP || pmap_logging)
3050 printf("pmap_enter(%p, %#lx, %p, %#x, %x, %d) -- asid=%lu curpid=%d name=%s\n",
3051 pmap, va, m, prot, flags, psind,
3052 pmap->pm_pid, curproc->p_pid,
3053 curproc->p_comm);
3054 #endif
3055 }
3056
3057 /*
3058 * Increment the counters.
3059 */
3060 if ((newpte & PG_W) != 0)
3061 pmap->pm_stats.wired_count++;
3062 pmap_resident_count_inc(pmap, 1);
3063 }
3064
3065 /*
3066 * Enter on the PV list if part of our managed memory.
3067 */
3068 if ((newpte & PG_MANAGED) != 0) {
3069 if (pv == NULL) {
3070 pv = get_pv_entry(pmap, &lock);
3071 pv->pv_va = va;
3072 }
3073 #ifdef VERBOSE_PV
3074 else
3075 printf("reassigning pv: %p to pmap: %p\n",
3076 pv, pmap);
3077 #endif
3078 CHANGE_PV_LIST_LOCK_TO_PHYS(&lock, pa);
3079 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_link);
3080 m->md.pv_gen++;
3081 if ((newpte & PG_RW) != 0)
3082 vm_page_aflag_set(m, PGA_WRITEABLE);
3083 }
3084
3085 /*
3086 * Update the PTE.
3087 */
3088 if ((origpte & PG_V) != 0) {
3089 validate:
3090 origpte = be64toh(pte_load_store(pte, htobe64(newpte)));
3091 KASSERT((origpte & PG_FRAME) == pa,
3092 ("pmap_enter: unexpected pa update for %#lx", va));
3093 if ((newpte & PG_M) == 0 && (origpte & (PG_M | PG_RW)) ==
3094 (PG_M | PG_RW)) {
3095 if ((origpte & PG_MANAGED) != 0)
3096 vm_page_dirty(m);
3097 invalidate_page = true;
3098
3099 /*
3100 * Although the PTE may still have PG_RW set, TLB
3101 * invalidation may nonetheless be required because
3102 * the PTE no longer has PG_M set.
3103 */
3104 } else if ((origpte & PG_X) != 0 || (newpte & PG_X) == 0) {
3105 /*
3106 * Removing capabilities requires invalidation on POWER
3107 */
3108 invalidate_page = true;
3109 goto unchanged;
3110 }
3111 if ((origpte & PG_A) != 0)
3112 invalidate_page = true;
3113 } else {
3114 pte_store(pte, newpte);
3115 ptesync();
3116 }
3117 unchanged:
3118
3119 #if VM_NRESERVLEVEL > 0
3120 /*
3121 * If both the page table page and the reservation are fully
3122 * populated, then attempt promotion.
3123 */
3124 if ((mpte == NULL || mpte->ref_count == NPTEPG) &&
3125 mmu_radix_ps_enabled(pmap) &&
3126 (m->flags & PG_FICTITIOUS) == 0 &&
3127 vm_reserv_level_iffullpop(m) == 0 &&
3128 pmap_promote_l3e(pmap, l3e, va, &lock) == 0)
3129 invalidate_all = true;
3130 #endif
3131 if (invalidate_all)
3132 pmap_invalidate_all(pmap);
3133 else if (invalidate_page)
3134 pmap_invalidate_page(pmap, va);
3135
3136 rv = KERN_SUCCESS;
3137 out:
3138 if (lock != NULL)
3139 rw_wunlock(lock);
3140 PMAP_UNLOCK(pmap);
3141
3142 return (rv);
3143 }
3144
3145 /*
3146 * Release a page table page reference after a failed attempt to create a
3147 * mapping.
3148 */
3149 static void
pmap_abort_ptp(pmap_t pmap,vm_offset_t va,vm_page_t pdpg)3150 pmap_abort_ptp(pmap_t pmap, vm_offset_t va, vm_page_t pdpg)
3151 {
3152 struct spglist free;
3153
3154 SLIST_INIT(&free);
3155 if (pmap_unwire_ptp(pmap, va, pdpg, &free)) {
3156 /*
3157 * Although "va" is not mapped, paging-
3158 * structure caches could nonetheless have
3159 * entries that refer to the freed page table
3160 * pages. Invalidate those entries.
3161 */
3162 pmap_invalidate_page(pmap, va);
3163 vm_page_free_pages_toq(&free, true);
3164 }
3165 }
3166
3167 /*
3168 * Tries to create a read- and/or execute-only 2MB page mapping. Returns true
3169 * if successful. Returns false if (1) a page table page cannot be allocated
3170 * without sleeping, (2) a mapping already exists at the specified virtual
3171 * address, or (3) a PV entry cannot be allocated without reclaiming another
3172 * PV entry.
3173 */
3174 static bool
pmap_enter_2mpage(pmap_t pmap,vm_offset_t va,vm_page_t m,vm_prot_t prot,struct rwlock ** lockp)3175 pmap_enter_2mpage(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,
3176 struct rwlock **lockp)
3177 {
3178 pml3_entry_t newpde;
3179
3180 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
3181 newpde = VM_PAGE_TO_PHYS(m) | pmap_cache_bits(m->md.mdpg_cache_attrs) |
3182 RPTE_LEAF | PG_V;
3183 if ((m->oflags & VPO_UNMANAGED) == 0)
3184 newpde |= PG_MANAGED;
3185 if (prot & VM_PROT_EXECUTE)
3186 newpde |= PG_X;
3187 if (prot & VM_PROT_READ)
3188 newpde |= RPTE_EAA_R;
3189 if (va >= DMAP_MIN_ADDRESS)
3190 newpde |= RPTE_EAA_P;
3191 return (pmap_enter_l3e(pmap, va, newpde, PMAP_ENTER_NOSLEEP |
3192 PMAP_ENTER_NOREPLACE | PMAP_ENTER_NORECLAIM, NULL, lockp) ==
3193 KERN_SUCCESS);
3194 }
3195
3196 /*
3197 * Tries to create the specified 2MB page mapping. Returns KERN_SUCCESS if
3198 * the mapping was created, and either KERN_FAILURE or KERN_RESOURCE_SHORTAGE
3199 * otherwise. Returns KERN_FAILURE if PMAP_ENTER_NOREPLACE was specified and
3200 * a mapping already exists at the specified virtual address. Returns
3201 * KERN_RESOURCE_SHORTAGE if PMAP_ENTER_NOSLEEP was specified and a page table
3202 * page allocation failed. Returns KERN_RESOURCE_SHORTAGE if
3203 * PMAP_ENTER_NORECLAIM was specified and a PV entry allocation failed.
3204 *
3205 * The parameter "m" is only used when creating a managed, writeable mapping.
3206 */
3207 static int
pmap_enter_l3e(pmap_t pmap,vm_offset_t va,pml3_entry_t newpde,u_int flags,vm_page_t m,struct rwlock ** lockp)3208 pmap_enter_l3e(pmap_t pmap, vm_offset_t va, pml3_entry_t newpde, u_int flags,
3209 vm_page_t m, struct rwlock **lockp)
3210 {
3211 struct spglist free;
3212 pml3_entry_t oldl3e, *l3e;
3213 vm_page_t mt, pdpg;
3214 vm_page_t uwptpg;
3215
3216 KASSERT((newpde & (PG_M | PG_RW)) != PG_RW,
3217 ("pmap_enter_pde: newpde is missing PG_M"));
3218 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
3219
3220 if ((pdpg = pmap_allocl3e(pmap, va, (flags & PMAP_ENTER_NOSLEEP) != 0 ?
3221 NULL : lockp)) == NULL) {
3222 CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx"
3223 " in pmap %p", va, pmap);
3224 return (KERN_RESOURCE_SHORTAGE);
3225 }
3226 l3e = VM_PAGE_TO_DMAP(pdpg);
3227 l3e = &l3e[pmap_pml3e_index(va)];
3228 oldl3e = be64toh(*l3e);
3229 if ((oldl3e & PG_V) != 0) {
3230 KASSERT(pdpg->ref_count > 1,
3231 ("pmap_enter_pde: pdpg's wire count is too low"));
3232 if ((flags & PMAP_ENTER_NOREPLACE) != 0) {
3233 pdpg->ref_count--;
3234 CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx"
3235 " in pmap %p", va, pmap);
3236 return (KERN_FAILURE);
3237 }
3238 /* Break the existing mapping(s). */
3239 SLIST_INIT(&free);
3240 if ((oldl3e & RPTE_LEAF) != 0) {
3241 /*
3242 * The reference to the PD page that was acquired by
3243 * pmap_allocl3e() ensures that it won't be freed.
3244 * However, if the PDE resulted from a promotion, then
3245 * a reserved PT page could be freed.
3246 */
3247 (void)pmap_remove_l3e(pmap, l3e, va, &free, lockp);
3248 pmap_invalidate_l3e_page(pmap, va, oldl3e);
3249 } else {
3250 if (pmap_remove_ptes(pmap, va, va + L3_PAGE_SIZE, l3e,
3251 &free, lockp))
3252 pmap_invalidate_all(pmap);
3253 }
3254 vm_page_free_pages_toq(&free, true);
3255 if (va >= VM_MAXUSER_ADDRESS) {
3256 mt = PHYS_TO_VM_PAGE(be64toh(*l3e) & PG_FRAME);
3257 if (pmap_insert_pt_page(pmap, mt)) {
3258 /*
3259 * XXX Currently, this can't happen because
3260 * we do not perform pmap_enter(psind == 1)
3261 * on the kernel pmap.
3262 */
3263 panic("pmap_enter_pde: trie insert failed");
3264 }
3265 } else
3266 KASSERT(be64toh(*l3e) == 0, ("pmap_enter_pde: non-zero pde %p",
3267 l3e));
3268 }
3269
3270 /*
3271 * Allocate leaf ptpage for wired userspace pages.
3272 */
3273 uwptpg = NULL;
3274 if ((newpde & PG_W) != 0 && pmap != kernel_pmap) {
3275 uwptpg = vm_page_alloc_noobj(VM_ALLOC_WIRED);
3276 if (uwptpg == NULL) {
3277 pmap_abort_ptp(pmap, va, pdpg);
3278 return (KERN_RESOURCE_SHORTAGE);
3279 }
3280 uwptpg->pindex = pmap_l3e_pindex(va);
3281 if (pmap_insert_pt_page(pmap, uwptpg)) {
3282 vm_page_unwire_noq(uwptpg);
3283 vm_page_free(uwptpg);
3284 pmap_abort_ptp(pmap, va, pdpg);
3285 return (KERN_RESOURCE_SHORTAGE);
3286 }
3287 pmap_resident_count_inc(pmap, 1);
3288 uwptpg->ref_count = NPTEPG;
3289 pmap_fill_ptp(VM_PAGE_TO_DMAP(uwptpg), newpde);
3290 }
3291 if ((newpde & PG_MANAGED) != 0) {
3292 /*
3293 * Abort this mapping if its PV entry could not be created.
3294 */
3295 if (!pmap_pv_insert_l3e(pmap, va, newpde, flags, lockp)) {
3296 pmap_abort_ptp(pmap, va, pdpg);
3297 if (uwptpg != NULL) {
3298 mt = pmap_remove_pt_page(pmap, va);
3299 KASSERT(mt == uwptpg,
3300 ("removed pt page %p, expected %p", mt,
3301 uwptpg));
3302 pmap_resident_count_dec(pmap, 1);
3303 uwptpg->ref_count = 1;
3304 vm_page_unwire_noq(uwptpg);
3305 vm_page_free(uwptpg);
3306 }
3307 CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx"
3308 " in pmap %p", va, pmap);
3309 return (KERN_RESOURCE_SHORTAGE);
3310 }
3311 if ((newpde & PG_RW) != 0) {
3312 for (mt = m; mt < &m[L3_PAGE_SIZE / PAGE_SIZE]; mt++)
3313 vm_page_aflag_set(mt, PGA_WRITEABLE);
3314 }
3315 }
3316
3317 /*
3318 * Increment counters.
3319 */
3320 if ((newpde & PG_W) != 0)
3321 pmap->pm_stats.wired_count += L3_PAGE_SIZE / PAGE_SIZE;
3322 pmap_resident_count_inc(pmap, L3_PAGE_SIZE / PAGE_SIZE);
3323
3324 /*
3325 * Map the superpage. (This is not a promoted mapping; there will not
3326 * be any lingering 4KB page mappings in the TLB.)
3327 */
3328 pte_store(l3e, newpde);
3329 ptesync();
3330
3331 counter_u64_add(pmap_l3e_mappings, 1);
3332 CTR2(KTR_PMAP, "pmap_enter_pde: success for va %#lx"
3333 " in pmap %p", va, pmap);
3334 return (KERN_SUCCESS);
3335 }
3336
3337 void
mmu_radix_enter_object(pmap_t pmap,vm_offset_t start,vm_offset_t end,vm_page_t m_start,vm_prot_t prot)3338 mmu_radix_enter_object(pmap_t pmap, vm_offset_t start,
3339 vm_offset_t end, vm_page_t m_start, vm_prot_t prot)
3340 {
3341 struct pctrie_iter pages;
3342 struct rwlock *lock;
3343 vm_offset_t va;
3344 vm_page_t m, mpte;
3345 bool invalidate;
3346
3347 VM_OBJECT_ASSERT_LOCKED(m_start->object);
3348
3349 CTR6(KTR_PMAP, "%s(%p, %#x, %#x, %p, %#x)", __func__, pmap, start,
3350 end, m_start, prot);
3351 invalidate = false;
3352 mpte = NULL;
3353 vm_page_iter_limit_init(&pages, m_start->object,
3354 m_start->pindex + atop(end - start));
3355 m = vm_radix_iter_lookup(&pages, m_start->pindex);
3356 lock = NULL;
3357 PMAP_LOCK(pmap);
3358 while (m != NULL) {
3359 va = start + ptoa(m->pindex - m_start->pindex);
3360 if ((va & L3_PAGE_MASK) == 0 && va + L3_PAGE_SIZE <= end &&
3361 m->psind == 1 && mmu_radix_ps_enabled(pmap) &&
3362 pmap_enter_2mpage(pmap, va, m, prot, &lock)) {
3363 m = vm_radix_iter_jump(&pages, L3_PAGE_SIZE / PAGE_SIZE);
3364 } else {
3365 mpte = mmu_radix_enter_quick_locked(pmap, va, m, prot,
3366 mpte, &lock, &invalidate);
3367 m = vm_radix_iter_step(&pages);
3368 }
3369 }
3370 ptesync();
3371 if (lock != NULL)
3372 rw_wunlock(lock);
3373 if (invalidate)
3374 pmap_invalidate_all(pmap);
3375 PMAP_UNLOCK(pmap);
3376 }
3377
3378 static vm_page_t
mmu_radix_enter_quick_locked(pmap_t pmap,vm_offset_t va,vm_page_t m,vm_prot_t prot,vm_page_t mpte,struct rwlock ** lockp,bool * invalidate)3379 mmu_radix_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m,
3380 vm_prot_t prot, vm_page_t mpte, struct rwlock **lockp, bool *invalidate)
3381 {
3382 struct spglist free;
3383 pt_entry_t *pte;
3384 vm_paddr_t pa;
3385
3386 KASSERT(!VA_IS_CLEANMAP(va) ||
3387 (m->oflags & VPO_UNMANAGED) != 0,
3388 ("mmu_radix_enter_quick_locked: managed mapping within the clean submap"));
3389 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
3390
3391 /*
3392 * In the case that a page table page is not
3393 * resident, we are creating it here.
3394 */
3395 if (va < VM_MAXUSER_ADDRESS) {
3396 vm_pindex_t ptepindex;
3397 pml3_entry_t *ptepa;
3398
3399 /*
3400 * Calculate pagetable page index
3401 */
3402 ptepindex = pmap_l3e_pindex(va);
3403 if (mpte && (mpte->pindex == ptepindex)) {
3404 mpte->ref_count++;
3405 } else {
3406 /*
3407 * Get the page directory entry
3408 */
3409 ptepa = pmap_pml3e(pmap, va);
3410
3411 /*
3412 * If the page table page is mapped, we just increment
3413 * the hold count, and activate it. Otherwise, we
3414 * attempt to allocate a page table page. If this
3415 * attempt fails, we don't retry. Instead, we give up.
3416 */
3417 if (ptepa && (be64toh(*ptepa) & PG_V) != 0) {
3418 if (be64toh(*ptepa) & RPTE_LEAF)
3419 return (NULL);
3420 mpte = PHYS_TO_VM_PAGE(be64toh(*ptepa) & PG_FRAME);
3421 mpte->ref_count++;
3422 } else {
3423 /*
3424 * Pass NULL instead of the PV list lock
3425 * pointer, because we don't intend to sleep.
3426 */
3427 mpte = _pmap_allocpte(pmap, ptepindex, NULL);
3428 if (mpte == NULL)
3429 return (mpte);
3430 }
3431 }
3432 pte = VM_PAGE_TO_DMAP(mpte);
3433 pte = &pte[pmap_pte_index(va)];
3434 } else {
3435 mpte = NULL;
3436 pte = pmap_pte(pmap, va);
3437 }
3438 if (be64toh(*pte)) {
3439 if (mpte != NULL) {
3440 mpte->ref_count--;
3441 mpte = NULL;
3442 }
3443 return (mpte);
3444 }
3445
3446 /*
3447 * Enter on the PV list if part of our managed memory.
3448 */
3449 if ((m->oflags & VPO_UNMANAGED) == 0 &&
3450 !pmap_try_insert_pv_entry(pmap, va, m, lockp)) {
3451 if (mpte != NULL) {
3452 SLIST_INIT(&free);
3453 if (pmap_unwire_ptp(pmap, va, mpte, &free)) {
3454 /*
3455 * Although "va" is not mapped, paging-
3456 * structure caches could nonetheless have
3457 * entries that refer to the freed page table
3458 * pages. Invalidate those entries.
3459 */
3460 *invalidate = true;
3461 vm_page_free_pages_toq(&free, true);
3462 }
3463 mpte = NULL;
3464 }
3465 return (mpte);
3466 }
3467
3468 /*
3469 * Increment counters
3470 */
3471 pmap_resident_count_inc(pmap, 1);
3472
3473 pa = VM_PAGE_TO_PHYS(m) | pmap_cache_bits(m->md.mdpg_cache_attrs);
3474 if (prot & VM_PROT_EXECUTE)
3475 pa |= PG_X;
3476 else
3477 pa |= RPTE_EAA_R;
3478 if ((m->oflags & VPO_UNMANAGED) == 0)
3479 pa |= PG_MANAGED;
3480
3481 pte_store(pte, pa);
3482 return (mpte);
3483 }
3484
3485 void
mmu_radix_enter_quick(pmap_t pmap,vm_offset_t va,vm_page_t m,vm_prot_t prot)3486 mmu_radix_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m,
3487 vm_prot_t prot)
3488 {
3489 struct rwlock *lock;
3490 bool invalidate;
3491
3492 lock = NULL;
3493 invalidate = false;
3494 PMAP_LOCK(pmap);
3495 mmu_radix_enter_quick_locked(pmap, va, m, prot, NULL, &lock,
3496 &invalidate);
3497 ptesync();
3498 if (lock != NULL)
3499 rw_wunlock(lock);
3500 if (invalidate)
3501 pmap_invalidate_all(pmap);
3502 PMAP_UNLOCK(pmap);
3503 }
3504
3505 static vm_paddr_t
mmu_radix_extract_locked(pmap_t pmap,vm_offset_t va)3506 mmu_radix_extract_locked(pmap_t pmap, vm_offset_t va)
3507 {
3508 pml3_entry_t *l3e;
3509 pt_entry_t *pte;
3510 vm_paddr_t pa;
3511
3512 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
3513 l3e = pmap_pml3e(pmap, va);
3514 if (__predict_false(l3e == NULL))
3515 return (0);
3516 if (be64toh(*l3e) & RPTE_LEAF) {
3517 pa = (be64toh(*l3e) & PG_PS_FRAME) | (va & L3_PAGE_MASK);
3518 } else {
3519 /*
3520 * Beware of a concurrent promotion that changes the
3521 * PDE at this point! For example, vtopte() must not
3522 * be used to access the PTE because it would use the
3523 * new PDE. It is, however, safe to use the old PDE
3524 * because the page table page is preserved by the
3525 * promotion.
3526 */
3527 pte = pmap_l3e_to_pte(l3e, va);
3528 if (__predict_false(pte == NULL))
3529 return (0);
3530 pa = be64toh(*pte);
3531 pa = (pa & PG_FRAME) | (va & PAGE_MASK);
3532 }
3533 return (pa);
3534 }
3535
3536 vm_paddr_t
mmu_radix_extract(pmap_t pmap,vm_offset_t va)3537 mmu_radix_extract(pmap_t pmap, vm_offset_t va)
3538 {
3539 vm_paddr_t pa;
3540
3541 PMAP_LOCK(pmap);
3542 pa = mmu_radix_extract_locked(pmap, va);
3543 PMAP_UNLOCK(pmap);
3544 return (pa);
3545 }
3546
3547 vm_page_t
mmu_radix_extract_and_hold(pmap_t pmap,vm_offset_t va,vm_prot_t prot)3548 mmu_radix_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot)
3549 {
3550 pml3_entry_t l3e, *l3ep;
3551 pt_entry_t pte;
3552 vm_page_t m;
3553
3554 m = NULL;
3555 CTR4(KTR_PMAP, "%s(%p, %#x, %#x)", __func__, pmap, va, prot);
3556 PMAP_LOCK(pmap);
3557 l3ep = pmap_pml3e(pmap, va);
3558 if (l3ep != NULL && (l3e = be64toh(*l3ep))) {
3559 if (l3e & RPTE_LEAF) {
3560 if ((l3e & PG_RW) || (prot & VM_PROT_WRITE) == 0)
3561 m = PHYS_TO_VM_PAGE((l3e & PG_PS_FRAME) |
3562 (va & L3_PAGE_MASK));
3563 } else {
3564 /* Native endian PTE, do not pass to pmap functions */
3565 pte = be64toh(*pmap_l3e_to_pte(l3ep, va));
3566 if ((pte & PG_V) &&
3567 ((pte & PG_RW) || (prot & VM_PROT_WRITE) == 0))
3568 m = PHYS_TO_VM_PAGE(pte & PG_FRAME);
3569 }
3570 if (m != NULL && !vm_page_wire_mapped(m))
3571 m = NULL;
3572 }
3573 PMAP_UNLOCK(pmap);
3574 return (m);
3575 }
3576
3577 static int
mmu_radix_growkernel(vm_offset_t addr)3578 mmu_radix_growkernel(vm_offset_t addr)
3579 {
3580 vm_paddr_t paddr;
3581 vm_page_t nkpg;
3582 pml3_entry_t *l3e;
3583 pml2_entry_t *l2e;
3584
3585 CTR2(KTR_PMAP, "%s(%#x)", __func__, addr);
3586 if (VM_MIN_KERNEL_ADDRESS < addr &&
3587 addr < (VM_MIN_KERNEL_ADDRESS + nkpt * L3_PAGE_SIZE))
3588 return (KERN_SUCCESS);
3589
3590 addr = roundup2(addr, L3_PAGE_SIZE);
3591 if (addr - 1 >= vm_map_max(kernel_map))
3592 addr = vm_map_max(kernel_map);
3593 while (kernel_vm_end < addr) {
3594 l2e = pmap_pml2e(kernel_pmap, kernel_vm_end);
3595 if ((be64toh(*l2e) & PG_V) == 0) {
3596 /* We need a new PDP entry */
3597 nkpg = vm_page_alloc_noobj(VM_ALLOC_INTERRUPT |
3598 VM_ALLOC_NOFREE | VM_ALLOC_WIRED | VM_ALLOC_ZERO);
3599 if (nkpg == NULL)
3600 return (KERN_RESOURCE_SHORTAGE);
3601 nkpg->pindex = kernel_vm_end >> L2_PAGE_SIZE_SHIFT;
3602 paddr = VM_PAGE_TO_PHYS(nkpg);
3603 pde_store(l2e, paddr);
3604 continue; /* try again */
3605 }
3606 l3e = pmap_l2e_to_l3e(l2e, kernel_vm_end);
3607 if ((be64toh(*l3e) & PG_V) != 0) {
3608 kernel_vm_end = (kernel_vm_end + L3_PAGE_SIZE) & ~L3_PAGE_MASK;
3609 if (kernel_vm_end - 1 >= vm_map_max(kernel_map)) {
3610 kernel_vm_end = vm_map_max(kernel_map);
3611 break;
3612 }
3613 continue;
3614 }
3615
3616 nkpg = vm_page_alloc_noobj(VM_ALLOC_INTERRUPT |
3617 VM_ALLOC_NOFREE | VM_ALLOC_WIRED | VM_ALLOC_ZERO);
3618 if (nkpg == NULL)
3619 return (KERN_RESOURCE_SHORTAGE);
3620 nkpg->pindex = pmap_l3e_pindex(kernel_vm_end);
3621 paddr = VM_PAGE_TO_PHYS(nkpg);
3622 pde_store(l3e, paddr);
3623
3624 kernel_vm_end = (kernel_vm_end + L3_PAGE_SIZE) & ~L3_PAGE_MASK;
3625 if (kernel_vm_end - 1 >= vm_map_max(kernel_map)) {
3626 kernel_vm_end = vm_map_max(kernel_map);
3627 break;
3628 }
3629 }
3630 ptesync();
3631 return (KERN_SUCCESS);
3632 }
3633
3634 static MALLOC_DEFINE(M_RADIX_PGD, "radix_pgd", "radix page table root directory");
3635 static uma_zone_t zone_radix_pgd;
3636
3637 static int
radix_pgd_import(void * arg __unused,void ** store,int count,int domain __unused,int flags)3638 radix_pgd_import(void *arg __unused, void **store, int count, int domain __unused,
3639 int flags)
3640 {
3641 int req;
3642
3643 req = VM_ALLOC_WIRED | malloc2vm_flags(flags);
3644 for (int i = 0; i < count; i++) {
3645 vm_page_t m = vm_page_alloc_noobj_contig(req,
3646 RADIX_PGD_SIZE / PAGE_SIZE,
3647 0, (vm_paddr_t)-1, RADIX_PGD_SIZE, L1_PAGE_SIZE,
3648 VM_MEMATTR_DEFAULT);
3649 store[i] = VM_PAGE_TO_DMAP(m);
3650 }
3651 return (count);
3652 }
3653
3654 static void
radix_pgd_release(void * arg __unused,void ** store,int count)3655 radix_pgd_release(void *arg __unused, void **store, int count)
3656 {
3657 vm_page_t m;
3658 struct spglist free;
3659 int page_count;
3660
3661 SLIST_INIT(&free);
3662 page_count = RADIX_PGD_SIZE/PAGE_SIZE;
3663
3664 for (int i = 0; i < count; i++) {
3665 /*
3666 * XXX selectively remove dmap and KVA entries so we don't
3667 * need to bzero
3668 */
3669 m = DMAP_TO_VM_PAGE(store[i]);
3670 for (int j = page_count-1; j >= 0; j--) {
3671 vm_page_unwire_noq(&m[j]);
3672 SLIST_INSERT_HEAD(&free, &m[j], plinks.s.ss);
3673 }
3674 vm_page_free_pages_toq(&free, false);
3675 }
3676 }
3677
3678 static void
mmu_radix_init(void)3679 mmu_radix_init(void)
3680 {
3681 vm_page_t mpte;
3682 vm_size_t s;
3683 int error, i, pv_npg;
3684
3685 /* XXX is this really needed for POWER? */
3686 /* L1TF, reserve page @0 unconditionally */
3687 vm_page_blacklist_add(0, bootverbose);
3688
3689 zone_radix_pgd = uma_zcache_create("radix_pgd_cache",
3690 RADIX_PGD_SIZE, NULL, NULL,
3691 #ifdef INVARIANTS
3692 trash_init, trash_fini,
3693 #else
3694 NULL, NULL,
3695 #endif
3696 radix_pgd_import, radix_pgd_release,
3697 NULL, UMA_ZONE_NOBUCKET);
3698
3699 /*
3700 * Initialize the vm page array entries for the kernel pmap's
3701 * page table pages.
3702 */
3703 PMAP_LOCK(kernel_pmap);
3704 for (i = 0; i < nkpt; i++) {
3705 mpte = PHYS_TO_VM_PAGE(KPTphys + (i << PAGE_SHIFT));
3706 KASSERT(mpte >= vm_page_array &&
3707 mpte < &vm_page_array[vm_page_array_size],
3708 ("pmap_init: page table page is out of range size: %lu",
3709 vm_page_array_size));
3710 mpte->pindex = pmap_l3e_pindex(VM_MIN_KERNEL_ADDRESS) + i;
3711 mpte->phys_addr = KPTphys + (i << PAGE_SHIFT);
3712 MPASS(PHYS_TO_VM_PAGE(mpte->phys_addr) == mpte);
3713 //pmap_insert_pt_page(kernel_pmap, mpte);
3714 mpte->ref_count = 1;
3715 }
3716 PMAP_UNLOCK(kernel_pmap);
3717 vm_wire_add(nkpt);
3718
3719 CTR1(KTR_PMAP, "%s()", __func__);
3720 TAILQ_INIT(&pv_dummy.pv_list);
3721
3722 /*
3723 * Are large page mappings enabled?
3724 */
3725 TUNABLE_INT_FETCH("vm.pmap.superpages_enabled", &superpages_enabled);
3726 if (superpages_enabled) {
3727 KASSERT(MAXPAGESIZES > 1 && pagesizes[1] == 0,
3728 ("pmap_init: can't assign to pagesizes[1]"));
3729 pagesizes[1] = L3_PAGE_SIZE;
3730 }
3731
3732 /*
3733 * Initialize the pv chunk list mutex.
3734 */
3735 mtx_init(&pv_chunks_mutex, "pmap pv chunk list", NULL, MTX_DEF);
3736
3737 /*
3738 * Initialize the pool of pv list locks.
3739 */
3740 for (i = 0; i < NPV_LIST_LOCKS; i++)
3741 rw_init(&pv_list_locks[i], "pmap pv list");
3742
3743 /*
3744 * Calculate the size of the pv head table for superpages.
3745 */
3746 pv_npg = howmany(vm_phys_segs[vm_phys_nsegs - 1].end, L3_PAGE_SIZE);
3747
3748 /*
3749 * Allocate memory for the pv head table for superpages.
3750 */
3751 s = (vm_size_t)(pv_npg * sizeof(struct md_page));
3752 s = round_page(s);
3753 pv_table = kmem_malloc(s, M_WAITOK | M_ZERO);
3754 for (i = 0; i < pv_npg; i++)
3755 TAILQ_INIT(&pv_table[i].pv_list);
3756 TAILQ_INIT(&pv_dummy.pv_list);
3757
3758 pmap_initialized = 1;
3759 mtx_init(&qframe_mtx, "qfrmlk", NULL, MTX_SPIN);
3760 error = vmem_alloc(kernel_arena, PAGE_SIZE, M_BESTFIT | M_WAITOK,
3761 (vmem_addr_t *)&qframe);
3762
3763 if (error != 0)
3764 panic("qframe allocation failed");
3765 asid_arena = vmem_create("ASID", isa3_base_pid + 1, (1<<isa3_pid_bits),
3766 1, 1, M_WAITOK);
3767 }
3768
3769 static bool
pmap_page_test_mappings(vm_page_t m,bool accessed,bool modified)3770 pmap_page_test_mappings(vm_page_t m, bool accessed, bool modified)
3771 {
3772 struct rwlock *lock;
3773 pv_entry_t pv;
3774 struct md_page *pvh;
3775 pt_entry_t *pte, mask;
3776 pmap_t pmap;
3777 int md_gen, pvh_gen;
3778 bool rv;
3779
3780 rv = false;
3781 lock = VM_PAGE_TO_PV_LIST_LOCK(m);
3782 rw_rlock(lock);
3783 restart:
3784 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) {
3785 pmap = PV_PMAP(pv);
3786 if (!PMAP_TRYLOCK(pmap)) {
3787 md_gen = m->md.pv_gen;
3788 rw_runlock(lock);
3789 PMAP_LOCK(pmap);
3790 rw_rlock(lock);
3791 if (md_gen != m->md.pv_gen) {
3792 PMAP_UNLOCK(pmap);
3793 goto restart;
3794 }
3795 }
3796 pte = pmap_pte(pmap, pv->pv_va);
3797 mask = 0;
3798 if (modified)
3799 mask |= PG_RW | PG_M;
3800 if (accessed)
3801 mask |= PG_V | PG_A;
3802 rv = (be64toh(*pte) & mask) == mask;
3803 PMAP_UNLOCK(pmap);
3804 if (rv)
3805 goto out;
3806 }
3807 if ((m->flags & PG_FICTITIOUS) == 0) {
3808 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
3809 TAILQ_FOREACH(pv, &pvh->pv_list, pv_link) {
3810 pmap = PV_PMAP(pv);
3811 if (!PMAP_TRYLOCK(pmap)) {
3812 md_gen = m->md.pv_gen;
3813 pvh_gen = pvh->pv_gen;
3814 rw_runlock(lock);
3815 PMAP_LOCK(pmap);
3816 rw_rlock(lock);
3817 if (md_gen != m->md.pv_gen ||
3818 pvh_gen != pvh->pv_gen) {
3819 PMAP_UNLOCK(pmap);
3820 goto restart;
3821 }
3822 }
3823 pte = pmap_pml3e(pmap, pv->pv_va);
3824 mask = 0;
3825 if (modified)
3826 mask |= PG_RW | PG_M;
3827 if (accessed)
3828 mask |= PG_V | PG_A;
3829 rv = (be64toh(*pte) & mask) == mask;
3830 PMAP_UNLOCK(pmap);
3831 if (rv)
3832 goto out;
3833 }
3834 }
3835 out:
3836 rw_runlock(lock);
3837 return (rv);
3838 }
3839
3840 /*
3841 * pmap_is_modified:
3842 *
3843 * Return whether or not the specified physical page was modified
3844 * in any physical maps.
3845 */
3846 bool
mmu_radix_is_modified(vm_page_t m)3847 mmu_radix_is_modified(vm_page_t m)
3848 {
3849
3850 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
3851 ("pmap_is_modified: page %p is not managed", m));
3852
3853 CTR2(KTR_PMAP, "%s(%p)", __func__, m);
3854 /*
3855 * If the page is not busied then this check is racy.
3856 */
3857 if (!pmap_page_is_write_mapped(m))
3858 return (false);
3859 return (pmap_page_test_mappings(m, false, true));
3860 }
3861
3862 bool
mmu_radix_is_prefaultable(pmap_t pmap,vm_offset_t addr)3863 mmu_radix_is_prefaultable(pmap_t pmap, vm_offset_t addr)
3864 {
3865 pml3_entry_t *l3e;
3866 pt_entry_t *pte;
3867 bool rv;
3868
3869 CTR3(KTR_PMAP, "%s(%p, %#x)", __func__, pmap, addr);
3870 rv = false;
3871 PMAP_LOCK(pmap);
3872 l3e = pmap_pml3e(pmap, addr);
3873 if (l3e != NULL && (be64toh(*l3e) & (RPTE_LEAF | PG_V)) == PG_V) {
3874 pte = pmap_l3e_to_pte(l3e, addr);
3875 rv = (be64toh(*pte) & PG_V) == 0;
3876 }
3877 PMAP_UNLOCK(pmap);
3878 return (rv);
3879 }
3880
3881 bool
mmu_radix_is_referenced(vm_page_t m)3882 mmu_radix_is_referenced(vm_page_t m)
3883 {
3884 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
3885 ("pmap_is_referenced: page %p is not managed", m));
3886 CTR2(KTR_PMAP, "%s(%p)", __func__, m);
3887 return (pmap_page_test_mappings(m, true, false));
3888 }
3889
3890 /*
3891 * pmap_ts_referenced:
3892 *
3893 * Return a count of reference bits for a page, clearing those bits.
3894 * It is not necessary for every reference bit to be cleared, but it
3895 * is necessary that 0 only be returned when there are truly no
3896 * reference bits set.
3897 *
3898 * As an optimization, update the page's dirty field if a modified bit is
3899 * found while counting reference bits. This opportunistic update can be
3900 * performed at low cost and can eliminate the need for some future calls
3901 * to pmap_is_modified(). However, since this function stops after
3902 * finding PMAP_TS_REFERENCED_MAX reference bits, it may not detect some
3903 * dirty pages. Those dirty pages will only be detected by a future call
3904 * to pmap_is_modified().
3905 *
3906 * A DI block is not needed within this function, because
3907 * invalidations are performed before the PV list lock is
3908 * released.
3909 */
3910 int
mmu_radix_ts_referenced(vm_page_t m)3911 mmu_radix_ts_referenced(vm_page_t m)
3912 {
3913 struct md_page *pvh;
3914 pv_entry_t pv, pvf;
3915 pmap_t pmap;
3916 struct rwlock *lock;
3917 pml3_entry_t oldl3e, *l3e;
3918 pt_entry_t *pte;
3919 vm_paddr_t pa;
3920 int cleared, md_gen, not_cleared, pvh_gen;
3921 struct spglist free;
3922
3923 CTR2(KTR_PMAP, "%s(%p)", __func__, m);
3924 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
3925 ("pmap_ts_referenced: page %p is not managed", m));
3926 SLIST_INIT(&free);
3927 cleared = 0;
3928 pa = VM_PAGE_TO_PHYS(m);
3929 lock = PHYS_TO_PV_LIST_LOCK(pa);
3930 pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy : pa_to_pvh(pa);
3931 rw_wlock(lock);
3932 retry:
3933 not_cleared = 0;
3934 if ((pvf = TAILQ_FIRST(&pvh->pv_list)) == NULL)
3935 goto small_mappings;
3936 pv = pvf;
3937 do {
3938 if (pvf == NULL)
3939 pvf = pv;
3940 pmap = PV_PMAP(pv);
3941 if (!PMAP_TRYLOCK(pmap)) {
3942 pvh_gen = pvh->pv_gen;
3943 rw_wunlock(lock);
3944 PMAP_LOCK(pmap);
3945 rw_wlock(lock);
3946 if (pvh_gen != pvh->pv_gen) {
3947 PMAP_UNLOCK(pmap);
3948 goto retry;
3949 }
3950 }
3951 l3e = pmap_pml3e(pmap, pv->pv_va);
3952 oldl3e = be64toh(*l3e);
3953 if ((oldl3e & (PG_M | PG_RW)) == (PG_M | PG_RW)) {
3954 /*
3955 * Although "oldpde" is mapping a 2MB page, because
3956 * this function is called at a 4KB page granularity,
3957 * we only update the 4KB page under test.
3958 */
3959 vm_page_dirty(m);
3960 }
3961 if ((oldl3e & PG_A) != 0) {
3962 /*
3963 * Since this reference bit is shared by 512 4KB
3964 * pages, it should not be cleared every time it is
3965 * tested. Apply a simple "hash" function on the
3966 * physical page number, the virtual superpage number,
3967 * and the pmap address to select one 4KB page out of
3968 * the 512 on which testing the reference bit will
3969 * result in clearing that reference bit. This
3970 * function is designed to avoid the selection of the
3971 * same 4KB page for every 2MB page mapping.
3972 *
3973 * On demotion, a mapping that hasn't been referenced
3974 * is simply destroyed. To avoid the possibility of a
3975 * subsequent page fault on a demoted wired mapping,
3976 * always leave its reference bit set. Moreover,
3977 * since the superpage is wired, the current state of
3978 * its reference bit won't affect page replacement.
3979 */
3980 if ((((pa >> PAGE_SHIFT) ^ (pv->pv_va >> L3_PAGE_SIZE_SHIFT) ^
3981 (uintptr_t)pmap) & (NPTEPG - 1)) == 0 &&
3982 (oldl3e & PG_W) == 0) {
3983 atomic_clear_long(l3e, htobe64(PG_A));
3984 pmap_invalidate_page(pmap, pv->pv_va);
3985 cleared++;
3986 KASSERT(lock == VM_PAGE_TO_PV_LIST_LOCK(m),
3987 ("inconsistent pv lock %p %p for page %p",
3988 lock, VM_PAGE_TO_PV_LIST_LOCK(m), m));
3989 } else
3990 not_cleared++;
3991 }
3992 PMAP_UNLOCK(pmap);
3993 /* Rotate the PV list if it has more than one entry. */
3994 if (pv != NULL && TAILQ_NEXT(pv, pv_link) != NULL) {
3995 TAILQ_REMOVE(&pvh->pv_list, pv, pv_link);
3996 TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_link);
3997 pvh->pv_gen++;
3998 }
3999 if (cleared + not_cleared >= PMAP_TS_REFERENCED_MAX)
4000 goto out;
4001 } while ((pv = TAILQ_FIRST(&pvh->pv_list)) != pvf);
4002 small_mappings:
4003 if ((pvf = TAILQ_FIRST(&m->md.pv_list)) == NULL)
4004 goto out;
4005 pv = pvf;
4006 do {
4007 if (pvf == NULL)
4008 pvf = pv;
4009 pmap = PV_PMAP(pv);
4010 if (!PMAP_TRYLOCK(pmap)) {
4011 pvh_gen = pvh->pv_gen;
4012 md_gen = m->md.pv_gen;
4013 rw_wunlock(lock);
4014 PMAP_LOCK(pmap);
4015 rw_wlock(lock);
4016 if (pvh_gen != pvh->pv_gen || md_gen != m->md.pv_gen) {
4017 PMAP_UNLOCK(pmap);
4018 goto retry;
4019 }
4020 }
4021 l3e = pmap_pml3e(pmap, pv->pv_va);
4022 KASSERT((be64toh(*l3e) & RPTE_LEAF) == 0,
4023 ("pmap_ts_referenced: found a 2mpage in page %p's pv list",
4024 m));
4025 pte = pmap_l3e_to_pte(l3e, pv->pv_va);
4026 if ((be64toh(*pte) & (PG_M | PG_RW)) == (PG_M | PG_RW))
4027 vm_page_dirty(m);
4028 if ((be64toh(*pte) & PG_A) != 0) {
4029 atomic_clear_long(pte, htobe64(PG_A));
4030 pmap_invalidate_page(pmap, pv->pv_va);
4031 cleared++;
4032 }
4033 PMAP_UNLOCK(pmap);
4034 /* Rotate the PV list if it has more than one entry. */
4035 if (pv != NULL && TAILQ_NEXT(pv, pv_link) != NULL) {
4036 TAILQ_REMOVE(&m->md.pv_list, pv, pv_link);
4037 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_link);
4038 m->md.pv_gen++;
4039 }
4040 } while ((pv = TAILQ_FIRST(&m->md.pv_list)) != pvf && cleared +
4041 not_cleared < PMAP_TS_REFERENCED_MAX);
4042 out:
4043 rw_wunlock(lock);
4044 vm_page_free_pages_toq(&free, true);
4045 return (cleared + not_cleared);
4046 }
4047
4048 static void *
mmu_radix_map(vm_offset_t * virt __unused,vm_paddr_t start,vm_paddr_t end,int prot __unused)4049 mmu_radix_map(vm_offset_t *virt __unused, vm_paddr_t start,
4050 vm_paddr_t end, int prot __unused)
4051 {
4052
4053 CTR5(KTR_PMAP, "%s(%p, %#x, %#x, %#x)", __func__, virt, start, end,
4054 prot);
4055 return (PHYS_TO_DMAP(start));
4056 }
4057
4058 void
mmu_radix_object_init_pt(pmap_t pmap,vm_offset_t addr,vm_object_t object,vm_pindex_t pindex,vm_size_t size)4059 mmu_radix_object_init_pt(pmap_t pmap, vm_offset_t addr,
4060 vm_object_t object, vm_pindex_t pindex, vm_size_t size)
4061 {
4062 struct pctrie_iter pages;
4063 pml3_entry_t *l3e;
4064 vm_paddr_t pa, ptepa;
4065 vm_page_t p, pdpg;
4066 vm_memattr_t ma;
4067
4068 CTR6(KTR_PMAP, "%s(%p, %#x, %p, %u, %#x)", __func__, pmap, addr,
4069 object, pindex, size);
4070 VM_OBJECT_ASSERT_WLOCKED(object);
4071 KASSERT(object->type == OBJT_DEVICE || object->type == OBJT_SG,
4072 ("pmap_object_init_pt: non-device object"));
4073 /* NB: size can be logically ored with addr here */
4074 if ((addr & L3_PAGE_MASK) == 0 && (size & L3_PAGE_MASK) == 0) {
4075 if (!mmu_radix_ps_enabled(pmap))
4076 return;
4077 if (!vm_object_populate(object, pindex, pindex + atop(size)))
4078 return;
4079 vm_page_iter_init(&pages, object);
4080 p = vm_radix_iter_lookup(&pages, pindex);
4081
4082 KASSERT(p->valid == VM_PAGE_BITS_ALL,
4083 ("pmap_object_init_pt: invalid page %p", p));
4084 ma = p->md.mdpg_cache_attrs;
4085
4086 /*
4087 * Abort the mapping if the first page is not physically
4088 * aligned to a 2MB page boundary.
4089 */
4090 ptepa = VM_PAGE_TO_PHYS(p);
4091 if (ptepa & L3_PAGE_MASK)
4092 return;
4093
4094 /*
4095 * Skip the first page. Abort the mapping if the rest of
4096 * the pages are not physically contiguous or have differing
4097 * memory attributes.
4098 */
4099 for (pa = ptepa + PAGE_SIZE; pa < ptepa + size;
4100 pa += PAGE_SIZE) {
4101 p = vm_radix_iter_next(&pages);
4102 KASSERT(p->valid == VM_PAGE_BITS_ALL,
4103 ("pmap_object_init_pt: invalid page %p", p));
4104 if (pa != VM_PAGE_TO_PHYS(p) ||
4105 ma != p->md.mdpg_cache_attrs)
4106 return;
4107 }
4108
4109 PMAP_LOCK(pmap);
4110 for (pa = ptepa | pmap_cache_bits(ma);
4111 pa < ptepa + size; pa += L3_PAGE_SIZE) {
4112 pdpg = pmap_allocl3e(pmap, addr, NULL);
4113 if (pdpg == NULL) {
4114 /*
4115 * The creation of mappings below is only an
4116 * optimization. If a page directory page
4117 * cannot be allocated without blocking,
4118 * continue on to the next mapping rather than
4119 * blocking.
4120 */
4121 addr += L3_PAGE_SIZE;
4122 continue;
4123 }
4124 l3e = VM_PAGE_TO_DMAP(pdpg);
4125 l3e = &l3e[pmap_pml3e_index(addr)];
4126 if ((be64toh(*l3e) & PG_V) == 0) {
4127 pa |= PG_M | PG_A | PG_RW;
4128 pte_store(l3e, pa);
4129 pmap_resident_count_inc(pmap, L3_PAGE_SIZE / PAGE_SIZE);
4130 counter_u64_add(pmap_l3e_mappings, 1);
4131 } else {
4132 /* Continue on if the PDE is already valid. */
4133 pdpg->ref_count--;
4134 KASSERT(pdpg->ref_count > 0,
4135 ("pmap_object_init_pt: missing reference "
4136 "to page directory page, va: 0x%lx", addr));
4137 }
4138 addr += L3_PAGE_SIZE;
4139 }
4140 ptesync();
4141 PMAP_UNLOCK(pmap);
4142 }
4143 }
4144
4145 bool
mmu_radix_page_exists_quick(pmap_t pmap,vm_page_t m)4146 mmu_radix_page_exists_quick(pmap_t pmap, vm_page_t m)
4147 {
4148 struct md_page *pvh;
4149 struct rwlock *lock;
4150 pv_entry_t pv;
4151 int loops = 0;
4152 bool rv;
4153
4154 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
4155 ("pmap_page_exists_quick: page %p is not managed", m));
4156 CTR3(KTR_PMAP, "%s(%p, %p)", __func__, pmap, m);
4157 rv = false;
4158 lock = VM_PAGE_TO_PV_LIST_LOCK(m);
4159 rw_rlock(lock);
4160 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) {
4161 if (PV_PMAP(pv) == pmap) {
4162 rv = true;
4163 break;
4164 }
4165 loops++;
4166 if (loops >= 16)
4167 break;
4168 }
4169 if (!rv && loops < 16 && (m->flags & PG_FICTITIOUS) == 0) {
4170 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
4171 TAILQ_FOREACH(pv, &pvh->pv_list, pv_link) {
4172 if (PV_PMAP(pv) == pmap) {
4173 rv = true;
4174 break;
4175 }
4176 loops++;
4177 if (loops >= 16)
4178 break;
4179 }
4180 }
4181 rw_runlock(lock);
4182 return (rv);
4183 }
4184
4185 void
mmu_radix_page_init(vm_page_t m)4186 mmu_radix_page_init(vm_page_t m)
4187 {
4188
4189 CTR2(KTR_PMAP, "%s(%p)", __func__, m);
4190 TAILQ_INIT(&m->md.pv_list);
4191 m->md.mdpg_cache_attrs = VM_MEMATTR_DEFAULT;
4192 }
4193
4194 int
mmu_radix_page_wired_mappings(vm_page_t m)4195 mmu_radix_page_wired_mappings(vm_page_t m)
4196 {
4197 struct rwlock *lock;
4198 struct md_page *pvh;
4199 pmap_t pmap;
4200 pt_entry_t *pte;
4201 pv_entry_t pv;
4202 int count, md_gen, pvh_gen;
4203
4204 if ((m->oflags & VPO_UNMANAGED) != 0)
4205 return (0);
4206 CTR2(KTR_PMAP, "%s(%p)", __func__, m);
4207 lock = VM_PAGE_TO_PV_LIST_LOCK(m);
4208 rw_rlock(lock);
4209 restart:
4210 count = 0;
4211 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) {
4212 pmap = PV_PMAP(pv);
4213 if (!PMAP_TRYLOCK(pmap)) {
4214 md_gen = m->md.pv_gen;
4215 rw_runlock(lock);
4216 PMAP_LOCK(pmap);
4217 rw_rlock(lock);
4218 if (md_gen != m->md.pv_gen) {
4219 PMAP_UNLOCK(pmap);
4220 goto restart;
4221 }
4222 }
4223 pte = pmap_pte(pmap, pv->pv_va);
4224 if ((be64toh(*pte) & PG_W) != 0)
4225 count++;
4226 PMAP_UNLOCK(pmap);
4227 }
4228 if ((m->flags & PG_FICTITIOUS) == 0) {
4229 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
4230 TAILQ_FOREACH(pv, &pvh->pv_list, pv_link) {
4231 pmap = PV_PMAP(pv);
4232 if (!PMAP_TRYLOCK(pmap)) {
4233 md_gen = m->md.pv_gen;
4234 pvh_gen = pvh->pv_gen;
4235 rw_runlock(lock);
4236 PMAP_LOCK(pmap);
4237 rw_rlock(lock);
4238 if (md_gen != m->md.pv_gen ||
4239 pvh_gen != pvh->pv_gen) {
4240 PMAP_UNLOCK(pmap);
4241 goto restart;
4242 }
4243 }
4244 pte = pmap_pml3e(pmap, pv->pv_va);
4245 if ((be64toh(*pte) & PG_W) != 0)
4246 count++;
4247 PMAP_UNLOCK(pmap);
4248 }
4249 }
4250 rw_runlock(lock);
4251 return (count);
4252 }
4253
4254 static void
mmu_radix_update_proctab(int pid,pml1_entry_t l1pa)4255 mmu_radix_update_proctab(int pid, pml1_entry_t l1pa)
4256 {
4257 isa3_proctab[pid].proctab0 = htobe64(RTS_SIZE | l1pa | RADIX_PGD_INDEX_SHIFT);
4258 }
4259
4260 int
mmu_radix_pinit(pmap_t pmap)4261 mmu_radix_pinit(pmap_t pmap)
4262 {
4263 vmem_addr_t pid;
4264 vm_paddr_t l1pa;
4265
4266 CTR2(KTR_PMAP, "%s(%p)", __func__, pmap);
4267
4268 /*
4269 * allocate the page directory page
4270 */
4271 pmap->pm_pml1 = uma_zalloc(zone_radix_pgd, M_WAITOK);
4272
4273 for (int j = 0; j < RADIX_PGD_SIZE_SHIFT; j++)
4274 pagezero((char *)pmap->pm_pml1 + j * PAGE_SIZE);
4275 vm_radix_init(&pmap->pm_radix);
4276 TAILQ_INIT(&pmap->pm_pvchunk);
4277 bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
4278 pmap->pm_flags = PMAP_PDE_SUPERPAGE;
4279 vmem_alloc(asid_arena, 1, M_FIRSTFIT|M_WAITOK, &pid);
4280
4281 pmap->pm_pid = pid;
4282 l1pa = DMAP_TO_PHYS(pmap->pm_pml1);
4283 mmu_radix_update_proctab(pid, l1pa);
4284 __asm __volatile("ptesync;isync" : : : "memory");
4285
4286 return (1);
4287 }
4288
4289 /*
4290 * This routine is called if the desired page table page does not exist.
4291 *
4292 * If page table page allocation fails, this routine may sleep before
4293 * returning NULL. It sleeps only if a lock pointer was given.
4294 *
4295 * Note: If a page allocation fails at page table level two or three,
4296 * one or two pages may be held during the wait, only to be released
4297 * afterwards. This conservative approach is easily argued to avoid
4298 * race conditions.
4299 */
4300 static vm_page_t
_pmap_allocpte(pmap_t pmap,vm_pindex_t ptepindex,struct rwlock ** lockp)4301 _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, struct rwlock **lockp)
4302 {
4303 vm_page_t m, pdppg, pdpg;
4304
4305 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
4306
4307 /*
4308 * Allocate a page table page.
4309 */
4310 if ((m = vm_page_alloc_noobj(VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) {
4311 if (lockp != NULL) {
4312 RELEASE_PV_LIST_LOCK(lockp);
4313 PMAP_UNLOCK(pmap);
4314 vm_wait(NULL);
4315 PMAP_LOCK(pmap);
4316 }
4317 /*
4318 * Indicate the need to retry. While waiting, the page table
4319 * page may have been allocated.
4320 */
4321 return (NULL);
4322 }
4323 m->pindex = ptepindex;
4324
4325 /*
4326 * Map the pagetable page into the process address space, if
4327 * it isn't already there.
4328 */
4329
4330 if (ptepindex >= (NUPDE + NUPDPE)) {
4331 pml1_entry_t *l1e;
4332 vm_pindex_t pml1index;
4333
4334 /* Wire up a new PDPE page */
4335 pml1index = ptepindex - (NUPDE + NUPDPE);
4336 l1e = &pmap->pm_pml1[pml1index];
4337 KASSERT((be64toh(*l1e) & PG_V) == 0,
4338 ("%s: L1 entry %#lx is valid", __func__, *l1e));
4339 pde_store(l1e, VM_PAGE_TO_PHYS(m));
4340 } else if (ptepindex >= NUPDE) {
4341 vm_pindex_t pml1index;
4342 vm_pindex_t pdpindex;
4343 pml1_entry_t *l1e;
4344 pml2_entry_t *l2e;
4345
4346 /* Wire up a new l2e page */
4347 pdpindex = ptepindex - NUPDE;
4348 pml1index = pdpindex >> RPTE_SHIFT;
4349
4350 l1e = &pmap->pm_pml1[pml1index];
4351 if ((be64toh(*l1e) & PG_V) == 0) {
4352 /* Have to allocate a new pdp, recurse */
4353 if (_pmap_allocpte(pmap, NUPDE + NUPDPE + pml1index,
4354 lockp) == NULL) {
4355 vm_page_unwire_noq(m);
4356 vm_page_free_zero(m);
4357 return (NULL);
4358 }
4359 } else {
4360 /* Add reference to l2e page */
4361 pdppg = PHYS_TO_VM_PAGE(be64toh(*l1e) & PG_FRAME);
4362 pdppg->ref_count++;
4363 }
4364 l2e = PHYS_TO_DMAP(be64toh(*l1e) & PG_FRAME);
4365
4366 /* Now find the pdp page */
4367 l2e = &l2e[pdpindex & RPTE_MASK];
4368 KASSERT((be64toh(*l2e) & PG_V) == 0,
4369 ("%s: L2 entry %#lx is valid", __func__, *l2e));
4370 pde_store(l2e, VM_PAGE_TO_PHYS(m));
4371 } else {
4372 vm_pindex_t pml1index;
4373 vm_pindex_t pdpindex;
4374 pml1_entry_t *l1e;
4375 pml2_entry_t *l2e;
4376 pml3_entry_t *l3e;
4377
4378 /* Wire up a new PTE page */
4379 pdpindex = ptepindex >> RPTE_SHIFT;
4380 pml1index = pdpindex >> RPTE_SHIFT;
4381
4382 /* First, find the pdp and check that its valid. */
4383 l1e = &pmap->pm_pml1[pml1index];
4384 if ((be64toh(*l1e) & PG_V) == 0) {
4385 /* Have to allocate a new pd, recurse */
4386 if (_pmap_allocpte(pmap, NUPDE + pdpindex,
4387 lockp) == NULL) {
4388 vm_page_unwire_noq(m);
4389 vm_page_free_zero(m);
4390 return (NULL);
4391 }
4392 l2e = PHYS_TO_DMAP(be64toh(*l1e) & PG_FRAME);
4393 l2e = &l2e[pdpindex & RPTE_MASK];
4394 } else {
4395 l2e = PHYS_TO_DMAP(be64toh(*l1e) & PG_FRAME);
4396 l2e = &l2e[pdpindex & RPTE_MASK];
4397 if ((be64toh(*l2e) & PG_V) == 0) {
4398 /* Have to allocate a new pd, recurse */
4399 if (_pmap_allocpte(pmap, NUPDE + pdpindex,
4400 lockp) == NULL) {
4401 vm_page_unwire_noq(m);
4402 vm_page_free_zero(m);
4403 return (NULL);
4404 }
4405 } else {
4406 /* Add reference to the pd page */
4407 pdpg = PHYS_TO_VM_PAGE(be64toh(*l2e) & PG_FRAME);
4408 pdpg->ref_count++;
4409 }
4410 }
4411 l3e = PHYS_TO_DMAP(be64toh(*l2e) & PG_FRAME);
4412
4413 /* Now we know where the page directory page is */
4414 l3e = &l3e[ptepindex & RPTE_MASK];
4415 KASSERT((be64toh(*l3e) & PG_V) == 0,
4416 ("%s: L3 entry %#lx is valid", __func__, *l3e));
4417 pde_store(l3e, VM_PAGE_TO_PHYS(m));
4418 }
4419
4420 pmap_resident_count_inc(pmap, 1);
4421 return (m);
4422 }
4423 static vm_page_t
pmap_allocl3e(pmap_t pmap,vm_offset_t va,struct rwlock ** lockp)4424 pmap_allocl3e(pmap_t pmap, vm_offset_t va, struct rwlock **lockp)
4425 {
4426 vm_pindex_t pdpindex, ptepindex;
4427 pml2_entry_t *pdpe;
4428 vm_page_t pdpg;
4429
4430 retry:
4431 pdpe = pmap_pml2e(pmap, va);
4432 if (pdpe != NULL && (be64toh(*pdpe) & PG_V) != 0) {
4433 /* Add a reference to the pd page. */
4434 pdpg = PHYS_TO_VM_PAGE(be64toh(*pdpe) & PG_FRAME);
4435 pdpg->ref_count++;
4436 } else {
4437 /* Allocate a pd page. */
4438 ptepindex = pmap_l3e_pindex(va);
4439 pdpindex = ptepindex >> RPTE_SHIFT;
4440 pdpg = _pmap_allocpte(pmap, NUPDE + pdpindex, lockp);
4441 if (pdpg == NULL && lockp != NULL)
4442 goto retry;
4443 }
4444 return (pdpg);
4445 }
4446
4447 static vm_page_t
pmap_allocpte(pmap_t pmap,vm_offset_t va,struct rwlock ** lockp)4448 pmap_allocpte(pmap_t pmap, vm_offset_t va, struct rwlock **lockp)
4449 {
4450 vm_pindex_t ptepindex;
4451 pml3_entry_t *pd;
4452 vm_page_t m;
4453
4454 /*
4455 * Calculate pagetable page index
4456 */
4457 ptepindex = pmap_l3e_pindex(va);
4458 retry:
4459 /*
4460 * Get the page directory entry
4461 */
4462 pd = pmap_pml3e(pmap, va);
4463
4464 /*
4465 * This supports switching from a 2MB page to a
4466 * normal 4K page.
4467 */
4468 if (pd != NULL && (be64toh(*pd) & (RPTE_LEAF | PG_V)) == (RPTE_LEAF | PG_V)) {
4469 if (!pmap_demote_l3e_locked(pmap, pd, va, lockp)) {
4470 /*
4471 * Invalidation of the 2MB page mapping may have caused
4472 * the deallocation of the underlying PD page.
4473 */
4474 pd = NULL;
4475 }
4476 }
4477
4478 /*
4479 * If the page table page is mapped, we just increment the
4480 * hold count, and activate it.
4481 */
4482 if (pd != NULL && (be64toh(*pd) & PG_V) != 0) {
4483 m = PHYS_TO_VM_PAGE(be64toh(*pd) & PG_FRAME);
4484 m->ref_count++;
4485 } else {
4486 /*
4487 * Here if the pte page isn't mapped, or if it has been
4488 * deallocated.
4489 */
4490 m = _pmap_allocpte(pmap, ptepindex, lockp);
4491 if (m == NULL && lockp != NULL)
4492 goto retry;
4493 }
4494 return (m);
4495 }
4496
4497 static void
mmu_radix_pinit0(pmap_t pmap)4498 mmu_radix_pinit0(pmap_t pmap)
4499 {
4500
4501 CTR2(KTR_PMAP, "%s(%p)", __func__, pmap);
4502 PMAP_LOCK_INIT(pmap);
4503 pmap->pm_pml1 = kernel_pmap->pm_pml1;
4504 pmap->pm_pid = kernel_pmap->pm_pid;
4505
4506 vm_radix_init(&pmap->pm_radix);
4507 TAILQ_INIT(&pmap->pm_pvchunk);
4508 bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
4509 kernel_pmap->pm_flags =
4510 pmap->pm_flags = PMAP_PDE_SUPERPAGE;
4511 }
4512 /*
4513 * pmap_protect_l3e: do the things to protect a 2mpage in a process
4514 */
4515 static bool
pmap_protect_l3e(pmap_t pmap,pt_entry_t * l3e,vm_offset_t sva,vm_prot_t prot)4516 pmap_protect_l3e(pmap_t pmap, pt_entry_t *l3e, vm_offset_t sva, vm_prot_t prot)
4517 {
4518 pt_entry_t newpde, oldpde;
4519 vm_offset_t eva, va;
4520 vm_page_t m;
4521 bool anychanged;
4522
4523 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
4524 KASSERT((sva & L3_PAGE_MASK) == 0,
4525 ("pmap_protect_l3e: sva is not 2mpage aligned"));
4526 anychanged = false;
4527 retry:
4528 oldpde = newpde = be64toh(*l3e);
4529 if ((oldpde & (PG_MANAGED | PG_M | PG_RW)) ==
4530 (PG_MANAGED | PG_M | PG_RW)) {
4531 eva = sva + L3_PAGE_SIZE;
4532 for (va = sva, m = PHYS_TO_VM_PAGE(oldpde & PG_PS_FRAME);
4533 va < eva; va += PAGE_SIZE, m++)
4534 vm_page_dirty(m);
4535 }
4536 if ((prot & VM_PROT_WRITE) == 0) {
4537 newpde &= ~(PG_RW | PG_M);
4538 newpde |= RPTE_EAA_R;
4539 }
4540 if (prot & VM_PROT_EXECUTE)
4541 newpde |= PG_X;
4542 if (newpde != oldpde) {
4543 /*
4544 * As an optimization to future operations on this PDE, clear
4545 * PG_PROMOTED. The impending invalidation will remove any
4546 * lingering 4KB page mappings from the TLB.
4547 */
4548 if (!atomic_cmpset_long(l3e, htobe64(oldpde), htobe64(newpde & ~PG_PROMOTED)))
4549 goto retry;
4550 anychanged = true;
4551 }
4552 return (anychanged);
4553 }
4554
4555 void
mmu_radix_protect(pmap_t pmap,vm_offset_t sva,vm_offset_t eva,vm_prot_t prot)4556 mmu_radix_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,
4557 vm_prot_t prot)
4558 {
4559 vm_offset_t va_next;
4560 pml1_entry_t *l1e;
4561 pml2_entry_t *l2e;
4562 pml3_entry_t ptpaddr, *l3e;
4563 pt_entry_t *pte;
4564 bool anychanged;
4565
4566 CTR5(KTR_PMAP, "%s(%p, %#x, %#x, %#x)", __func__, pmap, sva, eva,
4567 prot);
4568
4569 KASSERT((prot & ~VM_PROT_ALL) == 0, ("invalid prot %x", prot));
4570 if (prot == VM_PROT_NONE) {
4571 mmu_radix_remove(pmap, sva, eva);
4572 return;
4573 }
4574
4575 if ((prot & (VM_PROT_WRITE|VM_PROT_EXECUTE)) ==
4576 (VM_PROT_WRITE|VM_PROT_EXECUTE))
4577 return;
4578
4579 #ifdef INVARIANTS
4580 if (VERBOSE_PROTECT || pmap_logging)
4581 printf("pmap_protect(%p, %#lx, %#lx, %x) - asid: %lu\n",
4582 pmap, sva, eva, prot, pmap->pm_pid);
4583 #endif
4584 anychanged = false;
4585
4586 PMAP_LOCK(pmap);
4587 for (; sva < eva; sva = va_next) {
4588 l1e = pmap_pml1e(pmap, sva);
4589 if ((be64toh(*l1e) & PG_V) == 0) {
4590 va_next = (sva + L1_PAGE_SIZE) & ~L1_PAGE_MASK;
4591 if (va_next < sva)
4592 va_next = eva;
4593 continue;
4594 }
4595
4596 l2e = pmap_l1e_to_l2e(l1e, sva);
4597 if ((be64toh(*l2e) & PG_V) == 0) {
4598 va_next = (sva + L2_PAGE_SIZE) & ~L2_PAGE_MASK;
4599 if (va_next < sva)
4600 va_next = eva;
4601 continue;
4602 }
4603
4604 va_next = (sva + L3_PAGE_SIZE) & ~L3_PAGE_MASK;
4605 if (va_next < sva)
4606 va_next = eva;
4607
4608 l3e = pmap_l2e_to_l3e(l2e, sva);
4609 ptpaddr = be64toh(*l3e);
4610
4611 /*
4612 * Weed out invalid mappings.
4613 */
4614 if (ptpaddr == 0)
4615 continue;
4616
4617 /*
4618 * Check for large page.
4619 */
4620 if ((ptpaddr & RPTE_LEAF) != 0) {
4621 /*
4622 * Are we protecting the entire large page? If not,
4623 * demote the mapping and fall through.
4624 */
4625 if (sva + L3_PAGE_SIZE == va_next && eva >= va_next) {
4626 if (pmap_protect_l3e(pmap, l3e, sva, prot))
4627 anychanged = true;
4628 continue;
4629 } else if (!pmap_demote_l3e(pmap, l3e, sva)) {
4630 /*
4631 * The large page mapping was destroyed.
4632 */
4633 continue;
4634 }
4635 }
4636
4637 if (va_next > eva)
4638 va_next = eva;
4639
4640 for (pte = pmap_l3e_to_pte(l3e, sva); sva != va_next; pte++,
4641 sva += PAGE_SIZE) {
4642 pt_entry_t obits, pbits;
4643 vm_page_t m;
4644
4645 retry:
4646 MPASS(pte == pmap_pte(pmap, sva));
4647 obits = pbits = be64toh(*pte);
4648 if ((pbits & PG_V) == 0)
4649 continue;
4650
4651 if ((prot & VM_PROT_WRITE) == 0) {
4652 if ((pbits & (PG_MANAGED | PG_M | PG_RW)) ==
4653 (PG_MANAGED | PG_M | PG_RW)) {
4654 m = PHYS_TO_VM_PAGE(pbits & PG_FRAME);
4655 vm_page_dirty(m);
4656 }
4657 pbits &= ~(PG_RW | PG_M);
4658 pbits |= RPTE_EAA_R;
4659 }
4660 if (prot & VM_PROT_EXECUTE)
4661 pbits |= PG_X;
4662
4663 if (pbits != obits) {
4664 if (!atomic_cmpset_long(pte, htobe64(obits), htobe64(pbits)))
4665 goto retry;
4666 if (obits & (PG_A|PG_M)) {
4667 anychanged = true;
4668 #ifdef INVARIANTS
4669 if (VERBOSE_PROTECT || pmap_logging)
4670 printf("%#lx %#lx -> %#lx\n",
4671 sva, obits, pbits);
4672 #endif
4673 }
4674 }
4675 }
4676 }
4677 if (anychanged)
4678 pmap_invalidate_all(pmap);
4679 PMAP_UNLOCK(pmap);
4680 }
4681
4682 void
mmu_radix_qenter(void * sva,vm_page_t * ma,int count)4683 mmu_radix_qenter(void *sva, vm_page_t *ma, int count)
4684 {
4685 pt_entry_t oldpte, pa, *pte;
4686 vm_page_t m;
4687 uint64_t cache_bits, attr_bits;
4688 vm_offset_t va;
4689
4690 CTR4(KTR_PMAP, "%s(%p, %p, %d)", __func__, sva, ma, count);
4691 oldpte = 0;
4692 attr_bits = RPTE_EAA_R | RPTE_EAA_W | RPTE_EAA_P | PG_M | PG_A;
4693 va = (vm_offset_t)sva;
4694 pte = kvtopte(va);
4695 while (va < (vm_offset_t)sva + PAGE_SIZE * count) {
4696 if (__predict_false((va & L3_PAGE_MASK) == 0))
4697 pte = kvtopte(va);
4698 MPASS(pte == pmap_pte(kernel_pmap, va));
4699
4700 /*
4701 * XXX there has to be a more efficient way than traversing
4702 * the page table every time - but go for correctness for
4703 * today
4704 */
4705
4706 m = *ma++;
4707 cache_bits = pmap_cache_bits(m->md.mdpg_cache_attrs);
4708 pa = VM_PAGE_TO_PHYS(m) | cache_bits | attr_bits;
4709 if (be64toh(*pte) != pa) {
4710 oldpte |= be64toh(*pte);
4711 pte_store(pte, pa);
4712 }
4713 va += PAGE_SIZE;
4714 pte++;
4715 }
4716 if (__predict_false((oldpte & RPTE_VALID) != 0))
4717 pmap_invalidate_range(kernel_pmap, (vm_offset_t)sva,
4718 (vm_offset_t)sva + count * PAGE_SIZE);
4719 else
4720 ptesync();
4721 }
4722
4723 void
mmu_radix_qremove(void * sva,int count)4724 mmu_radix_qremove(void *sva, int count)
4725 {
4726 vm_offset_t va;
4727 pt_entry_t *pte;
4728
4729 va = (vm_offset_t)sva;
4730 CTR3(KTR_PMAP, "%s(%p, %d)", __func__, sva, count);
4731 KASSERT(va >= VM_MIN_KERNEL_ADDRESS, ("usermode or dmap va %p", sva));
4732
4733 pte = kvtopte(va);
4734 while (va < (vm_offset_t)sva + PAGE_SIZE * count) {
4735 if (__predict_false((va & L3_PAGE_MASK) == 0))
4736 pte = kvtopte(va);
4737 pte_clear(pte);
4738 pte++;
4739 va += PAGE_SIZE;
4740 }
4741 pmap_invalidate_range(kernel_pmap, (vm_offset_t)sva, va);
4742 }
4743
4744 /***************************************************
4745 * Page table page management routines.....
4746 ***************************************************/
4747 /*
4748 * Schedule the specified unused page table page to be freed. Specifically,
4749 * add the page to the specified list of pages that will be released to the
4750 * physical memory manager after the TLB has been updated.
4751 */
4752 static __inline void
pmap_add_delayed_free_list(vm_page_t m,struct spglist * free,bool set_PG_ZERO)4753 pmap_add_delayed_free_list(vm_page_t m, struct spglist *free, bool set_PG_ZERO)
4754 {
4755
4756 if (set_PG_ZERO)
4757 m->flags |= PG_ZERO;
4758 else
4759 m->flags &= ~PG_ZERO;
4760 SLIST_INSERT_HEAD(free, m, plinks.s.ss);
4761 }
4762
4763 /*
4764 * Inserts the specified page table page into the specified pmap's collection
4765 * of idle page table pages. Each of a pmap's page table pages is responsible
4766 * for mapping a distinct range of virtual addresses. The pmap's collection is
4767 * ordered by this virtual address range.
4768 */
4769 static __inline int
pmap_insert_pt_page(pmap_t pmap,vm_page_t mpte)4770 pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte)
4771 {
4772
4773 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
4774 return (vm_radix_insert(&pmap->pm_radix, mpte));
4775 }
4776
4777 /*
4778 * Removes the page table page mapping the specified virtual address from the
4779 * specified pmap's collection of idle page table pages, and returns it.
4780 * Otherwise, returns NULL if there is no page table page corresponding to the
4781 * specified virtual address.
4782 */
4783 static __inline vm_page_t
pmap_remove_pt_page(pmap_t pmap,vm_offset_t va)4784 pmap_remove_pt_page(pmap_t pmap, vm_offset_t va)
4785 {
4786
4787 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
4788 return (vm_radix_remove(&pmap->pm_radix, pmap_l3e_pindex(va)));
4789 }
4790
4791 /*
4792 * Decrements a page table page's wire count, which is used to record the
4793 * number of valid page table entries within the page. If the wire count
4794 * drops to zero, then the page table page is unmapped. Returns true if the
4795 * page table page was unmapped and false otherwise.
4796 */
4797 static inline bool
pmap_unwire_ptp(pmap_t pmap,vm_offset_t va,vm_page_t m,struct spglist * free)4798 pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free)
4799 {
4800
4801 --m->ref_count;
4802 if (m->ref_count == 0) {
4803 _pmap_unwire_ptp(pmap, va, m, free);
4804 return (true);
4805 } else
4806 return (false);
4807 }
4808
4809 static void
_pmap_unwire_ptp(pmap_t pmap,vm_offset_t va,vm_page_t m,struct spglist * free)4810 _pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free)
4811 {
4812
4813 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
4814 /*
4815 * unmap the page table page
4816 */
4817 if (m->pindex >= NUPDE + NUPDPE) {
4818 /* PDP page */
4819 pml1_entry_t *pml1;
4820 pml1 = pmap_pml1e(pmap, va);
4821 *pml1 = 0;
4822 } else if (m->pindex >= NUPDE) {
4823 /* PD page */
4824 pml2_entry_t *l2e;
4825 l2e = pmap_pml2e(pmap, va);
4826 *l2e = 0;
4827 } else {
4828 /* PTE page */
4829 pml3_entry_t *l3e;
4830 l3e = pmap_pml3e(pmap, va);
4831 *l3e = 0;
4832 }
4833 pmap_resident_count_dec(pmap, 1);
4834 if (m->pindex < NUPDE) {
4835 /* We just released a PT, unhold the matching PD */
4836 vm_page_t pdpg;
4837
4838 pdpg = PHYS_TO_VM_PAGE(be64toh(*pmap_pml2e(pmap, va)) & PG_FRAME);
4839 pmap_unwire_ptp(pmap, va, pdpg, free);
4840 }
4841 else if (m->pindex >= NUPDE && m->pindex < (NUPDE + NUPDPE)) {
4842 /* We just released a PD, unhold the matching PDP */
4843 vm_page_t pdppg;
4844
4845 pdppg = PHYS_TO_VM_PAGE(be64toh(*pmap_pml1e(pmap, va)) & PG_FRAME);
4846 pmap_unwire_ptp(pmap, va, pdppg, free);
4847 }
4848
4849 /*
4850 * Put page on a list so that it is released after
4851 * *ALL* TLB shootdown is done
4852 */
4853 pmap_add_delayed_free_list(m, free, true);
4854 }
4855
4856 /*
4857 * After removing a page table entry, this routine is used to
4858 * conditionally free the page, and manage the hold/wire counts.
4859 */
4860 static int
pmap_unuse_pt(pmap_t pmap,vm_offset_t va,pml3_entry_t ptepde,struct spglist * free)4861 pmap_unuse_pt(pmap_t pmap, vm_offset_t va, pml3_entry_t ptepde,
4862 struct spglist *free)
4863 {
4864 vm_page_t mpte;
4865
4866 if (va >= VM_MAXUSER_ADDRESS)
4867 return (0);
4868 KASSERT(ptepde != 0, ("pmap_unuse_pt: ptepde != 0"));
4869 mpte = PHYS_TO_VM_PAGE(ptepde & PG_FRAME);
4870 return (pmap_unwire_ptp(pmap, va, mpte, free));
4871 }
4872
4873 void
mmu_radix_release(pmap_t pmap)4874 mmu_radix_release(pmap_t pmap)
4875 {
4876
4877 CTR2(KTR_PMAP, "%s(%p)", __func__, pmap);
4878 KASSERT(pmap->pm_stats.resident_count == 0,
4879 ("pmap_release: pmap resident count %ld != 0",
4880 pmap->pm_stats.resident_count));
4881 KASSERT(vm_radix_is_empty(&pmap->pm_radix),
4882 ("pmap_release: pmap has reserved page table page(s)"));
4883
4884 pmap_invalidate_all(pmap);
4885 isa3_proctab[pmap->pm_pid].proctab0 = 0;
4886 uma_zfree(zone_radix_pgd, pmap->pm_pml1);
4887 vmem_free(asid_arena, pmap->pm_pid, 1);
4888 }
4889
4890 /*
4891 * Create the PV entry for a 2MB page mapping. Always returns true unless the
4892 * flag PMAP_ENTER_NORECLAIM is specified. If that flag is specified, returns
4893 * false if the PV entry cannot be allocated without resorting to reclamation.
4894 */
4895 static bool
pmap_pv_insert_l3e(pmap_t pmap,vm_offset_t va,pml3_entry_t pde,u_int flags,struct rwlock ** lockp)4896 pmap_pv_insert_l3e(pmap_t pmap, vm_offset_t va, pml3_entry_t pde, u_int flags,
4897 struct rwlock **lockp)
4898 {
4899 struct md_page *pvh;
4900 pv_entry_t pv;
4901 vm_paddr_t pa;
4902
4903 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
4904 /* Pass NULL instead of the lock pointer to disable reclamation. */
4905 if ((pv = get_pv_entry(pmap, (flags & PMAP_ENTER_NORECLAIM) != 0 ?
4906 NULL : lockp)) == NULL)
4907 return (false);
4908 pv->pv_va = va;
4909 pa = pde & PG_PS_FRAME;
4910 CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa);
4911 pvh = pa_to_pvh(pa);
4912 TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_link);
4913 pvh->pv_gen++;
4914 return (true);
4915 }
4916
4917 /*
4918 * Fills a page table page with mappings to consecutive physical pages.
4919 */
4920 static void
pmap_fill_ptp(pt_entry_t * firstpte,pt_entry_t newpte)4921 pmap_fill_ptp(pt_entry_t *firstpte, pt_entry_t newpte)
4922 {
4923 pt_entry_t *pte;
4924
4925 for (pte = firstpte; pte < firstpte + NPTEPG; pte++) {
4926 *pte = htobe64(newpte);
4927 newpte += PAGE_SIZE;
4928 }
4929 }
4930
4931 static bool
pmap_demote_l3e(pmap_t pmap,pml3_entry_t * pde,vm_offset_t va)4932 pmap_demote_l3e(pmap_t pmap, pml3_entry_t *pde, vm_offset_t va)
4933 {
4934 struct rwlock *lock;
4935 bool rv;
4936
4937 lock = NULL;
4938 rv = pmap_demote_l3e_locked(pmap, pde, va, &lock);
4939 if (lock != NULL)
4940 rw_wunlock(lock);
4941 return (rv);
4942 }
4943
4944 static bool
pmap_demote_l3e_locked(pmap_t pmap,pml3_entry_t * l3e,vm_offset_t va,struct rwlock ** lockp)4945 pmap_demote_l3e_locked(pmap_t pmap, pml3_entry_t *l3e, vm_offset_t va,
4946 struct rwlock **lockp)
4947 {
4948 pml3_entry_t oldpde;
4949 pt_entry_t *firstpte;
4950 vm_paddr_t mptepa;
4951 vm_page_t mpte;
4952 struct spglist free;
4953 vm_offset_t sva;
4954
4955 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
4956 oldpde = be64toh(*l3e);
4957 KASSERT((oldpde & (RPTE_LEAF | PG_V)) == (RPTE_LEAF | PG_V),
4958 ("pmap_demote_l3e: oldpde is missing RPTE_LEAF and/or PG_V %lx",
4959 oldpde));
4960 if ((oldpde & PG_A) == 0 || (mpte = pmap_remove_pt_page(pmap, va)) ==
4961 NULL) {
4962 KASSERT((oldpde & PG_W) == 0,
4963 ("pmap_demote_l3e: page table page for a wired mapping"
4964 " is missing"));
4965
4966 /*
4967 * Invalidate the 2MB page mapping and return "failure" if the
4968 * mapping was never accessed or the allocation of the new
4969 * page table page fails. If the 2MB page mapping belongs to
4970 * the direct map region of the kernel's address space, then
4971 * the page allocation request specifies the highest possible
4972 * priority (VM_ALLOC_INTERRUPT). Otherwise, the priority is
4973 * normal. Page table pages are preallocated for every other
4974 * part of the kernel address space, so the direct map region
4975 * is the only part of the kernel address space that must be
4976 * handled here.
4977 */
4978 if ((oldpde & PG_A) == 0 || (mpte = vm_page_alloc_noobj(
4979 (va >= DMAP_MIN_ADDRESS && va < DMAP_MAX_ADDRESS ?
4980 VM_ALLOC_INTERRUPT : 0) | VM_ALLOC_WIRED)) == NULL) {
4981 SLIST_INIT(&free);
4982 sva = trunc_2mpage(va);
4983 pmap_remove_l3e(pmap, l3e, sva, &free, lockp);
4984 pmap_invalidate_l3e_page(pmap, sva, oldpde);
4985 vm_page_free_pages_toq(&free, true);
4986 CTR2(KTR_PMAP, "pmap_demote_l3e: failure for va %#lx"
4987 " in pmap %p", va, pmap);
4988 return (false);
4989 }
4990 mpte->pindex = pmap_l3e_pindex(va);
4991 if (va < VM_MAXUSER_ADDRESS)
4992 pmap_resident_count_inc(pmap, 1);
4993 }
4994 mptepa = VM_PAGE_TO_PHYS(mpte);
4995 firstpte = PHYS_TO_DMAP(mptepa);
4996 KASSERT((oldpde & PG_A) != 0,
4997 ("pmap_demote_l3e: oldpde is missing PG_A"));
4998 KASSERT((oldpde & (PG_M | PG_RW)) != PG_RW,
4999 ("pmap_demote_l3e: oldpde is missing PG_M"));
5000
5001 /*
5002 * If the page table page is new, initialize it.
5003 */
5004 if (mpte->ref_count == 1) {
5005 mpte->ref_count = NPTEPG;
5006 pmap_fill_ptp(firstpte, oldpde);
5007 }
5008
5009 KASSERT((be64toh(*firstpte) & PG_FRAME) == (oldpde & PG_FRAME),
5010 ("pmap_demote_l3e: firstpte and newpte map different physical"
5011 " addresses"));
5012
5013 /*
5014 * If the mapping has changed attributes, update the page table
5015 * entries.
5016 */
5017 if ((be64toh(*firstpte) & PG_PTE_PROMOTE) != (oldpde & PG_PTE_PROMOTE))
5018 pmap_fill_ptp(firstpte, oldpde);
5019
5020 /*
5021 * The spare PV entries must be reserved prior to demoting the
5022 * mapping, that is, prior to changing the PDE. Otherwise, the state
5023 * of the PDE and the PV lists will be inconsistent, which can result
5024 * in reclaim_pv_chunk() attempting to remove a PV entry from the
5025 * wrong PV list and pmap_pv_demote_l3e() failing to find the expected
5026 * PV entry for the 2MB page mapping that is being demoted.
5027 */
5028 if ((oldpde & PG_MANAGED) != 0)
5029 reserve_pv_entries(pmap, NPTEPG - 1, lockp);
5030
5031 /*
5032 * Demote the mapping. This pmap is locked. The old PDE has
5033 * PG_A set. If the old PDE has PG_RW set, it also has PG_M
5034 * set. Thus, there is no danger of a race with another
5035 * processor changing the setting of PG_A and/or PG_M between
5036 * the read above and the store below.
5037 */
5038 pde_store(l3e, mptepa);
5039 pmap_invalidate_l3e_page(pmap, trunc_2mpage(va), oldpde);
5040 /*
5041 * Demote the PV entry.
5042 */
5043 if ((oldpde & PG_MANAGED) != 0)
5044 pmap_pv_demote_l3e(pmap, va, oldpde & PG_PS_FRAME, lockp);
5045
5046 counter_u64_add(pmap_l3e_demotions, 1);
5047 CTR2(KTR_PMAP, "pmap_demote_l3e: success for va %#lx"
5048 " in pmap %p", va, pmap);
5049 return (true);
5050 }
5051
5052 /*
5053 * pmap_remove_kernel_pde: Remove a kernel superpage mapping.
5054 */
5055 static void
pmap_remove_kernel_l3e(pmap_t pmap,pml3_entry_t * l3e,vm_offset_t va)5056 pmap_remove_kernel_l3e(pmap_t pmap, pml3_entry_t *l3e, vm_offset_t va)
5057 {
5058 vm_paddr_t mptepa;
5059 vm_page_t mpte;
5060
5061 KASSERT(pmap == kernel_pmap, ("pmap %p is not kernel_pmap", pmap));
5062 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
5063 mpte = pmap_remove_pt_page(pmap, va);
5064 if (mpte == NULL)
5065 panic("pmap_remove_kernel_pde: Missing pt page.");
5066
5067 mptepa = VM_PAGE_TO_PHYS(mpte);
5068
5069 /*
5070 * Initialize the page table page.
5071 */
5072 pagezero(PHYS_TO_DMAP(mptepa));
5073
5074 /*
5075 * Demote the mapping.
5076 */
5077 pde_store(l3e, mptepa);
5078 ptesync();
5079 }
5080
5081 /*
5082 * pmap_remove_l3e: do the things to unmap a superpage in a process
5083 */
5084 static int
pmap_remove_l3e(pmap_t pmap,pml3_entry_t * pdq,vm_offset_t sva,struct spglist * free,struct rwlock ** lockp)5085 pmap_remove_l3e(pmap_t pmap, pml3_entry_t *pdq, vm_offset_t sva,
5086 struct spglist *free, struct rwlock **lockp)
5087 {
5088 struct md_page *pvh;
5089 pml3_entry_t oldpde;
5090 vm_offset_t eva, va;
5091 vm_page_t m, mpte;
5092
5093 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
5094 KASSERT((sva & L3_PAGE_MASK) == 0,
5095 ("pmap_remove_l3e: sva is not 2mpage aligned"));
5096 oldpde = be64toh(pte_load_clear(pdq));
5097 if (oldpde & PG_W)
5098 pmap->pm_stats.wired_count -= (L3_PAGE_SIZE / PAGE_SIZE);
5099 pmap_resident_count_dec(pmap, L3_PAGE_SIZE / PAGE_SIZE);
5100 if (oldpde & PG_MANAGED) {
5101 CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, oldpde & PG_PS_FRAME);
5102 pvh = pa_to_pvh(oldpde & PG_PS_FRAME);
5103 pmap_pvh_free(pvh, pmap, sva);
5104 eva = sva + L3_PAGE_SIZE;
5105 for (va = sva, m = PHYS_TO_VM_PAGE(oldpde & PG_PS_FRAME);
5106 va < eva; va += PAGE_SIZE, m++) {
5107 if ((oldpde & (PG_M | PG_RW)) == (PG_M | PG_RW))
5108 vm_page_dirty(m);
5109 if (oldpde & PG_A)
5110 vm_page_aflag_set(m, PGA_REFERENCED);
5111 if (TAILQ_EMPTY(&m->md.pv_list) &&
5112 TAILQ_EMPTY(&pvh->pv_list))
5113 vm_page_aflag_clear(m, PGA_WRITEABLE);
5114 }
5115 }
5116 if (pmap == kernel_pmap) {
5117 pmap_remove_kernel_l3e(pmap, pdq, sva);
5118 } else {
5119 mpte = pmap_remove_pt_page(pmap, sva);
5120 if (mpte != NULL) {
5121 pmap_resident_count_dec(pmap, 1);
5122 KASSERT(mpte->ref_count == NPTEPG,
5123 ("pmap_remove_l3e: pte page wire count error"));
5124 mpte->ref_count = 0;
5125 pmap_add_delayed_free_list(mpte, free, false);
5126 }
5127 }
5128 return (pmap_unuse_pt(pmap, sva, be64toh(*pmap_pml2e(pmap, sva)), free));
5129 }
5130
5131 /*
5132 * pmap_remove_pte: do the things to unmap a page in a process
5133 */
5134 static int
pmap_remove_pte(pmap_t pmap,pt_entry_t * ptq,vm_offset_t va,pml3_entry_t ptepde,struct spglist * free,struct rwlock ** lockp)5135 pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t va,
5136 pml3_entry_t ptepde, struct spglist *free, struct rwlock **lockp)
5137 {
5138 struct md_page *pvh;
5139 pt_entry_t oldpte;
5140 vm_page_t m;
5141
5142 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
5143 oldpte = be64toh(pte_load_clear(ptq));
5144 if (oldpte & RPTE_WIRED)
5145 pmap->pm_stats.wired_count -= 1;
5146 pmap_resident_count_dec(pmap, 1);
5147 if (oldpte & RPTE_MANAGED) {
5148 m = PHYS_TO_VM_PAGE(oldpte & PG_FRAME);
5149 if ((oldpte & (PG_M | PG_RW)) == (PG_M | PG_RW))
5150 vm_page_dirty(m);
5151 if (oldpte & PG_A)
5152 vm_page_aflag_set(m, PGA_REFERENCED);
5153 CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m);
5154 pmap_pvh_free(&m->md, pmap, va);
5155 if (TAILQ_EMPTY(&m->md.pv_list) &&
5156 (m->flags & PG_FICTITIOUS) == 0) {
5157 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
5158 if (TAILQ_EMPTY(&pvh->pv_list))
5159 vm_page_aflag_clear(m, PGA_WRITEABLE);
5160 }
5161 }
5162 return (pmap_unuse_pt(pmap, va, ptepde, free));
5163 }
5164
5165 /*
5166 * Remove a single page from a process address space
5167 */
5168 static bool
pmap_remove_page(pmap_t pmap,vm_offset_t va,pml3_entry_t * l3e,struct spglist * free)5169 pmap_remove_page(pmap_t pmap, vm_offset_t va, pml3_entry_t *l3e,
5170 struct spglist *free)
5171 {
5172 struct rwlock *lock;
5173 pt_entry_t *pte;
5174 bool invalidate_all;
5175
5176 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
5177 if ((be64toh(*l3e) & RPTE_VALID) == 0) {
5178 return (false);
5179 }
5180 pte = pmap_l3e_to_pte(l3e, va);
5181 if ((be64toh(*pte) & RPTE_VALID) == 0) {
5182 return (false);
5183 }
5184 lock = NULL;
5185
5186 invalidate_all = pmap_remove_pte(pmap, pte, va, be64toh(*l3e), free, &lock);
5187 if (lock != NULL)
5188 rw_wunlock(lock);
5189 if (!invalidate_all)
5190 pmap_invalidate_page(pmap, va);
5191 return (invalidate_all);
5192 }
5193
5194 /*
5195 * Removes the specified range of addresses from the page table page.
5196 */
5197 static bool
pmap_remove_ptes(pmap_t pmap,vm_offset_t sva,vm_offset_t eva,pml3_entry_t * l3e,struct spglist * free,struct rwlock ** lockp)5198 pmap_remove_ptes(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,
5199 pml3_entry_t *l3e, struct spglist *free, struct rwlock **lockp)
5200 {
5201 pt_entry_t *pte;
5202 vm_offset_t va;
5203 bool anyvalid;
5204
5205 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
5206 anyvalid = false;
5207 va = eva;
5208 for (pte = pmap_l3e_to_pte(l3e, sva); sva != eva; pte++,
5209 sva += PAGE_SIZE) {
5210 MPASS(pte == pmap_pte(pmap, sva));
5211 if (*pte == 0) {
5212 if (va != eva) {
5213 anyvalid = true;
5214 va = eva;
5215 }
5216 continue;
5217 }
5218 if (va == eva)
5219 va = sva;
5220 if (pmap_remove_pte(pmap, pte, sva, be64toh(*l3e), free, lockp)) {
5221 anyvalid = true;
5222 sva += PAGE_SIZE;
5223 break;
5224 }
5225 }
5226 if (anyvalid)
5227 pmap_invalidate_all(pmap);
5228 else if (va != eva)
5229 pmap_invalidate_range(pmap, va, sva);
5230 return (anyvalid);
5231 }
5232
5233 void
mmu_radix_remove(pmap_t pmap,vm_offset_t sva,vm_offset_t eva)5234 mmu_radix_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
5235 {
5236 struct rwlock *lock;
5237 vm_offset_t va_next;
5238 pml1_entry_t *l1e;
5239 pml2_entry_t *l2e;
5240 pml3_entry_t ptpaddr, *l3e;
5241 struct spglist free;
5242 bool anyvalid;
5243
5244 CTR4(KTR_PMAP, "%s(%p, %#x, %#x)", __func__, pmap, sva, eva);
5245
5246 /*
5247 * Perform an unsynchronized read. This is, however, safe.
5248 */
5249 if (pmap->pm_stats.resident_count == 0)
5250 return;
5251
5252 anyvalid = false;
5253 SLIST_INIT(&free);
5254
5255 /* XXX something fishy here */
5256 sva = (sva + PAGE_MASK) & ~PAGE_MASK;
5257 eva = (eva + PAGE_MASK) & ~PAGE_MASK;
5258
5259 PMAP_LOCK(pmap);
5260
5261 /*
5262 * special handling of removing one page. a very
5263 * common operation and easy to short circuit some
5264 * code.
5265 */
5266 if (sva + PAGE_SIZE == eva) {
5267 l3e = pmap_pml3e(pmap, sva);
5268 if (l3e && (be64toh(*l3e) & RPTE_LEAF) == 0) {
5269 anyvalid = pmap_remove_page(pmap, sva, l3e, &free);
5270 goto out;
5271 }
5272 }
5273
5274 lock = NULL;
5275 for (; sva < eva; sva = va_next) {
5276 if (pmap->pm_stats.resident_count == 0)
5277 break;
5278 l1e = pmap_pml1e(pmap, sva);
5279 if (l1e == NULL || (be64toh(*l1e) & PG_V) == 0) {
5280 va_next = (sva + L1_PAGE_SIZE) & ~L1_PAGE_MASK;
5281 if (va_next < sva)
5282 va_next = eva;
5283 continue;
5284 }
5285
5286 l2e = pmap_l1e_to_l2e(l1e, sva);
5287 if (l2e == NULL || (be64toh(*l2e) & PG_V) == 0) {
5288 va_next = (sva + L2_PAGE_SIZE) & ~L2_PAGE_MASK;
5289 if (va_next < sva)
5290 va_next = eva;
5291 continue;
5292 }
5293
5294 /*
5295 * Calculate index for next page table.
5296 */
5297 va_next = (sva + L3_PAGE_SIZE) & ~L3_PAGE_MASK;
5298 if (va_next < sva)
5299 va_next = eva;
5300
5301 l3e = pmap_l2e_to_l3e(l2e, sva);
5302 ptpaddr = be64toh(*l3e);
5303
5304 /*
5305 * Weed out invalid mappings.
5306 */
5307 if (ptpaddr == 0)
5308 continue;
5309
5310 /*
5311 * Check for large page.
5312 */
5313 if ((ptpaddr & RPTE_LEAF) != 0) {
5314 /*
5315 * Are we removing the entire large page? If not,
5316 * demote the mapping and fall through.
5317 */
5318 if (sva + L3_PAGE_SIZE == va_next && eva >= va_next) {
5319 pmap_remove_l3e(pmap, l3e, sva, &free, &lock);
5320 anyvalid = true;
5321 continue;
5322 } else if (!pmap_demote_l3e_locked(pmap, l3e, sva,
5323 &lock)) {
5324 /* The large page mapping was destroyed. */
5325 continue;
5326 } else
5327 ptpaddr = be64toh(*l3e);
5328 }
5329
5330 /*
5331 * Limit our scan to either the end of the va represented
5332 * by the current page table page, or to the end of the
5333 * range being removed.
5334 */
5335 if (va_next > eva)
5336 va_next = eva;
5337
5338 if (pmap_remove_ptes(pmap, sva, va_next, l3e, &free, &lock))
5339 anyvalid = true;
5340 }
5341 if (lock != NULL)
5342 rw_wunlock(lock);
5343 out:
5344 if (anyvalid)
5345 pmap_invalidate_all(pmap);
5346 PMAP_UNLOCK(pmap);
5347 vm_page_free_pages_toq(&free, true);
5348 }
5349
5350 void
mmu_radix_remove_all(vm_page_t m)5351 mmu_radix_remove_all(vm_page_t m)
5352 {
5353 struct md_page *pvh;
5354 pv_entry_t pv;
5355 pmap_t pmap;
5356 struct rwlock *lock;
5357 pt_entry_t *pte, tpte;
5358 pml3_entry_t *l3e;
5359 vm_offset_t va;
5360 struct spglist free;
5361 int pvh_gen, md_gen;
5362
5363 CTR2(KTR_PMAP, "%s(%p)", __func__, m);
5364 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
5365 ("pmap_remove_all: page %p is not managed", m));
5366 SLIST_INIT(&free);
5367 lock = VM_PAGE_TO_PV_LIST_LOCK(m);
5368 pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy :
5369 pa_to_pvh(VM_PAGE_TO_PHYS(m));
5370 retry:
5371 rw_wlock(lock);
5372 while ((pv = TAILQ_FIRST(&pvh->pv_list)) != NULL) {
5373 pmap = PV_PMAP(pv);
5374 if (!PMAP_TRYLOCK(pmap)) {
5375 pvh_gen = pvh->pv_gen;
5376 rw_wunlock(lock);
5377 PMAP_LOCK(pmap);
5378 rw_wlock(lock);
5379 if (pvh_gen != pvh->pv_gen) {
5380 rw_wunlock(lock);
5381 PMAP_UNLOCK(pmap);
5382 goto retry;
5383 }
5384 }
5385 va = pv->pv_va;
5386 l3e = pmap_pml3e(pmap, va);
5387 (void)pmap_demote_l3e_locked(pmap, l3e, va, &lock);
5388 PMAP_UNLOCK(pmap);
5389 }
5390 while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) {
5391 pmap = PV_PMAP(pv);
5392 if (!PMAP_TRYLOCK(pmap)) {
5393 pvh_gen = pvh->pv_gen;
5394 md_gen = m->md.pv_gen;
5395 rw_wunlock(lock);
5396 PMAP_LOCK(pmap);
5397 rw_wlock(lock);
5398 if (pvh_gen != pvh->pv_gen || md_gen != m->md.pv_gen) {
5399 rw_wunlock(lock);
5400 PMAP_UNLOCK(pmap);
5401 goto retry;
5402 }
5403 }
5404 pmap_resident_count_dec(pmap, 1);
5405 l3e = pmap_pml3e(pmap, pv->pv_va);
5406 KASSERT((be64toh(*l3e) & RPTE_LEAF) == 0, ("pmap_remove_all: found"
5407 " a 2mpage in page %p's pv list", m));
5408 pte = pmap_l3e_to_pte(l3e, pv->pv_va);
5409 tpte = be64toh(pte_load_clear(pte));
5410 if (tpte & PG_W)
5411 pmap->pm_stats.wired_count--;
5412 if (tpte & PG_A)
5413 vm_page_aflag_set(m, PGA_REFERENCED);
5414
5415 /*
5416 * Update the vm_page_t clean and reference bits.
5417 */
5418 if ((tpte & (PG_M | PG_RW)) == (PG_M | PG_RW))
5419 vm_page_dirty(m);
5420 pmap_unuse_pt(pmap, pv->pv_va, be64toh(*l3e), &free);
5421 pmap_invalidate_page(pmap, pv->pv_va);
5422 TAILQ_REMOVE(&m->md.pv_list, pv, pv_link);
5423 m->md.pv_gen++;
5424 free_pv_entry(pmap, pv);
5425 PMAP_UNLOCK(pmap);
5426 }
5427 vm_page_aflag_clear(m, PGA_WRITEABLE);
5428 rw_wunlock(lock);
5429 vm_page_free_pages_toq(&free, true);
5430 }
5431
5432 /*
5433 * Destroy all managed, non-wired mappings in the given user-space
5434 * pmap. This pmap cannot be active on any processor besides the
5435 * caller.
5436 *
5437 * This function cannot be applied to the kernel pmap. Moreover, it
5438 * is not intended for general use. It is only to be used during
5439 * process termination. Consequently, it can be implemented in ways
5440 * that make it faster than pmap_remove(). First, it can more quickly
5441 * destroy mappings by iterating over the pmap's collection of PV
5442 * entries, rather than searching the page table. Second, it doesn't
5443 * have to test and clear the page table entries atomically, because
5444 * no processor is currently accessing the user address space. In
5445 * particular, a page table entry's dirty bit won't change state once
5446 * this function starts.
5447 *
5448 * Although this function destroys all of the pmap's managed,
5449 * non-wired mappings, it can delay and batch the invalidation of TLB
5450 * entries without calling pmap_delayed_invl_started() and
5451 * pmap_delayed_invl_finished(). Because the pmap is not active on
5452 * any other processor, none of these TLB entries will ever be used
5453 * before their eventual invalidation. Consequently, there is no need
5454 * for either pmap_remove_all() or pmap_remove_write() to wait for
5455 * that eventual TLB invalidation.
5456 */
5457
5458 void
mmu_radix_remove_pages(pmap_t pmap)5459 mmu_radix_remove_pages(pmap_t pmap)
5460 {
5461
5462 CTR2(KTR_PMAP, "%s(%p)", __func__, pmap);
5463 pml3_entry_t ptel3e;
5464 pt_entry_t *pte, tpte;
5465 struct spglist free;
5466 vm_page_t m, mpte, mt;
5467 pv_entry_t pv;
5468 struct md_page *pvh;
5469 struct pv_chunk *pc, *npc;
5470 struct rwlock *lock;
5471 int64_t bit;
5472 uint64_t inuse, bitmask;
5473 int allfree, field, idx;
5474 #ifdef PV_STATS
5475 int freed;
5476 #endif
5477 bool superpage;
5478 vm_paddr_t pa;
5479
5480 /*
5481 * Assert that the given pmap is only active on the current
5482 * CPU. Unfortunately, we cannot block another CPU from
5483 * activating the pmap while this function is executing.
5484 */
5485 KASSERT(pmap->pm_pid == mfspr(SPR_PID),
5486 ("non-current asid %lu - expected %lu", pmap->pm_pid,
5487 mfspr(SPR_PID)));
5488
5489 lock = NULL;
5490
5491 SLIST_INIT(&free);
5492 PMAP_LOCK(pmap);
5493 TAILQ_FOREACH_SAFE(pc, &pmap->pm_pvchunk, pc_list, npc) {
5494 allfree = 1;
5495 #ifdef PV_STATS
5496 freed = 0;
5497 #endif
5498 for (field = 0; field < _NPCM; field++) {
5499 inuse = ~pc->pc_map[field] & pc_freemask[field];
5500 while (inuse != 0) {
5501 bit = cnttzd(inuse);
5502 bitmask = 1UL << bit;
5503 idx = field * 64 + bit;
5504 pv = &pc->pc_pventry[idx];
5505 inuse &= ~bitmask;
5506
5507 pte = pmap_pml2e(pmap, pv->pv_va);
5508 ptel3e = be64toh(*pte);
5509 pte = pmap_l2e_to_l3e(pte, pv->pv_va);
5510 tpte = be64toh(*pte);
5511 if ((tpte & (RPTE_LEAF | PG_V)) == PG_V) {
5512 superpage = false;
5513 ptel3e = tpte;
5514 pte = PHYS_TO_DMAP(tpte & PG_FRAME);
5515 pte = &pte[pmap_pte_index(pv->pv_va)];
5516 tpte = be64toh(*pte);
5517 } else {
5518 /*
5519 * Keep track whether 'tpte' is a
5520 * superpage explicitly instead of
5521 * relying on RPTE_LEAF being set.
5522 *
5523 * This is because RPTE_LEAF is numerically
5524 * identical to PG_PTE_PAT and thus a
5525 * regular page could be mistaken for
5526 * a superpage.
5527 */
5528 superpage = true;
5529 }
5530
5531 if ((tpte & PG_V) == 0) {
5532 panic("bad pte va %lx pte %lx",
5533 pv->pv_va, tpte);
5534 }
5535
5536 /*
5537 * We cannot remove wired pages from a process' mapping at this time
5538 */
5539 if (tpte & PG_W) {
5540 allfree = 0;
5541 continue;
5542 }
5543
5544 if (superpage)
5545 pa = tpte & PG_PS_FRAME;
5546 else
5547 pa = tpte & PG_FRAME;
5548
5549 m = PHYS_TO_VM_PAGE(pa);
5550 KASSERT(m->phys_addr == pa,
5551 ("vm_page_t %p phys_addr mismatch %016jx %016jx",
5552 m, (uintmax_t)m->phys_addr,
5553 (uintmax_t)tpte));
5554
5555 KASSERT((m->flags & PG_FICTITIOUS) != 0 ||
5556 m < &vm_page_array[vm_page_array_size],
5557 ("pmap_remove_pages: bad tpte %#jx",
5558 (uintmax_t)tpte));
5559
5560 pte_clear(pte);
5561
5562 /*
5563 * Update the vm_page_t clean/reference bits.
5564 */
5565 if ((tpte & (PG_M | PG_RW)) == (PG_M | PG_RW)) {
5566 if (superpage) {
5567 for (mt = m; mt < &m[L3_PAGE_SIZE / PAGE_SIZE]; mt++)
5568 vm_page_dirty(mt);
5569 } else
5570 vm_page_dirty(m);
5571 }
5572
5573 CHANGE_PV_LIST_LOCK_TO_VM_PAGE(&lock, m);
5574
5575 /* Mark free */
5576 pc->pc_map[field] |= bitmask;
5577 if (superpage) {
5578 pmap_resident_count_dec(pmap, L3_PAGE_SIZE / PAGE_SIZE);
5579 pvh = pa_to_pvh(tpte & PG_PS_FRAME);
5580 TAILQ_REMOVE(&pvh->pv_list, pv, pv_link);
5581 pvh->pv_gen++;
5582 if (TAILQ_EMPTY(&pvh->pv_list)) {
5583 for (mt = m; mt < &m[L3_PAGE_SIZE / PAGE_SIZE]; mt++)
5584 if ((mt->a.flags & PGA_WRITEABLE) != 0 &&
5585 TAILQ_EMPTY(&mt->md.pv_list))
5586 vm_page_aflag_clear(mt, PGA_WRITEABLE);
5587 }
5588 mpte = pmap_remove_pt_page(pmap, pv->pv_va);
5589 if (mpte != NULL) {
5590 pmap_resident_count_dec(pmap, 1);
5591 KASSERT(mpte->ref_count == NPTEPG,
5592 ("pmap_remove_pages: pte page wire count error"));
5593 mpte->ref_count = 0;
5594 pmap_add_delayed_free_list(mpte, &free, false);
5595 }
5596 } else {
5597 pmap_resident_count_dec(pmap, 1);
5598 #ifdef VERBOSE_PV
5599 printf("freeing pv (%p, %p)\n",
5600 pmap, pv);
5601 #endif
5602 TAILQ_REMOVE(&m->md.pv_list, pv, pv_link);
5603 m->md.pv_gen++;
5604 if ((m->a.flags & PGA_WRITEABLE) != 0 &&
5605 TAILQ_EMPTY(&m->md.pv_list) &&
5606 (m->flags & PG_FICTITIOUS) == 0) {
5607 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
5608 if (TAILQ_EMPTY(&pvh->pv_list))
5609 vm_page_aflag_clear(m, PGA_WRITEABLE);
5610 }
5611 }
5612 pmap_unuse_pt(pmap, pv->pv_va, ptel3e, &free);
5613 #ifdef PV_STATS
5614 freed++;
5615 #endif
5616 }
5617 }
5618 PV_STAT(atomic_add_long(&pv_entry_frees, freed));
5619 PV_STAT(atomic_add_int(&pv_entry_spare, freed));
5620 PV_STAT(atomic_subtract_long(&pv_entry_count, freed));
5621 if (allfree) {
5622 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
5623 free_pv_chunk(pc);
5624 }
5625 }
5626 if (lock != NULL)
5627 rw_wunlock(lock);
5628 pmap_invalidate_all(pmap);
5629 PMAP_UNLOCK(pmap);
5630 vm_page_free_pages_toq(&free, true);
5631 }
5632
5633 void
mmu_radix_remove_write(vm_page_t m)5634 mmu_radix_remove_write(vm_page_t m)
5635 {
5636 struct md_page *pvh;
5637 pmap_t pmap;
5638 struct rwlock *lock;
5639 pv_entry_t next_pv, pv;
5640 pml3_entry_t *l3e;
5641 pt_entry_t oldpte, *pte;
5642 int pvh_gen, md_gen;
5643
5644 CTR2(KTR_PMAP, "%s(%p)", __func__, m);
5645 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
5646 ("pmap_remove_write: page %p is not managed", m));
5647 vm_page_assert_busied(m);
5648
5649 if (!pmap_page_is_write_mapped(m))
5650 return;
5651 lock = VM_PAGE_TO_PV_LIST_LOCK(m);
5652 pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy :
5653 pa_to_pvh(VM_PAGE_TO_PHYS(m));
5654 retry_pv_loop:
5655 rw_wlock(lock);
5656 TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_link, next_pv) {
5657 pmap = PV_PMAP(pv);
5658 if (!PMAP_TRYLOCK(pmap)) {
5659 pvh_gen = pvh->pv_gen;
5660 rw_wunlock(lock);
5661 PMAP_LOCK(pmap);
5662 rw_wlock(lock);
5663 if (pvh_gen != pvh->pv_gen) {
5664 PMAP_UNLOCK(pmap);
5665 rw_wunlock(lock);
5666 goto retry_pv_loop;
5667 }
5668 }
5669 l3e = pmap_pml3e(pmap, pv->pv_va);
5670 if ((be64toh(*l3e) & PG_RW) != 0)
5671 (void)pmap_demote_l3e_locked(pmap, l3e, pv->pv_va, &lock);
5672 KASSERT(lock == VM_PAGE_TO_PV_LIST_LOCK(m),
5673 ("inconsistent pv lock %p %p for page %p",
5674 lock, VM_PAGE_TO_PV_LIST_LOCK(m), m));
5675 PMAP_UNLOCK(pmap);
5676 }
5677 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) {
5678 pmap = PV_PMAP(pv);
5679 if (!PMAP_TRYLOCK(pmap)) {
5680 pvh_gen = pvh->pv_gen;
5681 md_gen = m->md.pv_gen;
5682 rw_wunlock(lock);
5683 PMAP_LOCK(pmap);
5684 rw_wlock(lock);
5685 if (pvh_gen != pvh->pv_gen ||
5686 md_gen != m->md.pv_gen) {
5687 PMAP_UNLOCK(pmap);
5688 rw_wunlock(lock);
5689 goto retry_pv_loop;
5690 }
5691 }
5692 l3e = pmap_pml3e(pmap, pv->pv_va);
5693 KASSERT((be64toh(*l3e) & RPTE_LEAF) == 0,
5694 ("pmap_remove_write: found a 2mpage in page %p's pv list",
5695 m));
5696 pte = pmap_l3e_to_pte(l3e, pv->pv_va);
5697 retry:
5698 oldpte = be64toh(*pte);
5699 if (oldpte & PG_RW) {
5700 if (!atomic_cmpset_long(pte, htobe64(oldpte),
5701 htobe64((oldpte | RPTE_EAA_R) & ~(PG_RW | PG_M))))
5702 goto retry;
5703 if ((oldpte & PG_M) != 0)
5704 vm_page_dirty(m);
5705 pmap_invalidate_page(pmap, pv->pv_va);
5706 }
5707 PMAP_UNLOCK(pmap);
5708 }
5709 rw_wunlock(lock);
5710 vm_page_aflag_clear(m, PGA_WRITEABLE);
5711 }
5712
5713 /*
5714 * Clear the wired attribute from the mappings for the specified range of
5715 * addresses in the given pmap. Every valid mapping within that range
5716 * must have the wired attribute set. In contrast, invalid mappings
5717 * cannot have the wired attribute set, so they are ignored.
5718 *
5719 * The wired attribute of the page table entry is not a hardware
5720 * feature, so there is no need to invalidate any TLB entries.
5721 * Since pmap_demote_l3e() for the wired entry must never fail,
5722 * pmap_delayed_invl_started()/finished() calls around the
5723 * function are not needed.
5724 */
5725 void
mmu_radix_unwire(pmap_t pmap,vm_offset_t sva,vm_offset_t eva)5726 mmu_radix_unwire(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
5727 {
5728 vm_offset_t va_next;
5729 pml1_entry_t *l1e;
5730 pml2_entry_t *l2e;
5731 pml3_entry_t *l3e;
5732 pt_entry_t *pte;
5733
5734 CTR4(KTR_PMAP, "%s(%p, %#x, %#x)", __func__, pmap, sva, eva);
5735 PMAP_LOCK(pmap);
5736 for (; sva < eva; sva = va_next) {
5737 l1e = pmap_pml1e(pmap, sva);
5738 if ((be64toh(*l1e) & PG_V) == 0) {
5739 va_next = (sva + L1_PAGE_SIZE) & ~L1_PAGE_MASK;
5740 if (va_next < sva)
5741 va_next = eva;
5742 continue;
5743 }
5744 l2e = pmap_l1e_to_l2e(l1e, sva);
5745 if ((be64toh(*l2e) & PG_V) == 0) {
5746 va_next = (sva + L2_PAGE_SIZE) & ~L2_PAGE_MASK;
5747 if (va_next < sva)
5748 va_next = eva;
5749 continue;
5750 }
5751 va_next = (sva + L3_PAGE_SIZE) & ~L3_PAGE_MASK;
5752 if (va_next < sva)
5753 va_next = eva;
5754 l3e = pmap_l2e_to_l3e(l2e, sva);
5755 if ((be64toh(*l3e) & PG_V) == 0)
5756 continue;
5757 if ((be64toh(*l3e) & RPTE_LEAF) != 0) {
5758 if ((be64toh(*l3e) & PG_W) == 0)
5759 panic("pmap_unwire: pde %#jx is missing PG_W",
5760 (uintmax_t)(be64toh(*l3e)));
5761
5762 /*
5763 * Are we unwiring the entire large page? If not,
5764 * demote the mapping and fall through.
5765 */
5766 if (sva + L3_PAGE_SIZE == va_next && eva >= va_next) {
5767 atomic_clear_long(l3e, htobe64(PG_W));
5768 pmap->pm_stats.wired_count -= L3_PAGE_SIZE /
5769 PAGE_SIZE;
5770 continue;
5771 } else if (!pmap_demote_l3e(pmap, l3e, sva))
5772 panic("pmap_unwire: demotion failed");
5773 }
5774 if (va_next > eva)
5775 va_next = eva;
5776 for (pte = pmap_l3e_to_pte(l3e, sva); sva != va_next; pte++,
5777 sva += PAGE_SIZE) {
5778 MPASS(pte == pmap_pte(pmap, sva));
5779 if ((be64toh(*pte) & PG_V) == 0)
5780 continue;
5781 if ((be64toh(*pte) & PG_W) == 0)
5782 panic("pmap_unwire: pte %#jx is missing PG_W",
5783 (uintmax_t)(be64toh(*pte)));
5784
5785 /*
5786 * PG_W must be cleared atomically. Although the pmap
5787 * lock synchronizes access to PG_W, another processor
5788 * could be setting PG_M and/or PG_A concurrently.
5789 */
5790 atomic_clear_long(pte, htobe64(PG_W));
5791 pmap->pm_stats.wired_count--;
5792 }
5793 }
5794 PMAP_UNLOCK(pmap);
5795 }
5796
5797 void
mmu_radix_zero_page(vm_page_t m)5798 mmu_radix_zero_page(vm_page_t m)
5799 {
5800 void *addr;
5801
5802 CTR2(KTR_PMAP, "%s(%p)", __func__, m);
5803 addr = VM_PAGE_TO_DMAP(m);
5804 if (__predict_true(m->md.mdpg_cache_attrs == VM_MEMATTR_DEFAULT))
5805 pagezero(addr);
5806 else
5807 bzero(addr, PAGE_SIZE);
5808 }
5809
5810 void
mmu_radix_zero_page_area(vm_page_t m,int off,int size)5811 mmu_radix_zero_page_area(vm_page_t m, int off, int size)
5812 {
5813 caddr_t addr;
5814
5815 CTR4(KTR_PMAP, "%s(%p, %d, %d)", __func__, m, off, size);
5816 MPASS(off + size <= PAGE_SIZE);
5817 addr = VM_PAGE_TO_DMAP(m);
5818 memset(addr + off, 0, size);
5819 }
5820
5821 static int
mmu_radix_mincore(pmap_t pmap,vm_offset_t addr,vm_paddr_t * locked_pa)5822 mmu_radix_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *locked_pa)
5823 {
5824 pml3_entry_t *l3ep;
5825 pt_entry_t pte;
5826 vm_paddr_t pa;
5827 int val;
5828
5829 CTR3(KTR_PMAP, "%s(%p, %#x)", __func__, pmap, addr);
5830 PMAP_LOCK(pmap);
5831
5832 l3ep = pmap_pml3e(pmap, addr);
5833 if (l3ep != NULL && (be64toh(*l3ep) & PG_V)) {
5834 if (be64toh(*l3ep) & RPTE_LEAF) {
5835 pte = be64toh(*l3ep);
5836 /* Compute the physical address of the 4KB page. */
5837 pa = ((be64toh(*l3ep) & PG_PS_FRAME) | (addr & L3_PAGE_MASK)) &
5838 PG_FRAME;
5839 val = MINCORE_PSIND(1);
5840 } else {
5841 /* Native endian PTE, do not pass to functions */
5842 pte = be64toh(*pmap_l3e_to_pte(l3ep, addr));
5843 pa = pte & PG_FRAME;
5844 val = 0;
5845 }
5846 } else {
5847 pte = 0;
5848 pa = 0;
5849 val = 0;
5850 }
5851 if ((pte & PG_V) != 0) {
5852 val |= MINCORE_INCORE;
5853 if ((pte & (PG_M | PG_RW)) == (PG_M | PG_RW))
5854 val |= MINCORE_MODIFIED | MINCORE_MODIFIED_OTHER;
5855 if ((pte & PG_A) != 0)
5856 val |= MINCORE_REFERENCED | MINCORE_REFERENCED_OTHER;
5857 }
5858 if ((val & (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER)) !=
5859 (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER) &&
5860 (pte & (PG_MANAGED | PG_V)) == (PG_MANAGED | PG_V)) {
5861 *locked_pa = pa;
5862 }
5863 PMAP_UNLOCK(pmap);
5864 return (val);
5865 }
5866
5867 void
mmu_radix_activate(struct thread * td)5868 mmu_radix_activate(struct thread *td)
5869 {
5870 pmap_t pmap;
5871 uint32_t curpid;
5872
5873 CTR2(KTR_PMAP, "%s(%p)", __func__, td);
5874 critical_enter();
5875 pmap = vmspace_pmap(td->td_proc->p_vmspace);
5876 curpid = mfspr(SPR_PID);
5877 if (pmap->pm_pid > isa3_base_pid &&
5878 curpid != pmap->pm_pid) {
5879 mmu_radix_pid_set(pmap);
5880 }
5881 critical_exit();
5882 }
5883
5884 /*
5885 * Increase the starting virtual address of the given mapping if a
5886 * different alignment might result in more superpage mappings.
5887 */
5888 void
mmu_radix_align_superpage(vm_object_t object,vm_ooffset_t offset,vm_offset_t * addr,vm_size_t size)5889 mmu_radix_align_superpage(vm_object_t object, vm_ooffset_t offset,
5890 vm_offset_t *addr, vm_size_t size)
5891 {
5892
5893 CTR5(KTR_PMAP, "%s(%p, %#x, %p, %#x)", __func__, object, offset, addr,
5894 size);
5895 vm_offset_t superpage_offset;
5896
5897 if (size < L3_PAGE_SIZE)
5898 return;
5899 if (object != NULL && (object->flags & OBJ_COLORED) != 0)
5900 offset += ptoa(object->pg_color);
5901 superpage_offset = offset & L3_PAGE_MASK;
5902 if (size - ((L3_PAGE_SIZE - superpage_offset) & L3_PAGE_MASK) < L3_PAGE_SIZE ||
5903 (*addr & L3_PAGE_MASK) == superpage_offset)
5904 return;
5905 if ((*addr & L3_PAGE_MASK) < superpage_offset)
5906 *addr = (*addr & ~L3_PAGE_MASK) + superpage_offset;
5907 else
5908 *addr = ((*addr + L3_PAGE_MASK) & ~L3_PAGE_MASK) + superpage_offset;
5909 }
5910
5911 static void *
mmu_radix_mapdev_attr(vm_paddr_t pa,vm_size_t size,vm_memattr_t attr)5912 mmu_radix_mapdev_attr(vm_paddr_t pa, vm_size_t size, vm_memattr_t attr)
5913 {
5914 char *va;
5915 vm_offset_t tmpva, ppa, offset;
5916
5917 ppa = trunc_page(pa);
5918 offset = pa & PAGE_MASK;
5919 size = roundup2(offset + size, PAGE_SIZE);
5920 if (pa < powerpc_ptob(Maxmem))
5921 panic("bad pa: %#lx less than Maxmem %#lx\n",
5922 pa, powerpc_ptob(Maxmem));
5923 va = kva_alloc(size);
5924 if (bootverbose)
5925 printf("%s(%#lx, %lu, %d)\n", __func__, pa, size, attr);
5926 KASSERT(size > 0, ("%s(%#lx, %lu, %d)", __func__, pa, size, attr));
5927
5928 if (va == NULL)
5929 panic("%s: Couldn't alloc kernel virtual memory", __func__);
5930
5931 for (tmpva = (vm_offset_t)va; size > 0;) {
5932 mmu_radix_kenter_attr(tmpva, ppa, attr);
5933 size -= PAGE_SIZE;
5934 tmpva += PAGE_SIZE;
5935 ppa += PAGE_SIZE;
5936 }
5937 ptesync();
5938
5939 return (va + offset);
5940 }
5941
5942 static void *
mmu_radix_mapdev(vm_paddr_t pa,vm_size_t size)5943 mmu_radix_mapdev(vm_paddr_t pa, vm_size_t size)
5944 {
5945
5946 CTR3(KTR_PMAP, "%s(%#x, %#x)", __func__, pa, size);
5947
5948 return (mmu_radix_mapdev_attr(pa, size, VM_MEMATTR_DEFAULT));
5949 }
5950
5951 void
mmu_radix_page_set_memattr(vm_page_t m,vm_memattr_t ma)5952 mmu_radix_page_set_memattr(vm_page_t m, vm_memattr_t ma)
5953 {
5954
5955 CTR3(KTR_PMAP, "%s(%p, %#x)", __func__, m, ma);
5956
5957 if (m->md.mdpg_cache_attrs == ma)
5958 return;
5959
5960 m->md.mdpg_cache_attrs = ma;
5961
5962 /*
5963 * If "m" is a normal page, update its direct mapping. This update
5964 * can be relied upon to perform any cache operations that are
5965 * required for data coherence.
5966 */
5967 if ((m->flags & PG_FICTITIOUS) == 0 &&
5968 mmu_radix_change_attr(VM_PAGE_TO_DMAP(m), PAGE_SIZE,
5969 m->md.mdpg_cache_attrs))
5970 panic("memory attribute change on the direct map failed");
5971 }
5972
5973 static void
mmu_radix_unmapdev(void * va,vm_size_t size)5974 mmu_radix_unmapdev(void *va, vm_size_t size)
5975 {
5976 vm_offset_t offset;
5977
5978 CTR3(KTR_PMAP, "%s(%p, %#x)", __func__, va, size);
5979
5980 /* If we gave a direct map region in pmap_mapdev, do nothing */
5981 if ((vm_offset_t)va >= DMAP_MIN_ADDRESS &&
5982 (vm_offset_t)va < DMAP_MAX_ADDRESS)
5983 return;
5984
5985 offset = (vm_offset_t)va & PAGE_MASK;
5986 size = round_page(offset + size);
5987 va = trunc_page(va);
5988
5989 if (pmap_initialized) {
5990 mmu_radix_qremove(va, atop(size));
5991 kva_free(va, size);
5992 }
5993 }
5994
5995 void
mmu_radix_sync_icache(pmap_t pm,vm_offset_t va,vm_size_t sz)5996 mmu_radix_sync_icache(pmap_t pm, vm_offset_t va, vm_size_t sz)
5997 {
5998 vm_paddr_t pa = 0;
5999 int sync_sz;
6000
6001 if (__predict_false(pm == NULL))
6002 pm = &curthread->td_proc->p_vmspace->vm_pmap;
6003
6004 PMAP_LOCK(pm);
6005 while (sz > 0) {
6006 pa = mmu_radix_extract_locked(pm, va);
6007 sync_sz = PAGE_SIZE - (va & PAGE_MASK);
6008 sync_sz = min(sync_sz, sz);
6009 if (pa != 0)
6010 __syncicache(PHYS_TO_DMAP(pa), sync_sz);
6011 va += sync_sz;
6012 sz -= sync_sz;
6013 }
6014 PMAP_UNLOCK(pm);
6015 }
6016
6017 static __inline void
pmap_pte_attr(pt_entry_t * pte,uint64_t cache_bits,uint64_t mask)6018 pmap_pte_attr(pt_entry_t *pte, uint64_t cache_bits, uint64_t mask)
6019 {
6020 uint64_t opte, npte;
6021
6022 /*
6023 * The cache mode bits are all in the low 32-bits of the
6024 * PTE, so we can just spin on updating the low 32-bits.
6025 */
6026 do {
6027 opte = be64toh(*pte);
6028 npte = opte & ~mask;
6029 npte |= cache_bits;
6030 } while (npte != opte && !atomic_cmpset_long(pte, htobe64(opte), htobe64(npte)));
6031 }
6032
6033 /*
6034 * Tries to demote a 1GB page mapping.
6035 */
6036 static bool
pmap_demote_l2e(pmap_t pmap,pml2_entry_t * l2e,vm_offset_t va)6037 pmap_demote_l2e(pmap_t pmap, pml2_entry_t *l2e, vm_offset_t va)
6038 {
6039 pml2_entry_t oldpdpe;
6040 pml3_entry_t *firstpde, newpde, *pde;
6041 vm_paddr_t pdpgpa;
6042 vm_page_t pdpg;
6043
6044 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
6045 oldpdpe = be64toh(*l2e);
6046 KASSERT((oldpdpe & (RPTE_LEAF | PG_V)) == (RPTE_LEAF | PG_V),
6047 ("pmap_demote_pdpe: oldpdpe is missing PG_PS and/or PG_V"));
6048 pdpg = vm_page_alloc_noobj(VM_ALLOC_INTERRUPT | VM_ALLOC_WIRED);
6049 if (pdpg == NULL) {
6050 CTR2(KTR_PMAP, "pmap_demote_pdpe: failure for va %#lx"
6051 " in pmap %p", va, pmap);
6052 return (false);
6053 }
6054 pdpg->pindex = va >> L2_PAGE_SIZE_SHIFT;
6055 pdpgpa = VM_PAGE_TO_PHYS(pdpg);
6056 firstpde = PHYS_TO_DMAP(pdpgpa);
6057 KASSERT((oldpdpe & PG_A) != 0,
6058 ("pmap_demote_pdpe: oldpdpe is missing PG_A"));
6059 KASSERT((oldpdpe & (PG_M | PG_RW)) != PG_RW,
6060 ("pmap_demote_pdpe: oldpdpe is missing PG_M"));
6061 newpde = oldpdpe;
6062
6063 /*
6064 * Initialize the page directory page.
6065 */
6066 for (pde = firstpde; pde < firstpde + NPDEPG; pde++) {
6067 *pde = htobe64(newpde);
6068 newpde += L3_PAGE_SIZE;
6069 }
6070
6071 /*
6072 * Demote the mapping.
6073 */
6074 pde_store(l2e, pdpgpa);
6075
6076 /*
6077 * Flush PWC --- XXX revisit
6078 */
6079 pmap_invalidate_all(pmap);
6080
6081 counter_u64_add(pmap_l2e_demotions, 1);
6082 CTR2(KTR_PMAP, "pmap_demote_pdpe: success for va %#lx"
6083 " in pmap %p", va, pmap);
6084 return (true);
6085 }
6086
6087 vm_paddr_t
mmu_radix_kextract(vm_offset_t va)6088 mmu_radix_kextract(vm_offset_t va)
6089 {
6090 pml3_entry_t l3e;
6091 vm_paddr_t pa;
6092
6093 CTR2(KTR_PMAP, "%s(%#x)", __func__, va);
6094 if (va >= DMAP_MIN_ADDRESS && va < DMAP_MAX_ADDRESS) {
6095 pa = DMAP_TO_PHYS(va);
6096 } else {
6097 /* Big-endian PTE on stack */
6098 l3e = *pmap_pml3e(kernel_pmap, va);
6099 if (be64toh(l3e) & RPTE_LEAF) {
6100 pa = (be64toh(l3e) & PG_PS_FRAME) | (va & L3_PAGE_MASK);
6101 pa |= (va & L3_PAGE_MASK);
6102 } else {
6103 /*
6104 * Beware of a concurrent promotion that changes the
6105 * PDE at this point! For example, vtopte() must not
6106 * be used to access the PTE because it would use the
6107 * new PDE. It is, however, safe to use the old PDE
6108 * because the page table page is preserved by the
6109 * promotion.
6110 */
6111 pa = be64toh(*pmap_l3e_to_pte(&l3e, va));
6112 pa = (pa & PG_FRAME) | (va & PAGE_MASK);
6113 pa |= (va & PAGE_MASK);
6114 }
6115 }
6116 return (pa);
6117 }
6118
6119 static pt_entry_t
mmu_radix_calc_wimg(vm_paddr_t pa,vm_memattr_t ma)6120 mmu_radix_calc_wimg(vm_paddr_t pa, vm_memattr_t ma)
6121 {
6122
6123 if (ma != VM_MEMATTR_DEFAULT) {
6124 return pmap_cache_bits(ma);
6125 }
6126
6127 /*
6128 * Assume the page is cache inhibited and access is guarded unless
6129 * it's in our available memory array.
6130 */
6131 for (int i = 0; i < pregions_sz; i++) {
6132 if ((pa >= pregions[i].mr_start) &&
6133 (pa < (pregions[i].mr_start + pregions[i].mr_size)))
6134 return (RPTE_ATTR_MEM);
6135 }
6136 return (RPTE_ATTR_GUARDEDIO);
6137 }
6138
6139 static void
mmu_radix_kenter_attr(vm_offset_t va,vm_paddr_t pa,vm_memattr_t ma)6140 mmu_radix_kenter_attr(vm_offset_t va, vm_paddr_t pa, vm_memattr_t ma)
6141 {
6142 pt_entry_t *pte, pteval;
6143 uint64_t cache_bits;
6144
6145 pte = kvtopte(va);
6146 MPASS(pte != NULL);
6147 pteval = pa | RPTE_EAA_R | RPTE_EAA_W | RPTE_EAA_P | PG_M | PG_A;
6148 cache_bits = mmu_radix_calc_wimg(pa, ma);
6149 pte_store(pte, pteval | cache_bits);
6150 }
6151
6152 void
mmu_radix_kremove(vm_offset_t va)6153 mmu_radix_kremove(vm_offset_t va)
6154 {
6155 pt_entry_t *pte;
6156
6157 CTR2(KTR_PMAP, "%s(%#x)", __func__, va);
6158
6159 pte = kvtopte(va);
6160 pte_clear(pte);
6161 }
6162
6163 int
mmu_radix_decode_kernel_ptr(vm_offset_t addr,int * is_user,vm_offset_t * decoded)6164 mmu_radix_decode_kernel_ptr(vm_offset_t addr,
6165 int *is_user, vm_offset_t *decoded)
6166 {
6167
6168 CTR2(KTR_PMAP, "%s(%#jx)", __func__, (uintmax_t)addr);
6169 *decoded = addr;
6170 *is_user = (addr < VM_MAXUSER_ADDRESS);
6171 return (0);
6172 }
6173
6174 static int
mmu_radix_dev_direct_mapped(vm_paddr_t pa,vm_size_t size)6175 mmu_radix_dev_direct_mapped(vm_paddr_t pa, vm_size_t size)
6176 {
6177
6178 CTR3(KTR_PMAP, "%s(%#x, %#x)", __func__, pa, size);
6179 return (mem_valid(pa, size));
6180 }
6181
6182 static void
mmu_radix_scan_init(void)6183 mmu_radix_scan_init(void)
6184 {
6185
6186 CTR1(KTR_PMAP, "%s()", __func__);
6187 UNIMPLEMENTED();
6188 }
6189
6190 static void
mmu_radix_dumpsys_map(vm_paddr_t pa,size_t sz,void ** va)6191 mmu_radix_dumpsys_map(vm_paddr_t pa, size_t sz,
6192 void **va)
6193 {
6194 CTR4(KTR_PMAP, "%s(%#jx, %#zx, %p)", __func__, (uintmax_t)pa, sz, va);
6195 UNIMPLEMENTED();
6196 }
6197
6198 void *
mmu_radix_quick_enter_page(vm_page_t m)6199 mmu_radix_quick_enter_page(vm_page_t m)
6200 {
6201 CTR2(KTR_PMAP, "%s(%p)", __func__, m);
6202 return (VM_PAGE_TO_DMAP(m));
6203 }
6204
6205 void
mmu_radix_quick_remove_page(void * addr __unused)6206 mmu_radix_quick_remove_page(void *addr __unused)
6207 {
6208 /* no work to do here */
6209 CTR2(KTR_PMAP, "%s(%p)", __func__, addr);
6210 }
6211
6212 static void
pmap_invalidate_cache_range(vm_offset_t sva,vm_offset_t eva)6213 pmap_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva)
6214 {
6215 cpu_flush_dcache((void *)sva, eva - sva);
6216 }
6217
6218 int
mmu_radix_change_attr(void * va,vm_size_t size,vm_memattr_t mode)6219 mmu_radix_change_attr(void *va, vm_size_t size, vm_memattr_t mode)
6220 {
6221 int error;
6222
6223 CTR4(KTR_PMAP, "%s(%p, %#zx, %d)", __func__, va, size, mode);
6224 PMAP_LOCK(kernel_pmap);
6225 error = pmap_change_attr_locked(va, size, mode, true);
6226 PMAP_UNLOCK(kernel_pmap);
6227 return (error);
6228 }
6229
6230 static int
pmap_change_attr_locked(void * addr,vm_size_t size,int mode,bool flush)6231 pmap_change_attr_locked(void *addr, vm_size_t size, int mode, bool flush)
6232 {
6233 vm_offset_t base, offset, tmpva, va;
6234 vm_paddr_t pa_start, pa_end, pa_end1;
6235 pml2_entry_t *l2e;
6236 pml3_entry_t *l3e;
6237 pt_entry_t *pte;
6238 int cache_bits, error;
6239 bool changed;
6240
6241 PMAP_LOCK_ASSERT(kernel_pmap, MA_OWNED);
6242 va = (vm_offset_t)addr;
6243 base = trunc_page(va);
6244 offset = va & PAGE_MASK;
6245 size = round_page(offset + size);
6246
6247 /*
6248 * Only supported on kernel virtual addresses, including the direct
6249 * map but excluding the recursive map.
6250 */
6251 if (base < DMAP_MIN_ADDRESS)
6252 return (EINVAL);
6253
6254 cache_bits = pmap_cache_bits(mode);
6255 changed = false;
6256
6257 /*
6258 * Pages that aren't mapped aren't supported. Also break down 2MB pages
6259 * into 4KB pages if required.
6260 */
6261 for (tmpva = base; tmpva < base + size; ) {
6262 l2e = pmap_pml2e(kernel_pmap, tmpva);
6263 if (l2e == NULL || *l2e == 0)
6264 return (EINVAL);
6265 if (be64toh(*l2e) & RPTE_LEAF) {
6266 /*
6267 * If the current 1GB page already has the required
6268 * memory type, then we need not demote this page. Just
6269 * increment tmpva to the next 1GB page frame.
6270 */
6271 if ((be64toh(*l2e) & RPTE_ATTR_MASK) == cache_bits) {
6272 tmpva = trunc_1gpage(tmpva) + L2_PAGE_SIZE;
6273 continue;
6274 }
6275
6276 /*
6277 * If the current offset aligns with a 1GB page frame
6278 * and there is at least 1GB left within the range, then
6279 * we need not break down this page into 2MB pages.
6280 */
6281 if ((tmpva & L2_PAGE_MASK) == 0 &&
6282 tmpva + L2_PAGE_MASK < base + size) {
6283 tmpva += L2_PAGE_MASK;
6284 continue;
6285 }
6286 if (!pmap_demote_l2e(kernel_pmap, l2e, tmpva))
6287 return (ENOMEM);
6288 }
6289 l3e = pmap_l2e_to_l3e(l2e, tmpva);
6290 KASSERT(l3e != NULL, ("no l3e entry for %#lx in %p\n",
6291 tmpva, l2e));
6292 if (*l3e == 0)
6293 return (EINVAL);
6294 if (be64toh(*l3e) & RPTE_LEAF) {
6295 /*
6296 * If the current 2MB page already has the required
6297 * memory type, then we need not demote this page. Just
6298 * increment tmpva to the next 2MB page frame.
6299 */
6300 if ((be64toh(*l3e) & RPTE_ATTR_MASK) == cache_bits) {
6301 tmpva = trunc_2mpage(tmpva) + L3_PAGE_SIZE;
6302 continue;
6303 }
6304
6305 /*
6306 * If the current offset aligns with a 2MB page frame
6307 * and there is at least 2MB left within the range, then
6308 * we need not break down this page into 4KB pages.
6309 */
6310 if ((tmpva & L3_PAGE_MASK) == 0 &&
6311 tmpva + L3_PAGE_MASK < base + size) {
6312 tmpva += L3_PAGE_SIZE;
6313 continue;
6314 }
6315 if (!pmap_demote_l3e(kernel_pmap, l3e, tmpva))
6316 return (ENOMEM);
6317 }
6318 pte = pmap_l3e_to_pte(l3e, tmpva);
6319 if (*pte == 0)
6320 return (EINVAL);
6321 tmpva += PAGE_SIZE;
6322 }
6323 error = 0;
6324
6325 /*
6326 * Ok, all the pages exist, so run through them updating their
6327 * cache mode if required.
6328 */
6329 pa_start = pa_end = 0;
6330 for (tmpva = base; tmpva < base + size; ) {
6331 l2e = pmap_pml2e(kernel_pmap, tmpva);
6332 if (be64toh(*l2e) & RPTE_LEAF) {
6333 if ((be64toh(*l2e) & RPTE_ATTR_MASK) != cache_bits) {
6334 pmap_pte_attr(l2e, cache_bits,
6335 RPTE_ATTR_MASK);
6336 changed = true;
6337 }
6338 if (tmpva >= VM_MIN_KERNEL_ADDRESS &&
6339 (*l2e & PG_PS_FRAME) < dmaplimit) {
6340 if (pa_start == pa_end) {
6341 /* Start physical address run. */
6342 pa_start = be64toh(*l2e) & PG_PS_FRAME;
6343 pa_end = pa_start + L2_PAGE_SIZE;
6344 } else if (pa_end == (be64toh(*l2e) & PG_PS_FRAME))
6345 pa_end += L2_PAGE_SIZE;
6346 else {
6347 /* Run ended, update direct map. */
6348 error = pmap_change_attr_locked(
6349 PHYS_TO_DMAP(pa_start),
6350 pa_end - pa_start, mode, flush);
6351 if (error != 0)
6352 break;
6353 /* Start physical address run. */
6354 pa_start = be64toh(*l2e) & PG_PS_FRAME;
6355 pa_end = pa_start + L2_PAGE_SIZE;
6356 }
6357 }
6358 tmpva = trunc_1gpage(tmpva) + L2_PAGE_SIZE;
6359 continue;
6360 }
6361 l3e = pmap_l2e_to_l3e(l2e, tmpva);
6362 if (be64toh(*l3e) & RPTE_LEAF) {
6363 if ((be64toh(*l3e) & RPTE_ATTR_MASK) != cache_bits) {
6364 pmap_pte_attr(l3e, cache_bits,
6365 RPTE_ATTR_MASK);
6366 changed = true;
6367 }
6368 if (tmpva >= VM_MIN_KERNEL_ADDRESS &&
6369 (be64toh(*l3e) & PG_PS_FRAME) < dmaplimit) {
6370 if (pa_start == pa_end) {
6371 /* Start physical address run. */
6372 pa_start = be64toh(*l3e) & PG_PS_FRAME;
6373 pa_end = pa_start + L3_PAGE_SIZE;
6374 } else if (pa_end == (be64toh(*l3e) & PG_PS_FRAME))
6375 pa_end += L3_PAGE_SIZE;
6376 else {
6377 /* Run ended, update direct map. */
6378 error = pmap_change_attr_locked(
6379 PHYS_TO_DMAP(pa_start),
6380 pa_end - pa_start, mode, flush);
6381 if (error != 0)
6382 break;
6383 /* Start physical address run. */
6384 pa_start = be64toh(*l3e) & PG_PS_FRAME;
6385 pa_end = pa_start + L3_PAGE_SIZE;
6386 }
6387 }
6388 tmpva = trunc_2mpage(tmpva) + L3_PAGE_SIZE;
6389 } else {
6390 pte = pmap_l3e_to_pte(l3e, tmpva);
6391 if ((be64toh(*pte) & RPTE_ATTR_MASK) != cache_bits) {
6392 pmap_pte_attr(pte, cache_bits,
6393 RPTE_ATTR_MASK);
6394 changed = true;
6395 }
6396 if (tmpva >= VM_MIN_KERNEL_ADDRESS &&
6397 (be64toh(*pte) & PG_FRAME) < dmaplimit) {
6398 if (pa_start == pa_end) {
6399 /* Start physical address run. */
6400 pa_start = be64toh(*pte) & PG_FRAME;
6401 pa_end = pa_start + PAGE_SIZE;
6402 } else if (pa_end == (be64toh(*pte) & PG_FRAME))
6403 pa_end += PAGE_SIZE;
6404 else {
6405 /* Run ended, update direct map. */
6406 error = pmap_change_attr_locked(
6407 PHYS_TO_DMAP(pa_start),
6408 pa_end - pa_start, mode, flush);
6409 if (error != 0)
6410 break;
6411 /* Start physical address run. */
6412 pa_start = be64toh(*pte) & PG_FRAME;
6413 pa_end = pa_start + PAGE_SIZE;
6414 }
6415 }
6416 tmpva += PAGE_SIZE;
6417 }
6418 }
6419 if (error == 0 && pa_start != pa_end && pa_start < dmaplimit) {
6420 pa_end1 = MIN(pa_end, dmaplimit);
6421 if (pa_start != pa_end1)
6422 error = pmap_change_attr_locked(PHYS_TO_DMAP(pa_start),
6423 pa_end1 - pa_start, mode, flush);
6424 }
6425
6426 /*
6427 * Flush CPU caches if required to make sure any data isn't cached that
6428 * shouldn't be, etc.
6429 */
6430 if (changed) {
6431 pmap_invalidate_all(kernel_pmap);
6432
6433 if (flush)
6434 pmap_invalidate_cache_range(base, tmpva);
6435 }
6436 return (error);
6437 }
6438
6439 /*
6440 * Allocate physical memory for the vm_page array and map it into KVA,
6441 * attempting to back the vm_pages with domain-local memory.
6442 */
6443 void
mmu_radix_page_array_startup(long pages)6444 mmu_radix_page_array_startup(long pages)
6445 {
6446 #ifdef notyet
6447 pml2_entry_t *l2e;
6448 pml3_entry_t *pde;
6449 pml3_entry_t newl3;
6450 vm_offset_t va;
6451 long pfn;
6452 int domain, i;
6453 #endif
6454 vm_paddr_t pa;
6455 vm_offset_t start, end;
6456
6457 vm_page_array_size = pages;
6458
6459 start = VM_MIN_KERNEL_ADDRESS;
6460 end = start + pages * sizeof(struct vm_page);
6461
6462 pa = vm_phys_early_alloc(-1, end - start);
6463
6464 start = (vm_offset_t)mmu_radix_map(&start, pa, end - start,
6465 VM_MEMATTR_DEFAULT);
6466 #ifdef notyet
6467 /* TODO: NUMA vm_page_array. Blocked out until then (copied from amd64). */
6468 for (va = start; va < end; va += L3_PAGE_SIZE) {
6469 pfn = first_page + (va - start) / sizeof(struct vm_page);
6470 domain = vm_phys_domain(ptoa(pfn));
6471 l2e = pmap_pml2e(kernel_pmap, va);
6472 if ((be64toh(*l2e) & PG_V) == 0) {
6473 pa = vm_phys_early_alloc(domain, PAGE_SIZE);
6474 dump_add_page(pa);
6475 pagezero(PHYS_TO_DMAP(pa));
6476 pde_store(l2e, (pml2_entry_t)pa);
6477 }
6478 pde = pmap_l2e_to_l3e(l2e, va);
6479 if ((be64toh(*pde) & PG_V) != 0)
6480 panic("Unexpected pde %p", pde);
6481 pa = vm_phys_early_alloc(domain, L3_PAGE_SIZE);
6482 for (i = 0; i < NPDEPG; i++)
6483 dump_add_page(pa + i * PAGE_SIZE);
6484 newl3 = (pml3_entry_t)(pa | RPTE_EAA_P | RPTE_EAA_R | RPTE_EAA_W);
6485 pte_store(pde, newl3);
6486 }
6487 #endif
6488 vm_page_array = (vm_page_t)start;
6489 }
6490
6491 #ifdef DDB
6492 #include <sys/kdb.h>
6493 #include <ddb/ddb.h>
6494
6495 static void
pmap_pte_walk(pml1_entry_t * l1,vm_offset_t va)6496 pmap_pte_walk(pml1_entry_t *l1, vm_offset_t va)
6497 {
6498 pml1_entry_t *l1e;
6499 pml2_entry_t *l2e;
6500 pml3_entry_t *l3e;
6501 pt_entry_t *pte;
6502
6503 l1e = &l1[pmap_pml1e_index(va)];
6504 db_printf("VA %#016lx l1e %#016lx", va, be64toh(*l1e));
6505 if ((be64toh(*l1e) & PG_V) == 0) {
6506 db_printf("\n");
6507 return;
6508 }
6509 l2e = pmap_l1e_to_l2e(l1e, va);
6510 db_printf(" l2e %#016lx", be64toh(*l2e));
6511 if ((be64toh(*l2e) & PG_V) == 0 || (be64toh(*l2e) & RPTE_LEAF) != 0) {
6512 db_printf("\n");
6513 return;
6514 }
6515 l3e = pmap_l2e_to_l3e(l2e, va);
6516 db_printf(" l3e %#016lx", be64toh(*l3e));
6517 if ((be64toh(*l3e) & PG_V) == 0 || (be64toh(*l3e) & RPTE_LEAF) != 0) {
6518 db_printf("\n");
6519 return;
6520 }
6521 pte = pmap_l3e_to_pte(l3e, va);
6522 db_printf(" pte %#016lx\n", be64toh(*pte));
6523 }
6524
6525 void
pmap_page_print_mappings(vm_page_t m)6526 pmap_page_print_mappings(vm_page_t m)
6527 {
6528 pmap_t pmap;
6529 pv_entry_t pv;
6530
6531 db_printf("page %p(%lx)\n", m, m->phys_addr);
6532 /* need to elide locks if running in ddb */
6533 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) {
6534 db_printf("pv: %p ", pv);
6535 db_printf("va: %#016lx ", pv->pv_va);
6536 pmap = PV_PMAP(pv);
6537 db_printf("pmap %p ", pmap);
6538 if (pmap != NULL) {
6539 db_printf("asid: %lu\n", pmap->pm_pid);
6540 pmap_pte_walk(pmap->pm_pml1, pv->pv_va);
6541 }
6542 }
6543 }
6544
DB_SHOW_COMMAND(pte,pmap_print_pte)6545 DB_SHOW_COMMAND(pte, pmap_print_pte)
6546 {
6547 vm_offset_t va;
6548 pmap_t pmap;
6549
6550 if (!have_addr) {
6551 db_printf("show pte addr\n");
6552 return;
6553 }
6554 va = (vm_offset_t)addr;
6555
6556 if (va >= DMAP_MIN_ADDRESS)
6557 pmap = kernel_pmap;
6558 else if (kdb_thread != NULL)
6559 pmap = vmspace_pmap(kdb_thread->td_proc->p_vmspace);
6560 else
6561 pmap = vmspace_pmap(curthread->td_proc->p_vmspace);
6562
6563 pmap_pte_walk(pmap->pm_pml1, va);
6564 }
6565
6566 #endif
6567