1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2018 Matthew Macy
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 *
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28 #include "opt_platform.h"
29
30 #include <sys/param.h>
31 #include <sys/kernel.h>
32 #include <sys/systm.h>
33 #include <sys/conf.h>
34 #include <sys/bitstring.h>
35 #include <sys/queue.h>
36 #include <sys/cpuset.h>
37 #include <sys/endian.h>
38 #include <sys/kerneldump.h>
39 #include <sys/ktr.h>
40 #include <sys/lock.h>
41 #include <sys/syslog.h>
42 #include <sys/msgbuf.h>
43 #include <sys/malloc.h>
44 #include <sys/mman.h>
45 #include <sys/mutex.h>
46 #include <sys/proc.h>
47 #include <sys/rwlock.h>
48 #include <sys/sched.h>
49 #include <sys/sysctl.h>
50 #include <sys/systm.h>
51 #include <sys/vmem.h>
52 #include <sys/vmmeter.h>
53 #include <sys/smp.h>
54
55 #include <sys/kdb.h>
56
57 #include <dev/ofw/openfirm.h>
58
59 #include <vm/vm.h>
60 #include <vm/pmap.h>
61 #include <vm/vm_param.h>
62 #include <vm/vm_kern.h>
63 #include <vm/vm_page.h>
64 #include <vm/vm_map.h>
65 #include <vm/vm_object.h>
66 #include <vm/vm_extern.h>
67 #include <vm/vm_pageout.h>
68 #include <vm/vm_phys.h>
69 #include <vm/vm_radix.h>
70 #include <vm/vm_reserv.h>
71 #include <vm/vm_dumpset.h>
72 #include <vm/uma.h>
73
74 #include <machine/_inttypes.h>
75 #include <machine/cpu.h>
76 #include <machine/platform.h>
77 #include <machine/frame.h>
78 #include <machine/md_var.h>
79 #include <machine/psl.h>
80 #include <machine/bat.h>
81 #include <machine/hid.h>
82 #include <machine/pte.h>
83 #include <machine/sr.h>
84 #include <machine/trap.h>
85 #include <machine/mmuvar.h>
86
87 /* For pseries bit. */
88 #include <powerpc/pseries/phyp-hvcall.h>
89
90 #ifdef INVARIANTS
91 #include <vm/uma_dbg.h>
92 #endif
93
94 #define PPC_BITLSHIFT(bit) (sizeof(long)*NBBY - 1 - (bit))
95 #define PPC_BIT(bit) (1UL << PPC_BITLSHIFT(bit))
96 #define PPC_BITLSHIFT_VAL(val, bit) ((val) << PPC_BITLSHIFT(bit))
97
98 #include "opt_ddb.h"
99
100 #ifdef DDB
101 static void pmap_pte_walk(pml1_entry_t *l1, vm_offset_t va);
102 #endif
103
104 #define PG_W RPTE_WIRED
105 #define PG_V RPTE_VALID
106 #define PG_MANAGED RPTE_MANAGED
107 #define PG_PROMOTED RPTE_PROMOTED
108 #define PG_M RPTE_C
109 #define PG_A RPTE_R
110 #define PG_X RPTE_EAA_X
111 #define PG_RW RPTE_EAA_W
112 #define PG_PTE_CACHE RPTE_ATTR_MASK
113
114 #define RPTE_SHIFT 9
115 #define NLS_MASK ((1UL<<5)-1)
116 #define RPTE_ENTRIES (1UL<<RPTE_SHIFT)
117 #define RPTE_MASK (RPTE_ENTRIES-1)
118
119 #define NLB_SHIFT 0
120 #define NLB_MASK (((1UL<<52)-1) << 8)
121
122 extern int nkpt;
123 extern caddr_t crashdumpmap;
124
125 #define RIC_FLUSH_TLB 0
126 #define RIC_FLUSH_PWC 1
127 #define RIC_FLUSH_ALL 2
128
129 #define POWER9_TLB_SETS_RADIX 128 /* # sets in POWER9 TLB Radix mode */
130
131 #define PPC_INST_TLBIE 0x7c000264
132 #define PPC_INST_TLBIEL 0x7c000224
133 #define PPC_INST_SLBIA 0x7c0003e4
134
135 #define ___PPC_RA(a) (((a) & 0x1f) << 16)
136 #define ___PPC_RB(b) (((b) & 0x1f) << 11)
137 #define ___PPC_RS(s) (((s) & 0x1f) << 21)
138 #define ___PPC_RT(t) ___PPC_RS(t)
139 #define ___PPC_R(r) (((r) & 0x1) << 16)
140 #define ___PPC_PRS(prs) (((prs) & 0x1) << 17)
141 #define ___PPC_RIC(ric) (((ric) & 0x3) << 18)
142
143 #define PPC_SLBIA(IH) __XSTRING(.long PPC_INST_SLBIA | \
144 ((IH & 0x7) << 21))
145 #define PPC_TLBIE_5(rb,rs,ric,prs,r) \
146 __XSTRING(.long PPC_INST_TLBIE | \
147 ___PPC_RB(rb) | ___PPC_RS(rs) | \
148 ___PPC_RIC(ric) | ___PPC_PRS(prs) | \
149 ___PPC_R(r))
150
151 #define PPC_TLBIEL(rb,rs,ric,prs,r) \
152 __XSTRING(.long PPC_INST_TLBIEL | \
153 ___PPC_RB(rb) | ___PPC_RS(rs) | \
154 ___PPC_RIC(ric) | ___PPC_PRS(prs) | \
155 ___PPC_R(r))
156
157 #define PPC_INVALIDATE_ERAT PPC_SLBIA(7)
158
159 static __inline void
ttusync(void)160 ttusync(void)
161 {
162 __asm __volatile("eieio; tlbsync; ptesync" ::: "memory");
163 }
164
165 #define TLBIEL_INVAL_SEL_MASK 0xc00 /* invalidation selector */
166 #define TLBIEL_INVAL_PAGE 0x000 /* invalidate a single page */
167 #define TLBIEL_INVAL_SET_PID 0x400 /* invalidate a set for the current PID */
168 #define TLBIEL_INVAL_SET_LPID 0x800 /* invalidate a set for current LPID */
169 #define TLBIEL_INVAL_SET 0xc00 /* invalidate a set for all LPIDs */
170
171 #define TLBIE_ACTUAL_PAGE_MASK 0xe0
172 #define TLBIE_ACTUAL_PAGE_4K 0x00
173 #define TLBIE_ACTUAL_PAGE_64K 0xa0
174 #define TLBIE_ACTUAL_PAGE_2M 0x20
175 #define TLBIE_ACTUAL_PAGE_1G 0x40
176
177 #define TLBIE_PRS_PARTITION_SCOPE 0x0
178 #define TLBIE_PRS_PROCESS_SCOPE 0x1
179
180 #define TLBIE_RIC_INVALIDATE_TLB 0x0 /* Invalidate just TLB */
181 #define TLBIE_RIC_INVALIDATE_PWC 0x1 /* Invalidate just PWC */
182 #define TLBIE_RIC_INVALIDATE_ALL 0x2 /* Invalidate TLB, PWC,
183 * cached {proc, part}tab entries
184 */
185 #define TLBIE_RIC_INVALIDATE_SEQ 0x3 /* HPT - only:
186 * Invalidate a range of translations
187 */
188
189 static __always_inline void
radix_tlbie(uint8_t ric,uint8_t prs,uint16_t is,uint32_t pid,uint32_t lpid,vm_offset_t va,uint16_t ap)190 radix_tlbie(uint8_t ric, uint8_t prs, uint16_t is, uint32_t pid, uint32_t lpid,
191 vm_offset_t va, uint16_t ap)
192 {
193 uint64_t rb, rs;
194
195 MPASS((va & PAGE_MASK) == 0);
196
197 rs = ((uint64_t)pid << 32) | lpid;
198 rb = va | is | ap;
199 __asm __volatile(PPC_TLBIE_5(%0, %1, %2, %3, 1) : :
200 "r" (rb), "r" (rs), "i" (ric), "i" (prs) : "memory");
201 }
202
203 static __inline void
radix_tlbie_fixup(uint32_t pid,vm_offset_t va,int ap)204 radix_tlbie_fixup(uint32_t pid, vm_offset_t va, int ap)
205 {
206
207 __asm __volatile("ptesync" ::: "memory");
208 radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE,
209 TLBIEL_INVAL_PAGE, 0, 0, va, ap);
210 __asm __volatile("ptesync" ::: "memory");
211 radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE,
212 TLBIEL_INVAL_PAGE, pid, 0, va, ap);
213 }
214
215 static __inline void
radix_tlbie_invlpg_user_4k(uint32_t pid,vm_offset_t va)216 radix_tlbie_invlpg_user_4k(uint32_t pid, vm_offset_t va)
217 {
218
219 radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE,
220 TLBIEL_INVAL_PAGE, pid, 0, va, TLBIE_ACTUAL_PAGE_4K);
221 radix_tlbie_fixup(pid, va, TLBIE_ACTUAL_PAGE_4K);
222 }
223
224 static __inline void
radix_tlbie_invlpg_user_2m(uint32_t pid,vm_offset_t va)225 radix_tlbie_invlpg_user_2m(uint32_t pid, vm_offset_t va)
226 {
227
228 radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE,
229 TLBIEL_INVAL_PAGE, pid, 0, va, TLBIE_ACTUAL_PAGE_2M);
230 radix_tlbie_fixup(pid, va, TLBIE_ACTUAL_PAGE_2M);
231 }
232
233 static __inline void
radix_tlbie_invlpwc_user(uint32_t pid)234 radix_tlbie_invlpwc_user(uint32_t pid)
235 {
236
237 radix_tlbie(TLBIE_RIC_INVALIDATE_PWC, TLBIE_PRS_PROCESS_SCOPE,
238 TLBIEL_INVAL_SET_PID, pid, 0, 0, 0);
239 }
240
241 static __inline void
radix_tlbie_flush_user(uint32_t pid)242 radix_tlbie_flush_user(uint32_t pid)
243 {
244
245 radix_tlbie(TLBIE_RIC_INVALIDATE_ALL, TLBIE_PRS_PROCESS_SCOPE,
246 TLBIEL_INVAL_SET_PID, pid, 0, 0, 0);
247 }
248
249 static __inline void
radix_tlbie_invlpg_kernel_4k(vm_offset_t va)250 radix_tlbie_invlpg_kernel_4k(vm_offset_t va)
251 {
252
253 radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE,
254 TLBIEL_INVAL_PAGE, 0, 0, va, TLBIE_ACTUAL_PAGE_4K);
255 radix_tlbie_fixup(0, va, TLBIE_ACTUAL_PAGE_4K);
256 }
257
258 static __inline void
radix_tlbie_invlpg_kernel_2m(vm_offset_t va)259 radix_tlbie_invlpg_kernel_2m(vm_offset_t va)
260 {
261
262 radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE,
263 TLBIEL_INVAL_PAGE, 0, 0, va, TLBIE_ACTUAL_PAGE_2M);
264 radix_tlbie_fixup(0, va, TLBIE_ACTUAL_PAGE_2M);
265 }
266
267 /* 1GB pages aren't currently supported. */
268 static __inline __unused void
radix_tlbie_invlpg_kernel_1g(vm_offset_t va)269 radix_tlbie_invlpg_kernel_1g(vm_offset_t va)
270 {
271
272 radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE,
273 TLBIEL_INVAL_PAGE, 0, 0, va, TLBIE_ACTUAL_PAGE_1G);
274 radix_tlbie_fixup(0, va, TLBIE_ACTUAL_PAGE_1G);
275 }
276
277 static __inline void
radix_tlbie_invlpwc_kernel(void)278 radix_tlbie_invlpwc_kernel(void)
279 {
280
281 radix_tlbie(TLBIE_RIC_INVALIDATE_PWC, TLBIE_PRS_PROCESS_SCOPE,
282 TLBIEL_INVAL_SET_LPID, 0, 0, 0, 0);
283 }
284
285 static __inline void
radix_tlbie_flush_kernel(void)286 radix_tlbie_flush_kernel(void)
287 {
288
289 radix_tlbie(TLBIE_RIC_INVALIDATE_ALL, TLBIE_PRS_PROCESS_SCOPE,
290 TLBIEL_INVAL_SET_LPID, 0, 0, 0, 0);
291 }
292
293 static __inline vm_pindex_t
pmap_l3e_pindex(vm_offset_t va)294 pmap_l3e_pindex(vm_offset_t va)
295 {
296 return ((va & PG_FRAME) >> L3_PAGE_SIZE_SHIFT);
297 }
298
299 static __inline vm_pindex_t
pmap_pml3e_index(vm_offset_t va)300 pmap_pml3e_index(vm_offset_t va)
301 {
302
303 return ((va >> L3_PAGE_SIZE_SHIFT) & RPTE_MASK);
304 }
305
306 static __inline vm_pindex_t
pmap_pml2e_index(vm_offset_t va)307 pmap_pml2e_index(vm_offset_t va)
308 {
309 return ((va >> L2_PAGE_SIZE_SHIFT) & RPTE_MASK);
310 }
311
312 static __inline vm_pindex_t
pmap_pml1e_index(vm_offset_t va)313 pmap_pml1e_index(vm_offset_t va)
314 {
315 return ((va & PG_FRAME) >> L1_PAGE_SIZE_SHIFT);
316 }
317
318 /* Return various clipped indexes for a given VA */
319 static __inline vm_pindex_t
pmap_pte_index(vm_offset_t va)320 pmap_pte_index(vm_offset_t va)
321 {
322
323 return ((va >> PAGE_SHIFT) & RPTE_MASK);
324 }
325
326 /* Return a pointer to the PT slot that corresponds to a VA */
327 static __inline pt_entry_t *
pmap_l3e_to_pte(pt_entry_t * l3e,vm_offset_t va)328 pmap_l3e_to_pte(pt_entry_t *l3e, vm_offset_t va)
329 {
330 pt_entry_t *pte;
331 vm_paddr_t ptepa;
332
333 ptepa = (be64toh(*l3e) & NLB_MASK);
334 pte = (pt_entry_t *)PHYS_TO_DMAP(ptepa);
335 return (&pte[pmap_pte_index(va)]);
336 }
337
338 /* Return a pointer to the PD slot that corresponds to a VA */
339 static __inline pt_entry_t *
pmap_l2e_to_l3e(pt_entry_t * l2e,vm_offset_t va)340 pmap_l2e_to_l3e(pt_entry_t *l2e, vm_offset_t va)
341 {
342 pt_entry_t *l3e;
343 vm_paddr_t l3pa;
344
345 l3pa = (be64toh(*l2e) & NLB_MASK);
346 l3e = (pml3_entry_t *)PHYS_TO_DMAP(l3pa);
347 return (&l3e[pmap_pml3e_index(va)]);
348 }
349
350 /* Return a pointer to the PD slot that corresponds to a VA */
351 static __inline pt_entry_t *
pmap_l1e_to_l2e(pt_entry_t * l1e,vm_offset_t va)352 pmap_l1e_to_l2e(pt_entry_t *l1e, vm_offset_t va)
353 {
354 pt_entry_t *l2e;
355 vm_paddr_t l2pa;
356
357 l2pa = (be64toh(*l1e) & NLB_MASK);
358
359 l2e = (pml2_entry_t *)PHYS_TO_DMAP(l2pa);
360 return (&l2e[pmap_pml2e_index(va)]);
361 }
362
363 static __inline pml1_entry_t *
pmap_pml1e(pmap_t pmap,vm_offset_t va)364 pmap_pml1e(pmap_t pmap, vm_offset_t va)
365 {
366
367 return (&pmap->pm_pml1[pmap_pml1e_index(va)]);
368 }
369
370 static pt_entry_t *
pmap_pml2e(pmap_t pmap,vm_offset_t va)371 pmap_pml2e(pmap_t pmap, vm_offset_t va)
372 {
373 pt_entry_t *l1e;
374
375 l1e = pmap_pml1e(pmap, va);
376 if (l1e == NULL || (be64toh(*l1e) & RPTE_VALID) == 0)
377 return (NULL);
378 return (pmap_l1e_to_l2e(l1e, va));
379 }
380
381 static __inline pt_entry_t *
pmap_pml3e(pmap_t pmap,vm_offset_t va)382 pmap_pml3e(pmap_t pmap, vm_offset_t va)
383 {
384 pt_entry_t *l2e;
385
386 l2e = pmap_pml2e(pmap, va);
387 if (l2e == NULL || (be64toh(*l2e) & RPTE_VALID) == 0)
388 return (NULL);
389 return (pmap_l2e_to_l3e(l2e, va));
390 }
391
392 static __inline pt_entry_t *
pmap_pte(pmap_t pmap,vm_offset_t va)393 pmap_pte(pmap_t pmap, vm_offset_t va)
394 {
395 pt_entry_t *l3e;
396
397 l3e = pmap_pml3e(pmap, va);
398 if (l3e == NULL || (be64toh(*l3e) & RPTE_VALID) == 0)
399 return (NULL);
400 return (pmap_l3e_to_pte(l3e, va));
401 }
402
403 int nkpt = 64;
404 SYSCTL_INT(_machdep, OID_AUTO, nkpt, CTLFLAG_RD, &nkpt, 0,
405 "Number of kernel page table pages allocated on bootup");
406
407 vm_paddr_t dmaplimit;
408
409 SYSCTL_DECL(_vm_pmap);
410
411 #ifdef INVARIANTS
412 #define VERBOSE_PMAP 0
413 #define VERBOSE_PROTECT 0
414 static int pmap_logging;
415 SYSCTL_INT(_vm_pmap, OID_AUTO, pmap_logging, CTLFLAG_RWTUN,
416 &pmap_logging, 0, "verbose debug logging");
417 #endif
418
419 static u_int64_t KPTphys; /* phys addr of kernel level 1 */
420
421 //static vm_paddr_t KERNend; /* phys addr of end of bootstrap data */
422
423 static vm_offset_t qframe = 0;
424 static struct mtx qframe_mtx;
425
426 void mmu_radix_activate(struct thread *);
427 void mmu_radix_advise(pmap_t, vm_offset_t, vm_offset_t, int);
428 void mmu_radix_align_superpage(vm_object_t, vm_ooffset_t, vm_offset_t *,
429 vm_size_t);
430 void mmu_radix_clear_modify(vm_page_t);
431 void mmu_radix_copy(pmap_t, pmap_t, vm_offset_t, vm_size_t, vm_offset_t);
432 int mmu_radix_decode_kernel_ptr(vm_offset_t, int *, vm_offset_t *);
433 int mmu_radix_enter(pmap_t, vm_offset_t, vm_page_t, vm_prot_t, u_int, int8_t);
434 void mmu_radix_enter_object(pmap_t, vm_offset_t, vm_offset_t, vm_page_t,
435 vm_prot_t);
436 void mmu_radix_enter_quick(pmap_t, vm_offset_t, vm_page_t, vm_prot_t);
437 vm_paddr_t mmu_radix_extract(pmap_t pmap, vm_offset_t va);
438 vm_page_t mmu_radix_extract_and_hold(pmap_t, vm_offset_t, vm_prot_t);
439 void mmu_radix_kenter(vm_offset_t, vm_paddr_t);
440 vm_paddr_t mmu_radix_kextract(vm_offset_t);
441 void mmu_radix_kremove(vm_offset_t);
442 bool mmu_radix_is_modified(vm_page_t);
443 bool mmu_radix_is_prefaultable(pmap_t, vm_offset_t);
444 bool mmu_radix_is_referenced(vm_page_t);
445 void mmu_radix_object_init_pt(pmap_t, vm_offset_t, vm_object_t,
446 vm_pindex_t, vm_size_t);
447 bool mmu_radix_page_exists_quick(pmap_t, vm_page_t);
448 void mmu_radix_page_init(vm_page_t);
449 bool mmu_radix_page_is_mapped(vm_page_t m);
450 void mmu_radix_page_set_memattr(vm_page_t, vm_memattr_t);
451 int mmu_radix_page_wired_mappings(vm_page_t);
452 int mmu_radix_pinit(pmap_t);
453 void mmu_radix_protect(pmap_t, vm_offset_t, vm_offset_t, vm_prot_t);
454 bool mmu_radix_ps_enabled(pmap_t);
455 void mmu_radix_qenter(vm_offset_t, vm_page_t *, int);
456 void mmu_radix_qremove(vm_offset_t, int);
457 vm_offset_t mmu_radix_quick_enter_page(vm_page_t);
458 void mmu_radix_quick_remove_page(vm_offset_t);
459 int mmu_radix_ts_referenced(vm_page_t);
460 void mmu_radix_release(pmap_t);
461 void mmu_radix_remove(pmap_t, vm_offset_t, vm_offset_t);
462 void mmu_radix_remove_all(vm_page_t);
463 void mmu_radix_remove_pages(pmap_t);
464 void mmu_radix_remove_write(vm_page_t);
465 void mmu_radix_sync_icache(pmap_t pm, vm_offset_t va, vm_size_t sz);
466 void mmu_radix_unwire(pmap_t, vm_offset_t, vm_offset_t);
467 void mmu_radix_zero_page(vm_page_t);
468 void mmu_radix_zero_page_area(vm_page_t, int, int);
469 int mmu_radix_change_attr(vm_offset_t, vm_size_t, vm_memattr_t);
470 void mmu_radix_page_array_startup(long pages);
471
472 #include "mmu_oea64.h"
473
474 /*
475 * Kernel MMU interface
476 */
477
478 static void mmu_radix_bootstrap(vm_offset_t, vm_offset_t);
479
480 static void mmu_radix_copy_page(vm_page_t, vm_page_t);
481 static void mmu_radix_copy_pages(vm_page_t *ma, vm_offset_t a_offset,
482 vm_page_t *mb, vm_offset_t b_offset, int xfersize);
483 static int mmu_radix_growkernel(vm_offset_t);
484 static void mmu_radix_init(void);
485 static int mmu_radix_mincore(pmap_t, vm_offset_t, vm_paddr_t *);
486 static vm_offset_t mmu_radix_map(vm_offset_t *, vm_paddr_t, vm_paddr_t, int);
487 static void mmu_radix_pinit0(pmap_t);
488
489 static void *mmu_radix_mapdev(vm_paddr_t, vm_size_t);
490 static void *mmu_radix_mapdev_attr(vm_paddr_t, vm_size_t, vm_memattr_t);
491 static void mmu_radix_unmapdev(void *, vm_size_t);
492 static void mmu_radix_kenter_attr(vm_offset_t, vm_paddr_t, vm_memattr_t ma);
493 static int mmu_radix_dev_direct_mapped(vm_paddr_t, vm_size_t);
494 static void mmu_radix_dumpsys_map(vm_paddr_t pa, size_t sz, void **va);
495 static void mmu_radix_scan_init(void);
496 static void mmu_radix_cpu_bootstrap(int ap);
497 static void mmu_radix_tlbie_all(void);
498
499 static struct pmap_funcs mmu_radix_methods = {
500 .bootstrap = mmu_radix_bootstrap,
501 .copy_page = mmu_radix_copy_page,
502 .copy_pages = mmu_radix_copy_pages,
503 .cpu_bootstrap = mmu_radix_cpu_bootstrap,
504 .growkernel_nopanic = mmu_radix_growkernel,
505 .init = mmu_radix_init,
506 .map = mmu_radix_map,
507 .mincore = mmu_radix_mincore,
508 .pinit = mmu_radix_pinit,
509 .pinit0 = mmu_radix_pinit0,
510
511 .mapdev = mmu_radix_mapdev,
512 .mapdev_attr = mmu_radix_mapdev_attr,
513 .unmapdev = mmu_radix_unmapdev,
514 .kenter_attr = mmu_radix_kenter_attr,
515 .dev_direct_mapped = mmu_radix_dev_direct_mapped,
516 .dumpsys_pa_init = mmu_radix_scan_init,
517 .dumpsys_map_chunk = mmu_radix_dumpsys_map,
518 .page_is_mapped = mmu_radix_page_is_mapped,
519 .ps_enabled = mmu_radix_ps_enabled,
520 .align_superpage = mmu_radix_align_superpage,
521 .object_init_pt = mmu_radix_object_init_pt,
522 .protect = mmu_radix_protect,
523 /* pmap dispatcher interface */
524 .clear_modify = mmu_radix_clear_modify,
525 .copy = mmu_radix_copy,
526 .enter = mmu_radix_enter,
527 .enter_object = mmu_radix_enter_object,
528 .enter_quick = mmu_radix_enter_quick,
529 .extract = mmu_radix_extract,
530 .extract_and_hold = mmu_radix_extract_and_hold,
531 .is_modified = mmu_radix_is_modified,
532 .is_prefaultable = mmu_radix_is_prefaultable,
533 .is_referenced = mmu_radix_is_referenced,
534 .ts_referenced = mmu_radix_ts_referenced,
535 .page_exists_quick = mmu_radix_page_exists_quick,
536 .page_init = mmu_radix_page_init,
537 .page_wired_mappings = mmu_radix_page_wired_mappings,
538 .qenter = mmu_radix_qenter,
539 .qremove = mmu_radix_qremove,
540 .release = mmu_radix_release,
541 .remove = mmu_radix_remove,
542 .remove_all = mmu_radix_remove_all,
543 .remove_write = mmu_radix_remove_write,
544 .sync_icache = mmu_radix_sync_icache,
545 .unwire = mmu_radix_unwire,
546 .zero_page = mmu_radix_zero_page,
547 .zero_page_area = mmu_radix_zero_page_area,
548 .activate = mmu_radix_activate,
549 .quick_enter_page = mmu_radix_quick_enter_page,
550 .quick_remove_page = mmu_radix_quick_remove_page,
551 .page_set_memattr = mmu_radix_page_set_memattr,
552 .page_array_startup = mmu_radix_page_array_startup,
553
554 /* Internal interfaces */
555 .kenter = mmu_radix_kenter,
556 .kextract = mmu_radix_kextract,
557 .kremove = mmu_radix_kremove,
558 .change_attr = mmu_radix_change_attr,
559 .decode_kernel_ptr = mmu_radix_decode_kernel_ptr,
560
561 .tlbie_all = mmu_radix_tlbie_all,
562 };
563
564 MMU_DEF(mmu_radix, MMU_TYPE_RADIX, mmu_radix_methods);
565
566 static bool pmap_demote_l3e_locked(pmap_t pmap, pml3_entry_t *l3e, vm_offset_t va,
567 struct rwlock **lockp);
568 static bool pmap_demote_l3e(pmap_t pmap, pml3_entry_t *pde, vm_offset_t va);
569 static int pmap_unuse_pt(pmap_t, vm_offset_t, pml3_entry_t, struct spglist *);
570 static int pmap_remove_l3e(pmap_t pmap, pml3_entry_t *pdq, vm_offset_t sva,
571 struct spglist *free, struct rwlock **lockp);
572 static int pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t sva,
573 pml3_entry_t ptepde, struct spglist *free, struct rwlock **lockp);
574 static vm_page_t pmap_remove_pt_page(pmap_t pmap, vm_offset_t va);
575 static bool pmap_remove_page(pmap_t pmap, vm_offset_t va, pml3_entry_t *pde,
576 struct spglist *free);
577 static bool pmap_remove_ptes(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,
578 pml3_entry_t *l3e, struct spglist *free, struct rwlock **lockp);
579
580 static bool pmap_pv_insert_l3e(pmap_t pmap, vm_offset_t va, pml3_entry_t l3e,
581 u_int flags, struct rwlock **lockp);
582 #if VM_NRESERVLEVEL > 0
583 static void pmap_pv_promote_l3e(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,
584 struct rwlock **lockp);
585 #endif
586 static void pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va);
587 static int pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte);
588 static vm_page_t mmu_radix_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m,
589 vm_prot_t prot, vm_page_t mpte, struct rwlock **lockp, bool *invalidate);
590
591 static bool pmap_enter_2mpage(pmap_t pmap, vm_offset_t va, vm_page_t m,
592 vm_prot_t prot, struct rwlock **lockp);
593 static int pmap_enter_l3e(pmap_t pmap, vm_offset_t va, pml3_entry_t newpde,
594 u_int flags, vm_page_t m, struct rwlock **lockp);
595
596 static vm_page_t reclaim_pv_chunk(pmap_t locked_pmap, struct rwlock **lockp);
597 static void free_pv_chunk(struct pv_chunk *pc);
598 static vm_page_t _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, struct rwlock **lockp);
599 static vm_page_t pmap_allocl3e(pmap_t pmap, vm_offset_t va,
600 struct rwlock **lockp);
601 static vm_page_t pmap_allocpte(pmap_t pmap, vm_offset_t va,
602 struct rwlock **lockp);
603 static void _pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m,
604 struct spglist *free);
605 static bool pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free);
606
607 static void pmap_invalidate_page(pmap_t pmap, vm_offset_t start);
608 static void pmap_invalidate_all(pmap_t pmap);
609 static int pmap_change_attr_locked(vm_offset_t va, vm_size_t size, int mode, bool flush);
610 static void pmap_fill_ptp(pt_entry_t *firstpte, pt_entry_t newpte);
611
612 /*
613 * Internal flags for pmap_enter()'s helper functions.
614 */
615 #define PMAP_ENTER_NORECLAIM 0x1000000 /* Don't reclaim PV entries. */
616 #define PMAP_ENTER_NOREPLACE 0x2000000 /* Don't replace mappings. */
617
618 #define UNIMPLEMENTED() panic("%s not implemented", __func__)
619 #define UNTESTED() panic("%s not yet tested", __func__)
620
621 /* Number of supported PID bits */
622 static unsigned int isa3_pid_bits;
623
624 /* PID to start allocating from */
625 static unsigned int isa3_base_pid;
626
627 #define PROCTAB_SIZE_SHIFT (isa3_pid_bits + 4)
628 #define PROCTAB_ENTRIES (1ul << isa3_pid_bits)
629
630 /*
631 * Map of physical memory regions.
632 */
633 static struct mem_region *regions, *pregions;
634 static struct numa_mem_region *numa_pregions;
635 static u_int phys_avail_count;
636 static int regions_sz, pregions_sz, numa_pregions_sz;
637 static struct pate *isa3_parttab;
638 static struct prte *isa3_proctab;
639 static vmem_t *asid_arena;
640
641 extern void bs_remap_earlyboot(void);
642
643 #define RADIX_PGD_SIZE_SHIFT 16
644 #define RADIX_PGD_SIZE (1UL << RADIX_PGD_SIZE_SHIFT)
645
646 #define RADIX_PGD_INDEX_SHIFT (RADIX_PGD_SIZE_SHIFT-3)
647 #define NL2EPG (PAGE_SIZE/sizeof(pml2_entry_t))
648 #define NL3EPG (PAGE_SIZE/sizeof(pml3_entry_t))
649
650 #define NUPML1E (RADIX_PGD_SIZE/sizeof(uint64_t)) /* number of userland PML1 pages */
651 #define NUPDPE (NUPML1E * NL2EPG)/* number of userland PDP pages */
652 #define NUPDE (NUPDPE * NL3EPG) /* number of userland PD entries */
653
654 /* POWER9 only permits a 64k partition table size. */
655 #define PARTTAB_SIZE_SHIFT 16
656 #define PARTTAB_SIZE (1UL << PARTTAB_SIZE_SHIFT)
657
658 #define PARTTAB_HR (1UL << 63) /* host uses radix */
659 #define PARTTAB_GR (1UL << 63) /* guest uses radix must match host */
660
661 /* TLB flush actions. Used as argument to tlbiel_flush() */
662 enum {
663 TLB_INVAL_SCOPE_LPID = 2, /* invalidate TLBs for current LPID */
664 TLB_INVAL_SCOPE_GLOBAL = 3, /* invalidate all TLBs */
665 };
666
667 #define NPV_LIST_LOCKS MAXCPU
668 static int pmap_initialized;
669 static vm_paddr_t proctab0pa;
670 static vm_paddr_t parttab_phys;
671 CTASSERT(sizeof(struct pv_chunk) == PAGE_SIZE);
672
673 /*
674 * Data for the pv entry allocation mechanism.
675 * Updates to pv_invl_gen are protected by the pv_list_locks[]
676 * elements, but reads are not.
677 */
678 static TAILQ_HEAD(pch, pv_chunk) pv_chunks = TAILQ_HEAD_INITIALIZER(pv_chunks);
679 static struct mtx __exclusive_cache_line pv_chunks_mutex;
680 static struct rwlock __exclusive_cache_line pv_list_locks[NPV_LIST_LOCKS];
681 static struct md_page *pv_table;
682 static struct md_page pv_dummy;
683
684 #ifdef PV_STATS
685 #define PV_STAT(x) do { x ; } while (0)
686 #else
687 #define PV_STAT(x) do { } while (0)
688 #endif
689
690 #define pa_radix_index(pa) ((pa) >> L3_PAGE_SIZE_SHIFT)
691 #define pa_to_pvh(pa) (&pv_table[pa_radix_index(pa)])
692
693 #define PHYS_TO_PV_LIST_LOCK(pa) \
694 (&pv_list_locks[pa_radix_index(pa) % NPV_LIST_LOCKS])
695
696 #define CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa) do { \
697 struct rwlock **_lockp = (lockp); \
698 struct rwlock *_new_lock; \
699 \
700 _new_lock = PHYS_TO_PV_LIST_LOCK(pa); \
701 if (_new_lock != *_lockp) { \
702 if (*_lockp != NULL) \
703 rw_wunlock(*_lockp); \
704 *_lockp = _new_lock; \
705 rw_wlock(*_lockp); \
706 } \
707 } while (0)
708
709 #define CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m) \
710 CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, VM_PAGE_TO_PHYS(m))
711
712 #define RELEASE_PV_LIST_LOCK(lockp) do { \
713 struct rwlock **_lockp = (lockp); \
714 \
715 if (*_lockp != NULL) { \
716 rw_wunlock(*_lockp); \
717 *_lockp = NULL; \
718 } \
719 } while (0)
720
721 #define VM_PAGE_TO_PV_LIST_LOCK(m) \
722 PHYS_TO_PV_LIST_LOCK(VM_PAGE_TO_PHYS(m))
723
724 /*
725 * We support 52 bits, hence:
726 * bits 52 - 31 = 21, 0b10101
727 * RTS encoding details
728 * bits 0 - 3 of rts -> bits 6 - 8 unsigned long
729 * bits 4 - 5 of rts -> bits 62 - 63 of unsigned long
730 */
731 #define RTS_SIZE ((0x2UL << 61) | (0x5UL << 5))
732
733 static int powernv_enabled = 1;
734
735 static __always_inline void
tlbiel_radix_set_isa300(uint32_t set,uint32_t is,uint32_t pid,uint32_t ric,uint32_t prs)736 tlbiel_radix_set_isa300(uint32_t set, uint32_t is,
737 uint32_t pid, uint32_t ric, uint32_t prs)
738 {
739 uint64_t rb;
740 uint64_t rs;
741
742 rb = PPC_BITLSHIFT_VAL(set, 51) | PPC_BITLSHIFT_VAL(is, 53);
743 rs = PPC_BITLSHIFT_VAL((uint64_t)pid, 31);
744
745 __asm __volatile(PPC_TLBIEL(%0, %1, %2, %3, 1)
746 : : "r"(rb), "r"(rs), "i"(ric), "i"(prs)
747 : "memory");
748 }
749
750 static void
tlbiel_flush_isa3(uint32_t num_sets,uint32_t is)751 tlbiel_flush_isa3(uint32_t num_sets, uint32_t is)
752 {
753 uint32_t set;
754
755 __asm __volatile("ptesync": : :"memory");
756
757 /*
758 * Flush the first set of the TLB, and the entire Page Walk Cache
759 * and partition table entries. Then flush the remaining sets of the
760 * TLB.
761 */
762 if (is == TLB_INVAL_SCOPE_GLOBAL) {
763 tlbiel_radix_set_isa300(0, is, 0, RIC_FLUSH_ALL, 0);
764 for (set = 1; set < num_sets; set++)
765 tlbiel_radix_set_isa300(set, is, 0, RIC_FLUSH_TLB, 0);
766 }
767
768 /* Do the same for process scoped entries. */
769 tlbiel_radix_set_isa300(0, is, 0, RIC_FLUSH_ALL, 1);
770 for (set = 1; set < num_sets; set++)
771 tlbiel_radix_set_isa300(set, is, 0, RIC_FLUSH_TLB, 1);
772
773 __asm __volatile("ptesync": : :"memory");
774 }
775
776 static void
mmu_radix_tlbiel_flush(int scope)777 mmu_radix_tlbiel_flush(int scope)
778 {
779 MPASS(scope == TLB_INVAL_SCOPE_LPID ||
780 scope == TLB_INVAL_SCOPE_GLOBAL);
781
782 tlbiel_flush_isa3(POWER9_TLB_SETS_RADIX, scope);
783 __asm __volatile(PPC_INVALIDATE_ERAT "; isync" : : :"memory");
784 }
785
786 static void
mmu_radix_tlbie_all(void)787 mmu_radix_tlbie_all(void)
788 {
789 if (powernv_enabled)
790 mmu_radix_tlbiel_flush(TLB_INVAL_SCOPE_GLOBAL);
791 else
792 mmu_radix_tlbiel_flush(TLB_INVAL_SCOPE_LPID);
793 }
794
795 static void
mmu_radix_init_amor(void)796 mmu_radix_init_amor(void)
797 {
798 /*
799 * In HV mode, we init AMOR (Authority Mask Override Register) so that
800 * the hypervisor and guest can setup IAMR (Instruction Authority Mask
801 * Register), enable key 0 and set it to 1.
802 *
803 * AMOR = 0b1100 .... 0000 (Mask for key 0 is 11)
804 */
805 mtspr(SPR_AMOR, (3ul << 62));
806 }
807
808 static void
mmu_radix_init_iamr(void)809 mmu_radix_init_iamr(void)
810 {
811 /*
812 * Radix always uses key0 of the IAMR to determine if an access is
813 * allowed. We set bit 0 (IBM bit 1) of key0, to prevent instruction
814 * fetch.
815 */
816 mtspr(SPR_IAMR, (1ul << 62));
817 }
818
819 static void
mmu_radix_pid_set(pmap_t pmap)820 mmu_radix_pid_set(pmap_t pmap)
821 {
822
823 mtspr(SPR_PID, pmap->pm_pid);
824 isync();
825 }
826
827 /* Quick sort callout for comparing physical addresses. */
828 static int
pa_cmp(const void * a,const void * b)829 pa_cmp(const void *a, const void *b)
830 {
831 const vm_paddr_t *pa = a, *pb = b;
832
833 if (*pa < *pb)
834 return (-1);
835 else if (*pa > *pb)
836 return (1);
837 else
838 return (0);
839 }
840
841 #define pte_load_store(ptep, pte) atomic_swap_long(ptep, pte)
842 #define pte_load_clear(ptep) atomic_swap_long(ptep, 0)
843 #define pte_store(ptep, pte) do { \
844 MPASS((pte) & (RPTE_EAA_R | RPTE_EAA_W | RPTE_EAA_X)); \
845 *(u_long *)(ptep) = htobe64((u_long)((pte) | PG_V | RPTE_LEAF)); \
846 } while (0)
847 /*
848 * NB: should only be used for adding directories - not for direct mappings
849 */
850 #define pde_store(ptep, pa) do { \
851 *(u_long *)(ptep) = htobe64((u_long)(pa|RPTE_VALID|RPTE_SHIFT)); \
852 } while (0)
853
854 #define pte_clear(ptep) do { \
855 *(u_long *)(ptep) = (u_long)(0); \
856 } while (0)
857
858 #define PMAP_PDE_SUPERPAGE (1 << 8) /* supports 2MB superpages */
859
860 /*
861 * Promotion to a 2MB (PDE) page mapping requires that the corresponding 4KB
862 * (PTE) page mappings have identical settings for the following fields:
863 */
864 #define PG_PTE_PROMOTE (PG_X | PG_MANAGED | PG_W | PG_PTE_CACHE | \
865 PG_M | PG_A | RPTE_EAA_MASK | PG_V)
866
867 static __inline void
pmap_resident_count_inc(pmap_t pmap,int count)868 pmap_resident_count_inc(pmap_t pmap, int count)
869 {
870
871 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
872 pmap->pm_stats.resident_count += count;
873 }
874
875 static __inline void
pmap_resident_count_dec(pmap_t pmap,int count)876 pmap_resident_count_dec(pmap_t pmap, int count)
877 {
878
879 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
880 KASSERT(pmap->pm_stats.resident_count >= count,
881 ("pmap %p resident count underflow %ld %d", pmap,
882 pmap->pm_stats.resident_count, count));
883 pmap->pm_stats.resident_count -= count;
884 }
885
886 static void
pagezero(vm_offset_t va)887 pagezero(vm_offset_t va)
888 {
889 va = trunc_page(va);
890
891 bzero((void *)va, PAGE_SIZE);
892 }
893
894 static uint64_t
allocpages(int n)895 allocpages(int n)
896 {
897 u_int64_t ret;
898
899 ret = moea64_bootstrap_alloc(n * PAGE_SIZE, PAGE_SIZE);
900 for (int i = 0; i < n; i++)
901 pagezero(PHYS_TO_DMAP(ret + i * PAGE_SIZE));
902 return (ret);
903 }
904
905 static pt_entry_t *
kvtopte(vm_offset_t va)906 kvtopte(vm_offset_t va)
907 {
908 pt_entry_t *l3e;
909
910 l3e = pmap_pml3e(kernel_pmap, va);
911 if (l3e == NULL || (be64toh(*l3e) & RPTE_VALID) == 0)
912 return (NULL);
913 return (pmap_l3e_to_pte(l3e, va));
914 }
915
916 void
mmu_radix_kenter(vm_offset_t va,vm_paddr_t pa)917 mmu_radix_kenter(vm_offset_t va, vm_paddr_t pa)
918 {
919 pt_entry_t *pte;
920
921 pte = kvtopte(va);
922 MPASS(pte != NULL);
923 *pte = htobe64(pa | RPTE_VALID | RPTE_LEAF | RPTE_EAA_R | \
924 RPTE_EAA_W | RPTE_EAA_P | PG_M | PG_A);
925 }
926
927 bool
mmu_radix_ps_enabled(pmap_t pmap)928 mmu_radix_ps_enabled(pmap_t pmap)
929 {
930 return (superpages_enabled && (pmap->pm_flags & PMAP_PDE_SUPERPAGE) != 0);
931 }
932
933 static pt_entry_t *
pmap_nofault_pte(pmap_t pmap,vm_offset_t va,int * is_l3e)934 pmap_nofault_pte(pmap_t pmap, vm_offset_t va, int *is_l3e)
935 {
936 pml3_entry_t *l3e;
937 pt_entry_t *pte;
938
939 va &= PG_PS_FRAME;
940 l3e = pmap_pml3e(pmap, va);
941 if (l3e == NULL || (be64toh(*l3e) & PG_V) == 0)
942 return (NULL);
943
944 if (be64toh(*l3e) & RPTE_LEAF) {
945 *is_l3e = 1;
946 return (l3e);
947 }
948 *is_l3e = 0;
949 va &= PG_FRAME;
950 pte = pmap_l3e_to_pte(l3e, va);
951 if (pte == NULL || (be64toh(*pte) & PG_V) == 0)
952 return (NULL);
953 return (pte);
954 }
955
956 int
pmap_nofault(pmap_t pmap,vm_offset_t va,vm_prot_t flags)957 pmap_nofault(pmap_t pmap, vm_offset_t va, vm_prot_t flags)
958 {
959 pt_entry_t *pte;
960 pt_entry_t startpte, origpte, newpte;
961 vm_page_t m;
962 int is_l3e;
963
964 startpte = 0;
965 retry:
966 if ((pte = pmap_nofault_pte(pmap, va, &is_l3e)) == NULL)
967 return (KERN_INVALID_ADDRESS);
968 origpte = newpte = be64toh(*pte);
969 if (startpte == 0) {
970 startpte = origpte;
971 if (((flags & VM_PROT_WRITE) && (startpte & PG_M)) ||
972 ((flags & VM_PROT_READ) && (startpte & PG_A))) {
973 pmap_invalidate_all(pmap);
974 #ifdef INVARIANTS
975 if (VERBOSE_PMAP || pmap_logging)
976 printf("%s(%p, %#lx, %#x) (%#lx) -- invalidate all\n",
977 __func__, pmap, va, flags, origpte);
978 #endif
979 return (KERN_FAILURE);
980 }
981 }
982 #ifdef INVARIANTS
983 if (VERBOSE_PMAP || pmap_logging)
984 printf("%s(%p, %#lx, %#x) (%#lx)\n", __func__, pmap, va,
985 flags, origpte);
986 #endif
987 PMAP_LOCK(pmap);
988 if ((pte = pmap_nofault_pte(pmap, va, &is_l3e)) == NULL ||
989 be64toh(*pte) != origpte) {
990 PMAP_UNLOCK(pmap);
991 return (KERN_FAILURE);
992 }
993 m = PHYS_TO_VM_PAGE(newpte & PG_FRAME);
994 MPASS(m != NULL);
995 switch (flags) {
996 case VM_PROT_READ:
997 if ((newpte & (RPTE_EAA_R|RPTE_EAA_X)) == 0)
998 goto protfail;
999 newpte |= PG_A;
1000 vm_page_aflag_set(m, PGA_REFERENCED);
1001 break;
1002 case VM_PROT_WRITE:
1003 if ((newpte & RPTE_EAA_W) == 0)
1004 goto protfail;
1005 if (is_l3e)
1006 goto protfail;
1007 newpte |= PG_M;
1008 vm_page_dirty(m);
1009 break;
1010 case VM_PROT_EXECUTE:
1011 if ((newpte & RPTE_EAA_X) == 0)
1012 goto protfail;
1013 newpte |= PG_A;
1014 vm_page_aflag_set(m, PGA_REFERENCED);
1015 break;
1016 }
1017
1018 if (!atomic_cmpset_long(pte, htobe64(origpte), htobe64(newpte)))
1019 goto retry;
1020 ptesync();
1021 PMAP_UNLOCK(pmap);
1022 if (startpte == newpte)
1023 return (KERN_FAILURE);
1024 return (0);
1025 protfail:
1026 PMAP_UNLOCK(pmap);
1027 return (KERN_PROTECTION_FAILURE);
1028 }
1029
1030 /*
1031 * Returns true if the given page is mapped individually or as part of
1032 * a 2mpage. Otherwise, returns false.
1033 */
1034 bool
mmu_radix_page_is_mapped(vm_page_t m)1035 mmu_radix_page_is_mapped(vm_page_t m)
1036 {
1037 struct rwlock *lock;
1038 bool rv;
1039
1040 if ((m->oflags & VPO_UNMANAGED) != 0)
1041 return (false);
1042 lock = VM_PAGE_TO_PV_LIST_LOCK(m);
1043 rw_rlock(lock);
1044 rv = !TAILQ_EMPTY(&m->md.pv_list) ||
1045 ((m->flags & PG_FICTITIOUS) == 0 &&
1046 !TAILQ_EMPTY(&pa_to_pvh(VM_PAGE_TO_PHYS(m))->pv_list));
1047 rw_runlock(lock);
1048 return (rv);
1049 }
1050
1051 /*
1052 * Determine the appropriate bits to set in a PTE or PDE for a specified
1053 * caching mode.
1054 */
1055 static int
pmap_cache_bits(vm_memattr_t ma)1056 pmap_cache_bits(vm_memattr_t ma)
1057 {
1058 if (ma != VM_MEMATTR_DEFAULT) {
1059 switch (ma) {
1060 case VM_MEMATTR_UNCACHEABLE:
1061 return (RPTE_ATTR_GUARDEDIO);
1062 case VM_MEMATTR_CACHEABLE:
1063 return (RPTE_ATTR_MEM);
1064 case VM_MEMATTR_WRITE_BACK:
1065 case VM_MEMATTR_PREFETCHABLE:
1066 case VM_MEMATTR_WRITE_COMBINING:
1067 return (RPTE_ATTR_UNGUARDEDIO);
1068 }
1069 }
1070 return (0);
1071 }
1072
1073 static void
pmap_invalidate_page(pmap_t pmap,vm_offset_t start)1074 pmap_invalidate_page(pmap_t pmap, vm_offset_t start)
1075 {
1076 ptesync();
1077 if (pmap == kernel_pmap)
1078 radix_tlbie_invlpg_kernel_4k(start);
1079 else
1080 radix_tlbie_invlpg_user_4k(pmap->pm_pid, start);
1081 ttusync();
1082 }
1083
1084 static void
pmap_invalidate_page_2m(pmap_t pmap,vm_offset_t start)1085 pmap_invalidate_page_2m(pmap_t pmap, vm_offset_t start)
1086 {
1087 ptesync();
1088 if (pmap == kernel_pmap)
1089 radix_tlbie_invlpg_kernel_2m(start);
1090 else
1091 radix_tlbie_invlpg_user_2m(pmap->pm_pid, start);
1092 ttusync();
1093 }
1094
1095 static void
pmap_invalidate_pwc(pmap_t pmap)1096 pmap_invalidate_pwc(pmap_t pmap)
1097 {
1098 ptesync();
1099 if (pmap == kernel_pmap)
1100 radix_tlbie_invlpwc_kernel();
1101 else
1102 radix_tlbie_invlpwc_user(pmap->pm_pid);
1103 ttusync();
1104 }
1105
1106 static void
pmap_invalidate_range(pmap_t pmap,vm_offset_t start,vm_offset_t end)1107 pmap_invalidate_range(pmap_t pmap, vm_offset_t start, vm_offset_t end)
1108 {
1109 if (((start - end) >> PAGE_SHIFT) > 8) {
1110 pmap_invalidate_all(pmap);
1111 return;
1112 }
1113 ptesync();
1114 if (pmap == kernel_pmap) {
1115 while (start < end) {
1116 radix_tlbie_invlpg_kernel_4k(start);
1117 start += PAGE_SIZE;
1118 }
1119 } else {
1120 while (start < end) {
1121 radix_tlbie_invlpg_user_4k(pmap->pm_pid, start);
1122 start += PAGE_SIZE;
1123 }
1124 }
1125 ttusync();
1126 }
1127
1128 static void
pmap_invalidate_all(pmap_t pmap)1129 pmap_invalidate_all(pmap_t pmap)
1130 {
1131 ptesync();
1132 if (pmap == kernel_pmap)
1133 radix_tlbie_flush_kernel();
1134 else
1135 radix_tlbie_flush_user(pmap->pm_pid);
1136 ttusync();
1137 }
1138
1139 static void
pmap_invalidate_l3e_page(pmap_t pmap,vm_offset_t va,pml3_entry_t l3e)1140 pmap_invalidate_l3e_page(pmap_t pmap, vm_offset_t va, pml3_entry_t l3e)
1141 {
1142
1143 /*
1144 * When the PDE has PG_PROMOTED set, the 2MB page mapping was created
1145 * by a promotion that did not invalidate the 512 4KB page mappings
1146 * that might exist in the TLB. Consequently, at this point, the TLB
1147 * may hold both 4KB and 2MB page mappings for the address range [va,
1148 * va + L3_PAGE_SIZE). Therefore, the entire range must be invalidated here.
1149 * In contrast, when PG_PROMOTED is clear, the TLB will not hold any
1150 * 4KB page mappings for the address range [va, va + L3_PAGE_SIZE), and so a
1151 * single INVLPG suffices to invalidate the 2MB page mapping from the
1152 * TLB.
1153 */
1154 ptesync();
1155 if ((l3e & PG_PROMOTED) != 0)
1156 pmap_invalidate_range(pmap, va, va + L3_PAGE_SIZE - 1);
1157 else
1158 pmap_invalidate_page_2m(pmap, va);
1159
1160 pmap_invalidate_pwc(pmap);
1161 }
1162
1163 static __inline struct pv_chunk *
pv_to_chunk(pv_entry_t pv)1164 pv_to_chunk(pv_entry_t pv)
1165 {
1166
1167 return ((struct pv_chunk *)((uintptr_t)pv & ~(uintptr_t)PAGE_MASK));
1168 }
1169
1170 #define PV_PMAP(pv) (pv_to_chunk(pv)->pc_pmap)
1171
1172 #define PC_FREE0 0xfffffffffffffffful
1173 #define PC_FREE1 ((1ul << (_NPCPV % 64)) - 1)
1174
1175 static const uint64_t pc_freemask[_NPCM] = { PC_FREE0, PC_FREE1 };
1176
1177 /*
1178 * Ensure that the number of spare PV entries in the specified pmap meets or
1179 * exceeds the given count, "needed".
1180 *
1181 * The given PV list lock may be released.
1182 */
1183 static void
reserve_pv_entries(pmap_t pmap,int needed,struct rwlock ** lockp)1184 reserve_pv_entries(pmap_t pmap, int needed, struct rwlock **lockp)
1185 {
1186 struct pch new_tail;
1187 struct pv_chunk *pc;
1188 vm_page_t m;
1189 int avail, free;
1190 bool reclaimed;
1191
1192 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
1193 KASSERT(lockp != NULL, ("reserve_pv_entries: lockp is NULL"));
1194
1195 /*
1196 * Newly allocated PV chunks must be stored in a private list until
1197 * the required number of PV chunks have been allocated. Otherwise,
1198 * reclaim_pv_chunk() could recycle one of these chunks. In
1199 * contrast, these chunks must be added to the pmap upon allocation.
1200 */
1201 TAILQ_INIT(&new_tail);
1202 retry:
1203 avail = 0;
1204 TAILQ_FOREACH(pc, &pmap->pm_pvchunk, pc_list) {
1205 // if ((cpu_feature2 & CPUID2_POPCNT) == 0)
1206 bit_count((bitstr_t *)pc->pc_map, 0,
1207 sizeof(pc->pc_map) * NBBY, &free);
1208 #if 0
1209 free = popcnt_pc_map_pq(pc->pc_map);
1210 #endif
1211 if (free == 0)
1212 break;
1213 avail += free;
1214 if (avail >= needed)
1215 break;
1216 }
1217 for (reclaimed = false; avail < needed; avail += _NPCPV) {
1218 m = vm_page_alloc_noobj(VM_ALLOC_WIRED);
1219 if (m == NULL) {
1220 m = reclaim_pv_chunk(pmap, lockp);
1221 if (m == NULL)
1222 goto retry;
1223 reclaimed = true;
1224 }
1225 PV_STAT(atomic_add_int(&pc_chunk_count, 1));
1226 PV_STAT(atomic_add_int(&pc_chunk_allocs, 1));
1227 dump_add_page(m->phys_addr);
1228 pc = (void *)PHYS_TO_DMAP(m->phys_addr);
1229 pc->pc_pmap = pmap;
1230 pc->pc_map[0] = PC_FREE0;
1231 pc->pc_map[1] = PC_FREE1;
1232 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);
1233 TAILQ_INSERT_TAIL(&new_tail, pc, pc_lru);
1234 PV_STAT(atomic_add_int(&pv_entry_spare, _NPCPV));
1235
1236 /*
1237 * The reclaim might have freed a chunk from the current pmap.
1238 * If that chunk contained available entries, we need to
1239 * re-count the number of available entries.
1240 */
1241 if (reclaimed)
1242 goto retry;
1243 }
1244 if (!TAILQ_EMPTY(&new_tail)) {
1245 mtx_lock(&pv_chunks_mutex);
1246 TAILQ_CONCAT(&pv_chunks, &new_tail, pc_lru);
1247 mtx_unlock(&pv_chunks_mutex);
1248 }
1249 }
1250
1251 /*
1252 * First find and then remove the pv entry for the specified pmap and virtual
1253 * address from the specified pv list. Returns the pv entry if found and NULL
1254 * otherwise. This operation can be performed on pv lists for either 4KB or
1255 * 2MB page mappings.
1256 */
1257 static __inline pv_entry_t
pmap_pvh_remove(struct md_page * pvh,pmap_t pmap,vm_offset_t va)1258 pmap_pvh_remove(struct md_page *pvh, pmap_t pmap, vm_offset_t va)
1259 {
1260 pv_entry_t pv;
1261
1262 TAILQ_FOREACH(pv, &pvh->pv_list, pv_link) {
1263 #ifdef INVARIANTS
1264 if (PV_PMAP(pv) == NULL) {
1265 printf("corrupted pv_chunk/pv %p\n", pv);
1266 printf("pv_chunk: %64D\n", pv_to_chunk(pv), ":");
1267 }
1268 MPASS(PV_PMAP(pv) != NULL);
1269 MPASS(pv->pv_va != 0);
1270 #endif
1271 if (pmap == PV_PMAP(pv) && va == pv->pv_va) {
1272 TAILQ_REMOVE(&pvh->pv_list, pv, pv_link);
1273 pvh->pv_gen++;
1274 break;
1275 }
1276 }
1277 return (pv);
1278 }
1279
1280 /*
1281 * After demotion from a 2MB page mapping to 512 4KB page mappings,
1282 * destroy the pv entry for the 2MB page mapping and reinstantiate the pv
1283 * entries for each of the 4KB page mappings.
1284 */
1285 static void
pmap_pv_demote_l3e(pmap_t pmap,vm_offset_t va,vm_paddr_t pa,struct rwlock ** lockp)1286 pmap_pv_demote_l3e(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,
1287 struct rwlock **lockp)
1288 {
1289 struct md_page *pvh;
1290 struct pv_chunk *pc;
1291 pv_entry_t pv;
1292 vm_offset_t va_last;
1293 vm_page_t m;
1294 int bit, field;
1295
1296 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
1297 KASSERT((pa & L3_PAGE_MASK) == 0,
1298 ("pmap_pv_demote_pde: pa is not 2mpage aligned"));
1299 CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa);
1300
1301 /*
1302 * Transfer the 2mpage's pv entry for this mapping to the first
1303 * page's pv list. Once this transfer begins, the pv list lock
1304 * must not be released until the last pv entry is reinstantiated.
1305 */
1306 pvh = pa_to_pvh(pa);
1307 va = trunc_2mpage(va);
1308 pv = pmap_pvh_remove(pvh, pmap, va);
1309 KASSERT(pv != NULL, ("pmap_pv_demote_pde: pv not found"));
1310 m = PHYS_TO_VM_PAGE(pa);
1311 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_link);
1312
1313 m->md.pv_gen++;
1314 /* Instantiate the remaining NPTEPG - 1 pv entries. */
1315 PV_STAT(atomic_add_long(&pv_entry_allocs, NPTEPG - 1));
1316 va_last = va + L3_PAGE_SIZE - PAGE_SIZE;
1317 for (;;) {
1318 pc = TAILQ_FIRST(&pmap->pm_pvchunk);
1319 KASSERT(pc->pc_map[0] != 0 || pc->pc_map[1] != 0
1320 , ("pmap_pv_demote_pde: missing spare"));
1321 for (field = 0; field < _NPCM; field++) {
1322 while (pc->pc_map[field]) {
1323 bit = cnttzd(pc->pc_map[field]);
1324 pc->pc_map[field] &= ~(1ul << bit);
1325 pv = &pc->pc_pventry[field * 64 + bit];
1326 va += PAGE_SIZE;
1327 pv->pv_va = va;
1328 m++;
1329 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
1330 ("pmap_pv_demote_pde: page %p is not managed", m));
1331 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_link);
1332
1333 m->md.pv_gen++;
1334 if (va == va_last)
1335 goto out;
1336 }
1337 }
1338 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
1339 TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list);
1340 }
1341 out:
1342 if (pc->pc_map[0] == 0 && pc->pc_map[1] == 0) {
1343 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
1344 TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list);
1345 }
1346 PV_STAT(atomic_add_long(&pv_entry_count, NPTEPG - 1));
1347 PV_STAT(atomic_subtract_int(&pv_entry_spare, NPTEPG - 1));
1348 }
1349
1350 static void
reclaim_pv_chunk_leave_pmap(pmap_t pmap,pmap_t locked_pmap)1351 reclaim_pv_chunk_leave_pmap(pmap_t pmap, pmap_t locked_pmap)
1352 {
1353
1354 if (pmap == NULL)
1355 return;
1356 pmap_invalidate_all(pmap);
1357 if (pmap != locked_pmap)
1358 PMAP_UNLOCK(pmap);
1359 }
1360
1361 /*
1362 * We are in a serious low memory condition. Resort to
1363 * drastic measures to free some pages so we can allocate
1364 * another pv entry chunk.
1365 *
1366 * Returns NULL if PV entries were reclaimed from the specified pmap.
1367 *
1368 * We do not, however, unmap 2mpages because subsequent accesses will
1369 * allocate per-page pv entries until repromotion occurs, thereby
1370 * exacerbating the shortage of free pv entries.
1371 */
1372 static int active_reclaims = 0;
1373 static vm_page_t
reclaim_pv_chunk(pmap_t locked_pmap,struct rwlock ** lockp)1374 reclaim_pv_chunk(pmap_t locked_pmap, struct rwlock **lockp)
1375 {
1376 struct pv_chunk *pc, *pc_marker, *pc_marker_end;
1377 struct pv_chunk_header pc_marker_b, pc_marker_end_b;
1378 struct md_page *pvh;
1379 pml3_entry_t *l3e;
1380 pmap_t next_pmap, pmap;
1381 pt_entry_t *pte, tpte;
1382 pv_entry_t pv;
1383 vm_offset_t va;
1384 vm_page_t m, m_pc;
1385 struct spglist free;
1386 uint64_t inuse;
1387 int bit, field, freed;
1388
1389 PMAP_LOCK_ASSERT(locked_pmap, MA_OWNED);
1390 KASSERT(lockp != NULL, ("reclaim_pv_chunk: lockp is NULL"));
1391 pmap = NULL;
1392 m_pc = NULL;
1393 SLIST_INIT(&free);
1394 bzero(&pc_marker_b, sizeof(pc_marker_b));
1395 bzero(&pc_marker_end_b, sizeof(pc_marker_end_b));
1396 pc_marker = (struct pv_chunk *)&pc_marker_b;
1397 pc_marker_end = (struct pv_chunk *)&pc_marker_end_b;
1398
1399 mtx_lock(&pv_chunks_mutex);
1400 active_reclaims++;
1401 TAILQ_INSERT_HEAD(&pv_chunks, pc_marker, pc_lru);
1402 TAILQ_INSERT_TAIL(&pv_chunks, pc_marker_end, pc_lru);
1403 while ((pc = TAILQ_NEXT(pc_marker, pc_lru)) != pc_marker_end &&
1404 SLIST_EMPTY(&free)) {
1405 next_pmap = pc->pc_pmap;
1406 if (next_pmap == NULL) {
1407 /*
1408 * The next chunk is a marker. However, it is
1409 * not our marker, so active_reclaims must be
1410 * > 1. Consequently, the next_chunk code
1411 * will not rotate the pv_chunks list.
1412 */
1413 goto next_chunk;
1414 }
1415 mtx_unlock(&pv_chunks_mutex);
1416
1417 /*
1418 * A pv_chunk can only be removed from the pc_lru list
1419 * when both pc_chunks_mutex is owned and the
1420 * corresponding pmap is locked.
1421 */
1422 if (pmap != next_pmap) {
1423 reclaim_pv_chunk_leave_pmap(pmap, locked_pmap);
1424 pmap = next_pmap;
1425 /* Avoid deadlock and lock recursion. */
1426 if (pmap > locked_pmap) {
1427 RELEASE_PV_LIST_LOCK(lockp);
1428 PMAP_LOCK(pmap);
1429 mtx_lock(&pv_chunks_mutex);
1430 continue;
1431 } else if (pmap != locked_pmap) {
1432 if (PMAP_TRYLOCK(pmap)) {
1433 mtx_lock(&pv_chunks_mutex);
1434 continue;
1435 } else {
1436 pmap = NULL; /* pmap is not locked */
1437 mtx_lock(&pv_chunks_mutex);
1438 pc = TAILQ_NEXT(pc_marker, pc_lru);
1439 if (pc == NULL ||
1440 pc->pc_pmap != next_pmap)
1441 continue;
1442 goto next_chunk;
1443 }
1444 }
1445 }
1446
1447 /*
1448 * Destroy every non-wired, 4 KB page mapping in the chunk.
1449 */
1450 freed = 0;
1451 for (field = 0; field < _NPCM; field++) {
1452 for (inuse = ~pc->pc_map[field] & pc_freemask[field];
1453 inuse != 0; inuse &= ~(1UL << bit)) {
1454 bit = cnttzd(inuse);
1455 pv = &pc->pc_pventry[field * 64 + bit];
1456 va = pv->pv_va;
1457 l3e = pmap_pml3e(pmap, va);
1458 if ((be64toh(*l3e) & RPTE_LEAF) != 0)
1459 continue;
1460 pte = pmap_l3e_to_pte(l3e, va);
1461 if ((be64toh(*pte) & PG_W) != 0)
1462 continue;
1463 tpte = be64toh(pte_load_clear(pte));
1464 m = PHYS_TO_VM_PAGE(tpte & PG_FRAME);
1465 if ((tpte & (PG_M | PG_RW)) == (PG_M | PG_RW))
1466 vm_page_dirty(m);
1467 if ((tpte & PG_A) != 0)
1468 vm_page_aflag_set(m, PGA_REFERENCED);
1469 CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m);
1470 TAILQ_REMOVE(&m->md.pv_list, pv, pv_link);
1471
1472 m->md.pv_gen++;
1473 if (TAILQ_EMPTY(&m->md.pv_list) &&
1474 (m->flags & PG_FICTITIOUS) == 0) {
1475 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
1476 if (TAILQ_EMPTY(&pvh->pv_list)) {
1477 vm_page_aflag_clear(m,
1478 PGA_WRITEABLE);
1479 }
1480 }
1481 pc->pc_map[field] |= 1UL << bit;
1482 pmap_unuse_pt(pmap, va, be64toh(*l3e), &free);
1483 freed++;
1484 }
1485 }
1486 if (freed == 0) {
1487 mtx_lock(&pv_chunks_mutex);
1488 goto next_chunk;
1489 }
1490 /* Every freed mapping is for a 4 KB page. */
1491 pmap_resident_count_dec(pmap, freed);
1492 PV_STAT(atomic_add_long(&pv_entry_frees, freed));
1493 PV_STAT(atomic_add_int(&pv_entry_spare, freed));
1494 PV_STAT(atomic_subtract_long(&pv_entry_count, freed));
1495 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
1496 if (pc->pc_map[0] == PC_FREE0 && pc->pc_map[1] == PC_FREE1) {
1497 PV_STAT(atomic_subtract_int(&pv_entry_spare, _NPCPV));
1498 PV_STAT(atomic_subtract_int(&pc_chunk_count, 1));
1499 PV_STAT(atomic_add_int(&pc_chunk_frees, 1));
1500 /* Entire chunk is free; return it. */
1501 m_pc = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pc));
1502 dump_drop_page(m_pc->phys_addr);
1503 mtx_lock(&pv_chunks_mutex);
1504 TAILQ_REMOVE(&pv_chunks, pc, pc_lru);
1505 break;
1506 }
1507 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);
1508 mtx_lock(&pv_chunks_mutex);
1509 /* One freed pv entry in locked_pmap is sufficient. */
1510 if (pmap == locked_pmap)
1511 break;
1512 next_chunk:
1513 TAILQ_REMOVE(&pv_chunks, pc_marker, pc_lru);
1514 TAILQ_INSERT_AFTER(&pv_chunks, pc, pc_marker, pc_lru);
1515 if (active_reclaims == 1 && pmap != NULL) {
1516 /*
1517 * Rotate the pv chunks list so that we do not
1518 * scan the same pv chunks that could not be
1519 * freed (because they contained a wired
1520 * and/or superpage mapping) on every
1521 * invocation of reclaim_pv_chunk().
1522 */
1523 while ((pc = TAILQ_FIRST(&pv_chunks)) != pc_marker) {
1524 MPASS(pc->pc_pmap != NULL);
1525 TAILQ_REMOVE(&pv_chunks, pc, pc_lru);
1526 TAILQ_INSERT_TAIL(&pv_chunks, pc, pc_lru);
1527 }
1528 }
1529 }
1530 TAILQ_REMOVE(&pv_chunks, pc_marker, pc_lru);
1531 TAILQ_REMOVE(&pv_chunks, pc_marker_end, pc_lru);
1532 active_reclaims--;
1533 mtx_unlock(&pv_chunks_mutex);
1534 reclaim_pv_chunk_leave_pmap(pmap, locked_pmap);
1535 if (m_pc == NULL && !SLIST_EMPTY(&free)) {
1536 m_pc = SLIST_FIRST(&free);
1537 SLIST_REMOVE_HEAD(&free, plinks.s.ss);
1538 /* Recycle a freed page table page. */
1539 m_pc->ref_count = 1;
1540 }
1541 vm_page_free_pages_toq(&free, true);
1542 return (m_pc);
1543 }
1544
1545 /*
1546 * free the pv_entry back to the free list
1547 */
1548 static void
free_pv_entry(pmap_t pmap,pv_entry_t pv)1549 free_pv_entry(pmap_t pmap, pv_entry_t pv)
1550 {
1551 struct pv_chunk *pc;
1552 int idx, field, bit;
1553
1554 #ifdef VERBOSE_PV
1555 if (pmap != kernel_pmap)
1556 printf("%s(%p, %p)\n", __func__, pmap, pv);
1557 #endif
1558 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
1559 PV_STAT(atomic_add_long(&pv_entry_frees, 1));
1560 PV_STAT(atomic_add_int(&pv_entry_spare, 1));
1561 PV_STAT(atomic_subtract_long(&pv_entry_count, 1));
1562 pc = pv_to_chunk(pv);
1563 idx = pv - &pc->pc_pventry[0];
1564 field = idx / 64;
1565 bit = idx % 64;
1566 pc->pc_map[field] |= 1ul << bit;
1567 if (pc->pc_map[0] != PC_FREE0 || pc->pc_map[1] != PC_FREE1) {
1568 /* 98% of the time, pc is already at the head of the list. */
1569 if (__predict_false(pc != TAILQ_FIRST(&pmap->pm_pvchunk))) {
1570 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
1571 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);
1572 }
1573 return;
1574 }
1575 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
1576 free_pv_chunk(pc);
1577 }
1578
1579 static void
free_pv_chunk(struct pv_chunk * pc)1580 free_pv_chunk(struct pv_chunk *pc)
1581 {
1582 vm_page_t m;
1583
1584 mtx_lock(&pv_chunks_mutex);
1585 TAILQ_REMOVE(&pv_chunks, pc, pc_lru);
1586 mtx_unlock(&pv_chunks_mutex);
1587 PV_STAT(atomic_subtract_int(&pv_entry_spare, _NPCPV));
1588 PV_STAT(atomic_subtract_int(&pc_chunk_count, 1));
1589 PV_STAT(atomic_add_int(&pc_chunk_frees, 1));
1590 /* entire chunk is free, return it */
1591 m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pc));
1592 dump_drop_page(m->phys_addr);
1593 vm_page_unwire_noq(m);
1594 vm_page_free(m);
1595 }
1596
1597 /*
1598 * Returns a new PV entry, allocating a new PV chunk from the system when
1599 * needed. If this PV chunk allocation fails and a PV list lock pointer was
1600 * given, a PV chunk is reclaimed from an arbitrary pmap. Otherwise, NULL is
1601 * returned.
1602 *
1603 * The given PV list lock may be released.
1604 */
1605 static pv_entry_t
get_pv_entry(pmap_t pmap,struct rwlock ** lockp)1606 get_pv_entry(pmap_t pmap, struct rwlock **lockp)
1607 {
1608 int bit, field;
1609 pv_entry_t pv;
1610 struct pv_chunk *pc;
1611 vm_page_t m;
1612
1613 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
1614 PV_STAT(atomic_add_long(&pv_entry_allocs, 1));
1615 retry:
1616 pc = TAILQ_FIRST(&pmap->pm_pvchunk);
1617 if (pc != NULL) {
1618 for (field = 0; field < _NPCM; field++) {
1619 if (pc->pc_map[field]) {
1620 bit = cnttzd(pc->pc_map[field]);
1621 break;
1622 }
1623 }
1624 if (field < _NPCM) {
1625 pv = &pc->pc_pventry[field * 64 + bit];
1626 pc->pc_map[field] &= ~(1ul << bit);
1627 /* If this was the last item, move it to tail */
1628 if (pc->pc_map[0] == 0 && pc->pc_map[1] == 0) {
1629 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
1630 TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc,
1631 pc_list);
1632 }
1633 PV_STAT(atomic_add_long(&pv_entry_count, 1));
1634 PV_STAT(atomic_subtract_int(&pv_entry_spare, 1));
1635 MPASS(PV_PMAP(pv) != NULL);
1636 return (pv);
1637 }
1638 }
1639 /* No free items, allocate another chunk */
1640 m = vm_page_alloc_noobj(VM_ALLOC_WIRED);
1641 if (m == NULL) {
1642 if (lockp == NULL) {
1643 PV_STAT(pc_chunk_tryfail++);
1644 return (NULL);
1645 }
1646 m = reclaim_pv_chunk(pmap, lockp);
1647 if (m == NULL)
1648 goto retry;
1649 }
1650 PV_STAT(atomic_add_int(&pc_chunk_count, 1));
1651 PV_STAT(atomic_add_int(&pc_chunk_allocs, 1));
1652 dump_add_page(m->phys_addr);
1653 pc = (void *)PHYS_TO_DMAP(m->phys_addr);
1654 pc->pc_pmap = pmap;
1655 pc->pc_map[0] = PC_FREE0 & ~1ul; /* preallocated bit 0 */
1656 pc->pc_map[1] = PC_FREE1;
1657 mtx_lock(&pv_chunks_mutex);
1658 TAILQ_INSERT_TAIL(&pv_chunks, pc, pc_lru);
1659 mtx_unlock(&pv_chunks_mutex);
1660 pv = &pc->pc_pventry[0];
1661 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);
1662 PV_STAT(atomic_add_long(&pv_entry_count, 1));
1663 PV_STAT(atomic_add_int(&pv_entry_spare, _NPCPV - 1));
1664 MPASS(PV_PMAP(pv) != NULL);
1665 return (pv);
1666 }
1667
1668 #if VM_NRESERVLEVEL > 0
1669 /*
1670 * After promotion from 512 4KB page mappings to a single 2MB page mapping,
1671 * replace the many pv entries for the 4KB page mappings by a single pv entry
1672 * for the 2MB page mapping.
1673 */
1674 static void
pmap_pv_promote_l3e(pmap_t pmap,vm_offset_t va,vm_paddr_t pa,struct rwlock ** lockp)1675 pmap_pv_promote_l3e(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,
1676 struct rwlock **lockp)
1677 {
1678 struct md_page *pvh;
1679 pv_entry_t pv;
1680 vm_offset_t va_last;
1681 vm_page_t m;
1682
1683 KASSERT((pa & L3_PAGE_MASK) == 0,
1684 ("pmap_pv_promote_pde: pa is not 2mpage aligned"));
1685 CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa);
1686
1687 /*
1688 * Transfer the first page's pv entry for this mapping to the 2mpage's
1689 * pv list. Aside from avoiding the cost of a call to get_pv_entry(),
1690 * a transfer avoids the possibility that get_pv_entry() calls
1691 * reclaim_pv_chunk() and that reclaim_pv_chunk() removes one of the
1692 * mappings that is being promoted.
1693 */
1694 m = PHYS_TO_VM_PAGE(pa);
1695 va = trunc_2mpage(va);
1696 pv = pmap_pvh_remove(&m->md, pmap, va);
1697 KASSERT(pv != NULL, ("pmap_pv_promote_pde: pv not found"));
1698 pvh = pa_to_pvh(pa);
1699 TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_link);
1700 pvh->pv_gen++;
1701 /* Free the remaining NPTEPG - 1 pv entries. */
1702 va_last = va + L3_PAGE_SIZE - PAGE_SIZE;
1703 do {
1704 m++;
1705 va += PAGE_SIZE;
1706 pmap_pvh_free(&m->md, pmap, va);
1707 } while (va < va_last);
1708 }
1709 #endif /* VM_NRESERVLEVEL > 0 */
1710
1711 /*
1712 * First find and then destroy the pv entry for the specified pmap and virtual
1713 * address. This operation can be performed on pv lists for either 4KB or 2MB
1714 * page mappings.
1715 */
1716 static void
pmap_pvh_free(struct md_page * pvh,pmap_t pmap,vm_offset_t va)1717 pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va)
1718 {
1719 pv_entry_t pv;
1720
1721 pv = pmap_pvh_remove(pvh, pmap, va);
1722 KASSERT(pv != NULL, ("pmap_pvh_free: pv not found"));
1723 free_pv_entry(pmap, pv);
1724 }
1725
1726 /*
1727 * Conditionally create the PV entry for a 4KB page mapping if the required
1728 * memory can be allocated without resorting to reclamation.
1729 */
1730 static bool
pmap_try_insert_pv_entry(pmap_t pmap,vm_offset_t va,vm_page_t m,struct rwlock ** lockp)1731 pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, vm_page_t m,
1732 struct rwlock **lockp)
1733 {
1734 pv_entry_t pv;
1735
1736 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
1737 /* Pass NULL instead of the lock pointer to disable reclamation. */
1738 if ((pv = get_pv_entry(pmap, NULL)) != NULL) {
1739 pv->pv_va = va;
1740 CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m);
1741 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_link);
1742 m->md.pv_gen++;
1743 return (true);
1744 } else
1745 return (false);
1746 }
1747
1748 vm_paddr_t phys_avail_debug[2 * VM_PHYSSEG_MAX];
1749 #ifdef INVARIANTS
1750 static void
validate_addr(vm_paddr_t addr,vm_size_t size)1751 validate_addr(vm_paddr_t addr, vm_size_t size)
1752 {
1753 vm_paddr_t end = addr + size;
1754 bool found = false;
1755
1756 for (int i = 0; i < 2 * phys_avail_count; i += 2) {
1757 if (addr >= phys_avail_debug[i] &&
1758 end <= phys_avail_debug[i + 1]) {
1759 found = true;
1760 break;
1761 }
1762 }
1763 KASSERT(found, ("%#lx-%#lx outside of initial phys_avail array",
1764 addr, end));
1765 }
1766 #else
validate_addr(vm_paddr_t addr,vm_size_t size)1767 static void validate_addr(vm_paddr_t addr, vm_size_t size) {}
1768 #endif
1769 #define DMAP_PAGE_BITS (RPTE_VALID | RPTE_LEAF | RPTE_EAA_MASK | PG_M | PG_A)
1770
1771 static vm_paddr_t
alloc_pt_page(void)1772 alloc_pt_page(void)
1773 {
1774 vm_paddr_t page;
1775
1776 page = allocpages(1);
1777 pagezero(PHYS_TO_DMAP(page));
1778 return (page);
1779 }
1780
1781 static void
mmu_radix_dmap_range(vm_paddr_t start,vm_paddr_t end)1782 mmu_radix_dmap_range(vm_paddr_t start, vm_paddr_t end)
1783 {
1784 pt_entry_t *pte, pteval;
1785 vm_paddr_t page;
1786
1787 if (bootverbose)
1788 printf("%s %lx -> %lx\n", __func__, start, end);
1789 while (start < end) {
1790 pteval = start | DMAP_PAGE_BITS;
1791 pte = pmap_pml1e(kernel_pmap, PHYS_TO_DMAP(start));
1792 if ((be64toh(*pte) & RPTE_VALID) == 0) {
1793 page = alloc_pt_page();
1794 pde_store(pte, page);
1795 }
1796 pte = pmap_l1e_to_l2e(pte, PHYS_TO_DMAP(start));
1797 if ((start & L2_PAGE_MASK) == 0 &&
1798 end - start >= L2_PAGE_SIZE) {
1799 start += L2_PAGE_SIZE;
1800 goto done;
1801 } else if ((be64toh(*pte) & RPTE_VALID) == 0) {
1802 page = alloc_pt_page();
1803 pde_store(pte, page);
1804 }
1805
1806 pte = pmap_l2e_to_l3e(pte, PHYS_TO_DMAP(start));
1807 if ((start & L3_PAGE_MASK) == 0 &&
1808 end - start >= L3_PAGE_SIZE) {
1809 start += L3_PAGE_SIZE;
1810 goto done;
1811 } else if ((be64toh(*pte) & RPTE_VALID) == 0) {
1812 page = alloc_pt_page();
1813 pde_store(pte, page);
1814 }
1815 pte = pmap_l3e_to_pte(pte, PHYS_TO_DMAP(start));
1816 start += PAGE_SIZE;
1817 done:
1818 pte_store(pte, pteval);
1819 }
1820 }
1821
1822 static void
mmu_radix_dmap_populate(vm_size_t hwphyssz)1823 mmu_radix_dmap_populate(vm_size_t hwphyssz)
1824 {
1825 vm_paddr_t start, end;
1826
1827 for (int i = 0; i < pregions_sz; i++) {
1828 start = pregions[i].mr_start;
1829 end = start + pregions[i].mr_size;
1830 if (hwphyssz && start >= hwphyssz)
1831 break;
1832 if (hwphyssz && hwphyssz < end)
1833 end = hwphyssz;
1834 mmu_radix_dmap_range(start, end);
1835 }
1836 }
1837
1838 static void
mmu_radix_setup_pagetables(vm_size_t hwphyssz)1839 mmu_radix_setup_pagetables(vm_size_t hwphyssz)
1840 {
1841 vm_paddr_t ptpages, pages;
1842 pt_entry_t *pte;
1843 vm_paddr_t l1phys;
1844
1845 bzero(kernel_pmap, sizeof(struct pmap));
1846 PMAP_LOCK_INIT(kernel_pmap);
1847 vm_radix_init(&kernel_pmap->pm_radix);
1848
1849 ptpages = allocpages(3);
1850 l1phys = moea64_bootstrap_alloc(RADIX_PGD_SIZE, RADIX_PGD_SIZE);
1851 validate_addr(l1phys, RADIX_PGD_SIZE);
1852 if (bootverbose)
1853 printf("l1phys=%lx\n", l1phys);
1854 MPASS((l1phys & (RADIX_PGD_SIZE-1)) == 0);
1855 for (int i = 0; i < RADIX_PGD_SIZE/PAGE_SIZE; i++)
1856 pagezero(PHYS_TO_DMAP(l1phys + i * PAGE_SIZE));
1857 kernel_pmap->pm_pml1 = (pml1_entry_t *)PHYS_TO_DMAP(l1phys);
1858
1859 mmu_radix_dmap_populate(hwphyssz);
1860
1861 /*
1862 * Create page tables for first 128MB of KVA
1863 */
1864 pages = ptpages;
1865 pte = pmap_pml1e(kernel_pmap, VM_MIN_KERNEL_ADDRESS);
1866 *pte = htobe64(pages | RPTE_VALID | RPTE_SHIFT);
1867 pages += PAGE_SIZE;
1868 pte = pmap_l1e_to_l2e(pte, VM_MIN_KERNEL_ADDRESS);
1869 *pte = htobe64(pages | RPTE_VALID | RPTE_SHIFT);
1870 pages += PAGE_SIZE;
1871 pte = pmap_l2e_to_l3e(pte, VM_MIN_KERNEL_ADDRESS);
1872 /*
1873 * the kernel page table pages need to be preserved in
1874 * phys_avail and not overlap with previous allocations
1875 */
1876 pages = allocpages(nkpt);
1877 if (bootverbose) {
1878 printf("phys_avail after dmap populate and nkpt allocation\n");
1879 for (int j = 0; j < 2 * phys_avail_count; j+=2)
1880 printf("phys_avail[%d]=%08lx - phys_avail[%d]=%08lx\n",
1881 j, phys_avail[j], j + 1, phys_avail[j + 1]);
1882 }
1883 KPTphys = pages;
1884 for (int i = 0; i < nkpt; i++, pte++, pages += PAGE_SIZE)
1885 *pte = htobe64(pages | RPTE_VALID | RPTE_SHIFT);
1886 kernel_vm_end = VM_MIN_KERNEL_ADDRESS + nkpt * L3_PAGE_SIZE;
1887 if (bootverbose)
1888 printf("kernel_pmap pml1 %p\n", kernel_pmap->pm_pml1);
1889 /*
1890 * Add a physical memory segment (vm_phys_seg) corresponding to the
1891 * preallocated kernel page table pages so that vm_page structures
1892 * representing these pages will be created. The vm_page structures
1893 * are required for promotion of the corresponding kernel virtual
1894 * addresses to superpage mappings.
1895 */
1896 vm_phys_add_seg(KPTphys, KPTphys + ptoa(nkpt));
1897 }
1898
1899 static void
mmu_radix_early_bootstrap(vm_offset_t start,vm_offset_t end)1900 mmu_radix_early_bootstrap(vm_offset_t start, vm_offset_t end)
1901 {
1902 vm_paddr_t kpstart, kpend;
1903 vm_size_t physsz, hwphyssz;
1904 //uint64_t l2virt;
1905 int rm_pavail, proctab_size;
1906 int i, j;
1907
1908 kpstart = start & ~DMAP_BASE_ADDRESS;
1909 kpend = end & ~DMAP_BASE_ADDRESS;
1910
1911 /* Get physical memory regions from firmware */
1912 mem_regions(&pregions, &pregions_sz, ®ions, ®ions_sz);
1913 CTR0(KTR_PMAP, "mmu_radix_early_bootstrap: physical memory");
1914
1915 if (2 * VM_PHYSSEG_MAX < regions_sz)
1916 panic("mmu_radix_early_bootstrap: phys_avail too small");
1917
1918 if (bootverbose)
1919 for (int i = 0; i < regions_sz; i++)
1920 printf("regions[%d].mr_start=%lx regions[%d].mr_size=%lx\n",
1921 i, regions[i].mr_start, i, regions[i].mr_size);
1922 /*
1923 * XXX workaround a simulator bug
1924 */
1925 for (int i = 0; i < regions_sz; i++)
1926 if (regions[i].mr_start & PAGE_MASK) {
1927 regions[i].mr_start += PAGE_MASK;
1928 regions[i].mr_start &= ~PAGE_MASK;
1929 regions[i].mr_size &= ~PAGE_MASK;
1930 }
1931 if (bootverbose)
1932 for (int i = 0; i < pregions_sz; i++)
1933 printf("pregions[%d].mr_start=%lx pregions[%d].mr_size=%lx\n",
1934 i, pregions[i].mr_start, i, pregions[i].mr_size);
1935
1936 phys_avail_count = 0;
1937 physsz = 0;
1938 hwphyssz = 0;
1939 TUNABLE_ULONG_FETCH("hw.physmem", (u_long *) &hwphyssz);
1940 for (i = 0, j = 0; i < regions_sz; i++) {
1941 if (bootverbose)
1942 printf("regions[%d].mr_start=%016lx regions[%d].mr_size=%016lx\n",
1943 i, regions[i].mr_start, i, regions[i].mr_size);
1944
1945 if (regions[i].mr_size < PAGE_SIZE)
1946 continue;
1947
1948 if (hwphyssz != 0 &&
1949 (physsz + regions[i].mr_size) >= hwphyssz) {
1950 if (physsz < hwphyssz) {
1951 phys_avail[j] = regions[i].mr_start;
1952 phys_avail[j + 1] = regions[i].mr_start +
1953 (hwphyssz - physsz);
1954 physsz = hwphyssz;
1955 phys_avail_count++;
1956 dump_avail[j] = phys_avail[j];
1957 dump_avail[j + 1] = phys_avail[j + 1];
1958 }
1959 break;
1960 }
1961 phys_avail[j] = regions[i].mr_start;
1962 phys_avail[j + 1] = regions[i].mr_start + regions[i].mr_size;
1963 dump_avail[j] = phys_avail[j];
1964 dump_avail[j + 1] = phys_avail[j + 1];
1965
1966 phys_avail_count++;
1967 physsz += regions[i].mr_size;
1968 j += 2;
1969 }
1970
1971 /* Check for overlap with the kernel and exception vectors */
1972 rm_pavail = 0;
1973 for (j = 0; j < 2 * phys_avail_count; j+=2) {
1974 if (phys_avail[j] < EXC_LAST)
1975 phys_avail[j] += EXC_LAST;
1976
1977 if (phys_avail[j] >= kpstart &&
1978 phys_avail[j + 1] <= kpend) {
1979 phys_avail[j] = phys_avail[j + 1] = ~0;
1980 rm_pavail++;
1981 continue;
1982 }
1983
1984 if (kpstart >= phys_avail[j] &&
1985 kpstart < phys_avail[j + 1]) {
1986 if (kpend < phys_avail[j + 1]) {
1987 phys_avail[2 * phys_avail_count] =
1988 (kpend & ~PAGE_MASK) + PAGE_SIZE;
1989 phys_avail[2 * phys_avail_count + 1] =
1990 phys_avail[j + 1];
1991 phys_avail_count++;
1992 }
1993
1994 phys_avail[j + 1] = kpstart & ~PAGE_MASK;
1995 }
1996
1997 if (kpend >= phys_avail[j] &&
1998 kpend < phys_avail[j + 1]) {
1999 if (kpstart > phys_avail[j]) {
2000 phys_avail[2 * phys_avail_count] = phys_avail[j];
2001 phys_avail[2 * phys_avail_count + 1] =
2002 kpstart & ~PAGE_MASK;
2003 phys_avail_count++;
2004 }
2005
2006 phys_avail[j] = (kpend & ~PAGE_MASK) +
2007 PAGE_SIZE;
2008 }
2009 }
2010 qsort(phys_avail, 2 * phys_avail_count, sizeof(phys_avail[0]), pa_cmp);
2011 for (i = 0; i < 2 * phys_avail_count; i++)
2012 phys_avail_debug[i] = phys_avail[i];
2013
2014 /* Remove physical available regions marked for removal (~0) */
2015 if (rm_pavail) {
2016 phys_avail_count -= rm_pavail;
2017 for (i = 2 * phys_avail_count;
2018 i < 2*(phys_avail_count + rm_pavail); i+=2)
2019 phys_avail[i] = phys_avail[i + 1] = 0;
2020 }
2021 if (bootverbose) {
2022 printf("phys_avail ranges after filtering:\n");
2023 for (j = 0; j < 2 * phys_avail_count; j+=2)
2024 printf("phys_avail[%d]=%08lx - phys_avail[%d]=%08lx\n",
2025 j, phys_avail[j], j + 1, phys_avail[j + 1]);
2026 }
2027 physmem = btoc(physsz);
2028
2029 /* XXX assume we're running non-virtualized and
2030 * we don't support BHYVE
2031 */
2032 if (isa3_pid_bits == 0)
2033 isa3_pid_bits = 20;
2034 if (powernv_enabled) {
2035 parttab_phys =
2036 moea64_bootstrap_alloc(PARTTAB_SIZE, PARTTAB_SIZE);
2037 validate_addr(parttab_phys, PARTTAB_SIZE);
2038 for (int i = 0; i < PARTTAB_SIZE/PAGE_SIZE; i++)
2039 pagezero(PHYS_TO_DMAP(parttab_phys + i * PAGE_SIZE));
2040
2041 }
2042 proctab_size = 1UL << PROCTAB_SIZE_SHIFT;
2043 proctab0pa = moea64_bootstrap_alloc(proctab_size, proctab_size);
2044 validate_addr(proctab0pa, proctab_size);
2045 for (int i = 0; i < proctab_size/PAGE_SIZE; i++)
2046 pagezero(PHYS_TO_DMAP(proctab0pa + i * PAGE_SIZE));
2047
2048 mmu_radix_setup_pagetables(hwphyssz);
2049 }
2050
2051 static void
mmu_radix_late_bootstrap(vm_offset_t start,vm_offset_t end)2052 mmu_radix_late_bootstrap(vm_offset_t start, vm_offset_t end)
2053 {
2054 int i;
2055 vm_paddr_t pa;
2056 void *dpcpu;
2057 vm_offset_t va;
2058
2059 /*
2060 * Set up the Open Firmware pmap and add its mappings if not in real
2061 * mode.
2062 */
2063 if (bootverbose)
2064 printf("%s enter\n", __func__);
2065
2066 /*
2067 * Calculate the last available physical address, and reserve the
2068 * vm_page_array (upper bound).
2069 */
2070 Maxmem = 0;
2071 for (i = 0; phys_avail[i + 1] != 0; i += 2)
2072 Maxmem = MAX(Maxmem, powerpc_btop(phys_avail[i + 1]));
2073
2074 /*
2075 * Remap any early IO mappings (console framebuffer, etc.)
2076 */
2077 bs_remap_earlyboot();
2078
2079 /*
2080 * Allocate a kernel stack with a guard page for thread0 and map it
2081 * into the kernel page map.
2082 */
2083 pa = allocpages(kstack_pages);
2084 va = virtual_avail + KSTACK_GUARD_PAGES * PAGE_SIZE;
2085 virtual_avail = va + kstack_pages * PAGE_SIZE;
2086 CTR2(KTR_PMAP, "moea64_bootstrap: kstack0 at %#x (%#x)", pa, va);
2087 thread0.td_kstack = va;
2088 for (i = 0; i < kstack_pages; i++) {
2089 mmu_radix_kenter(va, pa);
2090 pa += PAGE_SIZE;
2091 va += PAGE_SIZE;
2092 }
2093 thread0.td_kstack_pages = kstack_pages;
2094
2095 /*
2096 * Allocate virtual address space for the message buffer.
2097 */
2098 pa = msgbuf_phys = allocpages((msgbufsize + PAGE_MASK) >> PAGE_SHIFT);
2099 msgbufp = (struct msgbuf *)PHYS_TO_DMAP(pa);
2100
2101 /*
2102 * Allocate virtual address space for the dynamic percpu area.
2103 */
2104 pa = allocpages(DPCPU_SIZE >> PAGE_SHIFT);
2105 dpcpu = (void *)PHYS_TO_DMAP(pa);
2106 dpcpu_init(dpcpu, curcpu);
2107
2108 crashdumpmap = (caddr_t)virtual_avail;
2109 virtual_avail += MAXDUMPPGS * PAGE_SIZE;
2110
2111 /*
2112 * Reserve some special page table entries/VA space for temporary
2113 * mapping of pages.
2114 */
2115 }
2116
2117 static void
mmu_parttab_init(void)2118 mmu_parttab_init(void)
2119 {
2120 uint64_t ptcr;
2121
2122 isa3_parttab = (struct pate *)PHYS_TO_DMAP(parttab_phys);
2123
2124 if (bootverbose)
2125 printf("%s parttab: %p\n", __func__, isa3_parttab);
2126 ptcr = parttab_phys | (PARTTAB_SIZE_SHIFT-12);
2127 if (bootverbose)
2128 printf("setting ptcr %lx\n", ptcr);
2129 mtspr(SPR_PTCR, ptcr);
2130 }
2131
2132 static void
mmu_parttab_update(uint64_t lpid,uint64_t pagetab,uint64_t proctab)2133 mmu_parttab_update(uint64_t lpid, uint64_t pagetab, uint64_t proctab)
2134 {
2135 uint64_t prev;
2136
2137 if (bootverbose)
2138 printf("%s isa3_parttab %p lpid %lx pagetab %lx proctab %lx\n", __func__, isa3_parttab,
2139 lpid, pagetab, proctab);
2140 prev = be64toh(isa3_parttab[lpid].pagetab);
2141 isa3_parttab[lpid].pagetab = htobe64(pagetab);
2142 isa3_parttab[lpid].proctab = htobe64(proctab);
2143
2144 if (prev & PARTTAB_HR) {
2145 __asm __volatile(PPC_TLBIE_5(%0,%1,2,0,1) : :
2146 "r" (TLBIEL_INVAL_SET_LPID), "r" (lpid));
2147 __asm __volatile(PPC_TLBIE_5(%0,%1,2,1,1) : :
2148 "r" (TLBIEL_INVAL_SET_LPID), "r" (lpid));
2149 } else {
2150 __asm __volatile(PPC_TLBIE_5(%0,%1,2,0,0) : :
2151 "r" (TLBIEL_INVAL_SET_LPID), "r" (lpid));
2152 }
2153 ttusync();
2154 }
2155
2156 static void
mmu_radix_parttab_init(void)2157 mmu_radix_parttab_init(void)
2158 {
2159 uint64_t pagetab;
2160
2161 mmu_parttab_init();
2162 pagetab = RTS_SIZE | DMAP_TO_PHYS((vm_offset_t)kernel_pmap->pm_pml1) | \
2163 RADIX_PGD_INDEX_SHIFT | PARTTAB_HR;
2164 mmu_parttab_update(0, pagetab, 0);
2165 }
2166
2167 static void
mmu_radix_proctab_register(vm_paddr_t proctabpa,uint64_t table_size)2168 mmu_radix_proctab_register(vm_paddr_t proctabpa, uint64_t table_size)
2169 {
2170 uint64_t pagetab, proctab;
2171
2172 pagetab = be64toh(isa3_parttab[0].pagetab);
2173 proctab = proctabpa | table_size | PARTTAB_GR;
2174 mmu_parttab_update(0, pagetab, proctab);
2175 }
2176
2177 static void
mmu_radix_proctab_init(void)2178 mmu_radix_proctab_init(void)
2179 {
2180
2181 isa3_base_pid = 1;
2182
2183 isa3_proctab = (void*)PHYS_TO_DMAP(proctab0pa);
2184 isa3_proctab->proctab0 =
2185 htobe64(RTS_SIZE | DMAP_TO_PHYS((vm_offset_t)kernel_pmap->pm_pml1) |
2186 RADIX_PGD_INDEX_SHIFT);
2187
2188 if (powernv_enabled) {
2189 mmu_radix_proctab_register(proctab0pa, PROCTAB_SIZE_SHIFT - 12);
2190 __asm __volatile("ptesync" : : : "memory");
2191 __asm __volatile(PPC_TLBIE_5(%0,%1,2,1,1) : :
2192 "r" (TLBIEL_INVAL_SET_LPID), "r" (0));
2193 __asm __volatile("eieio; tlbsync; ptesync" : : : "memory");
2194 #ifdef PSERIES
2195 } else {
2196 int64_t rc;
2197
2198 rc = phyp_hcall(H_REGISTER_PROC_TBL,
2199 PROC_TABLE_NEW | PROC_TABLE_RADIX | PROC_TABLE_GTSE,
2200 proctab0pa, 0, PROCTAB_SIZE_SHIFT - 12);
2201 if (rc != H_SUCCESS)
2202 panic("mmu_radix_proctab_init: "
2203 "failed to register process table: rc=%jd",
2204 (intmax_t)rc);
2205 #endif
2206 }
2207
2208 if (bootverbose)
2209 printf("process table %p and kernel radix PDE: %p\n",
2210 isa3_proctab, kernel_pmap->pm_pml1);
2211 mtmsr(mfmsr() | PSL_DR );
2212 mtmsr(mfmsr() & ~PSL_DR);
2213 kernel_pmap->pm_pid = isa3_base_pid;
2214 isa3_base_pid++;
2215 }
2216
2217 void
mmu_radix_advise(pmap_t pmap,vm_offset_t sva,vm_offset_t eva,int advice)2218 mmu_radix_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,
2219 int advice)
2220 {
2221 struct rwlock *lock;
2222 pml1_entry_t *l1e;
2223 pml2_entry_t *l2e;
2224 pml3_entry_t oldl3e, *l3e;
2225 pt_entry_t *pte;
2226 vm_offset_t va, va_next;
2227 vm_page_t m;
2228 bool anychanged;
2229
2230 if (advice != MADV_DONTNEED && advice != MADV_FREE)
2231 return;
2232 anychanged = false;
2233 PMAP_LOCK(pmap);
2234 for (; sva < eva; sva = va_next) {
2235 l1e = pmap_pml1e(pmap, sva);
2236 if ((be64toh(*l1e) & PG_V) == 0) {
2237 va_next = (sva + L1_PAGE_SIZE) & ~L1_PAGE_MASK;
2238 if (va_next < sva)
2239 va_next = eva;
2240 continue;
2241 }
2242 l2e = pmap_l1e_to_l2e(l1e, sva);
2243 if ((be64toh(*l2e) & PG_V) == 0) {
2244 va_next = (sva + L2_PAGE_SIZE) & ~L2_PAGE_MASK;
2245 if (va_next < sva)
2246 va_next = eva;
2247 continue;
2248 }
2249 va_next = (sva + L3_PAGE_SIZE) & ~L3_PAGE_MASK;
2250 if (va_next < sva)
2251 va_next = eva;
2252 l3e = pmap_l2e_to_l3e(l2e, sva);
2253 oldl3e = be64toh(*l3e);
2254 if ((oldl3e & PG_V) == 0)
2255 continue;
2256 else if ((oldl3e & RPTE_LEAF) != 0) {
2257 if ((oldl3e & PG_MANAGED) == 0)
2258 continue;
2259 lock = NULL;
2260 if (!pmap_demote_l3e_locked(pmap, l3e, sva, &lock)) {
2261 if (lock != NULL)
2262 rw_wunlock(lock);
2263
2264 /*
2265 * The large page mapping was destroyed.
2266 */
2267 continue;
2268 }
2269
2270 /*
2271 * Unless the page mappings are wired, remove the
2272 * mapping to a single page so that a subsequent
2273 * access may repromote. Choosing the last page
2274 * within the address range [sva, min(va_next, eva))
2275 * generally results in more repromotions. Since the
2276 * underlying page table page is fully populated, this
2277 * removal never frees a page table page.
2278 */
2279 if ((oldl3e & PG_W) == 0) {
2280 va = eva;
2281 if (va > va_next)
2282 va = va_next;
2283 va -= PAGE_SIZE;
2284 KASSERT(va >= sva,
2285 ("mmu_radix_advise: no address gap"));
2286 pte = pmap_l3e_to_pte(l3e, va);
2287 KASSERT((be64toh(*pte) & PG_V) != 0,
2288 ("pmap_advise: invalid PTE"));
2289 pmap_remove_pte(pmap, pte, va, be64toh(*l3e), NULL,
2290 &lock);
2291 anychanged = true;
2292 }
2293 if (lock != NULL)
2294 rw_wunlock(lock);
2295 }
2296 if (va_next > eva)
2297 va_next = eva;
2298 va = va_next;
2299 for (pte = pmap_l3e_to_pte(l3e, sva); sva != va_next;
2300 pte++, sva += PAGE_SIZE) {
2301 MPASS(pte == pmap_pte(pmap, sva));
2302
2303 if ((be64toh(*pte) & (PG_MANAGED | PG_V)) != (PG_MANAGED | PG_V))
2304 goto maybe_invlrng;
2305 else if ((be64toh(*pte) & (PG_M | PG_RW)) == (PG_M | PG_RW)) {
2306 if (advice == MADV_DONTNEED) {
2307 /*
2308 * Future calls to pmap_is_modified()
2309 * can be avoided by making the page
2310 * dirty now.
2311 */
2312 m = PHYS_TO_VM_PAGE(be64toh(*pte) & PG_FRAME);
2313 vm_page_dirty(m);
2314 }
2315 atomic_clear_long(pte, htobe64(PG_M | PG_A));
2316 } else if ((be64toh(*pte) & PG_A) != 0)
2317 atomic_clear_long(pte, htobe64(PG_A));
2318 else
2319 goto maybe_invlrng;
2320 anychanged = true;
2321 continue;
2322 maybe_invlrng:
2323 if (va != va_next) {
2324 anychanged = true;
2325 va = va_next;
2326 }
2327 }
2328 if (va != va_next)
2329 anychanged = true;
2330 }
2331 if (anychanged)
2332 pmap_invalidate_all(pmap);
2333 PMAP_UNLOCK(pmap);
2334 }
2335
2336 /*
2337 * Routines used in machine-dependent code
2338 */
2339 static void
mmu_radix_bootstrap(vm_offset_t start,vm_offset_t end)2340 mmu_radix_bootstrap(vm_offset_t start, vm_offset_t end)
2341 {
2342 uint64_t lpcr;
2343
2344 if (bootverbose)
2345 printf("%s\n", __func__);
2346 hw_direct_map = 1;
2347 powernv_enabled = (mfmsr() & PSL_HV) ? 1 : 0;
2348 mmu_radix_early_bootstrap(start, end);
2349 if (bootverbose)
2350 printf("early bootstrap complete\n");
2351 if (powernv_enabled) {
2352 lpcr = mfspr(SPR_LPCR);
2353 mtspr(SPR_LPCR, lpcr | LPCR_UPRT | LPCR_HR);
2354 mmu_radix_parttab_init();
2355 mmu_radix_init_amor();
2356 if (bootverbose)
2357 printf("powernv init complete\n");
2358 }
2359 mmu_radix_init_iamr();
2360 mmu_radix_proctab_init();
2361 mmu_radix_pid_set(kernel_pmap);
2362 if (powernv_enabled)
2363 mmu_radix_tlbiel_flush(TLB_INVAL_SCOPE_GLOBAL);
2364 else
2365 mmu_radix_tlbiel_flush(TLB_INVAL_SCOPE_LPID);
2366
2367 mmu_radix_late_bootstrap(start, end);
2368 numa_mem_regions(&numa_pregions, &numa_pregions_sz);
2369 if (bootverbose)
2370 printf("%s done\n", __func__);
2371 pmap_bootstrapped = 1;
2372 dmaplimit = roundup2(powerpc_ptob(Maxmem), L2_PAGE_SIZE);
2373 PCPU_SET(flags, PCPU_GET(flags) | PC_FLAG_NOSRS);
2374 }
2375
2376 static void
mmu_radix_cpu_bootstrap(int ap)2377 mmu_radix_cpu_bootstrap(int ap)
2378 {
2379 uint64_t lpcr;
2380 uint64_t ptcr;
2381
2382 if (powernv_enabled) {
2383 lpcr = mfspr(SPR_LPCR);
2384 mtspr(SPR_LPCR, lpcr | LPCR_UPRT | LPCR_HR);
2385
2386 ptcr = parttab_phys | (PARTTAB_SIZE_SHIFT-12);
2387 mtspr(SPR_PTCR, ptcr);
2388 mmu_radix_init_amor();
2389 }
2390 mmu_radix_init_iamr();
2391 mmu_radix_pid_set(kernel_pmap);
2392 if (powernv_enabled)
2393 mmu_radix_tlbiel_flush(TLB_INVAL_SCOPE_GLOBAL);
2394 else
2395 mmu_radix_tlbiel_flush(TLB_INVAL_SCOPE_LPID);
2396 }
2397
2398 static SYSCTL_NODE(_vm_pmap, OID_AUTO, l3e, CTLFLAG_RD, 0,
2399 "2MB page mapping counters");
2400
2401 static COUNTER_U64_DEFINE_EARLY(pmap_l3e_demotions);
2402 SYSCTL_COUNTER_U64(_vm_pmap_l3e, OID_AUTO, demotions, CTLFLAG_RD,
2403 &pmap_l3e_demotions, "2MB page demotions");
2404
2405 static COUNTER_U64_DEFINE_EARLY(pmap_l3e_mappings);
2406 SYSCTL_COUNTER_U64(_vm_pmap_l3e, OID_AUTO, mappings, CTLFLAG_RD,
2407 &pmap_l3e_mappings, "2MB page mappings");
2408
2409 static COUNTER_U64_DEFINE_EARLY(pmap_l3e_p_failures);
2410 SYSCTL_COUNTER_U64(_vm_pmap_l3e, OID_AUTO, p_failures, CTLFLAG_RD,
2411 &pmap_l3e_p_failures, "2MB page promotion failures");
2412
2413 static COUNTER_U64_DEFINE_EARLY(pmap_l3e_promotions);
2414 SYSCTL_COUNTER_U64(_vm_pmap_l3e, OID_AUTO, promotions, CTLFLAG_RD,
2415 &pmap_l3e_promotions, "2MB page promotions");
2416
2417 static SYSCTL_NODE(_vm_pmap, OID_AUTO, l2e, CTLFLAG_RD, 0,
2418 "1GB page mapping counters");
2419
2420 static COUNTER_U64_DEFINE_EARLY(pmap_l2e_demotions);
2421 SYSCTL_COUNTER_U64(_vm_pmap_l2e, OID_AUTO, demotions, CTLFLAG_RD,
2422 &pmap_l2e_demotions, "1GB page demotions");
2423
2424 void
mmu_radix_clear_modify(vm_page_t m)2425 mmu_radix_clear_modify(vm_page_t m)
2426 {
2427 struct md_page *pvh;
2428 pmap_t pmap;
2429 pv_entry_t next_pv, pv;
2430 pml3_entry_t oldl3e, *l3e;
2431 pt_entry_t oldpte, *pte;
2432 struct rwlock *lock;
2433 vm_offset_t va;
2434 int md_gen, pvh_gen;
2435
2436 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
2437 ("pmap_clear_modify: page %p is not managed", m));
2438 vm_page_assert_busied(m);
2439 CTR2(KTR_PMAP, "%s(%p)", __func__, m);
2440
2441 /*
2442 * If the page is not PGA_WRITEABLE, then no PTEs can have PG_M set.
2443 * If the object containing the page is locked and the page is not
2444 * exclusive busied, then PGA_WRITEABLE cannot be concurrently set.
2445 */
2446 if ((m->a.flags & PGA_WRITEABLE) == 0)
2447 return;
2448 pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy :
2449 pa_to_pvh(VM_PAGE_TO_PHYS(m));
2450 lock = VM_PAGE_TO_PV_LIST_LOCK(m);
2451 rw_wlock(lock);
2452 restart:
2453 TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_link, next_pv) {
2454 pmap = PV_PMAP(pv);
2455 if (!PMAP_TRYLOCK(pmap)) {
2456 pvh_gen = pvh->pv_gen;
2457 rw_wunlock(lock);
2458 PMAP_LOCK(pmap);
2459 rw_wlock(lock);
2460 if (pvh_gen != pvh->pv_gen) {
2461 PMAP_UNLOCK(pmap);
2462 goto restart;
2463 }
2464 }
2465 va = pv->pv_va;
2466 l3e = pmap_pml3e(pmap, va);
2467 oldl3e = be64toh(*l3e);
2468 if ((oldl3e & PG_RW) != 0 &&
2469 pmap_demote_l3e_locked(pmap, l3e, va, &lock) &&
2470 (oldl3e & PG_W) == 0) {
2471 /*
2472 * Write protect the mapping to a
2473 * single page so that a subsequent
2474 * write access may repromote.
2475 */
2476 va += VM_PAGE_TO_PHYS(m) - (oldl3e &
2477 PG_PS_FRAME);
2478 pte = pmap_l3e_to_pte(l3e, va);
2479 oldpte = be64toh(*pte);
2480 while (!atomic_cmpset_long(pte,
2481 htobe64(oldpte),
2482 htobe64((oldpte | RPTE_EAA_R) & ~(PG_M | PG_RW))))
2483 oldpte = be64toh(*pte);
2484 vm_page_dirty(m);
2485 pmap_invalidate_page(pmap, va);
2486 }
2487 PMAP_UNLOCK(pmap);
2488 }
2489 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) {
2490 pmap = PV_PMAP(pv);
2491 if (!PMAP_TRYLOCK(pmap)) {
2492 md_gen = m->md.pv_gen;
2493 pvh_gen = pvh->pv_gen;
2494 rw_wunlock(lock);
2495 PMAP_LOCK(pmap);
2496 rw_wlock(lock);
2497 if (pvh_gen != pvh->pv_gen || md_gen != m->md.pv_gen) {
2498 PMAP_UNLOCK(pmap);
2499 goto restart;
2500 }
2501 }
2502 l3e = pmap_pml3e(pmap, pv->pv_va);
2503 KASSERT((be64toh(*l3e) & RPTE_LEAF) == 0, ("pmap_clear_modify: found"
2504 " a 2mpage in page %p's pv list", m));
2505 pte = pmap_l3e_to_pte(l3e, pv->pv_va);
2506 if ((be64toh(*pte) & (PG_M | PG_RW)) == (PG_M | PG_RW)) {
2507 atomic_clear_long(pte, htobe64(PG_M));
2508 pmap_invalidate_page(pmap, pv->pv_va);
2509 }
2510 PMAP_UNLOCK(pmap);
2511 }
2512 rw_wunlock(lock);
2513 }
2514
2515 void
mmu_radix_copy(pmap_t dst_pmap,pmap_t src_pmap,vm_offset_t dst_addr,vm_size_t len,vm_offset_t src_addr)2516 mmu_radix_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr,
2517 vm_size_t len, vm_offset_t src_addr)
2518 {
2519 struct rwlock *lock;
2520 struct spglist free;
2521 vm_offset_t addr;
2522 vm_offset_t end_addr = src_addr + len;
2523 vm_offset_t va_next;
2524 vm_page_t dst_pdpg, dstmpte, srcmpte;
2525 bool invalidate_all;
2526
2527 CTR6(KTR_PMAP,
2528 "%s(dst_pmap=%p, src_pmap=%p, dst_addr=%lx, len=%lu, src_addr=%lx)\n",
2529 __func__, dst_pmap, src_pmap, dst_addr, len, src_addr);
2530
2531 if (dst_addr != src_addr)
2532 return;
2533 lock = NULL;
2534 invalidate_all = false;
2535 if (dst_pmap < src_pmap) {
2536 PMAP_LOCK(dst_pmap);
2537 PMAP_LOCK(src_pmap);
2538 } else {
2539 PMAP_LOCK(src_pmap);
2540 PMAP_LOCK(dst_pmap);
2541 }
2542
2543 for (addr = src_addr; addr < end_addr; addr = va_next) {
2544 pml1_entry_t *l1e;
2545 pml2_entry_t *l2e;
2546 pml3_entry_t srcptepaddr, *l3e;
2547 pt_entry_t *src_pte, *dst_pte;
2548
2549 l1e = pmap_pml1e(src_pmap, addr);
2550 if ((be64toh(*l1e) & PG_V) == 0) {
2551 va_next = (addr + L1_PAGE_SIZE) & ~L1_PAGE_MASK;
2552 if (va_next < addr)
2553 va_next = end_addr;
2554 continue;
2555 }
2556
2557 l2e = pmap_l1e_to_l2e(l1e, addr);
2558 if ((be64toh(*l2e) & PG_V) == 0) {
2559 va_next = (addr + L2_PAGE_SIZE) & ~L2_PAGE_MASK;
2560 if (va_next < addr)
2561 va_next = end_addr;
2562 continue;
2563 }
2564
2565 va_next = (addr + L3_PAGE_SIZE) & ~L3_PAGE_MASK;
2566 if (va_next < addr)
2567 va_next = end_addr;
2568
2569 l3e = pmap_l2e_to_l3e(l2e, addr);
2570 srcptepaddr = be64toh(*l3e);
2571 if (srcptepaddr == 0)
2572 continue;
2573
2574 if (srcptepaddr & RPTE_LEAF) {
2575 if ((addr & L3_PAGE_MASK) != 0 ||
2576 addr + L3_PAGE_SIZE > end_addr)
2577 continue;
2578 dst_pdpg = pmap_allocl3e(dst_pmap, addr, NULL);
2579 if (dst_pdpg == NULL)
2580 break;
2581 l3e = (pml3_entry_t *)
2582 PHYS_TO_DMAP(VM_PAGE_TO_PHYS(dst_pdpg));
2583 l3e = &l3e[pmap_pml3e_index(addr)];
2584 if (be64toh(*l3e) == 0 && ((srcptepaddr & PG_MANAGED) == 0 ||
2585 pmap_pv_insert_l3e(dst_pmap, addr, srcptepaddr,
2586 PMAP_ENTER_NORECLAIM, &lock))) {
2587 *l3e = htobe64(srcptepaddr & ~PG_W);
2588 pmap_resident_count_inc(dst_pmap,
2589 L3_PAGE_SIZE / PAGE_SIZE);
2590 counter_u64_add(pmap_l3e_mappings, 1);
2591 } else
2592 dst_pdpg->ref_count--;
2593 continue;
2594 }
2595
2596 srcptepaddr &= PG_FRAME;
2597 srcmpte = PHYS_TO_VM_PAGE(srcptepaddr);
2598 KASSERT(srcmpte->ref_count > 0,
2599 ("pmap_copy: source page table page is unused"));
2600
2601 if (va_next > end_addr)
2602 va_next = end_addr;
2603
2604 src_pte = (pt_entry_t *)PHYS_TO_DMAP(srcptepaddr);
2605 src_pte = &src_pte[pmap_pte_index(addr)];
2606 dstmpte = NULL;
2607 while (addr < va_next) {
2608 pt_entry_t ptetemp;
2609 ptetemp = be64toh(*src_pte);
2610 /*
2611 * we only virtual copy managed pages
2612 */
2613 if ((ptetemp & PG_MANAGED) != 0) {
2614 if (dstmpte != NULL &&
2615 dstmpte->pindex == pmap_l3e_pindex(addr))
2616 dstmpte->ref_count++;
2617 else if ((dstmpte = pmap_allocpte(dst_pmap,
2618 addr, NULL)) == NULL)
2619 goto out;
2620 dst_pte = (pt_entry_t *)
2621 PHYS_TO_DMAP(VM_PAGE_TO_PHYS(dstmpte));
2622 dst_pte = &dst_pte[pmap_pte_index(addr)];
2623 if (be64toh(*dst_pte) == 0 &&
2624 pmap_try_insert_pv_entry(dst_pmap, addr,
2625 PHYS_TO_VM_PAGE(ptetemp & PG_FRAME),
2626 &lock)) {
2627 /*
2628 * Clear the wired, modified, and
2629 * accessed (referenced) bits
2630 * during the copy.
2631 */
2632 *dst_pte = htobe64(ptetemp & ~(PG_W | PG_M |
2633 PG_A));
2634 pmap_resident_count_inc(dst_pmap, 1);
2635 } else {
2636 SLIST_INIT(&free);
2637 if (pmap_unwire_ptp(dst_pmap, addr,
2638 dstmpte, &free)) {
2639 /*
2640 * Although "addr" is not
2641 * mapped, paging-structure
2642 * caches could nonetheless
2643 * have entries that refer to
2644 * the freed page table pages.
2645 * Invalidate those entries.
2646 */
2647 invalidate_all = true;
2648 vm_page_free_pages_toq(&free,
2649 true);
2650 }
2651 goto out;
2652 }
2653 if (dstmpte->ref_count >= srcmpte->ref_count)
2654 break;
2655 }
2656 addr += PAGE_SIZE;
2657 if (__predict_false((addr & L3_PAGE_MASK) == 0))
2658 src_pte = pmap_pte(src_pmap, addr);
2659 else
2660 src_pte++;
2661 }
2662 }
2663 out:
2664 if (invalidate_all)
2665 pmap_invalidate_all(dst_pmap);
2666 if (lock != NULL)
2667 rw_wunlock(lock);
2668 PMAP_UNLOCK(src_pmap);
2669 PMAP_UNLOCK(dst_pmap);
2670 }
2671
2672 static void
mmu_radix_copy_page(vm_page_t msrc,vm_page_t mdst)2673 mmu_radix_copy_page(vm_page_t msrc, vm_page_t mdst)
2674 {
2675 vm_offset_t src = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(msrc));
2676 vm_offset_t dst = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mdst));
2677
2678 CTR3(KTR_PMAP, "%s(%p, %p)", __func__, src, dst);
2679 /*
2680 * XXX slow
2681 */
2682 bcopy((void *)src, (void *)dst, PAGE_SIZE);
2683 }
2684
2685 static void
mmu_radix_copy_pages(vm_page_t ma[],vm_offset_t a_offset,vm_page_t mb[],vm_offset_t b_offset,int xfersize)2686 mmu_radix_copy_pages(vm_page_t ma[], vm_offset_t a_offset, vm_page_t mb[],
2687 vm_offset_t b_offset, int xfersize)
2688 {
2689 void *a_cp, *b_cp;
2690 vm_offset_t a_pg_offset, b_pg_offset;
2691 int cnt;
2692
2693 CTR6(KTR_PMAP, "%s(%p, %#x, %p, %#x, %#x)", __func__, ma,
2694 a_offset, mb, b_offset, xfersize);
2695
2696 while (xfersize > 0) {
2697 a_pg_offset = a_offset & PAGE_MASK;
2698 cnt = min(xfersize, PAGE_SIZE - a_pg_offset);
2699 a_cp = (char *)(uintptr_t)PHYS_TO_DMAP(
2700 VM_PAGE_TO_PHYS(ma[a_offset >> PAGE_SHIFT])) +
2701 a_pg_offset;
2702 b_pg_offset = b_offset & PAGE_MASK;
2703 cnt = min(cnt, PAGE_SIZE - b_pg_offset);
2704 b_cp = (char *)(uintptr_t)PHYS_TO_DMAP(
2705 VM_PAGE_TO_PHYS(mb[b_offset >> PAGE_SHIFT])) +
2706 b_pg_offset;
2707 bcopy(a_cp, b_cp, cnt);
2708 a_offset += cnt;
2709 b_offset += cnt;
2710 xfersize -= cnt;
2711 }
2712 }
2713
2714 #if VM_NRESERVLEVEL > 0
2715 /*
2716 * Tries to promote the 512, contiguous 4KB page mappings that are within a
2717 * single page table page (PTP) to a single 2MB page mapping. For promotion
2718 * to occur, two conditions must be met: (1) the 4KB page mappings must map
2719 * aligned, contiguous physical memory and (2) the 4KB page mappings must have
2720 * identical characteristics.
2721 */
2722 static int
pmap_promote_l3e(pmap_t pmap,pml3_entry_t * pde,vm_offset_t va,struct rwlock ** lockp)2723 pmap_promote_l3e(pmap_t pmap, pml3_entry_t *pde, vm_offset_t va,
2724 struct rwlock **lockp)
2725 {
2726 pml3_entry_t newpde;
2727 pt_entry_t *firstpte, oldpte, pa, *pte;
2728 vm_page_t mpte;
2729
2730 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
2731
2732 /*
2733 * Examine the first PTE in the specified PTP. Abort if this PTE is
2734 * either invalid, unused, or does not map the first 4KB physical page
2735 * within a 2MB page.
2736 */
2737 firstpte = (pt_entry_t *)PHYS_TO_DMAP(be64toh(*pde) & PG_FRAME);
2738 setpde:
2739 newpde = be64toh(*firstpte);
2740 if ((newpde & ((PG_FRAME & L3_PAGE_MASK) | PG_A | PG_V)) != (PG_A | PG_V)) {
2741 CTR2(KTR_PMAP, "pmap_promote_l3e: failure for va %#lx"
2742 " in pmap %p", va, pmap);
2743 goto fail;
2744 }
2745 if ((newpde & (PG_M | PG_RW)) == PG_RW) {
2746 /*
2747 * When PG_M is already clear, PG_RW can be cleared without
2748 * a TLB invalidation.
2749 */
2750 if (!atomic_cmpset_long(firstpte, htobe64(newpde), htobe64((newpde | RPTE_EAA_R) & ~RPTE_EAA_W)))
2751 goto setpde;
2752 newpde &= ~RPTE_EAA_W;
2753 }
2754
2755 /*
2756 * Examine each of the other PTEs in the specified PTP. Abort if this
2757 * PTE maps an unexpected 4KB physical page or does not have identical
2758 * characteristics to the first PTE.
2759 */
2760 pa = (newpde & (PG_PS_FRAME | PG_A | PG_V)) + L3_PAGE_SIZE - PAGE_SIZE;
2761 for (pte = firstpte + NPTEPG - 1; pte > firstpte; pte--) {
2762 setpte:
2763 oldpte = be64toh(*pte);
2764 if ((oldpte & (PG_FRAME | PG_A | PG_V)) != pa) {
2765 CTR2(KTR_PMAP, "pmap_promote_l3e: failure for va %#lx"
2766 " in pmap %p", va, pmap);
2767 goto fail;
2768 }
2769 if ((oldpte & (PG_M | PG_RW)) == PG_RW) {
2770 /*
2771 * When PG_M is already clear, PG_RW can be cleared
2772 * without a TLB invalidation.
2773 */
2774 if (!atomic_cmpset_long(pte, htobe64(oldpte), htobe64((oldpte | RPTE_EAA_R) & ~RPTE_EAA_W)))
2775 goto setpte;
2776 oldpte &= ~RPTE_EAA_W;
2777 CTR2(KTR_PMAP, "pmap_promote_l3e: protect for va %#lx"
2778 " in pmap %p", (oldpte & PG_FRAME & L3_PAGE_MASK) |
2779 (va & ~L3_PAGE_MASK), pmap);
2780 }
2781 if ((oldpte & PG_PTE_PROMOTE) != (newpde & PG_PTE_PROMOTE)) {
2782 CTR2(KTR_PMAP, "pmap_promote_l3e: failure for va %#lx"
2783 " in pmap %p", va, pmap);
2784 goto fail;
2785 }
2786 pa -= PAGE_SIZE;
2787 }
2788
2789 /*
2790 * Save the page table page in its current state until the PDE
2791 * mapping the superpage is demoted by pmap_demote_pde() or
2792 * destroyed by pmap_remove_pde().
2793 */
2794 mpte = PHYS_TO_VM_PAGE(be64toh(*pde) & PG_FRAME);
2795 KASSERT(mpte >= vm_page_array &&
2796 mpte < &vm_page_array[vm_page_array_size],
2797 ("pmap_promote_l3e: page table page is out of range"));
2798 KASSERT(mpte->pindex == pmap_l3e_pindex(va),
2799 ("pmap_promote_l3e: page table page's pindex is wrong"));
2800 if (pmap_insert_pt_page(pmap, mpte)) {
2801 CTR2(KTR_PMAP,
2802 "pmap_promote_l3e: failure for va %#lx in pmap %p", va,
2803 pmap);
2804 goto fail;
2805 }
2806
2807 /*
2808 * Promote the pv entries.
2809 */
2810 if ((newpde & PG_MANAGED) != 0)
2811 pmap_pv_promote_l3e(pmap, va, newpde & PG_PS_FRAME, lockp);
2812
2813 pte_store(pde, PG_PROMOTED | newpde);
2814 ptesync();
2815 counter_u64_add(pmap_l3e_promotions, 1);
2816 CTR2(KTR_PMAP, "pmap_promote_l3e: success for va %#lx"
2817 " in pmap %p", va, pmap);
2818 return (0);
2819 fail:
2820 counter_u64_add(pmap_l3e_p_failures, 1);
2821 return (KERN_FAILURE);
2822 }
2823 #endif /* VM_NRESERVLEVEL > 0 */
2824
2825 int
mmu_radix_enter(pmap_t pmap,vm_offset_t va,vm_page_t m,vm_prot_t prot,u_int flags,int8_t psind)2826 mmu_radix_enter(pmap_t pmap, vm_offset_t va, vm_page_t m,
2827 vm_prot_t prot, u_int flags, int8_t psind)
2828 {
2829 struct rwlock *lock;
2830 pml3_entry_t *l3e;
2831 pt_entry_t *pte;
2832 pt_entry_t newpte, origpte;
2833 pv_entry_t pv;
2834 vm_paddr_t opa, pa;
2835 vm_page_t mpte, om;
2836 int rv, retrycount;
2837 bool nosleep, invalidate_all, invalidate_page;
2838
2839 va = trunc_page(va);
2840 retrycount = 0;
2841 invalidate_page = invalidate_all = false;
2842 CTR6(KTR_PMAP, "pmap_enter(%p, %#lx, %p, %#x, %#x, %d)", pmap, va,
2843 m, prot, flags, psind);
2844 KASSERT(va <= VM_MAX_KERNEL_ADDRESS, ("pmap_enter: toobig"));
2845 KASSERT((m->oflags & VPO_UNMANAGED) != 0 || !VA_IS_CLEANMAP(va),
2846 ("pmap_enter: managed mapping within the clean submap"));
2847 if ((m->oflags & VPO_UNMANAGED) == 0)
2848 VM_PAGE_OBJECT_BUSY_ASSERT(m);
2849
2850 KASSERT((flags & PMAP_ENTER_RESERVED) == 0,
2851 ("pmap_enter: flags %u has reserved bits set", flags));
2852 pa = VM_PAGE_TO_PHYS(m);
2853 newpte = (pt_entry_t)(pa | PG_A | PG_V | RPTE_LEAF);
2854 if ((flags & VM_PROT_WRITE) != 0)
2855 newpte |= PG_M;
2856 if ((flags & VM_PROT_READ) != 0)
2857 newpte |= PG_A;
2858 if (prot & VM_PROT_READ)
2859 newpte |= RPTE_EAA_R;
2860 if ((prot & VM_PROT_WRITE) != 0)
2861 newpte |= RPTE_EAA_W;
2862 KASSERT((newpte & (PG_M | PG_RW)) != PG_M,
2863 ("pmap_enter: flags includes VM_PROT_WRITE but prot doesn't"));
2864
2865 if (prot & VM_PROT_EXECUTE)
2866 newpte |= PG_X;
2867 if ((flags & PMAP_ENTER_WIRED) != 0)
2868 newpte |= PG_W;
2869 if (va >= DMAP_MIN_ADDRESS)
2870 newpte |= RPTE_EAA_P;
2871 newpte |= pmap_cache_bits(m->md.mdpg_cache_attrs);
2872 /*
2873 * Set modified bit gratuitously for writeable mappings if
2874 * the page is unmanaged. We do not want to take a fault
2875 * to do the dirty bit accounting for these mappings.
2876 */
2877 if ((m->oflags & VPO_UNMANAGED) != 0) {
2878 if ((newpte & PG_RW) != 0)
2879 newpte |= PG_M;
2880 } else
2881 newpte |= PG_MANAGED;
2882
2883 lock = NULL;
2884 PMAP_LOCK(pmap);
2885 if (psind == 1) {
2886 /* Assert the required virtual and physical alignment. */
2887 KASSERT((va & L3_PAGE_MASK) == 0, ("pmap_enter: va unaligned"));
2888 KASSERT(m->psind > 0, ("pmap_enter: m->psind < psind"));
2889 rv = pmap_enter_l3e(pmap, va, newpte | RPTE_LEAF, flags, m, &lock);
2890 goto out;
2891 }
2892 mpte = NULL;
2893
2894 /*
2895 * In the case that a page table page is not
2896 * resident, we are creating it here.
2897 */
2898 retry:
2899 l3e = pmap_pml3e(pmap, va);
2900 if (l3e != NULL && (be64toh(*l3e) & PG_V) != 0 && ((be64toh(*l3e) & RPTE_LEAF) == 0 ||
2901 pmap_demote_l3e_locked(pmap, l3e, va, &lock))) {
2902 pte = pmap_l3e_to_pte(l3e, va);
2903 if (va < VM_MAXUSER_ADDRESS && mpte == NULL) {
2904 mpte = PHYS_TO_VM_PAGE(be64toh(*l3e) & PG_FRAME);
2905 mpte->ref_count++;
2906 }
2907 } else if (va < VM_MAXUSER_ADDRESS) {
2908 /*
2909 * Here if the pte page isn't mapped, or if it has been
2910 * deallocated.
2911 */
2912 nosleep = (flags & PMAP_ENTER_NOSLEEP) != 0;
2913 mpte = _pmap_allocpte(pmap, pmap_l3e_pindex(va),
2914 nosleep ? NULL : &lock);
2915 if (mpte == NULL && nosleep) {
2916 rv = KERN_RESOURCE_SHORTAGE;
2917 goto out;
2918 }
2919 if (__predict_false(retrycount++ == 6))
2920 panic("too many retries");
2921 invalidate_all = true;
2922 goto retry;
2923 } else
2924 panic("pmap_enter: invalid page directory va=%#lx", va);
2925
2926 origpte = be64toh(*pte);
2927 pv = NULL;
2928
2929 /*
2930 * Is the specified virtual address already mapped?
2931 */
2932 if ((origpte & PG_V) != 0) {
2933 #ifdef INVARIANTS
2934 if (VERBOSE_PMAP || pmap_logging) {
2935 printf("cow fault pmap_enter(%p, %#lx, %p, %#x, %x, %d) --"
2936 " asid=%lu curpid=%d name=%s origpte0x%lx\n",
2937 pmap, va, m, prot, flags, psind, pmap->pm_pid,
2938 curproc->p_pid, curproc->p_comm, origpte);
2939 #ifdef DDB
2940 pmap_pte_walk(pmap->pm_pml1, va);
2941 #endif
2942 }
2943 #endif
2944 /*
2945 * Wiring change, just update stats. We don't worry about
2946 * wiring PT pages as they remain resident as long as there
2947 * are valid mappings in them. Hence, if a user page is wired,
2948 * the PT page will be also.
2949 */
2950 if ((newpte & PG_W) != 0 && (origpte & PG_W) == 0)
2951 pmap->pm_stats.wired_count++;
2952 else if ((newpte & PG_W) == 0 && (origpte & PG_W) != 0)
2953 pmap->pm_stats.wired_count--;
2954
2955 /*
2956 * Remove the extra PT page reference.
2957 */
2958 if (mpte != NULL) {
2959 mpte->ref_count--;
2960 KASSERT(mpte->ref_count > 0,
2961 ("pmap_enter: missing reference to page table page,"
2962 " va: 0x%lx", va));
2963 }
2964
2965 /*
2966 * Has the physical page changed?
2967 */
2968 opa = origpte & PG_FRAME;
2969 if (opa == pa) {
2970 /*
2971 * No, might be a protection or wiring change.
2972 */
2973 if ((origpte & PG_MANAGED) != 0 &&
2974 (newpte & PG_RW) != 0)
2975 vm_page_aflag_set(m, PGA_WRITEABLE);
2976 if (((origpte ^ newpte) & ~(PG_M | PG_A)) == 0) {
2977 if ((newpte & (PG_A|PG_M)) != (origpte & (PG_A|PG_M))) {
2978 if (!atomic_cmpset_long(pte, htobe64(origpte), htobe64(newpte)))
2979 goto retry;
2980 if ((newpte & PG_M) != (origpte & PG_M))
2981 vm_page_dirty(m);
2982 if ((newpte & PG_A) != (origpte & PG_A))
2983 vm_page_aflag_set(m, PGA_REFERENCED);
2984 ptesync();
2985 } else
2986 invalidate_all = true;
2987 if (((origpte ^ newpte) & ~(PG_M | PG_A)) == 0)
2988 goto unchanged;
2989 }
2990 goto validate;
2991 }
2992
2993 /*
2994 * The physical page has changed. Temporarily invalidate
2995 * the mapping. This ensures that all threads sharing the
2996 * pmap keep a consistent view of the mapping, which is
2997 * necessary for the correct handling of COW faults. It
2998 * also permits reuse of the old mapping's PV entry,
2999 * avoiding an allocation.
3000 *
3001 * For consistency, handle unmanaged mappings the same way.
3002 */
3003 origpte = be64toh(pte_load_clear(pte));
3004 KASSERT((origpte & PG_FRAME) == opa,
3005 ("pmap_enter: unexpected pa update for %#lx", va));
3006 if ((origpte & PG_MANAGED) != 0) {
3007 om = PHYS_TO_VM_PAGE(opa);
3008
3009 /*
3010 * The pmap lock is sufficient to synchronize with
3011 * concurrent calls to pmap_page_test_mappings() and
3012 * pmap_ts_referenced().
3013 */
3014 if ((origpte & (PG_M | PG_RW)) == (PG_M | PG_RW))
3015 vm_page_dirty(om);
3016 if ((origpte & PG_A) != 0)
3017 vm_page_aflag_set(om, PGA_REFERENCED);
3018 CHANGE_PV_LIST_LOCK_TO_PHYS(&lock, opa);
3019 pv = pmap_pvh_remove(&om->md, pmap, va);
3020 if ((newpte & PG_MANAGED) == 0)
3021 free_pv_entry(pmap, pv);
3022 #ifdef INVARIANTS
3023 else if (origpte & PG_MANAGED) {
3024 if (pv == NULL) {
3025 #ifdef DDB
3026 pmap_page_print_mappings(om);
3027 #endif
3028 MPASS(pv != NULL);
3029 }
3030 }
3031 #endif
3032 if ((om->a.flags & PGA_WRITEABLE) != 0 &&
3033 TAILQ_EMPTY(&om->md.pv_list) &&
3034 ((om->flags & PG_FICTITIOUS) != 0 ||
3035 TAILQ_EMPTY(&pa_to_pvh(opa)->pv_list)))
3036 vm_page_aflag_clear(om, PGA_WRITEABLE);
3037 }
3038 if ((origpte & PG_A) != 0)
3039 invalidate_page = true;
3040 origpte = 0;
3041 } else {
3042 if (pmap != kernel_pmap) {
3043 #ifdef INVARIANTS
3044 if (VERBOSE_PMAP || pmap_logging)
3045 printf("pmap_enter(%p, %#lx, %p, %#x, %x, %d) -- asid=%lu curpid=%d name=%s\n",
3046 pmap, va, m, prot, flags, psind,
3047 pmap->pm_pid, curproc->p_pid,
3048 curproc->p_comm);
3049 #endif
3050 }
3051
3052 /*
3053 * Increment the counters.
3054 */
3055 if ((newpte & PG_W) != 0)
3056 pmap->pm_stats.wired_count++;
3057 pmap_resident_count_inc(pmap, 1);
3058 }
3059
3060 /*
3061 * Enter on the PV list if part of our managed memory.
3062 */
3063 if ((newpte & PG_MANAGED) != 0) {
3064 if (pv == NULL) {
3065 pv = get_pv_entry(pmap, &lock);
3066 pv->pv_va = va;
3067 }
3068 #ifdef VERBOSE_PV
3069 else
3070 printf("reassigning pv: %p to pmap: %p\n",
3071 pv, pmap);
3072 #endif
3073 CHANGE_PV_LIST_LOCK_TO_PHYS(&lock, pa);
3074 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_link);
3075 m->md.pv_gen++;
3076 if ((newpte & PG_RW) != 0)
3077 vm_page_aflag_set(m, PGA_WRITEABLE);
3078 }
3079
3080 /*
3081 * Update the PTE.
3082 */
3083 if ((origpte & PG_V) != 0) {
3084 validate:
3085 origpte = be64toh(pte_load_store(pte, htobe64(newpte)));
3086 KASSERT((origpte & PG_FRAME) == pa,
3087 ("pmap_enter: unexpected pa update for %#lx", va));
3088 if ((newpte & PG_M) == 0 && (origpte & (PG_M | PG_RW)) ==
3089 (PG_M | PG_RW)) {
3090 if ((origpte & PG_MANAGED) != 0)
3091 vm_page_dirty(m);
3092 invalidate_page = true;
3093
3094 /*
3095 * Although the PTE may still have PG_RW set, TLB
3096 * invalidation may nonetheless be required because
3097 * the PTE no longer has PG_M set.
3098 */
3099 } else if ((origpte & PG_X) != 0 || (newpte & PG_X) == 0) {
3100 /*
3101 * Removing capabilities requires invalidation on POWER
3102 */
3103 invalidate_page = true;
3104 goto unchanged;
3105 }
3106 if ((origpte & PG_A) != 0)
3107 invalidate_page = true;
3108 } else {
3109 pte_store(pte, newpte);
3110 ptesync();
3111 }
3112 unchanged:
3113
3114 #if VM_NRESERVLEVEL > 0
3115 /*
3116 * If both the page table page and the reservation are fully
3117 * populated, then attempt promotion.
3118 */
3119 if ((mpte == NULL || mpte->ref_count == NPTEPG) &&
3120 mmu_radix_ps_enabled(pmap) &&
3121 (m->flags & PG_FICTITIOUS) == 0 &&
3122 vm_reserv_level_iffullpop(m) == 0 &&
3123 pmap_promote_l3e(pmap, l3e, va, &lock) == 0)
3124 invalidate_all = true;
3125 #endif
3126 if (invalidate_all)
3127 pmap_invalidate_all(pmap);
3128 else if (invalidate_page)
3129 pmap_invalidate_page(pmap, va);
3130
3131 rv = KERN_SUCCESS;
3132 out:
3133 if (lock != NULL)
3134 rw_wunlock(lock);
3135 PMAP_UNLOCK(pmap);
3136
3137 return (rv);
3138 }
3139
3140 /*
3141 * Release a page table page reference after a failed attempt to create a
3142 * mapping.
3143 */
3144 static void
pmap_abort_ptp(pmap_t pmap,vm_offset_t va,vm_page_t pdpg)3145 pmap_abort_ptp(pmap_t pmap, vm_offset_t va, vm_page_t pdpg)
3146 {
3147 struct spglist free;
3148
3149 SLIST_INIT(&free);
3150 if (pmap_unwire_ptp(pmap, va, pdpg, &free)) {
3151 /*
3152 * Although "va" is not mapped, paging-
3153 * structure caches could nonetheless have
3154 * entries that refer to the freed page table
3155 * pages. Invalidate those entries.
3156 */
3157 pmap_invalidate_page(pmap, va);
3158 vm_page_free_pages_toq(&free, true);
3159 }
3160 }
3161
3162 /*
3163 * Tries to create a read- and/or execute-only 2MB page mapping. Returns true
3164 * if successful. Returns false if (1) a page table page cannot be allocated
3165 * without sleeping, (2) a mapping already exists at the specified virtual
3166 * address, or (3) a PV entry cannot be allocated without reclaiming another
3167 * PV entry.
3168 */
3169 static bool
pmap_enter_2mpage(pmap_t pmap,vm_offset_t va,vm_page_t m,vm_prot_t prot,struct rwlock ** lockp)3170 pmap_enter_2mpage(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,
3171 struct rwlock **lockp)
3172 {
3173 pml3_entry_t newpde;
3174
3175 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
3176 newpde = VM_PAGE_TO_PHYS(m) | pmap_cache_bits(m->md.mdpg_cache_attrs) |
3177 RPTE_LEAF | PG_V;
3178 if ((m->oflags & VPO_UNMANAGED) == 0)
3179 newpde |= PG_MANAGED;
3180 if (prot & VM_PROT_EXECUTE)
3181 newpde |= PG_X;
3182 if (prot & VM_PROT_READ)
3183 newpde |= RPTE_EAA_R;
3184 if (va >= DMAP_MIN_ADDRESS)
3185 newpde |= RPTE_EAA_P;
3186 return (pmap_enter_l3e(pmap, va, newpde, PMAP_ENTER_NOSLEEP |
3187 PMAP_ENTER_NOREPLACE | PMAP_ENTER_NORECLAIM, NULL, lockp) ==
3188 KERN_SUCCESS);
3189 }
3190
3191 /*
3192 * Tries to create the specified 2MB page mapping. Returns KERN_SUCCESS if
3193 * the mapping was created, and either KERN_FAILURE or KERN_RESOURCE_SHORTAGE
3194 * otherwise. Returns KERN_FAILURE if PMAP_ENTER_NOREPLACE was specified and
3195 * a mapping already exists at the specified virtual address. Returns
3196 * KERN_RESOURCE_SHORTAGE if PMAP_ENTER_NOSLEEP was specified and a page table
3197 * page allocation failed. Returns KERN_RESOURCE_SHORTAGE if
3198 * PMAP_ENTER_NORECLAIM was specified and a PV entry allocation failed.
3199 *
3200 * The parameter "m" is only used when creating a managed, writeable mapping.
3201 */
3202 static int
pmap_enter_l3e(pmap_t pmap,vm_offset_t va,pml3_entry_t newpde,u_int flags,vm_page_t m,struct rwlock ** lockp)3203 pmap_enter_l3e(pmap_t pmap, vm_offset_t va, pml3_entry_t newpde, u_int flags,
3204 vm_page_t m, struct rwlock **lockp)
3205 {
3206 struct spglist free;
3207 pml3_entry_t oldl3e, *l3e;
3208 vm_page_t mt, pdpg;
3209 vm_page_t uwptpg;
3210
3211 KASSERT((newpde & (PG_M | PG_RW)) != PG_RW,
3212 ("pmap_enter_pde: newpde is missing PG_M"));
3213 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
3214
3215 if ((pdpg = pmap_allocl3e(pmap, va, (flags & PMAP_ENTER_NOSLEEP) != 0 ?
3216 NULL : lockp)) == NULL) {
3217 CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx"
3218 " in pmap %p", va, pmap);
3219 return (KERN_RESOURCE_SHORTAGE);
3220 }
3221 l3e = (pml3_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pdpg));
3222 l3e = &l3e[pmap_pml3e_index(va)];
3223 oldl3e = be64toh(*l3e);
3224 if ((oldl3e & PG_V) != 0) {
3225 KASSERT(pdpg->ref_count > 1,
3226 ("pmap_enter_pde: pdpg's wire count is too low"));
3227 if ((flags & PMAP_ENTER_NOREPLACE) != 0) {
3228 pdpg->ref_count--;
3229 CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx"
3230 " in pmap %p", va, pmap);
3231 return (KERN_FAILURE);
3232 }
3233 /* Break the existing mapping(s). */
3234 SLIST_INIT(&free);
3235 if ((oldl3e & RPTE_LEAF) != 0) {
3236 /*
3237 * The reference to the PD page that was acquired by
3238 * pmap_allocl3e() ensures that it won't be freed.
3239 * However, if the PDE resulted from a promotion, then
3240 * a reserved PT page could be freed.
3241 */
3242 (void)pmap_remove_l3e(pmap, l3e, va, &free, lockp);
3243 pmap_invalidate_l3e_page(pmap, va, oldl3e);
3244 } else {
3245 if (pmap_remove_ptes(pmap, va, va + L3_PAGE_SIZE, l3e,
3246 &free, lockp))
3247 pmap_invalidate_all(pmap);
3248 }
3249 vm_page_free_pages_toq(&free, true);
3250 if (va >= VM_MAXUSER_ADDRESS) {
3251 mt = PHYS_TO_VM_PAGE(be64toh(*l3e) & PG_FRAME);
3252 if (pmap_insert_pt_page(pmap, mt)) {
3253 /*
3254 * XXX Currently, this can't happen because
3255 * we do not perform pmap_enter(psind == 1)
3256 * on the kernel pmap.
3257 */
3258 panic("pmap_enter_pde: trie insert failed");
3259 }
3260 } else
3261 KASSERT(be64toh(*l3e) == 0, ("pmap_enter_pde: non-zero pde %p",
3262 l3e));
3263 }
3264
3265 /*
3266 * Allocate leaf ptpage for wired userspace pages.
3267 */
3268 uwptpg = NULL;
3269 if ((newpde & PG_W) != 0 && pmap != kernel_pmap) {
3270 uwptpg = vm_page_alloc_noobj(VM_ALLOC_WIRED);
3271 if (uwptpg == NULL) {
3272 pmap_abort_ptp(pmap, va, pdpg);
3273 return (KERN_RESOURCE_SHORTAGE);
3274 }
3275 uwptpg->pindex = pmap_l3e_pindex(va);
3276 if (pmap_insert_pt_page(pmap, uwptpg)) {
3277 vm_page_unwire_noq(uwptpg);
3278 vm_page_free(uwptpg);
3279 pmap_abort_ptp(pmap, va, pdpg);
3280 return (KERN_RESOURCE_SHORTAGE);
3281 }
3282 pmap_resident_count_inc(pmap, 1);
3283 uwptpg->ref_count = NPTEPG;
3284 pmap_fill_ptp((pt_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(uwptpg)),
3285 newpde);
3286 }
3287 if ((newpde & PG_MANAGED) != 0) {
3288 /*
3289 * Abort this mapping if its PV entry could not be created.
3290 */
3291 if (!pmap_pv_insert_l3e(pmap, va, newpde, flags, lockp)) {
3292 pmap_abort_ptp(pmap, va, pdpg);
3293 if (uwptpg != NULL) {
3294 mt = pmap_remove_pt_page(pmap, va);
3295 KASSERT(mt == uwptpg,
3296 ("removed pt page %p, expected %p", mt,
3297 uwptpg));
3298 pmap_resident_count_dec(pmap, 1);
3299 uwptpg->ref_count = 1;
3300 vm_page_unwire_noq(uwptpg);
3301 vm_page_free(uwptpg);
3302 }
3303 CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx"
3304 " in pmap %p", va, pmap);
3305 return (KERN_RESOURCE_SHORTAGE);
3306 }
3307 if ((newpde & PG_RW) != 0) {
3308 for (mt = m; mt < &m[L3_PAGE_SIZE / PAGE_SIZE]; mt++)
3309 vm_page_aflag_set(mt, PGA_WRITEABLE);
3310 }
3311 }
3312
3313 /*
3314 * Increment counters.
3315 */
3316 if ((newpde & PG_W) != 0)
3317 pmap->pm_stats.wired_count += L3_PAGE_SIZE / PAGE_SIZE;
3318 pmap_resident_count_inc(pmap, L3_PAGE_SIZE / PAGE_SIZE);
3319
3320 /*
3321 * Map the superpage. (This is not a promoted mapping; there will not
3322 * be any lingering 4KB page mappings in the TLB.)
3323 */
3324 pte_store(l3e, newpde);
3325 ptesync();
3326
3327 counter_u64_add(pmap_l3e_mappings, 1);
3328 CTR2(KTR_PMAP, "pmap_enter_pde: success for va %#lx"
3329 " in pmap %p", va, pmap);
3330 return (KERN_SUCCESS);
3331 }
3332
3333 void
mmu_radix_enter_object(pmap_t pmap,vm_offset_t start,vm_offset_t end,vm_page_t m_start,vm_prot_t prot)3334 mmu_radix_enter_object(pmap_t pmap, vm_offset_t start,
3335 vm_offset_t end, vm_page_t m_start, vm_prot_t prot)
3336 {
3337 struct pctrie_iter pages;
3338 struct rwlock *lock;
3339 vm_offset_t va;
3340 vm_page_t m, mpte;
3341 bool invalidate;
3342
3343 VM_OBJECT_ASSERT_LOCKED(m_start->object);
3344
3345 CTR6(KTR_PMAP, "%s(%p, %#x, %#x, %p, %#x)", __func__, pmap, start,
3346 end, m_start, prot);
3347 invalidate = false;
3348 mpte = NULL;
3349 vm_page_iter_limit_init(&pages, m_start->object,
3350 m_start->pindex + atop(end - start));
3351 m = vm_radix_iter_lookup(&pages, m_start->pindex);
3352 lock = NULL;
3353 PMAP_LOCK(pmap);
3354 while (m != NULL) {
3355 va = start + ptoa(m->pindex - m_start->pindex);
3356 if ((va & L3_PAGE_MASK) == 0 && va + L3_PAGE_SIZE <= end &&
3357 m->psind == 1 && mmu_radix_ps_enabled(pmap) &&
3358 pmap_enter_2mpage(pmap, va, m, prot, &lock)) {
3359 m = vm_radix_iter_jump(&pages, L3_PAGE_SIZE / PAGE_SIZE);
3360 } else {
3361 mpte = mmu_radix_enter_quick_locked(pmap, va, m, prot,
3362 mpte, &lock, &invalidate);
3363 m = vm_radix_iter_step(&pages);
3364 }
3365 }
3366 ptesync();
3367 if (lock != NULL)
3368 rw_wunlock(lock);
3369 if (invalidate)
3370 pmap_invalidate_all(pmap);
3371 PMAP_UNLOCK(pmap);
3372 }
3373
3374 static vm_page_t
mmu_radix_enter_quick_locked(pmap_t pmap,vm_offset_t va,vm_page_t m,vm_prot_t prot,vm_page_t mpte,struct rwlock ** lockp,bool * invalidate)3375 mmu_radix_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m,
3376 vm_prot_t prot, vm_page_t mpte, struct rwlock **lockp, bool *invalidate)
3377 {
3378 struct spglist free;
3379 pt_entry_t *pte;
3380 vm_paddr_t pa;
3381
3382 KASSERT(!VA_IS_CLEANMAP(va) ||
3383 (m->oflags & VPO_UNMANAGED) != 0,
3384 ("mmu_radix_enter_quick_locked: managed mapping within the clean submap"));
3385 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
3386
3387 /*
3388 * In the case that a page table page is not
3389 * resident, we are creating it here.
3390 */
3391 if (va < VM_MAXUSER_ADDRESS) {
3392 vm_pindex_t ptepindex;
3393 pml3_entry_t *ptepa;
3394
3395 /*
3396 * Calculate pagetable page index
3397 */
3398 ptepindex = pmap_l3e_pindex(va);
3399 if (mpte && (mpte->pindex == ptepindex)) {
3400 mpte->ref_count++;
3401 } else {
3402 /*
3403 * Get the page directory entry
3404 */
3405 ptepa = pmap_pml3e(pmap, va);
3406
3407 /*
3408 * If the page table page is mapped, we just increment
3409 * the hold count, and activate it. Otherwise, we
3410 * attempt to allocate a page table page. If this
3411 * attempt fails, we don't retry. Instead, we give up.
3412 */
3413 if (ptepa && (be64toh(*ptepa) & PG_V) != 0) {
3414 if (be64toh(*ptepa) & RPTE_LEAF)
3415 return (NULL);
3416 mpte = PHYS_TO_VM_PAGE(be64toh(*ptepa) & PG_FRAME);
3417 mpte->ref_count++;
3418 } else {
3419 /*
3420 * Pass NULL instead of the PV list lock
3421 * pointer, because we don't intend to sleep.
3422 */
3423 mpte = _pmap_allocpte(pmap, ptepindex, NULL);
3424 if (mpte == NULL)
3425 return (mpte);
3426 }
3427 }
3428 pte = (pt_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mpte));
3429 pte = &pte[pmap_pte_index(va)];
3430 } else {
3431 mpte = NULL;
3432 pte = pmap_pte(pmap, va);
3433 }
3434 if (be64toh(*pte)) {
3435 if (mpte != NULL) {
3436 mpte->ref_count--;
3437 mpte = NULL;
3438 }
3439 return (mpte);
3440 }
3441
3442 /*
3443 * Enter on the PV list if part of our managed memory.
3444 */
3445 if ((m->oflags & VPO_UNMANAGED) == 0 &&
3446 !pmap_try_insert_pv_entry(pmap, va, m, lockp)) {
3447 if (mpte != NULL) {
3448 SLIST_INIT(&free);
3449 if (pmap_unwire_ptp(pmap, va, mpte, &free)) {
3450 /*
3451 * Although "va" is not mapped, paging-
3452 * structure caches could nonetheless have
3453 * entries that refer to the freed page table
3454 * pages. Invalidate those entries.
3455 */
3456 *invalidate = true;
3457 vm_page_free_pages_toq(&free, true);
3458 }
3459 mpte = NULL;
3460 }
3461 return (mpte);
3462 }
3463
3464 /*
3465 * Increment counters
3466 */
3467 pmap_resident_count_inc(pmap, 1);
3468
3469 pa = VM_PAGE_TO_PHYS(m) | pmap_cache_bits(m->md.mdpg_cache_attrs);
3470 if (prot & VM_PROT_EXECUTE)
3471 pa |= PG_X;
3472 else
3473 pa |= RPTE_EAA_R;
3474 if ((m->oflags & VPO_UNMANAGED) == 0)
3475 pa |= PG_MANAGED;
3476
3477 pte_store(pte, pa);
3478 return (mpte);
3479 }
3480
3481 void
mmu_radix_enter_quick(pmap_t pmap,vm_offset_t va,vm_page_t m,vm_prot_t prot)3482 mmu_radix_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m,
3483 vm_prot_t prot)
3484 {
3485 struct rwlock *lock;
3486 bool invalidate;
3487
3488 lock = NULL;
3489 invalidate = false;
3490 PMAP_LOCK(pmap);
3491 mmu_radix_enter_quick_locked(pmap, va, m, prot, NULL, &lock,
3492 &invalidate);
3493 ptesync();
3494 if (lock != NULL)
3495 rw_wunlock(lock);
3496 if (invalidate)
3497 pmap_invalidate_all(pmap);
3498 PMAP_UNLOCK(pmap);
3499 }
3500
3501 vm_paddr_t
mmu_radix_extract(pmap_t pmap,vm_offset_t va)3502 mmu_radix_extract(pmap_t pmap, vm_offset_t va)
3503 {
3504 pml3_entry_t *l3e;
3505 pt_entry_t *pte;
3506 vm_paddr_t pa;
3507
3508 l3e = pmap_pml3e(pmap, va);
3509 if (__predict_false(l3e == NULL))
3510 return (0);
3511 if (be64toh(*l3e) & RPTE_LEAF) {
3512 pa = (be64toh(*l3e) & PG_PS_FRAME) | (va & L3_PAGE_MASK);
3513 pa |= (va & L3_PAGE_MASK);
3514 } else {
3515 /*
3516 * Beware of a concurrent promotion that changes the
3517 * PDE at this point! For example, vtopte() must not
3518 * be used to access the PTE because it would use the
3519 * new PDE. It is, however, safe to use the old PDE
3520 * because the page table page is preserved by the
3521 * promotion.
3522 */
3523 pte = pmap_l3e_to_pte(l3e, va);
3524 if (__predict_false(pte == NULL))
3525 return (0);
3526 pa = be64toh(*pte);
3527 pa = (pa & PG_FRAME) | (va & PAGE_MASK);
3528 pa |= (va & PAGE_MASK);
3529 }
3530 return (pa);
3531 }
3532
3533 vm_page_t
mmu_radix_extract_and_hold(pmap_t pmap,vm_offset_t va,vm_prot_t prot)3534 mmu_radix_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot)
3535 {
3536 pml3_entry_t l3e, *l3ep;
3537 pt_entry_t pte;
3538 vm_page_t m;
3539
3540 m = NULL;
3541 CTR4(KTR_PMAP, "%s(%p, %#x, %#x)", __func__, pmap, va, prot);
3542 PMAP_LOCK(pmap);
3543 l3ep = pmap_pml3e(pmap, va);
3544 if (l3ep != NULL && (l3e = be64toh(*l3ep))) {
3545 if (l3e & RPTE_LEAF) {
3546 if ((l3e & PG_RW) || (prot & VM_PROT_WRITE) == 0)
3547 m = PHYS_TO_VM_PAGE((l3e & PG_PS_FRAME) |
3548 (va & L3_PAGE_MASK));
3549 } else {
3550 /* Native endian PTE, do not pass to pmap functions */
3551 pte = be64toh(*pmap_l3e_to_pte(l3ep, va));
3552 if ((pte & PG_V) &&
3553 ((pte & PG_RW) || (prot & VM_PROT_WRITE) == 0))
3554 m = PHYS_TO_VM_PAGE(pte & PG_FRAME);
3555 }
3556 if (m != NULL && !vm_page_wire_mapped(m))
3557 m = NULL;
3558 }
3559 PMAP_UNLOCK(pmap);
3560 return (m);
3561 }
3562
3563 static int
mmu_radix_growkernel(vm_offset_t addr)3564 mmu_radix_growkernel(vm_offset_t addr)
3565 {
3566 vm_paddr_t paddr;
3567 vm_page_t nkpg;
3568 pml3_entry_t *l3e;
3569 pml2_entry_t *l2e;
3570
3571 CTR2(KTR_PMAP, "%s(%#x)", __func__, addr);
3572 if (VM_MIN_KERNEL_ADDRESS < addr &&
3573 addr < (VM_MIN_KERNEL_ADDRESS + nkpt * L3_PAGE_SIZE))
3574 return (KERN_SUCCESS);
3575
3576 addr = roundup2(addr, L3_PAGE_SIZE);
3577 if (addr - 1 >= vm_map_max(kernel_map))
3578 addr = vm_map_max(kernel_map);
3579 while (kernel_vm_end < addr) {
3580 l2e = pmap_pml2e(kernel_pmap, kernel_vm_end);
3581 if ((be64toh(*l2e) & PG_V) == 0) {
3582 /* We need a new PDP entry */
3583 nkpg = vm_page_alloc_noobj(VM_ALLOC_INTERRUPT |
3584 VM_ALLOC_NOFREE | VM_ALLOC_WIRED | VM_ALLOC_ZERO);
3585 if (nkpg == NULL)
3586 return (KERN_RESOURCE_SHORTAGE);
3587 nkpg->pindex = kernel_vm_end >> L2_PAGE_SIZE_SHIFT;
3588 paddr = VM_PAGE_TO_PHYS(nkpg);
3589 pde_store(l2e, paddr);
3590 continue; /* try again */
3591 }
3592 l3e = pmap_l2e_to_l3e(l2e, kernel_vm_end);
3593 if ((be64toh(*l3e) & PG_V) != 0) {
3594 kernel_vm_end = (kernel_vm_end + L3_PAGE_SIZE) & ~L3_PAGE_MASK;
3595 if (kernel_vm_end - 1 >= vm_map_max(kernel_map)) {
3596 kernel_vm_end = vm_map_max(kernel_map);
3597 break;
3598 }
3599 continue;
3600 }
3601
3602 nkpg = vm_page_alloc_noobj(VM_ALLOC_INTERRUPT |
3603 VM_ALLOC_NOFREE | VM_ALLOC_WIRED | VM_ALLOC_ZERO);
3604 if (nkpg == NULL)
3605 return (KERN_RESOURCE_SHORTAGE);
3606 nkpg->pindex = pmap_l3e_pindex(kernel_vm_end);
3607 paddr = VM_PAGE_TO_PHYS(nkpg);
3608 pde_store(l3e, paddr);
3609
3610 kernel_vm_end = (kernel_vm_end + L3_PAGE_SIZE) & ~L3_PAGE_MASK;
3611 if (kernel_vm_end - 1 >= vm_map_max(kernel_map)) {
3612 kernel_vm_end = vm_map_max(kernel_map);
3613 break;
3614 }
3615 }
3616 ptesync();
3617 return (KERN_SUCCESS);
3618 }
3619
3620 static MALLOC_DEFINE(M_RADIX_PGD, "radix_pgd", "radix page table root directory");
3621 static uma_zone_t zone_radix_pgd;
3622
3623 static int
radix_pgd_import(void * arg __unused,void ** store,int count,int domain __unused,int flags)3624 radix_pgd_import(void *arg __unused, void **store, int count, int domain __unused,
3625 int flags)
3626 {
3627 int req;
3628
3629 req = VM_ALLOC_WIRED | malloc2vm_flags(flags);
3630 for (int i = 0; i < count; i++) {
3631 vm_page_t m = vm_page_alloc_noobj_contig(req,
3632 RADIX_PGD_SIZE / PAGE_SIZE,
3633 0, (vm_paddr_t)-1, RADIX_PGD_SIZE, L1_PAGE_SIZE,
3634 VM_MEMATTR_DEFAULT);
3635 store[i] = (void *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m));
3636 }
3637 return (count);
3638 }
3639
3640 static void
radix_pgd_release(void * arg __unused,void ** store,int count)3641 radix_pgd_release(void *arg __unused, void **store, int count)
3642 {
3643 vm_page_t m;
3644 struct spglist free;
3645 int page_count;
3646
3647 SLIST_INIT(&free);
3648 page_count = RADIX_PGD_SIZE/PAGE_SIZE;
3649
3650 for (int i = 0; i < count; i++) {
3651 /*
3652 * XXX selectively remove dmap and KVA entries so we don't
3653 * need to bzero
3654 */
3655 m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)store[i]));
3656 for (int j = page_count-1; j >= 0; j--) {
3657 vm_page_unwire_noq(&m[j]);
3658 SLIST_INSERT_HEAD(&free, &m[j], plinks.s.ss);
3659 }
3660 vm_page_free_pages_toq(&free, false);
3661 }
3662 }
3663
3664 static void
mmu_radix_init(void)3665 mmu_radix_init(void)
3666 {
3667 vm_page_t mpte;
3668 vm_size_t s;
3669 int error, i, pv_npg;
3670
3671 /* XXX is this really needed for POWER? */
3672 /* L1TF, reserve page @0 unconditionally */
3673 vm_page_blacklist_add(0, bootverbose);
3674
3675 zone_radix_pgd = uma_zcache_create("radix_pgd_cache",
3676 RADIX_PGD_SIZE, NULL, NULL,
3677 #ifdef INVARIANTS
3678 trash_init, trash_fini,
3679 #else
3680 NULL, NULL,
3681 #endif
3682 radix_pgd_import, radix_pgd_release,
3683 NULL, UMA_ZONE_NOBUCKET);
3684
3685 /*
3686 * Initialize the vm page array entries for the kernel pmap's
3687 * page table pages.
3688 */
3689 PMAP_LOCK(kernel_pmap);
3690 for (i = 0; i < nkpt; i++) {
3691 mpte = PHYS_TO_VM_PAGE(KPTphys + (i << PAGE_SHIFT));
3692 KASSERT(mpte >= vm_page_array &&
3693 mpte < &vm_page_array[vm_page_array_size],
3694 ("pmap_init: page table page is out of range size: %lu",
3695 vm_page_array_size));
3696 mpte->pindex = pmap_l3e_pindex(VM_MIN_KERNEL_ADDRESS) + i;
3697 mpte->phys_addr = KPTphys + (i << PAGE_SHIFT);
3698 MPASS(PHYS_TO_VM_PAGE(mpte->phys_addr) == mpte);
3699 //pmap_insert_pt_page(kernel_pmap, mpte);
3700 mpte->ref_count = 1;
3701 }
3702 PMAP_UNLOCK(kernel_pmap);
3703 vm_wire_add(nkpt);
3704
3705 CTR1(KTR_PMAP, "%s()", __func__);
3706 TAILQ_INIT(&pv_dummy.pv_list);
3707
3708 /*
3709 * Are large page mappings enabled?
3710 */
3711 TUNABLE_INT_FETCH("vm.pmap.superpages_enabled", &superpages_enabled);
3712 if (superpages_enabled) {
3713 KASSERT(MAXPAGESIZES > 1 && pagesizes[1] == 0,
3714 ("pmap_init: can't assign to pagesizes[1]"));
3715 pagesizes[1] = L3_PAGE_SIZE;
3716 }
3717
3718 /*
3719 * Initialize the pv chunk list mutex.
3720 */
3721 mtx_init(&pv_chunks_mutex, "pmap pv chunk list", NULL, MTX_DEF);
3722
3723 /*
3724 * Initialize the pool of pv list locks.
3725 */
3726 for (i = 0; i < NPV_LIST_LOCKS; i++)
3727 rw_init(&pv_list_locks[i], "pmap pv list");
3728
3729 /*
3730 * Calculate the size of the pv head table for superpages.
3731 */
3732 pv_npg = howmany(vm_phys_segs[vm_phys_nsegs - 1].end, L3_PAGE_SIZE);
3733
3734 /*
3735 * Allocate memory for the pv head table for superpages.
3736 */
3737 s = (vm_size_t)(pv_npg * sizeof(struct md_page));
3738 s = round_page(s);
3739 pv_table = kmem_malloc(s, M_WAITOK | M_ZERO);
3740 for (i = 0; i < pv_npg; i++)
3741 TAILQ_INIT(&pv_table[i].pv_list);
3742 TAILQ_INIT(&pv_dummy.pv_list);
3743
3744 pmap_initialized = 1;
3745 mtx_init(&qframe_mtx, "qfrmlk", NULL, MTX_SPIN);
3746 error = vmem_alloc(kernel_arena, PAGE_SIZE, M_BESTFIT | M_WAITOK,
3747 (vmem_addr_t *)&qframe);
3748
3749 if (error != 0)
3750 panic("qframe allocation failed");
3751 asid_arena = vmem_create("ASID", isa3_base_pid + 1, (1<<isa3_pid_bits),
3752 1, 1, M_WAITOK);
3753 }
3754
3755 static bool
pmap_page_test_mappings(vm_page_t m,bool accessed,bool modified)3756 pmap_page_test_mappings(vm_page_t m, bool accessed, bool modified)
3757 {
3758 struct rwlock *lock;
3759 pv_entry_t pv;
3760 struct md_page *pvh;
3761 pt_entry_t *pte, mask;
3762 pmap_t pmap;
3763 int md_gen, pvh_gen;
3764 bool rv;
3765
3766 rv = false;
3767 lock = VM_PAGE_TO_PV_LIST_LOCK(m);
3768 rw_rlock(lock);
3769 restart:
3770 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) {
3771 pmap = PV_PMAP(pv);
3772 if (!PMAP_TRYLOCK(pmap)) {
3773 md_gen = m->md.pv_gen;
3774 rw_runlock(lock);
3775 PMAP_LOCK(pmap);
3776 rw_rlock(lock);
3777 if (md_gen != m->md.pv_gen) {
3778 PMAP_UNLOCK(pmap);
3779 goto restart;
3780 }
3781 }
3782 pte = pmap_pte(pmap, pv->pv_va);
3783 mask = 0;
3784 if (modified)
3785 mask |= PG_RW | PG_M;
3786 if (accessed)
3787 mask |= PG_V | PG_A;
3788 rv = (be64toh(*pte) & mask) == mask;
3789 PMAP_UNLOCK(pmap);
3790 if (rv)
3791 goto out;
3792 }
3793 if ((m->flags & PG_FICTITIOUS) == 0) {
3794 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
3795 TAILQ_FOREACH(pv, &pvh->pv_list, pv_link) {
3796 pmap = PV_PMAP(pv);
3797 if (!PMAP_TRYLOCK(pmap)) {
3798 md_gen = m->md.pv_gen;
3799 pvh_gen = pvh->pv_gen;
3800 rw_runlock(lock);
3801 PMAP_LOCK(pmap);
3802 rw_rlock(lock);
3803 if (md_gen != m->md.pv_gen ||
3804 pvh_gen != pvh->pv_gen) {
3805 PMAP_UNLOCK(pmap);
3806 goto restart;
3807 }
3808 }
3809 pte = pmap_pml3e(pmap, pv->pv_va);
3810 mask = 0;
3811 if (modified)
3812 mask |= PG_RW | PG_M;
3813 if (accessed)
3814 mask |= PG_V | PG_A;
3815 rv = (be64toh(*pte) & mask) == mask;
3816 PMAP_UNLOCK(pmap);
3817 if (rv)
3818 goto out;
3819 }
3820 }
3821 out:
3822 rw_runlock(lock);
3823 return (rv);
3824 }
3825
3826 /*
3827 * pmap_is_modified:
3828 *
3829 * Return whether or not the specified physical page was modified
3830 * in any physical maps.
3831 */
3832 bool
mmu_radix_is_modified(vm_page_t m)3833 mmu_radix_is_modified(vm_page_t m)
3834 {
3835
3836 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
3837 ("pmap_is_modified: page %p is not managed", m));
3838
3839 CTR2(KTR_PMAP, "%s(%p)", __func__, m);
3840 /*
3841 * If the page is not busied then this check is racy.
3842 */
3843 if (!pmap_page_is_write_mapped(m))
3844 return (false);
3845 return (pmap_page_test_mappings(m, false, true));
3846 }
3847
3848 bool
mmu_radix_is_prefaultable(pmap_t pmap,vm_offset_t addr)3849 mmu_radix_is_prefaultable(pmap_t pmap, vm_offset_t addr)
3850 {
3851 pml3_entry_t *l3e;
3852 pt_entry_t *pte;
3853 bool rv;
3854
3855 CTR3(KTR_PMAP, "%s(%p, %#x)", __func__, pmap, addr);
3856 rv = false;
3857 PMAP_LOCK(pmap);
3858 l3e = pmap_pml3e(pmap, addr);
3859 if (l3e != NULL && (be64toh(*l3e) & (RPTE_LEAF | PG_V)) == PG_V) {
3860 pte = pmap_l3e_to_pte(l3e, addr);
3861 rv = (be64toh(*pte) & PG_V) == 0;
3862 }
3863 PMAP_UNLOCK(pmap);
3864 return (rv);
3865 }
3866
3867 bool
mmu_radix_is_referenced(vm_page_t m)3868 mmu_radix_is_referenced(vm_page_t m)
3869 {
3870 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
3871 ("pmap_is_referenced: page %p is not managed", m));
3872 CTR2(KTR_PMAP, "%s(%p)", __func__, m);
3873 return (pmap_page_test_mappings(m, true, false));
3874 }
3875
3876 /*
3877 * pmap_ts_referenced:
3878 *
3879 * Return a count of reference bits for a page, clearing those bits.
3880 * It is not necessary for every reference bit to be cleared, but it
3881 * is necessary that 0 only be returned when there are truly no
3882 * reference bits set.
3883 *
3884 * As an optimization, update the page's dirty field if a modified bit is
3885 * found while counting reference bits. This opportunistic update can be
3886 * performed at low cost and can eliminate the need for some future calls
3887 * to pmap_is_modified(). However, since this function stops after
3888 * finding PMAP_TS_REFERENCED_MAX reference bits, it may not detect some
3889 * dirty pages. Those dirty pages will only be detected by a future call
3890 * to pmap_is_modified().
3891 *
3892 * A DI block is not needed within this function, because
3893 * invalidations are performed before the PV list lock is
3894 * released.
3895 */
3896 int
mmu_radix_ts_referenced(vm_page_t m)3897 mmu_radix_ts_referenced(vm_page_t m)
3898 {
3899 struct md_page *pvh;
3900 pv_entry_t pv, pvf;
3901 pmap_t pmap;
3902 struct rwlock *lock;
3903 pml3_entry_t oldl3e, *l3e;
3904 pt_entry_t *pte;
3905 vm_paddr_t pa;
3906 int cleared, md_gen, not_cleared, pvh_gen;
3907 struct spglist free;
3908
3909 CTR2(KTR_PMAP, "%s(%p)", __func__, m);
3910 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
3911 ("pmap_ts_referenced: page %p is not managed", m));
3912 SLIST_INIT(&free);
3913 cleared = 0;
3914 pa = VM_PAGE_TO_PHYS(m);
3915 lock = PHYS_TO_PV_LIST_LOCK(pa);
3916 pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy : pa_to_pvh(pa);
3917 rw_wlock(lock);
3918 retry:
3919 not_cleared = 0;
3920 if ((pvf = TAILQ_FIRST(&pvh->pv_list)) == NULL)
3921 goto small_mappings;
3922 pv = pvf;
3923 do {
3924 if (pvf == NULL)
3925 pvf = pv;
3926 pmap = PV_PMAP(pv);
3927 if (!PMAP_TRYLOCK(pmap)) {
3928 pvh_gen = pvh->pv_gen;
3929 rw_wunlock(lock);
3930 PMAP_LOCK(pmap);
3931 rw_wlock(lock);
3932 if (pvh_gen != pvh->pv_gen) {
3933 PMAP_UNLOCK(pmap);
3934 goto retry;
3935 }
3936 }
3937 l3e = pmap_pml3e(pmap, pv->pv_va);
3938 oldl3e = be64toh(*l3e);
3939 if ((oldl3e & (PG_M | PG_RW)) == (PG_M | PG_RW)) {
3940 /*
3941 * Although "oldpde" is mapping a 2MB page, because
3942 * this function is called at a 4KB page granularity,
3943 * we only update the 4KB page under test.
3944 */
3945 vm_page_dirty(m);
3946 }
3947 if ((oldl3e & PG_A) != 0) {
3948 /*
3949 * Since this reference bit is shared by 512 4KB
3950 * pages, it should not be cleared every time it is
3951 * tested. Apply a simple "hash" function on the
3952 * physical page number, the virtual superpage number,
3953 * and the pmap address to select one 4KB page out of
3954 * the 512 on which testing the reference bit will
3955 * result in clearing that reference bit. This
3956 * function is designed to avoid the selection of the
3957 * same 4KB page for every 2MB page mapping.
3958 *
3959 * On demotion, a mapping that hasn't been referenced
3960 * is simply destroyed. To avoid the possibility of a
3961 * subsequent page fault on a demoted wired mapping,
3962 * always leave its reference bit set. Moreover,
3963 * since the superpage is wired, the current state of
3964 * its reference bit won't affect page replacement.
3965 */
3966 if ((((pa >> PAGE_SHIFT) ^ (pv->pv_va >> L3_PAGE_SIZE_SHIFT) ^
3967 (uintptr_t)pmap) & (NPTEPG - 1)) == 0 &&
3968 (oldl3e & PG_W) == 0) {
3969 atomic_clear_long(l3e, htobe64(PG_A));
3970 pmap_invalidate_page(pmap, pv->pv_va);
3971 cleared++;
3972 KASSERT(lock == VM_PAGE_TO_PV_LIST_LOCK(m),
3973 ("inconsistent pv lock %p %p for page %p",
3974 lock, VM_PAGE_TO_PV_LIST_LOCK(m), m));
3975 } else
3976 not_cleared++;
3977 }
3978 PMAP_UNLOCK(pmap);
3979 /* Rotate the PV list if it has more than one entry. */
3980 if (pv != NULL && TAILQ_NEXT(pv, pv_link) != NULL) {
3981 TAILQ_REMOVE(&pvh->pv_list, pv, pv_link);
3982 TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_link);
3983 pvh->pv_gen++;
3984 }
3985 if (cleared + not_cleared >= PMAP_TS_REFERENCED_MAX)
3986 goto out;
3987 } while ((pv = TAILQ_FIRST(&pvh->pv_list)) != pvf);
3988 small_mappings:
3989 if ((pvf = TAILQ_FIRST(&m->md.pv_list)) == NULL)
3990 goto out;
3991 pv = pvf;
3992 do {
3993 if (pvf == NULL)
3994 pvf = pv;
3995 pmap = PV_PMAP(pv);
3996 if (!PMAP_TRYLOCK(pmap)) {
3997 pvh_gen = pvh->pv_gen;
3998 md_gen = m->md.pv_gen;
3999 rw_wunlock(lock);
4000 PMAP_LOCK(pmap);
4001 rw_wlock(lock);
4002 if (pvh_gen != pvh->pv_gen || md_gen != m->md.pv_gen) {
4003 PMAP_UNLOCK(pmap);
4004 goto retry;
4005 }
4006 }
4007 l3e = pmap_pml3e(pmap, pv->pv_va);
4008 KASSERT((be64toh(*l3e) & RPTE_LEAF) == 0,
4009 ("pmap_ts_referenced: found a 2mpage in page %p's pv list",
4010 m));
4011 pte = pmap_l3e_to_pte(l3e, pv->pv_va);
4012 if ((be64toh(*pte) & (PG_M | PG_RW)) == (PG_M | PG_RW))
4013 vm_page_dirty(m);
4014 if ((be64toh(*pte) & PG_A) != 0) {
4015 atomic_clear_long(pte, htobe64(PG_A));
4016 pmap_invalidate_page(pmap, pv->pv_va);
4017 cleared++;
4018 }
4019 PMAP_UNLOCK(pmap);
4020 /* Rotate the PV list if it has more than one entry. */
4021 if (pv != NULL && TAILQ_NEXT(pv, pv_link) != NULL) {
4022 TAILQ_REMOVE(&m->md.pv_list, pv, pv_link);
4023 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_link);
4024 m->md.pv_gen++;
4025 }
4026 } while ((pv = TAILQ_FIRST(&m->md.pv_list)) != pvf && cleared +
4027 not_cleared < PMAP_TS_REFERENCED_MAX);
4028 out:
4029 rw_wunlock(lock);
4030 vm_page_free_pages_toq(&free, true);
4031 return (cleared + not_cleared);
4032 }
4033
4034 static vm_offset_t
mmu_radix_map(vm_offset_t * virt __unused,vm_paddr_t start,vm_paddr_t end,int prot __unused)4035 mmu_radix_map(vm_offset_t *virt __unused, vm_paddr_t start,
4036 vm_paddr_t end, int prot __unused)
4037 {
4038
4039 CTR5(KTR_PMAP, "%s(%p, %#x, %#x, %#x)", __func__, virt, start, end,
4040 prot);
4041 return (PHYS_TO_DMAP(start));
4042 }
4043
4044 void
mmu_radix_object_init_pt(pmap_t pmap,vm_offset_t addr,vm_object_t object,vm_pindex_t pindex,vm_size_t size)4045 mmu_radix_object_init_pt(pmap_t pmap, vm_offset_t addr,
4046 vm_object_t object, vm_pindex_t pindex, vm_size_t size)
4047 {
4048 struct pctrie_iter pages;
4049 pml3_entry_t *l3e;
4050 vm_paddr_t pa, ptepa;
4051 vm_page_t p, pdpg;
4052 vm_memattr_t ma;
4053
4054 CTR6(KTR_PMAP, "%s(%p, %#x, %p, %u, %#x)", __func__, pmap, addr,
4055 object, pindex, size);
4056 VM_OBJECT_ASSERT_WLOCKED(object);
4057 KASSERT(object->type == OBJT_DEVICE || object->type == OBJT_SG,
4058 ("pmap_object_init_pt: non-device object"));
4059 /* NB: size can be logically ored with addr here */
4060 if ((addr & L3_PAGE_MASK) == 0 && (size & L3_PAGE_MASK) == 0) {
4061 if (!mmu_radix_ps_enabled(pmap))
4062 return;
4063 if (!vm_object_populate(object, pindex, pindex + atop(size)))
4064 return;
4065 vm_page_iter_init(&pages, object);
4066 p = vm_radix_iter_lookup(&pages, pindex);
4067
4068 KASSERT(p->valid == VM_PAGE_BITS_ALL,
4069 ("pmap_object_init_pt: invalid page %p", p));
4070 ma = p->md.mdpg_cache_attrs;
4071
4072 /*
4073 * Abort the mapping if the first page is not physically
4074 * aligned to a 2MB page boundary.
4075 */
4076 ptepa = VM_PAGE_TO_PHYS(p);
4077 if (ptepa & L3_PAGE_MASK)
4078 return;
4079
4080 /*
4081 * Skip the first page. Abort the mapping if the rest of
4082 * the pages are not physically contiguous or have differing
4083 * memory attributes.
4084 */
4085 for (pa = ptepa + PAGE_SIZE; pa < ptepa + size;
4086 pa += PAGE_SIZE) {
4087 p = vm_radix_iter_next(&pages);
4088 KASSERT(p->valid == VM_PAGE_BITS_ALL,
4089 ("pmap_object_init_pt: invalid page %p", p));
4090 if (pa != VM_PAGE_TO_PHYS(p) ||
4091 ma != p->md.mdpg_cache_attrs)
4092 return;
4093 }
4094
4095 PMAP_LOCK(pmap);
4096 for (pa = ptepa | pmap_cache_bits(ma);
4097 pa < ptepa + size; pa += L3_PAGE_SIZE) {
4098 pdpg = pmap_allocl3e(pmap, addr, NULL);
4099 if (pdpg == NULL) {
4100 /*
4101 * The creation of mappings below is only an
4102 * optimization. If a page directory page
4103 * cannot be allocated without blocking,
4104 * continue on to the next mapping rather than
4105 * blocking.
4106 */
4107 addr += L3_PAGE_SIZE;
4108 continue;
4109 }
4110 l3e = (pml3_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pdpg));
4111 l3e = &l3e[pmap_pml3e_index(addr)];
4112 if ((be64toh(*l3e) & PG_V) == 0) {
4113 pa |= PG_M | PG_A | PG_RW;
4114 pte_store(l3e, pa);
4115 pmap_resident_count_inc(pmap, L3_PAGE_SIZE / PAGE_SIZE);
4116 counter_u64_add(pmap_l3e_mappings, 1);
4117 } else {
4118 /* Continue on if the PDE is already valid. */
4119 pdpg->ref_count--;
4120 KASSERT(pdpg->ref_count > 0,
4121 ("pmap_object_init_pt: missing reference "
4122 "to page directory page, va: 0x%lx", addr));
4123 }
4124 addr += L3_PAGE_SIZE;
4125 }
4126 ptesync();
4127 PMAP_UNLOCK(pmap);
4128 }
4129 }
4130
4131 bool
mmu_radix_page_exists_quick(pmap_t pmap,vm_page_t m)4132 mmu_radix_page_exists_quick(pmap_t pmap, vm_page_t m)
4133 {
4134 struct md_page *pvh;
4135 struct rwlock *lock;
4136 pv_entry_t pv;
4137 int loops = 0;
4138 bool rv;
4139
4140 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
4141 ("pmap_page_exists_quick: page %p is not managed", m));
4142 CTR3(KTR_PMAP, "%s(%p, %p)", __func__, pmap, m);
4143 rv = false;
4144 lock = VM_PAGE_TO_PV_LIST_LOCK(m);
4145 rw_rlock(lock);
4146 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) {
4147 if (PV_PMAP(pv) == pmap) {
4148 rv = true;
4149 break;
4150 }
4151 loops++;
4152 if (loops >= 16)
4153 break;
4154 }
4155 if (!rv && loops < 16 && (m->flags & PG_FICTITIOUS) == 0) {
4156 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
4157 TAILQ_FOREACH(pv, &pvh->pv_list, pv_link) {
4158 if (PV_PMAP(pv) == pmap) {
4159 rv = true;
4160 break;
4161 }
4162 loops++;
4163 if (loops >= 16)
4164 break;
4165 }
4166 }
4167 rw_runlock(lock);
4168 return (rv);
4169 }
4170
4171 void
mmu_radix_page_init(vm_page_t m)4172 mmu_radix_page_init(vm_page_t m)
4173 {
4174
4175 CTR2(KTR_PMAP, "%s(%p)", __func__, m);
4176 TAILQ_INIT(&m->md.pv_list);
4177 m->md.mdpg_cache_attrs = VM_MEMATTR_DEFAULT;
4178 }
4179
4180 int
mmu_radix_page_wired_mappings(vm_page_t m)4181 mmu_radix_page_wired_mappings(vm_page_t m)
4182 {
4183 struct rwlock *lock;
4184 struct md_page *pvh;
4185 pmap_t pmap;
4186 pt_entry_t *pte;
4187 pv_entry_t pv;
4188 int count, md_gen, pvh_gen;
4189
4190 if ((m->oflags & VPO_UNMANAGED) != 0)
4191 return (0);
4192 CTR2(KTR_PMAP, "%s(%p)", __func__, m);
4193 lock = VM_PAGE_TO_PV_LIST_LOCK(m);
4194 rw_rlock(lock);
4195 restart:
4196 count = 0;
4197 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) {
4198 pmap = PV_PMAP(pv);
4199 if (!PMAP_TRYLOCK(pmap)) {
4200 md_gen = m->md.pv_gen;
4201 rw_runlock(lock);
4202 PMAP_LOCK(pmap);
4203 rw_rlock(lock);
4204 if (md_gen != m->md.pv_gen) {
4205 PMAP_UNLOCK(pmap);
4206 goto restart;
4207 }
4208 }
4209 pte = pmap_pte(pmap, pv->pv_va);
4210 if ((be64toh(*pte) & PG_W) != 0)
4211 count++;
4212 PMAP_UNLOCK(pmap);
4213 }
4214 if ((m->flags & PG_FICTITIOUS) == 0) {
4215 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
4216 TAILQ_FOREACH(pv, &pvh->pv_list, pv_link) {
4217 pmap = PV_PMAP(pv);
4218 if (!PMAP_TRYLOCK(pmap)) {
4219 md_gen = m->md.pv_gen;
4220 pvh_gen = pvh->pv_gen;
4221 rw_runlock(lock);
4222 PMAP_LOCK(pmap);
4223 rw_rlock(lock);
4224 if (md_gen != m->md.pv_gen ||
4225 pvh_gen != pvh->pv_gen) {
4226 PMAP_UNLOCK(pmap);
4227 goto restart;
4228 }
4229 }
4230 pte = pmap_pml3e(pmap, pv->pv_va);
4231 if ((be64toh(*pte) & PG_W) != 0)
4232 count++;
4233 PMAP_UNLOCK(pmap);
4234 }
4235 }
4236 rw_runlock(lock);
4237 return (count);
4238 }
4239
4240 static void
mmu_radix_update_proctab(int pid,pml1_entry_t l1pa)4241 mmu_radix_update_proctab(int pid, pml1_entry_t l1pa)
4242 {
4243 isa3_proctab[pid].proctab0 = htobe64(RTS_SIZE | l1pa | RADIX_PGD_INDEX_SHIFT);
4244 }
4245
4246 int
mmu_radix_pinit(pmap_t pmap)4247 mmu_radix_pinit(pmap_t pmap)
4248 {
4249 vmem_addr_t pid;
4250 vm_paddr_t l1pa;
4251
4252 CTR2(KTR_PMAP, "%s(%p)", __func__, pmap);
4253
4254 /*
4255 * allocate the page directory page
4256 */
4257 pmap->pm_pml1 = uma_zalloc(zone_radix_pgd, M_WAITOK);
4258
4259 for (int j = 0; j < RADIX_PGD_SIZE_SHIFT; j++)
4260 pagezero((vm_offset_t)pmap->pm_pml1 + j * PAGE_SIZE);
4261 vm_radix_init(&pmap->pm_radix);
4262 TAILQ_INIT(&pmap->pm_pvchunk);
4263 bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
4264 pmap->pm_flags = PMAP_PDE_SUPERPAGE;
4265 vmem_alloc(asid_arena, 1, M_FIRSTFIT|M_WAITOK, &pid);
4266
4267 pmap->pm_pid = pid;
4268 l1pa = DMAP_TO_PHYS((vm_offset_t)pmap->pm_pml1);
4269 mmu_radix_update_proctab(pid, l1pa);
4270 __asm __volatile("ptesync;isync" : : : "memory");
4271
4272 return (1);
4273 }
4274
4275 /*
4276 * This routine is called if the desired page table page does not exist.
4277 *
4278 * If page table page allocation fails, this routine may sleep before
4279 * returning NULL. It sleeps only if a lock pointer was given.
4280 *
4281 * Note: If a page allocation fails at page table level two or three,
4282 * one or two pages may be held during the wait, only to be released
4283 * afterwards. This conservative approach is easily argued to avoid
4284 * race conditions.
4285 */
4286 static vm_page_t
_pmap_allocpte(pmap_t pmap,vm_pindex_t ptepindex,struct rwlock ** lockp)4287 _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, struct rwlock **lockp)
4288 {
4289 vm_page_t m, pdppg, pdpg;
4290
4291 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
4292
4293 /*
4294 * Allocate a page table page.
4295 */
4296 if ((m = vm_page_alloc_noobj(VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) {
4297 if (lockp != NULL) {
4298 RELEASE_PV_LIST_LOCK(lockp);
4299 PMAP_UNLOCK(pmap);
4300 vm_wait(NULL);
4301 PMAP_LOCK(pmap);
4302 }
4303 /*
4304 * Indicate the need to retry. While waiting, the page table
4305 * page may have been allocated.
4306 */
4307 return (NULL);
4308 }
4309 m->pindex = ptepindex;
4310
4311 /*
4312 * Map the pagetable page into the process address space, if
4313 * it isn't already there.
4314 */
4315
4316 if (ptepindex >= (NUPDE + NUPDPE)) {
4317 pml1_entry_t *l1e;
4318 vm_pindex_t pml1index;
4319
4320 /* Wire up a new PDPE page */
4321 pml1index = ptepindex - (NUPDE + NUPDPE);
4322 l1e = &pmap->pm_pml1[pml1index];
4323 KASSERT((be64toh(*l1e) & PG_V) == 0,
4324 ("%s: L1 entry %#lx is valid", __func__, *l1e));
4325 pde_store(l1e, VM_PAGE_TO_PHYS(m));
4326 } else if (ptepindex >= NUPDE) {
4327 vm_pindex_t pml1index;
4328 vm_pindex_t pdpindex;
4329 pml1_entry_t *l1e;
4330 pml2_entry_t *l2e;
4331
4332 /* Wire up a new l2e page */
4333 pdpindex = ptepindex - NUPDE;
4334 pml1index = pdpindex >> RPTE_SHIFT;
4335
4336 l1e = &pmap->pm_pml1[pml1index];
4337 if ((be64toh(*l1e) & PG_V) == 0) {
4338 /* Have to allocate a new pdp, recurse */
4339 if (_pmap_allocpte(pmap, NUPDE + NUPDPE + pml1index,
4340 lockp) == NULL) {
4341 vm_page_unwire_noq(m);
4342 vm_page_free_zero(m);
4343 return (NULL);
4344 }
4345 } else {
4346 /* Add reference to l2e page */
4347 pdppg = PHYS_TO_VM_PAGE(be64toh(*l1e) & PG_FRAME);
4348 pdppg->ref_count++;
4349 }
4350 l2e = (pml2_entry_t *)PHYS_TO_DMAP(be64toh(*l1e) & PG_FRAME);
4351
4352 /* Now find the pdp page */
4353 l2e = &l2e[pdpindex & RPTE_MASK];
4354 KASSERT((be64toh(*l2e) & PG_V) == 0,
4355 ("%s: L2 entry %#lx is valid", __func__, *l2e));
4356 pde_store(l2e, VM_PAGE_TO_PHYS(m));
4357 } else {
4358 vm_pindex_t pml1index;
4359 vm_pindex_t pdpindex;
4360 pml1_entry_t *l1e;
4361 pml2_entry_t *l2e;
4362 pml3_entry_t *l3e;
4363
4364 /* Wire up a new PTE page */
4365 pdpindex = ptepindex >> RPTE_SHIFT;
4366 pml1index = pdpindex >> RPTE_SHIFT;
4367
4368 /* First, find the pdp and check that its valid. */
4369 l1e = &pmap->pm_pml1[pml1index];
4370 if ((be64toh(*l1e) & PG_V) == 0) {
4371 /* Have to allocate a new pd, recurse */
4372 if (_pmap_allocpte(pmap, NUPDE + pdpindex,
4373 lockp) == NULL) {
4374 vm_page_unwire_noq(m);
4375 vm_page_free_zero(m);
4376 return (NULL);
4377 }
4378 l2e = (pml2_entry_t *)PHYS_TO_DMAP(be64toh(*l1e) & PG_FRAME);
4379 l2e = &l2e[pdpindex & RPTE_MASK];
4380 } else {
4381 l2e = (pml2_entry_t *)PHYS_TO_DMAP(be64toh(*l1e) & PG_FRAME);
4382 l2e = &l2e[pdpindex & RPTE_MASK];
4383 if ((be64toh(*l2e) & PG_V) == 0) {
4384 /* Have to allocate a new pd, recurse */
4385 if (_pmap_allocpte(pmap, NUPDE + pdpindex,
4386 lockp) == NULL) {
4387 vm_page_unwire_noq(m);
4388 vm_page_free_zero(m);
4389 return (NULL);
4390 }
4391 } else {
4392 /* Add reference to the pd page */
4393 pdpg = PHYS_TO_VM_PAGE(be64toh(*l2e) & PG_FRAME);
4394 pdpg->ref_count++;
4395 }
4396 }
4397 l3e = (pml3_entry_t *)PHYS_TO_DMAP(be64toh(*l2e) & PG_FRAME);
4398
4399 /* Now we know where the page directory page is */
4400 l3e = &l3e[ptepindex & RPTE_MASK];
4401 KASSERT((be64toh(*l3e) & PG_V) == 0,
4402 ("%s: L3 entry %#lx is valid", __func__, *l3e));
4403 pde_store(l3e, VM_PAGE_TO_PHYS(m));
4404 }
4405
4406 pmap_resident_count_inc(pmap, 1);
4407 return (m);
4408 }
4409 static vm_page_t
pmap_allocl3e(pmap_t pmap,vm_offset_t va,struct rwlock ** lockp)4410 pmap_allocl3e(pmap_t pmap, vm_offset_t va, struct rwlock **lockp)
4411 {
4412 vm_pindex_t pdpindex, ptepindex;
4413 pml2_entry_t *pdpe;
4414 vm_page_t pdpg;
4415
4416 retry:
4417 pdpe = pmap_pml2e(pmap, va);
4418 if (pdpe != NULL && (be64toh(*pdpe) & PG_V) != 0) {
4419 /* Add a reference to the pd page. */
4420 pdpg = PHYS_TO_VM_PAGE(be64toh(*pdpe) & PG_FRAME);
4421 pdpg->ref_count++;
4422 } else {
4423 /* Allocate a pd page. */
4424 ptepindex = pmap_l3e_pindex(va);
4425 pdpindex = ptepindex >> RPTE_SHIFT;
4426 pdpg = _pmap_allocpte(pmap, NUPDE + pdpindex, lockp);
4427 if (pdpg == NULL && lockp != NULL)
4428 goto retry;
4429 }
4430 return (pdpg);
4431 }
4432
4433 static vm_page_t
pmap_allocpte(pmap_t pmap,vm_offset_t va,struct rwlock ** lockp)4434 pmap_allocpte(pmap_t pmap, vm_offset_t va, struct rwlock **lockp)
4435 {
4436 vm_pindex_t ptepindex;
4437 pml3_entry_t *pd;
4438 vm_page_t m;
4439
4440 /*
4441 * Calculate pagetable page index
4442 */
4443 ptepindex = pmap_l3e_pindex(va);
4444 retry:
4445 /*
4446 * Get the page directory entry
4447 */
4448 pd = pmap_pml3e(pmap, va);
4449
4450 /*
4451 * This supports switching from a 2MB page to a
4452 * normal 4K page.
4453 */
4454 if (pd != NULL && (be64toh(*pd) & (RPTE_LEAF | PG_V)) == (RPTE_LEAF | PG_V)) {
4455 if (!pmap_demote_l3e_locked(pmap, pd, va, lockp)) {
4456 /*
4457 * Invalidation of the 2MB page mapping may have caused
4458 * the deallocation of the underlying PD page.
4459 */
4460 pd = NULL;
4461 }
4462 }
4463
4464 /*
4465 * If the page table page is mapped, we just increment the
4466 * hold count, and activate it.
4467 */
4468 if (pd != NULL && (be64toh(*pd) & PG_V) != 0) {
4469 m = PHYS_TO_VM_PAGE(be64toh(*pd) & PG_FRAME);
4470 m->ref_count++;
4471 } else {
4472 /*
4473 * Here if the pte page isn't mapped, or if it has been
4474 * deallocated.
4475 */
4476 m = _pmap_allocpte(pmap, ptepindex, lockp);
4477 if (m == NULL && lockp != NULL)
4478 goto retry;
4479 }
4480 return (m);
4481 }
4482
4483 static void
mmu_radix_pinit0(pmap_t pmap)4484 mmu_radix_pinit0(pmap_t pmap)
4485 {
4486
4487 CTR2(KTR_PMAP, "%s(%p)", __func__, pmap);
4488 PMAP_LOCK_INIT(pmap);
4489 pmap->pm_pml1 = kernel_pmap->pm_pml1;
4490 pmap->pm_pid = kernel_pmap->pm_pid;
4491
4492 vm_radix_init(&pmap->pm_radix);
4493 TAILQ_INIT(&pmap->pm_pvchunk);
4494 bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
4495 kernel_pmap->pm_flags =
4496 pmap->pm_flags = PMAP_PDE_SUPERPAGE;
4497 }
4498 /*
4499 * pmap_protect_l3e: do the things to protect a 2mpage in a process
4500 */
4501 static bool
pmap_protect_l3e(pmap_t pmap,pt_entry_t * l3e,vm_offset_t sva,vm_prot_t prot)4502 pmap_protect_l3e(pmap_t pmap, pt_entry_t *l3e, vm_offset_t sva, vm_prot_t prot)
4503 {
4504 pt_entry_t newpde, oldpde;
4505 vm_offset_t eva, va;
4506 vm_page_t m;
4507 bool anychanged;
4508
4509 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
4510 KASSERT((sva & L3_PAGE_MASK) == 0,
4511 ("pmap_protect_l3e: sva is not 2mpage aligned"));
4512 anychanged = false;
4513 retry:
4514 oldpde = newpde = be64toh(*l3e);
4515 if ((oldpde & (PG_MANAGED | PG_M | PG_RW)) ==
4516 (PG_MANAGED | PG_M | PG_RW)) {
4517 eva = sva + L3_PAGE_SIZE;
4518 for (va = sva, m = PHYS_TO_VM_PAGE(oldpde & PG_PS_FRAME);
4519 va < eva; va += PAGE_SIZE, m++)
4520 vm_page_dirty(m);
4521 }
4522 if ((prot & VM_PROT_WRITE) == 0) {
4523 newpde &= ~(PG_RW | PG_M);
4524 newpde |= RPTE_EAA_R;
4525 }
4526 if (prot & VM_PROT_EXECUTE)
4527 newpde |= PG_X;
4528 if (newpde != oldpde) {
4529 /*
4530 * As an optimization to future operations on this PDE, clear
4531 * PG_PROMOTED. The impending invalidation will remove any
4532 * lingering 4KB page mappings from the TLB.
4533 */
4534 if (!atomic_cmpset_long(l3e, htobe64(oldpde), htobe64(newpde & ~PG_PROMOTED)))
4535 goto retry;
4536 anychanged = true;
4537 }
4538 return (anychanged);
4539 }
4540
4541 void
mmu_radix_protect(pmap_t pmap,vm_offset_t sva,vm_offset_t eva,vm_prot_t prot)4542 mmu_radix_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,
4543 vm_prot_t prot)
4544 {
4545 vm_offset_t va_next;
4546 pml1_entry_t *l1e;
4547 pml2_entry_t *l2e;
4548 pml3_entry_t ptpaddr, *l3e;
4549 pt_entry_t *pte;
4550 bool anychanged;
4551
4552 CTR5(KTR_PMAP, "%s(%p, %#x, %#x, %#x)", __func__, pmap, sva, eva,
4553 prot);
4554
4555 KASSERT((prot & ~VM_PROT_ALL) == 0, ("invalid prot %x", prot));
4556 if (prot == VM_PROT_NONE) {
4557 mmu_radix_remove(pmap, sva, eva);
4558 return;
4559 }
4560
4561 if ((prot & (VM_PROT_WRITE|VM_PROT_EXECUTE)) ==
4562 (VM_PROT_WRITE|VM_PROT_EXECUTE))
4563 return;
4564
4565 #ifdef INVARIANTS
4566 if (VERBOSE_PROTECT || pmap_logging)
4567 printf("pmap_protect(%p, %#lx, %#lx, %x) - asid: %lu\n",
4568 pmap, sva, eva, prot, pmap->pm_pid);
4569 #endif
4570 anychanged = false;
4571
4572 PMAP_LOCK(pmap);
4573 for (; sva < eva; sva = va_next) {
4574 l1e = pmap_pml1e(pmap, sva);
4575 if ((be64toh(*l1e) & PG_V) == 0) {
4576 va_next = (sva + L1_PAGE_SIZE) & ~L1_PAGE_MASK;
4577 if (va_next < sva)
4578 va_next = eva;
4579 continue;
4580 }
4581
4582 l2e = pmap_l1e_to_l2e(l1e, sva);
4583 if ((be64toh(*l2e) & PG_V) == 0) {
4584 va_next = (sva + L2_PAGE_SIZE) & ~L2_PAGE_MASK;
4585 if (va_next < sva)
4586 va_next = eva;
4587 continue;
4588 }
4589
4590 va_next = (sva + L3_PAGE_SIZE) & ~L3_PAGE_MASK;
4591 if (va_next < sva)
4592 va_next = eva;
4593
4594 l3e = pmap_l2e_to_l3e(l2e, sva);
4595 ptpaddr = be64toh(*l3e);
4596
4597 /*
4598 * Weed out invalid mappings.
4599 */
4600 if (ptpaddr == 0)
4601 continue;
4602
4603 /*
4604 * Check for large page.
4605 */
4606 if ((ptpaddr & RPTE_LEAF) != 0) {
4607 /*
4608 * Are we protecting the entire large page? If not,
4609 * demote the mapping and fall through.
4610 */
4611 if (sva + L3_PAGE_SIZE == va_next && eva >= va_next) {
4612 if (pmap_protect_l3e(pmap, l3e, sva, prot))
4613 anychanged = true;
4614 continue;
4615 } else if (!pmap_demote_l3e(pmap, l3e, sva)) {
4616 /*
4617 * The large page mapping was destroyed.
4618 */
4619 continue;
4620 }
4621 }
4622
4623 if (va_next > eva)
4624 va_next = eva;
4625
4626 for (pte = pmap_l3e_to_pte(l3e, sva); sva != va_next; pte++,
4627 sva += PAGE_SIZE) {
4628 pt_entry_t obits, pbits;
4629 vm_page_t m;
4630
4631 retry:
4632 MPASS(pte == pmap_pte(pmap, sva));
4633 obits = pbits = be64toh(*pte);
4634 if ((pbits & PG_V) == 0)
4635 continue;
4636
4637 if ((prot & VM_PROT_WRITE) == 0) {
4638 if ((pbits & (PG_MANAGED | PG_M | PG_RW)) ==
4639 (PG_MANAGED | PG_M | PG_RW)) {
4640 m = PHYS_TO_VM_PAGE(pbits & PG_FRAME);
4641 vm_page_dirty(m);
4642 }
4643 pbits &= ~(PG_RW | PG_M);
4644 pbits |= RPTE_EAA_R;
4645 }
4646 if (prot & VM_PROT_EXECUTE)
4647 pbits |= PG_X;
4648
4649 if (pbits != obits) {
4650 if (!atomic_cmpset_long(pte, htobe64(obits), htobe64(pbits)))
4651 goto retry;
4652 if (obits & (PG_A|PG_M)) {
4653 anychanged = true;
4654 #ifdef INVARIANTS
4655 if (VERBOSE_PROTECT || pmap_logging)
4656 printf("%#lx %#lx -> %#lx\n",
4657 sva, obits, pbits);
4658 #endif
4659 }
4660 }
4661 }
4662 }
4663 if (anychanged)
4664 pmap_invalidate_all(pmap);
4665 PMAP_UNLOCK(pmap);
4666 }
4667
4668 void
mmu_radix_qenter(vm_offset_t sva,vm_page_t * ma,int count)4669 mmu_radix_qenter(vm_offset_t sva, vm_page_t *ma, int count)
4670 {
4671
4672 CTR4(KTR_PMAP, "%s(%#x, %p, %d)", __func__, sva, ma, count);
4673 pt_entry_t oldpte, pa, *pte;
4674 vm_page_t m;
4675 uint64_t cache_bits, attr_bits;
4676 vm_offset_t va;
4677
4678 oldpte = 0;
4679 attr_bits = RPTE_EAA_R | RPTE_EAA_W | RPTE_EAA_P | PG_M | PG_A;
4680 va = sva;
4681 pte = kvtopte(va);
4682 while (va < sva + PAGE_SIZE * count) {
4683 if (__predict_false((va & L3_PAGE_MASK) == 0))
4684 pte = kvtopte(va);
4685 MPASS(pte == pmap_pte(kernel_pmap, va));
4686
4687 /*
4688 * XXX there has to be a more efficient way than traversing
4689 * the page table every time - but go for correctness for
4690 * today
4691 */
4692
4693 m = *ma++;
4694 cache_bits = pmap_cache_bits(m->md.mdpg_cache_attrs);
4695 pa = VM_PAGE_TO_PHYS(m) | cache_bits | attr_bits;
4696 if (be64toh(*pte) != pa) {
4697 oldpte |= be64toh(*pte);
4698 pte_store(pte, pa);
4699 }
4700 va += PAGE_SIZE;
4701 pte++;
4702 }
4703 if (__predict_false((oldpte & RPTE_VALID) != 0))
4704 pmap_invalidate_range(kernel_pmap, sva, sva + count *
4705 PAGE_SIZE);
4706 else
4707 ptesync();
4708 }
4709
4710 void
mmu_radix_qremove(vm_offset_t sva,int count)4711 mmu_radix_qremove(vm_offset_t sva, int count)
4712 {
4713 vm_offset_t va;
4714 pt_entry_t *pte;
4715
4716 CTR3(KTR_PMAP, "%s(%#x, %d)", __func__, sva, count);
4717 KASSERT(sva >= VM_MIN_KERNEL_ADDRESS, ("usermode or dmap va %lx", sva));
4718
4719 va = sva;
4720 pte = kvtopte(va);
4721 while (va < sva + PAGE_SIZE * count) {
4722 if (__predict_false((va & L3_PAGE_MASK) == 0))
4723 pte = kvtopte(va);
4724 pte_clear(pte);
4725 pte++;
4726 va += PAGE_SIZE;
4727 }
4728 pmap_invalidate_range(kernel_pmap, sva, va);
4729 }
4730
4731 /***************************************************
4732 * Page table page management routines.....
4733 ***************************************************/
4734 /*
4735 * Schedule the specified unused page table page to be freed. Specifically,
4736 * add the page to the specified list of pages that will be released to the
4737 * physical memory manager after the TLB has been updated.
4738 */
4739 static __inline void
pmap_add_delayed_free_list(vm_page_t m,struct spglist * free,bool set_PG_ZERO)4740 pmap_add_delayed_free_list(vm_page_t m, struct spglist *free, bool set_PG_ZERO)
4741 {
4742
4743 if (set_PG_ZERO)
4744 m->flags |= PG_ZERO;
4745 else
4746 m->flags &= ~PG_ZERO;
4747 SLIST_INSERT_HEAD(free, m, plinks.s.ss);
4748 }
4749
4750 /*
4751 * Inserts the specified page table page into the specified pmap's collection
4752 * of idle page table pages. Each of a pmap's page table pages is responsible
4753 * for mapping a distinct range of virtual addresses. The pmap's collection is
4754 * ordered by this virtual address range.
4755 */
4756 static __inline int
pmap_insert_pt_page(pmap_t pmap,vm_page_t mpte)4757 pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte)
4758 {
4759
4760 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
4761 return (vm_radix_insert(&pmap->pm_radix, mpte));
4762 }
4763
4764 /*
4765 * Removes the page table page mapping the specified virtual address from the
4766 * specified pmap's collection of idle page table pages, and returns it.
4767 * Otherwise, returns NULL if there is no page table page corresponding to the
4768 * specified virtual address.
4769 */
4770 static __inline vm_page_t
pmap_remove_pt_page(pmap_t pmap,vm_offset_t va)4771 pmap_remove_pt_page(pmap_t pmap, vm_offset_t va)
4772 {
4773
4774 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
4775 return (vm_radix_remove(&pmap->pm_radix, pmap_l3e_pindex(va)));
4776 }
4777
4778 /*
4779 * Decrements a page table page's wire count, which is used to record the
4780 * number of valid page table entries within the page. If the wire count
4781 * drops to zero, then the page table page is unmapped. Returns true if the
4782 * page table page was unmapped and false otherwise.
4783 */
4784 static inline bool
pmap_unwire_ptp(pmap_t pmap,vm_offset_t va,vm_page_t m,struct spglist * free)4785 pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free)
4786 {
4787
4788 --m->ref_count;
4789 if (m->ref_count == 0) {
4790 _pmap_unwire_ptp(pmap, va, m, free);
4791 return (true);
4792 } else
4793 return (false);
4794 }
4795
4796 static void
_pmap_unwire_ptp(pmap_t pmap,vm_offset_t va,vm_page_t m,struct spglist * free)4797 _pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free)
4798 {
4799
4800 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
4801 /*
4802 * unmap the page table page
4803 */
4804 if (m->pindex >= NUPDE + NUPDPE) {
4805 /* PDP page */
4806 pml1_entry_t *pml1;
4807 pml1 = pmap_pml1e(pmap, va);
4808 *pml1 = 0;
4809 } else if (m->pindex >= NUPDE) {
4810 /* PD page */
4811 pml2_entry_t *l2e;
4812 l2e = pmap_pml2e(pmap, va);
4813 *l2e = 0;
4814 } else {
4815 /* PTE page */
4816 pml3_entry_t *l3e;
4817 l3e = pmap_pml3e(pmap, va);
4818 *l3e = 0;
4819 }
4820 pmap_resident_count_dec(pmap, 1);
4821 if (m->pindex < NUPDE) {
4822 /* We just released a PT, unhold the matching PD */
4823 vm_page_t pdpg;
4824
4825 pdpg = PHYS_TO_VM_PAGE(be64toh(*pmap_pml2e(pmap, va)) & PG_FRAME);
4826 pmap_unwire_ptp(pmap, va, pdpg, free);
4827 }
4828 else if (m->pindex >= NUPDE && m->pindex < (NUPDE + NUPDPE)) {
4829 /* We just released a PD, unhold the matching PDP */
4830 vm_page_t pdppg;
4831
4832 pdppg = PHYS_TO_VM_PAGE(be64toh(*pmap_pml1e(pmap, va)) & PG_FRAME);
4833 pmap_unwire_ptp(pmap, va, pdppg, free);
4834 }
4835
4836 /*
4837 * Put page on a list so that it is released after
4838 * *ALL* TLB shootdown is done
4839 */
4840 pmap_add_delayed_free_list(m, free, true);
4841 }
4842
4843 /*
4844 * After removing a page table entry, this routine is used to
4845 * conditionally free the page, and manage the hold/wire counts.
4846 */
4847 static int
pmap_unuse_pt(pmap_t pmap,vm_offset_t va,pml3_entry_t ptepde,struct spglist * free)4848 pmap_unuse_pt(pmap_t pmap, vm_offset_t va, pml3_entry_t ptepde,
4849 struct spglist *free)
4850 {
4851 vm_page_t mpte;
4852
4853 if (va >= VM_MAXUSER_ADDRESS)
4854 return (0);
4855 KASSERT(ptepde != 0, ("pmap_unuse_pt: ptepde != 0"));
4856 mpte = PHYS_TO_VM_PAGE(ptepde & PG_FRAME);
4857 return (pmap_unwire_ptp(pmap, va, mpte, free));
4858 }
4859
4860 void
mmu_radix_release(pmap_t pmap)4861 mmu_radix_release(pmap_t pmap)
4862 {
4863
4864 CTR2(KTR_PMAP, "%s(%p)", __func__, pmap);
4865 KASSERT(pmap->pm_stats.resident_count == 0,
4866 ("pmap_release: pmap resident count %ld != 0",
4867 pmap->pm_stats.resident_count));
4868 KASSERT(vm_radix_is_empty(&pmap->pm_radix),
4869 ("pmap_release: pmap has reserved page table page(s)"));
4870
4871 pmap_invalidate_all(pmap);
4872 isa3_proctab[pmap->pm_pid].proctab0 = 0;
4873 uma_zfree(zone_radix_pgd, pmap->pm_pml1);
4874 vmem_free(asid_arena, pmap->pm_pid, 1);
4875 }
4876
4877 /*
4878 * Create the PV entry for a 2MB page mapping. Always returns true unless the
4879 * flag PMAP_ENTER_NORECLAIM is specified. If that flag is specified, returns
4880 * false if the PV entry cannot be allocated without resorting to reclamation.
4881 */
4882 static bool
pmap_pv_insert_l3e(pmap_t pmap,vm_offset_t va,pml3_entry_t pde,u_int flags,struct rwlock ** lockp)4883 pmap_pv_insert_l3e(pmap_t pmap, vm_offset_t va, pml3_entry_t pde, u_int flags,
4884 struct rwlock **lockp)
4885 {
4886 struct md_page *pvh;
4887 pv_entry_t pv;
4888 vm_paddr_t pa;
4889
4890 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
4891 /* Pass NULL instead of the lock pointer to disable reclamation. */
4892 if ((pv = get_pv_entry(pmap, (flags & PMAP_ENTER_NORECLAIM) != 0 ?
4893 NULL : lockp)) == NULL)
4894 return (false);
4895 pv->pv_va = va;
4896 pa = pde & PG_PS_FRAME;
4897 CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa);
4898 pvh = pa_to_pvh(pa);
4899 TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_link);
4900 pvh->pv_gen++;
4901 return (true);
4902 }
4903
4904 /*
4905 * Fills a page table page with mappings to consecutive physical pages.
4906 */
4907 static void
pmap_fill_ptp(pt_entry_t * firstpte,pt_entry_t newpte)4908 pmap_fill_ptp(pt_entry_t *firstpte, pt_entry_t newpte)
4909 {
4910 pt_entry_t *pte;
4911
4912 for (pte = firstpte; pte < firstpte + NPTEPG; pte++) {
4913 *pte = htobe64(newpte);
4914 newpte += PAGE_SIZE;
4915 }
4916 }
4917
4918 static bool
pmap_demote_l3e(pmap_t pmap,pml3_entry_t * pde,vm_offset_t va)4919 pmap_demote_l3e(pmap_t pmap, pml3_entry_t *pde, vm_offset_t va)
4920 {
4921 struct rwlock *lock;
4922 bool rv;
4923
4924 lock = NULL;
4925 rv = pmap_demote_l3e_locked(pmap, pde, va, &lock);
4926 if (lock != NULL)
4927 rw_wunlock(lock);
4928 return (rv);
4929 }
4930
4931 static bool
pmap_demote_l3e_locked(pmap_t pmap,pml3_entry_t * l3e,vm_offset_t va,struct rwlock ** lockp)4932 pmap_demote_l3e_locked(pmap_t pmap, pml3_entry_t *l3e, vm_offset_t va,
4933 struct rwlock **lockp)
4934 {
4935 pml3_entry_t oldpde;
4936 pt_entry_t *firstpte;
4937 vm_paddr_t mptepa;
4938 vm_page_t mpte;
4939 struct spglist free;
4940 vm_offset_t sva;
4941
4942 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
4943 oldpde = be64toh(*l3e);
4944 KASSERT((oldpde & (RPTE_LEAF | PG_V)) == (RPTE_LEAF | PG_V),
4945 ("pmap_demote_l3e: oldpde is missing RPTE_LEAF and/or PG_V %lx",
4946 oldpde));
4947 if ((oldpde & PG_A) == 0 || (mpte = pmap_remove_pt_page(pmap, va)) ==
4948 NULL) {
4949 KASSERT((oldpde & PG_W) == 0,
4950 ("pmap_demote_l3e: page table page for a wired mapping"
4951 " is missing"));
4952
4953 /*
4954 * Invalidate the 2MB page mapping and return "failure" if the
4955 * mapping was never accessed or the allocation of the new
4956 * page table page fails. If the 2MB page mapping belongs to
4957 * the direct map region of the kernel's address space, then
4958 * the page allocation request specifies the highest possible
4959 * priority (VM_ALLOC_INTERRUPT). Otherwise, the priority is
4960 * normal. Page table pages are preallocated for every other
4961 * part of the kernel address space, so the direct map region
4962 * is the only part of the kernel address space that must be
4963 * handled here.
4964 */
4965 if ((oldpde & PG_A) == 0 || (mpte = vm_page_alloc_noobj(
4966 (va >= DMAP_MIN_ADDRESS && va < DMAP_MAX_ADDRESS ?
4967 VM_ALLOC_INTERRUPT : 0) | VM_ALLOC_WIRED)) == NULL) {
4968 SLIST_INIT(&free);
4969 sva = trunc_2mpage(va);
4970 pmap_remove_l3e(pmap, l3e, sva, &free, lockp);
4971 pmap_invalidate_l3e_page(pmap, sva, oldpde);
4972 vm_page_free_pages_toq(&free, true);
4973 CTR2(KTR_PMAP, "pmap_demote_l3e: failure for va %#lx"
4974 " in pmap %p", va, pmap);
4975 return (false);
4976 }
4977 mpte->pindex = pmap_l3e_pindex(va);
4978 if (va < VM_MAXUSER_ADDRESS)
4979 pmap_resident_count_inc(pmap, 1);
4980 }
4981 mptepa = VM_PAGE_TO_PHYS(mpte);
4982 firstpte = (pt_entry_t *)PHYS_TO_DMAP(mptepa);
4983 KASSERT((oldpde & PG_A) != 0,
4984 ("pmap_demote_l3e: oldpde is missing PG_A"));
4985 KASSERT((oldpde & (PG_M | PG_RW)) != PG_RW,
4986 ("pmap_demote_l3e: oldpde is missing PG_M"));
4987
4988 /*
4989 * If the page table page is new, initialize it.
4990 */
4991 if (mpte->ref_count == 1) {
4992 mpte->ref_count = NPTEPG;
4993 pmap_fill_ptp(firstpte, oldpde);
4994 }
4995
4996 KASSERT((be64toh(*firstpte) & PG_FRAME) == (oldpde & PG_FRAME),
4997 ("pmap_demote_l3e: firstpte and newpte map different physical"
4998 " addresses"));
4999
5000 /*
5001 * If the mapping has changed attributes, update the page table
5002 * entries.
5003 */
5004 if ((be64toh(*firstpte) & PG_PTE_PROMOTE) != (oldpde & PG_PTE_PROMOTE))
5005 pmap_fill_ptp(firstpte, oldpde);
5006
5007 /*
5008 * The spare PV entries must be reserved prior to demoting the
5009 * mapping, that is, prior to changing the PDE. Otherwise, the state
5010 * of the PDE and the PV lists will be inconsistent, which can result
5011 * in reclaim_pv_chunk() attempting to remove a PV entry from the
5012 * wrong PV list and pmap_pv_demote_l3e() failing to find the expected
5013 * PV entry for the 2MB page mapping that is being demoted.
5014 */
5015 if ((oldpde & PG_MANAGED) != 0)
5016 reserve_pv_entries(pmap, NPTEPG - 1, lockp);
5017
5018 /*
5019 * Demote the mapping. This pmap is locked. The old PDE has
5020 * PG_A set. If the old PDE has PG_RW set, it also has PG_M
5021 * set. Thus, there is no danger of a race with another
5022 * processor changing the setting of PG_A and/or PG_M between
5023 * the read above and the store below.
5024 */
5025 pde_store(l3e, mptepa);
5026 pmap_invalidate_l3e_page(pmap, trunc_2mpage(va), oldpde);
5027 /*
5028 * Demote the PV entry.
5029 */
5030 if ((oldpde & PG_MANAGED) != 0)
5031 pmap_pv_demote_l3e(pmap, va, oldpde & PG_PS_FRAME, lockp);
5032
5033 counter_u64_add(pmap_l3e_demotions, 1);
5034 CTR2(KTR_PMAP, "pmap_demote_l3e: success for va %#lx"
5035 " in pmap %p", va, pmap);
5036 return (true);
5037 }
5038
5039 /*
5040 * pmap_remove_kernel_pde: Remove a kernel superpage mapping.
5041 */
5042 static void
pmap_remove_kernel_l3e(pmap_t pmap,pml3_entry_t * l3e,vm_offset_t va)5043 pmap_remove_kernel_l3e(pmap_t pmap, pml3_entry_t *l3e, vm_offset_t va)
5044 {
5045 vm_paddr_t mptepa;
5046 vm_page_t mpte;
5047
5048 KASSERT(pmap == kernel_pmap, ("pmap %p is not kernel_pmap", pmap));
5049 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
5050 mpte = pmap_remove_pt_page(pmap, va);
5051 if (mpte == NULL)
5052 panic("pmap_remove_kernel_pde: Missing pt page.");
5053
5054 mptepa = VM_PAGE_TO_PHYS(mpte);
5055
5056 /*
5057 * Initialize the page table page.
5058 */
5059 pagezero(PHYS_TO_DMAP(mptepa));
5060
5061 /*
5062 * Demote the mapping.
5063 */
5064 pde_store(l3e, mptepa);
5065 ptesync();
5066 }
5067
5068 /*
5069 * pmap_remove_l3e: do the things to unmap a superpage in a process
5070 */
5071 static int
pmap_remove_l3e(pmap_t pmap,pml3_entry_t * pdq,vm_offset_t sva,struct spglist * free,struct rwlock ** lockp)5072 pmap_remove_l3e(pmap_t pmap, pml3_entry_t *pdq, vm_offset_t sva,
5073 struct spglist *free, struct rwlock **lockp)
5074 {
5075 struct md_page *pvh;
5076 pml3_entry_t oldpde;
5077 vm_offset_t eva, va;
5078 vm_page_t m, mpte;
5079
5080 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
5081 KASSERT((sva & L3_PAGE_MASK) == 0,
5082 ("pmap_remove_l3e: sva is not 2mpage aligned"));
5083 oldpde = be64toh(pte_load_clear(pdq));
5084 if (oldpde & PG_W)
5085 pmap->pm_stats.wired_count -= (L3_PAGE_SIZE / PAGE_SIZE);
5086 pmap_resident_count_dec(pmap, L3_PAGE_SIZE / PAGE_SIZE);
5087 if (oldpde & PG_MANAGED) {
5088 CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, oldpde & PG_PS_FRAME);
5089 pvh = pa_to_pvh(oldpde & PG_PS_FRAME);
5090 pmap_pvh_free(pvh, pmap, sva);
5091 eva = sva + L3_PAGE_SIZE;
5092 for (va = sva, m = PHYS_TO_VM_PAGE(oldpde & PG_PS_FRAME);
5093 va < eva; va += PAGE_SIZE, m++) {
5094 if ((oldpde & (PG_M | PG_RW)) == (PG_M | PG_RW))
5095 vm_page_dirty(m);
5096 if (oldpde & PG_A)
5097 vm_page_aflag_set(m, PGA_REFERENCED);
5098 if (TAILQ_EMPTY(&m->md.pv_list) &&
5099 TAILQ_EMPTY(&pvh->pv_list))
5100 vm_page_aflag_clear(m, PGA_WRITEABLE);
5101 }
5102 }
5103 if (pmap == kernel_pmap) {
5104 pmap_remove_kernel_l3e(pmap, pdq, sva);
5105 } else {
5106 mpte = pmap_remove_pt_page(pmap, sva);
5107 if (mpte != NULL) {
5108 pmap_resident_count_dec(pmap, 1);
5109 KASSERT(mpte->ref_count == NPTEPG,
5110 ("pmap_remove_l3e: pte page wire count error"));
5111 mpte->ref_count = 0;
5112 pmap_add_delayed_free_list(mpte, free, false);
5113 }
5114 }
5115 return (pmap_unuse_pt(pmap, sva, be64toh(*pmap_pml2e(pmap, sva)), free));
5116 }
5117
5118 /*
5119 * pmap_remove_pte: do the things to unmap a page in a process
5120 */
5121 static int
pmap_remove_pte(pmap_t pmap,pt_entry_t * ptq,vm_offset_t va,pml3_entry_t ptepde,struct spglist * free,struct rwlock ** lockp)5122 pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t va,
5123 pml3_entry_t ptepde, struct spglist *free, struct rwlock **lockp)
5124 {
5125 struct md_page *pvh;
5126 pt_entry_t oldpte;
5127 vm_page_t m;
5128
5129 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
5130 oldpte = be64toh(pte_load_clear(ptq));
5131 if (oldpte & RPTE_WIRED)
5132 pmap->pm_stats.wired_count -= 1;
5133 pmap_resident_count_dec(pmap, 1);
5134 if (oldpte & RPTE_MANAGED) {
5135 m = PHYS_TO_VM_PAGE(oldpte & PG_FRAME);
5136 if ((oldpte & (PG_M | PG_RW)) == (PG_M | PG_RW))
5137 vm_page_dirty(m);
5138 if (oldpte & PG_A)
5139 vm_page_aflag_set(m, PGA_REFERENCED);
5140 CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m);
5141 pmap_pvh_free(&m->md, pmap, va);
5142 if (TAILQ_EMPTY(&m->md.pv_list) &&
5143 (m->flags & PG_FICTITIOUS) == 0) {
5144 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
5145 if (TAILQ_EMPTY(&pvh->pv_list))
5146 vm_page_aflag_clear(m, PGA_WRITEABLE);
5147 }
5148 }
5149 return (pmap_unuse_pt(pmap, va, ptepde, free));
5150 }
5151
5152 /*
5153 * Remove a single page from a process address space
5154 */
5155 static bool
pmap_remove_page(pmap_t pmap,vm_offset_t va,pml3_entry_t * l3e,struct spglist * free)5156 pmap_remove_page(pmap_t pmap, vm_offset_t va, pml3_entry_t *l3e,
5157 struct spglist *free)
5158 {
5159 struct rwlock *lock;
5160 pt_entry_t *pte;
5161 bool invalidate_all;
5162
5163 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
5164 if ((be64toh(*l3e) & RPTE_VALID) == 0) {
5165 return (false);
5166 }
5167 pte = pmap_l3e_to_pte(l3e, va);
5168 if ((be64toh(*pte) & RPTE_VALID) == 0) {
5169 return (false);
5170 }
5171 lock = NULL;
5172
5173 invalidate_all = pmap_remove_pte(pmap, pte, va, be64toh(*l3e), free, &lock);
5174 if (lock != NULL)
5175 rw_wunlock(lock);
5176 if (!invalidate_all)
5177 pmap_invalidate_page(pmap, va);
5178 return (invalidate_all);
5179 }
5180
5181 /*
5182 * Removes the specified range of addresses from the page table page.
5183 */
5184 static bool
pmap_remove_ptes(pmap_t pmap,vm_offset_t sva,vm_offset_t eva,pml3_entry_t * l3e,struct spglist * free,struct rwlock ** lockp)5185 pmap_remove_ptes(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,
5186 pml3_entry_t *l3e, struct spglist *free, struct rwlock **lockp)
5187 {
5188 pt_entry_t *pte;
5189 vm_offset_t va;
5190 bool anyvalid;
5191
5192 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
5193 anyvalid = false;
5194 va = eva;
5195 for (pte = pmap_l3e_to_pte(l3e, sva); sva != eva; pte++,
5196 sva += PAGE_SIZE) {
5197 MPASS(pte == pmap_pte(pmap, sva));
5198 if (*pte == 0) {
5199 if (va != eva) {
5200 anyvalid = true;
5201 va = eva;
5202 }
5203 continue;
5204 }
5205 if (va == eva)
5206 va = sva;
5207 if (pmap_remove_pte(pmap, pte, sva, be64toh(*l3e), free, lockp)) {
5208 anyvalid = true;
5209 sva += PAGE_SIZE;
5210 break;
5211 }
5212 }
5213 if (anyvalid)
5214 pmap_invalidate_all(pmap);
5215 else if (va != eva)
5216 pmap_invalidate_range(pmap, va, sva);
5217 return (anyvalid);
5218 }
5219
5220 void
mmu_radix_remove(pmap_t pmap,vm_offset_t sva,vm_offset_t eva)5221 mmu_radix_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
5222 {
5223 struct rwlock *lock;
5224 vm_offset_t va_next;
5225 pml1_entry_t *l1e;
5226 pml2_entry_t *l2e;
5227 pml3_entry_t ptpaddr, *l3e;
5228 struct spglist free;
5229 bool anyvalid;
5230
5231 CTR4(KTR_PMAP, "%s(%p, %#x, %#x)", __func__, pmap, sva, eva);
5232
5233 /*
5234 * Perform an unsynchronized read. This is, however, safe.
5235 */
5236 if (pmap->pm_stats.resident_count == 0)
5237 return;
5238
5239 anyvalid = false;
5240 SLIST_INIT(&free);
5241
5242 /* XXX something fishy here */
5243 sva = (sva + PAGE_MASK) & ~PAGE_MASK;
5244 eva = (eva + PAGE_MASK) & ~PAGE_MASK;
5245
5246 PMAP_LOCK(pmap);
5247
5248 /*
5249 * special handling of removing one page. a very
5250 * common operation and easy to short circuit some
5251 * code.
5252 */
5253 if (sva + PAGE_SIZE == eva) {
5254 l3e = pmap_pml3e(pmap, sva);
5255 if (l3e && (be64toh(*l3e) & RPTE_LEAF) == 0) {
5256 anyvalid = pmap_remove_page(pmap, sva, l3e, &free);
5257 goto out;
5258 }
5259 }
5260
5261 lock = NULL;
5262 for (; sva < eva; sva = va_next) {
5263 if (pmap->pm_stats.resident_count == 0)
5264 break;
5265 l1e = pmap_pml1e(pmap, sva);
5266 if (l1e == NULL || (be64toh(*l1e) & PG_V) == 0) {
5267 va_next = (sva + L1_PAGE_SIZE) & ~L1_PAGE_MASK;
5268 if (va_next < sva)
5269 va_next = eva;
5270 continue;
5271 }
5272
5273 l2e = pmap_l1e_to_l2e(l1e, sva);
5274 if (l2e == NULL || (be64toh(*l2e) & PG_V) == 0) {
5275 va_next = (sva + L2_PAGE_SIZE) & ~L2_PAGE_MASK;
5276 if (va_next < sva)
5277 va_next = eva;
5278 continue;
5279 }
5280
5281 /*
5282 * Calculate index for next page table.
5283 */
5284 va_next = (sva + L3_PAGE_SIZE) & ~L3_PAGE_MASK;
5285 if (va_next < sva)
5286 va_next = eva;
5287
5288 l3e = pmap_l2e_to_l3e(l2e, sva);
5289 ptpaddr = be64toh(*l3e);
5290
5291 /*
5292 * Weed out invalid mappings.
5293 */
5294 if (ptpaddr == 0)
5295 continue;
5296
5297 /*
5298 * Check for large page.
5299 */
5300 if ((ptpaddr & RPTE_LEAF) != 0) {
5301 /*
5302 * Are we removing the entire large page? If not,
5303 * demote the mapping and fall through.
5304 */
5305 if (sva + L3_PAGE_SIZE == va_next && eva >= va_next) {
5306 pmap_remove_l3e(pmap, l3e, sva, &free, &lock);
5307 anyvalid = true;
5308 continue;
5309 } else if (!pmap_demote_l3e_locked(pmap, l3e, sva,
5310 &lock)) {
5311 /* The large page mapping was destroyed. */
5312 continue;
5313 } else
5314 ptpaddr = be64toh(*l3e);
5315 }
5316
5317 /*
5318 * Limit our scan to either the end of the va represented
5319 * by the current page table page, or to the end of the
5320 * range being removed.
5321 */
5322 if (va_next > eva)
5323 va_next = eva;
5324
5325 if (pmap_remove_ptes(pmap, sva, va_next, l3e, &free, &lock))
5326 anyvalid = true;
5327 }
5328 if (lock != NULL)
5329 rw_wunlock(lock);
5330 out:
5331 if (anyvalid)
5332 pmap_invalidate_all(pmap);
5333 PMAP_UNLOCK(pmap);
5334 vm_page_free_pages_toq(&free, true);
5335 }
5336
5337 void
mmu_radix_remove_all(vm_page_t m)5338 mmu_radix_remove_all(vm_page_t m)
5339 {
5340 struct md_page *pvh;
5341 pv_entry_t pv;
5342 pmap_t pmap;
5343 struct rwlock *lock;
5344 pt_entry_t *pte, tpte;
5345 pml3_entry_t *l3e;
5346 vm_offset_t va;
5347 struct spglist free;
5348 int pvh_gen, md_gen;
5349
5350 CTR2(KTR_PMAP, "%s(%p)", __func__, m);
5351 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
5352 ("pmap_remove_all: page %p is not managed", m));
5353 SLIST_INIT(&free);
5354 lock = VM_PAGE_TO_PV_LIST_LOCK(m);
5355 pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy :
5356 pa_to_pvh(VM_PAGE_TO_PHYS(m));
5357 retry:
5358 rw_wlock(lock);
5359 while ((pv = TAILQ_FIRST(&pvh->pv_list)) != NULL) {
5360 pmap = PV_PMAP(pv);
5361 if (!PMAP_TRYLOCK(pmap)) {
5362 pvh_gen = pvh->pv_gen;
5363 rw_wunlock(lock);
5364 PMAP_LOCK(pmap);
5365 rw_wlock(lock);
5366 if (pvh_gen != pvh->pv_gen) {
5367 rw_wunlock(lock);
5368 PMAP_UNLOCK(pmap);
5369 goto retry;
5370 }
5371 }
5372 va = pv->pv_va;
5373 l3e = pmap_pml3e(pmap, va);
5374 (void)pmap_demote_l3e_locked(pmap, l3e, va, &lock);
5375 PMAP_UNLOCK(pmap);
5376 }
5377 while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) {
5378 pmap = PV_PMAP(pv);
5379 if (!PMAP_TRYLOCK(pmap)) {
5380 pvh_gen = pvh->pv_gen;
5381 md_gen = m->md.pv_gen;
5382 rw_wunlock(lock);
5383 PMAP_LOCK(pmap);
5384 rw_wlock(lock);
5385 if (pvh_gen != pvh->pv_gen || md_gen != m->md.pv_gen) {
5386 rw_wunlock(lock);
5387 PMAP_UNLOCK(pmap);
5388 goto retry;
5389 }
5390 }
5391 pmap_resident_count_dec(pmap, 1);
5392 l3e = pmap_pml3e(pmap, pv->pv_va);
5393 KASSERT((be64toh(*l3e) & RPTE_LEAF) == 0, ("pmap_remove_all: found"
5394 " a 2mpage in page %p's pv list", m));
5395 pte = pmap_l3e_to_pte(l3e, pv->pv_va);
5396 tpte = be64toh(pte_load_clear(pte));
5397 if (tpte & PG_W)
5398 pmap->pm_stats.wired_count--;
5399 if (tpte & PG_A)
5400 vm_page_aflag_set(m, PGA_REFERENCED);
5401
5402 /*
5403 * Update the vm_page_t clean and reference bits.
5404 */
5405 if ((tpte & (PG_M | PG_RW)) == (PG_M | PG_RW))
5406 vm_page_dirty(m);
5407 pmap_unuse_pt(pmap, pv->pv_va, be64toh(*l3e), &free);
5408 pmap_invalidate_page(pmap, pv->pv_va);
5409 TAILQ_REMOVE(&m->md.pv_list, pv, pv_link);
5410 m->md.pv_gen++;
5411 free_pv_entry(pmap, pv);
5412 PMAP_UNLOCK(pmap);
5413 }
5414 vm_page_aflag_clear(m, PGA_WRITEABLE);
5415 rw_wunlock(lock);
5416 vm_page_free_pages_toq(&free, true);
5417 }
5418
5419 /*
5420 * Destroy all managed, non-wired mappings in the given user-space
5421 * pmap. This pmap cannot be active on any processor besides the
5422 * caller.
5423 *
5424 * This function cannot be applied to the kernel pmap. Moreover, it
5425 * is not intended for general use. It is only to be used during
5426 * process termination. Consequently, it can be implemented in ways
5427 * that make it faster than pmap_remove(). First, it can more quickly
5428 * destroy mappings by iterating over the pmap's collection of PV
5429 * entries, rather than searching the page table. Second, it doesn't
5430 * have to test and clear the page table entries atomically, because
5431 * no processor is currently accessing the user address space. In
5432 * particular, a page table entry's dirty bit won't change state once
5433 * this function starts.
5434 *
5435 * Although this function destroys all of the pmap's managed,
5436 * non-wired mappings, it can delay and batch the invalidation of TLB
5437 * entries without calling pmap_delayed_invl_started() and
5438 * pmap_delayed_invl_finished(). Because the pmap is not active on
5439 * any other processor, none of these TLB entries will ever be used
5440 * before their eventual invalidation. Consequently, there is no need
5441 * for either pmap_remove_all() or pmap_remove_write() to wait for
5442 * that eventual TLB invalidation.
5443 */
5444
5445 void
mmu_radix_remove_pages(pmap_t pmap)5446 mmu_radix_remove_pages(pmap_t pmap)
5447 {
5448
5449 CTR2(KTR_PMAP, "%s(%p)", __func__, pmap);
5450 pml3_entry_t ptel3e;
5451 pt_entry_t *pte, tpte;
5452 struct spglist free;
5453 vm_page_t m, mpte, mt;
5454 pv_entry_t pv;
5455 struct md_page *pvh;
5456 struct pv_chunk *pc, *npc;
5457 struct rwlock *lock;
5458 int64_t bit;
5459 uint64_t inuse, bitmask;
5460 int allfree, field, idx;
5461 #ifdef PV_STATS
5462 int freed;
5463 #endif
5464 bool superpage;
5465 vm_paddr_t pa;
5466
5467 /*
5468 * Assert that the given pmap is only active on the current
5469 * CPU. Unfortunately, we cannot block another CPU from
5470 * activating the pmap while this function is executing.
5471 */
5472 KASSERT(pmap->pm_pid == mfspr(SPR_PID),
5473 ("non-current asid %lu - expected %lu", pmap->pm_pid,
5474 mfspr(SPR_PID)));
5475
5476 lock = NULL;
5477
5478 SLIST_INIT(&free);
5479 PMAP_LOCK(pmap);
5480 TAILQ_FOREACH_SAFE(pc, &pmap->pm_pvchunk, pc_list, npc) {
5481 allfree = 1;
5482 #ifdef PV_STATS
5483 freed = 0;
5484 #endif
5485 for (field = 0; field < _NPCM; field++) {
5486 inuse = ~pc->pc_map[field] & pc_freemask[field];
5487 while (inuse != 0) {
5488 bit = cnttzd(inuse);
5489 bitmask = 1UL << bit;
5490 idx = field * 64 + bit;
5491 pv = &pc->pc_pventry[idx];
5492 inuse &= ~bitmask;
5493
5494 pte = pmap_pml2e(pmap, pv->pv_va);
5495 ptel3e = be64toh(*pte);
5496 pte = pmap_l2e_to_l3e(pte, pv->pv_va);
5497 tpte = be64toh(*pte);
5498 if ((tpte & (RPTE_LEAF | PG_V)) == PG_V) {
5499 superpage = false;
5500 ptel3e = tpte;
5501 pte = (pt_entry_t *)PHYS_TO_DMAP(tpte &
5502 PG_FRAME);
5503 pte = &pte[pmap_pte_index(pv->pv_va)];
5504 tpte = be64toh(*pte);
5505 } else {
5506 /*
5507 * Keep track whether 'tpte' is a
5508 * superpage explicitly instead of
5509 * relying on RPTE_LEAF being set.
5510 *
5511 * This is because RPTE_LEAF is numerically
5512 * identical to PG_PTE_PAT and thus a
5513 * regular page could be mistaken for
5514 * a superpage.
5515 */
5516 superpage = true;
5517 }
5518
5519 if ((tpte & PG_V) == 0) {
5520 panic("bad pte va %lx pte %lx",
5521 pv->pv_va, tpte);
5522 }
5523
5524 /*
5525 * We cannot remove wired pages from a process' mapping at this time
5526 */
5527 if (tpte & PG_W) {
5528 allfree = 0;
5529 continue;
5530 }
5531
5532 if (superpage)
5533 pa = tpte & PG_PS_FRAME;
5534 else
5535 pa = tpte & PG_FRAME;
5536
5537 m = PHYS_TO_VM_PAGE(pa);
5538 KASSERT(m->phys_addr == pa,
5539 ("vm_page_t %p phys_addr mismatch %016jx %016jx",
5540 m, (uintmax_t)m->phys_addr,
5541 (uintmax_t)tpte));
5542
5543 KASSERT((m->flags & PG_FICTITIOUS) != 0 ||
5544 m < &vm_page_array[vm_page_array_size],
5545 ("pmap_remove_pages: bad tpte %#jx",
5546 (uintmax_t)tpte));
5547
5548 pte_clear(pte);
5549
5550 /*
5551 * Update the vm_page_t clean/reference bits.
5552 */
5553 if ((tpte & (PG_M | PG_RW)) == (PG_M | PG_RW)) {
5554 if (superpage) {
5555 for (mt = m; mt < &m[L3_PAGE_SIZE / PAGE_SIZE]; mt++)
5556 vm_page_dirty(mt);
5557 } else
5558 vm_page_dirty(m);
5559 }
5560
5561 CHANGE_PV_LIST_LOCK_TO_VM_PAGE(&lock, m);
5562
5563 /* Mark free */
5564 pc->pc_map[field] |= bitmask;
5565 if (superpage) {
5566 pmap_resident_count_dec(pmap, L3_PAGE_SIZE / PAGE_SIZE);
5567 pvh = pa_to_pvh(tpte & PG_PS_FRAME);
5568 TAILQ_REMOVE(&pvh->pv_list, pv, pv_link);
5569 pvh->pv_gen++;
5570 if (TAILQ_EMPTY(&pvh->pv_list)) {
5571 for (mt = m; mt < &m[L3_PAGE_SIZE / PAGE_SIZE]; mt++)
5572 if ((mt->a.flags & PGA_WRITEABLE) != 0 &&
5573 TAILQ_EMPTY(&mt->md.pv_list))
5574 vm_page_aflag_clear(mt, PGA_WRITEABLE);
5575 }
5576 mpte = pmap_remove_pt_page(pmap, pv->pv_va);
5577 if (mpte != NULL) {
5578 pmap_resident_count_dec(pmap, 1);
5579 KASSERT(mpte->ref_count == NPTEPG,
5580 ("pmap_remove_pages: pte page wire count error"));
5581 mpte->ref_count = 0;
5582 pmap_add_delayed_free_list(mpte, &free, false);
5583 }
5584 } else {
5585 pmap_resident_count_dec(pmap, 1);
5586 #ifdef VERBOSE_PV
5587 printf("freeing pv (%p, %p)\n",
5588 pmap, pv);
5589 #endif
5590 TAILQ_REMOVE(&m->md.pv_list, pv, pv_link);
5591 m->md.pv_gen++;
5592 if ((m->a.flags & PGA_WRITEABLE) != 0 &&
5593 TAILQ_EMPTY(&m->md.pv_list) &&
5594 (m->flags & PG_FICTITIOUS) == 0) {
5595 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
5596 if (TAILQ_EMPTY(&pvh->pv_list))
5597 vm_page_aflag_clear(m, PGA_WRITEABLE);
5598 }
5599 }
5600 pmap_unuse_pt(pmap, pv->pv_va, ptel3e, &free);
5601 #ifdef PV_STATS
5602 freed++;
5603 #endif
5604 }
5605 }
5606 PV_STAT(atomic_add_long(&pv_entry_frees, freed));
5607 PV_STAT(atomic_add_int(&pv_entry_spare, freed));
5608 PV_STAT(atomic_subtract_long(&pv_entry_count, freed));
5609 if (allfree) {
5610 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
5611 free_pv_chunk(pc);
5612 }
5613 }
5614 if (lock != NULL)
5615 rw_wunlock(lock);
5616 pmap_invalidate_all(pmap);
5617 PMAP_UNLOCK(pmap);
5618 vm_page_free_pages_toq(&free, true);
5619 }
5620
5621 void
mmu_radix_remove_write(vm_page_t m)5622 mmu_radix_remove_write(vm_page_t m)
5623 {
5624 struct md_page *pvh;
5625 pmap_t pmap;
5626 struct rwlock *lock;
5627 pv_entry_t next_pv, pv;
5628 pml3_entry_t *l3e;
5629 pt_entry_t oldpte, *pte;
5630 int pvh_gen, md_gen;
5631
5632 CTR2(KTR_PMAP, "%s(%p)", __func__, m);
5633 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
5634 ("pmap_remove_write: page %p is not managed", m));
5635 vm_page_assert_busied(m);
5636
5637 if (!pmap_page_is_write_mapped(m))
5638 return;
5639 lock = VM_PAGE_TO_PV_LIST_LOCK(m);
5640 pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy :
5641 pa_to_pvh(VM_PAGE_TO_PHYS(m));
5642 retry_pv_loop:
5643 rw_wlock(lock);
5644 TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_link, next_pv) {
5645 pmap = PV_PMAP(pv);
5646 if (!PMAP_TRYLOCK(pmap)) {
5647 pvh_gen = pvh->pv_gen;
5648 rw_wunlock(lock);
5649 PMAP_LOCK(pmap);
5650 rw_wlock(lock);
5651 if (pvh_gen != pvh->pv_gen) {
5652 PMAP_UNLOCK(pmap);
5653 rw_wunlock(lock);
5654 goto retry_pv_loop;
5655 }
5656 }
5657 l3e = pmap_pml3e(pmap, pv->pv_va);
5658 if ((be64toh(*l3e) & PG_RW) != 0)
5659 (void)pmap_demote_l3e_locked(pmap, l3e, pv->pv_va, &lock);
5660 KASSERT(lock == VM_PAGE_TO_PV_LIST_LOCK(m),
5661 ("inconsistent pv lock %p %p for page %p",
5662 lock, VM_PAGE_TO_PV_LIST_LOCK(m), m));
5663 PMAP_UNLOCK(pmap);
5664 }
5665 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) {
5666 pmap = PV_PMAP(pv);
5667 if (!PMAP_TRYLOCK(pmap)) {
5668 pvh_gen = pvh->pv_gen;
5669 md_gen = m->md.pv_gen;
5670 rw_wunlock(lock);
5671 PMAP_LOCK(pmap);
5672 rw_wlock(lock);
5673 if (pvh_gen != pvh->pv_gen ||
5674 md_gen != m->md.pv_gen) {
5675 PMAP_UNLOCK(pmap);
5676 rw_wunlock(lock);
5677 goto retry_pv_loop;
5678 }
5679 }
5680 l3e = pmap_pml3e(pmap, pv->pv_va);
5681 KASSERT((be64toh(*l3e) & RPTE_LEAF) == 0,
5682 ("pmap_remove_write: found a 2mpage in page %p's pv list",
5683 m));
5684 pte = pmap_l3e_to_pte(l3e, pv->pv_va);
5685 retry:
5686 oldpte = be64toh(*pte);
5687 if (oldpte & PG_RW) {
5688 if (!atomic_cmpset_long(pte, htobe64(oldpte),
5689 htobe64((oldpte | RPTE_EAA_R) & ~(PG_RW | PG_M))))
5690 goto retry;
5691 if ((oldpte & PG_M) != 0)
5692 vm_page_dirty(m);
5693 pmap_invalidate_page(pmap, pv->pv_va);
5694 }
5695 PMAP_UNLOCK(pmap);
5696 }
5697 rw_wunlock(lock);
5698 vm_page_aflag_clear(m, PGA_WRITEABLE);
5699 }
5700
5701 /*
5702 * Clear the wired attribute from the mappings for the specified range of
5703 * addresses in the given pmap. Every valid mapping within that range
5704 * must have the wired attribute set. In contrast, invalid mappings
5705 * cannot have the wired attribute set, so they are ignored.
5706 *
5707 * The wired attribute of the page table entry is not a hardware
5708 * feature, so there is no need to invalidate any TLB entries.
5709 * Since pmap_demote_l3e() for the wired entry must never fail,
5710 * pmap_delayed_invl_started()/finished() calls around the
5711 * function are not needed.
5712 */
5713 void
mmu_radix_unwire(pmap_t pmap,vm_offset_t sva,vm_offset_t eva)5714 mmu_radix_unwire(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
5715 {
5716 vm_offset_t va_next;
5717 pml1_entry_t *l1e;
5718 pml2_entry_t *l2e;
5719 pml3_entry_t *l3e;
5720 pt_entry_t *pte;
5721
5722 CTR4(KTR_PMAP, "%s(%p, %#x, %#x)", __func__, pmap, sva, eva);
5723 PMAP_LOCK(pmap);
5724 for (; sva < eva; sva = va_next) {
5725 l1e = pmap_pml1e(pmap, sva);
5726 if ((be64toh(*l1e) & PG_V) == 0) {
5727 va_next = (sva + L1_PAGE_SIZE) & ~L1_PAGE_MASK;
5728 if (va_next < sva)
5729 va_next = eva;
5730 continue;
5731 }
5732 l2e = pmap_l1e_to_l2e(l1e, sva);
5733 if ((be64toh(*l2e) & PG_V) == 0) {
5734 va_next = (sva + L2_PAGE_SIZE) & ~L2_PAGE_MASK;
5735 if (va_next < sva)
5736 va_next = eva;
5737 continue;
5738 }
5739 va_next = (sva + L3_PAGE_SIZE) & ~L3_PAGE_MASK;
5740 if (va_next < sva)
5741 va_next = eva;
5742 l3e = pmap_l2e_to_l3e(l2e, sva);
5743 if ((be64toh(*l3e) & PG_V) == 0)
5744 continue;
5745 if ((be64toh(*l3e) & RPTE_LEAF) != 0) {
5746 if ((be64toh(*l3e) & PG_W) == 0)
5747 panic("pmap_unwire: pde %#jx is missing PG_W",
5748 (uintmax_t)(be64toh(*l3e)));
5749
5750 /*
5751 * Are we unwiring the entire large page? If not,
5752 * demote the mapping and fall through.
5753 */
5754 if (sva + L3_PAGE_SIZE == va_next && eva >= va_next) {
5755 atomic_clear_long(l3e, htobe64(PG_W));
5756 pmap->pm_stats.wired_count -= L3_PAGE_SIZE /
5757 PAGE_SIZE;
5758 continue;
5759 } else if (!pmap_demote_l3e(pmap, l3e, sva))
5760 panic("pmap_unwire: demotion failed");
5761 }
5762 if (va_next > eva)
5763 va_next = eva;
5764 for (pte = pmap_l3e_to_pte(l3e, sva); sva != va_next; pte++,
5765 sva += PAGE_SIZE) {
5766 MPASS(pte == pmap_pte(pmap, sva));
5767 if ((be64toh(*pte) & PG_V) == 0)
5768 continue;
5769 if ((be64toh(*pte) & PG_W) == 0)
5770 panic("pmap_unwire: pte %#jx is missing PG_W",
5771 (uintmax_t)(be64toh(*pte)));
5772
5773 /*
5774 * PG_W must be cleared atomically. Although the pmap
5775 * lock synchronizes access to PG_W, another processor
5776 * could be setting PG_M and/or PG_A concurrently.
5777 */
5778 atomic_clear_long(pte, htobe64(PG_W));
5779 pmap->pm_stats.wired_count--;
5780 }
5781 }
5782 PMAP_UNLOCK(pmap);
5783 }
5784
5785 void
mmu_radix_zero_page(vm_page_t m)5786 mmu_radix_zero_page(vm_page_t m)
5787 {
5788 vm_offset_t addr;
5789
5790 CTR2(KTR_PMAP, "%s(%p)", __func__, m);
5791 addr = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m));
5792 pagezero(addr);
5793 }
5794
5795 void
mmu_radix_zero_page_area(vm_page_t m,int off,int size)5796 mmu_radix_zero_page_area(vm_page_t m, int off, int size)
5797 {
5798 caddr_t addr;
5799
5800 CTR4(KTR_PMAP, "%s(%p, %d, %d)", __func__, m, off, size);
5801 MPASS(off + size <= PAGE_SIZE);
5802 addr = (caddr_t)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m));
5803 memset(addr + off, 0, size);
5804 }
5805
5806 static int
mmu_radix_mincore(pmap_t pmap,vm_offset_t addr,vm_paddr_t * locked_pa)5807 mmu_radix_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *locked_pa)
5808 {
5809 pml3_entry_t *l3ep;
5810 pt_entry_t pte;
5811 vm_paddr_t pa;
5812 int val;
5813
5814 CTR3(KTR_PMAP, "%s(%p, %#x)", __func__, pmap, addr);
5815 PMAP_LOCK(pmap);
5816
5817 l3ep = pmap_pml3e(pmap, addr);
5818 if (l3ep != NULL && (be64toh(*l3ep) & PG_V)) {
5819 if (be64toh(*l3ep) & RPTE_LEAF) {
5820 pte = be64toh(*l3ep);
5821 /* Compute the physical address of the 4KB page. */
5822 pa = ((be64toh(*l3ep) & PG_PS_FRAME) | (addr & L3_PAGE_MASK)) &
5823 PG_FRAME;
5824 val = MINCORE_PSIND(1);
5825 } else {
5826 /* Native endian PTE, do not pass to functions */
5827 pte = be64toh(*pmap_l3e_to_pte(l3ep, addr));
5828 pa = pte & PG_FRAME;
5829 val = 0;
5830 }
5831 } else {
5832 pte = 0;
5833 pa = 0;
5834 val = 0;
5835 }
5836 if ((pte & PG_V) != 0) {
5837 val |= MINCORE_INCORE;
5838 if ((pte & (PG_M | PG_RW)) == (PG_M | PG_RW))
5839 val |= MINCORE_MODIFIED | MINCORE_MODIFIED_OTHER;
5840 if ((pte & PG_A) != 0)
5841 val |= MINCORE_REFERENCED | MINCORE_REFERENCED_OTHER;
5842 }
5843 if ((val & (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER)) !=
5844 (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER) &&
5845 (pte & (PG_MANAGED | PG_V)) == (PG_MANAGED | PG_V)) {
5846 *locked_pa = pa;
5847 }
5848 PMAP_UNLOCK(pmap);
5849 return (val);
5850 }
5851
5852 void
mmu_radix_activate(struct thread * td)5853 mmu_radix_activate(struct thread *td)
5854 {
5855 pmap_t pmap;
5856 uint32_t curpid;
5857
5858 CTR2(KTR_PMAP, "%s(%p)", __func__, td);
5859 critical_enter();
5860 pmap = vmspace_pmap(td->td_proc->p_vmspace);
5861 curpid = mfspr(SPR_PID);
5862 if (pmap->pm_pid > isa3_base_pid &&
5863 curpid != pmap->pm_pid) {
5864 mmu_radix_pid_set(pmap);
5865 }
5866 critical_exit();
5867 }
5868
5869 /*
5870 * Increase the starting virtual address of the given mapping if a
5871 * different alignment might result in more superpage mappings.
5872 */
5873 void
mmu_radix_align_superpage(vm_object_t object,vm_ooffset_t offset,vm_offset_t * addr,vm_size_t size)5874 mmu_radix_align_superpage(vm_object_t object, vm_ooffset_t offset,
5875 vm_offset_t *addr, vm_size_t size)
5876 {
5877
5878 CTR5(KTR_PMAP, "%s(%p, %#x, %p, %#x)", __func__, object, offset, addr,
5879 size);
5880 vm_offset_t superpage_offset;
5881
5882 if (size < L3_PAGE_SIZE)
5883 return;
5884 if (object != NULL && (object->flags & OBJ_COLORED) != 0)
5885 offset += ptoa(object->pg_color);
5886 superpage_offset = offset & L3_PAGE_MASK;
5887 if (size - ((L3_PAGE_SIZE - superpage_offset) & L3_PAGE_MASK) < L3_PAGE_SIZE ||
5888 (*addr & L3_PAGE_MASK) == superpage_offset)
5889 return;
5890 if ((*addr & L3_PAGE_MASK) < superpage_offset)
5891 *addr = (*addr & ~L3_PAGE_MASK) + superpage_offset;
5892 else
5893 *addr = ((*addr + L3_PAGE_MASK) & ~L3_PAGE_MASK) + superpage_offset;
5894 }
5895
5896 static void *
mmu_radix_mapdev_attr(vm_paddr_t pa,vm_size_t size,vm_memattr_t attr)5897 mmu_radix_mapdev_attr(vm_paddr_t pa, vm_size_t size, vm_memattr_t attr)
5898 {
5899 vm_offset_t va, tmpva, ppa, offset;
5900
5901 ppa = trunc_page(pa);
5902 offset = pa & PAGE_MASK;
5903 size = roundup2(offset + size, PAGE_SIZE);
5904 if (pa < powerpc_ptob(Maxmem))
5905 panic("bad pa: %#lx less than Maxmem %#lx\n",
5906 pa, powerpc_ptob(Maxmem));
5907 va = kva_alloc(size);
5908 if (bootverbose)
5909 printf("%s(%#lx, %lu, %d)\n", __func__, pa, size, attr);
5910 KASSERT(size > 0, ("%s(%#lx, %lu, %d)", __func__, pa, size, attr));
5911
5912 if (!va)
5913 panic("%s: Couldn't alloc kernel virtual memory", __func__);
5914
5915 for (tmpva = va; size > 0;) {
5916 mmu_radix_kenter_attr(tmpva, ppa, attr);
5917 size -= PAGE_SIZE;
5918 tmpva += PAGE_SIZE;
5919 ppa += PAGE_SIZE;
5920 }
5921 ptesync();
5922
5923 return ((void *)(va + offset));
5924 }
5925
5926 static void *
mmu_radix_mapdev(vm_paddr_t pa,vm_size_t size)5927 mmu_radix_mapdev(vm_paddr_t pa, vm_size_t size)
5928 {
5929
5930 CTR3(KTR_PMAP, "%s(%#x, %#x)", __func__, pa, size);
5931
5932 return (mmu_radix_mapdev_attr(pa, size, VM_MEMATTR_DEFAULT));
5933 }
5934
5935 void
mmu_radix_page_set_memattr(vm_page_t m,vm_memattr_t ma)5936 mmu_radix_page_set_memattr(vm_page_t m, vm_memattr_t ma)
5937 {
5938
5939 CTR3(KTR_PMAP, "%s(%p, %#x)", __func__, m, ma);
5940 m->md.mdpg_cache_attrs = ma;
5941
5942 /*
5943 * If "m" is a normal page, update its direct mapping. This update
5944 * can be relied upon to perform any cache operations that are
5945 * required for data coherence.
5946 */
5947 if ((m->flags & PG_FICTITIOUS) == 0 &&
5948 mmu_radix_change_attr(PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)),
5949 PAGE_SIZE, m->md.mdpg_cache_attrs))
5950 panic("memory attribute change on the direct map failed");
5951 }
5952
5953 static void
mmu_radix_unmapdev(void * p,vm_size_t size)5954 mmu_radix_unmapdev(void *p, vm_size_t size)
5955 {
5956 vm_offset_t offset, va;
5957
5958 CTR3(KTR_PMAP, "%s(%p, %#x)", __func__, p, size);
5959
5960 /* If we gave a direct map region in pmap_mapdev, do nothing */
5961 va = (vm_offset_t)p;
5962 if (va >= DMAP_MIN_ADDRESS && va < DMAP_MAX_ADDRESS)
5963 return;
5964
5965 offset = va & PAGE_MASK;
5966 size = round_page(offset + size);
5967 va = trunc_page(va);
5968
5969 if (pmap_initialized) {
5970 mmu_radix_qremove(va, atop(size));
5971 kva_free(va, size);
5972 }
5973 }
5974
5975 void
mmu_radix_sync_icache(pmap_t pm,vm_offset_t va,vm_size_t sz)5976 mmu_radix_sync_icache(pmap_t pm, vm_offset_t va, vm_size_t sz)
5977 {
5978 vm_paddr_t pa = 0;
5979 int sync_sz;
5980
5981 if (__predict_false(pm == NULL))
5982 pm = &curthread->td_proc->p_vmspace->vm_pmap;
5983
5984 while (sz > 0) {
5985 pa = pmap_extract(pm, va);
5986 sync_sz = PAGE_SIZE - (va & PAGE_MASK);
5987 sync_sz = min(sync_sz, sz);
5988 if (pa != 0) {
5989 pa += (va & PAGE_MASK);
5990 __syncicache((void *)PHYS_TO_DMAP(pa), sync_sz);
5991 }
5992 va += sync_sz;
5993 sz -= sync_sz;
5994 }
5995 }
5996
5997 static __inline void
pmap_pte_attr(pt_entry_t * pte,uint64_t cache_bits,uint64_t mask)5998 pmap_pte_attr(pt_entry_t *pte, uint64_t cache_bits, uint64_t mask)
5999 {
6000 uint64_t opte, npte;
6001
6002 /*
6003 * The cache mode bits are all in the low 32-bits of the
6004 * PTE, so we can just spin on updating the low 32-bits.
6005 */
6006 do {
6007 opte = be64toh(*pte);
6008 npte = opte & ~mask;
6009 npte |= cache_bits;
6010 } while (npte != opte && !atomic_cmpset_long(pte, htobe64(opte), htobe64(npte)));
6011 }
6012
6013 /*
6014 * Tries to demote a 1GB page mapping.
6015 */
6016 static bool
pmap_demote_l2e(pmap_t pmap,pml2_entry_t * l2e,vm_offset_t va)6017 pmap_demote_l2e(pmap_t pmap, pml2_entry_t *l2e, vm_offset_t va)
6018 {
6019 pml2_entry_t oldpdpe;
6020 pml3_entry_t *firstpde, newpde, *pde;
6021 vm_paddr_t pdpgpa;
6022 vm_page_t pdpg;
6023
6024 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
6025 oldpdpe = be64toh(*l2e);
6026 KASSERT((oldpdpe & (RPTE_LEAF | PG_V)) == (RPTE_LEAF | PG_V),
6027 ("pmap_demote_pdpe: oldpdpe is missing PG_PS and/or PG_V"));
6028 pdpg = vm_page_alloc_noobj(VM_ALLOC_INTERRUPT | VM_ALLOC_WIRED);
6029 if (pdpg == NULL) {
6030 CTR2(KTR_PMAP, "pmap_demote_pdpe: failure for va %#lx"
6031 " in pmap %p", va, pmap);
6032 return (false);
6033 }
6034 pdpg->pindex = va >> L2_PAGE_SIZE_SHIFT;
6035 pdpgpa = VM_PAGE_TO_PHYS(pdpg);
6036 firstpde = (pml3_entry_t *)PHYS_TO_DMAP(pdpgpa);
6037 KASSERT((oldpdpe & PG_A) != 0,
6038 ("pmap_demote_pdpe: oldpdpe is missing PG_A"));
6039 KASSERT((oldpdpe & (PG_M | PG_RW)) != PG_RW,
6040 ("pmap_demote_pdpe: oldpdpe is missing PG_M"));
6041 newpde = oldpdpe;
6042
6043 /*
6044 * Initialize the page directory page.
6045 */
6046 for (pde = firstpde; pde < firstpde + NPDEPG; pde++) {
6047 *pde = htobe64(newpde);
6048 newpde += L3_PAGE_SIZE;
6049 }
6050
6051 /*
6052 * Demote the mapping.
6053 */
6054 pde_store(l2e, pdpgpa);
6055
6056 /*
6057 * Flush PWC --- XXX revisit
6058 */
6059 pmap_invalidate_all(pmap);
6060
6061 counter_u64_add(pmap_l2e_demotions, 1);
6062 CTR2(KTR_PMAP, "pmap_demote_pdpe: success for va %#lx"
6063 " in pmap %p", va, pmap);
6064 return (true);
6065 }
6066
6067 vm_paddr_t
mmu_radix_kextract(vm_offset_t va)6068 mmu_radix_kextract(vm_offset_t va)
6069 {
6070 pml3_entry_t l3e;
6071 vm_paddr_t pa;
6072
6073 CTR2(KTR_PMAP, "%s(%#x)", __func__, va);
6074 if (va >= DMAP_MIN_ADDRESS && va < DMAP_MAX_ADDRESS) {
6075 pa = DMAP_TO_PHYS(va);
6076 } else {
6077 /* Big-endian PTE on stack */
6078 l3e = *pmap_pml3e(kernel_pmap, va);
6079 if (be64toh(l3e) & RPTE_LEAF) {
6080 pa = (be64toh(l3e) & PG_PS_FRAME) | (va & L3_PAGE_MASK);
6081 pa |= (va & L3_PAGE_MASK);
6082 } else {
6083 /*
6084 * Beware of a concurrent promotion that changes the
6085 * PDE at this point! For example, vtopte() must not
6086 * be used to access the PTE because it would use the
6087 * new PDE. It is, however, safe to use the old PDE
6088 * because the page table page is preserved by the
6089 * promotion.
6090 */
6091 pa = be64toh(*pmap_l3e_to_pte(&l3e, va));
6092 pa = (pa & PG_FRAME) | (va & PAGE_MASK);
6093 pa |= (va & PAGE_MASK);
6094 }
6095 }
6096 return (pa);
6097 }
6098
6099 static pt_entry_t
mmu_radix_calc_wimg(vm_paddr_t pa,vm_memattr_t ma)6100 mmu_radix_calc_wimg(vm_paddr_t pa, vm_memattr_t ma)
6101 {
6102
6103 if (ma != VM_MEMATTR_DEFAULT) {
6104 return pmap_cache_bits(ma);
6105 }
6106
6107 /*
6108 * Assume the page is cache inhibited and access is guarded unless
6109 * it's in our available memory array.
6110 */
6111 for (int i = 0; i < pregions_sz; i++) {
6112 if ((pa >= pregions[i].mr_start) &&
6113 (pa < (pregions[i].mr_start + pregions[i].mr_size)))
6114 return (RPTE_ATTR_MEM);
6115 }
6116 return (RPTE_ATTR_GUARDEDIO);
6117 }
6118
6119 static void
mmu_radix_kenter_attr(vm_offset_t va,vm_paddr_t pa,vm_memattr_t ma)6120 mmu_radix_kenter_attr(vm_offset_t va, vm_paddr_t pa, vm_memattr_t ma)
6121 {
6122 pt_entry_t *pte, pteval;
6123 uint64_t cache_bits;
6124
6125 pte = kvtopte(va);
6126 MPASS(pte != NULL);
6127 pteval = pa | RPTE_EAA_R | RPTE_EAA_W | RPTE_EAA_P | PG_M | PG_A;
6128 cache_bits = mmu_radix_calc_wimg(pa, ma);
6129 pte_store(pte, pteval | cache_bits);
6130 }
6131
6132 void
mmu_radix_kremove(vm_offset_t va)6133 mmu_radix_kremove(vm_offset_t va)
6134 {
6135 pt_entry_t *pte;
6136
6137 CTR2(KTR_PMAP, "%s(%#x)", __func__, va);
6138
6139 pte = kvtopte(va);
6140 pte_clear(pte);
6141 }
6142
6143 int
mmu_radix_decode_kernel_ptr(vm_offset_t addr,int * is_user,vm_offset_t * decoded)6144 mmu_radix_decode_kernel_ptr(vm_offset_t addr,
6145 int *is_user, vm_offset_t *decoded)
6146 {
6147
6148 CTR2(KTR_PMAP, "%s(%#jx)", __func__, (uintmax_t)addr);
6149 *decoded = addr;
6150 *is_user = (addr < VM_MAXUSER_ADDRESS);
6151 return (0);
6152 }
6153
6154 static int
mmu_radix_dev_direct_mapped(vm_paddr_t pa,vm_size_t size)6155 mmu_radix_dev_direct_mapped(vm_paddr_t pa, vm_size_t size)
6156 {
6157
6158 CTR3(KTR_PMAP, "%s(%#x, %#x)", __func__, pa, size);
6159 return (mem_valid(pa, size));
6160 }
6161
6162 static void
mmu_radix_scan_init(void)6163 mmu_radix_scan_init(void)
6164 {
6165
6166 CTR1(KTR_PMAP, "%s()", __func__);
6167 UNIMPLEMENTED();
6168 }
6169
6170 static void
mmu_radix_dumpsys_map(vm_paddr_t pa,size_t sz,void ** va)6171 mmu_radix_dumpsys_map(vm_paddr_t pa, size_t sz,
6172 void **va)
6173 {
6174 CTR4(KTR_PMAP, "%s(%#jx, %#zx, %p)", __func__, (uintmax_t)pa, sz, va);
6175 UNIMPLEMENTED();
6176 }
6177
6178 vm_offset_t
mmu_radix_quick_enter_page(vm_page_t m)6179 mmu_radix_quick_enter_page(vm_page_t m)
6180 {
6181 vm_paddr_t paddr;
6182
6183 CTR2(KTR_PMAP, "%s(%p)", __func__, m);
6184 paddr = VM_PAGE_TO_PHYS(m);
6185 return (PHYS_TO_DMAP(paddr));
6186 }
6187
6188 void
mmu_radix_quick_remove_page(vm_offset_t addr __unused)6189 mmu_radix_quick_remove_page(vm_offset_t addr __unused)
6190 {
6191 /* no work to do here */
6192 CTR2(KTR_PMAP, "%s(%#x)", __func__, addr);
6193 }
6194
6195 static void
pmap_invalidate_cache_range(vm_offset_t sva,vm_offset_t eva)6196 pmap_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva)
6197 {
6198 cpu_flush_dcache((void *)sva, eva - sva);
6199 }
6200
6201 int
mmu_radix_change_attr(vm_offset_t va,vm_size_t size,vm_memattr_t mode)6202 mmu_radix_change_attr(vm_offset_t va, vm_size_t size,
6203 vm_memattr_t mode)
6204 {
6205 int error;
6206
6207 CTR4(KTR_PMAP, "%s(%#x, %#zx, %d)", __func__, va, size, mode);
6208 PMAP_LOCK(kernel_pmap);
6209 error = pmap_change_attr_locked(va, size, mode, true);
6210 PMAP_UNLOCK(kernel_pmap);
6211 return (error);
6212 }
6213
6214 static int
pmap_change_attr_locked(vm_offset_t va,vm_size_t size,int mode,bool flush)6215 pmap_change_attr_locked(vm_offset_t va, vm_size_t size, int mode, bool flush)
6216 {
6217 vm_offset_t base, offset, tmpva;
6218 vm_paddr_t pa_start, pa_end, pa_end1;
6219 pml2_entry_t *l2e;
6220 pml3_entry_t *l3e;
6221 pt_entry_t *pte;
6222 int cache_bits, error;
6223 bool changed;
6224
6225 PMAP_LOCK_ASSERT(kernel_pmap, MA_OWNED);
6226 base = trunc_page(va);
6227 offset = va & PAGE_MASK;
6228 size = round_page(offset + size);
6229
6230 /*
6231 * Only supported on kernel virtual addresses, including the direct
6232 * map but excluding the recursive map.
6233 */
6234 if (base < DMAP_MIN_ADDRESS)
6235 return (EINVAL);
6236
6237 cache_bits = pmap_cache_bits(mode);
6238 changed = false;
6239
6240 /*
6241 * Pages that aren't mapped aren't supported. Also break down 2MB pages
6242 * into 4KB pages if required.
6243 */
6244 for (tmpva = base; tmpva < base + size; ) {
6245 l2e = pmap_pml2e(kernel_pmap, tmpva);
6246 if (l2e == NULL || *l2e == 0)
6247 return (EINVAL);
6248 if (be64toh(*l2e) & RPTE_LEAF) {
6249 /*
6250 * If the current 1GB page already has the required
6251 * memory type, then we need not demote this page. Just
6252 * increment tmpva to the next 1GB page frame.
6253 */
6254 if ((be64toh(*l2e) & RPTE_ATTR_MASK) == cache_bits) {
6255 tmpva = trunc_1gpage(tmpva) + L2_PAGE_SIZE;
6256 continue;
6257 }
6258
6259 /*
6260 * If the current offset aligns with a 1GB page frame
6261 * and there is at least 1GB left within the range, then
6262 * we need not break down this page into 2MB pages.
6263 */
6264 if ((tmpva & L2_PAGE_MASK) == 0 &&
6265 tmpva + L2_PAGE_MASK < base + size) {
6266 tmpva += L2_PAGE_MASK;
6267 continue;
6268 }
6269 if (!pmap_demote_l2e(kernel_pmap, l2e, tmpva))
6270 return (ENOMEM);
6271 }
6272 l3e = pmap_l2e_to_l3e(l2e, tmpva);
6273 KASSERT(l3e != NULL, ("no l3e entry for %#lx in %p\n",
6274 tmpva, l2e));
6275 if (*l3e == 0)
6276 return (EINVAL);
6277 if (be64toh(*l3e) & RPTE_LEAF) {
6278 /*
6279 * If the current 2MB page already has the required
6280 * memory type, then we need not demote this page. Just
6281 * increment tmpva to the next 2MB page frame.
6282 */
6283 if ((be64toh(*l3e) & RPTE_ATTR_MASK) == cache_bits) {
6284 tmpva = trunc_2mpage(tmpva) + L3_PAGE_SIZE;
6285 continue;
6286 }
6287
6288 /*
6289 * If the current offset aligns with a 2MB page frame
6290 * and there is at least 2MB left within the range, then
6291 * we need not break down this page into 4KB pages.
6292 */
6293 if ((tmpva & L3_PAGE_MASK) == 0 &&
6294 tmpva + L3_PAGE_MASK < base + size) {
6295 tmpva += L3_PAGE_SIZE;
6296 continue;
6297 }
6298 if (!pmap_demote_l3e(kernel_pmap, l3e, tmpva))
6299 return (ENOMEM);
6300 }
6301 pte = pmap_l3e_to_pte(l3e, tmpva);
6302 if (*pte == 0)
6303 return (EINVAL);
6304 tmpva += PAGE_SIZE;
6305 }
6306 error = 0;
6307
6308 /*
6309 * Ok, all the pages exist, so run through them updating their
6310 * cache mode if required.
6311 */
6312 pa_start = pa_end = 0;
6313 for (tmpva = base; tmpva < base + size; ) {
6314 l2e = pmap_pml2e(kernel_pmap, tmpva);
6315 if (be64toh(*l2e) & RPTE_LEAF) {
6316 if ((be64toh(*l2e) & RPTE_ATTR_MASK) != cache_bits) {
6317 pmap_pte_attr(l2e, cache_bits,
6318 RPTE_ATTR_MASK);
6319 changed = true;
6320 }
6321 if (tmpva >= VM_MIN_KERNEL_ADDRESS &&
6322 (*l2e & PG_PS_FRAME) < dmaplimit) {
6323 if (pa_start == pa_end) {
6324 /* Start physical address run. */
6325 pa_start = be64toh(*l2e) & PG_PS_FRAME;
6326 pa_end = pa_start + L2_PAGE_SIZE;
6327 } else if (pa_end == (be64toh(*l2e) & PG_PS_FRAME))
6328 pa_end += L2_PAGE_SIZE;
6329 else {
6330 /* Run ended, update direct map. */
6331 error = pmap_change_attr_locked(
6332 PHYS_TO_DMAP(pa_start),
6333 pa_end - pa_start, mode, flush);
6334 if (error != 0)
6335 break;
6336 /* Start physical address run. */
6337 pa_start = be64toh(*l2e) & PG_PS_FRAME;
6338 pa_end = pa_start + L2_PAGE_SIZE;
6339 }
6340 }
6341 tmpva = trunc_1gpage(tmpva) + L2_PAGE_SIZE;
6342 continue;
6343 }
6344 l3e = pmap_l2e_to_l3e(l2e, tmpva);
6345 if (be64toh(*l3e) & RPTE_LEAF) {
6346 if ((be64toh(*l3e) & RPTE_ATTR_MASK) != cache_bits) {
6347 pmap_pte_attr(l3e, cache_bits,
6348 RPTE_ATTR_MASK);
6349 changed = true;
6350 }
6351 if (tmpva >= VM_MIN_KERNEL_ADDRESS &&
6352 (be64toh(*l3e) & PG_PS_FRAME) < dmaplimit) {
6353 if (pa_start == pa_end) {
6354 /* Start physical address run. */
6355 pa_start = be64toh(*l3e) & PG_PS_FRAME;
6356 pa_end = pa_start + L3_PAGE_SIZE;
6357 } else if (pa_end == (be64toh(*l3e) & PG_PS_FRAME))
6358 pa_end += L3_PAGE_SIZE;
6359 else {
6360 /* Run ended, update direct map. */
6361 error = pmap_change_attr_locked(
6362 PHYS_TO_DMAP(pa_start),
6363 pa_end - pa_start, mode, flush);
6364 if (error != 0)
6365 break;
6366 /* Start physical address run. */
6367 pa_start = be64toh(*l3e) & PG_PS_FRAME;
6368 pa_end = pa_start + L3_PAGE_SIZE;
6369 }
6370 }
6371 tmpva = trunc_2mpage(tmpva) + L3_PAGE_SIZE;
6372 } else {
6373 pte = pmap_l3e_to_pte(l3e, tmpva);
6374 if ((be64toh(*pte) & RPTE_ATTR_MASK) != cache_bits) {
6375 pmap_pte_attr(pte, cache_bits,
6376 RPTE_ATTR_MASK);
6377 changed = true;
6378 }
6379 if (tmpva >= VM_MIN_KERNEL_ADDRESS &&
6380 (be64toh(*pte) & PG_FRAME) < dmaplimit) {
6381 if (pa_start == pa_end) {
6382 /* Start physical address run. */
6383 pa_start = be64toh(*pte) & PG_FRAME;
6384 pa_end = pa_start + PAGE_SIZE;
6385 } else if (pa_end == (be64toh(*pte) & PG_FRAME))
6386 pa_end += PAGE_SIZE;
6387 else {
6388 /* Run ended, update direct map. */
6389 error = pmap_change_attr_locked(
6390 PHYS_TO_DMAP(pa_start),
6391 pa_end - pa_start, mode, flush);
6392 if (error != 0)
6393 break;
6394 /* Start physical address run. */
6395 pa_start = be64toh(*pte) & PG_FRAME;
6396 pa_end = pa_start + PAGE_SIZE;
6397 }
6398 }
6399 tmpva += PAGE_SIZE;
6400 }
6401 }
6402 if (error == 0 && pa_start != pa_end && pa_start < dmaplimit) {
6403 pa_end1 = MIN(pa_end, dmaplimit);
6404 if (pa_start != pa_end1)
6405 error = pmap_change_attr_locked(PHYS_TO_DMAP(pa_start),
6406 pa_end1 - pa_start, mode, flush);
6407 }
6408
6409 /*
6410 * Flush CPU caches if required to make sure any data isn't cached that
6411 * shouldn't be, etc.
6412 */
6413 if (changed) {
6414 pmap_invalidate_all(kernel_pmap);
6415
6416 if (flush)
6417 pmap_invalidate_cache_range(base, tmpva);
6418 }
6419 return (error);
6420 }
6421
6422 /*
6423 * Allocate physical memory for the vm_page array and map it into KVA,
6424 * attempting to back the vm_pages with domain-local memory.
6425 */
6426 void
mmu_radix_page_array_startup(long pages)6427 mmu_radix_page_array_startup(long pages)
6428 {
6429 #ifdef notyet
6430 pml2_entry_t *l2e;
6431 pml3_entry_t *pde;
6432 pml3_entry_t newl3;
6433 vm_offset_t va;
6434 long pfn;
6435 int domain, i;
6436 #endif
6437 vm_paddr_t pa;
6438 vm_offset_t start, end;
6439
6440 vm_page_array_size = pages;
6441
6442 start = VM_MIN_KERNEL_ADDRESS;
6443 end = start + pages * sizeof(struct vm_page);
6444
6445 pa = vm_phys_early_alloc(-1, end - start);
6446
6447 start = mmu_radix_map(&start, pa, end - start, VM_MEMATTR_DEFAULT);
6448 #ifdef notyet
6449 /* TODO: NUMA vm_page_array. Blocked out until then (copied from amd64). */
6450 for (va = start; va < end; va += L3_PAGE_SIZE) {
6451 pfn = first_page + (va - start) / sizeof(struct vm_page);
6452 domain = vm_phys_domain(ptoa(pfn));
6453 l2e = pmap_pml2e(kernel_pmap, va);
6454 if ((be64toh(*l2e) & PG_V) == 0) {
6455 pa = vm_phys_early_alloc(domain, PAGE_SIZE);
6456 dump_add_page(pa);
6457 pagezero(PHYS_TO_DMAP(pa));
6458 pde_store(l2e, (pml2_entry_t)pa);
6459 }
6460 pde = pmap_l2e_to_l3e(l2e, va);
6461 if ((be64toh(*pde) & PG_V) != 0)
6462 panic("Unexpected pde %p", pde);
6463 pa = vm_phys_early_alloc(domain, L3_PAGE_SIZE);
6464 for (i = 0; i < NPDEPG; i++)
6465 dump_add_page(pa + i * PAGE_SIZE);
6466 newl3 = (pml3_entry_t)(pa | RPTE_EAA_P | RPTE_EAA_R | RPTE_EAA_W);
6467 pte_store(pde, newl3);
6468 }
6469 #endif
6470 vm_page_array = (vm_page_t)start;
6471 }
6472
6473 #ifdef DDB
6474 #include <sys/kdb.h>
6475 #include <ddb/ddb.h>
6476
6477 static void
pmap_pte_walk(pml1_entry_t * l1,vm_offset_t va)6478 pmap_pte_walk(pml1_entry_t *l1, vm_offset_t va)
6479 {
6480 pml1_entry_t *l1e;
6481 pml2_entry_t *l2e;
6482 pml3_entry_t *l3e;
6483 pt_entry_t *pte;
6484
6485 l1e = &l1[pmap_pml1e_index(va)];
6486 db_printf("VA %#016lx l1e %#016lx", va, be64toh(*l1e));
6487 if ((be64toh(*l1e) & PG_V) == 0) {
6488 db_printf("\n");
6489 return;
6490 }
6491 l2e = pmap_l1e_to_l2e(l1e, va);
6492 db_printf(" l2e %#016lx", be64toh(*l2e));
6493 if ((be64toh(*l2e) & PG_V) == 0 || (be64toh(*l2e) & RPTE_LEAF) != 0) {
6494 db_printf("\n");
6495 return;
6496 }
6497 l3e = pmap_l2e_to_l3e(l2e, va);
6498 db_printf(" l3e %#016lx", be64toh(*l3e));
6499 if ((be64toh(*l3e) & PG_V) == 0 || (be64toh(*l3e) & RPTE_LEAF) != 0) {
6500 db_printf("\n");
6501 return;
6502 }
6503 pte = pmap_l3e_to_pte(l3e, va);
6504 db_printf(" pte %#016lx\n", be64toh(*pte));
6505 }
6506
6507 void
pmap_page_print_mappings(vm_page_t m)6508 pmap_page_print_mappings(vm_page_t m)
6509 {
6510 pmap_t pmap;
6511 pv_entry_t pv;
6512
6513 db_printf("page %p(%lx)\n", m, m->phys_addr);
6514 /* need to elide locks if running in ddb */
6515 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) {
6516 db_printf("pv: %p ", pv);
6517 db_printf("va: %#016lx ", pv->pv_va);
6518 pmap = PV_PMAP(pv);
6519 db_printf("pmap %p ", pmap);
6520 if (pmap != NULL) {
6521 db_printf("asid: %lu\n", pmap->pm_pid);
6522 pmap_pte_walk(pmap->pm_pml1, pv->pv_va);
6523 }
6524 }
6525 }
6526
DB_SHOW_COMMAND(pte,pmap_print_pte)6527 DB_SHOW_COMMAND(pte, pmap_print_pte)
6528 {
6529 vm_offset_t va;
6530 pmap_t pmap;
6531
6532 if (!have_addr) {
6533 db_printf("show pte addr\n");
6534 return;
6535 }
6536 va = (vm_offset_t)addr;
6537
6538 if (va >= DMAP_MIN_ADDRESS)
6539 pmap = kernel_pmap;
6540 else if (kdb_thread != NULL)
6541 pmap = vmspace_pmap(kdb_thread->td_proc->p_vmspace);
6542 else
6543 pmap = vmspace_pmap(curthread->td_proc->p_vmspace);
6544
6545 pmap_pte_walk(pmap->pm_pml1, va);
6546 }
6547
6548 #endif
6549