1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2018 Matthew Macy
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 *
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28 #include "opt_platform.h"
29
30 #include <sys/param.h>
31 #include <sys/kernel.h>
32 #include <sys/systm.h>
33 #include <sys/conf.h>
34 #include <sys/bitstring.h>
35 #include <sys/queue.h>
36 #include <sys/cpuset.h>
37 #include <sys/endian.h>
38 #include <sys/kerneldump.h>
39 #include <sys/ktr.h>
40 #include <sys/lock.h>
41 #include <sys/syslog.h>
42 #include <sys/msgbuf.h>
43 #include <sys/malloc.h>
44 #include <sys/mman.h>
45 #include <sys/mutex.h>
46 #include <sys/proc.h>
47 #include <sys/rwlock.h>
48 #include <sys/sched.h>
49 #include <sys/sysctl.h>
50 #include <sys/systm.h>
51 #include <sys/vmem.h>
52 #include <sys/vmmeter.h>
53 #include <sys/smp.h>
54
55 #include <sys/kdb.h>
56
57 #include <dev/ofw/openfirm.h>
58
59 #include <vm/vm.h>
60 #include <vm/pmap.h>
61 #include <vm/vm_param.h>
62 #include <vm/vm_kern.h>
63 #include <vm/vm_page.h>
64 #include <vm/vm_map.h>
65 #include <vm/vm_object.h>
66 #include <vm/vm_extern.h>
67 #include <vm/vm_pageout.h>
68 #include <vm/vm_phys.h>
69 #include <vm/vm_radix.h>
70 #include <vm/vm_reserv.h>
71 #include <vm/vm_dumpset.h>
72 #include <vm/uma.h>
73
74 #include <machine/_inttypes.h>
75 #include <machine/cpu.h>
76 #include <machine/platform.h>
77 #include <machine/frame.h>
78 #include <machine/md_var.h>
79 #include <machine/psl.h>
80 #include <machine/bat.h>
81 #include <machine/hid.h>
82 #include <machine/pte.h>
83 #include <machine/sr.h>
84 #include <machine/trap.h>
85 #include <machine/mmuvar.h>
86
87 /* For pseries bit. */
88 #include <powerpc/pseries/phyp-hvcall.h>
89
90 #ifdef INVARIANTS
91 #include <vm/uma_dbg.h>
92 #endif
93
94 #define PPC_BITLSHIFT(bit) (sizeof(long)*NBBY - 1 - (bit))
95 #define PPC_BIT(bit) (1UL << PPC_BITLSHIFT(bit))
96 #define PPC_BITLSHIFT_VAL(val, bit) ((val) << PPC_BITLSHIFT(bit))
97
98 #include "opt_ddb.h"
99
100 #ifdef DDB
101 static void pmap_pte_walk(pml1_entry_t *l1, vm_offset_t va);
102 #endif
103
104 #define PG_W RPTE_WIRED
105 #define PG_V RPTE_VALID
106 #define PG_MANAGED RPTE_MANAGED
107 #define PG_PROMOTED RPTE_PROMOTED
108 #define PG_M RPTE_C
109 #define PG_A RPTE_R
110 #define PG_X RPTE_EAA_X
111 #define PG_RW RPTE_EAA_W
112 #define PG_PTE_CACHE RPTE_ATTR_MASK
113
114 #define RPTE_SHIFT 9
115 #define NLS_MASK ((1UL<<5)-1)
116 #define RPTE_ENTRIES (1UL<<RPTE_SHIFT)
117 #define RPTE_MASK (RPTE_ENTRIES-1)
118
119 #define NLB_SHIFT 0
120 #define NLB_MASK (((1UL<<52)-1) << 8)
121
122 extern int nkpt;
123 extern caddr_t crashdumpmap;
124
125 #define RIC_FLUSH_TLB 0
126 #define RIC_FLUSH_PWC 1
127 #define RIC_FLUSH_ALL 2
128
129 #define POWER9_TLB_SETS_RADIX 128 /* # sets in POWER9 TLB Radix mode */
130
131 #define PPC_INST_TLBIE 0x7c000264
132 #define PPC_INST_TLBIEL 0x7c000224
133 #define PPC_INST_SLBIA 0x7c0003e4
134
135 #define ___PPC_RA(a) (((a) & 0x1f) << 16)
136 #define ___PPC_RB(b) (((b) & 0x1f) << 11)
137 #define ___PPC_RS(s) (((s) & 0x1f) << 21)
138 #define ___PPC_RT(t) ___PPC_RS(t)
139 #define ___PPC_R(r) (((r) & 0x1) << 16)
140 #define ___PPC_PRS(prs) (((prs) & 0x1) << 17)
141 #define ___PPC_RIC(ric) (((ric) & 0x3) << 18)
142
143 #define PPC_SLBIA(IH) __XSTRING(.long PPC_INST_SLBIA | \
144 ((IH & 0x7) << 21))
145 #define PPC_TLBIE_5(rb,rs,ric,prs,r) \
146 __XSTRING(.long PPC_INST_TLBIE | \
147 ___PPC_RB(rb) | ___PPC_RS(rs) | \
148 ___PPC_RIC(ric) | ___PPC_PRS(prs) | \
149 ___PPC_R(r))
150
151 #define PPC_TLBIEL(rb,rs,ric,prs,r) \
152 __XSTRING(.long PPC_INST_TLBIEL | \
153 ___PPC_RB(rb) | ___PPC_RS(rs) | \
154 ___PPC_RIC(ric) | ___PPC_PRS(prs) | \
155 ___PPC_R(r))
156
157 #define PPC_INVALIDATE_ERAT PPC_SLBIA(7)
158
159 static __inline void
ttusync(void)160 ttusync(void)
161 {
162 __asm __volatile("eieio; tlbsync; ptesync" ::: "memory");
163 }
164
165 #define TLBIEL_INVAL_SEL_MASK 0xc00 /* invalidation selector */
166 #define TLBIEL_INVAL_PAGE 0x000 /* invalidate a single page */
167 #define TLBIEL_INVAL_SET_PID 0x400 /* invalidate a set for the current PID */
168 #define TLBIEL_INVAL_SET_LPID 0x800 /* invalidate a set for current LPID */
169 #define TLBIEL_INVAL_SET 0xc00 /* invalidate a set for all LPIDs */
170
171 #define TLBIE_ACTUAL_PAGE_MASK 0xe0
172 #define TLBIE_ACTUAL_PAGE_4K 0x00
173 #define TLBIE_ACTUAL_PAGE_64K 0xa0
174 #define TLBIE_ACTUAL_PAGE_2M 0x20
175 #define TLBIE_ACTUAL_PAGE_1G 0x40
176
177 #define TLBIE_PRS_PARTITION_SCOPE 0x0
178 #define TLBIE_PRS_PROCESS_SCOPE 0x1
179
180 #define TLBIE_RIC_INVALIDATE_TLB 0x0 /* Invalidate just TLB */
181 #define TLBIE_RIC_INVALIDATE_PWC 0x1 /* Invalidate just PWC */
182 #define TLBIE_RIC_INVALIDATE_ALL 0x2 /* Invalidate TLB, PWC,
183 * cached {proc, part}tab entries
184 */
185 #define TLBIE_RIC_INVALIDATE_SEQ 0x3 /* HPT - only:
186 * Invalidate a range of translations
187 */
188
189 static __always_inline void
radix_tlbie(uint8_t ric,uint8_t prs,uint16_t is,uint32_t pid,uint32_t lpid,vm_offset_t va,uint16_t ap)190 radix_tlbie(uint8_t ric, uint8_t prs, uint16_t is, uint32_t pid, uint32_t lpid,
191 vm_offset_t va, uint16_t ap)
192 {
193 uint64_t rb, rs;
194
195 MPASS((va & PAGE_MASK) == 0);
196
197 rs = ((uint64_t)pid << 32) | lpid;
198 rb = va | is | ap;
199 __asm __volatile(PPC_TLBIE_5(%0, %1, %2, %3, 1) : :
200 "r" (rb), "r" (rs), "i" (ric), "i" (prs) : "memory");
201 }
202
203 static __inline void
radix_tlbie_fixup(uint32_t pid,vm_offset_t va,int ap)204 radix_tlbie_fixup(uint32_t pid, vm_offset_t va, int ap)
205 {
206
207 __asm __volatile("ptesync" ::: "memory");
208 radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE,
209 TLBIEL_INVAL_PAGE, 0, 0, va, ap);
210 __asm __volatile("ptesync" ::: "memory");
211 radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE,
212 TLBIEL_INVAL_PAGE, pid, 0, va, ap);
213 }
214
215 static __inline void
radix_tlbie_invlpg_user_4k(uint32_t pid,vm_offset_t va)216 radix_tlbie_invlpg_user_4k(uint32_t pid, vm_offset_t va)
217 {
218
219 radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE,
220 TLBIEL_INVAL_PAGE, pid, 0, va, TLBIE_ACTUAL_PAGE_4K);
221 radix_tlbie_fixup(pid, va, TLBIE_ACTUAL_PAGE_4K);
222 }
223
224 static __inline void
radix_tlbie_invlpg_user_2m(uint32_t pid,vm_offset_t va)225 radix_tlbie_invlpg_user_2m(uint32_t pid, vm_offset_t va)
226 {
227
228 radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE,
229 TLBIEL_INVAL_PAGE, pid, 0, va, TLBIE_ACTUAL_PAGE_2M);
230 radix_tlbie_fixup(pid, va, TLBIE_ACTUAL_PAGE_2M);
231 }
232
233 static __inline void
radix_tlbie_invlpwc_user(uint32_t pid)234 radix_tlbie_invlpwc_user(uint32_t pid)
235 {
236
237 radix_tlbie(TLBIE_RIC_INVALIDATE_PWC, TLBIE_PRS_PROCESS_SCOPE,
238 TLBIEL_INVAL_SET_PID, pid, 0, 0, 0);
239 }
240
241 static __inline void
radix_tlbie_flush_user(uint32_t pid)242 radix_tlbie_flush_user(uint32_t pid)
243 {
244
245 radix_tlbie(TLBIE_RIC_INVALIDATE_ALL, TLBIE_PRS_PROCESS_SCOPE,
246 TLBIEL_INVAL_SET_PID, pid, 0, 0, 0);
247 }
248
249 static __inline void
radix_tlbie_invlpg_kernel_4k(vm_offset_t va)250 radix_tlbie_invlpg_kernel_4k(vm_offset_t va)
251 {
252
253 radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE,
254 TLBIEL_INVAL_PAGE, 0, 0, va, TLBIE_ACTUAL_PAGE_4K);
255 radix_tlbie_fixup(0, va, TLBIE_ACTUAL_PAGE_4K);
256 }
257
258 static __inline void
radix_tlbie_invlpg_kernel_2m(vm_offset_t va)259 radix_tlbie_invlpg_kernel_2m(vm_offset_t va)
260 {
261
262 radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE,
263 TLBIEL_INVAL_PAGE, 0, 0, va, TLBIE_ACTUAL_PAGE_2M);
264 radix_tlbie_fixup(0, va, TLBIE_ACTUAL_PAGE_2M);
265 }
266
267 /* 1GB pages aren't currently supported. */
268 static __inline __unused void
radix_tlbie_invlpg_kernel_1g(vm_offset_t va)269 radix_tlbie_invlpg_kernel_1g(vm_offset_t va)
270 {
271
272 radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE,
273 TLBIEL_INVAL_PAGE, 0, 0, va, TLBIE_ACTUAL_PAGE_1G);
274 radix_tlbie_fixup(0, va, TLBIE_ACTUAL_PAGE_1G);
275 }
276
277 static __inline void
radix_tlbie_invlpwc_kernel(void)278 radix_tlbie_invlpwc_kernel(void)
279 {
280
281 radix_tlbie(TLBIE_RIC_INVALIDATE_PWC, TLBIE_PRS_PROCESS_SCOPE,
282 TLBIEL_INVAL_SET_LPID, 0, 0, 0, 0);
283 }
284
285 static __inline void
radix_tlbie_flush_kernel(void)286 radix_tlbie_flush_kernel(void)
287 {
288
289 radix_tlbie(TLBIE_RIC_INVALIDATE_ALL, TLBIE_PRS_PROCESS_SCOPE,
290 TLBIEL_INVAL_SET_LPID, 0, 0, 0, 0);
291 }
292
293 static __inline vm_pindex_t
pmap_l3e_pindex(vm_offset_t va)294 pmap_l3e_pindex(vm_offset_t va)
295 {
296 return ((va & PG_FRAME) >> L3_PAGE_SIZE_SHIFT);
297 }
298
299 static __inline vm_pindex_t
pmap_pml3e_index(vm_offset_t va)300 pmap_pml3e_index(vm_offset_t va)
301 {
302
303 return ((va >> L3_PAGE_SIZE_SHIFT) & RPTE_MASK);
304 }
305
306 static __inline vm_pindex_t
pmap_pml2e_index(vm_offset_t va)307 pmap_pml2e_index(vm_offset_t va)
308 {
309 return ((va >> L2_PAGE_SIZE_SHIFT) & RPTE_MASK);
310 }
311
312 static __inline vm_pindex_t
pmap_pml1e_index(vm_offset_t va)313 pmap_pml1e_index(vm_offset_t va)
314 {
315 return ((va & PG_FRAME) >> L1_PAGE_SIZE_SHIFT);
316 }
317
318 /* Return various clipped indexes for a given VA */
319 static __inline vm_pindex_t
pmap_pte_index(vm_offset_t va)320 pmap_pte_index(vm_offset_t va)
321 {
322
323 return ((va >> PAGE_SHIFT) & RPTE_MASK);
324 }
325
326 /* Return a pointer to the PT slot that corresponds to a VA */
327 static __inline pt_entry_t *
pmap_l3e_to_pte(pt_entry_t * l3e,vm_offset_t va)328 pmap_l3e_to_pte(pt_entry_t *l3e, vm_offset_t va)
329 {
330 pt_entry_t *pte;
331 vm_paddr_t ptepa;
332
333 ptepa = (be64toh(*l3e) & NLB_MASK);
334 pte = PHYS_TO_DMAP(ptepa);
335 return (&pte[pmap_pte_index(va)]);
336 }
337
338 /* Return a pointer to the PD slot that corresponds to a VA */
339 static __inline pt_entry_t *
pmap_l2e_to_l3e(pt_entry_t * l2e,vm_offset_t va)340 pmap_l2e_to_l3e(pt_entry_t *l2e, vm_offset_t va)
341 {
342 pt_entry_t *l3e;
343 vm_paddr_t l3pa;
344
345 l3pa = (be64toh(*l2e) & NLB_MASK);
346 l3e = PHYS_TO_DMAP(l3pa);
347 return (&l3e[pmap_pml3e_index(va)]);
348 }
349
350 /* Return a pointer to the PD slot that corresponds to a VA */
351 static __inline pt_entry_t *
pmap_l1e_to_l2e(pt_entry_t * l1e,vm_offset_t va)352 pmap_l1e_to_l2e(pt_entry_t *l1e, vm_offset_t va)
353 {
354 pt_entry_t *l2e;
355 vm_paddr_t l2pa;
356
357 l2pa = (be64toh(*l1e) & NLB_MASK);
358
359 l2e = PHYS_TO_DMAP(l2pa);
360 return (&l2e[pmap_pml2e_index(va)]);
361 }
362
363 static __inline pml1_entry_t *
pmap_pml1e(pmap_t pmap,vm_offset_t va)364 pmap_pml1e(pmap_t pmap, vm_offset_t va)
365 {
366
367 return (&pmap->pm_pml1[pmap_pml1e_index(va)]);
368 }
369
370 static pt_entry_t *
pmap_pml2e(pmap_t pmap,vm_offset_t va)371 pmap_pml2e(pmap_t pmap, vm_offset_t va)
372 {
373 pt_entry_t *l1e;
374
375 l1e = pmap_pml1e(pmap, va);
376 if (l1e == NULL || (be64toh(*l1e) & RPTE_VALID) == 0)
377 return (NULL);
378 return (pmap_l1e_to_l2e(l1e, va));
379 }
380
381 static __inline pt_entry_t *
pmap_pml3e(pmap_t pmap,vm_offset_t va)382 pmap_pml3e(pmap_t pmap, vm_offset_t va)
383 {
384 pt_entry_t *l2e;
385
386 l2e = pmap_pml2e(pmap, va);
387 if (l2e == NULL || (be64toh(*l2e) & RPTE_VALID) == 0)
388 return (NULL);
389 return (pmap_l2e_to_l3e(l2e, va));
390 }
391
392 static __inline pt_entry_t *
pmap_pte(pmap_t pmap,vm_offset_t va)393 pmap_pte(pmap_t pmap, vm_offset_t va)
394 {
395 pt_entry_t *l3e;
396
397 l3e = pmap_pml3e(pmap, va);
398 if (l3e == NULL || (be64toh(*l3e) & RPTE_VALID) == 0)
399 return (NULL);
400 return (pmap_l3e_to_pte(l3e, va));
401 }
402
403 int nkpt = 64;
404 SYSCTL_INT(_machdep, OID_AUTO, nkpt, CTLFLAG_RD, &nkpt, 0,
405 "Number of kernel page table pages allocated on bootup");
406
407 vm_paddr_t dmaplimit;
408
409 SYSCTL_DECL(_vm_pmap);
410
411 #ifdef INVARIANTS
412 #define VERBOSE_PMAP 0
413 #define VERBOSE_PROTECT 0
414 static int pmap_logging;
415 SYSCTL_INT(_vm_pmap, OID_AUTO, pmap_logging, CTLFLAG_RWTUN,
416 &pmap_logging, 0, "verbose debug logging");
417 #endif
418
419 static u_int64_t KPTphys; /* phys addr of kernel level 1 */
420
421 //static vm_paddr_t KERNend; /* phys addr of end of bootstrap data */
422
423 static vm_offset_t qframe = 0;
424 static struct mtx qframe_mtx;
425
426 void mmu_radix_activate(struct thread *);
427 void mmu_radix_advise(pmap_t, vm_offset_t, vm_offset_t, int);
428 void mmu_radix_align_superpage(vm_object_t, vm_ooffset_t, vm_offset_t *,
429 vm_size_t);
430 void mmu_radix_clear_modify(vm_page_t);
431 void mmu_radix_copy(pmap_t, pmap_t, vm_offset_t, vm_size_t, vm_offset_t);
432 int mmu_radix_decode_kernel_ptr(vm_offset_t, int *, vm_offset_t *);
433 int mmu_radix_enter(pmap_t, vm_offset_t, vm_page_t, vm_prot_t, u_int, int8_t);
434 void mmu_radix_enter_object(pmap_t, vm_offset_t, vm_offset_t, vm_page_t,
435 vm_prot_t);
436 void mmu_radix_enter_quick(pmap_t, vm_offset_t, vm_page_t, vm_prot_t);
437 vm_paddr_t mmu_radix_extract(pmap_t pmap, vm_offset_t va);
438 vm_page_t mmu_radix_extract_and_hold(pmap_t, vm_offset_t, vm_prot_t);
439 void mmu_radix_kenter(vm_offset_t, vm_paddr_t);
440 vm_paddr_t mmu_radix_kextract(vm_offset_t);
441 void mmu_radix_kremove(vm_offset_t);
442 bool mmu_radix_is_modified(vm_page_t);
443 bool mmu_radix_is_prefaultable(pmap_t, vm_offset_t);
444 bool mmu_radix_is_referenced(vm_page_t);
445 void mmu_radix_object_init_pt(pmap_t, vm_offset_t, vm_object_t,
446 vm_pindex_t, vm_size_t);
447 bool mmu_radix_page_exists_quick(pmap_t, vm_page_t);
448 void mmu_radix_page_init(vm_page_t);
449 bool mmu_radix_page_is_mapped(vm_page_t m);
450 void mmu_radix_page_set_memattr(vm_page_t, vm_memattr_t);
451 int mmu_radix_page_wired_mappings(vm_page_t);
452 int mmu_radix_pinit(pmap_t);
453 void mmu_radix_protect(pmap_t, vm_offset_t, vm_offset_t, vm_prot_t);
454 bool mmu_radix_ps_enabled(pmap_t);
455 void mmu_radix_qenter(void *, vm_page_t *, int);
456 void mmu_radix_qremove(void *, int);
457 void *mmu_radix_quick_enter_page(vm_page_t);
458 void mmu_radix_quick_remove_page(void *);
459 int mmu_radix_ts_referenced(vm_page_t);
460 void mmu_radix_release(pmap_t);
461 void mmu_radix_remove(pmap_t, vm_offset_t, vm_offset_t);
462 void mmu_radix_remove_all(vm_page_t);
463 void mmu_radix_remove_pages(pmap_t);
464 void mmu_radix_remove_write(vm_page_t);
465 void mmu_radix_sync_icache(pmap_t pm, vm_offset_t va, vm_size_t sz);
466 void mmu_radix_unwire(pmap_t, vm_offset_t, vm_offset_t);
467 void mmu_radix_zero_page(vm_page_t);
468 void mmu_radix_zero_page_area(vm_page_t, int, int);
469 int mmu_radix_change_attr(void *, vm_size_t, vm_memattr_t);
470 void mmu_radix_page_array_startup(long pages);
471
472 #include "mmu_oea64.h"
473
474 /*
475 * Kernel MMU interface
476 */
477
478 static void mmu_radix_bootstrap(vm_offset_t, vm_offset_t);
479
480 static void mmu_radix_copy_page(vm_page_t, vm_page_t);
481 static void mmu_radix_copy_pages(vm_page_t *ma, vm_offset_t a_offset,
482 vm_page_t *mb, vm_offset_t b_offset, int xfersize);
483 static int mmu_radix_growkernel(vm_offset_t);
484 static void mmu_radix_init(void);
485 static int mmu_radix_mincore(pmap_t, vm_offset_t, vm_paddr_t *);
486 static void *mmu_radix_map(vm_offset_t *, vm_paddr_t, vm_paddr_t, int);
487 static void mmu_radix_pinit0(pmap_t);
488
489 static void *mmu_radix_mapdev(vm_paddr_t, vm_size_t);
490 static void *mmu_radix_mapdev_attr(vm_paddr_t, vm_size_t, vm_memattr_t);
491 static void mmu_radix_unmapdev(void *, vm_size_t);
492 static void mmu_radix_kenter_attr(vm_offset_t, vm_paddr_t, vm_memattr_t ma);
493 static int mmu_radix_dev_direct_mapped(vm_paddr_t, vm_size_t);
494 static void mmu_radix_dumpsys_map(vm_paddr_t pa, size_t sz, void **va);
495 static void mmu_radix_scan_init(void);
496 static void mmu_radix_cpu_bootstrap(int ap);
497 static void mmu_radix_tlbie_all(void);
498
499 static struct pmap_funcs mmu_radix_methods = {
500 .bootstrap = mmu_radix_bootstrap,
501 .copy_page = mmu_radix_copy_page,
502 .copy_pages = mmu_radix_copy_pages,
503 .cpu_bootstrap = mmu_radix_cpu_bootstrap,
504 .growkernel_nopanic = mmu_radix_growkernel,
505 .init = mmu_radix_init,
506 .map = mmu_radix_map,
507 .mincore = mmu_radix_mincore,
508 .pinit = mmu_radix_pinit,
509 .pinit0 = mmu_radix_pinit0,
510
511 .mapdev = mmu_radix_mapdev,
512 .mapdev_attr = mmu_radix_mapdev_attr,
513 .unmapdev = mmu_radix_unmapdev,
514 .kenter_attr = mmu_radix_kenter_attr,
515 .dev_direct_mapped = mmu_radix_dev_direct_mapped,
516 .dumpsys_pa_init = mmu_radix_scan_init,
517 .dumpsys_map_chunk = mmu_radix_dumpsys_map,
518 .page_is_mapped = mmu_radix_page_is_mapped,
519 .ps_enabled = mmu_radix_ps_enabled,
520 .align_superpage = mmu_radix_align_superpage,
521 .object_init_pt = mmu_radix_object_init_pt,
522 .protect = mmu_radix_protect,
523 /* pmap dispatcher interface */
524 .clear_modify = mmu_radix_clear_modify,
525 .copy = mmu_radix_copy,
526 .enter = mmu_radix_enter,
527 .enter_object = mmu_radix_enter_object,
528 .enter_quick = mmu_radix_enter_quick,
529 .extract = mmu_radix_extract,
530 .extract_and_hold = mmu_radix_extract_and_hold,
531 .is_modified = mmu_radix_is_modified,
532 .is_prefaultable = mmu_radix_is_prefaultable,
533 .is_referenced = mmu_radix_is_referenced,
534 .ts_referenced = mmu_radix_ts_referenced,
535 .page_exists_quick = mmu_radix_page_exists_quick,
536 .page_init = mmu_radix_page_init,
537 .page_wired_mappings = mmu_radix_page_wired_mappings,
538 .qenter = mmu_radix_qenter,
539 .qremove = mmu_radix_qremove,
540 .release = mmu_radix_release,
541 .remove = mmu_radix_remove,
542 .remove_all = mmu_radix_remove_all,
543 .remove_write = mmu_radix_remove_write,
544 .sync_icache = mmu_radix_sync_icache,
545 .unwire = mmu_radix_unwire,
546 .zero_page = mmu_radix_zero_page,
547 .zero_page_area = mmu_radix_zero_page_area,
548 .activate = mmu_radix_activate,
549 .quick_enter_page = mmu_radix_quick_enter_page,
550 .quick_remove_page = mmu_radix_quick_remove_page,
551 .page_set_memattr = mmu_radix_page_set_memattr,
552 .page_array_startup = mmu_radix_page_array_startup,
553
554 /* Internal interfaces */
555 .kenter = mmu_radix_kenter,
556 .kextract = mmu_radix_kextract,
557 .kremove = mmu_radix_kremove,
558 .change_attr = mmu_radix_change_attr,
559 .decode_kernel_ptr = mmu_radix_decode_kernel_ptr,
560
561 .tlbie_all = mmu_radix_tlbie_all,
562 };
563
564 MMU_DEF(mmu_radix, MMU_TYPE_RADIX, mmu_radix_methods);
565
566 static bool pmap_demote_l3e_locked(pmap_t pmap, pml3_entry_t *l3e, vm_offset_t va,
567 struct rwlock **lockp);
568 static bool pmap_demote_l3e(pmap_t pmap, pml3_entry_t *pde, vm_offset_t va);
569 static int pmap_unuse_pt(pmap_t, vm_offset_t, pml3_entry_t, struct spglist *);
570 static int pmap_remove_l3e(pmap_t pmap, pml3_entry_t *pdq, vm_offset_t sva,
571 struct spglist *free, struct rwlock **lockp);
572 static int pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t sva,
573 pml3_entry_t ptepde, struct spglist *free, struct rwlock **lockp);
574 static vm_page_t pmap_remove_pt_page(pmap_t pmap, vm_offset_t va);
575 static bool pmap_remove_page(pmap_t pmap, vm_offset_t va, pml3_entry_t *pde,
576 struct spglist *free);
577 static bool pmap_remove_ptes(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,
578 pml3_entry_t *l3e, struct spglist *free, struct rwlock **lockp);
579
580 static bool pmap_pv_insert_l3e(pmap_t pmap, vm_offset_t va, pml3_entry_t l3e,
581 u_int flags, struct rwlock **lockp);
582 #if VM_NRESERVLEVEL > 0
583 static void pmap_pv_promote_l3e(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,
584 struct rwlock **lockp);
585 #endif
586 static void pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va);
587 static int pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte);
588 static vm_page_t mmu_radix_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m,
589 vm_prot_t prot, vm_page_t mpte, struct rwlock **lockp, bool *invalidate);
590
591 static bool pmap_enter_2mpage(pmap_t pmap, vm_offset_t va, vm_page_t m,
592 vm_prot_t prot, struct rwlock **lockp);
593 static int pmap_enter_l3e(pmap_t pmap, vm_offset_t va, pml3_entry_t newpde,
594 u_int flags, vm_page_t m, struct rwlock **lockp);
595
596 static vm_page_t reclaim_pv_chunk(pmap_t locked_pmap, struct rwlock **lockp);
597 static void free_pv_chunk(struct pv_chunk *pc);
598 static vm_page_t _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, struct rwlock **lockp);
599 static vm_page_t pmap_allocl3e(pmap_t pmap, vm_offset_t va,
600 struct rwlock **lockp);
601 static vm_page_t pmap_allocpte(pmap_t pmap, vm_offset_t va,
602 struct rwlock **lockp);
603 static void _pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m,
604 struct spglist *free);
605 static bool pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free);
606
607 static void pmap_invalidate_page(pmap_t pmap, vm_offset_t start);
608 static void pmap_invalidate_all(pmap_t pmap);
609 static int pmap_change_attr_locked(void *va, vm_size_t size, int mode, bool flush);
610 static void pmap_fill_ptp(pt_entry_t *firstpte, pt_entry_t newpte);
611
612 /*
613 * Internal flags for pmap_enter()'s helper functions.
614 */
615 #define PMAP_ENTER_NORECLAIM 0x1000000 /* Don't reclaim PV entries. */
616 #define PMAP_ENTER_NOREPLACE 0x2000000 /* Don't replace mappings. */
617
618 #define UNIMPLEMENTED() panic("%s not implemented", __func__)
619 #define UNTESTED() panic("%s not yet tested", __func__)
620
621 /* Number of supported PID bits */
622 static unsigned int isa3_pid_bits;
623
624 /* PID to start allocating from */
625 static unsigned int isa3_base_pid;
626
627 #define PROCTAB_SIZE_SHIFT (isa3_pid_bits + 4)
628 #define PROCTAB_ENTRIES (1ul << isa3_pid_bits)
629
630 /*
631 * Map of physical memory regions.
632 */
633 static struct mem_region *regions, *pregions;
634 static struct numa_mem_region *numa_pregions;
635 static int regions_sz, pregions_sz, numa_pregions_sz;
636 static struct pate *isa3_parttab;
637 static struct prte *isa3_proctab;
638 static vmem_t *asid_arena;
639
640 extern void bs_remap_earlyboot(void);
641
642 #define RADIX_PGD_SIZE_SHIFT 16
643 #define RADIX_PGD_SIZE (1UL << RADIX_PGD_SIZE_SHIFT)
644
645 #define RADIX_PGD_INDEX_SHIFT (RADIX_PGD_SIZE_SHIFT-3)
646 #define NL2EPG (PAGE_SIZE/sizeof(pml2_entry_t))
647 #define NL3EPG (PAGE_SIZE/sizeof(pml3_entry_t))
648
649 #define NUPML1E (RADIX_PGD_SIZE/sizeof(uint64_t)) /* number of userland PML1 pages */
650 #define NUPDPE (NUPML1E * NL2EPG)/* number of userland PDP pages */
651 #define NUPDE (NUPDPE * NL3EPG) /* number of userland PD entries */
652
653 /* POWER9 only permits a 64k partition table size. */
654 #define PARTTAB_SIZE_SHIFT 16
655 #define PARTTAB_SIZE (1UL << PARTTAB_SIZE_SHIFT)
656
657 #define PARTTAB_HR (1UL << 63) /* host uses radix */
658 #define PARTTAB_GR (1UL << 63) /* guest uses radix must match host */
659
660 /* TLB flush actions. Used as argument to tlbiel_flush() */
661 enum {
662 TLB_INVAL_SCOPE_LPID = 2, /* invalidate TLBs for current LPID */
663 TLB_INVAL_SCOPE_GLOBAL = 3, /* invalidate all TLBs */
664 };
665
666 #define NPV_LIST_LOCKS MAXCPU
667 static int pmap_initialized;
668 static vm_paddr_t proctab0pa;
669 static vm_paddr_t parttab_phys;
670 CTASSERT(sizeof(struct pv_chunk) == PAGE_SIZE);
671
672 /*
673 * Data for the pv entry allocation mechanism.
674 * Updates to pv_invl_gen are protected by the pv_list_locks[]
675 * elements, but reads are not.
676 */
677 static TAILQ_HEAD(pch, pv_chunk) pv_chunks = TAILQ_HEAD_INITIALIZER(pv_chunks);
678 static struct mtx __exclusive_cache_line pv_chunks_mutex;
679 static struct rwlock __exclusive_cache_line pv_list_locks[NPV_LIST_LOCKS];
680 static struct md_page *pv_table;
681 static struct md_page pv_dummy;
682
683 #ifdef PV_STATS
684 #define PV_STAT(x) do { x ; } while (0)
685 #else
686 #define PV_STAT(x) do { } while (0)
687 #endif
688
689 #define pa_radix_index(pa) ((pa) >> L3_PAGE_SIZE_SHIFT)
690 #define pa_to_pvh(pa) (&pv_table[pa_radix_index(pa)])
691
692 #define PHYS_TO_PV_LIST_LOCK(pa) \
693 (&pv_list_locks[pa_radix_index(pa) % NPV_LIST_LOCKS])
694
695 #define CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa) do { \
696 struct rwlock **_lockp = (lockp); \
697 struct rwlock *_new_lock; \
698 \
699 _new_lock = PHYS_TO_PV_LIST_LOCK(pa); \
700 if (_new_lock != *_lockp) { \
701 if (*_lockp != NULL) \
702 rw_wunlock(*_lockp); \
703 *_lockp = _new_lock; \
704 rw_wlock(*_lockp); \
705 } \
706 } while (0)
707
708 #define CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m) \
709 CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, VM_PAGE_TO_PHYS(m))
710
711 #define RELEASE_PV_LIST_LOCK(lockp) do { \
712 struct rwlock **_lockp = (lockp); \
713 \
714 if (*_lockp != NULL) { \
715 rw_wunlock(*_lockp); \
716 *_lockp = NULL; \
717 } \
718 } while (0)
719
720 #define VM_PAGE_TO_PV_LIST_LOCK(m) \
721 PHYS_TO_PV_LIST_LOCK(VM_PAGE_TO_PHYS(m))
722
723 /*
724 * We support 52 bits, hence:
725 * bits 52 - 31 = 21, 0b10101
726 * RTS encoding details
727 * bits 0 - 3 of rts -> bits 6 - 8 unsigned long
728 * bits 4 - 5 of rts -> bits 62 - 63 of unsigned long
729 */
730 #define RTS_SIZE ((0x2UL << 61) | (0x5UL << 5))
731
732 static int powernv_enabled = 1;
733
734 static __always_inline void
tlbiel_radix_set_isa300(uint32_t set,uint32_t is,uint32_t pid,uint32_t ric,uint32_t prs)735 tlbiel_radix_set_isa300(uint32_t set, uint32_t is,
736 uint32_t pid, uint32_t ric, uint32_t prs)
737 {
738 uint64_t rb;
739 uint64_t rs;
740
741 rb = PPC_BITLSHIFT_VAL(set, 51) | PPC_BITLSHIFT_VAL(is, 53);
742 rs = PPC_BITLSHIFT_VAL((uint64_t)pid, 31);
743
744 __asm __volatile(PPC_TLBIEL(%0, %1, %2, %3, 1)
745 : : "r"(rb), "r"(rs), "i"(ric), "i"(prs)
746 : "memory");
747 }
748
749 static void
tlbiel_flush_isa3(uint32_t num_sets,uint32_t is)750 tlbiel_flush_isa3(uint32_t num_sets, uint32_t is)
751 {
752 uint32_t set;
753
754 __asm __volatile("ptesync": : :"memory");
755
756 /*
757 * Flush the first set of the TLB, and the entire Page Walk Cache
758 * and partition table entries. Then flush the remaining sets of the
759 * TLB.
760 */
761 if (is == TLB_INVAL_SCOPE_GLOBAL) {
762 tlbiel_radix_set_isa300(0, is, 0, RIC_FLUSH_ALL, 0);
763 for (set = 1; set < num_sets; set++)
764 tlbiel_radix_set_isa300(set, is, 0, RIC_FLUSH_TLB, 0);
765 }
766
767 /* Do the same for process scoped entries. */
768 tlbiel_radix_set_isa300(0, is, 0, RIC_FLUSH_ALL, 1);
769 for (set = 1; set < num_sets; set++)
770 tlbiel_radix_set_isa300(set, is, 0, RIC_FLUSH_TLB, 1);
771
772 __asm __volatile("ptesync": : :"memory");
773 }
774
775 static void
mmu_radix_tlbiel_flush(int scope)776 mmu_radix_tlbiel_flush(int scope)
777 {
778 MPASS(scope == TLB_INVAL_SCOPE_LPID ||
779 scope == TLB_INVAL_SCOPE_GLOBAL);
780
781 tlbiel_flush_isa3(POWER9_TLB_SETS_RADIX, scope);
782 __asm __volatile(PPC_INVALIDATE_ERAT "; isync" : : :"memory");
783 }
784
785 static void
mmu_radix_tlbie_all(void)786 mmu_radix_tlbie_all(void)
787 {
788 if (powernv_enabled)
789 mmu_radix_tlbiel_flush(TLB_INVAL_SCOPE_GLOBAL);
790 else
791 mmu_radix_tlbiel_flush(TLB_INVAL_SCOPE_LPID);
792 }
793
794 static void
mmu_radix_init_amor(void)795 mmu_radix_init_amor(void)
796 {
797 /*
798 * In HV mode, we init AMOR (Authority Mask Override Register) so that
799 * the hypervisor and guest can setup IAMR (Instruction Authority Mask
800 * Register), enable key 0 and set it to 1.
801 *
802 * AMOR = 0b1100 .... 0000 (Mask for key 0 is 11)
803 */
804 mtspr(SPR_AMOR, (3ul << 62));
805 }
806
807 static void
mmu_radix_init_iamr(void)808 mmu_radix_init_iamr(void)
809 {
810 /*
811 * Radix always uses key0 of the IAMR to determine if an access is
812 * allowed. We set bit 0 (IBM bit 1) of key0, to prevent instruction
813 * fetch.
814 */
815 mtspr(SPR_IAMR, (1ul << 62));
816 }
817
818 static void
mmu_radix_pid_set(pmap_t pmap)819 mmu_radix_pid_set(pmap_t pmap)
820 {
821
822 mtspr(SPR_PID, pmap->pm_pid);
823 isync();
824 }
825
826 /* Quick sort callout for comparing physical addresses. */
827 static int
pa_cmp(const void * a,const void * b)828 pa_cmp(const void *a, const void *b)
829 {
830 const vm_paddr_t *pa = a, *pb = b;
831
832 if (*pa < *pb)
833 return (-1);
834 else if (*pa > *pb)
835 return (1);
836 else
837 return (0);
838 }
839
840 #define pte_load_store(ptep, pte) atomic_swap_long(ptep, pte)
841 #define pte_load_clear(ptep) atomic_swap_long(ptep, 0)
842 #define pte_store(ptep, pte) do { \
843 MPASS((pte) & (RPTE_EAA_R | RPTE_EAA_W | RPTE_EAA_X)); \
844 *(u_long *)(ptep) = htobe64((u_long)((pte) | PG_V | RPTE_LEAF)); \
845 } while (0)
846 /*
847 * NB: should only be used for adding directories - not for direct mappings
848 */
849 #define pde_store(ptep, pa) do { \
850 *(u_long *)(ptep) = htobe64((u_long)(pa|RPTE_VALID|RPTE_SHIFT)); \
851 } while (0)
852
853 #define pte_clear(ptep) do { \
854 *(u_long *)(ptep) = (u_long)(0); \
855 } while (0)
856
857 #define PMAP_PDE_SUPERPAGE (1 << 8) /* supports 2MB superpages */
858
859 /*
860 * Promotion to a 2MB (PDE) page mapping requires that the corresponding 4KB
861 * (PTE) page mappings have identical settings for the following fields:
862 */
863 #define PG_PTE_PROMOTE (PG_X | PG_MANAGED | PG_W | PG_PTE_CACHE | \
864 PG_M | PG_A | RPTE_EAA_MASK | PG_V)
865
866 static __inline void
pmap_resident_count_inc(pmap_t pmap,int count)867 pmap_resident_count_inc(pmap_t pmap, int count)
868 {
869
870 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
871 pmap->pm_stats.resident_count += count;
872 }
873
874 static __inline void
pmap_resident_count_dec(pmap_t pmap,int count)875 pmap_resident_count_dec(pmap_t pmap, int count)
876 {
877
878 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
879 KASSERT(pmap->pm_stats.resident_count >= count,
880 ("pmap %p resident count underflow %ld %d", pmap,
881 pmap->pm_stats.resident_count, count));
882 pmap->pm_stats.resident_count -= count;
883 }
884
885 static void
pagezero(void * va)886 pagezero(void *va)
887 {
888 va = trunc_page(va);
889
890 bzero(va, PAGE_SIZE);
891 }
892
893 static uint64_t
allocpages(int n)894 allocpages(int n)
895 {
896 vm_paddr_t ret;
897
898 ret = moea64_bootstrap_alloc(n * PAGE_SIZE, PAGE_SIZE);
899 for (int i = 0; i < n; i++)
900 pagezero(PHYS_TO_DMAP(ret + i * PAGE_SIZE));
901 return (ret);
902 }
903
904 static pt_entry_t *
kvtopte(vm_offset_t va)905 kvtopte(vm_offset_t va)
906 {
907 pt_entry_t *l3e;
908
909 l3e = pmap_pml3e(kernel_pmap, va);
910 if (l3e == NULL || (be64toh(*l3e) & RPTE_VALID) == 0)
911 return (NULL);
912 return (pmap_l3e_to_pte(l3e, va));
913 }
914
915 void
mmu_radix_kenter(vm_offset_t va,vm_paddr_t pa)916 mmu_radix_kenter(vm_offset_t va, vm_paddr_t pa)
917 {
918 pt_entry_t *pte;
919
920 pte = kvtopte(va);
921 MPASS(pte != NULL);
922 *pte = htobe64(pa | RPTE_VALID | RPTE_LEAF | RPTE_EAA_R | \
923 RPTE_EAA_W | RPTE_EAA_P | PG_M | PG_A);
924 }
925
926 bool
mmu_radix_ps_enabled(pmap_t pmap)927 mmu_radix_ps_enabled(pmap_t pmap)
928 {
929 return (superpages_enabled && (pmap->pm_flags & PMAP_PDE_SUPERPAGE) != 0);
930 }
931
932 static pt_entry_t *
pmap_nofault_pte(pmap_t pmap,vm_offset_t va,int * is_l3e)933 pmap_nofault_pte(pmap_t pmap, vm_offset_t va, int *is_l3e)
934 {
935 pml3_entry_t *l3e;
936 pt_entry_t *pte;
937
938 va &= PG_PS_FRAME;
939 l3e = pmap_pml3e(pmap, va);
940 if (l3e == NULL || (be64toh(*l3e) & PG_V) == 0)
941 return (NULL);
942
943 if (be64toh(*l3e) & RPTE_LEAF) {
944 *is_l3e = 1;
945 return (l3e);
946 }
947 *is_l3e = 0;
948 va &= PG_FRAME;
949 pte = pmap_l3e_to_pte(l3e, va);
950 if (pte == NULL || (be64toh(*pte) & PG_V) == 0)
951 return (NULL);
952 return (pte);
953 }
954
955 int
pmap_nofault(pmap_t pmap,vm_offset_t va,vm_prot_t flags)956 pmap_nofault(pmap_t pmap, vm_offset_t va, vm_prot_t flags)
957 {
958 pt_entry_t *pte;
959 pt_entry_t startpte, origpte, newpte;
960 vm_page_t m;
961 int is_l3e;
962
963 startpte = 0;
964 retry:
965 if ((pte = pmap_nofault_pte(pmap, va, &is_l3e)) == NULL)
966 return (KERN_INVALID_ADDRESS);
967 origpte = newpte = be64toh(*pte);
968 if (startpte == 0) {
969 startpte = origpte;
970 if (((flags & VM_PROT_WRITE) && (startpte & PG_M)) ||
971 ((flags & VM_PROT_READ) && (startpte & PG_A))) {
972 pmap_invalidate_all(pmap);
973 #ifdef INVARIANTS
974 if (VERBOSE_PMAP || pmap_logging)
975 printf("%s(%p, %#lx, %#x) (%#lx) -- invalidate all\n",
976 __func__, pmap, va, flags, origpte);
977 #endif
978 return (KERN_FAILURE);
979 }
980 }
981 #ifdef INVARIANTS
982 if (VERBOSE_PMAP || pmap_logging)
983 printf("%s(%p, %#lx, %#x) (%#lx)\n", __func__, pmap, va,
984 flags, origpte);
985 #endif
986 PMAP_LOCK(pmap);
987 if ((pte = pmap_nofault_pte(pmap, va, &is_l3e)) == NULL ||
988 be64toh(*pte) != origpte) {
989 PMAP_UNLOCK(pmap);
990 return (KERN_FAILURE);
991 }
992 m = PHYS_TO_VM_PAGE(newpte & PG_FRAME);
993 MPASS(m != NULL);
994 switch (flags) {
995 case VM_PROT_READ:
996 if ((newpte & (RPTE_EAA_R|RPTE_EAA_X)) == 0)
997 goto protfail;
998 newpte |= PG_A;
999 vm_page_aflag_set(m, PGA_REFERENCED);
1000 break;
1001 case VM_PROT_WRITE:
1002 if ((newpte & RPTE_EAA_W) == 0)
1003 goto protfail;
1004 if (is_l3e)
1005 goto protfail;
1006 newpte |= PG_M;
1007 vm_page_dirty(m);
1008 break;
1009 case VM_PROT_EXECUTE:
1010 if ((newpte & RPTE_EAA_X) == 0)
1011 goto protfail;
1012 newpte |= PG_A;
1013 vm_page_aflag_set(m, PGA_REFERENCED);
1014 break;
1015 }
1016
1017 if (!atomic_cmpset_long(pte, htobe64(origpte), htobe64(newpte)))
1018 goto retry;
1019 ptesync();
1020 PMAP_UNLOCK(pmap);
1021 if (startpte == newpte)
1022 return (KERN_FAILURE);
1023 return (0);
1024 protfail:
1025 PMAP_UNLOCK(pmap);
1026 return (KERN_PROTECTION_FAILURE);
1027 }
1028
1029 /*
1030 * Returns true if the given page is mapped individually or as part of
1031 * a 2mpage. Otherwise, returns false.
1032 */
1033 bool
mmu_radix_page_is_mapped(vm_page_t m)1034 mmu_radix_page_is_mapped(vm_page_t m)
1035 {
1036 struct rwlock *lock;
1037 bool rv;
1038
1039 if ((m->oflags & VPO_UNMANAGED) != 0)
1040 return (false);
1041 lock = VM_PAGE_TO_PV_LIST_LOCK(m);
1042 rw_rlock(lock);
1043 rv = !TAILQ_EMPTY(&m->md.pv_list) ||
1044 ((m->flags & PG_FICTITIOUS) == 0 &&
1045 !TAILQ_EMPTY(&pa_to_pvh(VM_PAGE_TO_PHYS(m))->pv_list));
1046 rw_runlock(lock);
1047 return (rv);
1048 }
1049
1050 /*
1051 * Determine the appropriate bits to set in a PTE or PDE for a specified
1052 * caching mode.
1053 */
1054 static int
pmap_cache_bits(vm_memattr_t ma)1055 pmap_cache_bits(vm_memattr_t ma)
1056 {
1057 if (ma != VM_MEMATTR_DEFAULT) {
1058 switch (ma) {
1059 case VM_MEMATTR_UNCACHEABLE:
1060 return (RPTE_ATTR_GUARDEDIO);
1061 case VM_MEMATTR_CACHEABLE:
1062 return (RPTE_ATTR_MEM);
1063 case VM_MEMATTR_WRITE_BACK:
1064 case VM_MEMATTR_PREFETCHABLE:
1065 case VM_MEMATTR_WRITE_COMBINING:
1066 return (RPTE_ATTR_UNGUARDEDIO);
1067 }
1068 }
1069 return (0);
1070 }
1071
1072 static void
pmap_invalidate_page(pmap_t pmap,vm_offset_t start)1073 pmap_invalidate_page(pmap_t pmap, vm_offset_t start)
1074 {
1075 ptesync();
1076 if (pmap == kernel_pmap)
1077 radix_tlbie_invlpg_kernel_4k(start);
1078 else
1079 radix_tlbie_invlpg_user_4k(pmap->pm_pid, start);
1080 ttusync();
1081 }
1082
1083 static void
pmap_invalidate_page_2m(pmap_t pmap,vm_offset_t start)1084 pmap_invalidate_page_2m(pmap_t pmap, vm_offset_t start)
1085 {
1086 ptesync();
1087 if (pmap == kernel_pmap)
1088 radix_tlbie_invlpg_kernel_2m(start);
1089 else
1090 radix_tlbie_invlpg_user_2m(pmap->pm_pid, start);
1091 ttusync();
1092 }
1093
1094 static void
pmap_invalidate_pwc(pmap_t pmap)1095 pmap_invalidate_pwc(pmap_t pmap)
1096 {
1097 ptesync();
1098 if (pmap == kernel_pmap)
1099 radix_tlbie_invlpwc_kernel();
1100 else
1101 radix_tlbie_invlpwc_user(pmap->pm_pid);
1102 ttusync();
1103 }
1104
1105 static void
pmap_invalidate_range(pmap_t pmap,vm_offset_t start,vm_offset_t end)1106 pmap_invalidate_range(pmap_t pmap, vm_offset_t start, vm_offset_t end)
1107 {
1108 if (((start - end) >> PAGE_SHIFT) > 8) {
1109 pmap_invalidate_all(pmap);
1110 return;
1111 }
1112 ptesync();
1113 if (pmap == kernel_pmap) {
1114 while (start < end) {
1115 radix_tlbie_invlpg_kernel_4k(start);
1116 start += PAGE_SIZE;
1117 }
1118 } else {
1119 while (start < end) {
1120 radix_tlbie_invlpg_user_4k(pmap->pm_pid, start);
1121 start += PAGE_SIZE;
1122 }
1123 }
1124 ttusync();
1125 }
1126
1127 static void
pmap_invalidate_all(pmap_t pmap)1128 pmap_invalidate_all(pmap_t pmap)
1129 {
1130 ptesync();
1131 if (pmap == kernel_pmap)
1132 radix_tlbie_flush_kernel();
1133 else
1134 radix_tlbie_flush_user(pmap->pm_pid);
1135 ttusync();
1136 }
1137
1138 static void
pmap_invalidate_l3e_page(pmap_t pmap,vm_offset_t va,pml3_entry_t l3e)1139 pmap_invalidate_l3e_page(pmap_t pmap, vm_offset_t va, pml3_entry_t l3e)
1140 {
1141
1142 /*
1143 * When the PDE has PG_PROMOTED set, the 2MB page mapping was created
1144 * by a promotion that did not invalidate the 512 4KB page mappings
1145 * that might exist in the TLB. Consequently, at this point, the TLB
1146 * may hold both 4KB and 2MB page mappings for the address range [va,
1147 * va + L3_PAGE_SIZE). Therefore, the entire range must be invalidated here.
1148 * In contrast, when PG_PROMOTED is clear, the TLB will not hold any
1149 * 4KB page mappings for the address range [va, va + L3_PAGE_SIZE), and so a
1150 * single INVLPG suffices to invalidate the 2MB page mapping from the
1151 * TLB.
1152 */
1153 ptesync();
1154 if ((l3e & PG_PROMOTED) != 0)
1155 pmap_invalidate_range(pmap, va, va + L3_PAGE_SIZE - 1);
1156 else
1157 pmap_invalidate_page_2m(pmap, va);
1158
1159 pmap_invalidate_pwc(pmap);
1160 }
1161
1162 static __inline struct pv_chunk *
pv_to_chunk(pv_entry_t pv)1163 pv_to_chunk(pv_entry_t pv)
1164 {
1165
1166 return ((struct pv_chunk *)((uintptr_t)pv & ~(uintptr_t)PAGE_MASK));
1167 }
1168
1169 #define PV_PMAP(pv) (pv_to_chunk(pv)->pc_pmap)
1170
1171 #define PC_FREE0 0xfffffffffffffffful
1172 #define PC_FREE1 ((1ul << (_NPCPV % 64)) - 1)
1173
1174 static const uint64_t pc_freemask[_NPCM] = { PC_FREE0, PC_FREE1 };
1175
1176 /*
1177 * Ensure that the number of spare PV entries in the specified pmap meets or
1178 * exceeds the given count, "needed".
1179 *
1180 * The given PV list lock may be released.
1181 */
1182 static void
reserve_pv_entries(pmap_t pmap,int needed,struct rwlock ** lockp)1183 reserve_pv_entries(pmap_t pmap, int needed, struct rwlock **lockp)
1184 {
1185 struct pch new_tail;
1186 struct pv_chunk *pc;
1187 vm_page_t m;
1188 int avail, free;
1189 bool reclaimed;
1190
1191 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
1192 KASSERT(lockp != NULL, ("reserve_pv_entries: lockp is NULL"));
1193
1194 /*
1195 * Newly allocated PV chunks must be stored in a private list until
1196 * the required number of PV chunks have been allocated. Otherwise,
1197 * reclaim_pv_chunk() could recycle one of these chunks. In
1198 * contrast, these chunks must be added to the pmap upon allocation.
1199 */
1200 TAILQ_INIT(&new_tail);
1201 retry:
1202 avail = 0;
1203 TAILQ_FOREACH(pc, &pmap->pm_pvchunk, pc_list) {
1204 // if ((cpu_feature2 & CPUID2_POPCNT) == 0)
1205 bit_count((bitstr_t *)pc->pc_map, 0,
1206 sizeof(pc->pc_map) * NBBY, &free);
1207 #if 0
1208 free = popcnt_pc_map_pq(pc->pc_map);
1209 #endif
1210 if (free == 0)
1211 break;
1212 avail += free;
1213 if (avail >= needed)
1214 break;
1215 }
1216 for (reclaimed = false; avail < needed; avail += _NPCPV) {
1217 m = vm_page_alloc_noobj(VM_ALLOC_WIRED);
1218 if (m == NULL) {
1219 m = reclaim_pv_chunk(pmap, lockp);
1220 if (m == NULL)
1221 goto retry;
1222 reclaimed = true;
1223 }
1224 PV_STAT(atomic_add_int(&pc_chunk_count, 1));
1225 PV_STAT(atomic_add_int(&pc_chunk_allocs, 1));
1226 dump_add_page(m->phys_addr);
1227 pc = VM_PAGE_TO_DMAP(m);
1228 pc->pc_pmap = pmap;
1229 pc->pc_map[0] = PC_FREE0;
1230 pc->pc_map[1] = PC_FREE1;
1231 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);
1232 TAILQ_INSERT_TAIL(&new_tail, pc, pc_lru);
1233 PV_STAT(atomic_add_int(&pv_entry_spare, _NPCPV));
1234
1235 /*
1236 * The reclaim might have freed a chunk from the current pmap.
1237 * If that chunk contained available entries, we need to
1238 * re-count the number of available entries.
1239 */
1240 if (reclaimed)
1241 goto retry;
1242 }
1243 if (!TAILQ_EMPTY(&new_tail)) {
1244 mtx_lock(&pv_chunks_mutex);
1245 TAILQ_CONCAT(&pv_chunks, &new_tail, pc_lru);
1246 mtx_unlock(&pv_chunks_mutex);
1247 }
1248 }
1249
1250 /*
1251 * First find and then remove the pv entry for the specified pmap and virtual
1252 * address from the specified pv list. Returns the pv entry if found and NULL
1253 * otherwise. This operation can be performed on pv lists for either 4KB or
1254 * 2MB page mappings.
1255 */
1256 static __inline pv_entry_t
pmap_pvh_remove(struct md_page * pvh,pmap_t pmap,vm_offset_t va)1257 pmap_pvh_remove(struct md_page *pvh, pmap_t pmap, vm_offset_t va)
1258 {
1259 pv_entry_t pv;
1260
1261 TAILQ_FOREACH(pv, &pvh->pv_list, pv_link) {
1262 #ifdef INVARIANTS
1263 if (PV_PMAP(pv) == NULL) {
1264 printf("corrupted pv_chunk/pv %p\n", pv);
1265 printf("pv_chunk: %64D\n", pv_to_chunk(pv), ":");
1266 }
1267 MPASS(PV_PMAP(pv) != NULL);
1268 MPASS(pv->pv_va != 0);
1269 #endif
1270 if (pmap == PV_PMAP(pv) && va == pv->pv_va) {
1271 TAILQ_REMOVE(&pvh->pv_list, pv, pv_link);
1272 pvh->pv_gen++;
1273 break;
1274 }
1275 }
1276 return (pv);
1277 }
1278
1279 /*
1280 * After demotion from a 2MB page mapping to 512 4KB page mappings,
1281 * destroy the pv entry for the 2MB page mapping and reinstantiate the pv
1282 * entries for each of the 4KB page mappings.
1283 */
1284 static void
pmap_pv_demote_l3e(pmap_t pmap,vm_offset_t va,vm_paddr_t pa,struct rwlock ** lockp)1285 pmap_pv_demote_l3e(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,
1286 struct rwlock **lockp)
1287 {
1288 struct md_page *pvh;
1289 struct pv_chunk *pc;
1290 pv_entry_t pv;
1291 vm_offset_t va_last;
1292 vm_page_t m;
1293 int bit, field;
1294
1295 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
1296 KASSERT((pa & L3_PAGE_MASK) == 0,
1297 ("pmap_pv_demote_pde: pa is not 2mpage aligned"));
1298 CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa);
1299
1300 /*
1301 * Transfer the 2mpage's pv entry for this mapping to the first
1302 * page's pv list. Once this transfer begins, the pv list lock
1303 * must not be released until the last pv entry is reinstantiated.
1304 */
1305 pvh = pa_to_pvh(pa);
1306 va = trunc_2mpage(va);
1307 pv = pmap_pvh_remove(pvh, pmap, va);
1308 KASSERT(pv != NULL, ("pmap_pv_demote_pde: pv not found"));
1309 m = PHYS_TO_VM_PAGE(pa);
1310 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_link);
1311
1312 m->md.pv_gen++;
1313 /* Instantiate the remaining NPTEPG - 1 pv entries. */
1314 PV_STAT(atomic_add_long(&pv_entry_allocs, NPTEPG - 1));
1315 va_last = va + L3_PAGE_SIZE - PAGE_SIZE;
1316 for (;;) {
1317 pc = TAILQ_FIRST(&pmap->pm_pvchunk);
1318 KASSERT(pc->pc_map[0] != 0 || pc->pc_map[1] != 0
1319 , ("pmap_pv_demote_pde: missing spare"));
1320 for (field = 0; field < _NPCM; field++) {
1321 while (pc->pc_map[field]) {
1322 bit = cnttzd(pc->pc_map[field]);
1323 pc->pc_map[field] &= ~(1ul << bit);
1324 pv = &pc->pc_pventry[field * 64 + bit];
1325 va += PAGE_SIZE;
1326 pv->pv_va = va;
1327 m++;
1328 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
1329 ("pmap_pv_demote_pde: page %p is not managed", m));
1330 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_link);
1331
1332 m->md.pv_gen++;
1333 if (va == va_last)
1334 goto out;
1335 }
1336 }
1337 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
1338 TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list);
1339 }
1340 out:
1341 if (pc->pc_map[0] == 0 && pc->pc_map[1] == 0) {
1342 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
1343 TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list);
1344 }
1345 PV_STAT(atomic_add_long(&pv_entry_count, NPTEPG - 1));
1346 PV_STAT(atomic_subtract_int(&pv_entry_spare, NPTEPG - 1));
1347 }
1348
1349 static void
reclaim_pv_chunk_leave_pmap(pmap_t pmap,pmap_t locked_pmap)1350 reclaim_pv_chunk_leave_pmap(pmap_t pmap, pmap_t locked_pmap)
1351 {
1352
1353 if (pmap == NULL)
1354 return;
1355 pmap_invalidate_all(pmap);
1356 if (pmap != locked_pmap)
1357 PMAP_UNLOCK(pmap);
1358 }
1359
1360 /*
1361 * We are in a serious low memory condition. Resort to
1362 * drastic measures to free some pages so we can allocate
1363 * another pv entry chunk.
1364 *
1365 * Returns NULL if PV entries were reclaimed from the specified pmap.
1366 *
1367 * We do not, however, unmap 2mpages because subsequent accesses will
1368 * allocate per-page pv entries until repromotion occurs, thereby
1369 * exacerbating the shortage of free pv entries.
1370 */
1371 static int active_reclaims = 0;
1372 static vm_page_t
reclaim_pv_chunk(pmap_t locked_pmap,struct rwlock ** lockp)1373 reclaim_pv_chunk(pmap_t locked_pmap, struct rwlock **lockp)
1374 {
1375 struct pv_chunk *pc, *pc_marker, *pc_marker_end;
1376 struct pv_chunk_header pc_marker_b, pc_marker_end_b;
1377 struct md_page *pvh;
1378 pml3_entry_t *l3e;
1379 pmap_t next_pmap, pmap;
1380 pt_entry_t *pte, tpte;
1381 pv_entry_t pv;
1382 vm_offset_t va;
1383 vm_page_t m, m_pc;
1384 struct spglist free;
1385 uint64_t inuse;
1386 int bit, field, freed;
1387
1388 PMAP_LOCK_ASSERT(locked_pmap, MA_OWNED);
1389 KASSERT(lockp != NULL, ("reclaim_pv_chunk: lockp is NULL"));
1390 pmap = NULL;
1391 m_pc = NULL;
1392 SLIST_INIT(&free);
1393 bzero(&pc_marker_b, sizeof(pc_marker_b));
1394 bzero(&pc_marker_end_b, sizeof(pc_marker_end_b));
1395 pc_marker = (struct pv_chunk *)&pc_marker_b;
1396 pc_marker_end = (struct pv_chunk *)&pc_marker_end_b;
1397
1398 mtx_lock(&pv_chunks_mutex);
1399 active_reclaims++;
1400 TAILQ_INSERT_HEAD(&pv_chunks, pc_marker, pc_lru);
1401 TAILQ_INSERT_TAIL(&pv_chunks, pc_marker_end, pc_lru);
1402 while ((pc = TAILQ_NEXT(pc_marker, pc_lru)) != pc_marker_end &&
1403 SLIST_EMPTY(&free)) {
1404 next_pmap = pc->pc_pmap;
1405 if (next_pmap == NULL) {
1406 /*
1407 * The next chunk is a marker. However, it is
1408 * not our marker, so active_reclaims must be
1409 * > 1. Consequently, the next_chunk code
1410 * will not rotate the pv_chunks list.
1411 */
1412 goto next_chunk;
1413 }
1414 mtx_unlock(&pv_chunks_mutex);
1415
1416 /*
1417 * A pv_chunk can only be removed from the pc_lru list
1418 * when both pc_chunks_mutex is owned and the
1419 * corresponding pmap is locked.
1420 */
1421 if (pmap != next_pmap) {
1422 reclaim_pv_chunk_leave_pmap(pmap, locked_pmap);
1423 pmap = next_pmap;
1424 /* Avoid deadlock and lock recursion. */
1425 if (pmap > locked_pmap) {
1426 RELEASE_PV_LIST_LOCK(lockp);
1427 PMAP_LOCK(pmap);
1428 mtx_lock(&pv_chunks_mutex);
1429 continue;
1430 } else if (pmap != locked_pmap) {
1431 if (PMAP_TRYLOCK(pmap)) {
1432 mtx_lock(&pv_chunks_mutex);
1433 continue;
1434 } else {
1435 pmap = NULL; /* pmap is not locked */
1436 mtx_lock(&pv_chunks_mutex);
1437 pc = TAILQ_NEXT(pc_marker, pc_lru);
1438 if (pc == NULL ||
1439 pc->pc_pmap != next_pmap)
1440 continue;
1441 goto next_chunk;
1442 }
1443 }
1444 }
1445
1446 /*
1447 * Destroy every non-wired, 4 KB page mapping in the chunk.
1448 */
1449 freed = 0;
1450 for (field = 0; field < _NPCM; field++) {
1451 for (inuse = ~pc->pc_map[field] & pc_freemask[field];
1452 inuse != 0; inuse &= ~(1UL << bit)) {
1453 bit = cnttzd(inuse);
1454 pv = &pc->pc_pventry[field * 64 + bit];
1455 va = pv->pv_va;
1456 l3e = pmap_pml3e(pmap, va);
1457 if ((be64toh(*l3e) & RPTE_LEAF) != 0)
1458 continue;
1459 pte = pmap_l3e_to_pte(l3e, va);
1460 if ((be64toh(*pte) & PG_W) != 0)
1461 continue;
1462 tpte = be64toh(pte_load_clear(pte));
1463 m = PHYS_TO_VM_PAGE(tpte & PG_FRAME);
1464 if ((tpte & (PG_M | PG_RW)) == (PG_M | PG_RW))
1465 vm_page_dirty(m);
1466 if ((tpte & PG_A) != 0)
1467 vm_page_aflag_set(m, PGA_REFERENCED);
1468 CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m);
1469 TAILQ_REMOVE(&m->md.pv_list, pv, pv_link);
1470
1471 m->md.pv_gen++;
1472 if (TAILQ_EMPTY(&m->md.pv_list) &&
1473 (m->flags & PG_FICTITIOUS) == 0) {
1474 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
1475 if (TAILQ_EMPTY(&pvh->pv_list)) {
1476 vm_page_aflag_clear(m,
1477 PGA_WRITEABLE);
1478 }
1479 }
1480 pc->pc_map[field] |= 1UL << bit;
1481 pmap_unuse_pt(pmap, va, be64toh(*l3e), &free);
1482 freed++;
1483 }
1484 }
1485 if (freed == 0) {
1486 mtx_lock(&pv_chunks_mutex);
1487 goto next_chunk;
1488 }
1489 /* Every freed mapping is for a 4 KB page. */
1490 pmap_resident_count_dec(pmap, freed);
1491 PV_STAT(atomic_add_long(&pv_entry_frees, freed));
1492 PV_STAT(atomic_add_int(&pv_entry_spare, freed));
1493 PV_STAT(atomic_subtract_long(&pv_entry_count, freed));
1494 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
1495 if (pc->pc_map[0] == PC_FREE0 && pc->pc_map[1] == PC_FREE1) {
1496 PV_STAT(atomic_subtract_int(&pv_entry_spare, _NPCPV));
1497 PV_STAT(atomic_subtract_int(&pc_chunk_count, 1));
1498 PV_STAT(atomic_add_int(&pc_chunk_frees, 1));
1499 /* Entire chunk is free; return it. */
1500 m_pc = DMAP_TO_VM_PAGE(pc);
1501 dump_drop_page(m_pc->phys_addr);
1502 mtx_lock(&pv_chunks_mutex);
1503 TAILQ_REMOVE(&pv_chunks, pc, pc_lru);
1504 break;
1505 }
1506 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);
1507 mtx_lock(&pv_chunks_mutex);
1508 /* One freed pv entry in locked_pmap is sufficient. */
1509 if (pmap == locked_pmap)
1510 break;
1511 next_chunk:
1512 TAILQ_REMOVE(&pv_chunks, pc_marker, pc_lru);
1513 TAILQ_INSERT_AFTER(&pv_chunks, pc, pc_marker, pc_lru);
1514 if (active_reclaims == 1 && pmap != NULL) {
1515 /*
1516 * Rotate the pv chunks list so that we do not
1517 * scan the same pv chunks that could not be
1518 * freed (because they contained a wired
1519 * and/or superpage mapping) on every
1520 * invocation of reclaim_pv_chunk().
1521 */
1522 while ((pc = TAILQ_FIRST(&pv_chunks)) != pc_marker) {
1523 MPASS(pc->pc_pmap != NULL);
1524 TAILQ_REMOVE(&pv_chunks, pc, pc_lru);
1525 TAILQ_INSERT_TAIL(&pv_chunks, pc, pc_lru);
1526 }
1527 }
1528 }
1529 TAILQ_REMOVE(&pv_chunks, pc_marker, pc_lru);
1530 TAILQ_REMOVE(&pv_chunks, pc_marker_end, pc_lru);
1531 active_reclaims--;
1532 mtx_unlock(&pv_chunks_mutex);
1533 reclaim_pv_chunk_leave_pmap(pmap, locked_pmap);
1534 if (m_pc == NULL && !SLIST_EMPTY(&free)) {
1535 m_pc = SLIST_FIRST(&free);
1536 SLIST_REMOVE_HEAD(&free, plinks.s.ss);
1537 /* Recycle a freed page table page. */
1538 m_pc->ref_count = 1;
1539 }
1540 vm_page_free_pages_toq(&free, true);
1541 return (m_pc);
1542 }
1543
1544 /*
1545 * free the pv_entry back to the free list
1546 */
1547 static void
free_pv_entry(pmap_t pmap,pv_entry_t pv)1548 free_pv_entry(pmap_t pmap, pv_entry_t pv)
1549 {
1550 struct pv_chunk *pc;
1551 int idx, field, bit;
1552
1553 #ifdef VERBOSE_PV
1554 if (pmap != kernel_pmap)
1555 printf("%s(%p, %p)\n", __func__, pmap, pv);
1556 #endif
1557 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
1558 PV_STAT(atomic_add_long(&pv_entry_frees, 1));
1559 PV_STAT(atomic_add_int(&pv_entry_spare, 1));
1560 PV_STAT(atomic_subtract_long(&pv_entry_count, 1));
1561 pc = pv_to_chunk(pv);
1562 idx = pv - &pc->pc_pventry[0];
1563 field = idx / 64;
1564 bit = idx % 64;
1565 pc->pc_map[field] |= 1ul << bit;
1566 if (pc->pc_map[0] != PC_FREE0 || pc->pc_map[1] != PC_FREE1) {
1567 /* 98% of the time, pc is already at the head of the list. */
1568 if (__predict_false(pc != TAILQ_FIRST(&pmap->pm_pvchunk))) {
1569 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
1570 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);
1571 }
1572 return;
1573 }
1574 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
1575 free_pv_chunk(pc);
1576 }
1577
1578 static void
free_pv_chunk(struct pv_chunk * pc)1579 free_pv_chunk(struct pv_chunk *pc)
1580 {
1581 vm_page_t m;
1582
1583 mtx_lock(&pv_chunks_mutex);
1584 TAILQ_REMOVE(&pv_chunks, pc, pc_lru);
1585 mtx_unlock(&pv_chunks_mutex);
1586 PV_STAT(atomic_subtract_int(&pv_entry_spare, _NPCPV));
1587 PV_STAT(atomic_subtract_int(&pc_chunk_count, 1));
1588 PV_STAT(atomic_add_int(&pc_chunk_frees, 1));
1589 /* entire chunk is free, return it */
1590 m = DMAP_TO_VM_PAGE(pc);
1591 dump_drop_page(m->phys_addr);
1592 vm_page_unwire_noq(m);
1593 vm_page_free(m);
1594 }
1595
1596 /*
1597 * Returns a new PV entry, allocating a new PV chunk from the system when
1598 * needed. If this PV chunk allocation fails and a PV list lock pointer was
1599 * given, a PV chunk is reclaimed from an arbitrary pmap. Otherwise, NULL is
1600 * returned.
1601 *
1602 * The given PV list lock may be released.
1603 */
1604 static pv_entry_t
get_pv_entry(pmap_t pmap,struct rwlock ** lockp)1605 get_pv_entry(pmap_t pmap, struct rwlock **lockp)
1606 {
1607 int bit, field;
1608 pv_entry_t pv;
1609 struct pv_chunk *pc;
1610 vm_page_t m;
1611
1612 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
1613 PV_STAT(atomic_add_long(&pv_entry_allocs, 1));
1614 retry:
1615 pc = TAILQ_FIRST(&pmap->pm_pvchunk);
1616 if (pc != NULL) {
1617 for (field = 0; field < _NPCM; field++) {
1618 if (pc->pc_map[field]) {
1619 bit = cnttzd(pc->pc_map[field]);
1620 break;
1621 }
1622 }
1623 if (field < _NPCM) {
1624 pv = &pc->pc_pventry[field * 64 + bit];
1625 pc->pc_map[field] &= ~(1ul << bit);
1626 /* If this was the last item, move it to tail */
1627 if (pc->pc_map[0] == 0 && pc->pc_map[1] == 0) {
1628 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
1629 TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc,
1630 pc_list);
1631 }
1632 PV_STAT(atomic_add_long(&pv_entry_count, 1));
1633 PV_STAT(atomic_subtract_int(&pv_entry_spare, 1));
1634 MPASS(PV_PMAP(pv) != NULL);
1635 return (pv);
1636 }
1637 }
1638 /* No free items, allocate another chunk */
1639 m = vm_page_alloc_noobj(VM_ALLOC_WIRED);
1640 if (m == NULL) {
1641 if (lockp == NULL) {
1642 PV_STAT(pc_chunk_tryfail++);
1643 return (NULL);
1644 }
1645 m = reclaim_pv_chunk(pmap, lockp);
1646 if (m == NULL)
1647 goto retry;
1648 }
1649 PV_STAT(atomic_add_int(&pc_chunk_count, 1));
1650 PV_STAT(atomic_add_int(&pc_chunk_allocs, 1));
1651 dump_add_page(m->phys_addr);
1652 pc = VM_PAGE_TO_DMAP(m);
1653 pc->pc_pmap = pmap;
1654 pc->pc_map[0] = PC_FREE0 & ~1ul; /* preallocated bit 0 */
1655 pc->pc_map[1] = PC_FREE1;
1656 mtx_lock(&pv_chunks_mutex);
1657 TAILQ_INSERT_TAIL(&pv_chunks, pc, pc_lru);
1658 mtx_unlock(&pv_chunks_mutex);
1659 pv = &pc->pc_pventry[0];
1660 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);
1661 PV_STAT(atomic_add_long(&pv_entry_count, 1));
1662 PV_STAT(atomic_add_int(&pv_entry_spare, _NPCPV - 1));
1663 MPASS(PV_PMAP(pv) != NULL);
1664 return (pv);
1665 }
1666
1667 #if VM_NRESERVLEVEL > 0
1668 /*
1669 * After promotion from 512 4KB page mappings to a single 2MB page mapping,
1670 * replace the many pv entries for the 4KB page mappings by a single pv entry
1671 * for the 2MB page mapping.
1672 */
1673 static void
pmap_pv_promote_l3e(pmap_t pmap,vm_offset_t va,vm_paddr_t pa,struct rwlock ** lockp)1674 pmap_pv_promote_l3e(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,
1675 struct rwlock **lockp)
1676 {
1677 struct md_page *pvh;
1678 pv_entry_t pv;
1679 vm_offset_t va_last;
1680 vm_page_t m;
1681
1682 KASSERT((pa & L3_PAGE_MASK) == 0,
1683 ("pmap_pv_promote_pde: pa is not 2mpage aligned"));
1684 CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa);
1685
1686 /*
1687 * Transfer the first page's pv entry for this mapping to the 2mpage's
1688 * pv list. Aside from avoiding the cost of a call to get_pv_entry(),
1689 * a transfer avoids the possibility that get_pv_entry() calls
1690 * reclaim_pv_chunk() and that reclaim_pv_chunk() removes one of the
1691 * mappings that is being promoted.
1692 */
1693 m = PHYS_TO_VM_PAGE(pa);
1694 va = trunc_2mpage(va);
1695 pv = pmap_pvh_remove(&m->md, pmap, va);
1696 KASSERT(pv != NULL, ("pmap_pv_promote_pde: pv not found"));
1697 pvh = pa_to_pvh(pa);
1698 TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_link);
1699 pvh->pv_gen++;
1700 /* Free the remaining NPTEPG - 1 pv entries. */
1701 va_last = va + L3_PAGE_SIZE - PAGE_SIZE;
1702 do {
1703 m++;
1704 va += PAGE_SIZE;
1705 pmap_pvh_free(&m->md, pmap, va);
1706 } while (va < va_last);
1707 }
1708 #endif /* VM_NRESERVLEVEL > 0 */
1709
1710 /*
1711 * First find and then destroy the pv entry for the specified pmap and virtual
1712 * address. This operation can be performed on pv lists for either 4KB or 2MB
1713 * page mappings.
1714 */
1715 static void
pmap_pvh_free(struct md_page * pvh,pmap_t pmap,vm_offset_t va)1716 pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va)
1717 {
1718 pv_entry_t pv;
1719
1720 pv = pmap_pvh_remove(pvh, pmap, va);
1721 KASSERT(pv != NULL, ("pmap_pvh_free: pv not found"));
1722 free_pv_entry(pmap, pv);
1723 }
1724
1725 /*
1726 * Conditionally create the PV entry for a 4KB page mapping if the required
1727 * memory can be allocated without resorting to reclamation.
1728 */
1729 static bool
pmap_try_insert_pv_entry(pmap_t pmap,vm_offset_t va,vm_page_t m,struct rwlock ** lockp)1730 pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, vm_page_t m,
1731 struct rwlock **lockp)
1732 {
1733 pv_entry_t pv;
1734
1735 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
1736 /* Pass NULL instead of the lock pointer to disable reclamation. */
1737 if ((pv = get_pv_entry(pmap, NULL)) != NULL) {
1738 pv->pv_va = va;
1739 CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m);
1740 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_link);
1741 m->md.pv_gen++;
1742 return (true);
1743 } else
1744 return (false);
1745 }
1746
1747 vm_paddr_t phys_avail_debug[2 * VM_PHYSSEG_MAX];
1748 #ifdef INVARIANTS
1749 static void
validate_addr(vm_paddr_t addr,vm_size_t size)1750 validate_addr(vm_paddr_t addr, vm_size_t size)
1751 {
1752 vm_paddr_t end = addr + size;
1753 bool found = false;
1754
1755 for (int i = 0; i < 2 * phys_avail_count; i += 2) {
1756 if (addr >= phys_avail_debug[i] &&
1757 end <= phys_avail_debug[i + 1]) {
1758 found = true;
1759 break;
1760 }
1761 }
1762 KASSERT(found, ("%#lx-%#lx outside of initial phys_avail array",
1763 addr, end));
1764 }
1765 #else
validate_addr(vm_paddr_t addr,vm_size_t size)1766 static void validate_addr(vm_paddr_t addr, vm_size_t size) {}
1767 #endif
1768 #define DMAP_PAGE_BITS (RPTE_VALID | RPTE_LEAF | RPTE_EAA_MASK | PG_M | PG_A)
1769
1770 static vm_paddr_t
alloc_pt_page(void)1771 alloc_pt_page(void)
1772 {
1773 vm_paddr_t page;
1774
1775 page = allocpages(1);
1776 pagezero(PHYS_TO_DMAP(page));
1777 return (page);
1778 }
1779
1780 static void
mmu_radix_dmap_range(vm_paddr_t start,vm_paddr_t end)1781 mmu_radix_dmap_range(vm_paddr_t start, vm_paddr_t end)
1782 {
1783 pt_entry_t *pte, pteval;
1784 vm_paddr_t page;
1785
1786 if (bootverbose)
1787 printf("%s %lx -> %lx\n", __func__, start, end);
1788 while (start < end) {
1789 pteval = start | DMAP_PAGE_BITS;
1790 pte = pmap_pml1e(kernel_pmap, PHYS_TO_DMAP_ADDR(start));
1791 if ((be64toh(*pte) & RPTE_VALID) == 0) {
1792 page = alloc_pt_page();
1793 pde_store(pte, page);
1794 }
1795 pte = pmap_l1e_to_l2e(pte, PHYS_TO_DMAP_ADDR(start));
1796 if ((start & L2_PAGE_MASK) == 0 &&
1797 end - start >= L2_PAGE_SIZE) {
1798 start += L2_PAGE_SIZE;
1799 goto done;
1800 } else if ((be64toh(*pte) & RPTE_VALID) == 0) {
1801 page = alloc_pt_page();
1802 pde_store(pte, page);
1803 }
1804
1805 pte = pmap_l2e_to_l3e(pte, PHYS_TO_DMAP_ADDR(start));
1806 if ((start & L3_PAGE_MASK) == 0 &&
1807 end - start >= L3_PAGE_SIZE) {
1808 start += L3_PAGE_SIZE;
1809 goto done;
1810 } else if ((be64toh(*pte) & RPTE_VALID) == 0) {
1811 page = alloc_pt_page();
1812 pde_store(pte, page);
1813 }
1814 pte = pmap_l3e_to_pte(pte, PHYS_TO_DMAP_ADDR(start));
1815 start += PAGE_SIZE;
1816 done:
1817 pte_store(pte, pteval);
1818 }
1819 }
1820
1821 static void
mmu_radix_dmap_populate(vm_size_t hwphyssz)1822 mmu_radix_dmap_populate(vm_size_t hwphyssz)
1823 {
1824 vm_paddr_t start, end;
1825
1826 for (int i = 0; i < pregions_sz; i++) {
1827 start = pregions[i].mr_start;
1828 end = start + pregions[i].mr_size;
1829 if (hwphyssz && start >= hwphyssz)
1830 break;
1831 if (hwphyssz && hwphyssz < end)
1832 end = hwphyssz;
1833 mmu_radix_dmap_range(start, end);
1834 }
1835 }
1836
1837 static void
mmu_radix_setup_pagetables(vm_size_t hwphyssz)1838 mmu_radix_setup_pagetables(vm_size_t hwphyssz)
1839 {
1840 vm_paddr_t ptpages, pages;
1841 pt_entry_t *pte;
1842 vm_paddr_t l1phys;
1843
1844 bzero(kernel_pmap, sizeof(struct pmap));
1845 mtx_init(&kernel_pmap->pm_mtx, "kernel pmap", NULL, MTX_DEF);
1846 vm_radix_init(&kernel_pmap->pm_radix);
1847
1848 ptpages = allocpages(3);
1849 l1phys = moea64_bootstrap_alloc(RADIX_PGD_SIZE, RADIX_PGD_SIZE);
1850 validate_addr(l1phys, RADIX_PGD_SIZE);
1851 if (bootverbose)
1852 printf("l1phys=%lx\n", l1phys);
1853 MPASS((l1phys & (RADIX_PGD_SIZE-1)) == 0);
1854 for (int i = 0; i < RADIX_PGD_SIZE/PAGE_SIZE; i++)
1855 pagezero(PHYS_TO_DMAP(l1phys + i * PAGE_SIZE));
1856 kernel_pmap->pm_pml1 = PHYS_TO_DMAP(l1phys);
1857
1858 mmu_radix_dmap_populate(hwphyssz);
1859
1860 /*
1861 * Create page tables for first 128MB of KVA
1862 */
1863 pages = ptpages;
1864 pte = pmap_pml1e(kernel_pmap, VM_MIN_KERNEL_ADDRESS);
1865 *pte = htobe64(pages | RPTE_VALID | RPTE_SHIFT);
1866 pages += PAGE_SIZE;
1867 pte = pmap_l1e_to_l2e(pte, VM_MIN_KERNEL_ADDRESS);
1868 *pte = htobe64(pages | RPTE_VALID | RPTE_SHIFT);
1869 pages += PAGE_SIZE;
1870 pte = pmap_l2e_to_l3e(pte, VM_MIN_KERNEL_ADDRESS);
1871 /*
1872 * the kernel page table pages need to be preserved in
1873 * phys_avail and not overlap with previous allocations
1874 */
1875 pages = allocpages(nkpt);
1876 if (bootverbose) {
1877 printf("phys_avail after dmap populate and nkpt allocation\n");
1878 for (int j = 0; j < 2 * phys_avail_count; j+=2)
1879 printf("phys_avail[%d]=%08lx - phys_avail[%d]=%08lx\n",
1880 j, phys_avail[j], j + 1, phys_avail[j + 1]);
1881 }
1882 KPTphys = pages;
1883 for (int i = 0; i < nkpt; i++, pte++, pages += PAGE_SIZE)
1884 *pte = htobe64(pages | RPTE_VALID | RPTE_SHIFT);
1885 kernel_vm_end = VM_MIN_KERNEL_ADDRESS + nkpt * L3_PAGE_SIZE;
1886 if (bootverbose)
1887 printf("kernel_pmap pml1 %p\n", kernel_pmap->pm_pml1);
1888 /*
1889 * Add a physical memory segment (vm_phys_seg) corresponding to the
1890 * preallocated kernel page table pages so that vm_page structures
1891 * representing these pages will be created. The vm_page structures
1892 * are required for promotion of the corresponding kernel virtual
1893 * addresses to superpage mappings.
1894 */
1895 vm_phys_add_seg(KPTphys, KPTphys + ptoa(nkpt));
1896 }
1897
1898 static void
mmu_radix_early_bootstrap(vm_offset_t start,vm_offset_t end)1899 mmu_radix_early_bootstrap(vm_offset_t start, vm_offset_t end)
1900 {
1901 vm_paddr_t kpstart, kpend;
1902 vm_size_t physsz, hwphyssz;
1903 //uint64_t l2virt;
1904 int rm_pavail, proctab_size;
1905 int i, j;
1906
1907 kpstart = start & ~DMAP_BASE_ADDRESS;
1908 kpend = end & ~DMAP_BASE_ADDRESS;
1909
1910 /* Get physical memory regions from firmware */
1911 mem_regions(&pregions, &pregions_sz, ®ions, ®ions_sz);
1912 CTR0(KTR_PMAP, "mmu_radix_early_bootstrap: physical memory");
1913
1914 if (2 * VM_PHYSSEG_MAX < regions_sz)
1915 panic("mmu_radix_early_bootstrap: phys_avail too small");
1916
1917 if (bootverbose)
1918 for (int i = 0; i < regions_sz; i++)
1919 printf("regions[%d].mr_start=%lx regions[%d].mr_size=%lx\n",
1920 i, regions[i].mr_start, i, regions[i].mr_size);
1921 /*
1922 * XXX workaround a simulator bug
1923 */
1924 for (int i = 0; i < regions_sz; i++)
1925 if (regions[i].mr_start & PAGE_MASK) {
1926 regions[i].mr_start += PAGE_MASK;
1927 regions[i].mr_start &= ~PAGE_MASK;
1928 regions[i].mr_size &= ~PAGE_MASK;
1929 }
1930 if (bootverbose)
1931 for (int i = 0; i < pregions_sz; i++)
1932 printf("pregions[%d].mr_start=%lx pregions[%d].mr_size=%lx\n",
1933 i, pregions[i].mr_start, i, pregions[i].mr_size);
1934
1935 phys_avail_count = 0;
1936 physsz = 0;
1937 hwphyssz = 0;
1938 TUNABLE_ULONG_FETCH("hw.physmem", (u_long *) &hwphyssz);
1939 for (i = 0, j = 0; i < regions_sz; i++) {
1940 if (bootverbose)
1941 printf("regions[%d].mr_start=%016lx regions[%d].mr_size=%016lx\n",
1942 i, regions[i].mr_start, i, regions[i].mr_size);
1943
1944 if (regions[i].mr_size < PAGE_SIZE)
1945 continue;
1946
1947 if (hwphyssz != 0 &&
1948 (physsz + regions[i].mr_size) >= hwphyssz) {
1949 if (physsz < hwphyssz) {
1950 phys_avail[j] = regions[i].mr_start;
1951 phys_avail[j + 1] = regions[i].mr_start +
1952 (hwphyssz - physsz);
1953 physsz = hwphyssz;
1954 phys_avail_count++;
1955 dump_avail[j] = phys_avail[j];
1956 dump_avail[j + 1] = phys_avail[j + 1];
1957 }
1958 break;
1959 }
1960 phys_avail[j] = regions[i].mr_start;
1961 phys_avail[j + 1] = regions[i].mr_start + regions[i].mr_size;
1962 dump_avail[j] = phys_avail[j];
1963 dump_avail[j + 1] = phys_avail[j + 1];
1964
1965 phys_avail_count++;
1966 physsz += regions[i].mr_size;
1967 j += 2;
1968 }
1969
1970 /* Check for overlap with the kernel and exception vectors */
1971 rm_pavail = 0;
1972 for (j = 0; j < 2 * phys_avail_count; j+=2) {
1973 if (phys_avail[j] < EXC_LAST)
1974 phys_avail[j] += EXC_LAST;
1975
1976 if (phys_avail[j] >= kpstart &&
1977 phys_avail[j + 1] <= kpend) {
1978 phys_avail[j] = phys_avail[j + 1] = ~0;
1979 rm_pavail++;
1980 continue;
1981 }
1982
1983 if (kpstart >= phys_avail[j] &&
1984 kpstart < phys_avail[j + 1]) {
1985 if (kpend < phys_avail[j + 1]) {
1986 phys_avail[2 * phys_avail_count] =
1987 (kpend & ~PAGE_MASK) + PAGE_SIZE;
1988 phys_avail[2 * phys_avail_count + 1] =
1989 phys_avail[j + 1];
1990 phys_avail_count++;
1991 }
1992
1993 phys_avail[j + 1] = kpstart & ~PAGE_MASK;
1994 }
1995
1996 if (kpend >= phys_avail[j] &&
1997 kpend < phys_avail[j + 1]) {
1998 if (kpstart > phys_avail[j]) {
1999 phys_avail[2 * phys_avail_count] = phys_avail[j];
2000 phys_avail[2 * phys_avail_count + 1] =
2001 kpstart & ~PAGE_MASK;
2002 phys_avail_count++;
2003 }
2004
2005 phys_avail[j] = (kpend & ~PAGE_MASK) +
2006 PAGE_SIZE;
2007 }
2008 }
2009 qsort(phys_avail, 2 * phys_avail_count, sizeof(phys_avail[0]), pa_cmp);
2010 for (i = 0; i < 2 * phys_avail_count; i++)
2011 phys_avail_debug[i] = phys_avail[i];
2012
2013 /* Remove physical available regions marked for removal (~0) */
2014 if (rm_pavail) {
2015 phys_avail_count -= rm_pavail;
2016 for (i = 2 * phys_avail_count;
2017 i < 2*(phys_avail_count + rm_pavail); i+=2)
2018 phys_avail[i] = phys_avail[i + 1] = 0;
2019 }
2020 if (bootverbose) {
2021 printf("phys_avail ranges after filtering:\n");
2022 for (j = 0; j < 2 * phys_avail_count; j+=2)
2023 printf("phys_avail[%d]=%08lx - phys_avail[%d]=%08lx\n",
2024 j, phys_avail[j], j + 1, phys_avail[j + 1]);
2025 }
2026 physmem = btoc(physsz);
2027
2028 /* XXX assume we're running non-virtualized and
2029 * we don't support BHYVE
2030 */
2031 if (isa3_pid_bits == 0)
2032 isa3_pid_bits = 20;
2033 if (powernv_enabled) {
2034 parttab_phys =
2035 moea64_bootstrap_alloc(PARTTAB_SIZE, PARTTAB_SIZE);
2036 validate_addr(parttab_phys, PARTTAB_SIZE);
2037 for (int i = 0; i < PARTTAB_SIZE/PAGE_SIZE; i++)
2038 pagezero(PHYS_TO_DMAP(parttab_phys + i * PAGE_SIZE));
2039
2040 }
2041 proctab_size = 1UL << PROCTAB_SIZE_SHIFT;
2042 proctab0pa = moea64_bootstrap_alloc(proctab_size, proctab_size);
2043 validate_addr(proctab0pa, proctab_size);
2044 for (int i = 0; i < proctab_size/PAGE_SIZE; i++)
2045 pagezero(PHYS_TO_DMAP(proctab0pa + i * PAGE_SIZE));
2046
2047 mmu_radix_setup_pagetables(hwphyssz);
2048 }
2049
2050 static void
mmu_radix_late_bootstrap(vm_offset_t start,vm_offset_t end)2051 mmu_radix_late_bootstrap(vm_offset_t start, vm_offset_t end)
2052 {
2053 int i;
2054 vm_paddr_t pa;
2055 void *dpcpu;
2056 vm_offset_t va;
2057
2058 /*
2059 * Set up the Open Firmware pmap and add its mappings if not in real
2060 * mode.
2061 */
2062 if (bootverbose)
2063 printf("%s enter\n", __func__);
2064
2065 /*
2066 * Calculate the last available physical address, and reserve the
2067 * vm_page_array (upper bound).
2068 */
2069 Maxmem = 0;
2070 for (i = 0; phys_avail[i + 1] != 0; i += 2)
2071 Maxmem = MAX(Maxmem, powerpc_btop(phys_avail[i + 1]));
2072
2073 /*
2074 * Remap any early IO mappings (console framebuffer, etc.)
2075 */
2076 bs_remap_earlyboot();
2077
2078 /*
2079 * Allocate a kernel stack with a guard page for thread0 and map it
2080 * into the kernel page map.
2081 */
2082 pa = allocpages(kstack_pages);
2083 va = virtual_avail + KSTACK_GUARD_PAGES * PAGE_SIZE;
2084 virtual_avail = va + kstack_pages * PAGE_SIZE;
2085 CTR2(KTR_PMAP, "moea64_bootstrap: kstack0 at %#x (%#x)", pa, va);
2086 thread0.td_kstack = (char *)va;
2087 for (i = 0; i < kstack_pages; i++) {
2088 mmu_radix_kenter(va, pa);
2089 pa += PAGE_SIZE;
2090 va += PAGE_SIZE;
2091 }
2092 thread0.td_kstack_pages = kstack_pages;
2093
2094 /*
2095 * Allocate virtual address space for the message buffer.
2096 */
2097 pa = msgbuf_phys = allocpages((msgbufsize + PAGE_MASK) >> PAGE_SHIFT);
2098 msgbufp = PHYS_TO_DMAP(pa);
2099
2100 /*
2101 * Allocate virtual address space for the dynamic percpu area.
2102 */
2103 pa = allocpages(DPCPU_SIZE >> PAGE_SHIFT);
2104 dpcpu = PHYS_TO_DMAP(pa);
2105 dpcpu_init(dpcpu, curcpu);
2106
2107 crashdumpmap = (caddr_t)virtual_avail;
2108 virtual_avail += MAXDUMPPGS * PAGE_SIZE;
2109
2110 /*
2111 * Reserve some special page table entries/VA space for temporary
2112 * mapping of pages.
2113 */
2114 }
2115
2116 static void
mmu_parttab_init(void)2117 mmu_parttab_init(void)
2118 {
2119 uint64_t ptcr;
2120
2121 isa3_parttab = PHYS_TO_DMAP(parttab_phys);
2122
2123 if (bootverbose)
2124 printf("%s parttab: %p\n", __func__, isa3_parttab);
2125 ptcr = parttab_phys | (PARTTAB_SIZE_SHIFT-12);
2126 if (bootverbose)
2127 printf("setting ptcr %lx\n", ptcr);
2128 mtspr(SPR_PTCR, ptcr);
2129 }
2130
2131 static void
mmu_parttab_update(uint64_t lpid,uint64_t pagetab,uint64_t proctab)2132 mmu_parttab_update(uint64_t lpid, uint64_t pagetab, uint64_t proctab)
2133 {
2134 uint64_t prev;
2135
2136 if (bootverbose)
2137 printf("%s isa3_parttab %p lpid %lx pagetab %lx proctab %lx\n", __func__, isa3_parttab,
2138 lpid, pagetab, proctab);
2139 prev = be64toh(isa3_parttab[lpid].pagetab);
2140 isa3_parttab[lpid].pagetab = htobe64(pagetab);
2141 isa3_parttab[lpid].proctab = htobe64(proctab);
2142
2143 if (prev & PARTTAB_HR) {
2144 __asm __volatile(PPC_TLBIE_5(%0,%1,2,0,1) : :
2145 "r" (TLBIEL_INVAL_SET_LPID), "r" (lpid));
2146 __asm __volatile(PPC_TLBIE_5(%0,%1,2,1,1) : :
2147 "r" (TLBIEL_INVAL_SET_LPID), "r" (lpid));
2148 } else {
2149 __asm __volatile(PPC_TLBIE_5(%0,%1,2,0,0) : :
2150 "r" (TLBIEL_INVAL_SET_LPID), "r" (lpid));
2151 }
2152 ttusync();
2153 }
2154
2155 static void
mmu_radix_parttab_init(void)2156 mmu_radix_parttab_init(void)
2157 {
2158 uint64_t pagetab;
2159
2160 mmu_parttab_init();
2161 pagetab = RTS_SIZE | DMAP_TO_PHYS(kernel_pmap->pm_pml1) | \
2162 RADIX_PGD_INDEX_SHIFT | PARTTAB_HR;
2163 mmu_parttab_update(0, pagetab, 0);
2164 }
2165
2166 static void
mmu_radix_proctab_register(vm_paddr_t proctabpa,uint64_t table_size)2167 mmu_radix_proctab_register(vm_paddr_t proctabpa, uint64_t table_size)
2168 {
2169 uint64_t pagetab, proctab;
2170
2171 pagetab = be64toh(isa3_parttab[0].pagetab);
2172 proctab = proctabpa | table_size | PARTTAB_GR;
2173 mmu_parttab_update(0, pagetab, proctab);
2174 }
2175
2176 static void
mmu_radix_proctab_init(void)2177 mmu_radix_proctab_init(void)
2178 {
2179
2180 isa3_base_pid = 1;
2181
2182 isa3_proctab = PHYS_TO_DMAP(proctab0pa);
2183 isa3_proctab->proctab0 =
2184 htobe64(RTS_SIZE | DMAP_TO_PHYS(kernel_pmap->pm_pml1) |
2185 RADIX_PGD_INDEX_SHIFT);
2186
2187 if (powernv_enabled) {
2188 mmu_radix_proctab_register(proctab0pa, PROCTAB_SIZE_SHIFT - 12);
2189 __asm __volatile("ptesync" : : : "memory");
2190 __asm __volatile(PPC_TLBIE_5(%0,%1,2,1,1) : :
2191 "r" (TLBIEL_INVAL_SET_LPID), "r" (0));
2192 __asm __volatile("eieio; tlbsync; ptesync" : : : "memory");
2193 #ifdef PSERIES
2194 } else {
2195 int64_t rc;
2196
2197 rc = phyp_hcall(H_REGISTER_PROC_TBL,
2198 PROC_TABLE_NEW | PROC_TABLE_RADIX | PROC_TABLE_GTSE,
2199 proctab0pa, 0, PROCTAB_SIZE_SHIFT - 12);
2200 if (rc != H_SUCCESS)
2201 panic("mmu_radix_proctab_init: "
2202 "failed to register process table: rc=%jd",
2203 (intmax_t)rc);
2204 #endif
2205 }
2206
2207 if (bootverbose)
2208 printf("process table %p and kernel radix PDE: %p\n",
2209 isa3_proctab, kernel_pmap->pm_pml1);
2210 mtmsr(mfmsr() | PSL_DR );
2211 mtmsr(mfmsr() & ~PSL_DR);
2212 kernel_pmap->pm_pid = isa3_base_pid;
2213 isa3_base_pid++;
2214 }
2215
2216 void
mmu_radix_advise(pmap_t pmap,vm_offset_t sva,vm_offset_t eva,int advice)2217 mmu_radix_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,
2218 int advice)
2219 {
2220 struct rwlock *lock;
2221 pml1_entry_t *l1e;
2222 pml2_entry_t *l2e;
2223 pml3_entry_t oldl3e, *l3e;
2224 pt_entry_t *pte;
2225 vm_offset_t va, va_next;
2226 vm_page_t m;
2227 bool anychanged;
2228
2229 if (advice != MADV_DONTNEED && advice != MADV_FREE)
2230 return;
2231 anychanged = false;
2232 PMAP_LOCK(pmap);
2233 for (; sva < eva; sva = va_next) {
2234 l1e = pmap_pml1e(pmap, sva);
2235 if ((be64toh(*l1e) & PG_V) == 0) {
2236 va_next = (sva + L1_PAGE_SIZE) & ~L1_PAGE_MASK;
2237 if (va_next < sva)
2238 va_next = eva;
2239 continue;
2240 }
2241 l2e = pmap_l1e_to_l2e(l1e, sva);
2242 if ((be64toh(*l2e) & PG_V) == 0) {
2243 va_next = (sva + L2_PAGE_SIZE) & ~L2_PAGE_MASK;
2244 if (va_next < sva)
2245 va_next = eva;
2246 continue;
2247 }
2248 va_next = (sva + L3_PAGE_SIZE) & ~L3_PAGE_MASK;
2249 if (va_next < sva)
2250 va_next = eva;
2251 l3e = pmap_l2e_to_l3e(l2e, sva);
2252 oldl3e = be64toh(*l3e);
2253 if ((oldl3e & PG_V) == 0)
2254 continue;
2255 else if ((oldl3e & RPTE_LEAF) != 0) {
2256 if ((oldl3e & PG_MANAGED) == 0)
2257 continue;
2258 lock = NULL;
2259 if (!pmap_demote_l3e_locked(pmap, l3e, sva, &lock)) {
2260 if (lock != NULL)
2261 rw_wunlock(lock);
2262
2263 /*
2264 * The large page mapping was destroyed.
2265 */
2266 continue;
2267 }
2268
2269 /*
2270 * Unless the page mappings are wired, remove the
2271 * mapping to a single page so that a subsequent
2272 * access may repromote. Choosing the last page
2273 * within the address range [sva, min(va_next, eva))
2274 * generally results in more repromotions. Since the
2275 * underlying page table page is fully populated, this
2276 * removal never frees a page table page.
2277 */
2278 if ((oldl3e & PG_W) == 0) {
2279 va = eva;
2280 if (va > va_next)
2281 va = va_next;
2282 va -= PAGE_SIZE;
2283 KASSERT(va >= sva,
2284 ("mmu_radix_advise: no address gap"));
2285 pte = pmap_l3e_to_pte(l3e, va);
2286 KASSERT((be64toh(*pte) & PG_V) != 0,
2287 ("pmap_advise: invalid PTE"));
2288 pmap_remove_pte(pmap, pte, va, be64toh(*l3e), NULL,
2289 &lock);
2290 anychanged = true;
2291 }
2292 if (lock != NULL)
2293 rw_wunlock(lock);
2294 }
2295 if (va_next > eva)
2296 va_next = eva;
2297 va = va_next;
2298 for (pte = pmap_l3e_to_pte(l3e, sva); sva != va_next;
2299 pte++, sva += PAGE_SIZE) {
2300 MPASS(pte == pmap_pte(pmap, sva));
2301
2302 if ((be64toh(*pte) & (PG_MANAGED | PG_V)) != (PG_MANAGED | PG_V))
2303 goto maybe_invlrng;
2304 else if ((be64toh(*pte) & (PG_M | PG_RW)) == (PG_M | PG_RW)) {
2305 if (advice == MADV_DONTNEED) {
2306 /*
2307 * Future calls to pmap_is_modified()
2308 * can be avoided by making the page
2309 * dirty now.
2310 */
2311 m = PHYS_TO_VM_PAGE(be64toh(*pte) & PG_FRAME);
2312 vm_page_dirty(m);
2313 }
2314 atomic_clear_long(pte, htobe64(PG_M | PG_A));
2315 } else if ((be64toh(*pte) & PG_A) != 0)
2316 atomic_clear_long(pte, htobe64(PG_A));
2317 else
2318 goto maybe_invlrng;
2319 anychanged = true;
2320 continue;
2321 maybe_invlrng:
2322 if (va != va_next) {
2323 anychanged = true;
2324 va = va_next;
2325 }
2326 }
2327 if (va != va_next)
2328 anychanged = true;
2329 }
2330 if (anychanged)
2331 pmap_invalidate_all(pmap);
2332 PMAP_UNLOCK(pmap);
2333 }
2334
2335 /*
2336 * Routines used in machine-dependent code
2337 */
2338 static void
mmu_radix_bootstrap(vm_offset_t start,vm_offset_t end)2339 mmu_radix_bootstrap(vm_offset_t start, vm_offset_t end)
2340 {
2341 uint64_t lpcr;
2342
2343 if (bootverbose)
2344 printf("%s\n", __func__);
2345 hw_direct_map = 1;
2346 powernv_enabled = (mfmsr() & PSL_HV) ? 1 : 0;
2347 mmu_radix_early_bootstrap(start, end);
2348 if (bootverbose)
2349 printf("early bootstrap complete\n");
2350 if (powernv_enabled) {
2351 lpcr = mfspr(SPR_LPCR);
2352 mtspr(SPR_LPCR, lpcr | LPCR_UPRT | LPCR_HR);
2353 mmu_radix_parttab_init();
2354 mmu_radix_init_amor();
2355 if (bootverbose)
2356 printf("powernv init complete\n");
2357 }
2358 mmu_radix_init_iamr();
2359 mmu_radix_proctab_init();
2360 mmu_radix_pid_set(kernel_pmap);
2361 if (powernv_enabled)
2362 mmu_radix_tlbiel_flush(TLB_INVAL_SCOPE_GLOBAL);
2363 else
2364 mmu_radix_tlbiel_flush(TLB_INVAL_SCOPE_LPID);
2365
2366 mmu_radix_late_bootstrap(start, end);
2367 numa_mem_regions(&numa_pregions, &numa_pregions_sz);
2368 if (bootverbose)
2369 printf("%s done\n", __func__);
2370 pmap_bootstrapped = 1;
2371 dmaplimit = roundup2(powerpc_ptob(Maxmem), L2_PAGE_SIZE);
2372 PCPU_SET(flags, PCPU_GET(flags) | PC_FLAG_NOSRS);
2373 }
2374
2375 static void
mmu_radix_cpu_bootstrap(int ap)2376 mmu_radix_cpu_bootstrap(int ap)
2377 {
2378 uint64_t lpcr;
2379 uint64_t ptcr;
2380
2381 if (powernv_enabled) {
2382 lpcr = mfspr(SPR_LPCR);
2383 mtspr(SPR_LPCR, lpcr | LPCR_UPRT | LPCR_HR);
2384
2385 ptcr = parttab_phys | (PARTTAB_SIZE_SHIFT-12);
2386 mtspr(SPR_PTCR, ptcr);
2387 mmu_radix_init_amor();
2388 }
2389 mmu_radix_init_iamr();
2390 mmu_radix_pid_set(kernel_pmap);
2391 if (powernv_enabled)
2392 mmu_radix_tlbiel_flush(TLB_INVAL_SCOPE_GLOBAL);
2393 else
2394 mmu_radix_tlbiel_flush(TLB_INVAL_SCOPE_LPID);
2395 }
2396
2397 static SYSCTL_NODE(_vm_pmap, OID_AUTO, l3e, CTLFLAG_RD, 0,
2398 "2MB page mapping counters");
2399
2400 static COUNTER_U64_DEFINE_EARLY(pmap_l3e_demotions);
2401 SYSCTL_COUNTER_U64(_vm_pmap_l3e, OID_AUTO, demotions, CTLFLAG_RD,
2402 &pmap_l3e_demotions, "2MB page demotions");
2403
2404 static COUNTER_U64_DEFINE_EARLY(pmap_l3e_mappings);
2405 SYSCTL_COUNTER_U64(_vm_pmap_l3e, OID_AUTO, mappings, CTLFLAG_RD,
2406 &pmap_l3e_mappings, "2MB page mappings");
2407
2408 static COUNTER_U64_DEFINE_EARLY(pmap_l3e_p_failures);
2409 SYSCTL_COUNTER_U64(_vm_pmap_l3e, OID_AUTO, p_failures, CTLFLAG_RD,
2410 &pmap_l3e_p_failures, "2MB page promotion failures");
2411
2412 static COUNTER_U64_DEFINE_EARLY(pmap_l3e_promotions);
2413 SYSCTL_COUNTER_U64(_vm_pmap_l3e, OID_AUTO, promotions, CTLFLAG_RD,
2414 &pmap_l3e_promotions, "2MB page promotions");
2415
2416 static SYSCTL_NODE(_vm_pmap, OID_AUTO, l2e, CTLFLAG_RD, 0,
2417 "1GB page mapping counters");
2418
2419 static COUNTER_U64_DEFINE_EARLY(pmap_l2e_demotions);
2420 SYSCTL_COUNTER_U64(_vm_pmap_l2e, OID_AUTO, demotions, CTLFLAG_RD,
2421 &pmap_l2e_demotions, "1GB page demotions");
2422
2423 void
mmu_radix_clear_modify(vm_page_t m)2424 mmu_radix_clear_modify(vm_page_t m)
2425 {
2426 struct md_page *pvh;
2427 pmap_t pmap;
2428 pv_entry_t next_pv, pv;
2429 pml3_entry_t oldl3e, *l3e;
2430 pt_entry_t oldpte, *pte;
2431 struct rwlock *lock;
2432 vm_offset_t va;
2433 int md_gen, pvh_gen;
2434
2435 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
2436 ("pmap_clear_modify: page %p is not managed", m));
2437 vm_page_assert_busied(m);
2438 CTR2(KTR_PMAP, "%s(%p)", __func__, m);
2439
2440 /*
2441 * If the page is not PGA_WRITEABLE, then no PTEs can have PG_M set.
2442 * If the object containing the page is locked and the page is not
2443 * exclusive busied, then PGA_WRITEABLE cannot be concurrently set.
2444 */
2445 if ((m->a.flags & PGA_WRITEABLE) == 0)
2446 return;
2447 pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy :
2448 pa_to_pvh(VM_PAGE_TO_PHYS(m));
2449 lock = VM_PAGE_TO_PV_LIST_LOCK(m);
2450 rw_wlock(lock);
2451 restart:
2452 TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_link, next_pv) {
2453 pmap = PV_PMAP(pv);
2454 if (!PMAP_TRYLOCK(pmap)) {
2455 pvh_gen = pvh->pv_gen;
2456 rw_wunlock(lock);
2457 PMAP_LOCK(pmap);
2458 rw_wlock(lock);
2459 if (pvh_gen != pvh->pv_gen) {
2460 PMAP_UNLOCK(pmap);
2461 goto restart;
2462 }
2463 }
2464 va = pv->pv_va;
2465 l3e = pmap_pml3e(pmap, va);
2466 oldl3e = be64toh(*l3e);
2467 if ((oldl3e & PG_RW) != 0 &&
2468 pmap_demote_l3e_locked(pmap, l3e, va, &lock) &&
2469 (oldl3e & PG_W) == 0) {
2470 /*
2471 * Write protect the mapping to a
2472 * single page so that a subsequent
2473 * write access may repromote.
2474 */
2475 va += VM_PAGE_TO_PHYS(m) - (oldl3e &
2476 PG_PS_FRAME);
2477 pte = pmap_l3e_to_pte(l3e, va);
2478 oldpte = be64toh(*pte);
2479 while (!atomic_cmpset_long(pte,
2480 htobe64(oldpte),
2481 htobe64((oldpte | RPTE_EAA_R) & ~(PG_M | PG_RW))))
2482 oldpte = be64toh(*pte);
2483 vm_page_dirty(m);
2484 pmap_invalidate_page(pmap, va);
2485 }
2486 PMAP_UNLOCK(pmap);
2487 }
2488 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) {
2489 pmap = PV_PMAP(pv);
2490 if (!PMAP_TRYLOCK(pmap)) {
2491 md_gen = m->md.pv_gen;
2492 pvh_gen = pvh->pv_gen;
2493 rw_wunlock(lock);
2494 PMAP_LOCK(pmap);
2495 rw_wlock(lock);
2496 if (pvh_gen != pvh->pv_gen || md_gen != m->md.pv_gen) {
2497 PMAP_UNLOCK(pmap);
2498 goto restart;
2499 }
2500 }
2501 l3e = pmap_pml3e(pmap, pv->pv_va);
2502 KASSERT((be64toh(*l3e) & RPTE_LEAF) == 0, ("pmap_clear_modify: found"
2503 " a 2mpage in page %p's pv list", m));
2504 pte = pmap_l3e_to_pte(l3e, pv->pv_va);
2505 if ((be64toh(*pte) & (PG_M | PG_RW)) == (PG_M | PG_RW)) {
2506 atomic_clear_long(pte, htobe64(PG_M));
2507 pmap_invalidate_page(pmap, pv->pv_va);
2508 }
2509 PMAP_UNLOCK(pmap);
2510 }
2511 rw_wunlock(lock);
2512 }
2513
2514 void
mmu_radix_copy(pmap_t dst_pmap,pmap_t src_pmap,vm_offset_t dst_addr,vm_size_t len,vm_offset_t src_addr)2515 mmu_radix_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr,
2516 vm_size_t len, vm_offset_t src_addr)
2517 {
2518 struct rwlock *lock;
2519 struct spglist free;
2520 vm_offset_t addr;
2521 vm_offset_t end_addr = src_addr + len;
2522 vm_offset_t va_next;
2523 vm_page_t dst_pdpg, dstmpte, srcmpte;
2524 bool invalidate_all;
2525
2526 CTR6(KTR_PMAP,
2527 "%s(dst_pmap=%p, src_pmap=%p, dst_addr=%lx, len=%lu, src_addr=%lx)\n",
2528 __func__, dst_pmap, src_pmap, dst_addr, len, src_addr);
2529
2530 if (dst_addr != src_addr)
2531 return;
2532 lock = NULL;
2533 invalidate_all = false;
2534 if (dst_pmap < src_pmap) {
2535 PMAP_LOCK(dst_pmap);
2536 PMAP_LOCK(src_pmap);
2537 } else {
2538 PMAP_LOCK(src_pmap);
2539 PMAP_LOCK(dst_pmap);
2540 }
2541
2542 for (addr = src_addr; addr < end_addr; addr = va_next) {
2543 pml1_entry_t *l1e;
2544 pml2_entry_t *l2e;
2545 pml3_entry_t srcptepaddr, *l3e;
2546 pt_entry_t *src_pte, *dst_pte;
2547
2548 l1e = pmap_pml1e(src_pmap, addr);
2549 if ((be64toh(*l1e) & PG_V) == 0) {
2550 va_next = (addr + L1_PAGE_SIZE) & ~L1_PAGE_MASK;
2551 if (va_next < addr)
2552 va_next = end_addr;
2553 continue;
2554 }
2555
2556 l2e = pmap_l1e_to_l2e(l1e, addr);
2557 if ((be64toh(*l2e) & PG_V) == 0) {
2558 va_next = (addr + L2_PAGE_SIZE) & ~L2_PAGE_MASK;
2559 if (va_next < addr)
2560 va_next = end_addr;
2561 continue;
2562 }
2563
2564 va_next = (addr + L3_PAGE_SIZE) & ~L3_PAGE_MASK;
2565 if (va_next < addr)
2566 va_next = end_addr;
2567
2568 l3e = pmap_l2e_to_l3e(l2e, addr);
2569 srcptepaddr = be64toh(*l3e);
2570 if (srcptepaddr == 0)
2571 continue;
2572
2573 if (srcptepaddr & RPTE_LEAF) {
2574 if ((addr & L3_PAGE_MASK) != 0 ||
2575 addr + L3_PAGE_SIZE > end_addr)
2576 continue;
2577 dst_pdpg = pmap_allocl3e(dst_pmap, addr, NULL);
2578 if (dst_pdpg == NULL)
2579 break;
2580 l3e = VM_PAGE_TO_DMAP(dst_pdpg);
2581 l3e = &l3e[pmap_pml3e_index(addr)];
2582 if (be64toh(*l3e) == 0 && ((srcptepaddr & PG_MANAGED) == 0 ||
2583 pmap_pv_insert_l3e(dst_pmap, addr, srcptepaddr,
2584 PMAP_ENTER_NORECLAIM, &lock))) {
2585 *l3e = htobe64(srcptepaddr & ~PG_W);
2586 pmap_resident_count_inc(dst_pmap,
2587 L3_PAGE_SIZE / PAGE_SIZE);
2588 counter_u64_add(pmap_l3e_mappings, 1);
2589 } else
2590 dst_pdpg->ref_count--;
2591 continue;
2592 }
2593
2594 srcptepaddr &= PG_FRAME;
2595 srcmpte = PHYS_TO_VM_PAGE(srcptepaddr);
2596 KASSERT(srcmpte->ref_count > 0,
2597 ("pmap_copy: source page table page is unused"));
2598
2599 if (va_next > end_addr)
2600 va_next = end_addr;
2601
2602 src_pte = PHYS_TO_DMAP(srcptepaddr);
2603 src_pte = &src_pte[pmap_pte_index(addr)];
2604 dstmpte = NULL;
2605 while (addr < va_next) {
2606 pt_entry_t ptetemp;
2607 ptetemp = be64toh(*src_pte);
2608 /*
2609 * we only virtual copy managed pages
2610 */
2611 if ((ptetemp & PG_MANAGED) != 0) {
2612 if (dstmpte != NULL &&
2613 dstmpte->pindex == pmap_l3e_pindex(addr))
2614 dstmpte->ref_count++;
2615 else if ((dstmpte = pmap_allocpte(dst_pmap,
2616 addr, NULL)) == NULL)
2617 goto out;
2618 dst_pte = VM_PAGE_TO_DMAP(dstmpte);
2619 dst_pte = &dst_pte[pmap_pte_index(addr)];
2620 if (be64toh(*dst_pte) == 0 &&
2621 pmap_try_insert_pv_entry(dst_pmap, addr,
2622 PHYS_TO_VM_PAGE(ptetemp & PG_FRAME),
2623 &lock)) {
2624 /*
2625 * Clear the wired, modified, and
2626 * accessed (referenced) bits
2627 * during the copy.
2628 */
2629 *dst_pte = htobe64(ptetemp & ~(PG_W | PG_M |
2630 PG_A));
2631 pmap_resident_count_inc(dst_pmap, 1);
2632 } else {
2633 SLIST_INIT(&free);
2634 if (pmap_unwire_ptp(dst_pmap, addr,
2635 dstmpte, &free)) {
2636 /*
2637 * Although "addr" is not
2638 * mapped, paging-structure
2639 * caches could nonetheless
2640 * have entries that refer to
2641 * the freed page table pages.
2642 * Invalidate those entries.
2643 */
2644 invalidate_all = true;
2645 vm_page_free_pages_toq(&free,
2646 true);
2647 }
2648 goto out;
2649 }
2650 if (dstmpte->ref_count >= srcmpte->ref_count)
2651 break;
2652 }
2653 addr += PAGE_SIZE;
2654 if (__predict_false((addr & L3_PAGE_MASK) == 0))
2655 src_pte = pmap_pte(src_pmap, addr);
2656 else
2657 src_pte++;
2658 }
2659 }
2660 out:
2661 if (invalidate_all)
2662 pmap_invalidate_all(dst_pmap);
2663 if (lock != NULL)
2664 rw_wunlock(lock);
2665 PMAP_UNLOCK(src_pmap);
2666 PMAP_UNLOCK(dst_pmap);
2667 }
2668
2669 static void
mmu_radix_copy_page(vm_page_t msrc,vm_page_t mdst)2670 mmu_radix_copy_page(vm_page_t msrc, vm_page_t mdst)
2671 {
2672 void *src = VM_PAGE_TO_DMAP(msrc);
2673 void *dst = VM_PAGE_TO_DMAP(mdst);
2674
2675 CTR3(KTR_PMAP, "%s(%p, %p)", __func__, src, dst);
2676 /*
2677 * XXX slow
2678 */
2679 bcopy(src, dst, PAGE_SIZE);
2680 }
2681
2682 static void
mmu_radix_copy_pages(vm_page_t ma[],vm_offset_t a_offset,vm_page_t mb[],vm_offset_t b_offset,int xfersize)2683 mmu_radix_copy_pages(vm_page_t ma[], vm_offset_t a_offset, vm_page_t mb[],
2684 vm_offset_t b_offset, int xfersize)
2685 {
2686 void *a_cp, *b_cp;
2687 vm_offset_t a_pg_offset, b_pg_offset;
2688 int cnt;
2689
2690 CTR6(KTR_PMAP, "%s(%p, %#x, %p, %#x, %#x)", __func__, ma,
2691 a_offset, mb, b_offset, xfersize);
2692
2693 while (xfersize > 0) {
2694 a_pg_offset = a_offset & PAGE_MASK;
2695 cnt = min(xfersize, PAGE_SIZE - a_pg_offset);
2696 a_cp = (char *)VM_PAGE_TO_DMAP(ma[a_offset >> PAGE_SHIFT]) +
2697 a_pg_offset;
2698 b_pg_offset = b_offset & PAGE_MASK;
2699 cnt = min(cnt, PAGE_SIZE - b_pg_offset);
2700 b_cp = (char *)VM_PAGE_TO_DMAP(mb[b_offset >> PAGE_SHIFT]) +
2701 b_pg_offset;
2702 bcopy(a_cp, b_cp, cnt);
2703 a_offset += cnt;
2704 b_offset += cnt;
2705 xfersize -= cnt;
2706 }
2707 }
2708
2709 #if VM_NRESERVLEVEL > 0
2710 /*
2711 * Tries to promote the 512, contiguous 4KB page mappings that are within a
2712 * single page table page (PTP) to a single 2MB page mapping. For promotion
2713 * to occur, two conditions must be met: (1) the 4KB page mappings must map
2714 * aligned, contiguous physical memory and (2) the 4KB page mappings must have
2715 * identical characteristics.
2716 */
2717 static int
pmap_promote_l3e(pmap_t pmap,pml3_entry_t * pde,vm_offset_t va,struct rwlock ** lockp)2718 pmap_promote_l3e(pmap_t pmap, pml3_entry_t *pde, vm_offset_t va,
2719 struct rwlock **lockp)
2720 {
2721 pml3_entry_t newpde;
2722 pt_entry_t *firstpte, oldpte, pa, *pte;
2723 vm_page_t mpte;
2724
2725 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
2726
2727 /*
2728 * Examine the first PTE in the specified PTP. Abort if this PTE is
2729 * either invalid, unused, or does not map the first 4KB physical page
2730 * within a 2MB page.
2731 */
2732 firstpte = PHYS_TO_DMAP(be64toh(*pde) & PG_FRAME);
2733 setpde:
2734 newpde = be64toh(*firstpte);
2735 if ((newpde & ((PG_FRAME & L3_PAGE_MASK) | PG_A | PG_V)) != (PG_A | PG_V)) {
2736 CTR2(KTR_PMAP, "pmap_promote_l3e: failure for va %#lx"
2737 " in pmap %p", va, pmap);
2738 goto fail;
2739 }
2740 if ((newpde & (PG_M | PG_RW)) == PG_RW) {
2741 /*
2742 * When PG_M is already clear, PG_RW can be cleared without
2743 * a TLB invalidation.
2744 */
2745 if (!atomic_cmpset_long(firstpte, htobe64(newpde), htobe64((newpde | RPTE_EAA_R) & ~RPTE_EAA_W)))
2746 goto setpde;
2747 newpde &= ~RPTE_EAA_W;
2748 }
2749
2750 /*
2751 * Examine each of the other PTEs in the specified PTP. Abort if this
2752 * PTE maps an unexpected 4KB physical page or does not have identical
2753 * characteristics to the first PTE.
2754 */
2755 pa = (newpde & (PG_PS_FRAME | PG_A | PG_V)) + L3_PAGE_SIZE - PAGE_SIZE;
2756 for (pte = firstpte + NPTEPG - 1; pte > firstpte; pte--) {
2757 setpte:
2758 oldpte = be64toh(*pte);
2759 if ((oldpte & (PG_FRAME | PG_A | PG_V)) != pa) {
2760 CTR2(KTR_PMAP, "pmap_promote_l3e: failure for va %#lx"
2761 " in pmap %p", va, pmap);
2762 goto fail;
2763 }
2764 if ((oldpte & (PG_M | PG_RW)) == PG_RW) {
2765 /*
2766 * When PG_M is already clear, PG_RW can be cleared
2767 * without a TLB invalidation.
2768 */
2769 if (!atomic_cmpset_long(pte, htobe64(oldpte), htobe64((oldpte | RPTE_EAA_R) & ~RPTE_EAA_W)))
2770 goto setpte;
2771 oldpte &= ~RPTE_EAA_W;
2772 CTR2(KTR_PMAP, "pmap_promote_l3e: protect for va %#lx"
2773 " in pmap %p", (oldpte & PG_FRAME & L3_PAGE_MASK) |
2774 (va & ~L3_PAGE_MASK), pmap);
2775 }
2776 if ((oldpte & PG_PTE_PROMOTE) != (newpde & PG_PTE_PROMOTE)) {
2777 CTR2(KTR_PMAP, "pmap_promote_l3e: failure for va %#lx"
2778 " in pmap %p", va, pmap);
2779 goto fail;
2780 }
2781 pa -= PAGE_SIZE;
2782 }
2783
2784 /*
2785 * Save the page table page in its current state until the PDE
2786 * mapping the superpage is demoted by pmap_demote_pde() or
2787 * destroyed by pmap_remove_pde().
2788 */
2789 mpte = PHYS_TO_VM_PAGE(be64toh(*pde) & PG_FRAME);
2790 KASSERT(mpte >= vm_page_array &&
2791 mpte < &vm_page_array[vm_page_array_size],
2792 ("pmap_promote_l3e: page table page is out of range"));
2793 KASSERT(mpte->pindex == pmap_l3e_pindex(va),
2794 ("pmap_promote_l3e: page table page's pindex is wrong"));
2795 if (pmap_insert_pt_page(pmap, mpte)) {
2796 CTR2(KTR_PMAP,
2797 "pmap_promote_l3e: failure for va %#lx in pmap %p", va,
2798 pmap);
2799 goto fail;
2800 }
2801
2802 /*
2803 * Promote the pv entries.
2804 */
2805 if ((newpde & PG_MANAGED) != 0)
2806 pmap_pv_promote_l3e(pmap, va, newpde & PG_PS_FRAME, lockp);
2807
2808 pte_store(pde, PG_PROMOTED | newpde);
2809 ptesync();
2810 counter_u64_add(pmap_l3e_promotions, 1);
2811 CTR2(KTR_PMAP, "pmap_promote_l3e: success for va %#lx"
2812 " in pmap %p", va, pmap);
2813 return (0);
2814 fail:
2815 counter_u64_add(pmap_l3e_p_failures, 1);
2816 return (KERN_FAILURE);
2817 }
2818 #endif /* VM_NRESERVLEVEL > 0 */
2819
2820 int
mmu_radix_enter(pmap_t pmap,vm_offset_t va,vm_page_t m,vm_prot_t prot,u_int flags,int8_t psind)2821 mmu_radix_enter(pmap_t pmap, vm_offset_t va, vm_page_t m,
2822 vm_prot_t prot, u_int flags, int8_t psind)
2823 {
2824 struct rwlock *lock;
2825 pml3_entry_t *l3e;
2826 pt_entry_t *pte;
2827 pt_entry_t newpte, origpte;
2828 pv_entry_t pv;
2829 vm_paddr_t opa, pa;
2830 vm_page_t mpte, om;
2831 int rv, retrycount;
2832 bool nosleep, invalidate_all, invalidate_page;
2833
2834 va = trunc_page(va);
2835 retrycount = 0;
2836 invalidate_page = invalidate_all = false;
2837 CTR6(KTR_PMAP, "pmap_enter(%p, %#lx, %p, %#x, %#x, %d)", pmap, va,
2838 m, prot, flags, psind);
2839 KASSERT(va <= VM_MAX_KERNEL_ADDRESS, ("pmap_enter: toobig"));
2840 KASSERT((m->oflags & VPO_UNMANAGED) != 0 || !VA_IS_CLEANMAP(va),
2841 ("pmap_enter: managed mapping within the clean submap"));
2842 if ((m->oflags & VPO_UNMANAGED) == 0)
2843 VM_PAGE_OBJECT_BUSY_ASSERT(m);
2844
2845 KASSERT((flags & PMAP_ENTER_RESERVED) == 0,
2846 ("pmap_enter: flags %u has reserved bits set", flags));
2847 pa = VM_PAGE_TO_PHYS(m);
2848 newpte = (pt_entry_t)(pa | PG_A | PG_V | RPTE_LEAF);
2849 if ((flags & VM_PROT_WRITE) != 0)
2850 newpte |= PG_M;
2851 if ((flags & VM_PROT_READ) != 0)
2852 newpte |= PG_A;
2853 if (prot & VM_PROT_READ)
2854 newpte |= RPTE_EAA_R;
2855 if ((prot & VM_PROT_WRITE) != 0)
2856 newpte |= RPTE_EAA_W;
2857 KASSERT((newpte & (PG_M | PG_RW)) != PG_M,
2858 ("pmap_enter: flags includes VM_PROT_WRITE but prot doesn't"));
2859
2860 if (prot & VM_PROT_EXECUTE)
2861 newpte |= PG_X;
2862 if ((flags & PMAP_ENTER_WIRED) != 0)
2863 newpte |= PG_W;
2864 if (va >= DMAP_MIN_ADDRESS)
2865 newpte |= RPTE_EAA_P;
2866 newpte |= pmap_cache_bits(m->md.mdpg_cache_attrs);
2867 /*
2868 * Set modified bit gratuitously for writeable mappings if
2869 * the page is unmanaged. We do not want to take a fault
2870 * to do the dirty bit accounting for these mappings.
2871 */
2872 if ((m->oflags & VPO_UNMANAGED) != 0) {
2873 if ((newpte & PG_RW) != 0)
2874 newpte |= PG_M;
2875 } else
2876 newpte |= PG_MANAGED;
2877
2878 lock = NULL;
2879 PMAP_LOCK(pmap);
2880 if (psind == 1) {
2881 /* Assert the required virtual and physical alignment. */
2882 KASSERT((va & L3_PAGE_MASK) == 0, ("pmap_enter: va unaligned"));
2883 KASSERT(m->psind > 0, ("pmap_enter: m->psind < psind"));
2884 rv = pmap_enter_l3e(pmap, va, newpte | RPTE_LEAF, flags, m, &lock);
2885 goto out;
2886 }
2887 mpte = NULL;
2888
2889 /*
2890 * In the case that a page table page is not
2891 * resident, we are creating it here.
2892 */
2893 retry:
2894 l3e = pmap_pml3e(pmap, va);
2895 if (l3e != NULL && (be64toh(*l3e) & PG_V) != 0 && ((be64toh(*l3e) & RPTE_LEAF) == 0 ||
2896 pmap_demote_l3e_locked(pmap, l3e, va, &lock))) {
2897 pte = pmap_l3e_to_pte(l3e, va);
2898 if (va < VM_MAXUSER_ADDRESS && mpte == NULL) {
2899 mpte = PHYS_TO_VM_PAGE(be64toh(*l3e) & PG_FRAME);
2900 mpte->ref_count++;
2901 }
2902 } else if (va < VM_MAXUSER_ADDRESS) {
2903 /*
2904 * Here if the pte page isn't mapped, or if it has been
2905 * deallocated.
2906 */
2907 nosleep = (flags & PMAP_ENTER_NOSLEEP) != 0;
2908 mpte = _pmap_allocpte(pmap, pmap_l3e_pindex(va),
2909 nosleep ? NULL : &lock);
2910 if (mpte == NULL && nosleep) {
2911 rv = KERN_RESOURCE_SHORTAGE;
2912 goto out;
2913 }
2914 if (__predict_false(retrycount++ == 6))
2915 panic("too many retries");
2916 invalidate_all = true;
2917 goto retry;
2918 } else
2919 panic("pmap_enter: invalid page directory va=%#lx", va);
2920
2921 origpte = be64toh(*pte);
2922 pv = NULL;
2923
2924 /*
2925 * Is the specified virtual address already mapped?
2926 */
2927 if ((origpte & PG_V) != 0) {
2928 #ifdef INVARIANTS
2929 if (VERBOSE_PMAP || pmap_logging) {
2930 printf("cow fault pmap_enter(%p, %#lx, %p, %#x, %x, %d) --"
2931 " asid=%lu curpid=%d name=%s origpte0x%lx\n",
2932 pmap, va, m, prot, flags, psind, pmap->pm_pid,
2933 curproc->p_pid, curproc->p_comm, origpte);
2934 #ifdef DDB
2935 pmap_pte_walk(pmap->pm_pml1, va);
2936 #endif
2937 }
2938 #endif
2939 /*
2940 * Wiring change, just update stats. We don't worry about
2941 * wiring PT pages as they remain resident as long as there
2942 * are valid mappings in them. Hence, if a user page is wired,
2943 * the PT page will be also.
2944 */
2945 if ((newpte & PG_W) != 0 && (origpte & PG_W) == 0)
2946 pmap->pm_stats.wired_count++;
2947 else if ((newpte & PG_W) == 0 && (origpte & PG_W) != 0)
2948 pmap->pm_stats.wired_count--;
2949
2950 /*
2951 * Remove the extra PT page reference.
2952 */
2953 if (mpte != NULL) {
2954 mpte->ref_count--;
2955 KASSERT(mpte->ref_count > 0,
2956 ("pmap_enter: missing reference to page table page,"
2957 " va: 0x%lx", va));
2958 }
2959
2960 /*
2961 * Has the physical page changed?
2962 */
2963 opa = origpte & PG_FRAME;
2964 if (opa == pa) {
2965 /*
2966 * No, might be a protection or wiring change.
2967 */
2968 if ((origpte & PG_MANAGED) != 0 &&
2969 (newpte & PG_RW) != 0)
2970 vm_page_aflag_set(m, PGA_WRITEABLE);
2971 if (((origpte ^ newpte) & ~(PG_M | PG_A)) == 0) {
2972 if ((newpte & (PG_A|PG_M)) != (origpte & (PG_A|PG_M))) {
2973 if (!atomic_cmpset_long(pte, htobe64(origpte), htobe64(newpte)))
2974 goto retry;
2975 if ((newpte & PG_M) != (origpte & PG_M))
2976 vm_page_dirty(m);
2977 if ((newpte & PG_A) != (origpte & PG_A))
2978 vm_page_aflag_set(m, PGA_REFERENCED);
2979 ptesync();
2980 } else
2981 invalidate_all = true;
2982 if (((origpte ^ newpte) & ~(PG_M | PG_A)) == 0)
2983 goto unchanged;
2984 }
2985 goto validate;
2986 }
2987
2988 /*
2989 * The physical page has changed. Temporarily invalidate
2990 * the mapping. This ensures that all threads sharing the
2991 * pmap keep a consistent view of the mapping, which is
2992 * necessary for the correct handling of COW faults. It
2993 * also permits reuse of the old mapping's PV entry,
2994 * avoiding an allocation.
2995 *
2996 * For consistency, handle unmanaged mappings the same way.
2997 */
2998 origpte = be64toh(pte_load_clear(pte));
2999 KASSERT((origpte & PG_FRAME) == opa,
3000 ("pmap_enter: unexpected pa update for %#lx", va));
3001 if ((origpte & PG_MANAGED) != 0) {
3002 om = PHYS_TO_VM_PAGE(opa);
3003
3004 /*
3005 * The pmap lock is sufficient to synchronize with
3006 * concurrent calls to pmap_page_test_mappings() and
3007 * pmap_ts_referenced().
3008 */
3009 if ((origpte & (PG_M | PG_RW)) == (PG_M | PG_RW))
3010 vm_page_dirty(om);
3011 if ((origpte & PG_A) != 0)
3012 vm_page_aflag_set(om, PGA_REFERENCED);
3013 CHANGE_PV_LIST_LOCK_TO_PHYS(&lock, opa);
3014 pv = pmap_pvh_remove(&om->md, pmap, va);
3015 if ((newpte & PG_MANAGED) == 0)
3016 free_pv_entry(pmap, pv);
3017 #ifdef INVARIANTS
3018 else if (origpte & PG_MANAGED) {
3019 if (pv == NULL) {
3020 #ifdef DDB
3021 pmap_page_print_mappings(om);
3022 #endif
3023 MPASS(pv != NULL);
3024 }
3025 }
3026 #endif
3027 if ((om->a.flags & PGA_WRITEABLE) != 0 &&
3028 TAILQ_EMPTY(&om->md.pv_list) &&
3029 ((om->flags & PG_FICTITIOUS) != 0 ||
3030 TAILQ_EMPTY(&pa_to_pvh(opa)->pv_list)))
3031 vm_page_aflag_clear(om, PGA_WRITEABLE);
3032 }
3033 if ((origpte & PG_A) != 0)
3034 invalidate_page = true;
3035 origpte = 0;
3036 } else {
3037 if (pmap != kernel_pmap) {
3038 #ifdef INVARIANTS
3039 if (VERBOSE_PMAP || pmap_logging)
3040 printf("pmap_enter(%p, %#lx, %p, %#x, %x, %d) -- asid=%lu curpid=%d name=%s\n",
3041 pmap, va, m, prot, flags, psind,
3042 pmap->pm_pid, curproc->p_pid,
3043 curproc->p_comm);
3044 #endif
3045 }
3046
3047 /*
3048 * Increment the counters.
3049 */
3050 if ((newpte & PG_W) != 0)
3051 pmap->pm_stats.wired_count++;
3052 pmap_resident_count_inc(pmap, 1);
3053 }
3054
3055 /*
3056 * Enter on the PV list if part of our managed memory.
3057 */
3058 if ((newpte & PG_MANAGED) != 0) {
3059 if (pv == NULL) {
3060 pv = get_pv_entry(pmap, &lock);
3061 pv->pv_va = va;
3062 }
3063 #ifdef VERBOSE_PV
3064 else
3065 printf("reassigning pv: %p to pmap: %p\n",
3066 pv, pmap);
3067 #endif
3068 CHANGE_PV_LIST_LOCK_TO_PHYS(&lock, pa);
3069 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_link);
3070 m->md.pv_gen++;
3071 if ((newpte & PG_RW) != 0)
3072 vm_page_aflag_set(m, PGA_WRITEABLE);
3073 }
3074
3075 /*
3076 * Update the PTE.
3077 */
3078 if ((origpte & PG_V) != 0) {
3079 validate:
3080 origpte = be64toh(pte_load_store(pte, htobe64(newpte)));
3081 KASSERT((origpte & PG_FRAME) == pa,
3082 ("pmap_enter: unexpected pa update for %#lx", va));
3083 if ((newpte & PG_M) == 0 && (origpte & (PG_M | PG_RW)) ==
3084 (PG_M | PG_RW)) {
3085 if ((origpte & PG_MANAGED) != 0)
3086 vm_page_dirty(m);
3087 invalidate_page = true;
3088
3089 /*
3090 * Although the PTE may still have PG_RW set, TLB
3091 * invalidation may nonetheless be required because
3092 * the PTE no longer has PG_M set.
3093 */
3094 } else if ((origpte & PG_X) != 0 || (newpte & PG_X) == 0) {
3095 /*
3096 * Removing capabilities requires invalidation on POWER
3097 */
3098 invalidate_page = true;
3099 goto unchanged;
3100 }
3101 if ((origpte & PG_A) != 0)
3102 invalidate_page = true;
3103 } else {
3104 pte_store(pte, newpte);
3105 ptesync();
3106 }
3107 unchanged:
3108
3109 #if VM_NRESERVLEVEL > 0
3110 /*
3111 * If both the page table page and the reservation are fully
3112 * populated, then attempt promotion.
3113 */
3114 if ((mpte == NULL || mpte->ref_count == NPTEPG) &&
3115 mmu_radix_ps_enabled(pmap) &&
3116 (m->flags & PG_FICTITIOUS) == 0 &&
3117 vm_reserv_level_iffullpop(m) == 0 &&
3118 pmap_promote_l3e(pmap, l3e, va, &lock) == 0)
3119 invalidate_all = true;
3120 #endif
3121 if (invalidate_all)
3122 pmap_invalidate_all(pmap);
3123 else if (invalidate_page)
3124 pmap_invalidate_page(pmap, va);
3125
3126 rv = KERN_SUCCESS;
3127 out:
3128 if (lock != NULL)
3129 rw_wunlock(lock);
3130 PMAP_UNLOCK(pmap);
3131
3132 return (rv);
3133 }
3134
3135 /*
3136 * Release a page table page reference after a failed attempt to create a
3137 * mapping.
3138 */
3139 static void
pmap_abort_ptp(pmap_t pmap,vm_offset_t va,vm_page_t pdpg)3140 pmap_abort_ptp(pmap_t pmap, vm_offset_t va, vm_page_t pdpg)
3141 {
3142 struct spglist free;
3143
3144 SLIST_INIT(&free);
3145 if (pmap_unwire_ptp(pmap, va, pdpg, &free)) {
3146 /*
3147 * Although "va" is not mapped, paging-
3148 * structure caches could nonetheless have
3149 * entries that refer to the freed page table
3150 * pages. Invalidate those entries.
3151 */
3152 pmap_invalidate_page(pmap, va);
3153 vm_page_free_pages_toq(&free, true);
3154 }
3155 }
3156
3157 /*
3158 * Tries to create a read- and/or execute-only 2MB page mapping. Returns true
3159 * if successful. Returns false if (1) a page table page cannot be allocated
3160 * without sleeping, (2) a mapping already exists at the specified virtual
3161 * address, or (3) a PV entry cannot be allocated without reclaiming another
3162 * PV entry.
3163 */
3164 static bool
pmap_enter_2mpage(pmap_t pmap,vm_offset_t va,vm_page_t m,vm_prot_t prot,struct rwlock ** lockp)3165 pmap_enter_2mpage(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,
3166 struct rwlock **lockp)
3167 {
3168 pml3_entry_t newpde;
3169
3170 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
3171 newpde = VM_PAGE_TO_PHYS(m) | pmap_cache_bits(m->md.mdpg_cache_attrs) |
3172 RPTE_LEAF | PG_V;
3173 if ((m->oflags & VPO_UNMANAGED) == 0)
3174 newpde |= PG_MANAGED;
3175 if (prot & VM_PROT_EXECUTE)
3176 newpde |= PG_X;
3177 if (prot & VM_PROT_READ)
3178 newpde |= RPTE_EAA_R;
3179 if (va >= DMAP_MIN_ADDRESS)
3180 newpde |= RPTE_EAA_P;
3181 return (pmap_enter_l3e(pmap, va, newpde, PMAP_ENTER_NOSLEEP |
3182 PMAP_ENTER_NOREPLACE | PMAP_ENTER_NORECLAIM, NULL, lockp) ==
3183 KERN_SUCCESS);
3184 }
3185
3186 /*
3187 * Tries to create the specified 2MB page mapping. Returns KERN_SUCCESS if
3188 * the mapping was created, and either KERN_FAILURE or KERN_RESOURCE_SHORTAGE
3189 * otherwise. Returns KERN_FAILURE if PMAP_ENTER_NOREPLACE was specified and
3190 * a mapping already exists at the specified virtual address. Returns
3191 * KERN_RESOURCE_SHORTAGE if PMAP_ENTER_NOSLEEP was specified and a page table
3192 * page allocation failed. Returns KERN_RESOURCE_SHORTAGE if
3193 * PMAP_ENTER_NORECLAIM was specified and a PV entry allocation failed.
3194 *
3195 * The parameter "m" is only used when creating a managed, writeable mapping.
3196 */
3197 static int
pmap_enter_l3e(pmap_t pmap,vm_offset_t va,pml3_entry_t newpde,u_int flags,vm_page_t m,struct rwlock ** lockp)3198 pmap_enter_l3e(pmap_t pmap, vm_offset_t va, pml3_entry_t newpde, u_int flags,
3199 vm_page_t m, struct rwlock **lockp)
3200 {
3201 struct spglist free;
3202 pml3_entry_t oldl3e, *l3e;
3203 vm_page_t mt, pdpg;
3204 vm_page_t uwptpg;
3205
3206 KASSERT((newpde & (PG_M | PG_RW)) != PG_RW,
3207 ("pmap_enter_pde: newpde is missing PG_M"));
3208 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
3209
3210 if ((pdpg = pmap_allocl3e(pmap, va, (flags & PMAP_ENTER_NOSLEEP) != 0 ?
3211 NULL : lockp)) == NULL) {
3212 CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx"
3213 " in pmap %p", va, pmap);
3214 return (KERN_RESOURCE_SHORTAGE);
3215 }
3216 l3e = VM_PAGE_TO_DMAP(pdpg);
3217 l3e = &l3e[pmap_pml3e_index(va)];
3218 oldl3e = be64toh(*l3e);
3219 if ((oldl3e & PG_V) != 0) {
3220 KASSERT(pdpg->ref_count > 1,
3221 ("pmap_enter_pde: pdpg's wire count is too low"));
3222 if ((flags & PMAP_ENTER_NOREPLACE) != 0) {
3223 pdpg->ref_count--;
3224 CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx"
3225 " in pmap %p", va, pmap);
3226 return (KERN_FAILURE);
3227 }
3228 /* Break the existing mapping(s). */
3229 SLIST_INIT(&free);
3230 if ((oldl3e & RPTE_LEAF) != 0) {
3231 /*
3232 * The reference to the PD page that was acquired by
3233 * pmap_allocl3e() ensures that it won't be freed.
3234 * However, if the PDE resulted from a promotion, then
3235 * a reserved PT page could be freed.
3236 */
3237 (void)pmap_remove_l3e(pmap, l3e, va, &free, lockp);
3238 pmap_invalidate_l3e_page(pmap, va, oldl3e);
3239 } else {
3240 if (pmap_remove_ptes(pmap, va, va + L3_PAGE_SIZE, l3e,
3241 &free, lockp))
3242 pmap_invalidate_all(pmap);
3243 }
3244 vm_page_free_pages_toq(&free, true);
3245 if (va >= VM_MAXUSER_ADDRESS) {
3246 mt = PHYS_TO_VM_PAGE(be64toh(*l3e) & PG_FRAME);
3247 if (pmap_insert_pt_page(pmap, mt)) {
3248 /*
3249 * XXX Currently, this can't happen because
3250 * we do not perform pmap_enter(psind == 1)
3251 * on the kernel pmap.
3252 */
3253 panic("pmap_enter_pde: trie insert failed");
3254 }
3255 } else
3256 KASSERT(be64toh(*l3e) == 0, ("pmap_enter_pde: non-zero pde %p",
3257 l3e));
3258 }
3259
3260 /*
3261 * Allocate leaf ptpage for wired userspace pages.
3262 */
3263 uwptpg = NULL;
3264 if ((newpde & PG_W) != 0 && pmap != kernel_pmap) {
3265 uwptpg = vm_page_alloc_noobj(VM_ALLOC_WIRED);
3266 if (uwptpg == NULL) {
3267 pmap_abort_ptp(pmap, va, pdpg);
3268 return (KERN_RESOURCE_SHORTAGE);
3269 }
3270 uwptpg->pindex = pmap_l3e_pindex(va);
3271 if (pmap_insert_pt_page(pmap, uwptpg)) {
3272 vm_page_unwire_noq(uwptpg);
3273 vm_page_free(uwptpg);
3274 pmap_abort_ptp(pmap, va, pdpg);
3275 return (KERN_RESOURCE_SHORTAGE);
3276 }
3277 pmap_resident_count_inc(pmap, 1);
3278 uwptpg->ref_count = NPTEPG;
3279 pmap_fill_ptp(VM_PAGE_TO_DMAP(uwptpg), newpde);
3280 }
3281 if ((newpde & PG_MANAGED) != 0) {
3282 /*
3283 * Abort this mapping if its PV entry could not be created.
3284 */
3285 if (!pmap_pv_insert_l3e(pmap, va, newpde, flags, lockp)) {
3286 pmap_abort_ptp(pmap, va, pdpg);
3287 if (uwptpg != NULL) {
3288 mt = pmap_remove_pt_page(pmap, va);
3289 KASSERT(mt == uwptpg,
3290 ("removed pt page %p, expected %p", mt,
3291 uwptpg));
3292 pmap_resident_count_dec(pmap, 1);
3293 uwptpg->ref_count = 1;
3294 vm_page_unwire_noq(uwptpg);
3295 vm_page_free(uwptpg);
3296 }
3297 CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx"
3298 " in pmap %p", va, pmap);
3299 return (KERN_RESOURCE_SHORTAGE);
3300 }
3301 if ((newpde & PG_RW) != 0) {
3302 for (mt = m; mt < &m[L3_PAGE_SIZE / PAGE_SIZE]; mt++)
3303 vm_page_aflag_set(mt, PGA_WRITEABLE);
3304 }
3305 }
3306
3307 /*
3308 * Increment counters.
3309 */
3310 if ((newpde & PG_W) != 0)
3311 pmap->pm_stats.wired_count += L3_PAGE_SIZE / PAGE_SIZE;
3312 pmap_resident_count_inc(pmap, L3_PAGE_SIZE / PAGE_SIZE);
3313
3314 /*
3315 * Map the superpage. (This is not a promoted mapping; there will not
3316 * be any lingering 4KB page mappings in the TLB.)
3317 */
3318 pte_store(l3e, newpde);
3319 ptesync();
3320
3321 counter_u64_add(pmap_l3e_mappings, 1);
3322 CTR2(KTR_PMAP, "pmap_enter_pde: success for va %#lx"
3323 " in pmap %p", va, pmap);
3324 return (KERN_SUCCESS);
3325 }
3326
3327 void
mmu_radix_enter_object(pmap_t pmap,vm_offset_t start,vm_offset_t end,vm_page_t m_start,vm_prot_t prot)3328 mmu_radix_enter_object(pmap_t pmap, vm_offset_t start,
3329 vm_offset_t end, vm_page_t m_start, vm_prot_t prot)
3330 {
3331 struct pctrie_iter pages;
3332 struct rwlock *lock;
3333 vm_offset_t va;
3334 vm_page_t m, mpte;
3335 bool invalidate;
3336
3337 VM_OBJECT_ASSERT_LOCKED(m_start->object);
3338
3339 CTR6(KTR_PMAP, "%s(%p, %#x, %#x, %p, %#x)", __func__, pmap, start,
3340 end, m_start, prot);
3341 invalidate = false;
3342 mpte = NULL;
3343 vm_page_iter_limit_init(&pages, m_start->object,
3344 m_start->pindex + atop(end - start));
3345 m = vm_radix_iter_lookup(&pages, m_start->pindex);
3346 lock = NULL;
3347 PMAP_LOCK(pmap);
3348 while (m != NULL) {
3349 va = start + ptoa(m->pindex - m_start->pindex);
3350 if ((va & L3_PAGE_MASK) == 0 && va + L3_PAGE_SIZE <= end &&
3351 m->psind == 1 && mmu_radix_ps_enabled(pmap) &&
3352 pmap_enter_2mpage(pmap, va, m, prot, &lock)) {
3353 m = vm_radix_iter_jump(&pages, L3_PAGE_SIZE / PAGE_SIZE);
3354 } else {
3355 mpte = mmu_radix_enter_quick_locked(pmap, va, m, prot,
3356 mpte, &lock, &invalidate);
3357 m = vm_radix_iter_step(&pages);
3358 }
3359 }
3360 ptesync();
3361 if (lock != NULL)
3362 rw_wunlock(lock);
3363 if (invalidate)
3364 pmap_invalidate_all(pmap);
3365 PMAP_UNLOCK(pmap);
3366 }
3367
3368 static vm_page_t
mmu_radix_enter_quick_locked(pmap_t pmap,vm_offset_t va,vm_page_t m,vm_prot_t prot,vm_page_t mpte,struct rwlock ** lockp,bool * invalidate)3369 mmu_radix_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m,
3370 vm_prot_t prot, vm_page_t mpte, struct rwlock **lockp, bool *invalidate)
3371 {
3372 struct spglist free;
3373 pt_entry_t *pte;
3374 vm_paddr_t pa;
3375
3376 KASSERT(!VA_IS_CLEANMAP(va) ||
3377 (m->oflags & VPO_UNMANAGED) != 0,
3378 ("mmu_radix_enter_quick_locked: managed mapping within the clean submap"));
3379 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
3380
3381 /*
3382 * In the case that a page table page is not
3383 * resident, we are creating it here.
3384 */
3385 if (va < VM_MAXUSER_ADDRESS) {
3386 vm_pindex_t ptepindex;
3387 pml3_entry_t *ptepa;
3388
3389 /*
3390 * Calculate pagetable page index
3391 */
3392 ptepindex = pmap_l3e_pindex(va);
3393 if (mpte && (mpte->pindex == ptepindex)) {
3394 mpte->ref_count++;
3395 } else {
3396 /*
3397 * Get the page directory entry
3398 */
3399 ptepa = pmap_pml3e(pmap, va);
3400
3401 /*
3402 * If the page table page is mapped, we just increment
3403 * the hold count, and activate it. Otherwise, we
3404 * attempt to allocate a page table page. If this
3405 * attempt fails, we don't retry. Instead, we give up.
3406 */
3407 if (ptepa && (be64toh(*ptepa) & PG_V) != 0) {
3408 if (be64toh(*ptepa) & RPTE_LEAF)
3409 return (NULL);
3410 mpte = PHYS_TO_VM_PAGE(be64toh(*ptepa) & PG_FRAME);
3411 mpte->ref_count++;
3412 } else {
3413 /*
3414 * Pass NULL instead of the PV list lock
3415 * pointer, because we don't intend to sleep.
3416 */
3417 mpte = _pmap_allocpte(pmap, ptepindex, NULL);
3418 if (mpte == NULL)
3419 return (mpte);
3420 }
3421 }
3422 pte = VM_PAGE_TO_DMAP(mpte);
3423 pte = &pte[pmap_pte_index(va)];
3424 } else {
3425 mpte = NULL;
3426 pte = pmap_pte(pmap, va);
3427 }
3428 if (be64toh(*pte)) {
3429 if (mpte != NULL) {
3430 mpte->ref_count--;
3431 mpte = NULL;
3432 }
3433 return (mpte);
3434 }
3435
3436 /*
3437 * Enter on the PV list if part of our managed memory.
3438 */
3439 if ((m->oflags & VPO_UNMANAGED) == 0 &&
3440 !pmap_try_insert_pv_entry(pmap, va, m, lockp)) {
3441 if (mpte != NULL) {
3442 SLIST_INIT(&free);
3443 if (pmap_unwire_ptp(pmap, va, mpte, &free)) {
3444 /*
3445 * Although "va" is not mapped, paging-
3446 * structure caches could nonetheless have
3447 * entries that refer to the freed page table
3448 * pages. Invalidate those entries.
3449 */
3450 *invalidate = true;
3451 vm_page_free_pages_toq(&free, true);
3452 }
3453 mpte = NULL;
3454 }
3455 return (mpte);
3456 }
3457
3458 /*
3459 * Increment counters
3460 */
3461 pmap_resident_count_inc(pmap, 1);
3462
3463 pa = VM_PAGE_TO_PHYS(m) | pmap_cache_bits(m->md.mdpg_cache_attrs);
3464 if (prot & VM_PROT_EXECUTE)
3465 pa |= PG_X;
3466 else
3467 pa |= RPTE_EAA_R;
3468 if ((m->oflags & VPO_UNMANAGED) == 0)
3469 pa |= PG_MANAGED;
3470
3471 pte_store(pte, pa);
3472 return (mpte);
3473 }
3474
3475 void
mmu_radix_enter_quick(pmap_t pmap,vm_offset_t va,vm_page_t m,vm_prot_t prot)3476 mmu_radix_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m,
3477 vm_prot_t prot)
3478 {
3479 struct rwlock *lock;
3480 bool invalidate;
3481
3482 lock = NULL;
3483 invalidate = false;
3484 PMAP_LOCK(pmap);
3485 mmu_radix_enter_quick_locked(pmap, va, m, prot, NULL, &lock,
3486 &invalidate);
3487 ptesync();
3488 if (lock != NULL)
3489 rw_wunlock(lock);
3490 if (invalidate)
3491 pmap_invalidate_all(pmap);
3492 PMAP_UNLOCK(pmap);
3493 }
3494
3495 vm_paddr_t
mmu_radix_extract(pmap_t pmap,vm_offset_t va)3496 mmu_radix_extract(pmap_t pmap, vm_offset_t va)
3497 {
3498 pml3_entry_t *l3e;
3499 pt_entry_t *pte;
3500 vm_paddr_t pa;
3501
3502 l3e = pmap_pml3e(pmap, va);
3503 if (__predict_false(l3e == NULL))
3504 return (0);
3505 if (be64toh(*l3e) & RPTE_LEAF) {
3506 pa = (be64toh(*l3e) & PG_PS_FRAME) | (va & L3_PAGE_MASK);
3507 pa |= (va & L3_PAGE_MASK);
3508 } else {
3509 /*
3510 * Beware of a concurrent promotion that changes the
3511 * PDE at this point! For example, vtopte() must not
3512 * be used to access the PTE because it would use the
3513 * new PDE. It is, however, safe to use the old PDE
3514 * because the page table page is preserved by the
3515 * promotion.
3516 */
3517 pte = pmap_l3e_to_pte(l3e, va);
3518 if (__predict_false(pte == NULL))
3519 return (0);
3520 pa = be64toh(*pte);
3521 pa = (pa & PG_FRAME) | (va & PAGE_MASK);
3522 pa |= (va & PAGE_MASK);
3523 }
3524 return (pa);
3525 }
3526
3527 vm_page_t
mmu_radix_extract_and_hold(pmap_t pmap,vm_offset_t va,vm_prot_t prot)3528 mmu_radix_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot)
3529 {
3530 pml3_entry_t l3e, *l3ep;
3531 pt_entry_t pte;
3532 vm_page_t m;
3533
3534 m = NULL;
3535 CTR4(KTR_PMAP, "%s(%p, %#x, %#x)", __func__, pmap, va, prot);
3536 PMAP_LOCK(pmap);
3537 l3ep = pmap_pml3e(pmap, va);
3538 if (l3ep != NULL && (l3e = be64toh(*l3ep))) {
3539 if (l3e & RPTE_LEAF) {
3540 if ((l3e & PG_RW) || (prot & VM_PROT_WRITE) == 0)
3541 m = PHYS_TO_VM_PAGE((l3e & PG_PS_FRAME) |
3542 (va & L3_PAGE_MASK));
3543 } else {
3544 /* Native endian PTE, do not pass to pmap functions */
3545 pte = be64toh(*pmap_l3e_to_pte(l3ep, va));
3546 if ((pte & PG_V) &&
3547 ((pte & PG_RW) || (prot & VM_PROT_WRITE) == 0))
3548 m = PHYS_TO_VM_PAGE(pte & PG_FRAME);
3549 }
3550 if (m != NULL && !vm_page_wire_mapped(m))
3551 m = NULL;
3552 }
3553 PMAP_UNLOCK(pmap);
3554 return (m);
3555 }
3556
3557 static int
mmu_radix_growkernel(vm_offset_t addr)3558 mmu_radix_growkernel(vm_offset_t addr)
3559 {
3560 vm_paddr_t paddr;
3561 vm_page_t nkpg;
3562 pml3_entry_t *l3e;
3563 pml2_entry_t *l2e;
3564
3565 CTR2(KTR_PMAP, "%s(%#x)", __func__, addr);
3566 if (VM_MIN_KERNEL_ADDRESS < addr &&
3567 addr < (VM_MIN_KERNEL_ADDRESS + nkpt * L3_PAGE_SIZE))
3568 return (KERN_SUCCESS);
3569
3570 addr = roundup2(addr, L3_PAGE_SIZE);
3571 if (addr - 1 >= vm_map_max(kernel_map))
3572 addr = vm_map_max(kernel_map);
3573 while (kernel_vm_end < addr) {
3574 l2e = pmap_pml2e(kernel_pmap, kernel_vm_end);
3575 if ((be64toh(*l2e) & PG_V) == 0) {
3576 /* We need a new PDP entry */
3577 nkpg = vm_page_alloc_noobj(VM_ALLOC_INTERRUPT |
3578 VM_ALLOC_NOFREE | VM_ALLOC_WIRED | VM_ALLOC_ZERO);
3579 if (nkpg == NULL)
3580 return (KERN_RESOURCE_SHORTAGE);
3581 nkpg->pindex = kernel_vm_end >> L2_PAGE_SIZE_SHIFT;
3582 paddr = VM_PAGE_TO_PHYS(nkpg);
3583 pde_store(l2e, paddr);
3584 continue; /* try again */
3585 }
3586 l3e = pmap_l2e_to_l3e(l2e, kernel_vm_end);
3587 if ((be64toh(*l3e) & PG_V) != 0) {
3588 kernel_vm_end = (kernel_vm_end + L3_PAGE_SIZE) & ~L3_PAGE_MASK;
3589 if (kernel_vm_end - 1 >= vm_map_max(kernel_map)) {
3590 kernel_vm_end = vm_map_max(kernel_map);
3591 break;
3592 }
3593 continue;
3594 }
3595
3596 nkpg = vm_page_alloc_noobj(VM_ALLOC_INTERRUPT |
3597 VM_ALLOC_NOFREE | VM_ALLOC_WIRED | VM_ALLOC_ZERO);
3598 if (nkpg == NULL)
3599 return (KERN_RESOURCE_SHORTAGE);
3600 nkpg->pindex = pmap_l3e_pindex(kernel_vm_end);
3601 paddr = VM_PAGE_TO_PHYS(nkpg);
3602 pde_store(l3e, paddr);
3603
3604 kernel_vm_end = (kernel_vm_end + L3_PAGE_SIZE) & ~L3_PAGE_MASK;
3605 if (kernel_vm_end - 1 >= vm_map_max(kernel_map)) {
3606 kernel_vm_end = vm_map_max(kernel_map);
3607 break;
3608 }
3609 }
3610 ptesync();
3611 return (KERN_SUCCESS);
3612 }
3613
3614 static MALLOC_DEFINE(M_RADIX_PGD, "radix_pgd", "radix page table root directory");
3615 static uma_zone_t zone_radix_pgd;
3616
3617 static int
radix_pgd_import(void * arg __unused,void ** store,int count,int domain __unused,int flags)3618 radix_pgd_import(void *arg __unused, void **store, int count, int domain __unused,
3619 int flags)
3620 {
3621 int req;
3622
3623 req = VM_ALLOC_WIRED | malloc2vm_flags(flags);
3624 for (int i = 0; i < count; i++) {
3625 vm_page_t m = vm_page_alloc_noobj_contig(req,
3626 RADIX_PGD_SIZE / PAGE_SIZE,
3627 0, (vm_paddr_t)-1, RADIX_PGD_SIZE, L1_PAGE_SIZE,
3628 VM_MEMATTR_DEFAULT);
3629 store[i] = VM_PAGE_TO_DMAP(m);
3630 }
3631 return (count);
3632 }
3633
3634 static void
radix_pgd_release(void * arg __unused,void ** store,int count)3635 radix_pgd_release(void *arg __unused, void **store, int count)
3636 {
3637 vm_page_t m;
3638 struct spglist free;
3639 int page_count;
3640
3641 SLIST_INIT(&free);
3642 page_count = RADIX_PGD_SIZE/PAGE_SIZE;
3643
3644 for (int i = 0; i < count; i++) {
3645 /*
3646 * XXX selectively remove dmap and KVA entries so we don't
3647 * need to bzero
3648 */
3649 m = DMAP_TO_VM_PAGE(store[i]);
3650 for (int j = page_count-1; j >= 0; j--) {
3651 vm_page_unwire_noq(&m[j]);
3652 SLIST_INSERT_HEAD(&free, &m[j], plinks.s.ss);
3653 }
3654 vm_page_free_pages_toq(&free, false);
3655 }
3656 }
3657
3658 static void
mmu_radix_init(void)3659 mmu_radix_init(void)
3660 {
3661 vm_page_t mpte;
3662 vm_size_t s;
3663 int error, i, pv_npg;
3664
3665 /* XXX is this really needed for POWER? */
3666 /* L1TF, reserve page @0 unconditionally */
3667 vm_page_blacklist_add(0, bootverbose);
3668
3669 zone_radix_pgd = uma_zcache_create("radix_pgd_cache",
3670 RADIX_PGD_SIZE, NULL, NULL,
3671 #ifdef INVARIANTS
3672 trash_init, trash_fini,
3673 #else
3674 NULL, NULL,
3675 #endif
3676 radix_pgd_import, radix_pgd_release,
3677 NULL, UMA_ZONE_NOBUCKET);
3678
3679 /*
3680 * Initialize the vm page array entries for the kernel pmap's
3681 * page table pages.
3682 */
3683 PMAP_LOCK(kernel_pmap);
3684 for (i = 0; i < nkpt; i++) {
3685 mpte = PHYS_TO_VM_PAGE(KPTphys + (i << PAGE_SHIFT));
3686 KASSERT(mpte >= vm_page_array &&
3687 mpte < &vm_page_array[vm_page_array_size],
3688 ("pmap_init: page table page is out of range size: %lu",
3689 vm_page_array_size));
3690 mpte->pindex = pmap_l3e_pindex(VM_MIN_KERNEL_ADDRESS) + i;
3691 mpte->phys_addr = KPTphys + (i << PAGE_SHIFT);
3692 MPASS(PHYS_TO_VM_PAGE(mpte->phys_addr) == mpte);
3693 //pmap_insert_pt_page(kernel_pmap, mpte);
3694 mpte->ref_count = 1;
3695 }
3696 PMAP_UNLOCK(kernel_pmap);
3697 vm_wire_add(nkpt);
3698
3699 CTR1(KTR_PMAP, "%s()", __func__);
3700 TAILQ_INIT(&pv_dummy.pv_list);
3701
3702 /*
3703 * Are large page mappings enabled?
3704 */
3705 TUNABLE_INT_FETCH("vm.pmap.superpages_enabled", &superpages_enabled);
3706 if (superpages_enabled) {
3707 KASSERT(MAXPAGESIZES > 1 && pagesizes[1] == 0,
3708 ("pmap_init: can't assign to pagesizes[1]"));
3709 pagesizes[1] = L3_PAGE_SIZE;
3710 }
3711
3712 /*
3713 * Initialize the pv chunk list mutex.
3714 */
3715 mtx_init(&pv_chunks_mutex, "pmap pv chunk list", NULL, MTX_DEF);
3716
3717 /*
3718 * Initialize the pool of pv list locks.
3719 */
3720 for (i = 0; i < NPV_LIST_LOCKS; i++)
3721 rw_init(&pv_list_locks[i], "pmap pv list");
3722
3723 /*
3724 * Calculate the size of the pv head table for superpages.
3725 */
3726 pv_npg = howmany(vm_phys_segs[vm_phys_nsegs - 1].end, L3_PAGE_SIZE);
3727
3728 /*
3729 * Allocate memory for the pv head table for superpages.
3730 */
3731 s = (vm_size_t)(pv_npg * sizeof(struct md_page));
3732 s = round_page(s);
3733 pv_table = kmem_malloc(s, M_WAITOK | M_ZERO);
3734 for (i = 0; i < pv_npg; i++)
3735 TAILQ_INIT(&pv_table[i].pv_list);
3736 TAILQ_INIT(&pv_dummy.pv_list);
3737
3738 pmap_initialized = 1;
3739 mtx_init(&qframe_mtx, "qfrmlk", NULL, MTX_SPIN);
3740 error = vmem_alloc(kernel_arena, PAGE_SIZE, M_BESTFIT | M_WAITOK,
3741 (vmem_addr_t *)&qframe);
3742
3743 if (error != 0)
3744 panic("qframe allocation failed");
3745 asid_arena = vmem_create("ASID", isa3_base_pid + 1, (1<<isa3_pid_bits),
3746 1, 1, M_WAITOK);
3747 }
3748
3749 static bool
pmap_page_test_mappings(vm_page_t m,bool accessed,bool modified)3750 pmap_page_test_mappings(vm_page_t m, bool accessed, bool modified)
3751 {
3752 struct rwlock *lock;
3753 pv_entry_t pv;
3754 struct md_page *pvh;
3755 pt_entry_t *pte, mask;
3756 pmap_t pmap;
3757 int md_gen, pvh_gen;
3758 bool rv;
3759
3760 rv = false;
3761 lock = VM_PAGE_TO_PV_LIST_LOCK(m);
3762 rw_rlock(lock);
3763 restart:
3764 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) {
3765 pmap = PV_PMAP(pv);
3766 if (!PMAP_TRYLOCK(pmap)) {
3767 md_gen = m->md.pv_gen;
3768 rw_runlock(lock);
3769 PMAP_LOCK(pmap);
3770 rw_rlock(lock);
3771 if (md_gen != m->md.pv_gen) {
3772 PMAP_UNLOCK(pmap);
3773 goto restart;
3774 }
3775 }
3776 pte = pmap_pte(pmap, pv->pv_va);
3777 mask = 0;
3778 if (modified)
3779 mask |= PG_RW | PG_M;
3780 if (accessed)
3781 mask |= PG_V | PG_A;
3782 rv = (be64toh(*pte) & mask) == mask;
3783 PMAP_UNLOCK(pmap);
3784 if (rv)
3785 goto out;
3786 }
3787 if ((m->flags & PG_FICTITIOUS) == 0) {
3788 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
3789 TAILQ_FOREACH(pv, &pvh->pv_list, pv_link) {
3790 pmap = PV_PMAP(pv);
3791 if (!PMAP_TRYLOCK(pmap)) {
3792 md_gen = m->md.pv_gen;
3793 pvh_gen = pvh->pv_gen;
3794 rw_runlock(lock);
3795 PMAP_LOCK(pmap);
3796 rw_rlock(lock);
3797 if (md_gen != m->md.pv_gen ||
3798 pvh_gen != pvh->pv_gen) {
3799 PMAP_UNLOCK(pmap);
3800 goto restart;
3801 }
3802 }
3803 pte = pmap_pml3e(pmap, pv->pv_va);
3804 mask = 0;
3805 if (modified)
3806 mask |= PG_RW | PG_M;
3807 if (accessed)
3808 mask |= PG_V | PG_A;
3809 rv = (be64toh(*pte) & mask) == mask;
3810 PMAP_UNLOCK(pmap);
3811 if (rv)
3812 goto out;
3813 }
3814 }
3815 out:
3816 rw_runlock(lock);
3817 return (rv);
3818 }
3819
3820 /*
3821 * pmap_is_modified:
3822 *
3823 * Return whether or not the specified physical page was modified
3824 * in any physical maps.
3825 */
3826 bool
mmu_radix_is_modified(vm_page_t m)3827 mmu_radix_is_modified(vm_page_t m)
3828 {
3829
3830 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
3831 ("pmap_is_modified: page %p is not managed", m));
3832
3833 CTR2(KTR_PMAP, "%s(%p)", __func__, m);
3834 /*
3835 * If the page is not busied then this check is racy.
3836 */
3837 if (!pmap_page_is_write_mapped(m))
3838 return (false);
3839 return (pmap_page_test_mappings(m, false, true));
3840 }
3841
3842 bool
mmu_radix_is_prefaultable(pmap_t pmap,vm_offset_t addr)3843 mmu_radix_is_prefaultable(pmap_t pmap, vm_offset_t addr)
3844 {
3845 pml3_entry_t *l3e;
3846 pt_entry_t *pte;
3847 bool rv;
3848
3849 CTR3(KTR_PMAP, "%s(%p, %#x)", __func__, pmap, addr);
3850 rv = false;
3851 PMAP_LOCK(pmap);
3852 l3e = pmap_pml3e(pmap, addr);
3853 if (l3e != NULL && (be64toh(*l3e) & (RPTE_LEAF | PG_V)) == PG_V) {
3854 pte = pmap_l3e_to_pte(l3e, addr);
3855 rv = (be64toh(*pte) & PG_V) == 0;
3856 }
3857 PMAP_UNLOCK(pmap);
3858 return (rv);
3859 }
3860
3861 bool
mmu_radix_is_referenced(vm_page_t m)3862 mmu_radix_is_referenced(vm_page_t m)
3863 {
3864 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
3865 ("pmap_is_referenced: page %p is not managed", m));
3866 CTR2(KTR_PMAP, "%s(%p)", __func__, m);
3867 return (pmap_page_test_mappings(m, true, false));
3868 }
3869
3870 /*
3871 * pmap_ts_referenced:
3872 *
3873 * Return a count of reference bits for a page, clearing those bits.
3874 * It is not necessary for every reference bit to be cleared, but it
3875 * is necessary that 0 only be returned when there are truly no
3876 * reference bits set.
3877 *
3878 * As an optimization, update the page's dirty field if a modified bit is
3879 * found while counting reference bits. This opportunistic update can be
3880 * performed at low cost and can eliminate the need for some future calls
3881 * to pmap_is_modified(). However, since this function stops after
3882 * finding PMAP_TS_REFERENCED_MAX reference bits, it may not detect some
3883 * dirty pages. Those dirty pages will only be detected by a future call
3884 * to pmap_is_modified().
3885 *
3886 * A DI block is not needed within this function, because
3887 * invalidations are performed before the PV list lock is
3888 * released.
3889 */
3890 int
mmu_radix_ts_referenced(vm_page_t m)3891 mmu_radix_ts_referenced(vm_page_t m)
3892 {
3893 struct md_page *pvh;
3894 pv_entry_t pv, pvf;
3895 pmap_t pmap;
3896 struct rwlock *lock;
3897 pml3_entry_t oldl3e, *l3e;
3898 pt_entry_t *pte;
3899 vm_paddr_t pa;
3900 int cleared, md_gen, not_cleared, pvh_gen;
3901 struct spglist free;
3902
3903 CTR2(KTR_PMAP, "%s(%p)", __func__, m);
3904 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
3905 ("pmap_ts_referenced: page %p is not managed", m));
3906 SLIST_INIT(&free);
3907 cleared = 0;
3908 pa = VM_PAGE_TO_PHYS(m);
3909 lock = PHYS_TO_PV_LIST_LOCK(pa);
3910 pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy : pa_to_pvh(pa);
3911 rw_wlock(lock);
3912 retry:
3913 not_cleared = 0;
3914 if ((pvf = TAILQ_FIRST(&pvh->pv_list)) == NULL)
3915 goto small_mappings;
3916 pv = pvf;
3917 do {
3918 if (pvf == NULL)
3919 pvf = pv;
3920 pmap = PV_PMAP(pv);
3921 if (!PMAP_TRYLOCK(pmap)) {
3922 pvh_gen = pvh->pv_gen;
3923 rw_wunlock(lock);
3924 PMAP_LOCK(pmap);
3925 rw_wlock(lock);
3926 if (pvh_gen != pvh->pv_gen) {
3927 PMAP_UNLOCK(pmap);
3928 goto retry;
3929 }
3930 }
3931 l3e = pmap_pml3e(pmap, pv->pv_va);
3932 oldl3e = be64toh(*l3e);
3933 if ((oldl3e & (PG_M | PG_RW)) == (PG_M | PG_RW)) {
3934 /*
3935 * Although "oldpde" is mapping a 2MB page, because
3936 * this function is called at a 4KB page granularity,
3937 * we only update the 4KB page under test.
3938 */
3939 vm_page_dirty(m);
3940 }
3941 if ((oldl3e & PG_A) != 0) {
3942 /*
3943 * Since this reference bit is shared by 512 4KB
3944 * pages, it should not be cleared every time it is
3945 * tested. Apply a simple "hash" function on the
3946 * physical page number, the virtual superpage number,
3947 * and the pmap address to select one 4KB page out of
3948 * the 512 on which testing the reference bit will
3949 * result in clearing that reference bit. This
3950 * function is designed to avoid the selection of the
3951 * same 4KB page for every 2MB page mapping.
3952 *
3953 * On demotion, a mapping that hasn't been referenced
3954 * is simply destroyed. To avoid the possibility of a
3955 * subsequent page fault on a demoted wired mapping,
3956 * always leave its reference bit set. Moreover,
3957 * since the superpage is wired, the current state of
3958 * its reference bit won't affect page replacement.
3959 */
3960 if ((((pa >> PAGE_SHIFT) ^ (pv->pv_va >> L3_PAGE_SIZE_SHIFT) ^
3961 (uintptr_t)pmap) & (NPTEPG - 1)) == 0 &&
3962 (oldl3e & PG_W) == 0) {
3963 atomic_clear_long(l3e, htobe64(PG_A));
3964 pmap_invalidate_page(pmap, pv->pv_va);
3965 cleared++;
3966 KASSERT(lock == VM_PAGE_TO_PV_LIST_LOCK(m),
3967 ("inconsistent pv lock %p %p for page %p",
3968 lock, VM_PAGE_TO_PV_LIST_LOCK(m), m));
3969 } else
3970 not_cleared++;
3971 }
3972 PMAP_UNLOCK(pmap);
3973 /* Rotate the PV list if it has more than one entry. */
3974 if (pv != NULL && TAILQ_NEXT(pv, pv_link) != NULL) {
3975 TAILQ_REMOVE(&pvh->pv_list, pv, pv_link);
3976 TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_link);
3977 pvh->pv_gen++;
3978 }
3979 if (cleared + not_cleared >= PMAP_TS_REFERENCED_MAX)
3980 goto out;
3981 } while ((pv = TAILQ_FIRST(&pvh->pv_list)) != pvf);
3982 small_mappings:
3983 if ((pvf = TAILQ_FIRST(&m->md.pv_list)) == NULL)
3984 goto out;
3985 pv = pvf;
3986 do {
3987 if (pvf == NULL)
3988 pvf = pv;
3989 pmap = PV_PMAP(pv);
3990 if (!PMAP_TRYLOCK(pmap)) {
3991 pvh_gen = pvh->pv_gen;
3992 md_gen = m->md.pv_gen;
3993 rw_wunlock(lock);
3994 PMAP_LOCK(pmap);
3995 rw_wlock(lock);
3996 if (pvh_gen != pvh->pv_gen || md_gen != m->md.pv_gen) {
3997 PMAP_UNLOCK(pmap);
3998 goto retry;
3999 }
4000 }
4001 l3e = pmap_pml3e(pmap, pv->pv_va);
4002 KASSERT((be64toh(*l3e) & RPTE_LEAF) == 0,
4003 ("pmap_ts_referenced: found a 2mpage in page %p's pv list",
4004 m));
4005 pte = pmap_l3e_to_pte(l3e, pv->pv_va);
4006 if ((be64toh(*pte) & (PG_M | PG_RW)) == (PG_M | PG_RW))
4007 vm_page_dirty(m);
4008 if ((be64toh(*pte) & PG_A) != 0) {
4009 atomic_clear_long(pte, htobe64(PG_A));
4010 pmap_invalidate_page(pmap, pv->pv_va);
4011 cleared++;
4012 }
4013 PMAP_UNLOCK(pmap);
4014 /* Rotate the PV list if it has more than one entry. */
4015 if (pv != NULL && TAILQ_NEXT(pv, pv_link) != NULL) {
4016 TAILQ_REMOVE(&m->md.pv_list, pv, pv_link);
4017 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_link);
4018 m->md.pv_gen++;
4019 }
4020 } while ((pv = TAILQ_FIRST(&m->md.pv_list)) != pvf && cleared +
4021 not_cleared < PMAP_TS_REFERENCED_MAX);
4022 out:
4023 rw_wunlock(lock);
4024 vm_page_free_pages_toq(&free, true);
4025 return (cleared + not_cleared);
4026 }
4027
4028 static void *
mmu_radix_map(vm_offset_t * virt __unused,vm_paddr_t start,vm_paddr_t end,int prot __unused)4029 mmu_radix_map(vm_offset_t *virt __unused, vm_paddr_t start,
4030 vm_paddr_t end, int prot __unused)
4031 {
4032
4033 CTR5(KTR_PMAP, "%s(%p, %#x, %#x, %#x)", __func__, virt, start, end,
4034 prot);
4035 return (PHYS_TO_DMAP(start));
4036 }
4037
4038 void
mmu_radix_object_init_pt(pmap_t pmap,vm_offset_t addr,vm_object_t object,vm_pindex_t pindex,vm_size_t size)4039 mmu_radix_object_init_pt(pmap_t pmap, vm_offset_t addr,
4040 vm_object_t object, vm_pindex_t pindex, vm_size_t size)
4041 {
4042 struct pctrie_iter pages;
4043 pml3_entry_t *l3e;
4044 vm_paddr_t pa, ptepa;
4045 vm_page_t p, pdpg;
4046 vm_memattr_t ma;
4047
4048 CTR6(KTR_PMAP, "%s(%p, %#x, %p, %u, %#x)", __func__, pmap, addr,
4049 object, pindex, size);
4050 VM_OBJECT_ASSERT_WLOCKED(object);
4051 KASSERT(object->type == OBJT_DEVICE || object->type == OBJT_SG,
4052 ("pmap_object_init_pt: non-device object"));
4053 /* NB: size can be logically ored with addr here */
4054 if ((addr & L3_PAGE_MASK) == 0 && (size & L3_PAGE_MASK) == 0) {
4055 if (!mmu_radix_ps_enabled(pmap))
4056 return;
4057 if (!vm_object_populate(object, pindex, pindex + atop(size)))
4058 return;
4059 vm_page_iter_init(&pages, object);
4060 p = vm_radix_iter_lookup(&pages, pindex);
4061
4062 KASSERT(p->valid == VM_PAGE_BITS_ALL,
4063 ("pmap_object_init_pt: invalid page %p", p));
4064 ma = p->md.mdpg_cache_attrs;
4065
4066 /*
4067 * Abort the mapping if the first page is not physically
4068 * aligned to a 2MB page boundary.
4069 */
4070 ptepa = VM_PAGE_TO_PHYS(p);
4071 if (ptepa & L3_PAGE_MASK)
4072 return;
4073
4074 /*
4075 * Skip the first page. Abort the mapping if the rest of
4076 * the pages are not physically contiguous or have differing
4077 * memory attributes.
4078 */
4079 for (pa = ptepa + PAGE_SIZE; pa < ptepa + size;
4080 pa += PAGE_SIZE) {
4081 p = vm_radix_iter_next(&pages);
4082 KASSERT(p->valid == VM_PAGE_BITS_ALL,
4083 ("pmap_object_init_pt: invalid page %p", p));
4084 if (pa != VM_PAGE_TO_PHYS(p) ||
4085 ma != p->md.mdpg_cache_attrs)
4086 return;
4087 }
4088
4089 PMAP_LOCK(pmap);
4090 for (pa = ptepa | pmap_cache_bits(ma);
4091 pa < ptepa + size; pa += L3_PAGE_SIZE) {
4092 pdpg = pmap_allocl3e(pmap, addr, NULL);
4093 if (pdpg == NULL) {
4094 /*
4095 * The creation of mappings below is only an
4096 * optimization. If a page directory page
4097 * cannot be allocated without blocking,
4098 * continue on to the next mapping rather than
4099 * blocking.
4100 */
4101 addr += L3_PAGE_SIZE;
4102 continue;
4103 }
4104 l3e = VM_PAGE_TO_DMAP(pdpg);
4105 l3e = &l3e[pmap_pml3e_index(addr)];
4106 if ((be64toh(*l3e) & PG_V) == 0) {
4107 pa |= PG_M | PG_A | PG_RW;
4108 pte_store(l3e, pa);
4109 pmap_resident_count_inc(pmap, L3_PAGE_SIZE / PAGE_SIZE);
4110 counter_u64_add(pmap_l3e_mappings, 1);
4111 } else {
4112 /* Continue on if the PDE is already valid. */
4113 pdpg->ref_count--;
4114 KASSERT(pdpg->ref_count > 0,
4115 ("pmap_object_init_pt: missing reference "
4116 "to page directory page, va: 0x%lx", addr));
4117 }
4118 addr += L3_PAGE_SIZE;
4119 }
4120 ptesync();
4121 PMAP_UNLOCK(pmap);
4122 }
4123 }
4124
4125 bool
mmu_radix_page_exists_quick(pmap_t pmap,vm_page_t m)4126 mmu_radix_page_exists_quick(pmap_t pmap, vm_page_t m)
4127 {
4128 struct md_page *pvh;
4129 struct rwlock *lock;
4130 pv_entry_t pv;
4131 int loops = 0;
4132 bool rv;
4133
4134 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
4135 ("pmap_page_exists_quick: page %p is not managed", m));
4136 CTR3(KTR_PMAP, "%s(%p, %p)", __func__, pmap, m);
4137 rv = false;
4138 lock = VM_PAGE_TO_PV_LIST_LOCK(m);
4139 rw_rlock(lock);
4140 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) {
4141 if (PV_PMAP(pv) == pmap) {
4142 rv = true;
4143 break;
4144 }
4145 loops++;
4146 if (loops >= 16)
4147 break;
4148 }
4149 if (!rv && loops < 16 && (m->flags & PG_FICTITIOUS) == 0) {
4150 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
4151 TAILQ_FOREACH(pv, &pvh->pv_list, pv_link) {
4152 if (PV_PMAP(pv) == pmap) {
4153 rv = true;
4154 break;
4155 }
4156 loops++;
4157 if (loops >= 16)
4158 break;
4159 }
4160 }
4161 rw_runlock(lock);
4162 return (rv);
4163 }
4164
4165 void
mmu_radix_page_init(vm_page_t m)4166 mmu_radix_page_init(vm_page_t m)
4167 {
4168
4169 CTR2(KTR_PMAP, "%s(%p)", __func__, m);
4170 TAILQ_INIT(&m->md.pv_list);
4171 m->md.mdpg_cache_attrs = VM_MEMATTR_DEFAULT;
4172 }
4173
4174 int
mmu_radix_page_wired_mappings(vm_page_t m)4175 mmu_radix_page_wired_mappings(vm_page_t m)
4176 {
4177 struct rwlock *lock;
4178 struct md_page *pvh;
4179 pmap_t pmap;
4180 pt_entry_t *pte;
4181 pv_entry_t pv;
4182 int count, md_gen, pvh_gen;
4183
4184 if ((m->oflags & VPO_UNMANAGED) != 0)
4185 return (0);
4186 CTR2(KTR_PMAP, "%s(%p)", __func__, m);
4187 lock = VM_PAGE_TO_PV_LIST_LOCK(m);
4188 rw_rlock(lock);
4189 restart:
4190 count = 0;
4191 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) {
4192 pmap = PV_PMAP(pv);
4193 if (!PMAP_TRYLOCK(pmap)) {
4194 md_gen = m->md.pv_gen;
4195 rw_runlock(lock);
4196 PMAP_LOCK(pmap);
4197 rw_rlock(lock);
4198 if (md_gen != m->md.pv_gen) {
4199 PMAP_UNLOCK(pmap);
4200 goto restart;
4201 }
4202 }
4203 pte = pmap_pte(pmap, pv->pv_va);
4204 if ((be64toh(*pte) & PG_W) != 0)
4205 count++;
4206 PMAP_UNLOCK(pmap);
4207 }
4208 if ((m->flags & PG_FICTITIOUS) == 0) {
4209 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
4210 TAILQ_FOREACH(pv, &pvh->pv_list, pv_link) {
4211 pmap = PV_PMAP(pv);
4212 if (!PMAP_TRYLOCK(pmap)) {
4213 md_gen = m->md.pv_gen;
4214 pvh_gen = pvh->pv_gen;
4215 rw_runlock(lock);
4216 PMAP_LOCK(pmap);
4217 rw_rlock(lock);
4218 if (md_gen != m->md.pv_gen ||
4219 pvh_gen != pvh->pv_gen) {
4220 PMAP_UNLOCK(pmap);
4221 goto restart;
4222 }
4223 }
4224 pte = pmap_pml3e(pmap, pv->pv_va);
4225 if ((be64toh(*pte) & PG_W) != 0)
4226 count++;
4227 PMAP_UNLOCK(pmap);
4228 }
4229 }
4230 rw_runlock(lock);
4231 return (count);
4232 }
4233
4234 static void
mmu_radix_update_proctab(int pid,pml1_entry_t l1pa)4235 mmu_radix_update_proctab(int pid, pml1_entry_t l1pa)
4236 {
4237 isa3_proctab[pid].proctab0 = htobe64(RTS_SIZE | l1pa | RADIX_PGD_INDEX_SHIFT);
4238 }
4239
4240 int
mmu_radix_pinit(pmap_t pmap)4241 mmu_radix_pinit(pmap_t pmap)
4242 {
4243 vmem_addr_t pid;
4244 vm_paddr_t l1pa;
4245
4246 CTR2(KTR_PMAP, "%s(%p)", __func__, pmap);
4247
4248 /*
4249 * allocate the page directory page
4250 */
4251 pmap->pm_pml1 = uma_zalloc(zone_radix_pgd, M_WAITOK);
4252
4253 for (int j = 0; j < RADIX_PGD_SIZE_SHIFT; j++)
4254 pagezero((char *)pmap->pm_pml1 + j * PAGE_SIZE);
4255 vm_radix_init(&pmap->pm_radix);
4256 TAILQ_INIT(&pmap->pm_pvchunk);
4257 bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
4258 pmap->pm_flags = PMAP_PDE_SUPERPAGE;
4259 vmem_alloc(asid_arena, 1, M_FIRSTFIT|M_WAITOK, &pid);
4260
4261 pmap->pm_pid = pid;
4262 l1pa = DMAP_TO_PHYS(pmap->pm_pml1);
4263 mmu_radix_update_proctab(pid, l1pa);
4264 __asm __volatile("ptesync;isync" : : : "memory");
4265
4266 return (1);
4267 }
4268
4269 /*
4270 * This routine is called if the desired page table page does not exist.
4271 *
4272 * If page table page allocation fails, this routine may sleep before
4273 * returning NULL. It sleeps only if a lock pointer was given.
4274 *
4275 * Note: If a page allocation fails at page table level two or three,
4276 * one or two pages may be held during the wait, only to be released
4277 * afterwards. This conservative approach is easily argued to avoid
4278 * race conditions.
4279 */
4280 static vm_page_t
_pmap_allocpte(pmap_t pmap,vm_pindex_t ptepindex,struct rwlock ** lockp)4281 _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, struct rwlock **lockp)
4282 {
4283 vm_page_t m, pdppg, pdpg;
4284
4285 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
4286
4287 /*
4288 * Allocate a page table page.
4289 */
4290 if ((m = vm_page_alloc_noobj(VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) {
4291 if (lockp != NULL) {
4292 RELEASE_PV_LIST_LOCK(lockp);
4293 PMAP_UNLOCK(pmap);
4294 vm_wait(NULL);
4295 PMAP_LOCK(pmap);
4296 }
4297 /*
4298 * Indicate the need to retry. While waiting, the page table
4299 * page may have been allocated.
4300 */
4301 return (NULL);
4302 }
4303 m->pindex = ptepindex;
4304
4305 /*
4306 * Map the pagetable page into the process address space, if
4307 * it isn't already there.
4308 */
4309
4310 if (ptepindex >= (NUPDE + NUPDPE)) {
4311 pml1_entry_t *l1e;
4312 vm_pindex_t pml1index;
4313
4314 /* Wire up a new PDPE page */
4315 pml1index = ptepindex - (NUPDE + NUPDPE);
4316 l1e = &pmap->pm_pml1[pml1index];
4317 KASSERT((be64toh(*l1e) & PG_V) == 0,
4318 ("%s: L1 entry %#lx is valid", __func__, *l1e));
4319 pde_store(l1e, VM_PAGE_TO_PHYS(m));
4320 } else if (ptepindex >= NUPDE) {
4321 vm_pindex_t pml1index;
4322 vm_pindex_t pdpindex;
4323 pml1_entry_t *l1e;
4324 pml2_entry_t *l2e;
4325
4326 /* Wire up a new l2e page */
4327 pdpindex = ptepindex - NUPDE;
4328 pml1index = pdpindex >> RPTE_SHIFT;
4329
4330 l1e = &pmap->pm_pml1[pml1index];
4331 if ((be64toh(*l1e) & PG_V) == 0) {
4332 /* Have to allocate a new pdp, recurse */
4333 if (_pmap_allocpte(pmap, NUPDE + NUPDPE + pml1index,
4334 lockp) == NULL) {
4335 vm_page_unwire_noq(m);
4336 vm_page_free_zero(m);
4337 return (NULL);
4338 }
4339 } else {
4340 /* Add reference to l2e page */
4341 pdppg = PHYS_TO_VM_PAGE(be64toh(*l1e) & PG_FRAME);
4342 pdppg->ref_count++;
4343 }
4344 l2e = PHYS_TO_DMAP(be64toh(*l1e) & PG_FRAME);
4345
4346 /* Now find the pdp page */
4347 l2e = &l2e[pdpindex & RPTE_MASK];
4348 KASSERT((be64toh(*l2e) & PG_V) == 0,
4349 ("%s: L2 entry %#lx is valid", __func__, *l2e));
4350 pde_store(l2e, VM_PAGE_TO_PHYS(m));
4351 } else {
4352 vm_pindex_t pml1index;
4353 vm_pindex_t pdpindex;
4354 pml1_entry_t *l1e;
4355 pml2_entry_t *l2e;
4356 pml3_entry_t *l3e;
4357
4358 /* Wire up a new PTE page */
4359 pdpindex = ptepindex >> RPTE_SHIFT;
4360 pml1index = pdpindex >> RPTE_SHIFT;
4361
4362 /* First, find the pdp and check that its valid. */
4363 l1e = &pmap->pm_pml1[pml1index];
4364 if ((be64toh(*l1e) & PG_V) == 0) {
4365 /* Have to allocate a new pd, recurse */
4366 if (_pmap_allocpte(pmap, NUPDE + pdpindex,
4367 lockp) == NULL) {
4368 vm_page_unwire_noq(m);
4369 vm_page_free_zero(m);
4370 return (NULL);
4371 }
4372 l2e = PHYS_TO_DMAP(be64toh(*l1e) & PG_FRAME);
4373 l2e = &l2e[pdpindex & RPTE_MASK];
4374 } else {
4375 l2e = PHYS_TO_DMAP(be64toh(*l1e) & PG_FRAME);
4376 l2e = &l2e[pdpindex & RPTE_MASK];
4377 if ((be64toh(*l2e) & PG_V) == 0) {
4378 /* Have to allocate a new pd, recurse */
4379 if (_pmap_allocpte(pmap, NUPDE + pdpindex,
4380 lockp) == NULL) {
4381 vm_page_unwire_noq(m);
4382 vm_page_free_zero(m);
4383 return (NULL);
4384 }
4385 } else {
4386 /* Add reference to the pd page */
4387 pdpg = PHYS_TO_VM_PAGE(be64toh(*l2e) & PG_FRAME);
4388 pdpg->ref_count++;
4389 }
4390 }
4391 l3e = PHYS_TO_DMAP(be64toh(*l2e) & PG_FRAME);
4392
4393 /* Now we know where the page directory page is */
4394 l3e = &l3e[ptepindex & RPTE_MASK];
4395 KASSERT((be64toh(*l3e) & PG_V) == 0,
4396 ("%s: L3 entry %#lx is valid", __func__, *l3e));
4397 pde_store(l3e, VM_PAGE_TO_PHYS(m));
4398 }
4399
4400 pmap_resident_count_inc(pmap, 1);
4401 return (m);
4402 }
4403 static vm_page_t
pmap_allocl3e(pmap_t pmap,vm_offset_t va,struct rwlock ** lockp)4404 pmap_allocl3e(pmap_t pmap, vm_offset_t va, struct rwlock **lockp)
4405 {
4406 vm_pindex_t pdpindex, ptepindex;
4407 pml2_entry_t *pdpe;
4408 vm_page_t pdpg;
4409
4410 retry:
4411 pdpe = pmap_pml2e(pmap, va);
4412 if (pdpe != NULL && (be64toh(*pdpe) & PG_V) != 0) {
4413 /* Add a reference to the pd page. */
4414 pdpg = PHYS_TO_VM_PAGE(be64toh(*pdpe) & PG_FRAME);
4415 pdpg->ref_count++;
4416 } else {
4417 /* Allocate a pd page. */
4418 ptepindex = pmap_l3e_pindex(va);
4419 pdpindex = ptepindex >> RPTE_SHIFT;
4420 pdpg = _pmap_allocpte(pmap, NUPDE + pdpindex, lockp);
4421 if (pdpg == NULL && lockp != NULL)
4422 goto retry;
4423 }
4424 return (pdpg);
4425 }
4426
4427 static vm_page_t
pmap_allocpte(pmap_t pmap,vm_offset_t va,struct rwlock ** lockp)4428 pmap_allocpte(pmap_t pmap, vm_offset_t va, struct rwlock **lockp)
4429 {
4430 vm_pindex_t ptepindex;
4431 pml3_entry_t *pd;
4432 vm_page_t m;
4433
4434 /*
4435 * Calculate pagetable page index
4436 */
4437 ptepindex = pmap_l3e_pindex(va);
4438 retry:
4439 /*
4440 * Get the page directory entry
4441 */
4442 pd = pmap_pml3e(pmap, va);
4443
4444 /*
4445 * This supports switching from a 2MB page to a
4446 * normal 4K page.
4447 */
4448 if (pd != NULL && (be64toh(*pd) & (RPTE_LEAF | PG_V)) == (RPTE_LEAF | PG_V)) {
4449 if (!pmap_demote_l3e_locked(pmap, pd, va, lockp)) {
4450 /*
4451 * Invalidation of the 2MB page mapping may have caused
4452 * the deallocation of the underlying PD page.
4453 */
4454 pd = NULL;
4455 }
4456 }
4457
4458 /*
4459 * If the page table page is mapped, we just increment the
4460 * hold count, and activate it.
4461 */
4462 if (pd != NULL && (be64toh(*pd) & PG_V) != 0) {
4463 m = PHYS_TO_VM_PAGE(be64toh(*pd) & PG_FRAME);
4464 m->ref_count++;
4465 } else {
4466 /*
4467 * Here if the pte page isn't mapped, or if it has been
4468 * deallocated.
4469 */
4470 m = _pmap_allocpte(pmap, ptepindex, lockp);
4471 if (m == NULL && lockp != NULL)
4472 goto retry;
4473 }
4474 return (m);
4475 }
4476
4477 static void
mmu_radix_pinit0(pmap_t pmap)4478 mmu_radix_pinit0(pmap_t pmap)
4479 {
4480
4481 CTR2(KTR_PMAP, "%s(%p)", __func__, pmap);
4482 PMAP_LOCK_INIT(pmap);
4483 pmap->pm_pml1 = kernel_pmap->pm_pml1;
4484 pmap->pm_pid = kernel_pmap->pm_pid;
4485
4486 vm_radix_init(&pmap->pm_radix);
4487 TAILQ_INIT(&pmap->pm_pvchunk);
4488 bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
4489 kernel_pmap->pm_flags =
4490 pmap->pm_flags = PMAP_PDE_SUPERPAGE;
4491 }
4492 /*
4493 * pmap_protect_l3e: do the things to protect a 2mpage in a process
4494 */
4495 static bool
pmap_protect_l3e(pmap_t pmap,pt_entry_t * l3e,vm_offset_t sva,vm_prot_t prot)4496 pmap_protect_l3e(pmap_t pmap, pt_entry_t *l3e, vm_offset_t sva, vm_prot_t prot)
4497 {
4498 pt_entry_t newpde, oldpde;
4499 vm_offset_t eva, va;
4500 vm_page_t m;
4501 bool anychanged;
4502
4503 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
4504 KASSERT((sva & L3_PAGE_MASK) == 0,
4505 ("pmap_protect_l3e: sva is not 2mpage aligned"));
4506 anychanged = false;
4507 retry:
4508 oldpde = newpde = be64toh(*l3e);
4509 if ((oldpde & (PG_MANAGED | PG_M | PG_RW)) ==
4510 (PG_MANAGED | PG_M | PG_RW)) {
4511 eva = sva + L3_PAGE_SIZE;
4512 for (va = sva, m = PHYS_TO_VM_PAGE(oldpde & PG_PS_FRAME);
4513 va < eva; va += PAGE_SIZE, m++)
4514 vm_page_dirty(m);
4515 }
4516 if ((prot & VM_PROT_WRITE) == 0) {
4517 newpde &= ~(PG_RW | PG_M);
4518 newpde |= RPTE_EAA_R;
4519 }
4520 if (prot & VM_PROT_EXECUTE)
4521 newpde |= PG_X;
4522 if (newpde != oldpde) {
4523 /*
4524 * As an optimization to future operations on this PDE, clear
4525 * PG_PROMOTED. The impending invalidation will remove any
4526 * lingering 4KB page mappings from the TLB.
4527 */
4528 if (!atomic_cmpset_long(l3e, htobe64(oldpde), htobe64(newpde & ~PG_PROMOTED)))
4529 goto retry;
4530 anychanged = true;
4531 }
4532 return (anychanged);
4533 }
4534
4535 void
mmu_radix_protect(pmap_t pmap,vm_offset_t sva,vm_offset_t eva,vm_prot_t prot)4536 mmu_radix_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,
4537 vm_prot_t prot)
4538 {
4539 vm_offset_t va_next;
4540 pml1_entry_t *l1e;
4541 pml2_entry_t *l2e;
4542 pml3_entry_t ptpaddr, *l3e;
4543 pt_entry_t *pte;
4544 bool anychanged;
4545
4546 CTR5(KTR_PMAP, "%s(%p, %#x, %#x, %#x)", __func__, pmap, sva, eva,
4547 prot);
4548
4549 KASSERT((prot & ~VM_PROT_ALL) == 0, ("invalid prot %x", prot));
4550 if (prot == VM_PROT_NONE) {
4551 mmu_radix_remove(pmap, sva, eva);
4552 return;
4553 }
4554
4555 if ((prot & (VM_PROT_WRITE|VM_PROT_EXECUTE)) ==
4556 (VM_PROT_WRITE|VM_PROT_EXECUTE))
4557 return;
4558
4559 #ifdef INVARIANTS
4560 if (VERBOSE_PROTECT || pmap_logging)
4561 printf("pmap_protect(%p, %#lx, %#lx, %x) - asid: %lu\n",
4562 pmap, sva, eva, prot, pmap->pm_pid);
4563 #endif
4564 anychanged = false;
4565
4566 PMAP_LOCK(pmap);
4567 for (; sva < eva; sva = va_next) {
4568 l1e = pmap_pml1e(pmap, sva);
4569 if ((be64toh(*l1e) & PG_V) == 0) {
4570 va_next = (sva + L1_PAGE_SIZE) & ~L1_PAGE_MASK;
4571 if (va_next < sva)
4572 va_next = eva;
4573 continue;
4574 }
4575
4576 l2e = pmap_l1e_to_l2e(l1e, sva);
4577 if ((be64toh(*l2e) & PG_V) == 0) {
4578 va_next = (sva + L2_PAGE_SIZE) & ~L2_PAGE_MASK;
4579 if (va_next < sva)
4580 va_next = eva;
4581 continue;
4582 }
4583
4584 va_next = (sva + L3_PAGE_SIZE) & ~L3_PAGE_MASK;
4585 if (va_next < sva)
4586 va_next = eva;
4587
4588 l3e = pmap_l2e_to_l3e(l2e, sva);
4589 ptpaddr = be64toh(*l3e);
4590
4591 /*
4592 * Weed out invalid mappings.
4593 */
4594 if (ptpaddr == 0)
4595 continue;
4596
4597 /*
4598 * Check for large page.
4599 */
4600 if ((ptpaddr & RPTE_LEAF) != 0) {
4601 /*
4602 * Are we protecting the entire large page? If not,
4603 * demote the mapping and fall through.
4604 */
4605 if (sva + L3_PAGE_SIZE == va_next && eva >= va_next) {
4606 if (pmap_protect_l3e(pmap, l3e, sva, prot))
4607 anychanged = true;
4608 continue;
4609 } else if (!pmap_demote_l3e(pmap, l3e, sva)) {
4610 /*
4611 * The large page mapping was destroyed.
4612 */
4613 continue;
4614 }
4615 }
4616
4617 if (va_next > eva)
4618 va_next = eva;
4619
4620 for (pte = pmap_l3e_to_pte(l3e, sva); sva != va_next; pte++,
4621 sva += PAGE_SIZE) {
4622 pt_entry_t obits, pbits;
4623 vm_page_t m;
4624
4625 retry:
4626 MPASS(pte == pmap_pte(pmap, sva));
4627 obits = pbits = be64toh(*pte);
4628 if ((pbits & PG_V) == 0)
4629 continue;
4630
4631 if ((prot & VM_PROT_WRITE) == 0) {
4632 if ((pbits & (PG_MANAGED | PG_M | PG_RW)) ==
4633 (PG_MANAGED | PG_M | PG_RW)) {
4634 m = PHYS_TO_VM_PAGE(pbits & PG_FRAME);
4635 vm_page_dirty(m);
4636 }
4637 pbits &= ~(PG_RW | PG_M);
4638 pbits |= RPTE_EAA_R;
4639 }
4640 if (prot & VM_PROT_EXECUTE)
4641 pbits |= PG_X;
4642
4643 if (pbits != obits) {
4644 if (!atomic_cmpset_long(pte, htobe64(obits), htobe64(pbits)))
4645 goto retry;
4646 if (obits & (PG_A|PG_M)) {
4647 anychanged = true;
4648 #ifdef INVARIANTS
4649 if (VERBOSE_PROTECT || pmap_logging)
4650 printf("%#lx %#lx -> %#lx\n",
4651 sva, obits, pbits);
4652 #endif
4653 }
4654 }
4655 }
4656 }
4657 if (anychanged)
4658 pmap_invalidate_all(pmap);
4659 PMAP_UNLOCK(pmap);
4660 }
4661
4662 void
mmu_radix_qenter(void * sva,vm_page_t * ma,int count)4663 mmu_radix_qenter(void *sva, vm_page_t *ma, int count)
4664 {
4665 pt_entry_t oldpte, pa, *pte;
4666 vm_page_t m;
4667 uint64_t cache_bits, attr_bits;
4668 vm_offset_t va;
4669
4670 CTR4(KTR_PMAP, "%s(%p, %p, %d)", __func__, sva, ma, count);
4671 oldpte = 0;
4672 attr_bits = RPTE_EAA_R | RPTE_EAA_W | RPTE_EAA_P | PG_M | PG_A;
4673 va = (vm_offset_t)sva;
4674 pte = kvtopte(va);
4675 while (va < (vm_offset_t)sva + PAGE_SIZE * count) {
4676 if (__predict_false((va & L3_PAGE_MASK) == 0))
4677 pte = kvtopte(va);
4678 MPASS(pte == pmap_pte(kernel_pmap, va));
4679
4680 /*
4681 * XXX there has to be a more efficient way than traversing
4682 * the page table every time - but go for correctness for
4683 * today
4684 */
4685
4686 m = *ma++;
4687 cache_bits = pmap_cache_bits(m->md.mdpg_cache_attrs);
4688 pa = VM_PAGE_TO_PHYS(m) | cache_bits | attr_bits;
4689 if (be64toh(*pte) != pa) {
4690 oldpte |= be64toh(*pte);
4691 pte_store(pte, pa);
4692 }
4693 va += PAGE_SIZE;
4694 pte++;
4695 }
4696 if (__predict_false((oldpte & RPTE_VALID) != 0))
4697 pmap_invalidate_range(kernel_pmap, (vm_offset_t)sva,
4698 (vm_offset_t)sva + count * PAGE_SIZE);
4699 else
4700 ptesync();
4701 }
4702
4703 void
mmu_radix_qremove(void * sva,int count)4704 mmu_radix_qremove(void *sva, int count)
4705 {
4706 vm_offset_t va;
4707 pt_entry_t *pte;
4708
4709 va = (vm_offset_t)sva;
4710 CTR3(KTR_PMAP, "%s(%p, %d)", __func__, sva, count);
4711 KASSERT(va >= VM_MIN_KERNEL_ADDRESS, ("usermode or dmap va %p", sva));
4712
4713 pte = kvtopte(va);
4714 while (va < (vm_offset_t)sva + PAGE_SIZE * count) {
4715 if (__predict_false((va & L3_PAGE_MASK) == 0))
4716 pte = kvtopte(va);
4717 pte_clear(pte);
4718 pte++;
4719 va += PAGE_SIZE;
4720 }
4721 pmap_invalidate_range(kernel_pmap, (vm_offset_t)sva, va);
4722 }
4723
4724 /***************************************************
4725 * Page table page management routines.....
4726 ***************************************************/
4727 /*
4728 * Schedule the specified unused page table page to be freed. Specifically,
4729 * add the page to the specified list of pages that will be released to the
4730 * physical memory manager after the TLB has been updated.
4731 */
4732 static __inline void
pmap_add_delayed_free_list(vm_page_t m,struct spglist * free,bool set_PG_ZERO)4733 pmap_add_delayed_free_list(vm_page_t m, struct spglist *free, bool set_PG_ZERO)
4734 {
4735
4736 if (set_PG_ZERO)
4737 m->flags |= PG_ZERO;
4738 else
4739 m->flags &= ~PG_ZERO;
4740 SLIST_INSERT_HEAD(free, m, plinks.s.ss);
4741 }
4742
4743 /*
4744 * Inserts the specified page table page into the specified pmap's collection
4745 * of idle page table pages. Each of a pmap's page table pages is responsible
4746 * for mapping a distinct range of virtual addresses. The pmap's collection is
4747 * ordered by this virtual address range.
4748 */
4749 static __inline int
pmap_insert_pt_page(pmap_t pmap,vm_page_t mpte)4750 pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte)
4751 {
4752
4753 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
4754 return (vm_radix_insert(&pmap->pm_radix, mpte));
4755 }
4756
4757 /*
4758 * Removes the page table page mapping the specified virtual address from the
4759 * specified pmap's collection of idle page table pages, and returns it.
4760 * Otherwise, returns NULL if there is no page table page corresponding to the
4761 * specified virtual address.
4762 */
4763 static __inline vm_page_t
pmap_remove_pt_page(pmap_t pmap,vm_offset_t va)4764 pmap_remove_pt_page(pmap_t pmap, vm_offset_t va)
4765 {
4766
4767 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
4768 return (vm_radix_remove(&pmap->pm_radix, pmap_l3e_pindex(va)));
4769 }
4770
4771 /*
4772 * Decrements a page table page's wire count, which is used to record the
4773 * number of valid page table entries within the page. If the wire count
4774 * drops to zero, then the page table page is unmapped. Returns true if the
4775 * page table page was unmapped and false otherwise.
4776 */
4777 static inline bool
pmap_unwire_ptp(pmap_t pmap,vm_offset_t va,vm_page_t m,struct spglist * free)4778 pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free)
4779 {
4780
4781 --m->ref_count;
4782 if (m->ref_count == 0) {
4783 _pmap_unwire_ptp(pmap, va, m, free);
4784 return (true);
4785 } else
4786 return (false);
4787 }
4788
4789 static void
_pmap_unwire_ptp(pmap_t pmap,vm_offset_t va,vm_page_t m,struct spglist * free)4790 _pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free)
4791 {
4792
4793 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
4794 /*
4795 * unmap the page table page
4796 */
4797 if (m->pindex >= NUPDE + NUPDPE) {
4798 /* PDP page */
4799 pml1_entry_t *pml1;
4800 pml1 = pmap_pml1e(pmap, va);
4801 *pml1 = 0;
4802 } else if (m->pindex >= NUPDE) {
4803 /* PD page */
4804 pml2_entry_t *l2e;
4805 l2e = pmap_pml2e(pmap, va);
4806 *l2e = 0;
4807 } else {
4808 /* PTE page */
4809 pml3_entry_t *l3e;
4810 l3e = pmap_pml3e(pmap, va);
4811 *l3e = 0;
4812 }
4813 pmap_resident_count_dec(pmap, 1);
4814 if (m->pindex < NUPDE) {
4815 /* We just released a PT, unhold the matching PD */
4816 vm_page_t pdpg;
4817
4818 pdpg = PHYS_TO_VM_PAGE(be64toh(*pmap_pml2e(pmap, va)) & PG_FRAME);
4819 pmap_unwire_ptp(pmap, va, pdpg, free);
4820 }
4821 else if (m->pindex >= NUPDE && m->pindex < (NUPDE + NUPDPE)) {
4822 /* We just released a PD, unhold the matching PDP */
4823 vm_page_t pdppg;
4824
4825 pdppg = PHYS_TO_VM_PAGE(be64toh(*pmap_pml1e(pmap, va)) & PG_FRAME);
4826 pmap_unwire_ptp(pmap, va, pdppg, free);
4827 }
4828
4829 /*
4830 * Put page on a list so that it is released after
4831 * *ALL* TLB shootdown is done
4832 */
4833 pmap_add_delayed_free_list(m, free, true);
4834 }
4835
4836 /*
4837 * After removing a page table entry, this routine is used to
4838 * conditionally free the page, and manage the hold/wire counts.
4839 */
4840 static int
pmap_unuse_pt(pmap_t pmap,vm_offset_t va,pml3_entry_t ptepde,struct spglist * free)4841 pmap_unuse_pt(pmap_t pmap, vm_offset_t va, pml3_entry_t ptepde,
4842 struct spglist *free)
4843 {
4844 vm_page_t mpte;
4845
4846 if (va >= VM_MAXUSER_ADDRESS)
4847 return (0);
4848 KASSERT(ptepde != 0, ("pmap_unuse_pt: ptepde != 0"));
4849 mpte = PHYS_TO_VM_PAGE(ptepde & PG_FRAME);
4850 return (pmap_unwire_ptp(pmap, va, mpte, free));
4851 }
4852
4853 void
mmu_radix_release(pmap_t pmap)4854 mmu_radix_release(pmap_t pmap)
4855 {
4856
4857 CTR2(KTR_PMAP, "%s(%p)", __func__, pmap);
4858 KASSERT(pmap->pm_stats.resident_count == 0,
4859 ("pmap_release: pmap resident count %ld != 0",
4860 pmap->pm_stats.resident_count));
4861 KASSERT(vm_radix_is_empty(&pmap->pm_radix),
4862 ("pmap_release: pmap has reserved page table page(s)"));
4863
4864 pmap_invalidate_all(pmap);
4865 isa3_proctab[pmap->pm_pid].proctab0 = 0;
4866 uma_zfree(zone_radix_pgd, pmap->pm_pml1);
4867 vmem_free(asid_arena, pmap->pm_pid, 1);
4868 }
4869
4870 /*
4871 * Create the PV entry for a 2MB page mapping. Always returns true unless the
4872 * flag PMAP_ENTER_NORECLAIM is specified. If that flag is specified, returns
4873 * false if the PV entry cannot be allocated without resorting to reclamation.
4874 */
4875 static bool
pmap_pv_insert_l3e(pmap_t pmap,vm_offset_t va,pml3_entry_t pde,u_int flags,struct rwlock ** lockp)4876 pmap_pv_insert_l3e(pmap_t pmap, vm_offset_t va, pml3_entry_t pde, u_int flags,
4877 struct rwlock **lockp)
4878 {
4879 struct md_page *pvh;
4880 pv_entry_t pv;
4881 vm_paddr_t pa;
4882
4883 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
4884 /* Pass NULL instead of the lock pointer to disable reclamation. */
4885 if ((pv = get_pv_entry(pmap, (flags & PMAP_ENTER_NORECLAIM) != 0 ?
4886 NULL : lockp)) == NULL)
4887 return (false);
4888 pv->pv_va = va;
4889 pa = pde & PG_PS_FRAME;
4890 CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa);
4891 pvh = pa_to_pvh(pa);
4892 TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_link);
4893 pvh->pv_gen++;
4894 return (true);
4895 }
4896
4897 /*
4898 * Fills a page table page with mappings to consecutive physical pages.
4899 */
4900 static void
pmap_fill_ptp(pt_entry_t * firstpte,pt_entry_t newpte)4901 pmap_fill_ptp(pt_entry_t *firstpte, pt_entry_t newpte)
4902 {
4903 pt_entry_t *pte;
4904
4905 for (pte = firstpte; pte < firstpte + NPTEPG; pte++) {
4906 *pte = htobe64(newpte);
4907 newpte += PAGE_SIZE;
4908 }
4909 }
4910
4911 static bool
pmap_demote_l3e(pmap_t pmap,pml3_entry_t * pde,vm_offset_t va)4912 pmap_demote_l3e(pmap_t pmap, pml3_entry_t *pde, vm_offset_t va)
4913 {
4914 struct rwlock *lock;
4915 bool rv;
4916
4917 lock = NULL;
4918 rv = pmap_demote_l3e_locked(pmap, pde, va, &lock);
4919 if (lock != NULL)
4920 rw_wunlock(lock);
4921 return (rv);
4922 }
4923
4924 static bool
pmap_demote_l3e_locked(pmap_t pmap,pml3_entry_t * l3e,vm_offset_t va,struct rwlock ** lockp)4925 pmap_demote_l3e_locked(pmap_t pmap, pml3_entry_t *l3e, vm_offset_t va,
4926 struct rwlock **lockp)
4927 {
4928 pml3_entry_t oldpde;
4929 pt_entry_t *firstpte;
4930 vm_paddr_t mptepa;
4931 vm_page_t mpte;
4932 struct spglist free;
4933 vm_offset_t sva;
4934
4935 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
4936 oldpde = be64toh(*l3e);
4937 KASSERT((oldpde & (RPTE_LEAF | PG_V)) == (RPTE_LEAF | PG_V),
4938 ("pmap_demote_l3e: oldpde is missing RPTE_LEAF and/or PG_V %lx",
4939 oldpde));
4940 if ((oldpde & PG_A) == 0 || (mpte = pmap_remove_pt_page(pmap, va)) ==
4941 NULL) {
4942 KASSERT((oldpde & PG_W) == 0,
4943 ("pmap_demote_l3e: page table page for a wired mapping"
4944 " is missing"));
4945
4946 /*
4947 * Invalidate the 2MB page mapping and return "failure" if the
4948 * mapping was never accessed or the allocation of the new
4949 * page table page fails. If the 2MB page mapping belongs to
4950 * the direct map region of the kernel's address space, then
4951 * the page allocation request specifies the highest possible
4952 * priority (VM_ALLOC_INTERRUPT). Otherwise, the priority is
4953 * normal. Page table pages are preallocated for every other
4954 * part of the kernel address space, so the direct map region
4955 * is the only part of the kernel address space that must be
4956 * handled here.
4957 */
4958 if ((oldpde & PG_A) == 0 || (mpte = vm_page_alloc_noobj(
4959 (va >= DMAP_MIN_ADDRESS && va < DMAP_MAX_ADDRESS ?
4960 VM_ALLOC_INTERRUPT : 0) | VM_ALLOC_WIRED)) == NULL) {
4961 SLIST_INIT(&free);
4962 sva = trunc_2mpage(va);
4963 pmap_remove_l3e(pmap, l3e, sva, &free, lockp);
4964 pmap_invalidate_l3e_page(pmap, sva, oldpde);
4965 vm_page_free_pages_toq(&free, true);
4966 CTR2(KTR_PMAP, "pmap_demote_l3e: failure for va %#lx"
4967 " in pmap %p", va, pmap);
4968 return (false);
4969 }
4970 mpte->pindex = pmap_l3e_pindex(va);
4971 if (va < VM_MAXUSER_ADDRESS)
4972 pmap_resident_count_inc(pmap, 1);
4973 }
4974 mptepa = VM_PAGE_TO_PHYS(mpte);
4975 firstpte = PHYS_TO_DMAP(mptepa);
4976 KASSERT((oldpde & PG_A) != 0,
4977 ("pmap_demote_l3e: oldpde is missing PG_A"));
4978 KASSERT((oldpde & (PG_M | PG_RW)) != PG_RW,
4979 ("pmap_demote_l3e: oldpde is missing PG_M"));
4980
4981 /*
4982 * If the page table page is new, initialize it.
4983 */
4984 if (mpte->ref_count == 1) {
4985 mpte->ref_count = NPTEPG;
4986 pmap_fill_ptp(firstpte, oldpde);
4987 }
4988
4989 KASSERT((be64toh(*firstpte) & PG_FRAME) == (oldpde & PG_FRAME),
4990 ("pmap_demote_l3e: firstpte and newpte map different physical"
4991 " addresses"));
4992
4993 /*
4994 * If the mapping has changed attributes, update the page table
4995 * entries.
4996 */
4997 if ((be64toh(*firstpte) & PG_PTE_PROMOTE) != (oldpde & PG_PTE_PROMOTE))
4998 pmap_fill_ptp(firstpte, oldpde);
4999
5000 /*
5001 * The spare PV entries must be reserved prior to demoting the
5002 * mapping, that is, prior to changing the PDE. Otherwise, the state
5003 * of the PDE and the PV lists will be inconsistent, which can result
5004 * in reclaim_pv_chunk() attempting to remove a PV entry from the
5005 * wrong PV list and pmap_pv_demote_l3e() failing to find the expected
5006 * PV entry for the 2MB page mapping that is being demoted.
5007 */
5008 if ((oldpde & PG_MANAGED) != 0)
5009 reserve_pv_entries(pmap, NPTEPG - 1, lockp);
5010
5011 /*
5012 * Demote the mapping. This pmap is locked. The old PDE has
5013 * PG_A set. If the old PDE has PG_RW set, it also has PG_M
5014 * set. Thus, there is no danger of a race with another
5015 * processor changing the setting of PG_A and/or PG_M between
5016 * the read above and the store below.
5017 */
5018 pde_store(l3e, mptepa);
5019 pmap_invalidate_l3e_page(pmap, trunc_2mpage(va), oldpde);
5020 /*
5021 * Demote the PV entry.
5022 */
5023 if ((oldpde & PG_MANAGED) != 0)
5024 pmap_pv_demote_l3e(pmap, va, oldpde & PG_PS_FRAME, lockp);
5025
5026 counter_u64_add(pmap_l3e_demotions, 1);
5027 CTR2(KTR_PMAP, "pmap_demote_l3e: success for va %#lx"
5028 " in pmap %p", va, pmap);
5029 return (true);
5030 }
5031
5032 /*
5033 * pmap_remove_kernel_pde: Remove a kernel superpage mapping.
5034 */
5035 static void
pmap_remove_kernel_l3e(pmap_t pmap,pml3_entry_t * l3e,vm_offset_t va)5036 pmap_remove_kernel_l3e(pmap_t pmap, pml3_entry_t *l3e, vm_offset_t va)
5037 {
5038 vm_paddr_t mptepa;
5039 vm_page_t mpte;
5040
5041 KASSERT(pmap == kernel_pmap, ("pmap %p is not kernel_pmap", pmap));
5042 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
5043 mpte = pmap_remove_pt_page(pmap, va);
5044 if (mpte == NULL)
5045 panic("pmap_remove_kernel_pde: Missing pt page.");
5046
5047 mptepa = VM_PAGE_TO_PHYS(mpte);
5048
5049 /*
5050 * Initialize the page table page.
5051 */
5052 pagezero(PHYS_TO_DMAP(mptepa));
5053
5054 /*
5055 * Demote the mapping.
5056 */
5057 pde_store(l3e, mptepa);
5058 ptesync();
5059 }
5060
5061 /*
5062 * pmap_remove_l3e: do the things to unmap a superpage in a process
5063 */
5064 static int
pmap_remove_l3e(pmap_t pmap,pml3_entry_t * pdq,vm_offset_t sva,struct spglist * free,struct rwlock ** lockp)5065 pmap_remove_l3e(pmap_t pmap, pml3_entry_t *pdq, vm_offset_t sva,
5066 struct spglist *free, struct rwlock **lockp)
5067 {
5068 struct md_page *pvh;
5069 pml3_entry_t oldpde;
5070 vm_offset_t eva, va;
5071 vm_page_t m, mpte;
5072
5073 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
5074 KASSERT((sva & L3_PAGE_MASK) == 0,
5075 ("pmap_remove_l3e: sva is not 2mpage aligned"));
5076 oldpde = be64toh(pte_load_clear(pdq));
5077 if (oldpde & PG_W)
5078 pmap->pm_stats.wired_count -= (L3_PAGE_SIZE / PAGE_SIZE);
5079 pmap_resident_count_dec(pmap, L3_PAGE_SIZE / PAGE_SIZE);
5080 if (oldpde & PG_MANAGED) {
5081 CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, oldpde & PG_PS_FRAME);
5082 pvh = pa_to_pvh(oldpde & PG_PS_FRAME);
5083 pmap_pvh_free(pvh, pmap, sva);
5084 eva = sva + L3_PAGE_SIZE;
5085 for (va = sva, m = PHYS_TO_VM_PAGE(oldpde & PG_PS_FRAME);
5086 va < eva; va += PAGE_SIZE, m++) {
5087 if ((oldpde & (PG_M | PG_RW)) == (PG_M | PG_RW))
5088 vm_page_dirty(m);
5089 if (oldpde & PG_A)
5090 vm_page_aflag_set(m, PGA_REFERENCED);
5091 if (TAILQ_EMPTY(&m->md.pv_list) &&
5092 TAILQ_EMPTY(&pvh->pv_list))
5093 vm_page_aflag_clear(m, PGA_WRITEABLE);
5094 }
5095 }
5096 if (pmap == kernel_pmap) {
5097 pmap_remove_kernel_l3e(pmap, pdq, sva);
5098 } else {
5099 mpte = pmap_remove_pt_page(pmap, sva);
5100 if (mpte != NULL) {
5101 pmap_resident_count_dec(pmap, 1);
5102 KASSERT(mpte->ref_count == NPTEPG,
5103 ("pmap_remove_l3e: pte page wire count error"));
5104 mpte->ref_count = 0;
5105 pmap_add_delayed_free_list(mpte, free, false);
5106 }
5107 }
5108 return (pmap_unuse_pt(pmap, sva, be64toh(*pmap_pml2e(pmap, sva)), free));
5109 }
5110
5111 /*
5112 * pmap_remove_pte: do the things to unmap a page in a process
5113 */
5114 static int
pmap_remove_pte(pmap_t pmap,pt_entry_t * ptq,vm_offset_t va,pml3_entry_t ptepde,struct spglist * free,struct rwlock ** lockp)5115 pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t va,
5116 pml3_entry_t ptepde, struct spglist *free, struct rwlock **lockp)
5117 {
5118 struct md_page *pvh;
5119 pt_entry_t oldpte;
5120 vm_page_t m;
5121
5122 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
5123 oldpte = be64toh(pte_load_clear(ptq));
5124 if (oldpte & RPTE_WIRED)
5125 pmap->pm_stats.wired_count -= 1;
5126 pmap_resident_count_dec(pmap, 1);
5127 if (oldpte & RPTE_MANAGED) {
5128 m = PHYS_TO_VM_PAGE(oldpte & PG_FRAME);
5129 if ((oldpte & (PG_M | PG_RW)) == (PG_M | PG_RW))
5130 vm_page_dirty(m);
5131 if (oldpte & PG_A)
5132 vm_page_aflag_set(m, PGA_REFERENCED);
5133 CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m);
5134 pmap_pvh_free(&m->md, pmap, va);
5135 if (TAILQ_EMPTY(&m->md.pv_list) &&
5136 (m->flags & PG_FICTITIOUS) == 0) {
5137 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
5138 if (TAILQ_EMPTY(&pvh->pv_list))
5139 vm_page_aflag_clear(m, PGA_WRITEABLE);
5140 }
5141 }
5142 return (pmap_unuse_pt(pmap, va, ptepde, free));
5143 }
5144
5145 /*
5146 * Remove a single page from a process address space
5147 */
5148 static bool
pmap_remove_page(pmap_t pmap,vm_offset_t va,pml3_entry_t * l3e,struct spglist * free)5149 pmap_remove_page(pmap_t pmap, vm_offset_t va, pml3_entry_t *l3e,
5150 struct spglist *free)
5151 {
5152 struct rwlock *lock;
5153 pt_entry_t *pte;
5154 bool invalidate_all;
5155
5156 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
5157 if ((be64toh(*l3e) & RPTE_VALID) == 0) {
5158 return (false);
5159 }
5160 pte = pmap_l3e_to_pte(l3e, va);
5161 if ((be64toh(*pte) & RPTE_VALID) == 0) {
5162 return (false);
5163 }
5164 lock = NULL;
5165
5166 invalidate_all = pmap_remove_pte(pmap, pte, va, be64toh(*l3e), free, &lock);
5167 if (lock != NULL)
5168 rw_wunlock(lock);
5169 if (!invalidate_all)
5170 pmap_invalidate_page(pmap, va);
5171 return (invalidate_all);
5172 }
5173
5174 /*
5175 * Removes the specified range of addresses from the page table page.
5176 */
5177 static bool
pmap_remove_ptes(pmap_t pmap,vm_offset_t sva,vm_offset_t eva,pml3_entry_t * l3e,struct spglist * free,struct rwlock ** lockp)5178 pmap_remove_ptes(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,
5179 pml3_entry_t *l3e, struct spglist *free, struct rwlock **lockp)
5180 {
5181 pt_entry_t *pte;
5182 vm_offset_t va;
5183 bool anyvalid;
5184
5185 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
5186 anyvalid = false;
5187 va = eva;
5188 for (pte = pmap_l3e_to_pte(l3e, sva); sva != eva; pte++,
5189 sva += PAGE_SIZE) {
5190 MPASS(pte == pmap_pte(pmap, sva));
5191 if (*pte == 0) {
5192 if (va != eva) {
5193 anyvalid = true;
5194 va = eva;
5195 }
5196 continue;
5197 }
5198 if (va == eva)
5199 va = sva;
5200 if (pmap_remove_pte(pmap, pte, sva, be64toh(*l3e), free, lockp)) {
5201 anyvalid = true;
5202 sva += PAGE_SIZE;
5203 break;
5204 }
5205 }
5206 if (anyvalid)
5207 pmap_invalidate_all(pmap);
5208 else if (va != eva)
5209 pmap_invalidate_range(pmap, va, sva);
5210 return (anyvalid);
5211 }
5212
5213 void
mmu_radix_remove(pmap_t pmap,vm_offset_t sva,vm_offset_t eva)5214 mmu_radix_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
5215 {
5216 struct rwlock *lock;
5217 vm_offset_t va_next;
5218 pml1_entry_t *l1e;
5219 pml2_entry_t *l2e;
5220 pml3_entry_t ptpaddr, *l3e;
5221 struct spglist free;
5222 bool anyvalid;
5223
5224 CTR4(KTR_PMAP, "%s(%p, %#x, %#x)", __func__, pmap, sva, eva);
5225
5226 /*
5227 * Perform an unsynchronized read. This is, however, safe.
5228 */
5229 if (pmap->pm_stats.resident_count == 0)
5230 return;
5231
5232 anyvalid = false;
5233 SLIST_INIT(&free);
5234
5235 /* XXX something fishy here */
5236 sva = (sva + PAGE_MASK) & ~PAGE_MASK;
5237 eva = (eva + PAGE_MASK) & ~PAGE_MASK;
5238
5239 PMAP_LOCK(pmap);
5240
5241 /*
5242 * special handling of removing one page. a very
5243 * common operation and easy to short circuit some
5244 * code.
5245 */
5246 if (sva + PAGE_SIZE == eva) {
5247 l3e = pmap_pml3e(pmap, sva);
5248 if (l3e && (be64toh(*l3e) & RPTE_LEAF) == 0) {
5249 anyvalid = pmap_remove_page(pmap, sva, l3e, &free);
5250 goto out;
5251 }
5252 }
5253
5254 lock = NULL;
5255 for (; sva < eva; sva = va_next) {
5256 if (pmap->pm_stats.resident_count == 0)
5257 break;
5258 l1e = pmap_pml1e(pmap, sva);
5259 if (l1e == NULL || (be64toh(*l1e) & PG_V) == 0) {
5260 va_next = (sva + L1_PAGE_SIZE) & ~L1_PAGE_MASK;
5261 if (va_next < sva)
5262 va_next = eva;
5263 continue;
5264 }
5265
5266 l2e = pmap_l1e_to_l2e(l1e, sva);
5267 if (l2e == NULL || (be64toh(*l2e) & PG_V) == 0) {
5268 va_next = (sva + L2_PAGE_SIZE) & ~L2_PAGE_MASK;
5269 if (va_next < sva)
5270 va_next = eva;
5271 continue;
5272 }
5273
5274 /*
5275 * Calculate index for next page table.
5276 */
5277 va_next = (sva + L3_PAGE_SIZE) & ~L3_PAGE_MASK;
5278 if (va_next < sva)
5279 va_next = eva;
5280
5281 l3e = pmap_l2e_to_l3e(l2e, sva);
5282 ptpaddr = be64toh(*l3e);
5283
5284 /*
5285 * Weed out invalid mappings.
5286 */
5287 if (ptpaddr == 0)
5288 continue;
5289
5290 /*
5291 * Check for large page.
5292 */
5293 if ((ptpaddr & RPTE_LEAF) != 0) {
5294 /*
5295 * Are we removing the entire large page? If not,
5296 * demote the mapping and fall through.
5297 */
5298 if (sva + L3_PAGE_SIZE == va_next && eva >= va_next) {
5299 pmap_remove_l3e(pmap, l3e, sva, &free, &lock);
5300 anyvalid = true;
5301 continue;
5302 } else if (!pmap_demote_l3e_locked(pmap, l3e, sva,
5303 &lock)) {
5304 /* The large page mapping was destroyed. */
5305 continue;
5306 } else
5307 ptpaddr = be64toh(*l3e);
5308 }
5309
5310 /*
5311 * Limit our scan to either the end of the va represented
5312 * by the current page table page, or to the end of the
5313 * range being removed.
5314 */
5315 if (va_next > eva)
5316 va_next = eva;
5317
5318 if (pmap_remove_ptes(pmap, sva, va_next, l3e, &free, &lock))
5319 anyvalid = true;
5320 }
5321 if (lock != NULL)
5322 rw_wunlock(lock);
5323 out:
5324 if (anyvalid)
5325 pmap_invalidate_all(pmap);
5326 PMAP_UNLOCK(pmap);
5327 vm_page_free_pages_toq(&free, true);
5328 }
5329
5330 void
mmu_radix_remove_all(vm_page_t m)5331 mmu_radix_remove_all(vm_page_t m)
5332 {
5333 struct md_page *pvh;
5334 pv_entry_t pv;
5335 pmap_t pmap;
5336 struct rwlock *lock;
5337 pt_entry_t *pte, tpte;
5338 pml3_entry_t *l3e;
5339 vm_offset_t va;
5340 struct spglist free;
5341 int pvh_gen, md_gen;
5342
5343 CTR2(KTR_PMAP, "%s(%p)", __func__, m);
5344 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
5345 ("pmap_remove_all: page %p is not managed", m));
5346 SLIST_INIT(&free);
5347 lock = VM_PAGE_TO_PV_LIST_LOCK(m);
5348 pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy :
5349 pa_to_pvh(VM_PAGE_TO_PHYS(m));
5350 retry:
5351 rw_wlock(lock);
5352 while ((pv = TAILQ_FIRST(&pvh->pv_list)) != NULL) {
5353 pmap = PV_PMAP(pv);
5354 if (!PMAP_TRYLOCK(pmap)) {
5355 pvh_gen = pvh->pv_gen;
5356 rw_wunlock(lock);
5357 PMAP_LOCK(pmap);
5358 rw_wlock(lock);
5359 if (pvh_gen != pvh->pv_gen) {
5360 rw_wunlock(lock);
5361 PMAP_UNLOCK(pmap);
5362 goto retry;
5363 }
5364 }
5365 va = pv->pv_va;
5366 l3e = pmap_pml3e(pmap, va);
5367 (void)pmap_demote_l3e_locked(pmap, l3e, va, &lock);
5368 PMAP_UNLOCK(pmap);
5369 }
5370 while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) {
5371 pmap = PV_PMAP(pv);
5372 if (!PMAP_TRYLOCK(pmap)) {
5373 pvh_gen = pvh->pv_gen;
5374 md_gen = m->md.pv_gen;
5375 rw_wunlock(lock);
5376 PMAP_LOCK(pmap);
5377 rw_wlock(lock);
5378 if (pvh_gen != pvh->pv_gen || md_gen != m->md.pv_gen) {
5379 rw_wunlock(lock);
5380 PMAP_UNLOCK(pmap);
5381 goto retry;
5382 }
5383 }
5384 pmap_resident_count_dec(pmap, 1);
5385 l3e = pmap_pml3e(pmap, pv->pv_va);
5386 KASSERT((be64toh(*l3e) & RPTE_LEAF) == 0, ("pmap_remove_all: found"
5387 " a 2mpage in page %p's pv list", m));
5388 pte = pmap_l3e_to_pte(l3e, pv->pv_va);
5389 tpte = be64toh(pte_load_clear(pte));
5390 if (tpte & PG_W)
5391 pmap->pm_stats.wired_count--;
5392 if (tpte & PG_A)
5393 vm_page_aflag_set(m, PGA_REFERENCED);
5394
5395 /*
5396 * Update the vm_page_t clean and reference bits.
5397 */
5398 if ((tpte & (PG_M | PG_RW)) == (PG_M | PG_RW))
5399 vm_page_dirty(m);
5400 pmap_unuse_pt(pmap, pv->pv_va, be64toh(*l3e), &free);
5401 pmap_invalidate_page(pmap, pv->pv_va);
5402 TAILQ_REMOVE(&m->md.pv_list, pv, pv_link);
5403 m->md.pv_gen++;
5404 free_pv_entry(pmap, pv);
5405 PMAP_UNLOCK(pmap);
5406 }
5407 vm_page_aflag_clear(m, PGA_WRITEABLE);
5408 rw_wunlock(lock);
5409 vm_page_free_pages_toq(&free, true);
5410 }
5411
5412 /*
5413 * Destroy all managed, non-wired mappings in the given user-space
5414 * pmap. This pmap cannot be active on any processor besides the
5415 * caller.
5416 *
5417 * This function cannot be applied to the kernel pmap. Moreover, it
5418 * is not intended for general use. It is only to be used during
5419 * process termination. Consequently, it can be implemented in ways
5420 * that make it faster than pmap_remove(). First, it can more quickly
5421 * destroy mappings by iterating over the pmap's collection of PV
5422 * entries, rather than searching the page table. Second, it doesn't
5423 * have to test and clear the page table entries atomically, because
5424 * no processor is currently accessing the user address space. In
5425 * particular, a page table entry's dirty bit won't change state once
5426 * this function starts.
5427 *
5428 * Although this function destroys all of the pmap's managed,
5429 * non-wired mappings, it can delay and batch the invalidation of TLB
5430 * entries without calling pmap_delayed_invl_started() and
5431 * pmap_delayed_invl_finished(). Because the pmap is not active on
5432 * any other processor, none of these TLB entries will ever be used
5433 * before their eventual invalidation. Consequently, there is no need
5434 * for either pmap_remove_all() or pmap_remove_write() to wait for
5435 * that eventual TLB invalidation.
5436 */
5437
5438 void
mmu_radix_remove_pages(pmap_t pmap)5439 mmu_radix_remove_pages(pmap_t pmap)
5440 {
5441
5442 CTR2(KTR_PMAP, "%s(%p)", __func__, pmap);
5443 pml3_entry_t ptel3e;
5444 pt_entry_t *pte, tpte;
5445 struct spglist free;
5446 vm_page_t m, mpte, mt;
5447 pv_entry_t pv;
5448 struct md_page *pvh;
5449 struct pv_chunk *pc, *npc;
5450 struct rwlock *lock;
5451 int64_t bit;
5452 uint64_t inuse, bitmask;
5453 int allfree, field, idx;
5454 #ifdef PV_STATS
5455 int freed;
5456 #endif
5457 bool superpage;
5458 vm_paddr_t pa;
5459
5460 /*
5461 * Assert that the given pmap is only active on the current
5462 * CPU. Unfortunately, we cannot block another CPU from
5463 * activating the pmap while this function is executing.
5464 */
5465 KASSERT(pmap->pm_pid == mfspr(SPR_PID),
5466 ("non-current asid %lu - expected %lu", pmap->pm_pid,
5467 mfspr(SPR_PID)));
5468
5469 lock = NULL;
5470
5471 SLIST_INIT(&free);
5472 PMAP_LOCK(pmap);
5473 TAILQ_FOREACH_SAFE(pc, &pmap->pm_pvchunk, pc_list, npc) {
5474 allfree = 1;
5475 #ifdef PV_STATS
5476 freed = 0;
5477 #endif
5478 for (field = 0; field < _NPCM; field++) {
5479 inuse = ~pc->pc_map[field] & pc_freemask[field];
5480 while (inuse != 0) {
5481 bit = cnttzd(inuse);
5482 bitmask = 1UL << bit;
5483 idx = field * 64 + bit;
5484 pv = &pc->pc_pventry[idx];
5485 inuse &= ~bitmask;
5486
5487 pte = pmap_pml2e(pmap, pv->pv_va);
5488 ptel3e = be64toh(*pte);
5489 pte = pmap_l2e_to_l3e(pte, pv->pv_va);
5490 tpte = be64toh(*pte);
5491 if ((tpte & (RPTE_LEAF | PG_V)) == PG_V) {
5492 superpage = false;
5493 ptel3e = tpte;
5494 pte = PHYS_TO_DMAP(tpte & PG_FRAME);
5495 pte = &pte[pmap_pte_index(pv->pv_va)];
5496 tpte = be64toh(*pte);
5497 } else {
5498 /*
5499 * Keep track whether 'tpte' is a
5500 * superpage explicitly instead of
5501 * relying on RPTE_LEAF being set.
5502 *
5503 * This is because RPTE_LEAF is numerically
5504 * identical to PG_PTE_PAT and thus a
5505 * regular page could be mistaken for
5506 * a superpage.
5507 */
5508 superpage = true;
5509 }
5510
5511 if ((tpte & PG_V) == 0) {
5512 panic("bad pte va %lx pte %lx",
5513 pv->pv_va, tpte);
5514 }
5515
5516 /*
5517 * We cannot remove wired pages from a process' mapping at this time
5518 */
5519 if (tpte & PG_W) {
5520 allfree = 0;
5521 continue;
5522 }
5523
5524 if (superpage)
5525 pa = tpte & PG_PS_FRAME;
5526 else
5527 pa = tpte & PG_FRAME;
5528
5529 m = PHYS_TO_VM_PAGE(pa);
5530 KASSERT(m->phys_addr == pa,
5531 ("vm_page_t %p phys_addr mismatch %016jx %016jx",
5532 m, (uintmax_t)m->phys_addr,
5533 (uintmax_t)tpte));
5534
5535 KASSERT((m->flags & PG_FICTITIOUS) != 0 ||
5536 m < &vm_page_array[vm_page_array_size],
5537 ("pmap_remove_pages: bad tpte %#jx",
5538 (uintmax_t)tpte));
5539
5540 pte_clear(pte);
5541
5542 /*
5543 * Update the vm_page_t clean/reference bits.
5544 */
5545 if ((tpte & (PG_M | PG_RW)) == (PG_M | PG_RW)) {
5546 if (superpage) {
5547 for (mt = m; mt < &m[L3_PAGE_SIZE / PAGE_SIZE]; mt++)
5548 vm_page_dirty(mt);
5549 } else
5550 vm_page_dirty(m);
5551 }
5552
5553 CHANGE_PV_LIST_LOCK_TO_VM_PAGE(&lock, m);
5554
5555 /* Mark free */
5556 pc->pc_map[field] |= bitmask;
5557 if (superpage) {
5558 pmap_resident_count_dec(pmap, L3_PAGE_SIZE / PAGE_SIZE);
5559 pvh = pa_to_pvh(tpte & PG_PS_FRAME);
5560 TAILQ_REMOVE(&pvh->pv_list, pv, pv_link);
5561 pvh->pv_gen++;
5562 if (TAILQ_EMPTY(&pvh->pv_list)) {
5563 for (mt = m; mt < &m[L3_PAGE_SIZE / PAGE_SIZE]; mt++)
5564 if ((mt->a.flags & PGA_WRITEABLE) != 0 &&
5565 TAILQ_EMPTY(&mt->md.pv_list))
5566 vm_page_aflag_clear(mt, PGA_WRITEABLE);
5567 }
5568 mpte = pmap_remove_pt_page(pmap, pv->pv_va);
5569 if (mpte != NULL) {
5570 pmap_resident_count_dec(pmap, 1);
5571 KASSERT(mpte->ref_count == NPTEPG,
5572 ("pmap_remove_pages: pte page wire count error"));
5573 mpte->ref_count = 0;
5574 pmap_add_delayed_free_list(mpte, &free, false);
5575 }
5576 } else {
5577 pmap_resident_count_dec(pmap, 1);
5578 #ifdef VERBOSE_PV
5579 printf("freeing pv (%p, %p)\n",
5580 pmap, pv);
5581 #endif
5582 TAILQ_REMOVE(&m->md.pv_list, pv, pv_link);
5583 m->md.pv_gen++;
5584 if ((m->a.flags & PGA_WRITEABLE) != 0 &&
5585 TAILQ_EMPTY(&m->md.pv_list) &&
5586 (m->flags & PG_FICTITIOUS) == 0) {
5587 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
5588 if (TAILQ_EMPTY(&pvh->pv_list))
5589 vm_page_aflag_clear(m, PGA_WRITEABLE);
5590 }
5591 }
5592 pmap_unuse_pt(pmap, pv->pv_va, ptel3e, &free);
5593 #ifdef PV_STATS
5594 freed++;
5595 #endif
5596 }
5597 }
5598 PV_STAT(atomic_add_long(&pv_entry_frees, freed));
5599 PV_STAT(atomic_add_int(&pv_entry_spare, freed));
5600 PV_STAT(atomic_subtract_long(&pv_entry_count, freed));
5601 if (allfree) {
5602 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
5603 free_pv_chunk(pc);
5604 }
5605 }
5606 if (lock != NULL)
5607 rw_wunlock(lock);
5608 pmap_invalidate_all(pmap);
5609 PMAP_UNLOCK(pmap);
5610 vm_page_free_pages_toq(&free, true);
5611 }
5612
5613 void
mmu_radix_remove_write(vm_page_t m)5614 mmu_radix_remove_write(vm_page_t m)
5615 {
5616 struct md_page *pvh;
5617 pmap_t pmap;
5618 struct rwlock *lock;
5619 pv_entry_t next_pv, pv;
5620 pml3_entry_t *l3e;
5621 pt_entry_t oldpte, *pte;
5622 int pvh_gen, md_gen;
5623
5624 CTR2(KTR_PMAP, "%s(%p)", __func__, m);
5625 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
5626 ("pmap_remove_write: page %p is not managed", m));
5627 vm_page_assert_busied(m);
5628
5629 if (!pmap_page_is_write_mapped(m))
5630 return;
5631 lock = VM_PAGE_TO_PV_LIST_LOCK(m);
5632 pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy :
5633 pa_to_pvh(VM_PAGE_TO_PHYS(m));
5634 retry_pv_loop:
5635 rw_wlock(lock);
5636 TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_link, next_pv) {
5637 pmap = PV_PMAP(pv);
5638 if (!PMAP_TRYLOCK(pmap)) {
5639 pvh_gen = pvh->pv_gen;
5640 rw_wunlock(lock);
5641 PMAP_LOCK(pmap);
5642 rw_wlock(lock);
5643 if (pvh_gen != pvh->pv_gen) {
5644 PMAP_UNLOCK(pmap);
5645 rw_wunlock(lock);
5646 goto retry_pv_loop;
5647 }
5648 }
5649 l3e = pmap_pml3e(pmap, pv->pv_va);
5650 if ((be64toh(*l3e) & PG_RW) != 0)
5651 (void)pmap_demote_l3e_locked(pmap, l3e, pv->pv_va, &lock);
5652 KASSERT(lock == VM_PAGE_TO_PV_LIST_LOCK(m),
5653 ("inconsistent pv lock %p %p for page %p",
5654 lock, VM_PAGE_TO_PV_LIST_LOCK(m), m));
5655 PMAP_UNLOCK(pmap);
5656 }
5657 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) {
5658 pmap = PV_PMAP(pv);
5659 if (!PMAP_TRYLOCK(pmap)) {
5660 pvh_gen = pvh->pv_gen;
5661 md_gen = m->md.pv_gen;
5662 rw_wunlock(lock);
5663 PMAP_LOCK(pmap);
5664 rw_wlock(lock);
5665 if (pvh_gen != pvh->pv_gen ||
5666 md_gen != m->md.pv_gen) {
5667 PMAP_UNLOCK(pmap);
5668 rw_wunlock(lock);
5669 goto retry_pv_loop;
5670 }
5671 }
5672 l3e = pmap_pml3e(pmap, pv->pv_va);
5673 KASSERT((be64toh(*l3e) & RPTE_LEAF) == 0,
5674 ("pmap_remove_write: found a 2mpage in page %p's pv list",
5675 m));
5676 pte = pmap_l3e_to_pte(l3e, pv->pv_va);
5677 retry:
5678 oldpte = be64toh(*pte);
5679 if (oldpte & PG_RW) {
5680 if (!atomic_cmpset_long(pte, htobe64(oldpte),
5681 htobe64((oldpte | RPTE_EAA_R) & ~(PG_RW | PG_M))))
5682 goto retry;
5683 if ((oldpte & PG_M) != 0)
5684 vm_page_dirty(m);
5685 pmap_invalidate_page(pmap, pv->pv_va);
5686 }
5687 PMAP_UNLOCK(pmap);
5688 }
5689 rw_wunlock(lock);
5690 vm_page_aflag_clear(m, PGA_WRITEABLE);
5691 }
5692
5693 /*
5694 * Clear the wired attribute from the mappings for the specified range of
5695 * addresses in the given pmap. Every valid mapping within that range
5696 * must have the wired attribute set. In contrast, invalid mappings
5697 * cannot have the wired attribute set, so they are ignored.
5698 *
5699 * The wired attribute of the page table entry is not a hardware
5700 * feature, so there is no need to invalidate any TLB entries.
5701 * Since pmap_demote_l3e() for the wired entry must never fail,
5702 * pmap_delayed_invl_started()/finished() calls around the
5703 * function are not needed.
5704 */
5705 void
mmu_radix_unwire(pmap_t pmap,vm_offset_t sva,vm_offset_t eva)5706 mmu_radix_unwire(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
5707 {
5708 vm_offset_t va_next;
5709 pml1_entry_t *l1e;
5710 pml2_entry_t *l2e;
5711 pml3_entry_t *l3e;
5712 pt_entry_t *pte;
5713
5714 CTR4(KTR_PMAP, "%s(%p, %#x, %#x)", __func__, pmap, sva, eva);
5715 PMAP_LOCK(pmap);
5716 for (; sva < eva; sva = va_next) {
5717 l1e = pmap_pml1e(pmap, sva);
5718 if ((be64toh(*l1e) & PG_V) == 0) {
5719 va_next = (sva + L1_PAGE_SIZE) & ~L1_PAGE_MASK;
5720 if (va_next < sva)
5721 va_next = eva;
5722 continue;
5723 }
5724 l2e = pmap_l1e_to_l2e(l1e, sva);
5725 if ((be64toh(*l2e) & PG_V) == 0) {
5726 va_next = (sva + L2_PAGE_SIZE) & ~L2_PAGE_MASK;
5727 if (va_next < sva)
5728 va_next = eva;
5729 continue;
5730 }
5731 va_next = (sva + L3_PAGE_SIZE) & ~L3_PAGE_MASK;
5732 if (va_next < sva)
5733 va_next = eva;
5734 l3e = pmap_l2e_to_l3e(l2e, sva);
5735 if ((be64toh(*l3e) & PG_V) == 0)
5736 continue;
5737 if ((be64toh(*l3e) & RPTE_LEAF) != 0) {
5738 if ((be64toh(*l3e) & PG_W) == 0)
5739 panic("pmap_unwire: pde %#jx is missing PG_W",
5740 (uintmax_t)(be64toh(*l3e)));
5741
5742 /*
5743 * Are we unwiring the entire large page? If not,
5744 * demote the mapping and fall through.
5745 */
5746 if (sva + L3_PAGE_SIZE == va_next && eva >= va_next) {
5747 atomic_clear_long(l3e, htobe64(PG_W));
5748 pmap->pm_stats.wired_count -= L3_PAGE_SIZE /
5749 PAGE_SIZE;
5750 continue;
5751 } else if (!pmap_demote_l3e(pmap, l3e, sva))
5752 panic("pmap_unwire: demotion failed");
5753 }
5754 if (va_next > eva)
5755 va_next = eva;
5756 for (pte = pmap_l3e_to_pte(l3e, sva); sva != va_next; pte++,
5757 sva += PAGE_SIZE) {
5758 MPASS(pte == pmap_pte(pmap, sva));
5759 if ((be64toh(*pte) & PG_V) == 0)
5760 continue;
5761 if ((be64toh(*pte) & PG_W) == 0)
5762 panic("pmap_unwire: pte %#jx is missing PG_W",
5763 (uintmax_t)(be64toh(*pte)));
5764
5765 /*
5766 * PG_W must be cleared atomically. Although the pmap
5767 * lock synchronizes access to PG_W, another processor
5768 * could be setting PG_M and/or PG_A concurrently.
5769 */
5770 atomic_clear_long(pte, htobe64(PG_W));
5771 pmap->pm_stats.wired_count--;
5772 }
5773 }
5774 PMAP_UNLOCK(pmap);
5775 }
5776
5777 void
mmu_radix_zero_page(vm_page_t m)5778 mmu_radix_zero_page(vm_page_t m)
5779 {
5780 void *addr;
5781
5782 CTR2(KTR_PMAP, "%s(%p)", __func__, m);
5783 addr = VM_PAGE_TO_DMAP(m);
5784 pagezero(addr);
5785 }
5786
5787 void
mmu_radix_zero_page_area(vm_page_t m,int off,int size)5788 mmu_radix_zero_page_area(vm_page_t m, int off, int size)
5789 {
5790 caddr_t addr;
5791
5792 CTR4(KTR_PMAP, "%s(%p, %d, %d)", __func__, m, off, size);
5793 MPASS(off + size <= PAGE_SIZE);
5794 addr = VM_PAGE_TO_DMAP(m);
5795 memset(addr + off, 0, size);
5796 }
5797
5798 static int
mmu_radix_mincore(pmap_t pmap,vm_offset_t addr,vm_paddr_t * locked_pa)5799 mmu_radix_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *locked_pa)
5800 {
5801 pml3_entry_t *l3ep;
5802 pt_entry_t pte;
5803 vm_paddr_t pa;
5804 int val;
5805
5806 CTR3(KTR_PMAP, "%s(%p, %#x)", __func__, pmap, addr);
5807 PMAP_LOCK(pmap);
5808
5809 l3ep = pmap_pml3e(pmap, addr);
5810 if (l3ep != NULL && (be64toh(*l3ep) & PG_V)) {
5811 if (be64toh(*l3ep) & RPTE_LEAF) {
5812 pte = be64toh(*l3ep);
5813 /* Compute the physical address of the 4KB page. */
5814 pa = ((be64toh(*l3ep) & PG_PS_FRAME) | (addr & L3_PAGE_MASK)) &
5815 PG_FRAME;
5816 val = MINCORE_PSIND(1);
5817 } else {
5818 /* Native endian PTE, do not pass to functions */
5819 pte = be64toh(*pmap_l3e_to_pte(l3ep, addr));
5820 pa = pte & PG_FRAME;
5821 val = 0;
5822 }
5823 } else {
5824 pte = 0;
5825 pa = 0;
5826 val = 0;
5827 }
5828 if ((pte & PG_V) != 0) {
5829 val |= MINCORE_INCORE;
5830 if ((pte & (PG_M | PG_RW)) == (PG_M | PG_RW))
5831 val |= MINCORE_MODIFIED | MINCORE_MODIFIED_OTHER;
5832 if ((pte & PG_A) != 0)
5833 val |= MINCORE_REFERENCED | MINCORE_REFERENCED_OTHER;
5834 }
5835 if ((val & (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER)) !=
5836 (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER) &&
5837 (pte & (PG_MANAGED | PG_V)) == (PG_MANAGED | PG_V)) {
5838 *locked_pa = pa;
5839 }
5840 PMAP_UNLOCK(pmap);
5841 return (val);
5842 }
5843
5844 void
mmu_radix_activate(struct thread * td)5845 mmu_radix_activate(struct thread *td)
5846 {
5847 pmap_t pmap;
5848 uint32_t curpid;
5849
5850 CTR2(KTR_PMAP, "%s(%p)", __func__, td);
5851 critical_enter();
5852 pmap = vmspace_pmap(td->td_proc->p_vmspace);
5853 curpid = mfspr(SPR_PID);
5854 if (pmap->pm_pid > isa3_base_pid &&
5855 curpid != pmap->pm_pid) {
5856 mmu_radix_pid_set(pmap);
5857 }
5858 critical_exit();
5859 }
5860
5861 /*
5862 * Increase the starting virtual address of the given mapping if a
5863 * different alignment might result in more superpage mappings.
5864 */
5865 void
mmu_radix_align_superpage(vm_object_t object,vm_ooffset_t offset,vm_offset_t * addr,vm_size_t size)5866 mmu_radix_align_superpage(vm_object_t object, vm_ooffset_t offset,
5867 vm_offset_t *addr, vm_size_t size)
5868 {
5869
5870 CTR5(KTR_PMAP, "%s(%p, %#x, %p, %#x)", __func__, object, offset, addr,
5871 size);
5872 vm_offset_t superpage_offset;
5873
5874 if (size < L3_PAGE_SIZE)
5875 return;
5876 if (object != NULL && (object->flags & OBJ_COLORED) != 0)
5877 offset += ptoa(object->pg_color);
5878 superpage_offset = offset & L3_PAGE_MASK;
5879 if (size - ((L3_PAGE_SIZE - superpage_offset) & L3_PAGE_MASK) < L3_PAGE_SIZE ||
5880 (*addr & L3_PAGE_MASK) == superpage_offset)
5881 return;
5882 if ((*addr & L3_PAGE_MASK) < superpage_offset)
5883 *addr = (*addr & ~L3_PAGE_MASK) + superpage_offset;
5884 else
5885 *addr = ((*addr + L3_PAGE_MASK) & ~L3_PAGE_MASK) + superpage_offset;
5886 }
5887
5888 static void *
mmu_radix_mapdev_attr(vm_paddr_t pa,vm_size_t size,vm_memattr_t attr)5889 mmu_radix_mapdev_attr(vm_paddr_t pa, vm_size_t size, vm_memattr_t attr)
5890 {
5891 char *va;
5892 vm_offset_t tmpva, ppa, offset;
5893
5894 ppa = trunc_page(pa);
5895 offset = pa & PAGE_MASK;
5896 size = roundup2(offset + size, PAGE_SIZE);
5897 if (pa < powerpc_ptob(Maxmem))
5898 panic("bad pa: %#lx less than Maxmem %#lx\n",
5899 pa, powerpc_ptob(Maxmem));
5900 va = kva_alloc(size);
5901 if (bootverbose)
5902 printf("%s(%#lx, %lu, %d)\n", __func__, pa, size, attr);
5903 KASSERT(size > 0, ("%s(%#lx, %lu, %d)", __func__, pa, size, attr));
5904
5905 if (va == NULL)
5906 panic("%s: Couldn't alloc kernel virtual memory", __func__);
5907
5908 for (tmpva = (vm_offset_t)va; size > 0;) {
5909 mmu_radix_kenter_attr(tmpva, ppa, attr);
5910 size -= PAGE_SIZE;
5911 tmpva += PAGE_SIZE;
5912 ppa += PAGE_SIZE;
5913 }
5914 ptesync();
5915
5916 return (va + offset);
5917 }
5918
5919 static void *
mmu_radix_mapdev(vm_paddr_t pa,vm_size_t size)5920 mmu_radix_mapdev(vm_paddr_t pa, vm_size_t size)
5921 {
5922
5923 CTR3(KTR_PMAP, "%s(%#x, %#x)", __func__, pa, size);
5924
5925 return (mmu_radix_mapdev_attr(pa, size, VM_MEMATTR_DEFAULT));
5926 }
5927
5928 void
mmu_radix_page_set_memattr(vm_page_t m,vm_memattr_t ma)5929 mmu_radix_page_set_memattr(vm_page_t m, vm_memattr_t ma)
5930 {
5931
5932 CTR3(KTR_PMAP, "%s(%p, %#x)", __func__, m, ma);
5933
5934 if (m->md.mdpg_cache_attrs == ma)
5935 return;
5936
5937 m->md.mdpg_cache_attrs = ma;
5938
5939 /*
5940 * If "m" is a normal page, update its direct mapping. This update
5941 * can be relied upon to perform any cache operations that are
5942 * required for data coherence.
5943 */
5944 if ((m->flags & PG_FICTITIOUS) == 0 &&
5945 mmu_radix_change_attr(VM_PAGE_TO_DMAP(m), PAGE_SIZE,
5946 m->md.mdpg_cache_attrs))
5947 panic("memory attribute change on the direct map failed");
5948 }
5949
5950 static void
mmu_radix_unmapdev(void * va,vm_size_t size)5951 mmu_radix_unmapdev(void *va, vm_size_t size)
5952 {
5953 vm_offset_t offset;
5954
5955 CTR3(KTR_PMAP, "%s(%p, %#x)", __func__, va, size);
5956
5957 /* If we gave a direct map region in pmap_mapdev, do nothing */
5958 if ((vm_offset_t)va >= DMAP_MIN_ADDRESS &&
5959 (vm_offset_t)va < DMAP_MAX_ADDRESS)
5960 return;
5961
5962 offset = (vm_offset_t)va & PAGE_MASK;
5963 size = round_page(offset + size);
5964 va = trunc_page(va);
5965
5966 if (pmap_initialized) {
5967 mmu_radix_qremove(va, atop(size));
5968 kva_free(va, size);
5969 }
5970 }
5971
5972 void
mmu_radix_sync_icache(pmap_t pm,vm_offset_t va,vm_size_t sz)5973 mmu_radix_sync_icache(pmap_t pm, vm_offset_t va, vm_size_t sz)
5974 {
5975 vm_paddr_t pa = 0;
5976 int sync_sz;
5977
5978 if (__predict_false(pm == NULL))
5979 pm = &curthread->td_proc->p_vmspace->vm_pmap;
5980
5981 while (sz > 0) {
5982 pa = pmap_extract(pm, va);
5983 sync_sz = PAGE_SIZE - (va & PAGE_MASK);
5984 sync_sz = min(sync_sz, sz);
5985 if (pa != 0) {
5986 pa += (va & PAGE_MASK);
5987 __syncicache(PHYS_TO_DMAP(pa), sync_sz);
5988 }
5989 va += sync_sz;
5990 sz -= sync_sz;
5991 }
5992 }
5993
5994 static __inline void
pmap_pte_attr(pt_entry_t * pte,uint64_t cache_bits,uint64_t mask)5995 pmap_pte_attr(pt_entry_t *pte, uint64_t cache_bits, uint64_t mask)
5996 {
5997 uint64_t opte, npte;
5998
5999 /*
6000 * The cache mode bits are all in the low 32-bits of the
6001 * PTE, so we can just spin on updating the low 32-bits.
6002 */
6003 do {
6004 opte = be64toh(*pte);
6005 npte = opte & ~mask;
6006 npte |= cache_bits;
6007 } while (npte != opte && !atomic_cmpset_long(pte, htobe64(opte), htobe64(npte)));
6008 }
6009
6010 /*
6011 * Tries to demote a 1GB page mapping.
6012 */
6013 static bool
pmap_demote_l2e(pmap_t pmap,pml2_entry_t * l2e,vm_offset_t va)6014 pmap_demote_l2e(pmap_t pmap, pml2_entry_t *l2e, vm_offset_t va)
6015 {
6016 pml2_entry_t oldpdpe;
6017 pml3_entry_t *firstpde, newpde, *pde;
6018 vm_paddr_t pdpgpa;
6019 vm_page_t pdpg;
6020
6021 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
6022 oldpdpe = be64toh(*l2e);
6023 KASSERT((oldpdpe & (RPTE_LEAF | PG_V)) == (RPTE_LEAF | PG_V),
6024 ("pmap_demote_pdpe: oldpdpe is missing PG_PS and/or PG_V"));
6025 pdpg = vm_page_alloc_noobj(VM_ALLOC_INTERRUPT | VM_ALLOC_WIRED);
6026 if (pdpg == NULL) {
6027 CTR2(KTR_PMAP, "pmap_demote_pdpe: failure for va %#lx"
6028 " in pmap %p", va, pmap);
6029 return (false);
6030 }
6031 pdpg->pindex = va >> L2_PAGE_SIZE_SHIFT;
6032 pdpgpa = VM_PAGE_TO_PHYS(pdpg);
6033 firstpde = PHYS_TO_DMAP(pdpgpa);
6034 KASSERT((oldpdpe & PG_A) != 0,
6035 ("pmap_demote_pdpe: oldpdpe is missing PG_A"));
6036 KASSERT((oldpdpe & (PG_M | PG_RW)) != PG_RW,
6037 ("pmap_demote_pdpe: oldpdpe is missing PG_M"));
6038 newpde = oldpdpe;
6039
6040 /*
6041 * Initialize the page directory page.
6042 */
6043 for (pde = firstpde; pde < firstpde + NPDEPG; pde++) {
6044 *pde = htobe64(newpde);
6045 newpde += L3_PAGE_SIZE;
6046 }
6047
6048 /*
6049 * Demote the mapping.
6050 */
6051 pde_store(l2e, pdpgpa);
6052
6053 /*
6054 * Flush PWC --- XXX revisit
6055 */
6056 pmap_invalidate_all(pmap);
6057
6058 counter_u64_add(pmap_l2e_demotions, 1);
6059 CTR2(KTR_PMAP, "pmap_demote_pdpe: success for va %#lx"
6060 " in pmap %p", va, pmap);
6061 return (true);
6062 }
6063
6064 vm_paddr_t
mmu_radix_kextract(vm_offset_t va)6065 mmu_radix_kextract(vm_offset_t va)
6066 {
6067 pml3_entry_t l3e;
6068 vm_paddr_t pa;
6069
6070 CTR2(KTR_PMAP, "%s(%#x)", __func__, va);
6071 if (va >= DMAP_MIN_ADDRESS && va < DMAP_MAX_ADDRESS) {
6072 pa = DMAP_TO_PHYS(va);
6073 } else {
6074 /* Big-endian PTE on stack */
6075 l3e = *pmap_pml3e(kernel_pmap, va);
6076 if (be64toh(l3e) & RPTE_LEAF) {
6077 pa = (be64toh(l3e) & PG_PS_FRAME) | (va & L3_PAGE_MASK);
6078 pa |= (va & L3_PAGE_MASK);
6079 } else {
6080 /*
6081 * Beware of a concurrent promotion that changes the
6082 * PDE at this point! For example, vtopte() must not
6083 * be used to access the PTE because it would use the
6084 * new PDE. It is, however, safe to use the old PDE
6085 * because the page table page is preserved by the
6086 * promotion.
6087 */
6088 pa = be64toh(*pmap_l3e_to_pte(&l3e, va));
6089 pa = (pa & PG_FRAME) | (va & PAGE_MASK);
6090 pa |= (va & PAGE_MASK);
6091 }
6092 }
6093 return (pa);
6094 }
6095
6096 static pt_entry_t
mmu_radix_calc_wimg(vm_paddr_t pa,vm_memattr_t ma)6097 mmu_radix_calc_wimg(vm_paddr_t pa, vm_memattr_t ma)
6098 {
6099
6100 if (ma != VM_MEMATTR_DEFAULT) {
6101 return pmap_cache_bits(ma);
6102 }
6103
6104 /*
6105 * Assume the page is cache inhibited and access is guarded unless
6106 * it's in our available memory array.
6107 */
6108 for (int i = 0; i < pregions_sz; i++) {
6109 if ((pa >= pregions[i].mr_start) &&
6110 (pa < (pregions[i].mr_start + pregions[i].mr_size)))
6111 return (RPTE_ATTR_MEM);
6112 }
6113 return (RPTE_ATTR_GUARDEDIO);
6114 }
6115
6116 static void
mmu_radix_kenter_attr(vm_offset_t va,vm_paddr_t pa,vm_memattr_t ma)6117 mmu_radix_kenter_attr(vm_offset_t va, vm_paddr_t pa, vm_memattr_t ma)
6118 {
6119 pt_entry_t *pte, pteval;
6120 uint64_t cache_bits;
6121
6122 pte = kvtopte(va);
6123 MPASS(pte != NULL);
6124 pteval = pa | RPTE_EAA_R | RPTE_EAA_W | RPTE_EAA_P | PG_M | PG_A;
6125 cache_bits = mmu_radix_calc_wimg(pa, ma);
6126 pte_store(pte, pteval | cache_bits);
6127 }
6128
6129 void
mmu_radix_kremove(vm_offset_t va)6130 mmu_radix_kremove(vm_offset_t va)
6131 {
6132 pt_entry_t *pte;
6133
6134 CTR2(KTR_PMAP, "%s(%#x)", __func__, va);
6135
6136 pte = kvtopte(va);
6137 pte_clear(pte);
6138 }
6139
6140 int
mmu_radix_decode_kernel_ptr(vm_offset_t addr,int * is_user,vm_offset_t * decoded)6141 mmu_radix_decode_kernel_ptr(vm_offset_t addr,
6142 int *is_user, vm_offset_t *decoded)
6143 {
6144
6145 CTR2(KTR_PMAP, "%s(%#jx)", __func__, (uintmax_t)addr);
6146 *decoded = addr;
6147 *is_user = (addr < VM_MAXUSER_ADDRESS);
6148 return (0);
6149 }
6150
6151 static int
mmu_radix_dev_direct_mapped(vm_paddr_t pa,vm_size_t size)6152 mmu_radix_dev_direct_mapped(vm_paddr_t pa, vm_size_t size)
6153 {
6154
6155 CTR3(KTR_PMAP, "%s(%#x, %#x)", __func__, pa, size);
6156 return (mem_valid(pa, size));
6157 }
6158
6159 static void
mmu_radix_scan_init(void)6160 mmu_radix_scan_init(void)
6161 {
6162
6163 CTR1(KTR_PMAP, "%s()", __func__);
6164 UNIMPLEMENTED();
6165 }
6166
6167 static void
mmu_radix_dumpsys_map(vm_paddr_t pa,size_t sz,void ** va)6168 mmu_radix_dumpsys_map(vm_paddr_t pa, size_t sz,
6169 void **va)
6170 {
6171 CTR4(KTR_PMAP, "%s(%#jx, %#zx, %p)", __func__, (uintmax_t)pa, sz, va);
6172 UNIMPLEMENTED();
6173 }
6174
6175 void *
mmu_radix_quick_enter_page(vm_page_t m)6176 mmu_radix_quick_enter_page(vm_page_t m)
6177 {
6178 CTR2(KTR_PMAP, "%s(%p)", __func__, m);
6179 return (VM_PAGE_TO_DMAP(m));
6180 }
6181
6182 void
mmu_radix_quick_remove_page(void * addr __unused)6183 mmu_radix_quick_remove_page(void *addr __unused)
6184 {
6185 /* no work to do here */
6186 CTR2(KTR_PMAP, "%s(%p)", __func__, addr);
6187 }
6188
6189 static void
pmap_invalidate_cache_range(vm_offset_t sva,vm_offset_t eva)6190 pmap_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva)
6191 {
6192 cpu_flush_dcache((void *)sva, eva - sva);
6193 }
6194
6195 int
mmu_radix_change_attr(void * va,vm_size_t size,vm_memattr_t mode)6196 mmu_radix_change_attr(void *va, vm_size_t size, vm_memattr_t mode)
6197 {
6198 int error;
6199
6200 CTR4(KTR_PMAP, "%s(%p, %#zx, %d)", __func__, va, size, mode);
6201 PMAP_LOCK(kernel_pmap);
6202 error = pmap_change_attr_locked(va, size, mode, true);
6203 PMAP_UNLOCK(kernel_pmap);
6204 return (error);
6205 }
6206
6207 static int
pmap_change_attr_locked(void * addr,vm_size_t size,int mode,bool flush)6208 pmap_change_attr_locked(void *addr, vm_size_t size, int mode, bool flush)
6209 {
6210 vm_offset_t base, offset, tmpva, va;
6211 vm_paddr_t pa_start, pa_end, pa_end1;
6212 pml2_entry_t *l2e;
6213 pml3_entry_t *l3e;
6214 pt_entry_t *pte;
6215 int cache_bits, error;
6216 bool changed;
6217
6218 PMAP_LOCK_ASSERT(kernel_pmap, MA_OWNED);
6219 va = (vm_offset_t)addr;
6220 base = trunc_page(va);
6221 offset = va & PAGE_MASK;
6222 size = round_page(offset + size);
6223
6224 /*
6225 * Only supported on kernel virtual addresses, including the direct
6226 * map but excluding the recursive map.
6227 */
6228 if (base < DMAP_MIN_ADDRESS)
6229 return (EINVAL);
6230
6231 cache_bits = pmap_cache_bits(mode);
6232 changed = false;
6233
6234 /*
6235 * Pages that aren't mapped aren't supported. Also break down 2MB pages
6236 * into 4KB pages if required.
6237 */
6238 for (tmpva = base; tmpva < base + size; ) {
6239 l2e = pmap_pml2e(kernel_pmap, tmpva);
6240 if (l2e == NULL || *l2e == 0)
6241 return (EINVAL);
6242 if (be64toh(*l2e) & RPTE_LEAF) {
6243 /*
6244 * If the current 1GB page already has the required
6245 * memory type, then we need not demote this page. Just
6246 * increment tmpva to the next 1GB page frame.
6247 */
6248 if ((be64toh(*l2e) & RPTE_ATTR_MASK) == cache_bits) {
6249 tmpva = trunc_1gpage(tmpva) + L2_PAGE_SIZE;
6250 continue;
6251 }
6252
6253 /*
6254 * If the current offset aligns with a 1GB page frame
6255 * and there is at least 1GB left within the range, then
6256 * we need not break down this page into 2MB pages.
6257 */
6258 if ((tmpva & L2_PAGE_MASK) == 0 &&
6259 tmpva + L2_PAGE_MASK < base + size) {
6260 tmpva += L2_PAGE_MASK;
6261 continue;
6262 }
6263 if (!pmap_demote_l2e(kernel_pmap, l2e, tmpva))
6264 return (ENOMEM);
6265 }
6266 l3e = pmap_l2e_to_l3e(l2e, tmpva);
6267 KASSERT(l3e != NULL, ("no l3e entry for %#lx in %p\n",
6268 tmpva, l2e));
6269 if (*l3e == 0)
6270 return (EINVAL);
6271 if (be64toh(*l3e) & RPTE_LEAF) {
6272 /*
6273 * If the current 2MB page already has the required
6274 * memory type, then we need not demote this page. Just
6275 * increment tmpva to the next 2MB page frame.
6276 */
6277 if ((be64toh(*l3e) & RPTE_ATTR_MASK) == cache_bits) {
6278 tmpva = trunc_2mpage(tmpva) + L3_PAGE_SIZE;
6279 continue;
6280 }
6281
6282 /*
6283 * If the current offset aligns with a 2MB page frame
6284 * and there is at least 2MB left within the range, then
6285 * we need not break down this page into 4KB pages.
6286 */
6287 if ((tmpva & L3_PAGE_MASK) == 0 &&
6288 tmpva + L3_PAGE_MASK < base + size) {
6289 tmpva += L3_PAGE_SIZE;
6290 continue;
6291 }
6292 if (!pmap_demote_l3e(kernel_pmap, l3e, tmpva))
6293 return (ENOMEM);
6294 }
6295 pte = pmap_l3e_to_pte(l3e, tmpva);
6296 if (*pte == 0)
6297 return (EINVAL);
6298 tmpva += PAGE_SIZE;
6299 }
6300 error = 0;
6301
6302 /*
6303 * Ok, all the pages exist, so run through them updating their
6304 * cache mode if required.
6305 */
6306 pa_start = pa_end = 0;
6307 for (tmpva = base; tmpva < base + size; ) {
6308 l2e = pmap_pml2e(kernel_pmap, tmpva);
6309 if (be64toh(*l2e) & RPTE_LEAF) {
6310 if ((be64toh(*l2e) & RPTE_ATTR_MASK) != cache_bits) {
6311 pmap_pte_attr(l2e, cache_bits,
6312 RPTE_ATTR_MASK);
6313 changed = true;
6314 }
6315 if (tmpva >= VM_MIN_KERNEL_ADDRESS &&
6316 (*l2e & PG_PS_FRAME) < dmaplimit) {
6317 if (pa_start == pa_end) {
6318 /* Start physical address run. */
6319 pa_start = be64toh(*l2e) & PG_PS_FRAME;
6320 pa_end = pa_start + L2_PAGE_SIZE;
6321 } else if (pa_end == (be64toh(*l2e) & PG_PS_FRAME))
6322 pa_end += L2_PAGE_SIZE;
6323 else {
6324 /* Run ended, update direct map. */
6325 error = pmap_change_attr_locked(
6326 PHYS_TO_DMAP(pa_start),
6327 pa_end - pa_start, mode, flush);
6328 if (error != 0)
6329 break;
6330 /* Start physical address run. */
6331 pa_start = be64toh(*l2e) & PG_PS_FRAME;
6332 pa_end = pa_start + L2_PAGE_SIZE;
6333 }
6334 }
6335 tmpva = trunc_1gpage(tmpva) + L2_PAGE_SIZE;
6336 continue;
6337 }
6338 l3e = pmap_l2e_to_l3e(l2e, tmpva);
6339 if (be64toh(*l3e) & RPTE_LEAF) {
6340 if ((be64toh(*l3e) & RPTE_ATTR_MASK) != cache_bits) {
6341 pmap_pte_attr(l3e, cache_bits,
6342 RPTE_ATTR_MASK);
6343 changed = true;
6344 }
6345 if (tmpva >= VM_MIN_KERNEL_ADDRESS &&
6346 (be64toh(*l3e) & PG_PS_FRAME) < dmaplimit) {
6347 if (pa_start == pa_end) {
6348 /* Start physical address run. */
6349 pa_start = be64toh(*l3e) & PG_PS_FRAME;
6350 pa_end = pa_start + L3_PAGE_SIZE;
6351 } else if (pa_end == (be64toh(*l3e) & PG_PS_FRAME))
6352 pa_end += L3_PAGE_SIZE;
6353 else {
6354 /* Run ended, update direct map. */
6355 error = pmap_change_attr_locked(
6356 PHYS_TO_DMAP(pa_start),
6357 pa_end - pa_start, mode, flush);
6358 if (error != 0)
6359 break;
6360 /* Start physical address run. */
6361 pa_start = be64toh(*l3e) & PG_PS_FRAME;
6362 pa_end = pa_start + L3_PAGE_SIZE;
6363 }
6364 }
6365 tmpva = trunc_2mpage(tmpva) + L3_PAGE_SIZE;
6366 } else {
6367 pte = pmap_l3e_to_pte(l3e, tmpva);
6368 if ((be64toh(*pte) & RPTE_ATTR_MASK) != cache_bits) {
6369 pmap_pte_attr(pte, cache_bits,
6370 RPTE_ATTR_MASK);
6371 changed = true;
6372 }
6373 if (tmpva >= VM_MIN_KERNEL_ADDRESS &&
6374 (be64toh(*pte) & PG_FRAME) < dmaplimit) {
6375 if (pa_start == pa_end) {
6376 /* Start physical address run. */
6377 pa_start = be64toh(*pte) & PG_FRAME;
6378 pa_end = pa_start + PAGE_SIZE;
6379 } else if (pa_end == (be64toh(*pte) & PG_FRAME))
6380 pa_end += PAGE_SIZE;
6381 else {
6382 /* Run ended, update direct map. */
6383 error = pmap_change_attr_locked(
6384 PHYS_TO_DMAP(pa_start),
6385 pa_end - pa_start, mode, flush);
6386 if (error != 0)
6387 break;
6388 /* Start physical address run. */
6389 pa_start = be64toh(*pte) & PG_FRAME;
6390 pa_end = pa_start + PAGE_SIZE;
6391 }
6392 }
6393 tmpva += PAGE_SIZE;
6394 }
6395 }
6396 if (error == 0 && pa_start != pa_end && pa_start < dmaplimit) {
6397 pa_end1 = MIN(pa_end, dmaplimit);
6398 if (pa_start != pa_end1)
6399 error = pmap_change_attr_locked(PHYS_TO_DMAP(pa_start),
6400 pa_end1 - pa_start, mode, flush);
6401 }
6402
6403 /*
6404 * Flush CPU caches if required to make sure any data isn't cached that
6405 * shouldn't be, etc.
6406 */
6407 if (changed) {
6408 pmap_invalidate_all(kernel_pmap);
6409
6410 if (flush)
6411 pmap_invalidate_cache_range(base, tmpva);
6412 }
6413 return (error);
6414 }
6415
6416 /*
6417 * Allocate physical memory for the vm_page array and map it into KVA,
6418 * attempting to back the vm_pages with domain-local memory.
6419 */
6420 void
mmu_radix_page_array_startup(long pages)6421 mmu_radix_page_array_startup(long pages)
6422 {
6423 #ifdef notyet
6424 pml2_entry_t *l2e;
6425 pml3_entry_t *pde;
6426 pml3_entry_t newl3;
6427 vm_offset_t va;
6428 long pfn;
6429 int domain, i;
6430 #endif
6431 vm_paddr_t pa;
6432 vm_offset_t start, end;
6433
6434 vm_page_array_size = pages;
6435
6436 start = VM_MIN_KERNEL_ADDRESS;
6437 end = start + pages * sizeof(struct vm_page);
6438
6439 pa = vm_phys_early_alloc(-1, end - start);
6440
6441 start = (vm_offset_t)mmu_radix_map(&start, pa, end - start,
6442 VM_MEMATTR_DEFAULT);
6443 #ifdef notyet
6444 /* TODO: NUMA vm_page_array. Blocked out until then (copied from amd64). */
6445 for (va = start; va < end; va += L3_PAGE_SIZE) {
6446 pfn = first_page + (va - start) / sizeof(struct vm_page);
6447 domain = vm_phys_domain(ptoa(pfn));
6448 l2e = pmap_pml2e(kernel_pmap, va);
6449 if ((be64toh(*l2e) & PG_V) == 0) {
6450 pa = vm_phys_early_alloc(domain, PAGE_SIZE);
6451 dump_add_page(pa);
6452 pagezero(PHYS_TO_DMAP(pa));
6453 pde_store(l2e, (pml2_entry_t)pa);
6454 }
6455 pde = pmap_l2e_to_l3e(l2e, va);
6456 if ((be64toh(*pde) & PG_V) != 0)
6457 panic("Unexpected pde %p", pde);
6458 pa = vm_phys_early_alloc(domain, L3_PAGE_SIZE);
6459 for (i = 0; i < NPDEPG; i++)
6460 dump_add_page(pa + i * PAGE_SIZE);
6461 newl3 = (pml3_entry_t)(pa | RPTE_EAA_P | RPTE_EAA_R | RPTE_EAA_W);
6462 pte_store(pde, newl3);
6463 }
6464 #endif
6465 vm_page_array = (vm_page_t)start;
6466 }
6467
6468 #ifdef DDB
6469 #include <sys/kdb.h>
6470 #include <ddb/ddb.h>
6471
6472 static void
pmap_pte_walk(pml1_entry_t * l1,vm_offset_t va)6473 pmap_pte_walk(pml1_entry_t *l1, vm_offset_t va)
6474 {
6475 pml1_entry_t *l1e;
6476 pml2_entry_t *l2e;
6477 pml3_entry_t *l3e;
6478 pt_entry_t *pte;
6479
6480 l1e = &l1[pmap_pml1e_index(va)];
6481 db_printf("VA %#016lx l1e %#016lx", va, be64toh(*l1e));
6482 if ((be64toh(*l1e) & PG_V) == 0) {
6483 db_printf("\n");
6484 return;
6485 }
6486 l2e = pmap_l1e_to_l2e(l1e, va);
6487 db_printf(" l2e %#016lx", be64toh(*l2e));
6488 if ((be64toh(*l2e) & PG_V) == 0 || (be64toh(*l2e) & RPTE_LEAF) != 0) {
6489 db_printf("\n");
6490 return;
6491 }
6492 l3e = pmap_l2e_to_l3e(l2e, va);
6493 db_printf(" l3e %#016lx", be64toh(*l3e));
6494 if ((be64toh(*l3e) & PG_V) == 0 || (be64toh(*l3e) & RPTE_LEAF) != 0) {
6495 db_printf("\n");
6496 return;
6497 }
6498 pte = pmap_l3e_to_pte(l3e, va);
6499 db_printf(" pte %#016lx\n", be64toh(*pte));
6500 }
6501
6502 void
pmap_page_print_mappings(vm_page_t m)6503 pmap_page_print_mappings(vm_page_t m)
6504 {
6505 pmap_t pmap;
6506 pv_entry_t pv;
6507
6508 db_printf("page %p(%lx)\n", m, m->phys_addr);
6509 /* need to elide locks if running in ddb */
6510 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) {
6511 db_printf("pv: %p ", pv);
6512 db_printf("va: %#016lx ", pv->pv_va);
6513 pmap = PV_PMAP(pv);
6514 db_printf("pmap %p ", pmap);
6515 if (pmap != NULL) {
6516 db_printf("asid: %lu\n", pmap->pm_pid);
6517 pmap_pte_walk(pmap->pm_pml1, pv->pv_va);
6518 }
6519 }
6520 }
6521
DB_SHOW_COMMAND(pte,pmap_print_pte)6522 DB_SHOW_COMMAND(pte, pmap_print_pte)
6523 {
6524 vm_offset_t va;
6525 pmap_t pmap;
6526
6527 if (!have_addr) {
6528 db_printf("show pte addr\n");
6529 return;
6530 }
6531 va = (vm_offset_t)addr;
6532
6533 if (va >= DMAP_MIN_ADDRESS)
6534 pmap = kernel_pmap;
6535 else if (kdb_thread != NULL)
6536 pmap = vmspace_pmap(kdb_thread->td_proc->p_vmspace);
6537 else
6538 pmap = vmspace_pmap(curthread->td_proc->p_vmspace);
6539
6540 pmap_pte_walk(pmap->pm_pml1, va);
6541 }
6542
6543 #endif
6544