xref: /freebsd/sys/powerpc/aim/mmu_radix.c (revision 3332f1b444d4a73238e9f59cca27bfc95fe936bd)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright (c) 2018 Matthew Macy
5  *
6  * Redistribution and use in source and binary forms, with or without
7  * modification, are permitted provided that the following conditions
8  * are met:
9  *
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice, this list of conditions and the following disclaimer.
12  * 2. Redistributions in binary form must reproduce the above copyright
13  *    notice, this list of conditions and the following disclaimer in the
14  *    documentation and/or other materials provided with the distribution.
15  *
16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
20  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
21  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
25  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26  */
27 
28 #include "opt_platform.h"
29 
30 #include <sys/cdefs.h>
31 __FBSDID("$FreeBSD$");
32 
33 #include <sys/param.h>
34 #include <sys/kernel.h>
35 #include <sys/systm.h>
36 #include <sys/conf.h>
37 #include <sys/bitstring.h>
38 #include <sys/queue.h>
39 #include <sys/cpuset.h>
40 #include <sys/endian.h>
41 #include <sys/kerneldump.h>
42 #include <sys/ktr.h>
43 #include <sys/lock.h>
44 #include <sys/syslog.h>
45 #include <sys/msgbuf.h>
46 #include <sys/malloc.h>
47 #include <sys/mman.h>
48 #include <sys/mutex.h>
49 #include <sys/proc.h>
50 #include <sys/rwlock.h>
51 #include <sys/sched.h>
52 #include <sys/sysctl.h>
53 #include <sys/systm.h>
54 #include <sys/vmem.h>
55 #include <sys/vmmeter.h>
56 #include <sys/smp.h>
57 
58 #include <sys/kdb.h>
59 
60 #include <dev/ofw/openfirm.h>
61 
62 #include <vm/vm.h>
63 #include <vm/pmap.h>
64 #include <vm/vm_param.h>
65 #include <vm/vm_kern.h>
66 #include <vm/vm_page.h>
67 #include <vm/vm_map.h>
68 #include <vm/vm_object.h>
69 #include <vm/vm_extern.h>
70 #include <vm/vm_pageout.h>
71 #include <vm/vm_phys.h>
72 #include <vm/vm_reserv.h>
73 #include <vm/vm_dumpset.h>
74 #include <vm/uma.h>
75 
76 #include <machine/_inttypes.h>
77 #include <machine/cpu.h>
78 #include <machine/platform.h>
79 #include <machine/frame.h>
80 #include <machine/md_var.h>
81 #include <machine/psl.h>
82 #include <machine/bat.h>
83 #include <machine/hid.h>
84 #include <machine/pte.h>
85 #include <machine/sr.h>
86 #include <machine/trap.h>
87 #include <machine/mmuvar.h>
88 
89 /* For pseries bit. */
90 #include <powerpc/pseries/phyp-hvcall.h>
91 
92 #ifdef INVARIANTS
93 #include <vm/uma_dbg.h>
94 #endif
95 
96 #define PPC_BITLSHIFT(bit)	(sizeof(long)*NBBY - 1 - (bit))
97 #define PPC_BIT(bit)		(1UL << PPC_BITLSHIFT(bit))
98 #define PPC_BITLSHIFT_VAL(val, bit) ((val) << PPC_BITLSHIFT(bit))
99 
100 #include "opt_ddb.h"
101 
102 #ifdef DDB
103 static void pmap_pte_walk(pml1_entry_t *l1, vm_offset_t va);
104 #endif
105 
106 #define PG_W	RPTE_WIRED
107 #define PG_V	RPTE_VALID
108 #define PG_MANAGED	RPTE_MANAGED
109 #define PG_PROMOTED	RPTE_PROMOTED
110 #define PG_M	RPTE_C
111 #define PG_A	RPTE_R
112 #define PG_X	RPTE_EAA_X
113 #define PG_RW	RPTE_EAA_W
114 #define PG_PTE_CACHE RPTE_ATTR_MASK
115 
116 #define RPTE_SHIFT 9
117 #define NLS_MASK ((1UL<<5)-1)
118 #define RPTE_ENTRIES (1UL<<RPTE_SHIFT)
119 #define RPTE_MASK (RPTE_ENTRIES-1)
120 
121 #define NLB_SHIFT 0
122 #define NLB_MASK (((1UL<<52)-1) << 8)
123 
124 extern int nkpt;
125 extern caddr_t crashdumpmap;
126 
127 #define RIC_FLUSH_TLB 0
128 #define RIC_FLUSH_PWC 1
129 #define RIC_FLUSH_ALL 2
130 
131 #define POWER9_TLB_SETS_RADIX	128	/* # sets in POWER9 TLB Radix mode */
132 
133 #define PPC_INST_TLBIE			0x7c000264
134 #define PPC_INST_TLBIEL			0x7c000224
135 #define PPC_INST_SLBIA			0x7c0003e4
136 
137 #define ___PPC_RA(a)	(((a) & 0x1f) << 16)
138 #define ___PPC_RB(b)	(((b) & 0x1f) << 11)
139 #define ___PPC_RS(s)	(((s) & 0x1f) << 21)
140 #define ___PPC_RT(t)	___PPC_RS(t)
141 #define ___PPC_R(r)	(((r) & 0x1) << 16)
142 #define ___PPC_PRS(prs)	(((prs) & 0x1) << 17)
143 #define ___PPC_RIC(ric)	(((ric) & 0x3) << 18)
144 
145 #define PPC_SLBIA(IH)	__XSTRING(.long PPC_INST_SLBIA | \
146 				       ((IH & 0x7) << 21))
147 #define	PPC_TLBIE_5(rb,rs,ric,prs,r)				\
148 	__XSTRING(.long PPC_INST_TLBIE |			\
149 			  ___PPC_RB(rb) | ___PPC_RS(rs) |	\
150 			  ___PPC_RIC(ric) | ___PPC_PRS(prs) |	\
151 			  ___PPC_R(r))
152 
153 #define	PPC_TLBIEL(rb,rs,ric,prs,r) \
154 	 __XSTRING(.long PPC_INST_TLBIEL | \
155 			   ___PPC_RB(rb) | ___PPC_RS(rs) |	\
156 			   ___PPC_RIC(ric) | ___PPC_PRS(prs) |	\
157 			   ___PPC_R(r))
158 
159 #define PPC_INVALIDATE_ERAT		PPC_SLBIA(7)
160 
161 static __inline void
162 ttusync(void)
163 {
164 	__asm __volatile("eieio; tlbsync; ptesync" ::: "memory");
165 }
166 
167 #define TLBIEL_INVAL_SEL_MASK	0xc00	/* invalidation selector */
168 #define  TLBIEL_INVAL_PAGE	0x000	/* invalidate a single page */
169 #define  TLBIEL_INVAL_SET_PID	0x400	/* invalidate a set for the current PID */
170 #define  TLBIEL_INVAL_SET_LPID	0x800	/* invalidate a set for current LPID */
171 #define  TLBIEL_INVAL_SET	0xc00	/* invalidate a set for all LPIDs */
172 
173 #define TLBIE_ACTUAL_PAGE_MASK		0xe0
174 #define  TLBIE_ACTUAL_PAGE_4K		0x00
175 #define  TLBIE_ACTUAL_PAGE_64K		0xa0
176 #define  TLBIE_ACTUAL_PAGE_2M		0x20
177 #define  TLBIE_ACTUAL_PAGE_1G		0x40
178 
179 #define TLBIE_PRS_PARTITION_SCOPE	0x0
180 #define TLBIE_PRS_PROCESS_SCOPE	0x1
181 
182 #define TLBIE_RIC_INVALIDATE_TLB	0x0	/* Invalidate just TLB */
183 #define TLBIE_RIC_INVALIDATE_PWC	0x1	/* Invalidate just PWC */
184 #define TLBIE_RIC_INVALIDATE_ALL	0x2	/* Invalidate TLB, PWC,
185 						 * cached {proc, part}tab entries
186 						 */
187 #define TLBIE_RIC_INVALIDATE_SEQ	0x3	/* HPT - only:
188 						 * Invalidate a range of translations
189 						 */
190 
191 static __always_inline void
192 radix_tlbie(uint8_t ric, uint8_t prs, uint16_t is, uint32_t pid, uint32_t lpid,
193 			vm_offset_t va, uint16_t ap)
194 {
195 	uint64_t rb, rs;
196 
197 	MPASS((va & PAGE_MASK) == 0);
198 
199 	rs = ((uint64_t)pid << 32) | lpid;
200 	rb = va | is | ap;
201 	__asm __volatile(PPC_TLBIE_5(%0, %1, %2, %3, 1) : :
202 		"r" (rb), "r" (rs), "i" (ric), "i" (prs) : "memory");
203 }
204 
205 static __inline void
206 radix_tlbie_fixup(uint32_t pid, vm_offset_t va, int ap)
207 {
208 
209 	__asm __volatile("ptesync" ::: "memory");
210 	radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE,
211 	    TLBIEL_INVAL_PAGE, 0, 0, va, ap);
212 	__asm __volatile("ptesync" ::: "memory");
213 	radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE,
214 	    TLBIEL_INVAL_PAGE, pid, 0, va, ap);
215 }
216 
217 static __inline void
218 radix_tlbie_invlpg_user_4k(uint32_t pid, vm_offset_t va)
219 {
220 
221 	radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE,
222 		TLBIEL_INVAL_PAGE, pid, 0, va, TLBIE_ACTUAL_PAGE_4K);
223 	radix_tlbie_fixup(pid, va, TLBIE_ACTUAL_PAGE_4K);
224 }
225 
226 static __inline void
227 radix_tlbie_invlpg_user_2m(uint32_t pid, vm_offset_t va)
228 {
229 
230 	radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE,
231 		TLBIEL_INVAL_PAGE, pid, 0, va, TLBIE_ACTUAL_PAGE_2M);
232 	radix_tlbie_fixup(pid, va, TLBIE_ACTUAL_PAGE_2M);
233 }
234 
235 static __inline void
236 radix_tlbie_invlpwc_user(uint32_t pid)
237 {
238 
239 	radix_tlbie(TLBIE_RIC_INVALIDATE_PWC, TLBIE_PRS_PROCESS_SCOPE,
240 		TLBIEL_INVAL_SET_PID, pid, 0, 0, 0);
241 }
242 
243 static __inline void
244 radix_tlbie_flush_user(uint32_t pid)
245 {
246 
247 	radix_tlbie(TLBIE_RIC_INVALIDATE_ALL, TLBIE_PRS_PROCESS_SCOPE,
248 		TLBIEL_INVAL_SET_PID, pid, 0, 0, 0);
249 }
250 
251 static __inline void
252 radix_tlbie_invlpg_kernel_4k(vm_offset_t va)
253 {
254 
255 	radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE,
256 	    TLBIEL_INVAL_PAGE, 0, 0, va, TLBIE_ACTUAL_PAGE_4K);
257 	radix_tlbie_fixup(0, va, TLBIE_ACTUAL_PAGE_4K);
258 }
259 
260 static __inline void
261 radix_tlbie_invlpg_kernel_2m(vm_offset_t va)
262 {
263 
264 	radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE,
265 	    TLBIEL_INVAL_PAGE, 0, 0, va, TLBIE_ACTUAL_PAGE_2M);
266 	radix_tlbie_fixup(0, va, TLBIE_ACTUAL_PAGE_2M);
267 }
268 
269 /* 1GB pages aren't currently supported. */
270 static __inline __unused void
271 radix_tlbie_invlpg_kernel_1g(vm_offset_t va)
272 {
273 
274 	radix_tlbie(TLBIE_RIC_INVALIDATE_TLB, TLBIE_PRS_PROCESS_SCOPE,
275 	    TLBIEL_INVAL_PAGE, 0, 0, va, TLBIE_ACTUAL_PAGE_1G);
276 	radix_tlbie_fixup(0, va, TLBIE_ACTUAL_PAGE_1G);
277 }
278 
279 static __inline void
280 radix_tlbie_invlpwc_kernel(void)
281 {
282 
283 	radix_tlbie(TLBIE_RIC_INVALIDATE_PWC, TLBIE_PRS_PROCESS_SCOPE,
284 	    TLBIEL_INVAL_SET_LPID, 0, 0, 0, 0);
285 }
286 
287 static __inline void
288 radix_tlbie_flush_kernel(void)
289 {
290 
291 	radix_tlbie(TLBIE_RIC_INVALIDATE_ALL, TLBIE_PRS_PROCESS_SCOPE,
292 	    TLBIEL_INVAL_SET_LPID, 0, 0, 0, 0);
293 }
294 
295 static __inline vm_pindex_t
296 pmap_l3e_pindex(vm_offset_t va)
297 {
298 	return ((va & PG_FRAME) >> L3_PAGE_SIZE_SHIFT);
299 }
300 
301 static __inline vm_pindex_t
302 pmap_pml3e_index(vm_offset_t va)
303 {
304 
305 	return ((va >> L3_PAGE_SIZE_SHIFT) & RPTE_MASK);
306 }
307 
308 static __inline vm_pindex_t
309 pmap_pml2e_index(vm_offset_t va)
310 {
311 	return ((va >> L2_PAGE_SIZE_SHIFT) & RPTE_MASK);
312 }
313 
314 static __inline vm_pindex_t
315 pmap_pml1e_index(vm_offset_t va)
316 {
317 	return ((va & PG_FRAME) >> L1_PAGE_SIZE_SHIFT);
318 }
319 
320 /* Return various clipped indexes for a given VA */
321 static __inline vm_pindex_t
322 pmap_pte_index(vm_offset_t va)
323 {
324 
325 	return ((va >> PAGE_SHIFT) & RPTE_MASK);
326 }
327 
328 /* Return a pointer to the PT slot that corresponds to a VA */
329 static __inline pt_entry_t *
330 pmap_l3e_to_pte(pt_entry_t *l3e, vm_offset_t va)
331 {
332 	pt_entry_t *pte;
333 	vm_paddr_t ptepa;
334 
335 	ptepa = (be64toh(*l3e) & NLB_MASK);
336 	pte = (pt_entry_t *)PHYS_TO_DMAP(ptepa);
337 	return (&pte[pmap_pte_index(va)]);
338 }
339 
340 /* Return a pointer to the PD slot that corresponds to a VA */
341 static __inline pt_entry_t *
342 pmap_l2e_to_l3e(pt_entry_t *l2e, vm_offset_t va)
343 {
344 	pt_entry_t *l3e;
345 	vm_paddr_t l3pa;
346 
347 	l3pa = (be64toh(*l2e) & NLB_MASK);
348 	l3e = (pml3_entry_t *)PHYS_TO_DMAP(l3pa);
349 	return (&l3e[pmap_pml3e_index(va)]);
350 }
351 
352 /* Return a pointer to the PD slot that corresponds to a VA */
353 static __inline pt_entry_t *
354 pmap_l1e_to_l2e(pt_entry_t *l1e, vm_offset_t va)
355 {
356 	pt_entry_t *l2e;
357 	vm_paddr_t l2pa;
358 
359 	l2pa = (be64toh(*l1e) & NLB_MASK);
360 
361 	l2e = (pml2_entry_t *)PHYS_TO_DMAP(l2pa);
362 	return (&l2e[pmap_pml2e_index(va)]);
363 }
364 
365 static __inline pml1_entry_t *
366 pmap_pml1e(pmap_t pmap, vm_offset_t va)
367 {
368 
369 	return (&pmap->pm_pml1[pmap_pml1e_index(va)]);
370 }
371 
372 static pt_entry_t *
373 pmap_pml2e(pmap_t pmap, vm_offset_t va)
374 {
375 	pt_entry_t *l1e;
376 
377 	l1e = pmap_pml1e(pmap, va);
378 	if (l1e == NULL || (be64toh(*l1e) & RPTE_VALID) == 0)
379 		return (NULL);
380 	return (pmap_l1e_to_l2e(l1e, va));
381 }
382 
383 static __inline pt_entry_t *
384 pmap_pml3e(pmap_t pmap, vm_offset_t va)
385 {
386 	pt_entry_t *l2e;
387 
388 	l2e = pmap_pml2e(pmap, va);
389 	if (l2e == NULL || (be64toh(*l2e) & RPTE_VALID) == 0)
390 		return (NULL);
391 	return (pmap_l2e_to_l3e(l2e, va));
392 }
393 
394 static __inline pt_entry_t *
395 pmap_pte(pmap_t pmap, vm_offset_t va)
396 {
397 	pt_entry_t *l3e;
398 
399 	l3e = pmap_pml3e(pmap, va);
400 	if (l3e == NULL || (be64toh(*l3e) & RPTE_VALID) == 0)
401 		return (NULL);
402 	return (pmap_l3e_to_pte(l3e, va));
403 }
404 
405 int nkpt = 64;
406 SYSCTL_INT(_machdep, OID_AUTO, nkpt, CTLFLAG_RD, &nkpt, 0,
407     "Number of kernel page table pages allocated on bootup");
408 
409 vm_paddr_t dmaplimit;
410 
411 SYSCTL_DECL(_vm_pmap);
412 
413 #ifdef INVARIANTS
414 #define VERBOSE_PMAP 0
415 #define VERBOSE_PROTECT 0
416 static int pmap_logging;
417 SYSCTL_INT(_vm_pmap, OID_AUTO, pmap_logging, CTLFLAG_RWTUN,
418     &pmap_logging, 0, "verbose debug logging");
419 #endif
420 
421 static u_int64_t	KPTphys;	/* phys addr of kernel level 1 */
422 
423 //static vm_paddr_t	KERNend;	/* phys addr of end of bootstrap data */
424 
425 static vm_offset_t qframe = 0;
426 static struct mtx qframe_mtx;
427 
428 void mmu_radix_activate(struct thread *);
429 void mmu_radix_advise(pmap_t, vm_offset_t, vm_offset_t, int);
430 void mmu_radix_align_superpage(vm_object_t, vm_ooffset_t, vm_offset_t *,
431     vm_size_t);
432 void mmu_radix_clear_modify(vm_page_t);
433 void mmu_radix_copy(pmap_t, pmap_t, vm_offset_t, vm_size_t, vm_offset_t);
434 int mmu_radix_decode_kernel_ptr(vm_offset_t, int *, vm_offset_t *);
435 int mmu_radix_enter(pmap_t, vm_offset_t, vm_page_t, vm_prot_t, u_int, int8_t);
436 void mmu_radix_enter_object(pmap_t, vm_offset_t, vm_offset_t, vm_page_t,
437 	vm_prot_t);
438 void mmu_radix_enter_quick(pmap_t, vm_offset_t, vm_page_t, vm_prot_t);
439 vm_paddr_t mmu_radix_extract(pmap_t pmap, vm_offset_t va);
440 vm_page_t mmu_radix_extract_and_hold(pmap_t, vm_offset_t, vm_prot_t);
441 void mmu_radix_kenter(vm_offset_t, vm_paddr_t);
442 vm_paddr_t mmu_radix_kextract(vm_offset_t);
443 void mmu_radix_kremove(vm_offset_t);
444 boolean_t mmu_radix_is_modified(vm_page_t);
445 boolean_t mmu_radix_is_prefaultable(pmap_t, vm_offset_t);
446 boolean_t mmu_radix_is_referenced(vm_page_t);
447 void mmu_radix_object_init_pt(pmap_t, vm_offset_t, vm_object_t,
448 	vm_pindex_t, vm_size_t);
449 boolean_t mmu_radix_page_exists_quick(pmap_t, vm_page_t);
450 void mmu_radix_page_init(vm_page_t);
451 boolean_t mmu_radix_page_is_mapped(vm_page_t m);
452 void mmu_radix_page_set_memattr(vm_page_t, vm_memattr_t);
453 int mmu_radix_page_wired_mappings(vm_page_t);
454 int mmu_radix_pinit(pmap_t);
455 void mmu_radix_protect(pmap_t, vm_offset_t, vm_offset_t, vm_prot_t);
456 bool mmu_radix_ps_enabled(pmap_t);
457 void mmu_radix_qenter(vm_offset_t, vm_page_t *, int);
458 void mmu_radix_qremove(vm_offset_t, int);
459 vm_offset_t mmu_radix_quick_enter_page(vm_page_t);
460 void mmu_radix_quick_remove_page(vm_offset_t);
461 boolean_t mmu_radix_ts_referenced(vm_page_t);
462 void mmu_radix_release(pmap_t);
463 void mmu_radix_remove(pmap_t, vm_offset_t, vm_offset_t);
464 void mmu_radix_remove_all(vm_page_t);
465 void mmu_radix_remove_pages(pmap_t);
466 void mmu_radix_remove_write(vm_page_t);
467 void mmu_radix_unwire(pmap_t, vm_offset_t, vm_offset_t);
468 void mmu_radix_zero_page(vm_page_t);
469 void mmu_radix_zero_page_area(vm_page_t, int, int);
470 int mmu_radix_change_attr(vm_offset_t, vm_size_t, vm_memattr_t);
471 void mmu_radix_page_array_startup(long pages);
472 
473 #include "mmu_oea64.h"
474 
475 /*
476  * Kernel MMU interface
477  */
478 
479 static void	mmu_radix_bootstrap(vm_offset_t, vm_offset_t);
480 
481 static void mmu_radix_copy_page(vm_page_t, vm_page_t);
482 static void mmu_radix_copy_pages(vm_page_t *ma, vm_offset_t a_offset,
483     vm_page_t *mb, vm_offset_t b_offset, int xfersize);
484 static void mmu_radix_growkernel(vm_offset_t);
485 static void mmu_radix_init(void);
486 static int mmu_radix_mincore(pmap_t, vm_offset_t, vm_paddr_t *);
487 static vm_offset_t mmu_radix_map(vm_offset_t *, vm_paddr_t, vm_paddr_t, int);
488 static void mmu_radix_pinit0(pmap_t);
489 
490 static void *mmu_radix_mapdev(vm_paddr_t, vm_size_t);
491 static void *mmu_radix_mapdev_attr(vm_paddr_t, vm_size_t, vm_memattr_t);
492 static void mmu_radix_unmapdev(vm_offset_t, vm_size_t);
493 static void mmu_radix_kenter_attr(vm_offset_t, vm_paddr_t, vm_memattr_t ma);
494 static boolean_t mmu_radix_dev_direct_mapped(vm_paddr_t, vm_size_t);
495 static void mmu_radix_dumpsys_map(vm_paddr_t pa, size_t sz, void **va);
496 static void mmu_radix_scan_init(void);
497 static void	mmu_radix_cpu_bootstrap(int ap);
498 static void	mmu_radix_tlbie_all(void);
499 
500 static struct pmap_funcs mmu_radix_methods = {
501 	.bootstrap = mmu_radix_bootstrap,
502 	.copy_page = mmu_radix_copy_page,
503 	.copy_pages = mmu_radix_copy_pages,
504 	.cpu_bootstrap = mmu_radix_cpu_bootstrap,
505 	.growkernel = mmu_radix_growkernel,
506 	.init = mmu_radix_init,
507 	.map =      		mmu_radix_map,
508 	.mincore =      	mmu_radix_mincore,
509 	.pinit = mmu_radix_pinit,
510 	.pinit0 = mmu_radix_pinit0,
511 
512 	.mapdev = mmu_radix_mapdev,
513 	.mapdev_attr = mmu_radix_mapdev_attr,
514 	.unmapdev = mmu_radix_unmapdev,
515 	.kenter_attr = mmu_radix_kenter_attr,
516 	.dev_direct_mapped = mmu_radix_dev_direct_mapped,
517 	.dumpsys_pa_init = mmu_radix_scan_init,
518 	.dumpsys_map_chunk = mmu_radix_dumpsys_map,
519 	.page_is_mapped = mmu_radix_page_is_mapped,
520 	.ps_enabled = mmu_radix_ps_enabled,
521 	.align_superpage = mmu_radix_align_superpage,
522 	.object_init_pt = mmu_radix_object_init_pt,
523 	.protect = mmu_radix_protect,
524 	/* pmap dispatcher interface */
525 	.clear_modify = mmu_radix_clear_modify,
526 	.copy = mmu_radix_copy,
527 	.enter = mmu_radix_enter,
528 	.enter_object = mmu_radix_enter_object,
529 	.enter_quick = mmu_radix_enter_quick,
530 	.extract = mmu_radix_extract,
531 	.extract_and_hold = mmu_radix_extract_and_hold,
532 	.is_modified = mmu_radix_is_modified,
533 	.is_prefaultable = mmu_radix_is_prefaultable,
534 	.is_referenced = mmu_radix_is_referenced,
535 	.ts_referenced = mmu_radix_ts_referenced,
536 	.page_exists_quick = mmu_radix_page_exists_quick,
537 	.page_init = mmu_radix_page_init,
538 	.page_wired_mappings =  mmu_radix_page_wired_mappings,
539 	.qenter = mmu_radix_qenter,
540 	.qremove = mmu_radix_qremove,
541 	.release = mmu_radix_release,
542 	.remove = mmu_radix_remove,
543 	.remove_all = mmu_radix_remove_all,
544 	.remove_write = mmu_radix_remove_write,
545 	.unwire = mmu_radix_unwire,
546 	.zero_page = mmu_radix_zero_page,
547 	.zero_page_area = mmu_radix_zero_page_area,
548 	.activate = mmu_radix_activate,
549 	.quick_enter_page =  mmu_radix_quick_enter_page,
550 	.quick_remove_page =  mmu_radix_quick_remove_page,
551 	.page_set_memattr = mmu_radix_page_set_memattr,
552 	.page_array_startup =  mmu_radix_page_array_startup,
553 
554 	/* Internal interfaces */
555 	.kenter = mmu_radix_kenter,
556 	.kextract = mmu_radix_kextract,
557 	.kremove = mmu_radix_kremove,
558 	.change_attr = mmu_radix_change_attr,
559 	.decode_kernel_ptr =  mmu_radix_decode_kernel_ptr,
560 
561 	.tlbie_all = mmu_radix_tlbie_all,
562 };
563 
564 MMU_DEF(mmu_radix, MMU_TYPE_RADIX, mmu_radix_methods);
565 
566 static boolean_t pmap_demote_l3e_locked(pmap_t pmap, pml3_entry_t *l3e, vm_offset_t va,
567 	struct rwlock **lockp);
568 static boolean_t pmap_demote_l3e(pmap_t pmap, pml3_entry_t *pde, vm_offset_t va);
569 static int pmap_unuse_pt(pmap_t, vm_offset_t, pml3_entry_t, struct spglist *);
570 static int pmap_remove_l3e(pmap_t pmap, pml3_entry_t *pdq, vm_offset_t sva,
571     struct spglist *free, struct rwlock **lockp);
572 static int pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t sva,
573     pml3_entry_t ptepde, struct spglist *free, struct rwlock **lockp);
574 static vm_page_t pmap_remove_pt_page(pmap_t pmap, vm_offset_t va);
575 static bool pmap_remove_page(pmap_t pmap, vm_offset_t va, pml3_entry_t *pde,
576     struct spglist *free);
577 static bool	pmap_remove_ptes(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,
578 	pml3_entry_t *l3e, struct spglist *free, struct rwlock **lockp);
579 
580 static bool	pmap_pv_insert_l3e(pmap_t pmap, vm_offset_t va, pml3_entry_t l3e,
581 		    u_int flags, struct rwlock **lockp);
582 #if VM_NRESERVLEVEL > 0
583 static void	pmap_pv_promote_l3e(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,
584 	struct rwlock **lockp);
585 #endif
586 static void	pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va);
587 static int pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte);
588 static vm_page_t mmu_radix_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m,
589 	vm_prot_t prot, vm_page_t mpte, struct rwlock **lockp, bool *invalidate);
590 
591 static bool	pmap_enter_2mpage(pmap_t pmap, vm_offset_t va, vm_page_t m,
592 	vm_prot_t prot, struct rwlock **lockp);
593 static int	pmap_enter_l3e(pmap_t pmap, vm_offset_t va, pml3_entry_t newpde,
594 	u_int flags, vm_page_t m, struct rwlock **lockp);
595 
596 static vm_page_t reclaim_pv_chunk(pmap_t locked_pmap, struct rwlock **lockp);
597 static void free_pv_chunk(struct pv_chunk *pc);
598 static vm_page_t _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, struct rwlock **lockp);
599 static vm_page_t pmap_allocl3e(pmap_t pmap, vm_offset_t va,
600 	struct rwlock **lockp);
601 static vm_page_t pmap_allocpte(pmap_t pmap, vm_offset_t va,
602 	struct rwlock **lockp);
603 static void _pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m,
604     struct spglist *free);
605 static boolean_t pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free);
606 
607 static void pmap_invalidate_page(pmap_t pmap, vm_offset_t start);
608 static void pmap_invalidate_all(pmap_t pmap);
609 static int pmap_change_attr_locked(vm_offset_t va, vm_size_t size, int mode, bool flush);
610 
611 /*
612  * Internal flags for pmap_enter()'s helper functions.
613  */
614 #define	PMAP_ENTER_NORECLAIM	0x1000000	/* Don't reclaim PV entries. */
615 #define	PMAP_ENTER_NOREPLACE	0x2000000	/* Don't replace mappings. */
616 
617 #define UNIMPLEMENTED() panic("%s not implemented", __func__)
618 #define UNTESTED() panic("%s not yet tested", __func__)
619 
620 /* Number of supported PID bits */
621 static unsigned int isa3_pid_bits;
622 
623 /* PID to start allocating from */
624 static unsigned int isa3_base_pid;
625 
626 #define PROCTAB_SIZE_SHIFT	(isa3_pid_bits + 4)
627 #define PROCTAB_ENTRIES	(1ul << isa3_pid_bits)
628 
629 /*
630  * Map of physical memory regions.
631  */
632 static struct	mem_region *regions, *pregions;
633 static struct	numa_mem_region *numa_pregions;
634 static u_int	phys_avail_count;
635 static int	regions_sz, pregions_sz, numa_pregions_sz;
636 static struct pate *isa3_parttab;
637 static struct prte *isa3_proctab;
638 static vmem_t *asid_arena;
639 
640 extern void bs_remap_earlyboot(void);
641 
642 #define	RADIX_PGD_SIZE_SHIFT	16
643 #define RADIX_PGD_SIZE	(1UL << RADIX_PGD_SIZE_SHIFT)
644 
645 #define	RADIX_PGD_INDEX_SHIFT	(RADIX_PGD_SIZE_SHIFT-3)
646 #define NL2EPG (PAGE_SIZE/sizeof(pml2_entry_t))
647 #define NL3EPG (PAGE_SIZE/sizeof(pml3_entry_t))
648 
649 #define	NUPML1E		(RADIX_PGD_SIZE/sizeof(uint64_t))	/* number of userland PML1 pages */
650 #define	NUPDPE		(NUPML1E * NL2EPG)/* number of userland PDP pages */
651 #define	NUPDE		(NUPDPE * NL3EPG)	/* number of userland PD entries */
652 
653 /* POWER9 only permits a 64k partition table size. */
654 #define	PARTTAB_SIZE_SHIFT	16
655 #define PARTTAB_SIZE	(1UL << PARTTAB_SIZE_SHIFT)
656 
657 #define PARTTAB_HR		(1UL << 63) /* host uses radix */
658 #define PARTTAB_GR		(1UL << 63) /* guest uses radix must match host */
659 
660 /* TLB flush actions. Used as argument to tlbiel_flush() */
661 enum {
662 	TLB_INVAL_SCOPE_LPID = 2,	/* invalidate TLBs for current LPID */
663 	TLB_INVAL_SCOPE_GLOBAL = 3,	/* invalidate all TLBs */
664 };
665 
666 #define	NPV_LIST_LOCKS	MAXCPU
667 static int pmap_initialized;
668 static vm_paddr_t proctab0pa;
669 static vm_paddr_t parttab_phys;
670 CTASSERT(sizeof(struct pv_chunk) == PAGE_SIZE);
671 
672 /*
673  * Data for the pv entry allocation mechanism.
674  * Updates to pv_invl_gen are protected by the pv_list_locks[]
675  * elements, but reads are not.
676  */
677 static TAILQ_HEAD(pch, pv_chunk) pv_chunks = TAILQ_HEAD_INITIALIZER(pv_chunks);
678 static struct mtx __exclusive_cache_line pv_chunks_mutex;
679 static struct rwlock __exclusive_cache_line pv_list_locks[NPV_LIST_LOCKS];
680 static struct md_page *pv_table;
681 static struct md_page pv_dummy;
682 
683 #ifdef PV_STATS
684 #define PV_STAT(x)	do { x ; } while (0)
685 #else
686 #define PV_STAT(x)	do { } while (0)
687 #endif
688 
689 #define	pa_radix_index(pa)	((pa) >> L3_PAGE_SIZE_SHIFT)
690 #define	pa_to_pvh(pa)	(&pv_table[pa_radix_index(pa)])
691 
692 #define	PHYS_TO_PV_LIST_LOCK(pa)	\
693 			(&pv_list_locks[pa_radix_index(pa) % NPV_LIST_LOCKS])
694 
695 #define	CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa)	do {	\
696 	struct rwlock **_lockp = (lockp);		\
697 	struct rwlock *_new_lock;			\
698 							\
699 	_new_lock = PHYS_TO_PV_LIST_LOCK(pa);		\
700 	if (_new_lock != *_lockp) {			\
701 		if (*_lockp != NULL)			\
702 			rw_wunlock(*_lockp);		\
703 		*_lockp = _new_lock;			\
704 		rw_wlock(*_lockp);			\
705 	}						\
706 } while (0)
707 
708 #define	CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m)	\
709 	CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, VM_PAGE_TO_PHYS(m))
710 
711 #define	RELEASE_PV_LIST_LOCK(lockp)		do {	\
712 	struct rwlock **_lockp = (lockp);		\
713 							\
714 	if (*_lockp != NULL) {				\
715 		rw_wunlock(*_lockp);			\
716 		*_lockp = NULL;				\
717 	}						\
718 } while (0)
719 
720 #define	VM_PAGE_TO_PV_LIST_LOCK(m)	\
721 	PHYS_TO_PV_LIST_LOCK(VM_PAGE_TO_PHYS(m))
722 
723 /*
724  * We support 52 bits, hence:
725  * bits 52 - 31 = 21, 0b10101
726  * RTS encoding details
727  * bits 0 - 3 of rts -> bits 6 - 8 unsigned long
728  * bits 4 - 5 of rts -> bits 62 - 63 of unsigned long
729  */
730 #define RTS_SIZE ((0x2UL << 61) | (0x5UL << 5))
731 
732 static int powernv_enabled = 1;
733 
734 static __always_inline void
735 tlbiel_radix_set_isa300(uint32_t set, uint32_t is,
736 	uint32_t pid, uint32_t ric, uint32_t prs)
737 {
738 	uint64_t rb;
739 	uint64_t rs;
740 
741 	rb = PPC_BITLSHIFT_VAL(set, 51) | PPC_BITLSHIFT_VAL(is, 53);
742 	rs = PPC_BITLSHIFT_VAL((uint64_t)pid, 31);
743 
744 	__asm __volatile(PPC_TLBIEL(%0, %1, %2, %3, 1)
745 		     : : "r"(rb), "r"(rs), "i"(ric), "i"(prs)
746 		     : "memory");
747 }
748 
749 static void
750 tlbiel_flush_isa3(uint32_t num_sets, uint32_t is)
751 {
752 	uint32_t set;
753 
754 	__asm __volatile("ptesync": : :"memory");
755 
756 	/*
757 	 * Flush the first set of the TLB, and the entire Page Walk Cache
758 	 * and partition table entries. Then flush the remaining sets of the
759 	 * TLB.
760 	 */
761 	if (is == TLB_INVAL_SCOPE_GLOBAL) {
762 		tlbiel_radix_set_isa300(0, is, 0, RIC_FLUSH_ALL, 0);
763 		for (set = 1; set < num_sets; set++)
764 			tlbiel_radix_set_isa300(set, is, 0, RIC_FLUSH_TLB, 0);
765 	}
766 
767 	/* Do the same for process scoped entries. */
768 	tlbiel_radix_set_isa300(0, is, 0, RIC_FLUSH_ALL, 1);
769 	for (set = 1; set < num_sets; set++)
770 		tlbiel_radix_set_isa300(set, is, 0, RIC_FLUSH_TLB, 1);
771 
772 	__asm __volatile("ptesync": : :"memory");
773 }
774 
775 static void
776 mmu_radix_tlbiel_flush(int scope)
777 {
778 	MPASS(scope == TLB_INVAL_SCOPE_LPID ||
779 		  scope == TLB_INVAL_SCOPE_GLOBAL);
780 
781 	tlbiel_flush_isa3(POWER9_TLB_SETS_RADIX, scope);
782 	__asm __volatile(PPC_INVALIDATE_ERAT "; isync" : : :"memory");
783 }
784 
785 static void
786 mmu_radix_tlbie_all()
787 {
788 	if (powernv_enabled)
789 		mmu_radix_tlbiel_flush(TLB_INVAL_SCOPE_GLOBAL);
790 	else
791 		mmu_radix_tlbiel_flush(TLB_INVAL_SCOPE_LPID);
792 }
793 
794 static void
795 mmu_radix_init_amor(void)
796 {
797 	/*
798 	* In HV mode, we init AMOR (Authority Mask Override Register) so that
799 	* the hypervisor and guest can setup IAMR (Instruction Authority Mask
800 	* Register), enable key 0 and set it to 1.
801 	*
802 	* AMOR = 0b1100 .... 0000 (Mask for key 0 is 11)
803 	*/
804 	mtspr(SPR_AMOR, (3ul << 62));
805 }
806 
807 static void
808 mmu_radix_init_iamr(void)
809 {
810 	/*
811 	 * Radix always uses key0 of the IAMR to determine if an access is
812 	 * allowed. We set bit 0 (IBM bit 1) of key0, to prevent instruction
813 	 * fetch.
814 	 */
815 	mtspr(SPR_IAMR, (1ul << 62));
816 }
817 
818 static void
819 mmu_radix_pid_set(pmap_t pmap)
820 {
821 
822 	mtspr(SPR_PID, pmap->pm_pid);
823 	isync();
824 }
825 
826 /* Quick sort callout for comparing physical addresses. */
827 static int
828 pa_cmp(const void *a, const void *b)
829 {
830 	const vm_paddr_t *pa = a, *pb = b;
831 
832 	if (*pa < *pb)
833 		return (-1);
834 	else if (*pa > *pb)
835 		return (1);
836 	else
837 		return (0);
838 }
839 
840 #define	pte_load_store(ptep, pte)	atomic_swap_long(ptep, pte)
841 #define	pte_load_clear(ptep)		atomic_swap_long(ptep, 0)
842 #define	pte_store(ptep, pte) do {	   \
843 	MPASS((pte) & (RPTE_EAA_R | RPTE_EAA_W | RPTE_EAA_X));	\
844 	*(u_long *)(ptep) = htobe64((u_long)((pte) | PG_V | RPTE_LEAF)); \
845 } while (0)
846 /*
847  * NB: should only be used for adding directories - not for direct mappings
848  */
849 #define	pde_store(ptep, pa) do {				\
850 	*(u_long *)(ptep) = htobe64((u_long)(pa|RPTE_VALID|RPTE_SHIFT)); \
851 } while (0)
852 
853 #define	pte_clear(ptep) do {					\
854 		*(u_long *)(ptep) = (u_long)(0);		\
855 } while (0)
856 
857 #define	PMAP_PDE_SUPERPAGE	(1 << 8)	/* supports 2MB superpages */
858 
859 /*
860  * Promotion to a 2MB (PDE) page mapping requires that the corresponding 4KB
861  * (PTE) page mappings have identical settings for the following fields:
862  */
863 #define	PG_PTE_PROMOTE	(PG_X | PG_MANAGED | PG_W | PG_PTE_CACHE | \
864 	    PG_M | PG_A | RPTE_EAA_MASK | PG_V)
865 
866 static __inline void
867 pmap_resident_count_inc(pmap_t pmap, int count)
868 {
869 
870 	PMAP_LOCK_ASSERT(pmap, MA_OWNED);
871 	pmap->pm_stats.resident_count += count;
872 }
873 
874 static __inline void
875 pmap_resident_count_dec(pmap_t pmap, int count)
876 {
877 
878 	PMAP_LOCK_ASSERT(pmap, MA_OWNED);
879 	KASSERT(pmap->pm_stats.resident_count >= count,
880 	    ("pmap %p resident count underflow %ld %d", pmap,
881 	    pmap->pm_stats.resident_count, count));
882 	pmap->pm_stats.resident_count -= count;
883 }
884 
885 static void
886 pagezero(vm_offset_t va)
887 {
888 	va = trunc_page(va);
889 
890 	bzero((void *)va, PAGE_SIZE);
891 }
892 
893 static uint64_t
894 allocpages(int n)
895 {
896 	u_int64_t ret;
897 
898 	ret = moea64_bootstrap_alloc(n * PAGE_SIZE, PAGE_SIZE);
899 	for (int i = 0; i < n; i++)
900 		pagezero(PHYS_TO_DMAP(ret + i * PAGE_SIZE));
901 	return (ret);
902 }
903 
904 static pt_entry_t *
905 kvtopte(vm_offset_t va)
906 {
907 	pt_entry_t *l3e;
908 
909 	l3e = pmap_pml3e(kernel_pmap, va);
910 	if (l3e == NULL || (be64toh(*l3e) & RPTE_VALID) == 0)
911 		return (NULL);
912 	return (pmap_l3e_to_pte(l3e, va));
913 }
914 
915 void
916 mmu_radix_kenter(vm_offset_t va, vm_paddr_t pa)
917 {
918 	pt_entry_t *pte;
919 
920 	pte = kvtopte(va);
921 	MPASS(pte != NULL);
922 	*pte = htobe64(pa | RPTE_VALID | RPTE_LEAF | RPTE_EAA_R | \
923 	    RPTE_EAA_W | RPTE_EAA_P | PG_M | PG_A);
924 }
925 
926 bool
927 mmu_radix_ps_enabled(pmap_t pmap)
928 {
929 	return (superpages_enabled && (pmap->pm_flags & PMAP_PDE_SUPERPAGE) != 0);
930 }
931 
932 static pt_entry_t *
933 pmap_nofault_pte(pmap_t pmap, vm_offset_t va, int *is_l3e)
934 {
935 	pml3_entry_t *l3e;
936 	pt_entry_t *pte;
937 
938 	va &= PG_PS_FRAME;
939 	l3e = pmap_pml3e(pmap, va);
940 	if (l3e == NULL || (be64toh(*l3e) & PG_V) == 0)
941 		return (NULL);
942 
943 	if (be64toh(*l3e) & RPTE_LEAF) {
944 		*is_l3e = 1;
945 		return (l3e);
946 	}
947 	*is_l3e = 0;
948 	va &= PG_FRAME;
949 	pte = pmap_l3e_to_pte(l3e, va);
950 	if (pte == NULL || (be64toh(*pte) & PG_V) == 0)
951 		return (NULL);
952 	return (pte);
953 }
954 
955 int
956 pmap_nofault(pmap_t pmap, vm_offset_t va, vm_prot_t flags)
957 {
958 	pt_entry_t *pte;
959 	pt_entry_t startpte, origpte, newpte;
960 	vm_page_t m;
961 	int is_l3e;
962 
963 	startpte = 0;
964  retry:
965 	if ((pte = pmap_nofault_pte(pmap, va, &is_l3e)) == NULL)
966 		return (KERN_INVALID_ADDRESS);
967 	origpte = newpte = be64toh(*pte);
968 	if (startpte == 0) {
969 		startpte = origpte;
970 		if (((flags & VM_PROT_WRITE) && (startpte & PG_M)) ||
971 		    ((flags & VM_PROT_READ) && (startpte & PG_A))) {
972 			pmap_invalidate_all(pmap);
973 #ifdef INVARIANTS
974 			if (VERBOSE_PMAP || pmap_logging)
975 				printf("%s(%p, %#lx, %#x) (%#lx) -- invalidate all\n",
976 				    __func__, pmap, va, flags, origpte);
977 #endif
978 			return (KERN_FAILURE);
979 		}
980 	}
981 #ifdef INVARIANTS
982 	if (VERBOSE_PMAP || pmap_logging)
983 		printf("%s(%p, %#lx, %#x) (%#lx)\n", __func__, pmap, va,
984 		    flags, origpte);
985 #endif
986 	PMAP_LOCK(pmap);
987 	if ((pte = pmap_nofault_pte(pmap, va, &is_l3e)) == NULL ||
988 	    be64toh(*pte) != origpte) {
989 		PMAP_UNLOCK(pmap);
990 		return (KERN_FAILURE);
991 	}
992 	m = PHYS_TO_VM_PAGE(newpte & PG_FRAME);
993 	MPASS(m != NULL);
994 	switch (flags) {
995 	case VM_PROT_READ:
996 		if ((newpte & (RPTE_EAA_R|RPTE_EAA_X)) == 0)
997 			goto protfail;
998 		newpte |= PG_A;
999 		vm_page_aflag_set(m, PGA_REFERENCED);
1000 		break;
1001 	case VM_PROT_WRITE:
1002 		if ((newpte & RPTE_EAA_W) == 0)
1003 			goto protfail;
1004 		if (is_l3e)
1005 			goto protfail;
1006 		newpte |= PG_M;
1007 		vm_page_dirty(m);
1008 		break;
1009 	case VM_PROT_EXECUTE:
1010 		if ((newpte & RPTE_EAA_X) == 0)
1011 			goto protfail;
1012 		newpte |= PG_A;
1013 		vm_page_aflag_set(m, PGA_REFERENCED);
1014 		break;
1015 	}
1016 
1017 	if (!atomic_cmpset_long(pte, htobe64(origpte), htobe64(newpte)))
1018 		goto retry;
1019 	ptesync();
1020 	PMAP_UNLOCK(pmap);
1021 	if (startpte == newpte)
1022 		return (KERN_FAILURE);
1023 	return (0);
1024  protfail:
1025 	PMAP_UNLOCK(pmap);
1026 	return (KERN_PROTECTION_FAILURE);
1027 }
1028 
1029 /*
1030  * Returns TRUE if the given page is mapped individually or as part of
1031  * a 2mpage.  Otherwise, returns FALSE.
1032  */
1033 boolean_t
1034 mmu_radix_page_is_mapped(vm_page_t m)
1035 {
1036 	struct rwlock *lock;
1037 	boolean_t rv;
1038 
1039 	if ((m->oflags & VPO_UNMANAGED) != 0)
1040 		return (FALSE);
1041 	lock = VM_PAGE_TO_PV_LIST_LOCK(m);
1042 	rw_rlock(lock);
1043 	rv = !TAILQ_EMPTY(&m->md.pv_list) ||
1044 	    ((m->flags & PG_FICTITIOUS) == 0 &&
1045 	    !TAILQ_EMPTY(&pa_to_pvh(VM_PAGE_TO_PHYS(m))->pv_list));
1046 	rw_runlock(lock);
1047 	return (rv);
1048 }
1049 
1050 /*
1051  * Determine the appropriate bits to set in a PTE or PDE for a specified
1052  * caching mode.
1053  */
1054 static int
1055 pmap_cache_bits(vm_memattr_t ma)
1056 {
1057 	if (ma != VM_MEMATTR_DEFAULT) {
1058 		switch (ma) {
1059 		case VM_MEMATTR_UNCACHEABLE:
1060 			return (RPTE_ATTR_GUARDEDIO);
1061 		case VM_MEMATTR_CACHEABLE:
1062 			return (RPTE_ATTR_MEM);
1063 		case VM_MEMATTR_WRITE_BACK:
1064 		case VM_MEMATTR_PREFETCHABLE:
1065 		case VM_MEMATTR_WRITE_COMBINING:
1066 			return (RPTE_ATTR_UNGUARDEDIO);
1067 		}
1068 	}
1069 	return (0);
1070 }
1071 
1072 static void
1073 pmap_invalidate_page(pmap_t pmap, vm_offset_t start)
1074 {
1075 	ptesync();
1076 	if (pmap == kernel_pmap)
1077 		radix_tlbie_invlpg_kernel_4k(start);
1078 	else
1079 		radix_tlbie_invlpg_user_4k(pmap->pm_pid, start);
1080 	ttusync();
1081 }
1082 
1083 static void
1084 pmap_invalidate_page_2m(pmap_t pmap, vm_offset_t start)
1085 {
1086 	ptesync();
1087 	if (pmap == kernel_pmap)
1088 		radix_tlbie_invlpg_kernel_2m(start);
1089 	else
1090 		radix_tlbie_invlpg_user_2m(pmap->pm_pid, start);
1091 	ttusync();
1092 }
1093 
1094 static void
1095 pmap_invalidate_pwc(pmap_t pmap)
1096 {
1097 	ptesync();
1098 	if (pmap == kernel_pmap)
1099 		radix_tlbie_invlpwc_kernel();
1100 	else
1101 		radix_tlbie_invlpwc_user(pmap->pm_pid);
1102 	ttusync();
1103 }
1104 
1105 static void
1106 pmap_invalidate_range(pmap_t pmap, vm_offset_t start, vm_offset_t end)
1107 {
1108 	if (((start - end) >> PAGE_SHIFT) > 8) {
1109 		pmap_invalidate_all(pmap);
1110 		return;
1111 	}
1112 	ptesync();
1113 	if (pmap == kernel_pmap) {
1114 		while (start < end) {
1115 			radix_tlbie_invlpg_kernel_4k(start);
1116 			start += PAGE_SIZE;
1117 		}
1118 	} else {
1119 		while (start < end) {
1120 			radix_tlbie_invlpg_user_4k(pmap->pm_pid, start);
1121 			start += PAGE_SIZE;
1122 		}
1123 	}
1124 	ttusync();
1125 }
1126 
1127 static void
1128 pmap_invalidate_all(pmap_t pmap)
1129 {
1130 	ptesync();
1131 	if (pmap == kernel_pmap)
1132 		radix_tlbie_flush_kernel();
1133 	else
1134 		radix_tlbie_flush_user(pmap->pm_pid);
1135 	ttusync();
1136 }
1137 
1138 static void
1139 pmap_invalidate_l3e_page(pmap_t pmap, vm_offset_t va, pml3_entry_t l3e)
1140 {
1141 
1142 	/*
1143 	 * When the PDE has PG_PROMOTED set, the 2MB page mapping was created
1144 	 * by a promotion that did not invalidate the 512 4KB page mappings
1145 	 * that might exist in the TLB.  Consequently, at this point, the TLB
1146 	 * may hold both 4KB and 2MB page mappings for the address range [va,
1147 	 * va + L3_PAGE_SIZE).  Therefore, the entire range must be invalidated here.
1148 	 * In contrast, when PG_PROMOTED is clear, the TLB will not hold any
1149 	 * 4KB page mappings for the address range [va, va + L3_PAGE_SIZE), and so a
1150 	 * single INVLPG suffices to invalidate the 2MB page mapping from the
1151 	 * TLB.
1152 	 */
1153 	ptesync();
1154 	if ((l3e & PG_PROMOTED) != 0)
1155 		pmap_invalidate_range(pmap, va, va + L3_PAGE_SIZE - 1);
1156 	else
1157 		pmap_invalidate_page_2m(pmap, va);
1158 
1159 	pmap_invalidate_pwc(pmap);
1160 }
1161 
1162 static __inline struct pv_chunk *
1163 pv_to_chunk(pv_entry_t pv)
1164 {
1165 
1166 	return ((struct pv_chunk *)((uintptr_t)pv & ~(uintptr_t)PAGE_MASK));
1167 }
1168 
1169 #define PV_PMAP(pv) (pv_to_chunk(pv)->pc_pmap)
1170 
1171 #define	PC_FREE0	0xfffffffffffffffful
1172 #define	PC_FREE1	0x3ffffffffffffffful
1173 
1174 static const uint64_t pc_freemask[_NPCM] = { PC_FREE0, PC_FREE1 };
1175 
1176 /*
1177  * Ensure that the number of spare PV entries in the specified pmap meets or
1178  * exceeds the given count, "needed".
1179  *
1180  * The given PV list lock may be released.
1181  */
1182 static void
1183 reserve_pv_entries(pmap_t pmap, int needed, struct rwlock **lockp)
1184 {
1185 	struct pch new_tail;
1186 	struct pv_chunk *pc;
1187 	vm_page_t m;
1188 	int avail, free;
1189 	bool reclaimed;
1190 
1191 	PMAP_LOCK_ASSERT(pmap, MA_OWNED);
1192 	KASSERT(lockp != NULL, ("reserve_pv_entries: lockp is NULL"));
1193 
1194 	/*
1195 	 * Newly allocated PV chunks must be stored in a private list until
1196 	 * the required number of PV chunks have been allocated.  Otherwise,
1197 	 * reclaim_pv_chunk() could recycle one of these chunks.  In
1198 	 * contrast, these chunks must be added to the pmap upon allocation.
1199 	 */
1200 	TAILQ_INIT(&new_tail);
1201 retry:
1202 	avail = 0;
1203 	TAILQ_FOREACH(pc, &pmap->pm_pvchunk, pc_list) {
1204 		//		if ((cpu_feature2 & CPUID2_POPCNT) == 0)
1205 		bit_count((bitstr_t *)pc->pc_map, 0,
1206 				  sizeof(pc->pc_map) * NBBY, &free);
1207 #if 0
1208 		free = popcnt_pc_map_pq(pc->pc_map);
1209 #endif
1210 		if (free == 0)
1211 			break;
1212 		avail += free;
1213 		if (avail >= needed)
1214 			break;
1215 	}
1216 	for (reclaimed = false; avail < needed; avail += _NPCPV) {
1217 		m = vm_page_alloc_noobj(VM_ALLOC_WIRED);
1218 		if (m == NULL) {
1219 			m = reclaim_pv_chunk(pmap, lockp);
1220 			if (m == NULL)
1221 				goto retry;
1222 			reclaimed = true;
1223 		}
1224 		PV_STAT(atomic_add_int(&pc_chunk_count, 1));
1225 		PV_STAT(atomic_add_int(&pc_chunk_allocs, 1));
1226 		dump_add_page(m->phys_addr);
1227 		pc = (void *)PHYS_TO_DMAP(m->phys_addr);
1228 		pc->pc_pmap = pmap;
1229 		pc->pc_map[0] = PC_FREE0;
1230 		pc->pc_map[1] = PC_FREE1;
1231 		TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);
1232 		TAILQ_INSERT_TAIL(&new_tail, pc, pc_lru);
1233 		PV_STAT(atomic_add_int(&pv_entry_spare, _NPCPV));
1234 
1235 		/*
1236 		 * The reclaim might have freed a chunk from the current pmap.
1237 		 * If that chunk contained available entries, we need to
1238 		 * re-count the number of available entries.
1239 		 */
1240 		if (reclaimed)
1241 			goto retry;
1242 	}
1243 	if (!TAILQ_EMPTY(&new_tail)) {
1244 		mtx_lock(&pv_chunks_mutex);
1245 		TAILQ_CONCAT(&pv_chunks, &new_tail, pc_lru);
1246 		mtx_unlock(&pv_chunks_mutex);
1247 	}
1248 }
1249 
1250 /*
1251  * First find and then remove the pv entry for the specified pmap and virtual
1252  * address from the specified pv list.  Returns the pv entry if found and NULL
1253  * otherwise.  This operation can be performed on pv lists for either 4KB or
1254  * 2MB page mappings.
1255  */
1256 static __inline pv_entry_t
1257 pmap_pvh_remove(struct md_page *pvh, pmap_t pmap, vm_offset_t va)
1258 {
1259 	pv_entry_t pv;
1260 
1261 	TAILQ_FOREACH(pv, &pvh->pv_list, pv_link) {
1262 #ifdef INVARIANTS
1263 		if (PV_PMAP(pv) == NULL) {
1264 			printf("corrupted pv_chunk/pv %p\n", pv);
1265 			printf("pv_chunk: %64D\n", pv_to_chunk(pv), ":");
1266 		}
1267 		MPASS(PV_PMAP(pv) != NULL);
1268 		MPASS(pv->pv_va != 0);
1269 #endif
1270 		if (pmap == PV_PMAP(pv) && va == pv->pv_va) {
1271 			TAILQ_REMOVE(&pvh->pv_list, pv, pv_link);
1272 			pvh->pv_gen++;
1273 			break;
1274 		}
1275 	}
1276 	return (pv);
1277 }
1278 
1279 /*
1280  * After demotion from a 2MB page mapping to 512 4KB page mappings,
1281  * destroy the pv entry for the 2MB page mapping and reinstantiate the pv
1282  * entries for each of the 4KB page mappings.
1283  */
1284 static void
1285 pmap_pv_demote_l3e(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,
1286     struct rwlock **lockp)
1287 {
1288 	struct md_page *pvh;
1289 	struct pv_chunk *pc;
1290 	pv_entry_t pv;
1291 	vm_offset_t va_last;
1292 	vm_page_t m;
1293 	int bit, field;
1294 
1295 	PMAP_LOCK_ASSERT(pmap, MA_OWNED);
1296 	KASSERT((pa & L3_PAGE_MASK) == 0,
1297 	    ("pmap_pv_demote_pde: pa is not 2mpage aligned"));
1298 	CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa);
1299 
1300 	/*
1301 	 * Transfer the 2mpage's pv entry for this mapping to the first
1302 	 * page's pv list.  Once this transfer begins, the pv list lock
1303 	 * must not be released until the last pv entry is reinstantiated.
1304 	 */
1305 	pvh = pa_to_pvh(pa);
1306 	va = trunc_2mpage(va);
1307 	pv = pmap_pvh_remove(pvh, pmap, va);
1308 	KASSERT(pv != NULL, ("pmap_pv_demote_pde: pv not found"));
1309 	m = PHYS_TO_VM_PAGE(pa);
1310 	TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_link);
1311 
1312 	m->md.pv_gen++;
1313 	/* Instantiate the remaining NPTEPG - 1 pv entries. */
1314 	PV_STAT(atomic_add_long(&pv_entry_allocs, NPTEPG - 1));
1315 	va_last = va + L3_PAGE_SIZE - PAGE_SIZE;
1316 	for (;;) {
1317 		pc = TAILQ_FIRST(&pmap->pm_pvchunk);
1318 		KASSERT(pc->pc_map[0] != 0 || pc->pc_map[1] != 0
1319 		    , ("pmap_pv_demote_pde: missing spare"));
1320 		for (field = 0; field < _NPCM; field++) {
1321 			while (pc->pc_map[field]) {
1322 				bit = cnttzd(pc->pc_map[field]);
1323 				pc->pc_map[field] &= ~(1ul << bit);
1324 				pv = &pc->pc_pventry[field * 64 + bit];
1325 				va += PAGE_SIZE;
1326 				pv->pv_va = va;
1327 				m++;
1328 				KASSERT((m->oflags & VPO_UNMANAGED) == 0,
1329 			    ("pmap_pv_demote_pde: page %p is not managed", m));
1330 				TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_link);
1331 
1332 				m->md.pv_gen++;
1333 				if (va == va_last)
1334 					goto out;
1335 			}
1336 		}
1337 		TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
1338 		TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list);
1339 	}
1340 out:
1341 	if (pc->pc_map[0] == 0 && pc->pc_map[1] == 0) {
1342 		TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
1343 		TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list);
1344 	}
1345 	PV_STAT(atomic_add_long(&pv_entry_count, NPTEPG - 1));
1346 	PV_STAT(atomic_subtract_int(&pv_entry_spare, NPTEPG - 1));
1347 }
1348 
1349 static void
1350 reclaim_pv_chunk_leave_pmap(pmap_t pmap, pmap_t locked_pmap)
1351 {
1352 
1353 	if (pmap == NULL)
1354 		return;
1355 	pmap_invalidate_all(pmap);
1356 	if (pmap != locked_pmap)
1357 		PMAP_UNLOCK(pmap);
1358 }
1359 
1360 /*
1361  * We are in a serious low memory condition.  Resort to
1362  * drastic measures to free some pages so we can allocate
1363  * another pv entry chunk.
1364  *
1365  * Returns NULL if PV entries were reclaimed from the specified pmap.
1366  *
1367  * We do not, however, unmap 2mpages because subsequent accesses will
1368  * allocate per-page pv entries until repromotion occurs, thereby
1369  * exacerbating the shortage of free pv entries.
1370  */
1371 static int active_reclaims = 0;
1372 static vm_page_t
1373 reclaim_pv_chunk(pmap_t locked_pmap, struct rwlock **lockp)
1374 {
1375 	struct pv_chunk *pc, *pc_marker, *pc_marker_end;
1376 	struct pv_chunk_header pc_marker_b, pc_marker_end_b;
1377 	struct md_page *pvh;
1378 	pml3_entry_t *l3e;
1379 	pmap_t next_pmap, pmap;
1380 	pt_entry_t *pte, tpte;
1381 	pv_entry_t pv;
1382 	vm_offset_t va;
1383 	vm_page_t m, m_pc;
1384 	struct spglist free;
1385 	uint64_t inuse;
1386 	int bit, field, freed;
1387 
1388 	PMAP_LOCK_ASSERT(locked_pmap, MA_OWNED);
1389 	KASSERT(lockp != NULL, ("reclaim_pv_chunk: lockp is NULL"));
1390 	pmap = NULL;
1391 	m_pc = NULL;
1392 	SLIST_INIT(&free);
1393 	bzero(&pc_marker_b, sizeof(pc_marker_b));
1394 	bzero(&pc_marker_end_b, sizeof(pc_marker_end_b));
1395 	pc_marker = (struct pv_chunk *)&pc_marker_b;
1396 	pc_marker_end = (struct pv_chunk *)&pc_marker_end_b;
1397 
1398 	mtx_lock(&pv_chunks_mutex);
1399 	active_reclaims++;
1400 	TAILQ_INSERT_HEAD(&pv_chunks, pc_marker, pc_lru);
1401 	TAILQ_INSERT_TAIL(&pv_chunks, pc_marker_end, pc_lru);
1402 	while ((pc = TAILQ_NEXT(pc_marker, pc_lru)) != pc_marker_end &&
1403 	    SLIST_EMPTY(&free)) {
1404 		next_pmap = pc->pc_pmap;
1405 		if (next_pmap == NULL) {
1406 			/*
1407 			 * The next chunk is a marker.  However, it is
1408 			 * not our marker, so active_reclaims must be
1409 			 * > 1.  Consequently, the next_chunk code
1410 			 * will not rotate the pv_chunks list.
1411 			 */
1412 			goto next_chunk;
1413 		}
1414 		mtx_unlock(&pv_chunks_mutex);
1415 
1416 		/*
1417 		 * A pv_chunk can only be removed from the pc_lru list
1418 		 * when both pc_chunks_mutex is owned and the
1419 		 * corresponding pmap is locked.
1420 		 */
1421 		if (pmap != next_pmap) {
1422 			reclaim_pv_chunk_leave_pmap(pmap, locked_pmap);
1423 			pmap = next_pmap;
1424 			/* Avoid deadlock and lock recursion. */
1425 			if (pmap > locked_pmap) {
1426 				RELEASE_PV_LIST_LOCK(lockp);
1427 				PMAP_LOCK(pmap);
1428 				mtx_lock(&pv_chunks_mutex);
1429 				continue;
1430 			} else if (pmap != locked_pmap) {
1431 				if (PMAP_TRYLOCK(pmap)) {
1432 					mtx_lock(&pv_chunks_mutex);
1433 					continue;
1434 				} else {
1435 					pmap = NULL; /* pmap is not locked */
1436 					mtx_lock(&pv_chunks_mutex);
1437 					pc = TAILQ_NEXT(pc_marker, pc_lru);
1438 					if (pc == NULL ||
1439 					    pc->pc_pmap != next_pmap)
1440 						continue;
1441 					goto next_chunk;
1442 				}
1443 			}
1444 		}
1445 
1446 		/*
1447 		 * Destroy every non-wired, 4 KB page mapping in the chunk.
1448 		 */
1449 		freed = 0;
1450 		for (field = 0; field < _NPCM; field++) {
1451 			for (inuse = ~pc->pc_map[field] & pc_freemask[field];
1452 			    inuse != 0; inuse &= ~(1UL << bit)) {
1453 				bit = cnttzd(inuse);
1454 				pv = &pc->pc_pventry[field * 64 + bit];
1455 				va = pv->pv_va;
1456 				l3e = pmap_pml3e(pmap, va);
1457 				if ((be64toh(*l3e) & RPTE_LEAF) != 0)
1458 					continue;
1459 				pte = pmap_l3e_to_pte(l3e, va);
1460 				if ((be64toh(*pte) & PG_W) != 0)
1461 					continue;
1462 				tpte = be64toh(pte_load_clear(pte));
1463 				m = PHYS_TO_VM_PAGE(tpte & PG_FRAME);
1464 				if ((tpte & (PG_M | PG_RW)) == (PG_M | PG_RW))
1465 					vm_page_dirty(m);
1466 				if ((tpte & PG_A) != 0)
1467 					vm_page_aflag_set(m, PGA_REFERENCED);
1468 				CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m);
1469 				TAILQ_REMOVE(&m->md.pv_list, pv, pv_link);
1470 
1471 				m->md.pv_gen++;
1472 				if (TAILQ_EMPTY(&m->md.pv_list) &&
1473 				    (m->flags & PG_FICTITIOUS) == 0) {
1474 					pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
1475 					if (TAILQ_EMPTY(&pvh->pv_list)) {
1476 						vm_page_aflag_clear(m,
1477 						    PGA_WRITEABLE);
1478 					}
1479 				}
1480 				pc->pc_map[field] |= 1UL << bit;
1481 				pmap_unuse_pt(pmap, va, be64toh(*l3e), &free);
1482 				freed++;
1483 			}
1484 		}
1485 		if (freed == 0) {
1486 			mtx_lock(&pv_chunks_mutex);
1487 			goto next_chunk;
1488 		}
1489 		/* Every freed mapping is for a 4 KB page. */
1490 		pmap_resident_count_dec(pmap, freed);
1491 		PV_STAT(atomic_add_long(&pv_entry_frees, freed));
1492 		PV_STAT(atomic_add_int(&pv_entry_spare, freed));
1493 		PV_STAT(atomic_subtract_long(&pv_entry_count, freed));
1494 		TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
1495 		if (pc->pc_map[0] == PC_FREE0 && pc->pc_map[1] == PC_FREE1) {
1496 			PV_STAT(atomic_subtract_int(&pv_entry_spare, _NPCPV));
1497 			PV_STAT(atomic_subtract_int(&pc_chunk_count, 1));
1498 			PV_STAT(atomic_add_int(&pc_chunk_frees, 1));
1499 			/* Entire chunk is free; return it. */
1500 			m_pc = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pc));
1501 			dump_drop_page(m_pc->phys_addr);
1502 			mtx_lock(&pv_chunks_mutex);
1503 			TAILQ_REMOVE(&pv_chunks, pc, pc_lru);
1504 			break;
1505 		}
1506 		TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);
1507 		mtx_lock(&pv_chunks_mutex);
1508 		/* One freed pv entry in locked_pmap is sufficient. */
1509 		if (pmap == locked_pmap)
1510 			break;
1511 next_chunk:
1512 		TAILQ_REMOVE(&pv_chunks, pc_marker, pc_lru);
1513 		TAILQ_INSERT_AFTER(&pv_chunks, pc, pc_marker, pc_lru);
1514 		if (active_reclaims == 1 && pmap != NULL) {
1515 			/*
1516 			 * Rotate the pv chunks list so that we do not
1517 			 * scan the same pv chunks that could not be
1518 			 * freed (because they contained a wired
1519 			 * and/or superpage mapping) on every
1520 			 * invocation of reclaim_pv_chunk().
1521 			 */
1522 			while ((pc = TAILQ_FIRST(&pv_chunks)) != pc_marker) {
1523 				MPASS(pc->pc_pmap != NULL);
1524 				TAILQ_REMOVE(&pv_chunks, pc, pc_lru);
1525 				TAILQ_INSERT_TAIL(&pv_chunks, pc, pc_lru);
1526 			}
1527 		}
1528 	}
1529 	TAILQ_REMOVE(&pv_chunks, pc_marker, pc_lru);
1530 	TAILQ_REMOVE(&pv_chunks, pc_marker_end, pc_lru);
1531 	active_reclaims--;
1532 	mtx_unlock(&pv_chunks_mutex);
1533 	reclaim_pv_chunk_leave_pmap(pmap, locked_pmap);
1534 	if (m_pc == NULL && !SLIST_EMPTY(&free)) {
1535 		m_pc = SLIST_FIRST(&free);
1536 		SLIST_REMOVE_HEAD(&free, plinks.s.ss);
1537 		/* Recycle a freed page table page. */
1538 		m_pc->ref_count = 1;
1539 	}
1540 	vm_page_free_pages_toq(&free, true);
1541 	return (m_pc);
1542 }
1543 
1544 /*
1545  * free the pv_entry back to the free list
1546  */
1547 static void
1548 free_pv_entry(pmap_t pmap, pv_entry_t pv)
1549 {
1550 	struct pv_chunk *pc;
1551 	int idx, field, bit;
1552 
1553 #ifdef VERBOSE_PV
1554 	if (pmap != kernel_pmap)
1555 		printf("%s(%p, %p)\n", __func__, pmap, pv);
1556 #endif
1557 	PMAP_LOCK_ASSERT(pmap, MA_OWNED);
1558 	PV_STAT(atomic_add_long(&pv_entry_frees, 1));
1559 	PV_STAT(atomic_add_int(&pv_entry_spare, 1));
1560 	PV_STAT(atomic_subtract_long(&pv_entry_count, 1));
1561 	pc = pv_to_chunk(pv);
1562 	idx = pv - &pc->pc_pventry[0];
1563 	field = idx / 64;
1564 	bit = idx % 64;
1565 	pc->pc_map[field] |= 1ul << bit;
1566 	if (pc->pc_map[0] != PC_FREE0 || pc->pc_map[1] != PC_FREE1) {
1567 		/* 98% of the time, pc is already at the head of the list. */
1568 		if (__predict_false(pc != TAILQ_FIRST(&pmap->pm_pvchunk))) {
1569 			TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
1570 			TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);
1571 		}
1572 		return;
1573 	}
1574 	TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
1575 	free_pv_chunk(pc);
1576 }
1577 
1578 static void
1579 free_pv_chunk(struct pv_chunk *pc)
1580 {
1581 	vm_page_t m;
1582 
1583 	mtx_lock(&pv_chunks_mutex);
1584  	TAILQ_REMOVE(&pv_chunks, pc, pc_lru);
1585 	mtx_unlock(&pv_chunks_mutex);
1586 	PV_STAT(atomic_subtract_int(&pv_entry_spare, _NPCPV));
1587 	PV_STAT(atomic_subtract_int(&pc_chunk_count, 1));
1588 	PV_STAT(atomic_add_int(&pc_chunk_frees, 1));
1589 	/* entire chunk is free, return it */
1590 	m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pc));
1591 	dump_drop_page(m->phys_addr);
1592 	vm_page_unwire_noq(m);
1593 	vm_page_free(m);
1594 }
1595 
1596 /*
1597  * Returns a new PV entry, allocating a new PV chunk from the system when
1598  * needed.  If this PV chunk allocation fails and a PV list lock pointer was
1599  * given, a PV chunk is reclaimed from an arbitrary pmap.  Otherwise, NULL is
1600  * returned.
1601  *
1602  * The given PV list lock may be released.
1603  */
1604 static pv_entry_t
1605 get_pv_entry(pmap_t pmap, struct rwlock **lockp)
1606 {
1607 	int bit, field;
1608 	pv_entry_t pv;
1609 	struct pv_chunk *pc;
1610 	vm_page_t m;
1611 
1612 	PMAP_LOCK_ASSERT(pmap, MA_OWNED);
1613 	PV_STAT(atomic_add_long(&pv_entry_allocs, 1));
1614 retry:
1615 	pc = TAILQ_FIRST(&pmap->pm_pvchunk);
1616 	if (pc != NULL) {
1617 		for (field = 0; field < _NPCM; field++) {
1618 			if (pc->pc_map[field]) {
1619 				bit = cnttzd(pc->pc_map[field]);
1620 				break;
1621 			}
1622 		}
1623 		if (field < _NPCM) {
1624 			pv = &pc->pc_pventry[field * 64 + bit];
1625 			pc->pc_map[field] &= ~(1ul << bit);
1626 			/* If this was the last item, move it to tail */
1627 			if (pc->pc_map[0] == 0 && pc->pc_map[1] == 0) {
1628 				TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
1629 				TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc,
1630 				    pc_list);
1631 			}
1632 			PV_STAT(atomic_add_long(&pv_entry_count, 1));
1633 			PV_STAT(atomic_subtract_int(&pv_entry_spare, 1));
1634 			MPASS(PV_PMAP(pv) != NULL);
1635 			return (pv);
1636 		}
1637 	}
1638 	/* No free items, allocate another chunk */
1639 	m = vm_page_alloc_noobj(VM_ALLOC_WIRED);
1640 	if (m == NULL) {
1641 		if (lockp == NULL) {
1642 			PV_STAT(pc_chunk_tryfail++);
1643 			return (NULL);
1644 		}
1645 		m = reclaim_pv_chunk(pmap, lockp);
1646 		if (m == NULL)
1647 			goto retry;
1648 	}
1649 	PV_STAT(atomic_add_int(&pc_chunk_count, 1));
1650 	PV_STAT(atomic_add_int(&pc_chunk_allocs, 1));
1651 	dump_add_page(m->phys_addr);
1652 	pc = (void *)PHYS_TO_DMAP(m->phys_addr);
1653 	pc->pc_pmap = pmap;
1654 	pc->pc_map[0] = PC_FREE0 & ~1ul;	/* preallocated bit 0 */
1655 	pc->pc_map[1] = PC_FREE1;
1656 	mtx_lock(&pv_chunks_mutex);
1657 	TAILQ_INSERT_TAIL(&pv_chunks, pc, pc_lru);
1658 	mtx_unlock(&pv_chunks_mutex);
1659 	pv = &pc->pc_pventry[0];
1660 	TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);
1661 	PV_STAT(atomic_add_long(&pv_entry_count, 1));
1662 	PV_STAT(atomic_add_int(&pv_entry_spare, _NPCPV - 1));
1663 	MPASS(PV_PMAP(pv) != NULL);
1664 	return (pv);
1665 }
1666 
1667 #if VM_NRESERVLEVEL > 0
1668 /*
1669  * After promotion from 512 4KB page mappings to a single 2MB page mapping,
1670  * replace the many pv entries for the 4KB page mappings by a single pv entry
1671  * for the 2MB page mapping.
1672  */
1673 static void
1674 pmap_pv_promote_l3e(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,
1675     struct rwlock **lockp)
1676 {
1677 	struct md_page *pvh;
1678 	pv_entry_t pv;
1679 	vm_offset_t va_last;
1680 	vm_page_t m;
1681 
1682 	KASSERT((pa & L3_PAGE_MASK) == 0,
1683 	    ("pmap_pv_promote_pde: pa is not 2mpage aligned"));
1684 	CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa);
1685 
1686 	/*
1687 	 * Transfer the first page's pv entry for this mapping to the 2mpage's
1688 	 * pv list.  Aside from avoiding the cost of a call to get_pv_entry(),
1689 	 * a transfer avoids the possibility that get_pv_entry() calls
1690 	 * reclaim_pv_chunk() and that reclaim_pv_chunk() removes one of the
1691 	 * mappings that is being promoted.
1692 	 */
1693 	m = PHYS_TO_VM_PAGE(pa);
1694 	va = trunc_2mpage(va);
1695 	pv = pmap_pvh_remove(&m->md, pmap, va);
1696 	KASSERT(pv != NULL, ("pmap_pv_promote_pde: pv not found"));
1697 	pvh = pa_to_pvh(pa);
1698 	TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_link);
1699 	pvh->pv_gen++;
1700 	/* Free the remaining NPTEPG - 1 pv entries. */
1701 	va_last = va + L3_PAGE_SIZE - PAGE_SIZE;
1702 	do {
1703 		m++;
1704 		va += PAGE_SIZE;
1705 		pmap_pvh_free(&m->md, pmap, va);
1706 	} while (va < va_last);
1707 }
1708 #endif /* VM_NRESERVLEVEL > 0 */
1709 
1710 /*
1711  * First find and then destroy the pv entry for the specified pmap and virtual
1712  * address.  This operation can be performed on pv lists for either 4KB or 2MB
1713  * page mappings.
1714  */
1715 static void
1716 pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va)
1717 {
1718 	pv_entry_t pv;
1719 
1720 	pv = pmap_pvh_remove(pvh, pmap, va);
1721 	KASSERT(pv != NULL, ("pmap_pvh_free: pv not found"));
1722 	free_pv_entry(pmap, pv);
1723 }
1724 
1725 /*
1726  * Conditionally create the PV entry for a 4KB page mapping if the required
1727  * memory can be allocated without resorting to reclamation.
1728  */
1729 static boolean_t
1730 pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, vm_page_t m,
1731     struct rwlock **lockp)
1732 {
1733 	pv_entry_t pv;
1734 
1735 	PMAP_LOCK_ASSERT(pmap, MA_OWNED);
1736 	/* Pass NULL instead of the lock pointer to disable reclamation. */
1737 	if ((pv = get_pv_entry(pmap, NULL)) != NULL) {
1738 		pv->pv_va = va;
1739 		CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m);
1740 		TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_link);
1741 		m->md.pv_gen++;
1742 		return (TRUE);
1743 	} else
1744 		return (FALSE);
1745 }
1746 
1747 vm_paddr_t phys_avail_debug[2 * VM_PHYSSEG_MAX];
1748 #ifdef INVARIANTS
1749 static void
1750 validate_addr(vm_paddr_t addr, vm_size_t size)
1751 {
1752 	vm_paddr_t end = addr + size;
1753 	bool found = false;
1754 
1755 	for (int i = 0; i < 2 * phys_avail_count; i += 2) {
1756 		if (addr >= phys_avail_debug[i] &&
1757 			end <= phys_avail_debug[i + 1]) {
1758 			found = true;
1759 			break;
1760 		}
1761 	}
1762 	KASSERT(found, ("%#lx-%#lx outside of initial phys_avail array",
1763 					addr, end));
1764 }
1765 #else
1766 static void validate_addr(vm_paddr_t addr, vm_size_t size) {}
1767 #endif
1768 #define DMAP_PAGE_BITS (RPTE_VALID | RPTE_LEAF | RPTE_EAA_MASK | PG_M | PG_A)
1769 
1770 static vm_paddr_t
1771 alloc_pt_page(void)
1772 {
1773 	vm_paddr_t page;
1774 
1775 	page = allocpages(1);
1776 	pagezero(PHYS_TO_DMAP(page));
1777 	return (page);
1778 }
1779 
1780 static void
1781 mmu_radix_dmap_range(vm_paddr_t start, vm_paddr_t end)
1782 {
1783 	pt_entry_t *pte, pteval;
1784 	vm_paddr_t page;
1785 
1786 	if (bootverbose)
1787 		printf("%s %lx -> %lx\n", __func__, start, end);
1788 	while (start < end) {
1789 		pteval = start | DMAP_PAGE_BITS;
1790 		pte = pmap_pml1e(kernel_pmap, PHYS_TO_DMAP(start));
1791 		if ((be64toh(*pte) & RPTE_VALID) == 0) {
1792 			page = alloc_pt_page();
1793 			pde_store(pte, page);
1794 		}
1795 		pte = pmap_l1e_to_l2e(pte, PHYS_TO_DMAP(start));
1796 		if ((start & L2_PAGE_MASK) == 0 &&
1797 			end - start >= L2_PAGE_SIZE) {
1798 			start += L2_PAGE_SIZE;
1799 			goto done;
1800 		} else if ((be64toh(*pte) & RPTE_VALID) == 0) {
1801 			page = alloc_pt_page();
1802 			pde_store(pte, page);
1803 		}
1804 
1805 		pte = pmap_l2e_to_l3e(pte, PHYS_TO_DMAP(start));
1806 		if ((start & L3_PAGE_MASK) == 0 &&
1807 			end - start >= L3_PAGE_SIZE) {
1808 			start += L3_PAGE_SIZE;
1809 			goto done;
1810 		} else if ((be64toh(*pte) & RPTE_VALID) == 0) {
1811 			page = alloc_pt_page();
1812 			pde_store(pte, page);
1813 		}
1814 		pte = pmap_l3e_to_pte(pte, PHYS_TO_DMAP(start));
1815 		start += PAGE_SIZE;
1816 	done:
1817 		pte_store(pte, pteval);
1818 	}
1819 }
1820 
1821 static void
1822 mmu_radix_dmap_populate(vm_size_t hwphyssz)
1823 {
1824 	vm_paddr_t start, end;
1825 
1826 	for (int i = 0; i < pregions_sz; i++) {
1827 		start = pregions[i].mr_start;
1828 		end = start + pregions[i].mr_size;
1829 		if (hwphyssz && start >= hwphyssz)
1830 			break;
1831 		if (hwphyssz && hwphyssz < end)
1832 			end = hwphyssz;
1833 		mmu_radix_dmap_range(start, end);
1834 	}
1835 }
1836 
1837 static void
1838 mmu_radix_setup_pagetables(vm_size_t hwphyssz)
1839 {
1840 	vm_paddr_t ptpages, pages;
1841 	pt_entry_t *pte;
1842 	vm_paddr_t l1phys;
1843 
1844 	bzero(kernel_pmap, sizeof(struct pmap));
1845 	PMAP_LOCK_INIT(kernel_pmap);
1846 
1847 	ptpages = allocpages(3);
1848 	l1phys = moea64_bootstrap_alloc(RADIX_PGD_SIZE, RADIX_PGD_SIZE);
1849 	validate_addr(l1phys, RADIX_PGD_SIZE);
1850 	if (bootverbose)
1851 		printf("l1phys=%lx\n", l1phys);
1852 	MPASS((l1phys & (RADIX_PGD_SIZE-1)) == 0);
1853 	for (int i = 0; i < RADIX_PGD_SIZE/PAGE_SIZE; i++)
1854 		pagezero(PHYS_TO_DMAP(l1phys + i * PAGE_SIZE));
1855 	kernel_pmap->pm_pml1 = (pml1_entry_t *)PHYS_TO_DMAP(l1phys);
1856 
1857 	mmu_radix_dmap_populate(hwphyssz);
1858 
1859 	/*
1860 	 * Create page tables for first 128MB of KVA
1861 	 */
1862 	pages = ptpages;
1863 	pte = pmap_pml1e(kernel_pmap, VM_MIN_KERNEL_ADDRESS);
1864 	*pte = htobe64(pages | RPTE_VALID | RPTE_SHIFT);
1865 	pages += PAGE_SIZE;
1866 	pte = pmap_l1e_to_l2e(pte, VM_MIN_KERNEL_ADDRESS);
1867 	*pte = htobe64(pages | RPTE_VALID | RPTE_SHIFT);
1868 	pages += PAGE_SIZE;
1869 	pte = pmap_l2e_to_l3e(pte, VM_MIN_KERNEL_ADDRESS);
1870 	/*
1871 	 * the kernel page table pages need to be preserved in
1872 	 * phys_avail and not overlap with previous  allocations
1873 	 */
1874 	pages = allocpages(nkpt);
1875 	if (bootverbose) {
1876 		printf("phys_avail after dmap populate and nkpt allocation\n");
1877 		for (int j = 0; j < 2 * phys_avail_count; j+=2)
1878 			printf("phys_avail[%d]=%08lx - phys_avail[%d]=%08lx\n",
1879 				   j, phys_avail[j], j + 1, phys_avail[j + 1]);
1880 	}
1881 	KPTphys = pages;
1882 	for (int i = 0; i < nkpt; i++, pte++, pages += PAGE_SIZE)
1883 		*pte = htobe64(pages | RPTE_VALID | RPTE_SHIFT);
1884 	kernel_vm_end = VM_MIN_KERNEL_ADDRESS + nkpt * L3_PAGE_SIZE;
1885 	if (bootverbose)
1886 		printf("kernel_pmap pml1 %p\n", kernel_pmap->pm_pml1);
1887 	/*
1888 	 * Add a physical memory segment (vm_phys_seg) corresponding to the
1889 	 * preallocated kernel page table pages so that vm_page structures
1890 	 * representing these pages will be created.  The vm_page structures
1891 	 * are required for promotion of the corresponding kernel virtual
1892 	 * addresses to superpage mappings.
1893 	 */
1894 	vm_phys_add_seg(KPTphys, KPTphys + ptoa(nkpt));
1895 }
1896 
1897 static void
1898 mmu_radix_early_bootstrap(vm_offset_t start, vm_offset_t end)
1899 {
1900 	vm_paddr_t	kpstart, kpend;
1901 	vm_size_t	physsz, hwphyssz;
1902 	//uint64_t	l2virt;
1903 	int		rm_pavail, proctab_size;
1904 	int		i, j;
1905 
1906 	kpstart = start & ~DMAP_BASE_ADDRESS;
1907 	kpend = end & ~DMAP_BASE_ADDRESS;
1908 
1909 	/* Get physical memory regions from firmware */
1910 	mem_regions(&pregions, &pregions_sz, &regions, &regions_sz);
1911 	CTR0(KTR_PMAP, "mmu_radix_early_bootstrap: physical memory");
1912 
1913 	if (2 * VM_PHYSSEG_MAX < regions_sz)
1914 		panic("mmu_radix_early_bootstrap: phys_avail too small");
1915 
1916 	if (bootverbose)
1917 		for (int i = 0; i < regions_sz; i++)
1918 			printf("regions[%d].mr_start=%lx regions[%d].mr_size=%lx\n",
1919 			    i, regions[i].mr_start, i, regions[i].mr_size);
1920 	/*
1921 	 * XXX workaround a simulator bug
1922 	 */
1923 	for (int i = 0; i < regions_sz; i++)
1924 		if (regions[i].mr_start & PAGE_MASK) {
1925 			regions[i].mr_start += PAGE_MASK;
1926 			regions[i].mr_start &= ~PAGE_MASK;
1927 			regions[i].mr_size &= ~PAGE_MASK;
1928 		}
1929 	if (bootverbose)
1930 		for (int i = 0; i < pregions_sz; i++)
1931 			printf("pregions[%d].mr_start=%lx pregions[%d].mr_size=%lx\n",
1932 			    i, pregions[i].mr_start, i, pregions[i].mr_size);
1933 
1934 	phys_avail_count = 0;
1935 	physsz = 0;
1936 	hwphyssz = 0;
1937 	TUNABLE_ULONG_FETCH("hw.physmem", (u_long *) &hwphyssz);
1938 	for (i = 0, j = 0; i < regions_sz; i++) {
1939 		if (bootverbose)
1940 			printf("regions[%d].mr_start=%016lx regions[%d].mr_size=%016lx\n",
1941 			    i, regions[i].mr_start, i, regions[i].mr_size);
1942 
1943 		if (regions[i].mr_size < PAGE_SIZE)
1944 			continue;
1945 
1946 		if (hwphyssz != 0 &&
1947 		    (physsz + regions[i].mr_size) >= hwphyssz) {
1948 			if (physsz < hwphyssz) {
1949 				phys_avail[j] = regions[i].mr_start;
1950 				phys_avail[j + 1] = regions[i].mr_start +
1951 				    (hwphyssz - physsz);
1952 				physsz = hwphyssz;
1953 				phys_avail_count++;
1954 				dump_avail[j] = phys_avail[j];
1955 				dump_avail[j + 1] = phys_avail[j + 1];
1956 			}
1957 			break;
1958 		}
1959 		phys_avail[j] = regions[i].mr_start;
1960 		phys_avail[j + 1] = regions[i].mr_start + regions[i].mr_size;
1961 		dump_avail[j] = phys_avail[j];
1962 		dump_avail[j + 1] = phys_avail[j + 1];
1963 
1964 		phys_avail_count++;
1965 		physsz += regions[i].mr_size;
1966 		j += 2;
1967 	}
1968 
1969 	/* Check for overlap with the kernel and exception vectors */
1970 	rm_pavail = 0;
1971 	for (j = 0; j < 2 * phys_avail_count; j+=2) {
1972 		if (phys_avail[j] < EXC_LAST)
1973 			phys_avail[j] += EXC_LAST;
1974 
1975 		if (phys_avail[j] >= kpstart &&
1976 		    phys_avail[j + 1] <= kpend) {
1977 			phys_avail[j] = phys_avail[j + 1] = ~0;
1978 			rm_pavail++;
1979 			continue;
1980 		}
1981 
1982 		if (kpstart >= phys_avail[j] &&
1983 		    kpstart < phys_avail[j + 1]) {
1984 			if (kpend < phys_avail[j + 1]) {
1985 				phys_avail[2 * phys_avail_count] =
1986 				    (kpend & ~PAGE_MASK) + PAGE_SIZE;
1987 				phys_avail[2 * phys_avail_count + 1] =
1988 				    phys_avail[j + 1];
1989 				phys_avail_count++;
1990 			}
1991 
1992 			phys_avail[j + 1] = kpstart & ~PAGE_MASK;
1993 		}
1994 
1995 		if (kpend >= phys_avail[j] &&
1996 		    kpend < phys_avail[j + 1]) {
1997 			if (kpstart > phys_avail[j]) {
1998 				phys_avail[2 * phys_avail_count] = phys_avail[j];
1999 				phys_avail[2 * phys_avail_count + 1] =
2000 				    kpstart & ~PAGE_MASK;
2001 				phys_avail_count++;
2002 			}
2003 
2004 			phys_avail[j] = (kpend & ~PAGE_MASK) +
2005 			    PAGE_SIZE;
2006 		}
2007 	}
2008 	qsort(phys_avail, 2 * phys_avail_count, sizeof(phys_avail[0]), pa_cmp);
2009 	for (i = 0; i < 2 * phys_avail_count; i++)
2010 		phys_avail_debug[i] = phys_avail[i];
2011 
2012 	/* Remove physical available regions marked for removal (~0) */
2013 	if (rm_pavail) {
2014 		phys_avail_count -= rm_pavail;
2015 		for (i = 2 * phys_avail_count;
2016 		     i < 2*(phys_avail_count + rm_pavail); i+=2)
2017 			phys_avail[i] = phys_avail[i + 1] = 0;
2018 	}
2019 	if (bootverbose) {
2020 		printf("phys_avail ranges after filtering:\n");
2021 		for (j = 0; j < 2 * phys_avail_count; j+=2)
2022 			printf("phys_avail[%d]=%08lx - phys_avail[%d]=%08lx\n",
2023 				   j, phys_avail[j], j + 1, phys_avail[j + 1]);
2024 	}
2025 	physmem = btoc(physsz);
2026 
2027 	/* XXX assume we're running non-virtualized and
2028 	 * we don't support BHYVE
2029 	 */
2030 	if (isa3_pid_bits == 0)
2031 		isa3_pid_bits = 20;
2032 	if (powernv_enabled) {
2033 		parttab_phys =
2034 		    moea64_bootstrap_alloc(PARTTAB_SIZE, PARTTAB_SIZE);
2035 		validate_addr(parttab_phys, PARTTAB_SIZE);
2036 		for (int i = 0; i < PARTTAB_SIZE/PAGE_SIZE; i++)
2037 			pagezero(PHYS_TO_DMAP(parttab_phys + i * PAGE_SIZE));
2038 
2039 	}
2040 	proctab_size = 1UL << PROCTAB_SIZE_SHIFT;
2041 	proctab0pa = moea64_bootstrap_alloc(proctab_size, proctab_size);
2042 	validate_addr(proctab0pa, proctab_size);
2043 	for (int i = 0; i < proctab_size/PAGE_SIZE; i++)
2044 		pagezero(PHYS_TO_DMAP(proctab0pa + i * PAGE_SIZE));
2045 
2046 	mmu_radix_setup_pagetables(hwphyssz);
2047 }
2048 
2049 static void
2050 mmu_radix_late_bootstrap(vm_offset_t start, vm_offset_t end)
2051 {
2052 	int		i;
2053 	vm_paddr_t	pa;
2054 	void		*dpcpu;
2055 	vm_offset_t va;
2056 
2057 	/*
2058 	 * Set up the Open Firmware pmap and add its mappings if not in real
2059 	 * mode.
2060 	 */
2061 	if (bootverbose)
2062 		printf("%s enter\n", __func__);
2063 
2064 	/*
2065 	 * Calculate the last available physical address, and reserve the
2066 	 * vm_page_array (upper bound).
2067 	 */
2068 	Maxmem = 0;
2069 	for (i = 0; phys_avail[i + 2] != 0; i += 2)
2070 		Maxmem = MAX(Maxmem, powerpc_btop(phys_avail[i + 1]));
2071 
2072 	/*
2073 	 * Remap any early IO mappings (console framebuffer, etc.)
2074 	 */
2075 	bs_remap_earlyboot();
2076 
2077 	/*
2078 	 * Allocate a kernel stack with a guard page for thread0 and map it
2079 	 * into the kernel page map.
2080 	 */
2081 	pa = allocpages(kstack_pages);
2082 	va = virtual_avail + KSTACK_GUARD_PAGES * PAGE_SIZE;
2083 	virtual_avail = va + kstack_pages * PAGE_SIZE;
2084 	CTR2(KTR_PMAP, "moea64_bootstrap: kstack0 at %#x (%#x)", pa, va);
2085 	thread0.td_kstack = va;
2086 	for (i = 0; i < kstack_pages; i++) {
2087 		mmu_radix_kenter(va, pa);
2088 		pa += PAGE_SIZE;
2089 		va += PAGE_SIZE;
2090 	}
2091 	thread0.td_kstack_pages = kstack_pages;
2092 
2093 	/*
2094 	 * Allocate virtual address space for the message buffer.
2095 	 */
2096 	pa = msgbuf_phys = allocpages((msgbufsize + PAGE_MASK)  >> PAGE_SHIFT);
2097 	msgbufp = (struct msgbuf *)PHYS_TO_DMAP(pa);
2098 
2099 	/*
2100 	 * Allocate virtual address space for the dynamic percpu area.
2101 	 */
2102 	pa = allocpages(DPCPU_SIZE >> PAGE_SHIFT);
2103 	dpcpu = (void *)PHYS_TO_DMAP(pa);
2104 	dpcpu_init(dpcpu, curcpu);
2105 
2106 	crashdumpmap = (caddr_t)virtual_avail;
2107 	virtual_avail += MAXDUMPPGS * PAGE_SIZE;
2108 
2109 	/*
2110 	 * Reserve some special page table entries/VA space for temporary
2111 	 * mapping of pages.
2112 	 */
2113 }
2114 
2115 static void
2116 mmu_parttab_init(void)
2117 {
2118 	uint64_t ptcr;
2119 
2120 	isa3_parttab = (struct pate *)PHYS_TO_DMAP(parttab_phys);
2121 
2122 	if (bootverbose)
2123 		printf("%s parttab: %p\n", __func__, isa3_parttab);
2124 	ptcr = parttab_phys | (PARTTAB_SIZE_SHIFT-12);
2125 	if (bootverbose)
2126 		printf("setting ptcr %lx\n", ptcr);
2127 	mtspr(SPR_PTCR, ptcr);
2128 }
2129 
2130 static void
2131 mmu_parttab_update(uint64_t lpid, uint64_t pagetab, uint64_t proctab)
2132 {
2133 	uint64_t prev;
2134 
2135 	if (bootverbose)
2136 		printf("%s isa3_parttab %p lpid %lx pagetab %lx proctab %lx\n", __func__, isa3_parttab,
2137 			   lpid, pagetab, proctab);
2138 	prev = be64toh(isa3_parttab[lpid].pagetab);
2139 	isa3_parttab[lpid].pagetab = htobe64(pagetab);
2140 	isa3_parttab[lpid].proctab = htobe64(proctab);
2141 
2142 	if (prev & PARTTAB_HR) {
2143 		__asm __volatile(PPC_TLBIE_5(%0,%1,2,0,1) : :
2144 			     "r" (TLBIEL_INVAL_SET_LPID), "r" (lpid));
2145 		__asm __volatile(PPC_TLBIE_5(%0,%1,2,1,1) : :
2146 			     "r" (TLBIEL_INVAL_SET_LPID), "r" (lpid));
2147 	} else {
2148 		__asm __volatile(PPC_TLBIE_5(%0,%1,2,0,0) : :
2149 			     "r" (TLBIEL_INVAL_SET_LPID), "r" (lpid));
2150 	}
2151 	ttusync();
2152 }
2153 
2154 static void
2155 mmu_radix_parttab_init(void)
2156 {
2157 	uint64_t pagetab;
2158 
2159 	mmu_parttab_init();
2160 	pagetab = RTS_SIZE | DMAP_TO_PHYS((vm_offset_t)kernel_pmap->pm_pml1) | \
2161 		         RADIX_PGD_INDEX_SHIFT | PARTTAB_HR;
2162 	mmu_parttab_update(0, pagetab, 0);
2163 }
2164 
2165 static void
2166 mmu_radix_proctab_register(vm_paddr_t proctabpa, uint64_t table_size)
2167 {
2168 	uint64_t pagetab, proctab;
2169 
2170 	pagetab = be64toh(isa3_parttab[0].pagetab);
2171 	proctab = proctabpa | table_size | PARTTAB_GR;
2172 	mmu_parttab_update(0, pagetab, proctab);
2173 }
2174 
2175 static void
2176 mmu_radix_proctab_init(void)
2177 {
2178 
2179 	isa3_base_pid = 1;
2180 
2181 	isa3_proctab = (void*)PHYS_TO_DMAP(proctab0pa);
2182 	isa3_proctab->proctab0 =
2183 	    htobe64(RTS_SIZE | DMAP_TO_PHYS((vm_offset_t)kernel_pmap->pm_pml1) |
2184 		RADIX_PGD_INDEX_SHIFT);
2185 
2186 	if (powernv_enabled) {
2187 		mmu_radix_proctab_register(proctab0pa, PROCTAB_SIZE_SHIFT - 12);
2188 		__asm __volatile("ptesync" : : : "memory");
2189 		__asm __volatile(PPC_TLBIE_5(%0,%1,2,1,1) : :
2190 			     "r" (TLBIEL_INVAL_SET_LPID), "r" (0));
2191 		__asm __volatile("eieio; tlbsync; ptesync" : : : "memory");
2192 #ifdef PSERIES
2193 	} else {
2194 		int64_t rc;
2195 
2196 		rc = phyp_hcall(H_REGISTER_PROC_TBL,
2197 		    PROC_TABLE_NEW | PROC_TABLE_RADIX | PROC_TABLE_GTSE,
2198 		    proctab0pa, 0, PROCTAB_SIZE_SHIFT - 12);
2199 		if (rc != H_SUCCESS)
2200 			panic("mmu_radix_proctab_init: "
2201 				"failed to register process table: rc=%jd",
2202 				(intmax_t)rc);
2203 #endif
2204 	}
2205 
2206 	if (bootverbose)
2207 		printf("process table %p and kernel radix PDE: %p\n",
2208 			   isa3_proctab, kernel_pmap->pm_pml1);
2209 	mtmsr(mfmsr() | PSL_DR );
2210 	mtmsr(mfmsr() &  ~PSL_DR);
2211 	kernel_pmap->pm_pid = isa3_base_pid;
2212 	isa3_base_pid++;
2213 }
2214 
2215 void
2216 mmu_radix_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,
2217     int advice)
2218 {
2219 	struct rwlock *lock;
2220 	pml1_entry_t *l1e;
2221 	pml2_entry_t *l2e;
2222 	pml3_entry_t oldl3e, *l3e;
2223 	pt_entry_t *pte;
2224 	vm_offset_t va, va_next;
2225 	vm_page_t m;
2226 	bool anychanged;
2227 
2228 	if (advice != MADV_DONTNEED && advice != MADV_FREE)
2229 		return;
2230 	anychanged = false;
2231 	PMAP_LOCK(pmap);
2232 	for (; sva < eva; sva = va_next) {
2233 		l1e = pmap_pml1e(pmap, sva);
2234 		if ((be64toh(*l1e) & PG_V) == 0) {
2235 			va_next = (sva + L1_PAGE_SIZE) & ~L1_PAGE_MASK;
2236 			if (va_next < sva)
2237 				va_next = eva;
2238 			continue;
2239 		}
2240 		l2e = pmap_l1e_to_l2e(l1e, sva);
2241 		if ((be64toh(*l2e) & PG_V) == 0) {
2242 			va_next = (sva + L2_PAGE_SIZE) & ~L2_PAGE_MASK;
2243 			if (va_next < sva)
2244 				va_next = eva;
2245 			continue;
2246 		}
2247 		va_next = (sva + L3_PAGE_SIZE) & ~L3_PAGE_MASK;
2248 		if (va_next < sva)
2249 			va_next = eva;
2250 		l3e = pmap_l2e_to_l3e(l2e, sva);
2251 		oldl3e = be64toh(*l3e);
2252 		if ((oldl3e & PG_V) == 0)
2253 			continue;
2254 		else if ((oldl3e & RPTE_LEAF) != 0) {
2255 			if ((oldl3e & PG_MANAGED) == 0)
2256 				continue;
2257 			lock = NULL;
2258 			if (!pmap_demote_l3e_locked(pmap, l3e, sva, &lock)) {
2259 				if (lock != NULL)
2260 					rw_wunlock(lock);
2261 
2262 				/*
2263 				 * The large page mapping was destroyed.
2264 				 */
2265 				continue;
2266 			}
2267 
2268 			/*
2269 			 * Unless the page mappings are wired, remove the
2270 			 * mapping to a single page so that a subsequent
2271 			 * access may repromote.  Choosing the last page
2272 			 * within the address range [sva, min(va_next, eva))
2273 			 * generally results in more repromotions.  Since the
2274 			 * underlying page table page is fully populated, this
2275 			 * removal never frees a page table page.
2276 			 */
2277 			if ((oldl3e & PG_W) == 0) {
2278 				va = eva;
2279 				if (va > va_next)
2280 					va = va_next;
2281 				va -= PAGE_SIZE;
2282 				KASSERT(va >= sva,
2283 				    ("mmu_radix_advise: no address gap"));
2284 				pte = pmap_l3e_to_pte(l3e, va);
2285 				KASSERT((be64toh(*pte) & PG_V) != 0,
2286 				    ("pmap_advise: invalid PTE"));
2287 				pmap_remove_pte(pmap, pte, va, be64toh(*l3e), NULL,
2288 				    &lock);
2289 				anychanged = true;
2290 			}
2291 			if (lock != NULL)
2292 				rw_wunlock(lock);
2293 		}
2294 		if (va_next > eva)
2295 			va_next = eva;
2296 		va = va_next;
2297 		for (pte = pmap_l3e_to_pte(l3e, sva); sva != va_next;
2298 			 pte++, sva += PAGE_SIZE) {
2299 			MPASS(pte == pmap_pte(pmap, sva));
2300 
2301 			if ((be64toh(*pte) & (PG_MANAGED | PG_V)) != (PG_MANAGED | PG_V))
2302 				goto maybe_invlrng;
2303 			else if ((be64toh(*pte) & (PG_M | PG_RW)) == (PG_M | PG_RW)) {
2304 				if (advice == MADV_DONTNEED) {
2305 					/*
2306 					 * Future calls to pmap_is_modified()
2307 					 * can be avoided by making the page
2308 					 * dirty now.
2309 					 */
2310 					m = PHYS_TO_VM_PAGE(be64toh(*pte) & PG_FRAME);
2311 					vm_page_dirty(m);
2312 				}
2313 				atomic_clear_long(pte, htobe64(PG_M | PG_A));
2314 			} else if ((be64toh(*pte) & PG_A) != 0)
2315 				atomic_clear_long(pte, htobe64(PG_A));
2316 			else
2317 				goto maybe_invlrng;
2318 			anychanged = true;
2319 			continue;
2320 maybe_invlrng:
2321 			if (va != va_next) {
2322 				anychanged = true;
2323 				va = va_next;
2324 			}
2325 		}
2326 		if (va != va_next)
2327 			anychanged = true;
2328 	}
2329 	if (anychanged)
2330 		pmap_invalidate_all(pmap);
2331 	PMAP_UNLOCK(pmap);
2332 }
2333 
2334 /*
2335  * Routines used in machine-dependent code
2336  */
2337 static void
2338 mmu_radix_bootstrap(vm_offset_t start, vm_offset_t end)
2339 {
2340 	uint64_t lpcr;
2341 
2342 	if (bootverbose)
2343 		printf("%s\n", __func__);
2344 	hw_direct_map = 1;
2345 	powernv_enabled = (mfmsr() & PSL_HV) ? 1 : 0;
2346 	mmu_radix_early_bootstrap(start, end);
2347 	if (bootverbose)
2348 		printf("early bootstrap complete\n");
2349 	if (powernv_enabled) {
2350 		lpcr = mfspr(SPR_LPCR);
2351 		mtspr(SPR_LPCR, lpcr | LPCR_UPRT | LPCR_HR);
2352 		mmu_radix_parttab_init();
2353 		mmu_radix_init_amor();
2354 		if (bootverbose)
2355 			printf("powernv init complete\n");
2356 	}
2357 	mmu_radix_init_iamr();
2358 	mmu_radix_proctab_init();
2359 	mmu_radix_pid_set(kernel_pmap);
2360 	if (powernv_enabled)
2361 		mmu_radix_tlbiel_flush(TLB_INVAL_SCOPE_GLOBAL);
2362 	else
2363 		mmu_radix_tlbiel_flush(TLB_INVAL_SCOPE_LPID);
2364 
2365 	mmu_radix_late_bootstrap(start, end);
2366 	numa_mem_regions(&numa_pregions, &numa_pregions_sz);
2367 	if (bootverbose)
2368 		printf("%s done\n", __func__);
2369 	pmap_bootstrapped = 1;
2370 	dmaplimit = roundup2(powerpc_ptob(Maxmem), L2_PAGE_SIZE);
2371 	PCPU_SET(flags, PCPU_GET(flags) | PC_FLAG_NOSRS);
2372 }
2373 
2374 static void
2375 mmu_radix_cpu_bootstrap(int ap)
2376 {
2377 	uint64_t lpcr;
2378 	uint64_t ptcr;
2379 
2380 	if (powernv_enabled) {
2381 		lpcr = mfspr(SPR_LPCR);
2382 		mtspr(SPR_LPCR, lpcr | LPCR_UPRT | LPCR_HR);
2383 
2384 		ptcr = parttab_phys | (PARTTAB_SIZE_SHIFT-12);
2385 		mtspr(SPR_PTCR, ptcr);
2386 		mmu_radix_init_amor();
2387 	}
2388 	mmu_radix_init_iamr();
2389 	mmu_radix_pid_set(kernel_pmap);
2390 	if (powernv_enabled)
2391 		mmu_radix_tlbiel_flush(TLB_INVAL_SCOPE_GLOBAL);
2392 	else
2393 		mmu_radix_tlbiel_flush(TLB_INVAL_SCOPE_LPID);
2394 }
2395 
2396 static SYSCTL_NODE(_vm_pmap, OID_AUTO, l3e, CTLFLAG_RD, 0,
2397     "2MB page mapping counters");
2398 
2399 static COUNTER_U64_DEFINE_EARLY(pmap_l3e_demotions);
2400 SYSCTL_COUNTER_U64(_vm_pmap_l3e, OID_AUTO, demotions, CTLFLAG_RD,
2401     &pmap_l3e_demotions, "2MB page demotions");
2402 
2403 static COUNTER_U64_DEFINE_EARLY(pmap_l3e_mappings);
2404 SYSCTL_COUNTER_U64(_vm_pmap_l3e, OID_AUTO, mappings, CTLFLAG_RD,
2405     &pmap_l3e_mappings, "2MB page mappings");
2406 
2407 static COUNTER_U64_DEFINE_EARLY(pmap_l3e_p_failures);
2408 SYSCTL_COUNTER_U64(_vm_pmap_l3e, OID_AUTO, p_failures, CTLFLAG_RD,
2409     &pmap_l3e_p_failures, "2MB page promotion failures");
2410 
2411 static COUNTER_U64_DEFINE_EARLY(pmap_l3e_promotions);
2412 SYSCTL_COUNTER_U64(_vm_pmap_l3e, OID_AUTO, promotions, CTLFLAG_RD,
2413     &pmap_l3e_promotions, "2MB page promotions");
2414 
2415 static SYSCTL_NODE(_vm_pmap, OID_AUTO, l2e, CTLFLAG_RD, 0,
2416     "1GB page mapping counters");
2417 
2418 static COUNTER_U64_DEFINE_EARLY(pmap_l2e_demotions);
2419 SYSCTL_COUNTER_U64(_vm_pmap_l2e, OID_AUTO, demotions, CTLFLAG_RD,
2420     &pmap_l2e_demotions, "1GB page demotions");
2421 
2422 void
2423 mmu_radix_clear_modify(vm_page_t m)
2424 {
2425 	struct md_page *pvh;
2426 	pmap_t pmap;
2427 	pv_entry_t next_pv, pv;
2428 	pml3_entry_t oldl3e, *l3e;
2429 	pt_entry_t oldpte, *pte;
2430 	struct rwlock *lock;
2431 	vm_offset_t va;
2432 	int md_gen, pvh_gen;
2433 
2434 	KASSERT((m->oflags & VPO_UNMANAGED) == 0,
2435 	    ("pmap_clear_modify: page %p is not managed", m));
2436 	vm_page_assert_busied(m);
2437 	CTR2(KTR_PMAP, "%s(%p)", __func__, m);
2438 
2439 	/*
2440 	 * If the page is not PGA_WRITEABLE, then no PTEs can have PG_M set.
2441 	 * If the object containing the page is locked and the page is not
2442 	 * exclusive busied, then PGA_WRITEABLE cannot be concurrently set.
2443 	 */
2444 	if ((m->a.flags & PGA_WRITEABLE) == 0)
2445 		return;
2446 	pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy :
2447 	    pa_to_pvh(VM_PAGE_TO_PHYS(m));
2448 	lock = VM_PAGE_TO_PV_LIST_LOCK(m);
2449 	rw_wlock(lock);
2450 restart:
2451 	TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_link, next_pv) {
2452 		pmap = PV_PMAP(pv);
2453 		if (!PMAP_TRYLOCK(pmap)) {
2454 			pvh_gen = pvh->pv_gen;
2455 			rw_wunlock(lock);
2456 			PMAP_LOCK(pmap);
2457 			rw_wlock(lock);
2458 			if (pvh_gen != pvh->pv_gen) {
2459 				PMAP_UNLOCK(pmap);
2460 				goto restart;
2461 			}
2462 		}
2463 		va = pv->pv_va;
2464 		l3e = pmap_pml3e(pmap, va);
2465 		oldl3e = be64toh(*l3e);
2466 		if ((oldl3e & PG_RW) != 0 &&
2467 		    pmap_demote_l3e_locked(pmap, l3e, va, &lock) &&
2468 		    (oldl3e & PG_W) == 0) {
2469 			/*
2470 			 * Write protect the mapping to a
2471 			 * single page so that a subsequent
2472 			 * write access may repromote.
2473 			 */
2474 			va += VM_PAGE_TO_PHYS(m) - (oldl3e &
2475 			    PG_PS_FRAME);
2476 			pte = pmap_l3e_to_pte(l3e, va);
2477 			oldpte = be64toh(*pte);
2478 			while (!atomic_cmpset_long(pte,
2479 			    htobe64(oldpte),
2480 				htobe64((oldpte | RPTE_EAA_R) & ~(PG_M | PG_RW))))
2481 				   oldpte = be64toh(*pte);
2482 			vm_page_dirty(m);
2483 			pmap_invalidate_page(pmap, va);
2484 		}
2485 		PMAP_UNLOCK(pmap);
2486 	}
2487 	TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) {
2488 		pmap = PV_PMAP(pv);
2489 		if (!PMAP_TRYLOCK(pmap)) {
2490 			md_gen = m->md.pv_gen;
2491 			pvh_gen = pvh->pv_gen;
2492 			rw_wunlock(lock);
2493 			PMAP_LOCK(pmap);
2494 			rw_wlock(lock);
2495 			if (pvh_gen != pvh->pv_gen || md_gen != m->md.pv_gen) {
2496 				PMAP_UNLOCK(pmap);
2497 				goto restart;
2498 			}
2499 		}
2500 		l3e = pmap_pml3e(pmap, pv->pv_va);
2501 		KASSERT((be64toh(*l3e) & RPTE_LEAF) == 0, ("pmap_clear_modify: found"
2502 		    " a 2mpage in page %p's pv list", m));
2503 		pte = pmap_l3e_to_pte(l3e, pv->pv_va);
2504 		if ((be64toh(*pte) & (PG_M | PG_RW)) == (PG_M | PG_RW)) {
2505 			atomic_clear_long(pte, htobe64(PG_M));
2506 			pmap_invalidate_page(pmap, pv->pv_va);
2507 		}
2508 		PMAP_UNLOCK(pmap);
2509 	}
2510 	rw_wunlock(lock);
2511 }
2512 
2513 void
2514 mmu_radix_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr,
2515     vm_size_t len, vm_offset_t src_addr)
2516 {
2517 	struct rwlock *lock;
2518 	struct spglist free;
2519 	vm_offset_t addr;
2520 	vm_offset_t end_addr = src_addr + len;
2521 	vm_offset_t va_next;
2522 	vm_page_t dst_pdpg, dstmpte, srcmpte;
2523 	bool invalidate_all;
2524 
2525 	CTR6(KTR_PMAP,
2526 	    "%s(dst_pmap=%p, src_pmap=%p, dst_addr=%lx, len=%lu, src_addr=%lx)\n",
2527 	    __func__, dst_pmap, src_pmap, dst_addr, len, src_addr);
2528 
2529 	if (dst_addr != src_addr)
2530 		return;
2531 	lock = NULL;
2532 	invalidate_all = false;
2533 	if (dst_pmap < src_pmap) {
2534 		PMAP_LOCK(dst_pmap);
2535 		PMAP_LOCK(src_pmap);
2536 	} else {
2537 		PMAP_LOCK(src_pmap);
2538 		PMAP_LOCK(dst_pmap);
2539 	}
2540 
2541 	for (addr = src_addr; addr < end_addr; addr = va_next) {
2542 		pml1_entry_t *l1e;
2543 		pml2_entry_t *l2e;
2544 		pml3_entry_t srcptepaddr, *l3e;
2545 		pt_entry_t *src_pte, *dst_pte;
2546 
2547 		l1e = pmap_pml1e(src_pmap, addr);
2548 		if ((be64toh(*l1e) & PG_V) == 0) {
2549 			va_next = (addr + L1_PAGE_SIZE) & ~L1_PAGE_MASK;
2550 			if (va_next < addr)
2551 				va_next = end_addr;
2552 			continue;
2553 		}
2554 
2555 		l2e = pmap_l1e_to_l2e(l1e, addr);
2556 		if ((be64toh(*l2e) & PG_V) == 0) {
2557 			va_next = (addr + L2_PAGE_SIZE) & ~L2_PAGE_MASK;
2558 			if (va_next < addr)
2559 				va_next = end_addr;
2560 			continue;
2561 		}
2562 
2563 		va_next = (addr + L3_PAGE_SIZE) & ~L3_PAGE_MASK;
2564 		if (va_next < addr)
2565 			va_next = end_addr;
2566 
2567 		l3e = pmap_l2e_to_l3e(l2e, addr);
2568 		srcptepaddr = be64toh(*l3e);
2569 		if (srcptepaddr == 0)
2570 			continue;
2571 
2572 		if (srcptepaddr & RPTE_LEAF) {
2573 			if ((addr & L3_PAGE_MASK) != 0 ||
2574 			    addr + L3_PAGE_SIZE > end_addr)
2575 				continue;
2576 			dst_pdpg = pmap_allocl3e(dst_pmap, addr, NULL);
2577 			if (dst_pdpg == NULL)
2578 				break;
2579 			l3e = (pml3_entry_t *)
2580 			    PHYS_TO_DMAP(VM_PAGE_TO_PHYS(dst_pdpg));
2581 			l3e = &l3e[pmap_pml3e_index(addr)];
2582 			if (be64toh(*l3e) == 0 && ((srcptepaddr & PG_MANAGED) == 0 ||
2583 			    pmap_pv_insert_l3e(dst_pmap, addr, srcptepaddr,
2584 			    PMAP_ENTER_NORECLAIM, &lock))) {
2585 				*l3e = htobe64(srcptepaddr & ~PG_W);
2586 				pmap_resident_count_inc(dst_pmap,
2587 				    L3_PAGE_SIZE / PAGE_SIZE);
2588 				counter_u64_add(pmap_l3e_mappings, 1);
2589 			} else
2590 				dst_pdpg->ref_count--;
2591 			continue;
2592 		}
2593 
2594 		srcptepaddr &= PG_FRAME;
2595 		srcmpte = PHYS_TO_VM_PAGE(srcptepaddr);
2596 		KASSERT(srcmpte->ref_count > 0,
2597 		    ("pmap_copy: source page table page is unused"));
2598 
2599 		if (va_next > end_addr)
2600 			va_next = end_addr;
2601 
2602 		src_pte = (pt_entry_t *)PHYS_TO_DMAP(srcptepaddr);
2603 		src_pte = &src_pte[pmap_pte_index(addr)];
2604 		dstmpte = NULL;
2605 		while (addr < va_next) {
2606 			pt_entry_t ptetemp;
2607 			ptetemp = be64toh(*src_pte);
2608 			/*
2609 			 * we only virtual copy managed pages
2610 			 */
2611 			if ((ptetemp & PG_MANAGED) != 0) {
2612 				if (dstmpte != NULL &&
2613 				    dstmpte->pindex == pmap_l3e_pindex(addr))
2614 					dstmpte->ref_count++;
2615 				else if ((dstmpte = pmap_allocpte(dst_pmap,
2616 				    addr, NULL)) == NULL)
2617 					goto out;
2618 				dst_pte = (pt_entry_t *)
2619 				    PHYS_TO_DMAP(VM_PAGE_TO_PHYS(dstmpte));
2620 				dst_pte = &dst_pte[pmap_pte_index(addr)];
2621 				if (be64toh(*dst_pte) == 0 &&
2622 				    pmap_try_insert_pv_entry(dst_pmap, addr,
2623 				    PHYS_TO_VM_PAGE(ptetemp & PG_FRAME),
2624 				    &lock)) {
2625 					/*
2626 					 * Clear the wired, modified, and
2627 					 * accessed (referenced) bits
2628 					 * during the copy.
2629 					 */
2630 					*dst_pte = htobe64(ptetemp & ~(PG_W | PG_M |
2631 					    PG_A));
2632 					pmap_resident_count_inc(dst_pmap, 1);
2633 				} else {
2634 					SLIST_INIT(&free);
2635 					if (pmap_unwire_ptp(dst_pmap, addr,
2636 					    dstmpte, &free)) {
2637 						/*
2638 						 * Although "addr" is not
2639 						 * mapped, paging-structure
2640 						 * caches could nonetheless
2641 						 * have entries that refer to
2642 						 * the freed page table pages.
2643 						 * Invalidate those entries.
2644 						 */
2645 						invalidate_all = true;
2646 						vm_page_free_pages_toq(&free,
2647 						    true);
2648 					}
2649 					goto out;
2650 				}
2651 				if (dstmpte->ref_count >= srcmpte->ref_count)
2652 					break;
2653 			}
2654 			addr += PAGE_SIZE;
2655 			if (__predict_false((addr & L3_PAGE_MASK) == 0))
2656 				src_pte = pmap_pte(src_pmap, addr);
2657 			else
2658 				src_pte++;
2659 		}
2660 	}
2661 out:
2662 	if (invalidate_all)
2663 		pmap_invalidate_all(dst_pmap);
2664 	if (lock != NULL)
2665 		rw_wunlock(lock);
2666 	PMAP_UNLOCK(src_pmap);
2667 	PMAP_UNLOCK(dst_pmap);
2668 }
2669 
2670 static void
2671 mmu_radix_copy_page(vm_page_t msrc, vm_page_t mdst)
2672 {
2673 	vm_offset_t src = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(msrc));
2674 	vm_offset_t dst = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mdst));
2675 
2676 	CTR3(KTR_PMAP, "%s(%p, %p)", __func__, src, dst);
2677 	/*
2678 	 * XXX slow
2679 	 */
2680 	bcopy((void *)src, (void *)dst, PAGE_SIZE);
2681 }
2682 
2683 static void
2684 mmu_radix_copy_pages(vm_page_t ma[], vm_offset_t a_offset, vm_page_t mb[],
2685     vm_offset_t b_offset, int xfersize)
2686 {
2687         void *a_cp, *b_cp;
2688         vm_offset_t a_pg_offset, b_pg_offset;
2689         int cnt;
2690 
2691 	CTR6(KTR_PMAP, "%s(%p, %#x, %p, %#x, %#x)", __func__, ma,
2692 	    a_offset, mb, b_offset, xfersize);
2693 
2694         while (xfersize > 0) {
2695                 a_pg_offset = a_offset & PAGE_MASK;
2696                 cnt = min(xfersize, PAGE_SIZE - a_pg_offset);
2697                 a_cp = (char *)(uintptr_t)PHYS_TO_DMAP(
2698                     VM_PAGE_TO_PHYS(ma[a_offset >> PAGE_SHIFT])) +
2699                     a_pg_offset;
2700                 b_pg_offset = b_offset & PAGE_MASK;
2701                 cnt = min(cnt, PAGE_SIZE - b_pg_offset);
2702                 b_cp = (char *)(uintptr_t)PHYS_TO_DMAP(
2703                     VM_PAGE_TO_PHYS(mb[b_offset >> PAGE_SHIFT])) +
2704                     b_pg_offset;
2705                 bcopy(a_cp, b_cp, cnt);
2706                 a_offset += cnt;
2707                 b_offset += cnt;
2708                 xfersize -= cnt;
2709         }
2710 }
2711 
2712 #if VM_NRESERVLEVEL > 0
2713 /*
2714  * Tries to promote the 512, contiguous 4KB page mappings that are within a
2715  * single page table page (PTP) to a single 2MB page mapping.  For promotion
2716  * to occur, two conditions must be met: (1) the 4KB page mappings must map
2717  * aligned, contiguous physical memory and (2) the 4KB page mappings must have
2718  * identical characteristics.
2719  */
2720 static int
2721 pmap_promote_l3e(pmap_t pmap, pml3_entry_t *pde, vm_offset_t va,
2722     struct rwlock **lockp)
2723 {
2724 	pml3_entry_t newpde;
2725 	pt_entry_t *firstpte, oldpte, pa, *pte;
2726 	vm_page_t mpte;
2727 
2728 	PMAP_LOCK_ASSERT(pmap, MA_OWNED);
2729 
2730 	/*
2731 	 * Examine the first PTE in the specified PTP.  Abort if this PTE is
2732 	 * either invalid, unused, or does not map the first 4KB physical page
2733 	 * within a 2MB page.
2734 	 */
2735 	firstpte = (pt_entry_t *)PHYS_TO_DMAP(be64toh(*pde) & PG_FRAME);
2736 setpde:
2737 	newpde = be64toh(*firstpte);
2738 	if ((newpde & ((PG_FRAME & L3_PAGE_MASK) | PG_A | PG_V)) != (PG_A | PG_V)) {
2739 		CTR2(KTR_PMAP, "pmap_promote_l3e: failure for va %#lx"
2740 		    " in pmap %p", va, pmap);
2741 		goto fail;
2742 	}
2743 	if ((newpde & (PG_M | PG_RW)) == PG_RW) {
2744 		/*
2745 		 * When PG_M is already clear, PG_RW can be cleared without
2746 		 * a TLB invalidation.
2747 		 */
2748 		if (!atomic_cmpset_long(firstpte, htobe64(newpde), htobe64((newpde | RPTE_EAA_R) & ~RPTE_EAA_W)))
2749 			goto setpde;
2750 		newpde &= ~RPTE_EAA_W;
2751 	}
2752 
2753 	/*
2754 	 * Examine each of the other PTEs in the specified PTP.  Abort if this
2755 	 * PTE maps an unexpected 4KB physical page or does not have identical
2756 	 * characteristics to the first PTE.
2757 	 */
2758 	pa = (newpde & (PG_PS_FRAME | PG_A | PG_V)) + L3_PAGE_SIZE - PAGE_SIZE;
2759 	for (pte = firstpte + NPTEPG - 1; pte > firstpte; pte--) {
2760 setpte:
2761 		oldpte = be64toh(*pte);
2762 		if ((oldpte & (PG_FRAME | PG_A | PG_V)) != pa) {
2763 			CTR2(KTR_PMAP, "pmap_promote_l3e: failure for va %#lx"
2764 			    " in pmap %p", va, pmap);
2765 			goto fail;
2766 		}
2767 		if ((oldpte & (PG_M | PG_RW)) == PG_RW) {
2768 			/*
2769 			 * When PG_M is already clear, PG_RW can be cleared
2770 			 * without a TLB invalidation.
2771 			 */
2772 			if (!atomic_cmpset_long(pte, htobe64(oldpte), htobe64((oldpte | RPTE_EAA_R) & ~RPTE_EAA_W)))
2773 				goto setpte;
2774 			oldpte &= ~RPTE_EAA_W;
2775 			CTR2(KTR_PMAP, "pmap_promote_l3e: protect for va %#lx"
2776 			    " in pmap %p", (oldpte & PG_FRAME & L3_PAGE_MASK) |
2777 			    (va & ~L3_PAGE_MASK), pmap);
2778 		}
2779 		if ((oldpte & PG_PTE_PROMOTE) != (newpde & PG_PTE_PROMOTE)) {
2780 			CTR2(KTR_PMAP, "pmap_promote_l3e: failure for va %#lx"
2781 			    " in pmap %p", va, pmap);
2782 			goto fail;
2783 		}
2784 		pa -= PAGE_SIZE;
2785 	}
2786 
2787 	/*
2788 	 * Save the page table page in its current state until the PDE
2789 	 * mapping the superpage is demoted by pmap_demote_pde() or
2790 	 * destroyed by pmap_remove_pde().
2791 	 */
2792 	mpte = PHYS_TO_VM_PAGE(be64toh(*pde) & PG_FRAME);
2793 	KASSERT(mpte >= vm_page_array &&
2794 	    mpte < &vm_page_array[vm_page_array_size],
2795 	    ("pmap_promote_l3e: page table page is out of range"));
2796 	KASSERT(mpte->pindex == pmap_l3e_pindex(va),
2797 	    ("pmap_promote_l3e: page table page's pindex is wrong"));
2798 	if (pmap_insert_pt_page(pmap, mpte)) {
2799 		CTR2(KTR_PMAP,
2800 		    "pmap_promote_l3e: failure for va %#lx in pmap %p", va,
2801 		    pmap);
2802 		goto fail;
2803 	}
2804 
2805 	/*
2806 	 * Promote the pv entries.
2807 	 */
2808 	if ((newpde & PG_MANAGED) != 0)
2809 		pmap_pv_promote_l3e(pmap, va, newpde & PG_PS_FRAME, lockp);
2810 
2811 	pte_store(pde, PG_PROMOTED | newpde);
2812 	ptesync();
2813 	counter_u64_add(pmap_l3e_promotions, 1);
2814 	CTR2(KTR_PMAP, "pmap_promote_l3e: success for va %#lx"
2815 	    " in pmap %p", va, pmap);
2816 	return (0);
2817  fail:
2818 	counter_u64_add(pmap_l3e_p_failures, 1);
2819 	return (KERN_FAILURE);
2820 }
2821 #endif /* VM_NRESERVLEVEL > 0 */
2822 
2823 int
2824 mmu_radix_enter(pmap_t pmap, vm_offset_t va, vm_page_t m,
2825     vm_prot_t prot, u_int flags, int8_t psind)
2826 {
2827 	struct rwlock *lock;
2828 	pml3_entry_t *l3e;
2829 	pt_entry_t *pte;
2830 	pt_entry_t newpte, origpte;
2831 	pv_entry_t pv;
2832 	vm_paddr_t opa, pa;
2833 	vm_page_t mpte, om;
2834 	int rv, retrycount;
2835 	boolean_t nosleep, invalidate_all, invalidate_page;
2836 
2837 	va = trunc_page(va);
2838 	retrycount = 0;
2839 	invalidate_page = invalidate_all = false;
2840 	CTR6(KTR_PMAP, "pmap_enter(%p, %#lx, %p, %#x, %#x, %d)", pmap, va,
2841 	    m, prot, flags, psind);
2842 	KASSERT(va <= VM_MAX_KERNEL_ADDRESS, ("pmap_enter: toobig"));
2843 	KASSERT((m->oflags & VPO_UNMANAGED) != 0 || !VA_IS_CLEANMAP(va),
2844 	    ("pmap_enter: managed mapping within the clean submap"));
2845 	if ((m->oflags & VPO_UNMANAGED) == 0)
2846 		VM_PAGE_OBJECT_BUSY_ASSERT(m);
2847 
2848 	KASSERT((flags & PMAP_ENTER_RESERVED) == 0,
2849 	    ("pmap_enter: flags %u has reserved bits set", flags));
2850 	pa = VM_PAGE_TO_PHYS(m);
2851 	newpte = (pt_entry_t)(pa | PG_A | PG_V | RPTE_LEAF);
2852 	if ((flags & VM_PROT_WRITE) != 0)
2853 		newpte |= PG_M;
2854 	if ((flags & VM_PROT_READ) != 0)
2855 		newpte |= PG_A;
2856 	if (prot & VM_PROT_READ)
2857 		newpte |= RPTE_EAA_R;
2858 	if ((prot & VM_PROT_WRITE) != 0)
2859 		newpte |= RPTE_EAA_W;
2860 	KASSERT((newpte & (PG_M | PG_RW)) != PG_M,
2861 	    ("pmap_enter: flags includes VM_PROT_WRITE but prot doesn't"));
2862 
2863 	if (prot & VM_PROT_EXECUTE)
2864 		newpte |= PG_X;
2865 	if ((flags & PMAP_ENTER_WIRED) != 0)
2866 		newpte |= PG_W;
2867 	if (va >= DMAP_MIN_ADDRESS)
2868 		newpte |= RPTE_EAA_P;
2869 	newpte |= pmap_cache_bits(m->md.mdpg_cache_attrs);
2870 	/*
2871 	 * Set modified bit gratuitously for writeable mappings if
2872 	 * the page is unmanaged. We do not want to take a fault
2873 	 * to do the dirty bit accounting for these mappings.
2874 	 */
2875 	if ((m->oflags & VPO_UNMANAGED) != 0) {
2876 		if ((newpte & PG_RW) != 0)
2877 			newpte |= PG_M;
2878 	} else
2879 		newpte |= PG_MANAGED;
2880 
2881 	lock = NULL;
2882 	PMAP_LOCK(pmap);
2883 	if (psind == 1) {
2884 		/* Assert the required virtual and physical alignment. */
2885 		KASSERT((va & L3_PAGE_MASK) == 0, ("pmap_enter: va unaligned"));
2886 		KASSERT(m->psind > 0, ("pmap_enter: m->psind < psind"));
2887 		rv = pmap_enter_l3e(pmap, va, newpte | RPTE_LEAF, flags, m, &lock);
2888 		goto out;
2889 	}
2890 	mpte = NULL;
2891 
2892 	/*
2893 	 * In the case that a page table page is not
2894 	 * resident, we are creating it here.
2895 	 */
2896 retry:
2897 	l3e = pmap_pml3e(pmap, va);
2898 	if (l3e != NULL && (be64toh(*l3e) & PG_V) != 0 && ((be64toh(*l3e) & RPTE_LEAF) == 0 ||
2899 	    pmap_demote_l3e_locked(pmap, l3e, va, &lock))) {
2900 		pte = pmap_l3e_to_pte(l3e, va);
2901 		if (va < VM_MAXUSER_ADDRESS && mpte == NULL) {
2902 			mpte = PHYS_TO_VM_PAGE(be64toh(*l3e) & PG_FRAME);
2903 			mpte->ref_count++;
2904 		}
2905 	} else if (va < VM_MAXUSER_ADDRESS) {
2906 		/*
2907 		 * Here if the pte page isn't mapped, or if it has been
2908 		 * deallocated.
2909 		 */
2910 		nosleep = (flags & PMAP_ENTER_NOSLEEP) != 0;
2911 		mpte = _pmap_allocpte(pmap, pmap_l3e_pindex(va),
2912 		    nosleep ? NULL : &lock);
2913 		if (mpte == NULL && nosleep) {
2914 			rv = KERN_RESOURCE_SHORTAGE;
2915 			goto out;
2916 		}
2917 		if (__predict_false(retrycount++ == 6))
2918 			panic("too many retries");
2919 		invalidate_all = true;
2920 		goto retry;
2921 	} else
2922 		panic("pmap_enter: invalid page directory va=%#lx", va);
2923 
2924 	origpte = be64toh(*pte);
2925 	pv = NULL;
2926 
2927 	/*
2928 	 * Is the specified virtual address already mapped?
2929 	 */
2930 	if ((origpte & PG_V) != 0) {
2931 #ifdef INVARIANTS
2932 		if (VERBOSE_PMAP || pmap_logging) {
2933 			printf("cow fault pmap_enter(%p, %#lx, %p, %#x, %x, %d) --"
2934 			    " asid=%lu curpid=%d name=%s origpte0x%lx\n",
2935 			    pmap, va, m, prot, flags, psind, pmap->pm_pid,
2936 			    curproc->p_pid, curproc->p_comm, origpte);
2937 			pmap_pte_walk(pmap->pm_pml1, va);
2938 		}
2939 #endif
2940 		/*
2941 		 * Wiring change, just update stats. We don't worry about
2942 		 * wiring PT pages as they remain resident as long as there
2943 		 * are valid mappings in them. Hence, if a user page is wired,
2944 		 * the PT page will be also.
2945 		 */
2946 		if ((newpte & PG_W) != 0 && (origpte & PG_W) == 0)
2947 			pmap->pm_stats.wired_count++;
2948 		else if ((newpte & PG_W) == 0 && (origpte & PG_W) != 0)
2949 			pmap->pm_stats.wired_count--;
2950 
2951 		/*
2952 		 * Remove the extra PT page reference.
2953 		 */
2954 		if (mpte != NULL) {
2955 			mpte->ref_count--;
2956 			KASSERT(mpte->ref_count > 0,
2957 			    ("pmap_enter: missing reference to page table page,"
2958 			     " va: 0x%lx", va));
2959 		}
2960 
2961 		/*
2962 		 * Has the physical page changed?
2963 		 */
2964 		opa = origpte & PG_FRAME;
2965 		if (opa == pa) {
2966 			/*
2967 			 * No, might be a protection or wiring change.
2968 			 */
2969 			if ((origpte & PG_MANAGED) != 0 &&
2970 			    (newpte & PG_RW) != 0)
2971 				vm_page_aflag_set(m, PGA_WRITEABLE);
2972 			if (((origpte ^ newpte) & ~(PG_M | PG_A)) == 0) {
2973 				if ((newpte & (PG_A|PG_M)) != (origpte & (PG_A|PG_M))) {
2974 					if (!atomic_cmpset_long(pte, htobe64(origpte), htobe64(newpte)))
2975 						goto retry;
2976 					if ((newpte & PG_M) != (origpte & PG_M))
2977 						vm_page_dirty(m);
2978 					if ((newpte & PG_A) != (origpte & PG_A))
2979 						vm_page_aflag_set(m, PGA_REFERENCED);
2980 					ptesync();
2981 				} else
2982 					invalidate_all = true;
2983 				if (((origpte ^ newpte) & ~(PG_M | PG_A)) == 0)
2984 					goto unchanged;
2985 			}
2986 			goto validate;
2987 		}
2988 
2989 		/*
2990 		 * The physical page has changed.  Temporarily invalidate
2991 		 * the mapping.  This ensures that all threads sharing the
2992 		 * pmap keep a consistent view of the mapping, which is
2993 		 * necessary for the correct handling of COW faults.  It
2994 		 * also permits reuse of the old mapping's PV entry,
2995 		 * avoiding an allocation.
2996 		 *
2997 		 * For consistency, handle unmanaged mappings the same way.
2998 		 */
2999 		origpte = be64toh(pte_load_clear(pte));
3000 		KASSERT((origpte & PG_FRAME) == opa,
3001 		    ("pmap_enter: unexpected pa update for %#lx", va));
3002 		if ((origpte & PG_MANAGED) != 0) {
3003 			om = PHYS_TO_VM_PAGE(opa);
3004 
3005 			/*
3006 			 * The pmap lock is sufficient to synchronize with
3007 			 * concurrent calls to pmap_page_test_mappings() and
3008 			 * pmap_ts_referenced().
3009 			 */
3010 			if ((origpte & (PG_M | PG_RW)) == (PG_M | PG_RW))
3011 				vm_page_dirty(om);
3012 			if ((origpte & PG_A) != 0)
3013 				vm_page_aflag_set(om, PGA_REFERENCED);
3014 			CHANGE_PV_LIST_LOCK_TO_PHYS(&lock, opa);
3015 			pv = pmap_pvh_remove(&om->md, pmap, va);
3016 			if ((newpte & PG_MANAGED) == 0)
3017 				free_pv_entry(pmap, pv);
3018 #ifdef INVARIANTS
3019 			else if (origpte & PG_MANAGED) {
3020 				if (pv == NULL) {
3021 					pmap_page_print_mappings(om);
3022 					MPASS(pv != NULL);
3023 				}
3024 			}
3025 #endif
3026 			if ((om->a.flags & PGA_WRITEABLE) != 0 &&
3027 			    TAILQ_EMPTY(&om->md.pv_list) &&
3028 			    ((om->flags & PG_FICTITIOUS) != 0 ||
3029 			    TAILQ_EMPTY(&pa_to_pvh(opa)->pv_list)))
3030 				vm_page_aflag_clear(om, PGA_WRITEABLE);
3031 		}
3032 		if ((origpte & PG_A) != 0)
3033 			invalidate_page = true;
3034 		origpte = 0;
3035 	} else {
3036 		if (pmap != kernel_pmap) {
3037 #ifdef INVARIANTS
3038 			if (VERBOSE_PMAP || pmap_logging)
3039 				printf("pmap_enter(%p, %#lx, %p, %#x, %x, %d) -- asid=%lu curpid=%d name=%s\n",
3040 				    pmap, va, m, prot, flags, psind,
3041 				    pmap->pm_pid, curproc->p_pid,
3042 				    curproc->p_comm);
3043 #endif
3044 		}
3045 
3046 		/*
3047 		 * Increment the counters.
3048 		 */
3049 		if ((newpte & PG_W) != 0)
3050 			pmap->pm_stats.wired_count++;
3051 		pmap_resident_count_inc(pmap, 1);
3052 	}
3053 
3054 	/*
3055 	 * Enter on the PV list if part of our managed memory.
3056 	 */
3057 	if ((newpte & PG_MANAGED) != 0) {
3058 		if (pv == NULL) {
3059 			pv = get_pv_entry(pmap, &lock);
3060 			pv->pv_va = va;
3061 		}
3062 #ifdef VERBOSE_PV
3063 		else
3064 			printf("reassigning pv: %p to pmap: %p\n",
3065 				   pv, pmap);
3066 #endif
3067 		CHANGE_PV_LIST_LOCK_TO_PHYS(&lock, pa);
3068 		TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_link);
3069 		m->md.pv_gen++;
3070 		if ((newpte & PG_RW) != 0)
3071 			vm_page_aflag_set(m, PGA_WRITEABLE);
3072 	}
3073 
3074 	/*
3075 	 * Update the PTE.
3076 	 */
3077 	if ((origpte & PG_V) != 0) {
3078 validate:
3079 		origpte = be64toh(pte_load_store(pte, htobe64(newpte)));
3080 		KASSERT((origpte & PG_FRAME) == pa,
3081 		    ("pmap_enter: unexpected pa update for %#lx", va));
3082 		if ((newpte & PG_M) == 0 && (origpte & (PG_M | PG_RW)) ==
3083 		    (PG_M | PG_RW)) {
3084 			if ((origpte & PG_MANAGED) != 0)
3085 				vm_page_dirty(m);
3086 			invalidate_page = true;
3087 
3088 			/*
3089 			 * Although the PTE may still have PG_RW set, TLB
3090 			 * invalidation may nonetheless be required because
3091 			 * the PTE no longer has PG_M set.
3092 			 */
3093 		} else if ((origpte & PG_X) != 0 || (newpte & PG_X) == 0) {
3094 			/*
3095 			 * Removing capabilities requires invalidation on POWER
3096 			 */
3097 			invalidate_page = true;
3098 			goto unchanged;
3099 		}
3100 		if ((origpte & PG_A) != 0)
3101 			invalidate_page = true;
3102 	} else {
3103 		pte_store(pte, newpte);
3104 		ptesync();
3105 	}
3106 unchanged:
3107 
3108 #if VM_NRESERVLEVEL > 0
3109 	/*
3110 	 * If both the page table page and the reservation are fully
3111 	 * populated, then attempt promotion.
3112 	 */
3113 	if ((mpte == NULL || mpte->ref_count == NPTEPG) &&
3114 	    mmu_radix_ps_enabled(pmap) &&
3115 	    (m->flags & PG_FICTITIOUS) == 0 &&
3116 	    vm_reserv_level_iffullpop(m) == 0 &&
3117 		pmap_promote_l3e(pmap, l3e, va, &lock) == 0)
3118 		invalidate_all = true;
3119 #endif
3120 	if (invalidate_all)
3121 		pmap_invalidate_all(pmap);
3122 	else if (invalidate_page)
3123 		pmap_invalidate_page(pmap, va);
3124 
3125 	rv = KERN_SUCCESS;
3126 out:
3127 	if (lock != NULL)
3128 		rw_wunlock(lock);
3129 	PMAP_UNLOCK(pmap);
3130 
3131 	return (rv);
3132 }
3133 
3134 /*
3135  * Tries to create a read- and/or execute-only 2MB page mapping.  Returns true
3136  * if successful.  Returns false if (1) a page table page cannot be allocated
3137  * without sleeping, (2) a mapping already exists at the specified virtual
3138  * address, or (3) a PV entry cannot be allocated without reclaiming another
3139  * PV entry.
3140  */
3141 static bool
3142 pmap_enter_2mpage(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,
3143     struct rwlock **lockp)
3144 {
3145 	pml3_entry_t newpde;
3146 
3147 	PMAP_LOCK_ASSERT(pmap, MA_OWNED);
3148 	newpde = VM_PAGE_TO_PHYS(m) | pmap_cache_bits(m->md.mdpg_cache_attrs) |
3149 	    RPTE_LEAF | PG_V;
3150 	if ((m->oflags & VPO_UNMANAGED) == 0)
3151 		newpde |= PG_MANAGED;
3152 	if (prot & VM_PROT_EXECUTE)
3153 		newpde |= PG_X;
3154 	if (prot & VM_PROT_READ)
3155 		newpde |= RPTE_EAA_R;
3156 	if (va >= DMAP_MIN_ADDRESS)
3157 		newpde |= RPTE_EAA_P;
3158 	return (pmap_enter_l3e(pmap, va, newpde, PMAP_ENTER_NOSLEEP |
3159 	    PMAP_ENTER_NOREPLACE | PMAP_ENTER_NORECLAIM, NULL, lockp) ==
3160 	    KERN_SUCCESS);
3161 }
3162 
3163 /*
3164  * Tries to create the specified 2MB page mapping.  Returns KERN_SUCCESS if
3165  * the mapping was created, and either KERN_FAILURE or KERN_RESOURCE_SHORTAGE
3166  * otherwise.  Returns KERN_FAILURE if PMAP_ENTER_NOREPLACE was specified and
3167  * a mapping already exists at the specified virtual address.  Returns
3168  * KERN_RESOURCE_SHORTAGE if PMAP_ENTER_NOSLEEP was specified and a page table
3169  * page allocation failed.  Returns KERN_RESOURCE_SHORTAGE if
3170  * PMAP_ENTER_NORECLAIM was specified and a PV entry allocation failed.
3171  *
3172  * The parameter "m" is only used when creating a managed, writeable mapping.
3173  */
3174 static int
3175 pmap_enter_l3e(pmap_t pmap, vm_offset_t va, pml3_entry_t newpde, u_int flags,
3176     vm_page_t m, struct rwlock **lockp)
3177 {
3178 	struct spglist free;
3179 	pml3_entry_t oldl3e, *l3e;
3180 	vm_page_t mt, pdpg;
3181 
3182 	KASSERT((newpde & (PG_M | PG_RW)) != PG_RW,
3183 	    ("pmap_enter_pde: newpde is missing PG_M"));
3184 	PMAP_LOCK_ASSERT(pmap, MA_OWNED);
3185 
3186 	if ((pdpg = pmap_allocl3e(pmap, va, (flags & PMAP_ENTER_NOSLEEP) != 0 ?
3187 	    NULL : lockp)) == NULL) {
3188 		CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx"
3189 		    " in pmap %p", va, pmap);
3190 		return (KERN_RESOURCE_SHORTAGE);
3191 	}
3192 	l3e = (pml3_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pdpg));
3193 	l3e = &l3e[pmap_pml3e_index(va)];
3194 	oldl3e = be64toh(*l3e);
3195 	if ((oldl3e & PG_V) != 0) {
3196 		KASSERT(pdpg->ref_count > 1,
3197 		    ("pmap_enter_pde: pdpg's wire count is too low"));
3198 		if ((flags & PMAP_ENTER_NOREPLACE) != 0) {
3199 			pdpg->ref_count--;
3200 			CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx"
3201 			    " in pmap %p", va, pmap);
3202 			return (KERN_FAILURE);
3203 		}
3204 		/* Break the existing mapping(s). */
3205 		SLIST_INIT(&free);
3206 		if ((oldl3e & RPTE_LEAF) != 0) {
3207 			/*
3208 			 * The reference to the PD page that was acquired by
3209 			 * pmap_allocl3e() ensures that it won't be freed.
3210 			 * However, if the PDE resulted from a promotion, then
3211 			 * a reserved PT page could be freed.
3212 			 */
3213 			(void)pmap_remove_l3e(pmap, l3e, va, &free, lockp);
3214 			pmap_invalidate_l3e_page(pmap, va, oldl3e);
3215 		} else {
3216 			if (pmap_remove_ptes(pmap, va, va + L3_PAGE_SIZE, l3e,
3217 			    &free, lockp))
3218 		               pmap_invalidate_all(pmap);
3219 		}
3220 		vm_page_free_pages_toq(&free, true);
3221 		if (va >= VM_MAXUSER_ADDRESS) {
3222 			mt = PHYS_TO_VM_PAGE(be64toh(*l3e) & PG_FRAME);
3223 			if (pmap_insert_pt_page(pmap, mt)) {
3224 				/*
3225 				 * XXX Currently, this can't happen because
3226 				 * we do not perform pmap_enter(psind == 1)
3227 				 * on the kernel pmap.
3228 				 */
3229 				panic("pmap_enter_pde: trie insert failed");
3230 			}
3231 		} else
3232 			KASSERT(be64toh(*l3e) == 0, ("pmap_enter_pde: non-zero pde %p",
3233 			    l3e));
3234 	}
3235 	if ((newpde & PG_MANAGED) != 0) {
3236 		/*
3237 		 * Abort this mapping if its PV entry could not be created.
3238 		 */
3239 		if (!pmap_pv_insert_l3e(pmap, va, newpde, flags, lockp)) {
3240 			SLIST_INIT(&free);
3241 			if (pmap_unwire_ptp(pmap, va, pdpg, &free)) {
3242 				/*
3243 				 * Although "va" is not mapped, paging-
3244 				 * structure caches could nonetheless have
3245 				 * entries that refer to the freed page table
3246 				 * pages.  Invalidate those entries.
3247 				 */
3248 				pmap_invalidate_page(pmap, va);
3249 				vm_page_free_pages_toq(&free, true);
3250 			}
3251 			CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx"
3252 			    " in pmap %p", va, pmap);
3253 			return (KERN_RESOURCE_SHORTAGE);
3254 		}
3255 		if ((newpde & PG_RW) != 0) {
3256 			for (mt = m; mt < &m[L3_PAGE_SIZE / PAGE_SIZE]; mt++)
3257 				vm_page_aflag_set(mt, PGA_WRITEABLE);
3258 		}
3259 	}
3260 
3261 	/*
3262 	 * Increment counters.
3263 	 */
3264 	if ((newpde & PG_W) != 0)
3265 		pmap->pm_stats.wired_count += L3_PAGE_SIZE / PAGE_SIZE;
3266 	pmap_resident_count_inc(pmap, L3_PAGE_SIZE / PAGE_SIZE);
3267 
3268 	/*
3269 	 * Map the superpage.  (This is not a promoted mapping; there will not
3270 	 * be any lingering 4KB page mappings in the TLB.)
3271 	 */
3272 	pte_store(l3e, newpde);
3273 	ptesync();
3274 
3275 	counter_u64_add(pmap_l3e_mappings, 1);
3276 	CTR2(KTR_PMAP, "pmap_enter_pde: success for va %#lx"
3277 	    " in pmap %p", va, pmap);
3278 	return (KERN_SUCCESS);
3279 }
3280 
3281 void
3282 mmu_radix_enter_object(pmap_t pmap, vm_offset_t start,
3283     vm_offset_t end, vm_page_t m_start, vm_prot_t prot)
3284 {
3285 
3286 	struct rwlock *lock;
3287 	vm_offset_t va;
3288 	vm_page_t m, mpte;
3289 	vm_pindex_t diff, psize;
3290 	bool invalidate;
3291 	VM_OBJECT_ASSERT_LOCKED(m_start->object);
3292 
3293 	CTR6(KTR_PMAP, "%s(%p, %#x, %#x, %p, %#x)", __func__, pmap, start,
3294 	    end, m_start, prot);
3295 
3296 	invalidate = false;
3297 	psize = atop(end - start);
3298 	mpte = NULL;
3299 	m = m_start;
3300 	lock = NULL;
3301 	PMAP_LOCK(pmap);
3302 	while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) {
3303 		va = start + ptoa(diff);
3304 		if ((va & L3_PAGE_MASK) == 0 && va + L3_PAGE_SIZE <= end &&
3305 		    m->psind == 1 && mmu_radix_ps_enabled(pmap) &&
3306 		    pmap_enter_2mpage(pmap, va, m, prot, &lock))
3307 			m = &m[L3_PAGE_SIZE / PAGE_SIZE - 1];
3308 		else
3309 			mpte = mmu_radix_enter_quick_locked(pmap, va, m, prot,
3310 			    mpte, &lock, &invalidate);
3311 		m = TAILQ_NEXT(m, listq);
3312 	}
3313 	ptesync();
3314 	if (lock != NULL)
3315 		rw_wunlock(lock);
3316 	if (invalidate)
3317 		pmap_invalidate_all(pmap);
3318 	PMAP_UNLOCK(pmap);
3319 }
3320 
3321 static vm_page_t
3322 mmu_radix_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m,
3323     vm_prot_t prot, vm_page_t mpte, struct rwlock **lockp, bool *invalidate)
3324 {
3325 	struct spglist free;
3326 	pt_entry_t *pte;
3327 	vm_paddr_t pa;
3328 
3329 	KASSERT(!VA_IS_CLEANMAP(va) ||
3330 	    (m->oflags & VPO_UNMANAGED) != 0,
3331 	    ("mmu_radix_enter_quick_locked: managed mapping within the clean submap"));
3332 	PMAP_LOCK_ASSERT(pmap, MA_OWNED);
3333 
3334 	/*
3335 	 * In the case that a page table page is not
3336 	 * resident, we are creating it here.
3337 	 */
3338 	if (va < VM_MAXUSER_ADDRESS) {
3339 		vm_pindex_t ptepindex;
3340 		pml3_entry_t *ptepa;
3341 
3342 		/*
3343 		 * Calculate pagetable page index
3344 		 */
3345 		ptepindex = pmap_l3e_pindex(va);
3346 		if (mpte && (mpte->pindex == ptepindex)) {
3347 			mpte->ref_count++;
3348 		} else {
3349 			/*
3350 			 * Get the page directory entry
3351 			 */
3352 			ptepa = pmap_pml3e(pmap, va);
3353 
3354 			/*
3355 			 * If the page table page is mapped, we just increment
3356 			 * the hold count, and activate it.  Otherwise, we
3357 			 * attempt to allocate a page table page.  If this
3358 			 * attempt fails, we don't retry.  Instead, we give up.
3359 			 */
3360 			if (ptepa && (be64toh(*ptepa) & PG_V) != 0) {
3361 				if (be64toh(*ptepa) & RPTE_LEAF)
3362 					return (NULL);
3363 				mpte = PHYS_TO_VM_PAGE(be64toh(*ptepa) & PG_FRAME);
3364 				mpte->ref_count++;
3365 			} else {
3366 				/*
3367 				 * Pass NULL instead of the PV list lock
3368 				 * pointer, because we don't intend to sleep.
3369 				 */
3370 				mpte = _pmap_allocpte(pmap, ptepindex, NULL);
3371 				if (mpte == NULL)
3372 					return (mpte);
3373 			}
3374 		}
3375 		pte = (pt_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mpte));
3376 		pte = &pte[pmap_pte_index(va)];
3377 	} else {
3378 		mpte = NULL;
3379 		pte = pmap_pte(pmap, va);
3380 	}
3381 	if (be64toh(*pte)) {
3382 		if (mpte != NULL) {
3383 			mpte->ref_count--;
3384 			mpte = NULL;
3385 		}
3386 		return (mpte);
3387 	}
3388 
3389 	/*
3390 	 * Enter on the PV list if part of our managed memory.
3391 	 */
3392 	if ((m->oflags & VPO_UNMANAGED) == 0 &&
3393 	    !pmap_try_insert_pv_entry(pmap, va, m, lockp)) {
3394 		if (mpte != NULL) {
3395 			SLIST_INIT(&free);
3396 			if (pmap_unwire_ptp(pmap, va, mpte, &free)) {
3397 				/*
3398 				 * Although "va" is not mapped, paging-
3399 				 * structure caches could nonetheless have
3400 				 * entries that refer to the freed page table
3401 				 * pages.  Invalidate those entries.
3402 				 */
3403 				*invalidate = true;
3404 				vm_page_free_pages_toq(&free, true);
3405 			}
3406 			mpte = NULL;
3407 		}
3408 		return (mpte);
3409 	}
3410 
3411 	/*
3412 	 * Increment counters
3413 	 */
3414 	pmap_resident_count_inc(pmap, 1);
3415 
3416 	pa = VM_PAGE_TO_PHYS(m) | pmap_cache_bits(m->md.mdpg_cache_attrs);
3417 	if (prot & VM_PROT_EXECUTE)
3418 		pa |= PG_X;
3419 	else
3420 		pa |= RPTE_EAA_R;
3421 	if ((m->oflags & VPO_UNMANAGED) == 0)
3422 		pa |= PG_MANAGED;
3423 
3424 	pte_store(pte, pa);
3425 	return (mpte);
3426 }
3427 
3428 void
3429 mmu_radix_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m,
3430     vm_prot_t prot)
3431 {
3432 	struct rwlock *lock;
3433 	bool invalidate;
3434 
3435 	lock = NULL;
3436 	invalidate = false;
3437 	PMAP_LOCK(pmap);
3438 	mmu_radix_enter_quick_locked(pmap, va, m, prot, NULL, &lock,
3439 	    &invalidate);
3440 	ptesync();
3441 	if (lock != NULL)
3442 		rw_wunlock(lock);
3443 	if (invalidate)
3444 		pmap_invalidate_all(pmap);
3445 	PMAP_UNLOCK(pmap);
3446 }
3447 
3448 vm_paddr_t
3449 mmu_radix_extract(pmap_t pmap, vm_offset_t va)
3450 {
3451 	pml3_entry_t *l3e;
3452 	pt_entry_t *pte;
3453 	vm_paddr_t pa;
3454 
3455 	l3e = pmap_pml3e(pmap, va);
3456 	if (__predict_false(l3e == NULL))
3457 		return (0);
3458 	if (be64toh(*l3e) & RPTE_LEAF) {
3459 		pa = (be64toh(*l3e) & PG_PS_FRAME) | (va & L3_PAGE_MASK);
3460 		pa |= (va & L3_PAGE_MASK);
3461 	} else {
3462 		/*
3463 		 * Beware of a concurrent promotion that changes the
3464 		 * PDE at this point!  For example, vtopte() must not
3465 		 * be used to access the PTE because it would use the
3466 		 * new PDE.  It is, however, safe to use the old PDE
3467 		 * because the page table page is preserved by the
3468 		 * promotion.
3469 		 */
3470 		pte = pmap_l3e_to_pte(l3e, va);
3471 		if (__predict_false(pte == NULL))
3472 			return (0);
3473 		pa = be64toh(*pte);
3474 		pa = (pa & PG_FRAME) | (va & PAGE_MASK);
3475 		pa |= (va & PAGE_MASK);
3476 	}
3477 	return (pa);
3478 }
3479 
3480 vm_page_t
3481 mmu_radix_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot)
3482 {
3483 	pml3_entry_t l3e, *l3ep;
3484 	pt_entry_t pte;
3485 	vm_page_t m;
3486 
3487 	m = NULL;
3488 	CTR4(KTR_PMAP, "%s(%p, %#x, %#x)", __func__, pmap, va, prot);
3489 	PMAP_LOCK(pmap);
3490 	l3ep = pmap_pml3e(pmap, va);
3491 	if (l3ep != NULL && (l3e = be64toh(*l3ep))) {
3492 		if (l3e & RPTE_LEAF) {
3493 			if ((l3e & PG_RW) || (prot & VM_PROT_WRITE) == 0)
3494 				m = PHYS_TO_VM_PAGE((l3e & PG_PS_FRAME) |
3495 				    (va & L3_PAGE_MASK));
3496 		} else {
3497 			/* Native endian PTE, do not pass to pmap functions */
3498 			pte = be64toh(*pmap_l3e_to_pte(l3ep, va));
3499 			if ((pte & PG_V) &&
3500 			    ((pte & PG_RW) || (prot & VM_PROT_WRITE) == 0))
3501 				m = PHYS_TO_VM_PAGE(pte & PG_FRAME);
3502 		}
3503 		if (m != NULL && !vm_page_wire_mapped(m))
3504 			m = NULL;
3505 	}
3506 	PMAP_UNLOCK(pmap);
3507 	return (m);
3508 }
3509 
3510 static void
3511 mmu_radix_growkernel(vm_offset_t addr)
3512 {
3513 	vm_paddr_t paddr;
3514 	vm_page_t nkpg;
3515 	pml3_entry_t *l3e;
3516 	pml2_entry_t *l2e;
3517 
3518 	CTR2(KTR_PMAP, "%s(%#x)", __func__, addr);
3519 	if (VM_MIN_KERNEL_ADDRESS < addr &&
3520 		addr < (VM_MIN_KERNEL_ADDRESS + nkpt * L3_PAGE_SIZE))
3521 		return;
3522 
3523 	addr = roundup2(addr, L3_PAGE_SIZE);
3524 	if (addr - 1 >= vm_map_max(kernel_map))
3525 		addr = vm_map_max(kernel_map);
3526 	while (kernel_vm_end < addr) {
3527 		l2e = pmap_pml2e(kernel_pmap, kernel_vm_end);
3528 		if ((be64toh(*l2e) & PG_V) == 0) {
3529 			/* We need a new PDP entry */
3530 			nkpg = vm_page_alloc_noobj(VM_ALLOC_INTERRUPT |
3531 			    VM_ALLOC_WIRED | VM_ALLOC_ZERO);
3532 			if (nkpg == NULL)
3533 				panic("pmap_growkernel: no memory to grow kernel");
3534 			nkpg->pindex = kernel_vm_end >> L2_PAGE_SIZE_SHIFT;
3535 			paddr = VM_PAGE_TO_PHYS(nkpg);
3536 			pde_store(l2e, paddr);
3537 			continue; /* try again */
3538 		}
3539 		l3e = pmap_l2e_to_l3e(l2e, kernel_vm_end);
3540 		if ((be64toh(*l3e) & PG_V) != 0) {
3541 			kernel_vm_end = (kernel_vm_end + L3_PAGE_SIZE) & ~L3_PAGE_MASK;
3542 			if (kernel_vm_end - 1 >= vm_map_max(kernel_map)) {
3543 				kernel_vm_end = vm_map_max(kernel_map);
3544 				break;
3545 			}
3546 			continue;
3547 		}
3548 
3549 		nkpg = vm_page_alloc_noobj(VM_ALLOC_INTERRUPT | VM_ALLOC_WIRED |
3550 		    VM_ALLOC_ZERO);
3551 		if (nkpg == NULL)
3552 			panic("pmap_growkernel: no memory to grow kernel");
3553 		nkpg->pindex = pmap_l3e_pindex(kernel_vm_end);
3554 		paddr = VM_PAGE_TO_PHYS(nkpg);
3555 		pde_store(l3e, paddr);
3556 
3557 		kernel_vm_end = (kernel_vm_end + L3_PAGE_SIZE) & ~L3_PAGE_MASK;
3558 		if (kernel_vm_end - 1 >= vm_map_max(kernel_map)) {
3559 			kernel_vm_end = vm_map_max(kernel_map);
3560 			break;
3561 		}
3562 	}
3563 	ptesync();
3564 }
3565 
3566 static MALLOC_DEFINE(M_RADIX_PGD, "radix_pgd", "radix page table root directory");
3567 static uma_zone_t zone_radix_pgd;
3568 
3569 static int
3570 radix_pgd_import(void *arg __unused, void **store, int count, int domain __unused,
3571     int flags)
3572 {
3573 	int req;
3574 
3575 	req = VM_ALLOC_WIRED | malloc2vm_flags(flags);
3576 	for (int i = 0; i < count; i++) {
3577 		vm_page_t m = vm_page_alloc_noobj_contig(req,
3578 		    RADIX_PGD_SIZE / PAGE_SIZE,
3579 		    0, (vm_paddr_t)-1, RADIX_PGD_SIZE, L1_PAGE_SIZE,
3580 		    VM_MEMATTR_DEFAULT);
3581 		store[i] = (void *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m));
3582 	}
3583 	return (count);
3584 }
3585 
3586 static void
3587 radix_pgd_release(void *arg __unused, void **store, int count)
3588 {
3589 	vm_page_t m;
3590 	struct spglist free;
3591 	int page_count;
3592 
3593 	SLIST_INIT(&free);
3594 	page_count = RADIX_PGD_SIZE/PAGE_SIZE;
3595 
3596 	for (int i = 0; i < count; i++) {
3597 		/*
3598 		 * XXX selectively remove dmap and KVA entries so we don't
3599 		 * need to bzero
3600 		 */
3601 		m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)store[i]));
3602 		for (int j = page_count-1; j >= 0; j--) {
3603 			vm_page_unwire_noq(&m[j]);
3604 			SLIST_INSERT_HEAD(&free, &m[j], plinks.s.ss);
3605 		}
3606 		vm_page_free_pages_toq(&free, false);
3607 	}
3608 }
3609 
3610 static void
3611 mmu_radix_init()
3612 {
3613 	vm_page_t mpte;
3614 	vm_size_t s;
3615 	int error, i, pv_npg;
3616 
3617 	/* XXX is this really needed for POWER? */
3618 	/* L1TF, reserve page @0 unconditionally */
3619 	vm_page_blacklist_add(0, bootverbose);
3620 
3621 	zone_radix_pgd = uma_zcache_create("radix_pgd_cache",
3622 		RADIX_PGD_SIZE, NULL, NULL,
3623 #ifdef INVARIANTS
3624 	    trash_init, trash_fini,
3625 #else
3626 	    NULL, NULL,
3627 #endif
3628 		radix_pgd_import, radix_pgd_release,
3629 		NULL, UMA_ZONE_NOBUCKET);
3630 
3631 	/*
3632 	 * Initialize the vm page array entries for the kernel pmap's
3633 	 * page table pages.
3634 	 */
3635 	PMAP_LOCK(kernel_pmap);
3636 	for (i = 0; i < nkpt; i++) {
3637 		mpte = PHYS_TO_VM_PAGE(KPTphys + (i << PAGE_SHIFT));
3638 		KASSERT(mpte >= vm_page_array &&
3639 		    mpte < &vm_page_array[vm_page_array_size],
3640 		    ("pmap_init: page table page is out of range size: %lu",
3641 		     vm_page_array_size));
3642 		mpte->pindex = pmap_l3e_pindex(VM_MIN_KERNEL_ADDRESS) + i;
3643 		mpte->phys_addr = KPTphys + (i << PAGE_SHIFT);
3644 		MPASS(PHYS_TO_VM_PAGE(mpte->phys_addr) == mpte);
3645 		//pmap_insert_pt_page(kernel_pmap, mpte);
3646 		mpte->ref_count = 1;
3647 	}
3648 	PMAP_UNLOCK(kernel_pmap);
3649 	vm_wire_add(nkpt);
3650 
3651 	CTR1(KTR_PMAP, "%s()", __func__);
3652 	TAILQ_INIT(&pv_dummy.pv_list);
3653 
3654 	/*
3655 	 * Are large page mappings enabled?
3656 	 */
3657 	TUNABLE_INT_FETCH("vm.pmap.superpages_enabled", &superpages_enabled);
3658 	if (superpages_enabled) {
3659 		KASSERT(MAXPAGESIZES > 1 && pagesizes[1] == 0,
3660 		    ("pmap_init: can't assign to pagesizes[1]"));
3661 		pagesizes[1] = L3_PAGE_SIZE;
3662 	}
3663 
3664 	/*
3665 	 * Initialize the pv chunk list mutex.
3666 	 */
3667 	mtx_init(&pv_chunks_mutex, "pmap pv chunk list", NULL, MTX_DEF);
3668 
3669 	/*
3670 	 * Initialize the pool of pv list locks.
3671 	 */
3672 	for (i = 0; i < NPV_LIST_LOCKS; i++)
3673 		rw_init(&pv_list_locks[i], "pmap pv list");
3674 
3675 	/*
3676 	 * Calculate the size of the pv head table for superpages.
3677 	 */
3678 	pv_npg = howmany(vm_phys_segs[vm_phys_nsegs - 1].end, L3_PAGE_SIZE);
3679 
3680 	/*
3681 	 * Allocate memory for the pv head table for superpages.
3682 	 */
3683 	s = (vm_size_t)(pv_npg * sizeof(struct md_page));
3684 	s = round_page(s);
3685 	pv_table = (struct md_page *)kmem_malloc(s, M_WAITOK | M_ZERO);
3686 	for (i = 0; i < pv_npg; i++)
3687 		TAILQ_INIT(&pv_table[i].pv_list);
3688 	TAILQ_INIT(&pv_dummy.pv_list);
3689 
3690 	pmap_initialized = 1;
3691 	mtx_init(&qframe_mtx, "qfrmlk", NULL, MTX_SPIN);
3692 	error = vmem_alloc(kernel_arena, PAGE_SIZE, M_BESTFIT | M_WAITOK,
3693 	    (vmem_addr_t *)&qframe);
3694 
3695 	if (error != 0)
3696 		panic("qframe allocation failed");
3697 	asid_arena = vmem_create("ASID", isa3_base_pid + 1, (1<<isa3_pid_bits),
3698 	    1, 1, M_WAITOK);
3699 }
3700 
3701 static boolean_t
3702 pmap_page_test_mappings(vm_page_t m, boolean_t accessed, boolean_t modified)
3703 {
3704 	struct rwlock *lock;
3705 	pv_entry_t pv;
3706 	struct md_page *pvh;
3707 	pt_entry_t *pte, mask;
3708 	pmap_t pmap;
3709 	int md_gen, pvh_gen;
3710 	boolean_t rv;
3711 
3712 	rv = FALSE;
3713 	lock = VM_PAGE_TO_PV_LIST_LOCK(m);
3714 	rw_rlock(lock);
3715 restart:
3716 	TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) {
3717 		pmap = PV_PMAP(pv);
3718 		if (!PMAP_TRYLOCK(pmap)) {
3719 			md_gen = m->md.pv_gen;
3720 			rw_runlock(lock);
3721 			PMAP_LOCK(pmap);
3722 			rw_rlock(lock);
3723 			if (md_gen != m->md.pv_gen) {
3724 				PMAP_UNLOCK(pmap);
3725 				goto restart;
3726 			}
3727 		}
3728 		pte = pmap_pte(pmap, pv->pv_va);
3729 		mask = 0;
3730 		if (modified)
3731 			mask |= PG_RW | PG_M;
3732 		if (accessed)
3733 			mask |= PG_V | PG_A;
3734 		rv = (be64toh(*pte) & mask) == mask;
3735 		PMAP_UNLOCK(pmap);
3736 		if (rv)
3737 			goto out;
3738 	}
3739 	if ((m->flags & PG_FICTITIOUS) == 0) {
3740 		pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
3741 		TAILQ_FOREACH(pv, &pvh->pv_list, pv_link) {
3742 			pmap = PV_PMAP(pv);
3743 			if (!PMAP_TRYLOCK(pmap)) {
3744 				md_gen = m->md.pv_gen;
3745 				pvh_gen = pvh->pv_gen;
3746 				rw_runlock(lock);
3747 				PMAP_LOCK(pmap);
3748 				rw_rlock(lock);
3749 				if (md_gen != m->md.pv_gen ||
3750 				    pvh_gen != pvh->pv_gen) {
3751 					PMAP_UNLOCK(pmap);
3752 					goto restart;
3753 				}
3754 			}
3755 			pte = pmap_pml3e(pmap, pv->pv_va);
3756 			mask = 0;
3757 			if (modified)
3758 				mask |= PG_RW | PG_M;
3759 			if (accessed)
3760 				mask |= PG_V | PG_A;
3761 			rv = (be64toh(*pte) & mask) == mask;
3762 			PMAP_UNLOCK(pmap);
3763 			if (rv)
3764 				goto out;
3765 		}
3766 	}
3767 out:
3768 	rw_runlock(lock);
3769 	return (rv);
3770 }
3771 
3772 /*
3773  *	pmap_is_modified:
3774  *
3775  *	Return whether or not the specified physical page was modified
3776  *	in any physical maps.
3777  */
3778 boolean_t
3779 mmu_radix_is_modified(vm_page_t m)
3780 {
3781 
3782 	KASSERT((m->oflags & VPO_UNMANAGED) == 0,
3783 	    ("pmap_is_modified: page %p is not managed", m));
3784 
3785 	CTR2(KTR_PMAP, "%s(%p)", __func__, m);
3786 	/*
3787 	 * If the page is not busied then this check is racy.
3788 	 */
3789 	if (!pmap_page_is_write_mapped(m))
3790 		return (FALSE);
3791 	return (pmap_page_test_mappings(m, FALSE, TRUE));
3792 }
3793 
3794 boolean_t
3795 mmu_radix_is_prefaultable(pmap_t pmap, vm_offset_t addr)
3796 {
3797 	pml3_entry_t *l3e;
3798 	pt_entry_t *pte;
3799 	boolean_t rv;
3800 
3801 	CTR3(KTR_PMAP, "%s(%p, %#x)", __func__, pmap, addr);
3802 	rv = FALSE;
3803 	PMAP_LOCK(pmap);
3804 	l3e = pmap_pml3e(pmap, addr);
3805 	if (l3e != NULL && (be64toh(*l3e) & (RPTE_LEAF | PG_V)) == PG_V) {
3806 		pte = pmap_l3e_to_pte(l3e, addr);
3807 		rv = (be64toh(*pte) & PG_V) == 0;
3808 	}
3809 	PMAP_UNLOCK(pmap);
3810 	return (rv);
3811 }
3812 
3813 boolean_t
3814 mmu_radix_is_referenced(vm_page_t m)
3815 {
3816 	KASSERT((m->oflags & VPO_UNMANAGED) == 0,
3817 	    ("pmap_is_referenced: page %p is not managed", m));
3818 	CTR2(KTR_PMAP, "%s(%p)", __func__, m);
3819 	return (pmap_page_test_mappings(m, TRUE, FALSE));
3820 }
3821 
3822 /*
3823  *	pmap_ts_referenced:
3824  *
3825  *	Return a count of reference bits for a page, clearing those bits.
3826  *	It is not necessary for every reference bit to be cleared, but it
3827  *	is necessary that 0 only be returned when there are truly no
3828  *	reference bits set.
3829  *
3830  *	As an optimization, update the page's dirty field if a modified bit is
3831  *	found while counting reference bits.  This opportunistic update can be
3832  *	performed at low cost and can eliminate the need for some future calls
3833  *	to pmap_is_modified().  However, since this function stops after
3834  *	finding PMAP_TS_REFERENCED_MAX reference bits, it may not detect some
3835  *	dirty pages.  Those dirty pages will only be detected by a future call
3836  *	to pmap_is_modified().
3837  *
3838  *	A DI block is not needed within this function, because
3839  *	invalidations are performed before the PV list lock is
3840  *	released.
3841  */
3842 boolean_t
3843 mmu_radix_ts_referenced(vm_page_t m)
3844 {
3845 	struct md_page *pvh;
3846 	pv_entry_t pv, pvf;
3847 	pmap_t pmap;
3848 	struct rwlock *lock;
3849 	pml3_entry_t oldl3e, *l3e;
3850 	pt_entry_t *pte;
3851 	vm_paddr_t pa;
3852 	int cleared, md_gen, not_cleared, pvh_gen;
3853 	struct spglist free;
3854 
3855 	CTR2(KTR_PMAP, "%s(%p)", __func__, m);
3856 	KASSERT((m->oflags & VPO_UNMANAGED) == 0,
3857 	    ("pmap_ts_referenced: page %p is not managed", m));
3858 	SLIST_INIT(&free);
3859 	cleared = 0;
3860 	pa = VM_PAGE_TO_PHYS(m);
3861 	lock = PHYS_TO_PV_LIST_LOCK(pa);
3862 	pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy : pa_to_pvh(pa);
3863 	rw_wlock(lock);
3864 retry:
3865 	not_cleared = 0;
3866 	if ((pvf = TAILQ_FIRST(&pvh->pv_list)) == NULL)
3867 		goto small_mappings;
3868 	pv = pvf;
3869 	do {
3870 		if (pvf == NULL)
3871 			pvf = pv;
3872 		pmap = PV_PMAP(pv);
3873 		if (!PMAP_TRYLOCK(pmap)) {
3874 			pvh_gen = pvh->pv_gen;
3875 			rw_wunlock(lock);
3876 			PMAP_LOCK(pmap);
3877 			rw_wlock(lock);
3878 			if (pvh_gen != pvh->pv_gen) {
3879 				PMAP_UNLOCK(pmap);
3880 				goto retry;
3881 			}
3882 		}
3883 		l3e = pmap_pml3e(pmap, pv->pv_va);
3884 		oldl3e = be64toh(*l3e);
3885 		if ((oldl3e & (PG_M | PG_RW)) == (PG_M | PG_RW)) {
3886 			/*
3887 			 * Although "oldpde" is mapping a 2MB page, because
3888 			 * this function is called at a 4KB page granularity,
3889 			 * we only update the 4KB page under test.
3890 			 */
3891 			vm_page_dirty(m);
3892 		}
3893 		if ((oldl3e & PG_A) != 0) {
3894 			/*
3895 			 * Since this reference bit is shared by 512 4KB
3896 			 * pages, it should not be cleared every time it is
3897 			 * tested.  Apply a simple "hash" function on the
3898 			 * physical page number, the virtual superpage number,
3899 			 * and the pmap address to select one 4KB page out of
3900 			 * the 512 on which testing the reference bit will
3901 			 * result in clearing that reference bit.  This
3902 			 * function is designed to avoid the selection of the
3903 			 * same 4KB page for every 2MB page mapping.
3904 			 *
3905 			 * On demotion, a mapping that hasn't been referenced
3906 			 * is simply destroyed.  To avoid the possibility of a
3907 			 * subsequent page fault on a demoted wired mapping,
3908 			 * always leave its reference bit set.  Moreover,
3909 			 * since the superpage is wired, the current state of
3910 			 * its reference bit won't affect page replacement.
3911 			 */
3912 			if ((((pa >> PAGE_SHIFT) ^ (pv->pv_va >> L3_PAGE_SIZE_SHIFT) ^
3913 			    (uintptr_t)pmap) & (NPTEPG - 1)) == 0 &&
3914 			    (oldl3e & PG_W) == 0) {
3915 				atomic_clear_long(l3e, htobe64(PG_A));
3916 				pmap_invalidate_page(pmap, pv->pv_va);
3917 				cleared++;
3918 				KASSERT(lock == VM_PAGE_TO_PV_LIST_LOCK(m),
3919 				    ("inconsistent pv lock %p %p for page %p",
3920 				    lock, VM_PAGE_TO_PV_LIST_LOCK(m), m));
3921 			} else
3922 				not_cleared++;
3923 		}
3924 		PMAP_UNLOCK(pmap);
3925 		/* Rotate the PV list if it has more than one entry. */
3926 		if (pv != NULL && TAILQ_NEXT(pv, pv_link) != NULL) {
3927 			TAILQ_REMOVE(&pvh->pv_list, pv, pv_link);
3928 			TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_link);
3929 			pvh->pv_gen++;
3930 		}
3931 		if (cleared + not_cleared >= PMAP_TS_REFERENCED_MAX)
3932 			goto out;
3933 	} while ((pv = TAILQ_FIRST(&pvh->pv_list)) != pvf);
3934 small_mappings:
3935 	if ((pvf = TAILQ_FIRST(&m->md.pv_list)) == NULL)
3936 		goto out;
3937 	pv = pvf;
3938 	do {
3939 		if (pvf == NULL)
3940 			pvf = pv;
3941 		pmap = PV_PMAP(pv);
3942 		if (!PMAP_TRYLOCK(pmap)) {
3943 			pvh_gen = pvh->pv_gen;
3944 			md_gen = m->md.pv_gen;
3945 			rw_wunlock(lock);
3946 			PMAP_LOCK(pmap);
3947 			rw_wlock(lock);
3948 			if (pvh_gen != pvh->pv_gen || md_gen != m->md.pv_gen) {
3949 				PMAP_UNLOCK(pmap);
3950 				goto retry;
3951 			}
3952 		}
3953 		l3e = pmap_pml3e(pmap, pv->pv_va);
3954 		KASSERT((be64toh(*l3e) & RPTE_LEAF) == 0,
3955 		    ("pmap_ts_referenced: found a 2mpage in page %p's pv list",
3956 		    m));
3957 		pte = pmap_l3e_to_pte(l3e, pv->pv_va);
3958 		if ((be64toh(*pte) & (PG_M | PG_RW)) == (PG_M | PG_RW))
3959 			vm_page_dirty(m);
3960 		if ((be64toh(*pte) & PG_A) != 0) {
3961 			atomic_clear_long(pte, htobe64(PG_A));
3962 			pmap_invalidate_page(pmap, pv->pv_va);
3963 			cleared++;
3964 		}
3965 		PMAP_UNLOCK(pmap);
3966 		/* Rotate the PV list if it has more than one entry. */
3967 		if (pv != NULL && TAILQ_NEXT(pv, pv_link) != NULL) {
3968 			TAILQ_REMOVE(&m->md.pv_list, pv, pv_link);
3969 			TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_link);
3970 			m->md.pv_gen++;
3971 		}
3972 	} while ((pv = TAILQ_FIRST(&m->md.pv_list)) != pvf && cleared +
3973 	    not_cleared < PMAP_TS_REFERENCED_MAX);
3974 out:
3975 	rw_wunlock(lock);
3976 	vm_page_free_pages_toq(&free, true);
3977 	return (cleared + not_cleared);
3978 }
3979 
3980 static vm_offset_t
3981 mmu_radix_map(vm_offset_t *virt __unused, vm_paddr_t start,
3982     vm_paddr_t end, int prot __unused)
3983 {
3984 
3985 	CTR5(KTR_PMAP, "%s(%p, %#x, %#x, %#x)", __func__, virt, start, end,
3986 		 prot);
3987 	return (PHYS_TO_DMAP(start));
3988 }
3989 
3990 void
3991 mmu_radix_object_init_pt(pmap_t pmap, vm_offset_t addr,
3992     vm_object_t object, vm_pindex_t pindex, vm_size_t size)
3993 {
3994 	pml3_entry_t *l3e;
3995 	vm_paddr_t pa, ptepa;
3996 	vm_page_t p, pdpg;
3997 	vm_memattr_t ma;
3998 
3999 	CTR6(KTR_PMAP, "%s(%p, %#x, %p, %u, %#x)", __func__, pmap, addr,
4000 	    object, pindex, size);
4001 	VM_OBJECT_ASSERT_WLOCKED(object);
4002 	KASSERT(object->type == OBJT_DEVICE || object->type == OBJT_SG,
4003 			("pmap_object_init_pt: non-device object"));
4004 	/* NB: size can be logically ored with addr here */
4005 	if ((addr & L3_PAGE_MASK) == 0 && (size & L3_PAGE_MASK) == 0) {
4006 		if (!mmu_radix_ps_enabled(pmap))
4007 			return;
4008 		if (!vm_object_populate(object, pindex, pindex + atop(size)))
4009 			return;
4010 		p = vm_page_lookup(object, pindex);
4011 		KASSERT(p->valid == VM_PAGE_BITS_ALL,
4012 		    ("pmap_object_init_pt: invalid page %p", p));
4013 		ma = p->md.mdpg_cache_attrs;
4014 
4015 		/*
4016 		 * Abort the mapping if the first page is not physically
4017 		 * aligned to a 2MB page boundary.
4018 		 */
4019 		ptepa = VM_PAGE_TO_PHYS(p);
4020 		if (ptepa & L3_PAGE_MASK)
4021 			return;
4022 
4023 		/*
4024 		 * Skip the first page.  Abort the mapping if the rest of
4025 		 * the pages are not physically contiguous or have differing
4026 		 * memory attributes.
4027 		 */
4028 		p = TAILQ_NEXT(p, listq);
4029 		for (pa = ptepa + PAGE_SIZE; pa < ptepa + size;
4030 		    pa += PAGE_SIZE) {
4031 			KASSERT(p->valid == VM_PAGE_BITS_ALL,
4032 			    ("pmap_object_init_pt: invalid page %p", p));
4033 			if (pa != VM_PAGE_TO_PHYS(p) ||
4034 			    ma != p->md.mdpg_cache_attrs)
4035 				return;
4036 			p = TAILQ_NEXT(p, listq);
4037 		}
4038 
4039 		PMAP_LOCK(pmap);
4040 		for (pa = ptepa | pmap_cache_bits(ma);
4041 		    pa < ptepa + size; pa += L3_PAGE_SIZE) {
4042 			pdpg = pmap_allocl3e(pmap, addr, NULL);
4043 			if (pdpg == NULL) {
4044 				/*
4045 				 * The creation of mappings below is only an
4046 				 * optimization.  If a page directory page
4047 				 * cannot be allocated without blocking,
4048 				 * continue on to the next mapping rather than
4049 				 * blocking.
4050 				 */
4051 				addr += L3_PAGE_SIZE;
4052 				continue;
4053 			}
4054 			l3e = (pml3_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pdpg));
4055 			l3e = &l3e[pmap_pml3e_index(addr)];
4056 			if ((be64toh(*l3e) & PG_V) == 0) {
4057 				pa |= PG_M | PG_A | PG_RW;
4058 				pte_store(l3e, pa);
4059 				pmap_resident_count_inc(pmap, L3_PAGE_SIZE / PAGE_SIZE);
4060 				counter_u64_add(pmap_l3e_mappings, 1);
4061 			} else {
4062 				/* Continue on if the PDE is already valid. */
4063 				pdpg->ref_count--;
4064 				KASSERT(pdpg->ref_count > 0,
4065 				    ("pmap_object_init_pt: missing reference "
4066 				    "to page directory page, va: 0x%lx", addr));
4067 			}
4068 			addr += L3_PAGE_SIZE;
4069 		}
4070 		ptesync();
4071 		PMAP_UNLOCK(pmap);
4072 	}
4073 }
4074 
4075 boolean_t
4076 mmu_radix_page_exists_quick(pmap_t pmap, vm_page_t m)
4077 {
4078 	struct md_page *pvh;
4079 	struct rwlock *lock;
4080 	pv_entry_t pv;
4081 	int loops = 0;
4082 	boolean_t rv;
4083 
4084 	KASSERT((m->oflags & VPO_UNMANAGED) == 0,
4085 	    ("pmap_page_exists_quick: page %p is not managed", m));
4086 	CTR3(KTR_PMAP, "%s(%p, %p)", __func__, pmap, m);
4087 	rv = FALSE;
4088 	lock = VM_PAGE_TO_PV_LIST_LOCK(m);
4089 	rw_rlock(lock);
4090 	TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) {
4091 		if (PV_PMAP(pv) == pmap) {
4092 			rv = TRUE;
4093 			break;
4094 		}
4095 		loops++;
4096 		if (loops >= 16)
4097 			break;
4098 	}
4099 	if (!rv && loops < 16 && (m->flags & PG_FICTITIOUS) == 0) {
4100 		pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
4101 		TAILQ_FOREACH(pv, &pvh->pv_list, pv_link) {
4102 			if (PV_PMAP(pv) == pmap) {
4103 				rv = TRUE;
4104 				break;
4105 			}
4106 			loops++;
4107 			if (loops >= 16)
4108 				break;
4109 		}
4110 	}
4111 	rw_runlock(lock);
4112 	return (rv);
4113 }
4114 
4115 void
4116 mmu_radix_page_init(vm_page_t m)
4117 {
4118 
4119 	CTR2(KTR_PMAP, "%s(%p)", __func__, m);
4120 	TAILQ_INIT(&m->md.pv_list);
4121 	m->md.mdpg_cache_attrs = VM_MEMATTR_DEFAULT;
4122 }
4123 
4124 int
4125 mmu_radix_page_wired_mappings(vm_page_t m)
4126 {
4127 	struct rwlock *lock;
4128 	struct md_page *pvh;
4129 	pmap_t pmap;
4130 	pt_entry_t *pte;
4131 	pv_entry_t pv;
4132 	int count, md_gen, pvh_gen;
4133 
4134 	if ((m->oflags & VPO_UNMANAGED) != 0)
4135 		return (0);
4136 	CTR2(KTR_PMAP, "%s(%p)", __func__, m);
4137 	lock = VM_PAGE_TO_PV_LIST_LOCK(m);
4138 	rw_rlock(lock);
4139 restart:
4140 	count = 0;
4141 	TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) {
4142 		pmap = PV_PMAP(pv);
4143 		if (!PMAP_TRYLOCK(pmap)) {
4144 			md_gen = m->md.pv_gen;
4145 			rw_runlock(lock);
4146 			PMAP_LOCK(pmap);
4147 			rw_rlock(lock);
4148 			if (md_gen != m->md.pv_gen) {
4149 				PMAP_UNLOCK(pmap);
4150 				goto restart;
4151 			}
4152 		}
4153 		pte = pmap_pte(pmap, pv->pv_va);
4154 		if ((be64toh(*pte) & PG_W) != 0)
4155 			count++;
4156 		PMAP_UNLOCK(pmap);
4157 	}
4158 	if ((m->flags & PG_FICTITIOUS) == 0) {
4159 		pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
4160 		TAILQ_FOREACH(pv, &pvh->pv_list, pv_link) {
4161 			pmap = PV_PMAP(pv);
4162 			if (!PMAP_TRYLOCK(pmap)) {
4163 				md_gen = m->md.pv_gen;
4164 				pvh_gen = pvh->pv_gen;
4165 				rw_runlock(lock);
4166 				PMAP_LOCK(pmap);
4167 				rw_rlock(lock);
4168 				if (md_gen != m->md.pv_gen ||
4169 				    pvh_gen != pvh->pv_gen) {
4170 					PMAP_UNLOCK(pmap);
4171 					goto restart;
4172 				}
4173 			}
4174 			pte = pmap_pml3e(pmap, pv->pv_va);
4175 			if ((be64toh(*pte) & PG_W) != 0)
4176 				count++;
4177 			PMAP_UNLOCK(pmap);
4178 		}
4179 	}
4180 	rw_runlock(lock);
4181 	return (count);
4182 }
4183 
4184 static void
4185 mmu_radix_update_proctab(int pid, pml1_entry_t l1pa)
4186 {
4187 	isa3_proctab[pid].proctab0 = htobe64(RTS_SIZE |  l1pa | RADIX_PGD_INDEX_SHIFT);
4188 }
4189 
4190 int
4191 mmu_radix_pinit(pmap_t pmap)
4192 {
4193 	vmem_addr_t pid;
4194 	vm_paddr_t l1pa;
4195 
4196 	CTR2(KTR_PMAP, "%s(%p)", __func__, pmap);
4197 
4198 	/*
4199 	 * allocate the page directory page
4200 	 */
4201 	pmap->pm_pml1 = uma_zalloc(zone_radix_pgd, M_WAITOK);
4202 
4203 	for (int j = 0; j <  RADIX_PGD_SIZE_SHIFT; j++)
4204 		pagezero((vm_offset_t)pmap->pm_pml1 + j * PAGE_SIZE);
4205 	vm_radix_init(&pmap->pm_radix);
4206 	TAILQ_INIT(&pmap->pm_pvchunk);
4207 	bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
4208 	pmap->pm_flags = PMAP_PDE_SUPERPAGE;
4209 	vmem_alloc(asid_arena, 1, M_FIRSTFIT|M_WAITOK, &pid);
4210 
4211 	pmap->pm_pid = pid;
4212 	l1pa = DMAP_TO_PHYS((vm_offset_t)pmap->pm_pml1);
4213 	mmu_radix_update_proctab(pid, l1pa);
4214 	__asm __volatile("ptesync;isync" : : : "memory");
4215 
4216 	return (1);
4217 }
4218 
4219 /*
4220  * This routine is called if the desired page table page does not exist.
4221  *
4222  * If page table page allocation fails, this routine may sleep before
4223  * returning NULL.  It sleeps only if a lock pointer was given.
4224  *
4225  * Note: If a page allocation fails at page table level two or three,
4226  * one or two pages may be held during the wait, only to be released
4227  * afterwards.  This conservative approach is easily argued to avoid
4228  * race conditions.
4229  */
4230 static vm_page_t
4231 _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, struct rwlock **lockp)
4232 {
4233 	vm_page_t m, pdppg, pdpg;
4234 
4235 	PMAP_LOCK_ASSERT(pmap, MA_OWNED);
4236 
4237 	/*
4238 	 * Allocate a page table page.
4239 	 */
4240 	if ((m = vm_page_alloc_noobj(VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) {
4241 		if (lockp != NULL) {
4242 			RELEASE_PV_LIST_LOCK(lockp);
4243 			PMAP_UNLOCK(pmap);
4244 			vm_wait(NULL);
4245 			PMAP_LOCK(pmap);
4246 		}
4247 		/*
4248 		 * Indicate the need to retry.  While waiting, the page table
4249 		 * page may have been allocated.
4250 		 */
4251 		return (NULL);
4252 	}
4253 	m->pindex = ptepindex;
4254 
4255 	/*
4256 	 * Map the pagetable page into the process address space, if
4257 	 * it isn't already there.
4258 	 */
4259 
4260 	if (ptepindex >= (NUPDE + NUPDPE)) {
4261 		pml1_entry_t *l1e;
4262 		vm_pindex_t pml1index;
4263 
4264 		/* Wire up a new PDPE page */
4265 		pml1index = ptepindex - (NUPDE + NUPDPE);
4266 		l1e = &pmap->pm_pml1[pml1index];
4267 		KASSERT((be64toh(*l1e) & PG_V) == 0,
4268 		    ("%s: L1 entry %#lx is valid", __func__, *l1e));
4269 		pde_store(l1e, VM_PAGE_TO_PHYS(m));
4270 	} else if (ptepindex >= NUPDE) {
4271 		vm_pindex_t pml1index;
4272 		vm_pindex_t pdpindex;
4273 		pml1_entry_t *l1e;
4274 		pml2_entry_t *l2e;
4275 
4276 		/* Wire up a new l2e page */
4277 		pdpindex = ptepindex - NUPDE;
4278 		pml1index = pdpindex >> RPTE_SHIFT;
4279 
4280 		l1e = &pmap->pm_pml1[pml1index];
4281 		if ((be64toh(*l1e) & PG_V) == 0) {
4282 			/* Have to allocate a new pdp, recurse */
4283 			if (_pmap_allocpte(pmap, NUPDE + NUPDPE + pml1index,
4284 				lockp) == NULL) {
4285 				vm_page_unwire_noq(m);
4286 				vm_page_free_zero(m);
4287 				return (NULL);
4288 			}
4289 		} else {
4290 			/* Add reference to l2e page */
4291 			pdppg = PHYS_TO_VM_PAGE(be64toh(*l1e) & PG_FRAME);
4292 			pdppg->ref_count++;
4293 		}
4294 		l2e = (pml2_entry_t *)PHYS_TO_DMAP(be64toh(*l1e) & PG_FRAME);
4295 
4296 		/* Now find the pdp page */
4297 		l2e = &l2e[pdpindex & RPTE_MASK];
4298 		KASSERT((be64toh(*l2e) & PG_V) == 0,
4299 		    ("%s: L2 entry %#lx is valid", __func__, *l2e));
4300 		pde_store(l2e, VM_PAGE_TO_PHYS(m));
4301 	} else {
4302 		vm_pindex_t pml1index;
4303 		vm_pindex_t pdpindex;
4304 		pml1_entry_t *l1e;
4305 		pml2_entry_t *l2e;
4306 		pml3_entry_t *l3e;
4307 
4308 		/* Wire up a new PTE page */
4309 		pdpindex = ptepindex >> RPTE_SHIFT;
4310 		pml1index = pdpindex >> RPTE_SHIFT;
4311 
4312 		/* First, find the pdp and check that its valid. */
4313 		l1e = &pmap->pm_pml1[pml1index];
4314 		if ((be64toh(*l1e) & PG_V) == 0) {
4315 			/* Have to allocate a new pd, recurse */
4316 			if (_pmap_allocpte(pmap, NUPDE + pdpindex,
4317 			    lockp) == NULL) {
4318 				vm_page_unwire_noq(m);
4319 				vm_page_free_zero(m);
4320 				return (NULL);
4321 			}
4322 			l2e = (pml2_entry_t *)PHYS_TO_DMAP(be64toh(*l1e) & PG_FRAME);
4323 			l2e = &l2e[pdpindex & RPTE_MASK];
4324 		} else {
4325 			l2e = (pml2_entry_t *)PHYS_TO_DMAP(be64toh(*l1e) & PG_FRAME);
4326 			l2e = &l2e[pdpindex & RPTE_MASK];
4327 			if ((be64toh(*l2e) & PG_V) == 0) {
4328 				/* Have to allocate a new pd, recurse */
4329 				if (_pmap_allocpte(pmap, NUPDE + pdpindex,
4330 				    lockp) == NULL) {
4331 					vm_page_unwire_noq(m);
4332 					vm_page_free_zero(m);
4333 					return (NULL);
4334 				}
4335 			} else {
4336 				/* Add reference to the pd page */
4337 				pdpg = PHYS_TO_VM_PAGE(be64toh(*l2e) & PG_FRAME);
4338 				pdpg->ref_count++;
4339 			}
4340 		}
4341 		l3e = (pml3_entry_t *)PHYS_TO_DMAP(be64toh(*l2e) & PG_FRAME);
4342 
4343 		/* Now we know where the page directory page is */
4344 		l3e = &l3e[ptepindex & RPTE_MASK];
4345 		KASSERT((be64toh(*l3e) & PG_V) == 0,
4346 		    ("%s: L3 entry %#lx is valid", __func__, *l3e));
4347 		pde_store(l3e, VM_PAGE_TO_PHYS(m));
4348 	}
4349 
4350 	pmap_resident_count_inc(pmap, 1);
4351 	return (m);
4352 }
4353 static vm_page_t
4354 pmap_allocl3e(pmap_t pmap, vm_offset_t va, struct rwlock **lockp)
4355 {
4356 	vm_pindex_t pdpindex, ptepindex;
4357 	pml2_entry_t *pdpe;
4358 	vm_page_t pdpg;
4359 
4360 retry:
4361 	pdpe = pmap_pml2e(pmap, va);
4362 	if (pdpe != NULL && (be64toh(*pdpe) & PG_V) != 0) {
4363 		/* Add a reference to the pd page. */
4364 		pdpg = PHYS_TO_VM_PAGE(be64toh(*pdpe) & PG_FRAME);
4365 		pdpg->ref_count++;
4366 	} else {
4367 		/* Allocate a pd page. */
4368 		ptepindex = pmap_l3e_pindex(va);
4369 		pdpindex = ptepindex >> RPTE_SHIFT;
4370 		pdpg = _pmap_allocpte(pmap, NUPDE + pdpindex, lockp);
4371 		if (pdpg == NULL && lockp != NULL)
4372 			goto retry;
4373 	}
4374 	return (pdpg);
4375 }
4376 
4377 static vm_page_t
4378 pmap_allocpte(pmap_t pmap, vm_offset_t va, struct rwlock **lockp)
4379 {
4380 	vm_pindex_t ptepindex;
4381 	pml3_entry_t *pd;
4382 	vm_page_t m;
4383 
4384 	/*
4385 	 * Calculate pagetable page index
4386 	 */
4387 	ptepindex = pmap_l3e_pindex(va);
4388 retry:
4389 	/*
4390 	 * Get the page directory entry
4391 	 */
4392 	pd = pmap_pml3e(pmap, va);
4393 
4394 	/*
4395 	 * This supports switching from a 2MB page to a
4396 	 * normal 4K page.
4397 	 */
4398 	if (pd != NULL && (be64toh(*pd) & (RPTE_LEAF | PG_V)) == (RPTE_LEAF | PG_V)) {
4399 		if (!pmap_demote_l3e_locked(pmap, pd, va, lockp)) {
4400 			/*
4401 			 * Invalidation of the 2MB page mapping may have caused
4402 			 * the deallocation of the underlying PD page.
4403 			 */
4404 			pd = NULL;
4405 		}
4406 	}
4407 
4408 	/*
4409 	 * If the page table page is mapped, we just increment the
4410 	 * hold count, and activate it.
4411 	 */
4412 	if (pd != NULL && (be64toh(*pd) & PG_V) != 0) {
4413 		m = PHYS_TO_VM_PAGE(be64toh(*pd) & PG_FRAME);
4414 		m->ref_count++;
4415 	} else {
4416 		/*
4417 		 * Here if the pte page isn't mapped, or if it has been
4418 		 * deallocated.
4419 		 */
4420 		m = _pmap_allocpte(pmap, ptepindex, lockp);
4421 		if (m == NULL && lockp != NULL)
4422 			goto retry;
4423 	}
4424 	return (m);
4425 }
4426 
4427 static void
4428 mmu_radix_pinit0(pmap_t pmap)
4429 {
4430 
4431 	CTR2(KTR_PMAP, "%s(%p)", __func__, pmap);
4432 	PMAP_LOCK_INIT(pmap);
4433 	pmap->pm_pml1 = kernel_pmap->pm_pml1;
4434 	pmap->pm_pid = kernel_pmap->pm_pid;
4435 
4436 	vm_radix_init(&pmap->pm_radix);
4437 	TAILQ_INIT(&pmap->pm_pvchunk);
4438 	bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
4439 	kernel_pmap->pm_flags =
4440 		pmap->pm_flags = PMAP_PDE_SUPERPAGE;
4441 }
4442 /*
4443  * pmap_protect_l3e: do the things to protect a 2mpage in a process
4444  */
4445 static boolean_t
4446 pmap_protect_l3e(pmap_t pmap, pt_entry_t *l3e, vm_offset_t sva, vm_prot_t prot)
4447 {
4448 	pt_entry_t newpde, oldpde;
4449 	vm_offset_t eva, va;
4450 	vm_page_t m;
4451 	boolean_t anychanged;
4452 
4453 	PMAP_LOCK_ASSERT(pmap, MA_OWNED);
4454 	KASSERT((sva & L3_PAGE_MASK) == 0,
4455 	    ("pmap_protect_l3e: sva is not 2mpage aligned"));
4456 	anychanged = FALSE;
4457 retry:
4458 	oldpde = newpde = be64toh(*l3e);
4459 	if ((oldpde & (PG_MANAGED | PG_M | PG_RW)) ==
4460 	    (PG_MANAGED | PG_M | PG_RW)) {
4461 		eva = sva + L3_PAGE_SIZE;
4462 		for (va = sva, m = PHYS_TO_VM_PAGE(oldpde & PG_PS_FRAME);
4463 		    va < eva; va += PAGE_SIZE, m++)
4464 			vm_page_dirty(m);
4465 	}
4466 	if ((prot & VM_PROT_WRITE) == 0) {
4467 		newpde &= ~(PG_RW | PG_M);
4468 		newpde |= RPTE_EAA_R;
4469 	}
4470 	if (prot & VM_PROT_EXECUTE)
4471 		newpde |= PG_X;
4472 	if (newpde != oldpde) {
4473 		/*
4474 		 * As an optimization to future operations on this PDE, clear
4475 		 * PG_PROMOTED.  The impending invalidation will remove any
4476 		 * lingering 4KB page mappings from the TLB.
4477 		 */
4478 		if (!atomic_cmpset_long(l3e, htobe64(oldpde), htobe64(newpde & ~PG_PROMOTED)))
4479 			goto retry;
4480 		anychanged = TRUE;
4481 	}
4482 	return (anychanged);
4483 }
4484 
4485 void
4486 mmu_radix_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,
4487     vm_prot_t prot)
4488 {
4489 	vm_offset_t va_next;
4490 	pml1_entry_t *l1e;
4491 	pml2_entry_t *l2e;
4492 	pml3_entry_t ptpaddr, *l3e;
4493 	pt_entry_t *pte;
4494 	boolean_t anychanged;
4495 
4496 	CTR5(KTR_PMAP, "%s(%p, %#x, %#x, %#x)", __func__, pmap, sva, eva,
4497 	    prot);
4498 
4499 	KASSERT((prot & ~VM_PROT_ALL) == 0, ("invalid prot %x", prot));
4500 	if (prot == VM_PROT_NONE) {
4501 		mmu_radix_remove(pmap, sva, eva);
4502 		return;
4503 	}
4504 
4505 	if ((prot & (VM_PROT_WRITE|VM_PROT_EXECUTE)) ==
4506 	    (VM_PROT_WRITE|VM_PROT_EXECUTE))
4507 		return;
4508 
4509 #ifdef INVARIANTS
4510 	if (VERBOSE_PROTECT || pmap_logging)
4511 		printf("pmap_protect(%p, %#lx, %#lx, %x) - asid: %lu\n",
4512 			   pmap, sva, eva, prot, pmap->pm_pid);
4513 #endif
4514 	anychanged = FALSE;
4515 
4516 	PMAP_LOCK(pmap);
4517 	for (; sva < eva; sva = va_next) {
4518 		l1e = pmap_pml1e(pmap, sva);
4519 		if ((be64toh(*l1e) & PG_V) == 0) {
4520 			va_next = (sva + L1_PAGE_SIZE) & ~L1_PAGE_MASK;
4521 			if (va_next < sva)
4522 				va_next = eva;
4523 			continue;
4524 		}
4525 
4526 		l2e = pmap_l1e_to_l2e(l1e, sva);
4527 		if ((be64toh(*l2e) & PG_V) == 0) {
4528 			va_next = (sva + L2_PAGE_SIZE) & ~L2_PAGE_MASK;
4529 			if (va_next < sva)
4530 				va_next = eva;
4531 			continue;
4532 		}
4533 
4534 		va_next = (sva + L3_PAGE_SIZE) & ~L3_PAGE_MASK;
4535 		if (va_next < sva)
4536 			va_next = eva;
4537 
4538 		l3e = pmap_l2e_to_l3e(l2e, sva);
4539 		ptpaddr = be64toh(*l3e);
4540 
4541 		/*
4542 		 * Weed out invalid mappings.
4543 		 */
4544 		if (ptpaddr == 0)
4545 			continue;
4546 
4547 		/*
4548 		 * Check for large page.
4549 		 */
4550 		if ((ptpaddr & RPTE_LEAF) != 0) {
4551 			/*
4552 			 * Are we protecting the entire large page?  If not,
4553 			 * demote the mapping and fall through.
4554 			 */
4555 			if (sva + L3_PAGE_SIZE == va_next && eva >= va_next) {
4556 				if (pmap_protect_l3e(pmap, l3e, sva, prot))
4557 					anychanged = TRUE;
4558 				continue;
4559 			} else if (!pmap_demote_l3e(pmap, l3e, sva)) {
4560 				/*
4561 				 * The large page mapping was destroyed.
4562 				 */
4563 				continue;
4564 			}
4565 		}
4566 
4567 		if (va_next > eva)
4568 			va_next = eva;
4569 
4570 		for (pte = pmap_l3e_to_pte(l3e, sva); sva != va_next; pte++,
4571 		    sva += PAGE_SIZE) {
4572 			pt_entry_t obits, pbits;
4573 			vm_page_t m;
4574 
4575 retry:
4576 			MPASS(pte == pmap_pte(pmap, sva));
4577 			obits = pbits = be64toh(*pte);
4578 			if ((pbits & PG_V) == 0)
4579 				continue;
4580 
4581 			if ((prot & VM_PROT_WRITE) == 0) {
4582 				if ((pbits & (PG_MANAGED | PG_M | PG_RW)) ==
4583 				    (PG_MANAGED | PG_M | PG_RW)) {
4584 					m = PHYS_TO_VM_PAGE(pbits & PG_FRAME);
4585 					vm_page_dirty(m);
4586 				}
4587 				pbits &= ~(PG_RW | PG_M);
4588 				pbits |= RPTE_EAA_R;
4589 			}
4590 			if (prot & VM_PROT_EXECUTE)
4591 				pbits |= PG_X;
4592 
4593 			if (pbits != obits) {
4594 				if (!atomic_cmpset_long(pte, htobe64(obits), htobe64(pbits)))
4595 					goto retry;
4596 				if (obits & (PG_A|PG_M)) {
4597 					anychanged = TRUE;
4598 #ifdef INVARIANTS
4599 					if (VERBOSE_PROTECT || pmap_logging)
4600 						printf("%#lx %#lx -> %#lx\n",
4601 						    sva, obits, pbits);
4602 #endif
4603 				}
4604 			}
4605 		}
4606 	}
4607 	if (anychanged)
4608 		pmap_invalidate_all(pmap);
4609 	PMAP_UNLOCK(pmap);
4610 }
4611 
4612 void
4613 mmu_radix_qenter(vm_offset_t sva, vm_page_t *ma, int count)
4614 {
4615 
4616 	CTR4(KTR_PMAP, "%s(%#x, %p, %d)", __func__, sva, ma, count);
4617 	pt_entry_t oldpte, pa, *pte;
4618 	vm_page_t m;
4619 	uint64_t cache_bits, attr_bits;
4620 	vm_offset_t va;
4621 
4622 	oldpte = 0;
4623 	attr_bits = RPTE_EAA_R | RPTE_EAA_W | RPTE_EAA_P | PG_M | PG_A;
4624 	va = sva;
4625 	pte = kvtopte(va);
4626 	while (va < sva + PAGE_SIZE * count) {
4627 		if (__predict_false((va & L3_PAGE_MASK) == 0))
4628 			pte = kvtopte(va);
4629 		MPASS(pte == pmap_pte(kernel_pmap, va));
4630 
4631 		/*
4632 		 * XXX there has to be a more efficient way than traversing
4633 		 * the page table every time - but go for correctness for
4634 		 * today
4635 		 */
4636 
4637 		m = *ma++;
4638 		cache_bits = pmap_cache_bits(m->md.mdpg_cache_attrs);
4639 		pa = VM_PAGE_TO_PHYS(m) | cache_bits | attr_bits;
4640 		if (be64toh(*pte) != pa) {
4641 			oldpte |= be64toh(*pte);
4642 			pte_store(pte, pa);
4643 		}
4644 		va += PAGE_SIZE;
4645 		pte++;
4646 	}
4647 	if (__predict_false((oldpte & RPTE_VALID) != 0))
4648 		pmap_invalidate_range(kernel_pmap, sva, sva + count *
4649 		    PAGE_SIZE);
4650 	else
4651 		ptesync();
4652 }
4653 
4654 void
4655 mmu_radix_qremove(vm_offset_t sva, int count)
4656 {
4657 	vm_offset_t va;
4658 	pt_entry_t *pte;
4659 
4660 	CTR3(KTR_PMAP, "%s(%#x, %d)", __func__, sva, count);
4661 	KASSERT(sva >= VM_MIN_KERNEL_ADDRESS, ("usermode or dmap va %lx", sva));
4662 
4663 	va = sva;
4664 	pte = kvtopte(va);
4665 	while (va < sva + PAGE_SIZE * count) {
4666 		if (__predict_false((va & L3_PAGE_MASK) == 0))
4667 			pte = kvtopte(va);
4668 		pte_clear(pte);
4669 		pte++;
4670 		va += PAGE_SIZE;
4671 	}
4672 	pmap_invalidate_range(kernel_pmap, sva, va);
4673 }
4674 
4675 /***************************************************
4676  * Page table page management routines.....
4677  ***************************************************/
4678 /*
4679  * Schedule the specified unused page table page to be freed.  Specifically,
4680  * add the page to the specified list of pages that will be released to the
4681  * physical memory manager after the TLB has been updated.
4682  */
4683 static __inline void
4684 pmap_add_delayed_free_list(vm_page_t m, struct spglist *free,
4685     boolean_t set_PG_ZERO)
4686 {
4687 
4688 	if (set_PG_ZERO)
4689 		m->flags |= PG_ZERO;
4690 	else
4691 		m->flags &= ~PG_ZERO;
4692 	SLIST_INSERT_HEAD(free, m, plinks.s.ss);
4693 }
4694 
4695 /*
4696  * Inserts the specified page table page into the specified pmap's collection
4697  * of idle page table pages.  Each of a pmap's page table pages is responsible
4698  * for mapping a distinct range of virtual addresses.  The pmap's collection is
4699  * ordered by this virtual address range.
4700  */
4701 static __inline int
4702 pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte)
4703 {
4704 
4705 	PMAP_LOCK_ASSERT(pmap, MA_OWNED);
4706 	return (vm_radix_insert(&pmap->pm_radix, mpte));
4707 }
4708 
4709 /*
4710  * Removes the page table page mapping the specified virtual address from the
4711  * specified pmap's collection of idle page table pages, and returns it.
4712  * Otherwise, returns NULL if there is no page table page corresponding to the
4713  * specified virtual address.
4714  */
4715 static __inline vm_page_t
4716 pmap_remove_pt_page(pmap_t pmap, vm_offset_t va)
4717 {
4718 
4719 	PMAP_LOCK_ASSERT(pmap, MA_OWNED);
4720 	return (vm_radix_remove(&pmap->pm_radix, pmap_l3e_pindex(va)));
4721 }
4722 
4723 /*
4724  * Decrements a page table page's wire count, which is used to record the
4725  * number of valid page table entries within the page.  If the wire count
4726  * drops to zero, then the page table page is unmapped.  Returns TRUE if the
4727  * page table page was unmapped and FALSE otherwise.
4728  */
4729 static inline boolean_t
4730 pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free)
4731 {
4732 
4733 	--m->ref_count;
4734 	if (m->ref_count == 0) {
4735 		_pmap_unwire_ptp(pmap, va, m, free);
4736 		return (TRUE);
4737 	} else
4738 		return (FALSE);
4739 }
4740 
4741 static void
4742 _pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free)
4743 {
4744 
4745 	PMAP_LOCK_ASSERT(pmap, MA_OWNED);
4746 	/*
4747 	 * unmap the page table page
4748 	 */
4749 	if (m->pindex >= NUPDE + NUPDPE) {
4750 		/* PDP page */
4751 		pml1_entry_t *pml1;
4752 		pml1 = pmap_pml1e(pmap, va);
4753 		*pml1 = 0;
4754 	} else if (m->pindex >= NUPDE) {
4755 		/* PD page */
4756 		pml2_entry_t *l2e;
4757 		l2e = pmap_pml2e(pmap, va);
4758 		*l2e = 0;
4759 	} else {
4760 		/* PTE page */
4761 		pml3_entry_t *l3e;
4762 		l3e = pmap_pml3e(pmap, va);
4763 		*l3e = 0;
4764 	}
4765 	pmap_resident_count_dec(pmap, 1);
4766 	if (m->pindex < NUPDE) {
4767 		/* We just released a PT, unhold the matching PD */
4768 		vm_page_t pdpg;
4769 
4770 		pdpg = PHYS_TO_VM_PAGE(be64toh(*pmap_pml2e(pmap, va)) & PG_FRAME);
4771 		pmap_unwire_ptp(pmap, va, pdpg, free);
4772 	}
4773 	else if (m->pindex >= NUPDE && m->pindex < (NUPDE + NUPDPE)) {
4774 		/* We just released a PD, unhold the matching PDP */
4775 		vm_page_t pdppg;
4776 
4777 		pdppg = PHYS_TO_VM_PAGE(be64toh(*pmap_pml1e(pmap, va)) & PG_FRAME);
4778 		pmap_unwire_ptp(pmap, va, pdppg, free);
4779 	}
4780 
4781 	/*
4782 	 * Put page on a list so that it is released after
4783 	 * *ALL* TLB shootdown is done
4784 	 */
4785 	pmap_add_delayed_free_list(m, free, TRUE);
4786 }
4787 
4788 /*
4789  * After removing a page table entry, this routine is used to
4790  * conditionally free the page, and manage the hold/wire counts.
4791  */
4792 static int
4793 pmap_unuse_pt(pmap_t pmap, vm_offset_t va, pml3_entry_t ptepde,
4794     struct spglist *free)
4795 {
4796 	vm_page_t mpte;
4797 
4798 	if (va >= VM_MAXUSER_ADDRESS)
4799 		return (0);
4800 	KASSERT(ptepde != 0, ("pmap_unuse_pt: ptepde != 0"));
4801 	mpte = PHYS_TO_VM_PAGE(ptepde & PG_FRAME);
4802 	return (pmap_unwire_ptp(pmap, va, mpte, free));
4803 }
4804 
4805 void
4806 mmu_radix_release(pmap_t pmap)
4807 {
4808 
4809 	CTR2(KTR_PMAP, "%s(%p)", __func__, pmap);
4810 	KASSERT(pmap->pm_stats.resident_count == 0,
4811 	    ("pmap_release: pmap resident count %ld != 0",
4812 	    pmap->pm_stats.resident_count));
4813 	KASSERT(vm_radix_is_empty(&pmap->pm_radix),
4814 	    ("pmap_release: pmap has reserved page table page(s)"));
4815 
4816 	pmap_invalidate_all(pmap);
4817 	isa3_proctab[pmap->pm_pid].proctab0 = 0;
4818 	uma_zfree(zone_radix_pgd, pmap->pm_pml1);
4819 	vmem_free(asid_arena, pmap->pm_pid, 1);
4820 }
4821 
4822 /*
4823  * Create the PV entry for a 2MB page mapping.  Always returns true unless the
4824  * flag PMAP_ENTER_NORECLAIM is specified.  If that flag is specified, returns
4825  * false if the PV entry cannot be allocated without resorting to reclamation.
4826  */
4827 static bool
4828 pmap_pv_insert_l3e(pmap_t pmap, vm_offset_t va, pml3_entry_t pde, u_int flags,
4829     struct rwlock **lockp)
4830 {
4831 	struct md_page *pvh;
4832 	pv_entry_t pv;
4833 	vm_paddr_t pa;
4834 
4835 	PMAP_LOCK_ASSERT(pmap, MA_OWNED);
4836 	/* Pass NULL instead of the lock pointer to disable reclamation. */
4837 	if ((pv = get_pv_entry(pmap, (flags & PMAP_ENTER_NORECLAIM) != 0 ?
4838 	    NULL : lockp)) == NULL)
4839 		return (false);
4840 	pv->pv_va = va;
4841 	pa = pde & PG_PS_FRAME;
4842 	CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa);
4843 	pvh = pa_to_pvh(pa);
4844 	TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_link);
4845 	pvh->pv_gen++;
4846 	return (true);
4847 }
4848 
4849 /*
4850  * Fills a page table page with mappings to consecutive physical pages.
4851  */
4852 static void
4853 pmap_fill_ptp(pt_entry_t *firstpte, pt_entry_t newpte)
4854 {
4855 	pt_entry_t *pte;
4856 
4857 	for (pte = firstpte; pte < firstpte + NPTEPG; pte++) {
4858 		*pte = htobe64(newpte);
4859 		newpte += PAGE_SIZE;
4860 	}
4861 }
4862 
4863 static boolean_t
4864 pmap_demote_l3e(pmap_t pmap, pml3_entry_t *pde, vm_offset_t va)
4865 {
4866 	struct rwlock *lock;
4867 	boolean_t rv;
4868 
4869 	lock = NULL;
4870 	rv = pmap_demote_l3e_locked(pmap, pde, va, &lock);
4871 	if (lock != NULL)
4872 		rw_wunlock(lock);
4873 	return (rv);
4874 }
4875 
4876 static boolean_t
4877 pmap_demote_l3e_locked(pmap_t pmap, pml3_entry_t *l3e, vm_offset_t va,
4878     struct rwlock **lockp)
4879 {
4880 	pml3_entry_t oldpde;
4881 	pt_entry_t *firstpte;
4882 	vm_paddr_t mptepa;
4883 	vm_page_t mpte;
4884 	struct spglist free;
4885 	vm_offset_t sva;
4886 
4887 	PMAP_LOCK_ASSERT(pmap, MA_OWNED);
4888 	oldpde = be64toh(*l3e);
4889 	KASSERT((oldpde & (RPTE_LEAF | PG_V)) == (RPTE_LEAF | PG_V),
4890 	    ("pmap_demote_l3e: oldpde is missing RPTE_LEAF and/or PG_V %lx",
4891 	    oldpde));
4892 	if ((oldpde & PG_A) == 0 || (mpte = pmap_remove_pt_page(pmap, va)) ==
4893 	    NULL) {
4894 		KASSERT((oldpde & PG_W) == 0,
4895 		    ("pmap_demote_l3e: page table page for a wired mapping"
4896 		    " is missing"));
4897 
4898 		/*
4899 		 * Invalidate the 2MB page mapping and return "failure" if the
4900 		 * mapping was never accessed or the allocation of the new
4901 		 * page table page fails.  If the 2MB page mapping belongs to
4902 		 * the direct map region of the kernel's address space, then
4903 		 * the page allocation request specifies the highest possible
4904 		 * priority (VM_ALLOC_INTERRUPT).  Otherwise, the priority is
4905 		 * normal.  Page table pages are preallocated for every other
4906 		 * part of the kernel address space, so the direct map region
4907 		 * is the only part of the kernel address space that must be
4908 		 * handled here.
4909 		 */
4910 		if ((oldpde & PG_A) == 0 || (mpte = vm_page_alloc_noobj(
4911 		    (va >= DMAP_MIN_ADDRESS && va < DMAP_MAX_ADDRESS ?
4912 		    VM_ALLOC_INTERRUPT : 0) | VM_ALLOC_WIRED)) == NULL) {
4913 			SLIST_INIT(&free);
4914 			sva = trunc_2mpage(va);
4915 			pmap_remove_l3e(pmap, l3e, sva, &free, lockp);
4916 			pmap_invalidate_l3e_page(pmap, sva, oldpde);
4917 			vm_page_free_pages_toq(&free, true);
4918 			CTR2(KTR_PMAP, "pmap_demote_l3e: failure for va %#lx"
4919 			    " in pmap %p", va, pmap);
4920 			return (FALSE);
4921 		}
4922 		mpte->pindex = pmap_l3e_pindex(va);
4923 		if (va < VM_MAXUSER_ADDRESS)
4924 			pmap_resident_count_inc(pmap, 1);
4925 	}
4926 	mptepa = VM_PAGE_TO_PHYS(mpte);
4927 	firstpte = (pt_entry_t *)PHYS_TO_DMAP(mptepa);
4928 	KASSERT((oldpde & PG_A) != 0,
4929 	    ("pmap_demote_l3e: oldpde is missing PG_A"));
4930 	KASSERT((oldpde & (PG_M | PG_RW)) != PG_RW,
4931 	    ("pmap_demote_l3e: oldpde is missing PG_M"));
4932 
4933 	/*
4934 	 * If the page table page is new, initialize it.
4935 	 */
4936 	if (mpte->ref_count == 1) {
4937 		mpte->ref_count = NPTEPG;
4938 		pmap_fill_ptp(firstpte, oldpde);
4939 	}
4940 
4941 	KASSERT((be64toh(*firstpte) & PG_FRAME) == (oldpde & PG_FRAME),
4942 	    ("pmap_demote_l3e: firstpte and newpte map different physical"
4943 	    " addresses"));
4944 
4945 	/*
4946 	 * If the mapping has changed attributes, update the page table
4947 	 * entries.
4948 	 */
4949 	if ((be64toh(*firstpte) & PG_PTE_PROMOTE) != (oldpde & PG_PTE_PROMOTE))
4950 		pmap_fill_ptp(firstpte, oldpde);
4951 
4952 	/*
4953 	 * The spare PV entries must be reserved prior to demoting the
4954 	 * mapping, that is, prior to changing the PDE.  Otherwise, the state
4955 	 * of the PDE and the PV lists will be inconsistent, which can result
4956 	 * in reclaim_pv_chunk() attempting to remove a PV entry from the
4957 	 * wrong PV list and pmap_pv_demote_l3e() failing to find the expected
4958 	 * PV entry for the 2MB page mapping that is being demoted.
4959 	 */
4960 	if ((oldpde & PG_MANAGED) != 0)
4961 		reserve_pv_entries(pmap, NPTEPG - 1, lockp);
4962 
4963 	/*
4964 	 * Demote the mapping.  This pmap is locked.  The old PDE has
4965 	 * PG_A set.  If the old PDE has PG_RW set, it also has PG_M
4966 	 * set.  Thus, there is no danger of a race with another
4967 	 * processor changing the setting of PG_A and/or PG_M between
4968 	 * the read above and the store below.
4969 	 */
4970 	pde_store(l3e, mptepa);
4971 	pmap_invalidate_l3e_page(pmap, trunc_2mpage(va), oldpde);
4972 	/*
4973 	 * Demote the PV entry.
4974 	 */
4975 	if ((oldpde & PG_MANAGED) != 0)
4976 		pmap_pv_demote_l3e(pmap, va, oldpde & PG_PS_FRAME, lockp);
4977 
4978 	counter_u64_add(pmap_l3e_demotions, 1);
4979 	CTR2(KTR_PMAP, "pmap_demote_l3e: success for va %#lx"
4980 	    " in pmap %p", va, pmap);
4981 	return (TRUE);
4982 }
4983 
4984 /*
4985  * pmap_remove_kernel_pde: Remove a kernel superpage mapping.
4986  */
4987 static void
4988 pmap_remove_kernel_l3e(pmap_t pmap, pml3_entry_t *l3e, vm_offset_t va)
4989 {
4990 	vm_paddr_t mptepa;
4991 	vm_page_t mpte;
4992 
4993 	KASSERT(pmap == kernel_pmap, ("pmap %p is not kernel_pmap", pmap));
4994 	PMAP_LOCK_ASSERT(pmap, MA_OWNED);
4995 	mpte = pmap_remove_pt_page(pmap, va);
4996 	if (mpte == NULL)
4997 		panic("pmap_remove_kernel_pde: Missing pt page.");
4998 
4999 	mptepa = VM_PAGE_TO_PHYS(mpte);
5000 
5001 	/*
5002 	 * Initialize the page table page.
5003 	 */
5004 	pagezero(PHYS_TO_DMAP(mptepa));
5005 
5006 	/*
5007 	 * Demote the mapping.
5008 	 */
5009 	pde_store(l3e, mptepa);
5010 	ptesync();
5011 }
5012 
5013 /*
5014  * pmap_remove_l3e: do the things to unmap a superpage in a process
5015  */
5016 static int
5017 pmap_remove_l3e(pmap_t pmap, pml3_entry_t *pdq, vm_offset_t sva,
5018     struct spglist *free, struct rwlock **lockp)
5019 {
5020 	struct md_page *pvh;
5021 	pml3_entry_t oldpde;
5022 	vm_offset_t eva, va;
5023 	vm_page_t m, mpte;
5024 
5025 	PMAP_LOCK_ASSERT(pmap, MA_OWNED);
5026 	KASSERT((sva & L3_PAGE_MASK) == 0,
5027 	    ("pmap_remove_l3e: sva is not 2mpage aligned"));
5028 	oldpde = be64toh(pte_load_clear(pdq));
5029 	if (oldpde & PG_W)
5030 		pmap->pm_stats.wired_count -= (L3_PAGE_SIZE / PAGE_SIZE);
5031 	pmap_resident_count_dec(pmap, L3_PAGE_SIZE / PAGE_SIZE);
5032 	if (oldpde & PG_MANAGED) {
5033 		CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, oldpde & PG_PS_FRAME);
5034 		pvh = pa_to_pvh(oldpde & PG_PS_FRAME);
5035 		pmap_pvh_free(pvh, pmap, sva);
5036 		eva = sva + L3_PAGE_SIZE;
5037 		for (va = sva, m = PHYS_TO_VM_PAGE(oldpde & PG_PS_FRAME);
5038 		    va < eva; va += PAGE_SIZE, m++) {
5039 			if ((oldpde & (PG_M | PG_RW)) == (PG_M | PG_RW))
5040 				vm_page_dirty(m);
5041 			if (oldpde & PG_A)
5042 				vm_page_aflag_set(m, PGA_REFERENCED);
5043 			if (TAILQ_EMPTY(&m->md.pv_list) &&
5044 			    TAILQ_EMPTY(&pvh->pv_list))
5045 				vm_page_aflag_clear(m, PGA_WRITEABLE);
5046 		}
5047 	}
5048 	if (pmap == kernel_pmap) {
5049 		pmap_remove_kernel_l3e(pmap, pdq, sva);
5050 	} else {
5051 		mpte = pmap_remove_pt_page(pmap, sva);
5052 		if (mpte != NULL) {
5053 			pmap_resident_count_dec(pmap, 1);
5054 			KASSERT(mpte->ref_count == NPTEPG,
5055 			    ("pmap_remove_l3e: pte page wire count error"));
5056 			mpte->ref_count = 0;
5057 			pmap_add_delayed_free_list(mpte, free, FALSE);
5058 		}
5059 	}
5060 	return (pmap_unuse_pt(pmap, sva, be64toh(*pmap_pml2e(pmap, sva)), free));
5061 }
5062 
5063 /*
5064  * pmap_remove_pte: do the things to unmap a page in a process
5065  */
5066 static int
5067 pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t va,
5068     pml3_entry_t ptepde, struct spglist *free, struct rwlock **lockp)
5069 {
5070 	struct md_page *pvh;
5071 	pt_entry_t oldpte;
5072 	vm_page_t m;
5073 
5074 	PMAP_LOCK_ASSERT(pmap, MA_OWNED);
5075 	oldpte = be64toh(pte_load_clear(ptq));
5076 	if (oldpte & RPTE_WIRED)
5077 		pmap->pm_stats.wired_count -= 1;
5078 	pmap_resident_count_dec(pmap, 1);
5079 	if (oldpte & RPTE_MANAGED) {
5080 		m = PHYS_TO_VM_PAGE(oldpte & PG_FRAME);
5081 		if ((oldpte & (PG_M | PG_RW)) == (PG_M | PG_RW))
5082 			vm_page_dirty(m);
5083 		if (oldpte & PG_A)
5084 			vm_page_aflag_set(m, PGA_REFERENCED);
5085 		CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m);
5086 		pmap_pvh_free(&m->md, pmap, va);
5087 		if (TAILQ_EMPTY(&m->md.pv_list) &&
5088 		    (m->flags & PG_FICTITIOUS) == 0) {
5089 			pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
5090 			if (TAILQ_EMPTY(&pvh->pv_list))
5091 				vm_page_aflag_clear(m, PGA_WRITEABLE);
5092 		}
5093 	}
5094 	return (pmap_unuse_pt(pmap, va, ptepde, free));
5095 }
5096 
5097 /*
5098  * Remove a single page from a process address space
5099  */
5100 static bool
5101 pmap_remove_page(pmap_t pmap, vm_offset_t va, pml3_entry_t *l3e,
5102     struct spglist *free)
5103 {
5104 	struct rwlock *lock;
5105 	pt_entry_t *pte;
5106 	bool invalidate_all;
5107 
5108 	PMAP_LOCK_ASSERT(pmap, MA_OWNED);
5109 	if ((be64toh(*l3e) & RPTE_VALID) == 0) {
5110 		return (false);
5111 	}
5112 	pte = pmap_l3e_to_pte(l3e, va);
5113 	if ((be64toh(*pte) & RPTE_VALID) == 0) {
5114 		return (false);
5115 	}
5116 	lock = NULL;
5117 
5118 	invalidate_all = pmap_remove_pte(pmap, pte, va, be64toh(*l3e), free, &lock);
5119 	if (lock != NULL)
5120 		rw_wunlock(lock);
5121 	if (!invalidate_all)
5122 		pmap_invalidate_page(pmap, va);
5123 	return (invalidate_all);
5124 }
5125 
5126 /*
5127  * Removes the specified range of addresses from the page table page.
5128  */
5129 static bool
5130 pmap_remove_ptes(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,
5131     pml3_entry_t *l3e, struct spglist *free, struct rwlock **lockp)
5132 {
5133 	pt_entry_t *pte;
5134 	vm_offset_t va;
5135 	bool anyvalid;
5136 
5137 	PMAP_LOCK_ASSERT(pmap, MA_OWNED);
5138 	anyvalid = false;
5139 	va = eva;
5140 	for (pte = pmap_l3e_to_pte(l3e, sva); sva != eva; pte++,
5141 	    sva += PAGE_SIZE) {
5142 		MPASS(pte == pmap_pte(pmap, sva));
5143 		if (*pte == 0) {
5144 			if (va != eva) {
5145 				anyvalid = true;
5146 				va = eva;
5147 			}
5148 			continue;
5149 		}
5150 		if (va == eva)
5151 			va = sva;
5152 		if (pmap_remove_pte(pmap, pte, sva, be64toh(*l3e), free, lockp)) {
5153 			anyvalid = true;
5154 			sva += PAGE_SIZE;
5155 			break;
5156 		}
5157 	}
5158 	if (anyvalid)
5159 		pmap_invalidate_all(pmap);
5160 	else if (va != eva)
5161 		pmap_invalidate_range(pmap, va, sva);
5162 	return (anyvalid);
5163 }
5164 
5165 void
5166 mmu_radix_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
5167 {
5168 	struct rwlock *lock;
5169 	vm_offset_t va_next;
5170 	pml1_entry_t *l1e;
5171 	pml2_entry_t *l2e;
5172 	pml3_entry_t ptpaddr, *l3e;
5173 	struct spglist free;
5174 	bool anyvalid;
5175 
5176 	CTR4(KTR_PMAP, "%s(%p, %#x, %#x)", __func__, pmap, sva, eva);
5177 
5178 	/*
5179 	 * Perform an unsynchronized read.  This is, however, safe.
5180 	 */
5181 	if (pmap->pm_stats.resident_count == 0)
5182 		return;
5183 
5184 	anyvalid = false;
5185 	SLIST_INIT(&free);
5186 
5187 	/* XXX something fishy here */
5188 	sva = (sva + PAGE_MASK) & ~PAGE_MASK;
5189 	eva = (eva + PAGE_MASK) & ~PAGE_MASK;
5190 
5191 	PMAP_LOCK(pmap);
5192 
5193 	/*
5194 	 * special handling of removing one page.  a very
5195 	 * common operation and easy to short circuit some
5196 	 * code.
5197 	 */
5198 	if (sva + PAGE_SIZE == eva) {
5199 		l3e = pmap_pml3e(pmap, sva);
5200 		if (l3e && (be64toh(*l3e) & RPTE_LEAF) == 0) {
5201 			anyvalid = pmap_remove_page(pmap, sva, l3e, &free);
5202 			goto out;
5203 		}
5204 	}
5205 
5206 	lock = NULL;
5207 	for (; sva < eva; sva = va_next) {
5208 		if (pmap->pm_stats.resident_count == 0)
5209 			break;
5210 		l1e = pmap_pml1e(pmap, sva);
5211 		if (l1e == NULL || (be64toh(*l1e) & PG_V) == 0) {
5212 			va_next = (sva + L1_PAGE_SIZE) & ~L1_PAGE_MASK;
5213 			if (va_next < sva)
5214 				va_next = eva;
5215 			continue;
5216 		}
5217 
5218 		l2e = pmap_l1e_to_l2e(l1e, sva);
5219 		if (l2e == NULL || (be64toh(*l2e) & PG_V) == 0) {
5220 			va_next = (sva + L2_PAGE_SIZE) & ~L2_PAGE_MASK;
5221 			if (va_next < sva)
5222 				va_next = eva;
5223 			continue;
5224 		}
5225 
5226 		/*
5227 		 * Calculate index for next page table.
5228 		 */
5229 		va_next = (sva + L3_PAGE_SIZE) & ~L3_PAGE_MASK;
5230 		if (va_next < sva)
5231 			va_next = eva;
5232 
5233 		l3e = pmap_l2e_to_l3e(l2e, sva);
5234 		ptpaddr = be64toh(*l3e);
5235 
5236 		/*
5237 		 * Weed out invalid mappings.
5238 		 */
5239 		if (ptpaddr == 0)
5240 			continue;
5241 
5242 		/*
5243 		 * Check for large page.
5244 		 */
5245 		if ((ptpaddr & RPTE_LEAF) != 0) {
5246 			/*
5247 			 * Are we removing the entire large page?  If not,
5248 			 * demote the mapping and fall through.
5249 			 */
5250 			if (sva + L3_PAGE_SIZE == va_next && eva >= va_next) {
5251 				pmap_remove_l3e(pmap, l3e, sva, &free, &lock);
5252 				anyvalid = true;
5253 				continue;
5254 			} else if (!pmap_demote_l3e_locked(pmap, l3e, sva,
5255 			    &lock)) {
5256 				/* The large page mapping was destroyed. */
5257 				continue;
5258 			} else
5259 				ptpaddr = be64toh(*l3e);
5260 		}
5261 
5262 		/*
5263 		 * Limit our scan to either the end of the va represented
5264 		 * by the current page table page, or to the end of the
5265 		 * range being removed.
5266 		 */
5267 		if (va_next > eva)
5268 			va_next = eva;
5269 
5270 		if (pmap_remove_ptes(pmap, sva, va_next, l3e, &free, &lock))
5271 			anyvalid = true;
5272 	}
5273 	if (lock != NULL)
5274 		rw_wunlock(lock);
5275 out:
5276 	if (anyvalid)
5277 		pmap_invalidate_all(pmap);
5278 	PMAP_UNLOCK(pmap);
5279 	vm_page_free_pages_toq(&free, true);
5280 }
5281 
5282 void
5283 mmu_radix_remove_all(vm_page_t m)
5284 {
5285 	struct md_page *pvh;
5286 	pv_entry_t pv;
5287 	pmap_t pmap;
5288 	struct rwlock *lock;
5289 	pt_entry_t *pte, tpte;
5290 	pml3_entry_t *l3e;
5291 	vm_offset_t va;
5292 	struct spglist free;
5293 	int pvh_gen, md_gen;
5294 
5295 	CTR2(KTR_PMAP, "%s(%p)", __func__, m);
5296 	KASSERT((m->oflags & VPO_UNMANAGED) == 0,
5297 	    ("pmap_remove_all: page %p is not managed", m));
5298 	SLIST_INIT(&free);
5299 	lock = VM_PAGE_TO_PV_LIST_LOCK(m);
5300 	pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy :
5301 	    pa_to_pvh(VM_PAGE_TO_PHYS(m));
5302 retry:
5303 	rw_wlock(lock);
5304 	while ((pv = TAILQ_FIRST(&pvh->pv_list)) != NULL) {
5305 		pmap = PV_PMAP(pv);
5306 		if (!PMAP_TRYLOCK(pmap)) {
5307 			pvh_gen = pvh->pv_gen;
5308 			rw_wunlock(lock);
5309 			PMAP_LOCK(pmap);
5310 			rw_wlock(lock);
5311 			if (pvh_gen != pvh->pv_gen) {
5312 				rw_wunlock(lock);
5313 				PMAP_UNLOCK(pmap);
5314 				goto retry;
5315 			}
5316 		}
5317 		va = pv->pv_va;
5318 		l3e = pmap_pml3e(pmap, va);
5319 		(void)pmap_demote_l3e_locked(pmap, l3e, va, &lock);
5320 		PMAP_UNLOCK(pmap);
5321 	}
5322 	while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) {
5323 		pmap = PV_PMAP(pv);
5324 		if (!PMAP_TRYLOCK(pmap)) {
5325 			pvh_gen = pvh->pv_gen;
5326 			md_gen = m->md.pv_gen;
5327 			rw_wunlock(lock);
5328 			PMAP_LOCK(pmap);
5329 			rw_wlock(lock);
5330 			if (pvh_gen != pvh->pv_gen || md_gen != m->md.pv_gen) {
5331 				rw_wunlock(lock);
5332 				PMAP_UNLOCK(pmap);
5333 				goto retry;
5334 			}
5335 		}
5336 		pmap_resident_count_dec(pmap, 1);
5337 		l3e = pmap_pml3e(pmap, pv->pv_va);
5338 		KASSERT((be64toh(*l3e) & RPTE_LEAF) == 0, ("pmap_remove_all: found"
5339 		    " a 2mpage in page %p's pv list", m));
5340 		pte = pmap_l3e_to_pte(l3e, pv->pv_va);
5341 		tpte = be64toh(pte_load_clear(pte));
5342 		if (tpte & PG_W)
5343 			pmap->pm_stats.wired_count--;
5344 		if (tpte & PG_A)
5345 			vm_page_aflag_set(m, PGA_REFERENCED);
5346 
5347 		/*
5348 		 * Update the vm_page_t clean and reference bits.
5349 		 */
5350 		if ((tpte & (PG_M | PG_RW)) == (PG_M | PG_RW))
5351 			vm_page_dirty(m);
5352 		pmap_unuse_pt(pmap, pv->pv_va, be64toh(*l3e), &free);
5353 		pmap_invalidate_page(pmap, pv->pv_va);
5354 		TAILQ_REMOVE(&m->md.pv_list, pv, pv_link);
5355 		m->md.pv_gen++;
5356 		free_pv_entry(pmap, pv);
5357 		PMAP_UNLOCK(pmap);
5358 	}
5359 	vm_page_aflag_clear(m, PGA_WRITEABLE);
5360 	rw_wunlock(lock);
5361 	vm_page_free_pages_toq(&free, true);
5362 }
5363 
5364 /*
5365  * Destroy all managed, non-wired mappings in the given user-space
5366  * pmap.  This pmap cannot be active on any processor besides the
5367  * caller.
5368  *
5369  * This function cannot be applied to the kernel pmap.  Moreover, it
5370  * is not intended for general use.  It is only to be used during
5371  * process termination.  Consequently, it can be implemented in ways
5372  * that make it faster than pmap_remove().  First, it can more quickly
5373  * destroy mappings by iterating over the pmap's collection of PV
5374  * entries, rather than searching the page table.  Second, it doesn't
5375  * have to test and clear the page table entries atomically, because
5376  * no processor is currently accessing the user address space.  In
5377  * particular, a page table entry's dirty bit won't change state once
5378  * this function starts.
5379  *
5380  * Although this function destroys all of the pmap's managed,
5381  * non-wired mappings, it can delay and batch the invalidation of TLB
5382  * entries without calling pmap_delayed_invl_started() and
5383  * pmap_delayed_invl_finished().  Because the pmap is not active on
5384  * any other processor, none of these TLB entries will ever be used
5385  * before their eventual invalidation.  Consequently, there is no need
5386  * for either pmap_remove_all() or pmap_remove_write() to wait for
5387  * that eventual TLB invalidation.
5388  */
5389 
5390 void
5391 mmu_radix_remove_pages(pmap_t pmap)
5392 {
5393 
5394 	CTR2(KTR_PMAP, "%s(%p)", __func__, pmap);
5395 	pml3_entry_t ptel3e;
5396 	pt_entry_t *pte, tpte;
5397 	struct spglist free;
5398 	vm_page_t m, mpte, mt;
5399 	pv_entry_t pv;
5400 	struct md_page *pvh;
5401 	struct pv_chunk *pc, *npc;
5402 	struct rwlock *lock;
5403 	int64_t bit;
5404 	uint64_t inuse, bitmask;
5405 	int allfree, field, freed, idx;
5406 	boolean_t superpage;
5407 	vm_paddr_t pa;
5408 
5409 	/*
5410 	 * Assert that the given pmap is only active on the current
5411 	 * CPU.  Unfortunately, we cannot block another CPU from
5412 	 * activating the pmap while this function is executing.
5413 	 */
5414 	KASSERT(pmap->pm_pid == mfspr(SPR_PID),
5415 	    ("non-current asid %lu - expected %lu", pmap->pm_pid,
5416 	    mfspr(SPR_PID)));
5417 
5418 	lock = NULL;
5419 
5420 	SLIST_INIT(&free);
5421 	PMAP_LOCK(pmap);
5422 	TAILQ_FOREACH_SAFE(pc, &pmap->pm_pvchunk, pc_list, npc) {
5423 		allfree = 1;
5424 		freed = 0;
5425 		for (field = 0; field < _NPCM; field++) {
5426 			inuse = ~pc->pc_map[field] & pc_freemask[field];
5427 			while (inuse != 0) {
5428 				bit = cnttzd(inuse);
5429 				bitmask = 1UL << bit;
5430 				idx = field * 64 + bit;
5431 				pv = &pc->pc_pventry[idx];
5432 				inuse &= ~bitmask;
5433 
5434 				pte = pmap_pml2e(pmap, pv->pv_va);
5435 				ptel3e = be64toh(*pte);
5436 				pte = pmap_l2e_to_l3e(pte, pv->pv_va);
5437 				tpte = be64toh(*pte);
5438 				if ((tpte & (RPTE_LEAF | PG_V)) == PG_V) {
5439 					superpage = FALSE;
5440 					ptel3e = tpte;
5441 					pte = (pt_entry_t *)PHYS_TO_DMAP(tpte &
5442 					    PG_FRAME);
5443 					pte = &pte[pmap_pte_index(pv->pv_va)];
5444 					tpte = be64toh(*pte);
5445 				} else {
5446 					/*
5447 					 * Keep track whether 'tpte' is a
5448 					 * superpage explicitly instead of
5449 					 * relying on RPTE_LEAF being set.
5450 					 *
5451 					 * This is because RPTE_LEAF is numerically
5452 					 * identical to PG_PTE_PAT and thus a
5453 					 * regular page could be mistaken for
5454 					 * a superpage.
5455 					 */
5456 					superpage = TRUE;
5457 				}
5458 
5459 				if ((tpte & PG_V) == 0) {
5460 					panic("bad pte va %lx pte %lx",
5461 					    pv->pv_va, tpte);
5462 				}
5463 
5464 /*
5465  * We cannot remove wired pages from a process' mapping at this time
5466  */
5467 				if (tpte & PG_W) {
5468 					allfree = 0;
5469 					continue;
5470 				}
5471 
5472 				if (superpage)
5473 					pa = tpte & PG_PS_FRAME;
5474 				else
5475 					pa = tpte & PG_FRAME;
5476 
5477 				m = PHYS_TO_VM_PAGE(pa);
5478 				KASSERT(m->phys_addr == pa,
5479 				    ("vm_page_t %p phys_addr mismatch %016jx %016jx",
5480 				    m, (uintmax_t)m->phys_addr,
5481 				    (uintmax_t)tpte));
5482 
5483 				KASSERT((m->flags & PG_FICTITIOUS) != 0 ||
5484 				    m < &vm_page_array[vm_page_array_size],
5485 				    ("pmap_remove_pages: bad tpte %#jx",
5486 				    (uintmax_t)tpte));
5487 
5488 				pte_clear(pte);
5489 
5490 				/*
5491 				 * Update the vm_page_t clean/reference bits.
5492 				 */
5493 				if ((tpte & (PG_M | PG_RW)) == (PG_M | PG_RW)) {
5494 					if (superpage) {
5495 						for (mt = m; mt < &m[L3_PAGE_SIZE / PAGE_SIZE]; mt++)
5496 							vm_page_dirty(mt);
5497 					} else
5498 						vm_page_dirty(m);
5499 				}
5500 
5501 				CHANGE_PV_LIST_LOCK_TO_VM_PAGE(&lock, m);
5502 
5503 				/* Mark free */
5504 				pc->pc_map[field] |= bitmask;
5505 				if (superpage) {
5506 					pmap_resident_count_dec(pmap, L3_PAGE_SIZE / PAGE_SIZE);
5507 					pvh = pa_to_pvh(tpte & PG_PS_FRAME);
5508 					TAILQ_REMOVE(&pvh->pv_list, pv, pv_link);
5509 					pvh->pv_gen++;
5510 					if (TAILQ_EMPTY(&pvh->pv_list)) {
5511 						for (mt = m; mt < &m[L3_PAGE_SIZE / PAGE_SIZE]; mt++)
5512 							if ((mt->a.flags & PGA_WRITEABLE) != 0 &&
5513 							    TAILQ_EMPTY(&mt->md.pv_list))
5514 								vm_page_aflag_clear(mt, PGA_WRITEABLE);
5515 					}
5516 					mpte = pmap_remove_pt_page(pmap, pv->pv_va);
5517 					if (mpte != NULL) {
5518 						pmap_resident_count_dec(pmap, 1);
5519 						KASSERT(mpte->ref_count == NPTEPG,
5520 						    ("pmap_remove_pages: pte page wire count error"));
5521 						mpte->ref_count = 0;
5522 						pmap_add_delayed_free_list(mpte, &free, FALSE);
5523 					}
5524 				} else {
5525 					pmap_resident_count_dec(pmap, 1);
5526 #ifdef VERBOSE_PV
5527 					printf("freeing pv (%p, %p)\n",
5528 						   pmap, pv);
5529 #endif
5530 					TAILQ_REMOVE(&m->md.pv_list, pv, pv_link);
5531 					m->md.pv_gen++;
5532 					if ((m->a.flags & PGA_WRITEABLE) != 0 &&
5533 					    TAILQ_EMPTY(&m->md.pv_list) &&
5534 					    (m->flags & PG_FICTITIOUS) == 0) {
5535 						pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
5536 						if (TAILQ_EMPTY(&pvh->pv_list))
5537 							vm_page_aflag_clear(m, PGA_WRITEABLE);
5538 					}
5539 				}
5540 				pmap_unuse_pt(pmap, pv->pv_va, ptel3e, &free);
5541 				freed++;
5542 			}
5543 		}
5544 		PV_STAT(atomic_add_long(&pv_entry_frees, freed));
5545 		PV_STAT(atomic_add_int(&pv_entry_spare, freed));
5546 		PV_STAT(atomic_subtract_long(&pv_entry_count, freed));
5547 		if (allfree) {
5548 			TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
5549 			free_pv_chunk(pc);
5550 		}
5551 	}
5552 	if (lock != NULL)
5553 		rw_wunlock(lock);
5554 	pmap_invalidate_all(pmap);
5555 	PMAP_UNLOCK(pmap);
5556 	vm_page_free_pages_toq(&free, true);
5557 }
5558 
5559 void
5560 mmu_radix_remove_write(vm_page_t m)
5561 {
5562 	struct md_page *pvh;
5563 	pmap_t pmap;
5564 	struct rwlock *lock;
5565 	pv_entry_t next_pv, pv;
5566 	pml3_entry_t *l3e;
5567 	pt_entry_t oldpte, *pte;
5568 	int pvh_gen, md_gen;
5569 
5570 	CTR2(KTR_PMAP, "%s(%p)", __func__, m);
5571 	KASSERT((m->oflags & VPO_UNMANAGED) == 0,
5572 	    ("pmap_remove_write: page %p is not managed", m));
5573 	vm_page_assert_busied(m);
5574 
5575 	if (!pmap_page_is_write_mapped(m))
5576 		return;
5577 	lock = VM_PAGE_TO_PV_LIST_LOCK(m);
5578 	pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy :
5579 	    pa_to_pvh(VM_PAGE_TO_PHYS(m));
5580 retry_pv_loop:
5581 	rw_wlock(lock);
5582 	TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_link, next_pv) {
5583 		pmap = PV_PMAP(pv);
5584 		if (!PMAP_TRYLOCK(pmap)) {
5585 			pvh_gen = pvh->pv_gen;
5586 			rw_wunlock(lock);
5587 			PMAP_LOCK(pmap);
5588 			rw_wlock(lock);
5589 			if (pvh_gen != pvh->pv_gen) {
5590 				PMAP_UNLOCK(pmap);
5591 				rw_wunlock(lock);
5592 				goto retry_pv_loop;
5593 			}
5594 		}
5595 		l3e = pmap_pml3e(pmap, pv->pv_va);
5596 		if ((be64toh(*l3e) & PG_RW) != 0)
5597 			(void)pmap_demote_l3e_locked(pmap, l3e, pv->pv_va, &lock);
5598 		KASSERT(lock == VM_PAGE_TO_PV_LIST_LOCK(m),
5599 		    ("inconsistent pv lock %p %p for page %p",
5600 		    lock, VM_PAGE_TO_PV_LIST_LOCK(m), m));
5601 		PMAP_UNLOCK(pmap);
5602 	}
5603 	TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) {
5604 		pmap = PV_PMAP(pv);
5605 		if (!PMAP_TRYLOCK(pmap)) {
5606 			pvh_gen = pvh->pv_gen;
5607 			md_gen = m->md.pv_gen;
5608 			rw_wunlock(lock);
5609 			PMAP_LOCK(pmap);
5610 			rw_wlock(lock);
5611 			if (pvh_gen != pvh->pv_gen ||
5612 			    md_gen != m->md.pv_gen) {
5613 				PMAP_UNLOCK(pmap);
5614 				rw_wunlock(lock);
5615 				goto retry_pv_loop;
5616 			}
5617 		}
5618 		l3e = pmap_pml3e(pmap, pv->pv_va);
5619 		KASSERT((be64toh(*l3e) & RPTE_LEAF) == 0,
5620 		    ("pmap_remove_write: found a 2mpage in page %p's pv list",
5621 		    m));
5622 		pte = pmap_l3e_to_pte(l3e, pv->pv_va);
5623 retry:
5624 		oldpte = be64toh(*pte);
5625 		if (oldpte & PG_RW) {
5626 			if (!atomic_cmpset_long(pte, htobe64(oldpte),
5627 			    htobe64((oldpte | RPTE_EAA_R) & ~(PG_RW | PG_M))))
5628 				goto retry;
5629 			if ((oldpte & PG_M) != 0)
5630 				vm_page_dirty(m);
5631 			pmap_invalidate_page(pmap, pv->pv_va);
5632 		}
5633 		PMAP_UNLOCK(pmap);
5634 	}
5635 	rw_wunlock(lock);
5636 	vm_page_aflag_clear(m, PGA_WRITEABLE);
5637 }
5638 
5639 /*
5640  *	Clear the wired attribute from the mappings for the specified range of
5641  *	addresses in the given pmap.  Every valid mapping within that range
5642  *	must have the wired attribute set.  In contrast, invalid mappings
5643  *	cannot have the wired attribute set, so they are ignored.
5644  *
5645  *	The wired attribute of the page table entry is not a hardware
5646  *	feature, so there is no need to invalidate any TLB entries.
5647  *	Since pmap_demote_l3e() for the wired entry must never fail,
5648  *	pmap_delayed_invl_started()/finished() calls around the
5649  *	function are not needed.
5650  */
5651 void
5652 mmu_radix_unwire(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
5653 {
5654 	vm_offset_t va_next;
5655 	pml1_entry_t *l1e;
5656 	pml2_entry_t *l2e;
5657 	pml3_entry_t *l3e;
5658 	pt_entry_t *pte;
5659 
5660 	CTR4(KTR_PMAP, "%s(%p, %#x, %#x)", __func__, pmap, sva, eva);
5661 	PMAP_LOCK(pmap);
5662 	for (; sva < eva; sva = va_next) {
5663 		l1e = pmap_pml1e(pmap, sva);
5664 		if ((be64toh(*l1e) & PG_V) == 0) {
5665 			va_next = (sva + L1_PAGE_SIZE) & ~L1_PAGE_MASK;
5666 			if (va_next < sva)
5667 				va_next = eva;
5668 			continue;
5669 		}
5670 		l2e = pmap_l1e_to_l2e(l1e, sva);
5671 		if ((be64toh(*l2e) & PG_V) == 0) {
5672 			va_next = (sva + L2_PAGE_SIZE) & ~L2_PAGE_MASK;
5673 			if (va_next < sva)
5674 				va_next = eva;
5675 			continue;
5676 		}
5677 		va_next = (sva + L3_PAGE_SIZE) & ~L3_PAGE_MASK;
5678 		if (va_next < sva)
5679 			va_next = eva;
5680 		l3e = pmap_l2e_to_l3e(l2e, sva);
5681 		if ((be64toh(*l3e) & PG_V) == 0)
5682 			continue;
5683 		if ((be64toh(*l3e) & RPTE_LEAF) != 0) {
5684 			if ((be64toh(*l3e) & PG_W) == 0)
5685 				panic("pmap_unwire: pde %#jx is missing PG_W",
5686 				    (uintmax_t)(be64toh(*l3e)));
5687 
5688 			/*
5689 			 * Are we unwiring the entire large page?  If not,
5690 			 * demote the mapping and fall through.
5691 			 */
5692 			if (sva + L3_PAGE_SIZE == va_next && eva >= va_next) {
5693 				atomic_clear_long(l3e, htobe64(PG_W));
5694 				pmap->pm_stats.wired_count -= L3_PAGE_SIZE /
5695 				    PAGE_SIZE;
5696 				continue;
5697 			} else if (!pmap_demote_l3e(pmap, l3e, sva))
5698 				panic("pmap_unwire: demotion failed");
5699 		}
5700 		if (va_next > eva)
5701 			va_next = eva;
5702 		for (pte = pmap_l3e_to_pte(l3e, sva); sva != va_next; pte++,
5703 		    sva += PAGE_SIZE) {
5704 			MPASS(pte == pmap_pte(pmap, sva));
5705 			if ((be64toh(*pte) & PG_V) == 0)
5706 				continue;
5707 			if ((be64toh(*pte) & PG_W) == 0)
5708 				panic("pmap_unwire: pte %#jx is missing PG_W",
5709 				    (uintmax_t)(be64toh(*pte)));
5710 
5711 			/*
5712 			 * PG_W must be cleared atomically.  Although the pmap
5713 			 * lock synchronizes access to PG_W, another processor
5714 			 * could be setting PG_M and/or PG_A concurrently.
5715 			 */
5716 			atomic_clear_long(pte, htobe64(PG_W));
5717 			pmap->pm_stats.wired_count--;
5718 		}
5719 	}
5720 	PMAP_UNLOCK(pmap);
5721 }
5722 
5723 void
5724 mmu_radix_zero_page(vm_page_t m)
5725 {
5726 	vm_offset_t addr;
5727 
5728 	CTR2(KTR_PMAP, "%s(%p)", __func__, m);
5729 	addr = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m));
5730 	pagezero(addr);
5731 }
5732 
5733 void
5734 mmu_radix_zero_page_area(vm_page_t m, int off, int size)
5735 {
5736 	caddr_t addr;
5737 
5738 	CTR4(KTR_PMAP, "%s(%p, %d, %d)", __func__, m, off, size);
5739 	MPASS(off + size <= PAGE_SIZE);
5740 	addr = (caddr_t)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m));
5741 	memset(addr + off, 0, size);
5742 }
5743 
5744 static int
5745 mmu_radix_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *locked_pa)
5746 {
5747 	pml3_entry_t *l3ep;
5748 	pt_entry_t pte;
5749 	vm_paddr_t pa;
5750 	int val;
5751 
5752 	CTR3(KTR_PMAP, "%s(%p, %#x)", __func__, pmap, addr);
5753 	PMAP_LOCK(pmap);
5754 
5755 	l3ep = pmap_pml3e(pmap, addr);
5756 	if (l3ep != NULL && (be64toh(*l3ep) & PG_V)) {
5757 		if (be64toh(*l3ep) & RPTE_LEAF) {
5758 			pte = be64toh(*l3ep);
5759 			/* Compute the physical address of the 4KB page. */
5760 			pa = ((be64toh(*l3ep) & PG_PS_FRAME) | (addr & L3_PAGE_MASK)) &
5761 			    PG_FRAME;
5762 			val = MINCORE_PSIND(1);
5763 		} else {
5764 			/* Native endian PTE, do not pass to functions */
5765 			pte = be64toh(*pmap_l3e_to_pte(l3ep, addr));
5766 			pa = pte & PG_FRAME;
5767 			val = 0;
5768 		}
5769 	} else {
5770 		pte = 0;
5771 		pa = 0;
5772 		val = 0;
5773 	}
5774 	if ((pte & PG_V) != 0) {
5775 		val |= MINCORE_INCORE;
5776 		if ((pte & (PG_M | PG_RW)) == (PG_M | PG_RW))
5777 			val |= MINCORE_MODIFIED | MINCORE_MODIFIED_OTHER;
5778 		if ((pte & PG_A) != 0)
5779 			val |= MINCORE_REFERENCED | MINCORE_REFERENCED_OTHER;
5780 	}
5781 	if ((val & (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER)) !=
5782 	    (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER) &&
5783 	    (pte & (PG_MANAGED | PG_V)) == (PG_MANAGED | PG_V)) {
5784 		*locked_pa = pa;
5785 	}
5786 	PMAP_UNLOCK(pmap);
5787 	return (val);
5788 }
5789 
5790 void
5791 mmu_radix_activate(struct thread *td)
5792 {
5793 	pmap_t pmap;
5794 	uint32_t curpid;
5795 
5796 	CTR2(KTR_PMAP, "%s(%p)", __func__, td);
5797 	critical_enter();
5798 	pmap = vmspace_pmap(td->td_proc->p_vmspace);
5799 	curpid = mfspr(SPR_PID);
5800 	if (pmap->pm_pid > isa3_base_pid &&
5801 		curpid != pmap->pm_pid) {
5802 		mmu_radix_pid_set(pmap);
5803 	}
5804 	critical_exit();
5805 }
5806 
5807 /*
5808  *	Increase the starting virtual address of the given mapping if a
5809  *	different alignment might result in more superpage mappings.
5810  */
5811 void
5812 mmu_radix_align_superpage(vm_object_t object, vm_ooffset_t offset,
5813     vm_offset_t *addr, vm_size_t size)
5814 {
5815 
5816 	CTR5(KTR_PMAP, "%s(%p, %#x, %p, %#x)", __func__, object, offset, addr,
5817 	    size);
5818 	vm_offset_t superpage_offset;
5819 
5820 	if (size < L3_PAGE_SIZE)
5821 		return;
5822 	if (object != NULL && (object->flags & OBJ_COLORED) != 0)
5823 		offset += ptoa(object->pg_color);
5824 	superpage_offset = offset & L3_PAGE_MASK;
5825 	if (size - ((L3_PAGE_SIZE - superpage_offset) & L3_PAGE_MASK) < L3_PAGE_SIZE ||
5826 	    (*addr & L3_PAGE_MASK) == superpage_offset)
5827 		return;
5828 	if ((*addr & L3_PAGE_MASK) < superpage_offset)
5829 		*addr = (*addr & ~L3_PAGE_MASK) + superpage_offset;
5830 	else
5831 		*addr = ((*addr + L3_PAGE_MASK) & ~L3_PAGE_MASK) + superpage_offset;
5832 }
5833 
5834 static void *
5835 mmu_radix_mapdev_attr(vm_paddr_t pa, vm_size_t size, vm_memattr_t attr)
5836 {
5837 	vm_offset_t va, tmpva, ppa, offset;
5838 
5839 	ppa = trunc_page(pa);
5840 	offset = pa & PAGE_MASK;
5841 	size = roundup2(offset + size, PAGE_SIZE);
5842 	if (pa < powerpc_ptob(Maxmem))
5843 		panic("bad pa: %#lx less than Maxmem %#lx\n",
5844 			  pa, powerpc_ptob(Maxmem));
5845 	va = kva_alloc(size);
5846 	if (bootverbose)
5847 		printf("%s(%#lx, %lu, %d)\n", __func__, pa, size, attr);
5848 	KASSERT(size > 0, ("%s(%#lx, %lu, %d)", __func__, pa, size, attr));
5849 
5850 	if (!va)
5851 		panic("%s: Couldn't alloc kernel virtual memory", __func__);
5852 
5853 	for (tmpva = va; size > 0;) {
5854 		mmu_radix_kenter_attr(tmpva, ppa, attr);
5855 		size -= PAGE_SIZE;
5856 		tmpva += PAGE_SIZE;
5857 		ppa += PAGE_SIZE;
5858 	}
5859 	ptesync();
5860 
5861 	return ((void *)(va + offset));
5862 }
5863 
5864 static void *
5865 mmu_radix_mapdev(vm_paddr_t pa, vm_size_t size)
5866 {
5867 
5868 	CTR3(KTR_PMAP, "%s(%#x, %#x)", __func__, pa, size);
5869 
5870 	return (mmu_radix_mapdev_attr(pa, size, VM_MEMATTR_DEFAULT));
5871 }
5872 
5873 void
5874 mmu_radix_page_set_memattr(vm_page_t m, vm_memattr_t ma)
5875 {
5876 
5877 	CTR3(KTR_PMAP, "%s(%p, %#x)", __func__, m, ma);
5878 	m->md.mdpg_cache_attrs = ma;
5879 
5880 	/*
5881 	 * If "m" is a normal page, update its direct mapping.  This update
5882 	 * can be relied upon to perform any cache operations that are
5883 	 * required for data coherence.
5884 	 */
5885 	if ((m->flags & PG_FICTITIOUS) == 0 &&
5886 	    mmu_radix_change_attr(PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)),
5887 	    PAGE_SIZE, m->md.mdpg_cache_attrs))
5888 		panic("memory attribute change on the direct map failed");
5889 }
5890 
5891 static void
5892 mmu_radix_unmapdev(vm_offset_t va, vm_size_t size)
5893 {
5894 	vm_offset_t offset;
5895 
5896 	CTR3(KTR_PMAP, "%s(%#x, %#x)", __func__, va, size);
5897 	/* If we gave a direct map region in pmap_mapdev, do nothing */
5898 	if (va >= DMAP_MIN_ADDRESS && va < DMAP_MAX_ADDRESS)
5899 		return;
5900 
5901 	offset = va & PAGE_MASK;
5902 	size = round_page(offset + size);
5903 	va = trunc_page(va);
5904 
5905 	if (pmap_initialized) {
5906 		mmu_radix_qremove(va, atop(size));
5907 		kva_free(va, size);
5908 	}
5909 }
5910 
5911 static __inline void
5912 pmap_pte_attr(pt_entry_t *pte, uint64_t cache_bits, uint64_t mask)
5913 {
5914 	uint64_t opte, npte;
5915 
5916 	/*
5917 	 * The cache mode bits are all in the low 32-bits of the
5918 	 * PTE, so we can just spin on updating the low 32-bits.
5919 	 */
5920 	do {
5921 		opte = be64toh(*pte);
5922 		npte = opte & ~mask;
5923 		npte |= cache_bits;
5924 	} while (npte != opte && !atomic_cmpset_long(pte, htobe64(opte), htobe64(npte)));
5925 }
5926 
5927 /*
5928  * Tries to demote a 1GB page mapping.
5929  */
5930 static boolean_t
5931 pmap_demote_l2e(pmap_t pmap, pml2_entry_t *l2e, vm_offset_t va)
5932 {
5933 	pml2_entry_t oldpdpe;
5934 	pml3_entry_t *firstpde, newpde, *pde;
5935 	vm_paddr_t pdpgpa;
5936 	vm_page_t pdpg;
5937 
5938 	PMAP_LOCK_ASSERT(pmap, MA_OWNED);
5939 	oldpdpe = be64toh(*l2e);
5940 	KASSERT((oldpdpe & (RPTE_LEAF | PG_V)) == (RPTE_LEAF | PG_V),
5941 	    ("pmap_demote_pdpe: oldpdpe is missing PG_PS and/or PG_V"));
5942 	pdpg = vm_page_alloc_noobj(VM_ALLOC_INTERRUPT | VM_ALLOC_WIRED);
5943 	if (pdpg == NULL) {
5944 		CTR2(KTR_PMAP, "pmap_demote_pdpe: failure for va %#lx"
5945 		    " in pmap %p", va, pmap);
5946 		return (FALSE);
5947 	}
5948 	pdpg->pindex = va >> L2_PAGE_SIZE_SHIFT;
5949 	pdpgpa = VM_PAGE_TO_PHYS(pdpg);
5950 	firstpde = (pml3_entry_t *)PHYS_TO_DMAP(pdpgpa);
5951 	KASSERT((oldpdpe & PG_A) != 0,
5952 	    ("pmap_demote_pdpe: oldpdpe is missing PG_A"));
5953 	KASSERT((oldpdpe & (PG_M | PG_RW)) != PG_RW,
5954 	    ("pmap_demote_pdpe: oldpdpe is missing PG_M"));
5955 	newpde = oldpdpe;
5956 
5957 	/*
5958 	 * Initialize the page directory page.
5959 	 */
5960 	for (pde = firstpde; pde < firstpde + NPDEPG; pde++) {
5961 		*pde = htobe64(newpde);
5962 		newpde += L3_PAGE_SIZE;
5963 	}
5964 
5965 	/*
5966 	 * Demote the mapping.
5967 	 */
5968 	pde_store(l2e, pdpgpa);
5969 
5970 	/*
5971 	 * Flush PWC --- XXX revisit
5972 	 */
5973 	pmap_invalidate_all(pmap);
5974 
5975 	counter_u64_add(pmap_l2e_demotions, 1);
5976 	CTR2(KTR_PMAP, "pmap_demote_pdpe: success for va %#lx"
5977 	    " in pmap %p", va, pmap);
5978 	return (TRUE);
5979 }
5980 
5981 vm_paddr_t
5982 mmu_radix_kextract(vm_offset_t va)
5983 {
5984 	pml3_entry_t l3e;
5985 	vm_paddr_t pa;
5986 
5987 	CTR2(KTR_PMAP, "%s(%#x)", __func__, va);
5988 	if (va >= DMAP_MIN_ADDRESS && va < DMAP_MAX_ADDRESS) {
5989 		pa = DMAP_TO_PHYS(va);
5990 	} else {
5991 		/* Big-endian PTE on stack */
5992 		l3e = *pmap_pml3e(kernel_pmap, va);
5993 		if (be64toh(l3e) & RPTE_LEAF) {
5994 			pa = (be64toh(l3e) & PG_PS_FRAME) | (va & L3_PAGE_MASK);
5995 			pa |= (va & L3_PAGE_MASK);
5996 		} else {
5997 			/*
5998 			 * Beware of a concurrent promotion that changes the
5999 			 * PDE at this point!  For example, vtopte() must not
6000 			 * be used to access the PTE because it would use the
6001 			 * new PDE.  It is, however, safe to use the old PDE
6002 			 * because the page table page is preserved by the
6003 			 * promotion.
6004 			 */
6005 			pa = be64toh(*pmap_l3e_to_pte(&l3e, va));
6006 			pa = (pa & PG_FRAME) | (va & PAGE_MASK);
6007 			pa |= (va & PAGE_MASK);
6008 		}
6009 	}
6010 	return (pa);
6011 }
6012 
6013 static pt_entry_t
6014 mmu_radix_calc_wimg(vm_paddr_t pa, vm_memattr_t ma)
6015 {
6016 
6017 	if (ma != VM_MEMATTR_DEFAULT) {
6018 		return pmap_cache_bits(ma);
6019 	}
6020 
6021 	/*
6022 	 * Assume the page is cache inhibited and access is guarded unless
6023 	 * it's in our available memory array.
6024 	 */
6025 	for (int i = 0; i < pregions_sz; i++) {
6026 		if ((pa >= pregions[i].mr_start) &&
6027 		    (pa < (pregions[i].mr_start + pregions[i].mr_size)))
6028 			return (RPTE_ATTR_MEM);
6029 	}
6030 	return (RPTE_ATTR_GUARDEDIO);
6031 }
6032 
6033 static void
6034 mmu_radix_kenter_attr(vm_offset_t va, vm_paddr_t pa, vm_memattr_t ma)
6035 {
6036 	pt_entry_t *pte, pteval;
6037 	uint64_t cache_bits;
6038 
6039 	pte = kvtopte(va);
6040 	MPASS(pte != NULL);
6041 	pteval = pa | RPTE_EAA_R | RPTE_EAA_W | RPTE_EAA_P | PG_M | PG_A;
6042 	cache_bits = mmu_radix_calc_wimg(pa, ma);
6043 	pte_store(pte, pteval | cache_bits);
6044 }
6045 
6046 void
6047 mmu_radix_kremove(vm_offset_t va)
6048 {
6049 	pt_entry_t *pte;
6050 
6051 	CTR2(KTR_PMAP, "%s(%#x)", __func__, va);
6052 
6053 	pte = kvtopte(va);
6054 	pte_clear(pte);
6055 }
6056 
6057 int
6058 mmu_radix_decode_kernel_ptr(vm_offset_t addr,
6059     int *is_user, vm_offset_t *decoded)
6060 {
6061 
6062 	CTR2(KTR_PMAP, "%s(%#jx)", __func__, (uintmax_t)addr);
6063 	*decoded = addr;
6064 	*is_user = (addr < VM_MAXUSER_ADDRESS);
6065 	return (0);
6066 }
6067 
6068 static boolean_t
6069 mmu_radix_dev_direct_mapped(vm_paddr_t pa, vm_size_t size)
6070 {
6071 
6072 	CTR3(KTR_PMAP, "%s(%#x, %#x)", __func__, pa, size);
6073 	return (mem_valid(pa, size));
6074 }
6075 
6076 static void
6077 mmu_radix_scan_init()
6078 {
6079 
6080 	CTR1(KTR_PMAP, "%s()", __func__);
6081 	UNIMPLEMENTED();
6082 }
6083 
6084 static void
6085 mmu_radix_dumpsys_map(vm_paddr_t pa, size_t sz,
6086 	void **va)
6087 {
6088 	CTR4(KTR_PMAP, "%s(%#jx, %#zx, %p)", __func__, (uintmax_t)pa, sz, va);
6089 	UNIMPLEMENTED();
6090 }
6091 
6092 vm_offset_t
6093 mmu_radix_quick_enter_page(vm_page_t m)
6094 {
6095 	vm_paddr_t paddr;
6096 
6097 	CTR2(KTR_PMAP, "%s(%p)", __func__, m);
6098 	paddr = VM_PAGE_TO_PHYS(m);
6099 	return (PHYS_TO_DMAP(paddr));
6100 }
6101 
6102 void
6103 mmu_radix_quick_remove_page(vm_offset_t addr __unused)
6104 {
6105 	/* no work to do here */
6106 	CTR2(KTR_PMAP, "%s(%#x)", __func__, addr);
6107 }
6108 
6109 static void
6110 pmap_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva)
6111 {
6112 	cpu_flush_dcache((void *)sva, eva - sva);
6113 }
6114 
6115 int
6116 mmu_radix_change_attr(vm_offset_t va, vm_size_t size,
6117     vm_memattr_t mode)
6118 {
6119 	int error;
6120 
6121 	CTR4(KTR_PMAP, "%s(%#x, %#zx, %d)", __func__, va, size, mode);
6122 	PMAP_LOCK(kernel_pmap);
6123 	error = pmap_change_attr_locked(va, size, mode, true);
6124 	PMAP_UNLOCK(kernel_pmap);
6125 	return (error);
6126 }
6127 
6128 static int
6129 pmap_change_attr_locked(vm_offset_t va, vm_size_t size, int mode, bool flush)
6130 {
6131 	vm_offset_t base, offset, tmpva;
6132 	vm_paddr_t pa_start, pa_end, pa_end1;
6133 	pml2_entry_t *l2e;
6134 	pml3_entry_t *l3e;
6135 	pt_entry_t *pte;
6136 	int cache_bits, error;
6137 	boolean_t changed;
6138 
6139 	PMAP_LOCK_ASSERT(kernel_pmap, MA_OWNED);
6140 	base = trunc_page(va);
6141 	offset = va & PAGE_MASK;
6142 	size = round_page(offset + size);
6143 
6144 	/*
6145 	 * Only supported on kernel virtual addresses, including the direct
6146 	 * map but excluding the recursive map.
6147 	 */
6148 	if (base < DMAP_MIN_ADDRESS)
6149 		return (EINVAL);
6150 
6151 	cache_bits = pmap_cache_bits(mode);
6152 	changed = FALSE;
6153 
6154 	/*
6155 	 * Pages that aren't mapped aren't supported.  Also break down 2MB pages
6156 	 * into 4KB pages if required.
6157 	 */
6158 	for (tmpva = base; tmpva < base + size; ) {
6159 		l2e = pmap_pml2e(kernel_pmap, tmpva);
6160 		if (l2e == NULL || *l2e == 0)
6161 			return (EINVAL);
6162 		if (be64toh(*l2e) & RPTE_LEAF) {
6163 			/*
6164 			 * If the current 1GB page already has the required
6165 			 * memory type, then we need not demote this page. Just
6166 			 * increment tmpva to the next 1GB page frame.
6167 			 */
6168 			if ((be64toh(*l2e) & RPTE_ATTR_MASK) == cache_bits) {
6169 				tmpva = trunc_1gpage(tmpva) + L2_PAGE_SIZE;
6170 				continue;
6171 			}
6172 
6173 			/*
6174 			 * If the current offset aligns with a 1GB page frame
6175 			 * and there is at least 1GB left within the range, then
6176 			 * we need not break down this page into 2MB pages.
6177 			 */
6178 			if ((tmpva & L2_PAGE_MASK) == 0 &&
6179 			    tmpva + L2_PAGE_MASK < base + size) {
6180 				tmpva += L2_PAGE_MASK;
6181 				continue;
6182 			}
6183 			if (!pmap_demote_l2e(kernel_pmap, l2e, tmpva))
6184 				return (ENOMEM);
6185 		}
6186 		l3e = pmap_l2e_to_l3e(l2e, tmpva);
6187 		KASSERT(l3e != NULL, ("no l3e entry for %#lx in %p\n",
6188 		    tmpva, l2e));
6189 		if (*l3e == 0)
6190 			return (EINVAL);
6191 		if (be64toh(*l3e) & RPTE_LEAF) {
6192 			/*
6193 			 * If the current 2MB page already has the required
6194 			 * memory type, then we need not demote this page. Just
6195 			 * increment tmpva to the next 2MB page frame.
6196 			 */
6197 			if ((be64toh(*l3e) & RPTE_ATTR_MASK) == cache_bits) {
6198 				tmpva = trunc_2mpage(tmpva) + L3_PAGE_SIZE;
6199 				continue;
6200 			}
6201 
6202 			/*
6203 			 * If the current offset aligns with a 2MB page frame
6204 			 * and there is at least 2MB left within the range, then
6205 			 * we need not break down this page into 4KB pages.
6206 			 */
6207 			if ((tmpva & L3_PAGE_MASK) == 0 &&
6208 			    tmpva + L3_PAGE_MASK < base + size) {
6209 				tmpva += L3_PAGE_SIZE;
6210 				continue;
6211 			}
6212 			if (!pmap_demote_l3e(kernel_pmap, l3e, tmpva))
6213 				return (ENOMEM);
6214 		}
6215 		pte = pmap_l3e_to_pte(l3e, tmpva);
6216 		if (*pte == 0)
6217 			return (EINVAL);
6218 		tmpva += PAGE_SIZE;
6219 	}
6220 	error = 0;
6221 
6222 	/*
6223 	 * Ok, all the pages exist, so run through them updating their
6224 	 * cache mode if required.
6225 	 */
6226 	pa_start = pa_end = 0;
6227 	for (tmpva = base; tmpva < base + size; ) {
6228 		l2e = pmap_pml2e(kernel_pmap, tmpva);
6229 		if (be64toh(*l2e) & RPTE_LEAF) {
6230 			if ((be64toh(*l2e) & RPTE_ATTR_MASK) != cache_bits) {
6231 				pmap_pte_attr(l2e, cache_bits,
6232 				    RPTE_ATTR_MASK);
6233 				changed = TRUE;
6234 			}
6235 			if (tmpva >= VM_MIN_KERNEL_ADDRESS &&
6236 			    (*l2e & PG_PS_FRAME) < dmaplimit) {
6237 				if (pa_start == pa_end) {
6238 					/* Start physical address run. */
6239 					pa_start = be64toh(*l2e) & PG_PS_FRAME;
6240 					pa_end = pa_start + L2_PAGE_SIZE;
6241 				} else if (pa_end == (be64toh(*l2e) & PG_PS_FRAME))
6242 					pa_end += L2_PAGE_SIZE;
6243 				else {
6244 					/* Run ended, update direct map. */
6245 					error = pmap_change_attr_locked(
6246 					    PHYS_TO_DMAP(pa_start),
6247 					    pa_end - pa_start, mode, flush);
6248 					if (error != 0)
6249 						break;
6250 					/* Start physical address run. */
6251 					pa_start = be64toh(*l2e) & PG_PS_FRAME;
6252 					pa_end = pa_start + L2_PAGE_SIZE;
6253 				}
6254 			}
6255 			tmpva = trunc_1gpage(tmpva) + L2_PAGE_SIZE;
6256 			continue;
6257 		}
6258 		l3e = pmap_l2e_to_l3e(l2e, tmpva);
6259 		if (be64toh(*l3e) & RPTE_LEAF) {
6260 			if ((be64toh(*l3e) & RPTE_ATTR_MASK) != cache_bits) {
6261 				pmap_pte_attr(l3e, cache_bits,
6262 				    RPTE_ATTR_MASK);
6263 				changed = TRUE;
6264 			}
6265 			if (tmpva >= VM_MIN_KERNEL_ADDRESS &&
6266 			    (be64toh(*l3e) & PG_PS_FRAME) < dmaplimit) {
6267 				if (pa_start == pa_end) {
6268 					/* Start physical address run. */
6269 					pa_start = be64toh(*l3e) & PG_PS_FRAME;
6270 					pa_end = pa_start + L3_PAGE_SIZE;
6271 				} else if (pa_end == (be64toh(*l3e) & PG_PS_FRAME))
6272 					pa_end += L3_PAGE_SIZE;
6273 				else {
6274 					/* Run ended, update direct map. */
6275 					error = pmap_change_attr_locked(
6276 					    PHYS_TO_DMAP(pa_start),
6277 					    pa_end - pa_start, mode, flush);
6278 					if (error != 0)
6279 						break;
6280 					/* Start physical address run. */
6281 					pa_start = be64toh(*l3e) & PG_PS_FRAME;
6282 					pa_end = pa_start + L3_PAGE_SIZE;
6283 				}
6284 			}
6285 			tmpva = trunc_2mpage(tmpva) + L3_PAGE_SIZE;
6286 		} else {
6287 			pte = pmap_l3e_to_pte(l3e, tmpva);
6288 			if ((be64toh(*pte) & RPTE_ATTR_MASK) != cache_bits) {
6289 				pmap_pte_attr(pte, cache_bits,
6290 				    RPTE_ATTR_MASK);
6291 				changed = TRUE;
6292 			}
6293 			if (tmpva >= VM_MIN_KERNEL_ADDRESS &&
6294 			    (be64toh(*pte) & PG_FRAME) < dmaplimit) {
6295 				if (pa_start == pa_end) {
6296 					/* Start physical address run. */
6297 					pa_start = be64toh(*pte) & PG_FRAME;
6298 					pa_end = pa_start + PAGE_SIZE;
6299 				} else if (pa_end == (be64toh(*pte) & PG_FRAME))
6300 					pa_end += PAGE_SIZE;
6301 				else {
6302 					/* Run ended, update direct map. */
6303 					error = pmap_change_attr_locked(
6304 					    PHYS_TO_DMAP(pa_start),
6305 					    pa_end - pa_start, mode, flush);
6306 					if (error != 0)
6307 						break;
6308 					/* Start physical address run. */
6309 					pa_start = be64toh(*pte) & PG_FRAME;
6310 					pa_end = pa_start + PAGE_SIZE;
6311 				}
6312 			}
6313 			tmpva += PAGE_SIZE;
6314 		}
6315 	}
6316 	if (error == 0 && pa_start != pa_end && pa_start < dmaplimit) {
6317 		pa_end1 = MIN(pa_end, dmaplimit);
6318 		if (pa_start != pa_end1)
6319 			error = pmap_change_attr_locked(PHYS_TO_DMAP(pa_start),
6320 			    pa_end1 - pa_start, mode, flush);
6321 	}
6322 
6323 	/*
6324 	 * Flush CPU caches if required to make sure any data isn't cached that
6325 	 * shouldn't be, etc.
6326 	 */
6327 	if (changed) {
6328 		pmap_invalidate_all(kernel_pmap);
6329 
6330 		if (flush)
6331 			pmap_invalidate_cache_range(base, tmpva);
6332 	}
6333 	return (error);
6334 }
6335 
6336 /*
6337  * Allocate physical memory for the vm_page array and map it into KVA,
6338  * attempting to back the vm_pages with domain-local memory.
6339  */
6340 void
6341 mmu_radix_page_array_startup(long pages)
6342 {
6343 #ifdef notyet
6344 	pml2_entry_t *l2e;
6345 	pml3_entry_t *pde;
6346 	pml3_entry_t newl3;
6347 	vm_offset_t va;
6348 	long pfn;
6349 	int domain, i;
6350 #endif
6351 	vm_paddr_t pa;
6352 	vm_offset_t start, end;
6353 
6354 	vm_page_array_size = pages;
6355 
6356 	start = VM_MIN_KERNEL_ADDRESS;
6357 	end = start + pages * sizeof(struct vm_page);
6358 
6359 	pa = vm_phys_early_alloc(0, end - start);
6360 
6361 	start = mmu_radix_map(&start, pa, end - start, VM_MEMATTR_DEFAULT);
6362 #ifdef notyet
6363 	/* TODO: NUMA vm_page_array.  Blocked out until then (copied from amd64). */
6364 	for (va = start; va < end; va += L3_PAGE_SIZE) {
6365 		pfn = first_page + (va - start) / sizeof(struct vm_page);
6366 		domain = vm_phys_domain(ptoa(pfn));
6367 		l2e = pmap_pml2e(kernel_pmap, va);
6368 		if ((be64toh(*l2e) & PG_V) == 0) {
6369 			pa = vm_phys_early_alloc(domain, PAGE_SIZE);
6370 			dump_add_page(pa);
6371 			pagezero(PHYS_TO_DMAP(pa));
6372 			pde_store(l2e, (pml2_entry_t)pa);
6373 		}
6374 		pde = pmap_l2e_to_l3e(l2e, va);
6375 		if ((be64toh(*pde) & PG_V) != 0)
6376 			panic("Unexpected pde %p", pde);
6377 		pa = vm_phys_early_alloc(domain, L3_PAGE_SIZE);
6378 		for (i = 0; i < NPDEPG; i++)
6379 			dump_add_page(pa + i * PAGE_SIZE);
6380 		newl3 = (pml3_entry_t)(pa | RPTE_EAA_P | RPTE_EAA_R | RPTE_EAA_W);
6381 		pte_store(pde, newl3);
6382 	}
6383 #endif
6384 	vm_page_array = (vm_page_t)start;
6385 }
6386 
6387 #ifdef DDB
6388 #include <sys/kdb.h>
6389 #include <ddb/ddb.h>
6390 
6391 static void
6392 pmap_pte_walk(pml1_entry_t *l1, vm_offset_t va)
6393 {
6394 	pml1_entry_t *l1e;
6395 	pml2_entry_t *l2e;
6396 	pml3_entry_t *l3e;
6397 	pt_entry_t *pte;
6398 
6399 	l1e = &l1[pmap_pml1e_index(va)];
6400 	db_printf("VA %#016lx l1e %#016lx", va, be64toh(*l1e));
6401 	if ((be64toh(*l1e) & PG_V) == 0) {
6402 		db_printf("\n");
6403 		return;
6404 	}
6405 	l2e = pmap_l1e_to_l2e(l1e, va);
6406 	db_printf(" l2e %#016lx", be64toh(*l2e));
6407 	if ((be64toh(*l2e) & PG_V) == 0 || (be64toh(*l2e) & RPTE_LEAF) != 0) {
6408 		db_printf("\n");
6409 		return;
6410 	}
6411 	l3e = pmap_l2e_to_l3e(l2e, va);
6412 	db_printf(" l3e %#016lx", be64toh(*l3e));
6413 	if ((be64toh(*l3e) & PG_V) == 0 || (be64toh(*l3e) & RPTE_LEAF) != 0) {
6414 		db_printf("\n");
6415 		return;
6416 	}
6417 	pte = pmap_l3e_to_pte(l3e, va);
6418 	db_printf(" pte %#016lx\n", be64toh(*pte));
6419 }
6420 
6421 void
6422 pmap_page_print_mappings(vm_page_t m)
6423 {
6424 	pmap_t pmap;
6425 	pv_entry_t pv;
6426 
6427 	db_printf("page %p(%lx)\n", m, m->phys_addr);
6428 	/* need to elide locks if running in ddb */
6429 	TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) {
6430 		db_printf("pv: %p ", pv);
6431 		db_printf("va: %#016lx ", pv->pv_va);
6432 		pmap = PV_PMAP(pv);
6433 		db_printf("pmap %p  ", pmap);
6434 		if (pmap != NULL) {
6435 			db_printf("asid: %lu\n", pmap->pm_pid);
6436 			pmap_pte_walk(pmap->pm_pml1, pv->pv_va);
6437 		}
6438 	}
6439 }
6440 
6441 DB_SHOW_COMMAND(pte, pmap_print_pte)
6442 {
6443 	vm_offset_t va;
6444 	pmap_t pmap;
6445 
6446 	if (!have_addr) {
6447 		db_printf("show pte addr\n");
6448 		return;
6449 	}
6450 	va = (vm_offset_t)addr;
6451 
6452 	if (va >= DMAP_MIN_ADDRESS)
6453 		pmap = kernel_pmap;
6454 	else if (kdb_thread != NULL)
6455 		pmap = vmspace_pmap(kdb_thread->td_proc->p_vmspace);
6456 	else
6457 		pmap = vmspace_pmap(curthread->td_proc->p_vmspace);
6458 
6459 	pmap_pte_walk(pmap->pm_pml1, va);
6460 }
6461 
6462 #endif
6463