1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2008-2015 Nathan Whitehorn
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 *
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 */
28
29 #include <sys/cdefs.h>
30 /*
31 * Manages physical address maps.
32 *
33 * Since the information managed by this module is also stored by the
34 * logical address mapping module, this module may throw away valid virtual
35 * to physical mappings at almost any time. However, invalidations of
36 * mappings must be done as requested.
37 *
38 * In order to cope with hardware architectures which make virtual to
39 * physical map invalidates expensive, this module may delay invalidate
40 * reduced protection operations until such time as they are actually
41 * necessary. This module is given full information as to which processors
42 * are currently using which maps, and to when physical maps must be made
43 * correct.
44 */
45
46 #include "opt_kstack_pages.h"
47
48 #include <sys/param.h>
49 #include <sys/kernel.h>
50 #include <sys/conf.h>
51 #include <sys/queue.h>
52 #include <sys/cpuset.h>
53 #include <sys/kerneldump.h>
54 #include <sys/ktr.h>
55 #include <sys/lock.h>
56 #include <sys/msgbuf.h>
57 #include <sys/malloc.h>
58 #include <sys/mman.h>
59 #include <sys/mutex.h>
60 #include <sys/proc.h>
61 #include <sys/rwlock.h>
62 #include <sys/sched.h>
63 #include <sys/sysctl.h>
64 #include <sys/systm.h>
65 #include <sys/vmmeter.h>
66 #include <sys/smp.h>
67 #include <sys/reboot.h>
68
69 #include <sys/kdb.h>
70
71 #include <dev/ofw/openfirm.h>
72
73 #include <vm/vm.h>
74 #include <vm/pmap.h>
75 #include <vm/vm_param.h>
76 #include <vm/vm_kern.h>
77 #include <vm/vm_page.h>
78 #include <vm/vm_phys.h>
79 #include <vm/vm_map.h>
80 #include <vm/vm_object.h>
81 #include <vm/vm_extern.h>
82 #include <vm/vm_pageout.h>
83 #include <vm/vm_dumpset.h>
84 #include <vm/vm_reserv.h>
85 #include <vm/uma.h>
86
87 #include <machine/_inttypes.h>
88 #include <machine/cpu.h>
89 #include <machine/ifunc.h>
90 #include <machine/platform.h>
91 #include <machine/frame.h>
92 #include <machine/md_var.h>
93 #include <machine/psl.h>
94 #include <machine/bat.h>
95 #include <machine/hid.h>
96 #include <machine/pte.h>
97 #include <machine/sr.h>
98 #include <machine/trap.h>
99 #include <machine/mmuvar.h>
100
101 #include "mmu_oea64.h"
102
103 void moea64_release_vsid(uint64_t vsid);
104 uintptr_t moea64_get_unique_vsid(void);
105
106 #define DISABLE_TRANS(msr) msr = mfmsr(); mtmsr(msr & ~PSL_DR)
107 #define ENABLE_TRANS(msr) mtmsr(msr)
108
109 #define VSID_MAKE(sr, hash) ((sr) | (((hash) & 0xfffff) << 4))
110 #define VSID_TO_HASH(vsid) (((vsid) >> 4) & 0xfffff)
111 #define VSID_HASH_MASK 0x0000007fffffffffULL
112
113 /*
114 * Locking semantics:
115 *
116 * There are two locks of interest: the page locks and the pmap locks, which
117 * protect their individual PVO lists and are locked in that order. The contents
118 * of all PVO entries are protected by the locks of their respective pmaps.
119 * The pmap of any PVO is guaranteed not to change so long as the PVO is linked
120 * into any list.
121 *
122 */
123
124 #define PV_LOCK_COUNT PA_LOCK_COUNT
125 static struct mtx_padalign pv_lock[PV_LOCK_COUNT];
126
127 /*
128 * Cheap NUMA-izing of the pv locks, to reduce contention across domains.
129 * NUMA domains on POWER9 appear to be indexed as sparse memory spaces, with the
130 * index at (N << 45).
131 */
132 #ifdef __powerpc64__
133 #define PV_LOCK_IDX(pa) ((pa_index(pa) * (((pa) >> 45) + 1)) % PV_LOCK_COUNT)
134 #else
135 #define PV_LOCK_IDX(pa) (pa_index(pa) % PV_LOCK_COUNT)
136 #endif
137 #define PV_LOCKPTR(pa) ((struct mtx *)(&pv_lock[PV_LOCK_IDX(pa)]))
138 #define PV_LOCK(pa) mtx_lock(PV_LOCKPTR(pa))
139 #define PV_UNLOCK(pa) mtx_unlock(PV_LOCKPTR(pa))
140 #define PV_LOCKASSERT(pa) mtx_assert(PV_LOCKPTR(pa), MA_OWNED)
141 #define PV_PAGE_LOCK(m) PV_LOCK(VM_PAGE_TO_PHYS(m))
142 #define PV_PAGE_UNLOCK(m) PV_UNLOCK(VM_PAGE_TO_PHYS(m))
143 #define PV_PAGE_LOCKASSERT(m) PV_LOCKASSERT(VM_PAGE_TO_PHYS(m))
144
145 /* Superpage PV lock */
146
147 #define PV_LOCK_SIZE (1<<PDRSHIFT)
148
149 static __always_inline void
moea64_sp_pv_lock(vm_paddr_t pa)150 moea64_sp_pv_lock(vm_paddr_t pa)
151 {
152 vm_paddr_t pa_end;
153
154 /* Note: breaking when pa_end is reached to avoid overflows */
155 pa_end = pa + (HPT_SP_SIZE - PV_LOCK_SIZE);
156 for (;;) {
157 mtx_lock_flags(PV_LOCKPTR(pa), MTX_DUPOK);
158 if (pa == pa_end)
159 break;
160 pa += PV_LOCK_SIZE;
161 }
162 }
163
164 static __always_inline void
moea64_sp_pv_unlock(vm_paddr_t pa)165 moea64_sp_pv_unlock(vm_paddr_t pa)
166 {
167 vm_paddr_t pa_end;
168
169 /* Note: breaking when pa_end is reached to avoid overflows */
170 pa_end = pa;
171 pa += HPT_SP_SIZE - PV_LOCK_SIZE;
172 for (;;) {
173 mtx_unlock_flags(PV_LOCKPTR(pa), MTX_DUPOK);
174 if (pa == pa_end)
175 break;
176 pa -= PV_LOCK_SIZE;
177 }
178 }
179
180 #define SP_PV_LOCK_ALIGNED(pa) moea64_sp_pv_lock(pa)
181 #define SP_PV_UNLOCK_ALIGNED(pa) moea64_sp_pv_unlock(pa)
182 #define SP_PV_LOCK(pa) moea64_sp_pv_lock((pa) & ~HPT_SP_MASK)
183 #define SP_PV_UNLOCK(pa) moea64_sp_pv_unlock((pa) & ~HPT_SP_MASK)
184 #define SP_PV_PAGE_LOCK(m) SP_PV_LOCK(VM_PAGE_TO_PHYS(m))
185 #define SP_PV_PAGE_UNLOCK(m) SP_PV_UNLOCK(VM_PAGE_TO_PHYS(m))
186
187 struct ofw_map {
188 cell_t om_va;
189 cell_t om_len;
190 uint64_t om_pa;
191 cell_t om_mode;
192 };
193
194 extern unsigned char _etext[];
195 extern unsigned char _end[];
196
197 extern void *slbtrap, *slbtrapend;
198
199 /*
200 * Map of physical memory regions.
201 */
202 static struct mem_region *regions;
203 static struct mem_region *pregions;
204 static struct numa_mem_region *numa_pregions;
205 static u_int phys_avail_count;
206 static int regions_sz, pregions_sz, numapregions_sz;
207
208 extern void bs_remap_earlyboot(void);
209
210 /*
211 * Lock for the SLB tables.
212 */
213 struct mtx moea64_slb_mutex;
214
215 /*
216 * PTEG data.
217 */
218 u_long moea64_pteg_count;
219 u_long moea64_pteg_mask;
220
221 /*
222 * PVO data.
223 */
224
225 uma_zone_t moea64_pvo_zone; /* zone for pvo entries */
226
227 static struct pvo_entry *moea64_bpvo_pool;
228 static int moea64_bpvo_pool_index = 0;
229 static int moea64_bpvo_pool_size = 0;
230 SYSCTL_INT(_machdep, OID_AUTO, moea64_allocated_bpvo_entries, CTLFLAG_RD,
231 &moea64_bpvo_pool_index, 0, "");
232
233 #define BPVO_POOL_SIZE 327680 /* Sensible historical default value */
234 #define BPVO_POOL_EXPANSION_FACTOR 3
235 #define VSID_NBPW (sizeof(u_int32_t) * 8)
236 #ifdef __powerpc64__
237 #define NVSIDS (NPMAPS * 16)
238 #define VSID_HASHMASK 0xffffffffUL
239 #else
240 #define NVSIDS NPMAPS
241 #define VSID_HASHMASK 0xfffffUL
242 #endif
243 static u_int moea64_vsid_bitmap[NVSIDS / VSID_NBPW];
244
245 static bool moea64_initialized = false;
246
247 #ifdef MOEA64_STATS
248 /*
249 * Statistics.
250 */
251 u_int moea64_pte_valid = 0;
252 u_int moea64_pte_overflow = 0;
253 u_int moea64_pvo_entries = 0;
254 u_int moea64_pvo_enter_calls = 0;
255 u_int moea64_pvo_remove_calls = 0;
256 SYSCTL_INT(_machdep, OID_AUTO, moea64_pte_valid, CTLFLAG_RD,
257 &moea64_pte_valid, 0, "");
258 SYSCTL_INT(_machdep, OID_AUTO, moea64_pte_overflow, CTLFLAG_RD,
259 &moea64_pte_overflow, 0, "");
260 SYSCTL_INT(_machdep, OID_AUTO, moea64_pvo_entries, CTLFLAG_RD,
261 &moea64_pvo_entries, 0, "");
262 SYSCTL_INT(_machdep, OID_AUTO, moea64_pvo_enter_calls, CTLFLAG_RD,
263 &moea64_pvo_enter_calls, 0, "");
264 SYSCTL_INT(_machdep, OID_AUTO, moea64_pvo_remove_calls, CTLFLAG_RD,
265 &moea64_pvo_remove_calls, 0, "");
266 #endif
267
268 vm_offset_t moea64_scratchpage_va[2];
269 struct pvo_entry *moea64_scratchpage_pvo[2];
270 struct mtx moea64_scratchpage_mtx;
271
272 uint64_t moea64_large_page_mask = 0;
273 uint64_t moea64_large_page_size = 0;
274 int moea64_large_page_shift = 0;
275 bool moea64_has_lp_4k_16m = false;
276
277 /*
278 * PVO calls.
279 */
280 static int moea64_pvo_enter(struct pvo_entry *pvo,
281 struct pvo_head *pvo_head, struct pvo_entry **oldpvo);
282 static void moea64_pvo_remove_from_pmap(struct pvo_entry *pvo);
283 static void moea64_pvo_remove_from_page(struct pvo_entry *pvo);
284 static void moea64_pvo_remove_from_page_locked(
285 struct pvo_entry *pvo, vm_page_t m);
286 static struct pvo_entry *moea64_pvo_find_va(pmap_t, vm_offset_t);
287
288 /*
289 * Utility routines.
290 */
291 static bool moea64_query_bit(vm_page_t, uint64_t);
292 static u_int moea64_clear_bit(vm_page_t, uint64_t);
293 static void moea64_kremove(vm_offset_t);
294 static void moea64_syncicache(pmap_t pmap, vm_offset_t va,
295 vm_paddr_t pa, vm_size_t sz);
296 static void moea64_pmap_init_qpages(void);
297 static void moea64_remove_locked(pmap_t, vm_offset_t,
298 vm_offset_t, struct pvo_dlist *);
299
300 /*
301 * Superpages data and routines.
302 */
303
304 /*
305 * PVO flags (in vaddr) that must match for promotion to succeed.
306 * Note that protection bits are checked separately, as they reside in
307 * another field.
308 */
309 #define PVO_FLAGS_PROMOTE (PVO_WIRED | PVO_MANAGED | PVO_PTEGIDX_VALID)
310
311 #define PVO_IS_SP(pvo) (((pvo)->pvo_vaddr & PVO_LARGE) && \
312 (pvo)->pvo_pmap != kernel_pmap)
313
314 /* Get physical address from PVO. */
315 #define PVO_PADDR(pvo) moea64_pvo_paddr(pvo)
316
317 /* MD page flag indicating that the page is a superpage. */
318 #define MDPG_ATTR_SP 0x40000000
319
320 SYSCTL_DECL(_vm_pmap);
321
322 static SYSCTL_NODE(_vm_pmap, OID_AUTO, sp, CTLFLAG_RD, 0,
323 "SP page mapping counters");
324
325 static u_long sp_demotions;
326 SYSCTL_ULONG(_vm_pmap_sp, OID_AUTO, demotions, CTLFLAG_RD,
327 &sp_demotions, 0, "SP page demotions");
328
329 static u_long sp_mappings;
330 SYSCTL_ULONG(_vm_pmap_sp, OID_AUTO, mappings, CTLFLAG_RD,
331 &sp_mappings, 0, "SP page mappings");
332
333 static u_long sp_p_failures;
334 SYSCTL_ULONG(_vm_pmap_sp, OID_AUTO, p_failures, CTLFLAG_RD,
335 &sp_p_failures, 0, "SP page promotion failures");
336
337 static u_long sp_p_fail_pa;
338 SYSCTL_ULONG(_vm_pmap_sp, OID_AUTO, p_fail_pa, CTLFLAG_RD,
339 &sp_p_fail_pa, 0, "SP page promotion failure: PAs don't match");
340
341 static u_long sp_p_fail_flags;
342 SYSCTL_ULONG(_vm_pmap_sp, OID_AUTO, p_fail_flags, CTLFLAG_RD,
343 &sp_p_fail_flags, 0, "SP page promotion failure: page flags don't match");
344
345 static u_long sp_p_fail_prot;
346 SYSCTL_ULONG(_vm_pmap_sp, OID_AUTO, p_fail_prot, CTLFLAG_RD,
347 &sp_p_fail_prot, 0,
348 "SP page promotion failure: page protections don't match");
349
350 static u_long sp_p_fail_wimg;
351 SYSCTL_ULONG(_vm_pmap_sp, OID_AUTO, p_fail_wimg, CTLFLAG_RD,
352 &sp_p_fail_wimg, 0, "SP page promotion failure: WIMG bits don't match");
353
354 static u_long sp_promotions;
355 SYSCTL_ULONG(_vm_pmap_sp, OID_AUTO, promotions, CTLFLAG_RD,
356 &sp_promotions, 0, "SP page promotions");
357
358 static bool moea64_ps_enabled(pmap_t);
359 static void moea64_align_superpage(vm_object_t, vm_ooffset_t,
360 vm_offset_t *, vm_size_t);
361
362 static int moea64_sp_enter(pmap_t pmap, vm_offset_t va,
363 vm_page_t m, vm_prot_t prot, u_int flags, int8_t psind);
364 static struct pvo_entry *moea64_sp_remove(struct pvo_entry *sp,
365 struct pvo_dlist *tofree);
366
367 static void moea64_sp_promote(pmap_t pmap, vm_offset_t va, vm_page_t m);
368 static void moea64_sp_demote_aligned(struct pvo_entry *sp);
369 static void moea64_sp_demote(struct pvo_entry *pvo);
370
371 static struct pvo_entry *moea64_sp_unwire(struct pvo_entry *sp);
372 static struct pvo_entry *moea64_sp_protect(struct pvo_entry *sp,
373 vm_prot_t prot);
374
375 static int64_t moea64_sp_query(struct pvo_entry *pvo, uint64_t ptebit);
376 static int64_t moea64_sp_clear(struct pvo_entry *pvo, vm_page_t m,
377 uint64_t ptebit);
378
379 static __inline bool moea64_sp_pvo_in_range(struct pvo_entry *pvo,
380 vm_offset_t sva, vm_offset_t eva);
381
382 /*
383 * Kernel MMU interface
384 */
385 void moea64_clear_modify(vm_page_t);
386 void moea64_copy_page(vm_page_t, vm_page_t);
387 void moea64_copy_page_dmap(vm_page_t, vm_page_t);
388 void moea64_copy_pages(vm_page_t *ma, vm_offset_t a_offset,
389 vm_page_t *mb, vm_offset_t b_offset, int xfersize);
390 void moea64_copy_pages_dmap(vm_page_t *ma, vm_offset_t a_offset,
391 vm_page_t *mb, vm_offset_t b_offset, int xfersize);
392 int moea64_enter(pmap_t, vm_offset_t, vm_page_t, vm_prot_t,
393 u_int flags, int8_t psind);
394 void moea64_enter_object(pmap_t, vm_offset_t, vm_offset_t, vm_page_t,
395 vm_prot_t);
396 void moea64_enter_quick(pmap_t, vm_offset_t, vm_page_t, vm_prot_t);
397 vm_paddr_t moea64_extract(pmap_t, vm_offset_t);
398 vm_page_t moea64_extract_and_hold(pmap_t, vm_offset_t, vm_prot_t);
399 void moea64_init(void);
400 bool moea64_is_modified(vm_page_t);
401 bool moea64_is_prefaultable(pmap_t, vm_offset_t);
402 bool moea64_is_referenced(vm_page_t);
403 int moea64_ts_referenced(vm_page_t);
404 vm_offset_t moea64_map(vm_offset_t *, vm_paddr_t, vm_paddr_t, int);
405 bool moea64_page_exists_quick(pmap_t, vm_page_t);
406 void moea64_page_init(vm_page_t);
407 int moea64_page_wired_mappings(vm_page_t);
408 int moea64_pinit(pmap_t);
409 void moea64_pinit0(pmap_t);
410 void moea64_protect(pmap_t, vm_offset_t, vm_offset_t, vm_prot_t);
411 void moea64_qenter(vm_offset_t, vm_page_t *, int);
412 void moea64_qremove(vm_offset_t, int);
413 void moea64_release(pmap_t);
414 void moea64_remove(pmap_t, vm_offset_t, vm_offset_t);
415 void moea64_remove_pages(pmap_t);
416 void moea64_remove_all(vm_page_t);
417 void moea64_remove_write(vm_page_t);
418 void moea64_unwire(pmap_t, vm_offset_t, vm_offset_t);
419 void moea64_zero_page(vm_page_t);
420 void moea64_zero_page_dmap(vm_page_t);
421 void moea64_zero_page_area(vm_page_t, int, int);
422 void moea64_activate(struct thread *);
423 void moea64_deactivate(struct thread *);
424 void *moea64_mapdev(vm_paddr_t, vm_size_t);
425 void *moea64_mapdev_attr(vm_paddr_t, vm_size_t, vm_memattr_t);
426 void moea64_unmapdev(void *, vm_size_t);
427 vm_paddr_t moea64_kextract(vm_offset_t);
428 void moea64_page_set_memattr(vm_page_t m, vm_memattr_t ma);
429 void moea64_kenter_attr(vm_offset_t, vm_paddr_t, vm_memattr_t ma);
430 void moea64_kenter(vm_offset_t, vm_paddr_t);
431 int moea64_dev_direct_mapped(vm_paddr_t, vm_size_t);
432 static void moea64_sync_icache(pmap_t, vm_offset_t, vm_size_t);
433 void moea64_dumpsys_map(vm_paddr_t pa, size_t sz,
434 void **va);
435 void moea64_scan_init(void);
436 vm_offset_t moea64_quick_enter_page(vm_page_t m);
437 vm_offset_t moea64_quick_enter_page_dmap(vm_page_t m);
438 void moea64_quick_remove_page(vm_offset_t addr);
439 bool moea64_page_is_mapped(vm_page_t m);
440 static int moea64_map_user_ptr(pmap_t pm,
441 volatile const void *uaddr, void **kaddr, size_t ulen, size_t *klen);
442 static int moea64_decode_kernel_ptr(vm_offset_t addr,
443 int *is_user, vm_offset_t *decoded_addr);
444 static size_t moea64_scan_pmap(struct bitset *dump_bitset);
445 static void *moea64_dump_pmap_init(unsigned blkpgs);
446 #ifdef __powerpc64__
447 static void moea64_page_array_startup(long);
448 #endif
449 static int moea64_mincore(pmap_t, vm_offset_t, vm_paddr_t *);
450
451 static struct pmap_funcs moea64_methods = {
452 .clear_modify = moea64_clear_modify,
453 .copy_page = moea64_copy_page,
454 .copy_pages = moea64_copy_pages,
455 .enter = moea64_enter,
456 .enter_object = moea64_enter_object,
457 .enter_quick = moea64_enter_quick,
458 .extract = moea64_extract,
459 .extract_and_hold = moea64_extract_and_hold,
460 .init = moea64_init,
461 .is_modified = moea64_is_modified,
462 .is_prefaultable = moea64_is_prefaultable,
463 .is_referenced = moea64_is_referenced,
464 .ts_referenced = moea64_ts_referenced,
465 .map = moea64_map,
466 .mincore = moea64_mincore,
467 .page_exists_quick = moea64_page_exists_quick,
468 .page_init = moea64_page_init,
469 .page_wired_mappings = moea64_page_wired_mappings,
470 .pinit = moea64_pinit,
471 .pinit0 = moea64_pinit0,
472 .protect = moea64_protect,
473 .qenter = moea64_qenter,
474 .qremove = moea64_qremove,
475 .release = moea64_release,
476 .remove = moea64_remove,
477 .remove_pages = moea64_remove_pages,
478 .remove_all = moea64_remove_all,
479 .remove_write = moea64_remove_write,
480 .sync_icache = moea64_sync_icache,
481 .unwire = moea64_unwire,
482 .zero_page = moea64_zero_page,
483 .zero_page_area = moea64_zero_page_area,
484 .activate = moea64_activate,
485 .deactivate = moea64_deactivate,
486 .page_set_memattr = moea64_page_set_memattr,
487 .quick_enter_page = moea64_quick_enter_page,
488 .quick_remove_page = moea64_quick_remove_page,
489 .page_is_mapped = moea64_page_is_mapped,
490 #ifdef __powerpc64__
491 .page_array_startup = moea64_page_array_startup,
492 #endif
493 .ps_enabled = moea64_ps_enabled,
494 .align_superpage = moea64_align_superpage,
495
496 /* Internal interfaces */
497 .mapdev = moea64_mapdev,
498 .mapdev_attr = moea64_mapdev_attr,
499 .unmapdev = moea64_unmapdev,
500 .kextract = moea64_kextract,
501 .kenter = moea64_kenter,
502 .kenter_attr = moea64_kenter_attr,
503 .dev_direct_mapped = moea64_dev_direct_mapped,
504 .dumpsys_pa_init = moea64_scan_init,
505 .dumpsys_scan_pmap = moea64_scan_pmap,
506 .dumpsys_dump_pmap_init = moea64_dump_pmap_init,
507 .dumpsys_map_chunk = moea64_dumpsys_map,
508 .map_user_ptr = moea64_map_user_ptr,
509 .decode_kernel_ptr = moea64_decode_kernel_ptr,
510 };
511
512 MMU_DEF(oea64_mmu, "mmu_oea64_base", moea64_methods);
513
514 /*
515 * Get physical address from PVO.
516 *
517 * For superpages, the lower bits are not stored on pvo_pte.pa and must be
518 * obtained from VA.
519 */
520 static __always_inline vm_paddr_t
moea64_pvo_paddr(struct pvo_entry * pvo)521 moea64_pvo_paddr(struct pvo_entry *pvo)
522 {
523 vm_paddr_t pa;
524
525 pa = (pvo)->pvo_pte.pa & LPTE_RPGN;
526
527 if (PVO_IS_SP(pvo)) {
528 pa &= ~HPT_SP_MASK; /* This is needed to clear LPTE_LP bits. */
529 pa |= PVO_VADDR(pvo) & HPT_SP_MASK;
530 }
531 return (pa);
532 }
533
534 static struct pvo_head *
vm_page_to_pvoh(vm_page_t m)535 vm_page_to_pvoh(vm_page_t m)
536 {
537
538 mtx_assert(PV_LOCKPTR(VM_PAGE_TO_PHYS(m)), MA_OWNED);
539 return (&m->md.mdpg_pvoh);
540 }
541
542 static struct pvo_entry *
alloc_pvo_entry(int bootstrap)543 alloc_pvo_entry(int bootstrap)
544 {
545 struct pvo_entry *pvo;
546
547 if (!moea64_initialized || bootstrap) {
548 if (moea64_bpvo_pool_index >= moea64_bpvo_pool_size) {
549 panic("%s: bpvo pool exhausted, index=%d, size=%d, bytes=%zd."
550 "Try setting machdep.moea64_bpvo_pool_size tunable",
551 __func__, moea64_bpvo_pool_index,
552 moea64_bpvo_pool_size,
553 moea64_bpvo_pool_size * sizeof(struct pvo_entry));
554 }
555 pvo = &moea64_bpvo_pool[
556 atomic_fetchadd_int(&moea64_bpvo_pool_index, 1)];
557 bzero(pvo, sizeof(*pvo));
558 pvo->pvo_vaddr = PVO_BOOTSTRAP;
559 } else
560 pvo = uma_zalloc(moea64_pvo_zone, M_NOWAIT | M_ZERO);
561
562 return (pvo);
563 }
564
565 static void
init_pvo_entry(struct pvo_entry * pvo,pmap_t pmap,vm_offset_t va)566 init_pvo_entry(struct pvo_entry *pvo, pmap_t pmap, vm_offset_t va)
567 {
568 uint64_t vsid;
569 uint64_t hash;
570 int shift;
571
572 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
573
574 pvo->pvo_pmap = pmap;
575 va &= ~ADDR_POFF;
576 pvo->pvo_vaddr |= va;
577 vsid = va_to_vsid(pmap, va);
578 pvo->pvo_vpn = (uint64_t)((va & ADDR_PIDX) >> ADDR_PIDX_SHFT)
579 | (vsid << 16);
580
581 if (pmap == kernel_pmap && (pvo->pvo_vaddr & PVO_LARGE) != 0)
582 shift = moea64_large_page_shift;
583 else
584 shift = ADDR_PIDX_SHFT;
585 hash = (vsid & VSID_HASH_MASK) ^ (((uint64_t)va & ADDR_PIDX) >> shift);
586 pvo->pvo_pte.slot = (hash & moea64_pteg_mask) << 3;
587 }
588
589 static void
free_pvo_entry(struct pvo_entry * pvo)590 free_pvo_entry(struct pvo_entry *pvo)
591 {
592
593 if (!(pvo->pvo_vaddr & PVO_BOOTSTRAP))
594 uma_zfree(moea64_pvo_zone, pvo);
595 }
596
597 void
moea64_pte_from_pvo(const struct pvo_entry * pvo,struct lpte * lpte)598 moea64_pte_from_pvo(const struct pvo_entry *pvo, struct lpte *lpte)
599 {
600
601 lpte->pte_hi = moea64_pte_vpn_from_pvo_vpn(pvo);
602 lpte->pte_hi |= LPTE_VALID;
603
604 if (pvo->pvo_vaddr & PVO_LARGE)
605 lpte->pte_hi |= LPTE_BIG;
606 if (pvo->pvo_vaddr & PVO_WIRED)
607 lpte->pte_hi |= LPTE_WIRED;
608 if (pvo->pvo_vaddr & PVO_HID)
609 lpte->pte_hi |= LPTE_HID;
610
611 lpte->pte_lo = pvo->pvo_pte.pa; /* Includes WIMG bits */
612 if (pvo->pvo_pte.prot & VM_PROT_WRITE)
613 lpte->pte_lo |= LPTE_BW;
614 else
615 lpte->pte_lo |= LPTE_BR;
616
617 if (!(pvo->pvo_pte.prot & VM_PROT_EXECUTE))
618 lpte->pte_lo |= LPTE_NOEXEC;
619 }
620
621 static __inline uint64_t
moea64_calc_wimg(vm_paddr_t pa,vm_memattr_t ma)622 moea64_calc_wimg(vm_paddr_t pa, vm_memattr_t ma)
623 {
624 uint64_t pte_lo;
625 int i;
626
627 if (ma != VM_MEMATTR_DEFAULT) {
628 switch (ma) {
629 case VM_MEMATTR_UNCACHEABLE:
630 return (LPTE_I | LPTE_G);
631 case VM_MEMATTR_CACHEABLE:
632 return (LPTE_M);
633 case VM_MEMATTR_WRITE_COMBINING:
634 case VM_MEMATTR_WRITE_BACK:
635 case VM_MEMATTR_PREFETCHABLE:
636 return (LPTE_I);
637 case VM_MEMATTR_WRITE_THROUGH:
638 return (LPTE_W | LPTE_M);
639 }
640 }
641
642 /*
643 * Assume the page is cache inhibited and access is guarded unless
644 * it's in our available memory array.
645 */
646 pte_lo = LPTE_I | LPTE_G;
647 for (i = 0; i < pregions_sz; i++) {
648 if ((pa >= pregions[i].mr_start) &&
649 (pa < (pregions[i].mr_start + pregions[i].mr_size))) {
650 pte_lo &= ~(LPTE_I | LPTE_G);
651 pte_lo |= LPTE_M;
652 break;
653 }
654 }
655
656 return pte_lo;
657 }
658
659 /*
660 * Quick sort callout for comparing memory regions.
661 */
662 static int om_cmp(const void *a, const void *b);
663
664 static int
om_cmp(const void * a,const void * b)665 om_cmp(const void *a, const void *b)
666 {
667 const struct ofw_map *mapa;
668 const struct ofw_map *mapb;
669
670 mapa = a;
671 mapb = b;
672 if (mapa->om_pa < mapb->om_pa)
673 return (-1);
674 else if (mapa->om_pa > mapb->om_pa)
675 return (1);
676 else
677 return (0);
678 }
679
680 static void
moea64_add_ofw_mappings(phandle_t mmu,size_t sz)681 moea64_add_ofw_mappings(phandle_t mmu, size_t sz)
682 {
683 struct ofw_map translations[sz/(4*sizeof(cell_t))]; /*>= 4 cells per */
684 pcell_t acells, trans_cells[sz/sizeof(cell_t)];
685 struct pvo_entry *pvo;
686 register_t msr;
687 vm_offset_t off;
688 vm_paddr_t pa_base;
689 int i, j;
690
691 bzero(translations, sz);
692 OF_getencprop(OF_finddevice("/"), "#address-cells", &acells,
693 sizeof(acells));
694 if (OF_getencprop(mmu, "translations", trans_cells, sz) == -1)
695 panic("moea64_bootstrap: can't get ofw translations");
696
697 CTR0(KTR_PMAP, "moea64_add_ofw_mappings: translations");
698 sz /= sizeof(cell_t);
699 for (i = 0, j = 0; i < sz; j++) {
700 translations[j].om_va = trans_cells[i++];
701 translations[j].om_len = trans_cells[i++];
702 translations[j].om_pa = trans_cells[i++];
703 if (acells == 2) {
704 translations[j].om_pa <<= 32;
705 translations[j].om_pa |= trans_cells[i++];
706 }
707 translations[j].om_mode = trans_cells[i++];
708 }
709 KASSERT(i == sz, ("Translations map has incorrect cell count (%d/%zd)",
710 i, sz));
711
712 sz = j;
713 qsort(translations, sz, sizeof (*translations), om_cmp);
714
715 for (i = 0; i < sz; i++) {
716 pa_base = translations[i].om_pa;
717 #ifndef __powerpc64__
718 if ((translations[i].om_pa >> 32) != 0)
719 panic("OFW translations above 32-bit boundary!");
720 #endif
721
722 if (pa_base % PAGE_SIZE)
723 panic("OFW translation not page-aligned (phys)!");
724 if (translations[i].om_va % PAGE_SIZE)
725 panic("OFW translation not page-aligned (virt)!");
726
727 CTR3(KTR_PMAP, "translation: pa=%#zx va=%#x len=%#x",
728 pa_base, translations[i].om_va, translations[i].om_len);
729
730 /* Now enter the pages for this mapping */
731
732 DISABLE_TRANS(msr);
733 for (off = 0; off < translations[i].om_len; off += PAGE_SIZE) {
734 /* If this address is direct-mapped, skip remapping */
735 if (hw_direct_map &&
736 translations[i].om_va == PHYS_TO_DMAP(pa_base) &&
737 moea64_calc_wimg(pa_base + off, VM_MEMATTR_DEFAULT)
738 == LPTE_M)
739 continue;
740
741 PMAP_LOCK(kernel_pmap);
742 pvo = moea64_pvo_find_va(kernel_pmap,
743 translations[i].om_va + off);
744 PMAP_UNLOCK(kernel_pmap);
745 if (pvo != NULL)
746 continue;
747
748 moea64_kenter(translations[i].om_va + off,
749 pa_base + off);
750 }
751 ENABLE_TRANS(msr);
752 }
753 }
754
755 #ifdef __powerpc64__
756 static void
moea64_probe_large_page(void)757 moea64_probe_large_page(void)
758 {
759 uint16_t pvr = mfpvr() >> 16;
760
761 switch (pvr) {
762 case IBM970:
763 case IBM970FX:
764 case IBM970MP:
765 powerpc_sync(); isync();
766 mtspr(SPR_HID4, mfspr(SPR_HID4) & ~HID4_970_DISABLE_LG_PG);
767 powerpc_sync(); isync();
768
769 /* FALLTHROUGH */
770 default:
771 if (moea64_large_page_size == 0) {
772 moea64_large_page_size = 0x1000000; /* 16 MB */
773 moea64_large_page_shift = 24;
774 }
775 }
776
777 moea64_large_page_mask = moea64_large_page_size - 1;
778 }
779
780 static void
moea64_bootstrap_slb_prefault(vm_offset_t va,int large)781 moea64_bootstrap_slb_prefault(vm_offset_t va, int large)
782 {
783 struct slb *cache;
784 struct slb entry;
785 uint64_t esid, slbe;
786 uint64_t i;
787
788 cache = PCPU_GET(aim.slb);
789 esid = va >> ADDR_SR_SHFT;
790 slbe = (esid << SLBE_ESID_SHIFT) | SLBE_VALID;
791
792 for (i = 0; i < 64; i++) {
793 if (cache[i].slbe == (slbe | i))
794 return;
795 }
796
797 entry.slbe = slbe;
798 entry.slbv = KERNEL_VSID(esid) << SLBV_VSID_SHIFT;
799 if (large)
800 entry.slbv |= SLBV_L;
801
802 slb_insert_kernel(entry.slbe, entry.slbv);
803 }
804 #endif
805
806 static int
moea64_kenter_large(vm_offset_t va,vm_paddr_t pa,uint64_t attr,int bootstrap)807 moea64_kenter_large(vm_offset_t va, vm_paddr_t pa, uint64_t attr, int bootstrap)
808 {
809 struct pvo_entry *pvo;
810 uint64_t pte_lo;
811 int error;
812
813 pte_lo = LPTE_M;
814 pte_lo |= attr;
815
816 pvo = alloc_pvo_entry(bootstrap);
817 pvo->pvo_vaddr |= PVO_WIRED | PVO_LARGE;
818 init_pvo_entry(pvo, kernel_pmap, va);
819
820 pvo->pvo_pte.prot = VM_PROT_READ | VM_PROT_WRITE |
821 VM_PROT_EXECUTE;
822 pvo->pvo_pte.pa = pa | pte_lo;
823 error = moea64_pvo_enter(pvo, NULL, NULL);
824 if (error != 0)
825 panic("Error %d inserting large page\n", error);
826 return (0);
827 }
828
829 static void
moea64_setup_direct_map(vm_offset_t kernelstart,vm_offset_t kernelend)830 moea64_setup_direct_map(vm_offset_t kernelstart,
831 vm_offset_t kernelend)
832 {
833 register_t msr;
834 vm_paddr_t pa, pkernelstart, pkernelend;
835 vm_offset_t size, off;
836 uint64_t pte_lo;
837 int i;
838
839 if (moea64_large_page_size == 0)
840 hw_direct_map = 0;
841
842 DISABLE_TRANS(msr);
843 if (hw_direct_map) {
844 PMAP_LOCK(kernel_pmap);
845 for (i = 0; i < pregions_sz; i++) {
846 for (pa = pregions[i].mr_start; pa < pregions[i].mr_start +
847 pregions[i].mr_size; pa += moea64_large_page_size) {
848 pte_lo = LPTE_M;
849 if (pa & moea64_large_page_mask) {
850 pa &= moea64_large_page_mask;
851 pte_lo |= LPTE_G;
852 }
853 if (pa + moea64_large_page_size >
854 pregions[i].mr_start + pregions[i].mr_size)
855 pte_lo |= LPTE_G;
856
857 moea64_kenter_large(PHYS_TO_DMAP(pa), pa, pte_lo, 1);
858 }
859 }
860 PMAP_UNLOCK(kernel_pmap);
861 }
862
863 /*
864 * Make sure the kernel and BPVO pool stay mapped on systems either
865 * without a direct map or on which the kernel is not already executing
866 * out of the direct-mapped region.
867 */
868 if (kernelstart < DMAP_BASE_ADDRESS) {
869 /*
870 * For pre-dmap execution, we need to use identity mapping
871 * because we will be operating with the mmu on but in the
872 * wrong address configuration until we __restartkernel().
873 */
874 for (pa = kernelstart & ~PAGE_MASK; pa < kernelend;
875 pa += PAGE_SIZE)
876 moea64_kenter(pa, pa);
877 } else if (!hw_direct_map) {
878 pkernelstart = kernelstart & ~DMAP_BASE_ADDRESS;
879 pkernelend = kernelend & ~DMAP_BASE_ADDRESS;
880 for (pa = pkernelstart & ~PAGE_MASK; pa < pkernelend;
881 pa += PAGE_SIZE)
882 moea64_kenter(pa | DMAP_BASE_ADDRESS, pa);
883 }
884
885 if (!hw_direct_map) {
886 size = moea64_bpvo_pool_size*sizeof(struct pvo_entry);
887 off = (vm_offset_t)(moea64_bpvo_pool);
888 for (pa = off; pa < off + size; pa += PAGE_SIZE)
889 moea64_kenter(pa, pa);
890
891 /* Map exception vectors */
892 for (pa = EXC_RSVD; pa < EXC_LAST; pa += PAGE_SIZE)
893 moea64_kenter(pa | DMAP_BASE_ADDRESS, pa);
894 }
895 ENABLE_TRANS(msr);
896
897 /*
898 * Allow user to override unmapped_buf_allowed for testing.
899 * XXXKIB Only direct map implementation was tested.
900 */
901 if (!TUNABLE_INT_FETCH("vfs.unmapped_buf_allowed",
902 &unmapped_buf_allowed))
903 unmapped_buf_allowed = hw_direct_map;
904 }
905
906 /* Quick sort callout for comparing physical addresses. */
907 static int
pa_cmp(const void * a,const void * b)908 pa_cmp(const void *a, const void *b)
909 {
910 const vm_paddr_t *pa = a, *pb = b;
911
912 if (*pa < *pb)
913 return (-1);
914 else if (*pa > *pb)
915 return (1);
916 else
917 return (0);
918 }
919
920 void
moea64_early_bootstrap(vm_offset_t kernelstart,vm_offset_t kernelend)921 moea64_early_bootstrap(vm_offset_t kernelstart, vm_offset_t kernelend)
922 {
923 int i, j;
924 vm_size_t physsz, hwphyssz;
925 vm_paddr_t kernelphysstart, kernelphysend;
926 int rm_pavail;
927
928 /* Level 0 reservations consist of 4096 pages (16MB superpage). */
929 vm_level_0_order = 12;
930
931 #ifndef __powerpc64__
932 /* We don't have a direct map since there is no BAT */
933 hw_direct_map = 0;
934
935 /* Make sure battable is zero, since we have no BAT */
936 for (i = 0; i < 16; i++) {
937 battable[i].batu = 0;
938 battable[i].batl = 0;
939 }
940 #else
941 /* Install trap handlers for SLBs */
942 bcopy(&slbtrap, (void *)EXC_DSE,(size_t)&slbtrapend - (size_t)&slbtrap);
943 bcopy(&slbtrap, (void *)EXC_ISE,(size_t)&slbtrapend - (size_t)&slbtrap);
944 __syncicache((void *)EXC_DSE, 0x80);
945 __syncicache((void *)EXC_ISE, 0x80);
946 #endif
947
948 kernelphysstart = kernelstart & ~DMAP_BASE_ADDRESS;
949 kernelphysend = kernelend & ~DMAP_BASE_ADDRESS;
950
951 /* Get physical memory regions from firmware */
952 mem_regions(&pregions, &pregions_sz, ®ions, ®ions_sz);
953 CTR0(KTR_PMAP, "moea64_bootstrap: physical memory");
954
955 if (PHYS_AVAIL_ENTRIES < regions_sz)
956 panic("moea64_bootstrap: phys_avail too small");
957
958 phys_avail_count = 0;
959 physsz = 0;
960 hwphyssz = 0;
961 TUNABLE_ULONG_FETCH("hw.physmem", (u_long *) &hwphyssz);
962 for (i = 0, j = 0; i < regions_sz; i++, j += 2) {
963 CTR3(KTR_PMAP, "region: %#zx - %#zx (%#zx)",
964 regions[i].mr_start, regions[i].mr_start +
965 regions[i].mr_size, regions[i].mr_size);
966 if (hwphyssz != 0 &&
967 (physsz + regions[i].mr_size) >= hwphyssz) {
968 if (physsz < hwphyssz) {
969 phys_avail[j] = regions[i].mr_start;
970 phys_avail[j + 1] = regions[i].mr_start +
971 hwphyssz - physsz;
972 physsz = hwphyssz;
973 phys_avail_count++;
974 dump_avail[j] = phys_avail[j];
975 dump_avail[j + 1] = phys_avail[j + 1];
976 }
977 break;
978 }
979 phys_avail[j] = regions[i].mr_start;
980 phys_avail[j + 1] = regions[i].mr_start + regions[i].mr_size;
981 phys_avail_count++;
982 physsz += regions[i].mr_size;
983 dump_avail[j] = phys_avail[j];
984 dump_avail[j + 1] = phys_avail[j + 1];
985 }
986
987 /* Check for overlap with the kernel and exception vectors */
988 rm_pavail = 0;
989 for (j = 0; j < 2*phys_avail_count; j+=2) {
990 if (phys_avail[j] < EXC_LAST)
991 phys_avail[j] += EXC_LAST;
992
993 if (phys_avail[j] >= kernelphysstart &&
994 phys_avail[j+1] <= kernelphysend) {
995 phys_avail[j] = phys_avail[j+1] = ~0;
996 rm_pavail++;
997 continue;
998 }
999
1000 if (kernelphysstart >= phys_avail[j] &&
1001 kernelphysstart < phys_avail[j+1]) {
1002 if (kernelphysend < phys_avail[j+1]) {
1003 phys_avail[2*phys_avail_count] =
1004 (kernelphysend & ~PAGE_MASK) + PAGE_SIZE;
1005 phys_avail[2*phys_avail_count + 1] =
1006 phys_avail[j+1];
1007 phys_avail_count++;
1008 }
1009
1010 phys_avail[j+1] = kernelphysstart & ~PAGE_MASK;
1011 }
1012
1013 if (kernelphysend >= phys_avail[j] &&
1014 kernelphysend < phys_avail[j+1]) {
1015 if (kernelphysstart > phys_avail[j]) {
1016 phys_avail[2*phys_avail_count] = phys_avail[j];
1017 phys_avail[2*phys_avail_count + 1] =
1018 kernelphysstart & ~PAGE_MASK;
1019 phys_avail_count++;
1020 }
1021
1022 phys_avail[j] = (kernelphysend & ~PAGE_MASK) +
1023 PAGE_SIZE;
1024 }
1025 }
1026
1027 /* Remove physical available regions marked for removal (~0) */
1028 if (rm_pavail) {
1029 qsort(phys_avail, 2*phys_avail_count, sizeof(phys_avail[0]),
1030 pa_cmp);
1031 phys_avail_count -= rm_pavail;
1032 for (i = 2*phys_avail_count;
1033 i < 2*(phys_avail_count + rm_pavail); i+=2)
1034 phys_avail[i] = phys_avail[i+1] = 0;
1035 }
1036
1037 physmem = btoc(physsz);
1038
1039 #ifdef PTEGCOUNT
1040 moea64_pteg_count = PTEGCOUNT;
1041 #else
1042 moea64_pteg_count = 0x1000;
1043
1044 while (moea64_pteg_count < physmem)
1045 moea64_pteg_count <<= 1;
1046
1047 moea64_pteg_count >>= 1;
1048 #endif /* PTEGCOUNT */
1049 }
1050
1051 void
moea64_mid_bootstrap(vm_offset_t kernelstart,vm_offset_t kernelend)1052 moea64_mid_bootstrap(vm_offset_t kernelstart, vm_offset_t kernelend)
1053 {
1054 int i;
1055
1056 /*
1057 * Set PTEG mask
1058 */
1059 moea64_pteg_mask = moea64_pteg_count - 1;
1060
1061 /*
1062 * Initialize SLB table lock and page locks
1063 */
1064 mtx_init(&moea64_slb_mutex, "SLB table", NULL, MTX_DEF);
1065 for (i = 0; i < PV_LOCK_COUNT; i++)
1066 mtx_init(&pv_lock[i], "page pv", NULL, MTX_DEF);
1067
1068 /*
1069 * Initialise the bootstrap pvo pool.
1070 */
1071 TUNABLE_INT_FETCH("machdep.moea64_bpvo_pool_size", &moea64_bpvo_pool_size);
1072 if (moea64_bpvo_pool_size == 0) {
1073 if (!hw_direct_map)
1074 moea64_bpvo_pool_size = ((ptoa((uintmax_t)physmem) * sizeof(struct vm_page)) /
1075 (PAGE_SIZE * PAGE_SIZE)) * BPVO_POOL_EXPANSION_FACTOR;
1076 else
1077 moea64_bpvo_pool_size = BPVO_POOL_SIZE;
1078 }
1079
1080 if (boothowto & RB_VERBOSE) {
1081 printf("mmu_oea64: bpvo pool entries = %d, bpvo pool size = %zu MB\n",
1082 moea64_bpvo_pool_size,
1083 moea64_bpvo_pool_size*sizeof(struct pvo_entry) / 1048576);
1084 }
1085
1086 moea64_bpvo_pool = (struct pvo_entry *)moea64_bootstrap_alloc(
1087 moea64_bpvo_pool_size*sizeof(struct pvo_entry), PAGE_SIZE);
1088 moea64_bpvo_pool_index = 0;
1089
1090 /* Place at address usable through the direct map */
1091 if (hw_direct_map)
1092 moea64_bpvo_pool = (struct pvo_entry *)
1093 PHYS_TO_DMAP((uintptr_t)moea64_bpvo_pool);
1094
1095 /*
1096 * Make sure kernel vsid is allocated as well as VSID 0.
1097 */
1098 #ifndef __powerpc64__
1099 moea64_vsid_bitmap[(KERNEL_VSIDBITS & (NVSIDS - 1)) / VSID_NBPW]
1100 |= 1 << (KERNEL_VSIDBITS % VSID_NBPW);
1101 moea64_vsid_bitmap[0] |= 1;
1102 #endif
1103
1104 /*
1105 * Initialize the kernel pmap (which is statically allocated).
1106 */
1107 #ifdef __powerpc64__
1108 for (i = 0; i < 64; i++) {
1109 pcpup->pc_aim.slb[i].slbv = 0;
1110 pcpup->pc_aim.slb[i].slbe = 0;
1111 }
1112 #else
1113 for (i = 0; i < 16; i++)
1114 kernel_pmap->pm_sr[i] = EMPTY_SEGMENT + i;
1115 #endif
1116
1117 kernel_pmap->pmap_phys = kernel_pmap;
1118 CPU_FILL(&kernel_pmap->pm_active);
1119 RB_INIT(&kernel_pmap->pmap_pvo);
1120
1121 PMAP_LOCK_INIT(kernel_pmap);
1122
1123 /*
1124 * Now map in all the other buffers we allocated earlier
1125 */
1126
1127 moea64_setup_direct_map(kernelstart, kernelend);
1128 }
1129
1130 void
moea64_late_bootstrap(vm_offset_t kernelstart,vm_offset_t kernelend)1131 moea64_late_bootstrap(vm_offset_t kernelstart, vm_offset_t kernelend)
1132 {
1133 ihandle_t mmui;
1134 phandle_t chosen;
1135 phandle_t mmu;
1136 ssize_t sz;
1137 int i;
1138 vm_offset_t pa, va;
1139 void *dpcpu;
1140
1141 /*
1142 * Set up the Open Firmware pmap and add its mappings if not in real
1143 * mode.
1144 */
1145
1146 chosen = OF_finddevice("/chosen");
1147 if (chosen != -1 && OF_getencprop(chosen, "mmu", &mmui, 4) != -1) {
1148 mmu = OF_instance_to_package(mmui);
1149 if (mmu == -1 ||
1150 (sz = OF_getproplen(mmu, "translations")) == -1)
1151 sz = 0;
1152 if (sz > 6144 /* tmpstksz - 2 KB headroom */)
1153 panic("moea64_bootstrap: too many ofw translations");
1154
1155 if (sz > 0)
1156 moea64_add_ofw_mappings(mmu, sz);
1157 }
1158
1159 /*
1160 * Calculate the last available physical address.
1161 */
1162 Maxmem = 0;
1163 for (i = 0; phys_avail[i + 1] != 0; i += 2)
1164 Maxmem = MAX(Maxmem, powerpc_btop(phys_avail[i + 1]));
1165
1166 /*
1167 * Initialize MMU.
1168 */
1169 pmap_cpu_bootstrap(0);
1170 mtmsr(mfmsr() | PSL_DR | PSL_IR);
1171 pmap_bootstrapped++;
1172
1173 /*
1174 * Set the start and end of kva.
1175 */
1176 virtual_avail = VM_MIN_KERNEL_ADDRESS;
1177 virtual_end = VM_MAX_SAFE_KERNEL_ADDRESS;
1178
1179 /*
1180 * Map the entire KVA range into the SLB. We must not fault there.
1181 */
1182 #ifdef __powerpc64__
1183 for (va = virtual_avail; va < virtual_end; va += SEGMENT_LENGTH)
1184 moea64_bootstrap_slb_prefault(va, 0);
1185 #endif
1186
1187 /*
1188 * Remap any early IO mappings (console framebuffer, etc.)
1189 */
1190 bs_remap_earlyboot();
1191
1192 /*
1193 * Figure out how far we can extend virtual_end into segment 16
1194 * without running into existing mappings. Segment 16 is guaranteed
1195 * to contain neither RAM nor devices (at least on Apple hardware),
1196 * but will generally contain some OFW mappings we should not
1197 * step on.
1198 */
1199
1200 #ifndef __powerpc64__ /* KVA is in high memory on PPC64 */
1201 PMAP_LOCK(kernel_pmap);
1202 while (virtual_end < VM_MAX_KERNEL_ADDRESS &&
1203 moea64_pvo_find_va(kernel_pmap, virtual_end+1) == NULL)
1204 virtual_end += PAGE_SIZE;
1205 PMAP_UNLOCK(kernel_pmap);
1206 #endif
1207
1208 /*
1209 * Allocate a kernel stack with a guard page for thread0 and map it
1210 * into the kernel page map.
1211 */
1212 pa = moea64_bootstrap_alloc(kstack_pages * PAGE_SIZE, PAGE_SIZE);
1213 va = virtual_avail + KSTACK_GUARD_PAGES * PAGE_SIZE;
1214 virtual_avail = va + kstack_pages * PAGE_SIZE;
1215 CTR2(KTR_PMAP, "moea64_bootstrap: kstack0 at %#x (%#x)", pa, va);
1216 thread0.td_kstack = va;
1217 thread0.td_kstack_pages = kstack_pages;
1218 for (i = 0; i < kstack_pages; i++) {
1219 moea64_kenter(va, pa);
1220 pa += PAGE_SIZE;
1221 va += PAGE_SIZE;
1222 }
1223
1224 /*
1225 * Allocate virtual address space for the message buffer.
1226 */
1227 pa = msgbuf_phys = moea64_bootstrap_alloc(msgbufsize, PAGE_SIZE);
1228 msgbufp = (struct msgbuf *)virtual_avail;
1229 va = virtual_avail;
1230 virtual_avail += round_page(msgbufsize);
1231 while (va < virtual_avail) {
1232 moea64_kenter(va, pa);
1233 pa += PAGE_SIZE;
1234 va += PAGE_SIZE;
1235 }
1236
1237 /*
1238 * Allocate virtual address space for the dynamic percpu area.
1239 */
1240 pa = moea64_bootstrap_alloc(DPCPU_SIZE, PAGE_SIZE);
1241 dpcpu = (void *)virtual_avail;
1242 va = virtual_avail;
1243 virtual_avail += DPCPU_SIZE;
1244 while (va < virtual_avail) {
1245 moea64_kenter(va, pa);
1246 pa += PAGE_SIZE;
1247 va += PAGE_SIZE;
1248 }
1249 dpcpu_init(dpcpu, curcpu);
1250
1251 crashdumpmap = (caddr_t)virtual_avail;
1252 virtual_avail += MAXDUMPPGS * PAGE_SIZE;
1253
1254 /*
1255 * Allocate some things for page zeroing. We put this directly
1256 * in the page table and use MOEA64_PTE_REPLACE to avoid any
1257 * of the PVO book-keeping or other parts of the VM system
1258 * from even knowing that this hack exists.
1259 */
1260
1261 if (!hw_direct_map) {
1262 mtx_init(&moea64_scratchpage_mtx, "pvo zero page", NULL,
1263 MTX_DEF);
1264 for (i = 0; i < 2; i++) {
1265 moea64_scratchpage_va[i] = (virtual_end+1) - PAGE_SIZE;
1266 virtual_end -= PAGE_SIZE;
1267
1268 moea64_kenter(moea64_scratchpage_va[i], 0);
1269
1270 PMAP_LOCK(kernel_pmap);
1271 moea64_scratchpage_pvo[i] = moea64_pvo_find_va(
1272 kernel_pmap, (vm_offset_t)moea64_scratchpage_va[i]);
1273 PMAP_UNLOCK(kernel_pmap);
1274 }
1275 }
1276
1277 numa_mem_regions(&numa_pregions, &numapregions_sz);
1278 }
1279
1280 static void
moea64_pmap_init_qpages(void)1281 moea64_pmap_init_qpages(void)
1282 {
1283 struct pcpu *pc;
1284 int i;
1285
1286 if (hw_direct_map)
1287 return;
1288
1289 CPU_FOREACH(i) {
1290 pc = pcpu_find(i);
1291 pc->pc_qmap_addr = kva_alloc(PAGE_SIZE);
1292 if (pc->pc_qmap_addr == 0)
1293 panic("pmap_init_qpages: unable to allocate KVA");
1294 PMAP_LOCK(kernel_pmap);
1295 pc->pc_aim.qmap_pvo =
1296 moea64_pvo_find_va(kernel_pmap, pc->pc_qmap_addr);
1297 PMAP_UNLOCK(kernel_pmap);
1298 mtx_init(&pc->pc_aim.qmap_lock, "qmap lock", NULL, MTX_DEF);
1299 }
1300 }
1301
1302 SYSINIT(qpages_init, SI_SUB_CPU, SI_ORDER_ANY, moea64_pmap_init_qpages, NULL);
1303
1304 /*
1305 * Activate a user pmap. This mostly involves setting some non-CPU
1306 * state.
1307 */
1308 void
moea64_activate(struct thread * td)1309 moea64_activate(struct thread *td)
1310 {
1311 pmap_t pm;
1312
1313 pm = &td->td_proc->p_vmspace->vm_pmap;
1314 CPU_SET(PCPU_GET(cpuid), &pm->pm_active);
1315
1316 #ifdef __powerpc64__
1317 PCPU_SET(aim.userslb, pm->pm_slb);
1318 __asm __volatile("slbmte %0, %1; isync" ::
1319 "r"(td->td_pcb->pcb_cpu.aim.usr_vsid), "r"(USER_SLB_SLBE));
1320 #else
1321 PCPU_SET(curpmap, pm->pmap_phys);
1322 mtsrin(USER_SR << ADDR_SR_SHFT, td->td_pcb->pcb_cpu.aim.usr_vsid);
1323 #endif
1324 }
1325
1326 void
moea64_deactivate(struct thread * td)1327 moea64_deactivate(struct thread *td)
1328 {
1329 pmap_t pm;
1330
1331 __asm __volatile("isync; slbie %0" :: "r"(USER_ADDR));
1332
1333 pm = &td->td_proc->p_vmspace->vm_pmap;
1334 CPU_CLR(PCPU_GET(cpuid), &pm->pm_active);
1335 #ifdef __powerpc64__
1336 PCPU_SET(aim.userslb, NULL);
1337 #else
1338 PCPU_SET(curpmap, NULL);
1339 #endif
1340 }
1341
1342 void
moea64_unwire(pmap_t pm,vm_offset_t sva,vm_offset_t eva)1343 moea64_unwire(pmap_t pm, vm_offset_t sva, vm_offset_t eva)
1344 {
1345 struct pvo_entry key, *pvo;
1346 vm_page_t m;
1347 int64_t refchg;
1348
1349 key.pvo_vaddr = sva;
1350 PMAP_LOCK(pm);
1351 for (pvo = RB_NFIND(pvo_tree, &pm->pmap_pvo, &key);
1352 pvo != NULL && PVO_VADDR(pvo) < eva;
1353 pvo = RB_NEXT(pvo_tree, &pm->pmap_pvo, pvo)) {
1354 if (PVO_IS_SP(pvo)) {
1355 if (moea64_sp_pvo_in_range(pvo, sva, eva)) {
1356 pvo = moea64_sp_unwire(pvo);
1357 continue;
1358 } else {
1359 CTR1(KTR_PMAP, "%s: demote before unwire",
1360 __func__);
1361 moea64_sp_demote(pvo);
1362 }
1363 }
1364
1365 if ((pvo->pvo_vaddr & PVO_WIRED) == 0)
1366 panic("moea64_unwire: pvo %p is missing PVO_WIRED",
1367 pvo);
1368 pvo->pvo_vaddr &= ~PVO_WIRED;
1369 refchg = moea64_pte_replace(pvo, 0 /* No invalidation */);
1370 if ((pvo->pvo_vaddr & PVO_MANAGED) &&
1371 (pvo->pvo_pte.prot & VM_PROT_WRITE)) {
1372 if (refchg < 0)
1373 refchg = LPTE_CHG;
1374 m = PHYS_TO_VM_PAGE(PVO_PADDR(pvo));
1375
1376 refchg |= atomic_readandclear_32(&m->md.mdpg_attrs);
1377 if (refchg & LPTE_CHG)
1378 vm_page_dirty(m);
1379 if (refchg & LPTE_REF)
1380 vm_page_aflag_set(m, PGA_REFERENCED);
1381 }
1382 pm->pm_stats.wired_count--;
1383 }
1384 PMAP_UNLOCK(pm);
1385 }
1386
1387 static int
moea64_mincore(pmap_t pmap,vm_offset_t addr,vm_paddr_t * pap)1388 moea64_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *pap)
1389 {
1390 struct pvo_entry *pvo;
1391 vm_paddr_t pa;
1392 vm_page_t m;
1393 int val;
1394 bool managed;
1395
1396 PMAP_LOCK(pmap);
1397
1398 pvo = moea64_pvo_find_va(pmap, addr);
1399 if (pvo != NULL) {
1400 pa = PVO_PADDR(pvo);
1401 m = PHYS_TO_VM_PAGE(pa);
1402 managed = (pvo->pvo_vaddr & PVO_MANAGED) == PVO_MANAGED;
1403 if (PVO_IS_SP(pvo))
1404 val = MINCORE_INCORE | MINCORE_PSIND(1);
1405 else
1406 val = MINCORE_INCORE;
1407 } else {
1408 PMAP_UNLOCK(pmap);
1409 return (0);
1410 }
1411
1412 PMAP_UNLOCK(pmap);
1413
1414 if (m == NULL)
1415 return (0);
1416
1417 if (managed) {
1418 if (moea64_is_modified(m))
1419 val |= MINCORE_MODIFIED | MINCORE_MODIFIED_OTHER;
1420
1421 if (moea64_is_referenced(m))
1422 val |= MINCORE_REFERENCED | MINCORE_REFERENCED_OTHER;
1423 }
1424
1425 if ((val & (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER)) !=
1426 (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER) &&
1427 managed) {
1428 *pap = pa;
1429 }
1430
1431 return (val);
1432 }
1433
1434 /*
1435 * This goes through and sets the physical address of our
1436 * special scratch PTE to the PA we want to zero or copy. Because
1437 * of locking issues (this can get called in pvo_enter() by
1438 * the UMA allocator), we can't use most other utility functions here
1439 */
1440
1441 static __inline
moea64_set_scratchpage_pa(int which,vm_paddr_t pa)1442 void moea64_set_scratchpage_pa(int which, vm_paddr_t pa)
1443 {
1444 struct pvo_entry *pvo;
1445
1446 KASSERT(!hw_direct_map, ("Using OEA64 scratchpage with a direct map!"));
1447 mtx_assert(&moea64_scratchpage_mtx, MA_OWNED);
1448
1449 pvo = moea64_scratchpage_pvo[which];
1450 PMAP_LOCK(pvo->pvo_pmap);
1451 pvo->pvo_pte.pa =
1452 moea64_calc_wimg(pa, VM_MEMATTR_DEFAULT) | (uint64_t)pa;
1453 moea64_pte_replace(pvo, MOEA64_PTE_INVALIDATE);
1454 PMAP_UNLOCK(pvo->pvo_pmap);
1455 isync();
1456 }
1457
1458 void
moea64_copy_page(vm_page_t msrc,vm_page_t mdst)1459 moea64_copy_page(vm_page_t msrc, vm_page_t mdst)
1460 {
1461 mtx_lock(&moea64_scratchpage_mtx);
1462
1463 moea64_set_scratchpage_pa(0, VM_PAGE_TO_PHYS(msrc));
1464 moea64_set_scratchpage_pa(1, VM_PAGE_TO_PHYS(mdst));
1465
1466 bcopy((void *)moea64_scratchpage_va[0],
1467 (void *)moea64_scratchpage_va[1], PAGE_SIZE);
1468
1469 mtx_unlock(&moea64_scratchpage_mtx);
1470 }
1471
1472 void
moea64_copy_page_dmap(vm_page_t msrc,vm_page_t mdst)1473 moea64_copy_page_dmap(vm_page_t msrc, vm_page_t mdst)
1474 {
1475 vm_offset_t dst;
1476 vm_offset_t src;
1477
1478 dst = VM_PAGE_TO_PHYS(mdst);
1479 src = VM_PAGE_TO_PHYS(msrc);
1480
1481 bcopy((void *)PHYS_TO_DMAP(src), (void *)PHYS_TO_DMAP(dst),
1482 PAGE_SIZE);
1483 }
1484
1485 inline void
moea64_copy_pages_dmap(vm_page_t * ma,vm_offset_t a_offset,vm_page_t * mb,vm_offset_t b_offset,int xfersize)1486 moea64_copy_pages_dmap(vm_page_t *ma, vm_offset_t a_offset,
1487 vm_page_t *mb, vm_offset_t b_offset, int xfersize)
1488 {
1489 void *a_cp, *b_cp;
1490 vm_offset_t a_pg_offset, b_pg_offset;
1491 int cnt;
1492
1493 while (xfersize > 0) {
1494 a_pg_offset = a_offset & PAGE_MASK;
1495 cnt = min(xfersize, PAGE_SIZE - a_pg_offset);
1496 a_cp = (char *)(uintptr_t)PHYS_TO_DMAP(
1497 VM_PAGE_TO_PHYS(ma[a_offset >> PAGE_SHIFT])) +
1498 a_pg_offset;
1499 b_pg_offset = b_offset & PAGE_MASK;
1500 cnt = min(cnt, PAGE_SIZE - b_pg_offset);
1501 b_cp = (char *)(uintptr_t)PHYS_TO_DMAP(
1502 VM_PAGE_TO_PHYS(mb[b_offset >> PAGE_SHIFT])) +
1503 b_pg_offset;
1504 bcopy(a_cp, b_cp, cnt);
1505 a_offset += cnt;
1506 b_offset += cnt;
1507 xfersize -= cnt;
1508 }
1509 }
1510
1511 void
moea64_copy_pages(vm_page_t * ma,vm_offset_t a_offset,vm_page_t * mb,vm_offset_t b_offset,int xfersize)1512 moea64_copy_pages(vm_page_t *ma, vm_offset_t a_offset,
1513 vm_page_t *mb, vm_offset_t b_offset, int xfersize)
1514 {
1515 void *a_cp, *b_cp;
1516 vm_offset_t a_pg_offset, b_pg_offset;
1517 int cnt;
1518
1519 mtx_lock(&moea64_scratchpage_mtx);
1520 while (xfersize > 0) {
1521 a_pg_offset = a_offset & PAGE_MASK;
1522 cnt = min(xfersize, PAGE_SIZE - a_pg_offset);
1523 moea64_set_scratchpage_pa(0,
1524 VM_PAGE_TO_PHYS(ma[a_offset >> PAGE_SHIFT]));
1525 a_cp = (char *)moea64_scratchpage_va[0] + a_pg_offset;
1526 b_pg_offset = b_offset & PAGE_MASK;
1527 cnt = min(cnt, PAGE_SIZE - b_pg_offset);
1528 moea64_set_scratchpage_pa(1,
1529 VM_PAGE_TO_PHYS(mb[b_offset >> PAGE_SHIFT]));
1530 b_cp = (char *)moea64_scratchpage_va[1] + b_pg_offset;
1531 bcopy(a_cp, b_cp, cnt);
1532 a_offset += cnt;
1533 b_offset += cnt;
1534 xfersize -= cnt;
1535 }
1536 mtx_unlock(&moea64_scratchpage_mtx);
1537 }
1538
1539 void
moea64_zero_page_area(vm_page_t m,int off,int size)1540 moea64_zero_page_area(vm_page_t m, int off, int size)
1541 {
1542 vm_paddr_t pa = VM_PAGE_TO_PHYS(m);
1543
1544 if (size + off > PAGE_SIZE)
1545 panic("moea64_zero_page: size + off > PAGE_SIZE");
1546
1547 if (hw_direct_map) {
1548 bzero((caddr_t)(uintptr_t)PHYS_TO_DMAP(pa) + off, size);
1549 } else {
1550 mtx_lock(&moea64_scratchpage_mtx);
1551 moea64_set_scratchpage_pa(0, pa);
1552 bzero((caddr_t)moea64_scratchpage_va[0] + off, size);
1553 mtx_unlock(&moea64_scratchpage_mtx);
1554 }
1555 }
1556
1557 /*
1558 * Zero a page of physical memory by temporarily mapping it
1559 */
1560 void
moea64_zero_page(vm_page_t m)1561 moea64_zero_page(vm_page_t m)
1562 {
1563 vm_paddr_t pa = VM_PAGE_TO_PHYS(m);
1564 vm_offset_t va, off;
1565
1566 mtx_lock(&moea64_scratchpage_mtx);
1567
1568 moea64_set_scratchpage_pa(0, pa);
1569 va = moea64_scratchpage_va[0];
1570
1571 for (off = 0; off < PAGE_SIZE; off += cacheline_size)
1572 __asm __volatile("dcbz 0,%0" :: "r"(va + off));
1573
1574 mtx_unlock(&moea64_scratchpage_mtx);
1575 }
1576
1577 void
moea64_zero_page_dmap(vm_page_t m)1578 moea64_zero_page_dmap(vm_page_t m)
1579 {
1580 vm_paddr_t pa = VM_PAGE_TO_PHYS(m);
1581 vm_offset_t va, off;
1582
1583 va = PHYS_TO_DMAP(pa);
1584 for (off = 0; off < PAGE_SIZE; off += cacheline_size)
1585 __asm __volatile("dcbz 0,%0" :: "r"(va + off));
1586 }
1587
1588 vm_offset_t
moea64_quick_enter_page(vm_page_t m)1589 moea64_quick_enter_page(vm_page_t m)
1590 {
1591 struct pvo_entry *pvo;
1592 vm_paddr_t pa = VM_PAGE_TO_PHYS(m);
1593
1594 /*
1595 * MOEA64_PTE_REPLACE does some locking, so we can't just grab
1596 * a critical section and access the PCPU data like on i386.
1597 * Instead, pin the thread and grab the PCPU lock to prevent
1598 * a preempting thread from using the same PCPU data.
1599 */
1600 sched_pin();
1601
1602 mtx_assert(PCPU_PTR(aim.qmap_lock), MA_NOTOWNED);
1603 pvo = PCPU_GET(aim.qmap_pvo);
1604
1605 mtx_lock(PCPU_PTR(aim.qmap_lock));
1606 pvo->pvo_pte.pa = moea64_calc_wimg(pa, pmap_page_get_memattr(m)) |
1607 (uint64_t)pa;
1608 moea64_pte_replace(pvo, MOEA64_PTE_INVALIDATE);
1609 isync();
1610
1611 return (PCPU_GET(qmap_addr));
1612 }
1613
1614 vm_offset_t
moea64_quick_enter_page_dmap(vm_page_t m)1615 moea64_quick_enter_page_dmap(vm_page_t m)
1616 {
1617
1618 return (PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)));
1619 }
1620
1621 void
moea64_quick_remove_page(vm_offset_t addr)1622 moea64_quick_remove_page(vm_offset_t addr)
1623 {
1624
1625 mtx_assert(PCPU_PTR(aim.qmap_lock), MA_OWNED);
1626 KASSERT(PCPU_GET(qmap_addr) == addr,
1627 ("moea64_quick_remove_page: invalid address"));
1628 mtx_unlock(PCPU_PTR(aim.qmap_lock));
1629 sched_unpin();
1630 }
1631
1632 bool
moea64_page_is_mapped(vm_page_t m)1633 moea64_page_is_mapped(vm_page_t m)
1634 {
1635 return (!LIST_EMPTY(&(m)->md.mdpg_pvoh));
1636 }
1637
1638 /*
1639 * Map the given physical page at the specified virtual address in the
1640 * target pmap with the protection requested. If specified the page
1641 * will be wired down.
1642 */
1643
1644 int
moea64_enter(pmap_t pmap,vm_offset_t va,vm_page_t m,vm_prot_t prot,u_int flags,int8_t psind)1645 moea64_enter(pmap_t pmap, vm_offset_t va, vm_page_t m,
1646 vm_prot_t prot, u_int flags, int8_t psind)
1647 {
1648 struct pvo_entry *pvo, *oldpvo, *tpvo;
1649 struct pvo_head *pvo_head;
1650 uint64_t pte_lo;
1651 int error;
1652 vm_paddr_t pa;
1653
1654 if ((m->oflags & VPO_UNMANAGED) == 0) {
1655 if ((flags & PMAP_ENTER_QUICK_LOCKED) == 0)
1656 VM_PAGE_OBJECT_BUSY_ASSERT(m);
1657 else
1658 VM_OBJECT_ASSERT_LOCKED(m->object);
1659 }
1660
1661 if (psind > 0)
1662 return (moea64_sp_enter(pmap, va, m, prot, flags, psind));
1663
1664 pvo = alloc_pvo_entry(0);
1665 if (pvo == NULL)
1666 return (KERN_RESOURCE_SHORTAGE);
1667 pvo->pvo_pmap = NULL; /* to be filled in later */
1668 pvo->pvo_pte.prot = prot;
1669
1670 pa = VM_PAGE_TO_PHYS(m);
1671 pte_lo = moea64_calc_wimg(pa, pmap_page_get_memattr(m));
1672 pvo->pvo_pte.pa = pa | pte_lo;
1673
1674 if ((flags & PMAP_ENTER_WIRED) != 0)
1675 pvo->pvo_vaddr |= PVO_WIRED;
1676
1677 if ((m->oflags & VPO_UNMANAGED) != 0 || !moea64_initialized) {
1678 pvo_head = NULL;
1679 } else {
1680 pvo_head = &m->md.mdpg_pvoh;
1681 pvo->pvo_vaddr |= PVO_MANAGED;
1682 }
1683
1684 PV_LOCK(pa);
1685 PMAP_LOCK(pmap);
1686 if (pvo->pvo_pmap == NULL)
1687 init_pvo_entry(pvo, pmap, va);
1688
1689 if (moea64_ps_enabled(pmap) &&
1690 (tpvo = moea64_pvo_find_va(pmap, va & ~HPT_SP_MASK)) != NULL &&
1691 PVO_IS_SP(tpvo)) {
1692 /* Demote SP before entering a regular page */
1693 CTR2(KTR_PMAP, "%s: demote before enter: va=%#jx",
1694 __func__, (uintmax_t)va);
1695 moea64_sp_demote_aligned(tpvo);
1696 }
1697
1698 if (prot & VM_PROT_WRITE)
1699 if (pmap_bootstrapped &&
1700 (m->oflags & VPO_UNMANAGED) == 0)
1701 vm_page_aflag_set(m, PGA_WRITEABLE);
1702
1703 error = moea64_pvo_enter(pvo, pvo_head, &oldpvo);
1704 if (error == EEXIST) {
1705 if (oldpvo->pvo_vaddr == pvo->pvo_vaddr &&
1706 oldpvo->pvo_pte.pa == pvo->pvo_pte.pa &&
1707 oldpvo->pvo_pte.prot == prot) {
1708 /* Identical mapping already exists */
1709 error = 0;
1710
1711 /* If not in page table, reinsert it */
1712 if (moea64_pte_synch(oldpvo) < 0) {
1713 STAT_MOEA64(moea64_pte_overflow--);
1714 moea64_pte_insert(oldpvo);
1715 }
1716
1717 /* Then just clean up and go home */
1718 PMAP_UNLOCK(pmap);
1719 PV_UNLOCK(pa);
1720 free_pvo_entry(pvo);
1721 pvo = NULL;
1722 goto out;
1723 } else {
1724 /* Otherwise, need to kill it first */
1725 KASSERT(oldpvo->pvo_pmap == pmap, ("pmap of old "
1726 "mapping does not match new mapping"));
1727 moea64_pvo_remove_from_pmap(oldpvo);
1728 moea64_pvo_enter(pvo, pvo_head, NULL);
1729 }
1730 }
1731 PMAP_UNLOCK(pmap);
1732 PV_UNLOCK(pa);
1733
1734 /* Free any dead pages */
1735 if (error == EEXIST) {
1736 moea64_pvo_remove_from_page(oldpvo);
1737 free_pvo_entry(oldpvo);
1738 }
1739
1740 out:
1741 /*
1742 * Flush the page from the instruction cache if this page is
1743 * mapped executable and cacheable.
1744 */
1745 if (pmap != kernel_pmap && (m->a.flags & PGA_EXECUTABLE) == 0 &&
1746 (pte_lo & (LPTE_I | LPTE_G | LPTE_NOEXEC)) == 0) {
1747 vm_page_aflag_set(m, PGA_EXECUTABLE);
1748 moea64_syncicache(pmap, va, pa, PAGE_SIZE);
1749 }
1750
1751 #if VM_NRESERVLEVEL > 0
1752 /*
1753 * Try to promote pages.
1754 *
1755 * If the VA of the entered page is not aligned with its PA,
1756 * don't try page promotion as it is not possible.
1757 * This reduces the number of promotion failures dramatically.
1758 *
1759 * Ignore VM_PROT_NO_PROMOTE unless PMAP_ENTER_QUICK_LOCKED.
1760 */
1761 if (moea64_ps_enabled(pmap) && pmap != kernel_pmap && pvo != NULL &&
1762 (pvo->pvo_vaddr & PVO_MANAGED) != 0 &&
1763 (va & HPT_SP_MASK) == (pa & HPT_SP_MASK) &&
1764 ((prot & VM_PROT_NO_PROMOTE) == 0 ||
1765 (flags & PMAP_ENTER_QUICK_LOCKED) == 0) &&
1766 (m->flags & PG_FICTITIOUS) == 0 &&
1767 vm_reserv_level_iffullpop(m) == 0)
1768 moea64_sp_promote(pmap, va, m);
1769 #endif
1770
1771 return (KERN_SUCCESS);
1772 }
1773
1774 static void
moea64_syncicache(pmap_t pmap,vm_offset_t va,vm_paddr_t pa,vm_size_t sz)1775 moea64_syncicache(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,
1776 vm_size_t sz)
1777 {
1778
1779 /*
1780 * This is much trickier than on older systems because
1781 * we can't sync the icache on physical addresses directly
1782 * without a direct map. Instead we check a couple of cases
1783 * where the memory is already mapped in and, failing that,
1784 * use the same trick we use for page zeroing to create
1785 * a temporary mapping for this physical address.
1786 */
1787
1788 if (!pmap_bootstrapped) {
1789 /*
1790 * If PMAP is not bootstrapped, we are likely to be
1791 * in real mode.
1792 */
1793 __syncicache((void *)(uintptr_t)pa, sz);
1794 } else if (pmap == kernel_pmap) {
1795 __syncicache((void *)va, sz);
1796 } else if (hw_direct_map) {
1797 __syncicache((void *)(uintptr_t)PHYS_TO_DMAP(pa), sz);
1798 } else {
1799 /* Use the scratch page to set up a temp mapping */
1800
1801 mtx_lock(&moea64_scratchpage_mtx);
1802
1803 moea64_set_scratchpage_pa(1, pa & ~ADDR_POFF);
1804 __syncicache((void *)(moea64_scratchpage_va[1] +
1805 (va & ADDR_POFF)), sz);
1806
1807 mtx_unlock(&moea64_scratchpage_mtx);
1808 }
1809 }
1810
1811 /*
1812 * Maps a sequence of resident pages belonging to the same object.
1813 * The sequence begins with the given page m_start. This page is
1814 * mapped at the given virtual address start. Each subsequent page is
1815 * mapped at a virtual address that is offset from start by the same
1816 * amount as the page is offset from m_start within the object. The
1817 * last page in the sequence is the page with the largest offset from
1818 * m_start that can be mapped at a virtual address less than the given
1819 * virtual address end. Not every virtual page between start and end
1820 * is mapped; only those for which a resident page exists with the
1821 * corresponding offset from m_start are mapped.
1822 */
1823 void
moea64_enter_object(pmap_t pm,vm_offset_t start,vm_offset_t end,vm_page_t m_start,vm_prot_t prot)1824 moea64_enter_object(pmap_t pm, vm_offset_t start, vm_offset_t end,
1825 vm_page_t m_start, vm_prot_t prot)
1826 {
1827 vm_page_t m;
1828 vm_pindex_t diff, psize;
1829 vm_offset_t va;
1830 int8_t psind;
1831
1832 VM_OBJECT_ASSERT_LOCKED(m_start->object);
1833
1834 psize = atop(end - start);
1835 m = m_start;
1836 while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) {
1837 va = start + ptoa(diff);
1838 if ((va & HPT_SP_MASK) == 0 && va + HPT_SP_SIZE <= end &&
1839 m->psind == 1 && moea64_ps_enabled(pm))
1840 psind = 1;
1841 else
1842 psind = 0;
1843 moea64_enter(pm, va, m, prot &
1844 (VM_PROT_READ | VM_PROT_EXECUTE),
1845 PMAP_ENTER_NOSLEEP | PMAP_ENTER_QUICK_LOCKED, psind);
1846 if (psind == 1)
1847 m = &m[HPT_SP_SIZE / PAGE_SIZE - 1];
1848 m = TAILQ_NEXT(m, listq);
1849 }
1850 }
1851
1852 void
moea64_enter_quick(pmap_t pm,vm_offset_t va,vm_page_t m,vm_prot_t prot)1853 moea64_enter_quick(pmap_t pm, vm_offset_t va, vm_page_t m,
1854 vm_prot_t prot)
1855 {
1856
1857 moea64_enter(pm, va, m, prot & (VM_PROT_READ | VM_PROT_EXECUTE |
1858 VM_PROT_NO_PROMOTE), PMAP_ENTER_NOSLEEP | PMAP_ENTER_QUICK_LOCKED,
1859 0);
1860 }
1861
1862 vm_paddr_t
moea64_extract(pmap_t pm,vm_offset_t va)1863 moea64_extract(pmap_t pm, vm_offset_t va)
1864 {
1865 struct pvo_entry *pvo;
1866 vm_paddr_t pa;
1867
1868 PMAP_LOCK(pm);
1869 pvo = moea64_pvo_find_va(pm, va);
1870 if (pvo == NULL)
1871 pa = 0;
1872 else
1873 pa = PVO_PADDR(pvo) | (va - PVO_VADDR(pvo));
1874 PMAP_UNLOCK(pm);
1875
1876 return (pa);
1877 }
1878
1879 /*
1880 * Atomically extract and hold the physical page with the given
1881 * pmap and virtual address pair if that mapping permits the given
1882 * protection.
1883 */
1884 vm_page_t
moea64_extract_and_hold(pmap_t pmap,vm_offset_t va,vm_prot_t prot)1885 moea64_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot)
1886 {
1887 struct pvo_entry *pvo;
1888 vm_page_t m;
1889
1890 m = NULL;
1891 PMAP_LOCK(pmap);
1892 pvo = moea64_pvo_find_va(pmap, va & ~ADDR_POFF);
1893 if (pvo != NULL && (pvo->pvo_pte.prot & prot) == prot) {
1894 m = PHYS_TO_VM_PAGE(PVO_PADDR(pvo));
1895 if (!vm_page_wire_mapped(m))
1896 m = NULL;
1897 }
1898 PMAP_UNLOCK(pmap);
1899 return (m);
1900 }
1901
1902 static void *
moea64_uma_page_alloc(uma_zone_t zone,vm_size_t bytes,int domain,uint8_t * flags,int wait)1903 moea64_uma_page_alloc(uma_zone_t zone, vm_size_t bytes, int domain,
1904 uint8_t *flags, int wait)
1905 {
1906 struct pvo_entry *pvo;
1907 vm_offset_t va;
1908 vm_page_t m;
1909 int needed_lock;
1910
1911 /*
1912 * This entire routine is a horrible hack to avoid bothering kmem
1913 * for new KVA addresses. Because this can get called from inside
1914 * kmem allocation routines, calling kmem for a new address here
1915 * can lead to multiply locking non-recursive mutexes.
1916 */
1917
1918 *flags = UMA_SLAB_PRIV;
1919 needed_lock = !PMAP_LOCKED(kernel_pmap);
1920
1921 m = vm_page_alloc_noobj_domain(domain, malloc2vm_flags(wait) |
1922 VM_ALLOC_WIRED);
1923 if (m == NULL)
1924 return (NULL);
1925
1926 va = VM_PAGE_TO_PHYS(m);
1927
1928 pvo = alloc_pvo_entry(1 /* bootstrap */);
1929
1930 pvo->pvo_pte.prot = VM_PROT_READ | VM_PROT_WRITE;
1931 pvo->pvo_pte.pa = VM_PAGE_TO_PHYS(m) | LPTE_M;
1932
1933 if (needed_lock)
1934 PMAP_LOCK(kernel_pmap);
1935
1936 init_pvo_entry(pvo, kernel_pmap, va);
1937 pvo->pvo_vaddr |= PVO_WIRED;
1938
1939 moea64_pvo_enter(pvo, NULL, NULL);
1940
1941 if (needed_lock)
1942 PMAP_UNLOCK(kernel_pmap);
1943
1944 return (void *)va;
1945 }
1946
1947 extern int elf32_nxstack;
1948
1949 void
moea64_init(void)1950 moea64_init(void)
1951 {
1952
1953 CTR0(KTR_PMAP, "moea64_init");
1954
1955 moea64_pvo_zone = uma_zcreate("UPVO entry", sizeof (struct pvo_entry),
1956 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR,
1957 UMA_ZONE_VM | UMA_ZONE_NOFREE);
1958
1959 /*
1960 * Are large page mappings enabled?
1961 *
1962 * While HPT superpages are not better tested, leave it disabled by
1963 * default.
1964 */
1965 superpages_enabled = 0;
1966 TUNABLE_INT_FETCH("vm.pmap.superpages_enabled", &superpages_enabled);
1967 if (superpages_enabled) {
1968 KASSERT(MAXPAGESIZES > 1 && pagesizes[1] == 0,
1969 ("moea64_init: can't assign to pagesizes[1]"));
1970
1971 if (moea64_large_page_size == 0) {
1972 printf("mmu_oea64: HW does not support large pages. "
1973 "Disabling superpages...\n");
1974 superpages_enabled = 0;
1975 } else if (!moea64_has_lp_4k_16m) {
1976 printf("mmu_oea64: "
1977 "HW does not support mixed 4KB/16MB page sizes. "
1978 "Disabling superpages...\n");
1979 superpages_enabled = 0;
1980 } else
1981 pagesizes[1] = HPT_SP_SIZE;
1982 }
1983
1984 if (!hw_direct_map) {
1985 uma_zone_set_allocf(moea64_pvo_zone, moea64_uma_page_alloc);
1986 }
1987
1988 #ifdef COMPAT_FREEBSD32
1989 elf32_nxstack = 1;
1990 #endif
1991
1992 moea64_initialized = true;
1993 }
1994
1995 bool
moea64_is_referenced(vm_page_t m)1996 moea64_is_referenced(vm_page_t m)
1997 {
1998
1999 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
2000 ("moea64_is_referenced: page %p is not managed", m));
2001
2002 return (moea64_query_bit(m, LPTE_REF));
2003 }
2004
2005 bool
moea64_is_modified(vm_page_t m)2006 moea64_is_modified(vm_page_t m)
2007 {
2008
2009 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
2010 ("moea64_is_modified: page %p is not managed", m));
2011
2012 /*
2013 * If the page is not busied then this check is racy.
2014 */
2015 if (!pmap_page_is_write_mapped(m))
2016 return (false);
2017
2018 return (moea64_query_bit(m, LPTE_CHG));
2019 }
2020
2021 bool
moea64_is_prefaultable(pmap_t pmap,vm_offset_t va)2022 moea64_is_prefaultable(pmap_t pmap, vm_offset_t va)
2023 {
2024 struct pvo_entry *pvo;
2025 bool rv = true;
2026
2027 PMAP_LOCK(pmap);
2028 pvo = moea64_pvo_find_va(pmap, va & ~ADDR_POFF);
2029 if (pvo != NULL)
2030 rv = false;
2031 PMAP_UNLOCK(pmap);
2032 return (rv);
2033 }
2034
2035 void
moea64_clear_modify(vm_page_t m)2036 moea64_clear_modify(vm_page_t m)
2037 {
2038
2039 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
2040 ("moea64_clear_modify: page %p is not managed", m));
2041 vm_page_assert_busied(m);
2042
2043 if (!pmap_page_is_write_mapped(m))
2044 return;
2045 moea64_clear_bit(m, LPTE_CHG);
2046 }
2047
2048 /*
2049 * Clear the write and modified bits in each of the given page's mappings.
2050 */
2051 void
moea64_remove_write(vm_page_t m)2052 moea64_remove_write(vm_page_t m)
2053 {
2054 struct pvo_entry *pvo;
2055 int64_t refchg, ret;
2056 pmap_t pmap;
2057
2058 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
2059 ("moea64_remove_write: page %p is not managed", m));
2060 vm_page_assert_busied(m);
2061
2062 if (!pmap_page_is_write_mapped(m))
2063 return;
2064
2065 powerpc_sync();
2066 PV_PAGE_LOCK(m);
2067 refchg = 0;
2068 LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) {
2069 pmap = pvo->pvo_pmap;
2070 PMAP_LOCK(pmap);
2071 if (!(pvo->pvo_vaddr & PVO_DEAD) &&
2072 (pvo->pvo_pte.prot & VM_PROT_WRITE)) {
2073 if (PVO_IS_SP(pvo)) {
2074 CTR1(KTR_PMAP, "%s: demote before remwr",
2075 __func__);
2076 moea64_sp_demote(pvo);
2077 }
2078 pvo->pvo_pte.prot &= ~VM_PROT_WRITE;
2079 ret = moea64_pte_replace(pvo, MOEA64_PTE_PROT_UPDATE);
2080 if (ret < 0)
2081 ret = LPTE_CHG;
2082 refchg |= ret;
2083 if (pvo->pvo_pmap == kernel_pmap)
2084 isync();
2085 }
2086 PMAP_UNLOCK(pmap);
2087 }
2088 if ((refchg | atomic_readandclear_32(&m->md.mdpg_attrs)) & LPTE_CHG)
2089 vm_page_dirty(m);
2090 vm_page_aflag_clear(m, PGA_WRITEABLE);
2091 PV_PAGE_UNLOCK(m);
2092 }
2093
2094 /*
2095 * moea64_ts_referenced:
2096 *
2097 * Return a count of reference bits for a page, clearing those bits.
2098 * It is not necessary for every reference bit to be cleared, but it
2099 * is necessary that 0 only be returned when there are truly no
2100 * reference bits set.
2101 *
2102 * XXX: The exact number of bits to check and clear is a matter that
2103 * should be tested and standardized at some point in the future for
2104 * optimal aging of shared pages.
2105 */
2106 int
moea64_ts_referenced(vm_page_t m)2107 moea64_ts_referenced(vm_page_t m)
2108 {
2109
2110 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
2111 ("moea64_ts_referenced: page %p is not managed", m));
2112 return (moea64_clear_bit(m, LPTE_REF));
2113 }
2114
2115 /*
2116 * Modify the WIMG settings of all mappings for a page.
2117 */
2118 void
moea64_page_set_memattr(vm_page_t m,vm_memattr_t ma)2119 moea64_page_set_memattr(vm_page_t m, vm_memattr_t ma)
2120 {
2121 struct pvo_entry *pvo;
2122 int64_t refchg;
2123 pmap_t pmap;
2124 uint64_t lo;
2125
2126 CTR3(KTR_PMAP, "%s: pa=%#jx, ma=%#x",
2127 __func__, (uintmax_t)VM_PAGE_TO_PHYS(m), ma);
2128
2129 if ((m->oflags & VPO_UNMANAGED) != 0) {
2130 m->md.mdpg_cache_attrs = ma;
2131 return;
2132 }
2133
2134 lo = moea64_calc_wimg(VM_PAGE_TO_PHYS(m), ma);
2135
2136 PV_PAGE_LOCK(m);
2137 LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) {
2138 pmap = pvo->pvo_pmap;
2139 PMAP_LOCK(pmap);
2140 if (!(pvo->pvo_vaddr & PVO_DEAD)) {
2141 if (PVO_IS_SP(pvo)) {
2142 CTR1(KTR_PMAP,
2143 "%s: demote before set_memattr", __func__);
2144 moea64_sp_demote(pvo);
2145 }
2146 pvo->pvo_pte.pa &= ~LPTE_WIMG;
2147 pvo->pvo_pte.pa |= lo;
2148 refchg = moea64_pte_replace(pvo, MOEA64_PTE_INVALIDATE);
2149 if (refchg < 0)
2150 refchg = (pvo->pvo_pte.prot & VM_PROT_WRITE) ?
2151 LPTE_CHG : 0;
2152 if ((pvo->pvo_vaddr & PVO_MANAGED) &&
2153 (pvo->pvo_pte.prot & VM_PROT_WRITE)) {
2154 refchg |=
2155 atomic_readandclear_32(&m->md.mdpg_attrs);
2156 if (refchg & LPTE_CHG)
2157 vm_page_dirty(m);
2158 if (refchg & LPTE_REF)
2159 vm_page_aflag_set(m, PGA_REFERENCED);
2160 }
2161 if (pvo->pvo_pmap == kernel_pmap)
2162 isync();
2163 }
2164 PMAP_UNLOCK(pmap);
2165 }
2166 m->md.mdpg_cache_attrs = ma;
2167 PV_PAGE_UNLOCK(m);
2168 }
2169
2170 /*
2171 * Map a wired page into kernel virtual address space.
2172 */
2173 void
moea64_kenter_attr(vm_offset_t va,vm_paddr_t pa,vm_memattr_t ma)2174 moea64_kenter_attr(vm_offset_t va, vm_paddr_t pa, vm_memattr_t ma)
2175 {
2176 int error;
2177 struct pvo_entry *pvo, *oldpvo;
2178
2179 do {
2180 pvo = alloc_pvo_entry(0);
2181 if (pvo == NULL)
2182 vm_wait(NULL);
2183 } while (pvo == NULL);
2184 pvo->pvo_pte.prot = VM_PROT_READ | VM_PROT_WRITE | VM_PROT_EXECUTE;
2185 pvo->pvo_pte.pa = (pa & ~ADDR_POFF) | moea64_calc_wimg(pa, ma);
2186 pvo->pvo_vaddr |= PVO_WIRED;
2187
2188 PMAP_LOCK(kernel_pmap);
2189 oldpvo = moea64_pvo_find_va(kernel_pmap, va);
2190 if (oldpvo != NULL)
2191 moea64_pvo_remove_from_pmap(oldpvo);
2192 init_pvo_entry(pvo, kernel_pmap, va);
2193 error = moea64_pvo_enter(pvo, NULL, NULL);
2194 PMAP_UNLOCK(kernel_pmap);
2195
2196 /* Free any dead pages */
2197 if (oldpvo != NULL) {
2198 moea64_pvo_remove_from_page(oldpvo);
2199 free_pvo_entry(oldpvo);
2200 }
2201
2202 if (error != 0)
2203 panic("moea64_kenter: failed to enter va %#zx pa %#jx: %d", va,
2204 (uintmax_t)pa, error);
2205 }
2206
2207 void
moea64_kenter(vm_offset_t va,vm_paddr_t pa)2208 moea64_kenter(vm_offset_t va, vm_paddr_t pa)
2209 {
2210
2211 moea64_kenter_attr(va, pa, VM_MEMATTR_DEFAULT);
2212 }
2213
2214 /*
2215 * Extract the physical page address associated with the given kernel virtual
2216 * address.
2217 */
2218 vm_paddr_t
moea64_kextract(vm_offset_t va)2219 moea64_kextract(vm_offset_t va)
2220 {
2221 struct pvo_entry *pvo;
2222 vm_paddr_t pa;
2223
2224 /*
2225 * Shortcut the direct-mapped case when applicable. We never put
2226 * anything but 1:1 (or 62-bit aliased) mappings below
2227 * VM_MIN_KERNEL_ADDRESS.
2228 */
2229 if (va < VM_MIN_KERNEL_ADDRESS)
2230 return (va & ~DMAP_BASE_ADDRESS);
2231
2232 PMAP_LOCK(kernel_pmap);
2233 pvo = moea64_pvo_find_va(kernel_pmap, va);
2234 KASSERT(pvo != NULL, ("moea64_kextract: no addr found for %#" PRIxPTR,
2235 va));
2236 pa = PVO_PADDR(pvo) | (va - PVO_VADDR(pvo));
2237 PMAP_UNLOCK(kernel_pmap);
2238 return (pa);
2239 }
2240
2241 /*
2242 * Remove a wired page from kernel virtual address space.
2243 */
2244 void
moea64_kremove(vm_offset_t va)2245 moea64_kremove(vm_offset_t va)
2246 {
2247 moea64_remove(kernel_pmap, va, va + PAGE_SIZE);
2248 }
2249
2250 /*
2251 * Provide a kernel pointer corresponding to a given userland pointer.
2252 * The returned pointer is valid until the next time this function is
2253 * called in this thread. This is used internally in copyin/copyout.
2254 */
2255 static int
moea64_map_user_ptr(pmap_t pm,volatile const void * uaddr,void ** kaddr,size_t ulen,size_t * klen)2256 moea64_map_user_ptr(pmap_t pm, volatile const void *uaddr,
2257 void **kaddr, size_t ulen, size_t *klen)
2258 {
2259 size_t l;
2260 #ifdef __powerpc64__
2261 struct slb *slb;
2262 #endif
2263 register_t slbv;
2264
2265 *kaddr = (char *)USER_ADDR + ((uintptr_t)uaddr & ~SEGMENT_MASK);
2266 l = ((char *)USER_ADDR + SEGMENT_LENGTH) - (char *)(*kaddr);
2267 if (l > ulen)
2268 l = ulen;
2269 if (klen)
2270 *klen = l;
2271 else if (l != ulen)
2272 return (EFAULT);
2273
2274 #ifdef __powerpc64__
2275 /* Try lockless look-up first */
2276 slb = user_va_to_slb_entry(pm, (vm_offset_t)uaddr);
2277
2278 if (slb == NULL) {
2279 /* If it isn't there, we need to pre-fault the VSID */
2280 PMAP_LOCK(pm);
2281 slbv = va_to_vsid(pm, (vm_offset_t)uaddr) << SLBV_VSID_SHIFT;
2282 PMAP_UNLOCK(pm);
2283 } else {
2284 slbv = slb->slbv;
2285 }
2286
2287 /* Mark segment no-execute */
2288 slbv |= SLBV_N;
2289 #else
2290 slbv = va_to_vsid(pm, (vm_offset_t)uaddr);
2291
2292 /* Mark segment no-execute */
2293 slbv |= SR_N;
2294 #endif
2295
2296 /* If we have already set this VSID, we can just return */
2297 if (curthread->td_pcb->pcb_cpu.aim.usr_vsid == slbv)
2298 return (0);
2299
2300 __asm __volatile("isync");
2301 curthread->td_pcb->pcb_cpu.aim.usr_segm =
2302 (uintptr_t)uaddr >> ADDR_SR_SHFT;
2303 curthread->td_pcb->pcb_cpu.aim.usr_vsid = slbv;
2304 #ifdef __powerpc64__
2305 __asm __volatile ("slbie %0; slbmte %1, %2; isync" ::
2306 "r"(USER_ADDR), "r"(slbv), "r"(USER_SLB_SLBE));
2307 #else
2308 __asm __volatile("mtsr %0,%1; isync" :: "n"(USER_SR), "r"(slbv));
2309 #endif
2310
2311 return (0);
2312 }
2313
2314 /*
2315 * Figure out where a given kernel pointer (usually in a fault) points
2316 * to from the VM's perspective, potentially remapping into userland's
2317 * address space.
2318 */
2319 static int
moea64_decode_kernel_ptr(vm_offset_t addr,int * is_user,vm_offset_t * decoded_addr)2320 moea64_decode_kernel_ptr(vm_offset_t addr, int *is_user,
2321 vm_offset_t *decoded_addr)
2322 {
2323 vm_offset_t user_sr;
2324
2325 if ((addr >> ADDR_SR_SHFT) == (USER_ADDR >> ADDR_SR_SHFT)) {
2326 user_sr = curthread->td_pcb->pcb_cpu.aim.usr_segm;
2327 addr &= ADDR_PIDX | ADDR_POFF;
2328 addr |= user_sr << ADDR_SR_SHFT;
2329 *decoded_addr = addr;
2330 *is_user = 1;
2331 } else {
2332 *decoded_addr = addr;
2333 *is_user = 0;
2334 }
2335
2336 return (0);
2337 }
2338
2339 /*
2340 * Map a range of physical addresses into kernel virtual address space.
2341 *
2342 * The value passed in *virt is a suggested virtual address for the mapping.
2343 * Architectures which can support a direct-mapped physical to virtual region
2344 * can return the appropriate address within that region, leaving '*virt'
2345 * unchanged. Other architectures should map the pages starting at '*virt' and
2346 * update '*virt' with the first usable address after the mapped region.
2347 */
2348 vm_offset_t
moea64_map(vm_offset_t * virt,vm_paddr_t pa_start,vm_paddr_t pa_end,int prot)2349 moea64_map(vm_offset_t *virt, vm_paddr_t pa_start,
2350 vm_paddr_t pa_end, int prot)
2351 {
2352 vm_offset_t sva, va;
2353
2354 if (hw_direct_map) {
2355 /*
2356 * Check if every page in the region is covered by the direct
2357 * map. The direct map covers all of physical memory. Use
2358 * moea64_calc_wimg() as a shortcut to see if the page is in
2359 * physical memory as a way to see if the direct map covers it.
2360 */
2361 for (va = pa_start; va < pa_end; va += PAGE_SIZE)
2362 if (moea64_calc_wimg(va, VM_MEMATTR_DEFAULT) != LPTE_M)
2363 break;
2364 if (va == pa_end)
2365 return (PHYS_TO_DMAP(pa_start));
2366 }
2367 sva = *virt;
2368 va = sva;
2369 /* XXX respect prot argument */
2370 for (; pa_start < pa_end; pa_start += PAGE_SIZE, va += PAGE_SIZE)
2371 moea64_kenter(va, pa_start);
2372 *virt = va;
2373
2374 return (sva);
2375 }
2376
2377 /*
2378 * Returns true if the pmap's pv is one of the first
2379 * 16 pvs linked to from this page. This count may
2380 * be changed upwards or downwards in the future; it
2381 * is only necessary that true be returned for a small
2382 * subset of pmaps for proper page aging.
2383 */
2384 bool
moea64_page_exists_quick(pmap_t pmap,vm_page_t m)2385 moea64_page_exists_quick(pmap_t pmap, vm_page_t m)
2386 {
2387 int loops;
2388 struct pvo_entry *pvo;
2389 bool rv;
2390
2391 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
2392 ("moea64_page_exists_quick: page %p is not managed", m));
2393 loops = 0;
2394 rv = false;
2395 PV_PAGE_LOCK(m);
2396 LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) {
2397 if (!(pvo->pvo_vaddr & PVO_DEAD) && pvo->pvo_pmap == pmap) {
2398 rv = true;
2399 break;
2400 }
2401 if (++loops >= 16)
2402 break;
2403 }
2404 PV_PAGE_UNLOCK(m);
2405 return (rv);
2406 }
2407
2408 void
moea64_page_init(vm_page_t m)2409 moea64_page_init(vm_page_t m)
2410 {
2411
2412 m->md.mdpg_attrs = 0;
2413 m->md.mdpg_cache_attrs = VM_MEMATTR_DEFAULT;
2414 LIST_INIT(&m->md.mdpg_pvoh);
2415 }
2416
2417 /*
2418 * Return the number of managed mappings to the given physical page
2419 * that are wired.
2420 */
2421 int
moea64_page_wired_mappings(vm_page_t m)2422 moea64_page_wired_mappings(vm_page_t m)
2423 {
2424 struct pvo_entry *pvo;
2425 int count;
2426
2427 count = 0;
2428 if ((m->oflags & VPO_UNMANAGED) != 0)
2429 return (count);
2430 PV_PAGE_LOCK(m);
2431 LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink)
2432 if ((pvo->pvo_vaddr & (PVO_DEAD | PVO_WIRED)) == PVO_WIRED)
2433 count++;
2434 PV_PAGE_UNLOCK(m);
2435 return (count);
2436 }
2437
2438 static uintptr_t moea64_vsidcontext;
2439
2440 uintptr_t
moea64_get_unique_vsid(void)2441 moea64_get_unique_vsid(void) {
2442 u_int entropy;
2443 register_t hash;
2444 uint32_t mask;
2445 int i;
2446
2447 entropy = 0;
2448 __asm __volatile("mftb %0" : "=r"(entropy));
2449
2450 mtx_lock(&moea64_slb_mutex);
2451 for (i = 0; i < NVSIDS; i += VSID_NBPW) {
2452 u_int n;
2453
2454 /*
2455 * Create a new value by multiplying by a prime and adding in
2456 * entropy from the timebase register. This is to make the
2457 * VSID more random so that the PT hash function collides
2458 * less often. (Note that the prime casues gcc to do shifts
2459 * instead of a multiply.)
2460 */
2461 moea64_vsidcontext = (moea64_vsidcontext * 0x1105) + entropy;
2462 hash = moea64_vsidcontext & (NVSIDS - 1);
2463 if (hash == 0) /* 0 is special, avoid it */
2464 continue;
2465 n = hash >> 5;
2466 mask = 1 << (hash & (VSID_NBPW - 1));
2467 hash = (moea64_vsidcontext & VSID_HASHMASK);
2468 if (moea64_vsid_bitmap[n] & mask) { /* collision? */
2469 /* anything free in this bucket? */
2470 if (moea64_vsid_bitmap[n] == 0xffffffff) {
2471 entropy = (moea64_vsidcontext >> 20);
2472 continue;
2473 }
2474 i = ffs(~moea64_vsid_bitmap[n]) - 1;
2475 mask = 1 << i;
2476 hash &= rounddown2(VSID_HASHMASK, VSID_NBPW);
2477 hash |= i;
2478 }
2479 if (hash == VSID_VRMA) /* also special, avoid this too */
2480 continue;
2481 KASSERT(!(moea64_vsid_bitmap[n] & mask),
2482 ("Allocating in-use VSID %#zx\n", hash));
2483 moea64_vsid_bitmap[n] |= mask;
2484 mtx_unlock(&moea64_slb_mutex);
2485 return (hash);
2486 }
2487
2488 mtx_unlock(&moea64_slb_mutex);
2489 panic("%s: out of segments",__func__);
2490 }
2491
2492 #ifdef __powerpc64__
2493 int
moea64_pinit(pmap_t pmap)2494 moea64_pinit(pmap_t pmap)
2495 {
2496
2497 RB_INIT(&pmap->pmap_pvo);
2498
2499 pmap->pm_slb_tree_root = slb_alloc_tree();
2500 pmap->pm_slb = slb_alloc_user_cache();
2501 pmap->pm_slb_len = 0;
2502
2503 return (1);
2504 }
2505 #else
2506 int
moea64_pinit(pmap_t pmap)2507 moea64_pinit(pmap_t pmap)
2508 {
2509 int i;
2510 uint32_t hash;
2511
2512 RB_INIT(&pmap->pmap_pvo);
2513
2514 if (pmap_bootstrapped)
2515 pmap->pmap_phys = (pmap_t)moea64_kextract((vm_offset_t)pmap);
2516 else
2517 pmap->pmap_phys = pmap;
2518
2519 /*
2520 * Allocate some segment registers for this pmap.
2521 */
2522 hash = moea64_get_unique_vsid();
2523
2524 for (i = 0; i < 16; i++)
2525 pmap->pm_sr[i] = VSID_MAKE(i, hash);
2526
2527 KASSERT(pmap->pm_sr[0] != 0, ("moea64_pinit: pm_sr[0] = 0"));
2528
2529 return (1);
2530 }
2531 #endif
2532
2533 /*
2534 * Initialize the pmap associated with process 0.
2535 */
2536 void
moea64_pinit0(pmap_t pm)2537 moea64_pinit0(pmap_t pm)
2538 {
2539
2540 PMAP_LOCK_INIT(pm);
2541 moea64_pinit(pm);
2542 bzero(&pm->pm_stats, sizeof(pm->pm_stats));
2543 }
2544
2545 /*
2546 * Set the physical protection on the specified range of this map as requested.
2547 */
2548 static void
moea64_pvo_protect(pmap_t pm,struct pvo_entry * pvo,vm_prot_t prot)2549 moea64_pvo_protect( pmap_t pm, struct pvo_entry *pvo, vm_prot_t prot)
2550 {
2551 struct vm_page *pg;
2552 vm_prot_t oldprot;
2553 int32_t refchg;
2554
2555 PMAP_LOCK_ASSERT(pm, MA_OWNED);
2556
2557 /*
2558 * Change the protection of the page.
2559 */
2560 oldprot = pvo->pvo_pte.prot;
2561 pvo->pvo_pte.prot = prot;
2562 pg = PHYS_TO_VM_PAGE(PVO_PADDR(pvo));
2563
2564 /*
2565 * If the PVO is in the page table, update mapping
2566 */
2567 refchg = moea64_pte_replace(pvo, MOEA64_PTE_PROT_UPDATE);
2568 if (refchg < 0)
2569 refchg = (oldprot & VM_PROT_WRITE) ? LPTE_CHG : 0;
2570
2571 if (pm != kernel_pmap && pg != NULL &&
2572 (pg->a.flags & PGA_EXECUTABLE) == 0 &&
2573 (pvo->pvo_pte.pa & (LPTE_I | LPTE_G | LPTE_NOEXEC)) == 0) {
2574 if ((pg->oflags & VPO_UNMANAGED) == 0)
2575 vm_page_aflag_set(pg, PGA_EXECUTABLE);
2576 moea64_syncicache(pm, PVO_VADDR(pvo),
2577 PVO_PADDR(pvo), PAGE_SIZE);
2578 }
2579
2580 /*
2581 * Update vm about the REF/CHG bits if the page is managed and we have
2582 * removed write access.
2583 */
2584 if (pg != NULL && (pvo->pvo_vaddr & PVO_MANAGED) &&
2585 (oldprot & VM_PROT_WRITE)) {
2586 refchg |= atomic_readandclear_32(&pg->md.mdpg_attrs);
2587 if (refchg & LPTE_CHG)
2588 vm_page_dirty(pg);
2589 if (refchg & LPTE_REF)
2590 vm_page_aflag_set(pg, PGA_REFERENCED);
2591 }
2592 }
2593
2594 void
moea64_protect(pmap_t pm,vm_offset_t sva,vm_offset_t eva,vm_prot_t prot)2595 moea64_protect(pmap_t pm, vm_offset_t sva, vm_offset_t eva,
2596 vm_prot_t prot)
2597 {
2598 struct pvo_entry *pvo, key;
2599
2600 CTR4(KTR_PMAP, "moea64_protect: pm=%p sva=%#x eva=%#x prot=%#x", pm,
2601 sva, eva, prot);
2602
2603 KASSERT(pm == &curproc->p_vmspace->vm_pmap || pm == kernel_pmap,
2604 ("moea64_protect: non current pmap"));
2605
2606 if ((prot & VM_PROT_READ) == VM_PROT_NONE) {
2607 moea64_remove(pm, sva, eva);
2608 return;
2609 }
2610
2611 PMAP_LOCK(pm);
2612 key.pvo_vaddr = sva;
2613 for (pvo = RB_NFIND(pvo_tree, &pm->pmap_pvo, &key);
2614 pvo != NULL && PVO_VADDR(pvo) < eva;
2615 pvo = RB_NEXT(pvo_tree, &pm->pmap_pvo, pvo)) {
2616 if (PVO_IS_SP(pvo)) {
2617 if (moea64_sp_pvo_in_range(pvo, sva, eva)) {
2618 pvo = moea64_sp_protect(pvo, prot);
2619 continue;
2620 } else {
2621 CTR1(KTR_PMAP, "%s: demote before protect",
2622 __func__);
2623 moea64_sp_demote(pvo);
2624 }
2625 }
2626 moea64_pvo_protect(pm, pvo, prot);
2627 }
2628 PMAP_UNLOCK(pm);
2629 }
2630
2631 /*
2632 * Map a list of wired pages into kernel virtual address space. This is
2633 * intended for temporary mappings which do not need page modification or
2634 * references recorded. Existing mappings in the region are overwritten.
2635 */
2636 void
moea64_qenter(vm_offset_t va,vm_page_t * m,int count)2637 moea64_qenter(vm_offset_t va, vm_page_t *m, int count)
2638 {
2639 while (count-- > 0) {
2640 moea64_kenter(va, VM_PAGE_TO_PHYS(*m));
2641 va += PAGE_SIZE;
2642 m++;
2643 }
2644 }
2645
2646 /*
2647 * Remove page mappings from kernel virtual address space. Intended for
2648 * temporary mappings entered by moea64_qenter.
2649 */
2650 void
moea64_qremove(vm_offset_t va,int count)2651 moea64_qremove(vm_offset_t va, int count)
2652 {
2653 while (count-- > 0) {
2654 moea64_kremove(va);
2655 va += PAGE_SIZE;
2656 }
2657 }
2658
2659 void
moea64_release_vsid(uint64_t vsid)2660 moea64_release_vsid(uint64_t vsid)
2661 {
2662 int idx, mask;
2663
2664 mtx_lock(&moea64_slb_mutex);
2665 idx = vsid & (NVSIDS-1);
2666 mask = 1 << (idx % VSID_NBPW);
2667 idx /= VSID_NBPW;
2668 KASSERT(moea64_vsid_bitmap[idx] & mask,
2669 ("Freeing unallocated VSID %#jx", vsid));
2670 moea64_vsid_bitmap[idx] &= ~mask;
2671 mtx_unlock(&moea64_slb_mutex);
2672 }
2673
2674 void
moea64_release(pmap_t pmap)2675 moea64_release(pmap_t pmap)
2676 {
2677
2678 /*
2679 * Free segment registers' VSIDs
2680 */
2681 #ifdef __powerpc64__
2682 slb_free_tree(pmap);
2683 slb_free_user_cache(pmap->pm_slb);
2684 #else
2685 KASSERT(pmap->pm_sr[0] != 0, ("moea64_release: pm_sr[0] = 0"));
2686
2687 moea64_release_vsid(VSID_TO_HASH(pmap->pm_sr[0]));
2688 #endif
2689 }
2690
2691 /*
2692 * Remove all pages mapped by the specified pmap
2693 */
2694 void
moea64_remove_pages(pmap_t pm)2695 moea64_remove_pages(pmap_t pm)
2696 {
2697 struct pvo_entry *pvo, *tpvo;
2698 struct pvo_dlist tofree;
2699
2700 SLIST_INIT(&tofree);
2701
2702 PMAP_LOCK(pm);
2703 RB_FOREACH_SAFE(pvo, pvo_tree, &pm->pmap_pvo, tpvo) {
2704 if (pvo->pvo_vaddr & PVO_WIRED)
2705 continue;
2706
2707 /*
2708 * For locking reasons, remove this from the page table and
2709 * pmap, but save delinking from the vm_page for a second
2710 * pass
2711 */
2712 moea64_pvo_remove_from_pmap(pvo);
2713 SLIST_INSERT_HEAD(&tofree, pvo, pvo_dlink);
2714 }
2715 PMAP_UNLOCK(pm);
2716
2717 while (!SLIST_EMPTY(&tofree)) {
2718 pvo = SLIST_FIRST(&tofree);
2719 SLIST_REMOVE_HEAD(&tofree, pvo_dlink);
2720 moea64_pvo_remove_from_page(pvo);
2721 free_pvo_entry(pvo);
2722 }
2723 }
2724
2725 static void
moea64_remove_locked(pmap_t pm,vm_offset_t sva,vm_offset_t eva,struct pvo_dlist * tofree)2726 moea64_remove_locked(pmap_t pm, vm_offset_t sva, vm_offset_t eva,
2727 struct pvo_dlist *tofree)
2728 {
2729 struct pvo_entry *pvo, *tpvo, key;
2730
2731 PMAP_LOCK_ASSERT(pm, MA_OWNED);
2732
2733 key.pvo_vaddr = sva;
2734 for (pvo = RB_NFIND(pvo_tree, &pm->pmap_pvo, &key);
2735 pvo != NULL && PVO_VADDR(pvo) < eva; pvo = tpvo) {
2736 if (PVO_IS_SP(pvo)) {
2737 if (moea64_sp_pvo_in_range(pvo, sva, eva)) {
2738 tpvo = moea64_sp_remove(pvo, tofree);
2739 continue;
2740 } else {
2741 CTR1(KTR_PMAP, "%s: demote before remove",
2742 __func__);
2743 moea64_sp_demote(pvo);
2744 }
2745 }
2746 tpvo = RB_NEXT(pvo_tree, &pm->pmap_pvo, pvo);
2747
2748 /*
2749 * For locking reasons, remove this from the page table and
2750 * pmap, but save delinking from the vm_page for a second
2751 * pass
2752 */
2753 moea64_pvo_remove_from_pmap(pvo);
2754 SLIST_INSERT_HEAD(tofree, pvo, pvo_dlink);
2755 }
2756 }
2757
2758 /*
2759 * Remove the given range of addresses from the specified map.
2760 */
2761 void
moea64_remove(pmap_t pm,vm_offset_t sva,vm_offset_t eva)2762 moea64_remove(pmap_t pm, vm_offset_t sva, vm_offset_t eva)
2763 {
2764 struct pvo_entry *pvo;
2765 struct pvo_dlist tofree;
2766
2767 /*
2768 * Perform an unsynchronized read. This is, however, safe.
2769 */
2770 if (pm->pm_stats.resident_count == 0)
2771 return;
2772
2773 SLIST_INIT(&tofree);
2774 PMAP_LOCK(pm);
2775 moea64_remove_locked(pm, sva, eva, &tofree);
2776 PMAP_UNLOCK(pm);
2777
2778 while (!SLIST_EMPTY(&tofree)) {
2779 pvo = SLIST_FIRST(&tofree);
2780 SLIST_REMOVE_HEAD(&tofree, pvo_dlink);
2781 moea64_pvo_remove_from_page(pvo);
2782 free_pvo_entry(pvo);
2783 }
2784 }
2785
2786 /*
2787 * Remove physical page from all pmaps in which it resides. moea64_pvo_remove()
2788 * will reflect changes in pte's back to the vm_page.
2789 */
2790 void
moea64_remove_all(vm_page_t m)2791 moea64_remove_all(vm_page_t m)
2792 {
2793 struct pvo_entry *pvo, *next_pvo;
2794 struct pvo_head freequeue;
2795 int wasdead;
2796 pmap_t pmap;
2797
2798 LIST_INIT(&freequeue);
2799
2800 PV_PAGE_LOCK(m);
2801 LIST_FOREACH_SAFE(pvo, vm_page_to_pvoh(m), pvo_vlink, next_pvo) {
2802 pmap = pvo->pvo_pmap;
2803 PMAP_LOCK(pmap);
2804 wasdead = (pvo->pvo_vaddr & PVO_DEAD);
2805 if (!wasdead) {
2806 if (PVO_IS_SP(pvo)) {
2807 CTR1(KTR_PMAP, "%s: demote before remove_all",
2808 __func__);
2809 moea64_sp_demote(pvo);
2810 }
2811 moea64_pvo_remove_from_pmap(pvo);
2812 }
2813 moea64_pvo_remove_from_page_locked(pvo, m);
2814 if (!wasdead)
2815 LIST_INSERT_HEAD(&freequeue, pvo, pvo_vlink);
2816 PMAP_UNLOCK(pmap);
2817
2818 }
2819 KASSERT(!pmap_page_is_mapped(m), ("Page still has mappings"));
2820 KASSERT((m->a.flags & PGA_WRITEABLE) == 0, ("Page still writable"));
2821 PV_PAGE_UNLOCK(m);
2822
2823 /* Clean up UMA allocations */
2824 LIST_FOREACH_SAFE(pvo, &freequeue, pvo_vlink, next_pvo)
2825 free_pvo_entry(pvo);
2826 }
2827
2828 /*
2829 * Allocate a physical page of memory directly from the phys_avail map.
2830 * Can only be called from moea64_bootstrap before avail start and end are
2831 * calculated.
2832 */
2833 vm_offset_t
moea64_bootstrap_alloc(vm_size_t size,vm_size_t align)2834 moea64_bootstrap_alloc(vm_size_t size, vm_size_t align)
2835 {
2836 vm_offset_t s, e;
2837 int i, j;
2838
2839 size = round_page(size);
2840 for (i = 0; phys_avail[i + 1] != 0; i += 2) {
2841 if (align != 0)
2842 s = roundup2(phys_avail[i], align);
2843 else
2844 s = phys_avail[i];
2845 e = s + size;
2846
2847 if (s < phys_avail[i] || e > phys_avail[i + 1])
2848 continue;
2849
2850 if (s + size > platform_real_maxaddr())
2851 continue;
2852
2853 if (s == phys_avail[i]) {
2854 phys_avail[i] += size;
2855 } else if (e == phys_avail[i + 1]) {
2856 phys_avail[i + 1] -= size;
2857 } else {
2858 for (j = phys_avail_count * 2; j > i; j -= 2) {
2859 phys_avail[j] = phys_avail[j - 2];
2860 phys_avail[j + 1] = phys_avail[j - 1];
2861 }
2862
2863 phys_avail[i + 3] = phys_avail[i + 1];
2864 phys_avail[i + 1] = s;
2865 phys_avail[i + 2] = e;
2866 phys_avail_count++;
2867 }
2868
2869 return (s);
2870 }
2871 panic("moea64_bootstrap_alloc: could not allocate memory");
2872 }
2873
2874 static int
moea64_pvo_enter(struct pvo_entry * pvo,struct pvo_head * pvo_head,struct pvo_entry ** oldpvop)2875 moea64_pvo_enter(struct pvo_entry *pvo, struct pvo_head *pvo_head,
2876 struct pvo_entry **oldpvop)
2877 {
2878 struct pvo_entry *old_pvo;
2879 int err;
2880
2881 PMAP_LOCK_ASSERT(pvo->pvo_pmap, MA_OWNED);
2882
2883 STAT_MOEA64(moea64_pvo_enter_calls++);
2884
2885 /*
2886 * Add to pmap list
2887 */
2888 old_pvo = RB_INSERT(pvo_tree, &pvo->pvo_pmap->pmap_pvo, pvo);
2889
2890 if (old_pvo != NULL) {
2891 if (oldpvop != NULL)
2892 *oldpvop = old_pvo;
2893 return (EEXIST);
2894 }
2895
2896 if (pvo_head != NULL) {
2897 LIST_INSERT_HEAD(pvo_head, pvo, pvo_vlink);
2898 }
2899
2900 if (pvo->pvo_vaddr & PVO_WIRED)
2901 pvo->pvo_pmap->pm_stats.wired_count++;
2902 pvo->pvo_pmap->pm_stats.resident_count++;
2903
2904 /*
2905 * Insert it into the hardware page table
2906 */
2907 err = moea64_pte_insert(pvo);
2908 if (err != 0) {
2909 panic("moea64_pvo_enter: overflow");
2910 }
2911
2912 STAT_MOEA64(moea64_pvo_entries++);
2913
2914 if (pvo->pvo_pmap == kernel_pmap)
2915 isync();
2916
2917 #ifdef __powerpc64__
2918 /*
2919 * Make sure all our bootstrap mappings are in the SLB as soon
2920 * as virtual memory is switched on.
2921 */
2922 if (!pmap_bootstrapped)
2923 moea64_bootstrap_slb_prefault(PVO_VADDR(pvo),
2924 pvo->pvo_vaddr & PVO_LARGE);
2925 #endif
2926
2927 return (0);
2928 }
2929
2930 static void
moea64_pvo_remove_from_pmap(struct pvo_entry * pvo)2931 moea64_pvo_remove_from_pmap(struct pvo_entry *pvo)
2932 {
2933 struct vm_page *pg;
2934 int32_t refchg;
2935
2936 KASSERT(pvo->pvo_pmap != NULL, ("Trying to remove PVO with no pmap"));
2937 PMAP_LOCK_ASSERT(pvo->pvo_pmap, MA_OWNED);
2938 KASSERT(!(pvo->pvo_vaddr & PVO_DEAD), ("Trying to remove dead PVO"));
2939
2940 /*
2941 * If there is an active pte entry, we need to deactivate it
2942 */
2943 refchg = moea64_pte_unset(pvo);
2944 if (refchg < 0) {
2945 /*
2946 * If it was evicted from the page table, be pessimistic and
2947 * dirty the page.
2948 */
2949 if (pvo->pvo_pte.prot & VM_PROT_WRITE)
2950 refchg = LPTE_CHG;
2951 else
2952 refchg = 0;
2953 }
2954
2955 /*
2956 * Update our statistics.
2957 */
2958 pvo->pvo_pmap->pm_stats.resident_count--;
2959 if (pvo->pvo_vaddr & PVO_WIRED)
2960 pvo->pvo_pmap->pm_stats.wired_count--;
2961
2962 /*
2963 * Remove this PVO from the pmap list.
2964 */
2965 RB_REMOVE(pvo_tree, &pvo->pvo_pmap->pmap_pvo, pvo);
2966
2967 /*
2968 * Mark this for the next sweep
2969 */
2970 pvo->pvo_vaddr |= PVO_DEAD;
2971
2972 /* Send RC bits to VM */
2973 if ((pvo->pvo_vaddr & PVO_MANAGED) &&
2974 (pvo->pvo_pte.prot & VM_PROT_WRITE)) {
2975 pg = PHYS_TO_VM_PAGE(PVO_PADDR(pvo));
2976 if (pg != NULL) {
2977 refchg |= atomic_readandclear_32(&pg->md.mdpg_attrs);
2978 if (refchg & LPTE_CHG)
2979 vm_page_dirty(pg);
2980 if (refchg & LPTE_REF)
2981 vm_page_aflag_set(pg, PGA_REFERENCED);
2982 }
2983 }
2984 }
2985
2986 static inline void
moea64_pvo_remove_from_page_locked(struct pvo_entry * pvo,vm_page_t m)2987 moea64_pvo_remove_from_page_locked(struct pvo_entry *pvo,
2988 vm_page_t m)
2989 {
2990
2991 KASSERT(pvo->pvo_vaddr & PVO_DEAD, ("Trying to delink live page"));
2992
2993 /* Use NULL pmaps as a sentinel for races in page deletion */
2994 if (pvo->pvo_pmap == NULL)
2995 return;
2996 pvo->pvo_pmap = NULL;
2997
2998 /*
2999 * Update vm about page writeability/executability if managed
3000 */
3001 PV_LOCKASSERT(PVO_PADDR(pvo));
3002 if (pvo->pvo_vaddr & PVO_MANAGED) {
3003 if (m != NULL) {
3004 LIST_REMOVE(pvo, pvo_vlink);
3005 if (LIST_EMPTY(vm_page_to_pvoh(m)))
3006 vm_page_aflag_clear(m,
3007 PGA_WRITEABLE | PGA_EXECUTABLE);
3008 }
3009 }
3010
3011 STAT_MOEA64(moea64_pvo_entries--);
3012 STAT_MOEA64(moea64_pvo_remove_calls++);
3013 }
3014
3015 static void
moea64_pvo_remove_from_page(struct pvo_entry * pvo)3016 moea64_pvo_remove_from_page(struct pvo_entry *pvo)
3017 {
3018 vm_page_t pg = NULL;
3019
3020 if (pvo->pvo_vaddr & PVO_MANAGED)
3021 pg = PHYS_TO_VM_PAGE(PVO_PADDR(pvo));
3022
3023 PV_LOCK(PVO_PADDR(pvo));
3024 moea64_pvo_remove_from_page_locked(pvo, pg);
3025 PV_UNLOCK(PVO_PADDR(pvo));
3026 }
3027
3028 static struct pvo_entry *
moea64_pvo_find_va(pmap_t pm,vm_offset_t va)3029 moea64_pvo_find_va(pmap_t pm, vm_offset_t va)
3030 {
3031 struct pvo_entry key;
3032
3033 PMAP_LOCK_ASSERT(pm, MA_OWNED);
3034
3035 key.pvo_vaddr = va & ~ADDR_POFF;
3036 return (RB_FIND(pvo_tree, &pm->pmap_pvo, &key));
3037 }
3038
3039 static bool
moea64_query_bit(vm_page_t m,uint64_t ptebit)3040 moea64_query_bit(vm_page_t m, uint64_t ptebit)
3041 {
3042 struct pvo_entry *pvo;
3043 int64_t ret;
3044 bool rv;
3045 vm_page_t sp;
3046
3047 /*
3048 * See if this bit is stored in the page already.
3049 *
3050 * For superpages, the bit is stored in the first vm page.
3051 */
3052 if ((m->md.mdpg_attrs & ptebit) != 0 ||
3053 ((sp = PHYS_TO_VM_PAGE(VM_PAGE_TO_PHYS(m) & ~HPT_SP_MASK)) != NULL &&
3054 (sp->md.mdpg_attrs & (ptebit | MDPG_ATTR_SP)) ==
3055 (ptebit | MDPG_ATTR_SP)))
3056 return (true);
3057
3058 /*
3059 * Examine each PTE. Sync so that any pending REF/CHG bits are
3060 * flushed to the PTEs.
3061 */
3062 rv = false;
3063 powerpc_sync();
3064 PV_PAGE_LOCK(m);
3065 LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) {
3066 if (PVO_IS_SP(pvo)) {
3067 ret = moea64_sp_query(pvo, ptebit);
3068 /*
3069 * If SP was not demoted, check its REF/CHG bits here.
3070 */
3071 if (ret != -1) {
3072 if ((ret & ptebit) != 0) {
3073 rv = true;
3074 break;
3075 }
3076 continue;
3077 }
3078 /* else, fallthrough */
3079 }
3080
3081 ret = 0;
3082
3083 /*
3084 * See if this pvo has a valid PTE. if so, fetch the
3085 * REF/CHG bits from the valid PTE. If the appropriate
3086 * ptebit is set, return success.
3087 */
3088 PMAP_LOCK(pvo->pvo_pmap);
3089 if (!(pvo->pvo_vaddr & PVO_DEAD))
3090 ret = moea64_pte_synch(pvo);
3091 PMAP_UNLOCK(pvo->pvo_pmap);
3092
3093 if (ret > 0) {
3094 atomic_set_32(&m->md.mdpg_attrs,
3095 ret & (LPTE_CHG | LPTE_REF));
3096 if (ret & ptebit) {
3097 rv = true;
3098 break;
3099 }
3100 }
3101 }
3102 PV_PAGE_UNLOCK(m);
3103
3104 return (rv);
3105 }
3106
3107 static u_int
moea64_clear_bit(vm_page_t m,u_int64_t ptebit)3108 moea64_clear_bit(vm_page_t m, u_int64_t ptebit)
3109 {
3110 u_int count;
3111 struct pvo_entry *pvo;
3112 int64_t ret;
3113
3114 /*
3115 * Sync so that any pending REF/CHG bits are flushed to the PTEs (so
3116 * we can reset the right ones).
3117 */
3118 powerpc_sync();
3119
3120 /*
3121 * For each pvo entry, clear the pte's ptebit.
3122 */
3123 count = 0;
3124 PV_PAGE_LOCK(m);
3125 LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) {
3126 if (PVO_IS_SP(pvo)) {
3127 if ((ret = moea64_sp_clear(pvo, m, ptebit)) != -1) {
3128 count += ret;
3129 continue;
3130 }
3131 }
3132 ret = 0;
3133
3134 PMAP_LOCK(pvo->pvo_pmap);
3135 if (!(pvo->pvo_vaddr & PVO_DEAD))
3136 ret = moea64_pte_clear(pvo, ptebit);
3137 PMAP_UNLOCK(pvo->pvo_pmap);
3138
3139 if (ret > 0 && (ret & ptebit))
3140 count++;
3141 }
3142 atomic_clear_32(&m->md.mdpg_attrs, ptebit);
3143 PV_PAGE_UNLOCK(m);
3144
3145 return (count);
3146 }
3147
3148 int
moea64_dev_direct_mapped(vm_paddr_t pa,vm_size_t size)3149 moea64_dev_direct_mapped(vm_paddr_t pa, vm_size_t size)
3150 {
3151 struct pvo_entry *pvo, key;
3152 vm_offset_t ppa;
3153 int error = 0;
3154
3155 if (hw_direct_map && mem_valid(pa, size) == 0)
3156 return (0);
3157
3158 PMAP_LOCK(kernel_pmap);
3159 ppa = pa & ~ADDR_POFF;
3160 key.pvo_vaddr = DMAP_BASE_ADDRESS + ppa;
3161 for (pvo = RB_FIND(pvo_tree, &kernel_pmap->pmap_pvo, &key);
3162 ppa < pa + size; ppa += PAGE_SIZE,
3163 pvo = RB_NEXT(pvo_tree, &kernel_pmap->pmap_pvo, pvo)) {
3164 if (pvo == NULL || PVO_PADDR(pvo) != ppa) {
3165 error = EFAULT;
3166 break;
3167 }
3168 }
3169 PMAP_UNLOCK(kernel_pmap);
3170
3171 return (error);
3172 }
3173
3174 /*
3175 * Map a set of physical memory pages into the kernel virtual
3176 * address space. Return a pointer to where it is mapped. This
3177 * routine is intended to be used for mapping device memory,
3178 * NOT real memory.
3179 */
3180 void *
moea64_mapdev_attr(vm_paddr_t pa,vm_size_t size,vm_memattr_t ma)3181 moea64_mapdev_attr(vm_paddr_t pa, vm_size_t size, vm_memattr_t ma)
3182 {
3183 vm_offset_t va, tmpva, ppa, offset;
3184
3185 ppa = trunc_page(pa);
3186 offset = pa & PAGE_MASK;
3187 size = roundup2(offset + size, PAGE_SIZE);
3188
3189 va = kva_alloc(size);
3190
3191 if (!va)
3192 panic("moea64_mapdev: Couldn't alloc kernel virtual memory");
3193
3194 for (tmpva = va; size > 0;) {
3195 moea64_kenter_attr(tmpva, ppa, ma);
3196 size -= PAGE_SIZE;
3197 tmpva += PAGE_SIZE;
3198 ppa += PAGE_SIZE;
3199 }
3200
3201 return ((void *)(va + offset));
3202 }
3203
3204 void *
moea64_mapdev(vm_paddr_t pa,vm_size_t size)3205 moea64_mapdev(vm_paddr_t pa, vm_size_t size)
3206 {
3207
3208 return moea64_mapdev_attr(pa, size, VM_MEMATTR_DEFAULT);
3209 }
3210
3211 void
moea64_unmapdev(void * p,vm_size_t size)3212 moea64_unmapdev(void *p, vm_size_t size)
3213 {
3214 vm_offset_t base, offset, va;
3215
3216 va = (vm_offset_t)p;
3217 base = trunc_page(va);
3218 offset = va & PAGE_MASK;
3219 size = roundup2(offset + size, PAGE_SIZE);
3220
3221 moea64_qremove(base, atop(size));
3222 kva_free(base, size);
3223 }
3224
3225 void
moea64_sync_icache(pmap_t pm,vm_offset_t va,vm_size_t sz)3226 moea64_sync_icache(pmap_t pm, vm_offset_t va, vm_size_t sz)
3227 {
3228 struct pvo_entry *pvo;
3229 vm_offset_t lim;
3230 vm_paddr_t pa;
3231 vm_size_t len;
3232
3233 if (__predict_false(pm == NULL))
3234 pm = &curthread->td_proc->p_vmspace->vm_pmap;
3235
3236 PMAP_LOCK(pm);
3237 while (sz > 0) {
3238 lim = round_page(va+1);
3239 len = MIN(lim - va, sz);
3240 pvo = moea64_pvo_find_va(pm, va & ~ADDR_POFF);
3241 if (pvo != NULL && !(pvo->pvo_pte.pa & LPTE_I)) {
3242 pa = PVO_PADDR(pvo) | (va & ADDR_POFF);
3243 moea64_syncicache(pm, va, pa, len);
3244 }
3245 va += len;
3246 sz -= len;
3247 }
3248 PMAP_UNLOCK(pm);
3249 }
3250
3251 void
moea64_dumpsys_map(vm_paddr_t pa,size_t sz,void ** va)3252 moea64_dumpsys_map(vm_paddr_t pa, size_t sz, void **va)
3253 {
3254
3255 *va = (void *)(uintptr_t)pa;
3256 }
3257
3258 extern struct dump_pa dump_map[PHYS_AVAIL_SZ + 1];
3259
3260 void
moea64_scan_init(void)3261 moea64_scan_init(void)
3262 {
3263 struct pvo_entry *pvo;
3264 vm_offset_t va;
3265 int i;
3266
3267 if (!do_minidump) {
3268 /* Initialize phys. segments for dumpsys(). */
3269 memset(&dump_map, 0, sizeof(dump_map));
3270 mem_regions(&pregions, &pregions_sz, ®ions, ®ions_sz);
3271 for (i = 0; i < pregions_sz; i++) {
3272 dump_map[i].pa_start = pregions[i].mr_start;
3273 dump_map[i].pa_size = pregions[i].mr_size;
3274 }
3275 return;
3276 }
3277
3278 /* Virtual segments for minidumps: */
3279 memset(&dump_map, 0, sizeof(dump_map));
3280
3281 /* 1st: kernel .data and .bss. */
3282 dump_map[0].pa_start = trunc_page((uintptr_t)_etext);
3283 dump_map[0].pa_size = round_page((uintptr_t)_end) -
3284 dump_map[0].pa_start;
3285
3286 /* 2nd: msgbuf and tables (see pmap_bootstrap()). */
3287 dump_map[1].pa_start = (vm_paddr_t)(uintptr_t)msgbufp->msg_ptr;
3288 dump_map[1].pa_size = round_page(msgbufp->msg_size);
3289
3290 /* 3rd: kernel VM. */
3291 va = dump_map[1].pa_start + dump_map[1].pa_size;
3292 /* Find start of next chunk (from va). */
3293 while (va < virtual_end) {
3294 /* Don't dump the buffer cache. */
3295 if (va >= kmi.buffer_sva && va < kmi.buffer_eva) {
3296 va = kmi.buffer_eva;
3297 continue;
3298 }
3299 pvo = moea64_pvo_find_va(kernel_pmap, va & ~ADDR_POFF);
3300 if (pvo != NULL && !(pvo->pvo_vaddr & PVO_DEAD))
3301 break;
3302 va += PAGE_SIZE;
3303 }
3304 if (va < virtual_end) {
3305 dump_map[2].pa_start = va;
3306 va += PAGE_SIZE;
3307 /* Find last page in chunk. */
3308 while (va < virtual_end) {
3309 /* Don't run into the buffer cache. */
3310 if (va == kmi.buffer_sva)
3311 break;
3312 pvo = moea64_pvo_find_va(kernel_pmap, va & ~ADDR_POFF);
3313 if (pvo == NULL || (pvo->pvo_vaddr & PVO_DEAD))
3314 break;
3315 va += PAGE_SIZE;
3316 }
3317 dump_map[2].pa_size = va - dump_map[2].pa_start;
3318 }
3319 }
3320
3321 #ifdef __powerpc64__
3322
3323 static size_t
moea64_scan_pmap(struct bitset * dump_bitset)3324 moea64_scan_pmap(struct bitset *dump_bitset)
3325 {
3326 struct pvo_entry *pvo;
3327 vm_paddr_t pa, pa_end;
3328 vm_offset_t va, pgva, kstart, kend, kstart_lp, kend_lp;
3329 uint64_t lpsize;
3330
3331 lpsize = moea64_large_page_size;
3332 kstart = trunc_page((vm_offset_t)_etext);
3333 kend = round_page((vm_offset_t)_end);
3334 kstart_lp = kstart & ~moea64_large_page_mask;
3335 kend_lp = (kend + moea64_large_page_mask) & ~moea64_large_page_mask;
3336
3337 CTR4(KTR_PMAP, "moea64_scan_pmap: kstart=0x%016lx, kend=0x%016lx, "
3338 "kstart_lp=0x%016lx, kend_lp=0x%016lx",
3339 kstart, kend, kstart_lp, kend_lp);
3340
3341 PMAP_LOCK(kernel_pmap);
3342 RB_FOREACH(pvo, pvo_tree, &kernel_pmap->pmap_pvo) {
3343 va = pvo->pvo_vaddr;
3344
3345 if (va & PVO_DEAD)
3346 continue;
3347
3348 /* Skip DMAP (except kernel area) */
3349 if (va >= DMAP_BASE_ADDRESS && va <= DMAP_MAX_ADDRESS) {
3350 if (va & PVO_LARGE) {
3351 pgva = va & ~moea64_large_page_mask;
3352 if (pgva < kstart_lp || pgva >= kend_lp)
3353 continue;
3354 } else {
3355 pgva = trunc_page(va);
3356 if (pgva < kstart || pgva >= kend)
3357 continue;
3358 }
3359 }
3360
3361 pa = PVO_PADDR(pvo);
3362
3363 if (va & PVO_LARGE) {
3364 pa_end = pa + lpsize;
3365 for (; pa < pa_end; pa += PAGE_SIZE) {
3366 if (vm_phys_is_dumpable(pa))
3367 vm_page_dump_add(dump_bitset, pa);
3368 }
3369 } else {
3370 if (vm_phys_is_dumpable(pa))
3371 vm_page_dump_add(dump_bitset, pa);
3372 }
3373 }
3374 PMAP_UNLOCK(kernel_pmap);
3375
3376 return (sizeof(struct lpte) * moea64_pteg_count * 8);
3377 }
3378
3379 static struct dump_context dump_ctx;
3380
3381 static void *
moea64_dump_pmap_init(unsigned blkpgs)3382 moea64_dump_pmap_init(unsigned blkpgs)
3383 {
3384 dump_ctx.ptex = 0;
3385 dump_ctx.ptex_end = moea64_pteg_count * 8;
3386 dump_ctx.blksz = blkpgs * PAGE_SIZE;
3387 return (&dump_ctx);
3388 }
3389
3390 #else
3391
3392 static size_t
moea64_scan_pmap(struct bitset * dump_bitset __unused)3393 moea64_scan_pmap(struct bitset *dump_bitset __unused)
3394 {
3395 return (0);
3396 }
3397
3398 static void *
moea64_dump_pmap_init(unsigned blkpgs)3399 moea64_dump_pmap_init(unsigned blkpgs)
3400 {
3401 return (NULL);
3402 }
3403
3404 #endif
3405
3406 #ifdef __powerpc64__
3407 static void
moea64_map_range(vm_offset_t va,vm_paddr_t pa,vm_size_t npages)3408 moea64_map_range(vm_offset_t va, vm_paddr_t pa, vm_size_t npages)
3409 {
3410
3411 for (; npages > 0; --npages) {
3412 if (moea64_large_page_size != 0 &&
3413 (pa & moea64_large_page_mask) == 0 &&
3414 (va & moea64_large_page_mask) == 0 &&
3415 npages >= (moea64_large_page_size >> PAGE_SHIFT)) {
3416 PMAP_LOCK(kernel_pmap);
3417 moea64_kenter_large(va, pa, 0, 0);
3418 PMAP_UNLOCK(kernel_pmap);
3419 pa += moea64_large_page_size;
3420 va += moea64_large_page_size;
3421 npages -= (moea64_large_page_size >> PAGE_SHIFT) - 1;
3422 } else {
3423 moea64_kenter(va, pa);
3424 pa += PAGE_SIZE;
3425 va += PAGE_SIZE;
3426 }
3427 }
3428 }
3429
3430 static void
moea64_page_array_startup(long pages)3431 moea64_page_array_startup(long pages)
3432 {
3433 long dom_pages[MAXMEMDOM];
3434 vm_paddr_t pa;
3435 vm_offset_t va, vm_page_base;
3436 vm_size_t needed, size;
3437 int domain;
3438 int i;
3439
3440 vm_page_base = 0xd000000000000000ULL;
3441
3442 /* Short-circuit single-domain systems. */
3443 if (vm_ndomains == 1) {
3444 size = round_page(pages * sizeof(struct vm_page));
3445 pa = vm_phys_early_alloc(0, size);
3446 vm_page_base = moea64_map(&vm_page_base,
3447 pa, pa + size, VM_PROT_READ | VM_PROT_WRITE);
3448 vm_page_array_size = pages;
3449 vm_page_array = (vm_page_t)vm_page_base;
3450 return;
3451 }
3452
3453 for (i = 0; i < MAXMEMDOM; i++)
3454 dom_pages[i] = 0;
3455
3456 /* Now get the number of pages required per domain. */
3457 for (i = 0; i < vm_phys_nsegs; i++) {
3458 domain = vm_phys_segs[i].domain;
3459 KASSERT(domain < MAXMEMDOM,
3460 ("Invalid vm_phys_segs NUMA domain %d!\n", domain));
3461 /* Get size of vm_page_array needed for this segment. */
3462 size = btoc(vm_phys_segs[i].end - vm_phys_segs[i].start);
3463 dom_pages[domain] += size;
3464 }
3465
3466 for (i = 0; phys_avail[i + 1] != 0; i+= 2) {
3467 domain = vm_phys_domain(phys_avail[i]);
3468 KASSERT(domain < MAXMEMDOM,
3469 ("Invalid phys_avail NUMA domain %d!\n", domain));
3470 size = btoc(phys_avail[i + 1] - phys_avail[i]);
3471 dom_pages[domain] += size;
3472 }
3473
3474 /*
3475 * Map in chunks that can get us all 16MB pages. There will be some
3476 * overlap between domains, but that's acceptable for now.
3477 */
3478 vm_page_array_size = 0;
3479 va = vm_page_base;
3480 for (i = 0; i < MAXMEMDOM && vm_page_array_size < pages; i++) {
3481 if (dom_pages[i] == 0)
3482 continue;
3483 size = ulmin(pages - vm_page_array_size, dom_pages[i]);
3484 size = round_page(size * sizeof(struct vm_page));
3485 needed = size;
3486 size = roundup2(size, moea64_large_page_size);
3487 pa = vm_phys_early_alloc(i, size);
3488 vm_page_array_size += size / sizeof(struct vm_page);
3489 moea64_map_range(va, pa, size >> PAGE_SHIFT);
3490 /* Scoot up domain 0, to reduce the domain page overlap. */
3491 if (i == 0)
3492 vm_page_base += size - needed;
3493 va += size;
3494 }
3495 vm_page_array = (vm_page_t)vm_page_base;
3496 vm_page_array_size = pages;
3497 }
3498 #endif
3499
3500 static int64_t
moea64_null_method(void)3501 moea64_null_method(void)
3502 {
3503 return (0);
3504 }
3505
moea64_pte_replace_default(struct pvo_entry * pvo,int flags)3506 static int64_t moea64_pte_replace_default(struct pvo_entry *pvo, int flags)
3507 {
3508 int64_t refchg;
3509
3510 refchg = moea64_pte_unset(pvo);
3511 moea64_pte_insert(pvo);
3512
3513 return (refchg);
3514 }
3515
3516 struct moea64_funcs *moea64_ops;
3517
3518 #define DEFINE_OEA64_IFUNC(ret, func, args, def) \
3519 DEFINE_IFUNC(, ret, moea64_##func, args) { \
3520 moea64_##func##_t f; \
3521 if (moea64_ops == NULL) \
3522 return ((moea64_##func##_t)def); \
3523 f = moea64_ops->func; \
3524 return (f != NULL ? f : (moea64_##func##_t)def);\
3525 }
3526
3527 void
moea64_install(void)3528 moea64_install(void)
3529 {
3530 #ifdef __powerpc64__
3531 if (hw_direct_map == -1) {
3532 moea64_probe_large_page();
3533
3534 /* Use a direct map if we have large page support */
3535 if (moea64_large_page_size > 0)
3536 hw_direct_map = 1;
3537 else
3538 hw_direct_map = 0;
3539 }
3540 #endif
3541
3542 /*
3543 * Default to non-DMAP, and switch over to DMAP functions once we know
3544 * we have DMAP.
3545 */
3546 if (hw_direct_map) {
3547 moea64_methods.quick_enter_page = moea64_quick_enter_page_dmap;
3548 moea64_methods.quick_remove_page = NULL;
3549 moea64_methods.copy_page = moea64_copy_page_dmap;
3550 moea64_methods.zero_page = moea64_zero_page_dmap;
3551 moea64_methods.copy_pages = moea64_copy_pages_dmap;
3552 }
3553 }
3554
3555 DEFINE_OEA64_IFUNC(int64_t, pte_replace, (struct pvo_entry *, int),
3556 moea64_pte_replace_default)
3557 DEFINE_OEA64_IFUNC(int64_t, pte_insert, (struct pvo_entry *), moea64_null_method)
3558 DEFINE_OEA64_IFUNC(int64_t, pte_unset, (struct pvo_entry *), moea64_null_method)
3559 DEFINE_OEA64_IFUNC(int64_t, pte_clear, (struct pvo_entry *, uint64_t),
3560 moea64_null_method)
3561 DEFINE_OEA64_IFUNC(int64_t, pte_synch, (struct pvo_entry *), moea64_null_method)
3562 DEFINE_OEA64_IFUNC(int64_t, pte_insert_sp, (struct pvo_entry *), moea64_null_method)
3563 DEFINE_OEA64_IFUNC(int64_t, pte_unset_sp, (struct pvo_entry *), moea64_null_method)
3564 DEFINE_OEA64_IFUNC(int64_t, pte_replace_sp, (struct pvo_entry *), moea64_null_method)
3565
3566 /* Superpage functions */
3567
3568 /* MMU interface */
3569
3570 static bool
moea64_ps_enabled(pmap_t pmap)3571 moea64_ps_enabled(pmap_t pmap)
3572 {
3573 return (superpages_enabled);
3574 }
3575
3576 static void
moea64_align_superpage(vm_object_t object,vm_ooffset_t offset,vm_offset_t * addr,vm_size_t size)3577 moea64_align_superpage(vm_object_t object, vm_ooffset_t offset,
3578 vm_offset_t *addr, vm_size_t size)
3579 {
3580 vm_offset_t sp_offset;
3581
3582 if (size < HPT_SP_SIZE)
3583 return;
3584
3585 CTR4(KTR_PMAP, "%s: offs=%#jx, addr=%p, size=%#jx",
3586 __func__, (uintmax_t)offset, addr, (uintmax_t)size);
3587
3588 if (object != NULL && (object->flags & OBJ_COLORED) != 0)
3589 offset += ptoa(object->pg_color);
3590 sp_offset = offset & HPT_SP_MASK;
3591 if (size - ((HPT_SP_SIZE - sp_offset) & HPT_SP_MASK) < HPT_SP_SIZE ||
3592 (*addr & HPT_SP_MASK) == sp_offset)
3593 return;
3594 if ((*addr & HPT_SP_MASK) < sp_offset)
3595 *addr = (*addr & ~HPT_SP_MASK) + sp_offset;
3596 else
3597 *addr = ((*addr + HPT_SP_MASK) & ~HPT_SP_MASK) + sp_offset;
3598 }
3599
3600 /* Helpers */
3601
3602 static __inline void
moea64_pvo_cleanup(struct pvo_dlist * tofree)3603 moea64_pvo_cleanup(struct pvo_dlist *tofree)
3604 {
3605 struct pvo_entry *pvo;
3606
3607 /* clean up */
3608 while (!SLIST_EMPTY(tofree)) {
3609 pvo = SLIST_FIRST(tofree);
3610 SLIST_REMOVE_HEAD(tofree, pvo_dlink);
3611 if (pvo->pvo_vaddr & PVO_DEAD)
3612 moea64_pvo_remove_from_page(pvo);
3613 free_pvo_entry(pvo);
3614 }
3615 }
3616
3617 static __inline uint16_t
pvo_to_vmpage_flags(struct pvo_entry * pvo)3618 pvo_to_vmpage_flags(struct pvo_entry *pvo)
3619 {
3620 uint16_t flags;
3621
3622 flags = 0;
3623 if ((pvo->pvo_pte.prot & VM_PROT_WRITE) != 0)
3624 flags |= PGA_WRITEABLE;
3625 if ((pvo->pvo_pte.prot & VM_PROT_EXECUTE) != 0)
3626 flags |= PGA_EXECUTABLE;
3627
3628 return (flags);
3629 }
3630
3631 /*
3632 * Check if the given pvo and its superpage are in sva-eva range.
3633 */
3634 static __inline bool
moea64_sp_pvo_in_range(struct pvo_entry * pvo,vm_offset_t sva,vm_offset_t eva)3635 moea64_sp_pvo_in_range(struct pvo_entry *pvo, vm_offset_t sva, vm_offset_t eva)
3636 {
3637 vm_offset_t spva;
3638
3639 spva = PVO_VADDR(pvo) & ~HPT_SP_MASK;
3640 if (spva >= sva && spva + HPT_SP_SIZE <= eva) {
3641 /*
3642 * Because this function is intended to be called from loops
3643 * that iterate over ordered pvo entries, if the condition
3644 * above is true then the pvo must be the first of its
3645 * superpage.
3646 */
3647 KASSERT(PVO_VADDR(pvo) == spva,
3648 ("%s: unexpected unaligned superpage pvo", __func__));
3649 return (true);
3650 }
3651 return (false);
3652 }
3653
3654 /*
3655 * Update vm about the REF/CHG bits if the superpage is managed and
3656 * has (or had) write access.
3657 */
3658 static void
moea64_sp_refchg_process(struct pvo_entry * sp,vm_page_t m,int64_t sp_refchg,vm_prot_t prot)3659 moea64_sp_refchg_process(struct pvo_entry *sp, vm_page_t m,
3660 int64_t sp_refchg, vm_prot_t prot)
3661 {
3662 vm_page_t m_end;
3663 int64_t refchg;
3664
3665 if ((sp->pvo_vaddr & PVO_MANAGED) != 0 && (prot & VM_PROT_WRITE) != 0) {
3666 for (m_end = &m[HPT_SP_PAGES]; m < m_end; m++) {
3667 refchg = sp_refchg |
3668 atomic_readandclear_32(&m->md.mdpg_attrs);
3669 if (refchg & LPTE_CHG)
3670 vm_page_dirty(m);
3671 if (refchg & LPTE_REF)
3672 vm_page_aflag_set(m, PGA_REFERENCED);
3673 }
3674 }
3675 }
3676
3677 /* Superpage ops */
3678
3679 static int
moea64_sp_enter(pmap_t pmap,vm_offset_t va,vm_page_t m,vm_prot_t prot,u_int flags,int8_t psind)3680 moea64_sp_enter(pmap_t pmap, vm_offset_t va, vm_page_t m,
3681 vm_prot_t prot, u_int flags, int8_t psind)
3682 {
3683 struct pvo_entry *pvo, **pvos;
3684 struct pvo_head *pvo_head;
3685 vm_offset_t sva;
3686 vm_page_t sm;
3687 vm_paddr_t pa, spa;
3688 bool sync;
3689 struct pvo_dlist tofree;
3690 int error __diagused, i;
3691 uint16_t aflags;
3692
3693 KASSERT((va & HPT_SP_MASK) == 0, ("%s: va %#jx unaligned",
3694 __func__, (uintmax_t)va));
3695 KASSERT(psind == 1, ("%s: invalid psind: %d", __func__, psind));
3696 KASSERT(m->psind == 1, ("%s: invalid m->psind: %d",
3697 __func__, m->psind));
3698 KASSERT(pmap != kernel_pmap,
3699 ("%s: function called with kernel pmap", __func__));
3700
3701 CTR5(KTR_PMAP, "%s: va=%#jx, pa=%#jx, prot=%#x, flags=%#x, psind=1",
3702 __func__, (uintmax_t)va, (uintmax_t)VM_PAGE_TO_PHYS(m),
3703 prot, flags);
3704
3705 SLIST_INIT(&tofree);
3706
3707 sva = va;
3708 sm = m;
3709 spa = pa = VM_PAGE_TO_PHYS(sm);
3710
3711 /* Try to allocate all PVOs first, to make failure handling easier. */
3712 pvos = malloc(HPT_SP_PAGES * sizeof(struct pvo_entry *), M_TEMP,
3713 M_NOWAIT);
3714 if (pvos == NULL) {
3715 CTR1(KTR_PMAP, "%s: failed to alloc pvo array", __func__);
3716 return (KERN_RESOURCE_SHORTAGE);
3717 }
3718
3719 for (i = 0; i < HPT_SP_PAGES; i++) {
3720 pvos[i] = alloc_pvo_entry(0);
3721 if (pvos[i] == NULL) {
3722 CTR1(KTR_PMAP, "%s: failed to alloc pvo", __func__);
3723 for (i = i - 1; i >= 0; i--)
3724 free_pvo_entry(pvos[i]);
3725 free(pvos, M_TEMP);
3726 return (KERN_RESOURCE_SHORTAGE);
3727 }
3728 }
3729
3730 SP_PV_LOCK_ALIGNED(spa);
3731 PMAP_LOCK(pmap);
3732
3733 /* Note: moea64_remove_locked() also clears cached REF/CHG bits. */
3734 moea64_remove_locked(pmap, va, va + HPT_SP_SIZE, &tofree);
3735
3736 /* Enter pages */
3737 for (i = 0; i < HPT_SP_PAGES;
3738 i++, va += PAGE_SIZE, pa += PAGE_SIZE, m++) {
3739 pvo = pvos[i];
3740
3741 pvo->pvo_pte.prot = prot;
3742 pvo->pvo_pte.pa = (pa & ~HPT_SP_MASK) | LPTE_LP_4K_16M |
3743 moea64_calc_wimg(pa, pmap_page_get_memattr(m));
3744
3745 if ((flags & PMAP_ENTER_WIRED) != 0)
3746 pvo->pvo_vaddr |= PVO_WIRED;
3747 pvo->pvo_vaddr |= PVO_LARGE;
3748
3749 if ((m->oflags & VPO_UNMANAGED) != 0)
3750 pvo_head = NULL;
3751 else {
3752 pvo_head = &m->md.mdpg_pvoh;
3753 pvo->pvo_vaddr |= PVO_MANAGED;
3754 }
3755
3756 init_pvo_entry(pvo, pmap, va);
3757
3758 error = moea64_pvo_enter(pvo, pvo_head, NULL);
3759 /*
3760 * All superpage PVOs were previously removed, so no errors
3761 * should occur while inserting the new ones.
3762 */
3763 KASSERT(error == 0, ("%s: unexpected error "
3764 "when inserting superpage PVO: %d",
3765 __func__, error));
3766 }
3767
3768 PMAP_UNLOCK(pmap);
3769 SP_PV_UNLOCK_ALIGNED(spa);
3770
3771 sync = (sm->a.flags & PGA_EXECUTABLE) == 0;
3772 /* Note: moea64_pvo_cleanup() also clears page prot. flags. */
3773 moea64_pvo_cleanup(&tofree);
3774 pvo = pvos[0];
3775
3776 /* Set vm page flags */
3777 aflags = pvo_to_vmpage_flags(pvo);
3778 if (aflags != 0)
3779 for (m = sm; m < &sm[HPT_SP_PAGES]; m++)
3780 vm_page_aflag_set(m, aflags);
3781
3782 /*
3783 * Flush the page from the instruction cache if this page is
3784 * mapped executable and cacheable.
3785 */
3786 if (sync && (pvo->pvo_pte.pa & (LPTE_I | LPTE_G | LPTE_NOEXEC)) == 0)
3787 moea64_syncicache(pmap, sva, spa, HPT_SP_SIZE);
3788
3789 atomic_add_long(&sp_mappings, 1);
3790 CTR3(KTR_PMAP, "%s: SP success for va %#jx in pmap %p",
3791 __func__, (uintmax_t)sva, pmap);
3792
3793 free(pvos, M_TEMP);
3794 return (KERN_SUCCESS);
3795 }
3796
3797 static void
moea64_sp_promote(pmap_t pmap,vm_offset_t va,vm_page_t m)3798 moea64_sp_promote(pmap_t pmap, vm_offset_t va, vm_page_t m)
3799 {
3800 struct pvo_entry *first, *pvo;
3801 vm_paddr_t pa, pa_end;
3802 vm_offset_t sva, va_end;
3803 int64_t sp_refchg;
3804
3805 /* This CTR may generate a lot of output. */
3806 /* CTR2(KTR_PMAP, "%s: va=%#jx", __func__, (uintmax_t)va); */
3807
3808 va &= ~HPT_SP_MASK;
3809 sva = va;
3810 /* Get superpage */
3811 pa = VM_PAGE_TO_PHYS(m) & ~HPT_SP_MASK;
3812 m = PHYS_TO_VM_PAGE(pa);
3813
3814 PMAP_LOCK(pmap);
3815
3816 /*
3817 * Check if all pages meet promotion criteria.
3818 *
3819 * XXX In some cases the loop below may be executed for each or most
3820 * of the entered pages of a superpage, which can be expensive
3821 * (although it was not profiled) and need some optimization.
3822 *
3823 * Some cases where this seems to happen are:
3824 * - When a superpage is first entered read-only and later becomes
3825 * read-write.
3826 * - When some of the superpage's virtual addresses map to previously
3827 * wired/cached pages while others map to pages allocated from a
3828 * different physical address range. A common scenario where this
3829 * happens is when mmap'ing a file that is already present in FS
3830 * block cache and doesn't fill a superpage.
3831 */
3832 first = pvo = moea64_pvo_find_va(pmap, sva);
3833 for (pa_end = pa + HPT_SP_SIZE;
3834 pa < pa_end; pa += PAGE_SIZE, va += PAGE_SIZE) {
3835 if (pvo == NULL || (pvo->pvo_vaddr & PVO_DEAD) != 0) {
3836 CTR3(KTR_PMAP,
3837 "%s: NULL or dead PVO: pmap=%p, va=%#jx",
3838 __func__, pmap, (uintmax_t)va);
3839 goto error;
3840 }
3841 if (PVO_PADDR(pvo) != pa) {
3842 CTR5(KTR_PMAP, "%s: PAs don't match: "
3843 "pmap=%p, va=%#jx, pvo_pa=%#jx, exp_pa=%#jx",
3844 __func__, pmap, (uintmax_t)va,
3845 (uintmax_t)PVO_PADDR(pvo), (uintmax_t)pa);
3846 atomic_add_long(&sp_p_fail_pa, 1);
3847 goto error;
3848 }
3849 if ((first->pvo_vaddr & PVO_FLAGS_PROMOTE) !=
3850 (pvo->pvo_vaddr & PVO_FLAGS_PROMOTE)) {
3851 CTR5(KTR_PMAP, "%s: PVO flags don't match: "
3852 "pmap=%p, va=%#jx, pvo_flags=%#jx, exp_flags=%#jx",
3853 __func__, pmap, (uintmax_t)va,
3854 (uintmax_t)(pvo->pvo_vaddr & PVO_FLAGS_PROMOTE),
3855 (uintmax_t)(first->pvo_vaddr & PVO_FLAGS_PROMOTE));
3856 atomic_add_long(&sp_p_fail_flags, 1);
3857 goto error;
3858 }
3859 if (first->pvo_pte.prot != pvo->pvo_pte.prot) {
3860 CTR5(KTR_PMAP, "%s: PVO protections don't match: "
3861 "pmap=%p, va=%#jx, pvo_prot=%#x, exp_prot=%#x",
3862 __func__, pmap, (uintmax_t)va,
3863 pvo->pvo_pte.prot, first->pvo_pte.prot);
3864 atomic_add_long(&sp_p_fail_prot, 1);
3865 goto error;
3866 }
3867 if ((first->pvo_pte.pa & LPTE_WIMG) !=
3868 (pvo->pvo_pte.pa & LPTE_WIMG)) {
3869 CTR5(KTR_PMAP, "%s: WIMG bits don't match: "
3870 "pmap=%p, va=%#jx, pvo_wimg=%#jx, exp_wimg=%#jx",
3871 __func__, pmap, (uintmax_t)va,
3872 (uintmax_t)(pvo->pvo_pte.pa & LPTE_WIMG),
3873 (uintmax_t)(first->pvo_pte.pa & LPTE_WIMG));
3874 atomic_add_long(&sp_p_fail_wimg, 1);
3875 goto error;
3876 }
3877
3878 pvo = RB_NEXT(pvo_tree, &pmap->pmap_pvo, pvo);
3879 }
3880
3881 /* All OK, promote. */
3882
3883 /*
3884 * Handle superpage REF/CHG bits. If REF or CHG is set in
3885 * any page, then it must be set in the superpage.
3886 *
3887 * Instead of querying each page, we take advantage of two facts:
3888 * 1- If a page is being promoted, it was referenced.
3889 * 2- If promoted pages are writable, they were modified.
3890 */
3891 sp_refchg = LPTE_REF |
3892 ((first->pvo_pte.prot & VM_PROT_WRITE) != 0 ? LPTE_CHG : 0);
3893
3894 /* Promote pages */
3895
3896 for (pvo = first, va_end = PVO_VADDR(pvo) + HPT_SP_SIZE;
3897 pvo != NULL && PVO_VADDR(pvo) < va_end;
3898 pvo = RB_NEXT(pvo_tree, &pmap->pmap_pvo, pvo)) {
3899 pvo->pvo_pte.pa &= ADDR_POFF | ~HPT_SP_MASK;
3900 pvo->pvo_pte.pa |= LPTE_LP_4K_16M;
3901 pvo->pvo_vaddr |= PVO_LARGE;
3902 }
3903 moea64_pte_replace_sp(first);
3904
3905 /* Send REF/CHG bits to VM */
3906 moea64_sp_refchg_process(first, m, sp_refchg, first->pvo_pte.prot);
3907
3908 /* Use first page to cache REF/CHG bits */
3909 atomic_set_32(&m->md.mdpg_attrs, sp_refchg | MDPG_ATTR_SP);
3910
3911 PMAP_UNLOCK(pmap);
3912
3913 atomic_add_long(&sp_mappings, 1);
3914 atomic_add_long(&sp_promotions, 1);
3915 CTR3(KTR_PMAP, "%s: success for va %#jx in pmap %p",
3916 __func__, (uintmax_t)sva, pmap);
3917 return;
3918
3919 error:
3920 atomic_add_long(&sp_p_failures, 1);
3921 PMAP_UNLOCK(pmap);
3922 }
3923
3924 static void
moea64_sp_demote_aligned(struct pvo_entry * sp)3925 moea64_sp_demote_aligned(struct pvo_entry *sp)
3926 {
3927 struct pvo_entry *pvo;
3928 vm_offset_t va, va_end;
3929 vm_paddr_t pa;
3930 vm_page_t m;
3931 pmap_t pmap __diagused;
3932 int64_t refchg;
3933
3934 CTR2(KTR_PMAP, "%s: va=%#jx", __func__, (uintmax_t)PVO_VADDR(sp));
3935
3936 pmap = sp->pvo_pmap;
3937 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
3938
3939 pvo = sp;
3940
3941 /* Demote pages */
3942
3943 va = PVO_VADDR(pvo);
3944 pa = PVO_PADDR(pvo);
3945 m = PHYS_TO_VM_PAGE(pa);
3946
3947 for (pvo = sp, va_end = va + HPT_SP_SIZE;
3948 pvo != NULL && PVO_VADDR(pvo) < va_end;
3949 pvo = RB_NEXT(pvo_tree, &pmap->pmap_pvo, pvo),
3950 va += PAGE_SIZE, pa += PAGE_SIZE) {
3951 KASSERT(pvo && PVO_VADDR(pvo) == va,
3952 ("%s: missing PVO for va %#jx", __func__, (uintmax_t)va));
3953
3954 pvo->pvo_vaddr &= ~PVO_LARGE;
3955 pvo->pvo_pte.pa &= ~LPTE_RPGN;
3956 pvo->pvo_pte.pa |= pa;
3957
3958 }
3959 refchg = moea64_pte_replace_sp(sp);
3960
3961 /*
3962 * Clear SP flag
3963 *
3964 * XXX It is possible that another pmap has this page mapped as
3965 * part of a superpage, but as the SP flag is used only for
3966 * caching SP REF/CHG bits, that will be queried if not set
3967 * in cache, it should be ok to clear it here.
3968 */
3969 atomic_clear_32(&m->md.mdpg_attrs, MDPG_ATTR_SP);
3970
3971 /*
3972 * Handle superpage REF/CHG bits. A bit set in the superpage
3973 * means all pages should consider it set.
3974 */
3975 moea64_sp_refchg_process(sp, m, refchg, sp->pvo_pte.prot);
3976
3977 atomic_add_long(&sp_demotions, 1);
3978 CTR3(KTR_PMAP, "%s: success for va %#jx in pmap %p",
3979 __func__, (uintmax_t)PVO_VADDR(sp), pmap);
3980 }
3981
3982 static void
moea64_sp_demote(struct pvo_entry * pvo)3983 moea64_sp_demote(struct pvo_entry *pvo)
3984 {
3985 PMAP_LOCK_ASSERT(pvo->pvo_pmap, MA_OWNED);
3986
3987 if ((PVO_VADDR(pvo) & HPT_SP_MASK) != 0) {
3988 pvo = moea64_pvo_find_va(pvo->pvo_pmap,
3989 PVO_VADDR(pvo) & ~HPT_SP_MASK);
3990 KASSERT(pvo != NULL, ("%s: missing PVO for va %#jx",
3991 __func__, (uintmax_t)(PVO_VADDR(pvo) & ~HPT_SP_MASK)));
3992 }
3993 moea64_sp_demote_aligned(pvo);
3994 }
3995
3996 static struct pvo_entry *
moea64_sp_unwire(struct pvo_entry * sp)3997 moea64_sp_unwire(struct pvo_entry *sp)
3998 {
3999 struct pvo_entry *pvo, *prev;
4000 vm_offset_t eva;
4001 pmap_t pm;
4002 int64_t ret, refchg;
4003
4004 CTR2(KTR_PMAP, "%s: va=%#jx", __func__, (uintmax_t)PVO_VADDR(sp));
4005
4006 pm = sp->pvo_pmap;
4007 PMAP_LOCK_ASSERT(pm, MA_OWNED);
4008
4009 eva = PVO_VADDR(sp) + HPT_SP_SIZE;
4010 refchg = 0;
4011 for (pvo = sp; pvo != NULL && PVO_VADDR(pvo) < eva;
4012 prev = pvo, pvo = RB_NEXT(pvo_tree, &pm->pmap_pvo, pvo)) {
4013 if ((pvo->pvo_vaddr & PVO_WIRED) == 0)
4014 panic("%s: pvo %p is missing PVO_WIRED",
4015 __func__, pvo);
4016 pvo->pvo_vaddr &= ~PVO_WIRED;
4017
4018 ret = moea64_pte_replace(pvo, 0 /* No invalidation */);
4019 if (ret < 0)
4020 refchg |= LPTE_CHG;
4021 else
4022 refchg |= ret;
4023
4024 pm->pm_stats.wired_count--;
4025 }
4026
4027 /* Send REF/CHG bits to VM */
4028 moea64_sp_refchg_process(sp, PHYS_TO_VM_PAGE(PVO_PADDR(sp)),
4029 refchg, sp->pvo_pte.prot);
4030
4031 return (prev);
4032 }
4033
4034 static struct pvo_entry *
moea64_sp_protect(struct pvo_entry * sp,vm_prot_t prot)4035 moea64_sp_protect(struct pvo_entry *sp, vm_prot_t prot)
4036 {
4037 struct pvo_entry *pvo, *prev;
4038 vm_offset_t eva;
4039 pmap_t pm;
4040 vm_page_t m, m_end;
4041 int64_t ret, refchg;
4042 vm_prot_t oldprot;
4043
4044 CTR3(KTR_PMAP, "%s: va=%#jx, prot=%x",
4045 __func__, (uintmax_t)PVO_VADDR(sp), prot);
4046
4047 pm = sp->pvo_pmap;
4048 PMAP_LOCK_ASSERT(pm, MA_OWNED);
4049
4050 oldprot = sp->pvo_pte.prot;
4051 m = PHYS_TO_VM_PAGE(PVO_PADDR(sp));
4052 KASSERT(m != NULL, ("%s: missing vm page for pa %#jx",
4053 __func__, (uintmax_t)PVO_PADDR(sp)));
4054 eva = PVO_VADDR(sp) + HPT_SP_SIZE;
4055 refchg = 0;
4056
4057 for (pvo = sp; pvo != NULL && PVO_VADDR(pvo) < eva;
4058 prev = pvo, pvo = RB_NEXT(pvo_tree, &pm->pmap_pvo, pvo)) {
4059 pvo->pvo_pte.prot = prot;
4060 /*
4061 * If the PVO is in the page table, update mapping
4062 */
4063 ret = moea64_pte_replace(pvo, MOEA64_PTE_PROT_UPDATE);
4064 if (ret < 0)
4065 refchg |= LPTE_CHG;
4066 else
4067 refchg |= ret;
4068 }
4069
4070 /* Send REF/CHG bits to VM */
4071 moea64_sp_refchg_process(sp, m, refchg, oldprot);
4072
4073 /* Handle pages that became executable */
4074 if ((m->a.flags & PGA_EXECUTABLE) == 0 &&
4075 (sp->pvo_pte.pa & (LPTE_I | LPTE_G | LPTE_NOEXEC)) == 0) {
4076 if ((m->oflags & VPO_UNMANAGED) == 0)
4077 for (m_end = &m[HPT_SP_PAGES]; m < m_end; m++)
4078 vm_page_aflag_set(m, PGA_EXECUTABLE);
4079 moea64_syncicache(pm, PVO_VADDR(sp), PVO_PADDR(sp),
4080 HPT_SP_SIZE);
4081 }
4082
4083 return (prev);
4084 }
4085
4086 static struct pvo_entry *
moea64_sp_remove(struct pvo_entry * sp,struct pvo_dlist * tofree)4087 moea64_sp_remove(struct pvo_entry *sp, struct pvo_dlist *tofree)
4088 {
4089 struct pvo_entry *pvo, *tpvo;
4090 vm_offset_t eva;
4091 pmap_t pm __diagused;
4092
4093 CTR2(KTR_PMAP, "%s: va=%#jx", __func__, (uintmax_t)PVO_VADDR(sp));
4094
4095 pm = sp->pvo_pmap;
4096 PMAP_LOCK_ASSERT(pm, MA_OWNED);
4097
4098 eva = PVO_VADDR(sp) + HPT_SP_SIZE;
4099 for (pvo = sp; pvo != NULL && PVO_VADDR(pvo) < eva; pvo = tpvo) {
4100 tpvo = RB_NEXT(pvo_tree, &pm->pmap_pvo, pvo);
4101
4102 /*
4103 * For locking reasons, remove this from the page table and
4104 * pmap, but save delinking from the vm_page for a second
4105 * pass
4106 */
4107 moea64_pvo_remove_from_pmap(pvo);
4108 SLIST_INSERT_HEAD(tofree, pvo, pvo_dlink);
4109 }
4110
4111 /*
4112 * Clear SP bit
4113 *
4114 * XXX See comment in moea64_sp_demote_aligned() for why it's
4115 * ok to always clear the SP bit on remove/demote.
4116 */
4117 atomic_clear_32(&PHYS_TO_VM_PAGE(PVO_PADDR(sp))->md.mdpg_attrs,
4118 MDPG_ATTR_SP);
4119
4120 return (tpvo);
4121 }
4122
4123 static int64_t
moea64_sp_query_locked(struct pvo_entry * pvo,uint64_t ptebit)4124 moea64_sp_query_locked(struct pvo_entry *pvo, uint64_t ptebit)
4125 {
4126 int64_t refchg, ret;
4127 vm_offset_t eva;
4128 vm_page_t m;
4129 pmap_t pmap;
4130 struct pvo_entry *sp;
4131
4132 pmap = pvo->pvo_pmap;
4133 PMAP_LOCK_ASSERT(pmap, MA_OWNED);
4134
4135 /* Get first SP PVO */
4136 if ((PVO_VADDR(pvo) & HPT_SP_MASK) != 0) {
4137 sp = moea64_pvo_find_va(pmap, PVO_VADDR(pvo) & ~HPT_SP_MASK);
4138 KASSERT(sp != NULL, ("%s: missing PVO for va %#jx",
4139 __func__, (uintmax_t)(PVO_VADDR(pvo) & ~HPT_SP_MASK)));
4140 } else
4141 sp = pvo;
4142 eva = PVO_VADDR(sp) + HPT_SP_SIZE;
4143
4144 refchg = 0;
4145 for (pvo = sp; pvo != NULL && PVO_VADDR(pvo) < eva;
4146 pvo = RB_NEXT(pvo_tree, &pmap->pmap_pvo, pvo)) {
4147 ret = moea64_pte_synch(pvo);
4148 if (ret > 0) {
4149 refchg |= ret & (LPTE_CHG | LPTE_REF);
4150 if ((refchg & ptebit) != 0)
4151 break;
4152 }
4153 }
4154
4155 /* Save results */
4156 if (refchg != 0) {
4157 m = PHYS_TO_VM_PAGE(PVO_PADDR(sp));
4158 atomic_set_32(&m->md.mdpg_attrs, refchg | MDPG_ATTR_SP);
4159 }
4160
4161 return (refchg);
4162 }
4163
4164 static int64_t
moea64_sp_query(struct pvo_entry * pvo,uint64_t ptebit)4165 moea64_sp_query(struct pvo_entry *pvo, uint64_t ptebit)
4166 {
4167 int64_t refchg;
4168 pmap_t pmap;
4169
4170 pmap = pvo->pvo_pmap;
4171 PMAP_LOCK(pmap);
4172
4173 /*
4174 * Check if SP was demoted/removed before pmap lock was acquired.
4175 */
4176 if (!PVO_IS_SP(pvo) || (pvo->pvo_vaddr & PVO_DEAD) != 0) {
4177 CTR2(KTR_PMAP, "%s: demoted/removed: pa=%#jx",
4178 __func__, (uintmax_t)PVO_PADDR(pvo));
4179 PMAP_UNLOCK(pmap);
4180 return (-1);
4181 }
4182
4183 refchg = moea64_sp_query_locked(pvo, ptebit);
4184 PMAP_UNLOCK(pmap);
4185
4186 CTR4(KTR_PMAP, "%s: va=%#jx, pa=%#jx: refchg=%#jx",
4187 __func__, (uintmax_t)PVO_VADDR(pvo),
4188 (uintmax_t)PVO_PADDR(pvo), (uintmax_t)refchg);
4189
4190 return (refchg);
4191 }
4192
4193 static int64_t
moea64_sp_pvo_clear(struct pvo_entry * pvo,uint64_t ptebit)4194 moea64_sp_pvo_clear(struct pvo_entry *pvo, uint64_t ptebit)
4195 {
4196 int64_t refchg, ret;
4197 pmap_t pmap;
4198 struct pvo_entry *sp;
4199 vm_offset_t eva;
4200 vm_page_t m;
4201
4202 pmap = pvo->pvo_pmap;
4203 PMAP_LOCK(pmap);
4204
4205 /*
4206 * Check if SP was demoted/removed before pmap lock was acquired.
4207 */
4208 if (!PVO_IS_SP(pvo) || (pvo->pvo_vaddr & PVO_DEAD) != 0) {
4209 CTR2(KTR_PMAP, "%s: demoted/removed: pa=%#jx",
4210 __func__, (uintmax_t)PVO_PADDR(pvo));
4211 PMAP_UNLOCK(pmap);
4212 return (-1);
4213 }
4214
4215 /* Get first SP PVO */
4216 if ((PVO_VADDR(pvo) & HPT_SP_MASK) != 0) {
4217 sp = moea64_pvo_find_va(pmap, PVO_VADDR(pvo) & ~HPT_SP_MASK);
4218 KASSERT(sp != NULL, ("%s: missing PVO for va %#jx",
4219 __func__, (uintmax_t)(PVO_VADDR(pvo) & ~HPT_SP_MASK)));
4220 } else
4221 sp = pvo;
4222 eva = PVO_VADDR(sp) + HPT_SP_SIZE;
4223
4224 refchg = 0;
4225 for (pvo = sp; pvo != NULL && PVO_VADDR(pvo) < eva;
4226 pvo = RB_NEXT(pvo_tree, &pmap->pmap_pvo, pvo)) {
4227 ret = moea64_pte_clear(pvo, ptebit);
4228 if (ret > 0)
4229 refchg |= ret & (LPTE_CHG | LPTE_REF);
4230 }
4231
4232 m = PHYS_TO_VM_PAGE(PVO_PADDR(sp));
4233 atomic_clear_32(&m->md.mdpg_attrs, ptebit);
4234 PMAP_UNLOCK(pmap);
4235
4236 CTR4(KTR_PMAP, "%s: va=%#jx, pa=%#jx: refchg=%#jx",
4237 __func__, (uintmax_t)PVO_VADDR(sp),
4238 (uintmax_t)PVO_PADDR(sp), (uintmax_t)refchg);
4239
4240 return (refchg);
4241 }
4242
4243 static int64_t
moea64_sp_clear(struct pvo_entry * pvo,vm_page_t m,uint64_t ptebit)4244 moea64_sp_clear(struct pvo_entry *pvo, vm_page_t m, uint64_t ptebit)
4245 {
4246 int64_t count, ret;
4247 pmap_t pmap;
4248
4249 count = 0;
4250 pmap = pvo->pvo_pmap;
4251
4252 /*
4253 * Since this reference bit is shared by 4096 4KB pages, it
4254 * should not be cleared every time it is tested. Apply a
4255 * simple "hash" function on the physical page number, the
4256 * virtual superpage number, and the pmap address to select
4257 * one 4KB page out of the 4096 on which testing the
4258 * reference bit will result in clearing that reference bit.
4259 * This function is designed to avoid the selection of the
4260 * same 4KB page for every 16MB page mapping.
4261 *
4262 * Always leave the reference bit of a wired mapping set, as
4263 * the current state of its reference bit won't affect page
4264 * replacement.
4265 */
4266 if (ptebit == LPTE_REF && (((VM_PAGE_TO_PHYS(m) >> PAGE_SHIFT) ^
4267 (PVO_VADDR(pvo) >> HPT_SP_SHIFT) ^ (uintptr_t)pmap) &
4268 (HPT_SP_PAGES - 1)) == 0 && (pvo->pvo_vaddr & PVO_WIRED) == 0) {
4269 if ((ret = moea64_sp_pvo_clear(pvo, ptebit)) == -1)
4270 return (-1);
4271
4272 if ((ret & ptebit) != 0)
4273 count++;
4274
4275 /*
4276 * If this page was not selected by the hash function, then assume
4277 * its REF bit was set.
4278 */
4279 } else if (ptebit == LPTE_REF) {
4280 count++;
4281
4282 /*
4283 * To clear the CHG bit of a single SP page, first it must be demoted.
4284 * But if no CHG bit is set, no bit clear and thus no SP demotion is
4285 * needed.
4286 */
4287 } else {
4288 CTR4(KTR_PMAP, "%s: ptebit=%#jx, va=%#jx, pa=%#jx",
4289 __func__, (uintmax_t)ptebit, (uintmax_t)PVO_VADDR(pvo),
4290 (uintmax_t)PVO_PADDR(pvo));
4291
4292 PMAP_LOCK(pmap);
4293
4294 /*
4295 * Make sure SP wasn't demoted/removed before pmap lock
4296 * was acquired.
4297 */
4298 if (!PVO_IS_SP(pvo) || (pvo->pvo_vaddr & PVO_DEAD) != 0) {
4299 CTR2(KTR_PMAP, "%s: demoted/removed: pa=%#jx",
4300 __func__, (uintmax_t)PVO_PADDR(pvo));
4301 PMAP_UNLOCK(pmap);
4302 return (-1);
4303 }
4304
4305 ret = moea64_sp_query_locked(pvo, ptebit);
4306 if ((ret & ptebit) != 0)
4307 count++;
4308 else {
4309 PMAP_UNLOCK(pmap);
4310 return (0);
4311 }
4312
4313 moea64_sp_demote(pvo);
4314 moea64_pte_clear(pvo, ptebit);
4315
4316 /*
4317 * Write protect the mapping to a single page so that a
4318 * subsequent write access may repromote.
4319 */
4320 if ((pvo->pvo_vaddr & PVO_WIRED) == 0)
4321 moea64_pvo_protect(pmap, pvo,
4322 pvo->pvo_pte.prot & ~VM_PROT_WRITE);
4323
4324 PMAP_UNLOCK(pmap);
4325 }
4326
4327 return (count);
4328 }
4329