xref: /freebsd/sys/powerpc/booke/pmap_32.c (revision 19fe57fdb4fd2c18a37f2a972617c8769609cdb8)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright (C) 2007-2009 Semihalf, Rafal Jaworowski <raj@semihalf.com>
5  * Copyright (C) 2006 Semihalf, Marian Balakowicz <m8@semihalf.com>
6  * All rights reserved.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in the
15  *    documentation and/or other materials provided with the distribution.
16  *
17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN
20  * NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
22  * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
23  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
24  * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
25  * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
26  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27  *
28  * Some hw specific parts of this pmap were derived or influenced
29  * by NetBSD's ibm4xx pmap module. More generic code is shared with
30  * a few other pmap modules from the FreeBSD tree.
31  */
32 
33  /*
34   * VM layout notes:
35   *
36   * Kernel and user threads run within one common virtual address space
37   * defined by AS=0.
38   *
39   * 32-bit pmap:
40   * Virtual address space layout:
41   * -----------------------------
42   * 0x0000_0000 - 0x7fff_ffff	: user process
43   * 0x8000_0000 - 0xbfff_ffff	: pmap_mapdev()-ed area (PCI/PCIE etc.)
44   * 0xc000_0000 - 0xffff_efff	: KVA
45   */
46 
47 #include <sys/cdefs.h>
48 __FBSDID("$FreeBSD$");
49 
50 #include "opt_ddb.h"
51 #include "opt_kstack_pages.h"
52 
53 #include <sys/param.h>
54 #include <sys/conf.h>
55 #include <sys/malloc.h>
56 #include <sys/ktr.h>
57 #include <sys/proc.h>
58 #include <sys/user.h>
59 #include <sys/queue.h>
60 #include <sys/systm.h>
61 #include <sys/kernel.h>
62 #include <sys/kerneldump.h>
63 #include <sys/linker.h>
64 #include <sys/msgbuf.h>
65 #include <sys/lock.h>
66 #include <sys/mutex.h>
67 #include <sys/rwlock.h>
68 #include <sys/sched.h>
69 #include <sys/smp.h>
70 #include <sys/vmmeter.h>
71 
72 #include <vm/vm.h>
73 #include <vm/vm_page.h>
74 #include <vm/vm_kern.h>
75 #include <vm/vm_pageout.h>
76 #include <vm/vm_extern.h>
77 #include <vm/vm_object.h>
78 #include <vm/vm_param.h>
79 #include <vm/vm_map.h>
80 #include <vm/vm_pager.h>
81 #include <vm/vm_phys.h>
82 #include <vm/vm_pagequeue.h>
83 #include <vm/uma.h>
84 
85 #include <machine/_inttypes.h>
86 #include <machine/cpu.h>
87 #include <machine/pcb.h>
88 #include <machine/platform.h>
89 
90 #include <machine/tlb.h>
91 #include <machine/spr.h>
92 #include <machine/md_var.h>
93 #include <machine/mmuvar.h>
94 #include <machine/pmap.h>
95 #include <machine/pte.h>
96 
97 #include <ddb/ddb.h>
98 
99 #include "mmu_if.h"
100 
101 #define	PRI0ptrX	"08x"
102 
103 /* Reserved KVA space and mutex for mmu_booke_zero_page. */
104 static vm_offset_t zero_page_va;
105 static struct mtx zero_page_mutex;
106 
107 /* Reserved KVA space and mutex for mmu_booke_copy_page. */
108 static vm_offset_t copy_page_src_va;
109 static vm_offset_t copy_page_dst_va;
110 static struct mtx copy_page_mutex;
111 
112 /**************************************************************************/
113 /* PMAP */
114 /**************************************************************************/
115 
116 #define	VM_MAPDEV_BASE	((vm_offset_t)VM_MAXUSER_ADDRESS + PAGE_SIZE)
117 
118 static void tid_flush(tlbtid_t tid);
119 static unsigned long ilog2(unsigned long);
120 
121 /**************************************************************************/
122 /* Page table management */
123 /**************************************************************************/
124 
125 #define PMAP_ROOT_SIZE	(sizeof(pte_t**) * PDIR_NENTRIES)
126 static void ptbl_init(void);
127 static struct ptbl_buf *ptbl_buf_alloc(void);
128 static void ptbl_buf_free(struct ptbl_buf *);
129 static void ptbl_free_pmap_ptbl(pmap_t, pte_t *);
130 
131 static pte_t *ptbl_alloc(mmu_t, pmap_t, unsigned int, boolean_t);
132 static void ptbl_free(mmu_t, pmap_t, unsigned int);
133 static void ptbl_hold(mmu_t, pmap_t, unsigned int);
134 static int ptbl_unhold(mmu_t, pmap_t, unsigned int);
135 
136 static vm_paddr_t pte_vatopa(mmu_t, pmap_t, vm_offset_t);
137 static int pte_enter(mmu_t, pmap_t, vm_page_t, vm_offset_t, uint32_t, boolean_t);
138 static int pte_remove(mmu_t, pmap_t, vm_offset_t, uint8_t);
139 static pte_t *pte_find(mmu_t, pmap_t, vm_offset_t);
140 static void kernel_pte_alloc(vm_offset_t, vm_offset_t, vm_offset_t);
141 
142 struct ptbl_buf {
143 	TAILQ_ENTRY(ptbl_buf) link;	/* list link */
144 	vm_offset_t kva;		/* va of mapping */
145 };
146 
147 /* Number of kva ptbl buffers, each covering one ptbl (PTBL_PAGES). */
148 #define PTBL_BUFS		(128 * 16)
149 
150 /* ptbl free list and a lock used for access synchronization. */
151 static TAILQ_HEAD(, ptbl_buf) ptbl_buf_freelist;
152 static struct mtx ptbl_buf_freelist_lock;
153 
154 /* Base address of kva space allocated fot ptbl bufs. */
155 static vm_offset_t ptbl_buf_pool_vabase;
156 
157 /* Pointer to ptbl_buf structures. */
158 static struct ptbl_buf *ptbl_bufs;
159 
160 /**************************************************************************/
161 /* Page table related */
162 /**************************************************************************/
163 
164 
165 /* Initialize pool of kva ptbl buffers. */
166 static void
167 ptbl_init(void)
168 {
169 	int i;
170 
171 	CTR3(KTR_PMAP, "%s: s (ptbl_bufs = 0x%08x size 0x%08x)", __func__,
172 	    (uint32_t)ptbl_bufs, sizeof(struct ptbl_buf) * PTBL_BUFS);
173 	CTR3(KTR_PMAP, "%s: s (ptbl_buf_pool_vabase = 0x%08x size = 0x%08x)",
174 	    __func__, ptbl_buf_pool_vabase, PTBL_BUFS * PTBL_PAGES * PAGE_SIZE);
175 
176 	mtx_init(&ptbl_buf_freelist_lock, "ptbl bufs lock", NULL, MTX_DEF);
177 	TAILQ_INIT(&ptbl_buf_freelist);
178 
179 	for (i = 0; i < PTBL_BUFS; i++) {
180 		ptbl_bufs[i].kva =
181 		    ptbl_buf_pool_vabase + i * PTBL_PAGES * PAGE_SIZE;
182 		TAILQ_INSERT_TAIL(&ptbl_buf_freelist, &ptbl_bufs[i], link);
183 	}
184 }
185 
186 /* Get a ptbl_buf from the freelist. */
187 static struct ptbl_buf *
188 ptbl_buf_alloc(void)
189 {
190 	struct ptbl_buf *buf;
191 
192 	mtx_lock(&ptbl_buf_freelist_lock);
193 	buf = TAILQ_FIRST(&ptbl_buf_freelist);
194 	if (buf != NULL)
195 		TAILQ_REMOVE(&ptbl_buf_freelist, buf, link);
196 	mtx_unlock(&ptbl_buf_freelist_lock);
197 
198 	CTR2(KTR_PMAP, "%s: buf = %p", __func__, buf);
199 
200 	return (buf);
201 }
202 
203 /* Return ptbl buff to free pool. */
204 static void
205 ptbl_buf_free(struct ptbl_buf *buf)
206 {
207 
208 	CTR2(KTR_PMAP, "%s: buf = %p", __func__, buf);
209 
210 	mtx_lock(&ptbl_buf_freelist_lock);
211 	TAILQ_INSERT_TAIL(&ptbl_buf_freelist, buf, link);
212 	mtx_unlock(&ptbl_buf_freelist_lock);
213 }
214 
215 /*
216  * Search the list of allocated ptbl bufs and find on list of allocated ptbls
217  */
218 static void
219 ptbl_free_pmap_ptbl(pmap_t pmap, pte_t *ptbl)
220 {
221 	struct ptbl_buf *pbuf;
222 
223 	CTR2(KTR_PMAP, "%s: ptbl = %p", __func__, ptbl);
224 
225 	PMAP_LOCK_ASSERT(pmap, MA_OWNED);
226 
227 	TAILQ_FOREACH(pbuf, &pmap->pm_ptbl_list, link)
228 		if (pbuf->kva == (vm_offset_t)ptbl) {
229 			/* Remove from pmap ptbl buf list. */
230 			TAILQ_REMOVE(&pmap->pm_ptbl_list, pbuf, link);
231 
232 			/* Free corresponding ptbl buf. */
233 			ptbl_buf_free(pbuf);
234 			break;
235 		}
236 }
237 
238 /* Allocate page table. */
239 static pte_t *
240 ptbl_alloc(mmu_t mmu, pmap_t pmap, unsigned int pdir_idx, boolean_t nosleep)
241 {
242 	vm_page_t mtbl[PTBL_PAGES];
243 	vm_page_t m;
244 	struct ptbl_buf *pbuf;
245 	unsigned int pidx;
246 	pte_t *ptbl;
247 	int i, j;
248 
249 	CTR4(KTR_PMAP, "%s: pmap = %p su = %d pdir_idx = %d", __func__, pmap,
250 	    (pmap == kernel_pmap), pdir_idx);
251 
252 	KASSERT((pdir_idx <= (VM_MAXUSER_ADDRESS / PDIR_SIZE)),
253 	    ("ptbl_alloc: invalid pdir_idx"));
254 	KASSERT((pmap->pm_pdir[pdir_idx] == NULL),
255 	    ("pte_alloc: valid ptbl entry exists!"));
256 
257 	pbuf = ptbl_buf_alloc();
258 	if (pbuf == NULL)
259 		panic("pte_alloc: couldn't alloc kernel virtual memory");
260 
261 	ptbl = (pte_t *)pbuf->kva;
262 
263 	CTR2(KTR_PMAP, "%s: ptbl kva = %p", __func__, ptbl);
264 
265 	for (i = 0; i < PTBL_PAGES; i++) {
266 		pidx = (PTBL_PAGES * pdir_idx) + i;
267 		while ((m = vm_page_alloc(NULL, pidx,
268 		    VM_ALLOC_NOOBJ | VM_ALLOC_WIRED)) == NULL) {
269 			if (nosleep) {
270 				ptbl_free_pmap_ptbl(pmap, ptbl);
271 				for (j = 0; j < i; j++)
272 					vm_page_free(mtbl[j]);
273 				vm_wire_sub(i);
274 				return (NULL);
275 			}
276 			PMAP_UNLOCK(pmap);
277 			rw_wunlock(&pvh_global_lock);
278 			vm_wait(NULL);
279 			rw_wlock(&pvh_global_lock);
280 			PMAP_LOCK(pmap);
281 		}
282 		mtbl[i] = m;
283 	}
284 
285 	/* Map allocated pages into kernel_pmap. */
286 	mmu_booke_qenter(mmu, (vm_offset_t)ptbl, mtbl, PTBL_PAGES);
287 
288 	/* Zero whole ptbl. */
289 	bzero((caddr_t)ptbl, PTBL_PAGES * PAGE_SIZE);
290 
291 	/* Add pbuf to the pmap ptbl bufs list. */
292 	TAILQ_INSERT_TAIL(&pmap->pm_ptbl_list, pbuf, link);
293 
294 	return (ptbl);
295 }
296 
297 /* Free ptbl pages and invalidate pdir entry. */
298 static void
299 ptbl_free(mmu_t mmu, pmap_t pmap, unsigned int pdir_idx)
300 {
301 	pte_t *ptbl;
302 	vm_paddr_t pa;
303 	vm_offset_t va;
304 	vm_page_t m;
305 	int i;
306 
307 	CTR4(KTR_PMAP, "%s: pmap = %p su = %d pdir_idx = %d", __func__, pmap,
308 	    (pmap == kernel_pmap), pdir_idx);
309 
310 	KASSERT((pdir_idx <= (VM_MAXUSER_ADDRESS / PDIR_SIZE)),
311 	    ("ptbl_free: invalid pdir_idx"));
312 
313 	ptbl = pmap->pm_pdir[pdir_idx];
314 
315 	CTR2(KTR_PMAP, "%s: ptbl = %p", __func__, ptbl);
316 
317 	KASSERT((ptbl != NULL), ("ptbl_free: null ptbl"));
318 
319 	/*
320 	 * Invalidate the pdir entry as soon as possible, so that other CPUs
321 	 * don't attempt to look up the page tables we are releasing.
322 	 */
323 	mtx_lock_spin(&tlbivax_mutex);
324 	tlb_miss_lock();
325 
326 	pmap->pm_pdir[pdir_idx] = NULL;
327 
328 	tlb_miss_unlock();
329 	mtx_unlock_spin(&tlbivax_mutex);
330 
331 	for (i = 0; i < PTBL_PAGES; i++) {
332 		va = ((vm_offset_t)ptbl + (i * PAGE_SIZE));
333 		pa = pte_vatopa(mmu, kernel_pmap, va);
334 		m = PHYS_TO_VM_PAGE(pa);
335 		vm_page_free_zero(m);
336 		vm_wire_sub(1);
337 		mmu_booke_kremove(mmu, va);
338 	}
339 
340 	ptbl_free_pmap_ptbl(pmap, ptbl);
341 }
342 
343 /*
344  * Decrement ptbl pages hold count and attempt to free ptbl pages.
345  * Called when removing pte entry from ptbl.
346  *
347  * Return 1 if ptbl pages were freed.
348  */
349 static int
350 ptbl_unhold(mmu_t mmu, pmap_t pmap, unsigned int pdir_idx)
351 {
352 	pte_t *ptbl;
353 	vm_paddr_t pa;
354 	vm_page_t m;
355 	int i;
356 
357 	CTR4(KTR_PMAP, "%s: pmap = %p su = %d pdir_idx = %d", __func__, pmap,
358 	    (pmap == kernel_pmap), pdir_idx);
359 
360 	KASSERT((pdir_idx <= (VM_MAXUSER_ADDRESS / PDIR_SIZE)),
361 	    ("ptbl_unhold: invalid pdir_idx"));
362 	KASSERT((pmap != kernel_pmap),
363 	    ("ptbl_unhold: unholding kernel ptbl!"));
364 
365 	ptbl = pmap->pm_pdir[pdir_idx];
366 
367 	//debugf("ptbl_unhold: ptbl = 0x%08x\n", (u_int32_t)ptbl);
368 	KASSERT(((vm_offset_t)ptbl >= VM_MIN_KERNEL_ADDRESS),
369 	    ("ptbl_unhold: non kva ptbl"));
370 
371 	/* decrement hold count */
372 	for (i = 0; i < PTBL_PAGES; i++) {
373 		pa = pte_vatopa(mmu, kernel_pmap,
374 		    (vm_offset_t)ptbl + (i * PAGE_SIZE));
375 		m = PHYS_TO_VM_PAGE(pa);
376 		m->ref_count--;
377 	}
378 
379 	/*
380 	 * Free ptbl pages if there are no pte etries in this ptbl.
381 	 * ref_count has the same value for all ptbl pages, so check the last
382 	 * page.
383 	 */
384 	if (m->ref_count == 0) {
385 		ptbl_free(mmu, pmap, pdir_idx);
386 
387 		//debugf("ptbl_unhold: e (freed ptbl)\n");
388 		return (1);
389 	}
390 
391 	return (0);
392 }
393 
394 /*
395  * Increment hold count for ptbl pages. This routine is used when a new pte
396  * entry is being inserted into the ptbl.
397  */
398 static void
399 ptbl_hold(mmu_t mmu, pmap_t pmap, unsigned int pdir_idx)
400 {
401 	vm_paddr_t pa;
402 	pte_t *ptbl;
403 	vm_page_t m;
404 	int i;
405 
406 	CTR3(KTR_PMAP, "%s: pmap = %p pdir_idx = %d", __func__, pmap,
407 	    pdir_idx);
408 
409 	KASSERT((pdir_idx <= (VM_MAXUSER_ADDRESS / PDIR_SIZE)),
410 	    ("ptbl_hold: invalid pdir_idx"));
411 	KASSERT((pmap != kernel_pmap),
412 	    ("ptbl_hold: holding kernel ptbl!"));
413 
414 	ptbl = pmap->pm_pdir[pdir_idx];
415 
416 	KASSERT((ptbl != NULL), ("ptbl_hold: null ptbl"));
417 
418 	for (i = 0; i < PTBL_PAGES; i++) {
419 		pa = pte_vatopa(mmu, kernel_pmap,
420 		    (vm_offset_t)ptbl + (i * PAGE_SIZE));
421 		m = PHYS_TO_VM_PAGE(pa);
422 		m->ref_count++;
423 	}
424 }
425 
426 /*
427  * Clean pte entry, try to free page table page if requested.
428  *
429  * Return 1 if ptbl pages were freed, otherwise return 0.
430  */
431 static int
432 pte_remove(mmu_t mmu, pmap_t pmap, vm_offset_t va, uint8_t flags)
433 {
434 	unsigned int pdir_idx = PDIR_IDX(va);
435 	unsigned int ptbl_idx = PTBL_IDX(va);
436 	vm_page_t m;
437 	pte_t *ptbl;
438 	pte_t *pte;
439 
440 	//int su = (pmap == kernel_pmap);
441 	//debugf("pte_remove: s (su = %d pmap = 0x%08x va = 0x%08x flags = %d)\n",
442 	//		su, (u_int32_t)pmap, va, flags);
443 
444 	ptbl = pmap->pm_pdir[pdir_idx];
445 	KASSERT(ptbl, ("pte_remove: null ptbl"));
446 
447 	pte = &ptbl[ptbl_idx];
448 
449 	if (pte == NULL || !PTE_ISVALID(pte))
450 		return (0);
451 
452 	if (PTE_ISWIRED(pte))
453 		pmap->pm_stats.wired_count--;
454 
455 	/* Get vm_page_t for mapped pte. */
456 	m = PHYS_TO_VM_PAGE(PTE_PA(pte));
457 
458 	/* Handle managed entry. */
459 	if (PTE_ISMANAGED(pte)) {
460 
461 		if (PTE_ISMODIFIED(pte))
462 			vm_page_dirty(m);
463 
464 		if (PTE_ISREFERENCED(pte))
465 			vm_page_aflag_set(m, PGA_REFERENCED);
466 
467 		pv_remove(pmap, va, m);
468 	} else if (pmap == kernel_pmap && m && m->md.pv_tracked) {
469 		/*
470 		 * Always pv_insert()/pv_remove() on MPC85XX, in case DPAA is
471 		 * used.  This is needed by the NCSW support code for fast
472 		 * VA<->PA translation.
473 		 */
474 		pv_remove(pmap, va, m);
475 		if (TAILQ_EMPTY(&m->md.pv_list))
476 			m->md.pv_tracked = false;
477 	}
478 
479 	mtx_lock_spin(&tlbivax_mutex);
480 	tlb_miss_lock();
481 
482 	tlb0_flush_entry(va);
483 	*pte = 0;
484 
485 	tlb_miss_unlock();
486 	mtx_unlock_spin(&tlbivax_mutex);
487 
488 	pmap->pm_stats.resident_count--;
489 
490 	if (flags & PTBL_UNHOLD) {
491 		//debugf("pte_remove: e (unhold)\n");
492 		return (ptbl_unhold(mmu, pmap, pdir_idx));
493 	}
494 
495 	//debugf("pte_remove: e\n");
496 	return (0);
497 }
498 
499 /*
500  * Insert PTE for a given page and virtual address.
501  */
502 static int
503 pte_enter(mmu_t mmu, pmap_t pmap, vm_page_t m, vm_offset_t va, uint32_t flags,
504     boolean_t nosleep)
505 {
506 	unsigned int pdir_idx = PDIR_IDX(va);
507 	unsigned int ptbl_idx = PTBL_IDX(va);
508 	pte_t *ptbl, *pte, pte_tmp;
509 
510 	CTR4(KTR_PMAP, "%s: su = %d pmap = %p va = %p", __func__,
511 	    pmap == kernel_pmap, pmap, va);
512 
513 	/* Get the page table pointer. */
514 	ptbl = pmap->pm_pdir[pdir_idx];
515 
516 	if (ptbl == NULL) {
517 		/* Allocate page table pages. */
518 		ptbl = ptbl_alloc(mmu, pmap, pdir_idx, nosleep);
519 		if (ptbl == NULL) {
520 			KASSERT(nosleep, ("nosleep and NULL ptbl"));
521 			return (ENOMEM);
522 		}
523 		pmap->pm_pdir[pdir_idx] = ptbl;
524 		pte = &ptbl[ptbl_idx];
525 	} else {
526 		/*
527 		 * Check if there is valid mapping for requested
528 		 * va, if there is, remove it.
529 		 */
530 		pte = &pmap->pm_pdir[pdir_idx][ptbl_idx];
531 		if (PTE_ISVALID(pte)) {
532 			pte_remove(mmu, pmap, va, PTBL_HOLD);
533 		} else {
534 			/*
535 			 * pte is not used, increment hold count
536 			 * for ptbl pages.
537 			 */
538 			if (pmap != kernel_pmap)
539 				ptbl_hold(mmu, pmap, pdir_idx);
540 		}
541 	}
542 
543 	/*
544 	 * Insert pv_entry into pv_list for mapped page if part of managed
545 	 * memory.
546 	 */
547 	if ((m->oflags & VPO_UNMANAGED) == 0) {
548 		flags |= PTE_MANAGED;
549 
550 		/* Create and insert pv entry. */
551 		pv_insert(pmap, va, m);
552 	}
553 
554 	pmap->pm_stats.resident_count++;
555 
556 	pte_tmp = PTE_RPN_FROM_PA(VM_PAGE_TO_PHYS(m));
557 	pte_tmp |= (PTE_VALID | flags | PTE_PS_4KB); /* 4KB pages only */
558 
559 	mtx_lock_spin(&tlbivax_mutex);
560 	tlb_miss_lock();
561 
562 	tlb0_flush_entry(va);
563 	*pte = pte_tmp;
564 
565 	tlb_miss_unlock();
566 	mtx_unlock_spin(&tlbivax_mutex);
567 	return (0);
568 }
569 
570 /* Return the pa for the given pmap/va. */
571 static vm_paddr_t
572 pte_vatopa(mmu_t mmu, pmap_t pmap, vm_offset_t va)
573 {
574 	vm_paddr_t pa = 0;
575 	pte_t *pte;
576 
577 	pte = pte_find(mmu, pmap, va);
578 	if ((pte != NULL) && PTE_ISVALID(pte))
579 		pa = (PTE_PA(pte) | (va & PTE_PA_MASK));
580 	return (pa);
581 }
582 
583 /* Get a pointer to a PTE in a page table. */
584 static pte_t *
585 pte_find(mmu_t mmu, pmap_t pmap, vm_offset_t va)
586 {
587 	unsigned int pdir_idx = PDIR_IDX(va);
588 	unsigned int ptbl_idx = PTBL_IDX(va);
589 
590 	KASSERT((pmap != NULL), ("pte_find: invalid pmap"));
591 
592 	if (pmap->pm_pdir[pdir_idx])
593 		return (&(pmap->pm_pdir[pdir_idx][ptbl_idx]));
594 
595 	return (NULL);
596 }
597 
598 /* Set up kernel page tables. */
599 static void
600 kernel_pte_alloc(vm_offset_t data_end, vm_offset_t addr, vm_offset_t pdir)
601 {
602 	int		i;
603 	vm_offset_t	va;
604 	pte_t		*pte;
605 
606 	/* Initialize kernel pdir */
607 	for (i = 0; i < kernel_ptbls; i++)
608 		kernel_pmap->pm_pdir[kptbl_min + i] =
609 		    (pte_t *)(pdir + (i * PAGE_SIZE * PTBL_PAGES));
610 
611 	/*
612 	 * Fill in PTEs covering kernel code and data. They are not required
613 	 * for address translation, as this area is covered by static TLB1
614 	 * entries, but for pte_vatopa() to work correctly with kernel area
615 	 * addresses.
616 	 */
617 	for (va = addr; va < data_end; va += PAGE_SIZE) {
618 		pte = &(kernel_pmap->pm_pdir[PDIR_IDX(va)][PTBL_IDX(va)]);
619 		*pte = PTE_RPN_FROM_PA(kernload + (va - kernstart));
620 		*pte |= PTE_M | PTE_SR | PTE_SW | PTE_SX | PTE_WIRED |
621 		    PTE_VALID | PTE_PS_4KB;
622 	}
623 }
624 
625 /*
626  * Initialize a preallocated and zeroed pmap structure,
627  * such as one in a vmspace structure.
628  */
629 static void
630 mmu_booke_pinit(mmu_t mmu, pmap_t pmap)
631 {
632 	int i;
633 
634 	CTR4(KTR_PMAP, "%s: pmap = %p, proc %d '%s'", __func__, pmap,
635 	    curthread->td_proc->p_pid, curthread->td_proc->p_comm);
636 
637 	KASSERT((pmap != kernel_pmap), ("pmap_pinit: initializing kernel_pmap"));
638 
639 	for (i = 0; i < MAXCPU; i++)
640 		pmap->pm_tid[i] = TID_NONE;
641 	CPU_ZERO(&kernel_pmap->pm_active);
642 	bzero(&pmap->pm_stats, sizeof(pmap->pm_stats));
643 	pmap->pm_pdir = uma_zalloc(ptbl_root_zone, M_WAITOK);
644 	bzero(pmap->pm_pdir, sizeof(pte_t *) * PDIR_NENTRIES);
645 	TAILQ_INIT(&pmap->pm_ptbl_list);
646 }
647 
648 /*
649  * Release any resources held by the given physical map.
650  * Called when a pmap initialized by mmu_booke_pinit is being released.
651  * Should only be called if the map contains no valid mappings.
652  */
653 static void
654 mmu_booke_release(mmu_t mmu, pmap_t pmap)
655 {
656 
657 	KASSERT(pmap->pm_stats.resident_count == 0,
658 	    ("pmap_release: pmap resident count %ld != 0",
659 	    pmap->pm_stats.resident_count));
660 	uma_zfree(ptbl_root_zone, pmap->pm_pdir);
661 }
662 
663 static void
664 mmu_booke_sync_icache(mmu_t mmu, pmap_t pm, vm_offset_t va, vm_size_t sz)
665 {
666 	pte_t *pte;
667 	vm_paddr_t pa = 0;
668 	int sync_sz, valid;
669 	pmap_t pmap;
670 	vm_page_t m;
671 	vm_offset_t addr;
672 	int active;
673 
674 	rw_wlock(&pvh_global_lock);
675 	pmap = PCPU_GET(curpmap);
676 	active = (pm == kernel_pmap || pm == pmap) ? 1 : 0;
677 	while (sz > 0) {
678 		PMAP_LOCK(pm);
679 		pte = pte_find(mmu, pm, va);
680 		valid = (pte != NULL && PTE_ISVALID(pte)) ? 1 : 0;
681 		if (valid)
682 			pa = PTE_PA(pte);
683 		PMAP_UNLOCK(pm);
684 		sync_sz = PAGE_SIZE - (va & PAGE_MASK);
685 		sync_sz = min(sync_sz, sz);
686 		if (valid) {
687 			if (!active) {
688 				/* Create a mapping in the active pmap. */
689 				addr = 0;
690 				m = PHYS_TO_VM_PAGE(pa);
691 				PMAP_LOCK(pmap);
692 				pte_enter(mmu, pmap, m, addr,
693 				    PTE_SR | PTE_VALID, FALSE);
694 				addr += (va & PAGE_MASK);
695 				__syncicache((void *)addr, sync_sz);
696 				pte_remove(mmu, pmap, addr, PTBL_UNHOLD);
697 				PMAP_UNLOCK(pmap);
698 			} else
699 				__syncicache((void *)va, sync_sz);
700 		}
701 		va += sync_sz;
702 		sz -= sync_sz;
703 	}
704 	rw_wunlock(&pvh_global_lock);
705 }
706 
707 /*
708  * mmu_booke_zero_page_area zeros the specified hardware page by
709  * mapping it into virtual memory and using bzero to clear
710  * its contents.
711  *
712  * off and size must reside within a single page.
713  */
714 static void
715 mmu_booke_zero_page_area(mmu_t mmu, vm_page_t m, int off, int size)
716 {
717 	vm_offset_t va;
718 
719 	/* XXX KASSERT off and size are within a single page? */
720 
721 	mtx_lock(&zero_page_mutex);
722 	va = zero_page_va;
723 
724 	mmu_booke_kenter(mmu, va, VM_PAGE_TO_PHYS(m));
725 	bzero((caddr_t)va + off, size);
726 	mmu_booke_kremove(mmu, va);
727 
728 	mtx_unlock(&zero_page_mutex);
729 }
730 
731 /*
732  * mmu_booke_zero_page zeros the specified hardware page.
733  */
734 static void
735 mmu_booke_zero_page(mmu_t mmu, vm_page_t m)
736 {
737 	vm_offset_t off, va;
738 
739 	va = zero_page_va;
740 	mtx_lock(&zero_page_mutex);
741 
742 	mmu_booke_kenter(mmu, va, VM_PAGE_TO_PHYS(m));
743 
744 	for (off = 0; off < PAGE_SIZE; off += cacheline_size)
745 		__asm __volatile("dcbz 0,%0" :: "r"(va + off));
746 
747 	mmu_booke_kremove(mmu, va);
748 
749 	mtx_unlock(&zero_page_mutex);
750 }
751 
752 /*
753  * mmu_booke_copy_page copies the specified (machine independent) page by
754  * mapping the page into virtual memory and using memcopy to copy the page,
755  * one machine dependent page at a time.
756  */
757 static void
758 mmu_booke_copy_page(mmu_t mmu, vm_page_t sm, vm_page_t dm)
759 {
760 	vm_offset_t sva, dva;
761 
762 	sva = copy_page_src_va;
763 	dva = copy_page_dst_va;
764 
765 	mtx_lock(&copy_page_mutex);
766 	mmu_booke_kenter(mmu, sva, VM_PAGE_TO_PHYS(sm));
767 	mmu_booke_kenter(mmu, dva, VM_PAGE_TO_PHYS(dm));
768 
769 	memcpy((caddr_t)dva, (caddr_t)sva, PAGE_SIZE);
770 
771 	mmu_booke_kremove(mmu, dva);
772 	mmu_booke_kremove(mmu, sva);
773 	mtx_unlock(&copy_page_mutex);
774 }
775 
776 static inline void
777 mmu_booke_copy_pages(mmu_t mmu, vm_page_t *ma, vm_offset_t a_offset,
778     vm_page_t *mb, vm_offset_t b_offset, int xfersize)
779 {
780 	void *a_cp, *b_cp;
781 	vm_offset_t a_pg_offset, b_pg_offset;
782 	int cnt;
783 
784 	mtx_lock(&copy_page_mutex);
785 	while (xfersize > 0) {
786 		a_pg_offset = a_offset & PAGE_MASK;
787 		cnt = min(xfersize, PAGE_SIZE - a_pg_offset);
788 		mmu_booke_kenter(mmu, copy_page_src_va,
789 		    VM_PAGE_TO_PHYS(ma[a_offset >> PAGE_SHIFT]));
790 		a_cp = (char *)copy_page_src_va + a_pg_offset;
791 		b_pg_offset = b_offset & PAGE_MASK;
792 		cnt = min(cnt, PAGE_SIZE - b_pg_offset);
793 		mmu_booke_kenter(mmu, copy_page_dst_va,
794 		    VM_PAGE_TO_PHYS(mb[b_offset >> PAGE_SHIFT]));
795 		b_cp = (char *)copy_page_dst_va + b_pg_offset;
796 		bcopy(a_cp, b_cp, cnt);
797 		mmu_booke_kremove(mmu, copy_page_dst_va);
798 		mmu_booke_kremove(mmu, copy_page_src_va);
799 		a_offset += cnt;
800 		b_offset += cnt;
801 		xfersize -= cnt;
802 	}
803 	mtx_unlock(&copy_page_mutex);
804 }
805 
806 static vm_offset_t
807 mmu_booke_quick_enter_page(mmu_t mmu, vm_page_t m)
808 {
809 	vm_paddr_t paddr;
810 	vm_offset_t qaddr;
811 	uint32_t flags;
812 	pte_t *pte;
813 
814 	paddr = VM_PAGE_TO_PHYS(m);
815 
816 	flags = PTE_SR | PTE_SW | PTE_SX | PTE_WIRED | PTE_VALID;
817 	flags |= tlb_calc_wimg(paddr, pmap_page_get_memattr(m)) << PTE_MAS2_SHIFT;
818 	flags |= PTE_PS_4KB;
819 
820 	critical_enter();
821 	qaddr = PCPU_GET(qmap_addr);
822 
823 	pte = pte_find(mmu, kernel_pmap, qaddr);
824 
825 	KASSERT(*pte == 0, ("mmu_booke_quick_enter_page: PTE busy"));
826 
827 	/*
828 	 * XXX: tlbivax is broadcast to other cores, but qaddr should
829  	 * not be present in other TLBs.  Is there a better instruction
830 	 * sequence to use? Or just forget it & use mmu_booke_kenter()...
831 	 */
832 	__asm __volatile("tlbivax 0, %0" :: "r"(qaddr & MAS2_EPN_MASK));
833 	__asm __volatile("isync; msync");
834 
835 	*pte = PTE_RPN_FROM_PA(paddr) | flags;
836 
837 	/* Flush the real memory from the instruction cache. */
838 	if ((flags & (PTE_I | PTE_G)) == 0)
839 		__syncicache((void *)qaddr, PAGE_SIZE);
840 
841 	return (qaddr);
842 }
843 
844 static void
845 mmu_booke_quick_remove_page(mmu_t mmu, vm_offset_t addr)
846 {
847 	pte_t *pte;
848 
849 	pte = pte_find(mmu, kernel_pmap, addr);
850 
851 	KASSERT(PCPU_GET(qmap_addr) == addr,
852 	    ("mmu_booke_quick_remove_page: invalid address"));
853 	KASSERT(*pte != 0,
854 	    ("mmu_booke_quick_remove_page: PTE not in use"));
855 
856 	*pte = 0;
857 	critical_exit();
858 }
859 
860 /**************************************************************************/
861 /* TID handling */
862 /**************************************************************************/
863 
864 /*
865  * Return the largest uint value log such that 2^log <= num.
866  */
867 static unsigned long
868 ilog2(unsigned long num)
869 {
870 	long lz;
871 
872 	__asm ("cntlzw %0, %1" : "=r" (lz) : "r" (num));
873 	return (31 - lz);
874 }
875 
876 /*
877  * Invalidate all TLB0 entries which match the given TID. Note this is
878  * dedicated for cases when invalidations should NOT be propagated to other
879  * CPUs.
880  */
881 static void
882 tid_flush(tlbtid_t tid)
883 {
884 	register_t msr;
885 	uint32_t mas0, mas1, mas2;
886 	int entry, way;
887 
888 
889 	/* Don't evict kernel translations */
890 	if (tid == TID_KERNEL)
891 		return;
892 
893 	msr = mfmsr();
894 	__asm __volatile("wrteei 0");
895 
896 	/*
897 	 * Newer (e500mc and later) have tlbilx, which doesn't broadcast, so use
898 	 * it for PID invalidation.
899 	 */
900 	switch ((mfpvr() >> 16) & 0xffff) {
901 	case FSL_E500mc:
902 	case FSL_E5500:
903 	case FSL_E6500:
904 		mtspr(SPR_MAS6, tid << MAS6_SPID0_SHIFT);
905 		/* tlbilxpid */
906 		__asm __volatile("isync; .long 0x7c200024; isync; msync");
907 		__asm __volatile("wrtee %0" :: "r"(msr));
908 		return;
909 	}
910 
911 	for (way = 0; way < TLB0_WAYS; way++)
912 		for (entry = 0; entry < TLB0_ENTRIES_PER_WAY; entry++) {
913 
914 			mas0 = MAS0_TLBSEL(0) | MAS0_ESEL(way);
915 			mtspr(SPR_MAS0, mas0);
916 
917 			mas2 = entry << MAS2_TLB0_ENTRY_IDX_SHIFT;
918 			mtspr(SPR_MAS2, mas2);
919 
920 			__asm __volatile("isync; tlbre");
921 
922 			mas1 = mfspr(SPR_MAS1);
923 
924 			if (!(mas1 & MAS1_VALID))
925 				continue;
926 			if (((mas1 & MAS1_TID_MASK) >> MAS1_TID_SHIFT) != tid)
927 				continue;
928 			mas1 &= ~MAS1_VALID;
929 			mtspr(SPR_MAS1, mas1);
930 			__asm __volatile("isync; tlbwe; isync; msync");
931 		}
932 	__asm __volatile("wrtee %0" :: "r"(msr));
933 }
934