xref: /freebsd/sys/powerpc/pseries/mmu_phyp.c (revision f6a3b357e9be4c6423c85eff9a847163a0d307c8)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright (C) 2010 Andreas Tobler
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice, this list of conditions and the following disclaimer.
12  * 2. Redistributions in binary form must reproduce the above copyright
13  *    notice, this list of conditions and the following disclaimer in the
14  *    documentation and/or other materials provided with the distribution.
15  *
16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19  * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
20  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
21  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
22  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
23  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
24  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
25  * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26  */
27 
28 #include <sys/cdefs.h>
29 __FBSDID("$FreeBSD$");
30 
31 #include <sys/param.h>
32 #include <sys/kernel.h>
33 #include <sys/ktr.h>
34 #include <sys/lock.h>
35 #include <sys/rmlock.h>
36 #include <sys/mutex.h>
37 #include <sys/proc.h>
38 #include <sys/sysctl.h>
39 #include <sys/systm.h>
40 #include <sys/vmmeter.h>
41 
42 #include <dev/ofw/openfirm.h>
43 #include <machine/ofw_machdep.h>
44 
45 #include <vm/vm.h>
46 #include <vm/vm_param.h>
47 #include <vm/vm_kern.h>
48 #include <vm/vm_page.h>
49 #include <vm/vm_map.h>
50 #include <vm/vm_object.h>
51 #include <vm/vm_extern.h>
52 #include <vm/vm_pageout.h>
53 #include <vm/uma.h>
54 
55 #include <powerpc/aim/mmu_oea64.h>
56 
57 #include "mmu_if.h"
58 #include "moea64_if.h"
59 
60 #include "phyp-hvcall.h"
61 
62 #define MMU_PHYP_DEBUG 0
63 #define MMU_PHYP_ID "mmu_phyp: "
64 #if MMU_PHYP_DEBUG
65 #define dprintf(fmt, ...) printf(fmt, ## __VA_ARGS__)
66 #define dprintf0(fmt, ...) dprintf(MMU_PHYP_ID fmt, ## __VA_ARGS__)
67 #else
68 #define dprintf(fmt, args...) do { ; } while(0)
69 #define dprintf0(fmt, args...) do { ; } while(0)
70 #endif
71 
72 static struct rmlock mphyp_eviction_lock;
73 
74 /*
75  * Kernel MMU interface
76  */
77 
78 static void	mphyp_bootstrap(mmu_t mmup, vm_offset_t kernelstart,
79 		    vm_offset_t kernelend);
80 static void	mphyp_cpu_bootstrap(mmu_t mmup, int ap);
81 static int64_t	mphyp_pte_synch(mmu_t, struct pvo_entry *pvo);
82 static int64_t	mphyp_pte_clear(mmu_t, struct pvo_entry *pvo, uint64_t ptebit);
83 static int64_t	mphyp_pte_unset(mmu_t, struct pvo_entry *pvo);
84 static int	mphyp_pte_insert(mmu_t, struct pvo_entry *pvo);
85 
86 static mmu_method_t mphyp_methods[] = {
87         MMUMETHOD(mmu_bootstrap,        mphyp_bootstrap),
88         MMUMETHOD(mmu_cpu_bootstrap,    mphyp_cpu_bootstrap),
89 
90 	MMUMETHOD(moea64_pte_synch,     mphyp_pte_synch),
91         MMUMETHOD(moea64_pte_clear,     mphyp_pte_clear),
92         MMUMETHOD(moea64_pte_unset,     mphyp_pte_unset),
93         MMUMETHOD(moea64_pte_insert,    mphyp_pte_insert),
94 
95 	/* XXX: pmap_copy_page, pmap_init_page with H_PAGE_INIT */
96 
97         { 0, 0 }
98 };
99 
100 MMU_DEF_INHERIT(pseries_mmu, "mmu_phyp", mphyp_methods, 0, oea64_mmu);
101 
102 static int brokenkvm = 0;
103 
104 static void
105 print_kvm_bug_warning(void *data)
106 {
107 
108 	if (brokenkvm)
109 		printf("WARNING: Running on a broken hypervisor that does "
110 		    "not support mandatory H_CLEAR_MOD and H_CLEAR_REF "
111 		    "hypercalls. Performance will be suboptimal.\n");
112 }
113 
114 SYSINIT(kvmbugwarn1, SI_SUB_COPYRIGHT, SI_ORDER_THIRD + 1,
115     print_kvm_bug_warning, NULL);
116 SYSINIT(kvmbugwarn2, SI_SUB_LAST, SI_ORDER_THIRD + 1, print_kvm_bug_warning,
117     NULL);
118 
119 static void
120 mphyp_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernelend)
121 {
122 	uint64_t final_pteg_count = 0;
123 	char buf[8];
124 	uint32_t prop[2];
125 	uint32_t nptlp, shift = 0, slb_encoding = 0;
126 	uint32_t lp_size, lp_encoding;
127 	struct lpte old;
128 	uint64_t vsid;
129 	phandle_t dev, node, root;
130 	int idx, len, res;
131 
132 	rm_init(&mphyp_eviction_lock, "pte eviction");
133 
134 	moea64_early_bootstrap(mmup, kernelstart, kernelend);
135 
136 	root = OF_peer(0);
137 
138         dev = OF_child(root);
139 	while (dev != 0) {
140                 res = OF_getprop(dev, "name", buf, sizeof(buf));
141                 if (res > 0 && strcmp(buf, "cpus") == 0)
142                         break;
143                 dev = OF_peer(dev);
144         }
145 
146 	node = OF_child(dev);
147 
148 	while (node != 0) {
149                 res = OF_getprop(node, "device_type", buf, sizeof(buf));
150                 if (res > 0 && strcmp(buf, "cpu") == 0)
151                         break;
152                 node = OF_peer(node);
153         }
154 
155 	res = OF_getencprop(node, "ibm,pft-size", prop, sizeof(prop));
156 	if (res <= 0)
157 		panic("mmu_phyp: unknown PFT size");
158 	final_pteg_count = 1 << prop[1];
159 	res = OF_getencprop(node, "ibm,slb-size", prop, sizeof(prop[0]));
160 	if (res > 0)
161 		n_slbs = prop[0];
162 	dprintf0("slb-size=%i\n", n_slbs);
163 
164 	moea64_pteg_count = final_pteg_count / sizeof(struct lpteg);
165 
166 	/* Clear any old page table entries */
167 	for (idx = 0; idx < moea64_pteg_count*8; idx++) {
168 		phyp_pft_hcall(H_READ, 0, idx, 0, 0, &old.pte_hi,
169 		    &old.pte_lo, &old.pte_lo);
170 		vsid = (old.pte_hi << (ADDR_API_SHFT64 - ADDR_PIDX_SHFT)) >> 28;
171 		if (vsid == VSID_VRMA || vsid == 0 /* Older VRMA */)
172 			continue;
173 
174 		if (old.pte_hi & LPTE_VALID)
175 			phyp_hcall(H_REMOVE, 0, idx, 0);
176 	}
177 
178 	/*
179 	 * Scan the large page size property for PAPR compatible machines.
180 	 * See PAPR D.5 Changes to Section 5.1.4, 'CPU Node Properties'
181 	 * for the encoding of the property.
182 	 */
183 
184 	len = OF_getproplen(node, "ibm,segment-page-sizes");
185 	if (len > 0) {
186 		/*
187 		 * We have to use a variable length array on the stack
188 		 * since we have very limited stack space.
189 		 */
190 		pcell_t arr[len/sizeof(cell_t)];
191 		res = OF_getencprop(node, "ibm,segment-page-sizes", arr,
192 		    sizeof(arr));
193 		len /= 4;
194 		idx = 0;
195 		while (len > 0) {
196 			shift = arr[idx];
197 			slb_encoding = arr[idx + 1];
198 			nptlp = arr[idx + 2];
199 
200 			dprintf0("Segment Page Size: "
201 			    "%uKB, slb_enc=0x%X: {size, encoding}[%u] =",
202 			    shift > 10? 1 << (shift-10) : 0,
203 			    slb_encoding, nptlp);
204 
205 			idx += 3;
206 			len -= 3;
207 			while (len > 0 && nptlp) {
208 				lp_size = arr[idx];
209 				lp_encoding = arr[idx+1];
210 
211 				dprintf(" {%uKB, 0x%X}",
212 				    lp_size > 10? 1 << (lp_size-10) : 0,
213 				    lp_encoding);
214 
215 				if (slb_encoding == SLBV_L && lp_encoding == 0)
216 					break;
217 
218 				idx += 2;
219 				len -= 2;
220 				nptlp--;
221 			}
222 			dprintf("\n");
223 			if (nptlp && slb_encoding == SLBV_L && lp_encoding == 0)
224 				break;
225 		}
226 
227 		if (len > 0) {
228 			moea64_large_page_shift = shift;
229 			moea64_large_page_size = 1ULL << lp_size;
230 			moea64_large_page_mask = moea64_large_page_size - 1;
231 			hw_direct_map = 1;
232 			printf(MMU_PHYP_ID
233 			    "Support for hugepages of %uKB detected\n",
234 			    moea64_large_page_shift > 10?
235 				1 << (moea64_large_page_shift-10) : 0);
236 		} else {
237 			moea64_large_page_size = 0;
238 			moea64_large_page_shift = 0;
239 			moea64_large_page_mask = 0;
240 			hw_direct_map = 0;
241 			printf(MMU_PHYP_ID
242 			    "Support for hugepages not found\n");
243 		}
244 	}
245 
246 	moea64_mid_bootstrap(mmup, kernelstart, kernelend);
247 	moea64_late_bootstrap(mmup, kernelstart, kernelend);
248 
249 	/* Test for broken versions of KVM that don't conform to the spec */
250 	if (phyp_hcall(H_CLEAR_MOD, 0, 0) == H_FUNCTION)
251 		brokenkvm = 1;
252 }
253 
254 static void
255 mphyp_cpu_bootstrap(mmu_t mmup, int ap)
256 {
257 	struct slb *slb = PCPU_GET(aim.slb);
258 	register_t seg0;
259 	int i;
260 
261 	/*
262 	 * Install kernel SLB entries
263 	 */
264 
265         __asm __volatile ("slbia");
266         __asm __volatile ("slbmfee %0,%1; slbie %0;" : "=r"(seg0) : "r"(0));
267 	for (i = 0; i < 64; i++) {
268 		if (!(slb[i].slbe & SLBE_VALID))
269 			continue;
270 
271 		__asm __volatile ("slbmte %0, %1" ::
272 		    "r"(slb[i].slbv), "r"(slb[i].slbe));
273 	}
274 }
275 
276 static int64_t
277 mphyp_pte_synch(mmu_t mmu, struct pvo_entry *pvo)
278 {
279 	struct lpte pte;
280 	uint64_t junk;
281 
282 	__asm __volatile("ptesync");
283 	phyp_pft_hcall(H_READ, 0, pvo->pvo_pte.slot, 0, 0, &pte.pte_hi,
284 	    &pte.pte_lo, &junk);
285 	if ((pte.pte_hi & LPTE_AVPN_MASK) !=
286 	    ((pvo->pvo_vpn >> (ADDR_API_SHFT64 - ADDR_PIDX_SHFT)) &
287 	    LPTE_AVPN_MASK))
288 		return (-1);
289 	if (!(pte.pte_hi & LPTE_VALID))
290 		return (-1);
291 
292 	return (pte.pte_lo & (LPTE_CHG | LPTE_REF));
293 }
294 
295 static int64_t
296 mphyp_pte_clear(mmu_t mmu, struct pvo_entry *pvo, uint64_t ptebit)
297 {
298 	struct rm_priotracker track;
299 	int64_t refchg;
300 	uint64_t ptelo, junk;
301 	int err;
302 
303 	/*
304 	 * This involves two steps (synch and clear) so we need the entry
305 	 * not to change in the middle. We are protected against deliberate
306 	 * unset by virtue of holding the pmap lock. Protection against
307 	 * incidental unset (page table eviction) comes from holding the
308 	 * shared eviction lock.
309 	 */
310 	PMAP_LOCK_ASSERT(pvo->pvo_pmap, MA_OWNED);
311 	rm_rlock(&mphyp_eviction_lock, &track);
312 
313 	refchg = mphyp_pte_synch(mmu, pvo);
314 	if (refchg < 0) {
315 		rm_runlock(&mphyp_eviction_lock, &track);
316 		return (refchg);
317 	}
318 
319 	if (brokenkvm) {
320 		/*
321 		 * No way to clear either bit, which is total madness.
322 		 * Pessimistically claim that, once modified, it stays so
323 		 * forever and that it is never referenced.
324 		 */
325 		rm_runlock(&mphyp_eviction_lock, &track);
326 		return (refchg & ~LPTE_REF);
327 	}
328 
329 	if (ptebit & LPTE_CHG) {
330 		err = phyp_pft_hcall(H_CLEAR_MOD, 0, pvo->pvo_pte.slot, 0, 0,
331 		    &ptelo, &junk, &junk);
332 		KASSERT(err == H_SUCCESS,
333 		    ("Error clearing page change bit: %d", err));
334 		refchg |= (ptelo & LPTE_CHG);
335 	}
336 	if (ptebit & LPTE_REF) {
337 		err = phyp_pft_hcall(H_CLEAR_REF, 0, pvo->pvo_pte.slot, 0, 0,
338 		    &ptelo, &junk, &junk);
339 		KASSERT(err == H_SUCCESS,
340 		    ("Error clearing page reference bit: %d", err));
341 		refchg |= (ptelo & LPTE_REF);
342 	}
343 
344 	rm_runlock(&mphyp_eviction_lock, &track);
345 
346 	return (refchg);
347 }
348 
349 static int64_t
350 mphyp_pte_unset(mmu_t mmu, struct pvo_entry *pvo)
351 {
352 	struct lpte pte;
353 	uint64_t junk;
354 	int err;
355 
356 	PMAP_LOCK_ASSERT(pvo->pvo_pmap, MA_OWNED);
357 
358 	moea64_pte_from_pvo(pvo, &pte);
359 
360 	err = phyp_pft_hcall(H_REMOVE, H_AVPN, pvo->pvo_pte.slot,
361 	    pte.pte_hi & LPTE_AVPN_MASK, 0, &pte.pte_hi, &pte.pte_lo,
362 	    &junk);
363 	KASSERT(err == H_SUCCESS || err == H_NOT_FOUND,
364 	    ("Error removing page: %d", err));
365 
366 	if (err == H_NOT_FOUND) {
367 		STAT_MOEA64(moea64_pte_overflow--);
368 		return (-1);
369 	}
370 
371 	return (pte.pte_lo & (LPTE_REF | LPTE_CHG));
372 }
373 
374 static uintptr_t
375 mphyp_pte_spillable_ident(uintptr_t ptegbase, struct lpte *to_evict)
376 {
377 	uint64_t slot, junk, k;
378 	struct lpte pt;
379 	int     i, j;
380 
381 	/* Start at a random slot */
382 	i = mftb() % 8;
383 	k = -1;
384 	for (j = 0; j < 8; j++) {
385 		slot = ptegbase + (i + j) % 8;
386 		phyp_pft_hcall(H_READ, 0, slot, 0, 0, &pt.pte_hi,
387 		    &pt.pte_lo, &junk);
388 
389 		if (pt.pte_hi & LPTE_WIRED)
390 			continue;
391 
392 		/* This is a candidate, so remember it */
393 		k = slot;
394 
395 		/* Try to get a page that has not been used lately */
396 		if (!(pt.pte_hi & LPTE_VALID) || !(pt.pte_lo & LPTE_REF)) {
397 			memcpy(to_evict, &pt, sizeof(struct lpte));
398 			return (k);
399 		}
400 	}
401 
402 	if (k == -1)
403 		return (k);
404 
405 	phyp_pft_hcall(H_READ, 0, k, 0, 0, &to_evict->pte_hi,
406 	    &to_evict->pte_lo, &junk);
407 	return (k);
408 }
409 
410 static int
411 mphyp_pte_insert(mmu_t mmu, struct pvo_entry *pvo)
412 {
413 	struct rm_priotracker track;
414 	int64_t result;
415 	struct lpte evicted, pte;
416 	uint64_t index, junk, lastptelo;
417 
418 	PMAP_LOCK_ASSERT(pvo->pvo_pmap, MA_OWNED);
419 
420 	/* Initialize PTE */
421 	moea64_pte_from_pvo(pvo, &pte);
422 	evicted.pte_hi = 0;
423 
424 	/* Make sure further insertion is locked out during evictions */
425 	rm_rlock(&mphyp_eviction_lock, &track);
426 
427 	/*
428 	 * First try primary hash.
429 	 */
430 	pvo->pvo_pte.slot &= ~7UL; /* Base slot address */
431 	result = phyp_pft_hcall(H_ENTER, 0, pvo->pvo_pte.slot, pte.pte_hi,
432 	    pte.pte_lo, &index, &evicted.pte_lo, &junk);
433 	if (result == H_SUCCESS) {
434 		rm_runlock(&mphyp_eviction_lock, &track);
435 		pvo->pvo_pte.slot = index;
436 		return (0);
437 	}
438 	KASSERT(result == H_PTEG_FULL, ("Page insertion error: %ld "
439 	    "(ptegidx: %#zx/%#lx, PTE %#lx/%#lx", result, pvo->pvo_pte.slot,
440 	    moea64_pteg_count, pte.pte_hi, pte.pte_lo));
441 
442 	/*
443 	 * Next try secondary hash.
444 	 */
445 	pvo->pvo_vaddr ^= PVO_HID;
446 	pte.pte_hi ^= LPTE_HID;
447 	pvo->pvo_pte.slot ^= (moea64_pteg_mask << 3);
448 
449 	result = phyp_pft_hcall(H_ENTER, 0, pvo->pvo_pte.slot,
450 	    pte.pte_hi, pte.pte_lo, &index, &evicted.pte_lo, &junk);
451 	if (result == H_SUCCESS) {
452 		rm_runlock(&mphyp_eviction_lock, &track);
453 		pvo->pvo_pte.slot = index;
454 		return (0);
455 	}
456 	KASSERT(result == H_PTEG_FULL, ("Secondary page insertion error: %ld",
457 	    result));
458 
459 	/*
460 	 * Out of luck. Find a PTE to sacrifice.
461 	 */
462 
463 	/* Lock out all insertions for a bit */
464 	rm_runlock(&mphyp_eviction_lock, &track);
465 	rm_wlock(&mphyp_eviction_lock);
466 
467 	index = mphyp_pte_spillable_ident(pvo->pvo_pte.slot, &evicted);
468 	if (index == -1L) {
469 		/* Try other hash table? */
470 		pvo->pvo_vaddr ^= PVO_HID;
471 		pte.pte_hi ^= LPTE_HID;
472 		pvo->pvo_pte.slot ^= (moea64_pteg_mask << 3);
473 		index = mphyp_pte_spillable_ident(pvo->pvo_pte.slot, &evicted);
474 	}
475 
476 	if (index == -1L) {
477 		/* No freeable slots in either PTEG? We're hosed. */
478 		rm_wunlock(&mphyp_eviction_lock);
479 		panic("mphyp_pte_insert: overflow");
480 		return (-1);
481 	}
482 
483 	/* Victim acquired: update page before waving goodbye */
484 	if (evicted.pte_hi & LPTE_VALID) {
485 		result = phyp_pft_hcall(H_REMOVE, H_AVPN, index,
486 		    evicted.pte_hi & LPTE_AVPN_MASK, 0, &junk, &lastptelo,
487 		    &junk);
488 		STAT_MOEA64(moea64_pte_overflow++);
489 		KASSERT(result == H_SUCCESS || result == H_NOT_FOUND,
490 		    ("Error evicting page: %d", (int)result));
491 	}
492 
493 	/*
494 	 * Set the new PTE.
495 	 */
496 	result = phyp_pft_hcall(H_ENTER, H_EXACT, index, pte.pte_hi,
497 	    pte.pte_lo, &index, &evicted.pte_lo, &junk);
498 	rm_wunlock(&mphyp_eviction_lock); /* All clear */
499 
500 	pvo->pvo_pte.slot = index;
501 	if (result == H_SUCCESS)
502 		return (0);
503 
504 	panic("Page replacement error: %ld", result);
505 	return (result);
506 }
507 
508