xref: /linux/arch/powerpc/kvm/book3s_hv_rm_mmu.c (revision c4ee0af3fa0dc65f690fc908f02b8355f9576ea0)
1 /*
2  * This program is free software; you can redistribute it and/or modify
3  * it under the terms of the GNU General Public License, version 2, as
4  * published by the Free Software Foundation.
5  *
6  * Copyright 2010-2011 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
7  */
8 
9 #include <linux/types.h>
10 #include <linux/string.h>
11 #include <linux/kvm.h>
12 #include <linux/kvm_host.h>
13 #include <linux/hugetlb.h>
14 #include <linux/module.h>
15 
16 #include <asm/tlbflush.h>
17 #include <asm/kvm_ppc.h>
18 #include <asm/kvm_book3s.h>
19 #include <asm/mmu-hash64.h>
20 #include <asm/hvcall.h>
21 #include <asm/synch.h>
22 #include <asm/ppc-opcode.h>
23 
24 /* Translate address of a vmalloc'd thing to a linear map address */
25 static void *real_vmalloc_addr(void *x)
26 {
27 	unsigned long addr = (unsigned long) x;
28 	pte_t *p;
29 
30 	p = find_linux_pte_or_hugepte(swapper_pg_dir, addr, NULL);
31 	if (!p || !pte_present(*p))
32 		return NULL;
33 	/* assume we don't have huge pages in vmalloc space... */
34 	addr = (pte_pfn(*p) << PAGE_SHIFT) | (addr & ~PAGE_MASK);
35 	return __va(addr);
36 }
37 
38 /* Return 1 if we need to do a global tlbie, 0 if we can use tlbiel */
39 static int global_invalidates(struct kvm *kvm, unsigned long flags)
40 {
41 	int global;
42 
43 	/*
44 	 * If there is only one vcore, and it's currently running,
45 	 * we can use tlbiel as long as we mark all other physical
46 	 * cores as potentially having stale TLB entries for this lpid.
47 	 * If we're not using MMU notifiers, we never take pages away
48 	 * from the guest, so we can use tlbiel if requested.
49 	 * Otherwise, don't use tlbiel.
50 	 */
51 	if (kvm->arch.online_vcores == 1 && local_paca->kvm_hstate.kvm_vcore)
52 		global = 0;
53 	else if (kvm->arch.using_mmu_notifiers)
54 		global = 1;
55 	else
56 		global = !(flags & H_LOCAL);
57 
58 	if (!global) {
59 		/* any other core might now have stale TLB entries... */
60 		smp_wmb();
61 		cpumask_setall(&kvm->arch.need_tlb_flush);
62 		cpumask_clear_cpu(local_paca->kvm_hstate.kvm_vcore->pcpu,
63 				  &kvm->arch.need_tlb_flush);
64 	}
65 
66 	return global;
67 }
68 
69 /*
70  * Add this HPTE into the chain for the real page.
71  * Must be called with the chain locked; it unlocks the chain.
72  */
73 void kvmppc_add_revmap_chain(struct kvm *kvm, struct revmap_entry *rev,
74 			     unsigned long *rmap, long pte_index, int realmode)
75 {
76 	struct revmap_entry *head, *tail;
77 	unsigned long i;
78 
79 	if (*rmap & KVMPPC_RMAP_PRESENT) {
80 		i = *rmap & KVMPPC_RMAP_INDEX;
81 		head = &kvm->arch.revmap[i];
82 		if (realmode)
83 			head = real_vmalloc_addr(head);
84 		tail = &kvm->arch.revmap[head->back];
85 		if (realmode)
86 			tail = real_vmalloc_addr(tail);
87 		rev->forw = i;
88 		rev->back = head->back;
89 		tail->forw = pte_index;
90 		head->back = pte_index;
91 	} else {
92 		rev->forw = rev->back = pte_index;
93 		*rmap = (*rmap & ~KVMPPC_RMAP_INDEX) |
94 			pte_index | KVMPPC_RMAP_PRESENT;
95 	}
96 	unlock_rmap(rmap);
97 }
98 EXPORT_SYMBOL_GPL(kvmppc_add_revmap_chain);
99 
100 /* Remove this HPTE from the chain for a real page */
101 static void remove_revmap_chain(struct kvm *kvm, long pte_index,
102 				struct revmap_entry *rev,
103 				unsigned long hpte_v, unsigned long hpte_r)
104 {
105 	struct revmap_entry *next, *prev;
106 	unsigned long gfn, ptel, head;
107 	struct kvm_memory_slot *memslot;
108 	unsigned long *rmap;
109 	unsigned long rcbits;
110 
111 	rcbits = hpte_r & (HPTE_R_R | HPTE_R_C);
112 	ptel = rev->guest_rpte |= rcbits;
113 	gfn = hpte_rpn(ptel, hpte_page_size(hpte_v, ptel));
114 	memslot = __gfn_to_memslot(kvm_memslots(kvm), gfn);
115 	if (!memslot)
116 		return;
117 
118 	rmap = real_vmalloc_addr(&memslot->arch.rmap[gfn - memslot->base_gfn]);
119 	lock_rmap(rmap);
120 
121 	head = *rmap & KVMPPC_RMAP_INDEX;
122 	next = real_vmalloc_addr(&kvm->arch.revmap[rev->forw]);
123 	prev = real_vmalloc_addr(&kvm->arch.revmap[rev->back]);
124 	next->back = rev->back;
125 	prev->forw = rev->forw;
126 	if (head == pte_index) {
127 		head = rev->forw;
128 		if (head == pte_index)
129 			*rmap &= ~(KVMPPC_RMAP_PRESENT | KVMPPC_RMAP_INDEX);
130 		else
131 			*rmap = (*rmap & ~KVMPPC_RMAP_INDEX) | head;
132 	}
133 	*rmap |= rcbits << KVMPPC_RMAP_RC_SHIFT;
134 	unlock_rmap(rmap);
135 }
136 
137 static pte_t lookup_linux_pte(pgd_t *pgdir, unsigned long hva,
138 			      int writing, unsigned long *pte_sizep)
139 {
140 	pte_t *ptep;
141 	unsigned long ps = *pte_sizep;
142 	unsigned int hugepage_shift;
143 
144 	ptep = find_linux_pte_or_hugepte(pgdir, hva, &hugepage_shift);
145 	if (!ptep)
146 		return __pte(0);
147 	if (hugepage_shift)
148 		*pte_sizep = 1ul << hugepage_shift;
149 	else
150 		*pte_sizep = PAGE_SIZE;
151 	if (ps > *pte_sizep)
152 		return __pte(0);
153 	return kvmppc_read_update_linux_pte(ptep, writing, hugepage_shift);
154 }
155 
156 static inline void unlock_hpte(unsigned long *hpte, unsigned long hpte_v)
157 {
158 	asm volatile(PPC_RELEASE_BARRIER "" : : : "memory");
159 	hpte[0] = hpte_v;
160 }
161 
162 long kvmppc_do_h_enter(struct kvm *kvm, unsigned long flags,
163 		       long pte_index, unsigned long pteh, unsigned long ptel,
164 		       pgd_t *pgdir, bool realmode, unsigned long *pte_idx_ret)
165 {
166 	unsigned long i, pa, gpa, gfn, psize;
167 	unsigned long slot_fn, hva;
168 	unsigned long *hpte;
169 	struct revmap_entry *rev;
170 	unsigned long g_ptel;
171 	struct kvm_memory_slot *memslot;
172 	unsigned long *physp, pte_size;
173 	unsigned long is_io;
174 	unsigned long *rmap;
175 	pte_t pte;
176 	unsigned int writing;
177 	unsigned long mmu_seq;
178 	unsigned long rcbits;
179 
180 	psize = hpte_page_size(pteh, ptel);
181 	if (!psize)
182 		return H_PARAMETER;
183 	writing = hpte_is_writable(ptel);
184 	pteh &= ~(HPTE_V_HVLOCK | HPTE_V_ABSENT | HPTE_V_VALID);
185 	ptel &= ~HPTE_GR_RESERVED;
186 	g_ptel = ptel;
187 
188 	/* used later to detect if we might have been invalidated */
189 	mmu_seq = kvm->mmu_notifier_seq;
190 	smp_rmb();
191 
192 	/* Find the memslot (if any) for this address */
193 	gpa = (ptel & HPTE_R_RPN) & ~(psize - 1);
194 	gfn = gpa >> PAGE_SHIFT;
195 	memslot = __gfn_to_memslot(kvm_memslots(kvm), gfn);
196 	pa = 0;
197 	is_io = ~0ul;
198 	rmap = NULL;
199 	if (!(memslot && !(memslot->flags & KVM_MEMSLOT_INVALID))) {
200 		/* PPC970 can't do emulated MMIO */
201 		if (!cpu_has_feature(CPU_FTR_ARCH_206))
202 			return H_PARAMETER;
203 		/* Emulated MMIO - mark this with key=31 */
204 		pteh |= HPTE_V_ABSENT;
205 		ptel |= HPTE_R_KEY_HI | HPTE_R_KEY_LO;
206 		goto do_insert;
207 	}
208 
209 	/* Check if the requested page fits entirely in the memslot. */
210 	if (!slot_is_aligned(memslot, psize))
211 		return H_PARAMETER;
212 	slot_fn = gfn - memslot->base_gfn;
213 	rmap = &memslot->arch.rmap[slot_fn];
214 
215 	if (!kvm->arch.using_mmu_notifiers) {
216 		physp = memslot->arch.slot_phys;
217 		if (!physp)
218 			return H_PARAMETER;
219 		physp += slot_fn;
220 		if (realmode)
221 			physp = real_vmalloc_addr(physp);
222 		pa = *physp;
223 		if (!pa)
224 			return H_TOO_HARD;
225 		is_io = pa & (HPTE_R_I | HPTE_R_W);
226 		pte_size = PAGE_SIZE << (pa & KVMPPC_PAGE_ORDER_MASK);
227 		pa &= PAGE_MASK;
228 		pa |= gpa & ~PAGE_MASK;
229 	} else {
230 		/* Translate to host virtual address */
231 		hva = __gfn_to_hva_memslot(memslot, gfn);
232 
233 		/* Look up the Linux PTE for the backing page */
234 		pte_size = psize;
235 		pte = lookup_linux_pte(pgdir, hva, writing, &pte_size);
236 		if (pte_present(pte)) {
237 			if (writing && !pte_write(pte))
238 				/* make the actual HPTE be read-only */
239 				ptel = hpte_make_readonly(ptel);
240 			is_io = hpte_cache_bits(pte_val(pte));
241 			pa = pte_pfn(pte) << PAGE_SHIFT;
242 			pa |= hva & (pte_size - 1);
243 			pa |= gpa & ~PAGE_MASK;
244 		}
245 	}
246 
247 	if (pte_size < psize)
248 		return H_PARAMETER;
249 
250 	ptel &= ~(HPTE_R_PP0 - psize);
251 	ptel |= pa;
252 
253 	if (pa)
254 		pteh |= HPTE_V_VALID;
255 	else
256 		pteh |= HPTE_V_ABSENT;
257 
258 	/* Check WIMG */
259 	if (is_io != ~0ul && !hpte_cache_flags_ok(ptel, is_io)) {
260 		if (is_io)
261 			return H_PARAMETER;
262 		/*
263 		 * Allow guest to map emulated device memory as
264 		 * uncacheable, but actually make it cacheable.
265 		 */
266 		ptel &= ~(HPTE_R_W|HPTE_R_I|HPTE_R_G);
267 		ptel |= HPTE_R_M;
268 	}
269 
270 	/* Find and lock the HPTEG slot to use */
271  do_insert:
272 	if (pte_index >= kvm->arch.hpt_npte)
273 		return H_PARAMETER;
274 	if (likely((flags & H_EXACT) == 0)) {
275 		pte_index &= ~7UL;
276 		hpte = (unsigned long *)(kvm->arch.hpt_virt + (pte_index << 4));
277 		for (i = 0; i < 8; ++i) {
278 			if ((*hpte & HPTE_V_VALID) == 0 &&
279 			    try_lock_hpte(hpte, HPTE_V_HVLOCK | HPTE_V_VALID |
280 					  HPTE_V_ABSENT))
281 				break;
282 			hpte += 2;
283 		}
284 		if (i == 8) {
285 			/*
286 			 * Since try_lock_hpte doesn't retry (not even stdcx.
287 			 * failures), it could be that there is a free slot
288 			 * but we transiently failed to lock it.  Try again,
289 			 * actually locking each slot and checking it.
290 			 */
291 			hpte -= 16;
292 			for (i = 0; i < 8; ++i) {
293 				while (!try_lock_hpte(hpte, HPTE_V_HVLOCK))
294 					cpu_relax();
295 				if (!(*hpte & (HPTE_V_VALID | HPTE_V_ABSENT)))
296 					break;
297 				*hpte &= ~HPTE_V_HVLOCK;
298 				hpte += 2;
299 			}
300 			if (i == 8)
301 				return H_PTEG_FULL;
302 		}
303 		pte_index += i;
304 	} else {
305 		hpte = (unsigned long *)(kvm->arch.hpt_virt + (pte_index << 4));
306 		if (!try_lock_hpte(hpte, HPTE_V_HVLOCK | HPTE_V_VALID |
307 				   HPTE_V_ABSENT)) {
308 			/* Lock the slot and check again */
309 			while (!try_lock_hpte(hpte, HPTE_V_HVLOCK))
310 				cpu_relax();
311 			if (*hpte & (HPTE_V_VALID | HPTE_V_ABSENT)) {
312 				*hpte &= ~HPTE_V_HVLOCK;
313 				return H_PTEG_FULL;
314 			}
315 		}
316 	}
317 
318 	/* Save away the guest's idea of the second HPTE dword */
319 	rev = &kvm->arch.revmap[pte_index];
320 	if (realmode)
321 		rev = real_vmalloc_addr(rev);
322 	if (rev) {
323 		rev->guest_rpte = g_ptel;
324 		note_hpte_modification(kvm, rev);
325 	}
326 
327 	/* Link HPTE into reverse-map chain */
328 	if (pteh & HPTE_V_VALID) {
329 		if (realmode)
330 			rmap = real_vmalloc_addr(rmap);
331 		lock_rmap(rmap);
332 		/* Check for pending invalidations under the rmap chain lock */
333 		if (kvm->arch.using_mmu_notifiers &&
334 		    mmu_notifier_retry(kvm, mmu_seq)) {
335 			/* inval in progress, write a non-present HPTE */
336 			pteh |= HPTE_V_ABSENT;
337 			pteh &= ~HPTE_V_VALID;
338 			unlock_rmap(rmap);
339 		} else {
340 			kvmppc_add_revmap_chain(kvm, rev, rmap, pte_index,
341 						realmode);
342 			/* Only set R/C in real HPTE if already set in *rmap */
343 			rcbits = *rmap >> KVMPPC_RMAP_RC_SHIFT;
344 			ptel &= rcbits | ~(HPTE_R_R | HPTE_R_C);
345 		}
346 	}
347 
348 	hpte[1] = ptel;
349 
350 	/* Write the first HPTE dword, unlocking the HPTE and making it valid */
351 	eieio();
352 	hpte[0] = pteh;
353 	asm volatile("ptesync" : : : "memory");
354 
355 	*pte_idx_ret = pte_index;
356 	return H_SUCCESS;
357 }
358 EXPORT_SYMBOL_GPL(kvmppc_do_h_enter);
359 
360 long kvmppc_h_enter(struct kvm_vcpu *vcpu, unsigned long flags,
361 		    long pte_index, unsigned long pteh, unsigned long ptel)
362 {
363 	return kvmppc_do_h_enter(vcpu->kvm, flags, pte_index, pteh, ptel,
364 				 vcpu->arch.pgdir, true, &vcpu->arch.gpr[4]);
365 }
366 
367 #ifdef __BIG_ENDIAN__
368 #define LOCK_TOKEN	(*(u32 *)(&get_paca()->lock_token))
369 #else
370 #define LOCK_TOKEN	(*(u32 *)(&get_paca()->paca_index))
371 #endif
372 
373 static inline int try_lock_tlbie(unsigned int *lock)
374 {
375 	unsigned int tmp, old;
376 	unsigned int token = LOCK_TOKEN;
377 
378 	asm volatile("1:lwarx	%1,0,%2\n"
379 		     "	cmpwi	cr0,%1,0\n"
380 		     "	bne	2f\n"
381 		     "  stwcx.	%3,0,%2\n"
382 		     "	bne-	1b\n"
383 		     "  isync\n"
384 		     "2:"
385 		     : "=&r" (tmp), "=&r" (old)
386 		     : "r" (lock), "r" (token)
387 		     : "cc", "memory");
388 	return old == 0;
389 }
390 
391 /*
392  * tlbie/tlbiel is a bit different on the PPC970 compared to later
393  * processors such as POWER7; the large page bit is in the instruction
394  * not RB, and the top 16 bits and the bottom 12 bits of the VA
395  * in RB must be 0.
396  */
397 static void do_tlbies_970(struct kvm *kvm, unsigned long *rbvalues,
398 			  long npages, int global, bool need_sync)
399 {
400 	long i;
401 
402 	if (global) {
403 		while (!try_lock_tlbie(&kvm->arch.tlbie_lock))
404 			cpu_relax();
405 		if (need_sync)
406 			asm volatile("ptesync" : : : "memory");
407 		for (i = 0; i < npages; ++i) {
408 			unsigned long rb = rbvalues[i];
409 
410 			if (rb & 1)		/* large page */
411 				asm volatile("tlbie %0,1" : :
412 					     "r" (rb & 0x0000fffffffff000ul));
413 			else
414 				asm volatile("tlbie %0,0" : :
415 					     "r" (rb & 0x0000fffffffff000ul));
416 		}
417 		asm volatile("eieio; tlbsync; ptesync" : : : "memory");
418 		kvm->arch.tlbie_lock = 0;
419 	} else {
420 		if (need_sync)
421 			asm volatile("ptesync" : : : "memory");
422 		for (i = 0; i < npages; ++i) {
423 			unsigned long rb = rbvalues[i];
424 
425 			if (rb & 1)		/* large page */
426 				asm volatile("tlbiel %0,1" : :
427 					     "r" (rb & 0x0000fffffffff000ul));
428 			else
429 				asm volatile("tlbiel %0,0" : :
430 					     "r" (rb & 0x0000fffffffff000ul));
431 		}
432 		asm volatile("ptesync" : : : "memory");
433 	}
434 }
435 
436 static void do_tlbies(struct kvm *kvm, unsigned long *rbvalues,
437 		      long npages, int global, bool need_sync)
438 {
439 	long i;
440 
441 	if (cpu_has_feature(CPU_FTR_ARCH_201)) {
442 		/* PPC970 tlbie instruction is a bit different */
443 		do_tlbies_970(kvm, rbvalues, npages, global, need_sync);
444 		return;
445 	}
446 	if (global) {
447 		while (!try_lock_tlbie(&kvm->arch.tlbie_lock))
448 			cpu_relax();
449 		if (need_sync)
450 			asm volatile("ptesync" : : : "memory");
451 		for (i = 0; i < npages; ++i)
452 			asm volatile(PPC_TLBIE(%1,%0) : :
453 				     "r" (rbvalues[i]), "r" (kvm->arch.lpid));
454 		asm volatile("eieio; tlbsync; ptesync" : : : "memory");
455 		kvm->arch.tlbie_lock = 0;
456 	} else {
457 		if (need_sync)
458 			asm volatile("ptesync" : : : "memory");
459 		for (i = 0; i < npages; ++i)
460 			asm volatile("tlbiel %0" : : "r" (rbvalues[i]));
461 		asm volatile("ptesync" : : : "memory");
462 	}
463 }
464 
465 long kvmppc_do_h_remove(struct kvm *kvm, unsigned long flags,
466 			unsigned long pte_index, unsigned long avpn,
467 			unsigned long *hpret)
468 {
469 	unsigned long *hpte;
470 	unsigned long v, r, rb;
471 	struct revmap_entry *rev;
472 
473 	if (pte_index >= kvm->arch.hpt_npte)
474 		return H_PARAMETER;
475 	hpte = (unsigned long *)(kvm->arch.hpt_virt + (pte_index << 4));
476 	while (!try_lock_hpte(hpte, HPTE_V_HVLOCK))
477 		cpu_relax();
478 	if ((hpte[0] & (HPTE_V_ABSENT | HPTE_V_VALID)) == 0 ||
479 	    ((flags & H_AVPN) && (hpte[0] & ~0x7fUL) != avpn) ||
480 	    ((flags & H_ANDCOND) && (hpte[0] & avpn) != 0)) {
481 		hpte[0] &= ~HPTE_V_HVLOCK;
482 		return H_NOT_FOUND;
483 	}
484 
485 	rev = real_vmalloc_addr(&kvm->arch.revmap[pte_index]);
486 	v = hpte[0] & ~HPTE_V_HVLOCK;
487 	if (v & HPTE_V_VALID) {
488 		hpte[0] &= ~HPTE_V_VALID;
489 		rb = compute_tlbie_rb(v, hpte[1], pte_index);
490 		do_tlbies(kvm, &rb, 1, global_invalidates(kvm, flags), true);
491 		/* Read PTE low word after tlbie to get final R/C values */
492 		remove_revmap_chain(kvm, pte_index, rev, v, hpte[1]);
493 	}
494 	r = rev->guest_rpte & ~HPTE_GR_RESERVED;
495 	note_hpte_modification(kvm, rev);
496 	unlock_hpte(hpte, 0);
497 
498 	hpret[0] = v;
499 	hpret[1] = r;
500 	return H_SUCCESS;
501 }
502 EXPORT_SYMBOL_GPL(kvmppc_do_h_remove);
503 
504 long kvmppc_h_remove(struct kvm_vcpu *vcpu, unsigned long flags,
505 		     unsigned long pte_index, unsigned long avpn)
506 {
507 	return kvmppc_do_h_remove(vcpu->kvm, flags, pte_index, avpn,
508 				  &vcpu->arch.gpr[4]);
509 }
510 
511 long kvmppc_h_bulk_remove(struct kvm_vcpu *vcpu)
512 {
513 	struct kvm *kvm = vcpu->kvm;
514 	unsigned long *args = &vcpu->arch.gpr[4];
515 	unsigned long *hp, *hptes[4], tlbrb[4];
516 	long int i, j, k, n, found, indexes[4];
517 	unsigned long flags, req, pte_index, rcbits;
518 	int global;
519 	long int ret = H_SUCCESS;
520 	struct revmap_entry *rev, *revs[4];
521 
522 	global = global_invalidates(kvm, 0);
523 	for (i = 0; i < 4 && ret == H_SUCCESS; ) {
524 		n = 0;
525 		for (; i < 4; ++i) {
526 			j = i * 2;
527 			pte_index = args[j];
528 			flags = pte_index >> 56;
529 			pte_index &= ((1ul << 56) - 1);
530 			req = flags >> 6;
531 			flags &= 3;
532 			if (req == 3) {		/* no more requests */
533 				i = 4;
534 				break;
535 			}
536 			if (req != 1 || flags == 3 ||
537 			    pte_index >= kvm->arch.hpt_npte) {
538 				/* parameter error */
539 				args[j] = ((0xa0 | flags) << 56) + pte_index;
540 				ret = H_PARAMETER;
541 				break;
542 			}
543 			hp = (unsigned long *)
544 				(kvm->arch.hpt_virt + (pte_index << 4));
545 			/* to avoid deadlock, don't spin except for first */
546 			if (!try_lock_hpte(hp, HPTE_V_HVLOCK)) {
547 				if (n)
548 					break;
549 				while (!try_lock_hpte(hp, HPTE_V_HVLOCK))
550 					cpu_relax();
551 			}
552 			found = 0;
553 			if (hp[0] & (HPTE_V_ABSENT | HPTE_V_VALID)) {
554 				switch (flags & 3) {
555 				case 0:		/* absolute */
556 					found = 1;
557 					break;
558 				case 1:		/* andcond */
559 					if (!(hp[0] & args[j + 1]))
560 						found = 1;
561 					break;
562 				case 2:		/* AVPN */
563 					if ((hp[0] & ~0x7fUL) == args[j + 1])
564 						found = 1;
565 					break;
566 				}
567 			}
568 			if (!found) {
569 				hp[0] &= ~HPTE_V_HVLOCK;
570 				args[j] = ((0x90 | flags) << 56) + pte_index;
571 				continue;
572 			}
573 
574 			args[j] = ((0x80 | flags) << 56) + pte_index;
575 			rev = real_vmalloc_addr(&kvm->arch.revmap[pte_index]);
576 			note_hpte_modification(kvm, rev);
577 
578 			if (!(hp[0] & HPTE_V_VALID)) {
579 				/* insert R and C bits from PTE */
580 				rcbits = rev->guest_rpte & (HPTE_R_R|HPTE_R_C);
581 				args[j] |= rcbits << (56 - 5);
582 				hp[0] = 0;
583 				continue;
584 			}
585 
586 			hp[0] &= ~HPTE_V_VALID;		/* leave it locked */
587 			tlbrb[n] = compute_tlbie_rb(hp[0], hp[1], pte_index);
588 			indexes[n] = j;
589 			hptes[n] = hp;
590 			revs[n] = rev;
591 			++n;
592 		}
593 
594 		if (!n)
595 			break;
596 
597 		/* Now that we've collected a batch, do the tlbies */
598 		do_tlbies(kvm, tlbrb, n, global, true);
599 
600 		/* Read PTE low words after tlbie to get final R/C values */
601 		for (k = 0; k < n; ++k) {
602 			j = indexes[k];
603 			pte_index = args[j] & ((1ul << 56) - 1);
604 			hp = hptes[k];
605 			rev = revs[k];
606 			remove_revmap_chain(kvm, pte_index, rev, hp[0], hp[1]);
607 			rcbits = rev->guest_rpte & (HPTE_R_R|HPTE_R_C);
608 			args[j] |= rcbits << (56 - 5);
609 			hp[0] = 0;
610 		}
611 	}
612 
613 	return ret;
614 }
615 
616 long kvmppc_h_protect(struct kvm_vcpu *vcpu, unsigned long flags,
617 		      unsigned long pte_index, unsigned long avpn,
618 		      unsigned long va)
619 {
620 	struct kvm *kvm = vcpu->kvm;
621 	unsigned long *hpte;
622 	struct revmap_entry *rev;
623 	unsigned long v, r, rb, mask, bits;
624 
625 	if (pte_index >= kvm->arch.hpt_npte)
626 		return H_PARAMETER;
627 
628 	hpte = (unsigned long *)(kvm->arch.hpt_virt + (pte_index << 4));
629 	while (!try_lock_hpte(hpte, HPTE_V_HVLOCK))
630 		cpu_relax();
631 	if ((hpte[0] & (HPTE_V_ABSENT | HPTE_V_VALID)) == 0 ||
632 	    ((flags & H_AVPN) && (hpte[0] & ~0x7fUL) != avpn)) {
633 		hpte[0] &= ~HPTE_V_HVLOCK;
634 		return H_NOT_FOUND;
635 	}
636 
637 	v = hpte[0];
638 	bits = (flags << 55) & HPTE_R_PP0;
639 	bits |= (flags << 48) & HPTE_R_KEY_HI;
640 	bits |= flags & (HPTE_R_PP | HPTE_R_N | HPTE_R_KEY_LO);
641 
642 	/* Update guest view of 2nd HPTE dword */
643 	mask = HPTE_R_PP0 | HPTE_R_PP | HPTE_R_N |
644 		HPTE_R_KEY_HI | HPTE_R_KEY_LO;
645 	rev = real_vmalloc_addr(&kvm->arch.revmap[pte_index]);
646 	if (rev) {
647 		r = (rev->guest_rpte & ~mask) | bits;
648 		rev->guest_rpte = r;
649 		note_hpte_modification(kvm, rev);
650 	}
651 	r = (hpte[1] & ~mask) | bits;
652 
653 	/* Update HPTE */
654 	if (v & HPTE_V_VALID) {
655 		rb = compute_tlbie_rb(v, r, pte_index);
656 		hpte[0] = v & ~HPTE_V_VALID;
657 		do_tlbies(kvm, &rb, 1, global_invalidates(kvm, flags), true);
658 		/*
659 		 * If the host has this page as readonly but the guest
660 		 * wants to make it read/write, reduce the permissions.
661 		 * Checking the host permissions involves finding the
662 		 * memslot and then the Linux PTE for the page.
663 		 */
664 		if (hpte_is_writable(r) && kvm->arch.using_mmu_notifiers) {
665 			unsigned long psize, gfn, hva;
666 			struct kvm_memory_slot *memslot;
667 			pgd_t *pgdir = vcpu->arch.pgdir;
668 			pte_t pte;
669 
670 			psize = hpte_page_size(v, r);
671 			gfn = ((r & HPTE_R_RPN) & ~(psize - 1)) >> PAGE_SHIFT;
672 			memslot = __gfn_to_memslot(kvm_memslots(kvm), gfn);
673 			if (memslot) {
674 				hva = __gfn_to_hva_memslot(memslot, gfn);
675 				pte = lookup_linux_pte(pgdir, hva, 1, &psize);
676 				if (pte_present(pte) && !pte_write(pte))
677 					r = hpte_make_readonly(r);
678 			}
679 		}
680 	}
681 	hpte[1] = r;
682 	eieio();
683 	hpte[0] = v & ~HPTE_V_HVLOCK;
684 	asm volatile("ptesync" : : : "memory");
685 	return H_SUCCESS;
686 }
687 
688 long kvmppc_h_read(struct kvm_vcpu *vcpu, unsigned long flags,
689 		   unsigned long pte_index)
690 {
691 	struct kvm *kvm = vcpu->kvm;
692 	unsigned long *hpte, v, r;
693 	int i, n = 1;
694 	struct revmap_entry *rev = NULL;
695 
696 	if (pte_index >= kvm->arch.hpt_npte)
697 		return H_PARAMETER;
698 	if (flags & H_READ_4) {
699 		pte_index &= ~3;
700 		n = 4;
701 	}
702 	rev = real_vmalloc_addr(&kvm->arch.revmap[pte_index]);
703 	for (i = 0; i < n; ++i, ++pte_index) {
704 		hpte = (unsigned long *)(kvm->arch.hpt_virt + (pte_index << 4));
705 		v = hpte[0] & ~HPTE_V_HVLOCK;
706 		r = hpte[1];
707 		if (v & HPTE_V_ABSENT) {
708 			v &= ~HPTE_V_ABSENT;
709 			v |= HPTE_V_VALID;
710 		}
711 		if (v & HPTE_V_VALID) {
712 			r = rev[i].guest_rpte | (r & (HPTE_R_R | HPTE_R_C));
713 			r &= ~HPTE_GR_RESERVED;
714 		}
715 		vcpu->arch.gpr[4 + i * 2] = v;
716 		vcpu->arch.gpr[5 + i * 2] = r;
717 	}
718 	return H_SUCCESS;
719 }
720 
721 void kvmppc_invalidate_hpte(struct kvm *kvm, unsigned long *hptep,
722 			unsigned long pte_index)
723 {
724 	unsigned long rb;
725 
726 	hptep[0] &= ~HPTE_V_VALID;
727 	rb = compute_tlbie_rb(hptep[0], hptep[1], pte_index);
728 	do_tlbies(kvm, &rb, 1, 1, true);
729 }
730 EXPORT_SYMBOL_GPL(kvmppc_invalidate_hpte);
731 
732 void kvmppc_clear_ref_hpte(struct kvm *kvm, unsigned long *hptep,
733 			   unsigned long pte_index)
734 {
735 	unsigned long rb;
736 	unsigned char rbyte;
737 
738 	rb = compute_tlbie_rb(hptep[0], hptep[1], pte_index);
739 	rbyte = (hptep[1] & ~HPTE_R_R) >> 8;
740 	/* modify only the second-last byte, which contains the ref bit */
741 	*((char *)hptep + 14) = rbyte;
742 	do_tlbies(kvm, &rb, 1, 1, false);
743 }
744 EXPORT_SYMBOL_GPL(kvmppc_clear_ref_hpte);
745 
746 static int slb_base_page_shift[4] = {
747 	24,	/* 16M */
748 	16,	/* 64k */
749 	34,	/* 16G */
750 	20,	/* 1M, unsupported */
751 };
752 
753 /* When called from virtmode, this func should be protected by
754  * preempt_disable(), otherwise, the holding of HPTE_V_HVLOCK
755  * can trigger deadlock issue.
756  */
757 long kvmppc_hv_find_lock_hpte(struct kvm *kvm, gva_t eaddr, unsigned long slb_v,
758 			      unsigned long valid)
759 {
760 	unsigned int i;
761 	unsigned int pshift;
762 	unsigned long somask;
763 	unsigned long vsid, hash;
764 	unsigned long avpn;
765 	unsigned long *hpte;
766 	unsigned long mask, val;
767 	unsigned long v, r;
768 
769 	/* Get page shift, work out hash and AVPN etc. */
770 	mask = SLB_VSID_B | HPTE_V_AVPN | HPTE_V_SECONDARY;
771 	val = 0;
772 	pshift = 12;
773 	if (slb_v & SLB_VSID_L) {
774 		mask |= HPTE_V_LARGE;
775 		val |= HPTE_V_LARGE;
776 		pshift = slb_base_page_shift[(slb_v & SLB_VSID_LP) >> 4];
777 	}
778 	if (slb_v & SLB_VSID_B_1T) {
779 		somask = (1UL << 40) - 1;
780 		vsid = (slb_v & ~SLB_VSID_B) >> SLB_VSID_SHIFT_1T;
781 		vsid ^= vsid << 25;
782 	} else {
783 		somask = (1UL << 28) - 1;
784 		vsid = (slb_v & ~SLB_VSID_B) >> SLB_VSID_SHIFT;
785 	}
786 	hash = (vsid ^ ((eaddr & somask) >> pshift)) & kvm->arch.hpt_mask;
787 	avpn = slb_v & ~(somask >> 16);	/* also includes B */
788 	avpn |= (eaddr & somask) >> 16;
789 
790 	if (pshift >= 24)
791 		avpn &= ~((1UL << (pshift - 16)) - 1);
792 	else
793 		avpn &= ~0x7fUL;
794 	val |= avpn;
795 
796 	for (;;) {
797 		hpte = (unsigned long *)(kvm->arch.hpt_virt + (hash << 7));
798 
799 		for (i = 0; i < 16; i += 2) {
800 			/* Read the PTE racily */
801 			v = hpte[i] & ~HPTE_V_HVLOCK;
802 
803 			/* Check valid/absent, hash, segment size and AVPN */
804 			if (!(v & valid) || (v & mask) != val)
805 				continue;
806 
807 			/* Lock the PTE and read it under the lock */
808 			while (!try_lock_hpte(&hpte[i], HPTE_V_HVLOCK))
809 				cpu_relax();
810 			v = hpte[i] & ~HPTE_V_HVLOCK;
811 			r = hpte[i+1];
812 
813 			/*
814 			 * Check the HPTE again, including large page size
815 			 * Since we don't currently allow any MPSS (mixed
816 			 * page-size segment) page sizes, it is sufficient
817 			 * to check against the actual page size.
818 			 */
819 			if ((v & valid) && (v & mask) == val &&
820 			    hpte_page_size(v, r) == (1ul << pshift))
821 				/* Return with the HPTE still locked */
822 				return (hash << 3) + (i >> 1);
823 
824 			/* Unlock and move on */
825 			hpte[i] = v;
826 		}
827 
828 		if (val & HPTE_V_SECONDARY)
829 			break;
830 		val |= HPTE_V_SECONDARY;
831 		hash = hash ^ kvm->arch.hpt_mask;
832 	}
833 	return -1;
834 }
835 EXPORT_SYMBOL(kvmppc_hv_find_lock_hpte);
836 
837 /*
838  * Called in real mode to check whether an HPTE not found fault
839  * is due to accessing a paged-out page or an emulated MMIO page,
840  * or if a protection fault is due to accessing a page that the
841  * guest wanted read/write access to but which we made read-only.
842  * Returns a possibly modified status (DSISR) value if not
843  * (i.e. pass the interrupt to the guest),
844  * -1 to pass the fault up to host kernel mode code, -2 to do that
845  * and also load the instruction word (for MMIO emulation),
846  * or 0 if we should make the guest retry the access.
847  */
848 long kvmppc_hpte_hv_fault(struct kvm_vcpu *vcpu, unsigned long addr,
849 			  unsigned long slb_v, unsigned int status, bool data)
850 {
851 	struct kvm *kvm = vcpu->kvm;
852 	long int index;
853 	unsigned long v, r, gr;
854 	unsigned long *hpte;
855 	unsigned long valid;
856 	struct revmap_entry *rev;
857 	unsigned long pp, key;
858 
859 	/* For protection fault, expect to find a valid HPTE */
860 	valid = HPTE_V_VALID;
861 	if (status & DSISR_NOHPTE)
862 		valid |= HPTE_V_ABSENT;
863 
864 	index = kvmppc_hv_find_lock_hpte(kvm, addr, slb_v, valid);
865 	if (index < 0) {
866 		if (status & DSISR_NOHPTE)
867 			return status;	/* there really was no HPTE */
868 		return 0;		/* for prot fault, HPTE disappeared */
869 	}
870 	hpte = (unsigned long *)(kvm->arch.hpt_virt + (index << 4));
871 	v = hpte[0] & ~HPTE_V_HVLOCK;
872 	r = hpte[1];
873 	rev = real_vmalloc_addr(&kvm->arch.revmap[index]);
874 	gr = rev->guest_rpte;
875 
876 	unlock_hpte(hpte, v);
877 
878 	/* For not found, if the HPTE is valid by now, retry the instruction */
879 	if ((status & DSISR_NOHPTE) && (v & HPTE_V_VALID))
880 		return 0;
881 
882 	/* Check access permissions to the page */
883 	pp = gr & (HPTE_R_PP0 | HPTE_R_PP);
884 	key = (vcpu->arch.shregs.msr & MSR_PR) ? SLB_VSID_KP : SLB_VSID_KS;
885 	status &= ~DSISR_NOHPTE;	/* DSISR_NOHPTE == SRR1_ISI_NOPT */
886 	if (!data) {
887 		if (gr & (HPTE_R_N | HPTE_R_G))
888 			return status | SRR1_ISI_N_OR_G;
889 		if (!hpte_read_permission(pp, slb_v & key))
890 			return status | SRR1_ISI_PROT;
891 	} else if (status & DSISR_ISSTORE) {
892 		/* check write permission */
893 		if (!hpte_write_permission(pp, slb_v & key))
894 			return status | DSISR_PROTFAULT;
895 	} else {
896 		if (!hpte_read_permission(pp, slb_v & key))
897 			return status | DSISR_PROTFAULT;
898 	}
899 
900 	/* Check storage key, if applicable */
901 	if (data && (vcpu->arch.shregs.msr & MSR_DR)) {
902 		unsigned int perm = hpte_get_skey_perm(gr, vcpu->arch.amr);
903 		if (status & DSISR_ISSTORE)
904 			perm >>= 1;
905 		if (perm & 1)
906 			return status | DSISR_KEYFAULT;
907 	}
908 
909 	/* Save HPTE info for virtual-mode handler */
910 	vcpu->arch.pgfault_addr = addr;
911 	vcpu->arch.pgfault_index = index;
912 	vcpu->arch.pgfault_hpte[0] = v;
913 	vcpu->arch.pgfault_hpte[1] = r;
914 
915 	/* Check the storage key to see if it is possibly emulated MMIO */
916 	if (data && (vcpu->arch.shregs.msr & MSR_IR) &&
917 	    (r & (HPTE_R_KEY_HI | HPTE_R_KEY_LO)) ==
918 	    (HPTE_R_KEY_HI | HPTE_R_KEY_LO))
919 		return -2;	/* MMIO emulation - load instr word */
920 
921 	return -1;		/* send fault up to host kernel mode */
922 }
923