xref: /linux/arch/mips/kvm/tlb.c (revision 1c506c9c104cf01d01a9633ad2e76f15f938c54c)
1d7d5b05fSDeng-Cheng Zhu /*
2d7d5b05fSDeng-Cheng Zhu  * This file is subject to the terms and conditions of the GNU General Public
3d7d5b05fSDeng-Cheng Zhu  * License.  See the file "COPYING" in the main directory of this archive
4d7d5b05fSDeng-Cheng Zhu  * for more details.
5d7d5b05fSDeng-Cheng Zhu  *
6d7d5b05fSDeng-Cheng Zhu  * KVM/MIPS TLB handling, this file is part of the Linux host kernel so that
7d7d5b05fSDeng-Cheng Zhu  * TLB handlers run from KSEG0
8d7d5b05fSDeng-Cheng Zhu  *
9d7d5b05fSDeng-Cheng Zhu  * Copyright (C) 2012  MIPS Technologies, Inc.  All rights reserved.
10d7d5b05fSDeng-Cheng Zhu  * Authors: Sanjay Lal <sanjayl@kymasys.com>
11d7d5b05fSDeng-Cheng Zhu  */
12d7d5b05fSDeng-Cheng Zhu 
13d7d5b05fSDeng-Cheng Zhu #include <linux/sched.h>
14d7d5b05fSDeng-Cheng Zhu #include <linux/smp.h>
15d7d5b05fSDeng-Cheng Zhu #include <linux/mm.h>
16d7d5b05fSDeng-Cheng Zhu #include <linux/delay.h>
17403015b3SJames Hogan #include <linux/export.h>
18d7d5b05fSDeng-Cheng Zhu #include <linux/kvm_host.h>
19d7d5b05fSDeng-Cheng Zhu #include <linux/srcu.h>
20d7d5b05fSDeng-Cheng Zhu 
21d7d5b05fSDeng-Cheng Zhu #include <asm/cpu.h>
22d7d5b05fSDeng-Cheng Zhu #include <asm/bootinfo.h>
23d7d5b05fSDeng-Cheng Zhu #include <asm/mmu_context.h>
24d7d5b05fSDeng-Cheng Zhu #include <asm/pgtable.h>
25d7d5b05fSDeng-Cheng Zhu #include <asm/cacheflush.h>
26d7d5b05fSDeng-Cheng Zhu #include <asm/tlb.h>
27e922a4cbSJames Hogan #include <asm/tlbdebug.h>
28d7d5b05fSDeng-Cheng Zhu 
29d7d5b05fSDeng-Cheng Zhu #undef CONFIG_MIPS_MT
30d7d5b05fSDeng-Cheng Zhu #include <asm/r4kcache.h>
31d7d5b05fSDeng-Cheng Zhu #define CONFIG_MIPS_MT
32d7d5b05fSDeng-Cheng Zhu 
33d7d5b05fSDeng-Cheng Zhu #define KVM_GUEST_PC_TLB    0
34d7d5b05fSDeng-Cheng Zhu #define KVM_GUEST_SP_TLB    1
35d7d5b05fSDeng-Cheng Zhu 
36372582a6SJames Hogan #ifdef CONFIG_KVM_MIPS_VZ
37c992a4f6SJames Hogan unsigned long GUESTID_MASK;
38c992a4f6SJames Hogan EXPORT_SYMBOL_GPL(GUESTID_MASK);
39c992a4f6SJames Hogan unsigned long GUESTID_FIRST_VERSION;
40c992a4f6SJames Hogan EXPORT_SYMBOL_GPL(GUESTID_FIRST_VERSION);
41c992a4f6SJames Hogan unsigned long GUESTID_VERSION_MASK;
42c992a4f6SJames Hogan EXPORT_SYMBOL_GPL(GUESTID_VERSION_MASK);
43c992a4f6SJames Hogan 
44372582a6SJames Hogan static u32 kvm_mips_get_root_asid(struct kvm_vcpu *vcpu)
45372582a6SJames Hogan {
46372582a6SJames Hogan 	struct mm_struct *gpa_mm = &vcpu->kvm->arch.gpa_mm;
47372582a6SJames Hogan 
48372582a6SJames Hogan 	if (cpu_has_guestid)
49372582a6SJames Hogan 		return 0;
50372582a6SJames Hogan 	else
51372582a6SJames Hogan 		return cpu_asid(smp_processor_id(), gpa_mm);
52372582a6SJames Hogan }
53372582a6SJames Hogan #endif
54372582a6SJames Hogan 
55403015b3SJames Hogan static u32 kvm_mips_get_kernel_asid(struct kvm_vcpu *vcpu)
56d7d5b05fSDeng-Cheng Zhu {
57c550d539SJames Hogan 	struct mm_struct *kern_mm = &vcpu->arch.guest_kernel_mm;
584edf00a4SPaul Burton 	int cpu = smp_processor_id();
594edf00a4SPaul Burton 
60c550d539SJames Hogan 	return cpu_asid(cpu, kern_mm);
61d7d5b05fSDeng-Cheng Zhu }
62d7d5b05fSDeng-Cheng Zhu 
63403015b3SJames Hogan static u32 kvm_mips_get_user_asid(struct kvm_vcpu *vcpu)
64d7d5b05fSDeng-Cheng Zhu {
65c550d539SJames Hogan 	struct mm_struct *user_mm = &vcpu->arch.guest_user_mm;
664edf00a4SPaul Burton 	int cpu = smp_processor_id();
674edf00a4SPaul Burton 
68c550d539SJames Hogan 	return cpu_asid(cpu, user_mm);
69d7d5b05fSDeng-Cheng Zhu }
70d7d5b05fSDeng-Cheng Zhu 
71d7d5b05fSDeng-Cheng Zhu /* Structure defining an tlb entry data set. */
72d7d5b05fSDeng-Cheng Zhu 
73d7d5b05fSDeng-Cheng Zhu void kvm_mips_dump_host_tlbs(void)
74d7d5b05fSDeng-Cheng Zhu {
75d7d5b05fSDeng-Cheng Zhu 	unsigned long flags;
76d7d5b05fSDeng-Cheng Zhu 
77d7d5b05fSDeng-Cheng Zhu 	local_irq_save(flags);
78d7d5b05fSDeng-Cheng Zhu 
79d7d5b05fSDeng-Cheng Zhu 	kvm_info("HOST TLBs:\n");
80e922a4cbSJames Hogan 	dump_tlb_regs();
81e922a4cbSJames Hogan 	pr_info("\n");
82e922a4cbSJames Hogan 	dump_tlb_all();
83d7d5b05fSDeng-Cheng Zhu 
84d7d5b05fSDeng-Cheng Zhu 	local_irq_restore(flags);
85d7d5b05fSDeng-Cheng Zhu }
86cb1b447fSJames Hogan EXPORT_SYMBOL_GPL(kvm_mips_dump_host_tlbs);
87d7d5b05fSDeng-Cheng Zhu 
88d7d5b05fSDeng-Cheng Zhu void kvm_mips_dump_guest_tlbs(struct kvm_vcpu *vcpu)
89d7d5b05fSDeng-Cheng Zhu {
90d7d5b05fSDeng-Cheng Zhu 	struct mips_coproc *cop0 = vcpu->arch.cop0;
91d7d5b05fSDeng-Cheng Zhu 	struct kvm_mips_tlb tlb;
92d7d5b05fSDeng-Cheng Zhu 	int i;
93d7d5b05fSDeng-Cheng Zhu 
94d7d5b05fSDeng-Cheng Zhu 	kvm_info("Guest TLBs:\n");
95d7d5b05fSDeng-Cheng Zhu 	kvm_info("Guest EntryHi: %#lx\n", kvm_read_c0_guest_entryhi(cop0));
96d7d5b05fSDeng-Cheng Zhu 
97d7d5b05fSDeng-Cheng Zhu 	for (i = 0; i < KVM_MIPS_GUEST_TLB_SIZE; i++) {
98d7d5b05fSDeng-Cheng Zhu 		tlb = vcpu->arch.guest_tlb[i];
99d7d5b05fSDeng-Cheng Zhu 		kvm_info("TLB%c%3d Hi 0x%08lx ",
100e6207bbeSJames Hogan 			 (tlb.tlb_lo[0] | tlb.tlb_lo[1]) & ENTRYLO_V
1019fbfb06aSJames Hogan 							? ' ' : '*',
102d7d5b05fSDeng-Cheng Zhu 			 i, tlb.tlb_hi);
1038cffd197SJames Hogan 		kvm_info("Lo0=0x%09llx %c%c attr %lx ",
1049fbfb06aSJames Hogan 			 (u64) mips3_tlbpfn_to_paddr(tlb.tlb_lo[0]),
105e6207bbeSJames Hogan 			 (tlb.tlb_lo[0] & ENTRYLO_D) ? 'D' : ' ',
106e6207bbeSJames Hogan 			 (tlb.tlb_lo[0] & ENTRYLO_G) ? 'G' : ' ',
107e6207bbeSJames Hogan 			 (tlb.tlb_lo[0] & ENTRYLO_C) >> ENTRYLO_C_SHIFT);
1088cffd197SJames Hogan 		kvm_info("Lo1=0x%09llx %c%c attr %lx sz=%lx\n",
1099fbfb06aSJames Hogan 			 (u64) mips3_tlbpfn_to_paddr(tlb.tlb_lo[1]),
110e6207bbeSJames Hogan 			 (tlb.tlb_lo[1] & ENTRYLO_D) ? 'D' : ' ',
111e6207bbeSJames Hogan 			 (tlb.tlb_lo[1] & ENTRYLO_G) ? 'G' : ' ',
112e6207bbeSJames Hogan 			 (tlb.tlb_lo[1] & ENTRYLO_C) >> ENTRYLO_C_SHIFT,
113e6207bbeSJames Hogan 			 tlb.tlb_mask);
114d7d5b05fSDeng-Cheng Zhu 	}
115d7d5b05fSDeng-Cheng Zhu }
116cb1b447fSJames Hogan EXPORT_SYMBOL_GPL(kvm_mips_dump_guest_tlbs);
117d7d5b05fSDeng-Cheng Zhu 
118d7d5b05fSDeng-Cheng Zhu int kvm_mips_guest_tlb_lookup(struct kvm_vcpu *vcpu, unsigned long entryhi)
119d7d5b05fSDeng-Cheng Zhu {
120d7d5b05fSDeng-Cheng Zhu 	int i;
121d7d5b05fSDeng-Cheng Zhu 	int index = -1;
122d7d5b05fSDeng-Cheng Zhu 	struct kvm_mips_tlb *tlb = vcpu->arch.guest_tlb;
123d7d5b05fSDeng-Cheng Zhu 
124d7d5b05fSDeng-Cheng Zhu 	for (i = 0; i < KVM_MIPS_GUEST_TLB_SIZE; i++) {
125d7d5b05fSDeng-Cheng Zhu 		if (TLB_HI_VPN2_HIT(tlb[i], entryhi) &&
126d7d5b05fSDeng-Cheng Zhu 		    TLB_HI_ASID_HIT(tlb[i], entryhi)) {
127d7d5b05fSDeng-Cheng Zhu 			index = i;
128d7d5b05fSDeng-Cheng Zhu 			break;
129d7d5b05fSDeng-Cheng Zhu 		}
130d7d5b05fSDeng-Cheng Zhu 	}
131d7d5b05fSDeng-Cheng Zhu 
132d7d5b05fSDeng-Cheng Zhu 	kvm_debug("%s: entryhi: %#lx, index: %d lo0: %#lx, lo1: %#lx\n",
1339fbfb06aSJames Hogan 		  __func__, entryhi, index, tlb[i].tlb_lo[0], tlb[i].tlb_lo[1]);
134d7d5b05fSDeng-Cheng Zhu 
135d7d5b05fSDeng-Cheng Zhu 	return index;
136d7d5b05fSDeng-Cheng Zhu }
137cb1b447fSJames Hogan EXPORT_SYMBOL_GPL(kvm_mips_guest_tlb_lookup);
138d7d5b05fSDeng-Cheng Zhu 
13957e3869cSJames Hogan static int _kvm_mips_host_tlb_inv(unsigned long entryhi)
140d7d5b05fSDeng-Cheng Zhu {
141d7d5b05fSDeng-Cheng Zhu 	int idx;
142d7d5b05fSDeng-Cheng Zhu 
14357e3869cSJames Hogan 	write_c0_entryhi(entryhi);
144d7d5b05fSDeng-Cheng Zhu 	mtc0_tlbw_hazard();
145d7d5b05fSDeng-Cheng Zhu 
146d7d5b05fSDeng-Cheng Zhu 	tlb_probe();
147d7d5b05fSDeng-Cheng Zhu 	tlb_probe_hazard();
148d7d5b05fSDeng-Cheng Zhu 	idx = read_c0_index();
149d7d5b05fSDeng-Cheng Zhu 
150d7d5b05fSDeng-Cheng Zhu 	if (idx >= current_cpu_data.tlbsize)
151d7d5b05fSDeng-Cheng Zhu 		BUG();
152d7d5b05fSDeng-Cheng Zhu 
153f3a8603fSJames Hogan 	if (idx >= 0) {
154d7d5b05fSDeng-Cheng Zhu 		write_c0_entryhi(UNIQUE_ENTRYHI(idx));
155d7d5b05fSDeng-Cheng Zhu 		write_c0_entrylo0(0);
156d7d5b05fSDeng-Cheng Zhu 		write_c0_entrylo1(0);
157d7d5b05fSDeng-Cheng Zhu 		mtc0_tlbw_hazard();
158d7d5b05fSDeng-Cheng Zhu 
159d7d5b05fSDeng-Cheng Zhu 		tlb_write_indexed();
160138f7ad9SJames Hogan 		tlbw_use_hazard();
161d7d5b05fSDeng-Cheng Zhu 	}
162d7d5b05fSDeng-Cheng Zhu 
16357e3869cSJames Hogan 	return idx;
16457e3869cSJames Hogan }
16557e3869cSJames Hogan 
16657e3869cSJames Hogan int kvm_mips_host_tlb_inv(struct kvm_vcpu *vcpu, unsigned long va,
16757e3869cSJames Hogan 			  bool user, bool kernel)
16857e3869cSJames Hogan {
16957e3869cSJames Hogan 	int idx_user, idx_kernel;
17057e3869cSJames Hogan 	unsigned long flags, old_entryhi;
17157e3869cSJames Hogan 
17257e3869cSJames Hogan 	local_irq_save(flags);
17357e3869cSJames Hogan 
17457e3869cSJames Hogan 	old_entryhi = read_c0_entryhi();
17557e3869cSJames Hogan 
17657e3869cSJames Hogan 	if (user)
17757e3869cSJames Hogan 		idx_user = _kvm_mips_host_tlb_inv((va & VPN2_MASK) |
17857e3869cSJames Hogan 						  kvm_mips_get_user_asid(vcpu));
17957e3869cSJames Hogan 	if (kernel)
18057e3869cSJames Hogan 		idx_kernel = _kvm_mips_host_tlb_inv((va & VPN2_MASK) |
18157e3869cSJames Hogan 						kvm_mips_get_kernel_asid(vcpu));
18257e3869cSJames Hogan 
183d7d5b05fSDeng-Cheng Zhu 	write_c0_entryhi(old_entryhi);
184d7d5b05fSDeng-Cheng Zhu 	mtc0_tlbw_hazard();
185d7d5b05fSDeng-Cheng Zhu 
186d7d5b05fSDeng-Cheng Zhu 	local_irq_restore(flags);
187d7d5b05fSDeng-Cheng Zhu 
188*1c506c9cSJames Hogan 	/*
189*1c506c9cSJames Hogan 	 * We don't want to get reserved instruction exceptions for missing tlb
190*1c506c9cSJames Hogan 	 * entries.
191*1c506c9cSJames Hogan 	 */
192*1c506c9cSJames Hogan 	if (cpu_has_vtag_icache)
193*1c506c9cSJames Hogan 		flush_icache_all();
194*1c506c9cSJames Hogan 
19557e3869cSJames Hogan 	if (user && idx_user >= 0)
19657e3869cSJames Hogan 		kvm_debug("%s: Invalidated guest user entryhi %#lx @ idx %d\n",
19757e3869cSJames Hogan 			  __func__, (va & VPN2_MASK) |
19857e3869cSJames Hogan 				    kvm_mips_get_user_asid(vcpu), idx_user);
19957e3869cSJames Hogan 	if (kernel && idx_kernel >= 0)
20057e3869cSJames Hogan 		kvm_debug("%s: Invalidated guest kernel entryhi %#lx @ idx %d\n",
20157e3869cSJames Hogan 			  __func__, (va & VPN2_MASK) |
20257e3869cSJames Hogan 				    kvm_mips_get_kernel_asid(vcpu), idx_kernel);
203d7d5b05fSDeng-Cheng Zhu 
204d7d5b05fSDeng-Cheng Zhu 	return 0;
205d7d5b05fSDeng-Cheng Zhu }
206cb1b447fSJames Hogan EXPORT_SYMBOL_GPL(kvm_mips_host_tlb_inv);
207d7d5b05fSDeng-Cheng Zhu 
208372582a6SJames Hogan #ifdef CONFIG_KVM_MIPS_VZ
209372582a6SJames Hogan 
210372582a6SJames Hogan /* GuestID management */
211372582a6SJames Hogan 
212372582a6SJames Hogan /**
213372582a6SJames Hogan  * clear_root_gid() - Set GuestCtl1.RID for normal root operation.
214372582a6SJames Hogan  */
215372582a6SJames Hogan static inline void clear_root_gid(void)
216372582a6SJames Hogan {
217372582a6SJames Hogan 	if (cpu_has_guestid) {
218372582a6SJames Hogan 		clear_c0_guestctl1(MIPS_GCTL1_RID);
219372582a6SJames Hogan 		mtc0_tlbw_hazard();
220372582a6SJames Hogan 	}
221372582a6SJames Hogan }
222372582a6SJames Hogan 
223372582a6SJames Hogan /**
224372582a6SJames Hogan  * set_root_gid_to_guest_gid() - Set GuestCtl1.RID to match GuestCtl1.ID.
225372582a6SJames Hogan  *
226372582a6SJames Hogan  * Sets the root GuestID to match the current guest GuestID, for TLB operation
227372582a6SJames Hogan  * on the GPA->RPA mappings in the root TLB.
228372582a6SJames Hogan  *
229372582a6SJames Hogan  * The caller must be sure to disable HTW while the root GID is set, and
230372582a6SJames Hogan  * possibly longer if TLB registers are modified.
231372582a6SJames Hogan  */
232372582a6SJames Hogan static inline void set_root_gid_to_guest_gid(void)
233372582a6SJames Hogan {
234372582a6SJames Hogan 	unsigned int guestctl1;
235372582a6SJames Hogan 
236372582a6SJames Hogan 	if (cpu_has_guestid) {
237372582a6SJames Hogan 		back_to_back_c0_hazard();
238372582a6SJames Hogan 		guestctl1 = read_c0_guestctl1();
239372582a6SJames Hogan 		guestctl1 = (guestctl1 & ~MIPS_GCTL1_RID) |
240372582a6SJames Hogan 			((guestctl1 & MIPS_GCTL1_ID) >> MIPS_GCTL1_ID_SHIFT)
241372582a6SJames Hogan 						     << MIPS_GCTL1_RID_SHIFT;
242372582a6SJames Hogan 		write_c0_guestctl1(guestctl1);
243372582a6SJames Hogan 		mtc0_tlbw_hazard();
244372582a6SJames Hogan 	}
245372582a6SJames Hogan }
246372582a6SJames Hogan 
247372582a6SJames Hogan int kvm_vz_host_tlb_inv(struct kvm_vcpu *vcpu, unsigned long va)
248372582a6SJames Hogan {
249372582a6SJames Hogan 	int idx;
250372582a6SJames Hogan 	unsigned long flags, old_entryhi;
251372582a6SJames Hogan 
252372582a6SJames Hogan 	local_irq_save(flags);
253372582a6SJames Hogan 	htw_stop();
254372582a6SJames Hogan 
255372582a6SJames Hogan 	/* Set root GuestID for root probe and write of guest TLB entry */
256372582a6SJames Hogan 	set_root_gid_to_guest_gid();
257372582a6SJames Hogan 
258372582a6SJames Hogan 	old_entryhi = read_c0_entryhi();
259372582a6SJames Hogan 
260372582a6SJames Hogan 	idx = _kvm_mips_host_tlb_inv((va & VPN2_MASK) |
261372582a6SJames Hogan 				     kvm_mips_get_root_asid(vcpu));
262372582a6SJames Hogan 
263372582a6SJames Hogan 	write_c0_entryhi(old_entryhi);
264372582a6SJames Hogan 	clear_root_gid();
265372582a6SJames Hogan 	mtc0_tlbw_hazard();
266372582a6SJames Hogan 
267372582a6SJames Hogan 	htw_start();
268372582a6SJames Hogan 	local_irq_restore(flags);
269372582a6SJames Hogan 
270*1c506c9cSJames Hogan 	/*
271*1c506c9cSJames Hogan 	 * We don't want to get reserved instruction exceptions for missing tlb
272*1c506c9cSJames Hogan 	 * entries.
273*1c506c9cSJames Hogan 	 */
274*1c506c9cSJames Hogan 	if (cpu_has_vtag_icache)
275*1c506c9cSJames Hogan 		flush_icache_all();
276*1c506c9cSJames Hogan 
277372582a6SJames Hogan 	if (idx > 0)
278372582a6SJames Hogan 		kvm_debug("%s: Invalidated root entryhi %#lx @ idx %d\n",
279372582a6SJames Hogan 			  __func__, (va & VPN2_MASK) |
280372582a6SJames Hogan 				    kvm_mips_get_root_asid(vcpu), idx);
281372582a6SJames Hogan 
282372582a6SJames Hogan 	return 0;
283372582a6SJames Hogan }
284372582a6SJames Hogan EXPORT_SYMBOL_GPL(kvm_vz_host_tlb_inv);
285372582a6SJames Hogan 
286372582a6SJames Hogan /**
287372582a6SJames Hogan  * kvm_vz_guest_tlb_lookup() - Lookup a guest VZ TLB mapping.
288372582a6SJames Hogan  * @vcpu:	KVM VCPU pointer.
289372582a6SJames Hogan  * @gpa:	Guest virtual address in a TLB mapped guest segment.
290372582a6SJames Hogan  * @gpa:	Ponter to output guest physical address it maps to.
291372582a6SJames Hogan  *
292372582a6SJames Hogan  * Converts a guest virtual address in a guest TLB mapped segment to a guest
293372582a6SJames Hogan  * physical address, by probing the guest TLB.
294372582a6SJames Hogan  *
295372582a6SJames Hogan  * Returns:	0 if guest TLB mapping exists for @gva. *@gpa will have been
296372582a6SJames Hogan  *		written.
297372582a6SJames Hogan  *		-EFAULT if no guest TLB mapping exists for @gva. *@gpa may not
298372582a6SJames Hogan  *		have been written.
299372582a6SJames Hogan  */
300372582a6SJames Hogan int kvm_vz_guest_tlb_lookup(struct kvm_vcpu *vcpu, unsigned long gva,
301372582a6SJames Hogan 			    unsigned long *gpa)
302372582a6SJames Hogan {
303372582a6SJames Hogan 	unsigned long o_entryhi, o_entrylo[2], o_pagemask;
304372582a6SJames Hogan 	unsigned int o_index;
305372582a6SJames Hogan 	unsigned long entrylo[2], pagemask, pagemaskbit, pa;
306372582a6SJames Hogan 	unsigned long flags;
307372582a6SJames Hogan 	int index;
308372582a6SJames Hogan 
309372582a6SJames Hogan 	/* Probe the guest TLB for a mapping */
310372582a6SJames Hogan 	local_irq_save(flags);
311372582a6SJames Hogan 	/* Set root GuestID for root probe of guest TLB entry */
312372582a6SJames Hogan 	htw_stop();
313372582a6SJames Hogan 	set_root_gid_to_guest_gid();
314372582a6SJames Hogan 
315372582a6SJames Hogan 	o_entryhi = read_gc0_entryhi();
316372582a6SJames Hogan 	o_index = read_gc0_index();
317372582a6SJames Hogan 
318372582a6SJames Hogan 	write_gc0_entryhi((o_entryhi & 0x3ff) | (gva & ~0xfffl));
319372582a6SJames Hogan 	mtc0_tlbw_hazard();
320372582a6SJames Hogan 	guest_tlb_probe();
321372582a6SJames Hogan 	tlb_probe_hazard();
322372582a6SJames Hogan 
323372582a6SJames Hogan 	index = read_gc0_index();
324372582a6SJames Hogan 	if (index < 0) {
325372582a6SJames Hogan 		/* No match, fail */
326372582a6SJames Hogan 		write_gc0_entryhi(o_entryhi);
327372582a6SJames Hogan 		write_gc0_index(o_index);
328372582a6SJames Hogan 
329372582a6SJames Hogan 		clear_root_gid();
330372582a6SJames Hogan 		htw_start();
331372582a6SJames Hogan 		local_irq_restore(flags);
332372582a6SJames Hogan 		return -EFAULT;
333372582a6SJames Hogan 	}
334372582a6SJames Hogan 
335372582a6SJames Hogan 	/* Match! read the TLB entry */
336372582a6SJames Hogan 	o_entrylo[0] = read_gc0_entrylo0();
337372582a6SJames Hogan 	o_entrylo[1] = read_gc0_entrylo1();
338372582a6SJames Hogan 	o_pagemask = read_gc0_pagemask();
339372582a6SJames Hogan 
340372582a6SJames Hogan 	mtc0_tlbr_hazard();
341372582a6SJames Hogan 	guest_tlb_read();
342372582a6SJames Hogan 	tlb_read_hazard();
343372582a6SJames Hogan 
344372582a6SJames Hogan 	entrylo[0] = read_gc0_entrylo0();
345372582a6SJames Hogan 	entrylo[1] = read_gc0_entrylo1();
346372582a6SJames Hogan 	pagemask = ~read_gc0_pagemask() & ~0x1fffl;
347372582a6SJames Hogan 
348372582a6SJames Hogan 	write_gc0_entryhi(o_entryhi);
349372582a6SJames Hogan 	write_gc0_index(o_index);
350372582a6SJames Hogan 	write_gc0_entrylo0(o_entrylo[0]);
351372582a6SJames Hogan 	write_gc0_entrylo1(o_entrylo[1]);
352372582a6SJames Hogan 	write_gc0_pagemask(o_pagemask);
353372582a6SJames Hogan 
354372582a6SJames Hogan 	clear_root_gid();
355372582a6SJames Hogan 	htw_start();
356372582a6SJames Hogan 	local_irq_restore(flags);
357372582a6SJames Hogan 
358372582a6SJames Hogan 	/* Select one of the EntryLo values and interpret the GPA */
359372582a6SJames Hogan 	pagemaskbit = (pagemask ^ (pagemask & (pagemask - 1))) >> 1;
360372582a6SJames Hogan 	pa = entrylo[!!(gva & pagemaskbit)];
361372582a6SJames Hogan 
362372582a6SJames Hogan 	/*
363372582a6SJames Hogan 	 * TLB entry may have become invalid since TLB probe if physical FTLB
364372582a6SJames Hogan 	 * entries are shared between threads (e.g. I6400).
365372582a6SJames Hogan 	 */
366372582a6SJames Hogan 	if (!(pa & ENTRYLO_V))
367372582a6SJames Hogan 		return -EFAULT;
368372582a6SJames Hogan 
369372582a6SJames Hogan 	/*
370372582a6SJames Hogan 	 * Note, this doesn't take guest MIPS32 XPA into account, where PFN is
371372582a6SJames Hogan 	 * split with XI/RI in the middle.
372372582a6SJames Hogan 	 */
373372582a6SJames Hogan 	pa = (pa << 6) & ~0xfffl;
374372582a6SJames Hogan 	pa |= gva & ~(pagemask | pagemaskbit);
375372582a6SJames Hogan 
376372582a6SJames Hogan 	*gpa = pa;
377372582a6SJames Hogan 	return 0;
378372582a6SJames Hogan }
379372582a6SJames Hogan EXPORT_SYMBOL_GPL(kvm_vz_guest_tlb_lookup);
380372582a6SJames Hogan 
381372582a6SJames Hogan /**
382372582a6SJames Hogan  * kvm_vz_local_flush_roottlb_all_guests() - Flush all root TLB entries for
383372582a6SJames Hogan  * guests.
384372582a6SJames Hogan  *
385372582a6SJames Hogan  * Invalidate all entries in root tlb which are GPA mappings.
386372582a6SJames Hogan  */
387372582a6SJames Hogan void kvm_vz_local_flush_roottlb_all_guests(void)
388372582a6SJames Hogan {
389372582a6SJames Hogan 	unsigned long flags;
390372582a6SJames Hogan 	unsigned long old_entryhi, old_pagemask, old_guestctl1;
391372582a6SJames Hogan 	int entry;
392372582a6SJames Hogan 
393372582a6SJames Hogan 	if (WARN_ON(!cpu_has_guestid))
394372582a6SJames Hogan 		return;
395372582a6SJames Hogan 
396372582a6SJames Hogan 	local_irq_save(flags);
397372582a6SJames Hogan 	htw_stop();
398372582a6SJames Hogan 
399372582a6SJames Hogan 	/* TLBR may clobber EntryHi.ASID, PageMask, and GuestCtl1.RID */
400372582a6SJames Hogan 	old_entryhi = read_c0_entryhi();
401372582a6SJames Hogan 	old_pagemask = read_c0_pagemask();
402372582a6SJames Hogan 	old_guestctl1 = read_c0_guestctl1();
403372582a6SJames Hogan 
404372582a6SJames Hogan 	/*
405372582a6SJames Hogan 	 * Invalidate guest entries in root TLB while leaving root entries
406372582a6SJames Hogan 	 * intact when possible.
407372582a6SJames Hogan 	 */
408372582a6SJames Hogan 	for (entry = 0; entry < current_cpu_data.tlbsize; entry++) {
409372582a6SJames Hogan 		write_c0_index(entry);
410372582a6SJames Hogan 		mtc0_tlbw_hazard();
411372582a6SJames Hogan 		tlb_read();
412372582a6SJames Hogan 		tlb_read_hazard();
413372582a6SJames Hogan 
414372582a6SJames Hogan 		/* Don't invalidate non-guest (RVA) mappings in the root TLB */
415372582a6SJames Hogan 		if (!(read_c0_guestctl1() & MIPS_GCTL1_RID))
416372582a6SJames Hogan 			continue;
417372582a6SJames Hogan 
418372582a6SJames Hogan 		/* Make sure all entries differ. */
419372582a6SJames Hogan 		write_c0_entryhi(UNIQUE_ENTRYHI(entry));
420372582a6SJames Hogan 		write_c0_entrylo0(0);
421372582a6SJames Hogan 		write_c0_entrylo1(0);
422372582a6SJames Hogan 		write_c0_guestctl1(0);
423372582a6SJames Hogan 		mtc0_tlbw_hazard();
424372582a6SJames Hogan 		tlb_write_indexed();
425372582a6SJames Hogan 	}
426372582a6SJames Hogan 
427372582a6SJames Hogan 	write_c0_entryhi(old_entryhi);
428372582a6SJames Hogan 	write_c0_pagemask(old_pagemask);
429372582a6SJames Hogan 	write_c0_guestctl1(old_guestctl1);
430372582a6SJames Hogan 	tlbw_use_hazard();
431372582a6SJames Hogan 
432372582a6SJames Hogan 	htw_start();
433372582a6SJames Hogan 	local_irq_restore(flags);
434372582a6SJames Hogan }
435372582a6SJames Hogan EXPORT_SYMBOL_GPL(kvm_vz_local_flush_roottlb_all_guests);
436372582a6SJames Hogan 
437372582a6SJames Hogan /**
438372582a6SJames Hogan  * kvm_vz_local_flush_guesttlb_all() - Flush all guest TLB entries.
439372582a6SJames Hogan  *
440372582a6SJames Hogan  * Invalidate all entries in guest tlb irrespective of guestid.
441372582a6SJames Hogan  */
442372582a6SJames Hogan void kvm_vz_local_flush_guesttlb_all(void)
443372582a6SJames Hogan {
444372582a6SJames Hogan 	unsigned long flags;
445372582a6SJames Hogan 	unsigned long old_index;
446372582a6SJames Hogan 	unsigned long old_entryhi;
447372582a6SJames Hogan 	unsigned long old_entrylo[2];
448372582a6SJames Hogan 	unsigned long old_pagemask;
449372582a6SJames Hogan 	int entry;
450372582a6SJames Hogan 
451372582a6SJames Hogan 	local_irq_save(flags);
452372582a6SJames Hogan 
453372582a6SJames Hogan 	/* Preserve all clobbered guest registers */
454372582a6SJames Hogan 	old_index = read_gc0_index();
455372582a6SJames Hogan 	old_entryhi = read_gc0_entryhi();
456372582a6SJames Hogan 	old_entrylo[0] = read_gc0_entrylo0();
457372582a6SJames Hogan 	old_entrylo[1] = read_gc0_entrylo1();
458372582a6SJames Hogan 	old_pagemask = read_gc0_pagemask();
459372582a6SJames Hogan 
460372582a6SJames Hogan 	/* Invalidate guest entries in guest TLB */
461372582a6SJames Hogan 	write_gc0_entrylo0(0);
462372582a6SJames Hogan 	write_gc0_entrylo1(0);
463372582a6SJames Hogan 	write_gc0_pagemask(0);
464372582a6SJames Hogan 	for (entry = 0; entry < current_cpu_data.guest.tlbsize; entry++) {
465372582a6SJames Hogan 		/* Make sure all entries differ. */
466372582a6SJames Hogan 		write_gc0_index(entry);
467372582a6SJames Hogan 		write_gc0_entryhi(UNIQUE_GUEST_ENTRYHI(entry));
468372582a6SJames Hogan 		mtc0_tlbw_hazard();
469372582a6SJames Hogan 		guest_tlb_write_indexed();
470372582a6SJames Hogan 	}
471372582a6SJames Hogan 	write_gc0_index(old_index);
472372582a6SJames Hogan 	write_gc0_entryhi(old_entryhi);
473372582a6SJames Hogan 	write_gc0_entrylo0(old_entrylo[0]);
474372582a6SJames Hogan 	write_gc0_entrylo1(old_entrylo[1]);
475372582a6SJames Hogan 	write_gc0_pagemask(old_pagemask);
476372582a6SJames Hogan 	tlbw_use_hazard();
477372582a6SJames Hogan 
478372582a6SJames Hogan 	local_irq_restore(flags);
479372582a6SJames Hogan }
480372582a6SJames Hogan EXPORT_SYMBOL_GPL(kvm_vz_local_flush_guesttlb_all);
481372582a6SJames Hogan 
482372582a6SJames Hogan /**
483372582a6SJames Hogan  * kvm_vz_save_guesttlb() - Save a range of guest TLB entries.
484372582a6SJames Hogan  * @buf:	Buffer to write TLB entries into.
485372582a6SJames Hogan  * @index:	Start index.
486372582a6SJames Hogan  * @count:	Number of entries to save.
487372582a6SJames Hogan  *
488372582a6SJames Hogan  * Save a range of guest TLB entries. The caller must ensure interrupts are
489372582a6SJames Hogan  * disabled.
490372582a6SJames Hogan  */
491372582a6SJames Hogan void kvm_vz_save_guesttlb(struct kvm_mips_tlb *buf, unsigned int index,
492372582a6SJames Hogan 			  unsigned int count)
493372582a6SJames Hogan {
494372582a6SJames Hogan 	unsigned int end = index + count;
495372582a6SJames Hogan 	unsigned long old_entryhi, old_entrylo0, old_entrylo1, old_pagemask;
496372582a6SJames Hogan 	unsigned int guestctl1 = 0;
497372582a6SJames Hogan 	int old_index, i;
498372582a6SJames Hogan 
499372582a6SJames Hogan 	/* Save registers we're about to clobber */
500372582a6SJames Hogan 	old_index = read_gc0_index();
501372582a6SJames Hogan 	old_entryhi = read_gc0_entryhi();
502372582a6SJames Hogan 	old_entrylo0 = read_gc0_entrylo0();
503372582a6SJames Hogan 	old_entrylo1 = read_gc0_entrylo1();
504372582a6SJames Hogan 	old_pagemask = read_gc0_pagemask();
505372582a6SJames Hogan 
506372582a6SJames Hogan 	/* Set root GuestID for root probe */
507372582a6SJames Hogan 	htw_stop();
508372582a6SJames Hogan 	set_root_gid_to_guest_gid();
509372582a6SJames Hogan 	if (cpu_has_guestid)
510372582a6SJames Hogan 		guestctl1 = read_c0_guestctl1();
511372582a6SJames Hogan 
512372582a6SJames Hogan 	/* Read each entry from guest TLB */
513372582a6SJames Hogan 	for (i = index; i < end; ++i, ++buf) {
514372582a6SJames Hogan 		write_gc0_index(i);
515372582a6SJames Hogan 
516372582a6SJames Hogan 		mtc0_tlbr_hazard();
517372582a6SJames Hogan 		guest_tlb_read();
518372582a6SJames Hogan 		tlb_read_hazard();
519372582a6SJames Hogan 
520372582a6SJames Hogan 		if (cpu_has_guestid &&
521372582a6SJames Hogan 		    (read_c0_guestctl1() ^ guestctl1) & MIPS_GCTL1_RID) {
522372582a6SJames Hogan 			/* Entry invalid or belongs to another guest */
523372582a6SJames Hogan 			buf->tlb_hi = UNIQUE_GUEST_ENTRYHI(i);
524372582a6SJames Hogan 			buf->tlb_lo[0] = 0;
525372582a6SJames Hogan 			buf->tlb_lo[1] = 0;
526372582a6SJames Hogan 			buf->tlb_mask = 0;
527372582a6SJames Hogan 		} else {
528372582a6SJames Hogan 			/* Entry belongs to the right guest */
529372582a6SJames Hogan 			buf->tlb_hi = read_gc0_entryhi();
530372582a6SJames Hogan 			buf->tlb_lo[0] = read_gc0_entrylo0();
531372582a6SJames Hogan 			buf->tlb_lo[1] = read_gc0_entrylo1();
532372582a6SJames Hogan 			buf->tlb_mask = read_gc0_pagemask();
533372582a6SJames Hogan 		}
534372582a6SJames Hogan 	}
535372582a6SJames Hogan 
536372582a6SJames Hogan 	/* Clear root GuestID again */
537372582a6SJames Hogan 	clear_root_gid();
538372582a6SJames Hogan 	htw_start();
539372582a6SJames Hogan 
540372582a6SJames Hogan 	/* Restore clobbered registers */
541372582a6SJames Hogan 	write_gc0_index(old_index);
542372582a6SJames Hogan 	write_gc0_entryhi(old_entryhi);
543372582a6SJames Hogan 	write_gc0_entrylo0(old_entrylo0);
544372582a6SJames Hogan 	write_gc0_entrylo1(old_entrylo1);
545372582a6SJames Hogan 	write_gc0_pagemask(old_pagemask);
546372582a6SJames Hogan 
547372582a6SJames Hogan 	tlbw_use_hazard();
548372582a6SJames Hogan }
549372582a6SJames Hogan EXPORT_SYMBOL_GPL(kvm_vz_save_guesttlb);
550372582a6SJames Hogan 
551372582a6SJames Hogan /**
552372582a6SJames Hogan  * kvm_vz_load_guesttlb() - Save a range of guest TLB entries.
553372582a6SJames Hogan  * @buf:	Buffer to read TLB entries from.
554372582a6SJames Hogan  * @index:	Start index.
555372582a6SJames Hogan  * @count:	Number of entries to load.
556372582a6SJames Hogan  *
557372582a6SJames Hogan  * Load a range of guest TLB entries. The caller must ensure interrupts are
558372582a6SJames Hogan  * disabled.
559372582a6SJames Hogan  */
560372582a6SJames Hogan void kvm_vz_load_guesttlb(const struct kvm_mips_tlb *buf, unsigned int index,
561372582a6SJames Hogan 			  unsigned int count)
562372582a6SJames Hogan {
563372582a6SJames Hogan 	unsigned int end = index + count;
564372582a6SJames Hogan 	unsigned long old_entryhi, old_entrylo0, old_entrylo1, old_pagemask;
565372582a6SJames Hogan 	int old_index, i;
566372582a6SJames Hogan 
567372582a6SJames Hogan 	/* Save registers we're about to clobber */
568372582a6SJames Hogan 	old_index = read_gc0_index();
569372582a6SJames Hogan 	old_entryhi = read_gc0_entryhi();
570372582a6SJames Hogan 	old_entrylo0 = read_gc0_entrylo0();
571372582a6SJames Hogan 	old_entrylo1 = read_gc0_entrylo1();
572372582a6SJames Hogan 	old_pagemask = read_gc0_pagemask();
573372582a6SJames Hogan 
574372582a6SJames Hogan 	/* Set root GuestID for root probe */
575372582a6SJames Hogan 	htw_stop();
576372582a6SJames Hogan 	set_root_gid_to_guest_gid();
577372582a6SJames Hogan 
578372582a6SJames Hogan 	/* Write each entry to guest TLB */
579372582a6SJames Hogan 	for (i = index; i < end; ++i, ++buf) {
580372582a6SJames Hogan 		write_gc0_index(i);
581372582a6SJames Hogan 		write_gc0_entryhi(buf->tlb_hi);
582372582a6SJames Hogan 		write_gc0_entrylo0(buf->tlb_lo[0]);
583372582a6SJames Hogan 		write_gc0_entrylo1(buf->tlb_lo[1]);
584372582a6SJames Hogan 		write_gc0_pagemask(buf->tlb_mask);
585372582a6SJames Hogan 
586372582a6SJames Hogan 		mtc0_tlbw_hazard();
587372582a6SJames Hogan 		guest_tlb_write_indexed();
588372582a6SJames Hogan 	}
589372582a6SJames Hogan 
590372582a6SJames Hogan 	/* Clear root GuestID again */
591372582a6SJames Hogan 	clear_root_gid();
592372582a6SJames Hogan 	htw_start();
593372582a6SJames Hogan 
594372582a6SJames Hogan 	/* Restore clobbered registers */
595372582a6SJames Hogan 	write_gc0_index(old_index);
596372582a6SJames Hogan 	write_gc0_entryhi(old_entryhi);
597372582a6SJames Hogan 	write_gc0_entrylo0(old_entrylo0);
598372582a6SJames Hogan 	write_gc0_entrylo1(old_entrylo1);
599372582a6SJames Hogan 	write_gc0_pagemask(old_pagemask);
600372582a6SJames Hogan 
601372582a6SJames Hogan 	tlbw_use_hazard();
602372582a6SJames Hogan }
603372582a6SJames Hogan EXPORT_SYMBOL_GPL(kvm_vz_load_guesttlb);
604372582a6SJames Hogan 
605372582a6SJames Hogan #endif
606372582a6SJames Hogan 
607a7ebb2e4SJames Hogan /**
608a7ebb2e4SJames Hogan  * kvm_mips_suspend_mm() - Suspend the active mm.
609a7ebb2e4SJames Hogan  * @cpu		The CPU we're running on.
610a7ebb2e4SJames Hogan  *
611a7ebb2e4SJames Hogan  * Suspend the active_mm, ready for a switch to a KVM guest virtual address
612a7ebb2e4SJames Hogan  * space. This is left active for the duration of guest context, including time
613a7ebb2e4SJames Hogan  * with interrupts enabled, so we need to be careful not to confuse e.g. cache
614a7ebb2e4SJames Hogan  * management IPIs.
615a7ebb2e4SJames Hogan  *
616a7ebb2e4SJames Hogan  * kvm_mips_resume_mm() should be called before context switching to a different
617a7ebb2e4SJames Hogan  * process so we don't need to worry about reference counting.
618a7ebb2e4SJames Hogan  *
619a7ebb2e4SJames Hogan  * This needs to be in static kernel code to avoid exporting init_mm.
620a7ebb2e4SJames Hogan  */
621a7ebb2e4SJames Hogan void kvm_mips_suspend_mm(int cpu)
622a7ebb2e4SJames Hogan {
623a7ebb2e4SJames Hogan 	cpumask_clear_cpu(cpu, mm_cpumask(current->active_mm));
624a7ebb2e4SJames Hogan 	current->active_mm = &init_mm;
625a7ebb2e4SJames Hogan }
626a7ebb2e4SJames Hogan EXPORT_SYMBOL_GPL(kvm_mips_suspend_mm);
627a7ebb2e4SJames Hogan 
628a7ebb2e4SJames Hogan /**
629a7ebb2e4SJames Hogan  * kvm_mips_resume_mm() - Resume the current process mm.
630a7ebb2e4SJames Hogan  * @cpu		The CPU we're running on.
631a7ebb2e4SJames Hogan  *
632a7ebb2e4SJames Hogan  * Resume the mm of the current process, after a switch back from a KVM guest
633a7ebb2e4SJames Hogan  * virtual address space (see kvm_mips_suspend_mm()).
634a7ebb2e4SJames Hogan  */
635a7ebb2e4SJames Hogan void kvm_mips_resume_mm(int cpu)
636a7ebb2e4SJames Hogan {
637a7ebb2e4SJames Hogan 	cpumask_set_cpu(cpu, mm_cpumask(current->mm));
638a7ebb2e4SJames Hogan 	current->active_mm = current->mm;
639a7ebb2e4SJames Hogan }
640a7ebb2e4SJames Hogan EXPORT_SYMBOL_GPL(kvm_mips_resume_mm);
641