1d2912cb1SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only
2b71c9e2fSAlexander Graf /*
3b71c9e2fSAlexander Graf * Copyright (C) 2008-2013 Freescale Semiconductor, Inc. All rights reserved.
4b71c9e2fSAlexander Graf *
5b71c9e2fSAlexander Graf * Author: Yu Liu, yu.liu@freescale.com
6b71c9e2fSAlexander Graf * Scott Wood, scottwood@freescale.com
7b71c9e2fSAlexander Graf * Ashish Kalra, ashish.kalra@freescale.com
8b71c9e2fSAlexander Graf * Varun Sethi, varun.sethi@freescale.com
9b71c9e2fSAlexander Graf * Alexander Graf, agraf@suse.de
10b71c9e2fSAlexander Graf *
11b71c9e2fSAlexander Graf * Description:
12b71c9e2fSAlexander Graf * This file is based on arch/powerpc/kvm/44x_tlb.c,
13b71c9e2fSAlexander Graf * by Hollis Blanchard <hollisb@us.ibm.com>.
14b71c9e2fSAlexander Graf */
15b71c9e2fSAlexander Graf
16b71c9e2fSAlexander Graf #include <linux/kernel.h>
17b71c9e2fSAlexander Graf #include <linux/types.h>
18b71c9e2fSAlexander Graf #include <linux/slab.h>
19b71c9e2fSAlexander Graf #include <linux/string.h>
20b71c9e2fSAlexander Graf #include <linux/kvm.h>
21b71c9e2fSAlexander Graf #include <linux/kvm_host.h>
22b71c9e2fSAlexander Graf #include <linux/highmem.h>
23b71c9e2fSAlexander Graf #include <linux/log2.h>
24b71c9e2fSAlexander Graf #include <linux/uaccess.h>
25589ee628SIngo Molnar #include <linux/sched/mm.h>
26b71c9e2fSAlexander Graf #include <linux/rwsem.h>
27b71c9e2fSAlexander Graf #include <linux/vmalloc.h>
28b71c9e2fSAlexander Graf #include <linux/hugetlb.h>
29b71c9e2fSAlexander Graf #include <asm/kvm_ppc.h>
3094171b19SAneesh Kumar K.V #include <asm/pte-walk.h>
31b71c9e2fSAlexander Graf
32b71c9e2fSAlexander Graf #include "e500.h"
33b71c9e2fSAlexander Graf #include "timing.h"
34b71c9e2fSAlexander Graf #include "e500_mmu_host.h"
35b71c9e2fSAlexander Graf
36dba291f2SAneesh Kumar K.V #include "trace_booke.h"
37dba291f2SAneesh Kumar K.V
38b71c9e2fSAlexander Graf #define to_htlb1_esel(esel) (host_tlb_params[1].entries - (esel) - 1)
39b71c9e2fSAlexander Graf
40b71c9e2fSAlexander Graf static struct kvmppc_e500_tlb_params host_tlb_params[E500_TLB_NUM];
41b71c9e2fSAlexander Graf
tlb1_max_shadow_size(void)42b71c9e2fSAlexander Graf static inline unsigned int tlb1_max_shadow_size(void)
43b71c9e2fSAlexander Graf {
44b71c9e2fSAlexander Graf /* reserve one entry for magic page */
45b71c9e2fSAlexander Graf return host_tlb_params[1].entries - tlbcam_index - 1;
46b71c9e2fSAlexander Graf }
47b71c9e2fSAlexander Graf
e500_shadow_mas3_attrib(u32 mas3,int usermode)48b71c9e2fSAlexander Graf static inline u32 e500_shadow_mas3_attrib(u32 mas3, int usermode)
49b71c9e2fSAlexander Graf {
50b71c9e2fSAlexander Graf /* Mask off reserved bits. */
51b71c9e2fSAlexander Graf mas3 &= MAS3_ATTRIB_MASK;
52b71c9e2fSAlexander Graf
53b71c9e2fSAlexander Graf #ifndef CONFIG_KVM_BOOKE_HV
54b71c9e2fSAlexander Graf if (!usermode) {
55b71c9e2fSAlexander Graf /* Guest is in supervisor mode,
56b71c9e2fSAlexander Graf * so we need to translate guest
57b71c9e2fSAlexander Graf * supervisor permissions into user permissions. */
58b71c9e2fSAlexander Graf mas3 &= ~E500_TLB_USER_PERM_MASK;
59b71c9e2fSAlexander Graf mas3 |= (mas3 & E500_TLB_SUPER_PERM_MASK) << 1;
60b71c9e2fSAlexander Graf }
61b71c9e2fSAlexander Graf mas3 |= E500_TLB_SUPER_PERM_MASK;
62b71c9e2fSAlexander Graf #endif
63b71c9e2fSAlexander Graf return mas3;
64b71c9e2fSAlexander Graf }
65b71c9e2fSAlexander Graf
66b71c9e2fSAlexander Graf /*
67b71c9e2fSAlexander Graf * writing shadow tlb entry to host TLB
68b71c9e2fSAlexander Graf */
__write_host_tlbe(struct kvm_book3e_206_tlb_entry * stlbe,uint32_t mas0,uint32_t lpid)69b71c9e2fSAlexander Graf static inline void __write_host_tlbe(struct kvm_book3e_206_tlb_entry *stlbe,
70188e267cSMihai Caraman uint32_t mas0,
71188e267cSMihai Caraman uint32_t lpid)
72b71c9e2fSAlexander Graf {
73b71c9e2fSAlexander Graf unsigned long flags;
74b71c9e2fSAlexander Graf
75b71c9e2fSAlexander Graf local_irq_save(flags);
76b71c9e2fSAlexander Graf mtspr(SPRN_MAS0, mas0);
77b71c9e2fSAlexander Graf mtspr(SPRN_MAS1, stlbe->mas1);
78b71c9e2fSAlexander Graf mtspr(SPRN_MAS2, (unsigned long)stlbe->mas2);
79b71c9e2fSAlexander Graf mtspr(SPRN_MAS3, (u32)stlbe->mas7_3);
80b71c9e2fSAlexander Graf mtspr(SPRN_MAS7, (u32)(stlbe->mas7_3 >> 32));
81b71c9e2fSAlexander Graf #ifdef CONFIG_KVM_BOOKE_HV
82188e267cSMihai Caraman mtspr(SPRN_MAS8, MAS8_TGS | get_thread_specific_lpid(lpid));
83b71c9e2fSAlexander Graf #endif
84b71c9e2fSAlexander Graf asm volatile("isync; tlbwe" : : : "memory");
85b71c9e2fSAlexander Graf
86b71c9e2fSAlexander Graf #ifdef CONFIG_KVM_BOOKE_HV
87b71c9e2fSAlexander Graf /* Must clear mas8 for other host tlbwe's */
88b71c9e2fSAlexander Graf mtspr(SPRN_MAS8, 0);
89b71c9e2fSAlexander Graf isync();
90b71c9e2fSAlexander Graf #endif
91b71c9e2fSAlexander Graf local_irq_restore(flags);
92b71c9e2fSAlexander Graf
93b71c9e2fSAlexander Graf trace_kvm_booke206_stlb_write(mas0, stlbe->mas8, stlbe->mas1,
94b71c9e2fSAlexander Graf stlbe->mas2, stlbe->mas7_3);
95b71c9e2fSAlexander Graf }
96b71c9e2fSAlexander Graf
97b71c9e2fSAlexander Graf /*
98b71c9e2fSAlexander Graf * Acquire a mas0 with victim hint, as if we just took a TLB miss.
99b71c9e2fSAlexander Graf *
100b71c9e2fSAlexander Graf * We don't care about the address we're searching for, other than that it's
101b71c9e2fSAlexander Graf * in the right set and is not present in the TLB. Using a zero PID and a
102b71c9e2fSAlexander Graf * userspace address means we don't have to set and then restore MAS5, or
103b71c9e2fSAlexander Graf * calculate a proper MAS6 value.
104b71c9e2fSAlexander Graf */
get_host_mas0(unsigned long eaddr)105b71c9e2fSAlexander Graf static u32 get_host_mas0(unsigned long eaddr)
106b71c9e2fSAlexander Graf {
107b71c9e2fSAlexander Graf unsigned long flags;
108b71c9e2fSAlexander Graf u32 mas0;
109d57cef91SMihai Caraman u32 mas4;
110b71c9e2fSAlexander Graf
111b71c9e2fSAlexander Graf local_irq_save(flags);
112b71c9e2fSAlexander Graf mtspr(SPRN_MAS6, 0);
113d57cef91SMihai Caraman mas4 = mfspr(SPRN_MAS4);
114d57cef91SMihai Caraman mtspr(SPRN_MAS4, mas4 & ~MAS4_TLBSEL_MASK);
115b71c9e2fSAlexander Graf asm volatile("tlbsx 0, %0" : : "b" (eaddr & ~CONFIG_PAGE_OFFSET));
116b71c9e2fSAlexander Graf mas0 = mfspr(SPRN_MAS0);
117d57cef91SMihai Caraman mtspr(SPRN_MAS4, mas4);
118b71c9e2fSAlexander Graf local_irq_restore(flags);
119b71c9e2fSAlexander Graf
120b71c9e2fSAlexander Graf return mas0;
121b71c9e2fSAlexander Graf }
122b71c9e2fSAlexander Graf
123b71c9e2fSAlexander Graf /* sesel is for tlb1 only */
write_host_tlbe(struct kvmppc_vcpu_e500 * vcpu_e500,int tlbsel,int sesel,struct kvm_book3e_206_tlb_entry * stlbe)124b71c9e2fSAlexander Graf static inline void write_host_tlbe(struct kvmppc_vcpu_e500 *vcpu_e500,
125b71c9e2fSAlexander Graf int tlbsel, int sesel, struct kvm_book3e_206_tlb_entry *stlbe)
126b71c9e2fSAlexander Graf {
127b71c9e2fSAlexander Graf u32 mas0;
128b71c9e2fSAlexander Graf
129b71c9e2fSAlexander Graf if (tlbsel == 0) {
130b71c9e2fSAlexander Graf mas0 = get_host_mas0(stlbe->mas2);
131188e267cSMihai Caraman __write_host_tlbe(stlbe, mas0, vcpu_e500->vcpu.kvm->arch.lpid);
132b71c9e2fSAlexander Graf } else {
133b71c9e2fSAlexander Graf __write_host_tlbe(stlbe,
134b71c9e2fSAlexander Graf MAS0_TLBSEL(1) |
135188e267cSMihai Caraman MAS0_ESEL(to_htlb1_esel(sesel)),
136188e267cSMihai Caraman vcpu_e500->vcpu.kvm->arch.lpid);
137b71c9e2fSAlexander Graf }
138b71c9e2fSAlexander Graf }
139b71c9e2fSAlexander Graf
140b71c9e2fSAlexander Graf /* sesel is for tlb1 only */
write_stlbe(struct kvmppc_vcpu_e500 * vcpu_e500,struct kvm_book3e_206_tlb_entry * gtlbe,struct kvm_book3e_206_tlb_entry * stlbe,int stlbsel,int sesel)141b71c9e2fSAlexander Graf static void write_stlbe(struct kvmppc_vcpu_e500 *vcpu_e500,
142b71c9e2fSAlexander Graf struct kvm_book3e_206_tlb_entry *gtlbe,
143b71c9e2fSAlexander Graf struct kvm_book3e_206_tlb_entry *stlbe,
144b71c9e2fSAlexander Graf int stlbsel, int sesel)
145b71c9e2fSAlexander Graf {
146b71c9e2fSAlexander Graf int stid;
147b71c9e2fSAlexander Graf
148b71c9e2fSAlexander Graf preempt_disable();
149b71c9e2fSAlexander Graf stid = kvmppc_e500_get_tlb_stid(&vcpu_e500->vcpu, gtlbe);
150b71c9e2fSAlexander Graf
151b71c9e2fSAlexander Graf stlbe->mas1 |= MAS1_TID(stid);
152b71c9e2fSAlexander Graf write_host_tlbe(vcpu_e500, stlbsel, sesel, stlbe);
153b71c9e2fSAlexander Graf preempt_enable();
154b71c9e2fSAlexander Graf }
155b71c9e2fSAlexander Graf
156b71c9e2fSAlexander Graf #ifdef CONFIG_KVM_E500V2
157b71c9e2fSAlexander Graf /* XXX should be a hook in the gva2hpa translation */
kvmppc_map_magic(struct kvm_vcpu * vcpu)158b71c9e2fSAlexander Graf void kvmppc_map_magic(struct kvm_vcpu *vcpu)
159b71c9e2fSAlexander Graf {
160b71c9e2fSAlexander Graf struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);
161b71c9e2fSAlexander Graf struct kvm_book3e_206_tlb_entry magic;
162b71c9e2fSAlexander Graf ulong shared_page = ((ulong)vcpu->arch.shared) & PAGE_MASK;
163b71c9e2fSAlexander Graf unsigned int stid;
164ba049e93SDan Williams kvm_pfn_t pfn;
165b71c9e2fSAlexander Graf
166ba049e93SDan Williams pfn = (kvm_pfn_t)virt_to_phys((void *)shared_page) >> PAGE_SHIFT;
167b71c9e2fSAlexander Graf get_page(pfn_to_page(pfn));
168b71c9e2fSAlexander Graf
169b71c9e2fSAlexander Graf preempt_disable();
170b71c9e2fSAlexander Graf stid = kvmppc_e500_get_sid(vcpu_e500, 0, 0, 0, 0);
171b71c9e2fSAlexander Graf
172b71c9e2fSAlexander Graf magic.mas1 = MAS1_VALID | MAS1_TS | MAS1_TID(stid) |
173b71c9e2fSAlexander Graf MAS1_TSIZE(BOOK3E_PAGESZ_4K);
174b71c9e2fSAlexander Graf magic.mas2 = vcpu->arch.magic_page_ea | MAS2_M;
175b71c9e2fSAlexander Graf magic.mas7_3 = ((u64)pfn << PAGE_SHIFT) |
176b71c9e2fSAlexander Graf MAS3_SW | MAS3_SR | MAS3_UW | MAS3_UR;
177b71c9e2fSAlexander Graf magic.mas8 = 0;
178b71c9e2fSAlexander Graf
179188e267cSMihai Caraman __write_host_tlbe(&magic, MAS0_TLBSEL(1) | MAS0_ESEL(tlbcam_index), 0);
180b71c9e2fSAlexander Graf preempt_enable();
181b71c9e2fSAlexander Graf }
182b71c9e2fSAlexander Graf #endif
183b71c9e2fSAlexander Graf
inval_gtlbe_on_host(struct kvmppc_vcpu_e500 * vcpu_e500,int tlbsel,int esel)184b71c9e2fSAlexander Graf void inval_gtlbe_on_host(struct kvmppc_vcpu_e500 *vcpu_e500, int tlbsel,
185b71c9e2fSAlexander Graf int esel)
186b71c9e2fSAlexander Graf {
187b71c9e2fSAlexander Graf struct kvm_book3e_206_tlb_entry *gtlbe =
188b71c9e2fSAlexander Graf get_entry(vcpu_e500, tlbsel, esel);
189b71c9e2fSAlexander Graf struct tlbe_ref *ref = &vcpu_e500->gtlb_priv[tlbsel][esel].ref;
190b71c9e2fSAlexander Graf
191b71c9e2fSAlexander Graf /* Don't bother with unmapped entries */
1924d2be6f7SScott Wood if (!(ref->flags & E500_TLB_VALID)) {
1934d2be6f7SScott Wood WARN(ref->flags & (E500_TLB_BITMAP | E500_TLB_TLB0),
1944d2be6f7SScott Wood "%s: flags %x\n", __func__, ref->flags);
1954d2be6f7SScott Wood WARN_ON(tlbsel == 1 && vcpu_e500->g2h_tlb1_map[esel]);
1964d2be6f7SScott Wood }
197b71c9e2fSAlexander Graf
198b71c9e2fSAlexander Graf if (tlbsel == 1 && ref->flags & E500_TLB_BITMAP) {
199b71c9e2fSAlexander Graf u64 tmp = vcpu_e500->g2h_tlb1_map[esel];
200b71c9e2fSAlexander Graf int hw_tlb_indx;
201b71c9e2fSAlexander Graf unsigned long flags;
202b71c9e2fSAlexander Graf
203b71c9e2fSAlexander Graf local_irq_save(flags);
204b71c9e2fSAlexander Graf while (tmp) {
205b71c9e2fSAlexander Graf hw_tlb_indx = __ilog2_u64(tmp & -tmp);
206b71c9e2fSAlexander Graf mtspr(SPRN_MAS0,
207b71c9e2fSAlexander Graf MAS0_TLBSEL(1) |
208b71c9e2fSAlexander Graf MAS0_ESEL(to_htlb1_esel(hw_tlb_indx)));
209b71c9e2fSAlexander Graf mtspr(SPRN_MAS1, 0);
210b71c9e2fSAlexander Graf asm volatile("tlbwe");
211b71c9e2fSAlexander Graf vcpu_e500->h2g_tlb1_rmap[hw_tlb_indx] = 0;
212b71c9e2fSAlexander Graf tmp &= tmp - 1;
213b71c9e2fSAlexander Graf }
214b71c9e2fSAlexander Graf mb();
215b71c9e2fSAlexander Graf vcpu_e500->g2h_tlb1_map[esel] = 0;
216b71c9e2fSAlexander Graf ref->flags &= ~(E500_TLB_BITMAP | E500_TLB_VALID);
217b71c9e2fSAlexander Graf local_irq_restore(flags);
218b71c9e2fSAlexander Graf }
219b71c9e2fSAlexander Graf
220c015c62bSAlexander Graf if (tlbsel == 1 && ref->flags & E500_TLB_TLB0) {
221c015c62bSAlexander Graf /*
222c015c62bSAlexander Graf * TLB1 entry is backed by 4k pages. This should happen
223c015c62bSAlexander Graf * rarely and is not worth optimizing. Invalidate everything.
224c015c62bSAlexander Graf */
225c015c62bSAlexander Graf kvmppc_e500_tlbil_all(vcpu_e500);
226c015c62bSAlexander Graf ref->flags &= ~(E500_TLB_TLB0 | E500_TLB_VALID);
227c015c62bSAlexander Graf }
228c015c62bSAlexander Graf
22930a91fe2SBharat Bhushan /*
23030a91fe2SBharat Bhushan * If TLB entry is still valid then it's a TLB0 entry, and thus
23130a91fe2SBharat Bhushan * backed by at most one host tlbe per shadow pid
23230a91fe2SBharat Bhushan */
23330a91fe2SBharat Bhushan if (ref->flags & E500_TLB_VALID)
234b71c9e2fSAlexander Graf kvmppc_e500_tlbil_one(vcpu_e500, gtlbe);
235b71c9e2fSAlexander Graf
236b71c9e2fSAlexander Graf /* Mark the TLB as not backed by the host anymore */
23730a91fe2SBharat Bhushan ref->flags = 0;
238b71c9e2fSAlexander Graf }
239b71c9e2fSAlexander Graf
tlbe_is_writable(struct kvm_book3e_206_tlb_entry * tlbe)240b71c9e2fSAlexander Graf static inline int tlbe_is_writable(struct kvm_book3e_206_tlb_entry *tlbe)
241b71c9e2fSAlexander Graf {
242b71c9e2fSAlexander Graf return tlbe->mas7_3 & (MAS3_SW|MAS3_UW);
243b71c9e2fSAlexander Graf }
244b71c9e2fSAlexander Graf
kvmppc_e500_ref_setup(struct tlbe_ref * ref,struct kvm_book3e_206_tlb_entry * gtlbe,kvm_pfn_t pfn,unsigned int wimg)245b71c9e2fSAlexander Graf static inline void kvmppc_e500_ref_setup(struct tlbe_ref *ref,
246b71c9e2fSAlexander Graf struct kvm_book3e_206_tlb_entry *gtlbe,
247ba049e93SDan Williams kvm_pfn_t pfn, unsigned int wimg)
248b71c9e2fSAlexander Graf {
249b71c9e2fSAlexander Graf ref->pfn = pfn;
25030a91fe2SBharat Bhushan ref->flags = E500_TLB_VALID;
251b71c9e2fSAlexander Graf
25208c9a188SBharat Bhushan /* Use guest supplied MAS2_G and MAS2_E */
25308c9a188SBharat Bhushan ref->flags |= (gtlbe->mas2 & MAS2_ATTRIB_MASK) | wimg;
25408c9a188SBharat Bhushan
25584e4d632SBharat Bhushan /* Mark the page accessed */
25684e4d632SBharat Bhushan kvm_set_pfn_accessed(pfn);
25784e4d632SBharat Bhushan
258b71c9e2fSAlexander Graf if (tlbe_is_writable(gtlbe))
259b71c9e2fSAlexander Graf kvm_set_pfn_dirty(pfn);
260b71c9e2fSAlexander Graf }
261b71c9e2fSAlexander Graf
kvmppc_e500_ref_release(struct tlbe_ref * ref)262b71c9e2fSAlexander Graf static inline void kvmppc_e500_ref_release(struct tlbe_ref *ref)
263b71c9e2fSAlexander Graf {
264b71c9e2fSAlexander Graf if (ref->flags & E500_TLB_VALID) {
2654d2be6f7SScott Wood /* FIXME: don't log bogus pfn for TLB1 */
266b71c9e2fSAlexander Graf trace_kvm_booke206_ref_release(ref->pfn, ref->flags);
267b71c9e2fSAlexander Graf ref->flags = 0;
268b71c9e2fSAlexander Graf }
269b71c9e2fSAlexander Graf }
270b71c9e2fSAlexander Graf
clear_tlb1_bitmap(struct kvmppc_vcpu_e500 * vcpu_e500)271483ba97cSAlexander Graf static void clear_tlb1_bitmap(struct kvmppc_vcpu_e500 *vcpu_e500)
272b71c9e2fSAlexander Graf {
273b71c9e2fSAlexander Graf if (vcpu_e500->g2h_tlb1_map)
274b71c9e2fSAlexander Graf memset(vcpu_e500->g2h_tlb1_map, 0,
275b71c9e2fSAlexander Graf sizeof(u64) * vcpu_e500->gtlb_params[1].entries);
276b71c9e2fSAlexander Graf if (vcpu_e500->h2g_tlb1_rmap)
277b71c9e2fSAlexander Graf memset(vcpu_e500->h2g_tlb1_rmap, 0,
278b71c9e2fSAlexander Graf sizeof(unsigned int) * host_tlb_params[1].entries);
279b71c9e2fSAlexander Graf }
280b71c9e2fSAlexander Graf
clear_tlb_privs(struct kvmppc_vcpu_e500 * vcpu_e500)281b71c9e2fSAlexander Graf static void clear_tlb_privs(struct kvmppc_vcpu_e500 *vcpu_e500)
282b71c9e2fSAlexander Graf {
2834d2be6f7SScott Wood int tlbsel;
284b71c9e2fSAlexander Graf int i;
285b71c9e2fSAlexander Graf
2864d2be6f7SScott Wood for (tlbsel = 0; tlbsel <= 1; tlbsel++) {
287b71c9e2fSAlexander Graf for (i = 0; i < vcpu_e500->gtlb_params[tlbsel].entries; i++) {
288b71c9e2fSAlexander Graf struct tlbe_ref *ref =
289b71c9e2fSAlexander Graf &vcpu_e500->gtlb_priv[tlbsel][i].ref;
290b71c9e2fSAlexander Graf kvmppc_e500_ref_release(ref);
291b71c9e2fSAlexander Graf }
292b71c9e2fSAlexander Graf }
293b71c9e2fSAlexander Graf }
294b71c9e2fSAlexander Graf
kvmppc_core_flush_tlb(struct kvm_vcpu * vcpu)295b71c9e2fSAlexander Graf void kvmppc_core_flush_tlb(struct kvm_vcpu *vcpu)
296b71c9e2fSAlexander Graf {
297b71c9e2fSAlexander Graf struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);
2984d2be6f7SScott Wood kvmppc_e500_tlbil_all(vcpu_e500);
2994d2be6f7SScott Wood clear_tlb_privs(vcpu_e500);
300b71c9e2fSAlexander Graf clear_tlb1_bitmap(vcpu_e500);
301b71c9e2fSAlexander Graf }
302b71c9e2fSAlexander Graf
303b71c9e2fSAlexander Graf /* TID must be supplied by the caller */
kvmppc_e500_setup_stlbe(struct kvm_vcpu * vcpu,struct kvm_book3e_206_tlb_entry * gtlbe,int tsize,struct tlbe_ref * ref,u64 gvaddr,struct kvm_book3e_206_tlb_entry * stlbe)304b71c9e2fSAlexander Graf static void kvmppc_e500_setup_stlbe(
305b71c9e2fSAlexander Graf struct kvm_vcpu *vcpu,
306b71c9e2fSAlexander Graf struct kvm_book3e_206_tlb_entry *gtlbe,
307b71c9e2fSAlexander Graf int tsize, struct tlbe_ref *ref, u64 gvaddr,
308b71c9e2fSAlexander Graf struct kvm_book3e_206_tlb_entry *stlbe)
309b71c9e2fSAlexander Graf {
310ba049e93SDan Williams kvm_pfn_t pfn = ref->pfn;
311b71c9e2fSAlexander Graf u32 pr = vcpu->arch.shared->msr & MSR_PR;
312b71c9e2fSAlexander Graf
313b71c9e2fSAlexander Graf BUG_ON(!(ref->flags & E500_TLB_VALID));
314b71c9e2fSAlexander Graf
315b71c9e2fSAlexander Graf /* Force IPROT=0 for all guest mappings. */
316b71c9e2fSAlexander Graf stlbe->mas1 = MAS1_TSIZE(tsize) | get_tlb_sts(gtlbe) | MAS1_VALID;
31708c9a188SBharat Bhushan stlbe->mas2 = (gvaddr & MAS2_EPN) | (ref->flags & E500_TLB_MAS2_ATTR);
318b71c9e2fSAlexander Graf stlbe->mas7_3 = ((u64)pfn << PAGE_SHIFT) |
319b71c9e2fSAlexander Graf e500_shadow_mas3_attrib(gtlbe->mas7_3, pr);
320b71c9e2fSAlexander Graf }
321b71c9e2fSAlexander Graf
kvmppc_e500_shadow_map(struct kvmppc_vcpu_e500 * vcpu_e500,u64 gvaddr,gfn_t gfn,struct kvm_book3e_206_tlb_entry * gtlbe,int tlbsel,struct kvm_book3e_206_tlb_entry * stlbe,struct tlbe_ref * ref)322b71c9e2fSAlexander Graf static inline int kvmppc_e500_shadow_map(struct kvmppc_vcpu_e500 *vcpu_e500,
323b71c9e2fSAlexander Graf u64 gvaddr, gfn_t gfn, struct kvm_book3e_206_tlb_entry *gtlbe,
324b71c9e2fSAlexander Graf int tlbsel, struct kvm_book3e_206_tlb_entry *stlbe,
325b71c9e2fSAlexander Graf struct tlbe_ref *ref)
326b71c9e2fSAlexander Graf {
327b71c9e2fSAlexander Graf struct kvm_memory_slot *slot;
328b71c9e2fSAlexander Graf unsigned long pfn = 0; /* silence GCC warning */
329b71c9e2fSAlexander Graf unsigned long hva;
330b71c9e2fSAlexander Graf int pfnmap = 0;
331b71c9e2fSAlexander Graf int tsize = BOOK3E_PAGESZ_4K;
33240fde70dSBharat Bhushan int ret = 0;
33340fde70dSBharat Bhushan unsigned long mmu_seq;
33440fde70dSBharat Bhushan struct kvm *kvm = vcpu_e500->vcpu.kvm;
33508c9a188SBharat Bhushan unsigned long tsize_pages = 0;
33608c9a188SBharat Bhushan pte_t *ptep;
33708c9a188SBharat Bhushan unsigned int wimg = 0;
33808c9a188SBharat Bhushan pgd_t *pgdir;
339691e95fdSAneesh Kumar K.V unsigned long flags;
34040fde70dSBharat Bhushan
34140fde70dSBharat Bhushan /* used to check for invalidations in progress */
34220ec3ebdSChao Peng mmu_seq = kvm->mmu_invalidate_seq;
34340fde70dSBharat Bhushan smp_rmb();
344b71c9e2fSAlexander Graf
345b71c9e2fSAlexander Graf /*
346b71c9e2fSAlexander Graf * Translate guest physical to true physical, acquiring
347b71c9e2fSAlexander Graf * a page reference if it is normal, non-reserved memory.
348b71c9e2fSAlexander Graf *
349b71c9e2fSAlexander Graf * gfn_to_memslot() must succeed because otherwise we wouldn't
350b71c9e2fSAlexander Graf * have gotten this far. Eventually we should just pass the slot
351b71c9e2fSAlexander Graf * pointer through from the first lookup.
352b71c9e2fSAlexander Graf */
353b71c9e2fSAlexander Graf slot = gfn_to_memslot(vcpu_e500->vcpu.kvm, gfn);
354b71c9e2fSAlexander Graf hva = gfn_to_hva_memslot(slot, gfn);
355b71c9e2fSAlexander Graf
356b71c9e2fSAlexander Graf if (tlbsel == 1) {
357b71c9e2fSAlexander Graf struct vm_area_struct *vma;
358d8ed45c5SMichel Lespinasse mmap_read_lock(kvm->mm);
359b71c9e2fSAlexander Graf
360f41c4989SLeonardo Bras vma = find_vma(kvm->mm, hva);
361b71c9e2fSAlexander Graf if (vma && hva >= vma->vm_start &&
362b71c9e2fSAlexander Graf (vma->vm_flags & VM_PFNMAP)) {
363b71c9e2fSAlexander Graf /*
364b71c9e2fSAlexander Graf * This VMA is a physically contiguous region (e.g.
365b71c9e2fSAlexander Graf * /dev/mem) that bypasses normal Linux page
366b71c9e2fSAlexander Graf * management. Find the overlap between the
367b71c9e2fSAlexander Graf * vma and the memslot.
368b71c9e2fSAlexander Graf */
369b71c9e2fSAlexander Graf
370b71c9e2fSAlexander Graf unsigned long start, end;
371b71c9e2fSAlexander Graf unsigned long slot_start, slot_end;
372b71c9e2fSAlexander Graf
373b71c9e2fSAlexander Graf pfnmap = 1;
374b71c9e2fSAlexander Graf
375b71c9e2fSAlexander Graf start = vma->vm_pgoff;
376b71c9e2fSAlexander Graf end = start +
3774bdcb701SThomas Meyer vma_pages(vma);
378b71c9e2fSAlexander Graf
379b71c9e2fSAlexander Graf pfn = start + ((hva - vma->vm_start) >> PAGE_SHIFT);
380b71c9e2fSAlexander Graf
381b71c9e2fSAlexander Graf slot_start = pfn - (gfn - slot->base_gfn);
382b71c9e2fSAlexander Graf slot_end = slot_start + slot->npages;
383b71c9e2fSAlexander Graf
384b71c9e2fSAlexander Graf if (start < slot_start)
385b71c9e2fSAlexander Graf start = slot_start;
386b71c9e2fSAlexander Graf if (end > slot_end)
387b71c9e2fSAlexander Graf end = slot_end;
388b71c9e2fSAlexander Graf
389b71c9e2fSAlexander Graf tsize = (gtlbe->mas1 & MAS1_TSIZE_MASK) >>
390b71c9e2fSAlexander Graf MAS1_TSIZE_SHIFT;
391b71c9e2fSAlexander Graf
392b71c9e2fSAlexander Graf /*
393b71c9e2fSAlexander Graf * e500 doesn't implement the lowest tsize bit,
394b71c9e2fSAlexander Graf * or 1K pages.
395b71c9e2fSAlexander Graf */
396b71c9e2fSAlexander Graf tsize = max(BOOK3E_PAGESZ_4K, tsize & ~1);
397b71c9e2fSAlexander Graf
398b71c9e2fSAlexander Graf /*
399b71c9e2fSAlexander Graf * Now find the largest tsize (up to what the guest
400b71c9e2fSAlexander Graf * requested) that will cover gfn, stay within the
401b71c9e2fSAlexander Graf * range, and for which gfn and pfn are mutually
402b71c9e2fSAlexander Graf * aligned.
403b71c9e2fSAlexander Graf */
404b71c9e2fSAlexander Graf
405b71c9e2fSAlexander Graf for (; tsize > BOOK3E_PAGESZ_4K; tsize -= 2) {
40608c9a188SBharat Bhushan unsigned long gfn_start, gfn_end;
407224f3632STudor Laurentiu tsize_pages = 1UL << (tsize - 2);
408b71c9e2fSAlexander Graf
409b71c9e2fSAlexander Graf gfn_start = gfn & ~(tsize_pages - 1);
410b71c9e2fSAlexander Graf gfn_end = gfn_start + tsize_pages;
411b71c9e2fSAlexander Graf
412b71c9e2fSAlexander Graf if (gfn_start + pfn - gfn < start)
413b71c9e2fSAlexander Graf continue;
414b71c9e2fSAlexander Graf if (gfn_end + pfn - gfn > end)
415b71c9e2fSAlexander Graf continue;
416b71c9e2fSAlexander Graf if ((gfn & (tsize_pages - 1)) !=
417b71c9e2fSAlexander Graf (pfn & (tsize_pages - 1)))
418b71c9e2fSAlexander Graf continue;
419b71c9e2fSAlexander Graf
420b71c9e2fSAlexander Graf gvaddr &= ~((tsize_pages << PAGE_SHIFT) - 1);
421b71c9e2fSAlexander Graf pfn &= ~(tsize_pages - 1);
422b71c9e2fSAlexander Graf break;
423b71c9e2fSAlexander Graf }
424b71c9e2fSAlexander Graf } else if (vma && hva >= vma->vm_start &&
42503911132SAnshuman Khandual is_vm_hugetlb_page(vma)) {
426b71c9e2fSAlexander Graf unsigned long psize = vma_kernel_pagesize(vma);
427b71c9e2fSAlexander Graf
428b71c9e2fSAlexander Graf tsize = (gtlbe->mas1 & MAS1_TSIZE_MASK) >>
429b71c9e2fSAlexander Graf MAS1_TSIZE_SHIFT;
430b71c9e2fSAlexander Graf
431b71c9e2fSAlexander Graf /*
432b71c9e2fSAlexander Graf * Take the largest page size that satisfies both host
433b71c9e2fSAlexander Graf * and guest mapping
434b71c9e2fSAlexander Graf */
435b71c9e2fSAlexander Graf tsize = min(__ilog2(psize) - 10, tsize);
436b71c9e2fSAlexander Graf
437b71c9e2fSAlexander Graf /*
438b71c9e2fSAlexander Graf * e500 doesn't implement the lowest tsize bit,
439b71c9e2fSAlexander Graf * or 1K pages.
440b71c9e2fSAlexander Graf */
441b71c9e2fSAlexander Graf tsize = max(BOOK3E_PAGESZ_4K, tsize & ~1);
442b71c9e2fSAlexander Graf }
443b71c9e2fSAlexander Graf
444d8ed45c5SMichel Lespinasse mmap_read_unlock(kvm->mm);
445b71c9e2fSAlexander Graf }
446b71c9e2fSAlexander Graf
447b71c9e2fSAlexander Graf if (likely(!pfnmap)) {
448224f3632STudor Laurentiu tsize_pages = 1UL << (tsize + 10 - PAGE_SHIFT);
449b71c9e2fSAlexander Graf pfn = gfn_to_pfn_memslot(slot, gfn);
450b71c9e2fSAlexander Graf if (is_error_noslot_pfn(pfn)) {
45108c9a188SBharat Bhushan if (printk_ratelimit())
45208c9a188SBharat Bhushan pr_err("%s: real page not found for gfn %lx\n",
45308c9a188SBharat Bhushan __func__, (long)gfn);
454b71c9e2fSAlexander Graf return -EINVAL;
455b71c9e2fSAlexander Graf }
456b71c9e2fSAlexander Graf
457b71c9e2fSAlexander Graf /* Align guest and physical address to page map boundaries */
458b71c9e2fSAlexander Graf pfn &= ~(tsize_pages - 1);
459b71c9e2fSAlexander Graf gvaddr &= ~((tsize_pages << PAGE_SHIFT) - 1);
460b71c9e2fSAlexander Graf }
461b71c9e2fSAlexander Graf
46240fde70dSBharat Bhushan spin_lock(&kvm->mmu_lock);
46320ec3ebdSChao Peng if (mmu_invalidate_retry(kvm, mmu_seq)) {
46440fde70dSBharat Bhushan ret = -EAGAIN;
46540fde70dSBharat Bhushan goto out;
46640fde70dSBharat Bhushan }
46740fde70dSBharat Bhushan
46808c9a188SBharat Bhushan
46908c9a188SBharat Bhushan pgdir = vcpu_e500->vcpu.arch.pgdir;
470691e95fdSAneesh Kumar K.V /*
471691e95fdSAneesh Kumar K.V * We are just looking at the wimg bits, so we don't
472691e95fdSAneesh Kumar K.V * care much about the trans splitting bit.
473691e95fdSAneesh Kumar K.V * We are holding kvm->mmu_lock so a notifier invalidate
474691e95fdSAneesh Kumar K.V * can't run hence pfn won't change.
475691e95fdSAneesh Kumar K.V */
476691e95fdSAneesh Kumar K.V local_irq_save(flags);
47794171b19SAneesh Kumar K.V ptep = find_linux_pte(pgdir, hva, NULL, NULL);
4785e1d44aeSAneesh Kumar K.V if (ptep) {
4795e1d44aeSAneesh Kumar K.V pte_t pte = READ_ONCE(*ptep);
4805e1d44aeSAneesh Kumar K.V
481691e95fdSAneesh Kumar K.V if (pte_present(pte)) {
4825e1d44aeSAneesh Kumar K.V wimg = (pte_val(pte) >> PTE_WIMGE_SHIFT) &
4835e1d44aeSAneesh Kumar K.V MAS2_WIMGE_MASK;
484691e95fdSAneesh Kumar K.V local_irq_restore(flags);
485691e95fdSAneesh Kumar K.V } else {
486691e95fdSAneesh Kumar K.V local_irq_restore(flags);
4875e1d44aeSAneesh Kumar K.V pr_err_ratelimited("%s: pte not present: gfn %lx,pfn %lx\n",
48808c9a188SBharat Bhushan __func__, (long)gfn, pfn);
489511c6681SMihai Caraman ret = -EINVAL;
490511c6681SMihai Caraman goto out;
49108c9a188SBharat Bhushan }
4925e1d44aeSAneesh Kumar K.V }
49308c9a188SBharat Bhushan kvmppc_e500_ref_setup(ref, gtlbe, pfn, wimg);
494b71c9e2fSAlexander Graf
495b71c9e2fSAlexander Graf kvmppc_e500_setup_stlbe(&vcpu_e500->vcpu, gtlbe, tsize,
496b71c9e2fSAlexander Graf ref, gvaddr, stlbe);
497b71c9e2fSAlexander Graf
498b71c9e2fSAlexander Graf /* Clear i-cache for new pages */
499b71c9e2fSAlexander Graf kvmppc_mmu_flush_icache(pfn);
500b71c9e2fSAlexander Graf
50140fde70dSBharat Bhushan out:
50240fde70dSBharat Bhushan spin_unlock(&kvm->mmu_lock);
50340fde70dSBharat Bhushan
504b71c9e2fSAlexander Graf /* Drop refcount on page, so that mmu notifiers can clear it */
505b71c9e2fSAlexander Graf kvm_release_pfn_clean(pfn);
506b71c9e2fSAlexander Graf
50740fde70dSBharat Bhushan return ret;
508b71c9e2fSAlexander Graf }
509b71c9e2fSAlexander Graf
510b71c9e2fSAlexander Graf /* XXX only map the one-one case, for now use TLB0 */
kvmppc_e500_tlb0_map(struct kvmppc_vcpu_e500 * vcpu_e500,int esel,struct kvm_book3e_206_tlb_entry * stlbe)511b71c9e2fSAlexander Graf static int kvmppc_e500_tlb0_map(struct kvmppc_vcpu_e500 *vcpu_e500, int esel,
512b71c9e2fSAlexander Graf struct kvm_book3e_206_tlb_entry *stlbe)
513b71c9e2fSAlexander Graf {
514b71c9e2fSAlexander Graf struct kvm_book3e_206_tlb_entry *gtlbe;
515b71c9e2fSAlexander Graf struct tlbe_ref *ref;
516b71c9e2fSAlexander Graf int stlbsel = 0;
517b71c9e2fSAlexander Graf int sesel = 0;
518b71c9e2fSAlexander Graf int r;
519b71c9e2fSAlexander Graf
520b71c9e2fSAlexander Graf gtlbe = get_entry(vcpu_e500, 0, esel);
521b71c9e2fSAlexander Graf ref = &vcpu_e500->gtlb_priv[0][esel].ref;
522b71c9e2fSAlexander Graf
523b71c9e2fSAlexander Graf r = kvmppc_e500_shadow_map(vcpu_e500, get_tlb_eaddr(gtlbe),
524b71c9e2fSAlexander Graf get_tlb_raddr(gtlbe) >> PAGE_SHIFT,
525b71c9e2fSAlexander Graf gtlbe, 0, stlbe, ref);
526b71c9e2fSAlexander Graf if (r)
527b71c9e2fSAlexander Graf return r;
528b71c9e2fSAlexander Graf
529b71c9e2fSAlexander Graf write_stlbe(vcpu_e500, gtlbe, stlbe, stlbsel, sesel);
530b71c9e2fSAlexander Graf
531b71c9e2fSAlexander Graf return 0;
532b71c9e2fSAlexander Graf }
533b71c9e2fSAlexander Graf
kvmppc_e500_tlb1_map_tlb1(struct kvmppc_vcpu_e500 * vcpu_e500,struct tlbe_ref * ref,int esel)534c015c62bSAlexander Graf static int kvmppc_e500_tlb1_map_tlb1(struct kvmppc_vcpu_e500 *vcpu_e500,
535c015c62bSAlexander Graf struct tlbe_ref *ref,
536c015c62bSAlexander Graf int esel)
537b71c9e2fSAlexander Graf {
538c015c62bSAlexander Graf unsigned int sesel = vcpu_e500->host_tlb1_nv++;
539b71c9e2fSAlexander Graf
540b71c9e2fSAlexander Graf if (unlikely(vcpu_e500->host_tlb1_nv >= tlb1_max_shadow_size()))
541b71c9e2fSAlexander Graf vcpu_e500->host_tlb1_nv = 0;
542b71c9e2fSAlexander Graf
543b71c9e2fSAlexander Graf if (vcpu_e500->h2g_tlb1_rmap[sesel]) {
5446b2ba1a9SScott Wood unsigned int idx = vcpu_e500->h2g_tlb1_rmap[sesel] - 1;
545b71c9e2fSAlexander Graf vcpu_e500->g2h_tlb1_map[idx] &= ~(1ULL << sesel);
546b71c9e2fSAlexander Graf }
54766a5fecdSScott Wood
54866a5fecdSScott Wood vcpu_e500->gtlb_priv[1][esel].ref.flags |= E500_TLB_BITMAP;
54966a5fecdSScott Wood vcpu_e500->g2h_tlb1_map[esel] |= (u64)1 << sesel;
5506b2ba1a9SScott Wood vcpu_e500->h2g_tlb1_rmap[sesel] = esel + 1;
5514d2be6f7SScott Wood WARN_ON(!(ref->flags & E500_TLB_VALID));
552b71c9e2fSAlexander Graf
553c015c62bSAlexander Graf return sesel;
554c015c62bSAlexander Graf }
555c015c62bSAlexander Graf
556c015c62bSAlexander Graf /* Caller must ensure that the specified guest TLB entry is safe to insert into
557c015c62bSAlexander Graf * the shadow TLB. */
558c015c62bSAlexander Graf /* For both one-one and one-to-many */
kvmppc_e500_tlb1_map(struct kvmppc_vcpu_e500 * vcpu_e500,u64 gvaddr,gfn_t gfn,struct kvm_book3e_206_tlb_entry * gtlbe,struct kvm_book3e_206_tlb_entry * stlbe,int esel)559c015c62bSAlexander Graf static int kvmppc_e500_tlb1_map(struct kvmppc_vcpu_e500 *vcpu_e500,
560c015c62bSAlexander Graf u64 gvaddr, gfn_t gfn, struct kvm_book3e_206_tlb_entry *gtlbe,
561c015c62bSAlexander Graf struct kvm_book3e_206_tlb_entry *stlbe, int esel)
562c015c62bSAlexander Graf {
5634d2be6f7SScott Wood struct tlbe_ref *ref = &vcpu_e500->gtlb_priv[1][esel].ref;
564c015c62bSAlexander Graf int sesel;
565c015c62bSAlexander Graf int r;
566c015c62bSAlexander Graf
567c015c62bSAlexander Graf r = kvmppc_e500_shadow_map(vcpu_e500, gvaddr, gfn, gtlbe, 1, stlbe,
5684d2be6f7SScott Wood ref);
569c015c62bSAlexander Graf if (r)
570c015c62bSAlexander Graf return r;
571c015c62bSAlexander Graf
572c015c62bSAlexander Graf /* Use TLB0 when we can only map a page with 4k */
573c015c62bSAlexander Graf if (get_tlb_tsize(stlbe) == BOOK3E_PAGESZ_4K) {
574c015c62bSAlexander Graf vcpu_e500->gtlb_priv[1][esel].ref.flags |= E500_TLB_TLB0;
575c015c62bSAlexander Graf write_stlbe(vcpu_e500, gtlbe, stlbe, 0, 0);
576c015c62bSAlexander Graf return 0;
577c015c62bSAlexander Graf }
578c015c62bSAlexander Graf
579c015c62bSAlexander Graf /* Otherwise map into TLB1 */
5804d2be6f7SScott Wood sesel = kvmppc_e500_tlb1_map_tlb1(vcpu_e500, ref, esel);
581c015c62bSAlexander Graf write_stlbe(vcpu_e500, gtlbe, stlbe, 1, sesel);
582b71c9e2fSAlexander Graf
583b71c9e2fSAlexander Graf return 0;
584b71c9e2fSAlexander Graf }
585b71c9e2fSAlexander Graf
kvmppc_mmu_map(struct kvm_vcpu * vcpu,u64 eaddr,gpa_t gpaddr,unsigned int index)586b71c9e2fSAlexander Graf void kvmppc_mmu_map(struct kvm_vcpu *vcpu, u64 eaddr, gpa_t gpaddr,
587b71c9e2fSAlexander Graf unsigned int index)
588b71c9e2fSAlexander Graf {
589b71c9e2fSAlexander Graf struct kvmppc_vcpu_e500 *vcpu_e500 = to_e500(vcpu);
590b71c9e2fSAlexander Graf struct tlbe_priv *priv;
591b71c9e2fSAlexander Graf struct kvm_book3e_206_tlb_entry *gtlbe, stlbe;
592b71c9e2fSAlexander Graf int tlbsel = tlbsel_of(index);
593b71c9e2fSAlexander Graf int esel = esel_of(index);
594b71c9e2fSAlexander Graf
595b71c9e2fSAlexander Graf gtlbe = get_entry(vcpu_e500, tlbsel, esel);
596b71c9e2fSAlexander Graf
597b71c9e2fSAlexander Graf switch (tlbsel) {
598b71c9e2fSAlexander Graf case 0:
599b71c9e2fSAlexander Graf priv = &vcpu_e500->gtlb_priv[tlbsel][esel];
600b71c9e2fSAlexander Graf
6014d2be6f7SScott Wood /* Triggers after clear_tlb_privs or on initial mapping */
602b71c9e2fSAlexander Graf if (!(priv->ref.flags & E500_TLB_VALID)) {
603b71c9e2fSAlexander Graf kvmppc_e500_tlb0_map(vcpu_e500, esel, &stlbe);
604b71c9e2fSAlexander Graf } else {
605b71c9e2fSAlexander Graf kvmppc_e500_setup_stlbe(vcpu, gtlbe, BOOK3E_PAGESZ_4K,
606b71c9e2fSAlexander Graf &priv->ref, eaddr, &stlbe);
607b71c9e2fSAlexander Graf write_stlbe(vcpu_e500, gtlbe, &stlbe, 0, 0);
608b71c9e2fSAlexander Graf }
609b71c9e2fSAlexander Graf break;
610b71c9e2fSAlexander Graf
611b71c9e2fSAlexander Graf case 1: {
612b71c9e2fSAlexander Graf gfn_t gfn = gpaddr >> PAGE_SHIFT;
613b71c9e2fSAlexander Graf kvmppc_e500_tlb1_map(vcpu_e500, eaddr, gfn, gtlbe, &stlbe,
614b71c9e2fSAlexander Graf esel);
615b71c9e2fSAlexander Graf break;
616b71c9e2fSAlexander Graf }
617b71c9e2fSAlexander Graf
618b71c9e2fSAlexander Graf default:
619b71c9e2fSAlexander Graf BUG();
620b71c9e2fSAlexander Graf break;
621b71c9e2fSAlexander Graf }
622b71c9e2fSAlexander Graf }
623b71c9e2fSAlexander Graf
624f5250471SMihai Caraman #ifdef CONFIG_KVM_BOOKE_HV
kvmppc_load_last_inst(struct kvm_vcpu * vcpu,enum instruction_fetch_type type,unsigned long * instr)62570923603SSimon Guo int kvmppc_load_last_inst(struct kvm_vcpu *vcpu,
626*953e3739SPaul Mackerras enum instruction_fetch_type type, unsigned long *instr)
627f5250471SMihai Caraman {
628f5250471SMihai Caraman gva_t geaddr;
629f5250471SMihai Caraman hpa_t addr;
630f5250471SMihai Caraman hfn_t pfn;
631f5250471SMihai Caraman hva_t eaddr;
632f5250471SMihai Caraman u32 mas1, mas2, mas3;
633f5250471SMihai Caraman u64 mas7_mas3;
634f5250471SMihai Caraman struct page *page;
635f5250471SMihai Caraman unsigned int addr_space, psize_shift;
636f5250471SMihai Caraman bool pr;
637f5250471SMihai Caraman unsigned long flags;
638f5250471SMihai Caraman
639f5250471SMihai Caraman /* Search TLB for guest pc to get the real address */
640f5250471SMihai Caraman geaddr = kvmppc_get_pc(vcpu);
641f5250471SMihai Caraman
642f5250471SMihai Caraman addr_space = (vcpu->arch.shared->msr & MSR_IS) >> MSR_IR_LG;
643f5250471SMihai Caraman
644f5250471SMihai Caraman local_irq_save(flags);
645f5250471SMihai Caraman mtspr(SPRN_MAS6, (vcpu->arch.pid << MAS6_SPID_SHIFT) | addr_space);
646188e267cSMihai Caraman mtspr(SPRN_MAS5, MAS5_SGS | get_lpid(vcpu));
647f5250471SMihai Caraman asm volatile("tlbsx 0, %[geaddr]\n" : :
648f5250471SMihai Caraman [geaddr] "r" (geaddr));
649f5250471SMihai Caraman mtspr(SPRN_MAS5, 0);
650f5250471SMihai Caraman mtspr(SPRN_MAS8, 0);
651f5250471SMihai Caraman mas1 = mfspr(SPRN_MAS1);
652f5250471SMihai Caraman mas2 = mfspr(SPRN_MAS2);
653f5250471SMihai Caraman mas3 = mfspr(SPRN_MAS3);
654f5250471SMihai Caraman #ifdef CONFIG_64BIT
655f5250471SMihai Caraman mas7_mas3 = mfspr(SPRN_MAS7_MAS3);
656f5250471SMihai Caraman #else
657f5250471SMihai Caraman mas7_mas3 = ((u64)mfspr(SPRN_MAS7) << 32) | mas3;
658f5250471SMihai Caraman #endif
659f5250471SMihai Caraman local_irq_restore(flags);
660f5250471SMihai Caraman
661f5250471SMihai Caraman /*
662f5250471SMihai Caraman * If the TLB entry for guest pc was evicted, return to the guest.
663f5250471SMihai Caraman * There are high chances to find a valid TLB entry next time.
664f5250471SMihai Caraman */
665f5250471SMihai Caraman if (!(mas1 & MAS1_VALID))
666f5250471SMihai Caraman return EMULATE_AGAIN;
667f5250471SMihai Caraman
668f5250471SMihai Caraman /*
669f5250471SMihai Caraman * Another thread may rewrite the TLB entry in parallel, don't
670f5250471SMihai Caraman * execute from the address if the execute permission is not set
671f5250471SMihai Caraman */
672f5250471SMihai Caraman pr = vcpu->arch.shared->msr & MSR_PR;
673f5250471SMihai Caraman if (unlikely((pr && !(mas3 & MAS3_UX)) ||
674f5250471SMihai Caraman (!pr && !(mas3 & MAS3_SX)))) {
675f5250471SMihai Caraman pr_err_ratelimited(
6766774def6SMasanari Iida "%s: Instruction emulation from guest address %08lx without execute permission\n",
677f5250471SMihai Caraman __func__, geaddr);
678f5250471SMihai Caraman return EMULATE_AGAIN;
679f5250471SMihai Caraman }
680f5250471SMihai Caraman
681f5250471SMihai Caraman /*
682f5250471SMihai Caraman * The real address will be mapped by a cacheable, memory coherent,
683f5250471SMihai Caraman * write-back page. Check for mismatches when LRAT is used.
684f5250471SMihai Caraman */
685f5250471SMihai Caraman if (has_feature(vcpu, VCPU_FTR_MMU_V2) &&
686f5250471SMihai Caraman unlikely((mas2 & MAS2_I) || (mas2 & MAS2_W) || !(mas2 & MAS2_M))) {
687f5250471SMihai Caraman pr_err_ratelimited(
6886774def6SMasanari Iida "%s: Instruction emulation from guest address %08lx mismatches storage attributes\n",
689f5250471SMihai Caraman __func__, geaddr);
690f5250471SMihai Caraman return EMULATE_AGAIN;
691f5250471SMihai Caraman }
692f5250471SMihai Caraman
693f5250471SMihai Caraman /* Get pfn */
694f5250471SMihai Caraman psize_shift = MAS1_GET_TSIZE(mas1) + 10;
695f5250471SMihai Caraman addr = (mas7_mas3 & (~0ULL << psize_shift)) |
696f5250471SMihai Caraman (geaddr & ((1ULL << psize_shift) - 1ULL));
697f5250471SMihai Caraman pfn = addr >> PAGE_SHIFT;
698f5250471SMihai Caraman
699f5250471SMihai Caraman /* Guard against emulation from devices area */
700f5250471SMihai Caraman if (unlikely(!page_is_ram(pfn))) {
7016774def6SMasanari Iida pr_err_ratelimited("%s: Instruction emulation from non-RAM host address %08llx is not supported\n",
702f5250471SMihai Caraman __func__, addr);
703f5250471SMihai Caraman return EMULATE_AGAIN;
704f5250471SMihai Caraman }
705f5250471SMihai Caraman
706f5250471SMihai Caraman /* Map a page and get guest's instruction */
707f5250471SMihai Caraman page = pfn_to_page(pfn);
708f5250471SMihai Caraman eaddr = (unsigned long)kmap_atomic(page);
709f5250471SMihai Caraman *instr = *(u32 *)(eaddr | (unsigned long)(addr & ~PAGE_MASK));
710f5250471SMihai Caraman kunmap_atomic((u32 *)eaddr);
711f5250471SMihai Caraman
712f5250471SMihai Caraman return EMULATE_DONE;
713f5250471SMihai Caraman }
714f5250471SMihai Caraman #else
kvmppc_load_last_inst(struct kvm_vcpu * vcpu,enum instruction_fetch_type type,unsigned long * instr)71570923603SSimon Guo int kvmppc_load_last_inst(struct kvm_vcpu *vcpu,
716*953e3739SPaul Mackerras enum instruction_fetch_type type, unsigned long *instr)
71751f04726SMihai Caraman {
71851f04726SMihai Caraman return EMULATE_AGAIN;
71951f04726SMihai Caraman }
720f5250471SMihai Caraman #endif
72151f04726SMihai Caraman
722b71c9e2fSAlexander Graf /************* MMU Notifiers *************/
723b71c9e2fSAlexander Graf
kvm_e500_mmu_unmap_gfn(struct kvm * kvm,struct kvm_gfn_range * range)724b1c5356eSSean Christopherson static bool kvm_e500_mmu_unmap_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
725b71c9e2fSAlexander Graf {
726b71c9e2fSAlexander Graf /*
727b71c9e2fSAlexander Graf * Flush all shadow tlb entries everywhere. This is slow, but
728b71c9e2fSAlexander Graf * we are 100% sure that we catch the to be unmapped page
729b71c9e2fSAlexander Graf */
730b1c5356eSSean Christopherson return true;
731b71c9e2fSAlexander Graf }
732b71c9e2fSAlexander Graf
kvm_unmap_gfn_range(struct kvm * kvm,struct kvm_gfn_range * range)733b1c5356eSSean Christopherson bool kvm_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range)
734b71c9e2fSAlexander Graf {
735b1c5356eSSean Christopherson return kvm_e500_mmu_unmap_gfn(kvm, range);
736b71c9e2fSAlexander Graf }
737b71c9e2fSAlexander Graf
kvm_age_gfn(struct kvm * kvm,struct kvm_gfn_range * range)738b1c5356eSSean Christopherson bool kvm_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
739b71c9e2fSAlexander Graf {
740b71c9e2fSAlexander Graf /* XXX could be more clever ;) */
741b1c5356eSSean Christopherson return false;
742b71c9e2fSAlexander Graf }
743b71c9e2fSAlexander Graf
kvm_test_age_gfn(struct kvm * kvm,struct kvm_gfn_range * range)744b1c5356eSSean Christopherson bool kvm_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
745b71c9e2fSAlexander Graf {
746b71c9e2fSAlexander Graf /* XXX could be more clever ;) */
747b1c5356eSSean Christopherson return false;
748b71c9e2fSAlexander Graf }
749b71c9e2fSAlexander Graf
750b71c9e2fSAlexander Graf /*****************************************/
751b71c9e2fSAlexander Graf
e500_mmu_host_init(struct kvmppc_vcpu_e500 * vcpu_e500)752b71c9e2fSAlexander Graf int e500_mmu_host_init(struct kvmppc_vcpu_e500 *vcpu_e500)
753b71c9e2fSAlexander Graf {
754b71c9e2fSAlexander Graf host_tlb_params[0].entries = mfspr(SPRN_TLB0CFG) & TLBnCFG_N_ENTRY;
755b71c9e2fSAlexander Graf host_tlb_params[1].entries = mfspr(SPRN_TLB1CFG) & TLBnCFG_N_ENTRY;
756b71c9e2fSAlexander Graf
757b71c9e2fSAlexander Graf /*
758b71c9e2fSAlexander Graf * This should never happen on real e500 hardware, but is
759b71c9e2fSAlexander Graf * architecturally possible -- e.g. in some weird nested
760b71c9e2fSAlexander Graf * virtualization case.
761b71c9e2fSAlexander Graf */
762b71c9e2fSAlexander Graf if (host_tlb_params[0].entries == 0 ||
763b71c9e2fSAlexander Graf host_tlb_params[1].entries == 0) {
764b71c9e2fSAlexander Graf pr_err("%s: need to know host tlb size\n", __func__);
765b71c9e2fSAlexander Graf return -ENODEV;
766b71c9e2fSAlexander Graf }
767b71c9e2fSAlexander Graf
768b71c9e2fSAlexander Graf host_tlb_params[0].ways = (mfspr(SPRN_TLB0CFG) & TLBnCFG_ASSOC) >>
769b71c9e2fSAlexander Graf TLBnCFG_ASSOC_SHIFT;
770b71c9e2fSAlexander Graf host_tlb_params[1].ways = host_tlb_params[1].entries;
771b71c9e2fSAlexander Graf
772b71c9e2fSAlexander Graf if (!is_power_of_2(host_tlb_params[0].entries) ||
773b71c9e2fSAlexander Graf !is_power_of_2(host_tlb_params[0].ways) ||
774b71c9e2fSAlexander Graf host_tlb_params[0].entries < host_tlb_params[0].ways ||
775b71c9e2fSAlexander Graf host_tlb_params[0].ways == 0) {
776b71c9e2fSAlexander Graf pr_err("%s: bad tlb0 host config: %u entries %u ways\n",
777b71c9e2fSAlexander Graf __func__, host_tlb_params[0].entries,
778b71c9e2fSAlexander Graf host_tlb_params[0].ways);
779b71c9e2fSAlexander Graf return -ENODEV;
780b71c9e2fSAlexander Graf }
781b71c9e2fSAlexander Graf
782b71c9e2fSAlexander Graf host_tlb_params[0].sets =
783b71c9e2fSAlexander Graf host_tlb_params[0].entries / host_tlb_params[0].ways;
784b71c9e2fSAlexander Graf host_tlb_params[1].sets = 1;
78537655490SMarkus Elfring vcpu_e500->h2g_tlb1_rmap = kcalloc(host_tlb_params[1].entries,
78637655490SMarkus Elfring sizeof(*vcpu_e500->h2g_tlb1_rmap),
787b71c9e2fSAlexander Graf GFP_KERNEL);
788b71c9e2fSAlexander Graf if (!vcpu_e500->h2g_tlb1_rmap)
7894d2be6f7SScott Wood return -EINVAL;
790b71c9e2fSAlexander Graf
791b71c9e2fSAlexander Graf return 0;
792b71c9e2fSAlexander Graf }
793b71c9e2fSAlexander Graf
e500_mmu_host_uninit(struct kvmppc_vcpu_e500 * vcpu_e500)794b71c9e2fSAlexander Graf void e500_mmu_host_uninit(struct kvmppc_vcpu_e500 *vcpu_e500)
795b71c9e2fSAlexander Graf {
796b71c9e2fSAlexander Graf kfree(vcpu_e500->h2g_tlb1_rmap);
797b71c9e2fSAlexander Graf }
798