1d7d5b05fSDeng-Cheng Zhu /* 2d7d5b05fSDeng-Cheng Zhu * This file is subject to the terms and conditions of the GNU General Public 3d7d5b05fSDeng-Cheng Zhu * License. See the file "COPYING" in the main directory of this archive 4d7d5b05fSDeng-Cheng Zhu * for more details. 5d7d5b05fSDeng-Cheng Zhu * 6d7d5b05fSDeng-Cheng Zhu * KVM/MIPS TLB handling, this file is part of the Linux host kernel so that 7d7d5b05fSDeng-Cheng Zhu * TLB handlers run from KSEG0 8d7d5b05fSDeng-Cheng Zhu * 9d7d5b05fSDeng-Cheng Zhu * Copyright (C) 2012 MIPS Technologies, Inc. All rights reserved. 10d7d5b05fSDeng-Cheng Zhu * Authors: Sanjay Lal <sanjayl@kymasys.com> 11d7d5b05fSDeng-Cheng Zhu */ 12d7d5b05fSDeng-Cheng Zhu 13d7d5b05fSDeng-Cheng Zhu #include <linux/sched.h> 14d7d5b05fSDeng-Cheng Zhu #include <linux/smp.h> 15d7d5b05fSDeng-Cheng Zhu #include <linux/mm.h> 16d7d5b05fSDeng-Cheng Zhu #include <linux/delay.h> 17403015b3SJames Hogan #include <linux/export.h> 18d7d5b05fSDeng-Cheng Zhu #include <linux/kvm_host.h> 19d7d5b05fSDeng-Cheng Zhu #include <linux/srcu.h> 20d7d5b05fSDeng-Cheng Zhu 21d7d5b05fSDeng-Cheng Zhu #include <asm/cpu.h> 22d7d5b05fSDeng-Cheng Zhu #include <asm/bootinfo.h> 23d7d5b05fSDeng-Cheng Zhu #include <asm/mmu_context.h> 24d7d5b05fSDeng-Cheng Zhu #include <asm/pgtable.h> 25d7d5b05fSDeng-Cheng Zhu #include <asm/cacheflush.h> 26d7d5b05fSDeng-Cheng Zhu #include <asm/tlb.h> 27e922a4cbSJames Hogan #include <asm/tlbdebug.h> 28d7d5b05fSDeng-Cheng Zhu 29d7d5b05fSDeng-Cheng Zhu #undef CONFIG_MIPS_MT 30d7d5b05fSDeng-Cheng Zhu #include <asm/r4kcache.h> 31d7d5b05fSDeng-Cheng Zhu #define CONFIG_MIPS_MT 32d7d5b05fSDeng-Cheng Zhu 33d7d5b05fSDeng-Cheng Zhu #define KVM_GUEST_PC_TLB 0 34d7d5b05fSDeng-Cheng Zhu #define KVM_GUEST_SP_TLB 1 35d7d5b05fSDeng-Cheng Zhu 36372582a6SJames Hogan #ifdef CONFIG_KVM_MIPS_VZ 37c992a4f6SJames Hogan unsigned long GUESTID_MASK; 38c992a4f6SJames Hogan EXPORT_SYMBOL_GPL(GUESTID_MASK); 39c992a4f6SJames Hogan unsigned long GUESTID_FIRST_VERSION; 40c992a4f6SJames Hogan EXPORT_SYMBOL_GPL(GUESTID_FIRST_VERSION); 41c992a4f6SJames Hogan unsigned long GUESTID_VERSION_MASK; 42c992a4f6SJames Hogan EXPORT_SYMBOL_GPL(GUESTID_VERSION_MASK); 43c992a4f6SJames Hogan 44372582a6SJames Hogan static u32 kvm_mips_get_root_asid(struct kvm_vcpu *vcpu) 45372582a6SJames Hogan { 46372582a6SJames Hogan struct mm_struct *gpa_mm = &vcpu->kvm->arch.gpa_mm; 47372582a6SJames Hogan 48372582a6SJames Hogan if (cpu_has_guestid) 49372582a6SJames Hogan return 0; 50372582a6SJames Hogan else 51372582a6SJames Hogan return cpu_asid(smp_processor_id(), gpa_mm); 52372582a6SJames Hogan } 53372582a6SJames Hogan #endif 54372582a6SJames Hogan 55403015b3SJames Hogan static u32 kvm_mips_get_kernel_asid(struct kvm_vcpu *vcpu) 56d7d5b05fSDeng-Cheng Zhu { 57c550d539SJames Hogan struct mm_struct *kern_mm = &vcpu->arch.guest_kernel_mm; 584edf00a4SPaul Burton int cpu = smp_processor_id(); 594edf00a4SPaul Burton 60c550d539SJames Hogan return cpu_asid(cpu, kern_mm); 61d7d5b05fSDeng-Cheng Zhu } 62d7d5b05fSDeng-Cheng Zhu 63403015b3SJames Hogan static u32 kvm_mips_get_user_asid(struct kvm_vcpu *vcpu) 64d7d5b05fSDeng-Cheng Zhu { 65c550d539SJames Hogan struct mm_struct *user_mm = &vcpu->arch.guest_user_mm; 664edf00a4SPaul Burton int cpu = smp_processor_id(); 674edf00a4SPaul Burton 68c550d539SJames Hogan return cpu_asid(cpu, user_mm); 69d7d5b05fSDeng-Cheng Zhu } 70d7d5b05fSDeng-Cheng Zhu 71d7d5b05fSDeng-Cheng Zhu /* Structure defining an tlb entry data set. */ 72d7d5b05fSDeng-Cheng Zhu 73d7d5b05fSDeng-Cheng Zhu void kvm_mips_dump_host_tlbs(void) 74d7d5b05fSDeng-Cheng Zhu { 75d7d5b05fSDeng-Cheng Zhu unsigned long flags; 76d7d5b05fSDeng-Cheng Zhu 77d7d5b05fSDeng-Cheng Zhu local_irq_save(flags); 78d7d5b05fSDeng-Cheng Zhu 79d7d5b05fSDeng-Cheng Zhu kvm_info("HOST TLBs:\n"); 80e922a4cbSJames Hogan dump_tlb_regs(); 81e922a4cbSJames Hogan pr_info("\n"); 82e922a4cbSJames Hogan dump_tlb_all(); 83d7d5b05fSDeng-Cheng Zhu 84d7d5b05fSDeng-Cheng Zhu local_irq_restore(flags); 85d7d5b05fSDeng-Cheng Zhu } 86cb1b447fSJames Hogan EXPORT_SYMBOL_GPL(kvm_mips_dump_host_tlbs); 87d7d5b05fSDeng-Cheng Zhu 88d7d5b05fSDeng-Cheng Zhu void kvm_mips_dump_guest_tlbs(struct kvm_vcpu *vcpu) 89d7d5b05fSDeng-Cheng Zhu { 90d7d5b05fSDeng-Cheng Zhu struct mips_coproc *cop0 = vcpu->arch.cop0; 91d7d5b05fSDeng-Cheng Zhu struct kvm_mips_tlb tlb; 92d7d5b05fSDeng-Cheng Zhu int i; 93d7d5b05fSDeng-Cheng Zhu 94d7d5b05fSDeng-Cheng Zhu kvm_info("Guest TLBs:\n"); 95d7d5b05fSDeng-Cheng Zhu kvm_info("Guest EntryHi: %#lx\n", kvm_read_c0_guest_entryhi(cop0)); 96d7d5b05fSDeng-Cheng Zhu 97d7d5b05fSDeng-Cheng Zhu for (i = 0; i < KVM_MIPS_GUEST_TLB_SIZE; i++) { 98d7d5b05fSDeng-Cheng Zhu tlb = vcpu->arch.guest_tlb[i]; 99d7d5b05fSDeng-Cheng Zhu kvm_info("TLB%c%3d Hi 0x%08lx ", 100e6207bbeSJames Hogan (tlb.tlb_lo[0] | tlb.tlb_lo[1]) & ENTRYLO_V 1019fbfb06aSJames Hogan ? ' ' : '*', 102d7d5b05fSDeng-Cheng Zhu i, tlb.tlb_hi); 1038cffd197SJames Hogan kvm_info("Lo0=0x%09llx %c%c attr %lx ", 1049fbfb06aSJames Hogan (u64) mips3_tlbpfn_to_paddr(tlb.tlb_lo[0]), 105e6207bbeSJames Hogan (tlb.tlb_lo[0] & ENTRYLO_D) ? 'D' : ' ', 106e6207bbeSJames Hogan (tlb.tlb_lo[0] & ENTRYLO_G) ? 'G' : ' ', 107e6207bbeSJames Hogan (tlb.tlb_lo[0] & ENTRYLO_C) >> ENTRYLO_C_SHIFT); 1088cffd197SJames Hogan kvm_info("Lo1=0x%09llx %c%c attr %lx sz=%lx\n", 1099fbfb06aSJames Hogan (u64) mips3_tlbpfn_to_paddr(tlb.tlb_lo[1]), 110e6207bbeSJames Hogan (tlb.tlb_lo[1] & ENTRYLO_D) ? 'D' : ' ', 111e6207bbeSJames Hogan (tlb.tlb_lo[1] & ENTRYLO_G) ? 'G' : ' ', 112e6207bbeSJames Hogan (tlb.tlb_lo[1] & ENTRYLO_C) >> ENTRYLO_C_SHIFT, 113e6207bbeSJames Hogan tlb.tlb_mask); 114d7d5b05fSDeng-Cheng Zhu } 115d7d5b05fSDeng-Cheng Zhu } 116cb1b447fSJames Hogan EXPORT_SYMBOL_GPL(kvm_mips_dump_guest_tlbs); 117d7d5b05fSDeng-Cheng Zhu 118d7d5b05fSDeng-Cheng Zhu int kvm_mips_guest_tlb_lookup(struct kvm_vcpu *vcpu, unsigned long entryhi) 119d7d5b05fSDeng-Cheng Zhu { 120d7d5b05fSDeng-Cheng Zhu int i; 121d7d5b05fSDeng-Cheng Zhu int index = -1; 122d7d5b05fSDeng-Cheng Zhu struct kvm_mips_tlb *tlb = vcpu->arch.guest_tlb; 123d7d5b05fSDeng-Cheng Zhu 124d7d5b05fSDeng-Cheng Zhu for (i = 0; i < KVM_MIPS_GUEST_TLB_SIZE; i++) { 125d7d5b05fSDeng-Cheng Zhu if (TLB_HI_VPN2_HIT(tlb[i], entryhi) && 126d7d5b05fSDeng-Cheng Zhu TLB_HI_ASID_HIT(tlb[i], entryhi)) { 127d7d5b05fSDeng-Cheng Zhu index = i; 128d7d5b05fSDeng-Cheng Zhu break; 129d7d5b05fSDeng-Cheng Zhu } 130d7d5b05fSDeng-Cheng Zhu } 131d7d5b05fSDeng-Cheng Zhu 132d7d5b05fSDeng-Cheng Zhu kvm_debug("%s: entryhi: %#lx, index: %d lo0: %#lx, lo1: %#lx\n", 1339fbfb06aSJames Hogan __func__, entryhi, index, tlb[i].tlb_lo[0], tlb[i].tlb_lo[1]); 134d7d5b05fSDeng-Cheng Zhu 135d7d5b05fSDeng-Cheng Zhu return index; 136d7d5b05fSDeng-Cheng Zhu } 137cb1b447fSJames Hogan EXPORT_SYMBOL_GPL(kvm_mips_guest_tlb_lookup); 138d7d5b05fSDeng-Cheng Zhu 13957e3869cSJames Hogan static int _kvm_mips_host_tlb_inv(unsigned long entryhi) 140d7d5b05fSDeng-Cheng Zhu { 141d7d5b05fSDeng-Cheng Zhu int idx; 142d7d5b05fSDeng-Cheng Zhu 14357e3869cSJames Hogan write_c0_entryhi(entryhi); 144d7d5b05fSDeng-Cheng Zhu mtc0_tlbw_hazard(); 145d7d5b05fSDeng-Cheng Zhu 146d7d5b05fSDeng-Cheng Zhu tlb_probe(); 147d7d5b05fSDeng-Cheng Zhu tlb_probe_hazard(); 148d7d5b05fSDeng-Cheng Zhu idx = read_c0_index(); 149d7d5b05fSDeng-Cheng Zhu 150d7d5b05fSDeng-Cheng Zhu if (idx >= current_cpu_data.tlbsize) 151d7d5b05fSDeng-Cheng Zhu BUG(); 152d7d5b05fSDeng-Cheng Zhu 153f3a8603fSJames Hogan if (idx >= 0) { 154d7d5b05fSDeng-Cheng Zhu write_c0_entryhi(UNIQUE_ENTRYHI(idx)); 155d7d5b05fSDeng-Cheng Zhu write_c0_entrylo0(0); 156d7d5b05fSDeng-Cheng Zhu write_c0_entrylo1(0); 157d7d5b05fSDeng-Cheng Zhu mtc0_tlbw_hazard(); 158d7d5b05fSDeng-Cheng Zhu 159d7d5b05fSDeng-Cheng Zhu tlb_write_indexed(); 160138f7ad9SJames Hogan tlbw_use_hazard(); 161d7d5b05fSDeng-Cheng Zhu } 162d7d5b05fSDeng-Cheng Zhu 16357e3869cSJames Hogan return idx; 16457e3869cSJames Hogan } 16557e3869cSJames Hogan 16657e3869cSJames Hogan int kvm_mips_host_tlb_inv(struct kvm_vcpu *vcpu, unsigned long va, 16757e3869cSJames Hogan bool user, bool kernel) 16857e3869cSJames Hogan { 16957e3869cSJames Hogan int idx_user, idx_kernel; 17057e3869cSJames Hogan unsigned long flags, old_entryhi; 17157e3869cSJames Hogan 17257e3869cSJames Hogan local_irq_save(flags); 17357e3869cSJames Hogan 17457e3869cSJames Hogan old_entryhi = read_c0_entryhi(); 17557e3869cSJames Hogan 17657e3869cSJames Hogan if (user) 17757e3869cSJames Hogan idx_user = _kvm_mips_host_tlb_inv((va & VPN2_MASK) | 17857e3869cSJames Hogan kvm_mips_get_user_asid(vcpu)); 17957e3869cSJames Hogan if (kernel) 18057e3869cSJames Hogan idx_kernel = _kvm_mips_host_tlb_inv((va & VPN2_MASK) | 18157e3869cSJames Hogan kvm_mips_get_kernel_asid(vcpu)); 18257e3869cSJames Hogan 183d7d5b05fSDeng-Cheng Zhu write_c0_entryhi(old_entryhi); 184d7d5b05fSDeng-Cheng Zhu mtc0_tlbw_hazard(); 185d7d5b05fSDeng-Cheng Zhu 186d7d5b05fSDeng-Cheng Zhu local_irq_restore(flags); 187d7d5b05fSDeng-Cheng Zhu 188*1c506c9cSJames Hogan /* 189*1c506c9cSJames Hogan * We don't want to get reserved instruction exceptions for missing tlb 190*1c506c9cSJames Hogan * entries. 191*1c506c9cSJames Hogan */ 192*1c506c9cSJames Hogan if (cpu_has_vtag_icache) 193*1c506c9cSJames Hogan flush_icache_all(); 194*1c506c9cSJames Hogan 19557e3869cSJames Hogan if (user && idx_user >= 0) 19657e3869cSJames Hogan kvm_debug("%s: Invalidated guest user entryhi %#lx @ idx %d\n", 19757e3869cSJames Hogan __func__, (va & VPN2_MASK) | 19857e3869cSJames Hogan kvm_mips_get_user_asid(vcpu), idx_user); 19957e3869cSJames Hogan if (kernel && idx_kernel >= 0) 20057e3869cSJames Hogan kvm_debug("%s: Invalidated guest kernel entryhi %#lx @ idx %d\n", 20157e3869cSJames Hogan __func__, (va & VPN2_MASK) | 20257e3869cSJames Hogan kvm_mips_get_kernel_asid(vcpu), idx_kernel); 203d7d5b05fSDeng-Cheng Zhu 204d7d5b05fSDeng-Cheng Zhu return 0; 205d7d5b05fSDeng-Cheng Zhu } 206cb1b447fSJames Hogan EXPORT_SYMBOL_GPL(kvm_mips_host_tlb_inv); 207d7d5b05fSDeng-Cheng Zhu 208372582a6SJames Hogan #ifdef CONFIG_KVM_MIPS_VZ 209372582a6SJames Hogan 210372582a6SJames Hogan /* GuestID management */ 211372582a6SJames Hogan 212372582a6SJames Hogan /** 213372582a6SJames Hogan * clear_root_gid() - Set GuestCtl1.RID for normal root operation. 214372582a6SJames Hogan */ 215372582a6SJames Hogan static inline void clear_root_gid(void) 216372582a6SJames Hogan { 217372582a6SJames Hogan if (cpu_has_guestid) { 218372582a6SJames Hogan clear_c0_guestctl1(MIPS_GCTL1_RID); 219372582a6SJames Hogan mtc0_tlbw_hazard(); 220372582a6SJames Hogan } 221372582a6SJames Hogan } 222372582a6SJames Hogan 223372582a6SJames Hogan /** 224372582a6SJames Hogan * set_root_gid_to_guest_gid() - Set GuestCtl1.RID to match GuestCtl1.ID. 225372582a6SJames Hogan * 226372582a6SJames Hogan * Sets the root GuestID to match the current guest GuestID, for TLB operation 227372582a6SJames Hogan * on the GPA->RPA mappings in the root TLB. 228372582a6SJames Hogan * 229372582a6SJames Hogan * The caller must be sure to disable HTW while the root GID is set, and 230372582a6SJames Hogan * possibly longer if TLB registers are modified. 231372582a6SJames Hogan */ 232372582a6SJames Hogan static inline void set_root_gid_to_guest_gid(void) 233372582a6SJames Hogan { 234372582a6SJames Hogan unsigned int guestctl1; 235372582a6SJames Hogan 236372582a6SJames Hogan if (cpu_has_guestid) { 237372582a6SJames Hogan back_to_back_c0_hazard(); 238372582a6SJames Hogan guestctl1 = read_c0_guestctl1(); 239372582a6SJames Hogan guestctl1 = (guestctl1 & ~MIPS_GCTL1_RID) | 240372582a6SJames Hogan ((guestctl1 & MIPS_GCTL1_ID) >> MIPS_GCTL1_ID_SHIFT) 241372582a6SJames Hogan << MIPS_GCTL1_RID_SHIFT; 242372582a6SJames Hogan write_c0_guestctl1(guestctl1); 243372582a6SJames Hogan mtc0_tlbw_hazard(); 244372582a6SJames Hogan } 245372582a6SJames Hogan } 246372582a6SJames Hogan 247372582a6SJames Hogan int kvm_vz_host_tlb_inv(struct kvm_vcpu *vcpu, unsigned long va) 248372582a6SJames Hogan { 249372582a6SJames Hogan int idx; 250372582a6SJames Hogan unsigned long flags, old_entryhi; 251372582a6SJames Hogan 252372582a6SJames Hogan local_irq_save(flags); 253372582a6SJames Hogan htw_stop(); 254372582a6SJames Hogan 255372582a6SJames Hogan /* Set root GuestID for root probe and write of guest TLB entry */ 256372582a6SJames Hogan set_root_gid_to_guest_gid(); 257372582a6SJames Hogan 258372582a6SJames Hogan old_entryhi = read_c0_entryhi(); 259372582a6SJames Hogan 260372582a6SJames Hogan idx = _kvm_mips_host_tlb_inv((va & VPN2_MASK) | 261372582a6SJames Hogan kvm_mips_get_root_asid(vcpu)); 262372582a6SJames Hogan 263372582a6SJames Hogan write_c0_entryhi(old_entryhi); 264372582a6SJames Hogan clear_root_gid(); 265372582a6SJames Hogan mtc0_tlbw_hazard(); 266372582a6SJames Hogan 267372582a6SJames Hogan htw_start(); 268372582a6SJames Hogan local_irq_restore(flags); 269372582a6SJames Hogan 270*1c506c9cSJames Hogan /* 271*1c506c9cSJames Hogan * We don't want to get reserved instruction exceptions for missing tlb 272*1c506c9cSJames Hogan * entries. 273*1c506c9cSJames Hogan */ 274*1c506c9cSJames Hogan if (cpu_has_vtag_icache) 275*1c506c9cSJames Hogan flush_icache_all(); 276*1c506c9cSJames Hogan 277372582a6SJames Hogan if (idx > 0) 278372582a6SJames Hogan kvm_debug("%s: Invalidated root entryhi %#lx @ idx %d\n", 279372582a6SJames Hogan __func__, (va & VPN2_MASK) | 280372582a6SJames Hogan kvm_mips_get_root_asid(vcpu), idx); 281372582a6SJames Hogan 282372582a6SJames Hogan return 0; 283372582a6SJames Hogan } 284372582a6SJames Hogan EXPORT_SYMBOL_GPL(kvm_vz_host_tlb_inv); 285372582a6SJames Hogan 286372582a6SJames Hogan /** 287372582a6SJames Hogan * kvm_vz_guest_tlb_lookup() - Lookup a guest VZ TLB mapping. 288372582a6SJames Hogan * @vcpu: KVM VCPU pointer. 289372582a6SJames Hogan * @gpa: Guest virtual address in a TLB mapped guest segment. 290372582a6SJames Hogan * @gpa: Ponter to output guest physical address it maps to. 291372582a6SJames Hogan * 292372582a6SJames Hogan * Converts a guest virtual address in a guest TLB mapped segment to a guest 293372582a6SJames Hogan * physical address, by probing the guest TLB. 294372582a6SJames Hogan * 295372582a6SJames Hogan * Returns: 0 if guest TLB mapping exists for @gva. *@gpa will have been 296372582a6SJames Hogan * written. 297372582a6SJames Hogan * -EFAULT if no guest TLB mapping exists for @gva. *@gpa may not 298372582a6SJames Hogan * have been written. 299372582a6SJames Hogan */ 300372582a6SJames Hogan int kvm_vz_guest_tlb_lookup(struct kvm_vcpu *vcpu, unsigned long gva, 301372582a6SJames Hogan unsigned long *gpa) 302372582a6SJames Hogan { 303372582a6SJames Hogan unsigned long o_entryhi, o_entrylo[2], o_pagemask; 304372582a6SJames Hogan unsigned int o_index; 305372582a6SJames Hogan unsigned long entrylo[2], pagemask, pagemaskbit, pa; 306372582a6SJames Hogan unsigned long flags; 307372582a6SJames Hogan int index; 308372582a6SJames Hogan 309372582a6SJames Hogan /* Probe the guest TLB for a mapping */ 310372582a6SJames Hogan local_irq_save(flags); 311372582a6SJames Hogan /* Set root GuestID for root probe of guest TLB entry */ 312372582a6SJames Hogan htw_stop(); 313372582a6SJames Hogan set_root_gid_to_guest_gid(); 314372582a6SJames Hogan 315372582a6SJames Hogan o_entryhi = read_gc0_entryhi(); 316372582a6SJames Hogan o_index = read_gc0_index(); 317372582a6SJames Hogan 318372582a6SJames Hogan write_gc0_entryhi((o_entryhi & 0x3ff) | (gva & ~0xfffl)); 319372582a6SJames Hogan mtc0_tlbw_hazard(); 320372582a6SJames Hogan guest_tlb_probe(); 321372582a6SJames Hogan tlb_probe_hazard(); 322372582a6SJames Hogan 323372582a6SJames Hogan index = read_gc0_index(); 324372582a6SJames Hogan if (index < 0) { 325372582a6SJames Hogan /* No match, fail */ 326372582a6SJames Hogan write_gc0_entryhi(o_entryhi); 327372582a6SJames Hogan write_gc0_index(o_index); 328372582a6SJames Hogan 329372582a6SJames Hogan clear_root_gid(); 330372582a6SJames Hogan htw_start(); 331372582a6SJames Hogan local_irq_restore(flags); 332372582a6SJames Hogan return -EFAULT; 333372582a6SJames Hogan } 334372582a6SJames Hogan 335372582a6SJames Hogan /* Match! read the TLB entry */ 336372582a6SJames Hogan o_entrylo[0] = read_gc0_entrylo0(); 337372582a6SJames Hogan o_entrylo[1] = read_gc0_entrylo1(); 338372582a6SJames Hogan o_pagemask = read_gc0_pagemask(); 339372582a6SJames Hogan 340372582a6SJames Hogan mtc0_tlbr_hazard(); 341372582a6SJames Hogan guest_tlb_read(); 342372582a6SJames Hogan tlb_read_hazard(); 343372582a6SJames Hogan 344372582a6SJames Hogan entrylo[0] = read_gc0_entrylo0(); 345372582a6SJames Hogan entrylo[1] = read_gc0_entrylo1(); 346372582a6SJames Hogan pagemask = ~read_gc0_pagemask() & ~0x1fffl; 347372582a6SJames Hogan 348372582a6SJames Hogan write_gc0_entryhi(o_entryhi); 349372582a6SJames Hogan write_gc0_index(o_index); 350372582a6SJames Hogan write_gc0_entrylo0(o_entrylo[0]); 351372582a6SJames Hogan write_gc0_entrylo1(o_entrylo[1]); 352372582a6SJames Hogan write_gc0_pagemask(o_pagemask); 353372582a6SJames Hogan 354372582a6SJames Hogan clear_root_gid(); 355372582a6SJames Hogan htw_start(); 356372582a6SJames Hogan local_irq_restore(flags); 357372582a6SJames Hogan 358372582a6SJames Hogan /* Select one of the EntryLo values and interpret the GPA */ 359372582a6SJames Hogan pagemaskbit = (pagemask ^ (pagemask & (pagemask - 1))) >> 1; 360372582a6SJames Hogan pa = entrylo[!!(gva & pagemaskbit)]; 361372582a6SJames Hogan 362372582a6SJames Hogan /* 363372582a6SJames Hogan * TLB entry may have become invalid since TLB probe if physical FTLB 364372582a6SJames Hogan * entries are shared between threads (e.g. I6400). 365372582a6SJames Hogan */ 366372582a6SJames Hogan if (!(pa & ENTRYLO_V)) 367372582a6SJames Hogan return -EFAULT; 368372582a6SJames Hogan 369372582a6SJames Hogan /* 370372582a6SJames Hogan * Note, this doesn't take guest MIPS32 XPA into account, where PFN is 371372582a6SJames Hogan * split with XI/RI in the middle. 372372582a6SJames Hogan */ 373372582a6SJames Hogan pa = (pa << 6) & ~0xfffl; 374372582a6SJames Hogan pa |= gva & ~(pagemask | pagemaskbit); 375372582a6SJames Hogan 376372582a6SJames Hogan *gpa = pa; 377372582a6SJames Hogan return 0; 378372582a6SJames Hogan } 379372582a6SJames Hogan EXPORT_SYMBOL_GPL(kvm_vz_guest_tlb_lookup); 380372582a6SJames Hogan 381372582a6SJames Hogan /** 382372582a6SJames Hogan * kvm_vz_local_flush_roottlb_all_guests() - Flush all root TLB entries for 383372582a6SJames Hogan * guests. 384372582a6SJames Hogan * 385372582a6SJames Hogan * Invalidate all entries in root tlb which are GPA mappings. 386372582a6SJames Hogan */ 387372582a6SJames Hogan void kvm_vz_local_flush_roottlb_all_guests(void) 388372582a6SJames Hogan { 389372582a6SJames Hogan unsigned long flags; 390372582a6SJames Hogan unsigned long old_entryhi, old_pagemask, old_guestctl1; 391372582a6SJames Hogan int entry; 392372582a6SJames Hogan 393372582a6SJames Hogan if (WARN_ON(!cpu_has_guestid)) 394372582a6SJames Hogan return; 395372582a6SJames Hogan 396372582a6SJames Hogan local_irq_save(flags); 397372582a6SJames Hogan htw_stop(); 398372582a6SJames Hogan 399372582a6SJames Hogan /* TLBR may clobber EntryHi.ASID, PageMask, and GuestCtl1.RID */ 400372582a6SJames Hogan old_entryhi = read_c0_entryhi(); 401372582a6SJames Hogan old_pagemask = read_c0_pagemask(); 402372582a6SJames Hogan old_guestctl1 = read_c0_guestctl1(); 403372582a6SJames Hogan 404372582a6SJames Hogan /* 405372582a6SJames Hogan * Invalidate guest entries in root TLB while leaving root entries 406372582a6SJames Hogan * intact when possible. 407372582a6SJames Hogan */ 408372582a6SJames Hogan for (entry = 0; entry < current_cpu_data.tlbsize; entry++) { 409372582a6SJames Hogan write_c0_index(entry); 410372582a6SJames Hogan mtc0_tlbw_hazard(); 411372582a6SJames Hogan tlb_read(); 412372582a6SJames Hogan tlb_read_hazard(); 413372582a6SJames Hogan 414372582a6SJames Hogan /* Don't invalidate non-guest (RVA) mappings in the root TLB */ 415372582a6SJames Hogan if (!(read_c0_guestctl1() & MIPS_GCTL1_RID)) 416372582a6SJames Hogan continue; 417372582a6SJames Hogan 418372582a6SJames Hogan /* Make sure all entries differ. */ 419372582a6SJames Hogan write_c0_entryhi(UNIQUE_ENTRYHI(entry)); 420372582a6SJames Hogan write_c0_entrylo0(0); 421372582a6SJames Hogan write_c0_entrylo1(0); 422372582a6SJames Hogan write_c0_guestctl1(0); 423372582a6SJames Hogan mtc0_tlbw_hazard(); 424372582a6SJames Hogan tlb_write_indexed(); 425372582a6SJames Hogan } 426372582a6SJames Hogan 427372582a6SJames Hogan write_c0_entryhi(old_entryhi); 428372582a6SJames Hogan write_c0_pagemask(old_pagemask); 429372582a6SJames Hogan write_c0_guestctl1(old_guestctl1); 430372582a6SJames Hogan tlbw_use_hazard(); 431372582a6SJames Hogan 432372582a6SJames Hogan htw_start(); 433372582a6SJames Hogan local_irq_restore(flags); 434372582a6SJames Hogan } 435372582a6SJames Hogan EXPORT_SYMBOL_GPL(kvm_vz_local_flush_roottlb_all_guests); 436372582a6SJames Hogan 437372582a6SJames Hogan /** 438372582a6SJames Hogan * kvm_vz_local_flush_guesttlb_all() - Flush all guest TLB entries. 439372582a6SJames Hogan * 440372582a6SJames Hogan * Invalidate all entries in guest tlb irrespective of guestid. 441372582a6SJames Hogan */ 442372582a6SJames Hogan void kvm_vz_local_flush_guesttlb_all(void) 443372582a6SJames Hogan { 444372582a6SJames Hogan unsigned long flags; 445372582a6SJames Hogan unsigned long old_index; 446372582a6SJames Hogan unsigned long old_entryhi; 447372582a6SJames Hogan unsigned long old_entrylo[2]; 448372582a6SJames Hogan unsigned long old_pagemask; 449372582a6SJames Hogan int entry; 450372582a6SJames Hogan 451372582a6SJames Hogan local_irq_save(flags); 452372582a6SJames Hogan 453372582a6SJames Hogan /* Preserve all clobbered guest registers */ 454372582a6SJames Hogan old_index = read_gc0_index(); 455372582a6SJames Hogan old_entryhi = read_gc0_entryhi(); 456372582a6SJames Hogan old_entrylo[0] = read_gc0_entrylo0(); 457372582a6SJames Hogan old_entrylo[1] = read_gc0_entrylo1(); 458372582a6SJames Hogan old_pagemask = read_gc0_pagemask(); 459372582a6SJames Hogan 460372582a6SJames Hogan /* Invalidate guest entries in guest TLB */ 461372582a6SJames Hogan write_gc0_entrylo0(0); 462372582a6SJames Hogan write_gc0_entrylo1(0); 463372582a6SJames Hogan write_gc0_pagemask(0); 464372582a6SJames Hogan for (entry = 0; entry < current_cpu_data.guest.tlbsize; entry++) { 465372582a6SJames Hogan /* Make sure all entries differ. */ 466372582a6SJames Hogan write_gc0_index(entry); 467372582a6SJames Hogan write_gc0_entryhi(UNIQUE_GUEST_ENTRYHI(entry)); 468372582a6SJames Hogan mtc0_tlbw_hazard(); 469372582a6SJames Hogan guest_tlb_write_indexed(); 470372582a6SJames Hogan } 471372582a6SJames Hogan write_gc0_index(old_index); 472372582a6SJames Hogan write_gc0_entryhi(old_entryhi); 473372582a6SJames Hogan write_gc0_entrylo0(old_entrylo[0]); 474372582a6SJames Hogan write_gc0_entrylo1(old_entrylo[1]); 475372582a6SJames Hogan write_gc0_pagemask(old_pagemask); 476372582a6SJames Hogan tlbw_use_hazard(); 477372582a6SJames Hogan 478372582a6SJames Hogan local_irq_restore(flags); 479372582a6SJames Hogan } 480372582a6SJames Hogan EXPORT_SYMBOL_GPL(kvm_vz_local_flush_guesttlb_all); 481372582a6SJames Hogan 482372582a6SJames Hogan /** 483372582a6SJames Hogan * kvm_vz_save_guesttlb() - Save a range of guest TLB entries. 484372582a6SJames Hogan * @buf: Buffer to write TLB entries into. 485372582a6SJames Hogan * @index: Start index. 486372582a6SJames Hogan * @count: Number of entries to save. 487372582a6SJames Hogan * 488372582a6SJames Hogan * Save a range of guest TLB entries. The caller must ensure interrupts are 489372582a6SJames Hogan * disabled. 490372582a6SJames Hogan */ 491372582a6SJames Hogan void kvm_vz_save_guesttlb(struct kvm_mips_tlb *buf, unsigned int index, 492372582a6SJames Hogan unsigned int count) 493372582a6SJames Hogan { 494372582a6SJames Hogan unsigned int end = index + count; 495372582a6SJames Hogan unsigned long old_entryhi, old_entrylo0, old_entrylo1, old_pagemask; 496372582a6SJames Hogan unsigned int guestctl1 = 0; 497372582a6SJames Hogan int old_index, i; 498372582a6SJames Hogan 499372582a6SJames Hogan /* Save registers we're about to clobber */ 500372582a6SJames Hogan old_index = read_gc0_index(); 501372582a6SJames Hogan old_entryhi = read_gc0_entryhi(); 502372582a6SJames Hogan old_entrylo0 = read_gc0_entrylo0(); 503372582a6SJames Hogan old_entrylo1 = read_gc0_entrylo1(); 504372582a6SJames Hogan old_pagemask = read_gc0_pagemask(); 505372582a6SJames Hogan 506372582a6SJames Hogan /* Set root GuestID for root probe */ 507372582a6SJames Hogan htw_stop(); 508372582a6SJames Hogan set_root_gid_to_guest_gid(); 509372582a6SJames Hogan if (cpu_has_guestid) 510372582a6SJames Hogan guestctl1 = read_c0_guestctl1(); 511372582a6SJames Hogan 512372582a6SJames Hogan /* Read each entry from guest TLB */ 513372582a6SJames Hogan for (i = index; i < end; ++i, ++buf) { 514372582a6SJames Hogan write_gc0_index(i); 515372582a6SJames Hogan 516372582a6SJames Hogan mtc0_tlbr_hazard(); 517372582a6SJames Hogan guest_tlb_read(); 518372582a6SJames Hogan tlb_read_hazard(); 519372582a6SJames Hogan 520372582a6SJames Hogan if (cpu_has_guestid && 521372582a6SJames Hogan (read_c0_guestctl1() ^ guestctl1) & MIPS_GCTL1_RID) { 522372582a6SJames Hogan /* Entry invalid or belongs to another guest */ 523372582a6SJames Hogan buf->tlb_hi = UNIQUE_GUEST_ENTRYHI(i); 524372582a6SJames Hogan buf->tlb_lo[0] = 0; 525372582a6SJames Hogan buf->tlb_lo[1] = 0; 526372582a6SJames Hogan buf->tlb_mask = 0; 527372582a6SJames Hogan } else { 528372582a6SJames Hogan /* Entry belongs to the right guest */ 529372582a6SJames Hogan buf->tlb_hi = read_gc0_entryhi(); 530372582a6SJames Hogan buf->tlb_lo[0] = read_gc0_entrylo0(); 531372582a6SJames Hogan buf->tlb_lo[1] = read_gc0_entrylo1(); 532372582a6SJames Hogan buf->tlb_mask = read_gc0_pagemask(); 533372582a6SJames Hogan } 534372582a6SJames Hogan } 535372582a6SJames Hogan 536372582a6SJames Hogan /* Clear root GuestID again */ 537372582a6SJames Hogan clear_root_gid(); 538372582a6SJames Hogan htw_start(); 539372582a6SJames Hogan 540372582a6SJames Hogan /* Restore clobbered registers */ 541372582a6SJames Hogan write_gc0_index(old_index); 542372582a6SJames Hogan write_gc0_entryhi(old_entryhi); 543372582a6SJames Hogan write_gc0_entrylo0(old_entrylo0); 544372582a6SJames Hogan write_gc0_entrylo1(old_entrylo1); 545372582a6SJames Hogan write_gc0_pagemask(old_pagemask); 546372582a6SJames Hogan 547372582a6SJames Hogan tlbw_use_hazard(); 548372582a6SJames Hogan } 549372582a6SJames Hogan EXPORT_SYMBOL_GPL(kvm_vz_save_guesttlb); 550372582a6SJames Hogan 551372582a6SJames Hogan /** 552372582a6SJames Hogan * kvm_vz_load_guesttlb() - Save a range of guest TLB entries. 553372582a6SJames Hogan * @buf: Buffer to read TLB entries from. 554372582a6SJames Hogan * @index: Start index. 555372582a6SJames Hogan * @count: Number of entries to load. 556372582a6SJames Hogan * 557372582a6SJames Hogan * Load a range of guest TLB entries. The caller must ensure interrupts are 558372582a6SJames Hogan * disabled. 559372582a6SJames Hogan */ 560372582a6SJames Hogan void kvm_vz_load_guesttlb(const struct kvm_mips_tlb *buf, unsigned int index, 561372582a6SJames Hogan unsigned int count) 562372582a6SJames Hogan { 563372582a6SJames Hogan unsigned int end = index + count; 564372582a6SJames Hogan unsigned long old_entryhi, old_entrylo0, old_entrylo1, old_pagemask; 565372582a6SJames Hogan int old_index, i; 566372582a6SJames Hogan 567372582a6SJames Hogan /* Save registers we're about to clobber */ 568372582a6SJames Hogan old_index = read_gc0_index(); 569372582a6SJames Hogan old_entryhi = read_gc0_entryhi(); 570372582a6SJames Hogan old_entrylo0 = read_gc0_entrylo0(); 571372582a6SJames Hogan old_entrylo1 = read_gc0_entrylo1(); 572372582a6SJames Hogan old_pagemask = read_gc0_pagemask(); 573372582a6SJames Hogan 574372582a6SJames Hogan /* Set root GuestID for root probe */ 575372582a6SJames Hogan htw_stop(); 576372582a6SJames Hogan set_root_gid_to_guest_gid(); 577372582a6SJames Hogan 578372582a6SJames Hogan /* Write each entry to guest TLB */ 579372582a6SJames Hogan for (i = index; i < end; ++i, ++buf) { 580372582a6SJames Hogan write_gc0_index(i); 581372582a6SJames Hogan write_gc0_entryhi(buf->tlb_hi); 582372582a6SJames Hogan write_gc0_entrylo0(buf->tlb_lo[0]); 583372582a6SJames Hogan write_gc0_entrylo1(buf->tlb_lo[1]); 584372582a6SJames Hogan write_gc0_pagemask(buf->tlb_mask); 585372582a6SJames Hogan 586372582a6SJames Hogan mtc0_tlbw_hazard(); 587372582a6SJames Hogan guest_tlb_write_indexed(); 588372582a6SJames Hogan } 589372582a6SJames Hogan 590372582a6SJames Hogan /* Clear root GuestID again */ 591372582a6SJames Hogan clear_root_gid(); 592372582a6SJames Hogan htw_start(); 593372582a6SJames Hogan 594372582a6SJames Hogan /* Restore clobbered registers */ 595372582a6SJames Hogan write_gc0_index(old_index); 596372582a6SJames Hogan write_gc0_entryhi(old_entryhi); 597372582a6SJames Hogan write_gc0_entrylo0(old_entrylo0); 598372582a6SJames Hogan write_gc0_entrylo1(old_entrylo1); 599372582a6SJames Hogan write_gc0_pagemask(old_pagemask); 600372582a6SJames Hogan 601372582a6SJames Hogan tlbw_use_hazard(); 602372582a6SJames Hogan } 603372582a6SJames Hogan EXPORT_SYMBOL_GPL(kvm_vz_load_guesttlb); 604372582a6SJames Hogan 605372582a6SJames Hogan #endif 606372582a6SJames Hogan 607a7ebb2e4SJames Hogan /** 608a7ebb2e4SJames Hogan * kvm_mips_suspend_mm() - Suspend the active mm. 609a7ebb2e4SJames Hogan * @cpu The CPU we're running on. 610a7ebb2e4SJames Hogan * 611a7ebb2e4SJames Hogan * Suspend the active_mm, ready for a switch to a KVM guest virtual address 612a7ebb2e4SJames Hogan * space. This is left active for the duration of guest context, including time 613a7ebb2e4SJames Hogan * with interrupts enabled, so we need to be careful not to confuse e.g. cache 614a7ebb2e4SJames Hogan * management IPIs. 615a7ebb2e4SJames Hogan * 616a7ebb2e4SJames Hogan * kvm_mips_resume_mm() should be called before context switching to a different 617a7ebb2e4SJames Hogan * process so we don't need to worry about reference counting. 618a7ebb2e4SJames Hogan * 619a7ebb2e4SJames Hogan * This needs to be in static kernel code to avoid exporting init_mm. 620a7ebb2e4SJames Hogan */ 621a7ebb2e4SJames Hogan void kvm_mips_suspend_mm(int cpu) 622a7ebb2e4SJames Hogan { 623a7ebb2e4SJames Hogan cpumask_clear_cpu(cpu, mm_cpumask(current->active_mm)); 624a7ebb2e4SJames Hogan current->active_mm = &init_mm; 625a7ebb2e4SJames Hogan } 626a7ebb2e4SJames Hogan EXPORT_SYMBOL_GPL(kvm_mips_suspend_mm); 627a7ebb2e4SJames Hogan 628a7ebb2e4SJames Hogan /** 629a7ebb2e4SJames Hogan * kvm_mips_resume_mm() - Resume the current process mm. 630a7ebb2e4SJames Hogan * @cpu The CPU we're running on. 631a7ebb2e4SJames Hogan * 632a7ebb2e4SJames Hogan * Resume the mm of the current process, after a switch back from a KVM guest 633a7ebb2e4SJames Hogan * virtual address space (see kvm_mips_suspend_mm()). 634a7ebb2e4SJames Hogan */ 635a7ebb2e4SJames Hogan void kvm_mips_resume_mm(int cpu) 636a7ebb2e4SJames Hogan { 637a7ebb2e4SJames Hogan cpumask_set_cpu(cpu, mm_cpumask(current->mm)); 638a7ebb2e4SJames Hogan current->active_mm = current->mm; 639a7ebb2e4SJames Hogan } 640a7ebb2e4SJames Hogan EXPORT_SYMBOL_GPL(kvm_mips_resume_mm); 641