1 /* SPDX-License-Identifier: GPL-2.0 */ 2 3 #ifndef __KVM_FPU_H_ 4 #define __KVM_FPU_H_ 5 6 #include <linux/kvm_host.h> 7 8 #include <trace/events/kvm.h> 9 10 #include <asm/fpu/api.h> 11 12 /* Swap (qemu) user FPU context for the guest FPU context. */ 13 static inline void kvm_load_guest_fpu(struct kvm_vcpu *vcpu) 14 { 15 if (KVM_BUG_ON(vcpu->arch.guest_fpu.fpstate->in_use, vcpu->kvm)) 16 return; 17 18 /* Exclude PKRU, it's restored separately immediately after VM-Exit. */ 19 fpu_swap_kvm_fpstate(&vcpu->arch.guest_fpu, true); 20 trace_kvm_fpu(1); 21 } 22 23 /* When vcpu_run ends, restore user space FPU context. */ 24 static inline void kvm_put_guest_fpu(struct kvm_vcpu *vcpu) 25 { 26 if (KVM_BUG_ON(!vcpu->arch.guest_fpu.fpstate->in_use, vcpu->kvm)) 27 return; 28 29 fpu_swap_kvm_fpstate(&vcpu->arch.guest_fpu, false); 30 ++vcpu->stat.fpu_reload; 31 trace_kvm_fpu(0); 32 } 33 34 typedef u32 __attribute__((vector_size(16))) sse128_t; 35 #define __sse128_u union { sse128_t vec; u64 as_u64[2]; u32 as_u32[4]; } 36 #define sse128_lo(x) ({ __sse128_u t; t.vec = x; t.as_u64[0]; }) 37 #define sse128_hi(x) ({ __sse128_u t; t.vec = x; t.as_u64[1]; }) 38 #define sse128_l0(x) ({ __sse128_u t; t.vec = x; t.as_u32[0]; }) 39 #define sse128_l1(x) ({ __sse128_u t; t.vec = x; t.as_u32[1]; }) 40 #define sse128_l2(x) ({ __sse128_u t; t.vec = x; t.as_u32[2]; }) 41 #define sse128_l3(x) ({ __sse128_u t; t.vec = x; t.as_u32[3]; }) 42 #define sse128(lo, hi) ({ __sse128_u t; t.as_u64[0] = lo; t.as_u64[1] = hi; t.vec; }) 43 44 typedef u32 __attribute__((vector_size(32))) avx256_t; 45 46 static inline void _kvm_read_avx_reg(int reg, avx256_t *data) 47 { 48 switch (reg) { 49 case 0: asm("vmovdqa %%ymm0, %0" : "=m"(*data)); break; 50 case 1: asm("vmovdqa %%ymm1, %0" : "=m"(*data)); break; 51 case 2: asm("vmovdqa %%ymm2, %0" : "=m"(*data)); break; 52 case 3: asm("vmovdqa %%ymm3, %0" : "=m"(*data)); break; 53 case 4: asm("vmovdqa %%ymm4, %0" : "=m"(*data)); break; 54 case 5: asm("vmovdqa %%ymm5, %0" : "=m"(*data)); break; 55 case 6: asm("vmovdqa %%ymm6, %0" : "=m"(*data)); break; 56 case 7: asm("vmovdqa %%ymm7, %0" : "=m"(*data)); break; 57 #ifdef CONFIG_X86_64 58 case 8: asm("vmovdqa %%ymm8, %0" : "=m"(*data)); break; 59 case 9: asm("vmovdqa %%ymm9, %0" : "=m"(*data)); break; 60 case 10: asm("vmovdqa %%ymm10, %0" : "=m"(*data)); break; 61 case 11: asm("vmovdqa %%ymm11, %0" : "=m"(*data)); break; 62 case 12: asm("vmovdqa %%ymm12, %0" : "=m"(*data)); break; 63 case 13: asm("vmovdqa %%ymm13, %0" : "=m"(*data)); break; 64 case 14: asm("vmovdqa %%ymm14, %0" : "=m"(*data)); break; 65 case 15: asm("vmovdqa %%ymm15, %0" : "=m"(*data)); break; 66 #endif 67 default: BUG(); 68 } 69 } 70 71 static inline void _kvm_write_avx_reg(int reg, const avx256_t *data) 72 { 73 switch (reg) { 74 case 0: asm("vmovdqa %0, %%ymm0" : : "m"(*data)); break; 75 case 1: asm("vmovdqa %0, %%ymm1" : : "m"(*data)); break; 76 case 2: asm("vmovdqa %0, %%ymm2" : : "m"(*data)); break; 77 case 3: asm("vmovdqa %0, %%ymm3" : : "m"(*data)); break; 78 case 4: asm("vmovdqa %0, %%ymm4" : : "m"(*data)); break; 79 case 5: asm("vmovdqa %0, %%ymm5" : : "m"(*data)); break; 80 case 6: asm("vmovdqa %0, %%ymm6" : : "m"(*data)); break; 81 case 7: asm("vmovdqa %0, %%ymm7" : : "m"(*data)); break; 82 #ifdef CONFIG_X86_64 83 case 8: asm("vmovdqa %0, %%ymm8" : : "m"(*data)); break; 84 case 9: asm("vmovdqa %0, %%ymm9" : : "m"(*data)); break; 85 case 10: asm("vmovdqa %0, %%ymm10" : : "m"(*data)); break; 86 case 11: asm("vmovdqa %0, %%ymm11" : : "m"(*data)); break; 87 case 12: asm("vmovdqa %0, %%ymm12" : : "m"(*data)); break; 88 case 13: asm("vmovdqa %0, %%ymm13" : : "m"(*data)); break; 89 case 14: asm("vmovdqa %0, %%ymm14" : : "m"(*data)); break; 90 case 15: asm("vmovdqa %0, %%ymm15" : : "m"(*data)); break; 91 #endif 92 default: BUG(); 93 } 94 } 95 96 static inline void _kvm_read_sse_reg(int reg, sse128_t *data) 97 { 98 switch (reg) { 99 case 0: asm("movdqa %%xmm0, %0" : "=m"(*data)); break; 100 case 1: asm("movdqa %%xmm1, %0" : "=m"(*data)); break; 101 case 2: asm("movdqa %%xmm2, %0" : "=m"(*data)); break; 102 case 3: asm("movdqa %%xmm3, %0" : "=m"(*data)); break; 103 case 4: asm("movdqa %%xmm4, %0" : "=m"(*data)); break; 104 case 5: asm("movdqa %%xmm5, %0" : "=m"(*data)); break; 105 case 6: asm("movdqa %%xmm6, %0" : "=m"(*data)); break; 106 case 7: asm("movdqa %%xmm7, %0" : "=m"(*data)); break; 107 #ifdef CONFIG_X86_64 108 case 8: asm("movdqa %%xmm8, %0" : "=m"(*data)); break; 109 case 9: asm("movdqa %%xmm9, %0" : "=m"(*data)); break; 110 case 10: asm("movdqa %%xmm10, %0" : "=m"(*data)); break; 111 case 11: asm("movdqa %%xmm11, %0" : "=m"(*data)); break; 112 case 12: asm("movdqa %%xmm12, %0" : "=m"(*data)); break; 113 case 13: asm("movdqa %%xmm13, %0" : "=m"(*data)); break; 114 case 14: asm("movdqa %%xmm14, %0" : "=m"(*data)); break; 115 case 15: asm("movdqa %%xmm15, %0" : "=m"(*data)); break; 116 #endif 117 default: BUG(); 118 } 119 } 120 121 static inline void _kvm_write_sse_reg(int reg, const sse128_t *data) 122 { 123 switch (reg) { 124 case 0: asm("movdqa %0, %%xmm0" : : "m"(*data)); break; 125 case 1: asm("movdqa %0, %%xmm1" : : "m"(*data)); break; 126 case 2: asm("movdqa %0, %%xmm2" : : "m"(*data)); break; 127 case 3: asm("movdqa %0, %%xmm3" : : "m"(*data)); break; 128 case 4: asm("movdqa %0, %%xmm4" : : "m"(*data)); break; 129 case 5: asm("movdqa %0, %%xmm5" : : "m"(*data)); break; 130 case 6: asm("movdqa %0, %%xmm6" : : "m"(*data)); break; 131 case 7: asm("movdqa %0, %%xmm7" : : "m"(*data)); break; 132 #ifdef CONFIG_X86_64 133 case 8: asm("movdqa %0, %%xmm8" : : "m"(*data)); break; 134 case 9: asm("movdqa %0, %%xmm9" : : "m"(*data)); break; 135 case 10: asm("movdqa %0, %%xmm10" : : "m"(*data)); break; 136 case 11: asm("movdqa %0, %%xmm11" : : "m"(*data)); break; 137 case 12: asm("movdqa %0, %%xmm12" : : "m"(*data)); break; 138 case 13: asm("movdqa %0, %%xmm13" : : "m"(*data)); break; 139 case 14: asm("movdqa %0, %%xmm14" : : "m"(*data)); break; 140 case 15: asm("movdqa %0, %%xmm15" : : "m"(*data)); break; 141 #endif 142 default: BUG(); 143 } 144 } 145 146 static inline void _kvm_read_mmx_reg(int reg, u64 *data) 147 { 148 switch (reg) { 149 case 0: asm("movq %%mm0, %0" : "=m"(*data)); break; 150 case 1: asm("movq %%mm1, %0" : "=m"(*data)); break; 151 case 2: asm("movq %%mm2, %0" : "=m"(*data)); break; 152 case 3: asm("movq %%mm3, %0" : "=m"(*data)); break; 153 case 4: asm("movq %%mm4, %0" : "=m"(*data)); break; 154 case 5: asm("movq %%mm5, %0" : "=m"(*data)); break; 155 case 6: asm("movq %%mm6, %0" : "=m"(*data)); break; 156 case 7: asm("movq %%mm7, %0" : "=m"(*data)); break; 157 default: BUG(); 158 } 159 } 160 161 static inline void _kvm_write_mmx_reg(int reg, const u64 *data) 162 { 163 switch (reg) { 164 case 0: asm("movq %0, %%mm0" : : "m"(*data)); break; 165 case 1: asm("movq %0, %%mm1" : : "m"(*data)); break; 166 case 2: asm("movq %0, %%mm2" : : "m"(*data)); break; 167 case 3: asm("movq %0, %%mm3" : : "m"(*data)); break; 168 case 4: asm("movq %0, %%mm4" : : "m"(*data)); break; 169 case 5: asm("movq %0, %%mm5" : : "m"(*data)); break; 170 case 6: asm("movq %0, %%mm6" : : "m"(*data)); break; 171 case 7: asm("movq %0, %%mm7" : : "m"(*data)); break; 172 default: BUG(); 173 } 174 } 175 176 static inline void kvm_fpu_get(void) 177 { 178 fpregs_lock(); 179 180 fpregs_assert_state_consistent(); 181 if (test_thread_flag(TIF_NEED_FPU_LOAD)) 182 switch_fpu_return(); 183 } 184 185 static inline void kvm_fpu_put(void) 186 { 187 fpregs_unlock(); 188 } 189 190 static inline void kvm_read_avx_reg(int reg, avx256_t *data) 191 { 192 kvm_fpu_get(); 193 _kvm_read_avx_reg(reg, data); 194 kvm_fpu_put(); 195 } 196 197 static inline void kvm_write_avx_reg(int reg, const avx256_t *data) 198 { 199 kvm_fpu_get(); 200 _kvm_write_avx_reg(reg, data); 201 kvm_fpu_put(); 202 } 203 204 static inline void kvm_read_sse_reg(int reg, sse128_t *data) 205 { 206 kvm_fpu_get(); 207 _kvm_read_sse_reg(reg, data); 208 kvm_fpu_put(); 209 } 210 211 static inline void kvm_write_sse_reg(int reg, const sse128_t *data) 212 { 213 kvm_fpu_get(); 214 _kvm_write_sse_reg(reg, data); 215 kvm_fpu_put(); 216 } 217 218 static inline void kvm_read_mmx_reg(int reg, u64 *data) 219 { 220 kvm_fpu_get(); 221 _kvm_read_mmx_reg(reg, data); 222 kvm_fpu_put(); 223 } 224 225 static inline void kvm_write_mmx_reg(int reg, const u64 *data) 226 { 227 kvm_fpu_get(); 228 _kvm_write_mmx_reg(reg, data); 229 kvm_fpu_put(); 230 } 231 232 #endif 233