1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * amx tests
4 *
5 * Copyright (C) 2021, Intel, Inc.
6 *
7 * Tests for amx #NM exception and save/restore.
8 */
9 #include <fcntl.h>
10 #include <stdio.h>
11 #include <stdlib.h>
12 #include <string.h>
13 #include <sys/ioctl.h>
14 #include <sys/syscall.h>
15
16 #include "test_util.h"
17
18 #include "kvm_util.h"
19 #include "processor.h"
20 #include "vmx.h"
21
22 #ifndef __x86_64__
23 # error This test is 64-bit only
24 #endif
25
26 #define NUM_TILES 8
27 #define TILE_SIZE 1024
28 #define XSAVE_SIZE ((NUM_TILES * TILE_SIZE) + PAGE_SIZE)
29
30 /* Tile configuration associated: */
31 #define PALETTE_TABLE_INDEX 1
32 #define MAX_TILES 16
33 #define RESERVED_BYTES 14
34
35 #define XSAVE_HDR_OFFSET 512
36
37 struct tile_config {
38 u8 palette_id;
39 u8 start_row;
40 u8 reserved[RESERVED_BYTES];
41 u16 colsb[MAX_TILES];
42 u8 rows[MAX_TILES];
43 };
44
45 struct tile_data {
46 u8 data[NUM_TILES * TILE_SIZE];
47 };
48
49 struct xtile_info {
50 u16 bytes_per_tile;
51 u16 bytes_per_row;
52 u16 max_names;
53 u16 max_rows;
54 u32 xsave_offset;
55 u32 xsave_size;
56 };
57
58 static struct xtile_info xtile;
59
__ldtilecfg(void * cfg)60 static inline void __ldtilecfg(void *cfg)
61 {
62 asm volatile(".byte 0xc4,0xe2,0x78,0x49,0x00"
63 : : "a"(cfg));
64 }
65
__tileloadd(void * tile)66 static inline void __tileloadd(void *tile)
67 {
68 asm volatile(".byte 0xc4,0xe2,0x7b,0x4b,0x04,0x10"
69 : : "a"(tile), "d"(0));
70 }
71
tileloadd_safe(void * tile)72 static inline int tileloadd_safe(void *tile)
73 {
74 return kvm_asm_safe(".byte 0xc4,0xe2,0x7b,0x4b,0x04,0x10",
75 "a"(tile), "d"(0));
76 }
77
__tilerelease(void)78 static inline void __tilerelease(void)
79 {
80 asm volatile(".byte 0xc4, 0xe2, 0x78, 0x49, 0xc0" ::);
81 }
82
__xsavec(struct xstate * xstate,uint64_t rfbm)83 static inline void __xsavec(struct xstate *xstate, uint64_t rfbm)
84 {
85 uint32_t rfbm_lo = rfbm;
86 uint32_t rfbm_hi = rfbm >> 32;
87
88 asm volatile("xsavec (%%rdi)"
89 : : "D" (xstate), "a" (rfbm_lo), "d" (rfbm_hi)
90 : "memory");
91 }
92
check_xtile_info(void)93 static void check_xtile_info(void)
94 {
95 GUEST_ASSERT((xgetbv(0) & XFEATURE_MASK_XTILE) == XFEATURE_MASK_XTILE);
96
97 GUEST_ASSERT(this_cpu_has_p(X86_PROPERTY_XSTATE_MAX_SIZE_XCR0));
98 GUEST_ASSERT(this_cpu_property(X86_PROPERTY_XSTATE_MAX_SIZE_XCR0) <= XSAVE_SIZE);
99
100 xtile.xsave_offset = this_cpu_property(X86_PROPERTY_XSTATE_TILE_OFFSET);
101 GUEST_ASSERT(xtile.xsave_offset == 2816);
102 xtile.xsave_size = this_cpu_property(X86_PROPERTY_XSTATE_TILE_SIZE);
103 GUEST_ASSERT(xtile.xsave_size == 8192);
104 GUEST_ASSERT(sizeof(struct tile_data) >= xtile.xsave_size);
105
106 GUEST_ASSERT(this_cpu_has_p(X86_PROPERTY_AMX_MAX_PALETTE_TABLES));
107 GUEST_ASSERT(this_cpu_property(X86_PROPERTY_AMX_MAX_PALETTE_TABLES) >=
108 PALETTE_TABLE_INDEX);
109
110 GUEST_ASSERT(this_cpu_has_p(X86_PROPERTY_AMX_NR_TILE_REGS));
111 xtile.max_names = this_cpu_property(X86_PROPERTY_AMX_NR_TILE_REGS);
112 GUEST_ASSERT(xtile.max_names == 8);
113 xtile.bytes_per_tile = this_cpu_property(X86_PROPERTY_AMX_BYTES_PER_TILE);
114 GUEST_ASSERT(xtile.bytes_per_tile == 1024);
115 xtile.bytes_per_row = this_cpu_property(X86_PROPERTY_AMX_BYTES_PER_ROW);
116 GUEST_ASSERT(xtile.bytes_per_row == 64);
117 xtile.max_rows = this_cpu_property(X86_PROPERTY_AMX_MAX_ROWS);
118 GUEST_ASSERT(xtile.max_rows == 16);
119 }
120
set_tilecfg(struct tile_config * cfg)121 static void set_tilecfg(struct tile_config *cfg)
122 {
123 int i;
124
125 /* Only palette id 1 */
126 cfg->palette_id = 1;
127 for (i = 0; i < xtile.max_names; i++) {
128 cfg->colsb[i] = xtile.bytes_per_row;
129 cfg->rows[i] = xtile.max_rows;
130 }
131 }
132
133 enum {
134 /* Retrieve TMM0 from guest, stash it for TEST_RESTORE_TILEDATA */
135 TEST_SAVE_TILEDATA = 1,
136
137 /* Check TMM0 against tiledata */
138 TEST_COMPARE_TILEDATA = 2,
139
140 /* Restore TMM0 from earlier save */
141 TEST_RESTORE_TILEDATA = 4,
142
143 /* Full VM save/restore */
144 TEST_SAVE_RESTORE = 8,
145 };
146
guest_code(struct tile_config * amx_cfg,struct tile_data * tiledata,struct xstate * xstate)147 static void __attribute__((__flatten__)) guest_code(struct tile_config *amx_cfg,
148 struct tile_data *tiledata,
149 struct xstate *xstate)
150 {
151 int vector;
152
153 GUEST_ASSERT(this_cpu_has(X86_FEATURE_XSAVE) &&
154 this_cpu_has(X86_FEATURE_OSXSAVE));
155 check_xtile_info();
156 GUEST_SYNC(TEST_SAVE_RESTORE);
157
158 /* xfd=0, enable amx */
159 wrmsr(MSR_IA32_XFD, 0);
160 GUEST_SYNC(TEST_SAVE_RESTORE);
161 GUEST_ASSERT(rdmsr(MSR_IA32_XFD) == 0);
162 set_tilecfg(amx_cfg);
163 __ldtilecfg(amx_cfg);
164 GUEST_SYNC(TEST_SAVE_RESTORE);
165 /* Check save/restore when trap to userspace */
166 __tileloadd(tiledata);
167 GUEST_SYNC(TEST_SAVE_TILEDATA | TEST_COMPARE_TILEDATA | TEST_SAVE_RESTORE);
168
169 /* xfd=0x40000, disable amx tiledata */
170 wrmsr(MSR_IA32_XFD, XFEATURE_MASK_XTILE_DATA);
171
172 /* host tries setting tiledata while guest XFD is set */
173 GUEST_SYNC(TEST_RESTORE_TILEDATA);
174 GUEST_SYNC(TEST_SAVE_RESTORE);
175
176 wrmsr(MSR_IA32_XFD, 0);
177 __tilerelease();
178 GUEST_SYNC(TEST_SAVE_RESTORE);
179 /*
180 * After XSAVEC, XTILEDATA is cleared in the xstate_bv but is set in
181 * the xcomp_bv.
182 */
183 xstate->header.xstate_bv = XFEATURE_MASK_XTILE_DATA;
184 __xsavec(xstate, XFEATURE_MASK_XTILE_DATA);
185 GUEST_ASSERT(!(xstate->header.xstate_bv & XFEATURE_MASK_XTILE_DATA));
186 GUEST_ASSERT(xstate->header.xcomp_bv & XFEATURE_MASK_XTILE_DATA);
187
188 /* #NM test */
189
190 /* xfd=0x40000, disable amx tiledata */
191 wrmsr(MSR_IA32_XFD, XFEATURE_MASK_XTILE_DATA);
192
193 /*
194 * XTILEDATA is cleared in xstate_bv but set in xcomp_bv, this property
195 * remains the same even when amx tiledata is disabled by IA32_XFD.
196 */
197 xstate->header.xstate_bv = XFEATURE_MASK_XTILE_DATA;
198 __xsavec(xstate, XFEATURE_MASK_XTILE_DATA);
199 GUEST_ASSERT(!(xstate->header.xstate_bv & XFEATURE_MASK_XTILE_DATA));
200 GUEST_ASSERT((xstate->header.xcomp_bv & XFEATURE_MASK_XTILE_DATA));
201
202 GUEST_SYNC(TEST_SAVE_RESTORE);
203 GUEST_ASSERT(rdmsr(MSR_IA32_XFD) == XFEATURE_MASK_XTILE_DATA);
204 set_tilecfg(amx_cfg);
205 __ldtilecfg(amx_cfg);
206
207 /* Trigger #NM exception */
208 vector = tileloadd_safe(tiledata);
209 __GUEST_ASSERT(vector == NM_VECTOR,
210 "Wanted #NM on tileloadd with XFD[18]=1, got %s",
211 ex_str(vector));
212
213 GUEST_ASSERT(!(get_cr0() & X86_CR0_TS));
214 GUEST_ASSERT(rdmsr(MSR_IA32_XFD_ERR) == XFEATURE_MASK_XTILE_DATA);
215 GUEST_ASSERT(rdmsr(MSR_IA32_XFD) == XFEATURE_MASK_XTILE_DATA);
216 GUEST_SYNC(TEST_SAVE_RESTORE);
217 GUEST_ASSERT(rdmsr(MSR_IA32_XFD_ERR) == XFEATURE_MASK_XTILE_DATA);
218 GUEST_ASSERT(rdmsr(MSR_IA32_XFD) == XFEATURE_MASK_XTILE_DATA);
219 /* Clear xfd_err */
220 wrmsr(MSR_IA32_XFD_ERR, 0);
221 /* xfd=0, enable amx */
222 wrmsr(MSR_IA32_XFD, 0);
223 GUEST_SYNC(TEST_SAVE_RESTORE);
224
225 __tileloadd(tiledata);
226 GUEST_SYNC(TEST_COMPARE_TILEDATA | TEST_SAVE_RESTORE);
227
228 GUEST_DONE();
229 }
230
main(int argc,char * argv[])231 int main(int argc, char *argv[])
232 {
233 struct kvm_regs regs1, regs2;
234 struct kvm_vcpu *vcpu;
235 struct kvm_vm *vm;
236 struct kvm_x86_state *state;
237 struct kvm_x86_state *tile_state = NULL;
238 int xsave_restore_size;
239 vm_vaddr_t amx_cfg, tiledata, xstate;
240 struct ucall uc;
241 int ret;
242
243 /*
244 * Note, all off-by-default features must be enabled before anything
245 * caches KVM_GET_SUPPORTED_CPUID, e.g. before using kvm_cpu_has().
246 */
247 vm_xsave_require_permission(XFEATURE_MASK_XTILE_DATA);
248
249 TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_XFD));
250 TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_XSAVE));
251 TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_AMX_TILE));
252 TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_XTILECFG));
253 TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_XTILEDATA));
254 TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_XTILEDATA_XFD));
255
256 /* Create VM */
257 vm = vm_create_with_one_vcpu(&vcpu, guest_code);
258
259 TEST_ASSERT(kvm_cpu_has_p(X86_PROPERTY_XSTATE_MAX_SIZE),
260 "KVM should enumerate max XSAVE size when XSAVE is supported");
261 xsave_restore_size = kvm_cpu_property(X86_PROPERTY_XSTATE_MAX_SIZE);
262
263 vcpu_regs_get(vcpu, ®s1);
264
265 /* amx cfg for guest_code */
266 amx_cfg = vm_vaddr_alloc_page(vm);
267 memset(addr_gva2hva(vm, amx_cfg), 0x0, getpagesize());
268
269 /* amx tiledata for guest_code */
270 tiledata = vm_vaddr_alloc_pages(vm, 2);
271 memset(addr_gva2hva(vm, tiledata), rand() | 1, 2 * getpagesize());
272
273 /* XSAVE state for guest_code */
274 xstate = vm_vaddr_alloc_pages(vm, DIV_ROUND_UP(XSAVE_SIZE, PAGE_SIZE));
275 memset(addr_gva2hva(vm, xstate), 0, PAGE_SIZE * DIV_ROUND_UP(XSAVE_SIZE, PAGE_SIZE));
276 vcpu_args_set(vcpu, 3, amx_cfg, tiledata, xstate);
277
278 int iter = 0;
279 for (;;) {
280 vcpu_run(vcpu);
281 TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_IO);
282
283 switch (get_ucall(vcpu, &uc)) {
284 case UCALL_ABORT:
285 REPORT_GUEST_ASSERT(uc);
286 /* NOT REACHED */
287 case UCALL_SYNC:
288 ++iter;
289 if (uc.args[1] & TEST_SAVE_TILEDATA) {
290 fprintf(stderr, "GUEST_SYNC #%d, save tiledata\n", iter);
291 tile_state = vcpu_save_state(vcpu);
292 }
293 if (uc.args[1] & TEST_COMPARE_TILEDATA) {
294 fprintf(stderr, "GUEST_SYNC #%d, check TMM0 contents\n", iter);
295
296 /* Compacted mode, get amx offset by xsave area
297 * size subtract 8K amx size.
298 */
299 u32 amx_offset = xsave_restore_size - NUM_TILES*TILE_SIZE;
300 void *amx_start = (void *)tile_state->xsave + amx_offset;
301 void *tiles_data = (void *)addr_gva2hva(vm, tiledata);
302 /* Only check TMM0 register, 1 tile */
303 ret = memcmp(amx_start, tiles_data, TILE_SIZE);
304 TEST_ASSERT(ret == 0, "memcmp failed, ret=%d", ret);
305 }
306 if (uc.args[1] & TEST_RESTORE_TILEDATA) {
307 fprintf(stderr, "GUEST_SYNC #%d, before KVM_SET_XSAVE\n", iter);
308 vcpu_xsave_set(vcpu, tile_state->xsave);
309 fprintf(stderr, "GUEST_SYNC #%d, after KVM_SET_XSAVE\n", iter);
310 }
311 if (uc.args[1] & TEST_SAVE_RESTORE) {
312 fprintf(stderr, "GUEST_SYNC #%d, save/restore VM state\n", iter);
313 state = vcpu_save_state(vcpu);
314 memset(®s1, 0, sizeof(regs1));
315 vcpu_regs_get(vcpu, ®s1);
316
317 kvm_vm_release(vm);
318
319 /* Restore state in a new VM. */
320 vcpu = vm_recreate_with_one_vcpu(vm);
321 vcpu_load_state(vcpu, state);
322 kvm_x86_state_cleanup(state);
323
324 memset(®s2, 0, sizeof(regs2));
325 vcpu_regs_get(vcpu, ®s2);
326 TEST_ASSERT(!memcmp(®s1, ®s2, sizeof(regs2)),
327 "Unexpected register values after vcpu_load_state; rdi: %lx rsi: %lx",
328 (ulong) regs2.rdi, (ulong) regs2.rsi);
329 }
330 break;
331 case UCALL_DONE:
332 fprintf(stderr, "UCALL_DONE\n");
333 goto done;
334 default:
335 TEST_FAIL("Unknown ucall %lu", uc.cmd);
336 }
337
338 }
339 done:
340 kvm_vm_free(vm);
341 }
342