1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2025 Intel Corporation
4 */
5
6 #include "xe_bo.h"
7 #include "xe_device.h"
8 #include "xe_guc_klv_helpers.h"
9 #include "xe_sriov_packet.h"
10 #include "xe_sriov_packet_types.h"
11 #include "xe_sriov_pf_helpers.h"
12 #include "xe_sriov_pf_migration.h"
13 #include "xe_sriov_printk.h"
14
pf_migration_mutex(struct xe_device * xe,unsigned int vfid)15 static struct mutex *pf_migration_mutex(struct xe_device *xe, unsigned int vfid)
16 {
17 xe_assert(xe, IS_SRIOV_PF(xe));
18 xe_assert(xe, vfid <= xe_sriov_pf_get_totalvfs(xe));
19
20 return &xe->sriov.pf.vfs[vfid].migration.lock;
21 }
22
pf_pick_pending(struct xe_device * xe,unsigned int vfid)23 static struct xe_sriov_packet **pf_pick_pending(struct xe_device *xe, unsigned int vfid)
24 {
25 xe_assert(xe, IS_SRIOV_PF(xe));
26 xe_assert(xe, vfid <= xe_sriov_pf_get_totalvfs(xe));
27 lockdep_assert_held(pf_migration_mutex(xe, vfid));
28
29 return &xe->sriov.pf.vfs[vfid].migration.pending;
30 }
31
32 static struct xe_sriov_packet **
pf_pick_descriptor(struct xe_device * xe,unsigned int vfid)33 pf_pick_descriptor(struct xe_device *xe, unsigned int vfid)
34 {
35 xe_assert(xe, IS_SRIOV_PF(xe));
36 xe_assert(xe, vfid <= xe_sriov_pf_get_totalvfs(xe));
37 lockdep_assert_held(pf_migration_mutex(xe, vfid));
38
39 return &xe->sriov.pf.vfs[vfid].migration.descriptor;
40 }
41
pf_pick_trailer(struct xe_device * xe,unsigned int vfid)42 static struct xe_sriov_packet **pf_pick_trailer(struct xe_device *xe, unsigned int vfid)
43 {
44 xe_assert(xe, IS_SRIOV_PF(xe));
45 xe_assert(xe, vfid <= xe_sriov_pf_get_totalvfs(xe));
46 lockdep_assert_held(pf_migration_mutex(xe, vfid));
47
48 return &xe->sriov.pf.vfs[vfid].migration.trailer;
49 }
50
pf_pick_read_packet(struct xe_device * xe,unsigned int vfid)51 static struct xe_sriov_packet **pf_pick_read_packet(struct xe_device *xe,
52 unsigned int vfid)
53 {
54 struct xe_sriov_packet **data;
55
56 data = pf_pick_descriptor(xe, vfid);
57 if (*data)
58 return data;
59
60 data = pf_pick_pending(xe, vfid);
61 if (!*data)
62 *data = xe_sriov_pf_migration_save_consume(xe, vfid);
63 if (*data)
64 return data;
65
66 data = pf_pick_trailer(xe, vfid);
67 if (*data)
68 return data;
69
70 return NULL;
71 }
72
pkt_needs_bo(struct xe_sriov_packet * data)73 static bool pkt_needs_bo(struct xe_sriov_packet *data)
74 {
75 return data->hdr.type == XE_SRIOV_PACKET_TYPE_VRAM;
76 }
77
78 /**
79 * xe_sriov_packet_alloc() - Allocate migration data packet
80 * @xe: the &xe_device
81 *
82 * Only allocates the "outer" structure, without initializing the migration
83 * data backing storage.
84 *
85 * Return: Pointer to &xe_sriov_packet on success,
86 * NULL in case of error.
87 */
xe_sriov_packet_alloc(struct xe_device * xe)88 struct xe_sriov_packet *xe_sriov_packet_alloc(struct xe_device *xe)
89 {
90 struct xe_sriov_packet *data;
91
92 data = kzalloc_obj(*data);
93 if (!data)
94 return NULL;
95
96 data->xe = xe;
97 data->hdr_remaining = sizeof(data->hdr);
98
99 return data;
100 }
101
102 /**
103 * xe_sriov_packet_free() - Free migration data packet.
104 * @data: the &xe_sriov_packet
105 */
xe_sriov_packet_free(struct xe_sriov_packet * data)106 void xe_sriov_packet_free(struct xe_sriov_packet *data)
107 {
108 if (IS_ERR_OR_NULL(data))
109 return;
110
111 if (pkt_needs_bo(data))
112 xe_bo_unpin_map_no_vm(data->bo);
113 else
114 kvfree(data->buff);
115
116 kfree(data);
117 }
118
pkt_init(struct xe_sriov_packet * data)119 static int pkt_init(struct xe_sriov_packet *data)
120 {
121 struct xe_gt *gt = xe_device_get_gt(data->xe, data->hdr.gt_id);
122
123 if (!gt)
124 return -EINVAL;
125
126 if (data->hdr.size == 0)
127 return 0;
128
129 if (pkt_needs_bo(data)) {
130 struct xe_bo *bo;
131
132 bo = xe_bo_create_pin_map_novm(data->xe, gt->tile, PAGE_ALIGN(data->hdr.size),
133 ttm_bo_type_kernel,
134 XE_BO_FLAG_SYSTEM | XE_BO_FLAG_PINNED, false);
135 if (IS_ERR(bo))
136 return PTR_ERR(bo);
137
138 data->bo = bo;
139 data->vaddr = bo->vmap.vaddr;
140 } else {
141 void *buff = kvzalloc(data->hdr.size, GFP_KERNEL);
142
143 if (!buff)
144 return -ENOMEM;
145
146 data->buff = buff;
147 data->vaddr = buff;
148 }
149
150 return 0;
151 }
152
153 #define XE_SRIOV_PACKET_SUPPORTED_VERSION 1
154
155 /**
156 * xe_sriov_packet_init() - Initialize migration packet header and backing storage.
157 * @data: the &xe_sriov_packet
158 * @tile_id: tile identifier
159 * @gt_id: GT identifier
160 * @type: &xe_sriov_packet_type
161 * @offset: offset of data packet payload (within wider resource)
162 * @size: size of data packet payload
163 *
164 * Return: 0 on success or a negative error code on failure.
165 */
xe_sriov_packet_init(struct xe_sriov_packet * data,u8 tile_id,u8 gt_id,enum xe_sriov_packet_type type,loff_t offset,size_t size)166 int xe_sriov_packet_init(struct xe_sriov_packet *data, u8 tile_id, u8 gt_id,
167 enum xe_sriov_packet_type type, loff_t offset, size_t size)
168 {
169 data->hdr.version = XE_SRIOV_PACKET_SUPPORTED_VERSION;
170 data->hdr.type = type;
171 data->hdr.tile_id = tile_id;
172 data->hdr.gt_id = gt_id;
173 data->hdr.offset = offset;
174 data->hdr.size = size;
175 data->remaining = size;
176
177 return pkt_init(data);
178 }
179
180 /**
181 * xe_sriov_packet_init_from_hdr() - Initialize migration packet backing storage based on header.
182 * @data: the &xe_sriov_packet
183 *
184 * Header data is expected to be filled prior to calling this function.
185 *
186 * Return: 0 on success or a negative error code on failure.
187 */
xe_sriov_packet_init_from_hdr(struct xe_sriov_packet * data)188 int xe_sriov_packet_init_from_hdr(struct xe_sriov_packet *data)
189 {
190 xe_assert(data->xe, !data->hdr_remaining);
191
192 if (data->hdr.version != XE_SRIOV_PACKET_SUPPORTED_VERSION)
193 return -EINVAL;
194
195 data->remaining = data->hdr.size;
196
197 return pkt_init(data);
198 }
199
pkt_hdr_read(struct xe_sriov_packet * data,char __user * buf,size_t len)200 static ssize_t pkt_hdr_read(struct xe_sriov_packet *data,
201 char __user *buf, size_t len)
202 {
203 loff_t offset = sizeof(data->hdr) - data->hdr_remaining;
204
205 if (!data->hdr_remaining)
206 return -EINVAL;
207
208 if (len > data->hdr_remaining)
209 len = data->hdr_remaining;
210
211 if (copy_to_user(buf, (void *)&data->hdr + offset, len))
212 return -EFAULT;
213
214 data->hdr_remaining -= len;
215
216 return len;
217 }
218
pkt_data_read(struct xe_sriov_packet * data,char __user * buf,size_t len)219 static ssize_t pkt_data_read(struct xe_sriov_packet *data,
220 char __user *buf, size_t len)
221 {
222 if (len > data->remaining)
223 len = data->remaining;
224
225 if (copy_to_user(buf, data->vaddr + (data->hdr.size - data->remaining), len))
226 return -EFAULT;
227
228 data->remaining -= len;
229
230 return len;
231 }
232
pkt_read_single(struct xe_sriov_packet ** data,unsigned int vfid,char __user * buf,size_t len)233 static ssize_t pkt_read_single(struct xe_sriov_packet **data,
234 unsigned int vfid, char __user *buf, size_t len)
235 {
236 ssize_t copied = 0;
237
238 if ((*data)->hdr_remaining)
239 copied = pkt_hdr_read(*data, buf, len);
240 else
241 copied = pkt_data_read(*data, buf, len);
242
243 if ((*data)->remaining == 0 && (*data)->hdr_remaining == 0) {
244 xe_sriov_packet_free(*data);
245 *data = NULL;
246 }
247
248 return copied;
249 }
250
251 /**
252 * xe_sriov_packet_read_single() - Read migration data from a single packet.
253 * @xe: the &xe_device
254 * @vfid: the VF identifier
255 * @buf: start address of userspace buffer
256 * @len: requested read size from userspace
257 *
258 * Return: number of bytes that has been successfully read,
259 * 0 if no more migration data is available,
260 * -errno on failure.
261 */
xe_sriov_packet_read_single(struct xe_device * xe,unsigned int vfid,char __user * buf,size_t len)262 ssize_t xe_sriov_packet_read_single(struct xe_device *xe, unsigned int vfid,
263 char __user *buf, size_t len)
264 {
265 struct xe_sriov_packet **data = pf_pick_read_packet(xe, vfid);
266
267 if (!data)
268 return -ENODATA;
269 if (IS_ERR(*data))
270 return PTR_ERR(*data);
271
272 return pkt_read_single(data, vfid, buf, len);
273 }
274
pkt_hdr_write(struct xe_sriov_packet * data,const char __user * buf,size_t len)275 static ssize_t pkt_hdr_write(struct xe_sriov_packet *data,
276 const char __user *buf, size_t len)
277 {
278 loff_t offset = sizeof(data->hdr) - data->hdr_remaining;
279 int ret;
280
281 if (len > data->hdr_remaining)
282 len = data->hdr_remaining;
283
284 if (copy_from_user((void *)&data->hdr + offset, buf, len))
285 return -EFAULT;
286
287 data->hdr_remaining -= len;
288
289 if (!data->hdr_remaining) {
290 ret = xe_sriov_packet_init_from_hdr(data);
291 if (ret)
292 return ret;
293 }
294
295 return len;
296 }
297
pkt_data_write(struct xe_sriov_packet * data,const char __user * buf,size_t len)298 static ssize_t pkt_data_write(struct xe_sriov_packet *data,
299 const char __user *buf, size_t len)
300 {
301 if (len > data->remaining)
302 len = data->remaining;
303
304 if (copy_from_user(data->vaddr + (data->hdr.size - data->remaining), buf, len))
305 return -EFAULT;
306
307 data->remaining -= len;
308
309 return len;
310 }
311
312 /**
313 * xe_sriov_packet_write_single() - Write migration data to a single packet.
314 * @xe: the &xe_device
315 * @vfid: the VF identifier
316 * @buf: start address of userspace buffer
317 * @len: requested write size from userspace
318 *
319 * Return: number of bytes that has been successfully written,
320 * -errno on failure.
321 */
xe_sriov_packet_write_single(struct xe_device * xe,unsigned int vfid,const char __user * buf,size_t len)322 ssize_t xe_sriov_packet_write_single(struct xe_device *xe, unsigned int vfid,
323 const char __user *buf, size_t len)
324 {
325 struct xe_sriov_packet **data = pf_pick_pending(xe, vfid);
326 int ret;
327 ssize_t copied;
328
329 if (IS_ERR_OR_NULL(*data)) {
330 *data = xe_sriov_packet_alloc(xe);
331 if (!*data)
332 return -ENOMEM;
333 }
334
335 if ((*data)->hdr_remaining)
336 copied = pkt_hdr_write(*data, buf, len);
337 else
338 copied = pkt_data_write(*data, buf, len);
339
340 if ((*data)->hdr_remaining == 0 && (*data)->remaining == 0) {
341 ret = xe_sriov_pf_migration_restore_produce(xe, vfid, *data);
342 if (ret) {
343 xe_sriov_packet_free(*data);
344 *data = NULL;
345
346 return ret;
347 }
348
349 *data = NULL;
350 }
351
352 return copied;
353 }
354
355 #define MIGRATION_KLV_DEVICE_DEVID_KEY 0xf001u
356 #define MIGRATION_KLV_DEVICE_DEVID_LEN 1u
357 #define MIGRATION_KLV_DEVICE_REVID_KEY 0xf002u
358 #define MIGRATION_KLV_DEVICE_REVID_LEN 1u
359
360 #define MIGRATION_DESCRIPTOR_DWORDS (GUC_KLV_LEN_MIN + MIGRATION_KLV_DEVICE_DEVID_LEN + \
361 GUC_KLV_LEN_MIN + MIGRATION_KLV_DEVICE_REVID_LEN)
pf_descriptor_init(struct xe_device * xe,unsigned int vfid)362 static int pf_descriptor_init(struct xe_device *xe, unsigned int vfid)
363 {
364 struct xe_sriov_packet **desc = pf_pick_descriptor(xe, vfid);
365 struct xe_sriov_packet *data;
366 unsigned int len = 0;
367 u32 *klvs;
368 int ret;
369
370 data = xe_sriov_packet_alloc(xe);
371 if (!data)
372 return -ENOMEM;
373
374 ret = xe_sriov_packet_init(data, 0, 0, XE_SRIOV_PACKET_TYPE_DESCRIPTOR,
375 0, MIGRATION_DESCRIPTOR_DWORDS * sizeof(u32));
376 if (ret) {
377 xe_sriov_packet_free(data);
378 return ret;
379 }
380
381 klvs = data->vaddr;
382 klvs[len++] = PREP_GUC_KLV_CONST(MIGRATION_KLV_DEVICE_DEVID_KEY,
383 MIGRATION_KLV_DEVICE_DEVID_LEN);
384 klvs[len++] = xe->info.devid;
385 klvs[len++] = PREP_GUC_KLV_CONST(MIGRATION_KLV_DEVICE_REVID_KEY,
386 MIGRATION_KLV_DEVICE_REVID_LEN);
387 klvs[len++] = xe->info.revid;
388
389 xe_assert(xe, len == MIGRATION_DESCRIPTOR_DWORDS);
390
391 *desc = data;
392
393 return 0;
394 }
395
396 /**
397 * xe_sriov_packet_process_descriptor() - Process migration data descriptor packet.
398 * @xe: the &xe_device
399 * @vfid: the VF identifier
400 * @data: the &xe_sriov_packet containing the descriptor
401 *
402 * The descriptor uses the same KLV format as GuC, and contains metadata used for
403 * checking migration data compatibility.
404 *
405 * Return: 0 on success, -errno on failure.
406 */
xe_sriov_packet_process_descriptor(struct xe_device * xe,unsigned int vfid,struct xe_sriov_packet * data)407 int xe_sriov_packet_process_descriptor(struct xe_device *xe, unsigned int vfid,
408 struct xe_sriov_packet *data)
409 {
410 u32 num_dwords = data->hdr.size / sizeof(u32);
411 u32 *klvs = data->vaddr;
412
413 xe_assert(xe, data->hdr.type == XE_SRIOV_PACKET_TYPE_DESCRIPTOR);
414
415 if (data->hdr.size % sizeof(u32)) {
416 xe_sriov_warn(xe, "Aborting migration, descriptor not in KLV format (size=%llu)\n",
417 data->hdr.size);
418 return -EINVAL;
419 }
420
421 while (num_dwords >= GUC_KLV_LEN_MIN) {
422 u32 key = FIELD_GET(GUC_KLV_0_KEY, klvs[0]);
423 u32 len = FIELD_GET(GUC_KLV_0_LEN, klvs[0]);
424
425 klvs += GUC_KLV_LEN_MIN;
426 num_dwords -= GUC_KLV_LEN_MIN;
427
428 if (len > num_dwords) {
429 xe_sriov_warn(xe, "Aborting migration, truncated KLV %#x, len %u\n",
430 key, len);
431 return -EINVAL;
432 }
433
434 switch (key) {
435 case MIGRATION_KLV_DEVICE_DEVID_KEY:
436 if (*klvs != xe->info.devid) {
437 xe_sriov_warn(xe,
438 "Aborting migration, devid mismatch %#06x!=%#06x\n",
439 *klvs, xe->info.devid);
440 return -ENODEV;
441 }
442 break;
443 case MIGRATION_KLV_DEVICE_REVID_KEY:
444 if (*klvs != xe->info.revid) {
445 xe_sriov_warn(xe,
446 "Aborting migration, revid mismatch %#06x!=%#06x\n",
447 *klvs, xe->info.revid);
448 return -ENODEV;
449 }
450 break;
451 default:
452 xe_sriov_dbg(xe,
453 "Skipping unknown migration KLV %#x, len=%u\n",
454 key, len);
455 print_hex_dump_bytes("desc: ", DUMP_PREFIX_OFFSET, klvs,
456 min(SZ_64, len * sizeof(u32)));
457 break;
458 }
459
460 klvs += len;
461 num_dwords -= len;
462 }
463
464 return 0;
465 }
466
pf_pending_init(struct xe_device * xe,unsigned int vfid)467 static void pf_pending_init(struct xe_device *xe, unsigned int vfid)
468 {
469 struct xe_sriov_packet **data = pf_pick_pending(xe, vfid);
470
471 *data = NULL;
472 }
473
474 #define MIGRATION_TRAILER_SIZE 0
pf_trailer_init(struct xe_device * xe,unsigned int vfid)475 static int pf_trailer_init(struct xe_device *xe, unsigned int vfid)
476 {
477 struct xe_sriov_packet **trailer = pf_pick_trailer(xe, vfid);
478 struct xe_sriov_packet *data;
479 int ret;
480
481 data = xe_sriov_packet_alloc(xe);
482 if (!data)
483 return -ENOMEM;
484
485 ret = xe_sriov_packet_init(data, 0, 0, XE_SRIOV_PACKET_TYPE_TRAILER,
486 0, MIGRATION_TRAILER_SIZE);
487 if (ret) {
488 xe_sriov_packet_free(data);
489 return ret;
490 }
491
492 *trailer = data;
493
494 return 0;
495 }
496
497 /**
498 * xe_sriov_packet_save_init() - Initialize the pending save migration packets.
499 * @xe: the &xe_device
500 * @vfid: the VF identifier
501 *
502 * Return: 0 on success, -errno on failure.
503 */
xe_sriov_packet_save_init(struct xe_device * xe,unsigned int vfid)504 int xe_sriov_packet_save_init(struct xe_device *xe, unsigned int vfid)
505 {
506 int ret;
507
508 scoped_cond_guard(mutex_intr, return -EINTR, pf_migration_mutex(xe, vfid)) {
509 ret = pf_descriptor_init(xe, vfid);
510 if (ret)
511 return ret;
512
513 ret = pf_trailer_init(xe, vfid);
514 if (ret)
515 return ret;
516
517 pf_pending_init(xe, vfid);
518 }
519
520 return 0;
521 }
522