1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2025 Intel Corporation 4 */ 5 6 #include "xe_bo.h" 7 #include "xe_device.h" 8 #include "xe_guc_klv_helpers.h" 9 #include "xe_sriov_packet.h" 10 #include "xe_sriov_packet_types.h" 11 #include "xe_sriov_pf_helpers.h" 12 #include "xe_sriov_pf_migration.h" 13 #include "xe_sriov_printk.h" 14 15 static struct mutex *pf_migration_mutex(struct xe_device *xe, unsigned int vfid) 16 { 17 xe_assert(xe, IS_SRIOV_PF(xe)); 18 xe_assert(xe, vfid <= xe_sriov_pf_get_totalvfs(xe)); 19 20 return &xe->sriov.pf.vfs[vfid].migration.lock; 21 } 22 23 static struct xe_sriov_packet **pf_pick_pending(struct xe_device *xe, unsigned int vfid) 24 { 25 xe_assert(xe, IS_SRIOV_PF(xe)); 26 xe_assert(xe, vfid <= xe_sriov_pf_get_totalvfs(xe)); 27 lockdep_assert_held(pf_migration_mutex(xe, vfid)); 28 29 return &xe->sriov.pf.vfs[vfid].migration.pending; 30 } 31 32 static struct xe_sriov_packet ** 33 pf_pick_descriptor(struct xe_device *xe, unsigned int vfid) 34 { 35 xe_assert(xe, IS_SRIOV_PF(xe)); 36 xe_assert(xe, vfid <= xe_sriov_pf_get_totalvfs(xe)); 37 lockdep_assert_held(pf_migration_mutex(xe, vfid)); 38 39 return &xe->sriov.pf.vfs[vfid].migration.descriptor; 40 } 41 42 static struct xe_sriov_packet **pf_pick_trailer(struct xe_device *xe, unsigned int vfid) 43 { 44 xe_assert(xe, IS_SRIOV_PF(xe)); 45 xe_assert(xe, vfid <= xe_sriov_pf_get_totalvfs(xe)); 46 lockdep_assert_held(pf_migration_mutex(xe, vfid)); 47 48 return &xe->sriov.pf.vfs[vfid].migration.trailer; 49 } 50 51 static struct xe_sriov_packet **pf_pick_read_packet(struct xe_device *xe, 52 unsigned int vfid) 53 { 54 struct xe_sriov_packet **data; 55 56 data = pf_pick_descriptor(xe, vfid); 57 if (*data) 58 return data; 59 60 data = pf_pick_pending(xe, vfid); 61 if (!*data) 62 *data = xe_sriov_pf_migration_save_consume(xe, vfid); 63 if (*data) 64 return data; 65 66 data = pf_pick_trailer(xe, vfid); 67 if (*data) 68 return data; 69 70 return NULL; 71 } 72 73 static bool pkt_needs_bo(struct xe_sriov_packet *data) 74 { 75 return data->hdr.type == XE_SRIOV_PACKET_TYPE_VRAM; 76 } 77 78 /** 79 * xe_sriov_packet_alloc() - Allocate migration data packet 80 * @xe: the &xe_device 81 * 82 * Only allocates the "outer" structure, without initializing the migration 83 * data backing storage. 84 * 85 * Return: Pointer to &xe_sriov_packet on success, 86 * NULL in case of error. 87 */ 88 struct xe_sriov_packet *xe_sriov_packet_alloc(struct xe_device *xe) 89 { 90 struct xe_sriov_packet *data; 91 92 data = kzalloc_obj(*data); 93 if (!data) 94 return NULL; 95 96 data->xe = xe; 97 data->hdr_remaining = sizeof(data->hdr); 98 99 return data; 100 } 101 102 /** 103 * xe_sriov_packet_free() - Free migration data packet. 104 * @data: the &xe_sriov_packet 105 */ 106 void xe_sriov_packet_free(struct xe_sriov_packet *data) 107 { 108 if (IS_ERR_OR_NULL(data)) 109 return; 110 111 if (pkt_needs_bo(data)) 112 xe_bo_unpin_map_no_vm(data->bo); 113 else 114 kvfree(data->buff); 115 116 kfree(data); 117 } 118 119 static int pkt_init(struct xe_sriov_packet *data) 120 { 121 struct xe_gt *gt = xe_device_get_gt(data->xe, data->hdr.gt_id); 122 123 if (!gt) 124 return -EINVAL; 125 126 if (data->hdr.size == 0) 127 return 0; 128 129 if (pkt_needs_bo(data)) { 130 struct xe_bo *bo; 131 132 bo = xe_bo_create_pin_map_novm(data->xe, gt->tile, PAGE_ALIGN(data->hdr.size), 133 ttm_bo_type_kernel, 134 XE_BO_FLAG_SYSTEM | XE_BO_FLAG_PINNED, false); 135 if (IS_ERR(bo)) 136 return PTR_ERR(bo); 137 138 data->bo = bo; 139 data->vaddr = bo->vmap.vaddr; 140 } else { 141 void *buff = kvzalloc(data->hdr.size, GFP_KERNEL); 142 143 if (!buff) 144 return -ENOMEM; 145 146 data->buff = buff; 147 data->vaddr = buff; 148 } 149 150 return 0; 151 } 152 153 #define XE_SRIOV_PACKET_SUPPORTED_VERSION 1 154 155 /** 156 * xe_sriov_packet_init() - Initialize migration packet header and backing storage. 157 * @data: the &xe_sriov_packet 158 * @tile_id: tile identifier 159 * @gt_id: GT identifier 160 * @type: &xe_sriov_packet_type 161 * @offset: offset of data packet payload (within wider resource) 162 * @size: size of data packet payload 163 * 164 * Return: 0 on success or a negative error code on failure. 165 */ 166 int xe_sriov_packet_init(struct xe_sriov_packet *data, u8 tile_id, u8 gt_id, 167 enum xe_sriov_packet_type type, loff_t offset, size_t size) 168 { 169 data->hdr.version = XE_SRIOV_PACKET_SUPPORTED_VERSION; 170 data->hdr.type = type; 171 data->hdr.tile_id = tile_id; 172 data->hdr.gt_id = gt_id; 173 data->hdr.offset = offset; 174 data->hdr.size = size; 175 data->remaining = size; 176 177 return pkt_init(data); 178 } 179 180 /** 181 * xe_sriov_packet_init_from_hdr() - Initialize migration packet backing storage based on header. 182 * @data: the &xe_sriov_packet 183 * 184 * Header data is expected to be filled prior to calling this function. 185 * 186 * Return: 0 on success or a negative error code on failure. 187 */ 188 int xe_sriov_packet_init_from_hdr(struct xe_sriov_packet *data) 189 { 190 xe_assert(data->xe, !data->hdr_remaining); 191 192 if (data->hdr.version != XE_SRIOV_PACKET_SUPPORTED_VERSION) 193 return -EINVAL; 194 195 data->remaining = data->hdr.size; 196 197 return pkt_init(data); 198 } 199 200 static ssize_t pkt_hdr_read(struct xe_sriov_packet *data, 201 char __user *buf, size_t len) 202 { 203 loff_t offset = sizeof(data->hdr) - data->hdr_remaining; 204 205 if (!data->hdr_remaining) 206 return -EINVAL; 207 208 if (len > data->hdr_remaining) 209 len = data->hdr_remaining; 210 211 if (copy_to_user(buf, (void *)&data->hdr + offset, len)) 212 return -EFAULT; 213 214 data->hdr_remaining -= len; 215 216 return len; 217 } 218 219 static ssize_t pkt_data_read(struct xe_sriov_packet *data, 220 char __user *buf, size_t len) 221 { 222 if (len > data->remaining) 223 len = data->remaining; 224 225 if (copy_to_user(buf, data->vaddr + (data->hdr.size - data->remaining), len)) 226 return -EFAULT; 227 228 data->remaining -= len; 229 230 return len; 231 } 232 233 static ssize_t pkt_read_single(struct xe_sriov_packet **data, 234 unsigned int vfid, char __user *buf, size_t len) 235 { 236 ssize_t copied = 0; 237 238 if ((*data)->hdr_remaining) 239 copied = pkt_hdr_read(*data, buf, len); 240 else 241 copied = pkt_data_read(*data, buf, len); 242 243 if ((*data)->remaining == 0 && (*data)->hdr_remaining == 0) { 244 xe_sriov_packet_free(*data); 245 *data = NULL; 246 } 247 248 return copied; 249 } 250 251 /** 252 * xe_sriov_packet_read_single() - Read migration data from a single packet. 253 * @xe: the &xe_device 254 * @vfid: the VF identifier 255 * @buf: start address of userspace buffer 256 * @len: requested read size from userspace 257 * 258 * Return: number of bytes that has been successfully read, 259 * 0 if no more migration data is available, 260 * -errno on failure. 261 */ 262 ssize_t xe_sriov_packet_read_single(struct xe_device *xe, unsigned int vfid, 263 char __user *buf, size_t len) 264 { 265 struct xe_sriov_packet **data = pf_pick_read_packet(xe, vfid); 266 267 if (!data) 268 return -ENODATA; 269 if (IS_ERR(*data)) 270 return PTR_ERR(*data); 271 272 return pkt_read_single(data, vfid, buf, len); 273 } 274 275 static ssize_t pkt_hdr_write(struct xe_sriov_packet *data, 276 const char __user *buf, size_t len) 277 { 278 loff_t offset = sizeof(data->hdr) - data->hdr_remaining; 279 int ret; 280 281 if (len > data->hdr_remaining) 282 len = data->hdr_remaining; 283 284 if (copy_from_user((void *)&data->hdr + offset, buf, len)) 285 return -EFAULT; 286 287 data->hdr_remaining -= len; 288 289 if (!data->hdr_remaining) { 290 ret = xe_sriov_packet_init_from_hdr(data); 291 if (ret) 292 return ret; 293 } 294 295 return len; 296 } 297 298 static ssize_t pkt_data_write(struct xe_sriov_packet *data, 299 const char __user *buf, size_t len) 300 { 301 if (len > data->remaining) 302 len = data->remaining; 303 304 if (copy_from_user(data->vaddr + (data->hdr.size - data->remaining), buf, len)) 305 return -EFAULT; 306 307 data->remaining -= len; 308 309 return len; 310 } 311 312 /** 313 * xe_sriov_packet_write_single() - Write migration data to a single packet. 314 * @xe: the &xe_device 315 * @vfid: the VF identifier 316 * @buf: start address of userspace buffer 317 * @len: requested write size from userspace 318 * 319 * Return: number of bytes that has been successfully written, 320 * -errno on failure. 321 */ 322 ssize_t xe_sriov_packet_write_single(struct xe_device *xe, unsigned int vfid, 323 const char __user *buf, size_t len) 324 { 325 struct xe_sriov_packet **data = pf_pick_pending(xe, vfid); 326 int ret; 327 ssize_t copied; 328 329 if (IS_ERR_OR_NULL(*data)) { 330 *data = xe_sriov_packet_alloc(xe); 331 if (!*data) 332 return -ENOMEM; 333 } 334 335 if ((*data)->hdr_remaining) 336 copied = pkt_hdr_write(*data, buf, len); 337 else 338 copied = pkt_data_write(*data, buf, len); 339 340 if ((*data)->hdr_remaining == 0 && (*data)->remaining == 0) { 341 ret = xe_sriov_pf_migration_restore_produce(xe, vfid, *data); 342 if (ret) { 343 xe_sriov_packet_free(*data); 344 return ret; 345 } 346 347 *data = NULL; 348 } 349 350 return copied; 351 } 352 353 #define MIGRATION_KLV_DEVICE_DEVID_KEY 0xf001u 354 #define MIGRATION_KLV_DEVICE_DEVID_LEN 1u 355 #define MIGRATION_KLV_DEVICE_REVID_KEY 0xf002u 356 #define MIGRATION_KLV_DEVICE_REVID_LEN 1u 357 358 #define MIGRATION_DESCRIPTOR_DWORDS (GUC_KLV_LEN_MIN + MIGRATION_KLV_DEVICE_DEVID_LEN + \ 359 GUC_KLV_LEN_MIN + MIGRATION_KLV_DEVICE_REVID_LEN) 360 static int pf_descriptor_init(struct xe_device *xe, unsigned int vfid) 361 { 362 struct xe_sriov_packet **desc = pf_pick_descriptor(xe, vfid); 363 struct xe_sriov_packet *data; 364 unsigned int len = 0; 365 u32 *klvs; 366 int ret; 367 368 data = xe_sriov_packet_alloc(xe); 369 if (!data) 370 return -ENOMEM; 371 372 ret = xe_sriov_packet_init(data, 0, 0, XE_SRIOV_PACKET_TYPE_DESCRIPTOR, 373 0, MIGRATION_DESCRIPTOR_DWORDS * sizeof(u32)); 374 if (ret) { 375 xe_sriov_packet_free(data); 376 return ret; 377 } 378 379 klvs = data->vaddr; 380 klvs[len++] = PREP_GUC_KLV_CONST(MIGRATION_KLV_DEVICE_DEVID_KEY, 381 MIGRATION_KLV_DEVICE_DEVID_LEN); 382 klvs[len++] = xe->info.devid; 383 klvs[len++] = PREP_GUC_KLV_CONST(MIGRATION_KLV_DEVICE_REVID_KEY, 384 MIGRATION_KLV_DEVICE_REVID_LEN); 385 klvs[len++] = xe->info.revid; 386 387 xe_assert(xe, len == MIGRATION_DESCRIPTOR_DWORDS); 388 389 *desc = data; 390 391 return 0; 392 } 393 394 /** 395 * xe_sriov_packet_process_descriptor() - Process migration data descriptor packet. 396 * @xe: the &xe_device 397 * @vfid: the VF identifier 398 * @data: the &xe_sriov_packet containing the descriptor 399 * 400 * The descriptor uses the same KLV format as GuC, and contains metadata used for 401 * checking migration data compatibility. 402 * 403 * Return: 0 on success, -errno on failure. 404 */ 405 int xe_sriov_packet_process_descriptor(struct xe_device *xe, unsigned int vfid, 406 struct xe_sriov_packet *data) 407 { 408 u32 num_dwords = data->hdr.size / sizeof(u32); 409 u32 *klvs = data->vaddr; 410 411 xe_assert(xe, data->hdr.type == XE_SRIOV_PACKET_TYPE_DESCRIPTOR); 412 413 if (data->hdr.size % sizeof(u32)) { 414 xe_sriov_warn(xe, "Aborting migration, descriptor not in KLV format (size=%llu)\n", 415 data->hdr.size); 416 return -EINVAL; 417 } 418 419 while (num_dwords >= GUC_KLV_LEN_MIN) { 420 u32 key = FIELD_GET(GUC_KLV_0_KEY, klvs[0]); 421 u32 len = FIELD_GET(GUC_KLV_0_LEN, klvs[0]); 422 423 klvs += GUC_KLV_LEN_MIN; 424 num_dwords -= GUC_KLV_LEN_MIN; 425 426 if (len > num_dwords) { 427 xe_sriov_warn(xe, "Aborting migration, truncated KLV %#x, len %u\n", 428 key, len); 429 return -EINVAL; 430 } 431 432 switch (key) { 433 case MIGRATION_KLV_DEVICE_DEVID_KEY: 434 if (*klvs != xe->info.devid) { 435 xe_sriov_warn(xe, 436 "Aborting migration, devid mismatch %#06x!=%#06x\n", 437 *klvs, xe->info.devid); 438 return -ENODEV; 439 } 440 break; 441 case MIGRATION_KLV_DEVICE_REVID_KEY: 442 if (*klvs != xe->info.revid) { 443 xe_sriov_warn(xe, 444 "Aborting migration, revid mismatch %#06x!=%#06x\n", 445 *klvs, xe->info.revid); 446 return -ENODEV; 447 } 448 break; 449 default: 450 xe_sriov_dbg(xe, 451 "Skipping unknown migration KLV %#x, len=%u\n", 452 key, len); 453 print_hex_dump_bytes("desc: ", DUMP_PREFIX_OFFSET, klvs, 454 min(SZ_64, len * sizeof(u32))); 455 break; 456 } 457 458 klvs += len; 459 num_dwords -= len; 460 } 461 462 return 0; 463 } 464 465 static void pf_pending_init(struct xe_device *xe, unsigned int vfid) 466 { 467 struct xe_sriov_packet **data = pf_pick_pending(xe, vfid); 468 469 *data = NULL; 470 } 471 472 #define MIGRATION_TRAILER_SIZE 0 473 static int pf_trailer_init(struct xe_device *xe, unsigned int vfid) 474 { 475 struct xe_sriov_packet **trailer = pf_pick_trailer(xe, vfid); 476 struct xe_sriov_packet *data; 477 int ret; 478 479 data = xe_sriov_packet_alloc(xe); 480 if (!data) 481 return -ENOMEM; 482 483 ret = xe_sriov_packet_init(data, 0, 0, XE_SRIOV_PACKET_TYPE_TRAILER, 484 0, MIGRATION_TRAILER_SIZE); 485 if (ret) { 486 xe_sriov_packet_free(data); 487 return ret; 488 } 489 490 *trailer = data; 491 492 return 0; 493 } 494 495 /** 496 * xe_sriov_packet_save_init() - Initialize the pending save migration packets. 497 * @xe: the &xe_device 498 * @vfid: the VF identifier 499 * 500 * Return: 0 on success, -errno on failure. 501 */ 502 int xe_sriov_packet_save_init(struct xe_device *xe, unsigned int vfid) 503 { 504 int ret; 505 506 scoped_cond_guard(mutex_intr, return -EINTR, pf_migration_mutex(xe, vfid)) { 507 ret = pf_descriptor_init(xe, vfid); 508 if (ret) 509 return ret; 510 511 ret = pf_trailer_init(xe, vfid); 512 if (ret) 513 return ret; 514 515 pf_pending_init(xe, vfid); 516 } 517 518 return 0; 519 } 520