1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2025 Intel Corporation 4 */ 5 6 #include "abi/xe_driver_klvs_abi.h" 7 8 #include "xe_bo.h" 9 #include "xe_device.h" 10 #include "xe_guc_klv_helpers.h" 11 #include "xe_sriov_packet.h" 12 #include "xe_sriov_packet_types.h" 13 #include "xe_sriov_pf_helpers.h" 14 #include "xe_sriov_pf_migration.h" 15 #include "xe_sriov_printk.h" 16 17 static struct mutex *pf_migration_mutex(struct xe_device *xe, unsigned int vfid) 18 { 19 xe_assert(xe, IS_SRIOV_PF(xe)); 20 xe_assert(xe, vfid <= xe_sriov_pf_get_totalvfs(xe)); 21 22 return &xe->sriov.pf.vfs[vfid].migration.lock; 23 } 24 25 static struct xe_sriov_packet **pf_pick_pending(struct xe_device *xe, unsigned int vfid) 26 { 27 xe_assert(xe, IS_SRIOV_PF(xe)); 28 xe_assert(xe, vfid <= xe_sriov_pf_get_totalvfs(xe)); 29 lockdep_assert_held(pf_migration_mutex(xe, vfid)); 30 31 return &xe->sriov.pf.vfs[vfid].migration.pending; 32 } 33 34 static struct xe_sriov_packet ** 35 pf_pick_descriptor(struct xe_device *xe, unsigned int vfid) 36 { 37 xe_assert(xe, IS_SRIOV_PF(xe)); 38 xe_assert(xe, vfid <= xe_sriov_pf_get_totalvfs(xe)); 39 lockdep_assert_held(pf_migration_mutex(xe, vfid)); 40 41 return &xe->sriov.pf.vfs[vfid].migration.descriptor; 42 } 43 44 static struct xe_sriov_packet **pf_pick_trailer(struct xe_device *xe, unsigned int vfid) 45 { 46 xe_assert(xe, IS_SRIOV_PF(xe)); 47 xe_assert(xe, vfid <= xe_sriov_pf_get_totalvfs(xe)); 48 lockdep_assert_held(pf_migration_mutex(xe, vfid)); 49 50 return &xe->sriov.pf.vfs[vfid].migration.trailer; 51 } 52 53 static struct xe_sriov_packet **pf_pick_read_packet(struct xe_device *xe, 54 unsigned int vfid) 55 { 56 struct xe_sriov_packet **data; 57 58 data = pf_pick_descriptor(xe, vfid); 59 if (*data) 60 return data; 61 62 data = pf_pick_pending(xe, vfid); 63 if (!*data) 64 *data = xe_sriov_pf_migration_save_consume(xe, vfid); 65 if (*data) 66 return data; 67 68 data = pf_pick_trailer(xe, vfid); 69 if (*data) 70 return data; 71 72 return NULL; 73 } 74 75 static bool pkt_needs_bo(struct xe_sriov_packet *data) 76 { 77 return data->hdr.type == XE_SRIOV_PACKET_TYPE_VRAM; 78 } 79 80 /** 81 * xe_sriov_packet_alloc() - Allocate migration data packet 82 * @xe: the &xe_device 83 * 84 * Only allocates the "outer" structure, without initializing the migration 85 * data backing storage. 86 * 87 * Return: Pointer to &xe_sriov_packet on success, 88 * NULL in case of error. 89 */ 90 struct xe_sriov_packet *xe_sriov_packet_alloc(struct xe_device *xe) 91 { 92 struct xe_sriov_packet *data; 93 94 data = kzalloc_obj(*data); 95 if (!data) 96 return NULL; 97 98 data->xe = xe; 99 data->hdr_remaining = sizeof(data->hdr); 100 101 return data; 102 } 103 104 /** 105 * xe_sriov_packet_free() - Free migration data packet. 106 * @data: the &xe_sriov_packet 107 */ 108 void xe_sriov_packet_free(struct xe_sriov_packet *data) 109 { 110 if (IS_ERR_OR_NULL(data)) 111 return; 112 113 if (pkt_needs_bo(data)) 114 xe_bo_unpin_map_no_vm(data->bo); 115 else 116 kvfree(data->buff); 117 118 kfree(data); 119 } 120 121 static int pkt_init(struct xe_sriov_packet *data) 122 { 123 struct xe_gt *gt = xe_device_get_gt(data->xe, data->hdr.gt_id); 124 125 if (!gt) 126 return -EINVAL; 127 128 if (data->hdr.size == 0) 129 return 0; 130 131 if (pkt_needs_bo(data)) { 132 struct xe_bo *bo; 133 134 bo = xe_bo_create_pin_map_novm(data->xe, gt->tile, PAGE_ALIGN(data->hdr.size), 135 ttm_bo_type_kernel, 136 XE_BO_FLAG_SYSTEM | XE_BO_FLAG_PINNED, false); 137 if (IS_ERR(bo)) 138 return PTR_ERR(bo); 139 140 data->bo = bo; 141 data->vaddr = bo->vmap.vaddr; 142 } else { 143 void *buff = kvzalloc(data->hdr.size, GFP_KERNEL); 144 145 if (!buff) 146 return -ENOMEM; 147 148 data->buff = buff; 149 data->vaddr = buff; 150 } 151 152 return 0; 153 } 154 155 #define XE_SRIOV_PACKET_SUPPORTED_VERSION 1 156 157 /** 158 * xe_sriov_packet_init() - Initialize migration packet header and backing storage. 159 * @data: the &xe_sriov_packet 160 * @tile_id: tile identifier 161 * @gt_id: GT identifier 162 * @type: &xe_sriov_packet_type 163 * @offset: offset of data packet payload (within wider resource) 164 * @size: size of data packet payload 165 * 166 * Return: 0 on success or a negative error code on failure. 167 */ 168 int xe_sriov_packet_init(struct xe_sriov_packet *data, u8 tile_id, u8 gt_id, 169 enum xe_sriov_packet_type type, loff_t offset, size_t size) 170 { 171 data->hdr.version = XE_SRIOV_PACKET_SUPPORTED_VERSION; 172 data->hdr.type = type; 173 data->hdr.tile_id = tile_id; 174 data->hdr.gt_id = gt_id; 175 data->hdr.offset = offset; 176 data->hdr.size = size; 177 data->remaining = size; 178 179 return pkt_init(data); 180 } 181 182 /** 183 * xe_sriov_packet_init_from_hdr() - Initialize migration packet backing storage based on header. 184 * @data: the &xe_sriov_packet 185 * 186 * Header data is expected to be filled prior to calling this function. 187 * 188 * Return: 0 on success or a negative error code on failure. 189 */ 190 int xe_sriov_packet_init_from_hdr(struct xe_sriov_packet *data) 191 { 192 xe_assert(data->xe, !data->hdr_remaining); 193 194 if (data->hdr.version != XE_SRIOV_PACKET_SUPPORTED_VERSION) 195 return -EINVAL; 196 197 data->remaining = data->hdr.size; 198 199 return pkt_init(data); 200 } 201 202 static ssize_t pkt_hdr_read(struct xe_sriov_packet *data, 203 char __user *buf, size_t len) 204 { 205 loff_t offset = sizeof(data->hdr) - data->hdr_remaining; 206 207 if (!data->hdr_remaining) 208 return -EINVAL; 209 210 if (len > data->hdr_remaining) 211 len = data->hdr_remaining; 212 213 if (copy_to_user(buf, (void *)&data->hdr + offset, len)) 214 return -EFAULT; 215 216 data->hdr_remaining -= len; 217 218 return len; 219 } 220 221 static ssize_t pkt_data_read(struct xe_sriov_packet *data, 222 char __user *buf, size_t len) 223 { 224 if (len > data->remaining) 225 len = data->remaining; 226 227 if (copy_to_user(buf, data->vaddr + (data->hdr.size - data->remaining), len)) 228 return -EFAULT; 229 230 data->remaining -= len; 231 232 return len; 233 } 234 235 static ssize_t pkt_read_single(struct xe_sriov_packet **data, 236 unsigned int vfid, char __user *buf, size_t len) 237 { 238 ssize_t copied = 0; 239 240 if ((*data)->hdr_remaining) 241 copied = pkt_hdr_read(*data, buf, len); 242 else 243 copied = pkt_data_read(*data, buf, len); 244 245 if ((*data)->remaining == 0 && (*data)->hdr_remaining == 0) { 246 xe_sriov_packet_free(*data); 247 *data = NULL; 248 } 249 250 return copied; 251 } 252 253 /** 254 * xe_sriov_packet_read_single() - Read migration data from a single packet. 255 * @xe: the &xe_device 256 * @vfid: the VF identifier 257 * @buf: start address of userspace buffer 258 * @len: requested read size from userspace 259 * 260 * Return: number of bytes that has been successfully read, 261 * 0 if no more migration data is available, 262 * -errno on failure. 263 */ 264 ssize_t xe_sriov_packet_read_single(struct xe_device *xe, unsigned int vfid, 265 char __user *buf, size_t len) 266 { 267 struct xe_sriov_packet **data = pf_pick_read_packet(xe, vfid); 268 269 if (!data) 270 return -ENODATA; 271 if (IS_ERR(*data)) 272 return PTR_ERR(*data); 273 274 return pkt_read_single(data, vfid, buf, len); 275 } 276 277 static ssize_t pkt_hdr_write(struct xe_sriov_packet *data, 278 const char __user *buf, size_t len) 279 { 280 loff_t offset = sizeof(data->hdr) - data->hdr_remaining; 281 int ret; 282 283 if (len > data->hdr_remaining) 284 len = data->hdr_remaining; 285 286 if (copy_from_user((void *)&data->hdr + offset, buf, len)) 287 return -EFAULT; 288 289 data->hdr_remaining -= len; 290 291 if (!data->hdr_remaining) { 292 ret = xe_sriov_packet_init_from_hdr(data); 293 if (ret) 294 return ret; 295 } 296 297 return len; 298 } 299 300 static ssize_t pkt_data_write(struct xe_sriov_packet *data, 301 const char __user *buf, size_t len) 302 { 303 if (len > data->remaining) 304 len = data->remaining; 305 306 if (copy_from_user(data->vaddr + (data->hdr.size - data->remaining), buf, len)) 307 return -EFAULT; 308 309 data->remaining -= len; 310 311 return len; 312 } 313 314 /** 315 * xe_sriov_packet_write_single() - Write migration data to a single packet. 316 * @xe: the &xe_device 317 * @vfid: the VF identifier 318 * @buf: start address of userspace buffer 319 * @len: requested write size from userspace 320 * 321 * Return: number of bytes that has been successfully written, 322 * -errno on failure. 323 */ 324 ssize_t xe_sriov_packet_write_single(struct xe_device *xe, unsigned int vfid, 325 const char __user *buf, size_t len) 326 { 327 struct xe_sriov_packet **data = pf_pick_pending(xe, vfid); 328 int ret; 329 ssize_t copied; 330 331 if (IS_ERR_OR_NULL(*data)) { 332 *data = xe_sriov_packet_alloc(xe); 333 if (!*data) 334 return -ENOMEM; 335 } 336 337 if ((*data)->hdr_remaining) 338 copied = pkt_hdr_write(*data, buf, len); 339 else 340 copied = pkt_data_write(*data, buf, len); 341 342 if ((*data)->hdr_remaining == 0 && (*data)->remaining == 0) { 343 ret = xe_sriov_pf_migration_restore_produce(xe, vfid, *data); 344 if (ret) { 345 xe_sriov_packet_free(*data); 346 *data = NULL; 347 348 return ret; 349 } 350 351 *data = NULL; 352 } 353 354 return copied; 355 } 356 357 #define MIGRATION_DESCRIPTOR_DWORDS (GUC_KLV_LEN_MIN + MIGRATION_KLV_DEVICE_DEVID_LEN + \ 358 GUC_KLV_LEN_MIN + MIGRATION_KLV_DEVICE_REVID_LEN) 359 static int pf_descriptor_init(struct xe_device *xe, unsigned int vfid) 360 { 361 struct xe_sriov_packet **desc = pf_pick_descriptor(xe, vfid); 362 struct xe_sriov_packet *data; 363 u32 *klvs, *end; 364 int ret; 365 366 data = xe_sriov_packet_alloc(xe); 367 if (!data) 368 return -ENOMEM; 369 370 ret = xe_sriov_packet_init(data, 0, 0, XE_SRIOV_PACKET_TYPE_DESCRIPTOR, 371 0, MIGRATION_DESCRIPTOR_DWORDS * sizeof(u32)); 372 if (ret) { 373 xe_sriov_packet_free(data); 374 return ret; 375 } 376 377 klvs = data->vaddr; 378 end = klvs + MIGRATION_DESCRIPTOR_DWORDS; 379 380 klvs = xe_guc_klv_encode_u32(klvs, end - klvs, 381 MIGRATION_KLV_DEVICE_DEVID_KEY, 382 xe->info.devid); 383 klvs = xe_guc_klv_encode_u32(klvs, end - klvs, 384 MIGRATION_KLV_DEVICE_REVID_KEY, 385 xe->info.revid); 386 xe_assert(xe, !IS_ERR(klvs)); 387 xe_assert(xe, klvs == end); 388 389 *desc = data; 390 391 return 0; 392 } 393 394 static int descriptor_decoder(void *arg, u16 key, u16 len, const u32 *value) 395 { 396 struct xe_device *xe = arg; 397 398 xe_sriov_dbg_verbose(xe, "found KLV %#x %s\n", key, xe_guc_klv_key_to_string(key)); 399 400 switch (key) { 401 case MIGRATION_KLV_DEVICE_DEVID_KEY: 402 if (*value != xe->info.devid) { 403 xe_sriov_warn(xe, "Aborting migration, devid mismatch %#06x!=%#06x\n", 404 *value, xe->info.devid); 405 return -ENODEV; 406 } 407 break; 408 case MIGRATION_KLV_DEVICE_REVID_KEY: 409 if (*value != xe->info.revid) { 410 xe_sriov_warn(xe, "Aborting migration, revid mismatch %#06x!=%#06x\n", 411 *value, xe->info.revid); 412 return -ENODEV; 413 } 414 break; 415 default: 416 if (IS_ENABLED(CONFIG_DRM_XE_DEBUG)) { 417 struct drm_printer p = xe_dbg_printer(xe); 418 419 xe_sriov_dbg(xe, "unexpected KLV %#x in descriptor!\n", key); 420 xe_guc_klv_print_one(key, len, value, &p); 421 } 422 return 0; 423 } 424 return 1; 425 } 426 427 /** 428 * xe_sriov_packet_process_descriptor() - Process migration data descriptor packet. 429 * @xe: the &xe_device 430 * @vfid: the VF identifier 431 * @data: the &xe_sriov_packet containing the descriptor 432 * 433 * The descriptor uses the same KLV format as GuC, and contains metadata used for 434 * checking migration data compatibility. 435 * 436 * Return: 0 on success, -errno on failure. 437 */ 438 int xe_sriov_packet_process_descriptor(struct xe_device *xe, unsigned int vfid, 439 struct xe_sriov_packet *data) 440 { 441 u32 num_dwords = data->hdr.size / sizeof(u32); 442 u32 *klvs = data->vaddr; 443 int ret; 444 445 xe_assert(xe, data->hdr.type == XE_SRIOV_PACKET_TYPE_DESCRIPTOR); 446 447 if (data->hdr.size % sizeof(u32)) { 448 xe_sriov_warn(xe, "Aborting migration, descriptor not in KLV format (size=%llu)\n", 449 data->hdr.size); 450 return -EINVAL; 451 } 452 453 ret = xe_guc_klv_count(klvs, num_dwords); 454 if (ret < 0) { 455 xe_sriov_warn(xe, "Aborting migration, corrupted descriptor KLVs (%pe)\n", 456 ERR_PTR(ret)); 457 return ret; 458 } 459 460 ret = xe_guc_klv_parser(klvs, num_dwords, xe, descriptor_decoder); 461 if (ret < 0) { 462 xe_sriov_warn(xe, "Aborting migration, descriptor parsing failed (%pe)\n", 463 ERR_PTR(ret)); 464 return ret; 465 } 466 467 return 0; 468 } 469 470 static void pf_pending_init(struct xe_device *xe, unsigned int vfid) 471 { 472 struct xe_sriov_packet **data = pf_pick_pending(xe, vfid); 473 474 *data = NULL; 475 } 476 477 #define MIGRATION_TRAILER_SIZE 0 478 static int pf_trailer_init(struct xe_device *xe, unsigned int vfid) 479 { 480 struct xe_sriov_packet **trailer = pf_pick_trailer(xe, vfid); 481 struct xe_sriov_packet *data; 482 int ret; 483 484 data = xe_sriov_packet_alloc(xe); 485 if (!data) 486 return -ENOMEM; 487 488 ret = xe_sriov_packet_init(data, 0, 0, XE_SRIOV_PACKET_TYPE_TRAILER, 489 0, MIGRATION_TRAILER_SIZE); 490 if (ret) { 491 xe_sriov_packet_free(data); 492 return ret; 493 } 494 495 *trailer = data; 496 497 return 0; 498 } 499 500 /** 501 * xe_sriov_packet_save_init() - Initialize the pending save migration packets. 502 * @xe: the &xe_device 503 * @vfid: the VF identifier 504 * 505 * Return: 0 on success, -errno on failure. 506 */ 507 int xe_sriov_packet_save_init(struct xe_device *xe, unsigned int vfid) 508 { 509 int ret; 510 511 scoped_cond_guard(mutex_intr, return -EINTR, pf_migration_mutex(xe, vfid)) { 512 ret = pf_descriptor_init(xe, vfid); 513 if (ret) 514 return ret; 515 516 ret = pf_trailer_init(xe, vfid); 517 if (ret) 518 return ret; 519 520 pf_pending_init(xe, vfid); 521 } 522 523 return 0; 524 } 525 526 #if IS_BUILTIN(CONFIG_DRM_XE_KUNIT_TEST) 527 #include "tests/xe_sriov_packet_kunit.c" 528 #endif 529