1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2025 Intel Corporation 4 */ 5 6 #include "xe_bo.h" 7 #include "xe_device.h" 8 #include "xe_guc_klv_helpers.h" 9 #include "xe_sriov_packet.h" 10 #include "xe_sriov_packet_types.h" 11 #include "xe_sriov_pf_helpers.h" 12 #include "xe_sriov_pf_migration.h" 13 #include "xe_sriov_printk.h" 14 15 static struct mutex *pf_migration_mutex(struct xe_device *xe, unsigned int vfid) 16 { 17 xe_assert(xe, IS_SRIOV_PF(xe)); 18 xe_assert(xe, vfid <= xe_sriov_pf_get_totalvfs(xe)); 19 20 return &xe->sriov.pf.vfs[vfid].migration.lock; 21 } 22 23 static struct xe_sriov_packet **pf_pick_pending(struct xe_device *xe, unsigned int vfid) 24 { 25 xe_assert(xe, IS_SRIOV_PF(xe)); 26 xe_assert(xe, vfid <= xe_sriov_pf_get_totalvfs(xe)); 27 lockdep_assert_held(pf_migration_mutex(xe, vfid)); 28 29 return &xe->sriov.pf.vfs[vfid].migration.pending; 30 } 31 32 static struct xe_sriov_packet ** 33 pf_pick_descriptor(struct xe_device *xe, unsigned int vfid) 34 { 35 xe_assert(xe, IS_SRIOV_PF(xe)); 36 xe_assert(xe, vfid <= xe_sriov_pf_get_totalvfs(xe)); 37 lockdep_assert_held(pf_migration_mutex(xe, vfid)); 38 39 return &xe->sriov.pf.vfs[vfid].migration.descriptor; 40 } 41 42 static struct xe_sriov_packet **pf_pick_trailer(struct xe_device *xe, unsigned int vfid) 43 { 44 xe_assert(xe, IS_SRIOV_PF(xe)); 45 xe_assert(xe, vfid <= xe_sriov_pf_get_totalvfs(xe)); 46 lockdep_assert_held(pf_migration_mutex(xe, vfid)); 47 48 return &xe->sriov.pf.vfs[vfid].migration.trailer; 49 } 50 51 static struct xe_sriov_packet **pf_pick_read_packet(struct xe_device *xe, 52 unsigned int vfid) 53 { 54 struct xe_sriov_packet **data; 55 56 data = pf_pick_descriptor(xe, vfid); 57 if (*data) 58 return data; 59 60 data = pf_pick_pending(xe, vfid); 61 if (!*data) 62 *data = xe_sriov_pf_migration_save_consume(xe, vfid); 63 if (*data) 64 return data; 65 66 data = pf_pick_trailer(xe, vfid); 67 if (*data) 68 return data; 69 70 return NULL; 71 } 72 73 static bool pkt_needs_bo(struct xe_sriov_packet *data) 74 { 75 return data->hdr.type == XE_SRIOV_PACKET_TYPE_VRAM; 76 } 77 78 /** 79 * xe_sriov_packet_alloc() - Allocate migration data packet 80 * @xe: the &xe_device 81 * 82 * Only allocates the "outer" structure, without initializing the migration 83 * data backing storage. 84 * 85 * Return: Pointer to &xe_sriov_packet on success, 86 * NULL in case of error. 87 */ 88 struct xe_sriov_packet *xe_sriov_packet_alloc(struct xe_device *xe) 89 { 90 struct xe_sriov_packet *data; 91 92 data = kzalloc_obj(*data); 93 if (!data) 94 return NULL; 95 96 data->xe = xe; 97 data->hdr_remaining = sizeof(data->hdr); 98 99 return data; 100 } 101 102 /** 103 * xe_sriov_packet_free() - Free migration data packet. 104 * @data: the &xe_sriov_packet 105 */ 106 void xe_sriov_packet_free(struct xe_sriov_packet *data) 107 { 108 if (IS_ERR_OR_NULL(data)) 109 return; 110 111 if (pkt_needs_bo(data)) 112 xe_bo_unpin_map_no_vm(data->bo); 113 else 114 kvfree(data->buff); 115 116 kfree(data); 117 } 118 119 static int pkt_init(struct xe_sriov_packet *data) 120 { 121 struct xe_gt *gt = xe_device_get_gt(data->xe, data->hdr.gt_id); 122 123 if (!gt) 124 return -EINVAL; 125 126 if (data->hdr.size == 0) 127 return 0; 128 129 if (pkt_needs_bo(data)) { 130 struct xe_bo *bo; 131 132 bo = xe_bo_create_pin_map_novm(data->xe, gt->tile, PAGE_ALIGN(data->hdr.size), 133 ttm_bo_type_kernel, 134 XE_BO_FLAG_SYSTEM | XE_BO_FLAG_PINNED, false); 135 if (IS_ERR(bo)) 136 return PTR_ERR(bo); 137 138 data->bo = bo; 139 data->vaddr = bo->vmap.vaddr; 140 } else { 141 void *buff = kvzalloc(data->hdr.size, GFP_KERNEL); 142 143 if (!buff) 144 return -ENOMEM; 145 146 data->buff = buff; 147 data->vaddr = buff; 148 } 149 150 return 0; 151 } 152 153 #define XE_SRIOV_PACKET_SUPPORTED_VERSION 1 154 155 /** 156 * xe_sriov_packet_init() - Initialize migration packet header and backing storage. 157 * @data: the &xe_sriov_packet 158 * @tile_id: tile identifier 159 * @gt_id: GT identifier 160 * @type: &xe_sriov_packet_type 161 * @offset: offset of data packet payload (within wider resource) 162 * @size: size of data packet payload 163 * 164 * Return: 0 on success or a negative error code on failure. 165 */ 166 int xe_sriov_packet_init(struct xe_sriov_packet *data, u8 tile_id, u8 gt_id, 167 enum xe_sriov_packet_type type, loff_t offset, size_t size) 168 { 169 data->hdr.version = XE_SRIOV_PACKET_SUPPORTED_VERSION; 170 data->hdr.type = type; 171 data->hdr.tile_id = tile_id; 172 data->hdr.gt_id = gt_id; 173 data->hdr.offset = offset; 174 data->hdr.size = size; 175 data->remaining = size; 176 177 return pkt_init(data); 178 } 179 180 /** 181 * xe_sriov_packet_init_from_hdr() - Initialize migration packet backing storage based on header. 182 * @data: the &xe_sriov_packet 183 * 184 * Header data is expected to be filled prior to calling this function. 185 * 186 * Return: 0 on success or a negative error code on failure. 187 */ 188 int xe_sriov_packet_init_from_hdr(struct xe_sriov_packet *data) 189 { 190 xe_assert(data->xe, !data->hdr_remaining); 191 192 if (data->hdr.version != XE_SRIOV_PACKET_SUPPORTED_VERSION) 193 return -EINVAL; 194 195 data->remaining = data->hdr.size; 196 197 return pkt_init(data); 198 } 199 200 static ssize_t pkt_hdr_read(struct xe_sriov_packet *data, 201 char __user *buf, size_t len) 202 { 203 loff_t offset = sizeof(data->hdr) - data->hdr_remaining; 204 205 if (!data->hdr_remaining) 206 return -EINVAL; 207 208 if (len > data->hdr_remaining) 209 len = data->hdr_remaining; 210 211 if (copy_to_user(buf, (void *)&data->hdr + offset, len)) 212 return -EFAULT; 213 214 data->hdr_remaining -= len; 215 216 return len; 217 } 218 219 static ssize_t pkt_data_read(struct xe_sriov_packet *data, 220 char __user *buf, size_t len) 221 { 222 if (len > data->remaining) 223 len = data->remaining; 224 225 if (copy_to_user(buf, data->vaddr + (data->hdr.size - data->remaining), len)) 226 return -EFAULT; 227 228 data->remaining -= len; 229 230 return len; 231 } 232 233 static ssize_t pkt_read_single(struct xe_sriov_packet **data, 234 unsigned int vfid, char __user *buf, size_t len) 235 { 236 ssize_t copied = 0; 237 238 if ((*data)->hdr_remaining) 239 copied = pkt_hdr_read(*data, buf, len); 240 else 241 copied = pkt_data_read(*data, buf, len); 242 243 if ((*data)->remaining == 0 && (*data)->hdr_remaining == 0) { 244 xe_sriov_packet_free(*data); 245 *data = NULL; 246 } 247 248 return copied; 249 } 250 251 /** 252 * xe_sriov_packet_read_single() - Read migration data from a single packet. 253 * @xe: the &xe_device 254 * @vfid: the VF identifier 255 * @buf: start address of userspace buffer 256 * @len: requested read size from userspace 257 * 258 * Return: number of bytes that has been successfully read, 259 * 0 if no more migration data is available, 260 * -errno on failure. 261 */ 262 ssize_t xe_sriov_packet_read_single(struct xe_device *xe, unsigned int vfid, 263 char __user *buf, size_t len) 264 { 265 struct xe_sriov_packet **data = pf_pick_read_packet(xe, vfid); 266 267 if (!data) 268 return -ENODATA; 269 if (IS_ERR(*data)) 270 return PTR_ERR(*data); 271 272 return pkt_read_single(data, vfid, buf, len); 273 } 274 275 static ssize_t pkt_hdr_write(struct xe_sriov_packet *data, 276 const char __user *buf, size_t len) 277 { 278 loff_t offset = sizeof(data->hdr) - data->hdr_remaining; 279 int ret; 280 281 if (len > data->hdr_remaining) 282 len = data->hdr_remaining; 283 284 if (copy_from_user((void *)&data->hdr + offset, buf, len)) 285 return -EFAULT; 286 287 data->hdr_remaining -= len; 288 289 if (!data->hdr_remaining) { 290 ret = xe_sriov_packet_init_from_hdr(data); 291 if (ret) 292 return ret; 293 } 294 295 return len; 296 } 297 298 static ssize_t pkt_data_write(struct xe_sriov_packet *data, 299 const char __user *buf, size_t len) 300 { 301 if (len > data->remaining) 302 len = data->remaining; 303 304 if (copy_from_user(data->vaddr + (data->hdr.size - data->remaining), buf, len)) 305 return -EFAULT; 306 307 data->remaining -= len; 308 309 return len; 310 } 311 312 /** 313 * xe_sriov_packet_write_single() - Write migration data to a single packet. 314 * @xe: the &xe_device 315 * @vfid: the VF identifier 316 * @buf: start address of userspace buffer 317 * @len: requested write size from userspace 318 * 319 * Return: number of bytes that has been successfully written, 320 * -errno on failure. 321 */ 322 ssize_t xe_sriov_packet_write_single(struct xe_device *xe, unsigned int vfid, 323 const char __user *buf, size_t len) 324 { 325 struct xe_sriov_packet **data = pf_pick_pending(xe, vfid); 326 int ret; 327 ssize_t copied; 328 329 if (IS_ERR_OR_NULL(*data)) { 330 *data = xe_sriov_packet_alloc(xe); 331 if (!*data) 332 return -ENOMEM; 333 } 334 335 if ((*data)->hdr_remaining) 336 copied = pkt_hdr_write(*data, buf, len); 337 else 338 copied = pkt_data_write(*data, buf, len); 339 340 if ((*data)->hdr_remaining == 0 && (*data)->remaining == 0) { 341 ret = xe_sriov_pf_migration_restore_produce(xe, vfid, *data); 342 if (ret) { 343 xe_sriov_packet_free(*data); 344 *data = NULL; 345 346 return ret; 347 } 348 349 *data = NULL; 350 } 351 352 return copied; 353 } 354 355 #define MIGRATION_KLV_DEVICE_DEVID_KEY 0xf001u 356 #define MIGRATION_KLV_DEVICE_DEVID_LEN 1u 357 #define MIGRATION_KLV_DEVICE_REVID_KEY 0xf002u 358 #define MIGRATION_KLV_DEVICE_REVID_LEN 1u 359 360 #define MIGRATION_DESCRIPTOR_DWORDS (GUC_KLV_LEN_MIN + MIGRATION_KLV_DEVICE_DEVID_LEN + \ 361 GUC_KLV_LEN_MIN + MIGRATION_KLV_DEVICE_REVID_LEN) 362 static int pf_descriptor_init(struct xe_device *xe, unsigned int vfid) 363 { 364 struct xe_sriov_packet **desc = pf_pick_descriptor(xe, vfid); 365 struct xe_sriov_packet *data; 366 unsigned int len = 0; 367 u32 *klvs; 368 int ret; 369 370 data = xe_sriov_packet_alloc(xe); 371 if (!data) 372 return -ENOMEM; 373 374 ret = xe_sriov_packet_init(data, 0, 0, XE_SRIOV_PACKET_TYPE_DESCRIPTOR, 375 0, MIGRATION_DESCRIPTOR_DWORDS * sizeof(u32)); 376 if (ret) { 377 xe_sriov_packet_free(data); 378 return ret; 379 } 380 381 klvs = data->vaddr; 382 klvs[len++] = PREP_GUC_KLV_CONST(MIGRATION_KLV_DEVICE_DEVID_KEY, 383 MIGRATION_KLV_DEVICE_DEVID_LEN); 384 klvs[len++] = xe->info.devid; 385 klvs[len++] = PREP_GUC_KLV_CONST(MIGRATION_KLV_DEVICE_REVID_KEY, 386 MIGRATION_KLV_DEVICE_REVID_LEN); 387 klvs[len++] = xe->info.revid; 388 389 xe_assert(xe, len == MIGRATION_DESCRIPTOR_DWORDS); 390 391 *desc = data; 392 393 return 0; 394 } 395 396 /** 397 * xe_sriov_packet_process_descriptor() - Process migration data descriptor packet. 398 * @xe: the &xe_device 399 * @vfid: the VF identifier 400 * @data: the &xe_sriov_packet containing the descriptor 401 * 402 * The descriptor uses the same KLV format as GuC, and contains metadata used for 403 * checking migration data compatibility. 404 * 405 * Return: 0 on success, -errno on failure. 406 */ 407 int xe_sriov_packet_process_descriptor(struct xe_device *xe, unsigned int vfid, 408 struct xe_sriov_packet *data) 409 { 410 u32 num_dwords = data->hdr.size / sizeof(u32); 411 u32 *klvs = data->vaddr; 412 413 xe_assert(xe, data->hdr.type == XE_SRIOV_PACKET_TYPE_DESCRIPTOR); 414 415 if (data->hdr.size % sizeof(u32)) { 416 xe_sriov_warn(xe, "Aborting migration, descriptor not in KLV format (size=%llu)\n", 417 data->hdr.size); 418 return -EINVAL; 419 } 420 421 while (num_dwords >= GUC_KLV_LEN_MIN) { 422 u32 key = FIELD_GET(GUC_KLV_0_KEY, klvs[0]); 423 u32 len = FIELD_GET(GUC_KLV_0_LEN, klvs[0]); 424 425 klvs += GUC_KLV_LEN_MIN; 426 num_dwords -= GUC_KLV_LEN_MIN; 427 428 if (len > num_dwords) { 429 xe_sriov_warn(xe, "Aborting migration, truncated KLV %#x, len %u\n", 430 key, len); 431 return -EINVAL; 432 } 433 434 switch (key) { 435 case MIGRATION_KLV_DEVICE_DEVID_KEY: 436 if (*klvs != xe->info.devid) { 437 xe_sriov_warn(xe, 438 "Aborting migration, devid mismatch %#06x!=%#06x\n", 439 *klvs, xe->info.devid); 440 return -ENODEV; 441 } 442 break; 443 case MIGRATION_KLV_DEVICE_REVID_KEY: 444 if (*klvs != xe->info.revid) { 445 xe_sriov_warn(xe, 446 "Aborting migration, revid mismatch %#06x!=%#06x\n", 447 *klvs, xe->info.revid); 448 return -ENODEV; 449 } 450 break; 451 default: 452 xe_sriov_dbg(xe, 453 "Skipping unknown migration KLV %#x, len=%u\n", 454 key, len); 455 print_hex_dump_bytes("desc: ", DUMP_PREFIX_OFFSET, klvs, 456 min(SZ_64, len * sizeof(u32))); 457 break; 458 } 459 460 klvs += len; 461 num_dwords -= len; 462 } 463 464 return 0; 465 } 466 467 static void pf_pending_init(struct xe_device *xe, unsigned int vfid) 468 { 469 struct xe_sriov_packet **data = pf_pick_pending(xe, vfid); 470 471 *data = NULL; 472 } 473 474 #define MIGRATION_TRAILER_SIZE 0 475 static int pf_trailer_init(struct xe_device *xe, unsigned int vfid) 476 { 477 struct xe_sriov_packet **trailer = pf_pick_trailer(xe, vfid); 478 struct xe_sriov_packet *data; 479 int ret; 480 481 data = xe_sriov_packet_alloc(xe); 482 if (!data) 483 return -ENOMEM; 484 485 ret = xe_sriov_packet_init(data, 0, 0, XE_SRIOV_PACKET_TYPE_TRAILER, 486 0, MIGRATION_TRAILER_SIZE); 487 if (ret) { 488 xe_sriov_packet_free(data); 489 return ret; 490 } 491 492 *trailer = data; 493 494 return 0; 495 } 496 497 /** 498 * xe_sriov_packet_save_init() - Initialize the pending save migration packets. 499 * @xe: the &xe_device 500 * @vfid: the VF identifier 501 * 502 * Return: 0 on success, -errno on failure. 503 */ 504 int xe_sriov_packet_save_init(struct xe_device *xe, unsigned int vfid) 505 { 506 int ret; 507 508 scoped_cond_guard(mutex_intr, return -EINTR, pf_migration_mutex(xe, vfid)) { 509 ret = pf_descriptor_init(xe, vfid); 510 if (ret) 511 return ret; 512 513 ret = pf_trailer_init(xe, vfid); 514 if (ret) 515 return ret; 516 517 pf_pending_init(xe, vfid); 518 } 519 520 return 0; 521 } 522