1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2025 Intel Corporation 4 */ 5 6 #include "xe_bo.h" 7 #include "xe_device.h" 8 #include "xe_guc_klv_helpers.h" 9 #include "xe_printk.h" 10 #include "xe_sriov_packet.h" 11 #include "xe_sriov_packet_types.h" 12 #include "xe_sriov_pf_helpers.h" 13 #include "xe_sriov_pf_migration.h" 14 #include "xe_sriov_printk.h" 15 16 static struct mutex *pf_migration_mutex(struct xe_device *xe, unsigned int vfid) 17 { 18 xe_assert(xe, IS_SRIOV_PF(xe)); 19 xe_assert(xe, vfid <= xe_sriov_pf_get_totalvfs(xe)); 20 21 return &xe->sriov.pf.vfs[vfid].migration.lock; 22 } 23 24 static struct xe_sriov_packet **pf_pick_pending(struct xe_device *xe, unsigned int vfid) 25 { 26 xe_assert(xe, IS_SRIOV_PF(xe)); 27 xe_assert(xe, vfid <= xe_sriov_pf_get_totalvfs(xe)); 28 lockdep_assert_held(pf_migration_mutex(xe, vfid)); 29 30 return &xe->sriov.pf.vfs[vfid].migration.pending; 31 } 32 33 static struct xe_sriov_packet ** 34 pf_pick_descriptor(struct xe_device *xe, unsigned int vfid) 35 { 36 xe_assert(xe, IS_SRIOV_PF(xe)); 37 xe_assert(xe, vfid <= xe_sriov_pf_get_totalvfs(xe)); 38 lockdep_assert_held(pf_migration_mutex(xe, vfid)); 39 40 return &xe->sriov.pf.vfs[vfid].migration.descriptor; 41 } 42 43 static struct xe_sriov_packet **pf_pick_trailer(struct xe_device *xe, unsigned int vfid) 44 { 45 xe_assert(xe, IS_SRIOV_PF(xe)); 46 xe_assert(xe, vfid <= xe_sriov_pf_get_totalvfs(xe)); 47 lockdep_assert_held(pf_migration_mutex(xe, vfid)); 48 49 return &xe->sriov.pf.vfs[vfid].migration.trailer; 50 } 51 52 static struct xe_sriov_packet **pf_pick_read_packet(struct xe_device *xe, 53 unsigned int vfid) 54 { 55 struct xe_sriov_packet **data; 56 57 data = pf_pick_descriptor(xe, vfid); 58 if (*data) 59 return data; 60 61 data = pf_pick_pending(xe, vfid); 62 if (!*data) 63 *data = xe_sriov_pf_migration_save_consume(xe, vfid); 64 if (*data) 65 return data; 66 67 data = pf_pick_trailer(xe, vfid); 68 if (*data) 69 return data; 70 71 return NULL; 72 } 73 74 static bool pkt_needs_bo(struct xe_sriov_packet *data) 75 { 76 return data->hdr.type == XE_SRIOV_PACKET_TYPE_VRAM; 77 } 78 79 /** 80 * xe_sriov_packet_alloc() - Allocate migration data packet 81 * @xe: the &xe_device 82 * 83 * Only allocates the "outer" structure, without initializing the migration 84 * data backing storage. 85 * 86 * Return: Pointer to &xe_sriov_packet on success, 87 * NULL in case of error. 88 */ 89 struct xe_sriov_packet *xe_sriov_packet_alloc(struct xe_device *xe) 90 { 91 struct xe_sriov_packet *data; 92 93 data = kzalloc(sizeof(*data), GFP_KERNEL); 94 if (!data) 95 return NULL; 96 97 data->xe = xe; 98 data->hdr_remaining = sizeof(data->hdr); 99 100 return data; 101 } 102 103 /** 104 * xe_sriov_packet_free() - Free migration data packet. 105 * @data: the &xe_sriov_packet 106 */ 107 void xe_sriov_packet_free(struct xe_sriov_packet *data) 108 { 109 if (IS_ERR_OR_NULL(data)) 110 return; 111 112 if (pkt_needs_bo(data)) 113 xe_bo_unpin_map_no_vm(data->bo); 114 else 115 kvfree(data->buff); 116 117 kfree(data); 118 } 119 120 static int pkt_init(struct xe_sriov_packet *data) 121 { 122 struct xe_gt *gt = xe_device_get_gt(data->xe, data->hdr.gt_id); 123 124 if (!gt) 125 return -EINVAL; 126 127 if (data->hdr.size == 0) 128 return 0; 129 130 if (pkt_needs_bo(data)) { 131 struct xe_bo *bo; 132 133 bo = xe_bo_create_pin_map_novm(data->xe, gt->tile, PAGE_ALIGN(data->hdr.size), 134 ttm_bo_type_kernel, 135 XE_BO_FLAG_SYSTEM | XE_BO_FLAG_PINNED, false); 136 if (IS_ERR(bo)) 137 return PTR_ERR(bo); 138 139 data->bo = bo; 140 data->vaddr = bo->vmap.vaddr; 141 } else { 142 void *buff = kvzalloc(data->hdr.size, GFP_KERNEL); 143 144 if (!buff) 145 return -ENOMEM; 146 147 data->buff = buff; 148 data->vaddr = buff; 149 } 150 151 return 0; 152 } 153 154 #define XE_SRIOV_PACKET_SUPPORTED_VERSION 1 155 156 /** 157 * xe_sriov_packet_init() - Initialize migration packet header and backing storage. 158 * @data: the &xe_sriov_packet 159 * @tile_id: tile identifier 160 * @gt_id: GT identifier 161 * @type: &xe_sriov_packet_type 162 * @offset: offset of data packet payload (within wider resource) 163 * @size: size of data packet payload 164 * 165 * Return: 0 on success or a negative error code on failure. 166 */ 167 int xe_sriov_packet_init(struct xe_sriov_packet *data, u8 tile_id, u8 gt_id, 168 enum xe_sriov_packet_type type, loff_t offset, size_t size) 169 { 170 data->hdr.version = XE_SRIOV_PACKET_SUPPORTED_VERSION; 171 data->hdr.type = type; 172 data->hdr.tile_id = tile_id; 173 data->hdr.gt_id = gt_id; 174 data->hdr.offset = offset; 175 data->hdr.size = size; 176 data->remaining = size; 177 178 return pkt_init(data); 179 } 180 181 /** 182 * xe_sriov_packet_init_from_hdr() - Initialize migration packet backing storage based on header. 183 * @data: the &xe_sriov_packet 184 * 185 * Header data is expected to be filled prior to calling this function. 186 * 187 * Return: 0 on success or a negative error code on failure. 188 */ 189 int xe_sriov_packet_init_from_hdr(struct xe_sriov_packet *data) 190 { 191 xe_assert(data->xe, !data->hdr_remaining); 192 193 if (data->hdr.version != XE_SRIOV_PACKET_SUPPORTED_VERSION) 194 return -EINVAL; 195 196 data->remaining = data->hdr.size; 197 198 return pkt_init(data); 199 } 200 201 static ssize_t pkt_hdr_read(struct xe_sriov_packet *data, 202 char __user *buf, size_t len) 203 { 204 loff_t offset = sizeof(data->hdr) - data->hdr_remaining; 205 206 if (!data->hdr_remaining) 207 return -EINVAL; 208 209 if (len > data->hdr_remaining) 210 len = data->hdr_remaining; 211 212 if (copy_to_user(buf, (void *)&data->hdr + offset, len)) 213 return -EFAULT; 214 215 data->hdr_remaining -= len; 216 217 return len; 218 } 219 220 static ssize_t pkt_data_read(struct xe_sriov_packet *data, 221 char __user *buf, size_t len) 222 { 223 if (len > data->remaining) 224 len = data->remaining; 225 226 if (copy_to_user(buf, data->vaddr + (data->hdr.size - data->remaining), len)) 227 return -EFAULT; 228 229 data->remaining -= len; 230 231 return len; 232 } 233 234 static ssize_t pkt_read_single(struct xe_sriov_packet **data, 235 unsigned int vfid, char __user *buf, size_t len) 236 { 237 ssize_t copied = 0; 238 239 if ((*data)->hdr_remaining) 240 copied = pkt_hdr_read(*data, buf, len); 241 else 242 copied = pkt_data_read(*data, buf, len); 243 244 if ((*data)->remaining == 0 && (*data)->hdr_remaining == 0) { 245 xe_sriov_packet_free(*data); 246 *data = NULL; 247 } 248 249 return copied; 250 } 251 252 /** 253 * xe_sriov_packet_read_single() - Read migration data from a single packet. 254 * @xe: the &xe_device 255 * @vfid: the VF identifier 256 * @buf: start address of userspace buffer 257 * @len: requested read size from userspace 258 * 259 * Return: number of bytes that has been successfully read, 260 * 0 if no more migration data is available, 261 * -errno on failure. 262 */ 263 ssize_t xe_sriov_packet_read_single(struct xe_device *xe, unsigned int vfid, 264 char __user *buf, size_t len) 265 { 266 struct xe_sriov_packet **data = pf_pick_read_packet(xe, vfid); 267 268 if (!data) 269 return -ENODATA; 270 if (IS_ERR(*data)) 271 return PTR_ERR(*data); 272 273 return pkt_read_single(data, vfid, buf, len); 274 } 275 276 static ssize_t pkt_hdr_write(struct xe_sriov_packet *data, 277 const char __user *buf, size_t len) 278 { 279 loff_t offset = sizeof(data->hdr) - data->hdr_remaining; 280 int ret; 281 282 if (len > data->hdr_remaining) 283 len = data->hdr_remaining; 284 285 if (copy_from_user((void *)&data->hdr + offset, buf, len)) 286 return -EFAULT; 287 288 data->hdr_remaining -= len; 289 290 if (!data->hdr_remaining) { 291 ret = xe_sriov_packet_init_from_hdr(data); 292 if (ret) 293 return ret; 294 } 295 296 return len; 297 } 298 299 static ssize_t pkt_data_write(struct xe_sriov_packet *data, 300 const char __user *buf, size_t len) 301 { 302 if (len > data->remaining) 303 len = data->remaining; 304 305 if (copy_from_user(data->vaddr + (data->hdr.size - data->remaining), buf, len)) 306 return -EFAULT; 307 308 data->remaining -= len; 309 310 return len; 311 } 312 313 /** 314 * xe_sriov_packet_write_single() - Write migration data to a single packet. 315 * @xe: the &xe_device 316 * @vfid: the VF identifier 317 * @buf: start address of userspace buffer 318 * @len: requested write size from userspace 319 * 320 * Return: number of bytes that has been successfully written, 321 * -errno on failure. 322 */ 323 ssize_t xe_sriov_packet_write_single(struct xe_device *xe, unsigned int vfid, 324 const char __user *buf, size_t len) 325 { 326 struct xe_sriov_packet **data = pf_pick_pending(xe, vfid); 327 int ret; 328 ssize_t copied; 329 330 if (IS_ERR_OR_NULL(*data)) { 331 *data = xe_sriov_packet_alloc(xe); 332 if (!*data) 333 return -ENOMEM; 334 } 335 336 if ((*data)->hdr_remaining) 337 copied = pkt_hdr_write(*data, buf, len); 338 else 339 copied = pkt_data_write(*data, buf, len); 340 341 if ((*data)->hdr_remaining == 0 && (*data)->remaining == 0) { 342 ret = xe_sriov_pf_migration_restore_produce(xe, vfid, *data); 343 if (ret) { 344 xe_sriov_packet_free(*data); 345 return ret; 346 } 347 348 *data = NULL; 349 } 350 351 return copied; 352 } 353 354 #define MIGRATION_KLV_DEVICE_DEVID_KEY 0xf001u 355 #define MIGRATION_KLV_DEVICE_DEVID_LEN 1u 356 #define MIGRATION_KLV_DEVICE_REVID_KEY 0xf002u 357 #define MIGRATION_KLV_DEVICE_REVID_LEN 1u 358 359 #define MIGRATION_DESCRIPTOR_DWORDS (GUC_KLV_LEN_MIN + MIGRATION_KLV_DEVICE_DEVID_LEN + \ 360 GUC_KLV_LEN_MIN + MIGRATION_KLV_DEVICE_REVID_LEN) 361 static size_t pf_descriptor_init(struct xe_device *xe, unsigned int vfid) 362 { 363 struct xe_sriov_packet **desc = pf_pick_descriptor(xe, vfid); 364 struct xe_sriov_packet *data; 365 unsigned int len = 0; 366 u32 *klvs; 367 int ret; 368 369 data = xe_sriov_packet_alloc(xe); 370 if (!data) 371 return -ENOMEM; 372 373 ret = xe_sriov_packet_init(data, 0, 0, XE_SRIOV_PACKET_TYPE_DESCRIPTOR, 374 0, MIGRATION_DESCRIPTOR_DWORDS * sizeof(u32)); 375 if (ret) { 376 xe_sriov_packet_free(data); 377 return ret; 378 } 379 380 klvs = data->vaddr; 381 klvs[len++] = PREP_GUC_KLV_CONST(MIGRATION_KLV_DEVICE_DEVID_KEY, 382 MIGRATION_KLV_DEVICE_DEVID_LEN); 383 klvs[len++] = xe->info.devid; 384 klvs[len++] = PREP_GUC_KLV_CONST(MIGRATION_KLV_DEVICE_REVID_KEY, 385 MIGRATION_KLV_DEVICE_REVID_LEN); 386 klvs[len++] = xe->info.revid; 387 388 xe_assert(xe, len == MIGRATION_DESCRIPTOR_DWORDS); 389 390 *desc = data; 391 392 return 0; 393 } 394 395 /** 396 * xe_sriov_packet_process_descriptor() - Process migration data descriptor packet. 397 * @xe: the &xe_device 398 * @vfid: the VF identifier 399 * @data: the &xe_sriov_packet containing the descriptor 400 * 401 * The descriptor uses the same KLV format as GuC, and contains metadata used for 402 * checking migration data compatibility. 403 * 404 * Return: 0 on success, -errno on failure. 405 */ 406 int xe_sriov_packet_process_descriptor(struct xe_device *xe, unsigned int vfid, 407 struct xe_sriov_packet *data) 408 { 409 u32 num_dwords = data->hdr.size / sizeof(u32); 410 u32 *klvs = data->vaddr; 411 412 xe_assert(xe, data->hdr.type == XE_SRIOV_PACKET_TYPE_DESCRIPTOR); 413 414 if (data->hdr.size % sizeof(u32)) { 415 xe_sriov_warn(xe, "Aborting migration, descriptor not in KLV format (size=%llu)\n", 416 data->hdr.size); 417 return -EINVAL; 418 } 419 420 while (num_dwords >= GUC_KLV_LEN_MIN) { 421 u32 key = FIELD_GET(GUC_KLV_0_KEY, klvs[0]); 422 u32 len = FIELD_GET(GUC_KLV_0_LEN, klvs[0]); 423 424 klvs += GUC_KLV_LEN_MIN; 425 num_dwords -= GUC_KLV_LEN_MIN; 426 427 if (len > num_dwords) { 428 xe_sriov_warn(xe, "Aborting migration, truncated KLV %#x, len %u\n", 429 key, len); 430 return -EINVAL; 431 } 432 433 switch (key) { 434 case MIGRATION_KLV_DEVICE_DEVID_KEY: 435 if (*klvs != xe->info.devid) { 436 xe_sriov_warn(xe, 437 "Aborting migration, devid mismatch %#06x!=%#06x\n", 438 *klvs, xe->info.devid); 439 return -ENODEV; 440 } 441 break; 442 case MIGRATION_KLV_DEVICE_REVID_KEY: 443 if (*klvs != xe->info.revid) { 444 xe_sriov_warn(xe, 445 "Aborting migration, revid mismatch %#06x!=%#06x\n", 446 *klvs, xe->info.revid); 447 return -ENODEV; 448 } 449 break; 450 default: 451 xe_sriov_dbg(xe, 452 "Skipping unknown migration KLV %#x, len=%u\n", 453 key, len); 454 print_hex_dump_bytes("desc: ", DUMP_PREFIX_OFFSET, klvs, 455 min(SZ_64, len * sizeof(u32))); 456 break; 457 } 458 459 klvs += len; 460 num_dwords -= len; 461 } 462 463 return 0; 464 } 465 466 static void pf_pending_init(struct xe_device *xe, unsigned int vfid) 467 { 468 struct xe_sriov_packet **data = pf_pick_pending(xe, vfid); 469 470 *data = NULL; 471 } 472 473 #define MIGRATION_TRAILER_SIZE 0 474 static int pf_trailer_init(struct xe_device *xe, unsigned int vfid) 475 { 476 struct xe_sriov_packet **trailer = pf_pick_trailer(xe, vfid); 477 struct xe_sriov_packet *data; 478 int ret; 479 480 data = xe_sriov_packet_alloc(xe); 481 if (!data) 482 return -ENOMEM; 483 484 ret = xe_sriov_packet_init(data, 0, 0, XE_SRIOV_PACKET_TYPE_TRAILER, 485 0, MIGRATION_TRAILER_SIZE); 486 if (ret) { 487 xe_sriov_packet_free(data); 488 return ret; 489 } 490 491 *trailer = data; 492 493 return 0; 494 } 495 496 /** 497 * xe_sriov_packet_save_init() - Initialize the pending save migration packets. 498 * @xe: the &xe_device 499 * @vfid: the VF identifier 500 * 501 * Return: 0 on success, -errno on failure. 502 */ 503 int xe_sriov_packet_save_init(struct xe_device *xe, unsigned int vfid) 504 { 505 int ret; 506 507 scoped_cond_guard(mutex_intr, return -EINTR, pf_migration_mutex(xe, vfid)) { 508 ret = pf_descriptor_init(xe, vfid); 509 if (ret) 510 return ret; 511 512 ret = pf_trailer_init(xe, vfid); 513 if (ret) 514 return ret; 515 516 pf_pending_init(xe, vfid); 517 } 518 519 return 0; 520 } 521