1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2025 Intel Corporation 4 */ 5 6 #include "instructions/xe_mi_commands.h" 7 #include "instructions/xe_gpu_commands.h" 8 #include "xe_bb.h" 9 #include "xe_bo.h" 10 #include "xe_device.h" 11 #include "xe_exec_queue.h" 12 #include "xe_exec_queue_types.h" 13 #include "xe_guc_submit.h" 14 #include "xe_lrc.h" 15 #include "xe_migrate.h" 16 #include "xe_pm.h" 17 #include "xe_sa.h" 18 #include "xe_sriov_printk.h" 19 #include "xe_sriov_vf.h" 20 #include "xe_sriov_vf_ccs.h" 21 #include "xe_sriov_vf_ccs_types.h" 22 23 /** 24 * DOC: VF save/restore of compression Meta Data 25 * 26 * VF KMD registers two special contexts/LRCAs. 27 * 28 * Save Context/LRCA: contain necessary cmds+page table to trigger Meta data / 29 * compression control surface (Aka CCS) save in regular System memory in VM. 30 * 31 * Restore Context/LRCA: contain necessary cmds+page table to trigger Meta data / 32 * compression control surface (Aka CCS) Restore from regular System memory in 33 * VM to corresponding CCS pool. 34 * 35 * Below diagram explain steps needed for VF save/Restore of compression Meta Data:: 36 * 37 * CCS Save CCS Restore VF KMD Guc BCS 38 * LRCA LRCA 39 * | | | | | 40 * | | | | | 41 * | Create Save LRCA | | | 42 * [ ]<----------------------------- [ ] | | 43 * | | | | | 44 * | | | | | 45 * | | | Register save LRCA | | 46 * | | | with Guc | | 47 * | | [ ]--------------------------->[ ] | 48 * | | | | | 49 * | | Create restore LRCA | | | 50 * | [ ]<------------------[ ] | | 51 * | | | | | 52 * | | | Register restore LRCA | | 53 * | | | with Guc | | 54 * | | [ ]--------------------------->[ ] | 55 * | | | | | 56 * | | | | | 57 * | | [ ]------------------------- | | 58 * | | [ ] Allocate main memory. | | | 59 * | | [ ] Allocate CCS memory. | | | 60 * | | [ ] Update Main memory & | | | 61 * [ ]<------------------------------[ ] CCS pages PPGTT + BB | | | 62 * | [ ]<------------------[ ] cmds to save & restore.| | | 63 * | | [ ]<------------------------ | | 64 * | | | | | 65 * | | | | | 66 * | | | | | 67 * : : : : : 68 * ---------------------------- VF Paused ------------------------------------- 69 * | | | | | 70 * | | | | | 71 * | | | |Schedule | 72 * | | | |CCS Save | 73 * | | | | LRCA | 74 * | | | [ ]------>[ ] 75 * | | | | | 76 * | | | | | 77 * | | | |CCS save | 78 * | | | |completed| 79 * | | | [ ]<------[ ] 80 * | | | | | 81 * : : : : : 82 * ---------------------------- VM Migrated ----------------------------------- 83 * | | | | | 84 * | | | | | 85 * : : : : : 86 * ---------------------------- VF Resumed ------------------------------------ 87 * | | | | | 88 * | | | | | 89 * | | [ ]-------------- | | 90 * | | [ ] Fix up GGTT | | | 91 * | | [ ]<------------- | | 92 * | | | | | 93 * | | | | | 94 * | | | Notify VF_RESFIX_DONE | | 95 * | | [ ]--------------------------->[ ] | 96 * | | | | | 97 * | | | |Schedule | 98 * | | | |CCS | 99 * | | | |Restore | 100 * | | | |LRCA | 101 * | | | [ ]------>[ ] 102 * | | | | | 103 * | | | | | 104 * | | | |CCS | 105 * | | | |restore | 106 * | | | |completed| 107 * | | | [ ]<------[ ] 108 * | | | | | 109 * | | | | | 110 * | | | VF_RESFIX_DONE complete | | 111 * | | | notification | | 112 * | | [ ]<---------------------------[ ] | 113 * | | | | | 114 * | | | | | 115 * : : : : : 116 * ------------------------- Continue VM restore ------------------------------ 117 */ 118 119 static u64 get_ccs_bb_pool_size(struct xe_device *xe) 120 { 121 u64 sys_mem_size, ccs_mem_size, ptes, bb_pool_size; 122 struct sysinfo si; 123 124 si_meminfo(&si); 125 sys_mem_size = si.totalram * si.mem_unit; 126 ccs_mem_size = div64_u64(sys_mem_size, NUM_BYTES_PER_CCS_BYTE(xe)); 127 ptes = DIV_ROUND_UP_ULL(sys_mem_size + ccs_mem_size, XE_PAGE_SIZE); 128 129 /** 130 * We need below BB size to hold PTE mappings and some DWs for copy 131 * command. In reality, we need space for many copy commands. So, let 132 * us allocate double the calculated size which is enough to holds GPU 133 * instructions for the whole region. 134 */ 135 bb_pool_size = ptes * sizeof(u32); 136 137 return round_up(bb_pool_size * 2, SZ_1M); 138 } 139 140 static int alloc_bb_pool(struct xe_tile *tile, struct xe_sriov_vf_ccs_ctx *ctx) 141 { 142 struct xe_device *xe = tile_to_xe(tile); 143 struct xe_sa_manager *sa_manager; 144 u64 bb_pool_size; 145 int offset, err; 146 147 bb_pool_size = get_ccs_bb_pool_size(xe); 148 xe_sriov_info(xe, "Allocating %s CCS BB pool size = %lldMB\n", 149 ctx->ctx_id ? "Restore" : "Save", bb_pool_size / SZ_1M); 150 151 sa_manager = xe_sa_bo_manager_init(tile, bb_pool_size, SZ_16); 152 153 if (IS_ERR(sa_manager)) { 154 xe_sriov_err(xe, "Suballocator init failed with error: %pe\n", 155 sa_manager); 156 err = PTR_ERR(sa_manager); 157 return err; 158 } 159 160 offset = 0; 161 xe_map_memset(xe, &sa_manager->bo->vmap, offset, MI_NOOP, 162 bb_pool_size); 163 164 offset = bb_pool_size - sizeof(u32); 165 xe_map_wr(xe, &sa_manager->bo->vmap, offset, u32, MI_BATCH_BUFFER_END); 166 167 ctx->mem.ccs_bb_pool = sa_manager; 168 169 return 0; 170 } 171 172 static void ccs_rw_update_ring(struct xe_sriov_vf_ccs_ctx *ctx) 173 { 174 u64 addr = xe_sa_manager_gpu_addr(ctx->mem.ccs_bb_pool); 175 struct xe_lrc *lrc = xe_exec_queue_lrc(ctx->mig_q); 176 u32 dw[10], i = 0; 177 178 /* 179 * XXX: Save/restore fixes — for some reason, the GuC only accepts the 180 * save/restore context if the LRC head pointer is zero. This is evident 181 * from repeated VF migrations failing when the LRC head pointer is 182 * non-zero. 183 */ 184 lrc->ring.tail = 0; 185 xe_lrc_set_ring_head(lrc, 0); 186 187 dw[i++] = MI_ARB_ON_OFF | MI_ARB_ENABLE; 188 dw[i++] = MI_BATCH_BUFFER_START | XE_INSTR_NUM_DW(3); 189 dw[i++] = lower_32_bits(addr); 190 dw[i++] = upper_32_bits(addr); 191 dw[i++] = MI_NOOP; 192 dw[i++] = MI_NOOP; 193 194 xe_lrc_write_ring(lrc, dw, i * sizeof(u32)); 195 xe_lrc_set_ring_tail(lrc, lrc->ring.tail); 196 } 197 198 /** 199 * xe_sriov_vf_ccs_rebase - Rebase GGTT addresses for CCS save / restore 200 * @xe: the &xe_device. 201 */ 202 void xe_sriov_vf_ccs_rebase(struct xe_device *xe) 203 { 204 enum xe_sriov_vf_ccs_rw_ctxs ctx_id; 205 206 if (!IS_VF_CCS_READY(xe)) 207 return; 208 209 for_each_ccs_rw_ctx(ctx_id) { 210 struct xe_sriov_vf_ccs_ctx *ctx = 211 &xe->sriov.vf.ccs.contexts[ctx_id]; 212 213 ccs_rw_update_ring(ctx); 214 } 215 } 216 217 static int register_save_restore_context(struct xe_sriov_vf_ccs_ctx *ctx) 218 { 219 int ctx_type; 220 221 switch (ctx->ctx_id) { 222 case XE_SRIOV_VF_CCS_READ_CTX: 223 ctx_type = GUC_CONTEXT_COMPRESSION_SAVE; 224 break; 225 case XE_SRIOV_VF_CCS_WRITE_CTX: 226 ctx_type = GUC_CONTEXT_COMPRESSION_RESTORE; 227 break; 228 default: 229 return -EINVAL; 230 } 231 232 xe_guc_register_vf_exec_queue(ctx->mig_q, ctx_type); 233 return 0; 234 } 235 236 /** 237 * xe_sriov_vf_ccs_register_context - Register read/write contexts with guc. 238 * @xe: the &xe_device to register contexts on. 239 * 240 * This function registers read and write contexts with Guc. Re-registration 241 * is needed whenever resuming from pm runtime suspend. 242 * 243 * Return: 0 on success. Negative error code on failure. 244 */ 245 int xe_sriov_vf_ccs_register_context(struct xe_device *xe) 246 { 247 enum xe_sriov_vf_ccs_rw_ctxs ctx_id; 248 struct xe_sriov_vf_ccs_ctx *ctx; 249 int err; 250 251 xe_assert(xe, IS_VF_CCS_READY(xe)); 252 253 for_each_ccs_rw_ctx(ctx_id) { 254 ctx = &xe->sriov.vf.ccs.contexts[ctx_id]; 255 err = register_save_restore_context(ctx); 256 if (err) 257 return err; 258 } 259 260 return err; 261 } 262 263 static void xe_sriov_vf_ccs_fini(void *arg) 264 { 265 struct xe_sriov_vf_ccs_ctx *ctx = arg; 266 struct xe_lrc *lrc = xe_exec_queue_lrc(ctx->mig_q); 267 268 /* 269 * Make TAIL = HEAD in the ring so that no issues are seen if Guc 270 * submits this context to HW on VF pause after unbinding device. 271 */ 272 xe_lrc_set_ring_tail(lrc, xe_lrc_ring_head(lrc)); 273 xe_exec_queue_put(ctx->mig_q); 274 } 275 276 /** 277 * xe_sriov_vf_ccs_init - Setup LRCA for save & restore. 278 * @xe: the &xe_device to start recovery on 279 * 280 * This function shall be called only by VF. It initializes 281 * LRCA and suballocator needed for CCS save & restore. 282 * 283 * Return: 0 on success. Negative error code on failure. 284 */ 285 int xe_sriov_vf_ccs_init(struct xe_device *xe) 286 { 287 struct xe_tile *tile = xe_device_get_root_tile(xe); 288 enum xe_sriov_vf_ccs_rw_ctxs ctx_id; 289 struct xe_sriov_vf_ccs_ctx *ctx; 290 struct xe_exec_queue *q; 291 u32 flags; 292 int err; 293 294 xe_assert(xe, IS_SRIOV_VF(xe)); 295 xe_assert(xe, xe_sriov_vf_migration_supported(xe)); 296 297 if (IS_DGFX(xe) || !xe_device_has_flat_ccs(xe)) 298 return 0; 299 300 for_each_ccs_rw_ctx(ctx_id) { 301 ctx = &xe->sriov.vf.ccs.contexts[ctx_id]; 302 ctx->ctx_id = ctx_id; 303 304 flags = EXEC_QUEUE_FLAG_KERNEL | 305 EXEC_QUEUE_FLAG_PERMANENT | 306 EXEC_QUEUE_FLAG_MIGRATE; 307 q = xe_exec_queue_create_bind(xe, tile, flags, 0); 308 if (IS_ERR(q)) { 309 err = PTR_ERR(q); 310 goto err_ret; 311 } 312 ctx->mig_q = q; 313 314 err = alloc_bb_pool(tile, ctx); 315 if (err) 316 goto err_free_queue; 317 318 ccs_rw_update_ring(ctx); 319 320 err = register_save_restore_context(ctx); 321 if (err) 322 goto err_free_queue; 323 324 err = devm_add_action_or_reset(xe->drm.dev, 325 xe_sriov_vf_ccs_fini, 326 ctx); 327 if (err) 328 goto err_ret; 329 } 330 331 xe->sriov.vf.ccs.initialized = 1; 332 333 return 0; 334 335 err_free_queue: 336 xe_exec_queue_put(q); 337 338 err_ret: 339 return err; 340 } 341 342 /** 343 * xe_sriov_vf_ccs_attach_bo - Insert CCS read write commands in the BO. 344 * @bo: the &buffer object to which batch buffer commands will be added. 345 * 346 * This function shall be called only by VF. It inserts the PTEs and copy 347 * command instructions in the BO by calling xe_migrate_ccs_rw_copy() 348 * function. 349 * 350 * Returns: 0 if successful, negative error code on failure. 351 */ 352 int xe_sriov_vf_ccs_attach_bo(struct xe_bo *bo) 353 { 354 struct xe_device *xe = xe_bo_device(bo); 355 enum xe_sriov_vf_ccs_rw_ctxs ctx_id; 356 struct xe_sriov_vf_ccs_ctx *ctx; 357 struct xe_tile *tile; 358 struct xe_bb *bb; 359 int err = 0; 360 361 xe_assert(xe, IS_VF_CCS_READY(xe)); 362 363 tile = xe_device_get_root_tile(xe); 364 365 for_each_ccs_rw_ctx(ctx_id) { 366 bb = bo->bb_ccs[ctx_id]; 367 /* bb should be NULL here. Assert if not NULL */ 368 xe_assert(xe, !bb); 369 370 ctx = &xe->sriov.vf.ccs.contexts[ctx_id]; 371 err = xe_migrate_ccs_rw_copy(tile, ctx->mig_q, bo, ctx_id); 372 } 373 return err; 374 } 375 376 /** 377 * xe_sriov_vf_ccs_detach_bo - Remove CCS read write commands from the BO. 378 * @bo: the &buffer object from which batch buffer commands will be removed. 379 * 380 * This function shall be called only by VF. It removes the PTEs and copy 381 * command instructions from the BO. Make sure to update the BB with MI_NOOP 382 * before freeing. 383 * 384 * Returns: 0 if successful. 385 */ 386 int xe_sriov_vf_ccs_detach_bo(struct xe_bo *bo) 387 { 388 struct xe_device *xe = xe_bo_device(bo); 389 enum xe_sriov_vf_ccs_rw_ctxs ctx_id; 390 struct xe_bb *bb; 391 392 xe_assert(xe, IS_VF_CCS_READY(xe)); 393 394 if (!xe_bo_has_valid_ccs_bb(bo)) 395 return 0; 396 397 for_each_ccs_rw_ctx(ctx_id) { 398 bb = bo->bb_ccs[ctx_id]; 399 if (!bb) 400 continue; 401 402 memset(bb->cs, MI_NOOP, bb->len * sizeof(u32)); 403 xe_bb_free(bb, NULL); 404 bo->bb_ccs[ctx_id] = NULL; 405 } 406 return 0; 407 } 408 409 /** 410 * xe_sriov_vf_ccs_print - Print VF CCS details. 411 * @xe: the &xe_device 412 * @p: the &drm_printer 413 * 414 * This function is for VF use only. 415 */ 416 void xe_sriov_vf_ccs_print(struct xe_device *xe, struct drm_printer *p) 417 { 418 struct xe_sa_manager *bb_pool; 419 enum xe_sriov_vf_ccs_rw_ctxs ctx_id; 420 421 if (!IS_VF_CCS_READY(xe)) 422 return; 423 424 xe_pm_runtime_get(xe); 425 426 for_each_ccs_rw_ctx(ctx_id) { 427 bb_pool = xe->sriov.vf.ccs.contexts[ctx_id].mem.ccs_bb_pool; 428 if (!bb_pool) 429 break; 430 431 drm_printf(p, "ccs %s bb suballoc info\n", ctx_id ? "write" : "read"); 432 drm_printf(p, "-------------------------\n"); 433 drm_suballoc_dump_debug_info(&bb_pool->base, p, xe_sa_manager_gpu_addr(bb_pool)); 434 drm_puts(p, "\n"); 435 } 436 437 xe_pm_runtime_put(xe); 438 } 439