1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2025 Intel Corporation
4 */
5
6 #include <drm/drm_drv.h>
7
8 #include "instructions/xe_mi_commands.h"
9 #include "instructions/xe_gpu_commands.h"
10 #include "xe_bb.h"
11 #include "xe_bo.h"
12 #include "xe_device.h"
13 #include "xe_exec_queue.h"
14 #include "xe_exec_queue_types.h"
15 #include "xe_gt_sriov_vf.h"
16 #include "xe_guc.h"
17 #include "xe_guc_submit.h"
18 #include "xe_lrc.h"
19 #include "xe_mem_pool.h"
20 #include "xe_migrate.h"
21 #include "xe_pm.h"
22 #include "xe_sriov_printk.h"
23 #include "xe_sriov_vf.h"
24 #include "xe_sriov_vf_ccs.h"
25 #include "xe_sriov_vf_ccs_types.h"
26
27 /**
28 * DOC: VF save/restore of compression Meta Data
29 *
30 * VF KMD registers two special contexts/LRCAs.
31 *
32 * Save Context/LRCA: contain necessary cmds+page table to trigger Meta data /
33 * compression control surface (Aka CCS) save in regular System memory in VM.
34 *
35 * Restore Context/LRCA: contain necessary cmds+page table to trigger Meta data /
36 * compression control surface (Aka CCS) Restore from regular System memory in
37 * VM to corresponding CCS pool.
38 *
39 * Below diagram explain steps needed for VF save/Restore of compression Meta Data::
40 *
41 * CCS Save CCS Restore VF KMD Guc BCS
42 * LRCA LRCA
43 * | | | | |
44 * | | | | |
45 * | Create Save LRCA | | |
46 * [ ]<----------------------------- [ ] | |
47 * | | | | |
48 * | | | | |
49 * | | | Register save LRCA | |
50 * | | | with Guc | |
51 * | | [ ]--------------------------->[ ] |
52 * | | | | |
53 * | | Create restore LRCA | | |
54 * | [ ]<------------------[ ] | |
55 * | | | | |
56 * | | | Register restore LRCA | |
57 * | | | with Guc | |
58 * | | [ ]--------------------------->[ ] |
59 * | | | | |
60 * | | | | |
61 * | | [ ]------------------------- | |
62 * | | [ ] Allocate main memory. | | |
63 * | | [ ] Allocate CCS memory. | | |
64 * | | [ ] Update Main memory & | | |
65 * [ ]<------------------------------[ ] CCS pages PPGTT + BB | | |
66 * | [ ]<------------------[ ] cmds to save & restore.| | |
67 * | | [ ]<------------------------ | |
68 * | | | | |
69 * | | | | |
70 * | | | | |
71 * : : : : :
72 * ---------------------------- VF Paused -------------------------------------
73 * | | | | |
74 * | | | | |
75 * | | | |Schedule |
76 * | | | |CCS Save |
77 * | | | | LRCA |
78 * | | | [ ]------>[ ]
79 * | | | | |
80 * | | | | |
81 * | | | |CCS save |
82 * | | | |completed|
83 * | | | [ ]<------[ ]
84 * | | | | |
85 * : : : : :
86 * ---------------------------- VM Migrated -----------------------------------
87 * | | | | |
88 * | | | | |
89 * : : : : :
90 * ---------------------------- VF Resumed ------------------------------------
91 * | | | | |
92 * | | | | |
93 * | | [ ]-------------- | |
94 * | | [ ] Fix up GGTT | | |
95 * | | [ ]<------------- | |
96 * | | | | |
97 * | | | | |
98 * | | | Notify VF_RESFIX_DONE | |
99 * | | [ ]--------------------------->[ ] |
100 * | | | | |
101 * | | | |Schedule |
102 * | | | |CCS |
103 * | | | |Restore |
104 * | | | |LRCA |
105 * | | | [ ]------>[ ]
106 * | | | | |
107 * | | | | |
108 * | | | |CCS |
109 * | | | |restore |
110 * | | | |completed|
111 * | | | [ ]<------[ ]
112 * | | | | |
113 * | | | | |
114 * | | | VF_RESFIX_DONE complete | |
115 * | | | notification | |
116 * | | [ ]<---------------------------[ ] |
117 * | | | | |
118 * | | | | |
119 * : : : : :
120 * ------------------------- Continue VM restore ------------------------------
121 */
122
get_ccs_bb_pool_size(struct xe_device * xe)123 static u64 get_ccs_bb_pool_size(struct xe_device *xe)
124 {
125 u64 sys_mem_size, ccs_mem_size, ptes, bb_pool_size;
126 struct sysinfo si;
127
128 si_meminfo(&si);
129 sys_mem_size = si.totalram * si.mem_unit;
130 ccs_mem_size = div64_u64(sys_mem_size, NUM_BYTES_PER_CCS_BYTE(xe));
131 ptes = DIV_ROUND_UP_ULL(sys_mem_size + ccs_mem_size, XE_PAGE_SIZE);
132
133 /**
134 * We need below BB size to hold PTE mappings and some DWs for copy
135 * command. In reality, we need space for many copy commands. So, let
136 * us allocate double the calculated size which is enough to holds GPU
137 * instructions for the whole region.
138 */
139 bb_pool_size = ptes * sizeof(u32);
140
141 return round_up(bb_pool_size * 2, SZ_1M);
142 }
143
alloc_bb_pool(struct xe_tile * tile,struct xe_sriov_vf_ccs_ctx * ctx)144 static int alloc_bb_pool(struct xe_tile *tile, struct xe_sriov_vf_ccs_ctx *ctx)
145 {
146 struct xe_mem_pool *pool;
147 struct xe_device *xe = tile_to_xe(tile);
148 u32 *pool_cpu_addr, *last_dw_addr;
149 u64 bb_pool_size;
150 int err;
151
152 bb_pool_size = get_ccs_bb_pool_size(xe);
153 xe_sriov_info(xe, "Allocating %s CCS BB pool size = %lldMB\n",
154 ctx->ctx_id ? "Restore" : "Save", bb_pool_size / SZ_1M);
155
156 pool = xe_mem_pool_init(tile, bb_pool_size, sizeof(u32),
157 XE_MEM_POOL_BO_FLAG_INIT_SHADOW_COPY);
158 if (IS_ERR(pool)) {
159 xe_sriov_err(xe, "xe_mem_pool_init failed with error: %pe\n",
160 pool);
161 err = PTR_ERR(pool);
162 return err;
163 }
164
165 pool_cpu_addr = xe_mem_pool_cpu_addr(pool);
166 memset(pool_cpu_addr, 0, bb_pool_size);
167
168 last_dw_addr = pool_cpu_addr + (bb_pool_size / sizeof(u32)) - 1;
169 *last_dw_addr = MI_BATCH_BUFFER_END;
170
171 /**
172 * Sync the main copy and shadow copy so that the shadow copy is
173 * replica of main copy. We sync only BBs after init part. So, we
174 * need to make sure the main pool and shadow copy are in sync after
175 * this point. This is needed as GuC may read the BB commands from
176 * shadow copy.
177 */
178 xe_mem_pool_sync(pool);
179
180 ctx->mem.ccs_bb_pool = pool;
181 return 0;
182 }
183
ccs_rw_update_ring(struct xe_sriov_vf_ccs_ctx * ctx)184 static void ccs_rw_update_ring(struct xe_sriov_vf_ccs_ctx *ctx)
185 {
186 u64 addr = xe_mem_pool_gpu_addr(ctx->mem.ccs_bb_pool);
187 struct xe_lrc *lrc = xe_exec_queue_lrc(ctx->mig_q);
188 u32 dw[10], i = 0;
189
190 /*
191 * XXX: Save/restore fixes — for some reason, the GuC only accepts the
192 * save/restore context if the LRC head pointer is zero. This is evident
193 * from repeated VF migrations failing when the LRC head pointer is
194 * non-zero.
195 */
196 lrc->ring.tail = 0;
197 xe_lrc_set_ring_head(lrc, 0);
198
199 dw[i++] = MI_ARB_ON_OFF | MI_ARB_ENABLE;
200 dw[i++] = MI_BATCH_BUFFER_START | XE_INSTR_NUM_DW(3);
201 dw[i++] = lower_32_bits(addr);
202 dw[i++] = upper_32_bits(addr);
203 dw[i++] = MI_NOOP;
204 dw[i++] = MI_NOOP;
205
206 xe_lrc_write_ring(lrc, dw, i * sizeof(u32));
207 xe_lrc_set_ring_tail(lrc, lrc->ring.tail);
208 }
209
210 /**
211 * xe_sriov_vf_ccs_rebase - Rebase GGTT addresses for CCS save / restore
212 * @xe: the &xe_device.
213 */
xe_sriov_vf_ccs_rebase(struct xe_device * xe)214 void xe_sriov_vf_ccs_rebase(struct xe_device *xe)
215 {
216 enum xe_sriov_vf_ccs_rw_ctxs ctx_id;
217
218 if (!IS_VF_CCS_READY(xe))
219 return;
220
221 for_each_ccs_rw_ctx(ctx_id) {
222 struct xe_sriov_vf_ccs_ctx *ctx =
223 &xe->sriov.vf.ccs.contexts[ctx_id];
224
225 ccs_rw_update_ring(ctx);
226 }
227 }
228
register_save_restore_context(struct xe_sriov_vf_ccs_ctx * ctx)229 static int register_save_restore_context(struct xe_sriov_vf_ccs_ctx *ctx)
230 {
231 int ctx_type;
232
233 switch (ctx->ctx_id) {
234 case XE_SRIOV_VF_CCS_READ_CTX:
235 ctx_type = GUC_CONTEXT_COMPRESSION_SAVE;
236 break;
237 case XE_SRIOV_VF_CCS_WRITE_CTX:
238 ctx_type = GUC_CONTEXT_COMPRESSION_RESTORE;
239 break;
240 default:
241 return -EINVAL;
242 }
243
244 xe_guc_register_vf_exec_queue(ctx->mig_q, ctx_type);
245 return 0;
246 }
247
248 /**
249 * xe_sriov_vf_ccs_register_context - Register read/write contexts with guc.
250 * @xe: the &xe_device to register contexts on.
251 *
252 * This function registers read and write contexts with Guc. Re-registration
253 * is needed whenever resuming from pm runtime suspend.
254 *
255 * Return: 0 on success. Negative error code on failure.
256 */
xe_sriov_vf_ccs_register_context(struct xe_device * xe)257 int xe_sriov_vf_ccs_register_context(struct xe_device *xe)
258 {
259 enum xe_sriov_vf_ccs_rw_ctxs ctx_id;
260 struct xe_sriov_vf_ccs_ctx *ctx;
261 int err;
262
263 xe_assert(xe, IS_VF_CCS_READY(xe));
264
265 for_each_ccs_rw_ctx(ctx_id) {
266 ctx = &xe->sriov.vf.ccs.contexts[ctx_id];
267 err = register_save_restore_context(ctx);
268 if (err)
269 return err;
270 }
271
272 return err;
273 }
274
275 /*
276 * Whether GuC requires CCS copy BBs for VF migration.
277 * @xe: the &xe_device instance.
278 *
279 * Only selected platforms require VF KMD to maintain CCS copy BBs and linked LRCAs.
280 *
281 * Return: true if VF driver must participate in the CCS migration, false otherwise.
282 */
vf_migration_ccs_bb_needed(struct xe_device * xe)283 static bool vf_migration_ccs_bb_needed(struct xe_device *xe)
284 {
285 xe_assert(xe, IS_SRIOV_VF(xe));
286
287 return !IS_DGFX(xe) && xe_device_has_flat_ccs(xe);
288 }
289
290 /*
291 * Check for disable migration due to no CCS BBs support in GuC FW.
292 * @xe: the &xe_device instance.
293 *
294 * Performs late disable of VF migration feature in case GuC FW cannot support it.
295 *
296 * Returns: True if VF migration with CCS BBs is supported, false otherwise.
297 */
vf_migration_ccs_bb_support_check(struct xe_device * xe)298 static bool vf_migration_ccs_bb_support_check(struct xe_device *xe)
299 {
300 struct xe_gt *gt = xe_root_mmio_gt(xe);
301 struct xe_uc_fw_version guc_version;
302
303 xe_gt_sriov_vf_guc_versions(gt, NULL, &guc_version);
304 if (MAKE_GUC_VER_STRUCT(guc_version) < MAKE_GUC_VER(1, 23, 0)) {
305 xe_sriov_vf_migration_disable(xe,
306 "CCS migration requires GuC ABI >= 1.23 but only %u.%u found",
307 guc_version.major, guc_version.minor);
308 return false;
309 }
310
311 return true;
312 }
313
xe_sriov_vf_ccs_fini(void * arg)314 static void xe_sriov_vf_ccs_fini(void *arg)
315 {
316 struct xe_sriov_vf_ccs_ctx *ctx = arg;
317 struct xe_lrc *lrc = xe_exec_queue_lrc(ctx->mig_q);
318
319 /*
320 * Make TAIL = HEAD in the ring so that no issues are seen if Guc
321 * submits this context to HW on VF pause after unbinding device.
322 */
323 xe_lrc_set_ring_tail(lrc, xe_lrc_ring_head(lrc));
324 xe_exec_queue_put(ctx->mig_q);
325 }
326
327 /**
328 * xe_sriov_vf_ccs_init - Setup LRCA for save & restore.
329 * @xe: the &xe_device to start recovery on
330 *
331 * This function shall be called only by VF. It initializes
332 * LRCA and suballocator needed for CCS save & restore.
333 *
334 * Return: 0 on success. Negative error code on failure.
335 */
xe_sriov_vf_ccs_init(struct xe_device * xe)336 int xe_sriov_vf_ccs_init(struct xe_device *xe)
337 {
338 struct xe_tile *tile = xe_device_get_root_tile(xe);
339 enum xe_sriov_vf_ccs_rw_ctxs ctx_id;
340 struct xe_sriov_vf_ccs_ctx *ctx;
341 struct xe_exec_queue *q;
342 u32 flags;
343 int err;
344
345 xe_assert(xe, IS_SRIOV_VF(xe));
346
347 if (!xe_sriov_vf_migration_supported(xe) ||
348 !vf_migration_ccs_bb_needed(xe) ||
349 !vf_migration_ccs_bb_support_check(xe))
350 return 0;
351
352 for_each_ccs_rw_ctx(ctx_id) {
353 ctx = &xe->sriov.vf.ccs.contexts[ctx_id];
354 ctx->ctx_id = ctx_id;
355
356 flags = EXEC_QUEUE_FLAG_KERNEL |
357 EXEC_QUEUE_FLAG_PERMANENT |
358 EXEC_QUEUE_FLAG_MIGRATE;
359 q = xe_exec_queue_create_bind(xe, tile, NULL, flags, 0);
360 if (IS_ERR(q)) {
361 err = PTR_ERR(q);
362 goto err_ret;
363 }
364 ctx->mig_q = q;
365
366 err = alloc_bb_pool(tile, ctx);
367 if (err)
368 goto err_free_queue;
369
370 ccs_rw_update_ring(ctx);
371
372 err = register_save_restore_context(ctx);
373 if (err)
374 goto err_free_queue;
375
376 err = devm_add_action_or_reset(xe->drm.dev,
377 xe_sriov_vf_ccs_fini,
378 ctx);
379 if (err)
380 goto err_ret;
381 }
382
383 xe->sriov.vf.ccs.initialized = 1;
384
385 return 0;
386
387 err_free_queue:
388 xe_exec_queue_put(q);
389
390 err_ret:
391 return err;
392 }
393
394 #define XE_SRIOV_VF_CCS_RW_BB_ADDR_OFFSET (2 * sizeof(u32))
xe_sriov_vf_ccs_rw_update_bb_addr(struct xe_sriov_vf_ccs_ctx * ctx)395 void xe_sriov_vf_ccs_rw_update_bb_addr(struct xe_sriov_vf_ccs_ctx *ctx)
396 {
397 u64 addr = xe_mem_pool_gpu_addr(ctx->mem.ccs_bb_pool);
398 struct xe_lrc *lrc = xe_exec_queue_lrc(ctx->mig_q);
399 struct xe_device *xe = gt_to_xe(ctx->mig_q->gt);
400
401 xe_device_wmb(xe);
402 xe_map_wr(xe, &lrc->bo->vmap, XE_SRIOV_VF_CCS_RW_BB_ADDR_OFFSET, u32, addr);
403 xe_device_wmb(xe);
404 }
405
406 /**
407 * xe_sriov_vf_ccs_attach_bo - Insert CCS read write commands in the BO.
408 * @bo: the &buffer object to which batch buffer commands will be added.
409 * @new_mem: the (not yet committed) destination resource @bo is being moved
410 * into; bo->ttm.resource is still the old resource at this point.
411 *
412 * This function shall be called only by VF. It inserts the PTEs and copy
413 * command instructions in the BO by calling xe_migrate_ccs_rw_copy()
414 * function.
415 *
416 * Returns: 0 if successful, negative error code on failure.
417 */
xe_sriov_vf_ccs_attach_bo(struct xe_bo * bo,struct ttm_resource * new_mem)418 int xe_sriov_vf_ccs_attach_bo(struct xe_bo *bo, struct ttm_resource *new_mem)
419 {
420 struct xe_device *xe = xe_bo_device(bo);
421 enum xe_sriov_vf_ccs_rw_ctxs ctx_id;
422 struct xe_sriov_vf_ccs_ctx *ctx;
423 struct xe_mem_pool_node *bb;
424 struct xe_tile *tile;
425 int err = 0;
426
427 xe_assert(xe, IS_VF_CCS_READY(xe));
428
429 tile = xe_device_get_root_tile(xe);
430
431 for_each_ccs_rw_ctx(ctx_id) {
432 bb = bo->bb_ccs[ctx_id];
433 /* bb should be NULL here. Assert if not NULL */
434 xe_assert(xe, !bb);
435
436 ctx = &xe->sriov.vf.ccs.contexts[ctx_id];
437 err = xe_migrate_ccs_rw_copy(tile, ctx->mig_q, bo, new_mem, ctx_id);
438 if (err)
439 goto err_unwind;
440 }
441 return 0;
442
443 err_unwind:
444 /*
445 * Clean up any contexts already attached. Can't reuse
446 * xe_sriov_vf_ccs_detach_bo() here as it requires both contexts
447 * attached before cleaning up either one.
448 */
449 for_each_ccs_rw_ctx(ctx_id) {
450 if (bo->bb_ccs[ctx_id])
451 xe_migrate_ccs_rw_copy_clear(bo, ctx_id, true);
452 }
453 return err;
454 }
455
456 /**
457 * xe_sriov_vf_ccs_detach_bo - Remove CCS read write commands from the BO.
458 * @bo: the &buffer object from which batch buffer commands will be removed.
459 *
460 * This function shall be called only by VF. It removes the PTEs and copy
461 * command instructions from the BO. Make sure to update the BB with MI_NOOP
462 * before freeing.
463 *
464 * Returns: 0 if successful.
465 */
xe_sriov_vf_ccs_detach_bo(struct xe_bo * bo)466 int xe_sriov_vf_ccs_detach_bo(struct xe_bo *bo)
467 {
468 struct xe_device *xe = xe_bo_device(bo);
469 enum xe_sriov_vf_ccs_rw_ctxs ctx_id;
470 struct xe_mem_pool_node *bb;
471 bool bound;
472 int idx;
473
474 xe_assert(xe, IS_VF_CCS_READY(xe));
475
476 if (!xe_bo_has_valid_ccs_bb(bo))
477 return 0;
478
479 bound = drm_dev_enter(&xe->drm, &idx);
480
481 for_each_ccs_rw_ctx(ctx_id) {
482 bb = bo->bb_ccs[ctx_id];
483 if (!bb)
484 continue;
485
486 xe_migrate_ccs_rw_copy_clear(bo, ctx_id, bound);
487 }
488
489 if (bound)
490 drm_dev_exit(idx);
491
492 return 0;
493 }
494
495 /**
496 * xe_sriov_vf_ccs_print - Print VF CCS details.
497 * @xe: the &xe_device
498 * @p: the &drm_printer
499 *
500 * This function is for VF use only.
501 */
xe_sriov_vf_ccs_print(struct xe_device * xe,struct drm_printer * p)502 void xe_sriov_vf_ccs_print(struct xe_device *xe, struct drm_printer *p)
503 {
504 enum xe_sriov_vf_ccs_rw_ctxs ctx_id;
505 struct xe_mem_pool *bb_pool;
506
507 if (!IS_VF_CCS_READY(xe))
508 return;
509
510 guard(xe_pm_runtime)(xe);
511 for_each_ccs_rw_ctx(ctx_id) {
512 bb_pool = xe->sriov.vf.ccs.contexts[ctx_id].mem.ccs_bb_pool;
513 if (!bb_pool)
514 break;
515
516 drm_printf(p, "ccs %s bb suballoc info\n", ctx_id ? "write" : "read");
517 drm_printf(p, "-------------------------\n");
518 xe_mem_pool_dump(bb_pool, p);
519 drm_puts(p, "\n");
520 }
521 }
522