1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2025 Intel Corporation
4 */
5
6 #include "instructions/xe_mi_commands.h"
7 #include "instructions/xe_gpu_commands.h"
8 #include "xe_bb.h"
9 #include "xe_bo.h"
10 #include "xe_device.h"
11 #include "xe_exec_queue.h"
12 #include "xe_exec_queue_types.h"
13 #include "xe_gt_sriov_vf.h"
14 #include "xe_guc.h"
15 #include "xe_guc_submit.h"
16 #include "xe_lrc.h"
17 #include "xe_migrate.h"
18 #include "xe_pm.h"
19 #include "xe_sa.h"
20 #include "xe_sriov_printk.h"
21 #include "xe_sriov_vf.h"
22 #include "xe_sriov_vf_ccs.h"
23 #include "xe_sriov_vf_ccs_types.h"
24
25 /**
26 * DOC: VF save/restore of compression Meta Data
27 *
28 * VF KMD registers two special contexts/LRCAs.
29 *
30 * Save Context/LRCA: contain necessary cmds+page table to trigger Meta data /
31 * compression control surface (Aka CCS) save in regular System memory in VM.
32 *
33 * Restore Context/LRCA: contain necessary cmds+page table to trigger Meta data /
34 * compression control surface (Aka CCS) Restore from regular System memory in
35 * VM to corresponding CCS pool.
36 *
37 * Below diagram explain steps needed for VF save/Restore of compression Meta Data::
38 *
39 * CCS Save CCS Restore VF KMD Guc BCS
40 * LRCA LRCA
41 * | | | | |
42 * | | | | |
43 * | Create Save LRCA | | |
44 * [ ]<----------------------------- [ ] | |
45 * | | | | |
46 * | | | | |
47 * | | | Register save LRCA | |
48 * | | | with Guc | |
49 * | | [ ]--------------------------->[ ] |
50 * | | | | |
51 * | | Create restore LRCA | | |
52 * | [ ]<------------------[ ] | |
53 * | | | | |
54 * | | | Register restore LRCA | |
55 * | | | with Guc | |
56 * | | [ ]--------------------------->[ ] |
57 * | | | | |
58 * | | | | |
59 * | | [ ]------------------------- | |
60 * | | [ ] Allocate main memory. | | |
61 * | | [ ] Allocate CCS memory. | | |
62 * | | [ ] Update Main memory & | | |
63 * [ ]<------------------------------[ ] CCS pages PPGTT + BB | | |
64 * | [ ]<------------------[ ] cmds to save & restore.| | |
65 * | | [ ]<------------------------ | |
66 * | | | | |
67 * | | | | |
68 * | | | | |
69 * : : : : :
70 * ---------------------------- VF Paused -------------------------------------
71 * | | | | |
72 * | | | | |
73 * | | | |Schedule |
74 * | | | |CCS Save |
75 * | | | | LRCA |
76 * | | | [ ]------>[ ]
77 * | | | | |
78 * | | | | |
79 * | | | |CCS save |
80 * | | | |completed|
81 * | | | [ ]<------[ ]
82 * | | | | |
83 * : : : : :
84 * ---------------------------- VM Migrated -----------------------------------
85 * | | | | |
86 * | | | | |
87 * : : : : :
88 * ---------------------------- VF Resumed ------------------------------------
89 * | | | | |
90 * | | | | |
91 * | | [ ]-------------- | |
92 * | | [ ] Fix up GGTT | | |
93 * | | [ ]<------------- | |
94 * | | | | |
95 * | | | | |
96 * | | | Notify VF_RESFIX_DONE | |
97 * | | [ ]--------------------------->[ ] |
98 * | | | | |
99 * | | | |Schedule |
100 * | | | |CCS |
101 * | | | |Restore |
102 * | | | |LRCA |
103 * | | | [ ]------>[ ]
104 * | | | | |
105 * | | | | |
106 * | | | |CCS |
107 * | | | |restore |
108 * | | | |completed|
109 * | | | [ ]<------[ ]
110 * | | | | |
111 * | | | | |
112 * | | | VF_RESFIX_DONE complete | |
113 * | | | notification | |
114 * | | [ ]<---------------------------[ ] |
115 * | | | | |
116 * | | | | |
117 * : : : : :
118 * ------------------------- Continue VM restore ------------------------------
119 */
120
get_ccs_bb_pool_size(struct xe_device * xe)121 static u64 get_ccs_bb_pool_size(struct xe_device *xe)
122 {
123 u64 sys_mem_size, ccs_mem_size, ptes, bb_pool_size;
124 struct sysinfo si;
125
126 si_meminfo(&si);
127 sys_mem_size = si.totalram * si.mem_unit;
128 ccs_mem_size = div64_u64(sys_mem_size, NUM_BYTES_PER_CCS_BYTE(xe));
129 ptes = DIV_ROUND_UP_ULL(sys_mem_size + ccs_mem_size, XE_PAGE_SIZE);
130
131 /**
132 * We need below BB size to hold PTE mappings and some DWs for copy
133 * command. In reality, we need space for many copy commands. So, let
134 * us allocate double the calculated size which is enough to holds GPU
135 * instructions for the whole region.
136 */
137 bb_pool_size = ptes * sizeof(u32);
138
139 return round_up(bb_pool_size * 2, SZ_1M);
140 }
141
alloc_bb_pool(struct xe_tile * tile,struct xe_sriov_vf_ccs_ctx * ctx)142 static int alloc_bb_pool(struct xe_tile *tile, struct xe_sriov_vf_ccs_ctx *ctx)
143 {
144 struct xe_device *xe = tile_to_xe(tile);
145 struct xe_sa_manager *sa_manager;
146 u64 bb_pool_size;
147 int offset, err;
148
149 bb_pool_size = get_ccs_bb_pool_size(xe);
150 xe_sriov_info(xe, "Allocating %s CCS BB pool size = %lldMB\n",
151 ctx->ctx_id ? "Restore" : "Save", bb_pool_size / SZ_1M);
152
153 sa_manager = xe_sa_bo_manager_init(tile, bb_pool_size, SZ_16);
154
155 if (IS_ERR(sa_manager)) {
156 xe_sriov_err(xe, "Suballocator init failed with error: %pe\n",
157 sa_manager);
158 err = PTR_ERR(sa_manager);
159 return err;
160 }
161
162 offset = 0;
163 xe_map_memset(xe, &sa_manager->bo->vmap, offset, MI_NOOP,
164 bb_pool_size);
165
166 offset = bb_pool_size - sizeof(u32);
167 xe_map_wr(xe, &sa_manager->bo->vmap, offset, u32, MI_BATCH_BUFFER_END);
168
169 ctx->mem.ccs_bb_pool = sa_manager;
170
171 return 0;
172 }
173
ccs_rw_update_ring(struct xe_sriov_vf_ccs_ctx * ctx)174 static void ccs_rw_update_ring(struct xe_sriov_vf_ccs_ctx *ctx)
175 {
176 u64 addr = xe_sa_manager_gpu_addr(ctx->mem.ccs_bb_pool);
177 struct xe_lrc *lrc = xe_exec_queue_lrc(ctx->mig_q);
178 u32 dw[10], i = 0;
179
180 /*
181 * XXX: Save/restore fixes — for some reason, the GuC only accepts the
182 * save/restore context if the LRC head pointer is zero. This is evident
183 * from repeated VF migrations failing when the LRC head pointer is
184 * non-zero.
185 */
186 lrc->ring.tail = 0;
187 xe_lrc_set_ring_head(lrc, 0);
188
189 dw[i++] = MI_ARB_ON_OFF | MI_ARB_ENABLE;
190 dw[i++] = MI_BATCH_BUFFER_START | XE_INSTR_NUM_DW(3);
191 dw[i++] = lower_32_bits(addr);
192 dw[i++] = upper_32_bits(addr);
193 dw[i++] = MI_NOOP;
194 dw[i++] = MI_NOOP;
195
196 xe_lrc_write_ring(lrc, dw, i * sizeof(u32));
197 xe_lrc_set_ring_tail(lrc, lrc->ring.tail);
198 }
199
200 /**
201 * xe_sriov_vf_ccs_rebase - Rebase GGTT addresses for CCS save / restore
202 * @xe: the &xe_device.
203 */
xe_sriov_vf_ccs_rebase(struct xe_device * xe)204 void xe_sriov_vf_ccs_rebase(struct xe_device *xe)
205 {
206 enum xe_sriov_vf_ccs_rw_ctxs ctx_id;
207
208 if (!IS_VF_CCS_READY(xe))
209 return;
210
211 for_each_ccs_rw_ctx(ctx_id) {
212 struct xe_sriov_vf_ccs_ctx *ctx =
213 &xe->sriov.vf.ccs.contexts[ctx_id];
214
215 ccs_rw_update_ring(ctx);
216 }
217 }
218
register_save_restore_context(struct xe_sriov_vf_ccs_ctx * ctx)219 static int register_save_restore_context(struct xe_sriov_vf_ccs_ctx *ctx)
220 {
221 int ctx_type;
222
223 switch (ctx->ctx_id) {
224 case XE_SRIOV_VF_CCS_READ_CTX:
225 ctx_type = GUC_CONTEXT_COMPRESSION_SAVE;
226 break;
227 case XE_SRIOV_VF_CCS_WRITE_CTX:
228 ctx_type = GUC_CONTEXT_COMPRESSION_RESTORE;
229 break;
230 default:
231 return -EINVAL;
232 }
233
234 xe_guc_register_vf_exec_queue(ctx->mig_q, ctx_type);
235 return 0;
236 }
237
238 /**
239 * xe_sriov_vf_ccs_register_context - Register read/write contexts with guc.
240 * @xe: the &xe_device to register contexts on.
241 *
242 * This function registers read and write contexts with Guc. Re-registration
243 * is needed whenever resuming from pm runtime suspend.
244 *
245 * Return: 0 on success. Negative error code on failure.
246 */
xe_sriov_vf_ccs_register_context(struct xe_device * xe)247 int xe_sriov_vf_ccs_register_context(struct xe_device *xe)
248 {
249 enum xe_sriov_vf_ccs_rw_ctxs ctx_id;
250 struct xe_sriov_vf_ccs_ctx *ctx;
251 int err;
252
253 xe_assert(xe, IS_VF_CCS_READY(xe));
254
255 for_each_ccs_rw_ctx(ctx_id) {
256 ctx = &xe->sriov.vf.ccs.contexts[ctx_id];
257 err = register_save_restore_context(ctx);
258 if (err)
259 return err;
260 }
261
262 return err;
263 }
264
265 /*
266 * Whether GuC requires CCS copy BBs for VF migration.
267 * @xe: the &xe_device instance.
268 *
269 * Only selected platforms require VF KMD to maintain CCS copy BBs and linked LRCAs.
270 *
271 * Return: true if VF driver must participate in the CCS migration, false otherwise.
272 */
vf_migration_ccs_bb_needed(struct xe_device * xe)273 static bool vf_migration_ccs_bb_needed(struct xe_device *xe)
274 {
275 xe_assert(xe, IS_SRIOV_VF(xe));
276
277 return !IS_DGFX(xe) && xe_device_has_flat_ccs(xe);
278 }
279
280 /*
281 * Check for disable migration due to no CCS BBs support in GuC FW.
282 * @xe: the &xe_device instance.
283 *
284 * Performs late disable of VF migration feature in case GuC FW cannot support it.
285 *
286 * Returns: True if VF migration with CCS BBs is supported, false otherwise.
287 */
vf_migration_ccs_bb_support_check(struct xe_device * xe)288 static bool vf_migration_ccs_bb_support_check(struct xe_device *xe)
289 {
290 struct xe_gt *gt = xe_root_mmio_gt(xe);
291 struct xe_uc_fw_version guc_version;
292
293 xe_gt_sriov_vf_guc_versions(gt, NULL, &guc_version);
294 if (MAKE_GUC_VER_STRUCT(guc_version) < MAKE_GUC_VER(1, 23, 0)) {
295 xe_sriov_vf_migration_disable(xe,
296 "CCS migration requires GuC ABI >= 1.23 but only %u.%u found",
297 guc_version.major, guc_version.minor);
298 return false;
299 }
300
301 return true;
302 }
303
xe_sriov_vf_ccs_fini(void * arg)304 static void xe_sriov_vf_ccs_fini(void *arg)
305 {
306 struct xe_sriov_vf_ccs_ctx *ctx = arg;
307 struct xe_lrc *lrc = xe_exec_queue_lrc(ctx->mig_q);
308
309 /*
310 * Make TAIL = HEAD in the ring so that no issues are seen if Guc
311 * submits this context to HW on VF pause after unbinding device.
312 */
313 xe_lrc_set_ring_tail(lrc, xe_lrc_ring_head(lrc));
314 xe_exec_queue_put(ctx->mig_q);
315 }
316
317 /**
318 * xe_sriov_vf_ccs_init - Setup LRCA for save & restore.
319 * @xe: the &xe_device to start recovery on
320 *
321 * This function shall be called only by VF. It initializes
322 * LRCA and suballocator needed for CCS save & restore.
323 *
324 * Return: 0 on success. Negative error code on failure.
325 */
xe_sriov_vf_ccs_init(struct xe_device * xe)326 int xe_sriov_vf_ccs_init(struct xe_device *xe)
327 {
328 struct xe_tile *tile = xe_device_get_root_tile(xe);
329 enum xe_sriov_vf_ccs_rw_ctxs ctx_id;
330 struct xe_sriov_vf_ccs_ctx *ctx;
331 struct xe_exec_queue *q;
332 u32 flags;
333 int err;
334
335 xe_assert(xe, IS_SRIOV_VF(xe));
336
337 if (!xe_sriov_vf_migration_supported(xe) ||
338 !vf_migration_ccs_bb_needed(xe) ||
339 !vf_migration_ccs_bb_support_check(xe))
340 return 0;
341
342 for_each_ccs_rw_ctx(ctx_id) {
343 ctx = &xe->sriov.vf.ccs.contexts[ctx_id];
344 ctx->ctx_id = ctx_id;
345
346 flags = EXEC_QUEUE_FLAG_KERNEL |
347 EXEC_QUEUE_FLAG_PERMANENT |
348 EXEC_QUEUE_FLAG_MIGRATE;
349 q = xe_exec_queue_create_bind(xe, tile, NULL, flags, 0);
350 if (IS_ERR(q)) {
351 err = PTR_ERR(q);
352 goto err_ret;
353 }
354 ctx->mig_q = q;
355
356 err = alloc_bb_pool(tile, ctx);
357 if (err)
358 goto err_free_queue;
359
360 ccs_rw_update_ring(ctx);
361
362 err = register_save_restore_context(ctx);
363 if (err)
364 goto err_free_queue;
365
366 err = devm_add_action_or_reset(xe->drm.dev,
367 xe_sriov_vf_ccs_fini,
368 ctx);
369 if (err)
370 goto err_ret;
371 }
372
373 xe->sriov.vf.ccs.initialized = 1;
374
375 return 0;
376
377 err_free_queue:
378 xe_exec_queue_put(q);
379
380 err_ret:
381 return err;
382 }
383
384 /**
385 * xe_sriov_vf_ccs_attach_bo - Insert CCS read write commands in the BO.
386 * @bo: the &buffer object to which batch buffer commands will be added.
387 *
388 * This function shall be called only by VF. It inserts the PTEs and copy
389 * command instructions in the BO by calling xe_migrate_ccs_rw_copy()
390 * function.
391 *
392 * Returns: 0 if successful, negative error code on failure.
393 */
xe_sriov_vf_ccs_attach_bo(struct xe_bo * bo)394 int xe_sriov_vf_ccs_attach_bo(struct xe_bo *bo)
395 {
396 struct xe_device *xe = xe_bo_device(bo);
397 enum xe_sriov_vf_ccs_rw_ctxs ctx_id;
398 struct xe_sriov_vf_ccs_ctx *ctx;
399 struct xe_tile *tile;
400 struct xe_bb *bb;
401 int err = 0;
402
403 xe_assert(xe, IS_VF_CCS_READY(xe));
404
405 tile = xe_device_get_root_tile(xe);
406
407 for_each_ccs_rw_ctx(ctx_id) {
408 bb = bo->bb_ccs[ctx_id];
409 /* bb should be NULL here. Assert if not NULL */
410 xe_assert(xe, !bb);
411
412 ctx = &xe->sriov.vf.ccs.contexts[ctx_id];
413 err = xe_migrate_ccs_rw_copy(tile, ctx->mig_q, bo, ctx_id);
414 }
415 return err;
416 }
417
418 /**
419 * xe_sriov_vf_ccs_detach_bo - Remove CCS read write commands from the BO.
420 * @bo: the &buffer object from which batch buffer commands will be removed.
421 *
422 * This function shall be called only by VF. It removes the PTEs and copy
423 * command instructions from the BO. Make sure to update the BB with MI_NOOP
424 * before freeing.
425 *
426 * Returns: 0 if successful.
427 */
xe_sriov_vf_ccs_detach_bo(struct xe_bo * bo)428 int xe_sriov_vf_ccs_detach_bo(struct xe_bo *bo)
429 {
430 struct xe_device *xe = xe_bo_device(bo);
431 enum xe_sriov_vf_ccs_rw_ctxs ctx_id;
432 struct xe_bb *bb;
433
434 xe_assert(xe, IS_VF_CCS_READY(xe));
435
436 if (!xe_bo_has_valid_ccs_bb(bo))
437 return 0;
438
439 for_each_ccs_rw_ctx(ctx_id) {
440 bb = bo->bb_ccs[ctx_id];
441 if (!bb)
442 continue;
443
444 memset(bb->cs, MI_NOOP, bb->len * sizeof(u32));
445 xe_bb_free(bb, NULL);
446 bo->bb_ccs[ctx_id] = NULL;
447 }
448 return 0;
449 }
450
451 /**
452 * xe_sriov_vf_ccs_print - Print VF CCS details.
453 * @xe: the &xe_device
454 * @p: the &drm_printer
455 *
456 * This function is for VF use only.
457 */
xe_sriov_vf_ccs_print(struct xe_device * xe,struct drm_printer * p)458 void xe_sriov_vf_ccs_print(struct xe_device *xe, struct drm_printer *p)
459 {
460 struct xe_sa_manager *bb_pool;
461 enum xe_sriov_vf_ccs_rw_ctxs ctx_id;
462
463 if (!IS_VF_CCS_READY(xe))
464 return;
465
466 xe_pm_runtime_get(xe);
467
468 for_each_ccs_rw_ctx(ctx_id) {
469 bb_pool = xe->sriov.vf.ccs.contexts[ctx_id].mem.ccs_bb_pool;
470 if (!bb_pool)
471 break;
472
473 drm_printf(p, "ccs %s bb suballoc info\n", ctx_id ? "write" : "read");
474 drm_printf(p, "-------------------------\n");
475 drm_suballoc_dump_debug_info(&bb_pool->base, p, xe_sa_manager_gpu_addr(bb_pool));
476 drm_puts(p, "\n");
477 }
478
479 xe_pm_runtime_put(xe);
480 }
481