1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2023-2024 Intel Corporation
4 */
5
6 #include <drm/drm_managed.h>
7
8 #include "xe_assert.h"
9 #include "xe_device.h"
10 #include "xe_gt.h"
11 #include "xe_gt_sriov_printk.h"
12 #include "xe_gt_sriov_vf.h"
13 #include "xe_guc_ct.h"
14 #include "xe_pm.h"
15 #include "xe_sriov.h"
16 #include "xe_sriov_printk.h"
17 #include "xe_sriov_vf.h"
18 #include "xe_tile_sriov_vf.h"
19
20 /**
21 * DOC: VF restore procedure in PF KMD and VF KMD
22 *
23 * Restoring previously saved state of a VF is one of core features of
24 * SR-IOV. All major VM Management applications allow saving and restoring
25 * the VM state, and doing that to a VM which uses SRIOV VF as one of
26 * the accessible devices requires support from KMD on both PF and VF side.
27 * VMM initiates all required operations through VFIO module, which then
28 * translates them into PF KMD calls. This description will focus on these
29 * calls, leaving out the module which initiates these steps (VFIO).
30 *
31 * In order to start the restore procedure, GuC needs to keep the VF in
32 * proper state. The PF driver can ensure GuC set it to VF_READY state
33 * by provisioning the VF, which in turn can be done after Function Level
34 * Reset of said VF (or after it was freshly created - in that case FLR
35 * is not needed). The FLR procedure ends with GuC sending message
36 * `GUC_PF_NOTIFY_VF_FLR_DONE`, and then provisioning data is sent to GuC.
37 * After the provisioning is completed, the VF needs to be paused, and
38 * at that point the actual restore can begin.
39 *
40 * During VF Restore, state of several resources is restored. These may
41 * include local memory content (system memory is restored by VMM itself),
42 * values of MMIO registers, stateless compression metadata and others.
43 * The final resource which also needs restoring is state of the VF
44 * submission maintained within GuC. For that, `GUC_PF_OPCODE_VF_RESTORE`
45 * message is used, with reference to the state blob to be consumed by
46 * GuC.
47 *
48 * Next, when VFIO is asked to set the VM into running state, the PF driver
49 * sends `GUC_PF_TRIGGER_VF_RESUME` to GuC. When sent after restore, this
50 * changes VF state within GuC to `VF_RESFIX_BLOCKED` rather than the
51 * usual `VF_RUNNING`. At this point GuC triggers an interrupt to inform
52 * the VF KMD within the VM that it was migrated.
53 *
54 * As soon as Virtual GPU of the VM starts, the VF driver within receives
55 * the MIGRATED interrupt and schedules post-migration recovery worker.
56 * That worker queries GuC for new provisioning (using MMIO communication),
57 * and applies fixups to any non-virtualized resources used by the VF.
58 *
59 * When the VF driver is ready to continue operation on the newly connected
60 * hardware, it sends `VF2GUC_NOTIFY_RESFIX_DONE` which causes it to
61 * enter the long awaited `VF_RUNNING` state, and therefore start handling
62 * CTB messages and scheduling workloads from the VF::
63 *
64 * PF GuC VF
65 * [ ] | |
66 * [ ] PF2GUC_VF_CONTROL(pause) | |
67 * [ ]---------------------------> [ ] |
68 * [ ] [ ] GuC sets new VF state to |
69 * [ ] [ ]------- VF_READY_PAUSED |
70 * [ ] [ ] | |
71 * [ ] [ ] <----- |
72 * [ ] success [ ] |
73 * [ ] <---------------------------[ ] |
74 * [ ] | |
75 * [ ] PF loads resources from the | |
76 * [ ]------- saved image supplied | |
77 * [ ] | | |
78 * [ ] <----- | |
79 * [ ] | |
80 * [ ] GUC_PF_OPCODE_VF_RESTORE | |
81 * [ ]---------------------------> [ ] |
82 * [ ] [ ] GuC loads contexts and CTB |
83 * [ ] [ ]------- state from image |
84 * [ ] [ ] | |
85 * [ ] [ ] <----- |
86 * [ ] [ ] |
87 * [ ] [ ] GuC sets new VF state to |
88 * [ ] [ ]------- VF_RESFIX_PAUSED |
89 * [ ] [ ] | |
90 * [ ] success [ ] <----- |
91 * [ ] <---------------------------[ ] |
92 * [ ] | |
93 * [ ] GUC_PF_TRIGGER_VF_RESUME | |
94 * [ ]---------------------------> [ ] |
95 * [ ] [ ] GuC sets new VF state to |
96 * [ ] [ ]------- VF_RESFIX_BLOCKED |
97 * [ ] [ ] | |
98 * [ ] [ ] <----- |
99 * [ ] [ ] |
100 * [ ] [ ] GUC_INTR_SW_INT_0 |
101 * [ ] success [ ]---------------------------> [ ]
102 * [ ] <---------------------------[ ] [ ]
103 * | | VF2GUC_QUERY_SINGLE_KLV [ ]
104 * | [ ] <---------------------------[ ]
105 * | [ ] [ ]
106 * | [ ] new VF provisioning [ ]
107 * | [ ]---------------------------> [ ]
108 * | | [ ]
109 * | | VF driver applies post [ ]
110 * | | migration fixups -------[ ]
111 * | | | [ ]
112 * | | -----> [ ]
113 * | | [ ]
114 * | | VF2GUC_NOTIFY_RESFIX_DONE [ ]
115 * | [ ] <---------------------------[ ]
116 * | [ ] [ ]
117 * | [ ] GuC sets new VF state to [ ]
118 * | [ ]------- VF_RUNNING [ ]
119 * | [ ] | [ ]
120 * | [ ] <----- [ ]
121 * | [ ] success [ ]
122 * | [ ]---------------------------> [ ]
123 * | | |
124 * | | |
125 */
126
vf_migration_supported(struct xe_device * xe)127 static bool vf_migration_supported(struct xe_device *xe)
128 {
129 /*
130 * TODO: Add conditions to allow specific platforms, when they're
131 * supported at production quality.
132 */
133 return IS_ENABLED(CONFIG_DRM_XE_DEBUG);
134 }
135
136 static void migration_worker_func(struct work_struct *w);
137
138 /**
139 * xe_sriov_vf_init_early - Initialize SR-IOV VF specific data.
140 * @xe: the &xe_device to initialize
141 */
xe_sriov_vf_init_early(struct xe_device * xe)142 void xe_sriov_vf_init_early(struct xe_device *xe)
143 {
144 INIT_WORK(&xe->sriov.vf.migration.worker, migration_worker_func);
145
146 if (!vf_migration_supported(xe))
147 xe_sriov_info(xe, "migration not supported by this module version\n");
148 }
149
gt_vf_post_migration_needed(struct xe_gt * gt)150 static bool gt_vf_post_migration_needed(struct xe_gt *gt)
151 {
152 return test_bit(gt->info.id, >_to_xe(gt)->sriov.vf.migration.gt_flags);
153 }
154
155 /*
156 * Notify GuCs marked in flags about resource fixups apply finished.
157 * @xe: the &xe_device struct instance
158 * @gt_flags: flags marking to which GTs the notification shall be sent
159 */
vf_post_migration_notify_resfix_done(struct xe_device * xe,unsigned long gt_flags)160 static int vf_post_migration_notify_resfix_done(struct xe_device *xe, unsigned long gt_flags)
161 {
162 struct xe_gt *gt;
163 unsigned int id;
164 int err = 0;
165
166 for_each_gt(gt, xe, id) {
167 if (!test_bit(id, >_flags))
168 continue;
169 /* skip asking GuC for RESFIX exit if new recovery request arrived */
170 if (gt_vf_post_migration_needed(gt))
171 continue;
172 err = xe_gt_sriov_vf_notify_resfix_done(gt);
173 if (err)
174 break;
175 clear_bit(id, >_flags);
176 }
177
178 if (gt_flags && !err)
179 drm_dbg(&xe->drm, "another recovery imminent, skipped some notifications\n");
180 return err;
181 }
182
vf_get_next_migrated_gt_id(struct xe_device * xe)183 static int vf_get_next_migrated_gt_id(struct xe_device *xe)
184 {
185 struct xe_gt *gt;
186 unsigned int id;
187
188 for_each_gt(gt, xe, id) {
189 if (test_and_clear_bit(id, &xe->sriov.vf.migration.gt_flags))
190 return id;
191 }
192 return -1;
193 }
194
195 /**
196 * Perform post-migration fixups on a single GT.
197 *
198 * After migration, GuC needs to be re-queried for VF configuration to check
199 * if it matches previous provisioning. Most of VF provisioning shall be the
200 * same, except GGTT range, since GGTT is not virtualized per-VF. If GGTT
201 * range has changed, we have to perform fixups - shift all GGTT references
202 * used anywhere within the driver. After the fixups in this function succeed,
203 * it is allowed to ask the GuC bound to this GT to continue normal operation.
204 *
205 * Returns: 0 if the operation completed successfully, or a negative error
206 * code otherwise.
207 */
gt_vf_post_migration_fixups(struct xe_gt * gt)208 static int gt_vf_post_migration_fixups(struct xe_gt *gt)
209 {
210 s64 shift;
211 int err;
212
213 err = xe_gt_sriov_vf_query_config(gt);
214 if (err)
215 return err;
216
217 shift = xe_gt_sriov_vf_ggtt_shift(gt);
218 if (shift) {
219 xe_tile_sriov_vf_fixup_ggtt_nodes(gt_to_tile(gt), shift);
220 /* FIXME: add the recovery steps */
221 xe_guc_ct_fixup_messages_with_ggtt(>->uc.guc.ct, shift);
222 }
223 return 0;
224 }
225
vf_post_migration_recovery(struct xe_device * xe)226 static void vf_post_migration_recovery(struct xe_device *xe)
227 {
228 unsigned long fixed_gts = 0;
229 int id, err;
230
231 drm_dbg(&xe->drm, "migration recovery in progress\n");
232 xe_pm_runtime_get(xe);
233
234 if (!vf_migration_supported(xe)) {
235 xe_sriov_err(xe, "migration not supported by this module version\n");
236 err = -ENOTRECOVERABLE;
237 goto fail;
238 }
239
240 while (id = vf_get_next_migrated_gt_id(xe), id >= 0) {
241 struct xe_gt *gt = xe_device_get_gt(xe, id);
242
243 err = gt_vf_post_migration_fixups(gt);
244 if (err)
245 goto fail;
246
247 set_bit(id, &fixed_gts);
248 }
249
250 err = vf_post_migration_notify_resfix_done(xe, fixed_gts);
251 if (err)
252 goto fail;
253
254 xe_pm_runtime_put(xe);
255 drm_notice(&xe->drm, "migration recovery ended\n");
256 return;
257 fail:
258 xe_pm_runtime_put(xe);
259 drm_err(&xe->drm, "migration recovery failed (%pe)\n", ERR_PTR(err));
260 xe_device_declare_wedged(xe);
261 }
262
migration_worker_func(struct work_struct * w)263 static void migration_worker_func(struct work_struct *w)
264 {
265 struct xe_device *xe = container_of(w, struct xe_device,
266 sriov.vf.migration.worker);
267
268 vf_post_migration_recovery(xe);
269 }
270
271 /*
272 * Check if post-restore recovery is coming on any of GTs.
273 * @xe: the &xe_device struct instance
274 *
275 * Return: True if migration recovery worker will soon be running. Any worker currently
276 * executing does not affect the result.
277 */
vf_ready_to_recovery_on_any_gts(struct xe_device * xe)278 static bool vf_ready_to_recovery_on_any_gts(struct xe_device *xe)
279 {
280 struct xe_gt *gt;
281 unsigned int id;
282
283 for_each_gt(gt, xe, id) {
284 if (test_bit(id, &xe->sriov.vf.migration.gt_flags))
285 return true;
286 }
287 return false;
288 }
289
290 /**
291 * xe_sriov_vf_start_migration_recovery - Start VF migration recovery.
292 * @xe: the &xe_device to start recovery on
293 *
294 * This function shall be called only by VF.
295 */
xe_sriov_vf_start_migration_recovery(struct xe_device * xe)296 void xe_sriov_vf_start_migration_recovery(struct xe_device *xe)
297 {
298 bool started;
299
300 xe_assert(xe, IS_SRIOV_VF(xe));
301
302 if (!vf_ready_to_recovery_on_any_gts(xe))
303 return;
304
305 started = queue_work(xe->sriov.wq, &xe->sriov.vf.migration.worker);
306 drm_info(&xe->drm, "VF migration recovery %s\n", started ?
307 "scheduled" : "already in progress");
308 }
309