1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2023-2024 Intel Corporation 4 */ 5 6 #include <drm/drm_managed.h> 7 8 #include "regs/xe_guc_regs.h" 9 #include "regs/xe_regs.h" 10 11 #include "xe_gt.h" 12 #include "xe_gt_sriov_pf.h" 13 #include "xe_gt_sriov_pf_config.h" 14 #include "xe_gt_sriov_pf_control.h" 15 #include "xe_gt_sriov_pf_helpers.h" 16 #include "xe_gt_sriov_pf_migration.h" 17 #include "xe_gt_sriov_pf_service.h" 18 #include "xe_gt_sriov_printk.h" 19 #include "xe_guc_submit.h" 20 #include "xe_mmio.h" 21 #include "xe_pm.h" 22 23 static void pf_worker_restart_func(struct work_struct *w); 24 25 /* 26 * VF's metadata is maintained in the flexible array where: 27 * - entry [0] contains metadata for the PF (only if applicable), 28 * - entries [1..n] contain metadata for VF1..VFn:: 29 * 30 * <--------------------------- 1 + total_vfs -----------> 31 * +-------+-------+-------+-----------------------+-------+ 32 * | 0 | 1 | 2 | | n | 33 * +-------+-------+-------+-----------------------+-------+ 34 * | PF | VF1 | VF2 | ... ... | VFn | 35 * +-------+-------+-------+-----------------------+-------+ 36 */ 37 static int pf_alloc_metadata(struct xe_gt *gt) 38 { 39 unsigned int num_vfs = xe_gt_sriov_pf_get_totalvfs(gt); 40 41 gt->sriov.pf.vfs = drmm_kcalloc(>_to_xe(gt)->drm, 1 + num_vfs, 42 sizeof(*gt->sriov.pf.vfs), GFP_KERNEL); 43 if (!gt->sriov.pf.vfs) 44 return -ENOMEM; 45 46 return 0; 47 } 48 49 static void pf_init_workers(struct xe_gt *gt) 50 { 51 xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); 52 INIT_WORK(>->sriov.pf.workers.restart, pf_worker_restart_func); 53 } 54 55 static void pf_fini_workers(struct xe_gt *gt) 56 { 57 xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); 58 59 if (disable_work_sync(>->sriov.pf.workers.restart)) { 60 xe_gt_sriov_dbg_verbose(gt, "pending restart disabled!\n"); 61 /* release an rpm reference taken on the worker's behalf */ 62 xe_pm_runtime_put(gt_to_xe(gt)); 63 } 64 } 65 66 /** 67 * xe_gt_sriov_pf_init_early - Prepare SR-IOV PF data structures on PF. 68 * @gt: the &xe_gt to initialize 69 * 70 * Early initialization of the PF data. 71 * 72 * Return: 0 on success or a negative error code on failure. 73 */ 74 int xe_gt_sriov_pf_init_early(struct xe_gt *gt) 75 { 76 int err; 77 78 err = pf_alloc_metadata(gt); 79 if (err) 80 return err; 81 82 err = xe_gt_sriov_pf_service_init(gt); 83 if (err) 84 return err; 85 86 err = xe_gt_sriov_pf_control_init(gt); 87 if (err) 88 return err; 89 90 pf_init_workers(gt); 91 92 return 0; 93 } 94 95 static void pf_fini_action(void *arg) 96 { 97 struct xe_gt *gt = arg; 98 99 pf_fini_workers(gt); 100 } 101 102 static int pf_init_late(struct xe_gt *gt) 103 { 104 struct xe_device *xe = gt_to_xe(gt); 105 106 xe_gt_assert(gt, IS_SRIOV_PF(xe)); 107 return devm_add_action_or_reset(xe->drm.dev, pf_fini_action, gt); 108 } 109 110 /** 111 * xe_gt_sriov_pf_init - Prepare SR-IOV PF data structures on PF. 112 * @gt: the &xe_gt to initialize 113 * 114 * Late one-time initialization of the PF data. 115 * 116 * Return: 0 on success or a negative error code on failure. 117 */ 118 int xe_gt_sriov_pf_init(struct xe_gt *gt) 119 { 120 int err; 121 122 err = xe_gt_sriov_pf_config_init(gt); 123 if (err) 124 return err; 125 126 err = xe_gt_sriov_pf_migration_init(gt); 127 if (err) 128 return err; 129 130 err = pf_init_late(gt); 131 if (err) 132 return err; 133 134 return 0; 135 } 136 137 static bool pf_needs_enable_ggtt_guest_update(struct xe_device *xe) 138 { 139 return GRAPHICS_VERx100(xe) == 1200; 140 } 141 142 static void pf_enable_ggtt_guest_update(struct xe_gt *gt) 143 { 144 xe_mmio_write32(>->mmio, VIRTUAL_CTRL_REG, GUEST_GTT_UPDATE_EN); 145 } 146 147 /** 148 * xe_gt_sriov_pf_init_hw - Initialize SR-IOV hardware support. 149 * @gt: the &xe_gt to initialize 150 * 151 * On some platforms the PF must explicitly enable VF's access to the GGTT. 152 */ 153 void xe_gt_sriov_pf_init_hw(struct xe_gt *gt) 154 { 155 if (pf_needs_enable_ggtt_guest_update(gt_to_xe(gt))) 156 pf_enable_ggtt_guest_update(gt); 157 158 xe_gt_sriov_pf_service_update(gt); 159 } 160 161 static u32 pf_get_vf_regs_stride(struct xe_device *xe) 162 { 163 return GRAPHICS_VERx100(xe) > 1200 ? 0x400 : 0x1000; 164 } 165 166 static struct xe_reg xe_reg_vf_to_pf(struct xe_reg vf_reg, unsigned int vfid, u32 stride) 167 { 168 struct xe_reg pf_reg = vf_reg; 169 170 pf_reg.vf = 0; 171 pf_reg.addr += stride * vfid; 172 173 return pf_reg; 174 } 175 176 static void pf_clear_vf_scratch_regs(struct xe_gt *gt, unsigned int vfid) 177 { 178 u32 stride = pf_get_vf_regs_stride(gt_to_xe(gt)); 179 struct xe_reg scratch; 180 int n, count; 181 182 if (xe_gt_is_media_type(gt)) { 183 count = MED_VF_SW_FLAG_COUNT; 184 for (n = 0; n < count; n++) { 185 scratch = xe_reg_vf_to_pf(MED_VF_SW_FLAG(n), vfid, stride); 186 xe_mmio_write32(>->mmio, scratch, 0); 187 } 188 } else { 189 count = VF_SW_FLAG_COUNT; 190 for (n = 0; n < count; n++) { 191 scratch = xe_reg_vf_to_pf(VF_SW_FLAG(n), vfid, stride); 192 xe_mmio_write32(>->mmio, scratch, 0); 193 } 194 } 195 } 196 197 /** 198 * xe_gt_sriov_pf_sanitize_hw() - Reset hardware state related to a VF. 199 * @gt: the &xe_gt 200 * @vfid: the VF identifier 201 * 202 * This function can only be called on PF. 203 */ 204 void xe_gt_sriov_pf_sanitize_hw(struct xe_gt *gt, unsigned int vfid) 205 { 206 xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); 207 208 pf_clear_vf_scratch_regs(gt, vfid); 209 } 210 211 static void pf_cancel_restart(struct xe_gt *gt) 212 { 213 xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); 214 215 if (cancel_work_sync(>->sriov.pf.workers.restart)) { 216 xe_gt_sriov_dbg_verbose(gt, "pending restart canceled!\n"); 217 /* release an rpm reference taken on the worker's behalf */ 218 xe_pm_runtime_put(gt_to_xe(gt)); 219 } 220 } 221 222 /** 223 * xe_gt_sriov_pf_stop_prepare() - Prepare to stop SR-IOV support. 224 * @gt: the &xe_gt 225 * 226 * This function can only be called on the PF. 227 */ 228 void xe_gt_sriov_pf_stop_prepare(struct xe_gt *gt) 229 { 230 pf_cancel_restart(gt); 231 } 232 233 static void pf_restart(struct xe_gt *gt) 234 { 235 struct xe_device *xe = gt_to_xe(gt); 236 237 xe_gt_assert(gt, !xe_pm_runtime_suspended(xe)); 238 239 xe_gt_sriov_pf_config_restart(gt); 240 xe_gt_sriov_pf_control_restart(gt); 241 242 /* release an rpm reference taken on our behalf */ 243 xe_pm_runtime_put(xe); 244 245 xe_gt_sriov_dbg(gt, "restart completed\n"); 246 } 247 248 static void pf_worker_restart_func(struct work_struct *w) 249 { 250 struct xe_gt *gt = container_of(w, typeof(*gt), sriov.pf.workers.restart); 251 252 pf_restart(gt); 253 } 254 255 static void pf_queue_restart(struct xe_gt *gt) 256 { 257 struct xe_device *xe = gt_to_xe(gt); 258 259 xe_gt_assert(gt, IS_SRIOV_PF(xe)); 260 261 /* take an rpm reference on behalf of the worker */ 262 xe_pm_runtime_get_noresume(xe); 263 264 if (!queue_work(xe->sriov.wq, >->sriov.pf.workers.restart)) { 265 xe_gt_sriov_dbg(gt, "restart already in queue!\n"); 266 xe_pm_runtime_put(xe); 267 } 268 } 269 270 /** 271 * xe_gt_sriov_pf_restart - Restart SR-IOV support after a GT reset. 272 * @gt: the &xe_gt 273 * 274 * This function can only be called on PF. 275 */ 276 void xe_gt_sriov_pf_restart(struct xe_gt *gt) 277 { 278 pf_queue_restart(gt); 279 } 280 281 static void pf_flush_restart(struct xe_gt *gt) 282 { 283 xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); 284 flush_work(>->sriov.pf.workers.restart); 285 } 286 287 /** 288 * xe_gt_sriov_pf_wait_ready() - Wait until per-GT PF SR-IOV support is ready. 289 * @gt: the &xe_gt 290 * 291 * This function can only be called on PF. 292 * 293 * Return: 0 on success or a negative error code on failure. 294 */ 295 int xe_gt_sriov_pf_wait_ready(struct xe_gt *gt) 296 { 297 /* don't wait if there is another ongoing reset */ 298 if (xe_guc_read_stopped(>->uc.guc)) 299 return -EBUSY; 300 301 pf_flush_restart(gt); 302 return 0; 303 } 304