1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2023 Intel Corporation 4 */ 5 6 #include <drm/drm_managed.h> 7 8 #include "regs/xe_gt_regs.h" 9 #include "regs/xe_guc_regs.h" 10 #include "regs/xe_regs.h" 11 12 #include "xe_assert.h" 13 #include "xe_bo.h" 14 #include "xe_device.h" 15 #include "xe_device_types.h" 16 #include "xe_gt.h" 17 #include "xe_gt_printk.h" 18 #include "xe_guc.h" 19 #include "xe_hw_engine.h" 20 #include "xe_map.h" 21 #include "xe_memirq.h" 22 #include "xe_sriov.h" 23 #include "xe_sriov_printk.h" 24 25 #define memirq_assert(m, condition) xe_tile_assert(memirq_to_tile(m), condition) 26 #define memirq_debug(m, msg...) xe_sriov_dbg_verbose(memirq_to_xe(m), "MEMIRQ: " msg) 27 28 static struct xe_tile *memirq_to_tile(struct xe_memirq *memirq) 29 { 30 return container_of(memirq, struct xe_tile, sriov.vf.memirq); 31 } 32 33 static struct xe_device *memirq_to_xe(struct xe_memirq *memirq) 34 { 35 return tile_to_xe(memirq_to_tile(memirq)); 36 } 37 38 static const char *guc_name(struct xe_guc *guc) 39 { 40 return xe_gt_is_media_type(guc_to_gt(guc)) ? "media GuC" : "GuC"; 41 } 42 43 /** 44 * DOC: Memory Based Interrupts 45 * 46 * MMIO register based interrupts infrastructure used for non-virtualized mode 47 * or SRIOV-8 (which supports 8 Virtual Functions) does not scale efficiently 48 * to allow delivering interrupts to a large number of Virtual machines or 49 * containers. Memory based interrupt status reporting provides an efficient 50 * and scalable infrastructure. 51 * 52 * For memory based interrupt status reporting hardware sequence is: 53 * * Engine writes the interrupt event to memory 54 * (Pointer to memory location is provided by SW. This memory surface must 55 * be mapped to system memory and must be marked as un-cacheable (UC) on 56 * Graphics IP Caches) 57 * * Engine triggers an interrupt to host. 58 */ 59 60 /** 61 * DOC: Memory Based Interrupts Page Layout 62 * 63 * `Memory Based Interrupts`_ requires three different objects, which are 64 * called "page" in the specs, even if they aren't page-sized or aligned. 65 * 66 * To simplify the code we allocate a single page size object and then use 67 * offsets to embedded "pages". The address of those "pages" are then 68 * programmed in the HW via LRI and LRM in the context image. 69 * 70 * - _`Interrupt Status Report Page`: this page contains the interrupt 71 * status vectors for each unit. Each bit in the interrupt vectors is 72 * converted to a byte, with the byte being set to 0xFF when an 73 * interrupt is triggered; interrupt vectors are 16b big so each unit 74 * gets 16B. One space is reserved for each bit in one of the 75 * GT_INTR_DWx registers, so this object needs a total of 1024B. 76 * This object needs to be 4KiB aligned. 77 * 78 * - _`Interrupt Source Report Page`: this is the equivalent of the 79 * GEN11_GT_INTR_DWx registers, with each bit in those registers being 80 * mapped to a byte here. The offsets are the same, just bytes instead 81 * of bits. This object needs to be cacheline aligned. 82 * 83 * - Interrupt Mask: the HW needs a location to fetch the interrupt 84 * mask vector to be used by the LRM in the context, so we just use 85 * the next available space in the interrupt page. 86 * 87 * :: 88 * 89 * 0x0000 +===========+ <== Interrupt Status Report Page 90 * | | 91 * | | ____ +----+----------------+ 92 * | | / | 0 | USER INTERRUPT | 93 * +-----------+ __/ | 1 | | 94 * | HWE(n) | __ | | CTX SWITCH | 95 * +-----------+ \ | | WAIT SEMAPHORE | 96 * | | \____ | 15 | | 97 * | | +----+----------------+ 98 * | | 99 * 0x0400 +===========+ <== Interrupt Source Report Page 100 * | HWE(0) | 101 * | HWE(1) | 102 * | | 103 * | HWE(x) | 104 * 0x0440 +===========+ <== Interrupt Enable Mask 105 * | | 106 * | | 107 * +-----------+ 108 */ 109 110 static void __release_xe_bo(struct drm_device *drm, void *arg) 111 { 112 struct xe_bo *bo = arg; 113 114 xe_bo_unpin_map_no_vm(bo); 115 } 116 117 static int memirq_alloc_pages(struct xe_memirq *memirq) 118 { 119 struct xe_device *xe = memirq_to_xe(memirq); 120 struct xe_tile *tile = memirq_to_tile(memirq); 121 struct xe_bo *bo; 122 int err; 123 124 BUILD_BUG_ON(!IS_ALIGNED(XE_MEMIRQ_SOURCE_OFFSET, SZ_64)); 125 BUILD_BUG_ON(!IS_ALIGNED(XE_MEMIRQ_STATUS_OFFSET, SZ_4K)); 126 127 /* XXX: convert to managed bo */ 128 bo = xe_bo_create_pin_map(xe, tile, NULL, SZ_4K, 129 ttm_bo_type_kernel, 130 XE_BO_FLAG_SYSTEM | 131 XE_BO_FLAG_GGTT | 132 XE_BO_FLAG_GGTT_INVALIDATE | 133 XE_BO_FLAG_NEEDS_UC | 134 XE_BO_FLAG_NEEDS_CPU_ACCESS); 135 if (IS_ERR(bo)) { 136 err = PTR_ERR(bo); 137 goto out; 138 } 139 140 memirq_assert(memirq, !xe_bo_is_vram(bo)); 141 memirq_assert(memirq, !memirq->bo); 142 143 iosys_map_memset(&bo->vmap, 0, 0, SZ_4K); 144 145 memirq->bo = bo; 146 memirq->source = IOSYS_MAP_INIT_OFFSET(&bo->vmap, XE_MEMIRQ_SOURCE_OFFSET); 147 memirq->status = IOSYS_MAP_INIT_OFFSET(&bo->vmap, XE_MEMIRQ_STATUS_OFFSET); 148 memirq->mask = IOSYS_MAP_INIT_OFFSET(&bo->vmap, XE_MEMIRQ_ENABLE_OFFSET); 149 150 memirq_assert(memirq, !memirq->source.is_iomem); 151 memirq_assert(memirq, !memirq->status.is_iomem); 152 memirq_assert(memirq, !memirq->mask.is_iomem); 153 154 memirq_debug(memirq, "page offsets: source %#x status %#x\n", 155 xe_memirq_source_ptr(memirq), xe_memirq_status_ptr(memirq)); 156 157 return drmm_add_action_or_reset(&xe->drm, __release_xe_bo, memirq->bo); 158 159 out: 160 xe_sriov_err(memirq_to_xe(memirq), 161 "Failed to allocate memirq page (%pe)\n", ERR_PTR(err)); 162 return err; 163 } 164 165 static void memirq_set_enable(struct xe_memirq *memirq, bool enable) 166 { 167 iosys_map_wr(&memirq->mask, 0, u32, enable ? GENMASK(15, 0) : 0); 168 169 memirq->enabled = enable; 170 } 171 172 /** 173 * xe_memirq_init - Initialize data used by `Memory Based Interrupts`_. 174 * @memirq: the &xe_memirq to initialize 175 * 176 * Allocate `Interrupt Source Report Page`_ and `Interrupt Status Report Page`_ 177 * used by `Memory Based Interrupts`_. 178 * 179 * These allocations are managed and will be implicitly released on unload. 180 * 181 * Note: This function shall be called only by the VF driver. 182 * 183 * If this function fails then VF driver won't be able to operate correctly. 184 * If `Memory Based Interrupts`_ are not used this function will return 0. 185 * 186 * Return: 0 on success or a negative error code on failure. 187 */ 188 int xe_memirq_init(struct xe_memirq *memirq) 189 { 190 struct xe_device *xe = memirq_to_xe(memirq); 191 int err; 192 193 memirq_assert(memirq, IS_SRIOV_VF(xe)); 194 195 if (!xe_device_has_memirq(xe)) 196 return 0; 197 198 err = memirq_alloc_pages(memirq); 199 if (unlikely(err)) 200 return err; 201 202 /* we need to start with all irqs enabled */ 203 memirq_set_enable(memirq, true); 204 205 return 0; 206 } 207 208 /** 209 * xe_memirq_source_ptr - Get GGTT's offset of the `Interrupt Source Report Page`_. 210 * @memirq: the &xe_memirq to query 211 * 212 * Shall be called only on VF driver when `Memory Based Interrupts`_ are used 213 * and xe_memirq_init() didn't fail. 214 * 215 * Return: GGTT's offset of the `Interrupt Source Report Page`_. 216 */ 217 u32 xe_memirq_source_ptr(struct xe_memirq *memirq) 218 { 219 memirq_assert(memirq, IS_SRIOV_VF(memirq_to_xe(memirq))); 220 memirq_assert(memirq, xe_device_has_memirq(memirq_to_xe(memirq))); 221 memirq_assert(memirq, memirq->bo); 222 223 return xe_bo_ggtt_addr(memirq->bo) + XE_MEMIRQ_SOURCE_OFFSET; 224 } 225 226 /** 227 * xe_memirq_status_ptr - Get GGTT's offset of the `Interrupt Status Report Page`_. 228 * @memirq: the &xe_memirq to query 229 * 230 * Shall be called only on VF driver when `Memory Based Interrupts`_ are used 231 * and xe_memirq_init() didn't fail. 232 * 233 * Return: GGTT's offset of the `Interrupt Status Report Page`_. 234 */ 235 u32 xe_memirq_status_ptr(struct xe_memirq *memirq) 236 { 237 memirq_assert(memirq, IS_SRIOV_VF(memirq_to_xe(memirq))); 238 memirq_assert(memirq, xe_device_has_memirq(memirq_to_xe(memirq))); 239 memirq_assert(memirq, memirq->bo); 240 241 return xe_bo_ggtt_addr(memirq->bo) + XE_MEMIRQ_STATUS_OFFSET; 242 } 243 244 /** 245 * xe_memirq_enable_ptr - Get GGTT's offset of the Interrupt Enable Mask. 246 * @memirq: the &xe_memirq to query 247 * 248 * Shall be called only on VF driver when `Memory Based Interrupts`_ are used 249 * and xe_memirq_init() didn't fail. 250 * 251 * Return: GGTT's offset of the Interrupt Enable Mask. 252 */ 253 u32 xe_memirq_enable_ptr(struct xe_memirq *memirq) 254 { 255 memirq_assert(memirq, IS_SRIOV_VF(memirq_to_xe(memirq))); 256 memirq_assert(memirq, xe_device_has_memirq(memirq_to_xe(memirq))); 257 memirq_assert(memirq, memirq->bo); 258 259 return xe_bo_ggtt_addr(memirq->bo) + XE_MEMIRQ_ENABLE_OFFSET; 260 } 261 262 /** 263 * xe_memirq_init_guc - Prepare GuC for `Memory Based Interrupts`_. 264 * @memirq: the &xe_memirq 265 * @guc: the &xe_guc to setup 266 * 267 * Register `Interrupt Source Report Page`_ and `Interrupt Status Report Page`_ 268 * to be used by the GuC when `Memory Based Interrupts`_ are required. 269 * 270 * Shall be called only on VF driver when `Memory Based Interrupts`_ are used 271 * and xe_memirq_init() didn't fail. 272 * 273 * Return: 0 on success or a negative error code on failure. 274 */ 275 int xe_memirq_init_guc(struct xe_memirq *memirq, struct xe_guc *guc) 276 { 277 bool is_media = xe_gt_is_media_type(guc_to_gt(guc)); 278 u32 offset = is_media ? ilog2(INTR_MGUC) : ilog2(INTR_GUC); 279 u32 source, status; 280 int err; 281 282 memirq_assert(memirq, IS_SRIOV_VF(memirq_to_xe(memirq))); 283 memirq_assert(memirq, xe_device_has_memirq(memirq_to_xe(memirq))); 284 memirq_assert(memirq, memirq->bo); 285 286 source = xe_memirq_source_ptr(memirq) + offset; 287 status = xe_memirq_status_ptr(memirq) + offset * SZ_16; 288 289 err = xe_guc_self_cfg64(guc, GUC_KLV_SELF_CFG_MEMIRQ_SOURCE_ADDR_KEY, 290 source); 291 if (unlikely(err)) 292 goto failed; 293 294 err = xe_guc_self_cfg64(guc, GUC_KLV_SELF_CFG_MEMIRQ_STATUS_ADDR_KEY, 295 status); 296 if (unlikely(err)) 297 goto failed; 298 299 return 0; 300 301 failed: 302 xe_sriov_err(memirq_to_xe(memirq), 303 "Failed to setup report pages in %s (%pe)\n", 304 guc_name(guc), ERR_PTR(err)); 305 return err; 306 } 307 308 /** 309 * xe_memirq_reset - Disable processing of `Memory Based Interrupts`_. 310 * @memirq: struct xe_memirq 311 * 312 * This is part of the driver IRQ setup flow. 313 * 314 * This function shall only be used by the VF driver on platforms that use 315 * `Memory Based Interrupts`_. 316 */ 317 void xe_memirq_reset(struct xe_memirq *memirq) 318 { 319 memirq_assert(memirq, IS_SRIOV_VF(memirq_to_xe(memirq))); 320 memirq_assert(memirq, xe_device_has_memirq(memirq_to_xe(memirq))); 321 322 if (memirq->bo) 323 memirq_set_enable(memirq, false); 324 } 325 326 /** 327 * xe_memirq_postinstall - Enable processing of `Memory Based Interrupts`_. 328 * @memirq: the &xe_memirq 329 * 330 * This is part of the driver IRQ setup flow. 331 * 332 * This function shall only be used by the VF driver on platforms that use 333 * `Memory Based Interrupts`_. 334 */ 335 void xe_memirq_postinstall(struct xe_memirq *memirq) 336 { 337 memirq_assert(memirq, IS_SRIOV_VF(memirq_to_xe(memirq))); 338 memirq_assert(memirq, xe_device_has_memirq(memirq_to_xe(memirq))); 339 340 if (memirq->bo) 341 memirq_set_enable(memirq, true); 342 } 343 344 static bool memirq_received(struct xe_memirq *memirq, struct iosys_map *vector, 345 u16 offset, const char *name) 346 { 347 u8 value; 348 349 value = iosys_map_rd(vector, offset, u8); 350 if (value) { 351 if (value != 0xff) 352 xe_sriov_err_ratelimited(memirq_to_xe(memirq), 353 "Unexpected memirq value %#x from %s at %u\n", 354 value, name, offset); 355 iosys_map_wr(vector, offset, u8, 0x00); 356 } 357 358 return value; 359 } 360 361 static void memirq_dispatch_engine(struct xe_memirq *memirq, struct iosys_map *status, 362 struct xe_hw_engine *hwe) 363 { 364 memirq_debug(memirq, "STATUS %s %*ph\n", hwe->name, 16, status->vaddr); 365 366 if (memirq_received(memirq, status, ilog2(GT_RENDER_USER_INTERRUPT), hwe->name)) 367 xe_hw_engine_handle_irq(hwe, GT_RENDER_USER_INTERRUPT); 368 } 369 370 static void memirq_dispatch_guc(struct xe_memirq *memirq, struct iosys_map *status, 371 struct xe_guc *guc) 372 { 373 const char *name = guc_name(guc); 374 375 memirq_debug(memirq, "STATUS %s %*ph\n", name, 16, status->vaddr); 376 377 if (memirq_received(memirq, status, ilog2(GUC_INTR_GUC2HOST), name)) 378 xe_guc_irq_handler(guc, GUC_INTR_GUC2HOST); 379 } 380 381 /** 382 * xe_memirq_handler - The `Memory Based Interrupts`_ Handler. 383 * @memirq: the &xe_memirq 384 * 385 * This function reads and dispatches `Memory Based Interrupts`. 386 */ 387 void xe_memirq_handler(struct xe_memirq *memirq) 388 { 389 struct xe_device *xe = memirq_to_xe(memirq); 390 struct xe_tile *tile = memirq_to_tile(memirq); 391 struct xe_hw_engine *hwe; 392 enum xe_hw_engine_id id; 393 struct iosys_map map; 394 unsigned int gtid; 395 struct xe_gt *gt; 396 397 if (!memirq->bo) 398 return; 399 400 memirq_assert(memirq, !memirq->source.is_iomem); 401 memirq_debug(memirq, "SOURCE %*ph\n", 32, memirq->source.vaddr); 402 memirq_debug(memirq, "SOURCE %*ph\n", 32, memirq->source.vaddr + 32); 403 404 for_each_gt(gt, xe, gtid) { 405 if (gt->tile != tile) 406 continue; 407 408 for_each_hw_engine(hwe, gt, id) { 409 if (memirq_received(memirq, &memirq->source, hwe->irq_offset, "SRC")) { 410 map = IOSYS_MAP_INIT_OFFSET(&memirq->status, 411 hwe->irq_offset * SZ_16); 412 memirq_dispatch_engine(memirq, &map, hwe); 413 } 414 } 415 } 416 417 /* GuC and media GuC (if present) must be checked separately */ 418 419 if (memirq_received(memirq, &memirq->source, ilog2(INTR_GUC), "SRC")) { 420 map = IOSYS_MAP_INIT_OFFSET(&memirq->status, ilog2(INTR_GUC) * SZ_16); 421 memirq_dispatch_guc(memirq, &map, &tile->primary_gt->uc.guc); 422 } 423 424 if (!tile->media_gt) 425 return; 426 427 if (memirq_received(memirq, &memirq->source, ilog2(INTR_MGUC), "SRC")) { 428 map = IOSYS_MAP_INIT_OFFSET(&memirq->status, ilog2(INTR_MGUC) * SZ_16); 429 memirq_dispatch_guc(memirq, &map, &tile->media_gt->uc.guc); 430 } 431 } 432