1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2023 Intel Corporation
4 */
5
6 #include <drm/drm_managed.h>
7
8 #include "regs/xe_guc_regs.h"
9 #include "regs/xe_irq_regs.h"
10 #include "regs/xe_regs.h"
11
12 #include "xe_assert.h"
13 #include "xe_bo.h"
14 #include "xe_device.h"
15 #include "xe_device_types.h"
16 #include "xe_gt.h"
17 #include "xe_gt_printk.h"
18 #include "xe_guc.h"
19 #include "xe_hw_engine.h"
20 #include "xe_map.h"
21 #include "xe_memirq.h"
22
23 #define memirq_assert(m, condition) xe_tile_assert(memirq_to_tile(m), condition)
24 #define memirq_printk(m, _level, _fmt, ...) \
25 drm_##_level(&memirq_to_xe(m)->drm, "MEMIRQ%u: " _fmt, \
26 memirq_to_tile(m)->id, ##__VA_ARGS__)
27
28 #ifdef CONFIG_DRM_XE_DEBUG_MEMIRQ
29 #define memirq_debug(m, _fmt, ...) memirq_printk(m, dbg, _fmt, ##__VA_ARGS__)
30 #else
31 #define memirq_debug(...)
32 #endif
33
34 #define memirq_err(m, _fmt, ...) memirq_printk(m, err, _fmt, ##__VA_ARGS__)
35 #define memirq_err_ratelimited(m, _fmt, ...) \
36 memirq_printk(m, err_ratelimited, _fmt, ##__VA_ARGS__)
37
memirq_to_tile(struct xe_memirq * memirq)38 static struct xe_tile *memirq_to_tile(struct xe_memirq *memirq)
39 {
40 return container_of(memirq, struct xe_tile, memirq);
41 }
42
memirq_to_xe(struct xe_memirq * memirq)43 static struct xe_device *memirq_to_xe(struct xe_memirq *memirq)
44 {
45 return tile_to_xe(memirq_to_tile(memirq));
46 }
47
guc_name(struct xe_guc * guc)48 static const char *guc_name(struct xe_guc *guc)
49 {
50 return xe_gt_is_media_type(guc_to_gt(guc)) ? "media GuC" : "GuC";
51 }
52
53 /**
54 * DOC: Memory Based Interrupts
55 *
56 * MMIO register based interrupts infrastructure used for non-virtualized mode
57 * or SRIOV-8 (which supports 8 Virtual Functions) does not scale efficiently
58 * to allow delivering interrupts to a large number of Virtual machines or
59 * containers. Memory based interrupt status reporting provides an efficient
60 * and scalable infrastructure.
61 *
62 * For memory based interrupt status reporting hardware sequence is:
63 * * Engine writes the interrupt event to memory
64 * (Pointer to memory location is provided by SW. This memory surface must
65 * be mapped to system memory and must be marked as un-cacheable (UC) on
66 * Graphics IP Caches)
67 * * Engine triggers an interrupt to host.
68 */
69
70 /**
71 * DOC: Memory Based Interrupts Page Layout
72 *
73 * `Memory Based Interrupts`_ requires three different objects, which are
74 * called "page" in the specs, even if they aren't page-sized or aligned.
75 *
76 * To simplify the code we allocate a single page size object and then use
77 * offsets to embedded "pages". The address of those "pages" are then
78 * programmed in the HW via LRI and LRM in the context image.
79 *
80 * - _`Interrupt Status Report Page`: this page contains the interrupt
81 * status vectors for each unit. Each bit in the interrupt vectors is
82 * converted to a byte, with the byte being set to 0xFF when an
83 * interrupt is triggered; interrupt vectors are 16b big so each unit
84 * gets 16B. One space is reserved for each bit in one of the
85 * GT_INTR_DWx registers, so this object needs a total of 1024B.
86 * This object needs to be 4KiB aligned.
87 *
88 * - _`Interrupt Source Report Page`: this is the equivalent of the
89 * GEN11_GT_INTR_DWx registers, with each bit in those registers being
90 * mapped to a byte here. The offsets are the same, just bytes instead
91 * of bits. This object needs to be cacheline aligned.
92 *
93 * - Interrupt Mask: the HW needs a location to fetch the interrupt
94 * mask vector to be used by the LRM in the context, so we just use
95 * the next available space in the interrupt page.
96 *
97 * ::
98 *
99 * 0x0000 +===========+ <== Interrupt Status Report Page
100 * | |
101 * | | ____ +----+----------------+
102 * | | / | 0 | USER INTERRUPT |
103 * +-----------+ __/ | 1 | |
104 * | HWE(n) | __ | | CTX SWITCH |
105 * +-----------+ \ | | WAIT SEMAPHORE |
106 * | | \____ | 15 | |
107 * | | +----+----------------+
108 * | |
109 * 0x0400 +===========+ <== Interrupt Source Report Page
110 * | HWE(0) |
111 * | HWE(1) |
112 * | |
113 * | HWE(x) |
114 * 0x0440 +===========+ <== Interrupt Enable Mask
115 * | |
116 * | |
117 * +-----------+
118 *
119 *
120 * MSI-X use case
121 *
122 * When using MSI-X, hw engines report interrupt status and source to engine
123 * instance 0. For this scenario, in order to differentiate between the
124 * engines, we need to pass different status/source pointers in the LRC.
125 *
126 * The requirements on those pointers are:
127 * - Interrupt status should be 4KiB aligned
128 * - Interrupt source should be 64 bytes aligned
129 *
130 * To accommodate this, we duplicate the memirq page layout above -
131 * allocating a page for each engine instance and pass this page in the LRC.
132 * Note that the same page can be reused for different engine types.
133 * For example, an LRC executing on CCS #x will have pointers to page #x,
134 * and an LRC executing on BCS #x will have the same pointers.
135 *
136 * ::
137 *
138 * 0x0000 +==============================+ <== page for instance 0 (BCS0, CCS0, etc.)
139 * | Interrupt Status Report Page |
140 * 0x0400 +==============================+
141 * | Interrupt Source Report Page |
142 * 0x0440 +==============================+
143 * | Interrupt Enable Mask |
144 * +==============================+
145 * | Not used |
146 * 0x1000 +==============================+ <== page for instance 1 (BCS1, CCS1, etc.)
147 * | Interrupt Status Report Page |
148 * 0x1400 +==============================+
149 * | Interrupt Source Report Page |
150 * 0x1440 +==============================+
151 * | Not used |
152 * 0x2000 +==============================+ <== page for instance 2 (BCS2, CCS2, etc.)
153 * | ... |
154 * +==============================+
155 *
156 */
157
hw_reports_to_instance_zero(struct xe_memirq * memirq)158 static inline bool hw_reports_to_instance_zero(struct xe_memirq *memirq)
159 {
160 /*
161 * When the HW engines are configured to use MSI-X,
162 * they report interrupt status and source to the offset of
163 * engine instance 0.
164 */
165 return xe_device_has_msix(memirq_to_xe(memirq));
166 }
167
memirq_alloc_pages(struct xe_memirq * memirq)168 static int memirq_alloc_pages(struct xe_memirq *memirq)
169 {
170 struct xe_device *xe = memirq_to_xe(memirq);
171 struct xe_tile *tile = memirq_to_tile(memirq);
172 size_t bo_size = hw_reports_to_instance_zero(memirq) ?
173 XE_HW_ENGINE_MAX_INSTANCE * SZ_4K : SZ_4K;
174 struct xe_bo *bo;
175 int err;
176
177 BUILD_BUG_ON(!IS_ALIGNED(XE_MEMIRQ_SOURCE_OFFSET(0), SZ_64));
178 BUILD_BUG_ON(!IS_ALIGNED(XE_MEMIRQ_STATUS_OFFSET(0), SZ_4K));
179
180 bo = xe_managed_bo_create_pin_map(xe, tile, bo_size,
181 XE_BO_FLAG_SYSTEM |
182 XE_BO_FLAG_GGTT |
183 XE_BO_FLAG_GGTT_INVALIDATE |
184 XE_BO_FLAG_NEEDS_UC |
185 XE_BO_FLAG_NEEDS_CPU_ACCESS);
186 if (IS_ERR(bo)) {
187 err = PTR_ERR(bo);
188 goto out;
189 }
190
191 memirq_assert(memirq, !xe_bo_is_vram(bo));
192 memirq_assert(memirq, !memirq->bo);
193
194 iosys_map_memset(&bo->vmap, 0, 0, bo_size);
195
196 memirq->bo = bo;
197 memirq->source = IOSYS_MAP_INIT_OFFSET(&bo->vmap, XE_MEMIRQ_SOURCE_OFFSET(0));
198 memirq->status = IOSYS_MAP_INIT_OFFSET(&bo->vmap, XE_MEMIRQ_STATUS_OFFSET(0));
199 memirq->mask = IOSYS_MAP_INIT_OFFSET(&bo->vmap, XE_MEMIRQ_ENABLE_OFFSET);
200
201 memirq_assert(memirq, !memirq->source.is_iomem);
202 memirq_assert(memirq, !memirq->status.is_iomem);
203 memirq_assert(memirq, !memirq->mask.is_iomem);
204
205 memirq_debug(memirq, "page offsets: bo %#x bo_size %zu source %#x status %#x\n",
206 xe_bo_ggtt_addr(bo), bo_size, XE_MEMIRQ_SOURCE_OFFSET(0),
207 XE_MEMIRQ_STATUS_OFFSET(0));
208
209 return 0;
210
211 out:
212 memirq_err(memirq, "Failed to allocate memirq page (%pe)\n", ERR_PTR(err));
213 return err;
214 }
215
memirq_set_enable(struct xe_memirq * memirq,bool enable)216 static void memirq_set_enable(struct xe_memirq *memirq, bool enable)
217 {
218 iosys_map_wr(&memirq->mask, 0, u32, enable ? GENMASK(15, 0) : 0);
219
220 memirq->enabled = enable;
221 }
222
223 /**
224 * xe_memirq_init - Initialize data used by `Memory Based Interrupts`_.
225 * @memirq: the &xe_memirq to initialize
226 *
227 * Allocate `Interrupt Source Report Page`_ and `Interrupt Status Report Page`_
228 * used by `Memory Based Interrupts`_.
229 *
230 * These allocations are managed and will be implicitly released on unload.
231 *
232 * If this function fails then the driver won't be able to operate correctly.
233 * If `Memory Based Interrupts`_ are not used this function will return 0.
234 *
235 * Return: 0 on success or a negative error code on failure.
236 */
xe_memirq_init(struct xe_memirq * memirq)237 int xe_memirq_init(struct xe_memirq *memirq)
238 {
239 struct xe_device *xe = memirq_to_xe(memirq);
240 int err;
241
242 if (!xe_device_uses_memirq(xe))
243 return 0;
244
245 err = memirq_alloc_pages(memirq);
246 if (unlikely(err))
247 return err;
248
249 /* we need to start with all irqs enabled */
250 memirq_set_enable(memirq, true);
251
252 return 0;
253 }
254
__memirq_source_page(struct xe_memirq * memirq,u16 instance)255 static u32 __memirq_source_page(struct xe_memirq *memirq, u16 instance)
256 {
257 memirq_assert(memirq, instance <= XE_HW_ENGINE_MAX_INSTANCE);
258 memirq_assert(memirq, memirq->bo);
259
260 instance = hw_reports_to_instance_zero(memirq) ? instance : 0;
261 return xe_bo_ggtt_addr(memirq->bo) + XE_MEMIRQ_SOURCE_OFFSET(instance);
262 }
263
264 /**
265 * xe_memirq_source_ptr - Get GGTT's offset of the `Interrupt Source Report Page`_.
266 * @memirq: the &xe_memirq to query
267 * @hwe: the hw engine for which we want the report page
268 *
269 * Shall be called when `Memory Based Interrupts`_ are used
270 * and xe_memirq_init() didn't fail.
271 *
272 * Return: GGTT's offset of the `Interrupt Source Report Page`_.
273 */
xe_memirq_source_ptr(struct xe_memirq * memirq,struct xe_hw_engine * hwe)274 u32 xe_memirq_source_ptr(struct xe_memirq *memirq, struct xe_hw_engine *hwe)
275 {
276 memirq_assert(memirq, xe_device_uses_memirq(memirq_to_xe(memirq)));
277
278 return __memirq_source_page(memirq, hwe->instance);
279 }
280
__memirq_status_page(struct xe_memirq * memirq,u16 instance)281 static u32 __memirq_status_page(struct xe_memirq *memirq, u16 instance)
282 {
283 memirq_assert(memirq, instance <= XE_HW_ENGINE_MAX_INSTANCE);
284 memirq_assert(memirq, memirq->bo);
285
286 instance = hw_reports_to_instance_zero(memirq) ? instance : 0;
287 return xe_bo_ggtt_addr(memirq->bo) + XE_MEMIRQ_STATUS_OFFSET(instance);
288 }
289
290 /**
291 * xe_memirq_status_ptr - Get GGTT's offset of the `Interrupt Status Report Page`_.
292 * @memirq: the &xe_memirq to query
293 * @hwe: the hw engine for which we want the report page
294 *
295 * Shall be called when `Memory Based Interrupts`_ are used
296 * and xe_memirq_init() didn't fail.
297 *
298 * Return: GGTT's offset of the `Interrupt Status Report Page`_.
299 */
xe_memirq_status_ptr(struct xe_memirq * memirq,struct xe_hw_engine * hwe)300 u32 xe_memirq_status_ptr(struct xe_memirq *memirq, struct xe_hw_engine *hwe)
301 {
302 memirq_assert(memirq, xe_device_uses_memirq(memirq_to_xe(memirq)));
303
304 return __memirq_status_page(memirq, hwe->instance);
305 }
306
307 /**
308 * xe_memirq_enable_ptr - Get GGTT's offset of the Interrupt Enable Mask.
309 * @memirq: the &xe_memirq to query
310 *
311 * Shall be called when `Memory Based Interrupts`_ are used
312 * and xe_memirq_init() didn't fail.
313 *
314 * Return: GGTT's offset of the Interrupt Enable Mask.
315 */
xe_memirq_enable_ptr(struct xe_memirq * memirq)316 u32 xe_memirq_enable_ptr(struct xe_memirq *memirq)
317 {
318 memirq_assert(memirq, xe_device_uses_memirq(memirq_to_xe(memirq)));
319 memirq_assert(memirq, memirq->bo);
320
321 return xe_bo_ggtt_addr(memirq->bo) + XE_MEMIRQ_ENABLE_OFFSET;
322 }
323
324 /**
325 * xe_memirq_init_guc - Prepare GuC for `Memory Based Interrupts`_.
326 * @memirq: the &xe_memirq
327 * @guc: the &xe_guc to setup
328 *
329 * Register `Interrupt Source Report Page`_ and `Interrupt Status Report Page`_
330 * to be used by the GuC when `Memory Based Interrupts`_ are required.
331 *
332 * Shall be called when `Memory Based Interrupts`_ are used
333 * and xe_memirq_init() didn't fail.
334 *
335 * Return: 0 on success or a negative error code on failure.
336 */
xe_memirq_init_guc(struct xe_memirq * memirq,struct xe_guc * guc)337 int xe_memirq_init_guc(struct xe_memirq *memirq, struct xe_guc *guc)
338 {
339 bool is_media = xe_gt_is_media_type(guc_to_gt(guc));
340 u32 offset = is_media ? ilog2(INTR_MGUC) : ilog2(INTR_GUC);
341 u32 source, status;
342 int err;
343
344 memirq_assert(memirq, xe_device_uses_memirq(memirq_to_xe(memirq)));
345
346 source = __memirq_source_page(memirq, 0) + offset;
347 status = __memirq_status_page(memirq, 0) + offset * SZ_16;
348
349 err = xe_guc_self_cfg64(guc, GUC_KLV_SELF_CFG_MEMIRQ_SOURCE_ADDR_KEY,
350 source);
351 if (unlikely(err))
352 goto failed;
353
354 err = xe_guc_self_cfg64(guc, GUC_KLV_SELF_CFG_MEMIRQ_STATUS_ADDR_KEY,
355 status);
356 if (unlikely(err))
357 goto failed;
358
359 return 0;
360
361 failed:
362 memirq_err(memirq, "Failed to setup report pages in %s (%pe)\n",
363 guc_name(guc), ERR_PTR(err));
364 return err;
365 }
366
367 /**
368 * xe_memirq_reset - Disable processing of `Memory Based Interrupts`_.
369 * @memirq: struct xe_memirq
370 *
371 * This is part of the driver IRQ setup flow.
372 *
373 * This function shall only be used on platforms that use
374 * `Memory Based Interrupts`_.
375 */
xe_memirq_reset(struct xe_memirq * memirq)376 void xe_memirq_reset(struct xe_memirq *memirq)
377 {
378 memirq_assert(memirq, xe_device_uses_memirq(memirq_to_xe(memirq)));
379
380 if (memirq->bo)
381 memirq_set_enable(memirq, false);
382 }
383
384 /**
385 * xe_memirq_postinstall - Enable processing of `Memory Based Interrupts`_.
386 * @memirq: the &xe_memirq
387 *
388 * This is part of the driver IRQ setup flow.
389 *
390 * This function shall only be used on platforms that use
391 * `Memory Based Interrupts`_.
392 */
xe_memirq_postinstall(struct xe_memirq * memirq)393 void xe_memirq_postinstall(struct xe_memirq *memirq)
394 {
395 memirq_assert(memirq, xe_device_uses_memirq(memirq_to_xe(memirq)));
396
397 if (memirq->bo)
398 memirq_set_enable(memirq, true);
399 }
400
memirq_received(struct xe_memirq * memirq,struct iosys_map * vector,u16 offset,const char * name)401 static bool memirq_received(struct xe_memirq *memirq, struct iosys_map *vector,
402 u16 offset, const char *name)
403 {
404 u8 value;
405
406 value = iosys_map_rd(vector, offset, u8);
407 if (value) {
408 if (value != 0xff)
409 memirq_err_ratelimited(memirq,
410 "Unexpected memirq value %#x from %s at %u\n",
411 value, name, offset);
412 iosys_map_wr(vector, offset, u8, 0x00);
413 }
414
415 return value;
416 }
417
memirq_dispatch_engine(struct xe_memirq * memirq,struct iosys_map * status,struct xe_hw_engine * hwe)418 static void memirq_dispatch_engine(struct xe_memirq *memirq, struct iosys_map *status,
419 struct xe_hw_engine *hwe)
420 {
421 memirq_debug(memirq, "STATUS %s %*ph\n", hwe->name, 16, status->vaddr);
422
423 if (memirq_received(memirq, status, ilog2(GT_RENDER_USER_INTERRUPT), hwe->name))
424 xe_hw_engine_handle_irq(hwe, GT_RENDER_USER_INTERRUPT);
425 }
426
memirq_dispatch_guc(struct xe_memirq * memirq,struct iosys_map * status,struct xe_guc * guc)427 static void memirq_dispatch_guc(struct xe_memirq *memirq, struct iosys_map *status,
428 struct xe_guc *guc)
429 {
430 const char *name = guc_name(guc);
431
432 memirq_debug(memirq, "STATUS %s %*ph\n", name, 16, status->vaddr);
433
434 if (memirq_received(memirq, status, ilog2(GUC_INTR_GUC2HOST), name))
435 xe_guc_irq_handler(guc, GUC_INTR_GUC2HOST);
436
437 if (memirq_received(memirq, status, ilog2(GUC_INTR_SW_INT_0), name))
438 xe_guc_irq_handler(guc, GUC_INTR_SW_INT_0);
439 }
440
441 /**
442 * xe_memirq_hwe_handler - Check and process interrupts for a specific HW engine.
443 * @memirq: the &xe_memirq
444 * @hwe: the hw engine to process
445 *
446 * This function reads and dispatches `Memory Based Interrupts` for the provided HW engine.
447 */
xe_memirq_hwe_handler(struct xe_memirq * memirq,struct xe_hw_engine * hwe)448 void xe_memirq_hwe_handler(struct xe_memirq *memirq, struct xe_hw_engine *hwe)
449 {
450 u16 offset = hwe->irq_offset;
451 u16 instance = hw_reports_to_instance_zero(memirq) ? hwe->instance : 0;
452 struct iosys_map src_offset = IOSYS_MAP_INIT_OFFSET(&memirq->bo->vmap,
453 XE_MEMIRQ_SOURCE_OFFSET(instance));
454
455 if (memirq_received(memirq, &src_offset, offset, "SRC")) {
456 struct iosys_map status_offset =
457 IOSYS_MAP_INIT_OFFSET(&memirq->bo->vmap,
458 XE_MEMIRQ_STATUS_OFFSET(instance) + offset * SZ_16);
459 memirq_dispatch_engine(memirq, &status_offset, hwe);
460 }
461 }
462
463 /**
464 * xe_memirq_handler - The `Memory Based Interrupts`_ Handler.
465 * @memirq: the &xe_memirq
466 *
467 * This function reads and dispatches `Memory Based Interrupts`.
468 */
xe_memirq_handler(struct xe_memirq * memirq)469 void xe_memirq_handler(struct xe_memirq *memirq)
470 {
471 struct xe_device *xe = memirq_to_xe(memirq);
472 struct xe_tile *tile = memirq_to_tile(memirq);
473 struct xe_hw_engine *hwe;
474 enum xe_hw_engine_id id;
475 struct iosys_map map;
476 unsigned int gtid;
477 struct xe_gt *gt;
478
479 if (!memirq->bo)
480 return;
481
482 memirq_assert(memirq, !memirq->source.is_iomem);
483 memirq_debug(memirq, "SOURCE %*ph\n", 32, memirq->source.vaddr);
484 memirq_debug(memirq, "SOURCE %*ph\n", 32, memirq->source.vaddr + 32);
485
486 for_each_gt(gt, xe, gtid) {
487 if (gt->tile != tile)
488 continue;
489
490 for_each_hw_engine(hwe, gt, id)
491 xe_memirq_hwe_handler(memirq, hwe);
492 }
493
494 /* GuC and media GuC (if present) must be checked separately */
495
496 if (memirq_received(memirq, &memirq->source, ilog2(INTR_GUC), "SRC")) {
497 map = IOSYS_MAP_INIT_OFFSET(&memirq->status, ilog2(INTR_GUC) * SZ_16);
498 memirq_dispatch_guc(memirq, &map, &tile->primary_gt->uc.guc);
499 }
500
501 if (!tile->media_gt)
502 return;
503
504 if (memirq_received(memirq, &memirq->source, ilog2(INTR_MGUC), "SRC")) {
505 map = IOSYS_MAP_INIT_OFFSET(&memirq->status, ilog2(INTR_MGUC) * SZ_16);
506 memirq_dispatch_guc(memirq, &map, &tile->media_gt->uc.guc);
507 }
508 }
509