1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2022 Intel Corporation
4 */
5
6 #include "xe_guc_ads.h"
7
8 #include <linux/fault-inject.h>
9
10 #include <drm/drm_managed.h>
11
12 #include <generated/xe_wa_oob.h>
13
14 #include "abi/guc_actions_abi.h"
15 #include "regs/xe_engine_regs.h"
16 #include "regs/xe_gt_regs.h"
17 #include "regs/xe_guc_regs.h"
18 #include "xe_bo.h"
19 #include "xe_gt.h"
20 #include "xe_gt_ccs_mode.h"
21 #include "xe_gt_printk.h"
22 #include "xe_guc.h"
23 #include "xe_guc_buf.h"
24 #include "xe_guc_capture.h"
25 #include "xe_guc_ct.h"
26 #include "xe_hw_engine.h"
27 #include "xe_lrc.h"
28 #include "xe_map.h"
29 #include "xe_mmio.h"
30 #include "xe_platform_types.h"
31 #include "xe_uc_fw.h"
32 #include "xe_wa.h"
33 #include "xe_gt_mcr.h"
34
35 /* Slack of a few additional entries per engine */
36 #define ADS_REGSET_EXTRA_MAX 8
37
38 static struct xe_guc *
ads_to_guc(struct xe_guc_ads * ads)39 ads_to_guc(struct xe_guc_ads *ads)
40 {
41 return container_of(ads, struct xe_guc, ads);
42 }
43
44 static struct xe_gt *
ads_to_gt(struct xe_guc_ads * ads)45 ads_to_gt(struct xe_guc_ads *ads)
46 {
47 return container_of(ads, struct xe_gt, uc.guc.ads);
48 }
49
50 static struct xe_device *
ads_to_xe(struct xe_guc_ads * ads)51 ads_to_xe(struct xe_guc_ads *ads)
52 {
53 return gt_to_xe(ads_to_gt(ads));
54 }
55
56 static struct iosys_map *
ads_to_map(struct xe_guc_ads * ads)57 ads_to_map(struct xe_guc_ads *ads)
58 {
59 return &ads->bo->vmap;
60 }
61
62 /* UM Queue parameters: */
63 #define GUC_UM_QUEUE_SIZE (SZ_64K)
64 #define GUC_PAGE_RES_TIMEOUT_US (-1)
65
66 /*
67 * The Additional Data Struct (ADS) has pointers for different buffers used by
68 * the GuC. One single gem object contains the ADS struct itself (guc_ads) and
69 * all the extra buffers indirectly linked via the ADS struct's entries.
70 *
71 * Layout of the ADS blob allocated for the GuC:
72 *
73 * +---------------------------------------+ <== base
74 * | guc_ads |
75 * +---------------------------------------+
76 * | guc_policies |
77 * +---------------------------------------+
78 * | guc_gt_system_info |
79 * +---------------------------------------+
80 * | guc_engine_usage |
81 * +---------------------------------------+
82 * | guc_um_init_params |
83 * +---------------------------------------+ <== static
84 * | guc_mmio_reg[countA] (engine 0.0) |
85 * | guc_mmio_reg[countB] (engine 0.1) |
86 * | guc_mmio_reg[countC] (engine 1.0) |
87 * | ... |
88 * +---------------------------------------+ <== dynamic
89 * | padding |
90 * +---------------------------------------+ <== 4K aligned
91 * | golden contexts |
92 * +---------------------------------------+
93 * | padding |
94 * +---------------------------------------+ <== 4K aligned
95 * | w/a KLVs |
96 * +---------------------------------------+
97 * | padding |
98 * +---------------------------------------+ <== 4K aligned
99 * | capture lists |
100 * +---------------------------------------+
101 * | padding |
102 * +---------------------------------------+ <== 4K aligned
103 * | UM queues |
104 * +---------------------------------------+
105 * | padding |
106 * +---------------------------------------+ <== 4K aligned
107 * | private data |
108 * +---------------------------------------+
109 * | padding |
110 * +---------------------------------------+ <== 4K aligned
111 */
112 struct __guc_ads_blob {
113 struct guc_ads ads;
114 struct guc_policies policies;
115 struct guc_gt_system_info system_info;
116 struct guc_engine_usage engine_usage;
117 struct guc_um_init_params um_init_params;
118 /* From here on, location is dynamic! Refer to above diagram. */
119 struct guc_mmio_reg regset[];
120 } __packed;
121
122 #define ads_blob_read(ads_, field_) \
123 xe_map_rd_field(ads_to_xe(ads_), ads_to_map(ads_), 0, \
124 struct __guc_ads_blob, field_)
125
126 #define ads_blob_write(ads_, field_, val_) \
127 xe_map_wr_field(ads_to_xe(ads_), ads_to_map(ads_), 0, \
128 struct __guc_ads_blob, field_, val_)
129
130 #define info_map_write(xe_, map_, field_, val_) \
131 xe_map_wr_field(xe_, map_, 0, struct guc_gt_system_info, field_, val_)
132
133 #define info_map_read(xe_, map_, field_) \
134 xe_map_rd_field(xe_, map_, 0, struct guc_gt_system_info, field_)
135
guc_ads_regset_size(struct xe_guc_ads * ads)136 static size_t guc_ads_regset_size(struct xe_guc_ads *ads)
137 {
138 struct xe_device *xe = ads_to_xe(ads);
139
140 xe_assert(xe, ads->regset_size);
141
142 return ads->regset_size;
143 }
144
guc_ads_golden_lrc_size(struct xe_guc_ads * ads)145 static size_t guc_ads_golden_lrc_size(struct xe_guc_ads *ads)
146 {
147 return PAGE_ALIGN(ads->golden_lrc_size);
148 }
149
guc_ads_waklv_size(struct xe_guc_ads * ads)150 static u32 guc_ads_waklv_size(struct xe_guc_ads *ads)
151 {
152 return PAGE_ALIGN(ads->ads_waklv_size);
153 }
154
guc_ads_capture_size(struct xe_guc_ads * ads)155 static size_t guc_ads_capture_size(struct xe_guc_ads *ads)
156 {
157 return PAGE_ALIGN(ads->capture_size);
158 }
159
guc_ads_um_queues_size(struct xe_guc_ads * ads)160 static size_t guc_ads_um_queues_size(struct xe_guc_ads *ads)
161 {
162 struct xe_device *xe = ads_to_xe(ads);
163
164 if (!xe->info.has_usm)
165 return 0;
166
167 return GUC_UM_QUEUE_SIZE * GUC_UM_HW_QUEUE_MAX;
168 }
169
guc_ads_private_data_size(struct xe_guc_ads * ads)170 static size_t guc_ads_private_data_size(struct xe_guc_ads *ads)
171 {
172 return PAGE_ALIGN(ads_to_guc(ads)->fw.private_data_size);
173 }
174
guc_ads_regset_offset(struct xe_guc_ads * ads)175 static size_t guc_ads_regset_offset(struct xe_guc_ads *ads)
176 {
177 return offsetof(struct __guc_ads_blob, regset);
178 }
179
guc_ads_golden_lrc_offset(struct xe_guc_ads * ads)180 static size_t guc_ads_golden_lrc_offset(struct xe_guc_ads *ads)
181 {
182 size_t offset;
183
184 offset = guc_ads_regset_offset(ads) +
185 guc_ads_regset_size(ads);
186
187 return PAGE_ALIGN(offset);
188 }
189
guc_ads_waklv_offset(struct xe_guc_ads * ads)190 static size_t guc_ads_waklv_offset(struct xe_guc_ads *ads)
191 {
192 u32 offset;
193
194 offset = guc_ads_golden_lrc_offset(ads) +
195 guc_ads_golden_lrc_size(ads);
196
197 return PAGE_ALIGN(offset);
198 }
199
guc_ads_capture_offset(struct xe_guc_ads * ads)200 static size_t guc_ads_capture_offset(struct xe_guc_ads *ads)
201 {
202 size_t offset;
203
204 offset = guc_ads_waklv_offset(ads) +
205 guc_ads_waklv_size(ads);
206
207 return PAGE_ALIGN(offset);
208 }
209
guc_ads_um_queues_offset(struct xe_guc_ads * ads)210 static size_t guc_ads_um_queues_offset(struct xe_guc_ads *ads)
211 {
212 u32 offset;
213
214 offset = guc_ads_capture_offset(ads) +
215 guc_ads_capture_size(ads);
216
217 return PAGE_ALIGN(offset);
218 }
219
guc_ads_private_data_offset(struct xe_guc_ads * ads)220 static size_t guc_ads_private_data_offset(struct xe_guc_ads *ads)
221 {
222 size_t offset;
223
224 offset = guc_ads_um_queues_offset(ads) +
225 guc_ads_um_queues_size(ads);
226
227 return PAGE_ALIGN(offset);
228 }
229
guc_ads_size(struct xe_guc_ads * ads)230 static size_t guc_ads_size(struct xe_guc_ads *ads)
231 {
232 return guc_ads_private_data_offset(ads) +
233 guc_ads_private_data_size(ads);
234 }
235
calculate_regset_size(struct xe_gt * gt)236 static size_t calculate_regset_size(struct xe_gt *gt)
237 {
238 struct xe_reg_sr_entry *sr_entry;
239 unsigned long sr_idx;
240 struct xe_hw_engine *hwe;
241 enum xe_hw_engine_id id;
242 unsigned int count = 0;
243
244 for_each_hw_engine(hwe, gt, id)
245 xa_for_each(&hwe->reg_sr.xa, sr_idx, sr_entry)
246 count++;
247
248 count += ADS_REGSET_EXTRA_MAX * XE_NUM_HW_ENGINES;
249
250 if (XE_WA(gt, 1607983814))
251 count += LNCFCMOCS_REG_COUNT;
252
253 return count * sizeof(struct guc_mmio_reg);
254 }
255
engine_enable_mask(struct xe_gt * gt,enum xe_engine_class class)256 static u32 engine_enable_mask(struct xe_gt *gt, enum xe_engine_class class)
257 {
258 struct xe_hw_engine *hwe;
259 enum xe_hw_engine_id id;
260 u32 mask = 0;
261
262 for_each_hw_engine(hwe, gt, id)
263 if (hwe->class == class)
264 mask |= BIT(hwe->instance);
265
266 return mask;
267 }
268
calculate_golden_lrc_size(struct xe_guc_ads * ads)269 static size_t calculate_golden_lrc_size(struct xe_guc_ads *ads)
270 {
271 struct xe_gt *gt = ads_to_gt(ads);
272 size_t total_size = 0, alloc_size, real_size;
273 int class;
274
275 for (class = 0; class < XE_ENGINE_CLASS_MAX; ++class) {
276 if (!engine_enable_mask(gt, class))
277 continue;
278
279 real_size = xe_gt_lrc_size(gt, class);
280 alloc_size = PAGE_ALIGN(real_size);
281 total_size += alloc_size;
282 }
283
284 return total_size;
285 }
286
guc_waklv_enable_one_word(struct xe_guc_ads * ads,enum xe_guc_klv_ids klv_id,u32 value,u32 * offset,u32 * remain)287 static void guc_waklv_enable_one_word(struct xe_guc_ads *ads,
288 enum xe_guc_klv_ids klv_id,
289 u32 value,
290 u32 *offset, u32 *remain)
291 {
292 u32 size;
293 u32 klv_entry[] = {
294 /* 16:16 key/length */
295 FIELD_PREP(GUC_KLV_0_KEY, klv_id) |
296 FIELD_PREP(GUC_KLV_0_LEN, 1),
297 value,
298 /* 1 dword data */
299 };
300
301 size = sizeof(klv_entry);
302
303 if (*remain < size) {
304 drm_warn(&ads_to_xe(ads)->drm,
305 "w/a klv buffer too small to add klv id %d\n", klv_id);
306 } else {
307 xe_map_memcpy_to(ads_to_xe(ads), ads_to_map(ads), *offset,
308 klv_entry, size);
309 *offset += size;
310 *remain -= size;
311 }
312 }
313
guc_waklv_enable_simple(struct xe_guc_ads * ads,enum xe_guc_klv_ids klv_id,u32 * offset,u32 * remain)314 static void guc_waklv_enable_simple(struct xe_guc_ads *ads,
315 enum xe_guc_klv_ids klv_id, u32 *offset, u32 *remain)
316 {
317 u32 klv_entry[] = {
318 /* 16:16 key/length */
319 FIELD_PREP(GUC_KLV_0_KEY, klv_id) |
320 FIELD_PREP(GUC_KLV_0_LEN, 0),
321 /* 0 dwords data */
322 };
323 u32 size;
324
325 size = sizeof(klv_entry);
326
327 if (xe_gt_WARN(ads_to_gt(ads), *remain < size,
328 "w/a klv buffer too small to add klv id %d\n", klv_id))
329 return;
330
331 xe_map_memcpy_to(ads_to_xe(ads), ads_to_map(ads), *offset,
332 klv_entry, size);
333 *offset += size;
334 *remain -= size;
335 }
336
guc_waklv_init(struct xe_guc_ads * ads)337 static void guc_waklv_init(struct xe_guc_ads *ads)
338 {
339 struct xe_gt *gt = ads_to_gt(ads);
340 u64 addr_ggtt;
341 u32 offset, remain, size;
342
343 offset = guc_ads_waklv_offset(ads);
344 remain = guc_ads_waklv_size(ads);
345
346 if (XE_WA(gt, 14019882105) || XE_WA(gt, 16021333562))
347 guc_waklv_enable_simple(ads,
348 GUC_WORKAROUND_KLV_BLOCK_INTERRUPTS_WHEN_MGSR_BLOCKED,
349 &offset, &remain);
350 if (XE_WA(gt, 18024947630))
351 guc_waklv_enable_simple(ads,
352 GUC_WORKAROUND_KLV_ID_GAM_PFQ_SHADOW_TAIL_POLLING,
353 &offset, &remain);
354 if (XE_WA(gt, 16022287689))
355 guc_waklv_enable_simple(ads,
356 GUC_WORKAROUND_KLV_ID_DISABLE_MTP_DURING_ASYNC_COMPUTE,
357 &offset, &remain);
358
359 if (XE_WA(gt, 14022866841))
360 guc_waklv_enable_simple(ads,
361 GUC_WA_KLV_WAKE_POWER_DOMAINS_FOR_OUTBOUND_MMIO,
362 &offset, &remain);
363
364 /*
365 * On RC6 exit, GuC will write register 0xB04 with the default value provided. As of now,
366 * the default value for this register is determined to be 0xC40. This could change in the
367 * future, so GuC depends on KMD to send it the correct value.
368 */
369 if (XE_WA(gt, 13011645652))
370 guc_waklv_enable_one_word(ads,
371 GUC_WA_KLV_NP_RD_WRITE_TO_CLEAR_RCSM_AT_CGP_LATE_RESTORE,
372 0xC40,
373 &offset, &remain);
374
375 if (XE_WA(gt, 14022293748) || XE_WA(gt, 22019794406))
376 guc_waklv_enable_simple(ads,
377 GUC_WORKAROUND_KLV_ID_BACK_TO_BACK_RCS_ENGINE_RESET,
378 &offset, &remain);
379
380 if (GUC_FIRMWARE_VER(>->uc.guc) >= MAKE_GUC_VER(70, 44, 0) && XE_WA(gt, 16026508708))
381 guc_waklv_enable_simple(ads,
382 GUC_WA_KLV_RESET_BB_STACK_PTR_ON_VF_SWITCH,
383 &offset, &remain);
384
385 size = guc_ads_waklv_size(ads) - remain;
386 if (!size)
387 return;
388
389 offset = guc_ads_waklv_offset(ads);
390 addr_ggtt = xe_bo_ggtt_addr(ads->bo) + offset;
391
392 ads_blob_write(ads, ads.wa_klv_addr_lo, lower_32_bits(addr_ggtt));
393 ads_blob_write(ads, ads.wa_klv_addr_hi, upper_32_bits(addr_ggtt));
394 ads_blob_write(ads, ads.wa_klv_size, size);
395 }
396
calculate_waklv_size(struct xe_guc_ads * ads)397 static int calculate_waklv_size(struct xe_guc_ads *ads)
398 {
399 /*
400 * A single page is both the minimum size possible and
401 * is sufficiently large enough for all current platforms.
402 */
403 return SZ_4K;
404 }
405
406 #define MAX_GOLDEN_LRC_SIZE (SZ_4K * 64)
407
xe_guc_ads_init(struct xe_guc_ads * ads)408 int xe_guc_ads_init(struct xe_guc_ads *ads)
409 {
410 struct xe_device *xe = ads_to_xe(ads);
411 struct xe_gt *gt = ads_to_gt(ads);
412 struct xe_tile *tile = gt_to_tile(gt);
413 struct xe_bo *bo;
414
415 ads->golden_lrc_size = calculate_golden_lrc_size(ads);
416 ads->capture_size = xe_guc_capture_ads_input_worst_size(ads_to_guc(ads));
417 ads->regset_size = calculate_regset_size(gt);
418 ads->ads_waklv_size = calculate_waklv_size(ads);
419
420 bo = xe_managed_bo_create_pin_map(xe, tile, guc_ads_size(ads) + MAX_GOLDEN_LRC_SIZE,
421 XE_BO_FLAG_SYSTEM |
422 XE_BO_FLAG_GGTT |
423 XE_BO_FLAG_GGTT_INVALIDATE |
424 XE_BO_FLAG_PINNED_NORESTORE);
425 if (IS_ERR(bo))
426 return PTR_ERR(bo);
427
428 ads->bo = bo;
429
430 return 0;
431 }
432 ALLOW_ERROR_INJECTION(xe_guc_ads_init, ERRNO); /* See xe_pci_probe() */
433
434 /**
435 * xe_guc_ads_init_post_hwconfig - initialize ADS post hwconfig load
436 * @ads: Additional data structures object
437 *
438 * Recalculate golden_lrc_size, capture_size and regset_size as the number
439 * hardware engines may have changed after the hwconfig was loaded. Also verify
440 * the new sizes fit in the already allocated ADS buffer object.
441 *
442 * Return: 0 on success, negative error code on error.
443 */
xe_guc_ads_init_post_hwconfig(struct xe_guc_ads * ads)444 int xe_guc_ads_init_post_hwconfig(struct xe_guc_ads *ads)
445 {
446 struct xe_gt *gt = ads_to_gt(ads);
447 u32 prev_regset_size = ads->regset_size;
448
449 xe_gt_assert(gt, ads->bo);
450
451 ads->golden_lrc_size = calculate_golden_lrc_size(ads);
452 /* Calculate Capture size with worst size */
453 ads->capture_size = xe_guc_capture_ads_input_worst_size(ads_to_guc(ads));
454 ads->regset_size = calculate_regset_size(gt);
455
456 xe_gt_assert(gt, ads->golden_lrc_size +
457 (ads->regset_size - prev_regset_size) <=
458 MAX_GOLDEN_LRC_SIZE);
459
460 return 0;
461 }
462
guc_policies_init(struct xe_guc_ads * ads)463 static void guc_policies_init(struct xe_guc_ads *ads)
464 {
465 struct xe_device *xe = ads_to_xe(ads);
466 u32 global_flags = 0;
467
468 ads_blob_write(ads, policies.dpc_promote_time,
469 GLOBAL_POLICY_DEFAULT_DPC_PROMOTE_TIME_US);
470 ads_blob_write(ads, policies.max_num_work_items,
471 GLOBAL_POLICY_MAX_NUM_WI);
472
473 if (xe->wedged.mode == 2)
474 global_flags |= GLOBAL_POLICY_DISABLE_ENGINE_RESET;
475
476 ads_blob_write(ads, policies.global_flags, global_flags);
477 ads_blob_write(ads, policies.is_valid, 1);
478 }
479
fill_engine_enable_masks(struct xe_gt * gt,struct iosys_map * info_map)480 static void fill_engine_enable_masks(struct xe_gt *gt,
481 struct iosys_map *info_map)
482 {
483 struct xe_device *xe = gt_to_xe(gt);
484
485 info_map_write(xe, info_map, engine_enabled_masks[GUC_RENDER_CLASS],
486 engine_enable_mask(gt, XE_ENGINE_CLASS_RENDER));
487 info_map_write(xe, info_map, engine_enabled_masks[GUC_BLITTER_CLASS],
488 engine_enable_mask(gt, XE_ENGINE_CLASS_COPY));
489 info_map_write(xe, info_map, engine_enabled_masks[GUC_VIDEO_CLASS],
490 engine_enable_mask(gt, XE_ENGINE_CLASS_VIDEO_DECODE));
491 info_map_write(xe, info_map,
492 engine_enabled_masks[GUC_VIDEOENHANCE_CLASS],
493 engine_enable_mask(gt, XE_ENGINE_CLASS_VIDEO_ENHANCE));
494 info_map_write(xe, info_map, engine_enabled_masks[GUC_COMPUTE_CLASS],
495 engine_enable_mask(gt, XE_ENGINE_CLASS_COMPUTE));
496 info_map_write(xe, info_map, engine_enabled_masks[GUC_GSC_OTHER_CLASS],
497 engine_enable_mask(gt, XE_ENGINE_CLASS_OTHER));
498 }
499
500 /*
501 * Write the offsets corresponding to the golden LRCs. The actual data is
502 * populated later by guc_golden_lrc_populate()
503 */
guc_golden_lrc_init(struct xe_guc_ads * ads)504 static void guc_golden_lrc_init(struct xe_guc_ads *ads)
505 {
506 struct xe_device *xe = ads_to_xe(ads);
507 struct xe_gt *gt = ads_to_gt(ads);
508 struct iosys_map info_map = IOSYS_MAP_INIT_OFFSET(ads_to_map(ads),
509 offsetof(struct __guc_ads_blob, system_info));
510 size_t alloc_size, real_size;
511 u32 addr_ggtt, offset;
512 int class;
513
514 offset = guc_ads_golden_lrc_offset(ads);
515 addr_ggtt = xe_bo_ggtt_addr(ads->bo) + offset;
516
517 for (class = 0; class < XE_ENGINE_CLASS_MAX; ++class) {
518 u8 guc_class;
519
520 guc_class = xe_engine_class_to_guc_class(class);
521
522 if (!info_map_read(xe, &info_map,
523 engine_enabled_masks[guc_class]))
524 continue;
525
526 real_size = xe_gt_lrc_size(gt, class);
527 alloc_size = PAGE_ALIGN(real_size);
528
529 /*
530 * This interface is slightly confusing. We need to pass the
531 * base address of the full golden context and the size of just
532 * the engine state, which is the section of the context image
533 * that starts after the execlists LRC registers. This is
534 * required to allow the GuC to restore just the engine state
535 * when a watchdog reset occurs.
536 * We calculate the engine state size by removing the size of
537 * what comes before it in the context image (which is identical
538 * on all engines).
539 */
540 ads_blob_write(ads, ads.eng_state_size[guc_class],
541 real_size - xe_lrc_skip_size(xe));
542 ads_blob_write(ads, ads.golden_context_lrca[guc_class],
543 addr_ggtt);
544
545 addr_ggtt += alloc_size;
546 }
547 }
548
guc_mapping_table_init_invalid(struct xe_gt * gt,struct iosys_map * info_map)549 static void guc_mapping_table_init_invalid(struct xe_gt *gt,
550 struct iosys_map *info_map)
551 {
552 struct xe_device *xe = gt_to_xe(gt);
553 unsigned int i, j;
554
555 /* Table must be set to invalid values for entries not used */
556 for (i = 0; i < GUC_MAX_ENGINE_CLASSES; ++i)
557 for (j = 0; j < GUC_MAX_INSTANCES_PER_CLASS; ++j)
558 info_map_write(xe, info_map, mapping_table[i][j],
559 GUC_MAX_INSTANCES_PER_CLASS);
560 }
561
guc_mapping_table_init(struct xe_gt * gt,struct iosys_map * info_map)562 static void guc_mapping_table_init(struct xe_gt *gt,
563 struct iosys_map *info_map)
564 {
565 struct xe_device *xe = gt_to_xe(gt);
566 struct xe_hw_engine *hwe;
567 enum xe_hw_engine_id id;
568
569 guc_mapping_table_init_invalid(gt, info_map);
570
571 for_each_hw_engine(hwe, gt, id) {
572 u8 guc_class;
573
574 guc_class = xe_engine_class_to_guc_class(hwe->class);
575 info_map_write(xe, info_map,
576 mapping_table[guc_class][hwe->logical_instance],
577 hwe->instance);
578 }
579 }
580
guc_get_capture_engine_mask(struct xe_gt * gt,struct iosys_map * info_map,enum guc_capture_list_class_type capture_class)581 static u32 guc_get_capture_engine_mask(struct xe_gt *gt, struct iosys_map *info_map,
582 enum guc_capture_list_class_type capture_class)
583 {
584 struct xe_device *xe = gt_to_xe(gt);
585 u32 mask;
586
587 switch (capture_class) {
588 case GUC_CAPTURE_LIST_CLASS_RENDER_COMPUTE:
589 mask = info_map_read(xe, info_map, engine_enabled_masks[GUC_RENDER_CLASS]);
590 mask |= info_map_read(xe, info_map, engine_enabled_masks[GUC_COMPUTE_CLASS]);
591 break;
592 case GUC_CAPTURE_LIST_CLASS_VIDEO:
593 mask = info_map_read(xe, info_map, engine_enabled_masks[GUC_VIDEO_CLASS]);
594 break;
595 case GUC_CAPTURE_LIST_CLASS_VIDEOENHANCE:
596 mask = info_map_read(xe, info_map, engine_enabled_masks[GUC_VIDEOENHANCE_CLASS]);
597 break;
598 case GUC_CAPTURE_LIST_CLASS_BLITTER:
599 mask = info_map_read(xe, info_map, engine_enabled_masks[GUC_BLITTER_CLASS]);
600 break;
601 case GUC_CAPTURE_LIST_CLASS_GSC_OTHER:
602 mask = info_map_read(xe, info_map, engine_enabled_masks[GUC_GSC_OTHER_CLASS]);
603 break;
604 default:
605 mask = 0;
606 }
607
608 return mask;
609 }
610
get_capture_list(struct xe_guc_ads * ads,struct xe_guc * guc,struct xe_gt * gt,int owner,int type,int class,u32 * total_size,size_t * size,void ** pptr)611 static inline bool get_capture_list(struct xe_guc_ads *ads, struct xe_guc *guc, struct xe_gt *gt,
612 int owner, int type, int class, u32 *total_size, size_t *size,
613 void **pptr)
614 {
615 *size = 0;
616
617 if (!xe_guc_capture_getlistsize(guc, owner, type, class, size)) {
618 if (*total_size + *size > ads->capture_size)
619 xe_gt_dbg(gt, "Capture size overflow :%zu vs %d\n",
620 *total_size + *size, ads->capture_size);
621 else if (!xe_guc_capture_getlist(guc, owner, type, class, pptr))
622 return false;
623 }
624
625 return true;
626 }
627
guc_capture_prep_lists(struct xe_guc_ads * ads)628 static int guc_capture_prep_lists(struct xe_guc_ads *ads)
629 {
630 struct xe_guc *guc = ads_to_guc(ads);
631 struct xe_gt *gt = ads_to_gt(ads);
632 u32 ads_ggtt, capture_offset, null_ggtt, total_size = 0;
633 struct iosys_map info_map;
634 size_t size = 0;
635 void *ptr;
636 int i, j;
637
638 /*
639 * GuC Capture's steered reg-list needs to be allocated and initialized
640 * after the GuC-hwconfig is available which guaranteed from here.
641 */
642 xe_guc_capture_steered_list_init(ads_to_guc(ads));
643
644 capture_offset = guc_ads_capture_offset(ads);
645 ads_ggtt = xe_bo_ggtt_addr(ads->bo);
646 info_map = IOSYS_MAP_INIT_OFFSET(ads_to_map(ads),
647 offsetof(struct __guc_ads_blob, system_info));
648
649 /* first, set aside the first page for a capture_list with zero descriptors */
650 total_size = PAGE_SIZE;
651 if (!xe_guc_capture_getnullheader(guc, &ptr, &size))
652 xe_map_memcpy_to(ads_to_xe(ads), ads_to_map(ads), capture_offset, ptr, size);
653
654 null_ggtt = ads_ggtt + capture_offset;
655 capture_offset += PAGE_SIZE;
656
657 /*
658 * Populate capture list : at this point adps is already allocated and
659 * mapped to worst case size
660 */
661 for (i = 0; i < GUC_CAPTURE_LIST_INDEX_MAX; i++) {
662 bool write_empty_list;
663
664 for (j = 0; j < GUC_CAPTURE_LIST_CLASS_MAX; j++) {
665 u32 engine_mask = guc_get_capture_engine_mask(gt, &info_map, j);
666 /* null list if we dont have said engine or list */
667 if (!engine_mask) {
668 ads_blob_write(ads, ads.capture_class[i][j], null_ggtt);
669 ads_blob_write(ads, ads.capture_instance[i][j], null_ggtt);
670 continue;
671 }
672
673 /* engine exists: start with engine-class registers */
674 write_empty_list = get_capture_list(ads, guc, gt, i,
675 GUC_STATE_CAPTURE_TYPE_ENGINE_CLASS,
676 j, &total_size, &size, &ptr);
677 if (!write_empty_list) {
678 ads_blob_write(ads, ads.capture_class[i][j],
679 ads_ggtt + capture_offset);
680 xe_map_memcpy_to(ads_to_xe(ads), ads_to_map(ads), capture_offset,
681 ptr, size);
682 total_size += size;
683 capture_offset += size;
684 } else {
685 ads_blob_write(ads, ads.capture_class[i][j], null_ggtt);
686 }
687
688 /* engine exists: next, engine-instance registers */
689 write_empty_list = get_capture_list(ads, guc, gt, i,
690 GUC_STATE_CAPTURE_TYPE_ENGINE_INSTANCE,
691 j, &total_size, &size, &ptr);
692 if (!write_empty_list) {
693 ads_blob_write(ads, ads.capture_instance[i][j],
694 ads_ggtt + capture_offset);
695 xe_map_memcpy_to(ads_to_xe(ads), ads_to_map(ads), capture_offset,
696 ptr, size);
697 total_size += size;
698 capture_offset += size;
699 } else {
700 ads_blob_write(ads, ads.capture_instance[i][j], null_ggtt);
701 }
702 }
703
704 /* global registers is last in our PF/VF loops */
705 write_empty_list = get_capture_list(ads, guc, gt, i,
706 GUC_STATE_CAPTURE_TYPE_GLOBAL,
707 0, &total_size, &size, &ptr);
708 if (!write_empty_list) {
709 ads_blob_write(ads, ads.capture_global[i], ads_ggtt + capture_offset);
710 xe_map_memcpy_to(ads_to_xe(ads), ads_to_map(ads), capture_offset, ptr,
711 size);
712 total_size += size;
713 capture_offset += size;
714 } else {
715 ads_blob_write(ads, ads.capture_global[i], null_ggtt);
716 }
717 }
718
719 if (ads->capture_size != PAGE_ALIGN(total_size))
720 xe_gt_dbg(gt, "Updated ADS capture size %d (was %d)\n",
721 PAGE_ALIGN(total_size), ads->capture_size);
722 return PAGE_ALIGN(total_size);
723 }
724
guc_mmio_regset_write_one(struct xe_guc_ads * ads,struct iosys_map * regset_map,struct xe_reg reg,unsigned int n_entry)725 static void guc_mmio_regset_write_one(struct xe_guc_ads *ads,
726 struct iosys_map *regset_map,
727 struct xe_reg reg,
728 unsigned int n_entry)
729 {
730 struct guc_mmio_reg entry = {
731 .offset = reg.addr,
732 .flags = reg.masked ? GUC_REGSET_MASKED : 0,
733 };
734
735 if (reg.mcr) {
736 struct xe_reg_mcr mcr_reg = XE_REG_MCR(reg.addr);
737 u8 group, instance;
738
739 bool steer = xe_gt_mcr_get_nonterminated_steering(ads_to_gt(ads), mcr_reg,
740 &group, &instance);
741
742 if (steer) {
743 entry.flags |= FIELD_PREP(GUC_REGSET_STEERING_GROUP, group);
744 entry.flags |= FIELD_PREP(GUC_REGSET_STEERING_INSTANCE, instance);
745 entry.flags |= GUC_REGSET_STEERING_NEEDED;
746 }
747 }
748
749 xe_map_memcpy_to(ads_to_xe(ads), regset_map, n_entry * sizeof(entry),
750 &entry, sizeof(entry));
751 }
752
guc_mmio_regset_write(struct xe_guc_ads * ads,struct iosys_map * regset_map,struct xe_hw_engine * hwe)753 static unsigned int guc_mmio_regset_write(struct xe_guc_ads *ads,
754 struct iosys_map *regset_map,
755 struct xe_hw_engine *hwe)
756 {
757 struct xe_hw_engine *hwe_rcs_reset_domain =
758 xe_gt_any_hw_engine_by_reset_domain(hwe->gt, XE_ENGINE_CLASS_RENDER);
759 struct xe_reg_sr_entry *entry;
760 unsigned long idx;
761 unsigned int count = 0;
762 const struct {
763 struct xe_reg reg;
764 bool skip;
765 } *e, extra_regs[] = {
766 { .reg = RING_MODE(hwe->mmio_base), },
767 { .reg = RING_HWS_PGA(hwe->mmio_base), },
768 { .reg = RING_IMR(hwe->mmio_base), },
769 { .reg = RCU_MODE, .skip = hwe != hwe_rcs_reset_domain },
770 { .reg = CCS_MODE,
771 .skip = hwe != hwe_rcs_reset_domain || !xe_gt_ccs_mode_enabled(hwe->gt) },
772 };
773 u32 i;
774
775 BUILD_BUG_ON(ARRAY_SIZE(extra_regs) > ADS_REGSET_EXTRA_MAX);
776
777 xa_for_each(&hwe->reg_sr.xa, idx, entry)
778 guc_mmio_regset_write_one(ads, regset_map, entry->reg, count++);
779
780 for (e = extra_regs; e < extra_regs + ARRAY_SIZE(extra_regs); e++) {
781 if (e->skip)
782 continue;
783
784 guc_mmio_regset_write_one(ads, regset_map, e->reg, count++);
785 }
786
787 if (XE_WA(hwe->gt, 1607983814) && hwe->class == XE_ENGINE_CLASS_RENDER) {
788 for (i = 0; i < LNCFCMOCS_REG_COUNT; i++) {
789 guc_mmio_regset_write_one(ads, regset_map,
790 XELP_LNCFCMOCS(i), count++);
791 }
792 }
793
794 return count;
795 }
796
guc_mmio_reg_state_init(struct xe_guc_ads * ads)797 static void guc_mmio_reg_state_init(struct xe_guc_ads *ads)
798 {
799 size_t regset_offset = guc_ads_regset_offset(ads);
800 struct xe_gt *gt = ads_to_gt(ads);
801 struct xe_hw_engine *hwe;
802 enum xe_hw_engine_id id;
803 u32 addr = xe_bo_ggtt_addr(ads->bo) + regset_offset;
804 struct iosys_map regset_map = IOSYS_MAP_INIT_OFFSET(ads_to_map(ads),
805 regset_offset);
806 unsigned int regset_used = 0;
807
808 for_each_hw_engine(hwe, gt, id) {
809 unsigned int count;
810 u8 gc;
811
812 /*
813 * 1. Write all MMIO entries for this exec queue to the table. No
814 * need to worry about fused-off engines and when there are
815 * entries in the regset: the reg_state_list has been zero'ed
816 * by xe_guc_ads_populate()
817 */
818 count = guc_mmio_regset_write(ads, ®set_map, hwe);
819 if (!count)
820 continue;
821
822 /*
823 * 2. Record in the header (ads.reg_state_list) the address
824 * location and number of entries
825 */
826 gc = xe_engine_class_to_guc_class(hwe->class);
827 ads_blob_write(ads, ads.reg_state_list[gc][hwe->instance].address, addr);
828 ads_blob_write(ads, ads.reg_state_list[gc][hwe->instance].count, count);
829
830 addr += count * sizeof(struct guc_mmio_reg);
831 iosys_map_incr(®set_map, count * sizeof(struct guc_mmio_reg));
832
833 regset_used += count * sizeof(struct guc_mmio_reg);
834 }
835
836 xe_gt_assert(gt, regset_used <= ads->regset_size);
837 }
838
guc_um_init_params(struct xe_guc_ads * ads)839 static void guc_um_init_params(struct xe_guc_ads *ads)
840 {
841 u32 um_queue_offset = guc_ads_um_queues_offset(ads);
842 u64 base_dpa;
843 u32 base_ggtt;
844 int i;
845
846 base_ggtt = xe_bo_ggtt_addr(ads->bo) + um_queue_offset;
847 base_dpa = xe_bo_main_addr(ads->bo, PAGE_SIZE) + um_queue_offset;
848
849 for (i = 0; i < GUC_UM_HW_QUEUE_MAX; ++i) {
850 ads_blob_write(ads, um_init_params.queue_params[i].base_dpa,
851 base_dpa + (i * GUC_UM_QUEUE_SIZE));
852 ads_blob_write(ads, um_init_params.queue_params[i].base_ggtt_address,
853 base_ggtt + (i * GUC_UM_QUEUE_SIZE));
854 ads_blob_write(ads, um_init_params.queue_params[i].size_in_bytes,
855 GUC_UM_QUEUE_SIZE);
856 }
857
858 ads_blob_write(ads, um_init_params.page_response_timeout_in_us,
859 GUC_PAGE_RES_TIMEOUT_US);
860 }
861
guc_doorbell_init(struct xe_guc_ads * ads)862 static void guc_doorbell_init(struct xe_guc_ads *ads)
863 {
864 struct xe_device *xe = ads_to_xe(ads);
865 struct xe_gt *gt = ads_to_gt(ads);
866
867 if (GRAPHICS_VER(xe) >= 12 && !IS_DGFX(xe)) {
868 u32 distdbreg =
869 xe_mmio_read32(>->mmio, DIST_DBS_POPULATED);
870
871 ads_blob_write(ads,
872 system_info.generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_DOORBELL_COUNT_PER_SQIDI],
873 REG_FIELD_GET(DOORBELLS_PER_SQIDI_MASK, distdbreg) + 1);
874 }
875 }
876
877 /**
878 * xe_guc_ads_populate_minimal - populate minimal ADS
879 * @ads: Additional data structures object
880 *
881 * This function populates a minimal ADS that does not support submissions but
882 * enough so the GuC can load and the hwconfig table can be read.
883 */
xe_guc_ads_populate_minimal(struct xe_guc_ads * ads)884 void xe_guc_ads_populate_minimal(struct xe_guc_ads *ads)
885 {
886 struct xe_gt *gt = ads_to_gt(ads);
887 struct iosys_map info_map = IOSYS_MAP_INIT_OFFSET(ads_to_map(ads),
888 offsetof(struct __guc_ads_blob, system_info));
889 u32 base = xe_bo_ggtt_addr(ads->bo);
890
891 xe_gt_assert(gt, ads->bo);
892
893 xe_map_memset(ads_to_xe(ads), ads_to_map(ads), 0, 0, xe_bo_size(ads->bo));
894 guc_policies_init(ads);
895 guc_golden_lrc_init(ads);
896 guc_mapping_table_init_invalid(gt, &info_map);
897 guc_doorbell_init(ads);
898
899 ads_blob_write(ads, ads.scheduler_policies, base +
900 offsetof(struct __guc_ads_blob, policies));
901 ads_blob_write(ads, ads.gt_system_info, base +
902 offsetof(struct __guc_ads_blob, system_info));
903 ads_blob_write(ads, ads.private_data, base +
904 guc_ads_private_data_offset(ads));
905 }
906
xe_guc_ads_populate(struct xe_guc_ads * ads)907 void xe_guc_ads_populate(struct xe_guc_ads *ads)
908 {
909 struct xe_device *xe = ads_to_xe(ads);
910 struct xe_gt *gt = ads_to_gt(ads);
911 struct iosys_map info_map = IOSYS_MAP_INIT_OFFSET(ads_to_map(ads),
912 offsetof(struct __guc_ads_blob, system_info));
913 u32 base = xe_bo_ggtt_addr(ads->bo);
914
915 xe_gt_assert(gt, ads->bo);
916
917 xe_map_memset(ads_to_xe(ads), ads_to_map(ads), 0, 0, xe_bo_size(ads->bo));
918 guc_policies_init(ads);
919 fill_engine_enable_masks(gt, &info_map);
920 guc_mmio_reg_state_init(ads);
921 guc_golden_lrc_init(ads);
922 guc_mapping_table_init(gt, &info_map);
923 guc_capture_prep_lists(ads);
924 guc_doorbell_init(ads);
925 guc_waklv_init(ads);
926
927 if (xe->info.has_usm) {
928 guc_um_init_params(ads);
929 ads_blob_write(ads, ads.um_init_data, base +
930 offsetof(struct __guc_ads_blob, um_init_params));
931 }
932
933 ads_blob_write(ads, ads.scheduler_policies, base +
934 offsetof(struct __guc_ads_blob, policies));
935 ads_blob_write(ads, ads.gt_system_info, base +
936 offsetof(struct __guc_ads_blob, system_info));
937 ads_blob_write(ads, ads.private_data, base +
938 guc_ads_private_data_offset(ads));
939 }
940
941 /*
942 * After the golden LRC's are recorded for each engine class by the first
943 * submission, copy them to the ADS, as initialized earlier by
944 * guc_golden_lrc_init().
945 */
guc_golden_lrc_populate(struct xe_guc_ads * ads)946 static void guc_golden_lrc_populate(struct xe_guc_ads *ads)
947 {
948 struct xe_device *xe = ads_to_xe(ads);
949 struct xe_gt *gt = ads_to_gt(ads);
950 struct iosys_map info_map = IOSYS_MAP_INIT_OFFSET(ads_to_map(ads),
951 offsetof(struct __guc_ads_blob, system_info));
952 size_t total_size = 0, alloc_size, real_size;
953 u32 offset;
954 int class;
955
956 offset = guc_ads_golden_lrc_offset(ads);
957
958 for (class = 0; class < XE_ENGINE_CLASS_MAX; ++class) {
959 u8 guc_class;
960
961 guc_class = xe_engine_class_to_guc_class(class);
962
963 if (!info_map_read(xe, &info_map,
964 engine_enabled_masks[guc_class]))
965 continue;
966
967 xe_gt_assert(gt, gt->default_lrc[class]);
968
969 real_size = xe_gt_lrc_size(gt, class);
970 alloc_size = PAGE_ALIGN(real_size);
971 total_size += alloc_size;
972
973 xe_map_memcpy_to(xe, ads_to_map(ads), offset,
974 gt->default_lrc[class], real_size);
975
976 offset += alloc_size;
977 }
978
979 xe_gt_assert(gt, total_size == ads->golden_lrc_size);
980 }
981
xe_guc_ads_populate_post_load(struct xe_guc_ads * ads)982 void xe_guc_ads_populate_post_load(struct xe_guc_ads *ads)
983 {
984 guc_golden_lrc_populate(ads);
985 }
986
guc_ads_action_update_policies(struct xe_guc_ads * ads,u32 policy_offset)987 static int guc_ads_action_update_policies(struct xe_guc_ads *ads, u32 policy_offset)
988 {
989 struct xe_guc_ct *ct = &ads_to_guc(ads)->ct;
990 u32 action[] = {
991 XE_GUC_ACTION_GLOBAL_SCHED_POLICY_CHANGE,
992 policy_offset
993 };
994
995 return xe_guc_ct_send(ct, action, ARRAY_SIZE(action), 0, 0);
996 }
997
998 /**
999 * xe_guc_ads_scheduler_policy_toggle_reset - Toggle reset policy
1000 * @ads: Additional data structures object
1001 *
1002 * This function update the GuC's engine reset policy based on wedged.mode.
1003 *
1004 * Return: 0 on success, and negative error code otherwise.
1005 */
xe_guc_ads_scheduler_policy_toggle_reset(struct xe_guc_ads * ads)1006 int xe_guc_ads_scheduler_policy_toggle_reset(struct xe_guc_ads *ads)
1007 {
1008 struct guc_policies *policies;
1009 struct xe_guc *guc = ads_to_guc(ads);
1010 struct xe_device *xe = ads_to_xe(ads);
1011 CLASS(xe_guc_buf, buf)(&guc->buf, sizeof(*policies));
1012
1013 if (!xe_guc_buf_is_valid(buf))
1014 return -ENOBUFS;
1015
1016 policies = xe_guc_buf_cpu_ptr(buf);
1017 memset(policies, 0, sizeof(*policies));
1018
1019 policies->dpc_promote_time = ads_blob_read(ads, policies.dpc_promote_time);
1020 policies->max_num_work_items = ads_blob_read(ads, policies.max_num_work_items);
1021 policies->is_valid = 1;
1022 if (xe->wedged.mode == 2)
1023 policies->global_flags |= GLOBAL_POLICY_DISABLE_ENGINE_RESET;
1024 else
1025 policies->global_flags &= ~GLOBAL_POLICY_DISABLE_ENGINE_RESET;
1026
1027 return guc_ads_action_update_policies(ads, xe_guc_buf_flush(buf));
1028 }
1029