1 /* SPDX-License-Identifier: MIT */
2 /*
3 * Copyright © 2014-2019 Intel Corporation
4 */
5
6 #ifndef _INTEL_GUC_H_
7 #define _INTEL_GUC_H_
8
9 #include <linux/delay.h>
10 #include <linux/iosys-map.h>
11 #include <linux/xarray.h>
12
13 #include "intel_guc_ct.h"
14 #include "intel_guc_fw.h"
15 #include "intel_guc_fwif.h"
16 #include "intel_guc_log.h"
17 #include "intel_guc_reg.h"
18 #include "intel_guc_slpc_types.h"
19 #include "intel_uc_fw.h"
20 #include "intel_uncore.h"
21 #include "i915_utils.h"
22 #include "i915_vma.h"
23
24 struct __guc_ads_blob;
25 struct intel_guc_state_capture;
26
27 /**
28 * struct intel_guc - Top level structure of GuC.
29 *
30 * It handles firmware loading and manages client pool. intel_guc owns an
31 * i915_sched_engine for submission.
32 */
33 struct intel_guc {
34 /** @fw: the GuC firmware */
35 struct intel_uc_fw fw;
36 /** @log: sub-structure containing GuC log related data and objects */
37 struct intel_guc_log log;
38 /** @ct: the command transport communication channel */
39 struct intel_guc_ct ct;
40 /** @slpc: sub-structure containing SLPC related data and objects */
41 struct intel_guc_slpc slpc;
42 /** @capture: the error-state-capture module's data and objects */
43 struct intel_guc_state_capture *capture;
44
45 /** @dbgfs_node: debugfs node */
46 struct dentry *dbgfs_node;
47
48 /** @sched_engine: Global engine used to submit requests to GuC */
49 struct i915_sched_engine *sched_engine;
50 /**
51 * @stalled_request: if GuC can't process a request for any reason, we
52 * save it until GuC restarts processing. No other request can be
53 * submitted until the stalled request is processed.
54 */
55 struct i915_request *stalled_request;
56 /**
57 * @submission_stall_reason: reason why submission is stalled
58 */
59 enum {
60 STALL_NONE,
61 STALL_REGISTER_CONTEXT,
62 STALL_MOVE_LRC_TAIL,
63 STALL_ADD_REQUEST,
64 } submission_stall_reason;
65
66 /* intel_guc_recv interrupt related state */
67 /** @irq_lock: protects GuC irq state */
68 spinlock_t irq_lock;
69 /**
70 * @msg_enabled_mask: mask of events that are processed when receiving
71 * an INTEL_GUC_ACTION_DEFAULT G2H message.
72 */
73 unsigned int msg_enabled_mask;
74
75 /**
76 * @outstanding_submission_g2h: number of outstanding GuC to Host
77 * responses related to GuC submission, used to determine if the GT is
78 * idle
79 */
80 atomic_t outstanding_submission_g2h;
81
82 /** @tlb_lookup: xarray to store all pending TLB invalidation requests */
83 struct xarray tlb_lookup;
84
85 /**
86 * @serial_slot: id to the initial waiter created in tlb_lookup,
87 * which is used only when failed to allocate new waiter.
88 */
89 u32 serial_slot;
90
91 /** @next_seqno: the next id (sequence number) to allocate. */
92 u32 next_seqno;
93
94 /** @interrupts: pointers to GuC interrupt-managing functions. */
95 struct {
96 bool enabled;
97 void (*reset)(struct intel_guc *guc);
98 void (*enable)(struct intel_guc *guc);
99 void (*disable)(struct intel_guc *guc);
100 } interrupts;
101
102 /**
103 * @submission_state: sub-structure for submission state protected by
104 * single lock
105 */
106 struct {
107 /**
108 * @submission_state.lock: protects everything in
109 * submission_state, ce->guc_id.id, and ce->guc_id.ref
110 * when transitioning in and out of zero
111 */
112 spinlock_t lock;
113 /**
114 * @submission_state.guc_ids: used to allocate new
115 * guc_ids, single-lrc
116 */
117 struct ida guc_ids;
118 /**
119 * @submission_state.num_guc_ids: Number of guc_ids, selftest
120 * feature to be able to reduce this number while testing.
121 */
122 int num_guc_ids;
123 /**
124 * @submission_state.guc_ids_bitmap: used to allocate
125 * new guc_ids, multi-lrc
126 */
127 unsigned long *guc_ids_bitmap;
128 /**
129 * @submission_state.guc_id_list: list of intel_context
130 * with valid guc_ids but no refs
131 */
132 struct list_head guc_id_list;
133 /**
134 * @submission_state.guc_ids_in_use: Number single-lrc
135 * guc_ids in use
136 */
137 unsigned int guc_ids_in_use;
138 /**
139 * @submission_state.destroyed_contexts: list of contexts
140 * waiting to be destroyed (deregistered with the GuC)
141 */
142 struct list_head destroyed_contexts;
143 /**
144 * @submission_state.destroyed_worker: worker to deregister
145 * contexts, need as we need to take a GT PM reference and
146 * can't from destroy function as it might be in an atomic
147 * context (no sleeping)
148 */
149 struct work_struct destroyed_worker;
150 /**
151 * @submission_state.reset_fail_worker: worker to trigger
152 * a GT reset after an engine reset fails
153 */
154 struct work_struct reset_fail_worker;
155 /**
156 * @submission_state.reset_fail_mask: mask of engines that
157 * failed to reset
158 */
159 intel_engine_mask_t reset_fail_mask;
160 /**
161 * @submission_state.sched_disable_delay_ms: schedule
162 * disable delay, in ms, for contexts
163 */
164 unsigned int sched_disable_delay_ms;
165 /**
166 * @submission_state.sched_disable_gucid_threshold:
167 * threshold of min remaining available guc_ids before
168 * we start bypassing the schedule disable delay
169 */
170 unsigned int sched_disable_gucid_threshold;
171 } submission_state;
172
173 /**
174 * @submission_supported: tracks whether we support GuC submission on
175 * the current platform
176 */
177 bool submission_supported;
178 /** @submission_selected: tracks whether the user enabled GuC submission */
179 bool submission_selected;
180 /** @submission_initialized: tracks whether GuC submission has been initialised */
181 bool submission_initialized;
182 /** @submission_version: Submission API version of the currently loaded firmware */
183 struct intel_uc_fw_ver submission_version;
184
185 /**
186 * @rc_supported: tracks whether we support GuC rc on the current platform
187 */
188 bool rc_supported;
189 /** @rc_selected: tracks whether the user enabled GuC rc */
190 bool rc_selected;
191
192 /** @ads_vma: object allocated to hold the GuC ADS */
193 struct i915_vma *ads_vma;
194 /** @ads_map: contents of the GuC ADS */
195 struct iosys_map ads_map;
196 /** @ads_regset_size: size of the save/restore regsets in the ADS */
197 u32 ads_regset_size;
198 /**
199 * @ads_regset_count: number of save/restore registers in the ADS for
200 * each engine
201 */
202 u32 ads_regset_count[I915_NUM_ENGINES];
203 /** @ads_regset: save/restore regsets in the ADS */
204 struct guc_mmio_reg *ads_regset;
205 /** @ads_golden_ctxt_size: size of the golden contexts in the ADS */
206 u32 ads_golden_ctxt_size;
207 /** @ads_waklv_size: size of workaround KLVs */
208 u32 ads_waklv_size;
209 /** @ads_capture_size: size of register lists in the ADS used for error capture */
210 u32 ads_capture_size;
211
212 /** @lrc_desc_pool_v69: object allocated to hold the GuC LRC descriptor pool */
213 struct i915_vma *lrc_desc_pool_v69;
214 /** @lrc_desc_pool_vaddr_v69: contents of the GuC LRC descriptor pool */
215 void *lrc_desc_pool_vaddr_v69;
216
217 /**
218 * @context_lookup: used to resolve intel_context from guc_id, if a
219 * context is present in this structure it is registered with the GuC
220 */
221 struct xarray context_lookup;
222
223 /** @params: Control params for fw initialization */
224 u32 params[GUC_CTL_MAX_DWORDS];
225
226 /** @send_regs: GuC's FW specific registers used for sending MMIO H2G */
227 struct {
228 u32 base;
229 unsigned int count;
230 enum forcewake_domains fw_domains;
231 } send_regs;
232
233 /** @notify_reg: register used to send interrupts to the GuC FW */
234 i915_reg_t notify_reg;
235
236 /**
237 * @mmio_msg: notification bitmask that the GuC writes in one of its
238 * registers when the CT channel is disabled, to be processed when the
239 * channel is back up.
240 */
241 u32 mmio_msg;
242
243 /** @send_mutex: used to serialize the intel_guc_send actions */
244 struct mutex send_mutex;
245
246 /**
247 * @timestamp: GT timestamp object that stores a copy of the timestamp
248 * and adjusts it for overflow using a worker.
249 */
250 struct {
251 /**
252 * @timestamp.lock: Lock protecting the below fields and
253 * the engine stats.
254 */
255 spinlock_t lock;
256
257 /**
258 * @timestamp.gt_stamp: 64-bit extended value of the GT
259 * timestamp.
260 */
261 u64 gt_stamp;
262
263 /**
264 * @timestamp.ping_delay: Period for polling the GT
265 * timestamp for overflow.
266 */
267 unsigned long ping_delay;
268
269 /**
270 * @timestamp.work: Periodic work to adjust GT timestamp,
271 * engine and context usage for overflows.
272 */
273 struct delayed_work work;
274
275 /**
276 * @timestamp.shift: Right shift value for the gpm timestamp
277 */
278 u32 shift;
279
280 /**
281 * @timestamp.last_stat_jiffies: jiffies at last actual
282 * stats collection time. We use this timestamp to ensure
283 * we don't oversample the stats because runtime power
284 * management events can trigger stats collection at much
285 * higher rates than required.
286 */
287 unsigned long last_stat_jiffies;
288 } timestamp;
289
290 /**
291 * @dead_guc_worker: Asynchronous worker thread for forcing a GuC reset.
292 * Specifically used when the G2H handler wants to issue a reset. Resets
293 * require flushing the G2H queue. So, the G2H processing itself must not
294 * trigger a reset directly. Instead, go via this worker.
295 */
296 struct work_struct dead_guc_worker;
297 /**
298 * @last_dead_guc_jiffies: timestamp of previous 'dead guc' occurrance
299 * used to prevent a fundamentally broken system from continuously
300 * reloading the GuC.
301 */
302 unsigned long last_dead_guc_jiffies;
303
304 #ifdef CONFIG_DRM_I915_SELFTEST
305 /**
306 * @number_guc_id_stolen: The number of guc_ids that have been stolen
307 */
308 int number_guc_id_stolen;
309 /**
310 * @fast_response_selftest: Backdoor to CT handler for fast response selftest
311 */
312 u32 fast_response_selftest;
313 #endif
314 };
315
316 struct intel_guc_tlb_wait {
317 struct wait_queue_head wq;
318 bool busy;
319 };
320
321 /*
322 * GuC version number components are only 8-bit, so converting to a 32bit 8.8.8
323 * integer works.
324 */
325 #define MAKE_GUC_VER(maj, min, pat) (((maj) << 16) | ((min) << 8) | (pat))
326 #define MAKE_GUC_VER_STRUCT(ver) MAKE_GUC_VER((ver).major, (ver).minor, (ver).patch)
327 #define GUC_SUBMIT_VER(guc) MAKE_GUC_VER_STRUCT((guc)->submission_version)
328 #define GUC_FIRMWARE_VER(guc) MAKE_GUC_VER_STRUCT((guc)->fw.file_selected.ver)
329
log_to_guc(struct intel_guc_log * log)330 static inline struct intel_guc *log_to_guc(struct intel_guc_log *log)
331 {
332 return container_of(log, struct intel_guc, log);
333 }
334
335 static
intel_guc_send(struct intel_guc * guc,const u32 * action,u32 len)336 inline int intel_guc_send(struct intel_guc *guc, const u32 *action, u32 len)
337 {
338 return intel_guc_ct_send(&guc->ct, action, len, NULL, 0, 0);
339 }
340
341 static
intel_guc_send_nb(struct intel_guc * guc,const u32 * action,u32 len,u32 g2h_len_dw)342 inline int intel_guc_send_nb(struct intel_guc *guc, const u32 *action, u32 len,
343 u32 g2h_len_dw)
344 {
345 return intel_guc_ct_send(&guc->ct, action, len, NULL, 0,
346 MAKE_SEND_FLAGS(g2h_len_dw));
347 }
348
349 static inline int
intel_guc_send_and_receive(struct intel_guc * guc,const u32 * action,u32 len,u32 * response_buf,u32 response_buf_size)350 intel_guc_send_and_receive(struct intel_guc *guc, const u32 *action, u32 len,
351 u32 *response_buf, u32 response_buf_size)
352 {
353 return intel_guc_ct_send(&guc->ct, action, len,
354 response_buf, response_buf_size, 0);
355 }
356
intel_guc_send_busy_loop(struct intel_guc * guc,const u32 * action,u32 len,u32 g2h_len_dw,bool loop)357 static inline int intel_guc_send_busy_loop(struct intel_guc *guc,
358 const u32 *action,
359 u32 len,
360 u32 g2h_len_dw,
361 bool loop)
362 {
363 int err;
364 unsigned int sleep_period_ms = 1;
365 bool not_atomic = !in_atomic() && !irqs_disabled();
366
367 /*
368 * FIXME: Have caller pass in if we are in an atomic context to avoid
369 * using in_atomic(). It is likely safe here as we check for irqs
370 * disabled which basically all the spin locks in the i915 do but
371 * regardless this should be cleaned up.
372 */
373
374 /* No sleeping with spin locks, just busy loop */
375 might_sleep_if(loop && not_atomic);
376
377 retry:
378 err = intel_guc_send_nb(guc, action, len, g2h_len_dw);
379 if (unlikely(err == -EBUSY && loop)) {
380 if (likely(not_atomic)) {
381 if (msleep_interruptible(sleep_period_ms))
382 return -EINTR;
383 sleep_period_ms = sleep_period_ms << 1;
384 } else {
385 cpu_relax();
386 }
387 goto retry;
388 }
389
390 return err;
391 }
392
393 /* Only call this from the interrupt handler code */
intel_guc_to_host_event_handler(struct intel_guc * guc)394 static inline void intel_guc_to_host_event_handler(struct intel_guc *guc)
395 {
396 if (guc->interrupts.enabled)
397 intel_guc_ct_event_handler(&guc->ct);
398 }
399
400 /* GuC addresses above GUC_GGTT_TOP also don't map through the GTT */
401 #define GUC_GGTT_TOP 0xFEE00000
402
403 /**
404 * intel_guc_ggtt_offset() - Get and validate the GGTT offset of @vma
405 * @guc: intel_guc structure.
406 * @vma: i915 graphics virtual memory area.
407 *
408 * GuC does not allow any gfx GGTT address that falls into range
409 * [0, ggtt.pin_bias), which is reserved for Boot ROM, SRAM and WOPCM.
410 * Currently, in order to exclude [0, ggtt.pin_bias) address space from
411 * GGTT, all gfx objects used by GuC are allocated with intel_guc_allocate_vma()
412 * and pinned with PIN_OFFSET_BIAS along with the value of ggtt.pin_bias.
413 *
414 * Return: GGTT offset of the @vma.
415 */
intel_guc_ggtt_offset(struct intel_guc * guc,struct i915_vma * vma)416 static inline u32 intel_guc_ggtt_offset(struct intel_guc *guc,
417 struct i915_vma *vma)
418 {
419 u32 offset = i915_ggtt_offset(vma);
420
421 GEM_BUG_ON(offset < i915_ggtt_pin_bias(vma));
422 GEM_BUG_ON(range_overflows_t(u64, offset, vma->size, GUC_GGTT_TOP));
423
424 return offset;
425 }
426
427 void intel_guc_init_early(struct intel_guc *guc);
428 void intel_guc_init_late(struct intel_guc *guc);
429 void intel_guc_init_send_regs(struct intel_guc *guc);
430 void intel_guc_write_params(struct intel_guc *guc);
431 int intel_guc_init(struct intel_guc *guc);
432 void intel_guc_fini(struct intel_guc *guc);
433 void intel_guc_notify(struct intel_guc *guc);
434 int intel_guc_send_mmio(struct intel_guc *guc, const u32 *action, u32 len,
435 u32 *response_buf, u32 response_buf_size);
436 int intel_guc_to_host_process_recv_msg(struct intel_guc *guc,
437 const u32 *payload, u32 len);
438 int intel_guc_auth_huc(struct intel_guc *guc, u32 rsa_offset);
439 int intel_guc_suspend(struct intel_guc *guc);
440 int intel_guc_resume(struct intel_guc *guc);
441 struct i915_vma *intel_guc_allocate_vma(struct intel_guc *guc, u32 size);
442 int intel_guc_allocate_and_map_vma(struct intel_guc *guc, u32 size,
443 struct i915_vma **out_vma, void **out_vaddr);
444 int intel_guc_self_cfg32(struct intel_guc *guc, u16 key, u32 value);
445 int intel_guc_self_cfg64(struct intel_guc *guc, u16 key, u64 value);
446
intel_guc_is_supported(struct intel_guc * guc)447 static inline bool intel_guc_is_supported(struct intel_guc *guc)
448 {
449 return intel_uc_fw_is_supported(&guc->fw);
450 }
451
intel_guc_is_wanted(struct intel_guc * guc)452 static inline bool intel_guc_is_wanted(struct intel_guc *guc)
453 {
454 return intel_uc_fw_is_enabled(&guc->fw);
455 }
456
intel_guc_is_used(struct intel_guc * guc)457 static inline bool intel_guc_is_used(struct intel_guc *guc)
458 {
459 GEM_BUG_ON(__intel_uc_fw_status(&guc->fw) == INTEL_UC_FIRMWARE_SELECTED);
460 return intel_uc_fw_is_available(&guc->fw);
461 }
462
intel_guc_is_fw_running(struct intel_guc * guc)463 static inline bool intel_guc_is_fw_running(struct intel_guc *guc)
464 {
465 return intel_uc_fw_is_running(&guc->fw);
466 }
467
intel_guc_is_ready(struct intel_guc * guc)468 static inline bool intel_guc_is_ready(struct intel_guc *guc)
469 {
470 return intel_guc_is_fw_running(guc) && intel_guc_ct_enabled(&guc->ct);
471 }
472
intel_guc_reset_interrupts(struct intel_guc * guc)473 static inline void intel_guc_reset_interrupts(struct intel_guc *guc)
474 {
475 guc->interrupts.reset(guc);
476 }
477
intel_guc_enable_interrupts(struct intel_guc * guc)478 static inline void intel_guc_enable_interrupts(struct intel_guc *guc)
479 {
480 guc->interrupts.enable(guc);
481 }
482
intel_guc_disable_interrupts(struct intel_guc * guc)483 static inline void intel_guc_disable_interrupts(struct intel_guc *guc)
484 {
485 guc->interrupts.disable(guc);
486 }
487
intel_guc_sanitize(struct intel_guc * guc)488 static inline int intel_guc_sanitize(struct intel_guc *guc)
489 {
490 intel_uc_fw_sanitize(&guc->fw);
491 intel_guc_disable_interrupts(guc);
492 intel_guc_ct_sanitize(&guc->ct);
493 guc->mmio_msg = 0;
494
495 return 0;
496 }
497
intel_guc_enable_msg(struct intel_guc * guc,u32 mask)498 static inline void intel_guc_enable_msg(struct intel_guc *guc, u32 mask)
499 {
500 spin_lock_irq(&guc->irq_lock);
501 guc->msg_enabled_mask |= mask;
502 spin_unlock_irq(&guc->irq_lock);
503 }
504
intel_guc_disable_msg(struct intel_guc * guc,u32 mask)505 static inline void intel_guc_disable_msg(struct intel_guc *guc, u32 mask)
506 {
507 spin_lock_irq(&guc->irq_lock);
508 guc->msg_enabled_mask &= ~mask;
509 spin_unlock_irq(&guc->irq_lock);
510 }
511
512 int intel_guc_wait_for_idle(struct intel_guc *guc, long timeout);
513
514 int intel_guc_deregister_done_process_msg(struct intel_guc *guc,
515 const u32 *msg, u32 len);
516 int intel_guc_sched_done_process_msg(struct intel_guc *guc,
517 const u32 *msg, u32 len);
518 int intel_guc_context_reset_process_msg(struct intel_guc *guc,
519 const u32 *msg, u32 len);
520 int intel_guc_engine_failure_process_msg(struct intel_guc *guc,
521 const u32 *msg, u32 len);
522 int intel_guc_error_capture_process_msg(struct intel_guc *guc,
523 const u32 *msg, u32 len);
524 int intel_guc_crash_process_msg(struct intel_guc *guc, u32 action);
525
526 struct intel_engine_cs *
527 intel_guc_lookup_engine(struct intel_guc *guc, u8 guc_class, u8 instance);
528
529 void intel_guc_find_hung_context(struct intel_engine_cs *engine);
530
531 int intel_guc_global_policies_update(struct intel_guc *guc);
532
533 void intel_guc_context_ban(struct intel_context *ce, struct i915_request *rq);
534
535 void intel_guc_submission_reset_prepare(struct intel_guc *guc);
536 void intel_guc_submission_reset(struct intel_guc *guc, intel_engine_mask_t stalled);
537 void intel_guc_submission_reset_finish(struct intel_guc *guc);
538 void intel_guc_submission_cancel_requests(struct intel_guc *guc);
539
540 void intel_guc_load_status(struct intel_guc *guc, struct drm_printer *p);
541
542 void intel_guc_write_barrier(struct intel_guc *guc);
543
544 void intel_guc_dump_time_info(struct intel_guc *guc, struct drm_printer *p);
545
546 int intel_guc_sched_disable_gucid_threshold_max(struct intel_guc *guc);
547
548 bool intel_guc_tlb_invalidation_is_available(struct intel_guc *guc);
549 int intel_guc_invalidate_tlb_engines(struct intel_guc *guc);
550 int intel_guc_invalidate_tlb_guc(struct intel_guc *guc);
551 int intel_guc_tlb_invalidation_done(struct intel_guc *guc,
552 const u32 *payload, u32 len);
553 void wake_up_all_tlb_invalidate(struct intel_guc *guc);
554 #endif
555