xref: /linux/drivers/gpu/drm/i915/gt/uc/intel_guc_ct.c (revision 41c177cf354126a22443b5c80cec9fdd313e67e1)
1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2016-2019 Intel Corporation
4  */
5 
6 #include <linux/circ_buf.h>
7 #include <linux/ktime.h>
8 #include <linux/time64.h>
9 #include <linux/string_helpers.h>
10 #include <linux/timekeeping.h>
11 
12 #include "i915_drv.h"
13 #include "intel_guc_ct.h"
14 #include "intel_guc_print.h"
15 
16 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GUC)
17 enum {
18 	CT_DEAD_ALIVE = 0,
19 	CT_DEAD_SETUP,
20 	CT_DEAD_WRITE,
21 	CT_DEAD_DEADLOCK,
22 	CT_DEAD_H2G_HAS_ROOM,
23 	CT_DEAD_READ,
24 	CT_DEAD_PROCESS_FAILED,
25 };
26 
27 static void ct_dead_ct_worker_func(struct work_struct *w);
28 
29 #define CT_DEAD(ct, reason)	\
30 	do { \
31 		if (!(ct)->dead_ct_reported) { \
32 			(ct)->dead_ct_reason |= 1 << CT_DEAD_##reason; \
33 			queue_work(system_unbound_wq, &(ct)->dead_ct_worker); \
34 		} \
35 	} while (0)
36 #else
37 #define CT_DEAD(ct, reason)	do { } while (0)
38 #endif
39 
ct_to_guc(struct intel_guc_ct * ct)40 static inline struct intel_guc *ct_to_guc(struct intel_guc_ct *ct)
41 {
42 	return container_of(ct, struct intel_guc, ct);
43 }
44 
45 #define CT_ERROR(_ct, _fmt, ...) \
46 	guc_err(ct_to_guc(_ct), "CT: " _fmt, ##__VA_ARGS__)
47 #ifdef CONFIG_DRM_I915_DEBUG_GUC
48 #define CT_DEBUG(_ct, _fmt, ...) \
49 	guc_dbg(ct_to_guc(_ct), "CT: " _fmt, ##__VA_ARGS__)
50 #else
51 #define CT_DEBUG(...)	do { } while (0)
52 #endif
53 #define CT_PROBE_ERROR(_ct, _fmt, ...) \
54 	guc_probe_error(ct_to_guc(ct), "CT: " _fmt, ##__VA_ARGS__)
55 
56 /**
57  * DOC: CTB Blob
58  *
59  * We allocate single blob to hold both CTB descriptors and buffers:
60  *
61  *      +--------+-----------------------------------------------+------+
62  *      | offset | contents                                      | size |
63  *      +========+===============================================+======+
64  *      | 0x0000 | H2G `CTB Descriptor`_ (send)                  |      |
65  *      +--------+-----------------------------------------------+  4K  |
66  *      | 0x0800 | G2H `CTB Descriptor`_ (recv)                  |      |
67  *      +--------+-----------------------------------------------+------+
68  *      | 0x1000 | H2G `CT Buffer`_ (send)                       | n*4K |
69  *      |        |                                               |      |
70  *      +--------+-----------------------------------------------+------+
71  *      | 0x1000 | G2H `CT Buffer`_ (recv)                       | m*4K |
72  *      | + n*4K |                                               |      |
73  *      +--------+-----------------------------------------------+------+
74  *
75  * Size of each `CT Buffer`_ must be multiple of 4K.
76  * We don't expect too many messages in flight at any time, unless we are
77  * using the GuC submission. In that case each request requires a minimum
78  * 2 dwords which gives us a maximum 256 queue'd requests. Hopefully this
79  * enough space to avoid backpressure on the driver. We increase the size
80  * of the receive buffer (relative to the send) to ensure a G2H response
81  * CTB has a landing spot.
82  */
83 #define CTB_DESC_SIZE		ALIGN(sizeof(struct guc_ct_buffer_desc), SZ_2K)
84 #define CTB_H2G_BUFFER_SIZE	(SZ_4K)
85 #define CTB_G2H_BUFFER_SIZE	(4 * CTB_H2G_BUFFER_SIZE)
86 #define G2H_ROOM_BUFFER_SIZE	(CTB_G2H_BUFFER_SIZE / 4)
87 
88 struct ct_request {
89 	struct list_head link;
90 	u32 fence;
91 	u32 status;
92 	u32 response_len;
93 	u32 *response_buf;
94 };
95 
96 struct ct_incoming_msg {
97 	struct list_head link;
98 	u32 size;
99 	u32 msg[] __counted_by(size);
100 };
101 
102 enum { CTB_SEND = 0, CTB_RECV = 1 };
103 
104 enum { CTB_OWNER_HOST = 0 };
105 
106 /*
107  * Some H2G commands involve a synchronous response that the driver needs
108  * to wait for. In such cases, a timeout is required to prevent the driver
109  * from waiting forever in the case of an error (either no error response
110  * is defined in the protocol or something has died and requires a reset).
111  * The specific command may be defined as having a time bound response but
112  * the CT is a queue and that time guarantee only starts from the point
113  * when the command reaches the head of the queue and is processed by GuC.
114  *
115  * Ideally there would be a helper to report the progress of a given
116  * command through the CT. However, that would require a significant
117  * amount of work in the CT layer. In the meantime, provide a reasonable
118  * estimation of the worst case latency it should take for the entire
119  * queue to drain. And therefore, how long a caller should wait before
120  * giving up on their request. The current estimate is based on empirical
121  * measurement of a test that fills the buffer with context creation and
122  * destruction requests as they seem to be the slowest operation.
123  */
intel_guc_ct_max_queue_time_jiffies(void)124 long intel_guc_ct_max_queue_time_jiffies(void)
125 {
126 	/*
127 	 * A 4KB buffer full of context destroy commands takes a little
128 	 * over a second to process so bump that to 2s to be super safe.
129 	 */
130 	return (CTB_H2G_BUFFER_SIZE * HZ) / SZ_2K;
131 }
132 
133 static void ct_receive_tasklet_func(struct tasklet_struct *t);
134 static void ct_incoming_request_worker_func(struct work_struct *w);
135 
136 /**
137  * intel_guc_ct_init_early - Initialize CT state without requiring device access
138  * @ct: pointer to CT struct
139  */
intel_guc_ct_init_early(struct intel_guc_ct * ct)140 void intel_guc_ct_init_early(struct intel_guc_ct *ct)
141 {
142 	spin_lock_init(&ct->ctbs.send.lock);
143 	spin_lock_init(&ct->ctbs.recv.lock);
144 	spin_lock_init(&ct->requests.lock);
145 	INIT_LIST_HEAD(&ct->requests.pending);
146 	INIT_LIST_HEAD(&ct->requests.incoming);
147 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GUC)
148 	INIT_WORK(&ct->dead_ct_worker, ct_dead_ct_worker_func);
149 #endif
150 	INIT_WORK(&ct->requests.worker, ct_incoming_request_worker_func);
151 	tasklet_setup(&ct->receive_tasklet, ct_receive_tasklet_func);
152 	init_waitqueue_head(&ct->wq);
153 }
154 
guc_ct_buffer_desc_init(struct guc_ct_buffer_desc * desc)155 static void guc_ct_buffer_desc_init(struct guc_ct_buffer_desc *desc)
156 {
157 	memset(desc, 0, sizeof(*desc));
158 }
159 
guc_ct_buffer_reset(struct intel_guc_ct_buffer * ctb)160 static void guc_ct_buffer_reset(struct intel_guc_ct_buffer *ctb)
161 {
162 	u32 space;
163 
164 	ctb->broken = false;
165 	ctb->tail = 0;
166 	ctb->head = 0;
167 	space = CIRC_SPACE(ctb->tail, ctb->head, ctb->size) - ctb->resv_space;
168 	atomic_set(&ctb->space, space);
169 
170 	guc_ct_buffer_desc_init(ctb->desc);
171 }
172 
guc_ct_buffer_init(struct intel_guc_ct_buffer * ctb,struct guc_ct_buffer_desc * desc,u32 * cmds,u32 size_in_bytes,u32 resv_space)173 static void guc_ct_buffer_init(struct intel_guc_ct_buffer *ctb,
174 			       struct guc_ct_buffer_desc *desc,
175 			       u32 *cmds, u32 size_in_bytes, u32 resv_space)
176 {
177 	GEM_BUG_ON(size_in_bytes % 4);
178 
179 	ctb->desc = desc;
180 	ctb->cmds = cmds;
181 	ctb->size = size_in_bytes / 4;
182 	ctb->resv_space = resv_space / 4;
183 
184 	guc_ct_buffer_reset(ctb);
185 }
186 
guc_action_control_ctb(struct intel_guc * guc,u32 control)187 static int guc_action_control_ctb(struct intel_guc *guc, u32 control)
188 {
189 	u32 request[HOST2GUC_CONTROL_CTB_REQUEST_MSG_LEN] = {
190 		FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) |
191 		FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) |
192 		FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION, GUC_ACTION_HOST2GUC_CONTROL_CTB),
193 		FIELD_PREP(HOST2GUC_CONTROL_CTB_REQUEST_MSG_1_CONTROL, control),
194 	};
195 	int ret;
196 
197 	GEM_BUG_ON(control != GUC_CTB_CONTROL_DISABLE && control != GUC_CTB_CONTROL_ENABLE);
198 
199 	/* CT control must go over MMIO */
200 	ret = intel_guc_send_mmio(guc, request, ARRAY_SIZE(request), NULL, 0);
201 
202 	return ret > 0 ? -EPROTO : ret;
203 }
204 
ct_control_enable(struct intel_guc_ct * ct,bool enable)205 static int ct_control_enable(struct intel_guc_ct *ct, bool enable)
206 {
207 	int err;
208 
209 	err = guc_action_control_ctb(ct_to_guc(ct), enable ?
210 				     GUC_CTB_CONTROL_ENABLE : GUC_CTB_CONTROL_DISABLE);
211 	if (unlikely(err))
212 		CT_PROBE_ERROR(ct, "Failed to control/%s CTB (%pe)\n",
213 			       str_enable_disable(enable), ERR_PTR(err));
214 
215 	return err;
216 }
217 
ct_register_buffer(struct intel_guc_ct * ct,bool send,u32 desc_addr,u32 buff_addr,u32 size)218 static int ct_register_buffer(struct intel_guc_ct *ct, bool send,
219 			      u32 desc_addr, u32 buff_addr, u32 size)
220 {
221 	int err;
222 
223 	err = intel_guc_self_cfg64(ct_to_guc(ct), send ?
224 				   GUC_KLV_SELF_CFG_H2G_CTB_DESCRIPTOR_ADDR_KEY :
225 				   GUC_KLV_SELF_CFG_G2H_CTB_DESCRIPTOR_ADDR_KEY,
226 				   desc_addr);
227 	if (unlikely(err))
228 		goto failed;
229 
230 	err = intel_guc_self_cfg64(ct_to_guc(ct), send ?
231 				   GUC_KLV_SELF_CFG_H2G_CTB_ADDR_KEY :
232 				   GUC_KLV_SELF_CFG_G2H_CTB_ADDR_KEY,
233 				   buff_addr);
234 	if (unlikely(err))
235 		goto failed;
236 
237 	err = intel_guc_self_cfg32(ct_to_guc(ct), send ?
238 				   GUC_KLV_SELF_CFG_H2G_CTB_SIZE_KEY :
239 				   GUC_KLV_SELF_CFG_G2H_CTB_SIZE_KEY,
240 				   size);
241 	if (unlikely(err))
242 failed:
243 		CT_PROBE_ERROR(ct, "Failed to register %s buffer (%pe)\n",
244 			       send ? "SEND" : "RECV", ERR_PTR(err));
245 
246 	return err;
247 }
248 
249 /**
250  * intel_guc_ct_init - Init buffer-based communication
251  * @ct: pointer to CT struct
252  *
253  * Allocate memory required for buffer-based communication.
254  *
255  * Return: 0 on success, a negative errno code on failure.
256  */
intel_guc_ct_init(struct intel_guc_ct * ct)257 int intel_guc_ct_init(struct intel_guc_ct *ct)
258 {
259 	struct intel_guc *guc = ct_to_guc(ct);
260 	struct guc_ct_buffer_desc *desc;
261 	u32 blob_size;
262 	u32 cmds_size;
263 	u32 resv_space;
264 	void *blob;
265 	u32 *cmds;
266 	int err;
267 
268 	err = i915_inject_probe_error(guc_to_i915(guc), -ENXIO);
269 	if (err)
270 		return err;
271 
272 	GEM_BUG_ON(ct->vma);
273 
274 	blob_size = 2 * CTB_DESC_SIZE + CTB_H2G_BUFFER_SIZE + CTB_G2H_BUFFER_SIZE;
275 	err = intel_guc_allocate_and_map_vma(guc, blob_size, &ct->vma, &blob);
276 	if (unlikely(err)) {
277 		CT_PROBE_ERROR(ct, "Failed to allocate %u for CTB data (%pe)\n",
278 			       blob_size, ERR_PTR(err));
279 		return err;
280 	}
281 
282 	CT_DEBUG(ct, "base=%#x size=%u\n", intel_guc_ggtt_offset(guc, ct->vma), blob_size);
283 
284 	/* store pointers to desc and cmds for send ctb */
285 	desc = blob;
286 	cmds = blob + 2 * CTB_DESC_SIZE;
287 	cmds_size = CTB_H2G_BUFFER_SIZE;
288 	resv_space = 0;
289 	CT_DEBUG(ct, "%s desc %#tx cmds %#tx size %u/%u\n", "send",
290 		 ptrdiff(desc, blob), ptrdiff(cmds, blob), cmds_size,
291 		 resv_space);
292 
293 	guc_ct_buffer_init(&ct->ctbs.send, desc, cmds, cmds_size, resv_space);
294 
295 	/* store pointers to desc and cmds for recv ctb */
296 	desc = blob + CTB_DESC_SIZE;
297 	cmds = blob + 2 * CTB_DESC_SIZE + CTB_H2G_BUFFER_SIZE;
298 	cmds_size = CTB_G2H_BUFFER_SIZE;
299 	resv_space = G2H_ROOM_BUFFER_SIZE;
300 	CT_DEBUG(ct, "%s desc %#tx cmds %#tx size %u/%u\n", "recv",
301 		 ptrdiff(desc, blob), ptrdiff(cmds, blob), cmds_size,
302 		 resv_space);
303 
304 	guc_ct_buffer_init(&ct->ctbs.recv, desc, cmds, cmds_size, resv_space);
305 
306 	return 0;
307 }
308 
309 /**
310  * intel_guc_ct_fini - Fini buffer-based communication
311  * @ct: pointer to CT struct
312  *
313  * Deallocate memory required for buffer-based communication.
314  */
intel_guc_ct_fini(struct intel_guc_ct * ct)315 void intel_guc_ct_fini(struct intel_guc_ct *ct)
316 {
317 	GEM_BUG_ON(ct->enabled);
318 
319 	tasklet_kill(&ct->receive_tasklet);
320 	i915_vma_unpin_and_release(&ct->vma, I915_VMA_RELEASE_MAP);
321 	memset(ct, 0, sizeof(*ct));
322 }
323 
324 /**
325  * intel_guc_ct_enable - Enable buffer based command transport.
326  * @ct: pointer to CT struct
327  *
328  * Return: 0 on success, a negative errno code on failure.
329  */
intel_guc_ct_enable(struct intel_guc_ct * ct)330 int intel_guc_ct_enable(struct intel_guc_ct *ct)
331 {
332 	struct intel_guc *guc = ct_to_guc(ct);
333 	u32 base, desc, cmds, size;
334 	void *blob;
335 	int err;
336 
337 	GEM_BUG_ON(ct->enabled);
338 
339 	/* vma should be already allocated and map'ed */
340 	GEM_BUG_ON(!ct->vma);
341 	GEM_BUG_ON(!i915_gem_object_has_pinned_pages(ct->vma->obj));
342 	base = intel_guc_ggtt_offset(guc, ct->vma);
343 
344 	/* blob should start with send descriptor */
345 	blob = __px_vaddr(ct->vma->obj);
346 	GEM_BUG_ON(blob != ct->ctbs.send.desc);
347 
348 	/* (re)initialize descriptors */
349 	guc_ct_buffer_reset(&ct->ctbs.send);
350 	guc_ct_buffer_reset(&ct->ctbs.recv);
351 
352 	/*
353 	 * Register both CT buffers starting with RECV buffer.
354 	 * Descriptors are in first half of the blob.
355 	 */
356 	desc = base + ptrdiff(ct->ctbs.recv.desc, blob);
357 	cmds = base + ptrdiff(ct->ctbs.recv.cmds, blob);
358 	size = ct->ctbs.recv.size * 4;
359 	err = ct_register_buffer(ct, false, desc, cmds, size);
360 	if (unlikely(err))
361 		goto err_out;
362 
363 	desc = base + ptrdiff(ct->ctbs.send.desc, blob);
364 	cmds = base + ptrdiff(ct->ctbs.send.cmds, blob);
365 	size = ct->ctbs.send.size * 4;
366 	err = ct_register_buffer(ct, true, desc, cmds, size);
367 	if (unlikely(err))
368 		goto err_out;
369 
370 	err = ct_control_enable(ct, true);
371 	if (unlikely(err))
372 		goto err_out;
373 
374 	ct->enabled = true;
375 	ct->stall_time = KTIME_MAX;
376 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GUC)
377 	ct->dead_ct_reported = false;
378 	ct->dead_ct_reason = CT_DEAD_ALIVE;
379 #endif
380 
381 	return 0;
382 
383 err_out:
384 	CT_PROBE_ERROR(ct, "Failed to enable CTB (%pe)\n", ERR_PTR(err));
385 	CT_DEAD(ct, SETUP);
386 	return err;
387 }
388 
389 /**
390  * intel_guc_ct_disable - Disable buffer based command transport.
391  * @ct: pointer to CT struct
392  */
intel_guc_ct_disable(struct intel_guc_ct * ct)393 void intel_guc_ct_disable(struct intel_guc_ct *ct)
394 {
395 	struct intel_guc *guc = ct_to_guc(ct);
396 
397 	GEM_BUG_ON(!ct->enabled);
398 
399 	ct->enabled = false;
400 
401 	if (intel_guc_is_fw_running(guc)) {
402 		ct_control_enable(ct, false);
403 	}
404 }
405 
406 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)
ct_track_lost_and_found(struct intel_guc_ct * ct,u32 fence,u32 action)407 static void ct_track_lost_and_found(struct intel_guc_ct *ct, u32 fence, u32 action)
408 {
409 	unsigned int lost = fence % ARRAY_SIZE(ct->requests.lost_and_found);
410 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GUC)
411 	unsigned long entries[SZ_32];
412 	unsigned int n;
413 
414 	n = stack_trace_save(entries, ARRAY_SIZE(entries), 1);
415 
416 	/* May be called under spinlock, so avoid sleeping */
417 	ct->requests.lost_and_found[lost].stack = stack_depot_save(entries, n, GFP_NOWAIT);
418 #endif
419 	ct->requests.lost_and_found[lost].fence = fence;
420 	ct->requests.lost_and_found[lost].action = action;
421 }
422 #endif
423 
ct_get_next_fence(struct intel_guc_ct * ct)424 static u32 ct_get_next_fence(struct intel_guc_ct *ct)
425 {
426 	/* For now it's trivial */
427 	return ++ct->requests.last_fence;
428 }
429 
ct_write(struct intel_guc_ct * ct,const u32 * action,u32 len,u32 fence,u32 flags)430 static int ct_write(struct intel_guc_ct *ct,
431 		    const u32 *action,
432 		    u32 len /* in dwords */,
433 		    u32 fence, u32 flags)
434 {
435 	struct intel_guc_ct_buffer *ctb = &ct->ctbs.send;
436 	struct guc_ct_buffer_desc *desc = ctb->desc;
437 	u32 tail = ctb->tail;
438 	u32 size = ctb->size;
439 	u32 header;
440 	u32 hxg;
441 	u32 type;
442 	u32 *cmds = ctb->cmds;
443 	unsigned int i;
444 
445 	if (unlikely(desc->status))
446 		goto corrupted;
447 
448 	GEM_BUG_ON(tail > size);
449 
450 #ifdef CONFIG_DRM_I915_DEBUG_GUC
451 	if (unlikely(tail != READ_ONCE(desc->tail))) {
452 		CT_ERROR(ct, "Tail was modified %u != %u\n",
453 			 desc->tail, tail);
454 		desc->status |= GUC_CTB_STATUS_MISMATCH;
455 		goto corrupted;
456 	}
457 	if (unlikely(READ_ONCE(desc->head) >= size)) {
458 		CT_ERROR(ct, "Invalid head offset %u >= %u)\n",
459 			 desc->head, size);
460 		desc->status |= GUC_CTB_STATUS_OVERFLOW;
461 		goto corrupted;
462 	}
463 #endif
464 
465 	/*
466 	 * dw0: CT header (including fence)
467 	 * dw1: HXG header (including action code)
468 	 * dw2+: action data
469 	 */
470 	header = FIELD_PREP(GUC_CTB_MSG_0_FORMAT, GUC_CTB_FORMAT_HXG) |
471 		 FIELD_PREP(GUC_CTB_MSG_0_NUM_DWORDS, len) |
472 		 FIELD_PREP(GUC_CTB_MSG_0_FENCE, fence);
473 
474 	type = (flags & INTEL_GUC_CT_SEND_NB) ? GUC_HXG_TYPE_FAST_REQUEST :
475 		GUC_HXG_TYPE_REQUEST;
476 	hxg = FIELD_PREP(GUC_HXG_MSG_0_TYPE, type) |
477 		FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION |
478 			   GUC_HXG_REQUEST_MSG_0_DATA0, action[0]);
479 
480 	CT_DEBUG(ct, "writing (tail %u) %*ph %*ph %*ph\n",
481 		 tail, 4, &header, 4, &hxg, 4 * (len - 1), &action[1]);
482 
483 	cmds[tail] = header;
484 	tail = (tail + 1) % size;
485 
486 	cmds[tail] = hxg;
487 	tail = (tail + 1) % size;
488 
489 	for (i = 1; i < len; i++) {
490 		cmds[tail] = action[i];
491 		tail = (tail + 1) % size;
492 	}
493 	GEM_BUG_ON(tail > size);
494 
495 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)
496 	ct_track_lost_and_found(ct, fence,
497 				FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, action[0]));
498 #endif
499 
500 	/*
501 	 * make sure H2G buffer update and LRC tail update (if this triggering a
502 	 * submission) are visible before updating the descriptor tail
503 	 */
504 	intel_guc_write_barrier(ct_to_guc(ct));
505 
506 	/* update local copies */
507 	ctb->tail = tail;
508 	GEM_BUG_ON(atomic_read(&ctb->space) < len + GUC_CTB_HDR_LEN);
509 	atomic_sub(len + GUC_CTB_HDR_LEN, &ctb->space);
510 
511 	/* now update descriptor */
512 	WRITE_ONCE(desc->tail, tail);
513 
514 	return 0;
515 
516 corrupted:
517 	CT_ERROR(ct, "Corrupted descriptor head=%u tail=%u status=%#x\n",
518 		 desc->head, desc->tail, desc->status);
519 	CT_DEAD(ct, WRITE);
520 	ctb->broken = true;
521 	return -EPIPE;
522 }
523 
524 /**
525  * wait_for_ct_request_update - Wait for CT request state update.
526  * @ct:		pointer to CT
527  * @req:	pointer to pending request
528  * @status:	placeholder for status
529  *
530  * For each sent request, GuC shall send back CT response message.
531  * Our message handler will update status of tracked request once
532  * response message with given fence is received. Wait here and
533  * check for valid response status value.
534  *
535  * Return:
536  * *	0 response received (status is valid)
537  * *	-ETIMEDOUT no response within hardcoded timeout
538  */
wait_for_ct_request_update(struct intel_guc_ct * ct,struct ct_request * req,u32 * status)539 static int wait_for_ct_request_update(struct intel_guc_ct *ct, struct ct_request *req, u32 *status)
540 {
541 	int err;
542 	bool ct_enabled;
543 
544 	/*
545 	 * Fast commands should complete in less than 10us, so sample quickly
546 	 * up to that length of time, then switch to a slower sleep-wait loop.
547 	 * No GuC command should ever take longer than 10ms but many GuC
548 	 * commands can be inflight at time, so use a 1s timeout on the slower
549 	 * sleep-wait loop.
550 	 */
551 #define GUC_CTB_RESPONSE_TIMEOUT_SHORT_MS 10
552 #define GUC_CTB_RESPONSE_TIMEOUT_LONG_MS 1000
553 #define done \
554 	(!(ct_enabled = intel_guc_ct_enabled(ct)) || \
555 	 FIELD_GET(GUC_HXG_MSG_0_ORIGIN, READ_ONCE(req->status)) == \
556 	 GUC_HXG_ORIGIN_GUC)
557 	err = wait_for_us(done, GUC_CTB_RESPONSE_TIMEOUT_SHORT_MS);
558 	if (err)
559 		err = wait_for(done, GUC_CTB_RESPONSE_TIMEOUT_LONG_MS);
560 #undef done
561 	if (!ct_enabled)
562 		err = -ENODEV;
563 
564 	*status = req->status;
565 	return err;
566 }
567 
568 #define GUC_CTB_TIMEOUT_MS	1500
ct_deadlocked(struct intel_guc_ct * ct)569 static inline bool ct_deadlocked(struct intel_guc_ct *ct)
570 {
571 	long timeout = GUC_CTB_TIMEOUT_MS;
572 	bool ret = ktime_ms_delta(ktime_get(), ct->stall_time) > timeout;
573 
574 	if (unlikely(ret)) {
575 		struct guc_ct_buffer_desc *send = ct->ctbs.send.desc;
576 		struct guc_ct_buffer_desc *recv = ct->ctbs.send.desc;
577 
578 		CT_ERROR(ct, "Communication stalled for %lld ms, desc status=%#x,%#x\n",
579 			 ktime_ms_delta(ktime_get(), ct->stall_time),
580 			 send->status, recv->status);
581 		CT_ERROR(ct, "H2G Space: %u (Bytes)\n",
582 			 atomic_read(&ct->ctbs.send.space) * 4);
583 		CT_ERROR(ct, "Head: %u (Dwords)\n", ct->ctbs.send.desc->head);
584 		CT_ERROR(ct, "Tail: %u (Dwords)\n", ct->ctbs.send.desc->tail);
585 		CT_ERROR(ct, "G2H Space: %u (Bytes)\n",
586 			 atomic_read(&ct->ctbs.recv.space) * 4);
587 		CT_ERROR(ct, "Head: %u\n (Dwords)", ct->ctbs.recv.desc->head);
588 		CT_ERROR(ct, "Tail: %u\n (Dwords)", ct->ctbs.recv.desc->tail);
589 
590 		CT_DEAD(ct, DEADLOCK);
591 		ct->ctbs.send.broken = true;
592 	}
593 
594 	return ret;
595 }
596 
g2h_has_room(struct intel_guc_ct * ct,u32 g2h_len_dw)597 static inline bool g2h_has_room(struct intel_guc_ct *ct, u32 g2h_len_dw)
598 {
599 	struct intel_guc_ct_buffer *ctb = &ct->ctbs.recv;
600 
601 	/*
602 	 * We leave a certain amount of space in the G2H CTB buffer for
603 	 * unexpected G2H CTBs (e.g. logging, engine hang, etc...)
604 	 */
605 	return !g2h_len_dw || atomic_read(&ctb->space) >= g2h_len_dw;
606 }
607 
g2h_reserve_space(struct intel_guc_ct * ct,u32 g2h_len_dw)608 static inline void g2h_reserve_space(struct intel_guc_ct *ct, u32 g2h_len_dw)
609 {
610 	lockdep_assert_held(&ct->ctbs.send.lock);
611 
612 	GEM_BUG_ON(!g2h_has_room(ct, g2h_len_dw));
613 
614 	if (g2h_len_dw)
615 		atomic_sub(g2h_len_dw, &ct->ctbs.recv.space);
616 }
617 
g2h_release_space(struct intel_guc_ct * ct,u32 g2h_len_dw)618 static inline void g2h_release_space(struct intel_guc_ct *ct, u32 g2h_len_dw)
619 {
620 	atomic_add(g2h_len_dw, &ct->ctbs.recv.space);
621 }
622 
h2g_has_room(struct intel_guc_ct * ct,u32 len_dw)623 static inline bool h2g_has_room(struct intel_guc_ct *ct, u32 len_dw)
624 {
625 	struct intel_guc_ct_buffer *ctb = &ct->ctbs.send;
626 	struct guc_ct_buffer_desc *desc = ctb->desc;
627 	u32 head;
628 	u32 space;
629 
630 	if (atomic_read(&ctb->space) >= len_dw)
631 		return true;
632 
633 	head = READ_ONCE(desc->head);
634 	if (unlikely(head > ctb->size)) {
635 		CT_ERROR(ct, "Invalid head offset %u >= %u)\n",
636 			 head, ctb->size);
637 		desc->status |= GUC_CTB_STATUS_OVERFLOW;
638 		ctb->broken = true;
639 		CT_DEAD(ct, H2G_HAS_ROOM);
640 		return false;
641 	}
642 
643 	space = CIRC_SPACE(ctb->tail, head, ctb->size);
644 	atomic_set(&ctb->space, space);
645 
646 	return space >= len_dw;
647 }
648 
has_room_nb(struct intel_guc_ct * ct,u32 h2g_dw,u32 g2h_dw)649 static int has_room_nb(struct intel_guc_ct *ct, u32 h2g_dw, u32 g2h_dw)
650 {
651 	bool h2g = h2g_has_room(ct, h2g_dw);
652 	bool g2h = g2h_has_room(ct, g2h_dw);
653 
654 	lockdep_assert_held(&ct->ctbs.send.lock);
655 
656 	if (unlikely(!h2g || !g2h)) {
657 		if (ct->stall_time == KTIME_MAX)
658 			ct->stall_time = ktime_get();
659 
660 		/* Be paranoid and kick G2H tasklet to free credits */
661 		if (!g2h)
662 			tasklet_hi_schedule(&ct->receive_tasklet);
663 
664 		if (unlikely(ct_deadlocked(ct)))
665 			return -EPIPE;
666 		else
667 			return -EBUSY;
668 	}
669 
670 	ct->stall_time = KTIME_MAX;
671 	return 0;
672 }
673 
674 #define G2H_LEN_DW(f) ({ \
675 	typeof(f) f_ = (f); \
676 	FIELD_GET(INTEL_GUC_CT_SEND_G2H_DW_MASK, f_) ? \
677 	FIELD_GET(INTEL_GUC_CT_SEND_G2H_DW_MASK, f_) + \
678 	GUC_CTB_HXG_MSG_MIN_LEN : 0; \
679 })
ct_send_nb(struct intel_guc_ct * ct,const u32 * action,u32 len,u32 flags)680 static int ct_send_nb(struct intel_guc_ct *ct,
681 		      const u32 *action,
682 		      u32 len,
683 		      u32 flags)
684 {
685 	struct intel_guc_ct_buffer *ctb = &ct->ctbs.send;
686 	unsigned long spin_flags;
687 	u32 g2h_len_dw = G2H_LEN_DW(flags);
688 	u32 fence;
689 	int ret;
690 
691 	spin_lock_irqsave(&ctb->lock, spin_flags);
692 
693 	ret = has_room_nb(ct, len + GUC_CTB_HDR_LEN, g2h_len_dw);
694 	if (unlikely(ret))
695 		goto out;
696 
697 	fence = ct_get_next_fence(ct);
698 	ret = ct_write(ct, action, len, fence, flags);
699 	if (unlikely(ret))
700 		goto out;
701 
702 	g2h_reserve_space(ct, g2h_len_dw);
703 	intel_guc_notify(ct_to_guc(ct));
704 
705 out:
706 	spin_unlock_irqrestore(&ctb->lock, spin_flags);
707 
708 	return ret;
709 }
710 
ct_send(struct intel_guc_ct * ct,const u32 * action,u32 len,u32 * response_buf,u32 response_buf_size,u32 * status)711 static int ct_send(struct intel_guc_ct *ct,
712 		   const u32 *action,
713 		   u32 len,
714 		   u32 *response_buf,
715 		   u32 response_buf_size,
716 		   u32 *status)
717 {
718 	struct intel_guc_ct_buffer *ctb = &ct->ctbs.send;
719 	struct ct_request request;
720 	unsigned long flags;
721 	unsigned int sleep_period_ms = 1;
722 	bool send_again;
723 	u32 fence;
724 	int err;
725 
726 	GEM_BUG_ON(!ct->enabled);
727 	GEM_BUG_ON(!len);
728 	GEM_BUG_ON(len > GUC_CTB_HXG_MSG_MAX_LEN - GUC_CTB_HDR_LEN);
729 	GEM_BUG_ON(!response_buf && response_buf_size);
730 	might_sleep();
731 
732 resend:
733 	send_again = false;
734 
735 	/*
736 	 * We use a lazy spin wait loop here as we believe that if the CT
737 	 * buffers are sized correctly the flow control condition should be
738 	 * rare. Reserving the maximum size in the G2H credits as we don't know
739 	 * how big the response is going to be.
740 	 */
741 retry:
742 	spin_lock_irqsave(&ctb->lock, flags);
743 	if (unlikely(!h2g_has_room(ct, len + GUC_CTB_HDR_LEN) ||
744 		     !g2h_has_room(ct, GUC_CTB_HXG_MSG_MAX_LEN))) {
745 		if (ct->stall_time == KTIME_MAX)
746 			ct->stall_time = ktime_get();
747 		spin_unlock_irqrestore(&ctb->lock, flags);
748 
749 		if (unlikely(ct_deadlocked(ct)))
750 			return -EPIPE;
751 
752 		if (msleep_interruptible(sleep_period_ms))
753 			return -EINTR;
754 		sleep_period_ms = sleep_period_ms << 1;
755 
756 		goto retry;
757 	}
758 
759 	ct->stall_time = KTIME_MAX;
760 
761 	fence = ct_get_next_fence(ct);
762 	request.fence = fence;
763 	request.status = 0;
764 	request.response_len = response_buf_size;
765 	request.response_buf = response_buf;
766 
767 	spin_lock(&ct->requests.lock);
768 	list_add_tail(&request.link, &ct->requests.pending);
769 	spin_unlock(&ct->requests.lock);
770 
771 	err = ct_write(ct, action, len, fence, 0);
772 	g2h_reserve_space(ct, GUC_CTB_HXG_MSG_MAX_LEN);
773 
774 	spin_unlock_irqrestore(&ctb->lock, flags);
775 
776 	if (unlikely(err))
777 		goto unlink;
778 
779 	intel_guc_notify(ct_to_guc(ct));
780 
781 	err = wait_for_ct_request_update(ct, &request, status);
782 	g2h_release_space(ct, GUC_CTB_HXG_MSG_MAX_LEN);
783 	if (unlikely(err)) {
784 		if (err == -ENODEV)
785 			/* wait_for_ct_request_update returns -ENODEV on reset/suspend in progress.
786 			 * In this case, output is debug rather than error info
787 			 */
788 			CT_DEBUG(ct, "Request %#x (fence %u) cancelled as CTB is disabled\n",
789 				 action[0], request.fence);
790 		else
791 			CT_ERROR(ct, "No response for request %#x (fence %u)\n",
792 				 action[0], request.fence);
793 		goto unlink;
794 	}
795 
796 	if (FIELD_GET(GUC_HXG_MSG_0_TYPE, *status) == GUC_HXG_TYPE_NO_RESPONSE_RETRY) {
797 		CT_DEBUG(ct, "retrying request %#x (%u)\n", *action,
798 			 FIELD_GET(GUC_HXG_RETRY_MSG_0_REASON, *status));
799 		send_again = true;
800 		goto unlink;
801 	}
802 
803 	if (FIELD_GET(GUC_HXG_MSG_0_TYPE, *status) != GUC_HXG_TYPE_RESPONSE_SUCCESS) {
804 		err = -EIO;
805 		goto unlink;
806 	}
807 
808 	if (response_buf) {
809 		/* There shall be no data in the status */
810 		WARN_ON(FIELD_GET(GUC_HXG_RESPONSE_MSG_0_DATA0, request.status));
811 		/* Return actual response len */
812 		err = request.response_len;
813 	} else {
814 		/* There shall be no response payload */
815 		WARN_ON(request.response_len);
816 		/* Return data decoded from the status dword */
817 		err = FIELD_GET(GUC_HXG_RESPONSE_MSG_0_DATA0, *status);
818 	}
819 
820 unlink:
821 	spin_lock_irqsave(&ct->requests.lock, flags);
822 	list_del(&request.link);
823 	spin_unlock_irqrestore(&ct->requests.lock, flags);
824 
825 	if (unlikely(send_again))
826 		goto resend;
827 
828 	return err;
829 }
830 
831 /*
832  * Command Transport (CT) buffer based GuC send function.
833  */
intel_guc_ct_send(struct intel_guc_ct * ct,const u32 * action,u32 len,u32 * response_buf,u32 response_buf_size,u32 flags)834 int intel_guc_ct_send(struct intel_guc_ct *ct, const u32 *action, u32 len,
835 		      u32 *response_buf, u32 response_buf_size, u32 flags)
836 {
837 	u32 status = ~0; /* undefined */
838 	int ret;
839 
840 	if (unlikely(!ct->enabled)) {
841 		struct intel_guc *guc = ct_to_guc(ct);
842 		struct intel_uc *uc = container_of(guc, struct intel_uc, guc);
843 
844 		WARN(!uc->reset_in_progress, "Unexpected send: action=%#x\n", *action);
845 		return -ENODEV;
846 	}
847 
848 	if (unlikely(ct->ctbs.send.broken))
849 		return -EPIPE;
850 
851 	if (flags & INTEL_GUC_CT_SEND_NB)
852 		return ct_send_nb(ct, action, len, flags);
853 
854 	ret = ct_send(ct, action, len, response_buf, response_buf_size, &status);
855 	if (unlikely(ret < 0)) {
856 		if (ret != -ENODEV)
857 			CT_ERROR(ct, "Sending action %#x failed (%pe) status=%#X\n",
858 				 action[0], ERR_PTR(ret), status);
859 	} else if (unlikely(ret)) {
860 		CT_DEBUG(ct, "send action %#x returned %d (%#x)\n",
861 			 action[0], ret, ret);
862 	}
863 
864 	return ret;
865 }
866 
ct_alloc_msg(u32 num_dwords)867 static struct ct_incoming_msg *ct_alloc_msg(u32 num_dwords)
868 {
869 	struct ct_incoming_msg *msg;
870 
871 	msg = kmalloc(struct_size(msg, msg, num_dwords), GFP_ATOMIC);
872 	if (msg)
873 		msg->size = num_dwords;
874 	return msg;
875 }
876 
ct_free_msg(struct ct_incoming_msg * msg)877 static void ct_free_msg(struct ct_incoming_msg *msg)
878 {
879 	kfree(msg);
880 }
881 
882 /*
883  * Return: number available remaining dwords to read (0 if empty)
884  *         or a negative error code on failure
885  */
ct_read(struct intel_guc_ct * ct,struct ct_incoming_msg ** msg)886 static int ct_read(struct intel_guc_ct *ct, struct ct_incoming_msg **msg)
887 {
888 	struct intel_guc_ct_buffer *ctb = &ct->ctbs.recv;
889 	struct guc_ct_buffer_desc *desc = ctb->desc;
890 	u32 head = ctb->head;
891 	u32 tail = READ_ONCE(desc->tail);
892 	u32 size = ctb->size;
893 	u32 *cmds = ctb->cmds;
894 	s32 available;
895 	unsigned int len;
896 	unsigned int i;
897 	u32 header;
898 
899 	if (unlikely(ctb->broken))
900 		return -EPIPE;
901 
902 	if (unlikely(desc->status)) {
903 		u32 status = desc->status;
904 
905 		if (status & GUC_CTB_STATUS_UNUSED) {
906 			/*
907 			 * Potentially valid if a CLIENT_RESET request resulted in
908 			 * contexts/engines being reset. But should never happen as
909 			 * no contexts should be active when CLIENT_RESET is sent.
910 			 */
911 			CT_ERROR(ct, "Unexpected G2H after GuC has stopped!\n");
912 			status &= ~GUC_CTB_STATUS_UNUSED;
913 		}
914 
915 		if (status)
916 			goto corrupted;
917 	}
918 
919 	GEM_BUG_ON(head > size);
920 
921 #ifdef CONFIG_DRM_I915_DEBUG_GUC
922 	if (unlikely(head != READ_ONCE(desc->head))) {
923 		CT_ERROR(ct, "Head was modified %u != %u\n",
924 			 desc->head, head);
925 		desc->status |= GUC_CTB_STATUS_MISMATCH;
926 		goto corrupted;
927 	}
928 #endif
929 	if (unlikely(tail >= size)) {
930 		CT_ERROR(ct, "Invalid tail offset %u >= %u)\n",
931 			 tail, size);
932 		desc->status |= GUC_CTB_STATUS_OVERFLOW;
933 		goto corrupted;
934 	}
935 
936 	/* tail == head condition indicates empty */
937 	available = tail - head;
938 	if (unlikely(available == 0)) {
939 		*msg = NULL;
940 		return 0;
941 	}
942 
943 	/* beware of buffer wrap case */
944 	if (unlikely(available < 0))
945 		available += size;
946 	CT_DEBUG(ct, "available %d (%u:%u:%u)\n", available, head, tail, size);
947 	GEM_BUG_ON(available < 0);
948 
949 	header = cmds[head];
950 	head = (head + 1) % size;
951 
952 	/* message len with header */
953 	len = FIELD_GET(GUC_CTB_MSG_0_NUM_DWORDS, header) + GUC_CTB_MSG_MIN_LEN;
954 	if (unlikely(len > (u32)available)) {
955 		CT_ERROR(ct, "Incomplete message %*ph %*ph %*ph\n",
956 			 4, &header,
957 			 4 * (head + available - 1 > size ?
958 			      size - head : available - 1), &cmds[head],
959 			 4 * (head + available - 1 > size ?
960 			      available - 1 - size + head : 0), &cmds[0]);
961 		desc->status |= GUC_CTB_STATUS_UNDERFLOW;
962 		goto corrupted;
963 	}
964 
965 	*msg = ct_alloc_msg(len);
966 	if (!*msg) {
967 		CT_ERROR(ct, "No memory for message %*ph %*ph %*ph\n",
968 			 4, &header,
969 			 4 * (head + available - 1 > size ?
970 			      size - head : available - 1), &cmds[head],
971 			 4 * (head + available - 1 > size ?
972 			      available - 1 - size + head : 0), &cmds[0]);
973 		return available;
974 	}
975 
976 	(*msg)->msg[0] = header;
977 
978 	for (i = 1; i < len; i++) {
979 		(*msg)->msg[i] = cmds[head];
980 		head = (head + 1) % size;
981 	}
982 	CT_DEBUG(ct, "received %*ph\n", 4 * len, (*msg)->msg);
983 
984 	/* update local copies */
985 	ctb->head = head;
986 
987 	/* now update descriptor */
988 	WRITE_ONCE(desc->head, head);
989 
990 	intel_guc_write_barrier(ct_to_guc(ct));
991 
992 	return available - len;
993 
994 corrupted:
995 	CT_ERROR(ct, "Corrupted descriptor head=%u tail=%u status=%#x\n",
996 		 desc->head, desc->tail, desc->status);
997 	ctb->broken = true;
998 	CT_DEAD(ct, READ);
999 	return -EPIPE;
1000 }
1001 
1002 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)
ct_check_lost_and_found(struct intel_guc_ct * ct,u32 fence)1003 static bool ct_check_lost_and_found(struct intel_guc_ct *ct, u32 fence)
1004 {
1005 	unsigned int n;
1006 	char *buf = NULL;
1007 	bool found = false;
1008 
1009 	lockdep_assert_held(&ct->requests.lock);
1010 
1011 	for (n = 0; n < ARRAY_SIZE(ct->requests.lost_and_found); n++) {
1012 		if (ct->requests.lost_and_found[n].fence != fence)
1013 			continue;
1014 		found = true;
1015 
1016 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GUC)
1017 		buf = kmalloc(SZ_4K, GFP_NOWAIT);
1018 		if (buf && stack_depot_snprint(ct->requests.lost_and_found[n].stack,
1019 					       buf, SZ_4K, 0)) {
1020 			CT_ERROR(ct, "Fence %u was used by action %#04x sent at\n%s",
1021 				 fence, ct->requests.lost_and_found[n].action, buf);
1022 			break;
1023 		}
1024 #endif
1025 		CT_ERROR(ct, "Fence %u was used by action %#04x\n",
1026 			 fence, ct->requests.lost_and_found[n].action);
1027 		break;
1028 	}
1029 	kfree(buf);
1030 	return found;
1031 }
1032 #else
ct_check_lost_and_found(struct intel_guc_ct * ct,u32 fence)1033 static bool ct_check_lost_and_found(struct intel_guc_ct *ct, u32 fence)
1034 {
1035 	return false;
1036 }
1037 #endif
1038 
ct_handle_response(struct intel_guc_ct * ct,struct ct_incoming_msg * response)1039 static int ct_handle_response(struct intel_guc_ct *ct, struct ct_incoming_msg *response)
1040 {
1041 	u32 len = FIELD_GET(GUC_CTB_MSG_0_NUM_DWORDS, response->msg[0]);
1042 	u32 fence = FIELD_GET(GUC_CTB_MSG_0_FENCE, response->msg[0]);
1043 	const u32 *hxg = &response->msg[GUC_CTB_MSG_MIN_LEN];
1044 	const u32 *data = &hxg[GUC_HXG_MSG_MIN_LEN];
1045 	u32 datalen = len - GUC_HXG_MSG_MIN_LEN;
1046 	struct ct_request *req;
1047 	unsigned long flags;
1048 	bool found = false;
1049 	int err = 0;
1050 
1051 	GEM_BUG_ON(len < GUC_HXG_MSG_MIN_LEN);
1052 	GEM_BUG_ON(FIELD_GET(GUC_HXG_MSG_0_ORIGIN, hxg[0]) != GUC_HXG_ORIGIN_GUC);
1053 	GEM_BUG_ON(FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_RESPONSE_SUCCESS &&
1054 		   FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_NO_RESPONSE_RETRY &&
1055 		   FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_RESPONSE_FAILURE);
1056 
1057 	CT_DEBUG(ct, "response fence %u status %#x\n", fence, hxg[0]);
1058 
1059 	spin_lock_irqsave(&ct->requests.lock, flags);
1060 	list_for_each_entry(req, &ct->requests.pending, link) {
1061 		if (unlikely(fence != req->fence)) {
1062 			CT_DEBUG(ct, "request %u awaits response\n",
1063 				 req->fence);
1064 			continue;
1065 		}
1066 		if (unlikely(datalen > req->response_len)) {
1067 			CT_ERROR(ct, "Response %u too long (datalen %u > %u)\n",
1068 				 req->fence, datalen, req->response_len);
1069 			datalen = min(datalen, req->response_len);
1070 			err = -EMSGSIZE;
1071 		}
1072 		if (datalen)
1073 			memcpy(req->response_buf, data, 4 * datalen);
1074 		req->response_len = datalen;
1075 		WRITE_ONCE(req->status, hxg[0]);
1076 		found = true;
1077 		break;
1078 	}
1079 
1080 #ifdef CONFIG_DRM_I915_SELFTEST
1081 	if (!found && ct_to_guc(ct)->fast_response_selftest) {
1082 		CT_DEBUG(ct, "Assuming unsolicited response due to FAST_REQUEST selftest\n");
1083 		ct_to_guc(ct)->fast_response_selftest++;
1084 		found = true;
1085 	}
1086 #endif
1087 
1088 	if (!found) {
1089 		CT_ERROR(ct, "Unsolicited response message: len %u, data %#x (fence %u, last %u)\n",
1090 			 len, hxg[0], fence, ct->requests.last_fence);
1091 		if (!ct_check_lost_and_found(ct, fence)) {
1092 			list_for_each_entry(req, &ct->requests.pending, link)
1093 				CT_ERROR(ct, "request %u awaits response\n",
1094 					 req->fence);
1095 		}
1096 		err = -ENOKEY;
1097 	}
1098 	spin_unlock_irqrestore(&ct->requests.lock, flags);
1099 
1100 	if (unlikely(err))
1101 		return err;
1102 
1103 	ct_free_msg(response);
1104 	return 0;
1105 }
1106 
ct_process_request(struct intel_guc_ct * ct,struct ct_incoming_msg * request)1107 static int ct_process_request(struct intel_guc_ct *ct, struct ct_incoming_msg *request)
1108 {
1109 	struct intel_guc *guc = ct_to_guc(ct);
1110 	const u32 *hxg;
1111 	const u32 *payload;
1112 	u32 hxg_len, action, len;
1113 	int ret;
1114 
1115 	hxg = &request->msg[GUC_CTB_MSG_MIN_LEN];
1116 	hxg_len = request->size - GUC_CTB_MSG_MIN_LEN;
1117 	payload = &hxg[GUC_HXG_MSG_MIN_LEN];
1118 	action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]);
1119 	len = hxg_len - GUC_HXG_MSG_MIN_LEN;
1120 
1121 	CT_DEBUG(ct, "request %x %*ph\n", action, 4 * len, payload);
1122 
1123 	switch (action) {
1124 	case INTEL_GUC_ACTION_DEFAULT:
1125 		ret = intel_guc_to_host_process_recv_msg(guc, payload, len);
1126 		break;
1127 	case INTEL_GUC_ACTION_DEREGISTER_CONTEXT_DONE:
1128 		ret = intel_guc_deregister_done_process_msg(guc, payload,
1129 							    len);
1130 		break;
1131 	case INTEL_GUC_ACTION_SCHED_CONTEXT_MODE_DONE:
1132 		ret = intel_guc_sched_done_process_msg(guc, payload, len);
1133 		break;
1134 	case INTEL_GUC_ACTION_CONTEXT_RESET_NOTIFICATION:
1135 		ret = intel_guc_context_reset_process_msg(guc, payload, len);
1136 		break;
1137 	case INTEL_GUC_ACTION_STATE_CAPTURE_NOTIFICATION:
1138 		ret = intel_guc_error_capture_process_msg(guc, payload, len);
1139 		if (unlikely(ret))
1140 			CT_ERROR(ct, "error capture notification failed %x %*ph\n",
1141 				 action, 4 * len, payload);
1142 		break;
1143 	case INTEL_GUC_ACTION_ENGINE_FAILURE_NOTIFICATION:
1144 		ret = intel_guc_engine_failure_process_msg(guc, payload, len);
1145 		break;
1146 	case INTEL_GUC_ACTION_NOTIFY_FLUSH_LOG_BUFFER_TO_FILE:
1147 		intel_guc_log_handle_flush_event(&guc->log);
1148 		ret = 0;
1149 		break;
1150 	case INTEL_GUC_ACTION_NOTIFY_CRASH_DUMP_POSTED:
1151 	case INTEL_GUC_ACTION_NOTIFY_EXCEPTION:
1152 		ret = intel_guc_crash_process_msg(guc, action);
1153 		break;
1154 	case INTEL_GUC_ACTION_TLB_INVALIDATION_DONE:
1155 		ret = intel_guc_tlb_invalidation_done(guc, payload, len);
1156 		break;
1157 	default:
1158 		ret = -EOPNOTSUPP;
1159 		break;
1160 	}
1161 
1162 	if (unlikely(ret)) {
1163 		CT_ERROR(ct, "Failed to process request %04x (%pe)\n",
1164 			 action, ERR_PTR(ret));
1165 		return ret;
1166 	}
1167 
1168 	ct_free_msg(request);
1169 	return 0;
1170 }
1171 
ct_process_incoming_requests(struct intel_guc_ct * ct)1172 static bool ct_process_incoming_requests(struct intel_guc_ct *ct)
1173 {
1174 	unsigned long flags;
1175 	struct ct_incoming_msg *request;
1176 	bool done;
1177 	int err;
1178 
1179 	spin_lock_irqsave(&ct->requests.lock, flags);
1180 	request = list_first_entry_or_null(&ct->requests.incoming,
1181 					   struct ct_incoming_msg, link);
1182 	if (request)
1183 		list_del(&request->link);
1184 	done = !!list_empty(&ct->requests.incoming);
1185 	spin_unlock_irqrestore(&ct->requests.lock, flags);
1186 
1187 	if (!request)
1188 		return true;
1189 
1190 	err = ct_process_request(ct, request);
1191 	if (unlikely(err)) {
1192 		CT_ERROR(ct, "Failed to process CT message (%pe) %*ph\n",
1193 			 ERR_PTR(err), 4 * request->size, request->msg);
1194 		CT_DEAD(ct, PROCESS_FAILED);
1195 		ct_free_msg(request);
1196 	}
1197 
1198 	return done;
1199 }
1200 
ct_incoming_request_worker_func(struct work_struct * w)1201 static void ct_incoming_request_worker_func(struct work_struct *w)
1202 {
1203 	struct intel_guc_ct *ct =
1204 		container_of(w, struct intel_guc_ct, requests.worker);
1205 	bool done;
1206 
1207 	do {
1208 		done = ct_process_incoming_requests(ct);
1209 	} while (!done);
1210 }
1211 
ct_handle_event(struct intel_guc_ct * ct,struct ct_incoming_msg * request)1212 static int ct_handle_event(struct intel_guc_ct *ct, struct ct_incoming_msg *request)
1213 {
1214 	const u32 *hxg = &request->msg[GUC_CTB_MSG_MIN_LEN];
1215 	u32 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]);
1216 	unsigned long flags;
1217 
1218 	GEM_BUG_ON(FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_EVENT);
1219 
1220 	/*
1221 	 * Adjusting the space must be done in IRQ or deadlock can occur as the
1222 	 * CTB processing in the below workqueue can send CTBs which creates a
1223 	 * circular dependency if the space was returned there.
1224 	 */
1225 	switch (action) {
1226 	case INTEL_GUC_ACTION_SCHED_CONTEXT_MODE_DONE:
1227 	case INTEL_GUC_ACTION_DEREGISTER_CONTEXT_DONE:
1228 	case INTEL_GUC_ACTION_TLB_INVALIDATION_DONE:
1229 		g2h_release_space(ct, request->size);
1230 	}
1231 
1232 	/*
1233 	 * TLB invalidation responses must be handled immediately as processing
1234 	 * of other G2H notifications may be blocked by an invalidation request.
1235 	 */
1236 	if (action == INTEL_GUC_ACTION_TLB_INVALIDATION_DONE)
1237 		return ct_process_request(ct, request);
1238 
1239 	spin_lock_irqsave(&ct->requests.lock, flags);
1240 	list_add_tail(&request->link, &ct->requests.incoming);
1241 	spin_unlock_irqrestore(&ct->requests.lock, flags);
1242 
1243 	queue_work(system_unbound_wq, &ct->requests.worker);
1244 	return 0;
1245 }
1246 
ct_handle_hxg(struct intel_guc_ct * ct,struct ct_incoming_msg * msg)1247 static int ct_handle_hxg(struct intel_guc_ct *ct, struct ct_incoming_msg *msg)
1248 {
1249 	u32 origin, type;
1250 	u32 *hxg;
1251 	int err;
1252 
1253 	if (unlikely(msg->size < GUC_CTB_HXG_MSG_MIN_LEN))
1254 		return -EBADMSG;
1255 
1256 	hxg = &msg->msg[GUC_CTB_MSG_MIN_LEN];
1257 
1258 	origin = FIELD_GET(GUC_HXG_MSG_0_ORIGIN, hxg[0]);
1259 	if (unlikely(origin != GUC_HXG_ORIGIN_GUC)) {
1260 		err = -EPROTO;
1261 		goto failed;
1262 	}
1263 
1264 	type = FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]);
1265 	switch (type) {
1266 	case GUC_HXG_TYPE_EVENT:
1267 		err = ct_handle_event(ct, msg);
1268 		break;
1269 	case GUC_HXG_TYPE_RESPONSE_SUCCESS:
1270 	case GUC_HXG_TYPE_RESPONSE_FAILURE:
1271 	case GUC_HXG_TYPE_NO_RESPONSE_RETRY:
1272 		err = ct_handle_response(ct, msg);
1273 		break;
1274 	default:
1275 		err = -EOPNOTSUPP;
1276 	}
1277 
1278 	if (unlikely(err)) {
1279 failed:
1280 		CT_ERROR(ct, "Failed to handle HXG message (%pe) %*ph\n",
1281 			 ERR_PTR(err), 4 * GUC_HXG_MSG_MIN_LEN, hxg);
1282 	}
1283 	return err;
1284 }
1285 
ct_handle_msg(struct intel_guc_ct * ct,struct ct_incoming_msg * msg)1286 static void ct_handle_msg(struct intel_guc_ct *ct, struct ct_incoming_msg *msg)
1287 {
1288 	u32 format = FIELD_GET(GUC_CTB_MSG_0_FORMAT, msg->msg[0]);
1289 	int err;
1290 
1291 	if (format == GUC_CTB_FORMAT_HXG)
1292 		err = ct_handle_hxg(ct, msg);
1293 	else
1294 		err = -EOPNOTSUPP;
1295 
1296 	if (unlikely(err)) {
1297 		CT_ERROR(ct, "Failed to process CT message (%pe) %*ph\n",
1298 			 ERR_PTR(err), 4 * msg->size, msg->msg);
1299 		ct_free_msg(msg);
1300 	}
1301 }
1302 
1303 /*
1304  * Return: number available remaining dwords to read (0 if empty)
1305  *         or a negative error code on failure
1306  */
ct_receive(struct intel_guc_ct * ct)1307 static int ct_receive(struct intel_guc_ct *ct)
1308 {
1309 	struct ct_incoming_msg *msg = NULL;
1310 	unsigned long flags;
1311 	int ret;
1312 
1313 	spin_lock_irqsave(&ct->ctbs.recv.lock, flags);
1314 	ret = ct_read(ct, &msg);
1315 	spin_unlock_irqrestore(&ct->ctbs.recv.lock, flags);
1316 	if (ret < 0)
1317 		return ret;
1318 
1319 	if (msg)
1320 		ct_handle_msg(ct, msg);
1321 
1322 	return ret;
1323 }
1324 
ct_try_receive_message(struct intel_guc_ct * ct)1325 static void ct_try_receive_message(struct intel_guc_ct *ct)
1326 {
1327 	int ret;
1328 
1329 	if (GEM_WARN_ON(!ct->enabled))
1330 		return;
1331 
1332 	ret = ct_receive(ct);
1333 	if (ret > 0)
1334 		tasklet_hi_schedule(&ct->receive_tasklet);
1335 }
1336 
ct_receive_tasklet_func(struct tasklet_struct * t)1337 static void ct_receive_tasklet_func(struct tasklet_struct *t)
1338 {
1339 	struct intel_guc_ct *ct = from_tasklet(ct, t, receive_tasklet);
1340 
1341 	ct_try_receive_message(ct);
1342 }
1343 
1344 /*
1345  * When we're communicating with the GuC over CT, GuC uses events
1346  * to notify us about new messages being posted on the RECV buffer.
1347  */
intel_guc_ct_event_handler(struct intel_guc_ct * ct)1348 void intel_guc_ct_event_handler(struct intel_guc_ct *ct)
1349 {
1350 	if (unlikely(!ct->enabled)) {
1351 		WARN(1, "Unexpected GuC event received while CT disabled!\n");
1352 		return;
1353 	}
1354 
1355 	ct_try_receive_message(ct);
1356 }
1357 
intel_guc_ct_print_info(struct intel_guc_ct * ct,struct drm_printer * p)1358 void intel_guc_ct_print_info(struct intel_guc_ct *ct,
1359 			     struct drm_printer *p)
1360 {
1361 	drm_printf(p, "CT %s\n", str_enabled_disabled(ct->enabled));
1362 
1363 	if (!ct->enabled)
1364 		return;
1365 
1366 	drm_printf(p, "H2G Space: %u\n",
1367 		   atomic_read(&ct->ctbs.send.space) * 4);
1368 	drm_printf(p, "Head: %u\n",
1369 		   ct->ctbs.send.desc->head);
1370 	drm_printf(p, "Tail: %u\n",
1371 		   ct->ctbs.send.desc->tail);
1372 	drm_printf(p, "G2H Space: %u\n",
1373 		   atomic_read(&ct->ctbs.recv.space) * 4);
1374 	drm_printf(p, "Head: %u\n",
1375 		   ct->ctbs.recv.desc->head);
1376 	drm_printf(p, "Tail: %u\n",
1377 		   ct->ctbs.recv.desc->tail);
1378 }
1379 
1380 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GUC)
ct_dead_ct_worker_func(struct work_struct * w)1381 static void ct_dead_ct_worker_func(struct work_struct *w)
1382 {
1383 	struct intel_guc_ct *ct = container_of(w, struct intel_guc_ct, dead_ct_worker);
1384 	struct intel_guc *guc = ct_to_guc(ct);
1385 
1386 	if (ct->dead_ct_reported)
1387 		return;
1388 
1389 	ct->dead_ct_reported = true;
1390 
1391 	guc_info(guc, "CTB is dead - reason=0x%X\n", ct->dead_ct_reason);
1392 	intel_klog_error_capture(guc_to_gt(guc), (intel_engine_mask_t)~0U);
1393 }
1394 #endif
1395