1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2023 Intel Corporation
4 */
5
6 #include <linux/bitfield.h>
7 #include <linux/delay.h>
8
9 #include <drm/drm_managed.h>
10
11 #include <kunit/static_stub.h>
12 #include <kunit/test-bug.h>
13
14 #include "abi/guc_actions_sriov_abi.h"
15 #include "abi/guc_relay_actions_abi.h"
16 #include "abi/guc_relay_communication_abi.h"
17
18 #include "xe_assert.h"
19 #include "xe_device.h"
20 #include "xe_gt.h"
21 #include "xe_gt_sriov_printk.h"
22 #include "xe_gt_sriov_pf_service.h"
23 #include "xe_guc.h"
24 #include "xe_guc_ct.h"
25 #include "xe_guc_hxg_helpers.h"
26 #include "xe_guc_relay.h"
27 #include "xe_guc_relay_types.h"
28 #include "xe_sriov.h"
29
30 /*
31 * How long should we wait for the response?
32 * XXX this value is subject for the profiling.
33 */
34 #define RELAY_TIMEOUT_MSEC (2500)
35
36 static void relays_worker_fn(struct work_struct *w);
37
relay_to_guc(struct xe_guc_relay * relay)38 static struct xe_guc *relay_to_guc(struct xe_guc_relay *relay)
39 {
40 return container_of(relay, struct xe_guc, relay);
41 }
42
relay_to_ct(struct xe_guc_relay * relay)43 static struct xe_guc_ct *relay_to_ct(struct xe_guc_relay *relay)
44 {
45 return &relay_to_guc(relay)->ct;
46 }
47
relay_to_gt(struct xe_guc_relay * relay)48 static struct xe_gt *relay_to_gt(struct xe_guc_relay *relay)
49 {
50 return guc_to_gt(relay_to_guc(relay));
51 }
52
relay_to_xe(struct xe_guc_relay * relay)53 static struct xe_device *relay_to_xe(struct xe_guc_relay *relay)
54 {
55 return gt_to_xe(relay_to_gt(relay));
56 }
57
58 #define relay_assert(relay, condition) xe_gt_assert(relay_to_gt(relay), condition)
59 #define relay_notice(relay, msg...) xe_gt_sriov_notice(relay_to_gt(relay), "relay: " msg)
60 #define relay_debug(relay, msg...) xe_gt_sriov_dbg_verbose(relay_to_gt(relay), "relay: " msg)
61
relay_get_totalvfs(struct xe_guc_relay * relay)62 static int relay_get_totalvfs(struct xe_guc_relay *relay)
63 {
64 struct xe_device *xe = relay_to_xe(relay);
65 struct pci_dev *pdev = to_pci_dev(xe->drm.dev);
66
67 KUNIT_STATIC_STUB_REDIRECT(relay_get_totalvfs, relay);
68 return IS_SRIOV_VF(xe) ? 0 : pci_sriov_get_totalvfs(pdev);
69 }
70
relay_is_ready(struct xe_guc_relay * relay)71 static bool relay_is_ready(struct xe_guc_relay *relay)
72 {
73 return mempool_initialized(&relay->pool);
74 }
75
relay_get_next_rid(struct xe_guc_relay * relay)76 static u32 relay_get_next_rid(struct xe_guc_relay *relay)
77 {
78 u32 rid;
79
80 spin_lock(&relay->lock);
81 rid = ++relay->last_rid;
82 spin_unlock(&relay->lock);
83
84 return rid;
85 }
86
87 /**
88 * struct relay_transaction - internal data used to handle transactions
89 *
90 * Relation between struct relay_transaction members::
91 *
92 * <-------------------- GUC_CTB_MAX_DWORDS -------------->
93 * <-------- GUC_RELAY_MSG_MAX_LEN --->
94 * <--- offset ---> <--- request_len ------->
95 * +----------------+-------------------------+----------+--+
96 * | | | | |
97 * +----------------+-------------------------+----------+--+
98 * ^ ^
99 * / /
100 * request_buf request
101 *
102 * <-------------------- GUC_CTB_MAX_DWORDS -------------->
103 * <-------- GUC_RELAY_MSG_MAX_LEN --->
104 * <--- offset ---> <--- response_len --->
105 * +----------------+----------------------+-------------+--+
106 * | | | | |
107 * +----------------+----------------------+-------------+--+
108 * ^ ^
109 * / /
110 * response_buf response
111 */
112 struct relay_transaction {
113 /**
114 * @incoming: indicates whether this transaction represents an incoming
115 * request from the remote VF/PF or this transaction
116 * represents outgoing request to the remote VF/PF.
117 */
118 bool incoming;
119
120 /**
121 * @remote: PF/VF identifier of the origin (or target) of the relay
122 * request message.
123 */
124 u32 remote;
125
126 /** @rid: identifier of the VF/PF relay message. */
127 u32 rid;
128
129 /**
130 * @request: points to the inner VF/PF request message, copied to the
131 * #response_buf starting at #offset.
132 */
133 u32 *request;
134
135 /** @request_len: length of the inner VF/PF request message. */
136 u32 request_len;
137
138 /**
139 * @response: points to the placeholder buffer where inner VF/PF
140 * response will be located, for outgoing transaction
141 * this could be caller's buffer (if provided) otherwise
142 * it points to the #response_buf starting at #offset.
143 */
144 u32 *response;
145
146 /**
147 * @response_len: length of the inner VF/PF response message (only
148 * if #status is 0), initially set to the size of the
149 * placeholder buffer where response message will be
150 * copied.
151 */
152 u32 response_len;
153
154 /**
155 * @offset: offset to the start of the inner VF/PF relay message inside
156 * buffers; this offset is equal the length of the outer GuC
157 * relay header message.
158 */
159 u32 offset;
160
161 /**
162 * @request_buf: buffer with VF/PF request message including outer
163 * transport message.
164 */
165 u32 request_buf[GUC_CTB_MAX_DWORDS];
166
167 /**
168 * @response_buf: buffer with VF/PF response message including outer
169 * transport message.
170 */
171 u32 response_buf[GUC_CTB_MAX_DWORDS];
172
173 /**
174 * @reply: status of the reply, 0 means that data pointed by the
175 * #response is valid.
176 */
177 int reply;
178
179 /** @done: completion of the outgoing transaction. */
180 struct completion done;
181
182 /** @link: transaction list link */
183 struct list_head link;
184 };
185
prepare_pf2guc(u32 * msg,u32 target,u32 rid)186 static u32 prepare_pf2guc(u32 *msg, u32 target, u32 rid)
187 {
188 msg[0] = FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) |
189 FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) |
190 FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION, XE_GUC_ACTION_PF2GUC_RELAY_TO_VF);
191 msg[1] = FIELD_PREP(PF2GUC_RELAY_TO_VF_REQUEST_MSG_1_VFID, target);
192 msg[2] = FIELD_PREP(PF2GUC_RELAY_TO_VF_REQUEST_MSG_2_RELAY_ID, rid);
193
194 return PF2GUC_RELAY_TO_VF_REQUEST_MSG_MIN_LEN;
195 }
196
prepare_vf2guc(u32 * msg,u32 rid)197 static u32 prepare_vf2guc(u32 *msg, u32 rid)
198 {
199 msg[0] = FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) |
200 FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) |
201 FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION, XE_GUC_ACTION_VF2GUC_RELAY_TO_PF);
202 msg[1] = FIELD_PREP(VF2GUC_RELAY_TO_PF_REQUEST_MSG_1_RELAY_ID, rid);
203
204 return VF2GUC_RELAY_TO_PF_REQUEST_MSG_MIN_LEN;
205 }
206
207 static struct relay_transaction *
__relay_get_transaction(struct xe_guc_relay * relay,bool incoming,u32 remote,u32 rid,const u32 * action,u32 action_len,u32 * resp,u32 resp_size)208 __relay_get_transaction(struct xe_guc_relay *relay, bool incoming, u32 remote, u32 rid,
209 const u32 *action, u32 action_len, u32 *resp, u32 resp_size)
210 {
211 struct relay_transaction *txn;
212
213 relay_assert(relay, action_len >= GUC_RELAY_MSG_MIN_LEN);
214 relay_assert(relay, action_len <= GUC_RELAY_MSG_MAX_LEN);
215 relay_assert(relay, !(!!resp ^ !!resp_size));
216 relay_assert(relay, resp_size <= GUC_RELAY_MSG_MAX_LEN);
217 relay_assert(relay, resp_size == 0 || resp_size >= GUC_RELAY_MSG_MIN_LEN);
218
219 if (unlikely(!relay_is_ready(relay)))
220 return ERR_PTR(-ENODEV);
221
222 /*
223 * For incoming requests we can't use GFP_KERNEL as those are delivered
224 * with CTB lock held which is marked as used in the reclaim path.
225 * Btw, that's one of the reason why we use mempool here!
226 */
227 txn = mempool_alloc(&relay->pool, incoming ? GFP_ATOMIC : GFP_KERNEL);
228 if (!txn)
229 return ERR_PTR(-ENOMEM);
230
231 txn->incoming = incoming;
232 txn->remote = remote;
233 txn->rid = rid;
234 txn->offset = remote ?
235 prepare_pf2guc(incoming ? txn->response_buf : txn->request_buf, remote, rid) :
236 prepare_vf2guc(incoming ? txn->response_buf : txn->request_buf, rid);
237
238 relay_assert(relay, txn->offset);
239 relay_assert(relay, txn->offset + GUC_RELAY_MSG_MAX_LEN <= ARRAY_SIZE(txn->request_buf));
240 relay_assert(relay, txn->offset + GUC_RELAY_MSG_MAX_LEN <= ARRAY_SIZE(txn->response_buf));
241
242 txn->request = txn->request_buf + txn->offset;
243 memcpy(&txn->request_buf[txn->offset], action, sizeof(u32) * action_len);
244 txn->request_len = action_len;
245
246 txn->response = resp ?: txn->response_buf + txn->offset;
247 txn->response_len = resp_size ?: GUC_RELAY_MSG_MAX_LEN;
248 txn->reply = -ENOMSG;
249 INIT_LIST_HEAD(&txn->link);
250 init_completion(&txn->done);
251
252 return txn;
253 }
254
255 static struct relay_transaction *
relay_new_transaction(struct xe_guc_relay * relay,u32 target,const u32 * action,u32 len,u32 * resp,u32 resp_size)256 relay_new_transaction(struct xe_guc_relay *relay, u32 target, const u32 *action, u32 len,
257 u32 *resp, u32 resp_size)
258 {
259 u32 rid = relay_get_next_rid(relay);
260
261 return __relay_get_transaction(relay, false, target, rid, action, len, resp, resp_size);
262 }
263
264 static struct relay_transaction *
relay_new_incoming_transaction(struct xe_guc_relay * relay,u32 origin,u32 rid,const u32 * action,u32 len)265 relay_new_incoming_transaction(struct xe_guc_relay *relay, u32 origin, u32 rid,
266 const u32 *action, u32 len)
267 {
268 return __relay_get_transaction(relay, true, origin, rid, action, len, NULL, 0);
269 }
270
relay_release_transaction(struct xe_guc_relay * relay,struct relay_transaction * txn)271 static void relay_release_transaction(struct xe_guc_relay *relay, struct relay_transaction *txn)
272 {
273 relay_assert(relay, list_empty(&txn->link));
274
275 txn->offset = 0;
276 txn->response = NULL;
277 txn->reply = -ESTALE;
278 mempool_free(txn, &relay->pool);
279 }
280
relay_send_transaction(struct xe_guc_relay * relay,struct relay_transaction * txn)281 static int relay_send_transaction(struct xe_guc_relay *relay, struct relay_transaction *txn)
282 {
283 u32 len = txn->incoming ? txn->response_len : txn->request_len;
284 u32 *buf = txn->incoming ? txn->response_buf : txn->request_buf;
285 u32 *msg = buf + txn->offset;
286 int ret;
287
288 relay_assert(relay, txn->offset);
289 relay_assert(relay, txn->offset + len <= GUC_CTB_MAX_DWORDS);
290 relay_assert(relay, len >= GUC_RELAY_MSG_MIN_LEN);
291 relay_assert(relay, len <= GUC_RELAY_MSG_MAX_LEN);
292
293 relay_debug(relay, "sending %s.%u to %u = %*ph\n",
294 guc_hxg_type_to_string(FIELD_GET(GUC_HXG_MSG_0_TYPE, msg[0])),
295 txn->rid, txn->remote, (int)sizeof(u32) * len, msg);
296
297 ret = xe_guc_ct_send_block(relay_to_ct(relay), buf, len + txn->offset);
298
299 if (unlikely(ret > 0)) {
300 relay_notice(relay, "Unexpected data=%d from GuC, wrong ABI?\n", ret);
301 ret = -EPROTO;
302 }
303 if (unlikely(ret < 0)) {
304 relay_notice(relay, "Failed to send %s.%x to GuC (%pe) %*ph ...\n",
305 guc_hxg_type_to_string(FIELD_GET(GUC_HXG_MSG_0_TYPE, buf[0])),
306 FIELD_GET(GUC_HXG_REQUEST_MSG_0_ACTION, buf[0]),
307 ERR_PTR(ret), (int)sizeof(u32) * txn->offset, buf);
308 relay_notice(relay, "Failed to send %s.%u to %u (%pe) %*ph\n",
309 guc_hxg_type_to_string(FIELD_GET(GUC_HXG_MSG_0_TYPE, msg[0])),
310 txn->rid, txn->remote, ERR_PTR(ret), (int)sizeof(u32) * len, msg);
311 }
312
313 return ret;
314 }
315
__fini_relay(struct drm_device * drm,void * arg)316 static void __fini_relay(struct drm_device *drm, void *arg)
317 {
318 struct xe_guc_relay *relay = arg;
319
320 mempool_exit(&relay->pool);
321 }
322
323 /**
324 * xe_guc_relay_init - Initialize a &xe_guc_relay
325 * @relay: the &xe_guc_relay to initialize
326 *
327 * Initialize remaining members of &xe_guc_relay that may depend
328 * on the SR-IOV mode.
329 *
330 * Return: 0 on success or a negative error code on failure.
331 */
xe_guc_relay_init(struct xe_guc_relay * relay)332 int xe_guc_relay_init(struct xe_guc_relay *relay)
333 {
334 const int XE_RELAY_MEMPOOL_MIN_NUM = 1;
335 struct xe_device *xe = relay_to_xe(relay);
336 int err;
337
338 relay_assert(relay, !relay_is_ready(relay));
339
340 if (!IS_SRIOV(xe))
341 return 0;
342
343 spin_lock_init(&relay->lock);
344 INIT_WORK(&relay->worker, relays_worker_fn);
345 INIT_LIST_HEAD(&relay->pending_relays);
346 INIT_LIST_HEAD(&relay->incoming_actions);
347
348 err = mempool_init_kmalloc_pool(&relay->pool, XE_RELAY_MEMPOOL_MIN_NUM +
349 relay_get_totalvfs(relay),
350 sizeof(struct relay_transaction));
351 if (err)
352 return err;
353
354 relay_debug(relay, "using mempool with %d elements\n", relay->pool.min_nr);
355
356 return drmm_add_action_or_reset(&xe->drm, __fini_relay, relay);
357 }
358
to_relay_error(int err)359 static u32 to_relay_error(int err)
360 {
361 /* XXX: assume that relay errors match errno codes */
362 return err < 0 ? -err : GUC_RELAY_ERROR_UNDISCLOSED;
363 }
364
from_relay_error(u32 error)365 static int from_relay_error(u32 error)
366 {
367 /* XXX: assume that relay errors match errno codes */
368 return error ? -error : -ENODATA;
369 }
370
sanitize_relay_error(u32 error)371 static u32 sanitize_relay_error(u32 error)
372 {
373 /* XXX TBD if generic error codes will be allowed */
374 if (!IS_ENABLED(CONFIG_DRM_XE_DEBUG))
375 error = GUC_RELAY_ERROR_UNDISCLOSED;
376 return error;
377 }
378
sanitize_relay_error_hint(u32 hint)379 static u32 sanitize_relay_error_hint(u32 hint)
380 {
381 /* XXX TBD if generic error codes will be allowed */
382 if (!IS_ENABLED(CONFIG_DRM_XE_DEBUG))
383 hint = 0;
384 return hint;
385 }
386
prepare_error_reply(u32 * msg,u32 error,u32 hint)387 static u32 prepare_error_reply(u32 *msg, u32 error, u32 hint)
388 {
389 msg[0] = FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) |
390 FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_RESPONSE_FAILURE) |
391 FIELD_PREP(GUC_HXG_FAILURE_MSG_0_HINT, hint) |
392 FIELD_PREP(GUC_HXG_FAILURE_MSG_0_ERROR, error);
393
394 XE_WARN_ON(!FIELD_FIT(GUC_HXG_FAILURE_MSG_0_ERROR, error));
395 XE_WARN_ON(!FIELD_FIT(GUC_HXG_FAILURE_MSG_0_HINT, hint));
396
397 return GUC_HXG_FAILURE_MSG_LEN;
398 }
399
relay_testonly_nop(struct xe_guc_relay * relay)400 static void relay_testonly_nop(struct xe_guc_relay *relay)
401 {
402 KUNIT_STATIC_STUB_REDIRECT(relay_testonly_nop, relay);
403 }
404
relay_send_message_and_wait(struct xe_guc_relay * relay,struct relay_transaction * txn,u32 * buf,u32 buf_size)405 static int relay_send_message_and_wait(struct xe_guc_relay *relay,
406 struct relay_transaction *txn,
407 u32 *buf, u32 buf_size)
408 {
409 unsigned long timeout = msecs_to_jiffies(RELAY_TIMEOUT_MSEC);
410 u32 *msg = &txn->request_buf[txn->offset];
411 u32 len = txn->request_len;
412 u32 type, action, data0;
413 int ret;
414 long n;
415
416 type = FIELD_GET(GUC_HXG_MSG_0_TYPE, msg[0]);
417 action = FIELD_GET(GUC_HXG_REQUEST_MSG_0_ACTION, msg[0]);
418 data0 = FIELD_GET(GUC_HXG_REQUEST_MSG_0_DATA0, msg[0]);
419
420 relay_debug(relay, "%s.%u to %u action %#x:%u\n",
421 guc_hxg_type_to_string(type),
422 txn->rid, txn->remote, action, data0);
423
424 /* list ordering does not need to match RID ordering */
425 spin_lock(&relay->lock);
426 list_add_tail(&txn->link, &relay->pending_relays);
427 spin_unlock(&relay->lock);
428
429 resend:
430 ret = relay_send_transaction(relay, txn);
431 if (unlikely(ret < 0))
432 goto unlink;
433
434 wait:
435 n = wait_for_completion_timeout(&txn->done, timeout);
436 if (unlikely(n == 0 && txn->reply)) {
437 ret = -ETIME;
438 goto unlink;
439 }
440
441 relay_debug(relay, "%u.%u reply %d after %u msec\n",
442 txn->remote, txn->rid, txn->reply, jiffies_to_msecs(timeout - n));
443 if (unlikely(txn->reply)) {
444 reinit_completion(&txn->done);
445 if (txn->reply == -EAGAIN)
446 goto resend;
447 if (txn->reply == -EBUSY) {
448 relay_testonly_nop(relay);
449 goto wait;
450 }
451 if (txn->reply > 0)
452 ret = from_relay_error(txn->reply);
453 else
454 ret = txn->reply;
455 goto unlink;
456 }
457
458 relay_debug(relay, "%u.%u response %*ph\n", txn->remote, txn->rid,
459 (int)sizeof(u32) * txn->response_len, txn->response);
460 relay_assert(relay, txn->response_len >= GUC_RELAY_MSG_MIN_LEN);
461 ret = txn->response_len;
462
463 unlink:
464 spin_lock(&relay->lock);
465 list_del_init(&txn->link);
466 spin_unlock(&relay->lock);
467
468 if (unlikely(ret < 0)) {
469 relay_notice(relay, "Unsuccessful %s.%u %#x:%u to %u (%pe) %*ph\n",
470 guc_hxg_type_to_string(type), txn->rid,
471 action, data0, txn->remote, ERR_PTR(ret),
472 (int)sizeof(u32) * len, msg);
473 }
474
475 return ret;
476 }
477
relay_send_to(struct xe_guc_relay * relay,u32 target,const u32 * msg,u32 len,u32 * buf,u32 buf_size)478 static int relay_send_to(struct xe_guc_relay *relay, u32 target,
479 const u32 *msg, u32 len, u32 *buf, u32 buf_size)
480 {
481 struct relay_transaction *txn;
482 int ret;
483
484 relay_assert(relay, len >= GUC_RELAY_MSG_MIN_LEN);
485 relay_assert(relay, len <= GUC_RELAY_MSG_MAX_LEN);
486 relay_assert(relay, FIELD_GET(GUC_HXG_MSG_0_ORIGIN, msg[0]) == GUC_HXG_ORIGIN_HOST);
487 relay_assert(relay, guc_hxg_type_is_action(FIELD_GET(GUC_HXG_MSG_0_TYPE, msg[0])));
488
489 if (unlikely(!relay_is_ready(relay)))
490 return -ENODEV;
491
492 txn = relay_new_transaction(relay, target, msg, len, buf, buf_size);
493 if (IS_ERR(txn))
494 return PTR_ERR(txn);
495
496 switch (FIELD_GET(GUC_HXG_MSG_0_TYPE, msg[0])) {
497 case GUC_HXG_TYPE_REQUEST:
498 ret = relay_send_message_and_wait(relay, txn, buf, buf_size);
499 break;
500 case GUC_HXG_TYPE_FAST_REQUEST:
501 relay_assert(relay, !GUC_HXG_TYPE_FAST_REQUEST);
502 fallthrough;
503 case GUC_HXG_TYPE_EVENT:
504 ret = relay_send_transaction(relay, txn);
505 break;
506 default:
507 ret = -EINVAL;
508 break;
509 }
510
511 relay_release_transaction(relay, txn);
512 return ret;
513 }
514
515 #ifdef CONFIG_PCI_IOV
516 /**
517 * xe_guc_relay_send_to_vf - Send a message to the VF.
518 * @relay: the &xe_guc_relay which will send the message
519 * @target: target VF number
520 * @msg: request message to be sent
521 * @len: length of the request message (in dwords, can't be 0)
522 * @buf: placeholder for the response message
523 * @buf_size: size of the response message placeholder (in dwords)
524 *
525 * This function can only be used by the driver running in the SR-IOV PF mode.
526 *
527 * Return: Non-negative response length (in dwords) or
528 * a negative error code on failure.
529 */
xe_guc_relay_send_to_vf(struct xe_guc_relay * relay,u32 target,const u32 * msg,u32 len,u32 * buf,u32 buf_size)530 int xe_guc_relay_send_to_vf(struct xe_guc_relay *relay, u32 target,
531 const u32 *msg, u32 len, u32 *buf, u32 buf_size)
532 {
533 relay_assert(relay, IS_SRIOV_PF(relay_to_xe(relay)));
534
535 return relay_send_to(relay, target, msg, len, buf, buf_size);
536 }
537 #endif
538
539 /**
540 * xe_guc_relay_send_to_pf - Send a message to the PF.
541 * @relay: the &xe_guc_relay which will send the message
542 * @msg: request message to be sent
543 * @len: length of the message (in dwords, can't be 0)
544 * @buf: placeholder for the response message
545 * @buf_size: size of the response message placeholder (in dwords)
546 *
547 * This function can only be used by driver running in SR-IOV VF mode.
548 *
549 * Return: Non-negative response length (in dwords) or
550 * a negative error code on failure.
551 */
xe_guc_relay_send_to_pf(struct xe_guc_relay * relay,const u32 * msg,u32 len,u32 * buf,u32 buf_size)552 int xe_guc_relay_send_to_pf(struct xe_guc_relay *relay,
553 const u32 *msg, u32 len, u32 *buf, u32 buf_size)
554 {
555 relay_assert(relay, IS_SRIOV_VF(relay_to_xe(relay)));
556
557 return relay_send_to(relay, PFID, msg, len, buf, buf_size);
558 }
559
relay_handle_reply(struct xe_guc_relay * relay,u32 origin,u32 rid,int reply,const u32 * msg,u32 len)560 static int relay_handle_reply(struct xe_guc_relay *relay, u32 origin,
561 u32 rid, int reply, const u32 *msg, u32 len)
562 {
563 struct relay_transaction *pending;
564 int err = -ESRCH;
565
566 spin_lock(&relay->lock);
567 list_for_each_entry(pending, &relay->pending_relays, link) {
568 if (pending->remote != origin || pending->rid != rid) {
569 relay_debug(relay, "%u.%u still awaits response\n",
570 pending->remote, pending->rid);
571 continue;
572 }
573 err = 0; /* found! */
574 if (reply == 0) {
575 if (len > pending->response_len) {
576 reply = -ENOBUFS;
577 err = -ENOBUFS;
578 } else {
579 memcpy(pending->response, msg, 4 * len);
580 pending->response_len = len;
581 }
582 }
583 pending->reply = reply;
584 complete_all(&pending->done);
585 break;
586 }
587 spin_unlock(&relay->lock);
588
589 return err;
590 }
591
relay_handle_failure(struct xe_guc_relay * relay,u32 origin,u32 rid,const u32 * msg,u32 len)592 static int relay_handle_failure(struct xe_guc_relay *relay, u32 origin,
593 u32 rid, const u32 *msg, u32 len)
594 {
595 int error = FIELD_GET(GUC_HXG_FAILURE_MSG_0_ERROR, msg[0]);
596 u32 hint __maybe_unused = FIELD_GET(GUC_HXG_FAILURE_MSG_0_HINT, msg[0]);
597
598 relay_assert(relay, len);
599 relay_debug(relay, "%u.%u error %#x (%pe) hint %u debug %*ph\n",
600 origin, rid, error, ERR_PTR(-error), hint, 4 * (len - 1), msg + 1);
601
602 return relay_handle_reply(relay, origin, rid, error ?: -EREMOTEIO, NULL, 0);
603 }
604
relay_testloop_action_handler(struct xe_guc_relay * relay,u32 origin,const u32 * msg,u32 len,u32 * response,u32 size)605 static int relay_testloop_action_handler(struct xe_guc_relay *relay, u32 origin,
606 const u32 *msg, u32 len, u32 *response, u32 size)
607 {
608 static ktime_t last_reply = 0;
609 u32 type = FIELD_GET(GUC_HXG_MSG_0_TYPE, msg[0]);
610 u32 action = FIELD_GET(GUC_HXG_REQUEST_MSG_0_ACTION, msg[0]);
611 u32 opcode = FIELD_GET(GUC_HXG_REQUEST_MSG_0_DATA0, msg[0]);
612 ktime_t now = ktime_get();
613 bool busy;
614 int ret;
615
616 relay_assert(relay, guc_hxg_type_is_action(type));
617 relay_assert(relay, action == GUC_RELAY_ACTION_VFXPF_TESTLOOP);
618
619 if (!IS_ENABLED(CONFIG_DRM_XE_DEBUG_SRIOV))
620 return -ECONNREFUSED;
621
622 if (!last_reply)
623 last_reply = now;
624 busy = ktime_before(now, ktime_add_ms(last_reply, 2 * RELAY_TIMEOUT_MSEC));
625 if (!busy)
626 last_reply = now;
627
628 switch (opcode) {
629 case VFXPF_TESTLOOP_OPCODE_NOP:
630 if (type == GUC_HXG_TYPE_EVENT)
631 return 0;
632 return guc_hxg_msg_encode_success(response, 0);
633 case VFXPF_TESTLOOP_OPCODE_BUSY:
634 if (type == GUC_HXG_TYPE_EVENT)
635 return -EPROTO;
636 msleep(RELAY_TIMEOUT_MSEC / 8);
637 if (busy)
638 return -EINPROGRESS;
639 return guc_hxg_msg_encode_success(response, 0);
640 case VFXPF_TESTLOOP_OPCODE_RETRY:
641 if (type == GUC_HXG_TYPE_EVENT)
642 return -EPROTO;
643 msleep(RELAY_TIMEOUT_MSEC / 8);
644 if (busy)
645 return guc_hxg_msg_encode_retry(response, 0);
646 return guc_hxg_msg_encode_success(response, 0);
647 case VFXPF_TESTLOOP_OPCODE_ECHO:
648 if (type == GUC_HXG_TYPE_EVENT)
649 return -EPROTO;
650 if (size < len)
651 return -ENOBUFS;
652 ret = guc_hxg_msg_encode_success(response, len);
653 memcpy(response + ret, msg + ret, (len - ret) * sizeof(u32));
654 return len;
655 case VFXPF_TESTLOOP_OPCODE_FAIL:
656 return -EHWPOISON;
657 default:
658 break;
659 }
660
661 relay_notice(relay, "Unexpected action %#x opcode %#x\n", action, opcode);
662 return -EBADRQC;
663 }
664
relay_action_handler(struct xe_guc_relay * relay,u32 origin,const u32 * msg,u32 len,u32 * response,u32 size)665 static int relay_action_handler(struct xe_guc_relay *relay, u32 origin,
666 const u32 *msg, u32 len, u32 *response, u32 size)
667 {
668 struct xe_gt *gt = relay_to_gt(relay);
669 u32 type;
670 int ret;
671
672 relay_assert(relay, len >= GUC_HXG_MSG_MIN_LEN);
673
674 if (FIELD_GET(GUC_HXG_REQUEST_MSG_0_ACTION, msg[0]) == GUC_RELAY_ACTION_VFXPF_TESTLOOP)
675 return relay_testloop_action_handler(relay, origin, msg, len, response, size);
676
677 type = FIELD_GET(GUC_HXG_MSG_0_TYPE, msg[0]);
678
679 if (IS_SRIOV_PF(relay_to_xe(relay)))
680 ret = xe_gt_sriov_pf_service_process_request(gt, origin, msg, len, response, size);
681 else
682 ret = -EOPNOTSUPP;
683
684 if (type == GUC_HXG_TYPE_EVENT)
685 relay_assert(relay, ret <= 0);
686
687 return ret;
688 }
689
relay_dequeue_transaction(struct xe_guc_relay * relay)690 static struct relay_transaction *relay_dequeue_transaction(struct xe_guc_relay *relay)
691 {
692 struct relay_transaction *txn;
693
694 spin_lock(&relay->lock);
695 txn = list_first_entry_or_null(&relay->incoming_actions, struct relay_transaction, link);
696 if (txn)
697 list_del_init(&txn->link);
698 spin_unlock(&relay->lock);
699
700 return txn;
701 }
702
relay_process_incoming_action(struct xe_guc_relay * relay)703 static void relay_process_incoming_action(struct xe_guc_relay *relay)
704 {
705 struct relay_transaction *txn;
706 bool again = false;
707 u32 type;
708 int ret;
709
710 txn = relay_dequeue_transaction(relay);
711 if (!txn)
712 return;
713
714 type = FIELD_GET(GUC_HXG_MSG_0_TYPE, txn->request_buf[txn->offset]);
715
716 ret = relay_action_handler(relay, txn->remote,
717 txn->request_buf + txn->offset, txn->request_len,
718 txn->response_buf + txn->offset,
719 ARRAY_SIZE(txn->response_buf) - txn->offset);
720
721 if (ret == -EINPROGRESS) {
722 again = true;
723 ret = guc_hxg_msg_encode_busy(txn->response_buf + txn->offset, 0);
724 }
725
726 if (ret > 0) {
727 txn->response_len = ret;
728 ret = relay_send_transaction(relay, txn);
729 }
730
731 if (ret < 0) {
732 u32 error = to_relay_error(ret);
733
734 relay_notice(relay, "Failed to handle %s.%u from %u (%pe) %*ph\n",
735 guc_hxg_type_to_string(type), txn->rid, txn->remote,
736 ERR_PTR(ret), 4 * txn->request_len, txn->request_buf + txn->offset);
737
738 txn->response_len = prepare_error_reply(txn->response_buf + txn->offset,
739 txn->remote ?
740 sanitize_relay_error(error) : error,
741 txn->remote ?
742 sanitize_relay_error_hint(-ret) : -ret);
743 ret = relay_send_transaction(relay, txn);
744 again = false;
745 }
746
747 if (again) {
748 spin_lock(&relay->lock);
749 list_add(&txn->link, &relay->incoming_actions);
750 spin_unlock(&relay->lock);
751 return;
752 }
753
754 if (unlikely(ret < 0))
755 relay_notice(relay, "Failed to process action.%u (%pe) %*ph\n",
756 txn->rid, ERR_PTR(ret), 4 * txn->request_len,
757 txn->request_buf + txn->offset);
758
759 relay_release_transaction(relay, txn);
760 }
761
relay_needs_worker(struct xe_guc_relay * relay)762 static bool relay_needs_worker(struct xe_guc_relay *relay)
763 {
764 bool is_empty;
765
766 spin_lock(&relay->lock);
767 is_empty = list_empty(&relay->incoming_actions);
768 spin_unlock(&relay->lock);
769
770 return !is_empty;
771
772 }
773
relay_kick_worker(struct xe_guc_relay * relay)774 static void relay_kick_worker(struct xe_guc_relay *relay)
775 {
776 KUNIT_STATIC_STUB_REDIRECT(relay_kick_worker, relay);
777 queue_work(relay_to_xe(relay)->sriov.wq, &relay->worker);
778 }
779
relays_worker_fn(struct work_struct * w)780 static void relays_worker_fn(struct work_struct *w)
781 {
782 struct xe_guc_relay *relay = container_of(w, struct xe_guc_relay, worker);
783
784 relay_process_incoming_action(relay);
785
786 if (relay_needs_worker(relay))
787 relay_kick_worker(relay);
788 }
789
relay_queue_action_msg(struct xe_guc_relay * relay,u32 origin,u32 rid,const u32 * msg,u32 len)790 static int relay_queue_action_msg(struct xe_guc_relay *relay, u32 origin, u32 rid,
791 const u32 *msg, u32 len)
792 {
793 struct relay_transaction *txn;
794
795 txn = relay_new_incoming_transaction(relay, origin, rid, msg, len);
796 if (IS_ERR(txn))
797 return PTR_ERR(txn);
798
799 spin_lock(&relay->lock);
800 list_add_tail(&txn->link, &relay->incoming_actions);
801 spin_unlock(&relay->lock);
802
803 relay_kick_worker(relay);
804 return 0;
805 }
806
relay_process_msg(struct xe_guc_relay * relay,u32 origin,u32 rid,const u32 * msg,u32 len)807 static int relay_process_msg(struct xe_guc_relay *relay, u32 origin, u32 rid,
808 const u32 *msg, u32 len)
809 {
810 u32 type;
811 int err;
812
813 if (unlikely(len < GUC_HXG_MSG_MIN_LEN))
814 return -EPROTO;
815
816 if (FIELD_GET(GUC_HXG_MSG_0_ORIGIN, msg[0]) != GUC_HXG_ORIGIN_HOST)
817 return -EPROTO;
818
819 type = FIELD_GET(GUC_HXG_MSG_0_TYPE, msg[0]);
820 relay_debug(relay, "received %s.%u from %u = %*ph\n",
821 guc_hxg_type_to_string(type), rid, origin, 4 * len, msg);
822
823 switch (type) {
824 case GUC_HXG_TYPE_REQUEST:
825 case GUC_HXG_TYPE_FAST_REQUEST:
826 case GUC_HXG_TYPE_EVENT:
827 err = relay_queue_action_msg(relay, origin, rid, msg, len);
828 break;
829 case GUC_HXG_TYPE_RESPONSE_SUCCESS:
830 err = relay_handle_reply(relay, origin, rid, 0, msg, len);
831 break;
832 case GUC_HXG_TYPE_NO_RESPONSE_BUSY:
833 err = relay_handle_reply(relay, origin, rid, -EBUSY, NULL, 0);
834 break;
835 case GUC_HXG_TYPE_NO_RESPONSE_RETRY:
836 err = relay_handle_reply(relay, origin, rid, -EAGAIN, NULL, 0);
837 break;
838 case GUC_HXG_TYPE_RESPONSE_FAILURE:
839 err = relay_handle_failure(relay, origin, rid, msg, len);
840 break;
841 default:
842 err = -EBADRQC;
843 }
844
845 if (unlikely(err))
846 relay_notice(relay, "Failed to process %s.%u from %u (%pe) %*ph\n",
847 guc_hxg_type_to_string(type), rid, origin,
848 ERR_PTR(err), 4 * len, msg);
849
850 return err;
851 }
852
853 /**
854 * xe_guc_relay_process_guc2vf - Handle relay notification message from the GuC.
855 * @relay: the &xe_guc_relay which will handle the message
856 * @msg: message to be handled
857 * @len: length of the message (in dwords)
858 *
859 * This function will handle relay messages received from the GuC.
860 *
861 * This function is can only be used if driver is running in SR-IOV mode.
862 *
863 * Return: 0 on success or a negative error code on failure.
864 */
xe_guc_relay_process_guc2vf(struct xe_guc_relay * relay,const u32 * msg,u32 len)865 int xe_guc_relay_process_guc2vf(struct xe_guc_relay *relay, const u32 *msg, u32 len)
866 {
867 u32 rid;
868
869 relay_assert(relay, len >= GUC_HXG_MSG_MIN_LEN);
870 relay_assert(relay, FIELD_GET(GUC_HXG_MSG_0_ORIGIN, msg[0]) == GUC_HXG_ORIGIN_GUC);
871 relay_assert(relay, FIELD_GET(GUC_HXG_MSG_0_TYPE, msg[0]) == GUC_HXG_TYPE_EVENT);
872 relay_assert(relay, FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, msg[0]) ==
873 XE_GUC_ACTION_GUC2VF_RELAY_FROM_PF);
874
875 if (unlikely(!IS_SRIOV_VF(relay_to_xe(relay)) && !kunit_get_current_test()))
876 return -EPERM;
877
878 if (unlikely(!relay_is_ready(relay)))
879 return -ENODEV;
880
881 if (unlikely(len < GUC2VF_RELAY_FROM_PF_EVENT_MSG_MIN_LEN))
882 return -EPROTO;
883
884 if (unlikely(len > GUC2VF_RELAY_FROM_PF_EVENT_MSG_MAX_LEN))
885 return -EMSGSIZE;
886
887 if (unlikely(FIELD_GET(GUC_HXG_EVENT_MSG_0_DATA0, msg[0])))
888 return -EPFNOSUPPORT;
889
890 rid = FIELD_GET(GUC2VF_RELAY_FROM_PF_EVENT_MSG_1_RELAY_ID, msg[1]);
891
892 return relay_process_msg(relay, PFID, rid,
893 msg + GUC2VF_RELAY_FROM_PF_EVENT_MSG_MIN_LEN,
894 len - GUC2VF_RELAY_FROM_PF_EVENT_MSG_MIN_LEN);
895 }
896
897 #ifdef CONFIG_PCI_IOV
898 /**
899 * xe_guc_relay_process_guc2pf - Handle relay notification message from the GuC.
900 * @relay: the &xe_guc_relay which will handle the message
901 * @msg: message to be handled
902 * @len: length of the message (in dwords)
903 *
904 * This function will handle relay messages received from the GuC.
905 *
906 * This function can only be used if driver is running in SR-IOV PF mode.
907 *
908 * Return: 0 on success or a negative error code on failure.
909 */
xe_guc_relay_process_guc2pf(struct xe_guc_relay * relay,const u32 * msg,u32 len)910 int xe_guc_relay_process_guc2pf(struct xe_guc_relay *relay, const u32 *msg, u32 len)
911 {
912 u32 origin, rid;
913 int err;
914
915 relay_assert(relay, len >= GUC_HXG_EVENT_MSG_MIN_LEN);
916 relay_assert(relay, FIELD_GET(GUC_HXG_MSG_0_ORIGIN, msg[0]) == GUC_HXG_ORIGIN_GUC);
917 relay_assert(relay, FIELD_GET(GUC_HXG_MSG_0_TYPE, msg[0]) == GUC_HXG_TYPE_EVENT);
918 relay_assert(relay, FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, msg[0]) ==
919 XE_GUC_ACTION_GUC2PF_RELAY_FROM_VF);
920
921 if (unlikely(!IS_SRIOV_PF(relay_to_xe(relay)) && !kunit_get_current_test()))
922 return -EPERM;
923
924 if (unlikely(!relay_is_ready(relay)))
925 return -ENODEV;
926
927 if (unlikely(len < GUC2PF_RELAY_FROM_VF_EVENT_MSG_MIN_LEN))
928 return -EPROTO;
929
930 if (unlikely(len > GUC2PF_RELAY_FROM_VF_EVENT_MSG_MAX_LEN))
931 return -EMSGSIZE;
932
933 if (unlikely(FIELD_GET(GUC_HXG_EVENT_MSG_0_DATA0, msg[0])))
934 return -EPFNOSUPPORT;
935
936 origin = FIELD_GET(GUC2PF_RELAY_FROM_VF_EVENT_MSG_1_VFID, msg[1]);
937 rid = FIELD_GET(GUC2PF_RELAY_FROM_VF_EVENT_MSG_2_RELAY_ID, msg[2]);
938
939 if (unlikely(origin > relay_get_totalvfs(relay)))
940 return -ENOENT;
941
942 err = relay_process_msg(relay, origin, rid,
943 msg + GUC2PF_RELAY_FROM_VF_EVENT_MSG_MIN_LEN,
944 len - GUC2PF_RELAY_FROM_VF_EVENT_MSG_MIN_LEN);
945
946 return err;
947 }
948 #endif
949
950 #if IS_BUILTIN(CONFIG_DRM_XE_KUNIT_TEST)
951 #include "tests/xe_guc_relay_test.c"
952 #endif
953