1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
24 * Copyright 2017 Joyent, Inc.
25 */
26
27 /*
28 * sol_uverbs_event.c
29 *
30 * OFED User Verbs Kernel Async Event funtions
31 *
32 */
33 #include <sys/file.h>
34 #include <sys/fcntl.h>
35 #include <sys/vfs.h>
36 #include <sys/errno.h>
37 #include <sys/cred.h>
38 #include <sys/uio.h>
39 #include <sys/semaphore.h>
40 #include <sys/ddi.h>
41 #include <sys/sunddi.h>
42
43 #include <sys/ib/ibtl/ibvti.h>
44 #include <sys/ib/clients/of/ofa_solaris.h>
45 #include <sys/ib/clients/of/sol_ofs/sol_ofs_common.h>
46 #include <sys/ib/clients/of/ofed_kernel.h>
47 #include <sys/ib/clients/of/sol_uverbs/sol_uverbs.h>
48 #include <sys/ib/clients/of/sol_uverbs/sol_uverbs_event.h>
49
50 extern char *sol_uverbs_dbg_str;
51
52 static void
53 uverbs_async_event_common(uverbs_uctxt_uobj_t *, uint64_t, uint32_t,
54 llist_head_t *, uint32_t *);
55
56 /*
57 * Function:
58 * sol_uverbs_event_file_close
59 * Input:
60 * ufile - Pointer to the event ufile
61 *
62 * Output:
63 * None
64 * Returns:
65 * Zero on success, else error code.
66 * Description:
67 * Called when all kernel references to the event file have been
68 * removed and the kernel (asynchronous) or user library (completion)
69 * have closed the file.
70 */
71 void
sol_uverbs_event_file_close(uverbs_ufile_uobj_t * ufile)72 sol_uverbs_event_file_close(uverbs_ufile_uobj_t *ufile)
73 {
74 if (!ufile) {
75 SOL_OFS_DPRINTF_L3(sol_uverbs_dbg_str,
76 "UFILE CLOSE: Ufile NULL\n");
77 return;
78 }
79
80 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
81 "UFILE CLOSE: Is async? %s",
82 ufile->is_async ? "yes" : "no");
83
84 /*
85 * Remove the user file from the user object table and
86 * releases appropriate references. The object resources
87 * are freed when it is no longer referenced.
88 *
89 * If sol_ofs_uobj_remove() returns NULL then the obj was already
90 * removed.
91 */
92 rw_enter(&(ufile->uobj.uo_lock), RW_WRITER);
93 if (sol_ofs_uobj_remove(&uverbs_ufile_uo_tbl, &ufile->uobj)) {
94 rw_exit(&(ufile->uobj.uo_lock));
95 sol_ofs_uobj_deref(&ufile->uobj, uverbs_release_event_file);
96 } else {
97 rw_exit(&(ufile->uobj.uo_lock));
98 }
99 }
100
101 /*
102 * Function:
103 * sol_uverbs_event_file_read
104 * Input:
105 * ufile - The user file pointer of the event channel.
106 * uiop - The user I/O pointer in which to place the event.
107 * cred - Pointer to the callers credentials.
108 * Output:
109 * uiop - Upon success the caller's buffer has been updated with
110 * the event details.
111 * Returns:
112 * Zero on success, else error code.
113 * EAGAIN - No event available and caller is non-
114 * blocking.
115 * ERESTART- A signal was received.
116 * EINVAL - Caller parameter/user buffer invalid.
117 * EFAULT - Failure copying data to user.
118 * Description:
119 * Perfrom a blocking read to retrieve an event (asynchronous or
120 * completion) for the user file specified. If an event is available it
121 * is immediately returned. If an event is not available, then the
122 * caller will block unless the open flags indicate it is a non-
123 * blocking open. If the caller does block it does so interruptable
124 * and therefore will return upon an event or receipt of a signal.
125 */
126 /* ARGSUSED */
127 int
sol_uverbs_event_file_read(uverbs_ufile_uobj_t * ufile,struct uio * uiop,cred_t * cred)128 sol_uverbs_event_file_read(uverbs_ufile_uobj_t *ufile, struct uio *uiop,
129 cred_t *cred)
130 {
131 int rc = 0;
132 uverbs_event_t *evt;
133 llist_head_t *entry;
134 int eventsz;
135 int ioflag;
136
137 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "event_file_read(%p), "
138 "ufile = %p, is_async =%d, uio_resid=%d",
139 ufile, ufile->is_async, uiop->uio_resid);
140
141 ioflag = uiop->uio_fmode & (FNONBLOCK | FNDELAY);
142
143 mutex_enter(&ufile->lock);
144
145 /*
146 * If Event list not empty and CQ event notification is disabled
147 * by sol_ucma, do not return events. Either return EAGAIN (if
148 * flag is O_NONBLOCK, or wait using cv_wait_sig().
149 */
150 if (ufile->ufile_notify_enabled == SOL_UVERBS2UCMA_CQ_NOTIFY_DISABLE &&
151 llist_empty(&ufile->event_list) != 0) {
152 if (ioflag) {
153 mutex_exit(&ufile->lock);
154 SOL_OFS_DPRINTF_L3(sol_uverbs_dbg_str,
155 "event_file_read - notify disabled, no block");
156 return (EAGAIN);
157 }
158
159 if (!cv_wait_sig(&ufile->poll_wait, &ufile->lock)) {
160 mutex_exit(&ufile->lock);
161 SOL_OFS_DPRINTF_L3(sol_uverbs_dbg_str,
162 "event_file_read - sig_wakeup");
163 return (ERESTART);
164 }
165 }
166
167 while (llist_empty(&ufile->event_list)) {
168 if (ioflag) {
169 mutex_exit(&ufile->lock);
170 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
171 "event_file_read - no events, no block");
172 return (EAGAIN);
173 }
174
175 if (!cv_wait_sig(&ufile->poll_wait, &ufile->lock)) {
176 mutex_exit(&ufile->lock);
177 SOL_OFS_DPRINTF_L3(sol_uverbs_dbg_str,
178 "event_file_read - sig_wakeup");
179 return (ERESTART);
180 }
181 }
182
183 entry = ufile->event_list.nxt;
184 evt = entry->ptr;
185
186 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "event_file_read: "
187 "Event entry found: entry:%p, event:%p, evt_list %p",
188 entry, evt, &evt->ev_list);
189
190 if (ufile->is_async) {
191 eventsz = sizeof (struct ib_uverbs_async_event_desc);
192 } else {
193 eventsz = sizeof (struct ib_uverbs_comp_event_desc);
194 }
195
196 if (eventsz > uiop->uio_resid) {
197 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
198 "event_file_read - Event too big");
199 rc = EINVAL;
200 evt = NULL;
201 } else {
202 llist_del(ufile->event_list.nxt);
203 if (evt->ev_counter) {
204 ++(*evt->ev_counter);
205 llist_del(&evt->ev_obj_list);
206 }
207 }
208
209 mutex_exit(&ufile->lock);
210
211 if (evt && (uiomove(evt, eventsz, UIO_READ, uiop) != 0)) {
212 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
213 "EVENT FILE READ: Error writing ev");
214 rc = EFAULT;
215 }
216
217 if (evt) {
218 kmem_free(evt, sizeof (*evt));
219 }
220
221 return (rc);
222 }
223
224 /*
225 * Function:
226 * sol_uverbs_event_file_poll
227 * Input:
228 * ufile - user file for desired completion channel event file
229 * events - The events that may occur.
230 * anyyet - A flag that is non-zero if any files in the set
231 * of descriptors has an event waiting.
232 * ct - Pointer to the callers context.
233 * Output:
234 * reventssp - A pointer updated to return a bitmask of events specified.
235 * phpp - A pointer to a pollhead pointer, updated to reflect the
236 * the event file's pollhead used for synchronization.
237 * Returns:
238 * Zero on success, else error code.
239 * EINVAL - Vnode does not point to valid event file.
240 * Description:
241 * Support for event channel polling interface, allows use of completion
242 * channel in asynchronous type environment. If events may be read
243 * without blocking indicate a POLLIN | POLLRDNORM event; otherwise if
244 * no other descriptors in the set have data waiting, set the pollhead
245 * pointer to our the associated completion event file's pollhead.
246 */
247 int
sol_uverbs_event_file_poll(uverbs_ufile_uobj_t * ufile,short events,int anyyet,short * reventsp,pollhead_t ** phpp)248 sol_uverbs_event_file_poll(uverbs_ufile_uobj_t *ufile, short events,
249 int anyyet, short *reventsp, pollhead_t **phpp)
250 {
251 short revent = 0;
252
253 #ifdef DEBUG
254 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "event_file_poll(%p, %x)",
255 ufile, events);
256 #endif
257
258 if (!ufile) {
259 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, "event_file_poll ",
260 "ufile %p", ufile);
261 return (EINVAL);
262 }
263
264 #ifdef DEBUG
265 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "event_file_poll "
266 "ufile = %p, is_async =%d", ufile, ufile->is_async);
267 #endif
268
269 mutex_enter(&ufile->lock);
270
271 /*
272 * If poll request and event is ready.
273 */
274 if ((events & (POLLIN | POLLRDNORM)) &&
275 !llist_empty(&ufile->event_list)) {
276
277 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "event_file_poll "
278 "Event entry available");
279
280 revent |= POLLIN | POLLRDNORM;
281 }
282
283 /*
284 * If we didn't get an event or are edge-triggered
285 */
286 if ((revent == 0 && !anyyet) || (events & POLLET)) {
287 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "event_file_poll "
288 "Event entry NOT available");
289
290 *phpp = &ufile->poll_head;
291 }
292
293 mutex_exit(&ufile->lock);
294 *reventsp = revent;
295 return (0);
296 }
297
298 /*
299 * Function:
300 * uverbs_alloc_event_file
301 * Input:
302 * uctxt - The Solaris User Verbs user context associated with the
303 * event channel.
304 * is_async - Indicates the file is for asynchronous events if non-zero;
305 * other wise it is for completion events.
306 * Output:
307 * None.
308 * Returns:
309 * New user verb event file object or NULL on error.
310 * Description:
311 * Allocate an asynchronous or completion event file
312 */
313 uverbs_ufile_uobj_t *
uverbs_alloc_event_file(uverbs_uctxt_uobj_t * uctxt,int is_async)314 uverbs_alloc_event_file(uverbs_uctxt_uobj_t *uctxt, int is_async)
315 {
316 uverbs_ufile_uobj_t *ufile;
317
318 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "alloc_event_file(%p, %x)",
319 uctxt, is_async);
320
321 ufile = kmem_zalloc(sizeof (*ufile), KM_NOSLEEP);
322 if (!ufile) {
323 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
324 "alloc_event_file: mem alloc fail");
325 return (NULL);
326 }
327 ufile->ufile_notify_enabled = SOL_UVERBS2UCMA_CQ_NOTIFY_ENABLE;
328 sol_ofs_uobj_init(&ufile->uobj, 0, SOL_UVERBS_UFILE_UOBJ_TYPE);
329 rw_enter(&ufile->uobj.uo_lock, RW_WRITER);
330
331 if (sol_ofs_uobj_add(&uverbs_ufile_uo_tbl, &ufile->uobj) != 0) {
332 /*
333 * The initialization routine set's the initial reference,
334 * we dereference the object here to clean it up.
335 */
336 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
337 "ALLOC UFILE: Object add failed");
338 rw_exit(&ufile->uobj.uo_lock);
339 ufile->uobj.uo_uobj_sz = sizeof (uverbs_ufile_uobj_t);
340 sol_ofs_uobj_deref(&ufile->uobj, sol_ofs_uobj_free);
341 return (NULL);
342 }
343
344 ufile->is_async = is_async ? 1 : 0;
345 llist_head_init(&ufile->event_list, NULL);
346
347 mutex_init(&ufile->lock, NULL, MUTEX_DRIVER, NULL);
348 cv_init(&ufile->poll_wait, NULL, CV_DRIVER, NULL);
349
350 ufile->uctxt = uctxt;
351 ufile->uobj.uo_live = 1;
352 rw_exit(&ufile->uobj.uo_lock);
353 return (ufile);
354 }
355
356 /*
357 * Function:
358 * uverbs_release_event_file
359 * Input:
360 * ufile - Pointer to the ufile user object that is being freed.
361 * Output:
362 * None.
363 * Returns:
364 * None.
365 * Description:
366 * Release/destroy event file resources before freeing memory. This
367 * routine should only be used if the event file is successfully
368 * created.
369 */
370 void
uverbs_release_event_file(sol_ofs_uobj_t * uobj)371 uverbs_release_event_file(sol_ofs_uobj_t *uobj)
372 {
373 uverbs_ufile_uobj_t *ufile = (uverbs_ufile_uobj_t *)uobj;
374 uverbs_event_t *evt;
375 llist_head_t *entry;
376 llist_head_t *list_tmp;
377
378 if (!ufile) {
379 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
380 "UFILE RELEASE: Ufile NULL\n");
381 return;
382 }
383
384 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
385 "UFILE RELEASE: Event file=%p, is async = %s",
386 ufile, ufile->is_async ? "yes" : "no");
387
388 /*
389 * Release any events still queued to the event file.
390 */
391 mutex_enter(&ufile->lock);
392
393 entry = ufile->event_list.nxt;
394 list_tmp = entry->nxt;
395 while (entry != &ufile->event_list) {
396 ASSERT(entry);
397 evt = (uverbs_event_t *)entry->ptr;
398
399 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
400 "UFILE RELEASE: Deleting event %p on event file %p",
401 evt, ufile);
402
403 llist_del(&evt->ev_list);
404 kmem_free(evt, sizeof (*evt));
405 entry = list_tmp;
406 list_tmp = entry->nxt;
407 }
408
409 mutex_exit(&ufile->lock);
410
411 pollhead_clean(&ufile->poll_head);
412 cv_destroy(&ufile->poll_wait);
413 mutex_destroy(&ufile->lock);
414 sol_ofs_uobj_free(uobj);
415 }
416
417 /*
418 * Function:
419 * uverbs_ibt_to_ofa_event_code
420 * Input:
421 * code - The OFA event code.
422 * Output:
423 * The OFED event code.
424 * Returns:
425 * Returns the OFA equivalent of an IBT Asynchronous Event code, -1 if
426 * a valid translation does not exist.
427 * Description:
428 * Map an IBT asynchronous event code to an OFED event code.
429 */
430 enum ib_event_type
uverbs_ibt_to_ofa_event_code(ibt_async_code_t code)431 uverbs_ibt_to_ofa_event_code(ibt_async_code_t code)
432 {
433 enum ib_event_type ofa_code;
434
435 switch (code) {
436 case IBT_EVENT_PATH_MIGRATED:
437 ofa_code = IB_EVENT_PATH_MIG;
438 break;
439
440 case IBT_EVENT_SQD:
441 ofa_code = IB_EVENT_SQ_DRAINED;
442 break;
443
444 case IBT_EVENT_COM_EST:
445 ofa_code = IB_EVENT_COMM_EST;
446 break;
447
448 case IBT_ERROR_CATASTROPHIC_CHAN:
449 case IBT_ERROR_LOCAL_CATASTROPHIC:
450 ofa_code = IB_EVENT_QP_FATAL;
451 break;
452
453 case IBT_ERROR_INVALID_REQUEST_CHAN:
454 ofa_code = IB_EVENT_QP_REQ_ERR;
455 break;
456
457 case IBT_ERROR_ACCESS_VIOLATION_CHAN:
458 ofa_code = IB_EVENT_QP_ACCESS_ERR;
459 break;
460
461 case IBT_ERROR_PATH_MIGRATE_REQ:
462 ofa_code = IB_EVENT_PATH_MIG_ERR;
463 break;
464
465 case IBT_ERROR_CQ:
466 ofa_code = IB_EVENT_CQ_ERR;
467 break;
468
469 case IBT_EVENT_PORT_UP:
470 ofa_code = IB_EVENT_PORT_ACTIVE;
471 break;
472
473 case IBT_ERROR_PORT_DOWN:
474 ofa_code = IB_EVENT_PORT_ERR;
475 break;
476
477 case IBT_HCA_ATTACH_EVENT:
478 ofa_code = IB_EVENT_CLIENT_REREGISTER;
479 break;
480
481 case IBT_EVENT_LIMIT_REACHED_SRQ:
482 ofa_code = IB_EVENT_SRQ_LIMIT_REACHED;
483 break;
484
485 case IBT_ERROR_CATASTROPHIC_SRQ:
486 ofa_code = IB_EVENT_SRQ_ERR;
487 break;
488
489 case IBT_EVENT_EMPTY_CHAN:
490 ofa_code = IB_EVENT_QP_LAST_WQE_REACHED;
491 break;
492
493 /*
494 * No mapping exists.
495 */
496 case IBT_ASYNC_OPAQUE1:
497 case IBT_ASYNC_OPAQUE2:
498 case IBT_ASYNC_OPAQUE3:
499 case IBT_ASYNC_OPAQUE4:
500 case IBT_HCA_DETACH_EVENT:
501 default:
502 ofa_code = -1;
503 }
504
505 return (ofa_code);
506 }
507
508 /*
509 * Function:
510 * uverbs_async_qp_event_handler
511 * Input:
512 * clnt_private - The IBT attach client handle.
513 * hca_hdl - The IBT hca handle associated with the notification.
514 * code - The OFED event identifier.
515 * event - The IBT event.
516 * Output:
517 * None
518 * Returns:
519 * None
520 * Description:
521 * Handle QP affiliated asynchronous event noficiation.
522 */
523 /* ARGSUSED */
524 void
uverbs_async_qp_event_handler(void * clnt_private,ibt_hca_hdl_t hca_hdl,enum ib_event_type code,ibt_async_event_t * event)525 uverbs_async_qp_event_handler(void *clnt_private, ibt_hca_hdl_t hca_hdl,
526 enum ib_event_type code, ibt_async_event_t *event)
527 {
528 uverbs_uqp_uobj_t *uqp;
529
530 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
531 "async_qp_event_handler()");
532
533 if (event->ev_chan_hdl == NULL) {
534 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
535 "async_qp_event_handler: event handle NULL");
536 return;
537 }
538 uqp = ibt_get_qp_private(event->ev_chan_hdl);
539 ASSERT(uqp);
540 if (uqp->uqp_free_state == SOL_UVERBS2UCMA_FREE_PENDING) {
541 SOL_OFS_DPRINTF_L3(sol_uverbs_dbg_str,
542 "async_qp_event_handler: User QP context has been freed");
543 return;
544 }
545 if (uqp->qp != event->ev_chan_hdl) {
546 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
547 "async_qp_event_handler: QP handle mismatch");
548 return;
549 }
550
551 uverbs_async_event_common(uqp->uctxt, uqp->uobj.uo_user_handle,
552 code, &uqp->async_list, &uqp->async_events_reported);
553
554 }
555
556 /*
557 * Function:
558 * uverbs_async_cq_event_handler
559 * Input:
560 * clnt_private - The IBT attach client handle.
561 * hca_hdl - The IBT hca handle associated with the notification.
562 * code - The OFED event identifier.
563 * event - The IBT event.
564 * Output:
565 * None
566 * Returns:
567 * None
568 * Description:
569 * Handle a CQ affiliated asynchronous event notification.
570 */
571 /* ARGSUSED */
572 void
uverbs_async_cq_event_handler(void * clnt_private,ibt_hca_hdl_t hca_hdl,enum ib_event_type code,ibt_async_event_t * event)573 uverbs_async_cq_event_handler(void *clnt_private, ibt_hca_hdl_t hca_hdl,
574 enum ib_event_type code, ibt_async_event_t *event)
575 {
576 uverbs_ucq_uobj_t *ucq;
577
578 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
579 "ASYNC CQ EVENT HANDLER:");
580
581 if (event->ev_cq_hdl == NULL) {
582 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
583 "ASYNC CQ EVENT HANDLER: event handle is NULL");
584 return;
585 }
586
587 ucq = ibt_get_cq_private(event->ev_cq_hdl);
588 if (ucq->cq != event->ev_cq_hdl) {
589 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
590 "ASYNC CQ EVENT HANDLER: CQ handle mismatch");
591 return;
592 }
593
594 uverbs_async_event_common(ucq->uctxt, ucq->uobj.uo_user_handle,
595 code, &ucq->async_list, &ucq->async_events_reported);
596 }
597
598 /*
599 * Function:
600 * uverbs_async_srq_event_handler
601 * Input:
602 * clnt_private - The IBT attach client handle.
603 * hca_hdl - The IBT hca handle associated with the notification.
604 * code - The OFED event identifier.
605 * event - The IBT event.
606 * Output:
607 * None
608 * Returns:
609 * None
610 * Description:
611 * Handle a shared receive queue asynchronous event notification.
612 */
613 /* ARGSUSED */
614 void
uverbs_async_srq_event_handler(void * clnt_private,ibt_hca_hdl_t hca_hdl,enum ib_event_type code,ibt_async_event_t * event)615 uverbs_async_srq_event_handler(void *clnt_private, ibt_hca_hdl_t hca_hdl,
616 enum ib_event_type code, ibt_async_event_t *event)
617 {
618 uverbs_usrq_uobj_t *usrq;
619
620 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
621 "ASYNC SRQ EVENT HANDLER:");
622
623 if (event->ev_srq_hdl == NULL) {
624 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
625 "ASYNC SRQ EVENT HANDLER: event handle is NULL");
626 return;
627 }
628
629 usrq = ibt_get_srq_private(event->ev_srq_hdl);
630 if (usrq->srq != event->ev_srq_hdl) {
631 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
632 "ASYNC SRQ EVENT HANDLER: SRQ handle mismatch");
633 return;
634 }
635
636 uverbs_async_event_common(usrq->uctxt, usrq->uobj.uo_user_handle,
637 code, &usrq->async_list, &usrq->async_events_reported);
638 }
639
640 /*
641 * Function:
642 * uverbs_async_unaff_event_handler
643 * Input:
644 * clnt_private - The IBT attach client handle.
645 * hca_hdl - The IBT hca handle associated with the notification.
646 * code - The OFED event identifier.
647 * event - The IBT event.
648 * Output:
649 * None
650 * Returns:
651 * None
652 * Description:
653 * Handle an unaffiliated asynchronous event notification.
654 */
655 /* ARGSUSED */
656 void
uverbs_async_unaff_event_handler(void * clnt_private,ibt_hca_hdl_t hca_hdl,enum ib_event_type code,ibt_async_event_t * event)657 uverbs_async_unaff_event_handler(void *clnt_private, ibt_hca_hdl_t hca_hdl,
658 enum ib_event_type code, ibt_async_event_t *event)
659 {
660 sol_ofs_uobj_table_t *uo_tbl = &uverbs_uctxt_uo_tbl;
661 sol_ofs_uobj_blk_t *blk;
662 uverbs_uctxt_uobj_t *uctxt;
663 int i, j;
664
665 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
666 "ASYNC UNAFF EVENT HANDLER:");
667
668 /*
669 * Unaffiliated events are returned at the IBT client level. We must
670 * return the event to all user context allocated for the specific
671 * HCA device specified.
672 */
673 rw_enter(&uo_tbl->uobj_tbl_lock, RW_READER);
674
675 for (i = 0; i < uo_tbl->uobj_tbl_used_blks; i++) {
676 blk = uo_tbl->uobj_tbl_uo_root[i];
677 if (blk == NULL) {
678 continue;
679 }
680 for (j = 0; j < SOL_OFS_UO_BLKSZ; j++) {
681 uctxt = (uverbs_uctxt_uobj_t *)blk->ofs_uoblk_blks[j];
682 if (uctxt == NULL) {
683 continue;
684 }
685 /*
686 * OK, check to see if this user context belongs
687 * to the idicated hca.
688 */
689 if (uctxt->hca->hdl == hca_hdl && uctxt->async_evfile) {
690 uverbs_async_event_common(uctxt,
691 event->ev_port, code, NULL, NULL);
692 }
693 }
694 }
695 rw_exit(&uo_tbl->uobj_tbl_lock);
696 }
697
698 /*
699 * Function:
700 * uverbs_async_event_handler
701 * Input:
702 * clnt_private - The IBT attach client handle.
703 * hca_hdl - The IBT hca handle associated with the notification.
704 * code - The OFED event identifier.
705 * event - The IBT event.
706 * Output:
707 * None
708 * Returns:
709 * None
710 * Description:
711 * Main IBT asynchronous event handler registered at ibt_attach.
712 * Convert to OFA event type and forward to the appropriate
713 * asynchronous handler.
714 */
715 void
uverbs_async_event_handler(void * clnt_private,ibt_hca_hdl_t hca_hdl,ibt_async_code_t code,ibt_async_event_t * event)716 uverbs_async_event_handler(void *clnt_private, ibt_hca_hdl_t hca_hdl,
717 ibt_async_code_t code, ibt_async_event_t *event)
718 {
719 enum ib_event_type ofa_type;
720 sol_uverbs_ib_event_handler_t *handler;
721 llist_head_t *entry;
722 sol_uverbs_hca_t *hca;
723
724 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
725 "ASYNNC EVENT HANDLER: entry, code=%d", code);
726
727 ofa_type = uverbs_ibt_to_ofa_event_code(code);
728
729 if ((int)ofa_type < 0) {
730 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
731 "ASYNC EVENT HANDLER:Event %d did not map to OFA "
732 "Event", code);
733 return;
734 }
735
736 switch (ofa_type) {
737 case IB_EVENT_QP_FATAL:
738 case IB_EVENT_QP_REQ_ERR:
739 case IB_EVENT_QP_ACCESS_ERR:
740 case IB_EVENT_QP_LAST_WQE_REACHED:
741 case IB_EVENT_SQ_DRAINED:
742 case IB_EVENT_PATH_MIG:
743 /*
744 * These events are related with a QP
745 */
746 uverbs_async_qp_event_handler(clnt_private, hca_hdl,
747 ofa_type, event);
748 break;
749
750 case IB_EVENT_CQ_ERR:
751 /*
752 * These events are related with a CQ
753 */
754 uverbs_async_cq_event_handler(clnt_private, hca_hdl,
755 ofa_type, event);
756 break;
757
758 case IB_EVENT_SRQ_ERR:
759 case IB_EVENT_SRQ_LIMIT_REACHED:
760 /*
761 * These events are related with a SRQ
762 */
763 uverbs_async_srq_event_handler(clnt_private, hca_hdl,
764 ofa_type, event);
765 break;
766
767
768 case IB_EVENT_PORT_ERR:
769 case IB_EVENT_PORT_ACTIVE:
770 case IB_EVENT_LID_CHANGE:
771 case IB_EVENT_PKEY_CHANGE:
772 case IB_EVENT_SM_CHANGE:
773 case IB_EVENT_CLIENT_REREGISTER:
774 case IB_EVENT_DEVICE_FATAL:
775 case IB_EVENT_PATH_MIG_ERR:
776 /*
777 * Unaffiliated asynchronous notifications.
778 */
779 uverbs_async_unaff_event_handler(clnt_private, hca_hdl,
780 ofa_type, event);
781 break;
782
783 default:
784 break;
785 }
786
787 /*
788 * Give other kernel agents a notification.
789 */
790 hca = sol_uverbs_ibt_hdl_to_hca(hca_hdl);
791 if (hca) {
792 mutex_enter(&hca->event_handler_lock);
793 list_for_each(entry, &hca->event_handler_list) {
794 handler = (sol_uverbs_ib_event_handler_t *)entry->ptr;
795
796 ASSERT(handler != NULL);
797 handler->handler(handler, hca_hdl, code, event);
798 }
799 mutex_exit(&hca->event_handler_lock);
800 }
801 }
802
803 /*
804 * Function:
805 * uverbs_async_event_common
806 * Input:
807 * uctxt - Pointer to the user context associated with the
808 * affiliated event.
809 * element - The users handle to the associated object.
810 * event - The event type.
811 * uobj_list - The list to enqueue the asynchronous event.
812 * counter - The counter to track the event delivery.
813 * Output:
814 * None
815 * Returns:
816 * None
817 * Description:
818 * Create an asyncronous event and enqueue it on the specified list.
819 * Then wake callers that may be blocked on the list.
820 */
821 static void
uverbs_async_event_common(uverbs_uctxt_uobj_t * uctxt,uint64_t element,uint32_t event,llist_head_t * obj_list,uint32_t * counter)822 uverbs_async_event_common(uverbs_uctxt_uobj_t *uctxt, uint64_t element,
823 uint32_t event, llist_head_t *obj_list, uint32_t *counter)
824 {
825 uverbs_ufile_uobj_t *ufile = uctxt->async_evfile;
826 uverbs_event_t *entry;
827
828 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "async_event_common(%p, "
829 "%llx, %llx, %p, %p)", uctxt, element, event, obj_list, counter);
830
831 if (!ufile) {
832 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, "async_event_common "
833 "ufile %p", ufile);
834 return;
835 }
836
837 mutex_enter(&ufile->lock);
838 entry = kmem_zalloc(sizeof (*entry), KM_NOSLEEP);
839 if (!entry) {
840 mutex_exit(&ufile->lock);
841 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, "async_event_common "
842 "kmem_zalloc failed");
843 return;
844 }
845
846 entry->ev_desc.async.element = element;
847 entry->ev_desc.async.event_type = event;
848 entry->ev_counter = counter;
849
850 llist_head_init(&entry->ev_list, entry);
851 llist_head_init(&entry->ev_obj_list, entry);
852
853 llist_add_tail(&entry->ev_list, &ufile->event_list);
854
855 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "async_event_common "
856 "adding ASYNC entry-ev_list=%p, entry %p",
857 &entry->ev_list, entry);
858
859 if (obj_list) {
860 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "async_event_common "
861 "adding ASYNC entry-ev_obj_list=%p, entry=%p",
862 &entry->ev_obj_list, entry);
863 llist_add_tail(&entry->ev_obj_list, obj_list);
864 }
865
866 mutex_exit(&ufile->lock);
867 cv_signal(&ufile->poll_wait);
868 pollwakeup(&ufile->poll_head, POLLIN | POLLRDNORM);
869 }
870
871 /*
872 * Function:
873 * uverbs_release_ucq_channel
874 * Input:
875 * uctxt - A pointer to the callers user context.
876 * ufile - A pointer to the event file associated with a CQ.
877 * ucq - A pointer to the user CQ object.
878 * Output:
879 * None
880 * Returns:
881 * None
882 * Description:
883 * Release any completion and asynchronous events that may
884 * be queued to the specified completion channel/UCQ but not
885 * yet reaped.
886 */
887 void
uverbs_release_ucq_channel(uverbs_uctxt_uobj_t * uctxt,uverbs_ufile_uobj_t * ufile,uverbs_ucq_uobj_t * ucq)888 uverbs_release_ucq_channel(uverbs_uctxt_uobj_t *uctxt,
889 uverbs_ufile_uobj_t *ufile, uverbs_ucq_uobj_t *ucq)
890 {
891 uverbs_event_t *evt;
892 llist_head_t *entry;
893 llist_head_t *list_tmp;
894
895 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
896 "RELEASE UCQ CHANNEL: uctxt=%p, ufile=%p, ucq=%p",
897 uctxt, ufile, ucq);
898
899 /*
900 * Release completion events that have been queued on the CQ completion
901 * eventlist.
902 */
903 if (ufile) {
904 rw_enter(&ufile->uobj.uo_lock, RW_WRITER);
905 ufile->ufile_cq_cnt--;
906 if (ufile->ufile_cq_cnt) {
907 rw_exit(&ufile->uobj.uo_lock);
908 uverbs_release_ucq_uevents(ufile, ucq);
909 return;
910 }
911 rw_exit(&ufile->uobj.uo_lock);
912 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
913 "release_ucq_chan : comp_list %p, prv %p, nxt %p",
914 &ucq->comp_list, ucq->comp_list.prv,
915 ucq->comp_list.nxt);
916 mutex_enter(&ufile->lock);
917
918 entry = ucq->comp_list.nxt;
919 list_tmp = entry->nxt;
920 while (entry != &ucq->comp_list) {
921 ASSERT(entry);
922 evt = (uverbs_event_t *)entry->ptr;
923
924 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
925 "RELEASE UCQ CHANNEL:Deleting event "
926 "on CQ comp list: %p", evt);
927 llist_del(&evt->ev_list);
928 llist_del(&evt->ev_obj_list);
929 kmem_free(evt, sizeof (*evt));
930 entry = list_tmp;
931 list_tmp = entry->nxt;
932 }
933 mutex_exit(&ufile->lock);
934
935 uverbs_release_ucq_uevents(ufile, ucq);
936 }
937
938 }
939
940 /*
941 * Function:
942 * uverbs_release_ucq_uevents
943 * Input:
944 * ufile - A pointer to the asynchronous event file associated with a QP.
945 * ucq - A pointer to the user CQ object.
946 * Output:
947 * None
948 * Returns:
949 * None
950 * Description:
951 * Free any user asynchronous events that have been queued for the
952 * user CQ object specified.
953 */
954 void
uverbs_release_ucq_uevents(uverbs_ufile_uobj_t * ufile,uverbs_ucq_uobj_t * ucq)955 uverbs_release_ucq_uevents(uverbs_ufile_uobj_t *ufile, uverbs_ucq_uobj_t *ucq)
956 {
957 uverbs_event_t *evt;
958 llist_head_t *entry;
959 llist_head_t *list_tmp;
960
961 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
962 "RELEASE UCQ ASYNC EVENTS: ufile=%p, ucq=%p", ufile, ucq);
963
964 if (ufile) {
965 mutex_enter(&ufile->lock);
966
967 entry = ucq->async_list.nxt;
968 list_tmp = entry->nxt;
969 while (entry != &ucq->async_list) {
970 ASSERT(entry);
971 evt = (uverbs_event_t *)entry->ptr;
972 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
973 "RELEASE UCQ EVENTS: Deleting event "
974 "on CQ async list: %p", evt);
975 llist_del(&evt->ev_list);
976 llist_del(&evt->ev_obj_list);
977 kmem_free(evt, sizeof (*evt));
978 entry = list_tmp;
979 list_tmp = entry->nxt;
980 }
981 mutex_exit(&ufile->lock);
982 }
983 }
984
985 /*
986 * Function:
987 * uverbs_release_uqp_uevents
988 * Input:
989 * ufile - A pointer to the asynchronous event file associated with a QP.
990 * uqp - A pointer to the user QP object.
991 * Output:
992 * None
993 * Returns:
994 * None
995 * Description:
996 * Free any user asynchronous events that have been queued for the
997 * user QP object specified.
998 */
999 void
uverbs_release_uqp_uevents(uverbs_ufile_uobj_t * ufile,uverbs_uqp_uobj_t * uqp)1000 uverbs_release_uqp_uevents(uverbs_ufile_uobj_t *ufile, uverbs_uqp_uobj_t *uqp)
1001 {
1002 uverbs_event_t *evt;
1003 llist_head_t *entry;
1004 llist_head_t *list_tmp;
1005
1006 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
1007 "RELEASE UQP EVENTS: ufile=%p, uqp=%p", ufile, uqp);
1008
1009 if (ufile) {
1010 mutex_enter(&ufile->lock);
1011 entry = uqp->async_list.nxt;
1012 list_tmp = entry->nxt;
1013 while (entry != &uqp->async_list) {
1014 ASSERT(entry);
1015 evt = (uverbs_event_t *)entry->ptr;
1016 ASSERT(evt);
1017
1018 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
1019 "RELEASE UQP EVENTS: Deleting event "
1020 "ON qp async list: %p", evt);
1021 llist_del(&evt->ev_list);
1022 llist_del(&evt->ev_obj_list);
1023 kmem_free(evt, sizeof (*evt));
1024 entry = list_tmp;
1025 list_tmp = entry->nxt;
1026 }
1027 mutex_exit(&ufile->lock);
1028 }
1029 }
1030
1031 /*
1032 * Function:
1033 * uverbs_release_usrq_uevents
1034 * Input:
1035 * ufile - A pointer to the asynchronous event file associated with a
1036 * SRQ.
1037 * uqp - A pointer to the user SRQ object.
1038 * Output:
1039 * None
1040 * Returns:
1041 * None
1042 * Description:
1043 * Free any user asynchronous events that have been queued for the
1044 * user SRQ object specified.
1045 */
1046 void
uverbs_release_usrq_uevents(uverbs_ufile_uobj_t * ufile,uverbs_usrq_uobj_t * usrq)1047 uverbs_release_usrq_uevents(uverbs_ufile_uobj_t *ufile,
1048 uverbs_usrq_uobj_t *usrq)
1049 {
1050 uverbs_event_t *evt;
1051 llist_head_t *entry;
1052 llist_head_t *list_tmp;
1053
1054 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
1055 "RELEASE USRQ EVENTS: ufile=%p, usrq=%p", ufile, usrq);
1056
1057 if (ufile) {
1058 mutex_enter(&ufile->lock);
1059
1060 entry = usrq->async_list.nxt;
1061 list_tmp = entry->nxt;
1062 while (entry != &usrq->async_list) {
1063 ASSERT(entry);
1064 evt = (uverbs_event_t *)entry->ptr;
1065 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
1066 "RELEASE SRQ EVENTS: Deleting event "
1067 "on SRQ async list: %p", evt);
1068 llist_del(&evt->ev_list);
1069 llist_del(&evt->ev_obj_list);
1070 kmem_free(evt, sizeof (*evt));
1071 entry = list_tmp;
1072 list_tmp = entry->nxt;
1073 }
1074 mutex_exit(&ufile->lock);
1075 }
1076 }
1077