1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
24 */
25 /*
26 * sol_uverbs_qp.c
27 *
28 * OFED User Verbs kernel agent QP implementation.
29 *
30 */
31 #include <sys/vfs.h>
32 #include <sys/vnode.h>
33 #include <sys/errno.h>
34 #include <sys/cred.h>
35 #include <sys/uio.h>
36 #include <sys/semaphore.h>
37 #include <sys/ddi.h>
38 #include <sys/sunddi.h>
39
40 #include <sys/ib/ibtl/ibvti.h>
41 #include <sys/ib/clients/of/ofa_solaris.h>
42 #include <sys/ib/clients/of/sol_ofs/sol_ofs_common.h>
43 #include <sys/ib/clients/of/ofed_kernel.h>
44 #include <sys/ib/clients/of/sol_uverbs/sol_uverbs.h>
45 #include <sys/ib/clients/of/sol_uverbs/sol_uverbs_qp.h>
46 #include <sys/ib/clients/of/sol_uverbs/sol_uverbs_event.h>
47
48
49 static uint32_t ibt_cep_flags2ibv(ibt_cep_flags_t);
50 static void uverbs_cq_ctrl(uverbs_ucq_uobj_t *, sol_uverbs_cq_ctrl_t);
51
52 extern char *sol_uverbs_dbg_str;
53
54 /*
55 * Multicast Element
56 */
57 typedef struct uverbs_mcast_entry {
58 llist_head_t list;
59 ibt_mcg_info_t mcg;
60 } uverbs_mcast_entry_t;
61
62 /*
63 * uverbs_qp_state_table
64 *
65 * Determine if the requested QP modify operation is valid. To maintain/ensure
66 * consistency with the semantics expected by OFED user verbs operaton, this
67 * table is used to verify a QP transition. A valid transition is one that is
68 * a legal state transition and meets the required/optional attributes
69 * associated with that transition for that QP type.
70 *
71 * Note: The OFED kern-abi (See ib_user_verbs.h) does not provide any
72 * mechanism to support queue resize, consequently the IBTA spec defined
73 * valid optional parameter to modify QP size (IB_QP_CAP) is not included
74 * in the state table.
75 */
76 typedef struct {
77 int valid;
78 enum ib_qp_attr_mask req_param[IB_QPT_RAW_ETY + 1];
79 enum ib_qp_attr_mask opt_param[IB_QPT_RAW_ETY + 1];
80 } uverbs_qp_state_tbl_t;
81
82 static uverbs_qp_state_tbl_t
83 uverbs_qp_state_table[IB_QPS_ERR + 1][IB_QPS_ERR + 1] = {
84 [IB_QPS_RESET] = {
85 [IB_QPS_RESET] = { .valid = 1 },
86 [IB_QPS_ERR] = { .valid = 1 },
87 [IB_QPS_INIT] = { .valid = 1,
88 .req_param = {
89 [IB_QPT_UD] = (IB_QP_PKEY_INDEX |
90 IB_QP_PORT | IB_QP_QKEY),
91 [IB_QPT_UC] = (IB_QP_PKEY_INDEX | IB_QP_PORT |
92 IB_QP_ACCESS_FLAGS),
93 [IB_QPT_RC] = (IB_QP_PKEY_INDEX | IB_QP_PORT |
94 IB_QP_ACCESS_FLAGS),
95 [IB_QPT_SMI] = (IB_QP_PKEY_INDEX | IB_QP_QKEY),
96 [IB_QPT_GSI] = (IB_QP_PKEY_INDEX | IB_QP_QKEY),
97 }
98 },
99 },
100 [IB_QPS_INIT] = {
101 [IB_QPS_RESET] = { .valid = 1 },
102 [IB_QPS_ERR] = { .valid = 1 },
103 [IB_QPS_INIT] = { .valid = 1,
104 .opt_param = {
105 [IB_QPT_UD] = (IB_QP_PKEY_INDEX | IB_QP_PORT |
106 IB_QP_QKEY),
107 [IB_QPT_UC] = (IB_QP_PKEY_INDEX | IB_QP_PORT |
108 IB_QP_ACCESS_FLAGS),
109 [IB_QPT_RC] = (IB_QP_PKEY_INDEX | IB_QP_PORT |
110 IB_QP_ACCESS_FLAGS),
111 [IB_QPT_SMI] = (IB_QP_PKEY_INDEX | IB_QP_QKEY),
112 [IB_QPT_GSI] = (IB_QP_PKEY_INDEX | IB_QP_QKEY),
113 }
114 },
115 [IB_QPS_RTR] = { .valid = 1,
116 .req_param = {
117 [IB_QPT_UC] = (IB_QP_AV | IB_QP_PATH_MTU |
118 IB_QP_DEST_QPN | IB_QP_RQ_PSN),
119 [IB_QPT_RC] = (IB_QP_AV | IB_QP_PATH_MTU |
120 IB_QP_DEST_QPN | IB_QP_RQ_PSN |
121 IB_QP_MAX_DEST_RD_ATOMIC |
122 IB_QP_MIN_RNR_TIMER),
123 },
124 .opt_param = {
125 [IB_QPT_UD] = (IB_QP_PKEY_INDEX | IB_QP_QKEY),
126 [IB_QPT_UC] = (IB_QP_ALT_PATH |
127 IB_QP_ACCESS_FLAGS |
128 IB_QP_PKEY_INDEX),
129 [IB_QPT_RC] = (IB_QP_ALT_PATH |
130 IB_QP_ACCESS_FLAGS |
131 IB_QP_PKEY_INDEX),
132 [IB_QPT_SMI] = (IB_QP_PKEY_INDEX | IB_QP_QKEY),
133 [IB_QPT_GSI] = (IB_QP_PKEY_INDEX | IB_QP_QKEY),
134 }
135 }
136 },
137 [IB_QPS_RTR] = {
138 [IB_QPS_RESET] = { .valid = 1 },
139 [IB_QPS_ERR] = { .valid = 1 },
140 [IB_QPS_RTS] = { .valid = 1,
141 .req_param = {
142 [IB_QPT_UD] = IB_QP_SQ_PSN,
143 [IB_QPT_UC] = IB_QP_SQ_PSN,
144 [IB_QPT_RC] = (IB_QP_TIMEOUT |
145 IB_QP_RETRY_CNT |
146 IB_QP_RNR_RETRY |
147 IB_QP_SQ_PSN |
148 IB_QP_MAX_QP_RD_ATOMIC),
149 [IB_QPT_SMI] = IB_QP_SQ_PSN,
150 [IB_QPT_GSI] = IB_QP_SQ_PSN,
151 },
152 .opt_param = {
153 [IB_QPT_UD] = (IB_QP_CUR_STATE | IB_QP_QKEY),
154 [IB_QPT_UC] = (IB_QP_CUR_STATE |
155 IB_QP_ALT_PATH |
156 IB_QP_ACCESS_FLAGS |
157 IB_QP_PATH_MIG_STATE),
158 [IB_QPT_RC] = (IB_QP_CUR_STATE |
159 IB_QP_ALT_PATH |
160 IB_QP_ACCESS_FLAGS |
161 IB_QP_MIN_RNR_TIMER |
162 IB_QP_PATH_MIG_STATE),
163 [IB_QPT_SMI] = (IB_QP_CUR_STATE | IB_QP_QKEY),
164 [IB_QPT_GSI] = (IB_QP_CUR_STATE | IB_QP_QKEY),
165 }
166 }
167 },
168 [IB_QPS_RTS] = {
169 [IB_QPS_RESET] = { .valid = 1 },
170 [IB_QPS_ERR] = { .valid = 1 },
171 [IB_QPS_RTS] = { .valid = 1,
172 .opt_param = {
173 [IB_QPT_UD] = (IB_QP_CUR_STATE | IB_QP_QKEY),
174 [IB_QPT_UC] = (IB_QP_CUR_STATE |
175 IB_QP_ACCESS_FLAGS |
176 IB_QP_ALT_PATH |
177 IB_QP_PATH_MIG_STATE),
178 [IB_QPT_RC] = (IB_QP_CUR_STATE |
179 IB_QP_ACCESS_FLAGS |
180 IB_QP_ALT_PATH |
181 IB_QP_PATH_MIG_STATE |
182 IB_QP_MIN_RNR_TIMER),
183 [IB_QPT_SMI] = (IB_QP_CUR_STATE | IB_QP_QKEY),
184 [IB_QPT_GSI] = (IB_QP_CUR_STATE | IB_QP_QKEY),
185 }
186 },
187 [IB_QPS_SQD] = { .valid = 1,
188 .opt_param = {
189 [IB_QPT_UD] = IB_QP_EN_SQD_ASYNC_NOTIFY,
190 [IB_QPT_UC] = IB_QP_EN_SQD_ASYNC_NOTIFY,
191 [IB_QPT_RC] = IB_QP_EN_SQD_ASYNC_NOTIFY,
192 [IB_QPT_SMI] = IB_QP_EN_SQD_ASYNC_NOTIFY,
193 [IB_QPT_GSI] = IB_QP_EN_SQD_ASYNC_NOTIFY
194 }
195 },
196 },
197 [IB_QPS_SQD] = {
198 [IB_QPS_RESET] = { .valid = 1 },
199 [IB_QPS_ERR] = { .valid = 1 },
200 [IB_QPS_RTS] = { .valid = 1,
201 .opt_param = {
202 [IB_QPT_UD] = (IB_QP_CUR_STATE | IB_QP_QKEY),
203 [IB_QPT_UC] = (IB_QP_CUR_STATE |
204 IB_QP_ALT_PATH |
205 IB_QP_ACCESS_FLAGS |
206 IB_QP_PATH_MIG_STATE),
207 [IB_QPT_RC] = (IB_QP_CUR_STATE |
208 IB_QP_ALT_PATH |
209 IB_QP_ACCESS_FLAGS |
210 IB_QP_MIN_RNR_TIMER |
211 IB_QP_PATH_MIG_STATE),
212 [IB_QPT_SMI] = (IB_QP_CUR_STATE | IB_QP_QKEY),
213 [IB_QPT_GSI] = (IB_QP_CUR_STATE | IB_QP_QKEY),
214 }
215 },
216 [IB_QPS_SQD] = { .valid = 1,
217 .opt_param = {
218 [IB_QPT_UD] = (IB_QP_PKEY_INDEX | IB_QP_QKEY),
219 [IB_QPT_UC] = (IB_QP_AV | IB_QP_ALT_PATH |
220 IB_QP_ACCESS_FLAGS |
221 IB_QP_PKEY_INDEX |
222 IB_QP_PATH_MIG_STATE),
223 [IB_QPT_RC] = (IB_QP_PORT | IB_QP_AV |
224 IB_QP_TIMEOUT |
225 IB_QP_RETRY_CNT |
226 IB_QP_RNR_RETRY |
227 IB_QP_MAX_QP_RD_ATOMIC |
228 IB_QP_MAX_DEST_RD_ATOMIC |
229 IB_QP_ALT_PATH |
230 IB_QP_ACCESS_FLAGS |
231 IB_QP_PKEY_INDEX |
232 IB_QP_MIN_RNR_TIMER |
233 IB_QP_PATH_MIG_STATE),
234 [IB_QPT_SMI] = (IB_QP_PKEY_INDEX | IB_QP_QKEY),
235 [IB_QPT_GSI] = (IB_QP_PKEY_INDEX | IB_QP_QKEY),
236 }
237 }
238 },
239 [IB_QPS_SQE] = {
240 [IB_QPS_RESET] = { .valid = 1 },
241 [IB_QPS_ERR] = { .valid = 1 },
242 [IB_QPS_RTS] = { .valid = 1,
243 .opt_param = {
244 [IB_QPT_UD] = (IB_QP_CUR_STATE | IB_QP_QKEY),
245 [IB_QPT_UC] = (IB_QP_CUR_STATE |
246 IB_QP_ACCESS_FLAGS),
247 [IB_QPT_SMI] = (IB_QP_CUR_STATE | IB_QP_QKEY),
248 [IB_QPT_GSI] = (IB_QP_CUR_STATE | IB_QP_QKEY),
249 }
250 }
251 },
252 [IB_QPS_ERR] = {
253 [IB_QPS_RESET] = { .valid = 1 },
254 [IB_QPS_ERR] = { .valid = 1 }
255 }
256 };
257
258 /*
259 * Function:
260 * uverbs_modify_qp_is_ok
261 * Input:
262 * cur_state - The current OFED QP state.
263 * next_state - The OFED QP state to transition to.
264 * type - The OFED QP transport type.
265 * mask - The OFED QP attribute mask for the QP transition.
266 * Output:
267 * None
268 * Returns:
269 * Returns 1 if the operation is valid; otherwise 0 for invalid
270 * operations.
271 * Description:
272 * Indicate whether the desired QP modify operation is a valid operation.
273 * To be valid, the state transition must be legal and the required and
274 * optional parameters must be valid for the transition and QP type.
275 */
276 static int
uverbs_modify_qp_is_ok(enum ib_qp_state cur_state,enum ib_qp_state next_state,enum ib_qp_type type,enum ib_qp_attr_mask * maskp)277 uverbs_modify_qp_is_ok(enum ib_qp_state cur_state,
278 enum ib_qp_state next_state, enum ib_qp_type type,
279 enum ib_qp_attr_mask *maskp)
280 {
281 enum ib_qp_attr_mask req_param, opt_param;
282 uverbs_qp_state_tbl_t *state_tblp;
283 enum ib_qp_attr_mask mask = *maskp;
284
285 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "modify_qp_is_ok"
286 "(%x, %x, %x, %x)", cur_state, next_state, type, mask);
287
288 if (cur_state < 0 || cur_state > IB_QPS_ERR ||
289 next_state < 0 || next_state > IB_QPS_ERR) {
290 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
291 "modify_qp_is_ok: bad state, cur %d, next %d",
292 cur_state, next_state);
293 return (0);
294 }
295
296 if (mask & IB_QP_CUR_STATE &&
297 cur_state != IB_QPS_RTR && cur_state != IB_QPS_RTS &&
298 cur_state != IB_QPS_SQD && cur_state != IB_QPS_SQE) {
299 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
300 "modify_qp_is_ok: cur_state %d is a bad state",
301 cur_state);
302 return (0);
303 }
304
305 state_tblp = &uverbs_qp_state_table[cur_state][next_state];
306 if (!state_tblp->valid) {
307 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, "modify_qp: "
308 "bad transition, cur = %d, next = %d", cur_state,
309 next_state);
310 return (0);
311 }
312
313 req_param = state_tblp->req_param[type];
314 opt_param = state_tblp->opt_param[type];
315
316 if ((mask & req_param) != req_param) {
317 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
318 "modify_qp_is_ok: cur %d, next %d, "
319 "missing required parms, spec = 0x%08X, "
320 "req = 0%08X", cur_state, next_state,
321 mask, req_param);
322 return (0);
323 }
324
325 if (mask & ~(req_param | opt_param | IB_QP_STATE)) {
326 SOL_OFS_DPRINTF_L3(sol_uverbs_dbg_str,
327 "modify_qp_is_ok: cur %d, next %d, "
328 "illegal optional parms, parms = 0x%08X, "
329 "illegal = 0x%08X", cur_state, next_state,
330 mask, mask & ~(req_param | opt_param | IB_QP_STATE));
331 *maskp = mask & (req_param | opt_param | IB_QP_STATE);
332 return (0);
333 }
334
335 return (1);
336 }
337
338
339 /*
340 * Function:
341 * sol_uverbs_create_qp
342 * Input:
343 * uctxt - Pointer to the callers user context.
344 * buf - Pointer to kernel buffer containing the create command.
345 * in_len - Length in bytes of input command buffer.
346 * out_len - Length in bytes of output response buffer.
347 * Output:
348 * The command output buffer is updated with command results.
349 * Returns:
350 * DDI_SUCCESS on success, else error code.
351 * Description:
352 * User verbs entry point to create a new device QP.
353 */
354 /* ARGSUSED */
355 int
sol_uverbs_create_qp(uverbs_uctxt_uobj_t * uctxt,char * buf,int in_len,int out_len)356 sol_uverbs_create_qp(uverbs_uctxt_uobj_t *uctxt, char *buf,
357 int in_len, int out_len)
358 {
359 struct ib_uverbs_create_qp cmd;
360 struct ib_uverbs_create_qp_resp resp;
361 uverbs_uqp_uobj_t *uqp;
362 ibt_qp_type_t qp_type;
363 ibt_qp_alloc_attr_t qp_attr;
364 ibt_chan_sizes_t qp_sizes;
365 int rc = 0;
366 uverbs_upd_uobj_t *upd;
367 uverbs_ucq_uobj_t *uscq;
368 uverbs_ucq_uobj_t *urcq;
369 uverbs_usrq_uobj_t *usrq = NULL;
370
371 (void) memcpy(&cmd, buf, sizeof (cmd));
372 (void) memset(&resp, 0, sizeof (resp));
373 (void) memset(&qp_attr, 0, sizeof (qp_attr));
374 (void) memset(&qp_sizes, 0, sizeof (qp_sizes));
375
376 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "create_qp(): entry");
377
378 switch (cmd.qp_type) {
379 case IB_QPT_UC:
380 qp_type = IBT_UC_RQP;
381 break;
382 case IB_QPT_UD:
383 qp_type = IBT_UD_RQP;
384 break;
385 case IB_QPT_RC:
386 qp_type = IBT_RC_RQP;
387 break;
388 default:
389 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
390 "create_qp(): Invalid qp type");
391 rc = EINVAL;
392 goto err_out;
393 }
394
395 qp_attr.qp_alloc_flags = IBT_QP_USER_MAP;
396
397 if (cmd.is_srq) {
398 qp_attr.qp_alloc_flags |= IBT_QP_USES_SRQ;
399 }
400
401 qp_attr.qp_flags = IBT_WR_SIGNALED;
402 if (cmd.sq_sig_all) {
403 qp_attr.qp_flags = IBT_ALL_SIGNALED;
404 }
405
406 uqp = kmem_zalloc(sizeof (*uqp), KM_NOSLEEP);
407 if (uqp == NULL) {
408 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
409 "create_qp(): User object allocation failed");
410 rc = ENOMEM;
411 goto err_out;
412 }
413 sol_ofs_uobj_init(&uqp->uobj, cmd.user_handle,
414 SOL_UVERBS_UQP_UOBJ_TYPE);
415 rw_enter(&uqp->uobj.uo_lock, RW_WRITER);
416 llist_head_init(&uqp->mcast_list, NULL);
417 llist_head_init(&uqp->async_list, NULL);
418
419 uqp->async_events_reported = 0;
420 uqp->uctxt = uctxt;
421 uqp->disable_qp_mod = FALSE;
422
423 if (cmd.is_srq) {
424 usrq = uverbs_uobj_get_usrq_read(cmd.srq_handle);
425 uqp->uqp_rcq_srq_valid |= SOL_UVERBS_UQP_SRQ_VALID;
426 uqp->uqp_srq_hdl = cmd.srq_handle;
427 }
428 upd = uverbs_uobj_get_upd_read(cmd.pd_handle);
429 uqp->uqp_pd_hdl = cmd.pd_handle;
430 uscq = uverbs_uobj_get_ucq_read(cmd.send_cq_handle);
431 uqp->uqp_scq_hdl = cmd.send_cq_handle;
432 uqp->uqp_rcq_hdl = cmd.recv_cq_handle;
433 if (cmd.recv_cq_handle != cmd.send_cq_handle) {
434 urcq = uverbs_uobj_get_ucq_read(cmd.recv_cq_handle);
435 uqp->uqp_rcq_srq_valid |= SOL_UVERBS_UQP_RCQ_VALID;
436 } else
437 urcq = uscq;
438
439 if (!upd || !uscq || !urcq || (cmd.is_srq && !usrq)) {
440 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
441 "create_qp(): Invalid resource handle");
442 rc = EINVAL;
443 goto err_put;
444 }
445 uqp->uqp_rcq = urcq;
446 uqp->uqp_scq = uscq;
447 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
448 "uqp %p, rcq %p. scq %p", uqp, urcq, uscq);
449
450 qp_attr.qp_pd_hdl = upd->pd;
451 if (usrq) {
452 qp_attr.qp_srq_hdl = usrq->srq;
453 }
454 qp_attr.qp_scq_hdl = uscq->cq;
455 qp_attr.qp_rcq_hdl = urcq->cq;
456 qp_attr.qp_sizes.cs_sq = cmd.max_send_wr;
457 qp_attr.qp_sizes.cs_rq = cmd.max_recv_wr;
458 qp_attr.qp_sizes.cs_sq_sgl = cmd.max_send_sge;
459 qp_attr.qp_sizes.cs_rq_sgl = cmd.max_recv_sge;
460
461 uqp->max_inline_data = cmd.max_inline_data;
462 uqp->ofa_qp_type = cmd.qp_type;
463
464 /*
465 * NOTE: We allocate the QP and leave it in the RESET state to follow
466 * usage semantics consistent with OFA verbs.
467 */
468 rc = ibt_alloc_qp(uctxt->hca->hdl, qp_type, &qp_attr, &qp_sizes,
469 &uqp->qp_num, &uqp->qp);
470 if (rc != IBT_SUCCESS) {
471 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
472 "create_qp(): Error in ibt_alloc_qp() (rc=%d)", rc);
473 rc = sol_uverbs_ibt_to_kernel_status(rc);
474 uqp->uobj.uo_uobj_sz = sizeof (uverbs_uqp_uobj_t);
475 goto err_put;
476 }
477
478 ibt_set_qp_private(uqp->qp, uqp);
479
480 /* Bump up the active_qp_cnt for CQ, SRQ & PD resources it is using */
481 upd->active_qp_cnt++;
482 uscq->active_qp_cnt++;
483 if (cmd.recv_cq_handle != cmd.send_cq_handle)
484 urcq->active_qp_cnt++;
485 if (usrq)
486 usrq->active_qp_cnt++;
487
488 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
489 "\treq cs_sq=%d, actual cs_sq=%d",
490 qp_attr.qp_sizes.cs_sq, qp_sizes.cs_sq);
491 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
492 "\treq cs_sq_sgl=%d, actual cs_sg_sgl=%d",
493 qp_attr.qp_sizes.cs_sq_sgl, qp_sizes.cs_sq_sgl);
494 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
495 "\treq cs_rq=%d, actual cs_rq=%d",
496 qp_attr.qp_sizes.cs_rq, qp_sizes.cs_rq);
497 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
498 "\treq cs_rq_sgl=%d, actual cs_rq_sgl=%d",
499 qp_attr.qp_sizes.cs_rq_sgl, qp_sizes.cs_rq_sgl);
500
501 /*
502 * Query underlying hardware for data used in mapping QP work
503 * queues back to user space, we will return this information
504 * in the user verbs command response.
505 */
506 rc = ibt_ci_data_out(uctxt->hca->hdl, IBT_CI_NO_FLAGS, IBT_HDL_CHANNEL,
507 (void *)uqp->qp, &resp.drv_out, sizeof (resp.drv_out));
508 if (rc != IBT_SUCCESS) {
509 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
510 "create_qp(): Error in ibt_ci_data_out() (rc=%d)", rc);
511 rc = EFAULT;
512 uqp->uobj.uo_uobj_sz = sizeof (uverbs_uqp_uobj_t);
513 goto err_qp_destroy;
514 }
515
516 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
517 "create_qp QP: ibt_ci_data_out:0x%016llx 0x%016llx",
518 resp.drv_out[0], resp.drv_out[1]);
519 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
520 " :0x%016llx 0x%016llx",
521 resp.drv_out[2], resp.drv_out[3]);
522 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
523 " :0x%016llx 0x%016llx",
524 resp.drv_out[4], resp.drv_out[5]);
525
526 if (sol_ofs_uobj_add(&uverbs_uqp_uo_tbl, &uqp->uobj) != 0) {
527 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
528 "create_qp(): User object add failed (rc=%d)", rc);
529 rc = ENOMEM;
530 goto err_qp_destroy;
531 }
532
533 resp.qp_handle = uqp->uobj.uo_id;
534 resp.qpn = uqp->qp_num;
535 resp.max_send_wr = qp_sizes.cs_sq;
536 resp.max_send_sge = qp_sizes.cs_sq_sgl;
537 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "create_qp() : "
538 "resp.qp_handle=0x%08x, resp.qpn=0x%08x", resp.qp_handle, resp.qpn);
539
540 /*
541 * In Solaris the receive work requests and sg entries are cleared
542 * when a SRQ is used since these values are ignored. To maintain
543 * consistency with OFED we return the requested values as is done
544 * in OFED, but these values will be ignored and SRQ valuves are
545 * used. MTHCA lib will extract the zeroed out value from the
546 * driver out data.
547 */
548 if (usrq) {
549 resp.max_recv_wr = cmd.max_recv_wr;
550 resp.max_recv_sge = cmd.max_recv_sge;
551 } else {
552 resp.max_recv_wr = qp_sizes.cs_rq;
553 resp.max_recv_sge = qp_sizes.cs_rq_sgl;
554 }
555 resp.max_inline_data = cmd.max_inline_data;
556
557 #ifdef _LP64
558 rc = copyout((void*)&resp, (void*)cmd.response.r_laddr, sizeof (resp));
559 #else
560 rc = copyout((void*)&resp, (void*)cmd.response.r_addr, sizeof (resp));
561 #endif
562 if (rc != 0) {
563 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
564 "create_qp(): Error writing resp data (rc=%d)", rc);
565 rc = EFAULT;
566 goto err_uo_delete;
567 }
568
569 mutex_enter(&uctxt->lock);
570 uqp->list_entry = add_genlist(&uctxt->qp_list, (uintptr_t)uqp,
571 (void*)uctxt);
572 mutex_exit(&uctxt->lock);
573
574 if (!uqp->list_entry) {
575 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
576 "create_qp(): Error adding uqp to qp_list\n");
577 rc = ENOMEM;
578 goto err_uo_delete;
579 }
580 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
581 "create_qp() - uqp %p", uqp);
582
583 uqp->uobj.uo_live = 1;
584
585 sol_ofs_uobj_put(&upd->uobj);
586 sol_ofs_uobj_put(&uscq->uobj);
587
588 if (urcq != uscq) {
589 sol_ofs_uobj_put(&urcq->uobj);
590 }
591 if (usrq) {
592 sol_ofs_uobj_put(&usrq->uobj);
593 }
594 rw_exit(&uqp->uobj.uo_lock);
595
596 return (DDI_SUCCESS);
597
598 err_uo_delete:
599 /*
600 * Need to set uo_live, so sol_ofs_uobj_remove() will
601 * remove the object from the object table.
602 */
603 uqp->uobj.uo_live = 1;
604 (void) sol_ofs_uobj_remove(&uverbs_uqp_uo_tbl, &uqp->uobj);
605
606 err_qp_destroy:
607 (void) ibt_free_qp(uqp->qp);
608
609 upd->active_qp_cnt--;
610 uscq->active_qp_cnt--;
611 if (cmd.recv_cq_handle != cmd.send_cq_handle)
612 urcq->active_qp_cnt--;
613 if (usrq)
614 usrq->active_qp_cnt--;
615
616 err_put:
617 if (upd) {
618 sol_ofs_uobj_put(&upd->uobj);
619 }
620 if (uscq) {
621 sol_ofs_uobj_put(&uscq->uobj);
622 }
623 if (urcq && urcq != uscq) {
624 sol_ofs_uobj_put(&urcq->uobj);
625 }
626 if (usrq) {
627 sol_ofs_uobj_put(&usrq->uobj);
628 }
629
630 rw_exit(&uqp->uobj.uo_lock);
631 sol_ofs_uobj_deref(&uqp->uobj, sol_ofs_uobj_free);
632
633 err_out:
634 return (rc);
635 }
636
637 /*
638 * Free the resources used by uqp. Return 0, if the free of all
639 * resources where succesful, return non-zero, if not.
640 *
641 * If other uQPs are holding the resources (PD, CQ, SRQ), do not
642 * free the resource, but return 0, so the free of uqp can be
643 * done. Return failure only if active_cnt cannot be decremented
644 * resource_free() fails.
645 */
646 static int
uverbs_uqp_rsrc_free(uverbs_uqp_uobj_t * uqp,uverbs_uctxt_uobj_t * uctxt)647 uverbs_uqp_rsrc_free(uverbs_uqp_uobj_t *uqp, uverbs_uctxt_uobj_t *uctxt)
648 {
649 uverbs_ucq_uobj_t *uscq = NULL, *urcq = NULL;
650 uverbs_usrq_uobj_t *usrq = NULL;
651 uverbs_upd_uobj_t *upd = NULL;
652 int ret, rc = -1;
653
654 /* Get uobj for PD, CQ & SRQ resources used. */
655 upd = uverbs_uobj_get_upd_write(uqp->uqp_pd_hdl);
656 if (upd == NULL) {
657 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
658 "uqp_rsrc_free: get_upd %d failed", uqp->uqp_pd_hdl);
659 goto err_free;
660 }
661 uscq = uverbs_uobj_get_ucq_write(uqp->uqp_scq_hdl);
662 if (uscq == NULL) {
663 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
664 "uqp_rsrc_free: get_ucq %x failed", uqp->uqp_scq_hdl);
665 goto err_free;
666 }
667 if (uqp->uqp_rcq_srq_valid & SOL_UVERBS_UQP_RCQ_VALID) {
668 urcq = uverbs_uobj_get_ucq_write(uqp->uqp_rcq_hdl);
669 if (urcq == NULL) {
670 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
671 "uqp_rsrc_free: get_ucq %x failed",
672 uqp->uqp_rcq_hdl);
673 goto err_free;
674 }
675 }
676 if (uqp->uqp_rcq_srq_valid & SOL_UVERBS_UQP_SRQ_VALID) {
677 usrq = uverbs_uobj_get_usrq_write(uqp->uqp_srq_hdl);
678 if (usrq == NULL) {
679 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
680 "uqp_rsrc_free: get_srq %x failed",
681 uqp->uqp_srq_hdl);
682 goto err_free;
683 }
684 }
685 rc = 0;
686
687 /* Decrement active_qp_cnt for resources used */
688 upd->active_qp_cnt--;
689 uscq->active_qp_cnt--;
690 if (urcq)
691 urcq->active_qp_cnt--;
692 if (usrq)
693 usrq->active_qp_cnt--;
694
695 /*
696 * Free the resources, if active_qp_cnt is 0 and userland free
697 * already been pending for the resource.
698 */
699 if (upd->active_qp_cnt == 0 && upd->free_pending) {
700 ret = uverbs_upd_free(upd, uctxt);
701 if (ret && rc == 0) {
702 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
703 "uqp_rsrc_free: upd_free failed");
704 rc = ret;
705 }
706 } else if (upd)
707 sol_ofs_uobj_put(&upd->uobj);
708 if (uscq && uscq->active_qp_cnt == 0 && uscq->free_pending) {
709 ret = uverbs_ucq_free(uscq, uctxt);
710 if (ret && rc == 0) {
711 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
712 "uqp_rsrc_free: ucq_free failed");
713 rc = ret;
714 }
715 } else if (uscq)
716 sol_ofs_uobj_put(&uscq->uobj);
717 if (urcq && urcq->active_qp_cnt == 0 && urcq->free_pending) {
718 ret = uverbs_ucq_free(urcq, uctxt);
719 if (ret && rc == 0) {
720 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
721 "uqp_rsrc_free: ucq_free failed");
722 rc = ret;
723 }
724 } else if (urcq)
725 sol_ofs_uobj_put(&urcq->uobj);
726 if (usrq && usrq->active_qp_cnt == 0 && usrq->free_pending) {
727 ret = uverbs_usrq_free(usrq, uctxt);
728 if (ret && rc == 0) {
729 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
730 "uqp_rsrc_free: usrq_free failed");
731 rc = ret;
732 }
733 } else if (usrq)
734 sol_ofs_uobj_put(&usrq->uobj);
735 return (rc);
736
737 err_free:
738 if (upd)
739 sol_ofs_uobj_put(&upd->uobj);
740 if (uscq)
741 sol_ofs_uobj_put(&uscq->uobj);
742 if (urcq)
743 sol_ofs_uobj_put(&urcq->uobj);
744
745 return (rc);
746 }
747
748 /*
749 * Free the resources held by the uqp.
750 * Call ibt_free_qp() to free the IBTF QP.
751 * Free uqp.
752 */
753 int
uverbs_uqp_free(uverbs_uqp_uobj_t * uqp,uverbs_uctxt_uobj_t * uctxt)754 uverbs_uqp_free(uverbs_uqp_uobj_t *uqp, uverbs_uctxt_uobj_t *uctxt)
755 {
756 int rc;
757 ibt_status_t status;
758
759 /* Detach Mcast entries, if any. */
760 uverbs_detach_uqp_mcast_entries(uqp);
761
762 if (!uqp->qp)
763 goto skip_ibt_free_qp;
764
765 status = ibt_free_qp(uqp->qp);
766 if (status != IBT_SUCCESS) {
767 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
768 "uqp_free: ibt_free_qp failed %d", status);
769 sol_ofs_uobj_put(&uqp->uobj);
770 return (status);
771 }
772 uqp->qp = NULL;
773
774 skip_ibt_free_qp :
775 rc = uverbs_uqp_rsrc_free(uqp, uctxt);
776 if (rc) {
777 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
778 "uqp_free: uqp_rcrc_free failed %d", rc);
779 sol_ofs_uobj_put(&uqp->uobj);
780 return (rc);
781 }
782
783 (void) sol_ofs_uobj_remove(&uverbs_uqp_uo_tbl, &uqp->uobj);
784 sol_ofs_uobj_put(&uqp->uobj);
785
786 if (uqp->list_entry) {
787 mutex_enter(&uctxt->lock);
788 delete_genlist(&uctxt->qp_list, uqp->list_entry);
789 uqp->list_entry = NULL;
790 mutex_exit(&uctxt->lock);
791 }
792
793 sol_ofs_uobj_deref(&uqp->uobj, sol_ofs_uobj_free);
794
795 if (uctxt->uctxt_free_pending && (uctxt->qp_list).count == 0) {
796 SOL_OFS_DPRINTF_L3(sol_uverbs_dbg_str,
797 "uqp_free: freeing uctxt %p", uctxt);
798 sol_ofs_uobj_deref(&uctxt->uobj, sol_ofs_uobj_free);
799 }
800 return (0);
801 }
802
803 /*
804 * Function:
805 * sol_uverbs_destroy_qp
806 * Input:
807 * uctxt - Pointer to the callers user context.
808 * buf - Pointer to kernel buffer containing the destroy command.
809 * in_len - Length in bytes of input command buffer.
810 * out_len - Length in bytes of output response buffer.
811 * Output:
812 * The command output buffer is updated with command results.
813 * Returns:
814 * DDI_SUCCESS on success, else error code.
815 * Description:
816 * User verbs entry point to destroy a device QP.
817 */
818 /* ARGSUSED */
819 int
sol_uverbs_destroy_qp(uverbs_uctxt_uobj_t * uctxt,char * buf,int in_len,int out_len)820 sol_uverbs_destroy_qp(uverbs_uctxt_uobj_t *uctxt, char *buf,
821 int in_len, int out_len)
822 {
823 struct ib_uverbs_destroy_qp cmd;
824 struct ib_uverbs_destroy_qp_resp resp;
825 uverbs_uqp_uobj_t *uqp;
826 int rc;
827
828 (void) memcpy(&cmd, buf, sizeof (cmd));
829 (void) memset(&resp, 0, sizeof (resp));
830
831 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "DESTROY QP: entry "
832 "(qp_handle=%d)", cmd.qp_handle);
833
834 uqp = uverbs_uobj_get_uqp_write(cmd.qp_handle);
835 if (uqp == NULL) {
836 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
837 "destroy_qp() : List lookup failure");
838 rc = EINVAL;
839 goto err_out;
840 }
841
842 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "DESTROY QP: qp_handle=%d, "
843 "uqp %p, qp_ptr %p", cmd.qp_handle, uqp, uqp->qp);
844
845 if (!llist_empty(&uqp->mcast_list)) {
846 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
847 "destroy_qp() called with attached MC group(s)");
848 rc = EBUSY;
849 goto err_busy;
850 }
851
852 uverbs_release_uqp_uevents(uctxt->async_evfile, uqp);
853 resp.events_reported = uqp->async_events_reported;
854
855 /*
856 * If ucma has disabled QP free for this QP, set FREE_PENDING
857 * flag so that the QP can be freed when UCMA enables QP_FREE.
858 * Call ibt_free_qp() is ucma has not disabled QP free.
859 */
860 if (uqp->uqp_free_state == SOL_UVERBS2UCMA_DISABLE_QP_FREE) {
861 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
862 "destroy_qp() - UCMA disabled");
863 uqp->uqp_free_state = SOL_UVERBS2UCMA_FREE_PENDING;
864 sol_ofs_uobj_put(&uqp->uobj);
865 rc = 0;
866 goto report_qp_evts;
867 } else {
868 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
869 "destroy_qp() - freeing QP : %p", uqp);
870 rc = uverbs_uqp_free(uqp, uctxt);
871 }
872
873 if (rc) {
874 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
875 "destroy_qp() - ibt_free_qp() fail %d", rc);
876 rc = sol_uverbs_ibt_to_kernel_status(rc);
877 goto err_out;
878 }
879
880 report_qp_evts:
881 #ifdef _LP64
882 rc = copyout((void*)&resp, (void*)cmd.response.r_laddr, sizeof (resp));
883 #else
884 rc = copyout((void*)&resp, (void*)cmd.response.r_addr, sizeof (resp));
885 #endif
886 if (rc != 0) {
887 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
888 "destroy_qp() : copyout failure %x", rc);
889 rc = EFAULT;
890 goto err_out;
891 }
892
893 return (DDI_SUCCESS);
894
895 err_busy:
896 sol_ofs_uobj_put(&uqp->uobj);
897
898 err_out:
899 return (rc);
900 }
901
902 /*
903 * Function:
904 * uverbs_copy_path_info_from_ibv
905 * Input:
906 * src_path - IB OFED path.
907 * Output:
908 * dest_path - IBT path.
909 * Returns:
910 * None
911 * Description:
912 * Helper to copy from the OFED path format to IBT path format.
913 */
914 static void
uverbs_copy_path_info_from_ibv(struct ib_uverbs_qp_dest * src_path,ibt_cep_path_t * dest_path)915 uverbs_copy_path_info_from_ibv(struct ib_uverbs_qp_dest *src_path,
916 ibt_cep_path_t *dest_path)
917 {
918 ASSERT(src_path != NULL);
919 ASSERT(dest_path != NULL);
920
921 (void) memcpy(&dest_path->cep_adds_vect.av_dgid,
922 &src_path->dgid[0], sizeof (src_path->dgid));
923
924 dest_path->cep_adds_vect.av_flow = src_path->flow_label;
925 dest_path->cep_adds_vect.av_dlid = src_path->dlid;
926 dest_path->cep_adds_vect.av_hop = src_path->hop_limit;
927 dest_path->cep_adds_vect.av_tclass = src_path->traffic_class;
928 dest_path->cep_adds_vect.av_srvl = src_path->sl & 0x0f;
929 dest_path->cep_adds_vect.av_port_num = src_path->port_num;
930 dest_path->cep_adds_vect.av_src_path = src_path->src_path_bits;
931 dest_path->cep_adds_vect.av_send_grh = src_path->is_global;
932 dest_path->cep_adds_vect.av_sgid_ix = src_path->sgid_index;
933 dest_path->cep_adds_vect.av_srate = src_path->static_rate;
934 }
935
936 /*
937 * Function:
938 * uverbs_modify_update
939 * Input:
940 * cmd - The user verbs modify command to be translated.
941 * cur_state - The current QP state
942 * new_state - The new QP state
943 * Output:
944 * qp_query_attr - The IBT QP attributes.
945 * flags - The IBT flags.
946 * Returns:
947 * None
948 * Description:
949 * Helper to convert OFED user verbs QP modify attributes to IBT
950 * QP modify attributes. Note that on required parameters, the
951 * individual IBT modify flags are not set (there is a global
952 * flag for the transition), only optional flags are set.
953 */
954 static void
uverbs_modify_update(struct ib_uverbs_modify_qp * cmd,enum ib_qp_state cur_state,enum ib_qp_state new_state,ibt_qp_query_attr_t * qp_query_attr,ibt_cep_modify_flags_t * flags)955 uverbs_modify_update(struct ib_uverbs_modify_qp *cmd,
956 enum ib_qp_state cur_state, enum ib_qp_state new_state,
957 ibt_qp_query_attr_t *qp_query_attr, ibt_cep_modify_flags_t *flags)
958 {
959 ibt_qp_info_t *qp_infop;
960 ibt_qp_rc_attr_t *rcp;
961 ibt_qp_uc_attr_t *ucp;
962 ibt_qp_ud_attr_t *udp;
963
964 *flags = IBT_CEP_SET_NOTHING;
965 qp_infop = &(qp_query_attr->qp_info);
966 rcp = &(qp_infop->qp_transport.rc);
967 ucp = &(qp_infop->qp_transport.uc);
968 udp = &(qp_infop->qp_transport.ud);
969
970 switch (cur_state) {
971 case IB_QPS_RESET:
972 qp_infop->qp_current_state = IBT_STATE_RESET;
973 break;
974 case IB_QPS_INIT:
975 qp_infop->qp_current_state = IBT_STATE_INIT;
976 break;
977 case IB_QPS_RTR:
978 qp_infop->qp_current_state = IBT_STATE_RTR;
979 break;
980 case IB_QPS_RTS:
981 qp_infop->qp_current_state = IBT_STATE_RTS;
982 break;
983 case IB_QPS_SQD:
984 qp_infop->qp_current_state = IBT_STATE_SQD;
985 break;
986 case IB_QPS_SQE:
987 qp_infop->qp_current_state = IBT_STATE_SQE;
988 break;
989 case IB_QPS_ERR:
990 qp_infop->qp_current_state = IBT_STATE_ERROR;
991 break;
992 }
993
994 if (cmd->attr_mask & IB_QP_STATE) {
995 switch (new_state) {
996 case IB_QPS_RESET:
997 qp_infop->qp_state = IBT_STATE_RESET;
998 *flags |= IBT_CEP_SET_STATE;
999 break;
1000
1001 case IB_QPS_INIT:
1002 qp_infop->qp_state = IBT_STATE_INIT;
1003 if (cur_state == IB_QPS_RESET) {
1004 *flags |= IBT_CEP_SET_RESET_INIT;
1005 } else {
1006 *flags |= IBT_CEP_SET_STATE;
1007 }
1008 break;
1009
1010 case IB_QPS_RTR:
1011 qp_infop->qp_state = IBT_STATE_RTR;
1012 if (cur_state == IB_QPS_INIT) {
1013 *flags |= IBT_CEP_SET_INIT_RTR;
1014 } else {
1015 *flags |= IBT_CEP_SET_STATE;
1016 }
1017 break;
1018
1019 case IB_QPS_RTS:
1020 qp_infop->qp_state = IBT_STATE_RTS;
1021
1022 /*
1023 * For RTS transitions other than RTR we must
1024 * specify the assumption for the qp state.
1025 */
1026 if (cur_state == IB_QPS_RTR) {
1027 *flags |= IBT_CEP_SET_RTR_RTS;
1028 } else {
1029 ibt_cep_state_t *ibt_curr =
1030 &qp_infop->qp_current_state;
1031
1032 switch (cur_state) {
1033 case IB_QPS_RTS:
1034 *ibt_curr = IBT_STATE_RTS;
1035 break;
1036
1037 case IB_QPS_SQD:
1038 *ibt_curr = IBT_STATE_SQD;
1039 break;
1040
1041 case IB_QPS_SQE:
1042 *ibt_curr = IBT_STATE_SQE;
1043 break;
1044 }
1045 *flags |= IBT_CEP_SET_STATE;
1046 }
1047 break;
1048
1049 case IB_QPS_SQD:
1050 qp_infop->qp_state = IBT_STATE_SQD;
1051 *flags |= IBT_CEP_SET_STATE;
1052 break;
1053
1054 case IB_QPS_SQE:
1055 qp_infop->qp_state = IBT_STATE_SQE;
1056 *flags |= IBT_CEP_SET_STATE;
1057 break;
1058
1059 case IB_QPS_ERR:
1060 qp_infop->qp_state = IBT_STATE_ERROR;
1061 *flags |= IBT_CEP_SET_STATE;
1062 break;
1063 }
1064 }
1065
1066 if (cmd->attr_mask & IB_QP_PKEY_INDEX) {
1067 if (qp_infop->qp_trans == IBT_UD_SRV) {
1068 udp->ud_pkey_ix = cmd->pkey_index;
1069 } else if (qp_infop->qp_trans == IBT_RC_SRV) {
1070 rcp->rc_path.cep_pkey_ix = cmd->pkey_index;
1071 }
1072 *flags |= IBT_CEP_SET_PKEY_IX;
1073 }
1074
1075
1076 if (cmd->attr_mask & IB_QP_AV) {
1077 if (qp_infop->qp_trans == IBT_RC_SRV) {
1078 uverbs_copy_path_info_from_ibv(&cmd->dest,
1079 &rcp->rc_path);
1080 }
1081 *flags |= IBT_CEP_SET_ADDS_VECT;
1082 }
1083
1084 if (qp_infop->qp_trans == IBT_RC_SRV) {
1085 if (cmd->attr_mask & IB_QP_TIMEOUT) {
1086 rcp->rc_path.cep_timeout = cmd->timeout;
1087 *flags |= IBT_CEP_SET_TIMEOUT;
1088 }
1089 }
1090
1091 if (cmd->attr_mask & IB_QP_PORT) {
1092 if (qp_infop->qp_trans == IBT_UD_SRV) {
1093 udp->ud_port = cmd->port_num;
1094 } else if (qp_infop->qp_trans == IBT_RC_SRV) {
1095 rcp->rc_path.cep_hca_port_num = cmd->port_num;
1096 }
1097 *flags |= IBT_CEP_SET_PORT;
1098 }
1099
1100 if (cmd->attr_mask & IB_QP_QKEY) {
1101 if (qp_infop->qp_trans == IBT_UD_SRV) {
1102 udp->ud_qkey = cmd->qkey;
1103 }
1104 if (qp_infop->qp_trans == IBT_RD_SRV) {
1105 qp_infop->qp_transport.rd.rd_qkey = cmd->qkey;
1106 }
1107 *flags |= IBT_CEP_SET_QKEY;
1108 }
1109
1110 if (cmd->attr_mask & IB_QP_PATH_MTU) {
1111 if (qp_infop->qp_trans == IBT_UC_SRV) {
1112 ucp->uc_path_mtu = cmd->path_mtu;
1113 }
1114 if (qp_infop->qp_trans == IBT_RC_SRV) {
1115 rcp->rc_path_mtu = cmd->path_mtu;
1116 }
1117 }
1118
1119 if (cmd->attr_mask & IB_QP_RETRY_CNT) {
1120 if (qp_infop->qp_trans == IBT_RC_SRV) {
1121 rcp->rc_retry_cnt = cmd->retry_cnt & 0x7;
1122 }
1123 *flags |= IBT_CEP_SET_RETRY;
1124 }
1125
1126 if (cmd->attr_mask & IB_QP_RNR_RETRY) {
1127 if (qp_infop->qp_trans == IBT_RC_SRV) {
1128 rcp->rc_rnr_retry_cnt = cmd->rnr_retry;
1129 }
1130 *flags |= IBT_CEP_SET_RNR_NAK_RETRY;
1131 }
1132
1133 if (cmd->attr_mask & IB_QP_MIN_RNR_TIMER) {
1134 if (qp_infop->qp_trans == IBT_RC_SRV) {
1135 rcp->rc_min_rnr_nak = cmd->min_rnr_timer;
1136 }
1137 *flags |= IBT_CEP_SET_MIN_RNR_NAK;
1138 }
1139
1140 if (cmd->attr_mask & IB_QP_RQ_PSN) {
1141 if (qp_infop->qp_trans == IBT_RC_SRV) {
1142 rcp->rc_rq_psn = cmd->rq_psn;
1143 }
1144 if (qp_infop->qp_trans == IBT_UC_SRV) {
1145 ucp->uc_rq_psn = cmd->rq_psn;
1146 }
1147 }
1148
1149 if (cmd->attr_mask & IB_QP_ALT_PATH) {
1150 if (qp_infop->qp_trans == IBT_RC_SRV) {
1151 uverbs_copy_path_info_from_ibv(&cmd->alt_dest,
1152 &rcp->rc_alt_path);
1153
1154 rcp->rc_alt_path.cep_hca_port_num = cmd->alt_port_num;
1155 rcp->rc_alt_path.cep_timeout = cmd->alt_timeout;
1156 }
1157
1158 if (qp_infop->qp_trans == IBT_UC_SRV) {
1159 uverbs_copy_path_info_from_ibv(&cmd->alt_dest,
1160 &ucp->uc_alt_path);
1161
1162 ucp->uc_alt_path.cep_hca_port_num = cmd->alt_port_num;
1163 ucp->uc_alt_path.cep_timeout = cmd->alt_timeout;
1164 }
1165
1166 *flags |= IBT_CEP_SET_ALT_PATH;
1167 }
1168
1169
1170 if (cmd->attr_mask & IB_QP_SQ_PSN) {
1171 if (qp_infop->qp_trans == IBT_UD_SRV) {
1172 udp->ud_sq_psn = cmd->sq_psn;
1173 }
1174 if (qp_infop->qp_trans == IBT_UC_SRV) {
1175 ucp->uc_sq_psn = cmd->sq_psn;
1176 }
1177 if (qp_infop->qp_trans == IBT_RC_SRV) {
1178 rcp->rc_sq_psn = cmd->sq_psn;
1179 }
1180 }
1181
1182 if (cmd->attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
1183 if (qp_infop->qp_trans == IBT_RC_SRV) {
1184 rcp->rc_rdma_ra_out = cmd->max_rd_atomic;
1185 }
1186 *flags |= IBT_CEP_SET_RDMARA_OUT | IBT_CEP_SET_STATE;
1187 }
1188
1189 if (cmd->attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
1190 if (qp_infop->qp_trans == IBT_RC_SRV) {
1191 rcp->rc_rdma_ra_in = cmd->max_dest_rd_atomic;
1192 }
1193 *flags |= IBT_CEP_SET_RDMARA_IN | IBT_CEP_SET_STATE;
1194 }
1195
1196 if (cmd->attr_mask & (IB_QP_ACCESS_FLAGS |
1197 IB_QP_MAX_DEST_RD_ATOMIC)) {
1198 if (qp_infop->qp_trans == IBT_RC_SRV) {
1199 uint32_t access_flags = IBT_CEP_NO_FLAGS;
1200
1201 if (rcp->rc_rdma_ra_in) {
1202 access_flags |= IBT_CEP_RDMA_WR;
1203 *flags |= IBT_CEP_SET_RDMA_W;
1204 }
1205
1206 if (cmd->attr_mask & IB_QP_ACCESS_FLAGS) {
1207 if (cmd->qp_access_flags &
1208 IB_ACCESS_REMOTE_WRITE) {
1209 access_flags |= IBT_CEP_RDMA_WR;
1210 *flags |= IBT_CEP_SET_RDMA_W;
1211 }
1212 if (cmd->qp_access_flags &
1213 IB_ACCESS_REMOTE_READ) {
1214 access_flags |= IBT_CEP_RDMA_RD;
1215 *flags |= IBT_CEP_SET_RDMA_R;
1216 }
1217 if (cmd->qp_access_flags &
1218 IB_ACCESS_REMOTE_ATOMIC) {
1219 access_flags |= IBT_CEP_ATOMIC;
1220 *flags |= IBT_CEP_SET_ATOMIC;
1221 }
1222 }
1223 qp_infop->qp_flags &= ~(IBT_CEP_RDMA_WR |
1224 IBT_CEP_RDMA_RD | IBT_CEP_ATOMIC);
1225 qp_infop->qp_flags |= access_flags;
1226 }
1227 }
1228
1229 if (cmd->attr_mask & IB_QP_PATH_MIG_STATE) {
1230 if (qp_infop->qp_trans == IBT_RC_SRV) {
1231 if (cmd->path_mig_state == IB_MIG_MIGRATED) {
1232 rcp->rc_mig_state = IBT_STATE_MIGRATED;
1233 }
1234 if (cmd->path_mig_state == IB_MIG_REARM) {
1235 rcp->rc_mig_state = IBT_STATE_REARMED;
1236 }
1237 if (cmd->path_mig_state == IB_MIG_ARMED) {
1238 rcp->rc_mig_state = IBT_STATE_ARMED;
1239 }
1240 }
1241
1242 if (qp_infop->qp_trans == IBT_UC_SRV) {
1243 if (cmd->path_mig_state == IB_MIG_MIGRATED) {
1244 ucp->uc_mig_state = IBT_STATE_MIGRATED;
1245 }
1246 if (cmd->path_mig_state == IB_MIG_REARM) {
1247 ucp->uc_mig_state = IBT_STATE_REARMED;
1248 }
1249 if (cmd->path_mig_state == IB_MIG_ARMED) {
1250 ucp->uc_mig_state = IBT_STATE_ARMED;
1251 }
1252 }
1253 *flags |= IBT_CEP_SET_MIG;
1254 }
1255
1256 if (cmd->attr_mask & IB_QP_DEST_QPN) {
1257 if (qp_infop->qp_trans == IBT_RC_SRV) {
1258 rcp->rc_dst_qpn = cmd->dest_qp_num;
1259 }
1260 if (qp_infop->qp_trans == IBT_UC_SRV) {
1261 ucp->uc_dst_qpn = cmd->dest_qp_num;
1262 }
1263 }
1264 }
1265
1266
1267 static void
uverbs_qp_print_path(ibt_cep_path_t * pathp)1268 uverbs_qp_print_path(ibt_cep_path_t *pathp)
1269 {
1270 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, "qp_print_pathp %p", pathp);
1271 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, "cep_pkey_ix %x, "
1272 "cep_hca_port_num %x", pathp->cep_pkey_ix, pathp->cep_hca_port_num);
1273 }
1274
1275 static void
uverbs_print_query_qp(ibt_qp_hdl_t qp_hdlp)1276 uverbs_print_query_qp(ibt_qp_hdl_t qp_hdlp)
1277 {
1278 ibt_qp_query_attr_t qp_query_attr;
1279 ibt_qp_info_t *qp_infop = &qp_query_attr.qp_info;
1280 ibt_qp_rc_attr_t *rcp = &((qp_infop->qp_transport).rc);
1281 ibt_status_t rc;
1282
1283 bzero(&qp_query_attr, sizeof (qp_query_attr));
1284 rc = ibt_query_qp(qp_hdlp, &qp_query_attr);
1285 if (rc != IBT_SUCCESS) {
1286 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, "print_query_qp -"
1287 "ibt_query_qp() failed - rc=%d", rc);
1288 return;
1289 }
1290 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, "qp_print %p", qp_hdlp);
1291
1292 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, "qp_sq_cq %p, qp_rq_cq %p, "
1293 "qp_qpn %x, qp_sq_sgl %x, qp_rq_sgl %x, qp_srq %p, qp_flags %x",
1294 qp_query_attr.qp_sq_cq, qp_query_attr.qp_rq_cq,
1295 qp_query_attr.qp_qpn, qp_query_attr.qp_sq_sgl,
1296 qp_query_attr.qp_rq_sgl, qp_query_attr.qp_srq,
1297 qp_query_attr.qp_flags);
1298
1299 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, "qp_sq_sz %x, qp_rq_sz %x, "
1300 "qp_state %x, qp_current_state %x, qp_flags %x, qp_trans %x",
1301 qp_infop->qp_sq_sz, qp_infop->qp_rq_sz, qp_infop->qp_state,
1302 qp_infop->qp_current_state, qp_infop->qp_flags,
1303 qp_infop->qp_trans);
1304
1305 if (qp_infop->qp_trans == IBT_RC_SRV) {
1306 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, "rc_sq_psn %x, rc_rq_psn %x, "
1307 "rc_dst_qpn %x, rc_mig_state %x, rc_rnr_retry_cnt %x,"
1308 "rc_retry_cnt %x rc_rdma_ra_out %x, rc_rdma_ra_in %x,"
1309 "rc_min_rnr_nak %x, rc_path_mtu %x",
1310 rcp->rc_sq_psn, rcp->rc_rq_psn, rcp->rc_dst_qpn, rcp->rc_mig_state,
1311 rcp->rc_rnr_retry_cnt, rcp->rc_retry_cnt, rcp->rc_rdma_ra_out,
1312 rcp->rc_rdma_ra_in, rcp->rc_min_rnr_nak, rcp->rc_path_mtu);
1313 uverbs_qp_print_path(&rcp->rc_path);
1314 uverbs_qp_print_path(&rcp->rc_alt_path);
1315 }
1316 }
1317
1318
1319 /*
1320 * Function:
1321 * sol_uverbs_modify_qp
1322 * Input:
1323 * uctxt - Pointer to the callers user context.
1324 * buf - Pointer to kernel buffer containing QP modify command.
1325 * in_len - Length in bytes of input command buffer.
1326 * out_len - Length in bytes of output response buffer.
1327 * Output:
1328 * The command output buffer is updated with command results.
1329 * Returns:
1330 * DDI_SUCCESS on success, else error code.
1331 * Description:
1332 * User verbs entry point to modify a device QP.
1333 */
1334 /* ARGSUSED */
1335 int
sol_uverbs_modify_qp(uverbs_uctxt_uobj_t * uctxt,char * buf,int in_len,int out_len)1336 sol_uverbs_modify_qp(uverbs_uctxt_uobj_t *uctxt, char *buf,
1337 int in_len, int out_len)
1338 {
1339 struct ib_uverbs_modify_qp cmd;
1340 uverbs_uqp_uobj_t *uqp;
1341 ibt_qp_query_attr_t qp_query_attr;
1342 ibt_cep_modify_flags_t flags;
1343 ibt_queue_sizes_t size;
1344 int rc;
1345 enum ib_qp_state cur_state, new_state;
1346
1347 (void) memcpy(&cmd, buf, sizeof (cmd));
1348
1349 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "modify_qp - qp_hdl %d, "
1350 "attr_mask %x", cmd.qp_handle, cmd.attr_mask);
1351
1352 uqp = uverbs_uobj_get_uqp_write(cmd.qp_handle);
1353 if (uqp == NULL) {
1354 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, "modify_qp -"
1355 "List lookup failure");
1356 rc = EINVAL;
1357 goto err_out;
1358 }
1359
1360 /*
1361 * Has the UCMA asked us to ignore QP modify operations?
1362 * This is required because of differences in the level of
1363 * abstraction fo CM processing between IBT and OFED.
1364 */
1365 if (uqp->disable_qp_mod == TRUE) {
1366 SOL_OFS_DPRINTF_L3(sol_uverbs_dbg_str, "modify_qp -"
1367 "qp_mod disabled");
1368 goto done;
1369 }
1370
1371 /*
1372 * Load the current QP attributes and then do a validation
1373 * based on OFA verbs expectations to see if the modify
1374 * should be performed.
1375 */
1376 bzero(&qp_query_attr, sizeof (qp_query_attr));
1377 rc = ibt_query_qp(uqp->qp, &qp_query_attr);
1378 if (rc != IBT_SUCCESS) {
1379 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, "modify_qp -"
1380 "ibt_query_qp() failed - rc=%d", rc);
1381 rc = sol_uverbs_ibt_to_kernel_status(rc);
1382 goto err_deref;
1383 }
1384
1385 if (cmd.attr_mask & IB_QP_CUR_STATE) {
1386 cur_state = cmd.cur_qp_state;
1387 } else {
1388 cur_state = IBT_TO_OFA_QP_STATE(qp_query_attr.qp_info.qp_state);
1389 }
1390
1391 new_state = cmd.attr_mask & IB_QP_STATE ? cmd.qp_state : cur_state;
1392
1393 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "modify_qp: ibt qp %p, handle "
1394 "%x, cur_state %x, new_state %x, qp_type %x, attr_mask %x", uqp->qp,
1395 cmd.qp_handle, cur_state, new_state, uqp->ofa_qp_type,
1396 cmd.attr_mask);
1397
1398 if (!uverbs_modify_qp_is_ok(cur_state, new_state, uqp->ofa_qp_type,
1399 (enum ib_qp_attr_mask *)&cmd.attr_mask)) {
1400 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, "modify_qp() -"
1401 "Failed modify OK test");
1402 rc = EINVAL;
1403 goto err_deref;
1404 }
1405
1406 if (!cmd.attr_mask) {
1407 SOL_OFS_DPRINTF_L3(sol_uverbs_dbg_str, "modify_qp() -"
1408 "attr_mask after modify OK test is 0");
1409 rc = 0;
1410 goto done;
1411 }
1412
1413 flags = 0;
1414
1415 switch (uqp->ofa_qp_type) {
1416 case IB_QPT_UC:
1417 qp_query_attr.qp_info.qp_trans = IBT_UC_SRV;
1418 break;
1419 case IB_QPT_UD:
1420 qp_query_attr.qp_info.qp_trans = IBT_UD_SRV;
1421 break;
1422 case IB_QPT_RC:
1423 qp_query_attr.qp_info.qp_trans = IBT_RC_SRV;
1424 break;
1425 default:
1426 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
1427 "modify_qp: Invalid QP type");
1428 rc = EINVAL;
1429 goto err_deref;
1430 }
1431
1432 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "modify_qp(): qp_info.qp_flags "
1433 "before modify update = 0%08x", qp_query_attr.qp_info.qp_flags);
1434
1435 uverbs_modify_update(&cmd, cur_state, new_state, &qp_query_attr,
1436 &flags);
1437
1438 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "modify_qp(): after "
1439 "modify_update hdl flags = 0x%08x, qp_info.qp_flags = 0%08x",
1440 flags, qp_query_attr.qp_info.qp_flags);
1441
1442 rc = ibt_modify_qp(uqp->qp, flags, &qp_query_attr.qp_info, &size);
1443
1444 if (rc != IBT_SUCCESS) {
1445 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
1446 "modify_qp: Error in ibt_modify_qp() (rc=%d)", rc);
1447 uverbs_print_query_qp(uqp->qp);
1448 rc = sol_uverbs_ibt_to_kernel_status(rc);
1449 goto err_deref;
1450 }
1451
1452 done:
1453 sol_ofs_uobj_put(&uqp->uobj);
1454 return (DDI_SUCCESS);
1455
1456 err_deref:
1457 sol_ofs_uobj_put(&uqp->uobj);
1458
1459 err_out:
1460 return (rc);
1461 }
1462
1463 /*
1464 * Function:
1465 * uverbs_copy_path_info_from_ibt
1466 * Input:
1467 * src_path - The IBT path.
1468 * Output:
1469 * dest_path - The OFED user verbs path.
1470 * Returns:
1471 * None
1472 * Description:
1473 * Helper to convert IBT path to OFED user verbs path.
1474 */
1475 static void
uverbs_copy_path_info_from_ibt(ibt_cep_path_t * src_path,struct ib_uverbs_qp_dest * dest_path)1476 uverbs_copy_path_info_from_ibt(ibt_cep_path_t *src_path,
1477 struct ib_uverbs_qp_dest *dest_path)
1478 {
1479 ASSERT(src_path != NULL);
1480 ASSERT(dest_path != NULL);
1481
1482 (void) memcpy(&dest_path->dgid[0],
1483 &src_path->cep_adds_vect.av_dgid, sizeof (dest_path->dgid));
1484
1485 dest_path->flow_label = src_path->cep_adds_vect.av_flow;
1486 dest_path->dlid = src_path->cep_adds_vect.av_dlid;
1487 dest_path->hop_limit = src_path->cep_adds_vect.av_hop;
1488 dest_path->traffic_class = src_path->cep_adds_vect.av_tclass;
1489 dest_path->sl = src_path->cep_adds_vect.av_srvl;
1490 dest_path->port_num = src_path->cep_adds_vect.av_port_num;
1491 dest_path->src_path_bits = src_path->cep_adds_vect.av_src_path;
1492 dest_path->is_global = src_path->cep_adds_vect.av_send_grh;
1493 dest_path->sgid_index = src_path->cep_adds_vect.av_sgid_ix;
1494 dest_path->static_rate = src_path->cep_adds_vect.av_srate;
1495 }
1496
1497 /*
1498 * Function:
1499 * uverbs_query_copy_rc
1500 * Input:
1501 * src - The IBT RC QP attributes.
1502 * Output:
1503 * dest - The OFED user verbs QP attributes.
1504 * Returns:
1505 * None
1506 * Description:
1507 * Helper to copy IBT RC QP attributes to OFED QP attributes.
1508 */
1509 static void
uverbs_query_copy_rc(struct ib_uverbs_query_qp_resp * dest,ibt_qp_rc_attr_t * src)1510 uverbs_query_copy_rc(struct ib_uverbs_query_qp_resp *dest,
1511 ibt_qp_rc_attr_t *src)
1512 {
1513 dest->sq_psn = src->rc_sq_psn;
1514 dest->rq_psn = src->rc_rq_psn;
1515 dest->dest_qp_num = src->rc_dst_qpn;
1516 dest->rnr_retry = src->rc_rnr_retry_cnt;
1517 dest->retry_cnt = src->rc_retry_cnt;
1518 dest->max_dest_rd_atomic = src->rc_rdma_ra_in;
1519 dest->max_rd_atomic = src->rc_rdma_ra_out;
1520 dest->min_rnr_timer = src->rc_min_rnr_nak;
1521 dest->path_mtu = src->rc_path_mtu;
1522 dest->timeout = src->rc_path.cep_timeout;
1523 dest->alt_timeout = src->rc_alt_path.cep_timeout;
1524 dest->port_num = src->rc_path.cep_hca_port_num;
1525 dest->alt_port_num = src->rc_alt_path.cep_hca_port_num;
1526
1527 if (src->rc_mig_state == IBT_STATE_MIGRATED) {
1528 dest->path_mig_state = IB_MIG_MIGRATED;
1529 }
1530 if (src->rc_mig_state == IBT_STATE_REARMED) {
1531 dest->path_mig_state = IB_MIG_REARM;
1532 }
1533 if (src->rc_mig_state == IBT_STATE_ARMED) {
1534 dest->path_mig_state = IB_MIG_ARMED;
1535 }
1536
1537 uverbs_copy_path_info_from_ibt(&src->rc_path, &dest->dest);
1538 uverbs_copy_path_info_from_ibt(&src->rc_alt_path, &dest->alt_dest);
1539 }
1540
1541 /*
1542 * Function:
1543 * uverbs_query_copy_uc
1544 * Input:
1545 * src - The IBT UC QP attributes.
1546 * Output:
1547 * dest - The OFED user verbs QP attributes.
1548 * Returns:
1549 * None
1550 * Description:
1551 * Helper to copy IBT UC QP attributes to OFED user verbs
1552 * QP attributes.
1553 */
1554 static void
uverbs_query_copy_uc(struct ib_uverbs_query_qp_resp * dest,ibt_qp_uc_attr_t * src)1555 uverbs_query_copy_uc(struct ib_uverbs_query_qp_resp *dest,
1556 ibt_qp_uc_attr_t *src)
1557 {
1558 dest->sq_psn = src->uc_sq_psn;
1559 dest->rq_psn = src->uc_rq_psn;
1560 dest->dest_qp_num = src->uc_dst_qpn;
1561 dest->path_mtu = src->uc_path_mtu;
1562
1563 if (src->uc_mig_state == IBT_STATE_MIGRATED) {
1564 dest->path_mig_state = IB_MIG_MIGRATED;
1565 }
1566 if (src->uc_mig_state == IBT_STATE_REARMED) {
1567 dest->path_mig_state = IB_MIG_REARM;
1568 }
1569 if (src->uc_mig_state == IBT_STATE_ARMED) {
1570 dest->path_mig_state = IB_MIG_ARMED;
1571 }
1572
1573 uverbs_copy_path_info_from_ibt(&src->uc_path, &dest->dest);
1574 uverbs_copy_path_info_from_ibt(&src->uc_alt_path, &dest->alt_dest);
1575 }
1576
1577 /*
1578 * Function:
1579 * uverbs_query_copy_rd
1580 * Input:
1581 * src - The IBT RD QP attributes.
1582 * Output:
1583 * dest - The OFED user verbs QP attributes.
1584 * Returns:
1585 * None
1586 * Description:
1587 * Helper to copy IBT RD QP attributes to OFED user verb QP attributes.
1588 */
1589 static void
uverbs_query_copy_rd(struct ib_uverbs_query_qp_resp * dest,ibt_qp_rd_attr_t * src)1590 uverbs_query_copy_rd(struct ib_uverbs_query_qp_resp *dest,
1591 ibt_qp_rd_attr_t *src)
1592 {
1593 dest->qkey = src->rd_qkey;
1594 dest->min_rnr_timer = src->rd_min_rnr_nak;
1595 }
1596
1597 /*
1598 * Function:
1599 * uverbs_query_copy_ud
1600 * Input:
1601 * src - The IBT UD QP attributes.
1602 * Output:
1603 * dest - The OFED user verbs QP attributes.
1604 * Returns:
1605 * None
1606 * Description:
1607 * Helper to copy IBT UD QP attributes to OFED user verbs QP attributes.
1608 */
1609 static void
uverbs_query_copy_ud(struct ib_uverbs_query_qp_resp * dest,ibt_qp_ud_attr_t * src)1610 uverbs_query_copy_ud(struct ib_uverbs_query_qp_resp *dest,
1611 ibt_qp_ud_attr_t *src)
1612 {
1613 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
1614 "query_copy_ud:entry - return UD info: qkey:%08X, "
1615 "psn:%d, pkey_idx:%d, port:%d", src->ud_qkey, src->ud_sq_psn,
1616 src->ud_pkey_ix, src->ud_port);
1617
1618 dest->qkey = src->ud_qkey;
1619 dest->sq_psn = src->ud_sq_psn;
1620 dest->pkey_index = src->ud_pkey_ix;
1621 dest->port_num = src->ud_port;
1622 }
1623
1624 /*
1625 * Function:
1626 * uverbs_query_copy_info
1627 * Input:
1628 * src - The IBT QP information.
1629 * Output:
1630 * dest - The OFED user verbs QP attributes.
1631 * Returns:
1632 * None
1633 * Description:
1634 * Helper to copy IBT QP info to OFED user verbs QP attributes.
1635 */
1636 static void
uverbs_query_copy_info(struct ib_uverbs_query_qp_resp * dest,ibt_qp_info_t * src)1637 uverbs_query_copy_info(struct ib_uverbs_query_qp_resp *dest,
1638 ibt_qp_info_t *src)
1639 {
1640
1641 dest->max_send_wr = src->qp_sq_sz;
1642 dest->max_recv_wr = src->qp_rq_sz;
1643 dest->qp_access_flags = ibt_cep_flags2ibv(src->qp_flags);
1644
1645 switch (src->qp_state) {
1646 case IBT_STATE_RESET:
1647 dest->qp_state = IB_QPS_RESET;
1648 break;
1649 case IBT_STATE_INIT:
1650 dest->qp_state = IB_QPS_INIT;
1651 break;
1652 case IBT_STATE_RTR:
1653 dest->qp_state = IB_QPS_RTR;
1654 break;
1655 case IBT_STATE_RTS:
1656 dest->qp_state = IB_QPS_RTS;
1657 break;
1658 case IBT_STATE_SQD:
1659 dest->qp_state = IB_QPS_SQD;
1660 break;
1661 case IBT_STATE_SQE:
1662 dest->qp_state = IB_QPS_SQE;
1663 break;
1664 case IBT_STATE_ERROR:
1665 default:
1666 dest->qp_state = IB_QPS_ERR;
1667 break;
1668 }
1669
1670 switch (src->qp_current_state) {
1671 case IBT_STATE_RESET:
1672 dest->cur_qp_state = IB_QPS_RESET;
1673 break;
1674 case IBT_STATE_INIT:
1675 dest->cur_qp_state = IB_QPS_INIT;
1676 break;
1677 case IBT_STATE_RTR:
1678 dest->cur_qp_state = IB_QPS_RTR;
1679 break;
1680 case IBT_STATE_RTS:
1681 dest->cur_qp_state = IB_QPS_RTS;
1682 break;
1683 case IBT_STATE_SQD:
1684 dest->cur_qp_state = IB_QPS_SQD;
1685 break;
1686 case IBT_STATE_SQE:
1687 dest->cur_qp_state = IB_QPS_SQE;
1688 break;
1689 case IBT_STATE_ERROR:
1690 default:
1691 dest->cur_qp_state = IB_QPS_ERR;
1692 break;
1693 }
1694
1695 if ((src->qp_flags & IBT_ALL_SIGNALED) == IBT_ALL_SIGNALED) {
1696 dest->sq_sig_all = 1;
1697 }
1698 }
1699
1700 /*
1701 * Function:
1702 * uverbs_query_copy_attr
1703 * Input:
1704 * src - The IBT QP information.
1705 * Output:
1706 * dest - The OFED user verbs QP attributes.
1707 * Returns:
1708 * None
1709 * Description:
1710 * Helper to copy IBT QP attributes to OFED user verbs QP attributes.
1711 */
1712 static void
uverbs_query_copy_attr(struct ib_uverbs_query_qp_resp * dest,ibt_qp_query_attr_t * src)1713 uverbs_query_copy_attr(struct ib_uverbs_query_qp_resp *dest,
1714 ibt_qp_query_attr_t *src)
1715 {
1716 dest->max_send_sge = src->qp_sq_sgl;
1717 dest->max_recv_sge = src->qp_rq_sgl;
1718 }
1719
1720 /*
1721 * Function:
1722 * sol_uverbs_query_qp
1723 * Input:
1724 * uctxt - Pointer to the callers user context.
1725 * buf - Pointer to kernel buffer containing query QP command.
1726 * in_len - Length in bytes of input command buffer.
1727 * out_len - Length in bytes of output response buffer.
1728 * Output:
1729 * The command output buffer is updated with command results.
1730 * Returns:
1731 * DDI_SUCCESS on success, else error code.
1732 * Description:
1733 * User verbs entry point to query a device QP properties.
1734 */
1735 /* ARGSUSED */
1736 int
sol_uverbs_query_qp(uverbs_uctxt_uobj_t * uctxt,char * buf,int in_len,int out_len)1737 sol_uverbs_query_qp(uverbs_uctxt_uobj_t *uctxt, char *buf,
1738 int in_len, int out_len)
1739 {
1740 struct ib_uverbs_query_qp cmd;
1741 struct ib_uverbs_query_qp_resp resp;
1742 uverbs_uqp_uobj_t *uqp;
1743 ibt_qp_query_attr_t qp_query_attr;
1744 int rc;
1745
1746 (void) memset(&resp, 0, sizeof (resp));
1747 (void) memcpy(&cmd, buf, sizeof (cmd));
1748
1749 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
1750 "query_qp: entry (qp_handle=%d)", cmd.qp_handle);
1751
1752 uqp = uverbs_uobj_get_uqp_read(cmd.qp_handle);
1753 if (uqp == NULL) {
1754 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
1755 "query_qp(): List lookup failure");
1756 rc = EINVAL;
1757 goto err_out;
1758 }
1759
1760 bzero(&qp_query_attr, sizeof (qp_query_attr));
1761 rc = ibt_query_qp(uqp->qp, &qp_query_attr);
1762 sol_ofs_uobj_put(&uqp->uobj);
1763
1764 if (rc != IBT_SUCCESS) {
1765 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
1766 "query_qp: Error in ibt_query_qp() (rc=%d)", rc);
1767 rc = sol_uverbs_ibt_to_kernel_status(rc);
1768 goto err_out;
1769 }
1770
1771 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
1772 "query_qp(): qp_query_attr.qp_info.qp_trans = %d",
1773 qp_query_attr.qp_info.qp_trans);
1774
1775 uverbs_query_copy_attr(&resp, &qp_query_attr);
1776 uverbs_query_copy_info(&resp, &qp_query_attr.qp_info);
1777
1778 if (qp_query_attr.qp_info.qp_trans == IBT_RC_SRV) {
1779 uverbs_query_copy_rc(&resp,
1780 &qp_query_attr.qp_info.qp_transport.rc);
1781 }
1782
1783 if (qp_query_attr.qp_info.qp_trans == IBT_UC_SRV) {
1784 uverbs_query_copy_uc(&resp,
1785 &qp_query_attr.qp_info.qp_transport.uc);
1786 }
1787
1788 if (qp_query_attr.qp_info.qp_trans == IBT_RD_SRV) {
1789 uverbs_query_copy_rd(&resp,
1790 &qp_query_attr.qp_info.qp_transport.rd);
1791 }
1792
1793
1794 if (qp_query_attr.qp_info.qp_trans == IBT_UD_SRV) {
1795 uverbs_query_copy_ud(&resp,
1796 &qp_query_attr.qp_info.qp_transport.ud);
1797 }
1798
1799 resp.max_inline_data = uqp->max_inline_data;
1800
1801 #ifdef _LP64
1802 rc = copyout((void*)&resp, (void*)cmd.response.r_laddr, sizeof (resp));
1803 #else
1804 rc = copyout((void*)&resp, (void*)cmd.response.r_addr, sizeof (resp));
1805 #endif
1806 if (rc != 0) {
1807 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
1808 "query_qp(): Error writing resp data (rc=%d)", rc);
1809 rc = EFAULT;
1810 goto err_out;
1811 }
1812
1813 return (DDI_SUCCESS);
1814
1815 err_out:
1816 return (rc);
1817 }
1818
1819 /*
1820 * Function:
1821 * sol_uverbs_create_srq
1822 * Input:
1823 * uctxt - Pointer to the callers user context.
1824 * buf - Pointer to kernel buffer containing command.
1825 * in_len - Length in bytes of input command buffer.
1826 * out_len - Length in bytes of output response buffer.
1827 * Output:
1828 * The command output buffer is updated with command results.
1829 * Returns:
1830 * DDI_SUCCESS on success, else error code.
1831 * Description:
1832 * User verbs entry point to create a device shared receive queue.
1833 */
1834 /* ARGSUSED */
1835 int
sol_uverbs_create_srq(uverbs_uctxt_uobj_t * uctxt,char * buf,int in_len,int out_len)1836 sol_uverbs_create_srq(uverbs_uctxt_uobj_t *uctxt, char *buf,
1837 int in_len, int out_len)
1838 {
1839 struct ib_uverbs_create_srq cmd;
1840 struct ib_uverbs_create_srq_resp resp;
1841 uverbs_usrq_uobj_t *usrq;
1842 uverbs_upd_uobj_t *upd;
1843 ibt_srq_flags_t flags = IBT_SRQ_USER_MAP;
1844 ibt_srq_sizes_t attr;
1845 ibt_srq_sizes_t real_attr;
1846 int rc;
1847
1848 (void) memcpy(&cmd, buf, sizeof (cmd));
1849 (void) memset(&resp, 0, sizeof (resp));
1850 (void) memset(&attr, 0, sizeof (attr));
1851
1852 attr.srq_wr_sz = cmd.max_wr;
1853 attr.srq_sgl_sz = cmd.max_sge;
1854
1855 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "create_srq: "
1856 "max_wr=%d, max_sge=%d, srq_limit=%d",
1857 cmd.max_wr, cmd.max_sge, cmd.srq_limit);
1858
1859 if (!attr.srq_wr_sz) {
1860 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
1861 "create_srq(): Invalid args, invalid work "
1862 "request size");
1863
1864 rc = EINVAL;
1865 goto err_out;
1866 }
1867
1868 if (attr.srq_wr_sz > uctxt->hca->attr.hca_max_srqs_sz ||
1869 attr.srq_sgl_sz > uctxt->hca->attr.hca_max_srq_sgl) {
1870 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
1871 "create_srq(): Invalid args, too large");
1872 rc = EINVAL;
1873 goto err_out;
1874 }
1875
1876 usrq = kmem_zalloc(sizeof (*usrq), KM_NOSLEEP);
1877 if (usrq == NULL) {
1878 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
1879 "create_srq(): User object alloc failed");
1880 rc = ENOMEM;
1881 goto err_out;
1882 }
1883 sol_ofs_uobj_init(&usrq->uobj, cmd.user_handle,
1884 SOL_UVERBS_USRQ_UOBJ_TYPE);
1885 rw_enter(&usrq->uobj.uo_lock, RW_WRITER);
1886 llist_head_init(&usrq->async_list, NULL);
1887 usrq->async_events_reported = 0;
1888 usrq->uctxt = uctxt;
1889
1890 upd = uverbs_uobj_get_upd_read(cmd.pd_handle);
1891 if (upd == NULL) {
1892 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
1893 "create_srq(): PD Invalid");
1894 rc = EINVAL;
1895 goto err_dealloc;
1896 }
1897
1898 rc = ibt_alloc_srq(uctxt->hca->hdl, flags, upd->pd, &attr, &usrq->srq,
1899 &real_attr);
1900
1901 if (rc != IBT_SUCCESS) {
1902 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
1903 "create_srq(): Error in ibt_alloc_srq() (rc=%d)", rc);
1904 usrq->srq = NULL;
1905 rc = sol_uverbs_ibt_to_kernel_status(rc);
1906 usrq->uobj.uo_uobj_sz = sizeof (uverbs_usrq_uobj_t);
1907 goto err_release_pd;
1908 }
1909
1910 ibt_set_srq_private(usrq->srq, usrq);
1911
1912 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
1913 "create_srq(): ib_alloc_srq()real wqe_sz=%d, real_sg_sz=%d",
1914 real_attr.srq_wr_sz, real_attr.srq_sgl_sz);
1915
1916 /*
1917 * Query underlying hardware for data used in mapping CQ back to user
1918 * space, we will return this information in the user verbs command
1919 * response.
1920 */
1921 rc = ibt_ci_data_out(uctxt->hca->hdl, IBT_CI_NO_FLAGS, IBT_HDL_SRQ,
1922 (void *)usrq->srq, &resp.drv_out, sizeof (resp.drv_out));
1923
1924 if (rc != IBT_SUCCESS) {
1925 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
1926 "create_srq(): Error in ibt_ci_data_out() (rc=%d)", rc);
1927 rc = EFAULT;
1928 usrq->uobj.uo_uobj_sz = sizeof (uverbs_usrq_uobj_t);
1929 goto err_srq_destroy;
1930 }
1931
1932 if (sol_ofs_uobj_add(&uverbs_usrq_uo_tbl, &usrq->uobj) != 0) {
1933 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
1934 "create_srq(): uobj add failed");
1935 rc = ENOMEM;
1936 goto err_srq_destroy;
1937 }
1938
1939 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
1940 "create_srq(): ibt_ci_data_out: 0x%16llx 0x%16llx 0x%16llx "
1941 "0x%16llx", resp.drv_out[0], resp.drv_out[1], resp.drv_out[2],
1942 resp.drv_out[3]);
1943
1944 resp.srq_handle = usrq->uobj.uo_id;
1945 resp.max_wr = real_attr.srq_wr_sz;
1946 resp.max_sge = real_attr.srq_sgl_sz;
1947
1948 #ifdef _LP64
1949 rc = copyout((void*)&resp, (void*)cmd.response.r_laddr, sizeof (resp));
1950 #else
1951 rc = copyout((void*)&resp, (void*)cmd.response.r_addr, sizeof (resp));
1952 #endif
1953 if (rc != 0) {
1954 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
1955 "create_srq(): Error writing resp data (rc=%d)", rc);
1956 rc = EFAULT;
1957 goto err_uo_delete;
1958 }
1959
1960 mutex_enter(&uctxt->lock);
1961 usrq->list_entry = add_genlist(&uctxt->srq_list, (uintptr_t)usrq,
1962 uctxt);
1963 mutex_exit(&uctxt->lock);
1964
1965 if (!usrq->list_entry) {
1966 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
1967 "create_srq() : Error adding usrq to srq_list failed");
1968 rc = ENOMEM;
1969 goto err_uo_delete;
1970 }
1971
1972 usrq->uobj.uo_live = 1;
1973 rw_exit(&usrq->uobj.uo_lock);
1974
1975 sol_ofs_uobj_put(&upd->uobj);
1976
1977 return (DDI_SUCCESS);
1978
1979 err_uo_delete:
1980 /*
1981 * Need to set uo_live, so sol_ofs_uobj_remove() will
1982 * remove the object from the object table.
1983 */
1984 usrq->uobj.uo_live = 1;
1985 (void) sol_ofs_uobj_remove(&uverbs_usrq_uo_tbl, &usrq->uobj);
1986
1987 err_srq_destroy:
1988 (void) ibt_free_srq(usrq->srq);
1989
1990 err_release_pd:
1991 sol_ofs_uobj_put(&upd->uobj);
1992
1993 err_dealloc:
1994 rw_exit(&usrq->uobj.uo_lock);
1995 sol_ofs_uobj_deref(&usrq->uobj, sol_ofs_uobj_free);
1996
1997 err_out:
1998 return (rc);
1999 }
2000
2001 /*
2002 * Function:
2003 * sol_uverbs_modify_srq
2004 * Input:
2005 * uctxt - Pointer to the callers user context.
2006 * buf - Pointer to kernel buffer containing SRQ modify command.
2007 * in_len - Length in bytes of input command buffer.
2008 * out_len - Length in bytes of output response buffer.
2009 * Output:
2010 * The command output buffer is updated with command results.
2011 * Returns:
2012 * DDI_SUCCESS on success, else error code.
2013 * Description:
2014 * User verbs entry point to modify a device shared receive queue.
2015 */
2016 /* ARGSUSED */
2017 int
sol_uverbs_modify_srq(uverbs_uctxt_uobj_t * uctxt,char * buf,int in_len,int out_len)2018 sol_uverbs_modify_srq(uverbs_uctxt_uobj_t *uctxt, char *buf,
2019 int in_len, int out_len)
2020 {
2021 struct ib_uverbs_modify_srq cmd;
2022 uverbs_usrq_uobj_t *usrq;
2023 uint_t limit = 0;
2024 uint_t size = 0;
2025 uint_t real_size = 0;
2026 ibt_srq_modify_flags_t flags = 0;
2027 int rc;
2028
2029 (void) memcpy(&cmd, buf, sizeof (cmd));
2030
2031 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
2032 "modify_srq(): entry (srq_handle=%d)", cmd.srq_handle);
2033
2034 usrq = uverbs_uobj_get_usrq_read(cmd.srq_handle);
2035 if (usrq == NULL) {
2036 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
2037 "modify_srq(): List lookup failure");
2038 rc = EINVAL;
2039 goto err_out;
2040 }
2041
2042 if (cmd.attr_mask & IB_SRQ_MAX_WR) {
2043 flags = IBT_SRQ_SET_SIZE;
2044 size = cmd.max_wr;
2045 }
2046
2047 if (cmd.attr_mask & IB_SRQ_LIMIT) {
2048 flags |= IBT_SRQ_SET_LIMIT;
2049 limit = cmd.srq_limit;
2050 }
2051
2052 rc = ibt_modify_srq(usrq->srq, flags, size, limit, &real_size);
2053 if (rc != IBT_SUCCESS) {
2054 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
2055 "modify_srq(): Error in ibt_modify_srq() (rc=%d)", rc);
2056 rc = sol_uverbs_ibt_to_kernel_status(rc);
2057 goto err_deref;
2058 }
2059
2060 done:
2061 sol_ofs_uobj_put(&usrq->uobj);
2062 return (DDI_SUCCESS);
2063
2064 err_deref:
2065 sol_ofs_uobj_put(&usrq->uobj);
2066
2067 err_out:
2068 return (rc);
2069 }
2070
2071 /*
2072 * Function:
2073 * sol_uverbs_query_srq
2074 * Input:
2075 * uctxt - Pointer to the callers user context.
2076 * buf - Pointer to kernel buffer containing command.
2077 * in_len - Length in bytes of input command buffer.
2078 * out_len - Length in bytes of output response buffer.
2079 * Output:
2080 * The command output buffer is updated with command results.
2081 * Returns:
2082 * DDI_SUCCESS on success, else error code.
2083 * Description:
2084 * User verbs entry point to query a device shared receive queue
2085 * properties.
2086 */
2087 /* ARGSUSED */
2088 int
sol_uverbs_query_srq(uverbs_uctxt_uobj_t * uctxt,char * buf,int in_len,int out_len)2089 sol_uverbs_query_srq(uverbs_uctxt_uobj_t *uctxt, char *buf,
2090 int in_len, int out_len)
2091 {
2092 struct ib_uverbs_query_srq cmd;
2093 struct ib_uverbs_query_srq_resp resp;
2094 uverbs_usrq_uobj_t *usrq;
2095 ibt_pd_hdl_t pd;
2096 int rc;
2097 ibt_srq_sizes_t attr;
2098 uint_t limit;
2099
2100 (void) memcpy(&cmd, buf, sizeof (cmd));
2101 (void) memset(&resp, 0, sizeof (resp));
2102
2103 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
2104 "query_srq(): entry (srq_handle=%d)", cmd.srq_handle);
2105
2106 usrq = uverbs_uobj_get_usrq_read(cmd.srq_handle);
2107 if (usrq == NULL) {
2108 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
2109 "query_srq(): Invalid handle: %d", cmd.srq_handle);
2110 rc = EINVAL;
2111 goto err_out;
2112 }
2113
2114 rc = ibt_query_srq(usrq->srq, &pd, &attr, &limit);
2115 sol_ofs_uobj_put(&usrq->uobj);
2116
2117 if (rc != IBT_SUCCESS) {
2118 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
2119 "query_srq(): Error in ibt_query_srq() (rc=%d)", rc);
2120 rc = sol_uverbs_ibt_to_kernel_status(rc);
2121 goto err_out;
2122 }
2123
2124 resp.max_wr = attr.srq_wr_sz;
2125 resp.max_sge = attr.srq_sgl_sz;
2126 resp.srq_limit = limit;
2127
2128 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "query_srq() - "
2129 "max_wr=%d, max_sge=%d, limit=%d", resp.max_wr,
2130 resp.max_sge, resp.srq_limit);
2131
2132 /*
2133 * Release the reference from the find above, we leave the initial
2134 * reference placed at SRQ creation time.
2135 */
2136
2137 #ifdef _LP64
2138 rc = copyout((void*)&resp, (void*)cmd.response.r_laddr, sizeof (resp));
2139 #else
2140 rc = copyout((void*)&resp, (void*)cmd.response.r_addr, sizeof (resp));
2141 #endif
2142 if (rc != 0) {
2143 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, "query_srq() - "
2144 "copyout failure %x", rc);
2145 rc = EFAULT;
2146 goto err_out;
2147 }
2148
2149 return (DDI_SUCCESS);
2150
2151 err_out:
2152 return (rc);
2153 }
2154
2155 int
uverbs_usrq_free(uverbs_usrq_uobj_t * usrq,uverbs_uctxt_uobj_t * uctxt)2156 uverbs_usrq_free(uverbs_usrq_uobj_t *usrq, uverbs_uctxt_uobj_t *uctxt)
2157 {
2158 int rc;
2159
2160 if (!usrq->srq)
2161 goto skip_ibt_free_srq;
2162
2163 rc = ibt_free_srq(usrq->srq);
2164 if (rc != IBT_SUCCESS) {
2165 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, "usrq_free() - "
2166 "Error in ibt_free_srq() (rc=%d)", rc);
2167 rc = sol_uverbs_ibt_to_kernel_status(rc);
2168 sol_ofs_uobj_put(&usrq->uobj);
2169 return (rc);
2170 }
2171 usrq->srq = NULL;
2172
2173 skip_ibt_free_srq :
2174 sol_ofs_uobj_put(&usrq->uobj);
2175 if (usrq->list_entry) {
2176 mutex_enter(&uctxt->lock);
2177 delete_genlist(&uctxt->srq_list, usrq->list_entry);
2178 mutex_exit(&uctxt->lock);
2179 (void) sol_ofs_uobj_remove(&uverbs_usrq_uo_tbl, &usrq->uobj);
2180 }
2181 sol_ofs_uobj_deref(&usrq->uobj, sol_ofs_uobj_free);
2182
2183 return (0);
2184 }
2185
2186 /*
2187 * Function:
2188 * sol_uverbs_destroy_srq
2189 * Input:
2190 * uctxt - Pointer to the callers user context.
2191 * buf - Pointer to kernel buffer containing command.
2192 * in_len - Length in bytes of input command buffer.
2193 * out_len - Length in bytes of output response buffer.
2194 * Output:
2195 * The command output buffer is updated with command results.
2196 * Returns:
2197 * DDI_SUCCESS on success, else error code.
2198 * Description:
2199 * User verbs entry point to destroy a device shared receive queue.
2200 */
2201 /* ARGSUSED */
2202 int
sol_uverbs_destroy_srq(uverbs_uctxt_uobj_t * uctxt,char * buf,int in_len,int out_len)2203 sol_uverbs_destroy_srq(uverbs_uctxt_uobj_t *uctxt, char *buf,
2204 int in_len, int out_len)
2205 {
2206 struct ib_uverbs_destroy_srq cmd;
2207 struct ib_uverbs_destroy_srq_resp resp;
2208 uverbs_usrq_uobj_t *usrq;
2209 int rc;
2210
2211 (void) memcpy(&cmd, buf, sizeof (cmd));
2212 (void) memset(&resp, 0, sizeof (resp));
2213 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "destroy_srq() - "
2214 "srq_handle %d", cmd.srq_handle);
2215
2216 usrq = uverbs_uobj_get_usrq_write(cmd.srq_handle);
2217 if (usrq == NULL) {
2218 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
2219 "destroy_srq() : inavlid hdl %d", cmd.srq_handle);
2220 rc = EINVAL;
2221 goto err_out;
2222 }
2223
2224 uverbs_release_usrq_uevents(uctxt->async_evfile, usrq);
2225 resp.events_reported = usrq->async_events_reported;
2226 if (usrq->active_qp_cnt) {
2227 sol_ofs_uobj_put(&usrq->uobj);
2228 return (EBUSY);
2229 } else {
2230 rc = uverbs_usrq_free(usrq, uctxt);
2231 if (rc)
2232 goto err_out;
2233 }
2234
2235 #ifdef _LP64
2236 rc = copyout((void*)&resp, (void*)cmd.response.r_laddr, sizeof (resp));
2237 #else
2238 rc = copyout((void*)&resp, (void*)cmd.response.r_addr, sizeof (resp));
2239 #endif
2240 if (rc != 0) {
2241 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
2242 "destroy_srq() : copyout failure %x", rc);
2243 rc = EFAULT;
2244 goto err_out;
2245 }
2246
2247 return (DDI_SUCCESS);
2248
2249 err_out:
2250 return (rc);
2251 }
2252
2253 /*
2254 * Function:
2255 * sol_uverbs_attach_mcast
2256 * Input:
2257 * uctxt - Pointer to the callers user context.
2258 * buf - Pointer to kernel buffer containing command.
2259 * in_len - Length in bytes of input command buffer.
2260 * out_len - Length in bytes of output response buffer.
2261 * Output:
2262 * The command output buffer is updated with command results.
2263 * Returns:
2264 * DDI_SUCCESS on success, else error code.
2265 * Description:
2266 * User verbs entry point to attach a QP to a multicast group
2267 */
2268 /* ARGSUSED */
2269 int
sol_uverbs_attach_mcast(uverbs_uctxt_uobj_t * uctxt,char * buf,int in_len,int out_len)2270 sol_uverbs_attach_mcast(uverbs_uctxt_uobj_t *uctxt, char *buf,
2271 int in_len, int out_len)
2272 {
2273 struct ib_uverbs_attach_mcast cmd;
2274 uverbs_uqp_uobj_t *uqp;
2275 uverbs_mcast_entry_t *mc;
2276 llist_head_t *entry;
2277 int rc;
2278 ib_gid_t mc_gid;
2279
2280 (void) memcpy(&cmd, buf, sizeof (cmd));
2281
2282 /*
2283 * API specifies gid in network order, Solaris expects the gid
2284 * in host order, do the conversion if required.
2285 */
2286 mc_gid.gid_prefix = b2h64(*((uint64_t *)&cmd.gid[0]));
2287 mc_gid.gid_guid = b2h64(*((uint64_t *)&cmd.gid[8]));
2288
2289 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "attach_mcast(qp_handle=%d, "
2290 "mlid=0x%04x, gid=%016llx:%016llx", cmd.qp_handle, cmd.mlid,
2291 mc_gid.gid_prefix, mc_gid.gid_guid);
2292
2293 /*
2294 * Take object write to protect MC list.
2295 */
2296 uqp = uverbs_uobj_get_uqp_write(cmd.qp_handle);
2297 if (uqp == NULL) {
2298 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
2299 "attach_mcast QP not found");
2300 rc = EINVAL;
2301 goto err_out;
2302 }
2303
2304 /*
2305 * Look to see if we are already attached and if so no need to attach
2306 * again, just return good status.
2307 */
2308 list_for_each(entry, &uqp->mcast_list) {
2309 mc = (uverbs_mcast_entry_t *)entry->ptr;
2310
2311 if (cmd.mlid == mc->mcg.mc_adds_vect.av_dlid &&
2312 !memcmp(&mc_gid.gid, &mc->mcg.mc_adds_vect.av_dgid,
2313 sizeof (mc_gid.gid))) {
2314 SOL_OFS_DPRINTF_L4(sol_uverbs_dbg_str,
2315 "attach_mcast: match entry found");
2316 rc = DDI_SUCCESS;
2317 goto out_put;
2318 }
2319 }
2320
2321 mc = kmem_zalloc(sizeof (*mc), KM_NOSLEEP);
2322 if (mc == NULL) {
2323 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
2324 "attach_mcast: kmem_zalloc fail");
2325 rc = ENOMEM;
2326 goto out_put;
2327 }
2328
2329 llist_head_init(&mc->list, mc);
2330 mc->mcg.mc_adds_vect.av_dlid = cmd.mlid;
2331 bcopy(&mc_gid, &(mc->mcg.mc_adds_vect.av_dgid), sizeof (mc_gid));
2332
2333 rc = ibt_attach_mcg(uqp->qp, &mc->mcg);
2334 if (rc != IBT_SUCCESS) {
2335 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
2336 "attach_mcast: ibt_attach_mcq failed (r=%d)", rc);
2337 rc = sol_uverbs_ibt_to_kernel_status(rc);
2338 goto err_free;
2339 }
2340
2341 llist_add_tail(&mc->list, &uqp->mcast_list);
2342 sol_ofs_uobj_put(&uqp->uobj);
2343
2344 return (DDI_SUCCESS);
2345
2346 err_free:
2347 kmem_free(mc, sizeof (*mc));
2348 out_put:
2349 sol_ofs_uobj_put(&uqp->uobj);
2350 err_out:
2351 return (rc);
2352 }
2353
2354 /*
2355 * Function:
2356 * sol_uverbs_detach_mcast
2357 * Input:
2358 * uctxt - Pointer to the callers user context.
2359 * buf - Pointer to kernel buffer containing command.
2360 * in_len - Length in bytes of input command buffer.
2361 * out_len - Length in bytes of output response buffer.
2362 * Output:
2363 * The command output buffer is updated with command results.
2364 * Returns:
2365 * DDI_SUCCESS on success, else error code.
2366 * Description:
2367 * User verbs entry point to detach a QP from a multicast group
2368 */
2369 /* ARGSUSED */
2370 int
sol_uverbs_detach_mcast(uverbs_uctxt_uobj_t * uctxt,char * buf,int in_len,int out_len)2371 sol_uverbs_detach_mcast(uverbs_uctxt_uobj_t *uctxt, char *buf,
2372 int in_len, int out_len)
2373 {
2374 struct ib_uverbs_detach_mcast cmd;
2375 uverbs_uqp_uobj_t *uqp;
2376 ibt_mcg_info_t mcg;
2377 int rc;
2378 uverbs_mcast_entry_t *mc;
2379 llist_head_t *entry;
2380 llist_head_t *temp;
2381 ib_gid_t mc_gid;
2382
2383 (void) memcpy(&cmd, buf, sizeof (cmd));
2384
2385 /*
2386 * API specifies gid in network order, Solaris expects the gid
2387 * in host order, do the conversion if required.
2388 */
2389 mc_gid.gid_prefix = b2h64(*((uint64_t *)&cmd.gid[0]));
2390 mc_gid.gid_guid = b2h64(*((uint64_t *)&cmd.gid[8]));
2391
2392 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
2393 "detach_mcast: entry (qp_handle=%d, mlid=0x%04x,"
2394 "gid=%016llx:%016llx", cmd.qp_handle, cmd.mlid, mc_gid.gid_prefix,
2395 mc_gid.gid_guid);
2396
2397 (void) memset(&mcg, 0, sizeof (mcg));
2398
2399 /*
2400 * Get object write to protect mcast list.
2401 */
2402 uqp = uverbs_uobj_get_uqp_write(cmd.qp_handle);
2403 if (uqp == NULL) {
2404 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
2405 "detach_mcast(): QP hdl %x not found", cmd.qp_handle);
2406 rc = EINVAL;
2407 goto err_out;
2408 }
2409
2410 mcg.mc_adds_vect.av_dlid = cmd.mlid;
2411 mcg.mc_adds_vect.av_dgid = mc_gid;
2412
2413 rc = ibt_detach_mcg(uqp->qp, &mcg);
2414 if (rc != IBT_SUCCESS) {
2415 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
2416 "deatch_mcast(): ibt_attach_mcq failed (r=%d)", rc);
2417 rc = sol_uverbs_ibt_to_kernel_status(rc);
2418 goto err_put;
2419 }
2420
2421 /*
2422 * Find and delete MC group from the QP multicast list.
2423 */
2424 entry = uqp->mcast_list.nxt;
2425 temp = entry->nxt;
2426 while (entry != &uqp->mcast_list) {
2427 ASSERT(entry);
2428 mc = (uverbs_mcast_entry_t *)entry->ptr;
2429 ASSERT(mc);
2430
2431 if (cmd.mlid == mc->mcg.mc_adds_vect.av_dlid &&
2432 !memcmp(&mc_gid.gid, &mc->mcg.mc_adds_vect.av_dgid,
2433 sizeof (mc_gid.gid))) {
2434 llist_del(&mc->list);
2435 kmem_free(mc, sizeof (*mc));
2436 break;
2437 }
2438 entry = temp;
2439 temp = entry->nxt;
2440 }
2441
2442 sol_ofs_uobj_put(&uqp->uobj);
2443
2444 return (DDI_SUCCESS);
2445
2446 err_put:
2447 sol_ofs_uobj_put(&uqp->uobj);
2448
2449 err_out:
2450 return (rc);
2451 }
2452
2453 /*
2454 * Function:
2455 * uverbs_release_uqp_mcast_entries
2456 * Input:
2457 * uctxt - Pointer to the callers user context.
2458 * uqp - Pointer to the user QP object for which the multicast
2459 * list should be flushed.
2460 * Output:
2461 * None
2462 * Returns:
2463 * None
2464 * Description:
2465 * Release any multicast resources held by this QP. The
2466 * user context associated with the QP should be locked
2467 * externally to this routine to protect the updates to the
2468 * multicast list.
2469 */
2470 void
uverbs_detach_uqp_mcast_entries(uverbs_uqp_uobj_t * uqp)2471 uverbs_detach_uqp_mcast_entries(uverbs_uqp_uobj_t *uqp)
2472 {
2473 int rc;
2474 uverbs_mcast_entry_t *mc;
2475 llist_head_t *entry;
2476 llist_head_t *temp;
2477
2478 /*
2479 * Find and delete MC group from the QP multicast list.
2480 */
2481 entry = uqp->mcast_list.nxt;
2482 temp = entry->nxt;
2483 while (entry != &uqp->mcast_list) {
2484 ASSERT(entry);
2485 mc = (uverbs_mcast_entry_t *)entry->ptr;
2486 ASSERT(mc);
2487
2488 rc = ibt_detach_mcg(uqp->qp, &mc->mcg);
2489 if (rc != IBT_SUCCESS) {
2490 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
2491 "detach_mcast() : "
2492 "ibt_detach_mcq failed (r=%d)", rc);
2493 }
2494 llist_del(&mc->list);
2495 entry = temp;
2496 temp = entry->nxt;
2497 }
2498 }
2499
2500 /*
2501 * Function:
2502 * sol_uverbs_uqpid_to_ibt_handle
2503 * Input:
2504 * uqpid - A user verbs QP id, i.e. a QP handle that was
2505 * created via libibverbs and sol_uverbs.
2506 * Output:
2507 * None
2508 * Returns:
2509 * The ibt_qp_hdl_t associated with the user space QP handle.
2510 * -1 is returned if the id is not found.
2511 * Description:
2512 * Map the user verbs QP id to the associated IBT QP handle.
2513 */
2514 ibt_qp_hdl_t
sol_uverbs_uqpid_to_ibt_handle(uint32_t uqpid)2515 sol_uverbs_uqpid_to_ibt_handle(uint32_t uqpid)
2516 {
2517 uverbs_uqp_uobj_t *uqp;
2518 void *qphdl;
2519
2520 uqp = uverbs_uobj_get_uqp_read(uqpid);
2521 if (uqp == NULL) {
2522 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
2523 "uqpid2ibthdl: QP lookup failure: id %d", uqpid);
2524 return (NULL);
2525 }
2526 qphdl = (void *)uqp->qp;
2527 sol_ofs_uobj_put(&uqp->uobj);
2528 return (qphdl);
2529 }
2530
2531 /*
2532 * Function:
2533 * sol_uverbs_disable_user_qp_modify
2534 * Input:
2535 * uqpid - A user verbs QP id, i.e. a QP handle that was
2536 * created via libibverbs and sol_uverbs.
2537 * Output:
2538 * None
2539 * Returns:
2540 * 0 on success, EINVAL if associated QP is not found.
2541 * Description:
2542 * Inform sol_uverbs driver to ignore user qp modify
2543 * operations it receives for the specified qp. To re-enable
2544 * this capability see the function sol_uverbs_enable_user_qp_modify.
2545 */
2546 int
sol_uverbs_disable_user_qp_modify(uint32_t uqpid)2547 sol_uverbs_disable_user_qp_modify(uint32_t uqpid)
2548 {
2549 uverbs_uqp_uobj_t *uqp;
2550
2551 uqp = uverbs_uobj_get_uqp_write(uqpid);
2552 if (uqp == NULL) {
2553 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
2554 "disable_uqp_modify(%d) -lookup failure", uqpid);
2555 return (EINVAL);
2556 }
2557 uqp->disable_qp_mod = TRUE;
2558 sol_ofs_uobj_put(&uqp->uobj);
2559 return (0);
2560 }
2561
2562 /*
2563 * Function:
2564 * sol_uverbs_enable_user_qp_modify
2565 * Input:
2566 * uqpid - A user verbs QP id, i.e. a QP handle that was
2567 * created via libibverbs and sol_uverbs.
2568 * Output:
2569 * None
2570 * Returns:
2571 * 0 on success, EINVAL if associated QP is not found.
2572 * Description:
2573 * Inform sol_uverbs driver to process user qp modify
2574 * operations it receives for the specified qp. This is
2575 * the default and this routine need only be invoked if
2576 * user QP modify operations have explicitly been disabled
2577 * with sol_uverbs_disable_user_qp_modify.
2578 */
2579 int
sol_uverbs_enable_user_qp_modify(uint32_t uqpid)2580 sol_uverbs_enable_user_qp_modify(uint32_t uqpid)
2581 {
2582 uverbs_uqp_uobj_t *uqp;
2583
2584 uqp = uverbs_uobj_get_uqp_write(uqpid);
2585 if (uqp == NULL) {
2586 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
2587 "enable_uqp_modify(%d) -lookup failure", uqpid);
2588 return (EINVAL);
2589 }
2590 uqp->disable_qp_mod = FALSE;
2591 sol_ofs_uobj_put(&uqp->uobj);
2592 return (0);
2593 }
2594
2595 int
uverbs_uqpn_cq_ctrl(uint32_t uqpid,sol_uverbs_cq_ctrl_t ctrl)2596 uverbs_uqpn_cq_ctrl(uint32_t uqpid, sol_uverbs_cq_ctrl_t ctrl)
2597 {
2598 uverbs_uqp_uobj_t *uqp;
2599 uverbs_ucq_uobj_t *uscq;
2600 uverbs_ucq_uobj_t *urcq;
2601
2602 uqp = uverbs_uobj_get_uqp_write(uqpid);
2603 if (uqp == NULL) {
2604 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
2605 "uqpn_cq_ctrl(%d) -lookup failure", uqpid);
2606 return (EINVAL);
2607 }
2608 uscq = uqp->uqp_scq;
2609 urcq = uqp->uqp_rcq;
2610 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
2611 "ctrl - uqp %p, rcq %p. scq %p", uqp, urcq, uscq);
2612
2613 ASSERT(uscq);
2614 ASSERT(urcq);
2615 uverbs_cq_ctrl(uscq, ctrl);
2616 if (uscq != urcq)
2617 uverbs_cq_ctrl(urcq, ctrl);
2618 sol_ofs_uobj_put(&uqp->uobj);
2619 return (0);
2620 }
2621
2622 extern uint32_t sol_uverbs_qpnum2uqpid(uint32_t);
2623
2624 void
sol_uverbs_flush_qp(uint32_t qpnum)2625 sol_uverbs_flush_qp(uint32_t qpnum)
2626 {
2627 int32_t uqpid;
2628 ibt_status_t status;
2629 uverbs_uqp_uobj_t *uqp;
2630
2631 uqpid = sol_uverbs_qpnum2uqpid(qpnum);
2632 if (uqpid == DDI_FAILURE) {
2633 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
2634 "sol_uverbs_flush_qp(%x) - Invalid qpnum",
2635 qpnum);
2636 return;
2637 }
2638 uqp = uverbs_uobj_get_uqp_write(uqpid);
2639 if (uqp == NULL) {
2640 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
2641 "sol_uverbs_flush_qp(%x) - Invalid "
2642 "uqpid %x", qpnum, uqpid);
2643 return;
2644 }
2645
2646 if (uqp->qp) {
2647 status = ibt_flush_qp(uqp->qp);
2648 if (status != IBT_SUCCESS)
2649 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
2650 "sol_uverbs_flush_qp(%x) - "
2651 "ibt_flush_qp(%p) failed - status %d",
2652 qpnum, uqp->qp, status);
2653 sol_ofs_uobj_put(&uqp->uobj);
2654 return;
2655 } else {
2656 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
2657 "sol_uverbs_flush_qp(%x), uqpid %x -"
2658 "uqp->qp is NULL!!", qpnum, uqpid);
2659 sol_ofs_uobj_put(&uqp->uobj);
2660 return;
2661 }
2662 }
2663 static uint32_t
ibt_cep_flags2ibv(ibt_cep_flags_t ibt_flags)2664 ibt_cep_flags2ibv(ibt_cep_flags_t ibt_flags)
2665 {
2666 uint32_t ib_flags = 0;
2667
2668 if (ibt_flags & IBT_CEP_RDMA_WR)
2669 ib_flags |= IB_ACCESS_REMOTE_WRITE;
2670 if (ibt_flags & IBT_CEP_RDMA_RD)
2671 ib_flags |= IB_ACCESS_REMOTE_READ;
2672 if (ibt_flags & IBT_CEP_ATOMIC)
2673 ib_flags |= IB_ACCESS_REMOTE_ATOMIC;
2674
2675 return (ib_flags);
2676 }
2677
2678 static void
uverbs_cq_ctrl(uverbs_ucq_uobj_t * ucq,sol_uverbs_cq_ctrl_t ctrl)2679 uverbs_cq_ctrl(uverbs_ucq_uobj_t *ucq, sol_uverbs_cq_ctrl_t ctrl)
2680 {
2681 uverbs_ufile_uobj_t *ufile;
2682
2683 ufile = ucq->comp_chan;
2684 if (!ufile) {
2685 SOL_OFS_DPRINTF_L3(sol_uverbs_dbg_str,
2686 "cq_ctrl(%p), ufile NULL", ucq, ufile);
2687 return;
2688 }
2689
2690 mutex_enter(&ufile->lock);
2691 ufile->ufile_notify_enabled = ctrl;
2692
2693 if (ctrl == SOL_UVERBS2UCMA_CQ_NOTIFY_ENABLE) {
2694 if (!llist_empty(&ufile->event_list)) {
2695 cv_signal(&ufile->poll_wait);
2696 pollwakeup(&ufile->poll_head,
2697 POLLIN | POLLRDNORM);
2698 }
2699 }
2700 mutex_exit(&ufile->lock);
2701 }
2702