1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 */
26
27 #include "dapl.h"
28 #include "dapl_adapter_util.h"
29 #include "dapl_lmr_util.h"
30 #include "dapl_rmr_util.h"
31 #include "dapl_cookie.h"
32
33 #include "dapl_tavor_ibtf_impl.h"
34
35 /*
36 *
37 *
38 * MODULE: dapl_tavor_ibtf_dto.c
39 *
40 * PURPOSE: Utility routines for data transfer operations
41 *
42 */
43
44
45 /*
46 * dapls_ib_post_recv
47 *
48 * Provider specific Post RECV function
49 */
50 DAT_RETURN
dapls_ib_post_recv(IN DAPL_EP * ep_ptr,IN DAPL_COOKIE * cookie,IN DAT_COUNT num_segments,IN DAT_LMR_TRIPLET * local_iov,IN DAT_COMPLETION_FLAGS completion_flags)51 dapls_ib_post_recv(
52 IN DAPL_EP *ep_ptr,
53 IN DAPL_COOKIE *cookie,
54 IN DAT_COUNT num_segments,
55 IN DAT_LMR_TRIPLET *local_iov,
56 IN DAT_COMPLETION_FLAGS completion_flags)
57 {
58 ibt_recv_wr_t pr_wr;
59 ibt_wr_ds_t pr_sgl_arr[DAPL_MAX_IOV];
60 ibt_wr_ds_t *pr_sgl;
61 boolean_t suppress_notification;
62 DAT_COUNT total_len;
63 int retval;
64 int i;
65
66 total_len = 0;
67
68 if (ep_ptr->qp_handle == NULL) {
69 dapl_dbg_log(DAPL_DBG_TYPE_EP, "dapls_ib_post_recv: "
70 "qp_handle == NULL\n");
71 return (DAT_INVALID_PARAMETER);
72 }
73
74 /* allocate scatter-gather list on the heap if its large */
75 if (num_segments > DAPL_MAX_IOV) {
76 pr_sgl = dapl_os_alloc(num_segments * sizeof (ibt_wr_ds_t));
77 if (NULL == pr_sgl) {
78 dapl_dbg_log(DAPL_DBG_TYPE_ERR,
79 "dapls_ib_post_recv: pr_sgl alloc failed");
80 return (DAT_INSUFFICIENT_RESOURCES);
81 }
82 } else {
83 pr_sgl = pr_sgl_arr;
84 }
85
86 for (i = 0; i < num_segments; i++) {
87 pr_sgl[i].ds_va = (ib_vaddr_t)local_iov[i].virtual_address;
88 pr_sgl[i].ds_key = (ibt_lkey_t)local_iov[i].lmr_context;
89 pr_sgl[i].ds_len = (ib_msglen_t)local_iov[i].segment_length;
90
91 total_len += pr_sgl[i].ds_len;
92 dapl_dbg_log(DAPL_DBG_TYPE_EP, "dapls_ib_post_recv: "
93 "i(%d) va(%p), lmrctxt(0x%x), len(%llu)\n", i,
94 pr_sgl[i].ds_va, pr_sgl[i].ds_key, pr_sgl[i].ds_len);
95 }
96
97 if (cookie != NULL) {
98 cookie->val.dto.size = total_len;
99 dapl_dbg_log(DAPL_DBG_TYPE_EP,
100 "dapls_ib_post_recv: dto_cookie(%p), num_seg(%d), "
101 "size(%d) hkey(%016llx)\n", cookie, num_segments,
102 cookie->val.dto.size, ep_ptr->qp_handle->ep_hkey);
103 }
104
105 pr_wr.wr_id = (ibt_wrid_t)(uintptr_t)cookie;
106 pr_wr.wr_nds = (uint32_t)num_segments;
107 if (num_segments > 0) {
108 pr_wr.wr_sgl = &pr_sgl[0];
109 } else {
110 pr_wr.wr_sgl = NULL;
111 }
112
113 if (ep_ptr->param.ep_attr.recv_completion_flags &
114 DAT_COMPLETION_UNSIGNALLED_FLAG) {
115 /* This flag is used to control notification of completions */
116 suppress_notification = (completion_flags &
117 DAT_COMPLETION_UNSIGNALLED_FLAG) ? B_TRUE : B_FALSE;
118 } else {
119 /*
120 * The evd waiter will use threshold to control wakeups
121 * Hence the event notification will be done via arming the
122 * CQ so we do not need special notification generation
123 * hence set suppression to true
124 */
125 suppress_notification = B_TRUE;
126 }
127
128 retval = DAPL_RECV(ep_ptr)(ep_ptr, &pr_wr, suppress_notification);
129
130 if (retval != 0) {
131 dapl_dbg_log(DAPL_DBG_TYPE_EP,
132 "dapls_ib_post_recv: post_recv failed %s\n",
133 strerror(errno));
134 }
135
136 /* free the pr_sgl if we had allocated it */
137 if (num_segments > DAPL_MAX_IOV) {
138 dapl_os_free(pr_sgl, num_segments*sizeof (ibt_wr_ds_t));
139 }
140
141 return (retval);
142 }
143
144 /*
145 * dapls_ib_post_recv_one
146 *
147 * Provider specific Post RECV function
148 */
149 DAT_RETURN
dapls_ib_post_recv_one(IN DAPL_EP * ep_ptr,IN DAPL_COOKIE * cookie,IN DAT_LMR_TRIPLET * local_iov)150 dapls_ib_post_recv_one(
151 IN DAPL_EP *ep_ptr,
152 IN DAPL_COOKIE *cookie,
153 IN DAT_LMR_TRIPLET *local_iov)
154 {
155 ibt_recv_wr_t pr_wr;
156 ibt_wr_ds_t pr_sgl;
157 boolean_t suppress_notification;
158 DAT_COUNT total_len;
159 int retval;
160
161 if (ep_ptr->qp_handle == NULL) {
162 dapl_dbg_log(DAPL_DBG_TYPE_EP, "dapls_ib_post_recv_one: "
163 "qp_handle == NULL\n");
164 return (DAT_INVALID_PARAMETER);
165 }
166
167 pr_sgl.ds_va = (ib_vaddr_t)local_iov->virtual_address;
168 pr_sgl.ds_key = (ibt_lkey_t)local_iov->lmr_context;
169 pr_sgl.ds_len = (ib_msglen_t)local_iov->segment_length;
170
171 total_len = pr_sgl.ds_len;
172 dapl_dbg_log(DAPL_DBG_TYPE_EP, "dapls_ib_post_recv_one: "
173 "va(%p), lmrctxt(0x%x), len(%llu)\n",
174 pr_sgl.ds_va, pr_sgl.ds_key, pr_sgl.ds_len);
175
176 if (cookie != NULL) {
177 cookie->val.dto.size = total_len;
178 dapl_dbg_log(DAPL_DBG_TYPE_EP,
179 "dapls_ib_post_recv_one: dto_cookie(%p), num_seg(1), "
180 "size(%d) hkey(%016llx)\n", cookie,
181 cookie->val.dto.size, ep_ptr->qp_handle->ep_hkey);
182 }
183
184 pr_wr.wr_id = (ibt_wrid_t)(uintptr_t)cookie;
185 pr_wr.wr_nds = 1;
186 pr_wr.wr_sgl = &pr_sgl;
187
188 if (ep_ptr->param.ep_attr.recv_completion_flags &
189 DAT_COMPLETION_UNSIGNALLED_FLAG) {
190 /* This flag is used to control notification of completions */
191 suppress_notification = B_FALSE;
192 } else {
193 /*
194 * The evd waiter will use threshold to control wakeups
195 * Hence the event notification will be done via arming the
196 * CQ so we do not need special notification generation
197 * hence set suppression to true
198 */
199 suppress_notification = B_TRUE;
200 }
201
202 retval = DAPL_RECV(ep_ptr)(ep_ptr, &pr_wr, suppress_notification);
203
204 if (retval != 0) {
205 dapl_dbg_log(DAPL_DBG_TYPE_EP,
206 "dapls_ib_post_recv_one: post_recv failed %s\n",
207 strerror(errno));
208 }
209
210 return (retval);
211 }
212
213 /*
214 * dapls_ib_srq_post_recv
215 *
216 * Provider specific SRQ Post RECV function
217 */
218 DAT_RETURN
dapls_ib_post_srq(IN DAPL_SRQ * srq_ptr,IN DAPL_COOKIE * cookie,IN DAT_COUNT num_segments,IN DAT_LMR_TRIPLET * local_iov)219 dapls_ib_post_srq(
220 IN DAPL_SRQ *srq_ptr,
221 IN DAPL_COOKIE *cookie,
222 IN DAT_COUNT num_segments,
223 IN DAT_LMR_TRIPLET *local_iov)
224 {
225 ibt_recv_wr_t pr_wr;
226 ibt_wr_ds_t pr_sgl_arr[DAPL_MAX_IOV];
227 ibt_wr_ds_t *pr_sgl;
228 DAT_COUNT total_len;
229 int retval;
230 int i;
231
232 total_len = 0;
233
234 if (srq_ptr->srq_handle == NULL) {
235 dapl_dbg_log(DAPL_DBG_TYPE_EP, "dapls_ib_post_srq: "
236 "srq_handle == NULL\n");
237 return (DAT_INVALID_PARAMETER);
238 }
239
240 /* allocate scatter-gather list on the heap if its large */
241 if (num_segments > DAPL_MAX_IOV) {
242 pr_sgl = dapl_os_alloc(num_segments * sizeof (ibt_wr_ds_t));
243 if (NULL == pr_sgl) {
244 dapl_dbg_log(DAPL_DBG_TYPE_ERR,
245 "dapls_ib_post_srq: pr_sgl alloc failed");
246 return (DAT_INSUFFICIENT_RESOURCES);
247 }
248 } else {
249 pr_sgl = pr_sgl_arr;
250 }
251
252 for (i = 0; i < num_segments; i++) {
253 pr_sgl[i].ds_va = (ib_vaddr_t)local_iov[i].virtual_address;
254 pr_sgl[i].ds_key = (ibt_lkey_t)local_iov[i].lmr_context;
255 pr_sgl[i].ds_len = (ib_msglen_t)local_iov[i].segment_length;
256
257 total_len += pr_sgl[i].ds_len;
258 dapl_dbg_log(DAPL_DBG_TYPE_EP, "dapls_ib_post_srq: "
259 "i(%d) va(%p), lmrctxt(0x%x), len(%u)\n", i,
260 pr_sgl[i].ds_va, pr_sgl[i].ds_key, pr_sgl[i].ds_len);
261 }
262
263 if (cookie != NULL) {
264 cookie->val.dto.size = total_len;
265 dapl_dbg_log(DAPL_DBG_TYPE_EP,
266 "dapls_ib_post_srq: dto_cookie(%p), num_seg(%d), "
267 "size(%d) hkey(%016llx)\n", cookie, num_segments,
268 cookie->val.dto.size, srq_ptr->srq_handle->srq_hkey);
269 }
270
271 pr_wr.wr_id = (ibt_wrid_t)(uintptr_t)cookie;
272 pr_wr.wr_nds = (uint32_t)num_segments;
273 if (num_segments > 0) {
274 pr_wr.wr_sgl = &pr_sgl[0];
275 } else {
276 pr_wr.wr_sgl = NULL;
277 }
278
279 retval = DAPL_SRECV(srq_ptr)(srq_ptr, &pr_wr, B_TRUE);
280
281 if (retval != 0) {
282 dapl_dbg_log(DAPL_DBG_TYPE_EP,
283 "dapls_ib_post_srq: post_recv failed %s\n",
284 strerror(errno));
285 }
286
287 /* free the pr_sgl if we had allocated it */
288 if (num_segments > DAPL_MAX_IOV) {
289 dapl_os_free(pr_sgl, num_segments*sizeof (ibt_wr_ds_t));
290 }
291
292 return (retval);
293 }
294
295 /*
296 * dapls_ib_post_send
297 *
298 * Provider specific Post SEND function
299 */
300 DAT_RETURN
dapls_ib_post_send(IN DAPL_EP * ep_ptr,IN ib_send_op_type_t op_type,IN DAPL_COOKIE * cookie,IN DAT_COUNT num_segments,IN DAT_LMR_TRIPLET * local_iov,IN const DAT_RMR_TRIPLET * remote_iov,IN DAT_COMPLETION_FLAGS completion_flags)301 dapls_ib_post_send(IN DAPL_EP *ep_ptr,
302 IN ib_send_op_type_t op_type,
303 IN DAPL_COOKIE *cookie,
304 IN DAT_COUNT num_segments,
305 IN DAT_LMR_TRIPLET *local_iov,
306 IN const DAT_RMR_TRIPLET *remote_iov,
307 IN DAT_COMPLETION_FLAGS completion_flags)
308 {
309 ibt_send_wr_t ps_wr;
310 ibt_wr_ds_t ps_sgl_arr[DAPL_MAX_IOV];
311 ibt_wr_ds_t *ps_sgl;
312 DAT_COUNT total_len;
313 boolean_t suppress_notification;
314 int retval;
315 int i;
316
317 total_len = 0;
318 retval = DAT_SUCCESS;
319
320 if (ep_ptr->qp_handle == NULL) {
321 dapl_dbg_log(DAPL_DBG_TYPE_EP, "dapls_ib_post_send: "
322 "qp_handle == NULL\n");
323 return (DAT_INVALID_PARAMETER);
324 }
325
326 /* allocate scatter-gather list on the heap if its large */
327 if (num_segments > DAPL_MAX_IOV) {
328 ps_sgl = dapl_os_alloc(num_segments * sizeof (ibt_wr_ds_t));
329 if (NULL == ps_sgl) {
330 dapl_dbg_log(DAPL_DBG_TYPE_ERR,
331 "dapls_ib_post_send: pr_sgl alloc failed");
332 return (DAT_INSUFFICIENT_RESOURCES);
333 }
334 } else {
335 ps_sgl = ps_sgl_arr;
336 }
337
338 for (i = 0; i < num_segments; i++) {
339 ps_sgl[i].ds_va = (ib_vaddr_t)local_iov[i].virtual_address;
340 ps_sgl[i].ds_key = (ibt_lkey_t)local_iov[i].lmr_context;
341 ps_sgl[i].ds_len = (ib_msglen_t)local_iov[i].segment_length;
342 total_len += ps_sgl[i].ds_len;
343
344 dapl_dbg_log(DAPL_DBG_TYPE_EP, "dapls_ib_post_send: "
345 "i(%d), va(0x%llx), lmrctxt(0x%x), len(%u)\n",
346 i, ps_sgl[i].ds_va, ps_sgl[i].ds_key, ps_sgl[i].ds_len);
347 }
348
349 if (cookie != NULL) {
350 cookie->val.dto.size = total_len;
351 dapl_dbg_log(DAPL_DBG_TYPE_EVD,
352 "dapls_ib_post_send: op_type(%d), cookie(%p) "
353 "num_seg(%d) size(%d) hkey(%016llx)\n", op_type,
354 cookie, num_segments, cookie->val.dto.size,
355 ep_ptr->qp_handle->ep_hkey);
356 }
357
358 ps_wr.wr_id = (ibt_wrid_t)(uintptr_t)cookie;
359 /* Translate dapl flags */
360 ps_wr.wr_flags = (DAT_COMPLETION_BARRIER_FENCE_FLAG &
361 completion_flags) ? IBT_WR_SEND_FENCE : 0;
362 /* suppress completions */
363 ps_wr.wr_flags |= (DAT_COMPLETION_SUPPRESS_FLAG &
364 completion_flags) ? 0 : IBT_WR_SEND_SIGNAL;
365
366 /* Solicited wait flag is valid only for post_send */
367 if (op_type == OP_SEND) {
368 ps_wr.wr_flags |= (DAT_COMPLETION_SOLICITED_WAIT_FLAG &
369 completion_flags) ? IBT_WR_SEND_SOLICIT : 0;
370 }
371
372 ps_wr.wr_opcode = (ibt_wrc_opcode_t)op_type;
373 ps_wr.wr_nds = (uint32_t)num_segments;
374 if (num_segments > 0) {
375 ps_wr.wr_sgl = &ps_sgl[0];
376 if (op_type == OP_RDMA_READ || op_type == OP_RDMA_WRITE) {
377 if (remote_iov == NULL) {
378 /* free the ps_sgl if we had allocated it */
379 if (num_segments > DAPL_MAX_IOV) {
380 dapl_os_free(ps_sgl,
381 num_segments*sizeof (ibt_wr_ds_t));
382 }
383 dapl_dbg_log(DAPL_DBG_TYPE_EP,
384 "dapls_ib_post_send: "
385 "remote_iov == NULL\n");
386 return (DAT_INVALID_PARAMETER);
387 }
388
389 if (remote_iov->segment_length != (DAT_VLEN)total_len) {
390 /* free the ps_sgl if we had allocated it */
391 if (num_segments > DAPL_MAX_IOV) {
392 dapl_os_free(ps_sgl,
393 num_segments*sizeof (ibt_wr_ds_t));
394 }
395 dapl_dbg_log(DAPL_DBG_TYPE_EP,
396 "dapls_ib_post_send: "
397 "remote_iov length(%llu != %llu)\n",
398 (DAT_VLEN)total_len,
399 remote_iov->segment_length);
400 return (DAT_LENGTH_ERROR);
401 }
402
403 ps_wr.wr.rc.rcwr.rdma.rdma_raddr =
404 (ib_vaddr_t)remote_iov->target_address;
405 ps_wr.wr.rc.rcwr.rdma.rdma_rkey =
406 (ibt_rkey_t)remote_iov->rmr_context;
407
408 dapl_dbg_log(DAPL_DBG_TYPE_EP,
409 "dapls_ib_post_send: remote_iov taddr(0x%llx), "
410 "rmr(0x%x)\n", remote_iov->target_address,
411 remote_iov->rmr_context);
412 }
413 } else {
414 ps_wr.wr_sgl = NULL;
415 }
416
417 if (ep_ptr->param.ep_attr.recv_completion_flags &
418 DAT_COMPLETION_UNSIGNALLED_FLAG) {
419 /* This flag is used to control notification of completions */
420 suppress_notification = (completion_flags &
421 DAT_COMPLETION_UNSIGNALLED_FLAG) ? B_TRUE : B_FALSE;
422 } else {
423 /*
424 * The evd waiter will use threshold to control wakeups
425 * Hence the event notification will be done via arming the
426 * CQ so we do not need special notification generation
427 * hence set suppression to true
428 */
429 suppress_notification = B_TRUE;
430 }
431
432 retval = DAPL_SEND(ep_ptr)(ep_ptr, &ps_wr, suppress_notification);
433
434 if (retval != 0) {
435 dapl_dbg_log(DAPL_DBG_TYPE_EP,
436 "dapls_ib_post_send: post_send failed %d\n", retval);
437 }
438
439 /* free the pr_sgl if we had allocated it */
440 if (num_segments > DAPL_MAX_IOV) {
441 dapl_os_free(ps_sgl, num_segments*sizeof (ibt_wr_ds_t));
442 }
443
444 return (retval);
445 }
446
447 /*
448 * dapls_ib_post_send_one
449 *
450 * Provider specific Post SEND function - special case for the common case of
451 * sgl num_segments == 1 and completion_flags == DAT_COMPLETION_DEFAULT_FLAG.
452 */
453 DAT_RETURN
dapls_ib_post_send_one(IN DAPL_EP * ep_ptr,IN ib_send_op_type_t op_type,IN DAPL_COOKIE * cookie,IN DAT_LMR_TRIPLET * local_iov,IN const DAT_RMR_TRIPLET * remote_iov)454 dapls_ib_post_send_one(IN DAPL_EP *ep_ptr,
455 IN ib_send_op_type_t op_type,
456 IN DAPL_COOKIE *cookie,
457 IN DAT_LMR_TRIPLET *local_iov,
458 IN const DAT_RMR_TRIPLET *remote_iov)
459 {
460 ibt_send_wr_t ps_wr;
461 ibt_wr_ds_t ps_sgl;
462 boolean_t suppress_notification;
463 int retval;
464
465 if (ep_ptr->qp_handle == NULL) {
466 dapl_dbg_log(DAPL_DBG_TYPE_EP, "dapls_ib_post_send_one: "
467 "qp_handle == NULL\n");
468 return (DAT_INVALID_PARAMETER);
469 }
470
471 ps_sgl.ds_va = (ib_vaddr_t)local_iov[0].virtual_address;
472 ps_sgl.ds_key = (ibt_lkey_t)local_iov[0].lmr_context;
473 ps_sgl.ds_len = (ib_msglen_t)local_iov[0].segment_length;
474
475 dapl_dbg_log(DAPL_DBG_TYPE_EP, "dapls_ib_post_send_one: "
476 "i(%d), va(0x%llx), lmrctxt(0x%x), len(%u)\n",
477 0, ps_sgl.ds_va, ps_sgl.ds_key, ps_sgl.ds_len);
478
479 cookie->val.dto.size = ps_sgl.ds_len;
480 dapl_dbg_log(DAPL_DBG_TYPE_EVD,
481 "dapls_ib_post_send_one: op_type(%d), cookie(%p) "
482 "num_seg(%d) size(%d) hkey(%016llx)\n", op_type,
483 cookie, 1, cookie->val.dto.size,
484 ep_ptr->qp_handle->ep_hkey);
485
486 ps_wr.wr_id = (ibt_wrid_t)(uintptr_t)cookie;
487 /* suppress completions */
488 ps_wr.wr_flags = IBT_WR_SEND_SIGNAL;
489
490 ps_wr.wr_opcode = (ibt_wrc_opcode_t)op_type;
491 ps_wr.wr_nds = 1;
492
493 ps_wr.wr_sgl = &ps_sgl;
494 if (op_type == OP_RDMA_READ || op_type == OP_RDMA_WRITE) {
495 if (remote_iov == NULL) {
496 /* free the ps_sgl if we had allocated it */
497 dapl_dbg_log(DAPL_DBG_TYPE_EP,
498 "dapls_ib_post_send_one: "
499 "remote_iov == NULL\n");
500 return (DAT_INVALID_PARAMETER);
501 }
502
503 if (remote_iov->segment_length != (DAT_VLEN)ps_sgl.ds_len) {
504 dapl_dbg_log(DAPL_DBG_TYPE_EP,
505 "dapls_ib_post_send_one: "
506 "remote_iov length(%llu != %llu)\n",
507 (DAT_VLEN)ps_sgl.ds_len,
508 remote_iov->segment_length);
509 return (DAT_LENGTH_ERROR);
510 }
511
512 ps_wr.wr.rc.rcwr.rdma.rdma_raddr =
513 (ib_vaddr_t)remote_iov->target_address;
514 ps_wr.wr.rc.rcwr.rdma.rdma_rkey =
515 (ibt_rkey_t)remote_iov->rmr_context;
516
517 dapl_dbg_log(DAPL_DBG_TYPE_EP,
518 "dapls_ib_post_send_one: remote_iov taddr(0x%llx), "
519 "rmr(0x%x)\n", remote_iov->target_address,
520 remote_iov->rmr_context);
521 }
522
523 if (ep_ptr->param.ep_attr.recv_completion_flags &
524 DAT_COMPLETION_UNSIGNALLED_FLAG) {
525 /* This flag is used to control notification of completions */
526 suppress_notification = B_FALSE;
527 } else {
528 /*
529 * The evd waiter will use threshold to control wakeups
530 * Hence the event notification will be done via arming the
531 * CQ so we do not need special notification generation
532 * hence set suppression to true
533 */
534 suppress_notification = B_TRUE;
535 }
536
537 retval = DAPL_SEND(ep_ptr)(ep_ptr, &ps_wr, suppress_notification);
538
539 if (retval != 0) {
540 dapl_dbg_log(DAPL_DBG_TYPE_EP,
541 "dapls_ib_post_send_one: post_send failed %d\n", retval);
542 }
543
544 return (retval);
545 }
546