1 /*
2 * Copyright (c) 2005 Topspin Communications. All rights reserved.
3 * Copyright (c) 2005 PathScale, Inc. All rights reserved.
4 * Copyright (c) 2006 Cisco Systems, Inc. All rights reserved.
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 */
34
35 #include <config.h>
36
37 #include <stdio.h>
38 #include <unistd.h>
39 #include <stdlib.h>
40 #include <errno.h>
41 #include <string.h>
42
43 #include "ibverbs.h"
44 #include <sys/param.h>
45
ibv_cmd_get_context(struct ibv_context * context,struct ibv_get_context * cmd,size_t cmd_size,struct ibv_get_context_resp * resp,size_t resp_size)46 int ibv_cmd_get_context(struct ibv_context *context, struct ibv_get_context *cmd,
47 size_t cmd_size, struct ibv_get_context_resp *resp,
48 size_t resp_size)
49 {
50 if (abi_ver < IB_USER_VERBS_MIN_ABI_VERSION)
51 return ENOSYS;
52
53 IBV_INIT_CMD_RESP(cmd, cmd_size, GET_CONTEXT, resp, resp_size);
54
55 if (write(context->cmd_fd, cmd, cmd_size) != cmd_size)
56 return errno;
57
58 (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size);
59
60 context->async_fd = resp->async_fd;
61 context->num_comp_vectors = resp->num_comp_vectors;
62
63 return 0;
64 }
65
copy_query_dev_fields(struct ibv_device_attr * device_attr,struct ibv_query_device_resp * resp,uint64_t * raw_fw_ver)66 static void copy_query_dev_fields(struct ibv_device_attr *device_attr,
67 struct ibv_query_device_resp *resp,
68 uint64_t *raw_fw_ver)
69 {
70 *raw_fw_ver = resp->fw_ver;
71 device_attr->node_guid = resp->node_guid;
72 device_attr->sys_image_guid = resp->sys_image_guid;
73 device_attr->max_mr_size = resp->max_mr_size;
74 device_attr->page_size_cap = resp->page_size_cap;
75 device_attr->vendor_id = resp->vendor_id;
76 device_attr->vendor_part_id = resp->vendor_part_id;
77 device_attr->hw_ver = resp->hw_ver;
78 device_attr->max_qp = resp->max_qp;
79 device_attr->max_qp_wr = resp->max_qp_wr;
80 device_attr->device_cap_flags = resp->device_cap_flags;
81 device_attr->max_sge = resp->max_sge;
82 device_attr->max_sge_rd = resp->max_sge_rd;
83 device_attr->max_cq = resp->max_cq;
84 device_attr->max_cqe = resp->max_cqe;
85 device_attr->max_mr = resp->max_mr;
86 device_attr->max_pd = resp->max_pd;
87 device_attr->max_qp_rd_atom = resp->max_qp_rd_atom;
88 device_attr->max_ee_rd_atom = resp->max_ee_rd_atom;
89 device_attr->max_res_rd_atom = resp->max_res_rd_atom;
90 device_attr->max_qp_init_rd_atom = resp->max_qp_init_rd_atom;
91 device_attr->max_ee_init_rd_atom = resp->max_ee_init_rd_atom;
92 device_attr->atomic_cap = resp->atomic_cap;
93 device_attr->max_ee = resp->max_ee;
94 device_attr->max_rdd = resp->max_rdd;
95 device_attr->max_mw = resp->max_mw;
96 device_attr->max_raw_ipv6_qp = resp->max_raw_ipv6_qp;
97 device_attr->max_raw_ethy_qp = resp->max_raw_ethy_qp;
98 device_attr->max_mcast_grp = resp->max_mcast_grp;
99 device_attr->max_mcast_qp_attach = resp->max_mcast_qp_attach;
100 device_attr->max_total_mcast_qp_attach = resp->max_total_mcast_qp_attach;
101 device_attr->max_ah = resp->max_ah;
102 device_attr->max_fmr = resp->max_fmr;
103 device_attr->max_map_per_fmr = resp->max_map_per_fmr;
104 device_attr->max_srq = resp->max_srq;
105 device_attr->max_srq_wr = resp->max_srq_wr;
106 device_attr->max_srq_sge = resp->max_srq_sge;
107 device_attr->max_pkeys = resp->max_pkeys;
108 device_attr->local_ca_ack_delay = resp->local_ca_ack_delay;
109 device_attr->phys_port_cnt = resp->phys_port_cnt;
110 }
111
ibv_cmd_query_device(struct ibv_context * context,struct ibv_device_attr * device_attr,uint64_t * raw_fw_ver,struct ibv_query_device * cmd,size_t cmd_size)112 int ibv_cmd_query_device(struct ibv_context *context,
113 struct ibv_device_attr *device_attr,
114 uint64_t *raw_fw_ver,
115 struct ibv_query_device *cmd, size_t cmd_size)
116 {
117 struct ibv_query_device_resp resp;
118
119 IBV_INIT_CMD_RESP(cmd, cmd_size, QUERY_DEVICE, &resp, sizeof resp);
120
121 if (write(context->cmd_fd, cmd, cmd_size) != cmd_size)
122 return errno;
123
124 (void) VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof resp);
125
126 memset(device_attr->fw_ver, 0, sizeof device_attr->fw_ver);
127 copy_query_dev_fields(device_attr, &resp, raw_fw_ver);
128
129 return 0;
130 }
131
ibv_cmd_query_device_ex(struct ibv_context * context,const struct ibv_query_device_ex_input * input,struct ibv_device_attr_ex * attr,size_t attr_size,uint64_t * raw_fw_ver,struct ibv_query_device_ex * cmd,size_t cmd_core_size,size_t cmd_size,struct ibv_query_device_resp_ex * resp,size_t resp_core_size,size_t resp_size)132 int ibv_cmd_query_device_ex(struct ibv_context *context,
133 const struct ibv_query_device_ex_input *input,
134 struct ibv_device_attr_ex *attr, size_t attr_size,
135 uint64_t *raw_fw_ver,
136 struct ibv_query_device_ex *cmd,
137 size_t cmd_core_size,
138 size_t cmd_size,
139 struct ibv_query_device_resp_ex *resp,
140 size_t resp_core_size,
141 size_t resp_size)
142 {
143 int err;
144
145 if (input && input->comp_mask)
146 return EINVAL;
147
148 if (attr_size < offsetof(struct ibv_device_attr_ex, comp_mask) +
149 sizeof(attr->comp_mask))
150 return EINVAL;
151
152 if (resp_core_size < offsetof(struct ibv_query_device_resp_ex,
153 response_length) +
154 sizeof(resp->response_length))
155 return EINVAL;
156
157 IBV_INIT_CMD_RESP_EX_V(cmd, cmd_core_size, cmd_size,
158 QUERY_DEVICE_EX, resp, resp_core_size,
159 resp_size);
160 cmd->comp_mask = 0;
161 cmd->reserved = 0;
162 memset(attr->orig_attr.fw_ver, 0, sizeof(attr->orig_attr.fw_ver));
163 memset(&attr->comp_mask, 0, attr_size - sizeof(attr->orig_attr));
164 err = write(context->cmd_fd, cmd, cmd_size);
165 if (err != cmd_size)
166 return errno;
167
168 (void)VALGRIND_MAKE_MEM_DEFINED(resp, resp_size);
169 copy_query_dev_fields(&attr->orig_attr, &resp->base, raw_fw_ver);
170 /* Report back supported comp_mask bits. For now no comp_mask bit is
171 * defined */
172 attr->comp_mask = resp->comp_mask & 0;
173 if (attr_size >= offsetof(struct ibv_device_attr_ex, odp_caps) +
174 sizeof(attr->odp_caps)) {
175 if (resp->response_length >=
176 offsetof(struct ibv_query_device_resp_ex, odp_caps) +
177 sizeof(resp->odp_caps)) {
178 attr->odp_caps.general_caps = resp->odp_caps.general_caps;
179 attr->odp_caps.per_transport_caps.rc_odp_caps =
180 resp->odp_caps.per_transport_caps.rc_odp_caps;
181 attr->odp_caps.per_transport_caps.uc_odp_caps =
182 resp->odp_caps.per_transport_caps.uc_odp_caps;
183 attr->odp_caps.per_transport_caps.ud_odp_caps =
184 resp->odp_caps.per_transport_caps.ud_odp_caps;
185 }
186 }
187
188 if (attr_size >= offsetof(struct ibv_device_attr_ex,
189 completion_timestamp_mask) +
190 sizeof(attr->completion_timestamp_mask)) {
191 if (resp->response_length >=
192 offsetof(struct ibv_query_device_resp_ex, timestamp_mask) +
193 sizeof(resp->timestamp_mask))
194 attr->completion_timestamp_mask = resp->timestamp_mask;
195 }
196
197 if (attr_size >= offsetof(struct ibv_device_attr_ex, hca_core_clock) +
198 sizeof(attr->hca_core_clock)) {
199 if (resp->response_length >=
200 offsetof(struct ibv_query_device_resp_ex, hca_core_clock) +
201 sizeof(resp->hca_core_clock))
202 attr->hca_core_clock = resp->hca_core_clock;
203 }
204
205 if (attr_size >= offsetof(struct ibv_device_attr_ex, device_cap_flags_ex) +
206 sizeof(attr->device_cap_flags_ex)) {
207 if (resp->response_length >=
208 offsetof(struct ibv_query_device_resp_ex, device_cap_flags_ex) +
209 sizeof(resp->device_cap_flags_ex))
210 attr->device_cap_flags_ex = resp->device_cap_flags_ex;
211 }
212
213 if (attr_size >= offsetof(struct ibv_device_attr_ex, rss_caps) +
214 sizeof(attr->rss_caps)) {
215 if (resp->response_length >=
216 offsetof(struct ibv_query_device_resp_ex, rss_caps) +
217 sizeof(resp->rss_caps)) {
218 attr->rss_caps.supported_qpts = resp->rss_caps.supported_qpts;
219 attr->rss_caps.max_rwq_indirection_tables = resp->rss_caps.max_rwq_indirection_tables;
220 attr->rss_caps.max_rwq_indirection_table_size = resp->rss_caps.max_rwq_indirection_table_size;
221 }
222 }
223
224 if (attr_size >= offsetof(struct ibv_device_attr_ex, max_wq_type_rq) +
225 sizeof(attr->max_wq_type_rq)) {
226 if (resp->response_length >=
227 offsetof(struct ibv_query_device_resp_ex, max_wq_type_rq) +
228 sizeof(resp->max_wq_type_rq))
229 attr->max_wq_type_rq = resp->max_wq_type_rq;
230 }
231
232 if (attr_size >= offsetof(struct ibv_device_attr_ex, raw_packet_caps) +
233 sizeof(attr->raw_packet_caps)) {
234 if (resp->response_length >=
235 offsetof(struct ibv_query_device_resp_ex, raw_packet_caps) +
236 sizeof(resp->raw_packet_caps))
237 attr->raw_packet_caps = resp->raw_packet_caps;
238 }
239
240 return 0;
241 }
242
ibv_cmd_query_port(struct ibv_context * context,uint8_t port_num,struct ibv_port_attr * port_attr,struct ibv_query_port * cmd,size_t cmd_size)243 int ibv_cmd_query_port(struct ibv_context *context, uint8_t port_num,
244 struct ibv_port_attr *port_attr,
245 struct ibv_query_port *cmd, size_t cmd_size)
246 {
247 struct ibv_query_port_resp resp;
248
249 IBV_INIT_CMD_RESP(cmd, cmd_size, QUERY_PORT, &resp, sizeof resp);
250 cmd->port_num = port_num;
251 memset(cmd->reserved, 0, sizeof cmd->reserved);
252
253 if (write(context->cmd_fd, cmd, cmd_size) != cmd_size)
254 return errno;
255
256 (void) VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof resp);
257
258 port_attr->state = resp.state;
259 port_attr->max_mtu = resp.max_mtu;
260 port_attr->active_mtu = resp.active_mtu;
261 port_attr->gid_tbl_len = resp.gid_tbl_len;
262 port_attr->port_cap_flags = resp.port_cap_flags;
263 port_attr->max_msg_sz = resp.max_msg_sz;
264 port_attr->bad_pkey_cntr = resp.bad_pkey_cntr;
265 port_attr->qkey_viol_cntr = resp.qkey_viol_cntr;
266 port_attr->pkey_tbl_len = resp.pkey_tbl_len;
267 port_attr->lid = resp.lid;
268 port_attr->sm_lid = resp.sm_lid;
269 port_attr->lmc = resp.lmc;
270 port_attr->max_vl_num = resp.max_vl_num;
271 port_attr->sm_sl = resp.sm_sl;
272 port_attr->subnet_timeout = resp.subnet_timeout;
273 port_attr->init_type_reply = resp.init_type_reply;
274 port_attr->active_width = resp.active_width;
275 port_attr->active_speed = resp.active_speed;
276 port_attr->phys_state = resp.phys_state;
277 port_attr->link_layer = resp.link_layer;
278
279 return 0;
280 }
281
ibv_cmd_alloc_pd(struct ibv_context * context,struct ibv_pd * pd,struct ibv_alloc_pd * cmd,size_t cmd_size,struct ibv_alloc_pd_resp * resp,size_t resp_size)282 int ibv_cmd_alloc_pd(struct ibv_context *context, struct ibv_pd *pd,
283 struct ibv_alloc_pd *cmd, size_t cmd_size,
284 struct ibv_alloc_pd_resp *resp, size_t resp_size)
285 {
286 IBV_INIT_CMD_RESP(cmd, cmd_size, ALLOC_PD, resp, resp_size);
287
288 if (write(context->cmd_fd, cmd, cmd_size) != cmd_size)
289 return errno;
290
291 (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size);
292
293 pd->handle = resp->pd_handle;
294 pd->context = context;
295
296 return 0;
297 }
298
ibv_cmd_dealloc_pd(struct ibv_pd * pd)299 int ibv_cmd_dealloc_pd(struct ibv_pd *pd)
300 {
301 struct ibv_dealloc_pd cmd;
302
303 IBV_INIT_CMD(&cmd, sizeof cmd, DEALLOC_PD);
304 cmd.pd_handle = pd->handle;
305
306 if (write(pd->context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd)
307 return errno;
308
309 return 0;
310 }
311
ibv_cmd_open_xrcd(struct ibv_context * context,struct verbs_xrcd * xrcd,int vxrcd_size,struct ibv_xrcd_init_attr * attr,struct ibv_open_xrcd * cmd,size_t cmd_size,struct ibv_open_xrcd_resp * resp,size_t resp_size)312 int ibv_cmd_open_xrcd(struct ibv_context *context, struct verbs_xrcd *xrcd,
313 int vxrcd_size,
314 struct ibv_xrcd_init_attr *attr,
315 struct ibv_open_xrcd *cmd, size_t cmd_size,
316 struct ibv_open_xrcd_resp *resp, size_t resp_size)
317 {
318 IBV_INIT_CMD_RESP(cmd, cmd_size, OPEN_XRCD, resp, resp_size);
319
320 if (attr->comp_mask >= IBV_XRCD_INIT_ATTR_RESERVED)
321 return ENOSYS;
322
323 if (!(attr->comp_mask & IBV_XRCD_INIT_ATTR_FD) ||
324 !(attr->comp_mask & IBV_XRCD_INIT_ATTR_OFLAGS))
325 return EINVAL;
326
327 cmd->fd = attr->fd;
328 cmd->oflags = attr->oflags;
329 if (write(context->cmd_fd, cmd, cmd_size) != cmd_size)
330 return errno;
331
332 (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size);
333
334 xrcd->xrcd.context = context;
335 xrcd->comp_mask = 0;
336 if (vext_field_avail(struct verbs_xrcd, handle, vxrcd_size)) {
337 xrcd->comp_mask = VERBS_XRCD_HANDLE;
338 xrcd->handle = resp->xrcd_handle;
339 }
340
341 return 0;
342 }
343
ibv_cmd_close_xrcd(struct verbs_xrcd * xrcd)344 int ibv_cmd_close_xrcd(struct verbs_xrcd *xrcd)
345 {
346 struct ibv_close_xrcd cmd;
347
348 IBV_INIT_CMD(&cmd, sizeof cmd, CLOSE_XRCD);
349 cmd.xrcd_handle = xrcd->handle;
350
351 if (write(xrcd->xrcd.context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd)
352 return errno;
353
354 return 0;
355 }
356
ibv_cmd_reg_mr(struct ibv_pd * pd,void * addr,size_t length,uint64_t hca_va,int access,struct ibv_mr * mr,struct ibv_reg_mr * cmd,size_t cmd_size,struct ibv_reg_mr_resp * resp,size_t resp_size)357 int ibv_cmd_reg_mr(struct ibv_pd *pd, void *addr, size_t length,
358 uint64_t hca_va, int access,
359 struct ibv_mr *mr, struct ibv_reg_mr *cmd,
360 size_t cmd_size,
361 struct ibv_reg_mr_resp *resp, size_t resp_size)
362 {
363
364 IBV_INIT_CMD_RESP(cmd, cmd_size, REG_MR, resp, resp_size);
365
366 cmd->start = (uintptr_t) addr;
367 cmd->length = length;
368 cmd->hca_va = hca_va;
369 cmd->pd_handle = pd->handle;
370 cmd->access_flags = access;
371
372 if (write(pd->context->cmd_fd, cmd, cmd_size) != cmd_size)
373 return errno;
374
375 (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size);
376
377 mr->handle = resp->mr_handle;
378 mr->lkey = resp->lkey;
379 mr->rkey = resp->rkey;
380 mr->context = pd->context;
381
382 return 0;
383 }
384
ibv_cmd_rereg_mr(struct ibv_mr * mr,uint32_t flags,void * addr,size_t length,uint64_t hca_va,int access,struct ibv_pd * pd,struct ibv_rereg_mr * cmd,size_t cmd_sz,struct ibv_rereg_mr_resp * resp,size_t resp_sz)385 int ibv_cmd_rereg_mr(struct ibv_mr *mr, uint32_t flags, void *addr,
386 size_t length, uint64_t hca_va, int access,
387 struct ibv_pd *pd, struct ibv_rereg_mr *cmd,
388 size_t cmd_sz, struct ibv_rereg_mr_resp *resp,
389 size_t resp_sz)
390 {
391 IBV_INIT_CMD_RESP(cmd, cmd_sz, REREG_MR, resp, resp_sz);
392
393 cmd->mr_handle = mr->handle;
394 cmd->flags = flags;
395 cmd->start = (uintptr_t)addr;
396 cmd->length = length;
397 cmd->hca_va = hca_va;
398 cmd->pd_handle = (flags & IBV_REREG_MR_CHANGE_PD) ? pd->handle : 0;
399 cmd->access_flags = access;
400
401 if (write(mr->context->cmd_fd, cmd, cmd_sz) != cmd_sz)
402 return errno;
403
404 (void)VALGRIND_MAKE_MEM_DEFINED(resp, resp_sz);
405
406 mr->lkey = resp->lkey;
407 mr->rkey = resp->rkey;
408 if (flags & IBV_REREG_MR_CHANGE_PD)
409 mr->context = pd->context;
410
411 return 0;
412 }
413
ibv_cmd_dereg_mr(struct ibv_mr * mr)414 int ibv_cmd_dereg_mr(struct ibv_mr *mr)
415 {
416 struct ibv_dereg_mr cmd;
417
418 IBV_INIT_CMD(&cmd, sizeof cmd, DEREG_MR);
419 cmd.mr_handle = mr->handle;
420
421 if (write(mr->context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd)
422 return errno;
423
424 return 0;
425 }
426
ibv_cmd_alloc_mw(struct ibv_pd * pd,enum ibv_mw_type type,struct ibv_mw * mw,struct ibv_alloc_mw * cmd,size_t cmd_size,struct ibv_alloc_mw_resp * resp,size_t resp_size)427 int ibv_cmd_alloc_mw(struct ibv_pd *pd, enum ibv_mw_type type,
428 struct ibv_mw *mw, struct ibv_alloc_mw *cmd,
429 size_t cmd_size,
430 struct ibv_alloc_mw_resp *resp, size_t resp_size)
431 {
432 IBV_INIT_CMD_RESP(cmd, cmd_size, ALLOC_MW, resp, resp_size);
433 cmd->pd_handle = pd->handle;
434 cmd->mw_type = type;
435 memset(cmd->reserved, 0, sizeof(cmd->reserved));
436
437 if (write(pd->context->cmd_fd, cmd, cmd_size) != cmd_size)
438 return errno;
439
440 (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size);
441
442 mw->context = pd->context;
443 mw->pd = pd;
444 mw->rkey = resp->rkey;
445 mw->handle = resp->mw_handle;
446 mw->type = type;
447
448 return 0;
449 }
450
ibv_cmd_dealloc_mw(struct ibv_mw * mw,struct ibv_dealloc_mw * cmd,size_t cmd_size)451 int ibv_cmd_dealloc_mw(struct ibv_mw *mw,
452 struct ibv_dealloc_mw *cmd, size_t cmd_size)
453 {
454 IBV_INIT_CMD(cmd, cmd_size, DEALLOC_MW);
455 cmd->mw_handle = mw->handle;
456 cmd->reserved = 0;
457
458 if (write(mw->context->cmd_fd, cmd, cmd_size) != cmd_size)
459 return errno;
460
461 return 0;
462 }
463
ibv_cmd_create_cq(struct ibv_context * context,int cqe,struct ibv_comp_channel * channel,int comp_vector,struct ibv_cq * cq,struct ibv_create_cq * cmd,size_t cmd_size,struct ibv_create_cq_resp * resp,size_t resp_size)464 int ibv_cmd_create_cq(struct ibv_context *context, int cqe,
465 struct ibv_comp_channel *channel,
466 int comp_vector, struct ibv_cq *cq,
467 struct ibv_create_cq *cmd, size_t cmd_size,
468 struct ibv_create_cq_resp *resp, size_t resp_size)
469 {
470 IBV_INIT_CMD_RESP(cmd, cmd_size, CREATE_CQ, resp, resp_size);
471 cmd->user_handle = (uintptr_t) cq;
472 cmd->cqe = cqe;
473 cmd->comp_vector = comp_vector;
474 cmd->comp_channel = channel ? channel->fd : -1;
475 cmd->reserved = 0;
476
477 if (write(context->cmd_fd, cmd, cmd_size) != cmd_size)
478 return errno;
479
480 (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size);
481
482 cq->handle = resp->cq_handle;
483 cq->cqe = resp->cqe;
484 cq->context = context;
485
486 return 0;
487 }
488
ibv_cmd_create_cq_ex(struct ibv_context * context,struct ibv_cq_init_attr_ex * cq_attr,struct ibv_cq_ex * cq,struct ibv_create_cq_ex * cmd,size_t cmd_core_size,size_t cmd_size,struct ibv_create_cq_resp_ex * resp,size_t resp_core_size,size_t resp_size)489 int ibv_cmd_create_cq_ex(struct ibv_context *context,
490 struct ibv_cq_init_attr_ex *cq_attr,
491 struct ibv_cq_ex *cq,
492 struct ibv_create_cq_ex *cmd,
493 size_t cmd_core_size,
494 size_t cmd_size,
495 struct ibv_create_cq_resp_ex *resp,
496 size_t resp_core_size,
497 size_t resp_size)
498 {
499 int err;
500
501 memset(cmd, 0, cmd_core_size);
502 IBV_INIT_CMD_RESP_EX_V(cmd, cmd_core_size, cmd_size, CREATE_CQ_EX, resp,
503 resp_core_size, resp_size);
504
505 if (cq_attr->comp_mask & ~(IBV_CQ_INIT_ATTR_MASK_RESERVED - 1))
506 return EINVAL;
507
508 cmd->user_handle = (uintptr_t)cq;
509 cmd->cqe = cq_attr->cqe;
510 cmd->comp_vector = cq_attr->comp_vector;
511 cmd->comp_channel = cq_attr->channel ? cq_attr->channel->fd : -1;
512 cmd->comp_mask = 0;
513
514 if (cmd_core_size >= offsetof(struct ibv_create_cq_ex, flags) +
515 sizeof(cmd->flags)) {
516 if ((cq_attr->comp_mask & IBV_CQ_INIT_ATTR_MASK_FLAGS) &&
517 (cq_attr->flags & ~(IBV_CREATE_CQ_ATTR_RESERVED - 1)))
518 return EOPNOTSUPP;
519
520 if (cq_attr->wc_flags & IBV_WC_EX_WITH_COMPLETION_TIMESTAMP)
521 cmd->flags |= IBV_CREATE_CQ_EX_KERNEL_FLAG_COMPLETION_TIMESTAMP;
522 }
523
524 err = write(context->cmd_fd, cmd, cmd_size);
525 if (err != cmd_size)
526 return errno;
527
528 (void)VALGRIND_MAKE_MEM_DEFINED(resp, resp_size);
529
530 cq->handle = resp->base.cq_handle;
531 cq->cqe = resp->base.cqe;
532 cq->context = context;
533
534 return 0;
535 }
536
ibv_cmd_poll_cq(struct ibv_cq * ibcq,int ne,struct ibv_wc * wc)537 int ibv_cmd_poll_cq(struct ibv_cq *ibcq, int ne, struct ibv_wc *wc)
538 {
539 struct ibv_poll_cq cmd;
540 struct ibv_poll_cq_resp *resp;
541 int i;
542 int rsize;
543 int ret;
544
545 rsize = sizeof *resp + ne * sizeof(struct ibv_kern_wc);
546 resp = malloc(rsize);
547 if (!resp)
548 return -1;
549
550 IBV_INIT_CMD_RESP(&cmd, sizeof cmd, POLL_CQ, resp, rsize);
551 cmd.cq_handle = ibcq->handle;
552 cmd.ne = ne;
553
554 if (write(ibcq->context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd) {
555 ret = -1;
556 goto out;
557 }
558
559 (void) VALGRIND_MAKE_MEM_DEFINED(resp, rsize);
560
561 for (i = 0; i < resp->count; i++) {
562 wc[i].wr_id = resp->wc[i].wr_id;
563 wc[i].status = resp->wc[i].status;
564 wc[i].opcode = resp->wc[i].opcode;
565 wc[i].vendor_err = resp->wc[i].vendor_err;
566 wc[i].byte_len = resp->wc[i].byte_len;
567 wc[i].imm_data = resp->wc[i].imm_data;
568 wc[i].qp_num = resp->wc[i].qp_num;
569 wc[i].src_qp = resp->wc[i].src_qp;
570 wc[i].wc_flags = resp->wc[i].wc_flags;
571 wc[i].pkey_index = resp->wc[i].pkey_index;
572 wc[i].slid = resp->wc[i].slid;
573 wc[i].sl = resp->wc[i].sl;
574 wc[i].dlid_path_bits = resp->wc[i].dlid_path_bits;
575 }
576
577 ret = resp->count;
578
579 out:
580 free(resp);
581 return ret;
582 }
583
ibv_cmd_req_notify_cq(struct ibv_cq * ibcq,int solicited_only)584 int ibv_cmd_req_notify_cq(struct ibv_cq *ibcq, int solicited_only)
585 {
586 struct ibv_req_notify_cq cmd;
587
588 IBV_INIT_CMD(&cmd, sizeof cmd, REQ_NOTIFY_CQ);
589 cmd.cq_handle = ibcq->handle;
590 cmd.solicited = !!solicited_only;
591
592 if (write(ibcq->context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd)
593 return errno;
594
595 return 0;
596 }
597
ibv_cmd_resize_cq(struct ibv_cq * cq,int cqe,struct ibv_resize_cq * cmd,size_t cmd_size,struct ibv_resize_cq_resp * resp,size_t resp_size)598 int ibv_cmd_resize_cq(struct ibv_cq *cq, int cqe,
599 struct ibv_resize_cq *cmd, size_t cmd_size,
600 struct ibv_resize_cq_resp *resp, size_t resp_size)
601 {
602 IBV_INIT_CMD_RESP(cmd, cmd_size, RESIZE_CQ, resp, resp_size);
603 cmd->cq_handle = cq->handle;
604 cmd->cqe = cqe;
605
606 if (write(cq->context->cmd_fd, cmd, cmd_size) != cmd_size)
607 return errno;
608
609 (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size);
610
611 cq->cqe = resp->cqe;
612
613 return 0;
614 }
615
ibv_cmd_destroy_cq(struct ibv_cq * cq)616 int ibv_cmd_destroy_cq(struct ibv_cq *cq)
617 {
618 struct ibv_destroy_cq cmd;
619 struct ibv_destroy_cq_resp resp;
620
621 IBV_INIT_CMD_RESP(&cmd, sizeof cmd, DESTROY_CQ, &resp, sizeof resp);
622 cmd.cq_handle = cq->handle;
623 cmd.reserved = 0;
624
625 if (write(cq->context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd)
626 return errno;
627
628 (void) VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof resp);
629
630 pthread_mutex_lock(&cq->mutex);
631 while (cq->comp_events_completed != resp.comp_events_reported ||
632 cq->async_events_completed != resp.async_events_reported)
633 pthread_cond_wait(&cq->cond, &cq->mutex);
634 pthread_mutex_unlock(&cq->mutex);
635
636 return 0;
637 }
638
ibv_cmd_create_srq(struct ibv_pd * pd,struct ibv_srq * srq,struct ibv_srq_init_attr * attr,struct ibv_create_srq * cmd,size_t cmd_size,struct ibv_create_srq_resp * resp,size_t resp_size)639 int ibv_cmd_create_srq(struct ibv_pd *pd,
640 struct ibv_srq *srq, struct ibv_srq_init_attr *attr,
641 struct ibv_create_srq *cmd, size_t cmd_size,
642 struct ibv_create_srq_resp *resp, size_t resp_size)
643 {
644 IBV_INIT_CMD_RESP(cmd, cmd_size, CREATE_SRQ, resp, resp_size);
645 cmd->user_handle = (uintptr_t) srq;
646 cmd->pd_handle = pd->handle;
647 cmd->max_wr = attr->attr.max_wr;
648 cmd->max_sge = attr->attr.max_sge;
649 cmd->srq_limit = attr->attr.srq_limit;
650
651 if (write(pd->context->cmd_fd, cmd, cmd_size) != cmd_size)
652 return errno;
653
654 (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size);
655
656 srq->handle = resp->srq_handle;
657 srq->context = pd->context;
658
659 if (abi_ver > 5) {
660 attr->attr.max_wr = resp->max_wr;
661 attr->attr.max_sge = resp->max_sge;
662 } else {
663 struct ibv_create_srq_resp_v5 *resp_v5 =
664 (struct ibv_create_srq_resp_v5 *) resp;
665
666 memmove((void *) resp + sizeof *resp,
667 (void *) resp_v5 + sizeof *resp_v5,
668 resp_size - sizeof *resp);
669 }
670
671 return 0;
672 }
673
ibv_cmd_create_srq_ex(struct ibv_context * context,struct verbs_srq * srq,int vsrq_sz,struct ibv_srq_init_attr_ex * attr_ex,struct ibv_create_xsrq * cmd,size_t cmd_size,struct ibv_create_srq_resp * resp,size_t resp_size)674 int ibv_cmd_create_srq_ex(struct ibv_context *context,
675 struct verbs_srq *srq, int vsrq_sz,
676 struct ibv_srq_init_attr_ex *attr_ex,
677 struct ibv_create_xsrq *cmd, size_t cmd_size,
678 struct ibv_create_srq_resp *resp, size_t resp_size)
679 {
680 struct verbs_xrcd *vxrcd = NULL;
681 int ret = 0;
682
683 IBV_INIT_CMD_RESP(cmd, cmd_size, CREATE_XSRQ, resp, resp_size);
684
685 if (attr_ex->comp_mask >= IBV_SRQ_INIT_ATTR_RESERVED)
686 return ENOSYS;
687
688 if (!(attr_ex->comp_mask & IBV_SRQ_INIT_ATTR_PD))
689 return EINVAL;
690
691 cmd->user_handle = (uintptr_t) srq;
692 cmd->pd_handle = attr_ex->pd->handle;
693 cmd->max_wr = attr_ex->attr.max_wr;
694 cmd->max_sge = attr_ex->attr.max_sge;
695 cmd->srq_limit = attr_ex->attr.srq_limit;
696
697 cmd->srq_type = (attr_ex->comp_mask & IBV_SRQ_INIT_ATTR_TYPE) ?
698 attr_ex->srq_type : IBV_SRQT_BASIC;
699 if (attr_ex->comp_mask & IBV_SRQ_INIT_ATTR_XRCD) {
700 if (!(attr_ex->comp_mask & IBV_SRQ_INIT_ATTR_CQ))
701 return EINVAL;
702
703 vxrcd = container_of(attr_ex->xrcd, struct verbs_xrcd, xrcd);
704 cmd->xrcd_handle = vxrcd->handle;
705 cmd->cq_handle = attr_ex->cq->handle;
706 }
707
708 ret = pthread_mutex_init(&srq->srq.mutex, NULL);
709 if (ret)
710 goto err;
711 ret = pthread_cond_init(&srq->srq.cond, NULL);
712 if (ret)
713 goto err_mutex;
714
715 if (write(context->cmd_fd, cmd, cmd_size) != cmd_size) {
716 ret = errno;
717 goto err_cond;
718 }
719
720 (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size);
721
722 srq->srq.handle = resp->srq_handle;
723 srq->srq.context = context;
724 srq->srq.srq_context = attr_ex->srq_context;
725 srq->srq.pd = attr_ex->pd;
726 srq->srq.events_completed = 0;
727
728 /*
729 * check that the last field is available.
730 * If it is than all the others exist as well
731 */
732 if (vext_field_avail(struct verbs_srq, srq_num, vsrq_sz)) {
733 srq->comp_mask = IBV_SRQ_INIT_ATTR_TYPE;
734 srq->srq_type = (attr_ex->comp_mask & IBV_SRQ_INIT_ATTR_TYPE) ?
735 attr_ex->srq_type : IBV_SRQT_BASIC;
736 if (srq->srq_type == IBV_SRQT_XRC) {
737 srq->comp_mask |= VERBS_SRQ_NUM;
738 srq->srq_num = resp->srqn;
739 }
740 if (attr_ex->comp_mask & IBV_SRQ_INIT_ATTR_XRCD) {
741 srq->comp_mask |= VERBS_SRQ_XRCD;
742 srq->xrcd = vxrcd;
743 }
744 if (attr_ex->comp_mask & IBV_SRQ_INIT_ATTR_CQ) {
745 srq->comp_mask |= VERBS_SRQ_CQ;
746 srq->cq = attr_ex->cq;
747 }
748 }
749
750 attr_ex->attr.max_wr = resp->max_wr;
751 attr_ex->attr.max_sge = resp->max_sge;
752
753 return 0;
754 err_cond:
755 pthread_cond_destroy(&srq->srq.cond);
756 err_mutex:
757 pthread_mutex_destroy(&srq->srq.mutex);
758 err:
759 return ret;
760 }
761
762
ibv_cmd_modify_srq_v3(struct ibv_srq * srq,struct ibv_srq_attr * srq_attr,int srq_attr_mask,struct ibv_modify_srq * new_cmd,size_t new_cmd_size)763 static int ibv_cmd_modify_srq_v3(struct ibv_srq *srq,
764 struct ibv_srq_attr *srq_attr,
765 int srq_attr_mask,
766 struct ibv_modify_srq *new_cmd,
767 size_t new_cmd_size)
768 {
769 struct ibv_modify_srq_v3 *cmd;
770 size_t cmd_size;
771
772 cmd_size = sizeof *cmd + new_cmd_size - sizeof *new_cmd;
773 cmd = alloca(cmd_size);
774 memcpy(cmd->driver_data, new_cmd->driver_data, new_cmd_size - sizeof *new_cmd);
775
776 IBV_INIT_CMD(cmd, cmd_size, MODIFY_SRQ);
777
778 cmd->srq_handle = srq->handle;
779 cmd->attr_mask = srq_attr_mask;
780 cmd->max_wr = srq_attr->max_wr;
781 cmd->srq_limit = srq_attr->srq_limit;
782 cmd->max_sge = 0;
783 cmd->reserved = 0;
784
785 if (write(srq->context->cmd_fd, cmd, cmd_size) != cmd_size)
786 return errno;
787
788 return 0;
789 }
790
ibv_cmd_modify_srq(struct ibv_srq * srq,struct ibv_srq_attr * srq_attr,int srq_attr_mask,struct ibv_modify_srq * cmd,size_t cmd_size)791 int ibv_cmd_modify_srq(struct ibv_srq *srq,
792 struct ibv_srq_attr *srq_attr,
793 int srq_attr_mask,
794 struct ibv_modify_srq *cmd, size_t cmd_size)
795 {
796 if (abi_ver == 3)
797 return ibv_cmd_modify_srq_v3(srq, srq_attr, srq_attr_mask,
798 cmd, cmd_size);
799
800 IBV_INIT_CMD(cmd, cmd_size, MODIFY_SRQ);
801
802 cmd->srq_handle = srq->handle;
803 cmd->attr_mask = srq_attr_mask;
804 cmd->max_wr = srq_attr->max_wr;
805 cmd->srq_limit = srq_attr->srq_limit;
806
807 if (write(srq->context->cmd_fd, cmd, cmd_size) != cmd_size)
808 return errno;
809
810 return 0;
811 }
812
ibv_cmd_query_srq(struct ibv_srq * srq,struct ibv_srq_attr * srq_attr,struct ibv_query_srq * cmd,size_t cmd_size)813 int ibv_cmd_query_srq(struct ibv_srq *srq, struct ibv_srq_attr *srq_attr,
814 struct ibv_query_srq *cmd, size_t cmd_size)
815 {
816 struct ibv_query_srq_resp resp;
817
818 IBV_INIT_CMD_RESP(cmd, cmd_size, QUERY_SRQ, &resp, sizeof resp);
819 cmd->srq_handle = srq->handle;
820 cmd->reserved = 0;
821
822 if (write(srq->context->cmd_fd, cmd, cmd_size) != cmd_size)
823 return errno;
824
825 (void) VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof resp);
826
827 srq_attr->max_wr = resp.max_wr;
828 srq_attr->max_sge = resp.max_sge;
829 srq_attr->srq_limit = resp.srq_limit;
830
831 return 0;
832 }
833
ibv_cmd_destroy_srq(struct ibv_srq * srq)834 int ibv_cmd_destroy_srq(struct ibv_srq *srq)
835 {
836 struct ibv_destroy_srq cmd;
837 struct ibv_destroy_srq_resp resp;
838
839 IBV_INIT_CMD_RESP(&cmd, sizeof cmd, DESTROY_SRQ, &resp, sizeof resp);
840 cmd.srq_handle = srq->handle;
841 cmd.reserved = 0;
842
843 if (write(srq->context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd)
844 return errno;
845
846 (void) VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof resp);
847
848 pthread_mutex_lock(&srq->mutex);
849 while (srq->events_completed != resp.events_reported)
850 pthread_cond_wait(&srq->cond, &srq->mutex);
851 pthread_mutex_unlock(&srq->mutex);
852
853 pthread_cond_destroy(&srq->cond);
854 pthread_mutex_destroy(&srq->mutex);
855
856 return 0;
857 }
858
create_qp_ex_common(struct verbs_qp * qp,struct ibv_qp_init_attr_ex * qp_attr,struct verbs_xrcd * vxrcd,struct ibv_create_qp_common * cmd)859 static int create_qp_ex_common(struct verbs_qp *qp,
860 struct ibv_qp_init_attr_ex *qp_attr,
861 struct verbs_xrcd *vxrcd,
862 struct ibv_create_qp_common *cmd)
863 {
864 cmd->user_handle = (uintptr_t)qp;
865
866 if (qp_attr->comp_mask & IBV_QP_INIT_ATTR_XRCD) {
867 vxrcd = container_of(qp_attr->xrcd, struct verbs_xrcd, xrcd);
868 cmd->pd_handle = vxrcd->handle;
869 } else {
870 if (!(qp_attr->comp_mask & IBV_QP_INIT_ATTR_PD))
871 return EINVAL;
872
873 cmd->pd_handle = qp_attr->pd->handle;
874 if (qp_attr->comp_mask & IBV_QP_INIT_ATTR_IND_TABLE) {
875 if (cmd->max_recv_wr || cmd->max_recv_sge ||
876 cmd->recv_cq_handle || qp_attr->srq)
877 return EINVAL;
878
879 /* send_cq is optinal */
880 if (qp_attr->cap.max_send_wr)
881 cmd->send_cq_handle = qp_attr->send_cq->handle;
882 } else {
883 cmd->send_cq_handle = qp_attr->send_cq->handle;
884
885 if (qp_attr->qp_type != IBV_QPT_XRC_SEND) {
886 cmd->recv_cq_handle = qp_attr->recv_cq->handle;
887 cmd->srq_handle = qp_attr->srq ? qp_attr->srq->handle :
888 0;
889 }
890 }
891 }
892
893 cmd->max_send_wr = qp_attr->cap.max_send_wr;
894 cmd->max_recv_wr = qp_attr->cap.max_recv_wr;
895 cmd->max_send_sge = qp_attr->cap.max_send_sge;
896 cmd->max_recv_sge = qp_attr->cap.max_recv_sge;
897 cmd->max_inline_data = qp_attr->cap.max_inline_data;
898 cmd->sq_sig_all = qp_attr->sq_sig_all;
899 cmd->qp_type = qp_attr->qp_type;
900 cmd->is_srq = !!qp_attr->srq;
901 cmd->reserved = 0;
902
903 return 0;
904 }
905
create_qp_handle_resp_common_cleanup(struct verbs_qp * qp)906 static void create_qp_handle_resp_common_cleanup(struct verbs_qp *qp)
907 {
908 pthread_cond_destroy(&qp->qp.cond);
909 pthread_mutex_destroy(&qp->qp.mutex);
910 }
911
create_qp_handle_resp_common_init(struct verbs_qp * qp)912 static int create_qp_handle_resp_common_init(struct verbs_qp *qp)
913 {
914 int ret = 0;
915
916 ret = pthread_mutex_init(&qp->qp.mutex, NULL);
917 if (ret)
918 return ret;
919 ret = pthread_cond_init(&qp->qp.cond, NULL);
920 if (ret)
921 goto err;
922
923 return ret;
924
925 err:
926 pthread_mutex_destroy(&qp->qp.mutex);
927
928 return ret;
929 }
930
create_qp_handle_resp_common(struct ibv_context * context,struct verbs_qp * qp,struct ibv_qp_init_attr_ex * qp_attr,struct ibv_create_qp_resp * resp,struct verbs_xrcd * vxrcd,int vqp_sz)931 static void create_qp_handle_resp_common(struct ibv_context *context,
932 struct verbs_qp *qp,
933 struct ibv_qp_init_attr_ex *qp_attr,
934 struct ibv_create_qp_resp *resp,
935 struct verbs_xrcd *vxrcd,
936 int vqp_sz)
937 {
938 if (abi_ver > 3) {
939 qp_attr->cap.max_recv_sge = resp->max_recv_sge;
940 qp_attr->cap.max_send_sge = resp->max_send_sge;
941 qp_attr->cap.max_recv_wr = resp->max_recv_wr;
942 qp_attr->cap.max_send_wr = resp->max_send_wr;
943 qp_attr->cap.max_inline_data = resp->max_inline_data;
944 }
945
946 qp->qp.handle = resp->qp_handle;
947 qp->qp.qp_num = resp->qpn;
948 qp->qp.context = context;
949 qp->qp.qp_context = qp_attr->qp_context;
950 qp->qp.pd = qp_attr->pd;
951 qp->qp.send_cq = qp_attr->send_cq;
952 qp->qp.recv_cq = qp_attr->recv_cq;
953 qp->qp.srq = qp_attr->srq;
954 qp->qp.qp_type = qp_attr->qp_type;
955 qp->qp.state = IBV_QPS_RESET;
956 qp->qp.events_completed = 0;
957
958 qp->comp_mask = 0;
959 if (vext_field_avail(struct verbs_qp, xrcd, vqp_sz) &&
960 (qp_attr->comp_mask & IBV_QP_INIT_ATTR_XRCD)) {
961 qp->comp_mask |= VERBS_QP_XRCD;
962 qp->xrcd = vxrcd;
963 }
964 }
965
966 enum {
967 CREATE_QP_EX2_SUP_CREATE_FLAGS = IBV_QP_CREATE_BLOCK_SELF_MCAST_LB |
968 IBV_QP_CREATE_SCATTER_FCS |
969 IBV_QP_CREATE_CVLAN_STRIPPING,
970 };
971
ibv_cmd_create_qp_ex2(struct ibv_context * context,struct verbs_qp * qp,int vqp_sz,struct ibv_qp_init_attr_ex * qp_attr,struct ibv_create_qp_ex * cmd,size_t cmd_core_size,size_t cmd_size,struct ibv_create_qp_resp_ex * resp,size_t resp_core_size,size_t resp_size)972 int ibv_cmd_create_qp_ex2(struct ibv_context *context,
973 struct verbs_qp *qp, int vqp_sz,
974 struct ibv_qp_init_attr_ex *qp_attr,
975 struct ibv_create_qp_ex *cmd,
976 size_t cmd_core_size,
977 size_t cmd_size,
978 struct ibv_create_qp_resp_ex *resp,
979 size_t resp_core_size,
980 size_t resp_size)
981 {
982 struct verbs_xrcd *vxrcd = NULL;
983 int err;
984
985 if (qp_attr->comp_mask >= IBV_QP_INIT_ATTR_RESERVED)
986 return EINVAL;
987
988 if (resp_core_size <
989 offsetof(struct ibv_create_qp_resp_ex, response_length) +
990 sizeof(resp->response_length))
991 return EINVAL;
992
993 memset(cmd, 0, cmd_core_size);
994
995 IBV_INIT_CMD_RESP_EX_V(cmd, cmd_core_size, cmd_size, CREATE_QP_EX, resp,
996 resp_core_size, resp_size);
997
998 err = create_qp_ex_common(qp, qp_attr, vxrcd, &cmd->base);
999 if (err)
1000 return err;
1001
1002 if (qp_attr->comp_mask & IBV_QP_INIT_ATTR_CREATE_FLAGS) {
1003 if (qp_attr->create_flags & ~CREATE_QP_EX2_SUP_CREATE_FLAGS)
1004 return EINVAL;
1005 if (cmd_core_size < offsetof(struct ibv_create_qp_ex, create_flags) +
1006 sizeof(qp_attr->create_flags))
1007 return EINVAL;
1008 cmd->create_flags = qp_attr->create_flags;
1009 }
1010
1011 if (qp_attr->comp_mask & IBV_QP_INIT_ATTR_IND_TABLE) {
1012 if (cmd_core_size < offsetof(struct ibv_create_qp_ex, ind_tbl_handle) +
1013 sizeof(cmd->ind_tbl_handle))
1014 return EINVAL;
1015 cmd->ind_tbl_handle = qp_attr->rwq_ind_tbl->ind_tbl_handle;
1016 cmd->comp_mask = IBV_CREATE_QP_EX_KERNEL_MASK_IND_TABLE;
1017 }
1018
1019 err = create_qp_handle_resp_common_init(qp);
1020 if (err)
1021 return err;
1022
1023 err = write(context->cmd_fd, cmd, cmd_size);
1024 if (err != cmd_size) {
1025 err = errno;
1026 goto err;
1027 }
1028
1029 (void)VALGRIND_MAKE_MEM_DEFINED(resp, resp_size);
1030
1031 create_qp_handle_resp_common(context, qp, qp_attr, &resp->base, vxrcd,
1032 vqp_sz);
1033
1034 return 0;
1035
1036 err:
1037 create_qp_handle_resp_common_cleanup(qp);
1038
1039 return err;
1040 }
1041
ibv_cmd_create_qp_ex(struct ibv_context * context,struct verbs_qp * qp,int vqp_sz,struct ibv_qp_init_attr_ex * attr_ex,struct ibv_create_qp * cmd,size_t cmd_size,struct ibv_create_qp_resp * resp,size_t resp_size)1042 int ibv_cmd_create_qp_ex(struct ibv_context *context,
1043 struct verbs_qp *qp, int vqp_sz,
1044 struct ibv_qp_init_attr_ex *attr_ex,
1045 struct ibv_create_qp *cmd, size_t cmd_size,
1046 struct ibv_create_qp_resp *resp, size_t resp_size)
1047 {
1048 struct verbs_xrcd *vxrcd = NULL;
1049 int err;
1050
1051 IBV_INIT_CMD_RESP(cmd, cmd_size, CREATE_QP, resp, resp_size);
1052
1053 if (attr_ex->comp_mask > (IBV_QP_INIT_ATTR_XRCD | IBV_QP_INIT_ATTR_PD))
1054 return ENOSYS;
1055
1056 err = create_qp_ex_common(qp, attr_ex, vxrcd,
1057 (struct ibv_create_qp_common *)&cmd->user_handle);
1058 if (err)
1059 return err;
1060
1061 err = create_qp_handle_resp_common_init(qp);
1062 if (err)
1063 return err;
1064
1065 err = write(context->cmd_fd, cmd, cmd_size);
1066 if (err != cmd_size) {
1067 err = errno;
1068 goto err;
1069 }
1070
1071 (void)VALGRIND_MAKE_MEM_DEFINED(resp, resp_size);
1072
1073 if (abi_ver == 4) {
1074 struct ibv_create_qp_resp_v4 *resp_v4 =
1075 (struct ibv_create_qp_resp_v4 *)resp;
1076
1077 memmove((void *)resp + sizeof *resp,
1078 (void *)resp_v4 + sizeof *resp_v4,
1079 resp_size - sizeof *resp);
1080 } else if (abi_ver <= 3) {
1081 struct ibv_create_qp_resp_v3 *resp_v3 =
1082 (struct ibv_create_qp_resp_v3 *)resp;
1083
1084 memmove((void *)resp + sizeof *resp,
1085 (void *)resp_v3 + sizeof *resp_v3,
1086 resp_size - sizeof *resp);
1087 }
1088
1089 create_qp_handle_resp_common(context, qp, attr_ex, resp, vxrcd, vqp_sz);
1090
1091 return 0;
1092
1093 err:
1094 create_qp_handle_resp_common_cleanup(qp);
1095
1096 return err;
1097 }
1098
ibv_cmd_create_qp(struct ibv_pd * pd,struct ibv_qp * qp,struct ibv_qp_init_attr * attr,struct ibv_create_qp * cmd,size_t cmd_size,struct ibv_create_qp_resp * resp,size_t resp_size)1099 int ibv_cmd_create_qp(struct ibv_pd *pd,
1100 struct ibv_qp *qp, struct ibv_qp_init_attr *attr,
1101 struct ibv_create_qp *cmd, size_t cmd_size,
1102 struct ibv_create_qp_resp *resp, size_t resp_size)
1103 {
1104 IBV_INIT_CMD_RESP(cmd, cmd_size, CREATE_QP, resp, resp_size);
1105
1106 cmd->user_handle = (uintptr_t) qp;
1107 cmd->pd_handle = pd->handle;
1108 cmd->send_cq_handle = attr->send_cq->handle;
1109 cmd->recv_cq_handle = attr->recv_cq->handle;
1110 cmd->srq_handle = attr->srq ? attr->srq->handle : 0;
1111 cmd->max_send_wr = attr->cap.max_send_wr;
1112 cmd->max_recv_wr = attr->cap.max_recv_wr;
1113 cmd->max_send_sge = attr->cap.max_send_sge;
1114 cmd->max_recv_sge = attr->cap.max_recv_sge;
1115 cmd->max_inline_data = attr->cap.max_inline_data;
1116 cmd->sq_sig_all = attr->sq_sig_all;
1117 cmd->qp_type = attr->qp_type;
1118 cmd->is_srq = !!attr->srq;
1119 cmd->reserved = 0;
1120
1121 if (write(pd->context->cmd_fd, cmd, cmd_size) != cmd_size)
1122 return errno;
1123
1124 (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size);
1125
1126 qp->handle = resp->qp_handle;
1127 qp->qp_num = resp->qpn;
1128 qp->context = pd->context;
1129
1130 if (abi_ver > 3) {
1131 attr->cap.max_recv_sge = resp->max_recv_sge;
1132 attr->cap.max_send_sge = resp->max_send_sge;
1133 attr->cap.max_recv_wr = resp->max_recv_wr;
1134 attr->cap.max_send_wr = resp->max_send_wr;
1135 attr->cap.max_inline_data = resp->max_inline_data;
1136 }
1137
1138 if (abi_ver == 4) {
1139 struct ibv_create_qp_resp_v4 *resp_v4 =
1140 (struct ibv_create_qp_resp_v4 *) resp;
1141
1142 memmove((void *) resp + sizeof *resp,
1143 (void *) resp_v4 + sizeof *resp_v4,
1144 resp_size - sizeof *resp);
1145 } else if (abi_ver <= 3) {
1146 struct ibv_create_qp_resp_v3 *resp_v3 =
1147 (struct ibv_create_qp_resp_v3 *) resp;
1148
1149 memmove((void *) resp + sizeof *resp,
1150 (void *) resp_v3 + sizeof *resp_v3,
1151 resp_size - sizeof *resp);
1152 }
1153
1154 return 0;
1155 }
1156
ibv_cmd_open_qp(struct ibv_context * context,struct verbs_qp * qp,int vqp_sz,struct ibv_qp_open_attr * attr,struct ibv_open_qp * cmd,size_t cmd_size,struct ibv_create_qp_resp * resp,size_t resp_size)1157 int ibv_cmd_open_qp(struct ibv_context *context, struct verbs_qp *qp,
1158 int vqp_sz,
1159 struct ibv_qp_open_attr *attr,
1160 struct ibv_open_qp *cmd, size_t cmd_size,
1161 struct ibv_create_qp_resp *resp, size_t resp_size)
1162 {
1163 int err = 0;
1164 struct verbs_xrcd *xrcd;
1165 IBV_INIT_CMD_RESP(cmd, cmd_size, OPEN_QP, resp, resp_size);
1166
1167 if (attr->comp_mask >= IBV_QP_OPEN_ATTR_RESERVED)
1168 return ENOSYS;
1169
1170 if (!(attr->comp_mask & IBV_QP_OPEN_ATTR_XRCD) ||
1171 !(attr->comp_mask & IBV_QP_OPEN_ATTR_NUM) ||
1172 !(attr->comp_mask & IBV_QP_OPEN_ATTR_TYPE))
1173 return EINVAL;
1174
1175 xrcd = container_of(attr->xrcd, struct verbs_xrcd, xrcd);
1176 cmd->user_handle = (uintptr_t) qp;
1177 cmd->pd_handle = xrcd->handle;
1178 cmd->qpn = attr->qp_num;
1179 cmd->qp_type = attr->qp_type;
1180
1181 err = pthread_mutex_init(&qp->qp.mutex, NULL);
1182 if (err)
1183 return err;
1184 err = pthread_cond_init(&qp->qp.cond, NULL);
1185 if (err)
1186 goto err_mutex;
1187
1188 err = write(context->cmd_fd, cmd, cmd_size);
1189 if (err != cmd_size) {
1190 err = errno;
1191 goto err_cond;
1192 }
1193
1194 (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size);
1195
1196 qp->qp.handle = resp->qp_handle;
1197 qp->qp.context = context;
1198 qp->qp.qp_context = attr->qp_context;
1199 qp->qp.pd = NULL;
1200 qp->qp.send_cq = NULL;
1201 qp->qp.recv_cq = NULL;
1202 qp->qp.srq = NULL;
1203 qp->qp.qp_num = attr->qp_num;
1204 qp->qp.qp_type = attr->qp_type;
1205 qp->qp.state = IBV_QPS_UNKNOWN;
1206 qp->qp.events_completed = 0;
1207 qp->comp_mask = 0;
1208 if (vext_field_avail(struct verbs_qp, xrcd, vqp_sz)) {
1209 qp->comp_mask = VERBS_QP_XRCD;
1210 qp->xrcd = xrcd;
1211 }
1212
1213 return 0;
1214
1215 err_cond:
1216 pthread_cond_destroy(&qp->qp.cond);
1217 err_mutex:
1218 pthread_mutex_destroy(&qp->qp.mutex);
1219
1220 return err;
1221 }
1222
ibv_cmd_query_qp(struct ibv_qp * qp,struct ibv_qp_attr * attr,int attr_mask,struct ibv_qp_init_attr * init_attr,struct ibv_query_qp * cmd,size_t cmd_size)1223 int ibv_cmd_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
1224 int attr_mask,
1225 struct ibv_qp_init_attr *init_attr,
1226 struct ibv_query_qp *cmd, size_t cmd_size)
1227 {
1228 struct ibv_query_qp_resp resp;
1229
1230 /*
1231 * Masks over IBV_QP_DEST_QPN are not supported by
1232 * that not extended command.
1233 */
1234 if (attr_mask & ~((IBV_QP_DEST_QPN << 1) - 1))
1235 return EOPNOTSUPP;
1236
1237 IBV_INIT_CMD_RESP(cmd, cmd_size, QUERY_QP, &resp, sizeof resp);
1238 cmd->qp_handle = qp->handle;
1239 cmd->attr_mask = attr_mask;
1240
1241 if (write(qp->context->cmd_fd, cmd, cmd_size) != cmd_size)
1242 return errno;
1243
1244 (void) VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof resp);
1245
1246 attr->qkey = resp.qkey;
1247 attr->rq_psn = resp.rq_psn;
1248 attr->sq_psn = resp.sq_psn;
1249 attr->dest_qp_num = resp.dest_qp_num;
1250 attr->qp_access_flags = resp.qp_access_flags;
1251 attr->pkey_index = resp.pkey_index;
1252 attr->alt_pkey_index = resp.alt_pkey_index;
1253 attr->qp_state = resp.qp_state;
1254 attr->cur_qp_state = resp.cur_qp_state;
1255 attr->path_mtu = resp.path_mtu;
1256 attr->path_mig_state = resp.path_mig_state;
1257 attr->sq_draining = resp.sq_draining;
1258 attr->max_rd_atomic = resp.max_rd_atomic;
1259 attr->max_dest_rd_atomic = resp.max_dest_rd_atomic;
1260 attr->min_rnr_timer = resp.min_rnr_timer;
1261 attr->port_num = resp.port_num;
1262 attr->timeout = resp.timeout;
1263 attr->retry_cnt = resp.retry_cnt;
1264 attr->rnr_retry = resp.rnr_retry;
1265 attr->alt_port_num = resp.alt_port_num;
1266 attr->alt_timeout = resp.alt_timeout;
1267 attr->cap.max_send_wr = resp.max_send_wr;
1268 attr->cap.max_recv_wr = resp.max_recv_wr;
1269 attr->cap.max_send_sge = resp.max_send_sge;
1270 attr->cap.max_recv_sge = resp.max_recv_sge;
1271 attr->cap.max_inline_data = resp.max_inline_data;
1272
1273 memcpy(attr->ah_attr.grh.dgid.raw, resp.dest.dgid, 16);
1274 attr->ah_attr.grh.flow_label = resp.dest.flow_label;
1275 attr->ah_attr.dlid = resp.dest.dlid;
1276 attr->ah_attr.grh.sgid_index = resp.dest.sgid_index;
1277 attr->ah_attr.grh.hop_limit = resp.dest.hop_limit;
1278 attr->ah_attr.grh.traffic_class = resp.dest.traffic_class;
1279 attr->ah_attr.sl = resp.dest.sl;
1280 attr->ah_attr.src_path_bits = resp.dest.src_path_bits;
1281 attr->ah_attr.static_rate = resp.dest.static_rate;
1282 attr->ah_attr.is_global = resp.dest.is_global;
1283 attr->ah_attr.port_num = resp.dest.port_num;
1284
1285 memcpy(attr->alt_ah_attr.grh.dgid.raw, resp.alt_dest.dgid, 16);
1286 attr->alt_ah_attr.grh.flow_label = resp.alt_dest.flow_label;
1287 attr->alt_ah_attr.dlid = resp.alt_dest.dlid;
1288 attr->alt_ah_attr.grh.sgid_index = resp.alt_dest.sgid_index;
1289 attr->alt_ah_attr.grh.hop_limit = resp.alt_dest.hop_limit;
1290 attr->alt_ah_attr.grh.traffic_class = resp.alt_dest.traffic_class;
1291 attr->alt_ah_attr.sl = resp.alt_dest.sl;
1292 attr->alt_ah_attr.src_path_bits = resp.alt_dest.src_path_bits;
1293 attr->alt_ah_attr.static_rate = resp.alt_dest.static_rate;
1294 attr->alt_ah_attr.is_global = resp.alt_dest.is_global;
1295 attr->alt_ah_attr.port_num = resp.alt_dest.port_num;
1296
1297 init_attr->qp_context = qp->qp_context;
1298 init_attr->send_cq = qp->send_cq;
1299 init_attr->recv_cq = qp->recv_cq;
1300 init_attr->srq = qp->srq;
1301 init_attr->qp_type = qp->qp_type;
1302 init_attr->cap.max_send_wr = resp.max_send_wr;
1303 init_attr->cap.max_recv_wr = resp.max_recv_wr;
1304 init_attr->cap.max_send_sge = resp.max_send_sge;
1305 init_attr->cap.max_recv_sge = resp.max_recv_sge;
1306 init_attr->cap.max_inline_data = resp.max_inline_data;
1307 init_attr->sq_sig_all = resp.sq_sig_all;
1308
1309 return 0;
1310 }
1311
copy_modify_qp_fields(struct ibv_qp * qp,struct ibv_qp_attr * attr,int attr_mask,struct ibv_modify_qp_common * cmd)1312 static void copy_modify_qp_fields(struct ibv_qp *qp, struct ibv_qp_attr *attr,
1313 int attr_mask,
1314 struct ibv_modify_qp_common *cmd)
1315 {
1316 cmd->qp_handle = qp->handle;
1317 cmd->attr_mask = attr_mask;
1318
1319 if (attr_mask & IBV_QP_STATE)
1320 cmd->qp_state = attr->qp_state;
1321 if (attr_mask & IBV_QP_CUR_STATE)
1322 cmd->cur_qp_state = attr->cur_qp_state;
1323 if (attr_mask & IBV_QP_EN_SQD_ASYNC_NOTIFY)
1324 cmd->en_sqd_async_notify = attr->en_sqd_async_notify;
1325 if (attr_mask & IBV_QP_ACCESS_FLAGS)
1326 cmd->qp_access_flags = attr->qp_access_flags;
1327 if (attr_mask & IBV_QP_PKEY_INDEX)
1328 cmd->pkey_index = attr->pkey_index;
1329 if (attr_mask & IBV_QP_PORT)
1330 cmd->port_num = attr->port_num;
1331 if (attr_mask & IBV_QP_QKEY)
1332 cmd->qkey = attr->qkey;
1333
1334 if (attr_mask & IBV_QP_AV) {
1335 memcpy(cmd->dest.dgid, attr->ah_attr.grh.dgid.raw, 16);
1336 cmd->dest.flow_label = attr->ah_attr.grh.flow_label;
1337 cmd->dest.dlid = attr->ah_attr.dlid;
1338 cmd->dest.reserved = 0;
1339 cmd->dest.sgid_index = attr->ah_attr.grh.sgid_index;
1340 cmd->dest.hop_limit = attr->ah_attr.grh.hop_limit;
1341 cmd->dest.traffic_class = attr->ah_attr.grh.traffic_class;
1342 cmd->dest.sl = attr->ah_attr.sl;
1343 cmd->dest.src_path_bits = attr->ah_attr.src_path_bits;
1344 cmd->dest.static_rate = attr->ah_attr.static_rate;
1345 cmd->dest.is_global = attr->ah_attr.is_global;
1346 cmd->dest.port_num = attr->ah_attr.port_num;
1347 }
1348
1349 if (attr_mask & IBV_QP_PATH_MTU)
1350 cmd->path_mtu = attr->path_mtu;
1351 if (attr_mask & IBV_QP_TIMEOUT)
1352 cmd->timeout = attr->timeout;
1353 if (attr_mask & IBV_QP_RETRY_CNT)
1354 cmd->retry_cnt = attr->retry_cnt;
1355 if (attr_mask & IBV_QP_RNR_RETRY)
1356 cmd->rnr_retry = attr->rnr_retry;
1357 if (attr_mask & IBV_QP_RQ_PSN)
1358 cmd->rq_psn = attr->rq_psn;
1359 if (attr_mask & IBV_QP_MAX_QP_RD_ATOMIC)
1360 cmd->max_rd_atomic = attr->max_rd_atomic;
1361
1362 if (attr_mask & IBV_QP_ALT_PATH) {
1363 cmd->alt_pkey_index = attr->alt_pkey_index;
1364 cmd->alt_port_num = attr->alt_port_num;
1365 cmd->alt_timeout = attr->alt_timeout;
1366
1367 memcpy(cmd->alt_dest.dgid, attr->alt_ah_attr.grh.dgid.raw, 16);
1368 cmd->alt_dest.flow_label = attr->alt_ah_attr.grh.flow_label;
1369 cmd->alt_dest.dlid = attr->alt_ah_attr.dlid;
1370 cmd->alt_dest.reserved = 0;
1371 cmd->alt_dest.sgid_index = attr->alt_ah_attr.grh.sgid_index;
1372 cmd->alt_dest.hop_limit = attr->alt_ah_attr.grh.hop_limit;
1373 cmd->alt_dest.traffic_class =
1374 attr->alt_ah_attr.grh.traffic_class;
1375 cmd->alt_dest.sl = attr->alt_ah_attr.sl;
1376 cmd->alt_dest.src_path_bits = attr->alt_ah_attr.src_path_bits;
1377 cmd->alt_dest.static_rate = attr->alt_ah_attr.static_rate;
1378 cmd->alt_dest.is_global = attr->alt_ah_attr.is_global;
1379 cmd->alt_dest.port_num = attr->alt_ah_attr.port_num;
1380 }
1381
1382 if (attr_mask & IBV_QP_MIN_RNR_TIMER)
1383 cmd->min_rnr_timer = attr->min_rnr_timer;
1384 if (attr_mask & IBV_QP_SQ_PSN)
1385 cmd->sq_psn = attr->sq_psn;
1386 if (attr_mask & IBV_QP_MAX_DEST_RD_ATOMIC)
1387 cmd->max_dest_rd_atomic = attr->max_dest_rd_atomic;
1388 if (attr_mask & IBV_QP_PATH_MIG_STATE)
1389 cmd->path_mig_state = attr->path_mig_state;
1390 if (attr_mask & IBV_QP_DEST_QPN)
1391 cmd->dest_qp_num = attr->dest_qp_num;
1392
1393 cmd->reserved[0] = cmd->reserved[1] = 0;
1394 }
1395
ibv_cmd_modify_qp(struct ibv_qp * qp,struct ibv_qp_attr * attr,int attr_mask,struct ibv_modify_qp * cmd,size_t cmd_size)1396 int ibv_cmd_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
1397 int attr_mask,
1398 struct ibv_modify_qp *cmd, size_t cmd_size)
1399 {
1400 /*
1401 * Masks over IBV_QP_DEST_QPN are only supported by
1402 * ibv_cmd_modify_qp_ex.
1403 */
1404 if (attr_mask & ~((IBV_QP_DEST_QPN << 1) - 1))
1405 return EOPNOTSUPP;
1406
1407 IBV_INIT_CMD(cmd, cmd_size, MODIFY_QP);
1408
1409 copy_modify_qp_fields(qp, attr, attr_mask, &cmd->base);
1410
1411 if (write(qp->context->cmd_fd, cmd, cmd_size) != cmd_size)
1412 return errno;
1413
1414 return 0;
1415 }
1416
ibv_cmd_modify_qp_ex(struct ibv_qp * qp,struct ibv_qp_attr * attr,int attr_mask,struct ibv_modify_qp_ex * cmd,size_t cmd_core_size,size_t cmd_size,struct ibv_modify_qp_resp_ex * resp,size_t resp_core_size,size_t resp_size)1417 int ibv_cmd_modify_qp_ex(struct ibv_qp *qp, struct ibv_qp_attr *attr,
1418 int attr_mask, struct ibv_modify_qp_ex *cmd,
1419 size_t cmd_core_size, size_t cmd_size,
1420 struct ibv_modify_qp_resp_ex *resp,
1421 size_t resp_core_size, size_t resp_size)
1422 {
1423 if (resp_core_size < offsetof(struct ibv_modify_qp_resp_ex,
1424 response_length) + sizeof(resp->response_length))
1425 return EINVAL;
1426
1427 IBV_INIT_CMD_RESP_EX_V(cmd, cmd_core_size, cmd_size, MODIFY_QP_EX,
1428 resp, resp_core_size, resp_size);
1429
1430 copy_modify_qp_fields(qp, attr, attr_mask, &cmd->base);
1431
1432 if (attr_mask & IBV_QP_RATE_LIMIT) {
1433 if (cmd_size >= offsetof(struct ibv_modify_qp_ex, rate_limit) +
1434 sizeof(cmd->rate_limit))
1435 cmd->rate_limit = attr->rate_limit;
1436 else
1437 return EINVAL;
1438 }
1439
1440 if (write(qp->context->cmd_fd, cmd, cmd_size) != cmd_size)
1441 return errno;
1442
1443 (void)VALGRIND_MAKE_MEM_DEFINED(resp, resp_size);
1444
1445 return 0;
1446 }
1447
ibv_cmd_post_send(struct ibv_qp * ibqp,struct ibv_send_wr * wr,struct ibv_send_wr ** bad_wr)1448 int ibv_cmd_post_send(struct ibv_qp *ibqp, struct ibv_send_wr *wr,
1449 struct ibv_send_wr **bad_wr)
1450 {
1451 struct ibv_post_send *cmd;
1452 struct ibv_post_send_resp resp;
1453 struct ibv_send_wr *i;
1454 struct ibv_kern_send_wr *n, *tmp;
1455 struct ibv_sge *s;
1456 unsigned wr_count = 0;
1457 unsigned sge_count = 0;
1458 int cmd_size;
1459 int ret = 0;
1460
1461 for (i = wr; i; i = i->next) {
1462 wr_count++;
1463 sge_count += i->num_sge;
1464 }
1465
1466 cmd_size = sizeof *cmd + wr_count * sizeof *n + sge_count * sizeof *s;
1467 cmd = alloca(cmd_size);
1468
1469 IBV_INIT_CMD_RESP(cmd, cmd_size, POST_SEND, &resp, sizeof resp);
1470 cmd->qp_handle = ibqp->handle;
1471 cmd->wr_count = wr_count;
1472 cmd->sge_count = sge_count;
1473 cmd->wqe_size = sizeof *n;
1474
1475 n = (struct ibv_kern_send_wr *) ((void *) cmd + sizeof *cmd);
1476 s = (struct ibv_sge *) (n + wr_count);
1477
1478 tmp = n;
1479 for (i = wr; i; i = i->next) {
1480 tmp->wr_id = i->wr_id;
1481 tmp->num_sge = i->num_sge;
1482 tmp->opcode = i->opcode;
1483 tmp->send_flags = i->send_flags;
1484 tmp->imm_data = i->imm_data;
1485 if (ibqp->qp_type == IBV_QPT_UD) {
1486 tmp->wr.ud.ah = i->wr.ud.ah->handle;
1487 tmp->wr.ud.remote_qpn = i->wr.ud.remote_qpn;
1488 tmp->wr.ud.remote_qkey = i->wr.ud.remote_qkey;
1489 } else {
1490 switch (i->opcode) {
1491 case IBV_WR_RDMA_WRITE:
1492 case IBV_WR_RDMA_WRITE_WITH_IMM:
1493 case IBV_WR_RDMA_READ:
1494 tmp->wr.rdma.remote_addr =
1495 i->wr.rdma.remote_addr;
1496 tmp->wr.rdma.rkey = i->wr.rdma.rkey;
1497 break;
1498 case IBV_WR_ATOMIC_CMP_AND_SWP:
1499 case IBV_WR_ATOMIC_FETCH_AND_ADD:
1500 tmp->wr.atomic.remote_addr =
1501 i->wr.atomic.remote_addr;
1502 tmp->wr.atomic.compare_add =
1503 i->wr.atomic.compare_add;
1504 tmp->wr.atomic.swap = i->wr.atomic.swap;
1505 tmp->wr.atomic.rkey = i->wr.atomic.rkey;
1506 break;
1507 default:
1508 break;
1509 }
1510 }
1511
1512 if (tmp->num_sge) {
1513 memcpy(s, i->sg_list, tmp->num_sge * sizeof *s);
1514 s += tmp->num_sge;
1515 }
1516
1517 tmp++;
1518 }
1519
1520 resp.bad_wr = 0;
1521 if (write(ibqp->context->cmd_fd, cmd, cmd_size) != cmd_size)
1522 ret = errno;
1523
1524 (void) VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof resp);
1525
1526 wr_count = resp.bad_wr;
1527 if (wr_count) {
1528 i = wr;
1529 while (--wr_count)
1530 i = i->next;
1531 *bad_wr = i;
1532 } else if (ret)
1533 *bad_wr = wr;
1534
1535 return ret;
1536 }
1537
ibv_cmd_post_recv(struct ibv_qp * ibqp,struct ibv_recv_wr * wr,struct ibv_recv_wr ** bad_wr)1538 int ibv_cmd_post_recv(struct ibv_qp *ibqp, struct ibv_recv_wr *wr,
1539 struct ibv_recv_wr **bad_wr)
1540 {
1541 struct ibv_post_recv *cmd;
1542 struct ibv_post_recv_resp resp;
1543 struct ibv_recv_wr *i;
1544 struct ibv_kern_recv_wr *n, *tmp;
1545 struct ibv_sge *s;
1546 unsigned wr_count = 0;
1547 unsigned sge_count = 0;
1548 int cmd_size;
1549 int ret = 0;
1550
1551 for (i = wr; i; i = i->next) {
1552 wr_count++;
1553 sge_count += i->num_sge;
1554 }
1555
1556 cmd_size = sizeof *cmd + wr_count * sizeof *n + sge_count * sizeof *s;
1557 cmd = alloca(cmd_size);
1558
1559 IBV_INIT_CMD_RESP(cmd, cmd_size, POST_RECV, &resp, sizeof resp);
1560 cmd->qp_handle = ibqp->handle;
1561 cmd->wr_count = wr_count;
1562 cmd->sge_count = sge_count;
1563 cmd->wqe_size = sizeof *n;
1564
1565 n = (struct ibv_kern_recv_wr *) ((void *) cmd + sizeof *cmd);
1566 s = (struct ibv_sge *) (n + wr_count);
1567
1568 tmp = n;
1569 for (i = wr; i; i = i->next) {
1570 tmp->wr_id = i->wr_id;
1571 tmp->num_sge = i->num_sge;
1572
1573 if (tmp->num_sge) {
1574 memcpy(s, i->sg_list, tmp->num_sge * sizeof *s);
1575 s += tmp->num_sge;
1576 }
1577
1578 tmp++;
1579 }
1580
1581 resp.bad_wr = 0;
1582 if (write(ibqp->context->cmd_fd, cmd, cmd_size) != cmd_size)
1583 ret = errno;
1584
1585 (void) VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof resp);
1586
1587 wr_count = resp.bad_wr;
1588 if (wr_count) {
1589 i = wr;
1590 while (--wr_count)
1591 i = i->next;
1592 *bad_wr = i;
1593 } else if (ret)
1594 *bad_wr = wr;
1595
1596 return ret;
1597 }
1598
ibv_cmd_post_srq_recv(struct ibv_srq * srq,struct ibv_recv_wr * wr,struct ibv_recv_wr ** bad_wr)1599 int ibv_cmd_post_srq_recv(struct ibv_srq *srq, struct ibv_recv_wr *wr,
1600 struct ibv_recv_wr **bad_wr)
1601 {
1602 struct ibv_post_srq_recv *cmd;
1603 struct ibv_post_srq_recv_resp resp;
1604 struct ibv_recv_wr *i;
1605 struct ibv_kern_recv_wr *n, *tmp;
1606 struct ibv_sge *s;
1607 unsigned wr_count = 0;
1608 unsigned sge_count = 0;
1609 int cmd_size;
1610 int ret = 0;
1611
1612 for (i = wr; i; i = i->next) {
1613 wr_count++;
1614 sge_count += i->num_sge;
1615 }
1616
1617 cmd_size = sizeof *cmd + wr_count * sizeof *n + sge_count * sizeof *s;
1618 cmd = alloca(cmd_size);
1619
1620 IBV_INIT_CMD_RESP(cmd, cmd_size, POST_SRQ_RECV, &resp, sizeof resp);
1621 cmd->srq_handle = srq->handle;
1622 cmd->wr_count = wr_count;
1623 cmd->sge_count = sge_count;
1624 cmd->wqe_size = sizeof *n;
1625
1626 n = (struct ibv_kern_recv_wr *) ((void *) cmd + sizeof *cmd);
1627 s = (struct ibv_sge *) (n + wr_count);
1628
1629 tmp = n;
1630 for (i = wr; i; i = i->next) {
1631 tmp->wr_id = i->wr_id;
1632 tmp->num_sge = i->num_sge;
1633
1634 if (tmp->num_sge) {
1635 memcpy(s, i->sg_list, tmp->num_sge * sizeof *s);
1636 s += tmp->num_sge;
1637 }
1638
1639 tmp++;
1640 }
1641
1642 resp.bad_wr = 0;
1643 if (write(srq->context->cmd_fd, cmd, cmd_size) != cmd_size)
1644 ret = errno;
1645
1646 (void) VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof resp);
1647
1648 wr_count = resp.bad_wr;
1649 if (wr_count) {
1650 i = wr;
1651 while (--wr_count)
1652 i = i->next;
1653 *bad_wr = i;
1654 } else if (ret)
1655 *bad_wr = wr;
1656
1657 return ret;
1658 }
1659
ibv_cmd_create_ah(struct ibv_pd * pd,struct ibv_ah * ah,struct ibv_ah_attr * attr,struct ibv_create_ah_resp * resp,size_t resp_size)1660 int ibv_cmd_create_ah(struct ibv_pd *pd, struct ibv_ah *ah,
1661 struct ibv_ah_attr *attr,
1662 struct ibv_create_ah_resp *resp,
1663 size_t resp_size)
1664 {
1665 struct ibv_create_ah cmd;
1666
1667 IBV_INIT_CMD_RESP(&cmd, sizeof cmd, CREATE_AH, resp, resp_size);
1668 cmd.user_handle = (uintptr_t) ah;
1669 cmd.pd_handle = pd->handle;
1670 cmd.attr.dlid = attr->dlid;
1671 cmd.attr.sl = attr->sl;
1672 cmd.attr.src_path_bits = attr->src_path_bits;
1673 cmd.attr.static_rate = attr->static_rate;
1674 cmd.attr.is_global = attr->is_global;
1675 cmd.attr.port_num = attr->port_num;
1676 cmd.attr.grh.flow_label = attr->grh.flow_label;
1677 cmd.attr.grh.sgid_index = attr->grh.sgid_index;
1678 cmd.attr.grh.hop_limit = attr->grh.hop_limit;
1679 cmd.attr.grh.traffic_class = attr->grh.traffic_class;
1680 memcpy(cmd.attr.grh.dgid, attr->grh.dgid.raw, 16);
1681
1682 if (write(pd->context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd)
1683 return errno;
1684
1685 (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size);
1686
1687 ah->handle = resp->handle;
1688 ah->context = pd->context;
1689
1690 return 0;
1691 }
1692
ibv_cmd_destroy_ah(struct ibv_ah * ah)1693 int ibv_cmd_destroy_ah(struct ibv_ah *ah)
1694 {
1695 struct ibv_destroy_ah cmd;
1696
1697 IBV_INIT_CMD(&cmd, sizeof cmd, DESTROY_AH);
1698 cmd.ah_handle = ah->handle;
1699
1700 if (write(ah->context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd)
1701 return errno;
1702
1703 return 0;
1704 }
1705
ibv_cmd_destroy_qp(struct ibv_qp * qp)1706 int ibv_cmd_destroy_qp(struct ibv_qp *qp)
1707 {
1708 struct ibv_destroy_qp cmd;
1709 struct ibv_destroy_qp_resp resp;
1710
1711 IBV_INIT_CMD_RESP(&cmd, sizeof cmd, DESTROY_QP, &resp, sizeof resp);
1712 cmd.qp_handle = qp->handle;
1713 cmd.reserved = 0;
1714
1715 if (write(qp->context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd)
1716 return errno;
1717
1718 (void) VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof resp);
1719
1720 pthread_mutex_lock(&qp->mutex);
1721 while (qp->events_completed != resp.events_reported)
1722 pthread_cond_wait(&qp->cond, &qp->mutex);
1723 pthread_mutex_unlock(&qp->mutex);
1724
1725 pthread_cond_destroy(&qp->cond);
1726 pthread_mutex_destroy(&qp->mutex);
1727
1728 return 0;
1729 }
1730
ibv_cmd_attach_mcast(struct ibv_qp * qp,const union ibv_gid * gid,uint16_t lid)1731 int ibv_cmd_attach_mcast(struct ibv_qp *qp, const union ibv_gid *gid, uint16_t lid)
1732 {
1733 struct ibv_attach_mcast cmd;
1734
1735 IBV_INIT_CMD(&cmd, sizeof cmd, ATTACH_MCAST);
1736 memcpy(cmd.gid, gid->raw, sizeof cmd.gid);
1737 cmd.qp_handle = qp->handle;
1738 cmd.mlid = lid;
1739 cmd.reserved = 0;
1740
1741 if (write(qp->context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd)
1742 return errno;
1743
1744 return 0;
1745 }
1746
ibv_cmd_detach_mcast(struct ibv_qp * qp,const union ibv_gid * gid,uint16_t lid)1747 int ibv_cmd_detach_mcast(struct ibv_qp *qp, const union ibv_gid *gid, uint16_t lid)
1748 {
1749 struct ibv_detach_mcast cmd;
1750
1751 IBV_INIT_CMD(&cmd, sizeof cmd, DETACH_MCAST);
1752 memcpy(cmd.gid, gid->raw, sizeof cmd.gid);
1753 cmd.qp_handle = qp->handle;
1754 cmd.mlid = lid;
1755 cmd.reserved = 0;
1756
1757 if (write(qp->context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd)
1758 return errno;
1759
1760 return 0;
1761 }
1762
buffer_is_zero(char * addr,ssize_t size)1763 static int buffer_is_zero(char *addr, ssize_t size)
1764 {
1765 return addr[0] == 0 && !memcmp(addr, addr + 1, size - 1);
1766 }
1767
get_filters_size(struct ibv_flow_spec * ib_spec,struct ibv_kern_spec * kern_spec,int * ib_filter_size,int * kern_filter_size,enum ibv_flow_spec_type type)1768 static int get_filters_size(struct ibv_flow_spec *ib_spec,
1769 struct ibv_kern_spec *kern_spec,
1770 int *ib_filter_size, int *kern_filter_size,
1771 enum ibv_flow_spec_type type)
1772 {
1773 void *ib_spec_filter_mask;
1774 int curr_kern_filter_size;
1775 int min_filter_size;
1776
1777 *ib_filter_size = (ib_spec->hdr.size - sizeof(ib_spec->hdr)) / 2;
1778
1779 switch (type) {
1780 case IBV_FLOW_SPEC_IPV4_EXT:
1781 min_filter_size =
1782 offsetof(struct ibv_kern_ipv4_ext_filter, flags) +
1783 sizeof(kern_spec->ipv4_ext.mask.flags);
1784 curr_kern_filter_size = min_filter_size;
1785 ib_spec_filter_mask = (void *)&ib_spec->ipv4_ext.val +
1786 *ib_filter_size;
1787 break;
1788 case IBV_FLOW_SPEC_IPV6:
1789 min_filter_size =
1790 offsetof(struct ibv_kern_ipv6_filter, hop_limit) +
1791 sizeof(kern_spec->ipv6.mask.hop_limit);
1792 curr_kern_filter_size = min_filter_size;
1793 ib_spec_filter_mask = (void *)&ib_spec->ipv6.val +
1794 *ib_filter_size;
1795 break;
1796 case IBV_FLOW_SPEC_VXLAN_TUNNEL:
1797 min_filter_size =
1798 offsetof(struct ibv_kern_tunnel_filter,
1799 tunnel_id) +
1800 sizeof(kern_spec->tunnel.mask.tunnel_id);
1801 curr_kern_filter_size = min_filter_size;
1802 ib_spec_filter_mask = (void *)&ib_spec->tunnel.val +
1803 *ib_filter_size;
1804 break;
1805 default:
1806 return EINVAL;
1807 }
1808
1809 if (*ib_filter_size < min_filter_size)
1810 return EINVAL;
1811
1812 if (*ib_filter_size > curr_kern_filter_size &&
1813 !buffer_is_zero(ib_spec_filter_mask + curr_kern_filter_size,
1814 *ib_filter_size - curr_kern_filter_size))
1815 return EOPNOTSUPP;
1816
1817 *kern_filter_size = min_t(int, curr_kern_filter_size, *ib_filter_size);
1818
1819 return 0;
1820 }
1821
ib_spec_to_kern_spec(struct ibv_flow_spec * ib_spec,struct ibv_kern_spec * kern_spec)1822 static int ib_spec_to_kern_spec(struct ibv_flow_spec *ib_spec,
1823 struct ibv_kern_spec *kern_spec)
1824 {
1825 int kern_filter_size;
1826 int ib_filter_size;
1827 int ret;
1828
1829 kern_spec->hdr.type = ib_spec->hdr.type;
1830
1831 switch (kern_spec->hdr.type) {
1832 case IBV_FLOW_SPEC_ETH:
1833 case IBV_FLOW_SPEC_ETH | IBV_FLOW_SPEC_INNER:
1834 kern_spec->eth.size = sizeof(struct ibv_kern_spec_eth);
1835 memcpy(&kern_spec->eth.val, &ib_spec->eth.val,
1836 sizeof(struct ibv_flow_eth_filter));
1837 memcpy(&kern_spec->eth.mask, &ib_spec->eth.mask,
1838 sizeof(struct ibv_flow_eth_filter));
1839 break;
1840 case IBV_FLOW_SPEC_IPV4:
1841 case IBV_FLOW_SPEC_IPV4 | IBV_FLOW_SPEC_INNER:
1842 kern_spec->ipv4.size = sizeof(struct ibv_kern_spec_ipv4);
1843 memcpy(&kern_spec->ipv4.val, &ib_spec->ipv4.val,
1844 sizeof(struct ibv_flow_ipv4_filter));
1845 memcpy(&kern_spec->ipv4.mask, &ib_spec->ipv4.mask,
1846 sizeof(struct ibv_flow_ipv4_filter));
1847 break;
1848 case IBV_FLOW_SPEC_IPV4_EXT:
1849 case IBV_FLOW_SPEC_IPV4_EXT | IBV_FLOW_SPEC_INNER:
1850 ret = get_filters_size(ib_spec, kern_spec,
1851 &ib_filter_size, &kern_filter_size,
1852 IBV_FLOW_SPEC_IPV4_EXT);
1853 if (ret)
1854 return ret;
1855
1856 kern_spec->hdr.type = IBV_FLOW_SPEC_IPV4 |
1857 (IBV_FLOW_SPEC_INNER & ib_spec->hdr.type);
1858 kern_spec->ipv4_ext.size = sizeof(struct
1859 ibv_kern_spec_ipv4_ext);
1860 memcpy(&kern_spec->ipv4_ext.val, &ib_spec->ipv4_ext.val,
1861 kern_filter_size);
1862 memcpy(&kern_spec->ipv4_ext.mask, (void *)&ib_spec->ipv4_ext.val
1863 + ib_filter_size, kern_filter_size);
1864 break;
1865 case IBV_FLOW_SPEC_IPV6:
1866 case IBV_FLOW_SPEC_IPV6 | IBV_FLOW_SPEC_INNER:
1867 ret = get_filters_size(ib_spec, kern_spec,
1868 &ib_filter_size, &kern_filter_size,
1869 IBV_FLOW_SPEC_IPV6);
1870 if (ret)
1871 return ret;
1872
1873 kern_spec->ipv6.size = sizeof(struct ibv_kern_spec_ipv6);
1874 memcpy(&kern_spec->ipv6.val, &ib_spec->ipv6.val,
1875 kern_filter_size);
1876 memcpy(&kern_spec->ipv6.mask, (void *)&ib_spec->ipv6.val
1877 + ib_filter_size, kern_filter_size);
1878 break;
1879 case IBV_FLOW_SPEC_TCP:
1880 case IBV_FLOW_SPEC_UDP:
1881 case IBV_FLOW_SPEC_TCP | IBV_FLOW_SPEC_INNER:
1882 case IBV_FLOW_SPEC_UDP | IBV_FLOW_SPEC_INNER:
1883 kern_spec->tcp_udp.size = sizeof(struct ibv_kern_spec_tcp_udp);
1884 memcpy(&kern_spec->tcp_udp.val, &ib_spec->tcp_udp.val,
1885 sizeof(struct ibv_flow_ipv4_filter));
1886 memcpy(&kern_spec->tcp_udp.mask, &ib_spec->tcp_udp.mask,
1887 sizeof(struct ibv_flow_tcp_udp_filter));
1888 break;
1889 case IBV_FLOW_SPEC_VXLAN_TUNNEL:
1890 ret = get_filters_size(ib_spec, kern_spec,
1891 &ib_filter_size, &kern_filter_size,
1892 IBV_FLOW_SPEC_VXLAN_TUNNEL);
1893 if (ret)
1894 return ret;
1895
1896 kern_spec->tunnel.size = sizeof(struct ibv_kern_spec_tunnel);
1897 memcpy(&kern_spec->tunnel.val, &ib_spec->tunnel.val,
1898 kern_filter_size);
1899 memcpy(&kern_spec->tunnel.mask, (void *)&ib_spec->tunnel.val
1900 + ib_filter_size, kern_filter_size);
1901 break;
1902 case IBV_FLOW_SPEC_ACTION_TAG:
1903 kern_spec->flow_tag.size =
1904 sizeof(struct ibv_kern_spec_action_tag);
1905 kern_spec->flow_tag.tag_id = ib_spec->flow_tag.tag_id;
1906 break;
1907 case IBV_FLOW_SPEC_ACTION_DROP:
1908 kern_spec->drop.size = sizeof(struct ibv_kern_spec_action_drop);
1909 break;
1910 default:
1911 return EINVAL;
1912 }
1913 return 0;
1914 }
1915
ibv_cmd_create_flow(struct ibv_qp * qp,struct ibv_flow_attr * flow_attr)1916 struct ibv_flow *ibv_cmd_create_flow(struct ibv_qp *qp,
1917 struct ibv_flow_attr *flow_attr)
1918 {
1919 struct ibv_create_flow *cmd;
1920 struct ibv_create_flow_resp resp;
1921 struct ibv_flow *flow_id;
1922 size_t cmd_size;
1923 size_t written_size;
1924 int i, err;
1925 void *kern_spec;
1926 void *ib_spec;
1927
1928 cmd_size = sizeof(*cmd) + (flow_attr->num_of_specs *
1929 sizeof(struct ibv_kern_spec));
1930 cmd = alloca(cmd_size);
1931 flow_id = malloc(sizeof(*flow_id));
1932 if (!flow_id)
1933 return NULL;
1934 memset(cmd, 0, cmd_size);
1935
1936 cmd->qp_handle = qp->handle;
1937
1938 cmd->flow_attr.type = flow_attr->type;
1939 cmd->flow_attr.priority = flow_attr->priority;
1940 cmd->flow_attr.num_of_specs = flow_attr->num_of_specs;
1941 cmd->flow_attr.port = flow_attr->port;
1942 cmd->flow_attr.flags = flow_attr->flags;
1943
1944 kern_spec = cmd + 1;
1945 ib_spec = flow_attr + 1;
1946 for (i = 0; i < flow_attr->num_of_specs; i++) {
1947 err = ib_spec_to_kern_spec(ib_spec, kern_spec);
1948 if (err) {
1949 errno = err;
1950 goto err;
1951 }
1952 cmd->flow_attr.size +=
1953 ((struct ibv_kern_spec *)kern_spec)->hdr.size;
1954 kern_spec += ((struct ibv_kern_spec *)kern_spec)->hdr.size;
1955 ib_spec += ((struct ibv_flow_spec *)ib_spec)->hdr.size;
1956 }
1957
1958 written_size = sizeof(*cmd) + cmd->flow_attr.size;
1959 IBV_INIT_CMD_RESP_EX_VCMD(cmd, written_size, written_size, CREATE_FLOW,
1960 &resp, sizeof(resp));
1961 if (write(qp->context->cmd_fd, cmd, written_size) != written_size)
1962 goto err;
1963
1964 (void) VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof(resp));
1965
1966 flow_id->context = qp->context;
1967 flow_id->handle = resp.flow_handle;
1968 return flow_id;
1969 err:
1970 free(flow_id);
1971 return NULL;
1972 }
1973
ibv_cmd_destroy_flow(struct ibv_flow * flow_id)1974 int ibv_cmd_destroy_flow(struct ibv_flow *flow_id)
1975 {
1976 struct ibv_destroy_flow cmd;
1977 int ret = 0;
1978
1979 memset(&cmd, 0, sizeof(cmd));
1980 IBV_INIT_CMD_EX(&cmd, sizeof(cmd), DESTROY_FLOW);
1981 cmd.flow_handle = flow_id->handle;
1982
1983 if (write(flow_id->context->cmd_fd, &cmd, sizeof(cmd)) != sizeof(cmd))
1984 ret = errno;
1985 free(flow_id);
1986 return ret;
1987 }
1988
ibv_cmd_create_wq(struct ibv_context * context,struct ibv_wq_init_attr * wq_init_attr,struct ibv_wq * wq,struct ibv_create_wq * cmd,size_t cmd_core_size,size_t cmd_size,struct ibv_create_wq_resp * resp,size_t resp_core_size,size_t resp_size)1989 int ibv_cmd_create_wq(struct ibv_context *context,
1990 struct ibv_wq_init_attr *wq_init_attr,
1991 struct ibv_wq *wq,
1992 struct ibv_create_wq *cmd,
1993 size_t cmd_core_size,
1994 size_t cmd_size,
1995 struct ibv_create_wq_resp *resp,
1996 size_t resp_core_size,
1997 size_t resp_size)
1998 {
1999 int err;
2000
2001 if (wq_init_attr->comp_mask >= IBV_WQ_INIT_ATTR_RESERVED)
2002 return EINVAL;
2003
2004 IBV_INIT_CMD_RESP_EX_V(cmd, cmd_core_size, cmd_size,
2005 CREATE_WQ, resp,
2006 resp_core_size, resp_size);
2007
2008 cmd->user_handle = (uintptr_t)wq;
2009 cmd->pd_handle = wq_init_attr->pd->handle;
2010 cmd->cq_handle = wq_init_attr->cq->handle;
2011 cmd->wq_type = wq_init_attr->wq_type;
2012 cmd->max_sge = wq_init_attr->max_sge;
2013 cmd->max_wr = wq_init_attr->max_wr;
2014 cmd->comp_mask = 0;
2015
2016 if (cmd_core_size >= offsetof(struct ibv_create_wq, create_flags) +
2017 sizeof(cmd->create_flags)) {
2018 if (wq_init_attr->comp_mask & IBV_WQ_INIT_ATTR_FLAGS) {
2019 if (wq_init_attr->create_flags & ~(IBV_WQ_FLAGS_RESERVED - 1))
2020 return EOPNOTSUPP;
2021 cmd->create_flags = wq_init_attr->create_flags;
2022 }
2023 }
2024
2025 err = write(context->cmd_fd, cmd, cmd_size);
2026 if (err != cmd_size)
2027 return errno;
2028
2029 (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size);
2030
2031 if (resp->response_length < resp_core_size)
2032 return EINVAL;
2033
2034 wq->handle = resp->wq_handle;
2035 wq_init_attr->max_wr = resp->max_wr;
2036 wq_init_attr->max_sge = resp->max_sge;
2037 wq->wq_num = resp->wqn;
2038 wq->context = context;
2039 wq->cq = wq_init_attr->cq;
2040 wq->pd = wq_init_attr->pd;
2041 wq->wq_type = wq_init_attr->wq_type;
2042
2043 return 0;
2044 }
2045
ibv_cmd_modify_wq(struct ibv_wq * wq,struct ibv_wq_attr * attr,struct ibv_modify_wq * cmd,size_t cmd_core_size,size_t cmd_size)2046 int ibv_cmd_modify_wq(struct ibv_wq *wq, struct ibv_wq_attr *attr,
2047 struct ibv_modify_wq *cmd, size_t cmd_core_size,
2048 size_t cmd_size)
2049 {
2050 if (attr->attr_mask >= IBV_WQ_ATTR_RESERVED)
2051 return EINVAL;
2052
2053 memset(cmd, 0, cmd_core_size);
2054 IBV_INIT_CMD_EX(cmd, cmd_size, MODIFY_WQ);
2055
2056 cmd->curr_wq_state = attr->curr_wq_state;
2057 cmd->wq_state = attr->wq_state;
2058 if (cmd_core_size >= offsetof(struct ibv_modify_wq, flags_mask) +
2059 sizeof(cmd->flags_mask)) {
2060 if (attr->attr_mask & IBV_WQ_ATTR_FLAGS) {
2061 if (attr->flags_mask & ~(IBV_WQ_FLAGS_RESERVED - 1))
2062 return EOPNOTSUPP;
2063 cmd->flags = attr->flags;
2064 cmd->flags_mask = attr->flags_mask;
2065 }
2066 }
2067 cmd->wq_handle = wq->handle;
2068 cmd->attr_mask = attr->attr_mask;
2069
2070 if (write(wq->context->cmd_fd, cmd, cmd_size) != cmd_size)
2071 return errno;
2072
2073 if (attr->attr_mask & IBV_WQ_ATTR_STATE)
2074 wq->state = attr->wq_state;
2075
2076 return 0;
2077 }
2078
ibv_cmd_destroy_wq(struct ibv_wq * wq)2079 int ibv_cmd_destroy_wq(struct ibv_wq *wq)
2080 {
2081 struct ibv_destroy_wq cmd;
2082 struct ibv_destroy_wq_resp resp;
2083 int ret = 0;
2084
2085 memset(&cmd, 0, sizeof(cmd));
2086 memset(&resp, 0, sizeof(resp));
2087
2088 IBV_INIT_CMD_RESP_EX(&cmd, sizeof(cmd), DESTROY_WQ, &resp, sizeof(resp));
2089 cmd.wq_handle = wq->handle;
2090
2091 if (write(wq->context->cmd_fd, &cmd, sizeof(cmd)) != sizeof(cmd))
2092 return errno;
2093
2094 if (resp.response_length < sizeof(resp))
2095 return EINVAL;
2096
2097 pthread_mutex_lock(&wq->mutex);
2098 while (wq->events_completed != resp.events_reported)
2099 pthread_cond_wait(&wq->cond, &wq->mutex);
2100 pthread_mutex_unlock(&wq->mutex);
2101
2102 return ret;
2103 }
2104
ibv_cmd_create_rwq_ind_table(struct ibv_context * context,struct ibv_rwq_ind_table_init_attr * init_attr,struct ibv_rwq_ind_table * rwq_ind_table,struct ibv_create_rwq_ind_table * cmd,size_t cmd_core_size,size_t cmd_size,struct ibv_create_rwq_ind_table_resp * resp,size_t resp_core_size,size_t resp_size)2105 int ibv_cmd_create_rwq_ind_table(struct ibv_context *context,
2106 struct ibv_rwq_ind_table_init_attr *init_attr,
2107 struct ibv_rwq_ind_table *rwq_ind_table,
2108 struct ibv_create_rwq_ind_table *cmd,
2109 size_t cmd_core_size,
2110 size_t cmd_size,
2111 struct ibv_create_rwq_ind_table_resp *resp,
2112 size_t resp_core_size,
2113 size_t resp_size)
2114 {
2115 int err, i;
2116 uint32_t required_tbl_size, alloc_tbl_size;
2117 uint32_t *tbl_start;
2118 int num_tbl_entries;
2119
2120 if (init_attr->comp_mask >= IBV_CREATE_IND_TABLE_RESERVED)
2121 return EINVAL;
2122
2123 alloc_tbl_size = cmd_core_size - sizeof(*cmd);
2124 num_tbl_entries = 1 << init_attr->log_ind_tbl_size;
2125
2126 /* Data must be u64 aligned */
2127 required_tbl_size = (num_tbl_entries * sizeof(uint32_t)) < sizeof(uint64_t) ?
2128 sizeof(uint64_t) : (num_tbl_entries * sizeof(uint32_t));
2129
2130 if (alloc_tbl_size < required_tbl_size)
2131 return EINVAL;
2132
2133 tbl_start = (uint32_t *)((uint8_t *)cmd + sizeof(*cmd));
2134 for (i = 0; i < num_tbl_entries; i++)
2135 tbl_start[i] = init_attr->ind_tbl[i]->handle;
2136
2137 IBV_INIT_CMD_RESP_EX_V(cmd, cmd_core_size, cmd_size,
2138 CREATE_RWQ_IND_TBL, resp,
2139 resp_core_size, resp_size);
2140 cmd->log_ind_tbl_size = init_attr->log_ind_tbl_size;
2141 cmd->comp_mask = 0;
2142
2143 err = write(context->cmd_fd, cmd, cmd_size);
2144 if (err != cmd_size)
2145 return errno;
2146
2147 (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size);
2148
2149 if (resp->response_length < resp_core_size)
2150 return EINVAL;
2151
2152 rwq_ind_table->ind_tbl_handle = resp->ind_tbl_handle;
2153 rwq_ind_table->ind_tbl_num = resp->ind_tbl_num;
2154 rwq_ind_table->context = context;
2155 return 0;
2156 }
2157
ibv_cmd_destroy_rwq_ind_table(struct ibv_rwq_ind_table * rwq_ind_table)2158 int ibv_cmd_destroy_rwq_ind_table(struct ibv_rwq_ind_table *rwq_ind_table)
2159 {
2160 struct ibv_destroy_rwq_ind_table cmd;
2161 int ret = 0;
2162
2163 memset(&cmd, 0, sizeof(cmd));
2164 IBV_INIT_CMD_EX(&cmd, sizeof(cmd), DESTROY_RWQ_IND_TBL);
2165 cmd.ind_tbl_handle = rwq_ind_table->ind_tbl_handle;
2166
2167 if (write(rwq_ind_table->context->cmd_fd, &cmd, sizeof(cmd)) != sizeof(cmd))
2168 ret = errno;
2169
2170 return ret;
2171 }
2172