1 /*
2 * Copyright (c) 2005 Topspin Communications. All rights reserved.
3 * Copyright (c) 2005 PathScale, Inc. All rights reserved.
4 * Copyright (c) 2006 Cisco Systems, Inc. All rights reserved.
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 */
34
35 #include <config.h>
36
37 #include <stdio.h>
38 #include <unistd.h>
39 #include <stdlib.h>
40 #include <errno.h>
41 #include <alloca.h>
42 #include <string.h>
43
44 #include "ibverbs.h"
45 #include <sys/param.h>
46
ibv_cmd_get_context(struct ibv_context * context,struct ibv_get_context * cmd,size_t cmd_size,struct ibv_get_context_resp * resp,size_t resp_size)47 int ibv_cmd_get_context(struct ibv_context *context, struct ibv_get_context *cmd,
48 size_t cmd_size, struct ibv_get_context_resp *resp,
49 size_t resp_size)
50 {
51 if (abi_ver < IB_USER_VERBS_MIN_ABI_VERSION)
52 return ENOSYS;
53
54 IBV_INIT_CMD_RESP(cmd, cmd_size, GET_CONTEXT, resp, resp_size);
55
56 if (write(context->cmd_fd, cmd, cmd_size) != cmd_size)
57 return errno;
58
59 (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size);
60
61 context->async_fd = resp->async_fd;
62 context->num_comp_vectors = resp->num_comp_vectors;
63
64 return 0;
65 }
66
copy_query_dev_fields(struct ibv_device_attr * device_attr,struct ibv_query_device_resp * resp,uint64_t * raw_fw_ver)67 static void copy_query_dev_fields(struct ibv_device_attr *device_attr,
68 struct ibv_query_device_resp *resp,
69 uint64_t *raw_fw_ver)
70 {
71 *raw_fw_ver = resp->fw_ver;
72 device_attr->node_guid = resp->node_guid;
73 device_attr->sys_image_guid = resp->sys_image_guid;
74 device_attr->max_mr_size = resp->max_mr_size;
75 device_attr->page_size_cap = resp->page_size_cap;
76 device_attr->vendor_id = resp->vendor_id;
77 device_attr->vendor_part_id = resp->vendor_part_id;
78 device_attr->hw_ver = resp->hw_ver;
79 device_attr->max_qp = resp->max_qp;
80 device_attr->max_qp_wr = resp->max_qp_wr;
81 device_attr->device_cap_flags = resp->device_cap_flags;
82 device_attr->max_sge = resp->max_sge;
83 device_attr->max_sge_rd = resp->max_sge_rd;
84 device_attr->max_cq = resp->max_cq;
85 device_attr->max_cqe = resp->max_cqe;
86 device_attr->max_mr = resp->max_mr;
87 device_attr->max_pd = resp->max_pd;
88 device_attr->max_qp_rd_atom = resp->max_qp_rd_atom;
89 device_attr->max_ee_rd_atom = resp->max_ee_rd_atom;
90 device_attr->max_res_rd_atom = resp->max_res_rd_atom;
91 device_attr->max_qp_init_rd_atom = resp->max_qp_init_rd_atom;
92 device_attr->max_ee_init_rd_atom = resp->max_ee_init_rd_atom;
93 device_attr->atomic_cap = resp->atomic_cap;
94 device_attr->max_ee = resp->max_ee;
95 device_attr->max_rdd = resp->max_rdd;
96 device_attr->max_mw = resp->max_mw;
97 device_attr->max_raw_ipv6_qp = resp->max_raw_ipv6_qp;
98 device_attr->max_raw_ethy_qp = resp->max_raw_ethy_qp;
99 device_attr->max_mcast_grp = resp->max_mcast_grp;
100 device_attr->max_mcast_qp_attach = resp->max_mcast_qp_attach;
101 device_attr->max_total_mcast_qp_attach = resp->max_total_mcast_qp_attach;
102 device_attr->max_ah = resp->max_ah;
103 device_attr->max_fmr = resp->max_fmr;
104 device_attr->max_map_per_fmr = resp->max_map_per_fmr;
105 device_attr->max_srq = resp->max_srq;
106 device_attr->max_srq_wr = resp->max_srq_wr;
107 device_attr->max_srq_sge = resp->max_srq_sge;
108 device_attr->max_pkeys = resp->max_pkeys;
109 device_attr->local_ca_ack_delay = resp->local_ca_ack_delay;
110 device_attr->phys_port_cnt = resp->phys_port_cnt;
111 }
112
ibv_cmd_query_device(struct ibv_context * context,struct ibv_device_attr * device_attr,uint64_t * raw_fw_ver,struct ibv_query_device * cmd,size_t cmd_size)113 int ibv_cmd_query_device(struct ibv_context *context,
114 struct ibv_device_attr *device_attr,
115 uint64_t *raw_fw_ver,
116 struct ibv_query_device *cmd, size_t cmd_size)
117 {
118 struct ibv_query_device_resp resp;
119
120 IBV_INIT_CMD_RESP(cmd, cmd_size, QUERY_DEVICE, &resp, sizeof resp);
121
122 if (write(context->cmd_fd, cmd, cmd_size) != cmd_size)
123 return errno;
124
125 (void) VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof resp);
126
127 memset(device_attr->fw_ver, 0, sizeof device_attr->fw_ver);
128 copy_query_dev_fields(device_attr, &resp, raw_fw_ver);
129
130 return 0;
131 }
132
ibv_cmd_query_device_ex(struct ibv_context * context,const struct ibv_query_device_ex_input * input,struct ibv_device_attr_ex * attr,size_t attr_size,uint64_t * raw_fw_ver,struct ibv_query_device_ex * cmd,size_t cmd_core_size,size_t cmd_size,struct ibv_query_device_resp_ex * resp,size_t resp_core_size,size_t resp_size)133 int ibv_cmd_query_device_ex(struct ibv_context *context,
134 const struct ibv_query_device_ex_input *input,
135 struct ibv_device_attr_ex *attr, size_t attr_size,
136 uint64_t *raw_fw_ver,
137 struct ibv_query_device_ex *cmd,
138 size_t cmd_core_size,
139 size_t cmd_size,
140 struct ibv_query_device_resp_ex *resp,
141 size_t resp_core_size,
142 size_t resp_size)
143 {
144 int err;
145
146 if (input && input->comp_mask)
147 return EINVAL;
148
149 if (attr_size < offsetof(struct ibv_device_attr_ex, comp_mask) +
150 sizeof(attr->comp_mask))
151 return EINVAL;
152
153 if (resp_core_size < offsetof(struct ibv_query_device_resp_ex,
154 response_length) +
155 sizeof(resp->response_length))
156 return EINVAL;
157
158 IBV_INIT_CMD_RESP_EX_V(cmd, cmd_core_size, cmd_size,
159 QUERY_DEVICE_EX, resp, resp_core_size,
160 resp_size);
161 cmd->comp_mask = 0;
162 cmd->reserved = 0;
163 memset(attr->orig_attr.fw_ver, 0, sizeof(attr->orig_attr.fw_ver));
164 memset(&attr->comp_mask, 0, attr_size - sizeof(attr->orig_attr));
165 err = write(context->cmd_fd, cmd, cmd_size);
166 if (err != cmd_size)
167 return errno;
168
169 (void)VALGRIND_MAKE_MEM_DEFINED(resp, resp_size);
170 copy_query_dev_fields(&attr->orig_attr, &resp->base, raw_fw_ver);
171 /* Report back supported comp_mask bits. For now no comp_mask bit is
172 * defined */
173 attr->comp_mask = resp->comp_mask & 0;
174 if (attr_size >= offsetof(struct ibv_device_attr_ex, odp_caps) +
175 sizeof(attr->odp_caps)) {
176 if (resp->response_length >=
177 offsetof(struct ibv_query_device_resp_ex, odp_caps) +
178 sizeof(resp->odp_caps)) {
179 attr->odp_caps.general_caps = resp->odp_caps.general_caps;
180 attr->odp_caps.per_transport_caps.rc_odp_caps =
181 resp->odp_caps.per_transport_caps.rc_odp_caps;
182 attr->odp_caps.per_transport_caps.uc_odp_caps =
183 resp->odp_caps.per_transport_caps.uc_odp_caps;
184 attr->odp_caps.per_transport_caps.ud_odp_caps =
185 resp->odp_caps.per_transport_caps.ud_odp_caps;
186 }
187 }
188
189 if (attr_size >= offsetof(struct ibv_device_attr_ex,
190 completion_timestamp_mask) +
191 sizeof(attr->completion_timestamp_mask)) {
192 if (resp->response_length >=
193 offsetof(struct ibv_query_device_resp_ex, timestamp_mask) +
194 sizeof(resp->timestamp_mask))
195 attr->completion_timestamp_mask = resp->timestamp_mask;
196 }
197
198 if (attr_size >= offsetof(struct ibv_device_attr_ex, hca_core_clock) +
199 sizeof(attr->hca_core_clock)) {
200 if (resp->response_length >=
201 offsetof(struct ibv_query_device_resp_ex, hca_core_clock) +
202 sizeof(resp->hca_core_clock))
203 attr->hca_core_clock = resp->hca_core_clock;
204 }
205
206 if (attr_size >= offsetof(struct ibv_device_attr_ex, device_cap_flags_ex) +
207 sizeof(attr->device_cap_flags_ex)) {
208 if (resp->response_length >=
209 offsetof(struct ibv_query_device_resp_ex, device_cap_flags_ex) +
210 sizeof(resp->device_cap_flags_ex))
211 attr->device_cap_flags_ex = resp->device_cap_flags_ex;
212 }
213
214 if (attr_size >= offsetof(struct ibv_device_attr_ex, rss_caps) +
215 sizeof(attr->rss_caps)) {
216 if (resp->response_length >=
217 offsetof(struct ibv_query_device_resp_ex, rss_caps) +
218 sizeof(resp->rss_caps)) {
219 attr->rss_caps.supported_qpts = resp->rss_caps.supported_qpts;
220 attr->rss_caps.max_rwq_indirection_tables = resp->rss_caps.max_rwq_indirection_tables;
221 attr->rss_caps.max_rwq_indirection_table_size = resp->rss_caps.max_rwq_indirection_table_size;
222 }
223 }
224
225 if (attr_size >= offsetof(struct ibv_device_attr_ex, max_wq_type_rq) +
226 sizeof(attr->max_wq_type_rq)) {
227 if (resp->response_length >=
228 offsetof(struct ibv_query_device_resp_ex, max_wq_type_rq) +
229 sizeof(resp->max_wq_type_rq))
230 attr->max_wq_type_rq = resp->max_wq_type_rq;
231 }
232
233 if (attr_size >= offsetof(struct ibv_device_attr_ex, raw_packet_caps) +
234 sizeof(attr->raw_packet_caps)) {
235 if (resp->response_length >=
236 offsetof(struct ibv_query_device_resp_ex, raw_packet_caps) +
237 sizeof(resp->raw_packet_caps))
238 attr->raw_packet_caps = resp->raw_packet_caps;
239 }
240
241 return 0;
242 }
243
ibv_cmd_query_port(struct ibv_context * context,uint8_t port_num,struct ibv_port_attr * port_attr,struct ibv_query_port * cmd,size_t cmd_size)244 int ibv_cmd_query_port(struct ibv_context *context, uint8_t port_num,
245 struct ibv_port_attr *port_attr,
246 struct ibv_query_port *cmd, size_t cmd_size)
247 {
248 struct ibv_query_port_resp resp;
249
250 IBV_INIT_CMD_RESP(cmd, cmd_size, QUERY_PORT, &resp, sizeof resp);
251 cmd->port_num = port_num;
252 memset(cmd->reserved, 0, sizeof cmd->reserved);
253
254 if (write(context->cmd_fd, cmd, cmd_size) != cmd_size)
255 return errno;
256
257 (void) VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof resp);
258
259 port_attr->state = resp.state;
260 port_attr->max_mtu = resp.max_mtu;
261 port_attr->active_mtu = resp.active_mtu;
262 port_attr->gid_tbl_len = resp.gid_tbl_len;
263 port_attr->port_cap_flags = resp.port_cap_flags;
264 port_attr->max_msg_sz = resp.max_msg_sz;
265 port_attr->bad_pkey_cntr = resp.bad_pkey_cntr;
266 port_attr->qkey_viol_cntr = resp.qkey_viol_cntr;
267 port_attr->pkey_tbl_len = resp.pkey_tbl_len;
268 port_attr->lid = resp.lid;
269 port_attr->sm_lid = resp.sm_lid;
270 port_attr->lmc = resp.lmc;
271 port_attr->max_vl_num = resp.max_vl_num;
272 port_attr->sm_sl = resp.sm_sl;
273 port_attr->subnet_timeout = resp.subnet_timeout;
274 port_attr->init_type_reply = resp.init_type_reply;
275 port_attr->active_width = resp.active_width;
276 port_attr->active_speed = resp.active_speed;
277 port_attr->phys_state = resp.phys_state;
278 port_attr->link_layer = resp.link_layer;
279
280 return 0;
281 }
282
ibv_cmd_alloc_pd(struct ibv_context * context,struct ibv_pd * pd,struct ibv_alloc_pd * cmd,size_t cmd_size,struct ibv_alloc_pd_resp * resp,size_t resp_size)283 int ibv_cmd_alloc_pd(struct ibv_context *context, struct ibv_pd *pd,
284 struct ibv_alloc_pd *cmd, size_t cmd_size,
285 struct ibv_alloc_pd_resp *resp, size_t resp_size)
286 {
287 IBV_INIT_CMD_RESP(cmd, cmd_size, ALLOC_PD, resp, resp_size);
288
289 if (write(context->cmd_fd, cmd, cmd_size) != cmd_size)
290 return errno;
291
292 (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size);
293
294 pd->handle = resp->pd_handle;
295 pd->context = context;
296
297 return 0;
298 }
299
ibv_cmd_dealloc_pd(struct ibv_pd * pd)300 int ibv_cmd_dealloc_pd(struct ibv_pd *pd)
301 {
302 struct ibv_dealloc_pd cmd;
303
304 IBV_INIT_CMD(&cmd, sizeof cmd, DEALLOC_PD);
305 cmd.pd_handle = pd->handle;
306
307 if (write(pd->context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd)
308 return errno;
309
310 return 0;
311 }
312
ibv_cmd_open_xrcd(struct ibv_context * context,struct verbs_xrcd * xrcd,int vxrcd_size,struct ibv_xrcd_init_attr * attr,struct ibv_open_xrcd * cmd,size_t cmd_size,struct ibv_open_xrcd_resp * resp,size_t resp_size)313 int ibv_cmd_open_xrcd(struct ibv_context *context, struct verbs_xrcd *xrcd,
314 int vxrcd_size,
315 struct ibv_xrcd_init_attr *attr,
316 struct ibv_open_xrcd *cmd, size_t cmd_size,
317 struct ibv_open_xrcd_resp *resp, size_t resp_size)
318 {
319 IBV_INIT_CMD_RESP(cmd, cmd_size, OPEN_XRCD, resp, resp_size);
320
321 if (attr->comp_mask >= IBV_XRCD_INIT_ATTR_RESERVED)
322 return ENOSYS;
323
324 if (!(attr->comp_mask & IBV_XRCD_INIT_ATTR_FD) ||
325 !(attr->comp_mask & IBV_XRCD_INIT_ATTR_OFLAGS))
326 return EINVAL;
327
328 cmd->fd = attr->fd;
329 cmd->oflags = attr->oflags;
330 if (write(context->cmd_fd, cmd, cmd_size) != cmd_size)
331 return errno;
332
333 (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size);
334
335 xrcd->xrcd.context = context;
336 xrcd->comp_mask = 0;
337 if (vext_field_avail(struct verbs_xrcd, handle, vxrcd_size)) {
338 xrcd->comp_mask = VERBS_XRCD_HANDLE;
339 xrcd->handle = resp->xrcd_handle;
340 }
341
342 return 0;
343 }
344
ibv_cmd_close_xrcd(struct verbs_xrcd * xrcd)345 int ibv_cmd_close_xrcd(struct verbs_xrcd *xrcd)
346 {
347 struct ibv_close_xrcd cmd;
348
349 IBV_INIT_CMD(&cmd, sizeof cmd, CLOSE_XRCD);
350 cmd.xrcd_handle = xrcd->handle;
351
352 if (write(xrcd->xrcd.context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd)
353 return errno;
354
355 return 0;
356 }
357
ibv_cmd_reg_mr(struct ibv_pd * pd,void * addr,size_t length,uint64_t hca_va,int access,struct ibv_mr * mr,struct ibv_reg_mr * cmd,size_t cmd_size,struct ibv_reg_mr_resp * resp,size_t resp_size)358 int ibv_cmd_reg_mr(struct ibv_pd *pd, void *addr, size_t length,
359 uint64_t hca_va, int access,
360 struct ibv_mr *mr, struct ibv_reg_mr *cmd,
361 size_t cmd_size,
362 struct ibv_reg_mr_resp *resp, size_t resp_size)
363 {
364
365 IBV_INIT_CMD_RESP(cmd, cmd_size, REG_MR, resp, resp_size);
366
367 cmd->start = (uintptr_t) addr;
368 cmd->length = length;
369 cmd->hca_va = hca_va;
370 cmd->pd_handle = pd->handle;
371 cmd->access_flags = access;
372
373 if (write(pd->context->cmd_fd, cmd, cmd_size) != cmd_size)
374 return errno;
375
376 (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size);
377
378 mr->handle = resp->mr_handle;
379 mr->lkey = resp->lkey;
380 mr->rkey = resp->rkey;
381 mr->context = pd->context;
382
383 return 0;
384 }
385
ibv_cmd_rereg_mr(struct ibv_mr * mr,uint32_t flags,void * addr,size_t length,uint64_t hca_va,int access,struct ibv_pd * pd,struct ibv_rereg_mr * cmd,size_t cmd_sz,struct ibv_rereg_mr_resp * resp,size_t resp_sz)386 int ibv_cmd_rereg_mr(struct ibv_mr *mr, uint32_t flags, void *addr,
387 size_t length, uint64_t hca_va, int access,
388 struct ibv_pd *pd, struct ibv_rereg_mr *cmd,
389 size_t cmd_sz, struct ibv_rereg_mr_resp *resp,
390 size_t resp_sz)
391 {
392 IBV_INIT_CMD_RESP(cmd, cmd_sz, REREG_MR, resp, resp_sz);
393
394 cmd->mr_handle = mr->handle;
395 cmd->flags = flags;
396 cmd->start = (uintptr_t)addr;
397 cmd->length = length;
398 cmd->hca_va = hca_va;
399 cmd->pd_handle = (flags & IBV_REREG_MR_CHANGE_PD) ? pd->handle : 0;
400 cmd->access_flags = access;
401
402 if (write(mr->context->cmd_fd, cmd, cmd_sz) != cmd_sz)
403 return errno;
404
405 (void)VALGRIND_MAKE_MEM_DEFINED(resp, resp_sz);
406
407 mr->lkey = resp->lkey;
408 mr->rkey = resp->rkey;
409 if (flags & IBV_REREG_MR_CHANGE_PD)
410 mr->context = pd->context;
411
412 return 0;
413 }
414
ibv_cmd_dereg_mr(struct ibv_mr * mr)415 int ibv_cmd_dereg_mr(struct ibv_mr *mr)
416 {
417 struct ibv_dereg_mr cmd;
418
419 IBV_INIT_CMD(&cmd, sizeof cmd, DEREG_MR);
420 cmd.mr_handle = mr->handle;
421
422 if (write(mr->context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd)
423 return errno;
424
425 return 0;
426 }
427
ibv_cmd_alloc_mw(struct ibv_pd * pd,enum ibv_mw_type type,struct ibv_mw * mw,struct ibv_alloc_mw * cmd,size_t cmd_size,struct ibv_alloc_mw_resp * resp,size_t resp_size)428 int ibv_cmd_alloc_mw(struct ibv_pd *pd, enum ibv_mw_type type,
429 struct ibv_mw *mw, struct ibv_alloc_mw *cmd,
430 size_t cmd_size,
431 struct ibv_alloc_mw_resp *resp, size_t resp_size)
432 {
433 IBV_INIT_CMD_RESP(cmd, cmd_size, ALLOC_MW, resp, resp_size);
434 cmd->pd_handle = pd->handle;
435 cmd->mw_type = type;
436 memset(cmd->reserved, 0, sizeof(cmd->reserved));
437
438 if (write(pd->context->cmd_fd, cmd, cmd_size) != cmd_size)
439 return errno;
440
441 (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size);
442
443 mw->context = pd->context;
444 mw->pd = pd;
445 mw->rkey = resp->rkey;
446 mw->handle = resp->mw_handle;
447 mw->type = type;
448
449 return 0;
450 }
451
ibv_cmd_dealloc_mw(struct ibv_mw * mw,struct ibv_dealloc_mw * cmd,size_t cmd_size)452 int ibv_cmd_dealloc_mw(struct ibv_mw *mw,
453 struct ibv_dealloc_mw *cmd, size_t cmd_size)
454 {
455 IBV_INIT_CMD(cmd, cmd_size, DEALLOC_MW);
456 cmd->mw_handle = mw->handle;
457 cmd->reserved = 0;
458
459 if (write(mw->context->cmd_fd, cmd, cmd_size) != cmd_size)
460 return errno;
461
462 return 0;
463 }
464
ibv_cmd_create_cq(struct ibv_context * context,int cqe,struct ibv_comp_channel * channel,int comp_vector,struct ibv_cq * cq,struct ibv_create_cq * cmd,size_t cmd_size,struct ibv_create_cq_resp * resp,size_t resp_size)465 int ibv_cmd_create_cq(struct ibv_context *context, int cqe,
466 struct ibv_comp_channel *channel,
467 int comp_vector, struct ibv_cq *cq,
468 struct ibv_create_cq *cmd, size_t cmd_size,
469 struct ibv_create_cq_resp *resp, size_t resp_size)
470 {
471 IBV_INIT_CMD_RESP(cmd, cmd_size, CREATE_CQ, resp, resp_size);
472 cmd->user_handle = (uintptr_t) cq;
473 cmd->cqe = cqe;
474 cmd->comp_vector = comp_vector;
475 cmd->comp_channel = channel ? channel->fd : -1;
476 cmd->reserved = 0;
477
478 if (write(context->cmd_fd, cmd, cmd_size) != cmd_size)
479 return errno;
480
481 (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size);
482
483 cq->handle = resp->cq_handle;
484 cq->cqe = resp->cqe;
485 cq->context = context;
486
487 return 0;
488 }
489
ibv_cmd_create_cq_ex(struct ibv_context * context,struct ibv_cq_init_attr_ex * cq_attr,struct ibv_cq_ex * cq,struct ibv_create_cq_ex * cmd,size_t cmd_core_size,size_t cmd_size,struct ibv_create_cq_resp_ex * resp,size_t resp_core_size,size_t resp_size)490 int ibv_cmd_create_cq_ex(struct ibv_context *context,
491 struct ibv_cq_init_attr_ex *cq_attr,
492 struct ibv_cq_ex *cq,
493 struct ibv_create_cq_ex *cmd,
494 size_t cmd_core_size,
495 size_t cmd_size,
496 struct ibv_create_cq_resp_ex *resp,
497 size_t resp_core_size,
498 size_t resp_size)
499 {
500 int err;
501
502 memset(cmd, 0, cmd_core_size);
503 IBV_INIT_CMD_RESP_EX_V(cmd, cmd_core_size, cmd_size, CREATE_CQ_EX, resp,
504 resp_core_size, resp_size);
505
506 if (cq_attr->comp_mask & ~(IBV_CQ_INIT_ATTR_MASK_RESERVED - 1))
507 return EINVAL;
508
509 cmd->user_handle = (uintptr_t)cq;
510 cmd->cqe = cq_attr->cqe;
511 cmd->comp_vector = cq_attr->comp_vector;
512 cmd->comp_channel = cq_attr->channel ? cq_attr->channel->fd : -1;
513 cmd->comp_mask = 0;
514
515 if (cmd_core_size >= offsetof(struct ibv_create_cq_ex, flags) +
516 sizeof(cmd->flags)) {
517 if ((cq_attr->comp_mask & IBV_CQ_INIT_ATTR_MASK_FLAGS) &&
518 (cq_attr->flags & ~(IBV_CREATE_CQ_ATTR_RESERVED - 1)))
519 return EOPNOTSUPP;
520
521 if (cq_attr->wc_flags & IBV_WC_EX_WITH_COMPLETION_TIMESTAMP)
522 cmd->flags |= IBV_CREATE_CQ_EX_KERNEL_FLAG_COMPLETION_TIMESTAMP;
523 }
524
525 err = write(context->cmd_fd, cmd, cmd_size);
526 if (err != cmd_size)
527 return errno;
528
529 (void)VALGRIND_MAKE_MEM_DEFINED(resp, resp_size);
530
531 cq->handle = resp->base.cq_handle;
532 cq->cqe = resp->base.cqe;
533 cq->context = context;
534
535 return 0;
536 }
537
ibv_cmd_poll_cq(struct ibv_cq * ibcq,int ne,struct ibv_wc * wc)538 int ibv_cmd_poll_cq(struct ibv_cq *ibcq, int ne, struct ibv_wc *wc)
539 {
540 struct ibv_poll_cq cmd;
541 struct ibv_poll_cq_resp *resp;
542 int i;
543 int rsize;
544 int ret;
545
546 rsize = sizeof *resp + ne * sizeof(struct ibv_kern_wc);
547 resp = malloc(rsize);
548 if (!resp)
549 return -1;
550
551 IBV_INIT_CMD_RESP(&cmd, sizeof cmd, POLL_CQ, resp, rsize);
552 cmd.cq_handle = ibcq->handle;
553 cmd.ne = ne;
554
555 if (write(ibcq->context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd) {
556 ret = -1;
557 goto out;
558 }
559
560 (void) VALGRIND_MAKE_MEM_DEFINED(resp, rsize);
561
562 for (i = 0; i < resp->count; i++) {
563 wc[i].wr_id = resp->wc[i].wr_id;
564 wc[i].status = resp->wc[i].status;
565 wc[i].opcode = resp->wc[i].opcode;
566 wc[i].vendor_err = resp->wc[i].vendor_err;
567 wc[i].byte_len = resp->wc[i].byte_len;
568 wc[i].imm_data = resp->wc[i].imm_data;
569 wc[i].qp_num = resp->wc[i].qp_num;
570 wc[i].src_qp = resp->wc[i].src_qp;
571 wc[i].wc_flags = resp->wc[i].wc_flags;
572 wc[i].pkey_index = resp->wc[i].pkey_index;
573 wc[i].slid = resp->wc[i].slid;
574 wc[i].sl = resp->wc[i].sl;
575 wc[i].dlid_path_bits = resp->wc[i].dlid_path_bits;
576 }
577
578 ret = resp->count;
579
580 out:
581 free(resp);
582 return ret;
583 }
584
ibv_cmd_req_notify_cq(struct ibv_cq * ibcq,int solicited_only)585 int ibv_cmd_req_notify_cq(struct ibv_cq *ibcq, int solicited_only)
586 {
587 struct ibv_req_notify_cq cmd;
588
589 IBV_INIT_CMD(&cmd, sizeof cmd, REQ_NOTIFY_CQ);
590 cmd.cq_handle = ibcq->handle;
591 cmd.solicited = !!solicited_only;
592
593 if (write(ibcq->context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd)
594 return errno;
595
596 return 0;
597 }
598
ibv_cmd_resize_cq(struct ibv_cq * cq,int cqe,struct ibv_resize_cq * cmd,size_t cmd_size,struct ibv_resize_cq_resp * resp,size_t resp_size)599 int ibv_cmd_resize_cq(struct ibv_cq *cq, int cqe,
600 struct ibv_resize_cq *cmd, size_t cmd_size,
601 struct ibv_resize_cq_resp *resp, size_t resp_size)
602 {
603 IBV_INIT_CMD_RESP(cmd, cmd_size, RESIZE_CQ, resp, resp_size);
604 cmd->cq_handle = cq->handle;
605 cmd->cqe = cqe;
606
607 if (write(cq->context->cmd_fd, cmd, cmd_size) != cmd_size)
608 return errno;
609
610 (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size);
611
612 cq->cqe = resp->cqe;
613
614 return 0;
615 }
616
ibv_cmd_destroy_cq(struct ibv_cq * cq)617 int ibv_cmd_destroy_cq(struct ibv_cq *cq)
618 {
619 struct ibv_destroy_cq cmd;
620 struct ibv_destroy_cq_resp resp;
621
622 IBV_INIT_CMD_RESP(&cmd, sizeof cmd, DESTROY_CQ, &resp, sizeof resp);
623 cmd.cq_handle = cq->handle;
624 cmd.reserved = 0;
625
626 if (write(cq->context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd)
627 return errno;
628
629 (void) VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof resp);
630
631 pthread_mutex_lock(&cq->mutex);
632 while (cq->comp_events_completed != resp.comp_events_reported ||
633 cq->async_events_completed != resp.async_events_reported)
634 pthread_cond_wait(&cq->cond, &cq->mutex);
635 pthread_mutex_unlock(&cq->mutex);
636
637 return 0;
638 }
639
ibv_cmd_create_srq(struct ibv_pd * pd,struct ibv_srq * srq,struct ibv_srq_init_attr * attr,struct ibv_create_srq * cmd,size_t cmd_size,struct ibv_create_srq_resp * resp,size_t resp_size)640 int ibv_cmd_create_srq(struct ibv_pd *pd,
641 struct ibv_srq *srq, struct ibv_srq_init_attr *attr,
642 struct ibv_create_srq *cmd, size_t cmd_size,
643 struct ibv_create_srq_resp *resp, size_t resp_size)
644 {
645 IBV_INIT_CMD_RESP(cmd, cmd_size, CREATE_SRQ, resp, resp_size);
646 cmd->user_handle = (uintptr_t) srq;
647 cmd->pd_handle = pd->handle;
648 cmd->max_wr = attr->attr.max_wr;
649 cmd->max_sge = attr->attr.max_sge;
650 cmd->srq_limit = attr->attr.srq_limit;
651
652 if (write(pd->context->cmd_fd, cmd, cmd_size) != cmd_size)
653 return errno;
654
655 (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size);
656
657 srq->handle = resp->srq_handle;
658 srq->context = pd->context;
659
660 if (abi_ver > 5) {
661 attr->attr.max_wr = resp->max_wr;
662 attr->attr.max_sge = resp->max_sge;
663 } else {
664 struct ibv_create_srq_resp_v5 *resp_v5 =
665 (struct ibv_create_srq_resp_v5 *) resp;
666
667 memmove((void *) resp + sizeof *resp,
668 (void *) resp_v5 + sizeof *resp_v5,
669 resp_size - sizeof *resp);
670 }
671
672 return 0;
673 }
674
ibv_cmd_create_srq_ex(struct ibv_context * context,struct verbs_srq * srq,int vsrq_sz,struct ibv_srq_init_attr_ex * attr_ex,struct ibv_create_xsrq * cmd,size_t cmd_size,struct ibv_create_srq_resp * resp,size_t resp_size)675 int ibv_cmd_create_srq_ex(struct ibv_context *context,
676 struct verbs_srq *srq, int vsrq_sz,
677 struct ibv_srq_init_attr_ex *attr_ex,
678 struct ibv_create_xsrq *cmd, size_t cmd_size,
679 struct ibv_create_srq_resp *resp, size_t resp_size)
680 {
681 struct verbs_xrcd *vxrcd = NULL;
682 int ret = 0;
683
684 IBV_INIT_CMD_RESP(cmd, cmd_size, CREATE_XSRQ, resp, resp_size);
685
686 if (attr_ex->comp_mask >= IBV_SRQ_INIT_ATTR_RESERVED)
687 return ENOSYS;
688
689 if (!(attr_ex->comp_mask & IBV_SRQ_INIT_ATTR_PD))
690 return EINVAL;
691
692 cmd->user_handle = (uintptr_t) srq;
693 cmd->pd_handle = attr_ex->pd->handle;
694 cmd->max_wr = attr_ex->attr.max_wr;
695 cmd->max_sge = attr_ex->attr.max_sge;
696 cmd->srq_limit = attr_ex->attr.srq_limit;
697
698 cmd->srq_type = (attr_ex->comp_mask & IBV_SRQ_INIT_ATTR_TYPE) ?
699 attr_ex->srq_type : IBV_SRQT_BASIC;
700 if (attr_ex->comp_mask & IBV_SRQ_INIT_ATTR_XRCD) {
701 if (!(attr_ex->comp_mask & IBV_SRQ_INIT_ATTR_CQ))
702 return EINVAL;
703
704 vxrcd = container_of(attr_ex->xrcd, struct verbs_xrcd, xrcd);
705 cmd->xrcd_handle = vxrcd->handle;
706 cmd->cq_handle = attr_ex->cq->handle;
707 }
708
709 ret = pthread_mutex_init(&srq->srq.mutex, NULL);
710 if (ret)
711 goto err;
712 ret = pthread_cond_init(&srq->srq.cond, NULL);
713 if (ret)
714 goto err_mutex;
715
716 if (write(context->cmd_fd, cmd, cmd_size) != cmd_size) {
717 ret = errno;
718 goto err_cond;
719 }
720
721 (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size);
722
723 srq->srq.handle = resp->srq_handle;
724 srq->srq.context = context;
725 srq->srq.srq_context = attr_ex->srq_context;
726 srq->srq.pd = attr_ex->pd;
727 srq->srq.events_completed = 0;
728
729 /*
730 * check that the last field is available.
731 * If it is than all the others exist as well
732 */
733 if (vext_field_avail(struct verbs_srq, srq_num, vsrq_sz)) {
734 srq->comp_mask = IBV_SRQ_INIT_ATTR_TYPE;
735 srq->srq_type = (attr_ex->comp_mask & IBV_SRQ_INIT_ATTR_TYPE) ?
736 attr_ex->srq_type : IBV_SRQT_BASIC;
737 if (srq->srq_type == IBV_SRQT_XRC) {
738 srq->comp_mask |= VERBS_SRQ_NUM;
739 srq->srq_num = resp->srqn;
740 }
741 if (attr_ex->comp_mask & IBV_SRQ_INIT_ATTR_XRCD) {
742 srq->comp_mask |= VERBS_SRQ_XRCD;
743 srq->xrcd = vxrcd;
744 }
745 if (attr_ex->comp_mask & IBV_SRQ_INIT_ATTR_CQ) {
746 srq->comp_mask |= VERBS_SRQ_CQ;
747 srq->cq = attr_ex->cq;
748 }
749 }
750
751 attr_ex->attr.max_wr = resp->max_wr;
752 attr_ex->attr.max_sge = resp->max_sge;
753
754 return 0;
755 err_cond:
756 pthread_cond_destroy(&srq->srq.cond);
757 err_mutex:
758 pthread_mutex_destroy(&srq->srq.mutex);
759 err:
760 return ret;
761 }
762
763
ibv_cmd_modify_srq_v3(struct ibv_srq * srq,struct ibv_srq_attr * srq_attr,int srq_attr_mask,struct ibv_modify_srq * new_cmd,size_t new_cmd_size)764 static int ibv_cmd_modify_srq_v3(struct ibv_srq *srq,
765 struct ibv_srq_attr *srq_attr,
766 int srq_attr_mask,
767 struct ibv_modify_srq *new_cmd,
768 size_t new_cmd_size)
769 {
770 struct ibv_modify_srq_v3 *cmd;
771 size_t cmd_size;
772
773 cmd_size = sizeof *cmd + new_cmd_size - sizeof *new_cmd;
774 cmd = alloca(cmd_size);
775 memcpy(cmd->driver_data, new_cmd->driver_data, new_cmd_size - sizeof *new_cmd);
776
777 IBV_INIT_CMD(cmd, cmd_size, MODIFY_SRQ);
778
779 cmd->srq_handle = srq->handle;
780 cmd->attr_mask = srq_attr_mask;
781 cmd->max_wr = srq_attr->max_wr;
782 cmd->srq_limit = srq_attr->srq_limit;
783 cmd->max_sge = 0;
784 cmd->reserved = 0;
785
786 if (write(srq->context->cmd_fd, cmd, cmd_size) != cmd_size)
787 return errno;
788
789 return 0;
790 }
791
ibv_cmd_modify_srq(struct ibv_srq * srq,struct ibv_srq_attr * srq_attr,int srq_attr_mask,struct ibv_modify_srq * cmd,size_t cmd_size)792 int ibv_cmd_modify_srq(struct ibv_srq *srq,
793 struct ibv_srq_attr *srq_attr,
794 int srq_attr_mask,
795 struct ibv_modify_srq *cmd, size_t cmd_size)
796 {
797 if (abi_ver == 3)
798 return ibv_cmd_modify_srq_v3(srq, srq_attr, srq_attr_mask,
799 cmd, cmd_size);
800
801 IBV_INIT_CMD(cmd, cmd_size, MODIFY_SRQ);
802
803 cmd->srq_handle = srq->handle;
804 cmd->attr_mask = srq_attr_mask;
805 cmd->max_wr = srq_attr->max_wr;
806 cmd->srq_limit = srq_attr->srq_limit;
807
808 if (write(srq->context->cmd_fd, cmd, cmd_size) != cmd_size)
809 return errno;
810
811 return 0;
812 }
813
ibv_cmd_query_srq(struct ibv_srq * srq,struct ibv_srq_attr * srq_attr,struct ibv_query_srq * cmd,size_t cmd_size)814 int ibv_cmd_query_srq(struct ibv_srq *srq, struct ibv_srq_attr *srq_attr,
815 struct ibv_query_srq *cmd, size_t cmd_size)
816 {
817 struct ibv_query_srq_resp resp;
818
819 IBV_INIT_CMD_RESP(cmd, cmd_size, QUERY_SRQ, &resp, sizeof resp);
820 cmd->srq_handle = srq->handle;
821 cmd->reserved = 0;
822
823 if (write(srq->context->cmd_fd, cmd, cmd_size) != cmd_size)
824 return errno;
825
826 (void) VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof resp);
827
828 srq_attr->max_wr = resp.max_wr;
829 srq_attr->max_sge = resp.max_sge;
830 srq_attr->srq_limit = resp.srq_limit;
831
832 return 0;
833 }
834
ibv_cmd_destroy_srq(struct ibv_srq * srq)835 int ibv_cmd_destroy_srq(struct ibv_srq *srq)
836 {
837 struct ibv_destroy_srq cmd;
838 struct ibv_destroy_srq_resp resp;
839
840 IBV_INIT_CMD_RESP(&cmd, sizeof cmd, DESTROY_SRQ, &resp, sizeof resp);
841 cmd.srq_handle = srq->handle;
842 cmd.reserved = 0;
843
844 if (write(srq->context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd)
845 return errno;
846
847 (void) VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof resp);
848
849 pthread_mutex_lock(&srq->mutex);
850 while (srq->events_completed != resp.events_reported)
851 pthread_cond_wait(&srq->cond, &srq->mutex);
852 pthread_mutex_unlock(&srq->mutex);
853
854 pthread_cond_destroy(&srq->cond);
855 pthread_mutex_destroy(&srq->mutex);
856
857 return 0;
858 }
859
create_qp_ex_common(struct verbs_qp * qp,struct ibv_qp_init_attr_ex * qp_attr,struct verbs_xrcd * vxrcd,struct ibv_create_qp_common * cmd)860 static int create_qp_ex_common(struct verbs_qp *qp,
861 struct ibv_qp_init_attr_ex *qp_attr,
862 struct verbs_xrcd *vxrcd,
863 struct ibv_create_qp_common *cmd)
864 {
865 cmd->user_handle = (uintptr_t)qp;
866
867 if (qp_attr->comp_mask & IBV_QP_INIT_ATTR_XRCD) {
868 vxrcd = container_of(qp_attr->xrcd, struct verbs_xrcd, xrcd);
869 cmd->pd_handle = vxrcd->handle;
870 } else {
871 if (!(qp_attr->comp_mask & IBV_QP_INIT_ATTR_PD))
872 return EINVAL;
873
874 cmd->pd_handle = qp_attr->pd->handle;
875 if (qp_attr->comp_mask & IBV_QP_INIT_ATTR_IND_TABLE) {
876 if (cmd->max_recv_wr || cmd->max_recv_sge ||
877 cmd->recv_cq_handle || qp_attr->srq)
878 return EINVAL;
879
880 /* send_cq is optinal */
881 if (qp_attr->cap.max_send_wr)
882 cmd->send_cq_handle = qp_attr->send_cq->handle;
883 } else {
884 cmd->send_cq_handle = qp_attr->send_cq->handle;
885
886 if (qp_attr->qp_type != IBV_QPT_XRC_SEND) {
887 cmd->recv_cq_handle = qp_attr->recv_cq->handle;
888 cmd->srq_handle = qp_attr->srq ? qp_attr->srq->handle :
889 0;
890 }
891 }
892 }
893
894 cmd->max_send_wr = qp_attr->cap.max_send_wr;
895 cmd->max_recv_wr = qp_attr->cap.max_recv_wr;
896 cmd->max_send_sge = qp_attr->cap.max_send_sge;
897 cmd->max_recv_sge = qp_attr->cap.max_recv_sge;
898 cmd->max_inline_data = qp_attr->cap.max_inline_data;
899 cmd->sq_sig_all = qp_attr->sq_sig_all;
900 cmd->qp_type = qp_attr->qp_type;
901 cmd->is_srq = !!qp_attr->srq;
902 cmd->reserved = 0;
903
904 return 0;
905 }
906
create_qp_handle_resp_common_cleanup(struct verbs_qp * qp)907 static void create_qp_handle_resp_common_cleanup(struct verbs_qp *qp)
908 {
909 pthread_cond_destroy(&qp->qp.cond);
910 pthread_mutex_destroy(&qp->qp.mutex);
911 }
912
create_qp_handle_resp_common_init(struct verbs_qp * qp)913 static int create_qp_handle_resp_common_init(struct verbs_qp *qp)
914 {
915 int ret = 0;
916
917 ret = pthread_mutex_init(&qp->qp.mutex, NULL);
918 if (ret)
919 return ret;
920 ret = pthread_cond_init(&qp->qp.cond, NULL);
921 if (ret)
922 goto err;
923
924 return ret;
925
926 err:
927 pthread_mutex_destroy(&qp->qp.mutex);
928
929 return ret;
930 }
931
create_qp_handle_resp_common(struct ibv_context * context,struct verbs_qp * qp,struct ibv_qp_init_attr_ex * qp_attr,struct ibv_create_qp_resp * resp,struct verbs_xrcd * vxrcd,int vqp_sz)932 static void create_qp_handle_resp_common(struct ibv_context *context,
933 struct verbs_qp *qp,
934 struct ibv_qp_init_attr_ex *qp_attr,
935 struct ibv_create_qp_resp *resp,
936 struct verbs_xrcd *vxrcd,
937 int vqp_sz)
938 {
939 if (abi_ver > 3) {
940 qp_attr->cap.max_recv_sge = resp->max_recv_sge;
941 qp_attr->cap.max_send_sge = resp->max_send_sge;
942 qp_attr->cap.max_recv_wr = resp->max_recv_wr;
943 qp_attr->cap.max_send_wr = resp->max_send_wr;
944 qp_attr->cap.max_inline_data = resp->max_inline_data;
945 }
946
947 qp->qp.handle = resp->qp_handle;
948 qp->qp.qp_num = resp->qpn;
949 qp->qp.context = context;
950 qp->qp.qp_context = qp_attr->qp_context;
951 qp->qp.pd = qp_attr->pd;
952 qp->qp.send_cq = qp_attr->send_cq;
953 qp->qp.recv_cq = qp_attr->recv_cq;
954 qp->qp.srq = qp_attr->srq;
955 qp->qp.qp_type = qp_attr->qp_type;
956 qp->qp.state = IBV_QPS_RESET;
957 qp->qp.events_completed = 0;
958
959 qp->comp_mask = 0;
960 if (vext_field_avail(struct verbs_qp, xrcd, vqp_sz) &&
961 (qp_attr->comp_mask & IBV_QP_INIT_ATTR_XRCD)) {
962 qp->comp_mask |= VERBS_QP_XRCD;
963 qp->xrcd = vxrcd;
964 }
965 }
966
967 enum {
968 CREATE_QP_EX2_SUP_CREATE_FLAGS = IBV_QP_CREATE_BLOCK_SELF_MCAST_LB |
969 IBV_QP_CREATE_SCATTER_FCS |
970 IBV_QP_CREATE_CVLAN_STRIPPING,
971 };
972
ibv_cmd_create_qp_ex2(struct ibv_context * context,struct verbs_qp * qp,int vqp_sz,struct ibv_qp_init_attr_ex * qp_attr,struct ibv_create_qp_ex * cmd,size_t cmd_core_size,size_t cmd_size,struct ibv_create_qp_resp_ex * resp,size_t resp_core_size,size_t resp_size)973 int ibv_cmd_create_qp_ex2(struct ibv_context *context,
974 struct verbs_qp *qp, int vqp_sz,
975 struct ibv_qp_init_attr_ex *qp_attr,
976 struct ibv_create_qp_ex *cmd,
977 size_t cmd_core_size,
978 size_t cmd_size,
979 struct ibv_create_qp_resp_ex *resp,
980 size_t resp_core_size,
981 size_t resp_size)
982 {
983 struct verbs_xrcd *vxrcd = NULL;
984 int err;
985
986 if (qp_attr->comp_mask >= IBV_QP_INIT_ATTR_RESERVED)
987 return EINVAL;
988
989 if (resp_core_size <
990 offsetof(struct ibv_create_qp_resp_ex, response_length) +
991 sizeof(resp->response_length))
992 return EINVAL;
993
994 memset(cmd, 0, cmd_core_size);
995
996 IBV_INIT_CMD_RESP_EX_V(cmd, cmd_core_size, cmd_size, CREATE_QP_EX, resp,
997 resp_core_size, resp_size);
998
999 err = create_qp_ex_common(qp, qp_attr, vxrcd, &cmd->base);
1000 if (err)
1001 return err;
1002
1003 if (qp_attr->comp_mask & IBV_QP_INIT_ATTR_CREATE_FLAGS) {
1004 if (qp_attr->create_flags & ~CREATE_QP_EX2_SUP_CREATE_FLAGS)
1005 return EINVAL;
1006 if (cmd_core_size < offsetof(struct ibv_create_qp_ex, create_flags) +
1007 sizeof(qp_attr->create_flags))
1008 return EINVAL;
1009 cmd->create_flags = qp_attr->create_flags;
1010 }
1011
1012 if (qp_attr->comp_mask & IBV_QP_INIT_ATTR_IND_TABLE) {
1013 if (cmd_core_size < offsetof(struct ibv_create_qp_ex, ind_tbl_handle) +
1014 sizeof(cmd->ind_tbl_handle))
1015 return EINVAL;
1016 cmd->ind_tbl_handle = qp_attr->rwq_ind_tbl->ind_tbl_handle;
1017 cmd->comp_mask = IBV_CREATE_QP_EX_KERNEL_MASK_IND_TABLE;
1018 }
1019
1020 err = create_qp_handle_resp_common_init(qp);
1021 if (err)
1022 return err;
1023
1024 err = write(context->cmd_fd, cmd, cmd_size);
1025 if (err != cmd_size) {
1026 err = errno;
1027 goto err;
1028 }
1029
1030 (void)VALGRIND_MAKE_MEM_DEFINED(resp, resp_size);
1031
1032 create_qp_handle_resp_common(context, qp, qp_attr, &resp->base, vxrcd,
1033 vqp_sz);
1034
1035 return 0;
1036
1037 err:
1038 create_qp_handle_resp_common_cleanup(qp);
1039
1040 return err;
1041 }
1042
ibv_cmd_create_qp_ex(struct ibv_context * context,struct verbs_qp * qp,int vqp_sz,struct ibv_qp_init_attr_ex * attr_ex,struct ibv_create_qp * cmd,size_t cmd_size,struct ibv_create_qp_resp * resp,size_t resp_size)1043 int ibv_cmd_create_qp_ex(struct ibv_context *context,
1044 struct verbs_qp *qp, int vqp_sz,
1045 struct ibv_qp_init_attr_ex *attr_ex,
1046 struct ibv_create_qp *cmd, size_t cmd_size,
1047 struct ibv_create_qp_resp *resp, size_t resp_size)
1048 {
1049 struct verbs_xrcd *vxrcd = NULL;
1050 int err;
1051
1052 IBV_INIT_CMD_RESP(cmd, cmd_size, CREATE_QP, resp, resp_size);
1053
1054 if (attr_ex->comp_mask > (IBV_QP_INIT_ATTR_XRCD | IBV_QP_INIT_ATTR_PD))
1055 return ENOSYS;
1056
1057 err = create_qp_ex_common(qp, attr_ex, vxrcd,
1058 (struct ibv_create_qp_common *)&cmd->user_handle);
1059 if (err)
1060 return err;
1061
1062 err = create_qp_handle_resp_common_init(qp);
1063 if (err)
1064 return err;
1065
1066 err = write(context->cmd_fd, cmd, cmd_size);
1067 if (err != cmd_size) {
1068 err = errno;
1069 goto err;
1070 }
1071
1072 (void)VALGRIND_MAKE_MEM_DEFINED(resp, resp_size);
1073
1074 if (abi_ver == 4) {
1075 struct ibv_create_qp_resp_v4 *resp_v4 =
1076 (struct ibv_create_qp_resp_v4 *)resp;
1077
1078 memmove((void *)resp + sizeof *resp,
1079 (void *)resp_v4 + sizeof *resp_v4,
1080 resp_size - sizeof *resp);
1081 } else if (abi_ver <= 3) {
1082 struct ibv_create_qp_resp_v3 *resp_v3 =
1083 (struct ibv_create_qp_resp_v3 *)resp;
1084
1085 memmove((void *)resp + sizeof *resp,
1086 (void *)resp_v3 + sizeof *resp_v3,
1087 resp_size - sizeof *resp);
1088 }
1089
1090 create_qp_handle_resp_common(context, qp, attr_ex, resp, vxrcd, vqp_sz);
1091
1092 return 0;
1093
1094 err:
1095 create_qp_handle_resp_common_cleanup(qp);
1096
1097 return err;
1098 }
1099
ibv_cmd_create_qp(struct ibv_pd * pd,struct ibv_qp * qp,struct ibv_qp_init_attr * attr,struct ibv_create_qp * cmd,size_t cmd_size,struct ibv_create_qp_resp * resp,size_t resp_size)1100 int ibv_cmd_create_qp(struct ibv_pd *pd,
1101 struct ibv_qp *qp, struct ibv_qp_init_attr *attr,
1102 struct ibv_create_qp *cmd, size_t cmd_size,
1103 struct ibv_create_qp_resp *resp, size_t resp_size)
1104 {
1105 IBV_INIT_CMD_RESP(cmd, cmd_size, CREATE_QP, resp, resp_size);
1106
1107 cmd->user_handle = (uintptr_t) qp;
1108 cmd->pd_handle = pd->handle;
1109 cmd->send_cq_handle = attr->send_cq->handle;
1110 cmd->recv_cq_handle = attr->recv_cq->handle;
1111 cmd->srq_handle = attr->srq ? attr->srq->handle : 0;
1112 cmd->max_send_wr = attr->cap.max_send_wr;
1113 cmd->max_recv_wr = attr->cap.max_recv_wr;
1114 cmd->max_send_sge = attr->cap.max_send_sge;
1115 cmd->max_recv_sge = attr->cap.max_recv_sge;
1116 cmd->max_inline_data = attr->cap.max_inline_data;
1117 cmd->sq_sig_all = attr->sq_sig_all;
1118 cmd->qp_type = attr->qp_type;
1119 cmd->is_srq = !!attr->srq;
1120 cmd->reserved = 0;
1121
1122 if (write(pd->context->cmd_fd, cmd, cmd_size) != cmd_size)
1123 return errno;
1124
1125 (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size);
1126
1127 qp->handle = resp->qp_handle;
1128 qp->qp_num = resp->qpn;
1129 qp->context = pd->context;
1130
1131 if (abi_ver > 3) {
1132 attr->cap.max_recv_sge = resp->max_recv_sge;
1133 attr->cap.max_send_sge = resp->max_send_sge;
1134 attr->cap.max_recv_wr = resp->max_recv_wr;
1135 attr->cap.max_send_wr = resp->max_send_wr;
1136 attr->cap.max_inline_data = resp->max_inline_data;
1137 }
1138
1139 if (abi_ver == 4) {
1140 struct ibv_create_qp_resp_v4 *resp_v4 =
1141 (struct ibv_create_qp_resp_v4 *) resp;
1142
1143 memmove((void *) resp + sizeof *resp,
1144 (void *) resp_v4 + sizeof *resp_v4,
1145 resp_size - sizeof *resp);
1146 } else if (abi_ver <= 3) {
1147 struct ibv_create_qp_resp_v3 *resp_v3 =
1148 (struct ibv_create_qp_resp_v3 *) resp;
1149
1150 memmove((void *) resp + sizeof *resp,
1151 (void *) resp_v3 + sizeof *resp_v3,
1152 resp_size - sizeof *resp);
1153 }
1154
1155 return 0;
1156 }
1157
ibv_cmd_open_qp(struct ibv_context * context,struct verbs_qp * qp,int vqp_sz,struct ibv_qp_open_attr * attr,struct ibv_open_qp * cmd,size_t cmd_size,struct ibv_create_qp_resp * resp,size_t resp_size)1158 int ibv_cmd_open_qp(struct ibv_context *context, struct verbs_qp *qp,
1159 int vqp_sz,
1160 struct ibv_qp_open_attr *attr,
1161 struct ibv_open_qp *cmd, size_t cmd_size,
1162 struct ibv_create_qp_resp *resp, size_t resp_size)
1163 {
1164 int err = 0;
1165 struct verbs_xrcd *xrcd;
1166 IBV_INIT_CMD_RESP(cmd, cmd_size, OPEN_QP, resp, resp_size);
1167
1168 if (attr->comp_mask >= IBV_QP_OPEN_ATTR_RESERVED)
1169 return ENOSYS;
1170
1171 if (!(attr->comp_mask & IBV_QP_OPEN_ATTR_XRCD) ||
1172 !(attr->comp_mask & IBV_QP_OPEN_ATTR_NUM) ||
1173 !(attr->comp_mask & IBV_QP_OPEN_ATTR_TYPE))
1174 return EINVAL;
1175
1176 xrcd = container_of(attr->xrcd, struct verbs_xrcd, xrcd);
1177 cmd->user_handle = (uintptr_t) qp;
1178 cmd->pd_handle = xrcd->handle;
1179 cmd->qpn = attr->qp_num;
1180 cmd->qp_type = attr->qp_type;
1181
1182 err = pthread_mutex_init(&qp->qp.mutex, NULL);
1183 if (err)
1184 return err;
1185 err = pthread_cond_init(&qp->qp.cond, NULL);
1186 if (err)
1187 goto err_mutex;
1188
1189 err = write(context->cmd_fd, cmd, cmd_size);
1190 if (err != cmd_size) {
1191 err = errno;
1192 goto err_cond;
1193 }
1194
1195 (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size);
1196
1197 qp->qp.handle = resp->qp_handle;
1198 qp->qp.context = context;
1199 qp->qp.qp_context = attr->qp_context;
1200 qp->qp.pd = NULL;
1201 qp->qp.send_cq = NULL;
1202 qp->qp.recv_cq = NULL;
1203 qp->qp.srq = NULL;
1204 qp->qp.qp_num = attr->qp_num;
1205 qp->qp.qp_type = attr->qp_type;
1206 qp->qp.state = IBV_QPS_UNKNOWN;
1207 qp->qp.events_completed = 0;
1208 qp->comp_mask = 0;
1209 if (vext_field_avail(struct verbs_qp, xrcd, vqp_sz)) {
1210 qp->comp_mask = VERBS_QP_XRCD;
1211 qp->xrcd = xrcd;
1212 }
1213
1214 return 0;
1215
1216 err_cond:
1217 pthread_cond_destroy(&qp->qp.cond);
1218 err_mutex:
1219 pthread_mutex_destroy(&qp->qp.mutex);
1220
1221 return err;
1222 }
1223
ibv_cmd_query_qp(struct ibv_qp * qp,struct ibv_qp_attr * attr,int attr_mask,struct ibv_qp_init_attr * init_attr,struct ibv_query_qp * cmd,size_t cmd_size)1224 int ibv_cmd_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
1225 int attr_mask,
1226 struct ibv_qp_init_attr *init_attr,
1227 struct ibv_query_qp *cmd, size_t cmd_size)
1228 {
1229 struct ibv_query_qp_resp resp;
1230
1231 /*
1232 * Masks over IBV_QP_DEST_QPN are not supported by
1233 * that not extended command.
1234 */
1235 if (attr_mask & ~((IBV_QP_DEST_QPN << 1) - 1))
1236 return EOPNOTSUPP;
1237
1238 IBV_INIT_CMD_RESP(cmd, cmd_size, QUERY_QP, &resp, sizeof resp);
1239 cmd->qp_handle = qp->handle;
1240 cmd->attr_mask = attr_mask;
1241
1242 if (write(qp->context->cmd_fd, cmd, cmd_size) != cmd_size)
1243 return errno;
1244
1245 (void) VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof resp);
1246
1247 attr->qkey = resp.qkey;
1248 attr->rq_psn = resp.rq_psn;
1249 attr->sq_psn = resp.sq_psn;
1250 attr->dest_qp_num = resp.dest_qp_num;
1251 attr->qp_access_flags = resp.qp_access_flags;
1252 attr->pkey_index = resp.pkey_index;
1253 attr->alt_pkey_index = resp.alt_pkey_index;
1254 attr->qp_state = resp.qp_state;
1255 attr->cur_qp_state = resp.cur_qp_state;
1256 attr->path_mtu = resp.path_mtu;
1257 attr->path_mig_state = resp.path_mig_state;
1258 attr->sq_draining = resp.sq_draining;
1259 attr->max_rd_atomic = resp.max_rd_atomic;
1260 attr->max_dest_rd_atomic = resp.max_dest_rd_atomic;
1261 attr->min_rnr_timer = resp.min_rnr_timer;
1262 attr->port_num = resp.port_num;
1263 attr->timeout = resp.timeout;
1264 attr->retry_cnt = resp.retry_cnt;
1265 attr->rnr_retry = resp.rnr_retry;
1266 attr->alt_port_num = resp.alt_port_num;
1267 attr->alt_timeout = resp.alt_timeout;
1268 attr->cap.max_send_wr = resp.max_send_wr;
1269 attr->cap.max_recv_wr = resp.max_recv_wr;
1270 attr->cap.max_send_sge = resp.max_send_sge;
1271 attr->cap.max_recv_sge = resp.max_recv_sge;
1272 attr->cap.max_inline_data = resp.max_inline_data;
1273
1274 memcpy(attr->ah_attr.grh.dgid.raw, resp.dest.dgid, 16);
1275 attr->ah_attr.grh.flow_label = resp.dest.flow_label;
1276 attr->ah_attr.dlid = resp.dest.dlid;
1277 attr->ah_attr.grh.sgid_index = resp.dest.sgid_index;
1278 attr->ah_attr.grh.hop_limit = resp.dest.hop_limit;
1279 attr->ah_attr.grh.traffic_class = resp.dest.traffic_class;
1280 attr->ah_attr.sl = resp.dest.sl;
1281 attr->ah_attr.src_path_bits = resp.dest.src_path_bits;
1282 attr->ah_attr.static_rate = resp.dest.static_rate;
1283 attr->ah_attr.is_global = resp.dest.is_global;
1284 attr->ah_attr.port_num = resp.dest.port_num;
1285
1286 memcpy(attr->alt_ah_attr.grh.dgid.raw, resp.alt_dest.dgid, 16);
1287 attr->alt_ah_attr.grh.flow_label = resp.alt_dest.flow_label;
1288 attr->alt_ah_attr.dlid = resp.alt_dest.dlid;
1289 attr->alt_ah_attr.grh.sgid_index = resp.alt_dest.sgid_index;
1290 attr->alt_ah_attr.grh.hop_limit = resp.alt_dest.hop_limit;
1291 attr->alt_ah_attr.grh.traffic_class = resp.alt_dest.traffic_class;
1292 attr->alt_ah_attr.sl = resp.alt_dest.sl;
1293 attr->alt_ah_attr.src_path_bits = resp.alt_dest.src_path_bits;
1294 attr->alt_ah_attr.static_rate = resp.alt_dest.static_rate;
1295 attr->alt_ah_attr.is_global = resp.alt_dest.is_global;
1296 attr->alt_ah_attr.port_num = resp.alt_dest.port_num;
1297
1298 init_attr->qp_context = qp->qp_context;
1299 init_attr->send_cq = qp->send_cq;
1300 init_attr->recv_cq = qp->recv_cq;
1301 init_attr->srq = qp->srq;
1302 init_attr->qp_type = qp->qp_type;
1303 init_attr->cap.max_send_wr = resp.max_send_wr;
1304 init_attr->cap.max_recv_wr = resp.max_recv_wr;
1305 init_attr->cap.max_send_sge = resp.max_send_sge;
1306 init_attr->cap.max_recv_sge = resp.max_recv_sge;
1307 init_attr->cap.max_inline_data = resp.max_inline_data;
1308 init_attr->sq_sig_all = resp.sq_sig_all;
1309
1310 return 0;
1311 }
1312
copy_modify_qp_fields(struct ibv_qp * qp,struct ibv_qp_attr * attr,int attr_mask,struct ibv_modify_qp_common * cmd)1313 static void copy_modify_qp_fields(struct ibv_qp *qp, struct ibv_qp_attr *attr,
1314 int attr_mask,
1315 struct ibv_modify_qp_common *cmd)
1316 {
1317 cmd->qp_handle = qp->handle;
1318 cmd->attr_mask = attr_mask;
1319
1320 if (attr_mask & IBV_QP_STATE)
1321 cmd->qp_state = attr->qp_state;
1322 if (attr_mask & IBV_QP_CUR_STATE)
1323 cmd->cur_qp_state = attr->cur_qp_state;
1324 if (attr_mask & IBV_QP_EN_SQD_ASYNC_NOTIFY)
1325 cmd->en_sqd_async_notify = attr->en_sqd_async_notify;
1326 if (attr_mask & IBV_QP_ACCESS_FLAGS)
1327 cmd->qp_access_flags = attr->qp_access_flags;
1328 if (attr_mask & IBV_QP_PKEY_INDEX)
1329 cmd->pkey_index = attr->pkey_index;
1330 if (attr_mask & IBV_QP_PORT)
1331 cmd->port_num = attr->port_num;
1332 if (attr_mask & IBV_QP_QKEY)
1333 cmd->qkey = attr->qkey;
1334
1335 if (attr_mask & IBV_QP_AV) {
1336 memcpy(cmd->dest.dgid, attr->ah_attr.grh.dgid.raw, 16);
1337 cmd->dest.flow_label = attr->ah_attr.grh.flow_label;
1338 cmd->dest.dlid = attr->ah_attr.dlid;
1339 cmd->dest.reserved = 0;
1340 cmd->dest.sgid_index = attr->ah_attr.grh.sgid_index;
1341 cmd->dest.hop_limit = attr->ah_attr.grh.hop_limit;
1342 cmd->dest.traffic_class = attr->ah_attr.grh.traffic_class;
1343 cmd->dest.sl = attr->ah_attr.sl;
1344 cmd->dest.src_path_bits = attr->ah_attr.src_path_bits;
1345 cmd->dest.static_rate = attr->ah_attr.static_rate;
1346 cmd->dest.is_global = attr->ah_attr.is_global;
1347 cmd->dest.port_num = attr->ah_attr.port_num;
1348 }
1349
1350 if (attr_mask & IBV_QP_PATH_MTU)
1351 cmd->path_mtu = attr->path_mtu;
1352 if (attr_mask & IBV_QP_TIMEOUT)
1353 cmd->timeout = attr->timeout;
1354 if (attr_mask & IBV_QP_RETRY_CNT)
1355 cmd->retry_cnt = attr->retry_cnt;
1356 if (attr_mask & IBV_QP_RNR_RETRY)
1357 cmd->rnr_retry = attr->rnr_retry;
1358 if (attr_mask & IBV_QP_RQ_PSN)
1359 cmd->rq_psn = attr->rq_psn;
1360 if (attr_mask & IBV_QP_MAX_QP_RD_ATOMIC)
1361 cmd->max_rd_atomic = attr->max_rd_atomic;
1362
1363 if (attr_mask & IBV_QP_ALT_PATH) {
1364 cmd->alt_pkey_index = attr->alt_pkey_index;
1365 cmd->alt_port_num = attr->alt_port_num;
1366 cmd->alt_timeout = attr->alt_timeout;
1367
1368 memcpy(cmd->alt_dest.dgid, attr->alt_ah_attr.grh.dgid.raw, 16);
1369 cmd->alt_dest.flow_label = attr->alt_ah_attr.grh.flow_label;
1370 cmd->alt_dest.dlid = attr->alt_ah_attr.dlid;
1371 cmd->alt_dest.reserved = 0;
1372 cmd->alt_dest.sgid_index = attr->alt_ah_attr.grh.sgid_index;
1373 cmd->alt_dest.hop_limit = attr->alt_ah_attr.grh.hop_limit;
1374 cmd->alt_dest.traffic_class =
1375 attr->alt_ah_attr.grh.traffic_class;
1376 cmd->alt_dest.sl = attr->alt_ah_attr.sl;
1377 cmd->alt_dest.src_path_bits = attr->alt_ah_attr.src_path_bits;
1378 cmd->alt_dest.static_rate = attr->alt_ah_attr.static_rate;
1379 cmd->alt_dest.is_global = attr->alt_ah_attr.is_global;
1380 cmd->alt_dest.port_num = attr->alt_ah_attr.port_num;
1381 }
1382
1383 if (attr_mask & IBV_QP_MIN_RNR_TIMER)
1384 cmd->min_rnr_timer = attr->min_rnr_timer;
1385 if (attr_mask & IBV_QP_SQ_PSN)
1386 cmd->sq_psn = attr->sq_psn;
1387 if (attr_mask & IBV_QP_MAX_DEST_RD_ATOMIC)
1388 cmd->max_dest_rd_atomic = attr->max_dest_rd_atomic;
1389 if (attr_mask & IBV_QP_PATH_MIG_STATE)
1390 cmd->path_mig_state = attr->path_mig_state;
1391 if (attr_mask & IBV_QP_DEST_QPN)
1392 cmd->dest_qp_num = attr->dest_qp_num;
1393
1394 cmd->reserved[0] = cmd->reserved[1] = 0;
1395 }
1396
ibv_cmd_modify_qp(struct ibv_qp * qp,struct ibv_qp_attr * attr,int attr_mask,struct ibv_modify_qp * cmd,size_t cmd_size)1397 int ibv_cmd_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
1398 int attr_mask,
1399 struct ibv_modify_qp *cmd, size_t cmd_size)
1400 {
1401 /*
1402 * Masks over IBV_QP_DEST_QPN are only supported by
1403 * ibv_cmd_modify_qp_ex.
1404 */
1405 if (attr_mask & ~((IBV_QP_DEST_QPN << 1) - 1))
1406 return EOPNOTSUPP;
1407
1408 IBV_INIT_CMD(cmd, cmd_size, MODIFY_QP);
1409
1410 copy_modify_qp_fields(qp, attr, attr_mask, &cmd->base);
1411
1412 if (write(qp->context->cmd_fd, cmd, cmd_size) != cmd_size)
1413 return errno;
1414
1415 return 0;
1416 }
1417
ibv_cmd_modify_qp_ex(struct ibv_qp * qp,struct ibv_qp_attr * attr,int attr_mask,struct ibv_modify_qp_ex * cmd,size_t cmd_core_size,size_t cmd_size,struct ibv_modify_qp_resp_ex * resp,size_t resp_core_size,size_t resp_size)1418 int ibv_cmd_modify_qp_ex(struct ibv_qp *qp, struct ibv_qp_attr *attr,
1419 int attr_mask, struct ibv_modify_qp_ex *cmd,
1420 size_t cmd_core_size, size_t cmd_size,
1421 struct ibv_modify_qp_resp_ex *resp,
1422 size_t resp_core_size, size_t resp_size)
1423 {
1424 if (resp_core_size < offsetof(struct ibv_modify_qp_resp_ex,
1425 response_length) + sizeof(resp->response_length))
1426 return EINVAL;
1427
1428 IBV_INIT_CMD_RESP_EX_V(cmd, cmd_core_size, cmd_size, MODIFY_QP_EX,
1429 resp, resp_core_size, resp_size);
1430
1431 copy_modify_qp_fields(qp, attr, attr_mask, &cmd->base);
1432
1433 if (attr_mask & IBV_QP_RATE_LIMIT) {
1434 if (cmd_size >= offsetof(struct ibv_modify_qp_ex, rate_limit) +
1435 sizeof(cmd->rate_limit))
1436 cmd->rate_limit = attr->rate_limit;
1437 else
1438 return EINVAL;
1439 }
1440
1441 if (write(qp->context->cmd_fd, cmd, cmd_size) != cmd_size)
1442 return errno;
1443
1444 (void)VALGRIND_MAKE_MEM_DEFINED(resp, resp_size);
1445
1446 return 0;
1447 }
1448
ibv_cmd_post_send(struct ibv_qp * ibqp,struct ibv_send_wr * wr,struct ibv_send_wr ** bad_wr)1449 int ibv_cmd_post_send(struct ibv_qp *ibqp, struct ibv_send_wr *wr,
1450 struct ibv_send_wr **bad_wr)
1451 {
1452 struct ibv_post_send *cmd;
1453 struct ibv_post_send_resp resp;
1454 struct ibv_send_wr *i;
1455 struct ibv_kern_send_wr *n, *tmp;
1456 struct ibv_sge *s;
1457 unsigned wr_count = 0;
1458 unsigned sge_count = 0;
1459 int cmd_size;
1460 int ret = 0;
1461
1462 for (i = wr; i; i = i->next) {
1463 wr_count++;
1464 sge_count += i->num_sge;
1465 }
1466
1467 cmd_size = sizeof *cmd + wr_count * sizeof *n + sge_count * sizeof *s;
1468 cmd = alloca(cmd_size);
1469
1470 IBV_INIT_CMD_RESP(cmd, cmd_size, POST_SEND, &resp, sizeof resp);
1471 cmd->qp_handle = ibqp->handle;
1472 cmd->wr_count = wr_count;
1473 cmd->sge_count = sge_count;
1474 cmd->wqe_size = sizeof *n;
1475
1476 n = (struct ibv_kern_send_wr *) ((void *) cmd + sizeof *cmd);
1477 s = (struct ibv_sge *) (n + wr_count);
1478
1479 tmp = n;
1480 for (i = wr; i; i = i->next) {
1481 tmp->wr_id = i->wr_id;
1482 tmp->num_sge = i->num_sge;
1483 tmp->opcode = i->opcode;
1484 tmp->send_flags = i->send_flags;
1485 tmp->imm_data = i->imm_data;
1486 if (ibqp->qp_type == IBV_QPT_UD) {
1487 tmp->wr.ud.ah = i->wr.ud.ah->handle;
1488 tmp->wr.ud.remote_qpn = i->wr.ud.remote_qpn;
1489 tmp->wr.ud.remote_qkey = i->wr.ud.remote_qkey;
1490 } else {
1491 switch (i->opcode) {
1492 case IBV_WR_RDMA_WRITE:
1493 case IBV_WR_RDMA_WRITE_WITH_IMM:
1494 case IBV_WR_RDMA_READ:
1495 tmp->wr.rdma.remote_addr =
1496 i->wr.rdma.remote_addr;
1497 tmp->wr.rdma.rkey = i->wr.rdma.rkey;
1498 break;
1499 case IBV_WR_ATOMIC_CMP_AND_SWP:
1500 case IBV_WR_ATOMIC_FETCH_AND_ADD:
1501 tmp->wr.atomic.remote_addr =
1502 i->wr.atomic.remote_addr;
1503 tmp->wr.atomic.compare_add =
1504 i->wr.atomic.compare_add;
1505 tmp->wr.atomic.swap = i->wr.atomic.swap;
1506 tmp->wr.atomic.rkey = i->wr.atomic.rkey;
1507 break;
1508 default:
1509 break;
1510 }
1511 }
1512
1513 if (tmp->num_sge) {
1514 memcpy(s, i->sg_list, tmp->num_sge * sizeof *s);
1515 s += tmp->num_sge;
1516 }
1517
1518 tmp++;
1519 }
1520
1521 resp.bad_wr = 0;
1522 if (write(ibqp->context->cmd_fd, cmd, cmd_size) != cmd_size)
1523 ret = errno;
1524
1525 (void) VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof resp);
1526
1527 wr_count = resp.bad_wr;
1528 if (wr_count) {
1529 i = wr;
1530 while (--wr_count)
1531 i = i->next;
1532 *bad_wr = i;
1533 } else if (ret)
1534 *bad_wr = wr;
1535
1536 return ret;
1537 }
1538
ibv_cmd_post_recv(struct ibv_qp * ibqp,struct ibv_recv_wr * wr,struct ibv_recv_wr ** bad_wr)1539 int ibv_cmd_post_recv(struct ibv_qp *ibqp, struct ibv_recv_wr *wr,
1540 struct ibv_recv_wr **bad_wr)
1541 {
1542 struct ibv_post_recv *cmd;
1543 struct ibv_post_recv_resp resp;
1544 struct ibv_recv_wr *i;
1545 struct ibv_kern_recv_wr *n, *tmp;
1546 struct ibv_sge *s;
1547 unsigned wr_count = 0;
1548 unsigned sge_count = 0;
1549 int cmd_size;
1550 int ret = 0;
1551
1552 for (i = wr; i; i = i->next) {
1553 wr_count++;
1554 sge_count += i->num_sge;
1555 }
1556
1557 cmd_size = sizeof *cmd + wr_count * sizeof *n + sge_count * sizeof *s;
1558 cmd = alloca(cmd_size);
1559
1560 IBV_INIT_CMD_RESP(cmd, cmd_size, POST_RECV, &resp, sizeof resp);
1561 cmd->qp_handle = ibqp->handle;
1562 cmd->wr_count = wr_count;
1563 cmd->sge_count = sge_count;
1564 cmd->wqe_size = sizeof *n;
1565
1566 n = (struct ibv_kern_recv_wr *) ((void *) cmd + sizeof *cmd);
1567 s = (struct ibv_sge *) (n + wr_count);
1568
1569 tmp = n;
1570 for (i = wr; i; i = i->next) {
1571 tmp->wr_id = i->wr_id;
1572 tmp->num_sge = i->num_sge;
1573
1574 if (tmp->num_sge) {
1575 memcpy(s, i->sg_list, tmp->num_sge * sizeof *s);
1576 s += tmp->num_sge;
1577 }
1578
1579 tmp++;
1580 }
1581
1582 resp.bad_wr = 0;
1583 if (write(ibqp->context->cmd_fd, cmd, cmd_size) != cmd_size)
1584 ret = errno;
1585
1586 (void) VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof resp);
1587
1588 wr_count = resp.bad_wr;
1589 if (wr_count) {
1590 i = wr;
1591 while (--wr_count)
1592 i = i->next;
1593 *bad_wr = i;
1594 } else if (ret)
1595 *bad_wr = wr;
1596
1597 return ret;
1598 }
1599
ibv_cmd_post_srq_recv(struct ibv_srq * srq,struct ibv_recv_wr * wr,struct ibv_recv_wr ** bad_wr)1600 int ibv_cmd_post_srq_recv(struct ibv_srq *srq, struct ibv_recv_wr *wr,
1601 struct ibv_recv_wr **bad_wr)
1602 {
1603 struct ibv_post_srq_recv *cmd;
1604 struct ibv_post_srq_recv_resp resp;
1605 struct ibv_recv_wr *i;
1606 struct ibv_kern_recv_wr *n, *tmp;
1607 struct ibv_sge *s;
1608 unsigned wr_count = 0;
1609 unsigned sge_count = 0;
1610 int cmd_size;
1611 int ret = 0;
1612
1613 for (i = wr; i; i = i->next) {
1614 wr_count++;
1615 sge_count += i->num_sge;
1616 }
1617
1618 cmd_size = sizeof *cmd + wr_count * sizeof *n + sge_count * sizeof *s;
1619 cmd = alloca(cmd_size);
1620
1621 IBV_INIT_CMD_RESP(cmd, cmd_size, POST_SRQ_RECV, &resp, sizeof resp);
1622 cmd->srq_handle = srq->handle;
1623 cmd->wr_count = wr_count;
1624 cmd->sge_count = sge_count;
1625 cmd->wqe_size = sizeof *n;
1626
1627 n = (struct ibv_kern_recv_wr *) ((void *) cmd + sizeof *cmd);
1628 s = (struct ibv_sge *) (n + wr_count);
1629
1630 tmp = n;
1631 for (i = wr; i; i = i->next) {
1632 tmp->wr_id = i->wr_id;
1633 tmp->num_sge = i->num_sge;
1634
1635 if (tmp->num_sge) {
1636 memcpy(s, i->sg_list, tmp->num_sge * sizeof *s);
1637 s += tmp->num_sge;
1638 }
1639
1640 tmp++;
1641 }
1642
1643 resp.bad_wr = 0;
1644 if (write(srq->context->cmd_fd, cmd, cmd_size) != cmd_size)
1645 ret = errno;
1646
1647 (void) VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof resp);
1648
1649 wr_count = resp.bad_wr;
1650 if (wr_count) {
1651 i = wr;
1652 while (--wr_count)
1653 i = i->next;
1654 *bad_wr = i;
1655 } else if (ret)
1656 *bad_wr = wr;
1657
1658 return ret;
1659 }
1660
ibv_cmd_create_ah(struct ibv_pd * pd,struct ibv_ah * ah,struct ibv_ah_attr * attr,struct ibv_create_ah_resp * resp,size_t resp_size)1661 int ibv_cmd_create_ah(struct ibv_pd *pd, struct ibv_ah *ah,
1662 struct ibv_ah_attr *attr,
1663 struct ibv_create_ah_resp *resp,
1664 size_t resp_size)
1665 {
1666 struct ibv_create_ah cmd;
1667
1668 IBV_INIT_CMD_RESP(&cmd, sizeof cmd, CREATE_AH, resp, resp_size);
1669 cmd.user_handle = (uintptr_t) ah;
1670 cmd.pd_handle = pd->handle;
1671 cmd.attr.dlid = attr->dlid;
1672 cmd.attr.sl = attr->sl;
1673 cmd.attr.src_path_bits = attr->src_path_bits;
1674 cmd.attr.static_rate = attr->static_rate;
1675 cmd.attr.is_global = attr->is_global;
1676 cmd.attr.port_num = attr->port_num;
1677 cmd.attr.grh.flow_label = attr->grh.flow_label;
1678 cmd.attr.grh.sgid_index = attr->grh.sgid_index;
1679 cmd.attr.grh.hop_limit = attr->grh.hop_limit;
1680 cmd.attr.grh.traffic_class = attr->grh.traffic_class;
1681 memcpy(cmd.attr.grh.dgid, attr->grh.dgid.raw, 16);
1682
1683 if (write(pd->context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd)
1684 return errno;
1685
1686 (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size);
1687
1688 ah->handle = resp->handle;
1689 ah->context = pd->context;
1690
1691 return 0;
1692 }
1693
ibv_cmd_destroy_ah(struct ibv_ah * ah)1694 int ibv_cmd_destroy_ah(struct ibv_ah *ah)
1695 {
1696 struct ibv_destroy_ah cmd;
1697
1698 IBV_INIT_CMD(&cmd, sizeof cmd, DESTROY_AH);
1699 cmd.ah_handle = ah->handle;
1700
1701 if (write(ah->context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd)
1702 return errno;
1703
1704 return 0;
1705 }
1706
ibv_cmd_destroy_qp(struct ibv_qp * qp)1707 int ibv_cmd_destroy_qp(struct ibv_qp *qp)
1708 {
1709 struct ibv_destroy_qp cmd;
1710 struct ibv_destroy_qp_resp resp;
1711
1712 IBV_INIT_CMD_RESP(&cmd, sizeof cmd, DESTROY_QP, &resp, sizeof resp);
1713 cmd.qp_handle = qp->handle;
1714 cmd.reserved = 0;
1715
1716 if (write(qp->context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd)
1717 return errno;
1718
1719 (void) VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof resp);
1720
1721 pthread_mutex_lock(&qp->mutex);
1722 while (qp->events_completed != resp.events_reported)
1723 pthread_cond_wait(&qp->cond, &qp->mutex);
1724 pthread_mutex_unlock(&qp->mutex);
1725
1726 pthread_cond_destroy(&qp->cond);
1727 pthread_mutex_destroy(&qp->mutex);
1728
1729 return 0;
1730 }
1731
ibv_cmd_attach_mcast(struct ibv_qp * qp,const union ibv_gid * gid,uint16_t lid)1732 int ibv_cmd_attach_mcast(struct ibv_qp *qp, const union ibv_gid *gid, uint16_t lid)
1733 {
1734 struct ibv_attach_mcast cmd;
1735
1736 IBV_INIT_CMD(&cmd, sizeof cmd, ATTACH_MCAST);
1737 memcpy(cmd.gid, gid->raw, sizeof cmd.gid);
1738 cmd.qp_handle = qp->handle;
1739 cmd.mlid = lid;
1740 cmd.reserved = 0;
1741
1742 if (write(qp->context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd)
1743 return errno;
1744
1745 return 0;
1746 }
1747
ibv_cmd_detach_mcast(struct ibv_qp * qp,const union ibv_gid * gid,uint16_t lid)1748 int ibv_cmd_detach_mcast(struct ibv_qp *qp, const union ibv_gid *gid, uint16_t lid)
1749 {
1750 struct ibv_detach_mcast cmd;
1751
1752 IBV_INIT_CMD(&cmd, sizeof cmd, DETACH_MCAST);
1753 memcpy(cmd.gid, gid->raw, sizeof cmd.gid);
1754 cmd.qp_handle = qp->handle;
1755 cmd.mlid = lid;
1756 cmd.reserved = 0;
1757
1758 if (write(qp->context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd)
1759 return errno;
1760
1761 return 0;
1762 }
1763
buffer_is_zero(char * addr,ssize_t size)1764 static int buffer_is_zero(char *addr, ssize_t size)
1765 {
1766 return addr[0] == 0 && !memcmp(addr, addr + 1, size - 1);
1767 }
1768
get_filters_size(struct ibv_flow_spec * ib_spec,struct ibv_kern_spec * kern_spec,int * ib_filter_size,int * kern_filter_size,enum ibv_flow_spec_type type)1769 static int get_filters_size(struct ibv_flow_spec *ib_spec,
1770 struct ibv_kern_spec *kern_spec,
1771 int *ib_filter_size, int *kern_filter_size,
1772 enum ibv_flow_spec_type type)
1773 {
1774 void *ib_spec_filter_mask;
1775 int curr_kern_filter_size;
1776 int min_filter_size;
1777
1778 *ib_filter_size = (ib_spec->hdr.size - sizeof(ib_spec->hdr)) / 2;
1779
1780 switch (type) {
1781 case IBV_FLOW_SPEC_IPV4_EXT:
1782 min_filter_size =
1783 offsetof(struct ibv_kern_ipv4_ext_filter, flags) +
1784 sizeof(kern_spec->ipv4_ext.mask.flags);
1785 curr_kern_filter_size = min_filter_size;
1786 ib_spec_filter_mask = (void *)&ib_spec->ipv4_ext.val +
1787 *ib_filter_size;
1788 break;
1789 case IBV_FLOW_SPEC_IPV6:
1790 min_filter_size =
1791 offsetof(struct ibv_kern_ipv6_filter, hop_limit) +
1792 sizeof(kern_spec->ipv6.mask.hop_limit);
1793 curr_kern_filter_size = min_filter_size;
1794 ib_spec_filter_mask = (void *)&ib_spec->ipv6.val +
1795 *ib_filter_size;
1796 break;
1797 case IBV_FLOW_SPEC_VXLAN_TUNNEL:
1798 min_filter_size =
1799 offsetof(struct ibv_kern_tunnel_filter,
1800 tunnel_id) +
1801 sizeof(kern_spec->tunnel.mask.tunnel_id);
1802 curr_kern_filter_size = min_filter_size;
1803 ib_spec_filter_mask = (void *)&ib_spec->tunnel.val +
1804 *ib_filter_size;
1805 break;
1806 default:
1807 return EINVAL;
1808 }
1809
1810 if (*ib_filter_size < min_filter_size)
1811 return EINVAL;
1812
1813 if (*ib_filter_size > curr_kern_filter_size &&
1814 !buffer_is_zero(ib_spec_filter_mask + curr_kern_filter_size,
1815 *ib_filter_size - curr_kern_filter_size))
1816 return EOPNOTSUPP;
1817
1818 *kern_filter_size = min_t(int, curr_kern_filter_size, *ib_filter_size);
1819
1820 return 0;
1821 }
1822
ib_spec_to_kern_spec(struct ibv_flow_spec * ib_spec,struct ibv_kern_spec * kern_spec)1823 static int ib_spec_to_kern_spec(struct ibv_flow_spec *ib_spec,
1824 struct ibv_kern_spec *kern_spec)
1825 {
1826 int kern_filter_size;
1827 int ib_filter_size;
1828 int ret;
1829
1830 kern_spec->hdr.type = ib_spec->hdr.type;
1831
1832 switch (kern_spec->hdr.type) {
1833 case IBV_FLOW_SPEC_ETH:
1834 case IBV_FLOW_SPEC_ETH | IBV_FLOW_SPEC_INNER:
1835 kern_spec->eth.size = sizeof(struct ibv_kern_spec_eth);
1836 memcpy(&kern_spec->eth.val, &ib_spec->eth.val,
1837 sizeof(struct ibv_flow_eth_filter));
1838 memcpy(&kern_spec->eth.mask, &ib_spec->eth.mask,
1839 sizeof(struct ibv_flow_eth_filter));
1840 break;
1841 case IBV_FLOW_SPEC_IPV4:
1842 case IBV_FLOW_SPEC_IPV4 | IBV_FLOW_SPEC_INNER:
1843 kern_spec->ipv4.size = sizeof(struct ibv_kern_spec_ipv4);
1844 memcpy(&kern_spec->ipv4.val, &ib_spec->ipv4.val,
1845 sizeof(struct ibv_flow_ipv4_filter));
1846 memcpy(&kern_spec->ipv4.mask, &ib_spec->ipv4.mask,
1847 sizeof(struct ibv_flow_ipv4_filter));
1848 break;
1849 case IBV_FLOW_SPEC_IPV4_EXT:
1850 case IBV_FLOW_SPEC_IPV4_EXT | IBV_FLOW_SPEC_INNER:
1851 ret = get_filters_size(ib_spec, kern_spec,
1852 &ib_filter_size, &kern_filter_size,
1853 IBV_FLOW_SPEC_IPV4_EXT);
1854 if (ret)
1855 return ret;
1856
1857 kern_spec->hdr.type = IBV_FLOW_SPEC_IPV4 |
1858 (IBV_FLOW_SPEC_INNER & ib_spec->hdr.type);
1859 kern_spec->ipv4_ext.size = sizeof(struct
1860 ibv_kern_spec_ipv4_ext);
1861 memcpy(&kern_spec->ipv4_ext.val, &ib_spec->ipv4_ext.val,
1862 kern_filter_size);
1863 memcpy(&kern_spec->ipv4_ext.mask, (void *)&ib_spec->ipv4_ext.val
1864 + ib_filter_size, kern_filter_size);
1865 break;
1866 case IBV_FLOW_SPEC_IPV6:
1867 case IBV_FLOW_SPEC_IPV6 | IBV_FLOW_SPEC_INNER:
1868 ret = get_filters_size(ib_spec, kern_spec,
1869 &ib_filter_size, &kern_filter_size,
1870 IBV_FLOW_SPEC_IPV6);
1871 if (ret)
1872 return ret;
1873
1874 kern_spec->ipv6.size = sizeof(struct ibv_kern_spec_ipv6);
1875 memcpy(&kern_spec->ipv6.val, &ib_spec->ipv6.val,
1876 kern_filter_size);
1877 memcpy(&kern_spec->ipv6.mask, (void *)&ib_spec->ipv6.val
1878 + ib_filter_size, kern_filter_size);
1879 break;
1880 case IBV_FLOW_SPEC_TCP:
1881 case IBV_FLOW_SPEC_UDP:
1882 case IBV_FLOW_SPEC_TCP | IBV_FLOW_SPEC_INNER:
1883 case IBV_FLOW_SPEC_UDP | IBV_FLOW_SPEC_INNER:
1884 kern_spec->tcp_udp.size = sizeof(struct ibv_kern_spec_tcp_udp);
1885 memcpy(&kern_spec->tcp_udp.val, &ib_spec->tcp_udp.val,
1886 sizeof(struct ibv_flow_ipv4_filter));
1887 memcpy(&kern_spec->tcp_udp.mask, &ib_spec->tcp_udp.mask,
1888 sizeof(struct ibv_flow_tcp_udp_filter));
1889 break;
1890 case IBV_FLOW_SPEC_VXLAN_TUNNEL:
1891 ret = get_filters_size(ib_spec, kern_spec,
1892 &ib_filter_size, &kern_filter_size,
1893 IBV_FLOW_SPEC_VXLAN_TUNNEL);
1894 if (ret)
1895 return ret;
1896
1897 kern_spec->tunnel.size = sizeof(struct ibv_kern_spec_tunnel);
1898 memcpy(&kern_spec->tunnel.val, &ib_spec->tunnel.val,
1899 kern_filter_size);
1900 memcpy(&kern_spec->tunnel.mask, (void *)&ib_spec->tunnel.val
1901 + ib_filter_size, kern_filter_size);
1902 break;
1903 case IBV_FLOW_SPEC_ACTION_TAG:
1904 kern_spec->flow_tag.size =
1905 sizeof(struct ibv_kern_spec_action_tag);
1906 kern_spec->flow_tag.tag_id = ib_spec->flow_tag.tag_id;
1907 break;
1908 case IBV_FLOW_SPEC_ACTION_DROP:
1909 kern_spec->drop.size = sizeof(struct ibv_kern_spec_action_drop);
1910 break;
1911 default:
1912 return EINVAL;
1913 }
1914 return 0;
1915 }
1916
ibv_cmd_create_flow(struct ibv_qp * qp,struct ibv_flow_attr * flow_attr)1917 struct ibv_flow *ibv_cmd_create_flow(struct ibv_qp *qp,
1918 struct ibv_flow_attr *flow_attr)
1919 {
1920 struct ibv_create_flow *cmd;
1921 struct ibv_create_flow_resp resp;
1922 struct ibv_flow *flow_id;
1923 size_t cmd_size;
1924 size_t written_size;
1925 int i, err;
1926 void *kern_spec;
1927 void *ib_spec;
1928
1929 cmd_size = sizeof(*cmd) + (flow_attr->num_of_specs *
1930 sizeof(struct ibv_kern_spec));
1931 cmd = alloca(cmd_size);
1932 flow_id = malloc(sizeof(*flow_id));
1933 if (!flow_id)
1934 return NULL;
1935 memset(cmd, 0, cmd_size);
1936
1937 cmd->qp_handle = qp->handle;
1938
1939 cmd->flow_attr.type = flow_attr->type;
1940 cmd->flow_attr.priority = flow_attr->priority;
1941 cmd->flow_attr.num_of_specs = flow_attr->num_of_specs;
1942 cmd->flow_attr.port = flow_attr->port;
1943 cmd->flow_attr.flags = flow_attr->flags;
1944
1945 kern_spec = cmd + 1;
1946 ib_spec = flow_attr + 1;
1947 for (i = 0; i < flow_attr->num_of_specs; i++) {
1948 err = ib_spec_to_kern_spec(ib_spec, kern_spec);
1949 if (err) {
1950 errno = err;
1951 goto err;
1952 }
1953 cmd->flow_attr.size +=
1954 ((struct ibv_kern_spec *)kern_spec)->hdr.size;
1955 kern_spec += ((struct ibv_kern_spec *)kern_spec)->hdr.size;
1956 ib_spec += ((struct ibv_flow_spec *)ib_spec)->hdr.size;
1957 }
1958
1959 written_size = sizeof(*cmd) + cmd->flow_attr.size;
1960 IBV_INIT_CMD_RESP_EX_VCMD(cmd, written_size, written_size, CREATE_FLOW,
1961 &resp, sizeof(resp));
1962 if (write(qp->context->cmd_fd, cmd, written_size) != written_size)
1963 goto err;
1964
1965 (void) VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof(resp));
1966
1967 flow_id->context = qp->context;
1968 flow_id->handle = resp.flow_handle;
1969 return flow_id;
1970 err:
1971 free(flow_id);
1972 return NULL;
1973 }
1974
ibv_cmd_destroy_flow(struct ibv_flow * flow_id)1975 int ibv_cmd_destroy_flow(struct ibv_flow *flow_id)
1976 {
1977 struct ibv_destroy_flow cmd;
1978 int ret = 0;
1979
1980 memset(&cmd, 0, sizeof(cmd));
1981 IBV_INIT_CMD_EX(&cmd, sizeof(cmd), DESTROY_FLOW);
1982 cmd.flow_handle = flow_id->handle;
1983
1984 if (write(flow_id->context->cmd_fd, &cmd, sizeof(cmd)) != sizeof(cmd))
1985 ret = errno;
1986 free(flow_id);
1987 return ret;
1988 }
1989
ibv_cmd_create_wq(struct ibv_context * context,struct ibv_wq_init_attr * wq_init_attr,struct ibv_wq * wq,struct ibv_create_wq * cmd,size_t cmd_core_size,size_t cmd_size,struct ibv_create_wq_resp * resp,size_t resp_core_size,size_t resp_size)1990 int ibv_cmd_create_wq(struct ibv_context *context,
1991 struct ibv_wq_init_attr *wq_init_attr,
1992 struct ibv_wq *wq,
1993 struct ibv_create_wq *cmd,
1994 size_t cmd_core_size,
1995 size_t cmd_size,
1996 struct ibv_create_wq_resp *resp,
1997 size_t resp_core_size,
1998 size_t resp_size)
1999 {
2000 int err;
2001
2002 if (wq_init_attr->comp_mask >= IBV_WQ_INIT_ATTR_RESERVED)
2003 return EINVAL;
2004
2005 IBV_INIT_CMD_RESP_EX_V(cmd, cmd_core_size, cmd_size,
2006 CREATE_WQ, resp,
2007 resp_core_size, resp_size);
2008
2009 cmd->user_handle = (uintptr_t)wq;
2010 cmd->pd_handle = wq_init_attr->pd->handle;
2011 cmd->cq_handle = wq_init_attr->cq->handle;
2012 cmd->wq_type = wq_init_attr->wq_type;
2013 cmd->max_sge = wq_init_attr->max_sge;
2014 cmd->max_wr = wq_init_attr->max_wr;
2015 cmd->comp_mask = 0;
2016
2017 if (cmd_core_size >= offsetof(struct ibv_create_wq, create_flags) +
2018 sizeof(cmd->create_flags)) {
2019 if (wq_init_attr->comp_mask & IBV_WQ_INIT_ATTR_FLAGS) {
2020 if (wq_init_attr->create_flags & ~(IBV_WQ_FLAGS_RESERVED - 1))
2021 return EOPNOTSUPP;
2022 cmd->create_flags = wq_init_attr->create_flags;
2023 }
2024 }
2025
2026 err = write(context->cmd_fd, cmd, cmd_size);
2027 if (err != cmd_size)
2028 return errno;
2029
2030 (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size);
2031
2032 if (resp->response_length < resp_core_size)
2033 return EINVAL;
2034
2035 wq->handle = resp->wq_handle;
2036 wq_init_attr->max_wr = resp->max_wr;
2037 wq_init_attr->max_sge = resp->max_sge;
2038 wq->wq_num = resp->wqn;
2039 wq->context = context;
2040 wq->cq = wq_init_attr->cq;
2041 wq->pd = wq_init_attr->pd;
2042 wq->wq_type = wq_init_attr->wq_type;
2043
2044 return 0;
2045 }
2046
ibv_cmd_modify_wq(struct ibv_wq * wq,struct ibv_wq_attr * attr,struct ibv_modify_wq * cmd,size_t cmd_core_size,size_t cmd_size)2047 int ibv_cmd_modify_wq(struct ibv_wq *wq, struct ibv_wq_attr *attr,
2048 struct ibv_modify_wq *cmd, size_t cmd_core_size,
2049 size_t cmd_size)
2050 {
2051 if (attr->attr_mask >= IBV_WQ_ATTR_RESERVED)
2052 return EINVAL;
2053
2054 memset(cmd, 0, cmd_core_size);
2055 IBV_INIT_CMD_EX(cmd, cmd_size, MODIFY_WQ);
2056
2057 cmd->curr_wq_state = attr->curr_wq_state;
2058 cmd->wq_state = attr->wq_state;
2059 if (cmd_core_size >= offsetof(struct ibv_modify_wq, flags_mask) +
2060 sizeof(cmd->flags_mask)) {
2061 if (attr->attr_mask & IBV_WQ_ATTR_FLAGS) {
2062 if (attr->flags_mask & ~(IBV_WQ_FLAGS_RESERVED - 1))
2063 return EOPNOTSUPP;
2064 cmd->flags = attr->flags;
2065 cmd->flags_mask = attr->flags_mask;
2066 }
2067 }
2068 cmd->wq_handle = wq->handle;
2069 cmd->attr_mask = attr->attr_mask;
2070
2071 if (write(wq->context->cmd_fd, cmd, cmd_size) != cmd_size)
2072 return errno;
2073
2074 if (attr->attr_mask & IBV_WQ_ATTR_STATE)
2075 wq->state = attr->wq_state;
2076
2077 return 0;
2078 }
2079
ibv_cmd_destroy_wq(struct ibv_wq * wq)2080 int ibv_cmd_destroy_wq(struct ibv_wq *wq)
2081 {
2082 struct ibv_destroy_wq cmd;
2083 struct ibv_destroy_wq_resp resp;
2084 int ret = 0;
2085
2086 memset(&cmd, 0, sizeof(cmd));
2087 memset(&resp, 0, sizeof(resp));
2088
2089 IBV_INIT_CMD_RESP_EX(&cmd, sizeof(cmd), DESTROY_WQ, &resp, sizeof(resp));
2090 cmd.wq_handle = wq->handle;
2091
2092 if (write(wq->context->cmd_fd, &cmd, sizeof(cmd)) != sizeof(cmd))
2093 return errno;
2094
2095 if (resp.response_length < sizeof(resp))
2096 return EINVAL;
2097
2098 pthread_mutex_lock(&wq->mutex);
2099 while (wq->events_completed != resp.events_reported)
2100 pthread_cond_wait(&wq->cond, &wq->mutex);
2101 pthread_mutex_unlock(&wq->mutex);
2102
2103 return ret;
2104 }
2105
ibv_cmd_create_rwq_ind_table(struct ibv_context * context,struct ibv_rwq_ind_table_init_attr * init_attr,struct ibv_rwq_ind_table * rwq_ind_table,struct ibv_create_rwq_ind_table * cmd,size_t cmd_core_size,size_t cmd_size,struct ibv_create_rwq_ind_table_resp * resp,size_t resp_core_size,size_t resp_size)2106 int ibv_cmd_create_rwq_ind_table(struct ibv_context *context,
2107 struct ibv_rwq_ind_table_init_attr *init_attr,
2108 struct ibv_rwq_ind_table *rwq_ind_table,
2109 struct ibv_create_rwq_ind_table *cmd,
2110 size_t cmd_core_size,
2111 size_t cmd_size,
2112 struct ibv_create_rwq_ind_table_resp *resp,
2113 size_t resp_core_size,
2114 size_t resp_size)
2115 {
2116 int err, i;
2117 uint32_t required_tbl_size, alloc_tbl_size;
2118 uint32_t *tbl_start;
2119 int num_tbl_entries;
2120
2121 if (init_attr->comp_mask >= IBV_CREATE_IND_TABLE_RESERVED)
2122 return EINVAL;
2123
2124 alloc_tbl_size = cmd_core_size - sizeof(*cmd);
2125 num_tbl_entries = 1 << init_attr->log_ind_tbl_size;
2126
2127 /* Data must be u64 aligned */
2128 required_tbl_size = (num_tbl_entries * sizeof(uint32_t)) < sizeof(uint64_t) ?
2129 sizeof(uint64_t) : (num_tbl_entries * sizeof(uint32_t));
2130
2131 if (alloc_tbl_size < required_tbl_size)
2132 return EINVAL;
2133
2134 tbl_start = (uint32_t *)((uint8_t *)cmd + sizeof(*cmd));
2135 for (i = 0; i < num_tbl_entries; i++)
2136 tbl_start[i] = init_attr->ind_tbl[i]->handle;
2137
2138 IBV_INIT_CMD_RESP_EX_V(cmd, cmd_core_size, cmd_size,
2139 CREATE_RWQ_IND_TBL, resp,
2140 resp_core_size, resp_size);
2141 cmd->log_ind_tbl_size = init_attr->log_ind_tbl_size;
2142 cmd->comp_mask = 0;
2143
2144 err = write(context->cmd_fd, cmd, cmd_size);
2145 if (err != cmd_size)
2146 return errno;
2147
2148 (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size);
2149
2150 if (resp->response_length < resp_core_size)
2151 return EINVAL;
2152
2153 rwq_ind_table->ind_tbl_handle = resp->ind_tbl_handle;
2154 rwq_ind_table->ind_tbl_num = resp->ind_tbl_num;
2155 rwq_ind_table->context = context;
2156 return 0;
2157 }
2158
ibv_cmd_destroy_rwq_ind_table(struct ibv_rwq_ind_table * rwq_ind_table)2159 int ibv_cmd_destroy_rwq_ind_table(struct ibv_rwq_ind_table *rwq_ind_table)
2160 {
2161 struct ibv_destroy_rwq_ind_table cmd;
2162 int ret = 0;
2163
2164 memset(&cmd, 0, sizeof(cmd));
2165 IBV_INIT_CMD_EX(&cmd, sizeof(cmd), DESTROY_RWQ_IND_TBL);
2166 cmd.ind_tbl_handle = rwq_ind_table->ind_tbl_handle;
2167
2168 if (write(rwq_ind_table->context->cmd_fd, &cmd, sizeof(cmd)) != sizeof(cmd))
2169 ret = errno;
2170
2171 return ret;
2172 }
2173