1 /* 2 * Copyright (c) 2005 Topspin Communications. All rights reserved. 3 * Copyright (c) 2005 PathScale, Inc. All rights reserved. 4 * Copyright (c) 2006 Cisco Systems, Inc. All rights reserved. 5 * 6 * This software is available to you under a choice of one of two 7 * licenses. You may choose to be licensed under the terms of the GNU 8 * General Public License (GPL) Version 2, available from the file 9 * COPYING in the main directory of this source tree, or the 10 * OpenIB.org BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or 13 * without modification, are permitted provided that the following 14 * conditions are met: 15 * 16 * - Redistributions of source code must retain the above 17 * copyright notice, this list of conditions and the following 18 * disclaimer. 19 * 20 * - Redistributions in binary form must reproduce the above 21 * copyright notice, this list of conditions and the following 22 * disclaimer in the documentation and/or other materials 23 * provided with the distribution. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32 * SOFTWARE. 33 */ 34 35 #include <config.h> 36 37 #include <stdio.h> 38 #include <unistd.h> 39 #include <stdlib.h> 40 #include <errno.h> 41 #include <string.h> 42 43 #include "ibverbs.h" 44 #include <sys/param.h> 45 46 int ibv_cmd_get_context(struct ibv_context *context, struct ibv_get_context *cmd, 47 size_t cmd_size, struct ibv_get_context_resp *resp, 48 size_t resp_size) 49 { 50 if (abi_ver < IB_USER_VERBS_MIN_ABI_VERSION) 51 return ENOSYS; 52 53 IBV_INIT_CMD_RESP(cmd, cmd_size, GET_CONTEXT, resp, resp_size); 54 55 if (write(context->cmd_fd, cmd, cmd_size) != cmd_size) 56 return errno; 57 58 (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size); 59 60 context->async_fd = resp->async_fd; 61 context->num_comp_vectors = resp->num_comp_vectors; 62 63 return 0; 64 } 65 66 static void copy_query_dev_fields(struct ibv_device_attr *device_attr, 67 struct ibv_query_device_resp *resp, 68 uint64_t *raw_fw_ver) 69 { 70 *raw_fw_ver = resp->fw_ver; 71 device_attr->node_guid = resp->node_guid; 72 device_attr->sys_image_guid = resp->sys_image_guid; 73 device_attr->max_mr_size = resp->max_mr_size; 74 device_attr->page_size_cap = resp->page_size_cap; 75 device_attr->vendor_id = resp->vendor_id; 76 device_attr->vendor_part_id = resp->vendor_part_id; 77 device_attr->hw_ver = resp->hw_ver; 78 device_attr->max_qp = resp->max_qp; 79 device_attr->max_qp_wr = resp->max_qp_wr; 80 device_attr->device_cap_flags = resp->device_cap_flags; 81 device_attr->max_sge = resp->max_sge; 82 device_attr->max_sge_rd = resp->max_sge_rd; 83 device_attr->max_cq = resp->max_cq; 84 device_attr->max_cqe = resp->max_cqe; 85 device_attr->max_mr = resp->max_mr; 86 device_attr->max_pd = resp->max_pd; 87 device_attr->max_qp_rd_atom = resp->max_qp_rd_atom; 88 device_attr->max_ee_rd_atom = resp->max_ee_rd_atom; 89 device_attr->max_res_rd_atom = resp->max_res_rd_atom; 90 device_attr->max_qp_init_rd_atom = resp->max_qp_init_rd_atom; 91 device_attr->max_ee_init_rd_atom = resp->max_ee_init_rd_atom; 92 device_attr->atomic_cap = resp->atomic_cap; 93 device_attr->max_ee = resp->max_ee; 94 device_attr->max_rdd = resp->max_rdd; 95 device_attr->max_mw = resp->max_mw; 96 device_attr->max_raw_ipv6_qp = resp->max_raw_ipv6_qp; 97 device_attr->max_raw_ethy_qp = resp->max_raw_ethy_qp; 98 device_attr->max_mcast_grp = resp->max_mcast_grp; 99 device_attr->max_mcast_qp_attach = resp->max_mcast_qp_attach; 100 device_attr->max_total_mcast_qp_attach = resp->max_total_mcast_qp_attach; 101 device_attr->max_ah = resp->max_ah; 102 device_attr->max_fmr = resp->max_fmr; 103 device_attr->max_map_per_fmr = resp->max_map_per_fmr; 104 device_attr->max_srq = resp->max_srq; 105 device_attr->max_srq_wr = resp->max_srq_wr; 106 device_attr->max_srq_sge = resp->max_srq_sge; 107 device_attr->max_pkeys = resp->max_pkeys; 108 device_attr->local_ca_ack_delay = resp->local_ca_ack_delay; 109 device_attr->phys_port_cnt = resp->phys_port_cnt; 110 } 111 112 int ibv_cmd_query_device(struct ibv_context *context, 113 struct ibv_device_attr *device_attr, 114 uint64_t *raw_fw_ver, 115 struct ibv_query_device *cmd, size_t cmd_size) 116 { 117 struct ibv_query_device_resp resp; 118 119 IBV_INIT_CMD_RESP(cmd, cmd_size, QUERY_DEVICE, &resp, sizeof resp); 120 121 if (write(context->cmd_fd, cmd, cmd_size) != cmd_size) 122 return errno; 123 124 (void) VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof resp); 125 126 memset(device_attr->fw_ver, 0, sizeof device_attr->fw_ver); 127 copy_query_dev_fields(device_attr, &resp, raw_fw_ver); 128 129 return 0; 130 } 131 132 int ibv_cmd_query_device_ex(struct ibv_context *context, 133 const struct ibv_query_device_ex_input *input, 134 struct ibv_device_attr_ex *attr, size_t attr_size, 135 uint64_t *raw_fw_ver, 136 struct ibv_query_device_ex *cmd, 137 size_t cmd_core_size, 138 size_t cmd_size, 139 struct ibv_query_device_resp_ex *resp, 140 size_t resp_core_size, 141 size_t resp_size) 142 { 143 int err; 144 145 if (input && input->comp_mask) 146 return EINVAL; 147 148 if (attr_size < offsetof(struct ibv_device_attr_ex, comp_mask) + 149 sizeof(attr->comp_mask)) 150 return EINVAL; 151 152 if (resp_core_size < offsetof(struct ibv_query_device_resp_ex, 153 response_length) + 154 sizeof(resp->response_length)) 155 return EINVAL; 156 157 IBV_INIT_CMD_RESP_EX_V(cmd, cmd_core_size, cmd_size, 158 QUERY_DEVICE_EX, resp, resp_core_size, 159 resp_size); 160 cmd->comp_mask = 0; 161 cmd->reserved = 0; 162 memset(attr->orig_attr.fw_ver, 0, sizeof(attr->orig_attr.fw_ver)); 163 memset(&attr->comp_mask, 0, attr_size - sizeof(attr->orig_attr)); 164 err = write(context->cmd_fd, cmd, cmd_size); 165 if (err != cmd_size) 166 return errno; 167 168 (void)VALGRIND_MAKE_MEM_DEFINED(resp, resp_size); 169 copy_query_dev_fields(&attr->orig_attr, &resp->base, raw_fw_ver); 170 /* Report back supported comp_mask bits. For now no comp_mask bit is 171 * defined */ 172 attr->comp_mask = resp->comp_mask & 0; 173 if (attr_size >= offsetof(struct ibv_device_attr_ex, odp_caps) + 174 sizeof(attr->odp_caps)) { 175 if (resp->response_length >= 176 offsetof(struct ibv_query_device_resp_ex, odp_caps) + 177 sizeof(resp->odp_caps)) { 178 attr->odp_caps.general_caps = resp->odp_caps.general_caps; 179 attr->odp_caps.per_transport_caps.rc_odp_caps = 180 resp->odp_caps.per_transport_caps.rc_odp_caps; 181 attr->odp_caps.per_transport_caps.uc_odp_caps = 182 resp->odp_caps.per_transport_caps.uc_odp_caps; 183 attr->odp_caps.per_transport_caps.ud_odp_caps = 184 resp->odp_caps.per_transport_caps.ud_odp_caps; 185 } 186 } 187 188 if (attr_size >= offsetof(struct ibv_device_attr_ex, 189 completion_timestamp_mask) + 190 sizeof(attr->completion_timestamp_mask)) { 191 if (resp->response_length >= 192 offsetof(struct ibv_query_device_resp_ex, timestamp_mask) + 193 sizeof(resp->timestamp_mask)) 194 attr->completion_timestamp_mask = resp->timestamp_mask; 195 } 196 197 if (attr_size >= offsetof(struct ibv_device_attr_ex, hca_core_clock) + 198 sizeof(attr->hca_core_clock)) { 199 if (resp->response_length >= 200 offsetof(struct ibv_query_device_resp_ex, hca_core_clock) + 201 sizeof(resp->hca_core_clock)) 202 attr->hca_core_clock = resp->hca_core_clock; 203 } 204 205 if (attr_size >= offsetof(struct ibv_device_attr_ex, device_cap_flags_ex) + 206 sizeof(attr->device_cap_flags_ex)) { 207 if (resp->response_length >= 208 offsetof(struct ibv_query_device_resp_ex, device_cap_flags_ex) + 209 sizeof(resp->device_cap_flags_ex)) 210 attr->device_cap_flags_ex = resp->device_cap_flags_ex; 211 } 212 213 if (attr_size >= offsetof(struct ibv_device_attr_ex, rss_caps) + 214 sizeof(attr->rss_caps)) { 215 if (resp->response_length >= 216 offsetof(struct ibv_query_device_resp_ex, rss_caps) + 217 sizeof(resp->rss_caps)) { 218 attr->rss_caps.supported_qpts = resp->rss_caps.supported_qpts; 219 attr->rss_caps.max_rwq_indirection_tables = resp->rss_caps.max_rwq_indirection_tables; 220 attr->rss_caps.max_rwq_indirection_table_size = resp->rss_caps.max_rwq_indirection_table_size; 221 } 222 } 223 224 if (attr_size >= offsetof(struct ibv_device_attr_ex, max_wq_type_rq) + 225 sizeof(attr->max_wq_type_rq)) { 226 if (resp->response_length >= 227 offsetof(struct ibv_query_device_resp_ex, max_wq_type_rq) + 228 sizeof(resp->max_wq_type_rq)) 229 attr->max_wq_type_rq = resp->max_wq_type_rq; 230 } 231 232 if (attr_size >= offsetof(struct ibv_device_attr_ex, raw_packet_caps) + 233 sizeof(attr->raw_packet_caps)) { 234 if (resp->response_length >= 235 offsetof(struct ibv_query_device_resp_ex, raw_packet_caps) + 236 sizeof(resp->raw_packet_caps)) 237 attr->raw_packet_caps = resp->raw_packet_caps; 238 } 239 240 if (attr_size >= offsetof(struct ibv_device_attr_ex, tm_caps) + 241 sizeof(attr->tm_caps)) { 242 if (resp->response_length >= 243 offsetof(struct ibv_query_device_resp_ex, tm_caps) + 244 sizeof(resp->tm_caps)) { 245 attr->tm_caps.max_rndv_hdr_size = 246 resp->tm_caps.max_rndv_hdr_size; 247 attr->tm_caps.max_num_tags = 248 resp->tm_caps.max_num_tags; 249 attr->tm_caps.flags = resp->tm_caps.flags; 250 attr->tm_caps.max_ops = 251 resp->tm_caps.max_ops; 252 attr->tm_caps.max_sge = 253 resp->tm_caps.max_sge; 254 } 255 } 256 257 return 0; 258 } 259 260 int ibv_cmd_query_port(struct ibv_context *context, uint8_t port_num, 261 struct ibv_port_attr *port_attr, 262 struct ibv_query_port *cmd, size_t cmd_size) 263 { 264 struct ibv_query_port_resp resp; 265 266 IBV_INIT_CMD_RESP(cmd, cmd_size, QUERY_PORT, &resp, sizeof resp); 267 cmd->port_num = port_num; 268 memset(cmd->reserved, 0, sizeof cmd->reserved); 269 270 if (write(context->cmd_fd, cmd, cmd_size) != cmd_size) 271 return errno; 272 273 (void) VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof resp); 274 275 port_attr->state = resp.state; 276 port_attr->max_mtu = resp.max_mtu; 277 port_attr->active_mtu = resp.active_mtu; 278 port_attr->gid_tbl_len = resp.gid_tbl_len; 279 port_attr->port_cap_flags = resp.port_cap_flags; 280 port_attr->max_msg_sz = resp.max_msg_sz; 281 port_attr->bad_pkey_cntr = resp.bad_pkey_cntr; 282 port_attr->qkey_viol_cntr = resp.qkey_viol_cntr; 283 port_attr->pkey_tbl_len = resp.pkey_tbl_len; 284 port_attr->lid = resp.lid; 285 port_attr->sm_lid = resp.sm_lid; 286 port_attr->lmc = resp.lmc; 287 port_attr->max_vl_num = resp.max_vl_num; 288 port_attr->sm_sl = resp.sm_sl; 289 port_attr->subnet_timeout = resp.subnet_timeout; 290 port_attr->init_type_reply = resp.init_type_reply; 291 port_attr->active_width = resp.active_width; 292 port_attr->active_speed = resp.active_speed; 293 port_attr->phys_state = resp.phys_state; 294 port_attr->link_layer = resp.link_layer; 295 296 return 0; 297 } 298 299 int ibv_cmd_alloc_pd(struct ibv_context *context, struct ibv_pd *pd, 300 struct ibv_alloc_pd *cmd, size_t cmd_size, 301 struct ibv_alloc_pd_resp *resp, size_t resp_size) 302 { 303 IBV_INIT_CMD_RESP(cmd, cmd_size, ALLOC_PD, resp, resp_size); 304 305 if (write(context->cmd_fd, cmd, cmd_size) != cmd_size) 306 return errno; 307 308 (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size); 309 310 pd->handle = resp->pd_handle; 311 pd->context = context; 312 313 return 0; 314 } 315 316 int ibv_cmd_dealloc_pd(struct ibv_pd *pd) 317 { 318 struct ibv_dealloc_pd cmd; 319 320 IBV_INIT_CMD(&cmd, sizeof cmd, DEALLOC_PD); 321 cmd.pd_handle = pd->handle; 322 323 if (write(pd->context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd) 324 return errno; 325 326 return 0; 327 } 328 329 int ibv_cmd_open_xrcd(struct ibv_context *context, struct verbs_xrcd *xrcd, 330 int vxrcd_size, 331 struct ibv_xrcd_init_attr *attr, 332 struct ibv_open_xrcd *cmd, size_t cmd_size, 333 struct ibv_open_xrcd_resp *resp, size_t resp_size) 334 { 335 IBV_INIT_CMD_RESP(cmd, cmd_size, OPEN_XRCD, resp, resp_size); 336 337 if (attr->comp_mask >= IBV_XRCD_INIT_ATTR_RESERVED) 338 return ENOSYS; 339 340 if (!(attr->comp_mask & IBV_XRCD_INIT_ATTR_FD) || 341 !(attr->comp_mask & IBV_XRCD_INIT_ATTR_OFLAGS)) 342 return EINVAL; 343 344 cmd->fd = attr->fd; 345 cmd->oflags = attr->oflags; 346 if (write(context->cmd_fd, cmd, cmd_size) != cmd_size) 347 return errno; 348 349 (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size); 350 351 xrcd->xrcd.context = context; 352 xrcd->comp_mask = 0; 353 if (vext_field_avail(struct verbs_xrcd, handle, vxrcd_size)) { 354 xrcd->comp_mask = VERBS_XRCD_HANDLE; 355 xrcd->handle = resp->xrcd_handle; 356 } 357 358 return 0; 359 } 360 361 int ibv_cmd_close_xrcd(struct verbs_xrcd *xrcd) 362 { 363 struct ibv_close_xrcd cmd; 364 365 IBV_INIT_CMD(&cmd, sizeof cmd, CLOSE_XRCD); 366 cmd.xrcd_handle = xrcd->handle; 367 368 if (write(xrcd->xrcd.context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd) 369 return errno; 370 371 return 0; 372 } 373 374 int ibv_cmd_reg_mr(struct ibv_pd *pd, void *addr, size_t length, 375 uint64_t hca_va, int access, 376 struct ibv_mr *mr, struct ibv_reg_mr *cmd, 377 size_t cmd_size, 378 struct ibv_reg_mr_resp *resp, size_t resp_size) 379 { 380 381 IBV_INIT_CMD_RESP(cmd, cmd_size, REG_MR, resp, resp_size); 382 383 cmd->start = (uintptr_t) addr; 384 cmd->length = length; 385 cmd->hca_va = hca_va; 386 cmd->pd_handle = pd->handle; 387 cmd->access_flags = access; 388 389 if (write(pd->context->cmd_fd, cmd, cmd_size) != cmd_size) 390 return errno; 391 392 (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size); 393 394 mr->handle = resp->mr_handle; 395 mr->lkey = resp->lkey; 396 mr->rkey = resp->rkey; 397 mr->context = pd->context; 398 399 return 0; 400 } 401 402 int ibv_cmd_rereg_mr(struct ibv_mr *mr, uint32_t flags, void *addr, 403 size_t length, uint64_t hca_va, int access, 404 struct ibv_pd *pd, struct ibv_rereg_mr *cmd, 405 size_t cmd_sz, struct ibv_rereg_mr_resp *resp, 406 size_t resp_sz) 407 { 408 IBV_INIT_CMD_RESP(cmd, cmd_sz, REREG_MR, resp, resp_sz); 409 410 cmd->mr_handle = mr->handle; 411 cmd->flags = flags; 412 cmd->start = (uintptr_t)addr; 413 cmd->length = length; 414 cmd->hca_va = hca_va; 415 cmd->pd_handle = (flags & IBV_REREG_MR_CHANGE_PD) ? pd->handle : 0; 416 cmd->access_flags = access; 417 418 if (write(mr->context->cmd_fd, cmd, cmd_sz) != cmd_sz) 419 return errno; 420 421 (void)VALGRIND_MAKE_MEM_DEFINED(resp, resp_sz); 422 423 mr->lkey = resp->lkey; 424 mr->rkey = resp->rkey; 425 if (flags & IBV_REREG_MR_CHANGE_PD) 426 mr->context = pd->context; 427 428 return 0; 429 } 430 431 int ibv_cmd_dereg_mr(struct ibv_mr *mr) 432 { 433 struct ibv_dereg_mr cmd; 434 435 IBV_INIT_CMD(&cmd, sizeof cmd, DEREG_MR); 436 cmd.mr_handle = mr->handle; 437 438 if (write(mr->context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd) 439 return errno; 440 441 return 0; 442 } 443 444 int ibv_cmd_alloc_mw(struct ibv_pd *pd, enum ibv_mw_type type, 445 struct ibv_mw *mw, struct ibv_alloc_mw *cmd, 446 size_t cmd_size, 447 struct ibv_alloc_mw_resp *resp, size_t resp_size) 448 { 449 IBV_INIT_CMD_RESP(cmd, cmd_size, ALLOC_MW, resp, resp_size); 450 cmd->pd_handle = pd->handle; 451 cmd->mw_type = type; 452 memset(cmd->reserved, 0, sizeof(cmd->reserved)); 453 454 if (write(pd->context->cmd_fd, cmd, cmd_size) != cmd_size) 455 return errno; 456 457 (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size); 458 459 mw->context = pd->context; 460 mw->pd = pd; 461 mw->rkey = resp->rkey; 462 mw->handle = resp->mw_handle; 463 mw->type = type; 464 465 return 0; 466 } 467 468 int ibv_cmd_dealloc_mw(struct ibv_mw *mw, 469 struct ibv_dealloc_mw *cmd, size_t cmd_size) 470 { 471 IBV_INIT_CMD(cmd, cmd_size, DEALLOC_MW); 472 cmd->mw_handle = mw->handle; 473 cmd->reserved = 0; 474 475 if (write(mw->context->cmd_fd, cmd, cmd_size) != cmd_size) 476 return errno; 477 478 return 0; 479 } 480 481 int ibv_cmd_create_cq(struct ibv_context *context, int cqe, 482 struct ibv_comp_channel *channel, 483 int comp_vector, struct ibv_cq *cq, 484 struct ibv_create_cq *cmd, size_t cmd_size, 485 struct ibv_create_cq_resp *resp, size_t resp_size) 486 { 487 IBV_INIT_CMD_RESP(cmd, cmd_size, CREATE_CQ, resp, resp_size); 488 cmd->user_handle = (uintptr_t) cq; 489 cmd->cqe = cqe; 490 cmd->comp_vector = comp_vector; 491 cmd->comp_channel = channel ? channel->fd : -1; 492 cmd->reserved = 0; 493 494 if (write(context->cmd_fd, cmd, cmd_size) != cmd_size) 495 return errno; 496 497 (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size); 498 499 cq->handle = resp->cq_handle; 500 cq->cqe = resp->cqe; 501 cq->context = context; 502 503 return 0; 504 } 505 506 int ibv_cmd_create_cq_ex(struct ibv_context *context, 507 struct ibv_cq_init_attr_ex *cq_attr, 508 struct ibv_cq_ex *cq, 509 struct ibv_create_cq_ex *cmd, 510 size_t cmd_core_size, 511 size_t cmd_size, 512 struct ibv_create_cq_resp_ex *resp, 513 size_t resp_core_size, 514 size_t resp_size) 515 { 516 int err; 517 518 memset(cmd, 0, cmd_core_size); 519 IBV_INIT_CMD_RESP_EX_V(cmd, cmd_core_size, cmd_size, CREATE_CQ_EX, resp, 520 resp_core_size, resp_size); 521 522 if (cq_attr->comp_mask & ~(IBV_CQ_INIT_ATTR_MASK_RESERVED - 1)) 523 return EINVAL; 524 525 cmd->user_handle = (uintptr_t)cq; 526 cmd->cqe = cq_attr->cqe; 527 cmd->comp_vector = cq_attr->comp_vector; 528 cmd->comp_channel = cq_attr->channel ? cq_attr->channel->fd : -1; 529 cmd->comp_mask = 0; 530 531 if (cmd_core_size >= offsetof(struct ibv_create_cq_ex, flags) + 532 sizeof(cmd->flags)) { 533 if ((cq_attr->comp_mask & IBV_CQ_INIT_ATTR_MASK_FLAGS) && 534 (cq_attr->flags & ~(IBV_CREATE_CQ_ATTR_RESERVED - 1))) 535 return EOPNOTSUPP; 536 537 if (cq_attr->wc_flags & IBV_WC_EX_WITH_COMPLETION_TIMESTAMP) 538 cmd->flags |= IBV_CREATE_CQ_EX_KERNEL_FLAG_COMPLETION_TIMESTAMP; 539 } 540 541 err = write(context->cmd_fd, cmd, cmd_size); 542 if (err != cmd_size) 543 return errno; 544 545 (void)VALGRIND_MAKE_MEM_DEFINED(resp, resp_size); 546 547 cq->handle = resp->base.cq_handle; 548 cq->cqe = resp->base.cqe; 549 cq->context = context; 550 551 return 0; 552 } 553 554 int ibv_cmd_poll_cq(struct ibv_cq *ibcq, int ne, struct ibv_wc *wc) 555 { 556 struct ibv_poll_cq cmd; 557 struct ibv_poll_cq_resp *resp; 558 int i; 559 int rsize; 560 int ret; 561 562 rsize = sizeof *resp + ne * sizeof(struct ibv_kern_wc); 563 resp = malloc(rsize); 564 if (!resp) 565 return -1; 566 567 IBV_INIT_CMD_RESP(&cmd, sizeof cmd, POLL_CQ, resp, rsize); 568 cmd.cq_handle = ibcq->handle; 569 cmd.ne = ne; 570 571 if (write(ibcq->context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd) { 572 ret = -1; 573 goto out; 574 } 575 576 (void) VALGRIND_MAKE_MEM_DEFINED(resp, rsize); 577 578 for (i = 0; i < resp->count; i++) { 579 wc[i].wr_id = resp->wc[i].wr_id; 580 wc[i].status = resp->wc[i].status; 581 wc[i].opcode = resp->wc[i].opcode; 582 wc[i].vendor_err = resp->wc[i].vendor_err; 583 wc[i].byte_len = resp->wc[i].byte_len; 584 wc[i].imm_data = resp->wc[i].imm_data; 585 wc[i].qp_num = resp->wc[i].qp_num; 586 wc[i].src_qp = resp->wc[i].src_qp; 587 wc[i].wc_flags = resp->wc[i].wc_flags; 588 wc[i].pkey_index = resp->wc[i].pkey_index; 589 wc[i].slid = resp->wc[i].slid; 590 wc[i].sl = resp->wc[i].sl; 591 wc[i].dlid_path_bits = resp->wc[i].dlid_path_bits; 592 } 593 594 ret = resp->count; 595 596 out: 597 free(resp); 598 return ret; 599 } 600 601 int ibv_cmd_req_notify_cq(struct ibv_cq *ibcq, int solicited_only) 602 { 603 struct ibv_req_notify_cq cmd; 604 605 IBV_INIT_CMD(&cmd, sizeof cmd, REQ_NOTIFY_CQ); 606 cmd.cq_handle = ibcq->handle; 607 cmd.solicited = !!solicited_only; 608 609 if (write(ibcq->context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd) 610 return errno; 611 612 return 0; 613 } 614 615 int ibv_cmd_resize_cq(struct ibv_cq *cq, int cqe, 616 struct ibv_resize_cq *cmd, size_t cmd_size, 617 struct ibv_resize_cq_resp *resp, size_t resp_size) 618 { 619 IBV_INIT_CMD_RESP(cmd, cmd_size, RESIZE_CQ, resp, resp_size); 620 cmd->cq_handle = cq->handle; 621 cmd->cqe = cqe; 622 623 if (write(cq->context->cmd_fd, cmd, cmd_size) != cmd_size) 624 return errno; 625 626 (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size); 627 628 cq->cqe = resp->cqe; 629 630 return 0; 631 } 632 633 int ibv_cmd_destroy_cq(struct ibv_cq *cq) 634 { 635 struct ibv_destroy_cq cmd; 636 struct ibv_destroy_cq_resp resp; 637 638 IBV_INIT_CMD_RESP(&cmd, sizeof cmd, DESTROY_CQ, &resp, sizeof resp); 639 cmd.cq_handle = cq->handle; 640 cmd.reserved = 0; 641 642 if (write(cq->context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd) 643 return errno; 644 645 (void) VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof resp); 646 647 pthread_mutex_lock(&cq->mutex); 648 while (cq->comp_events_completed != resp.comp_events_reported || 649 cq->async_events_completed != resp.async_events_reported) 650 pthread_cond_wait(&cq->cond, &cq->mutex); 651 pthread_mutex_unlock(&cq->mutex); 652 653 return 0; 654 } 655 656 int ibv_cmd_create_srq(struct ibv_pd *pd, 657 struct ibv_srq *srq, struct ibv_srq_init_attr *attr, 658 struct ibv_create_srq *cmd, size_t cmd_size, 659 struct ibv_create_srq_resp *resp, size_t resp_size) 660 { 661 IBV_INIT_CMD_RESP(cmd, cmd_size, CREATE_SRQ, resp, resp_size); 662 cmd->user_handle = (uintptr_t) srq; 663 cmd->pd_handle = pd->handle; 664 cmd->max_wr = attr->attr.max_wr; 665 cmd->max_sge = attr->attr.max_sge; 666 cmd->srq_limit = attr->attr.srq_limit; 667 668 if (write(pd->context->cmd_fd, cmd, cmd_size) != cmd_size) 669 return errno; 670 671 (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size); 672 673 srq->handle = resp->srq_handle; 674 srq->context = pd->context; 675 676 if (abi_ver > 5) { 677 attr->attr.max_wr = resp->max_wr; 678 attr->attr.max_sge = resp->max_sge; 679 } else { 680 struct ibv_create_srq_resp_v5 *resp_v5 = 681 (struct ibv_create_srq_resp_v5 *) resp; 682 683 memmove((void *) resp + sizeof *resp, 684 (void *) resp_v5 + sizeof *resp_v5, 685 resp_size - sizeof *resp); 686 } 687 688 return 0; 689 } 690 691 int ibv_cmd_create_srq_ex(struct ibv_context *context, 692 struct verbs_srq *srq, int vsrq_sz, 693 struct ibv_srq_init_attr_ex *attr_ex, 694 struct ibv_create_xsrq *cmd, size_t cmd_size, 695 struct ibv_create_srq_resp *resp, size_t resp_size) 696 { 697 struct verbs_xrcd *vxrcd = NULL; 698 int ret = 0; 699 700 IBV_INIT_CMD_RESP(cmd, cmd_size, CREATE_XSRQ, resp, resp_size); 701 702 if (attr_ex->comp_mask >= IBV_SRQ_INIT_ATTR_RESERVED) 703 return ENOSYS; 704 705 if (!(attr_ex->comp_mask & IBV_SRQ_INIT_ATTR_PD)) 706 return EINVAL; 707 708 cmd->user_handle = (uintptr_t) srq; 709 cmd->pd_handle = attr_ex->pd->handle; 710 cmd->max_wr = attr_ex->attr.max_wr; 711 cmd->max_sge = attr_ex->attr.max_sge; 712 cmd->srq_limit = attr_ex->attr.srq_limit; 713 714 cmd->srq_type = (attr_ex->comp_mask & IBV_SRQ_INIT_ATTR_TYPE) ? 715 attr_ex->srq_type : IBV_SRQT_BASIC; 716 if (attr_ex->comp_mask & IBV_SRQ_INIT_ATTR_XRCD) { 717 if (!(attr_ex->comp_mask & IBV_SRQ_INIT_ATTR_CQ)) 718 return EINVAL; 719 720 vxrcd = container_of(attr_ex->xrcd, struct verbs_xrcd, xrcd); 721 cmd->xrcd_handle = vxrcd->handle; 722 cmd->cq_handle = attr_ex->cq->handle; 723 } else if (attr_ex->comp_mask & IBV_SRQ_INIT_ATTR_TM) { 724 if (cmd->srq_type != IBV_SRQT_TM) 725 return EINVAL; 726 if (!(attr_ex->comp_mask & IBV_SRQ_INIT_ATTR_CQ) || 727 !attr_ex->tm_cap.max_num_tags) 728 return EINVAL; 729 730 cmd->cq_handle = attr_ex->cq->handle; 731 cmd->max_num_tags = attr_ex->tm_cap.max_num_tags; 732 } else if (cmd->srq_type != IBV_SRQT_BASIC) { 733 return EINVAL; 734 } 735 736 ret = pthread_mutex_init(&srq->srq.mutex, NULL); 737 if (ret) 738 goto err; 739 ret = pthread_cond_init(&srq->srq.cond, NULL); 740 if (ret) 741 goto err_mutex; 742 743 if (write(context->cmd_fd, cmd, cmd_size) != cmd_size) { 744 ret = errno; 745 goto err_cond; 746 } 747 748 (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size); 749 750 srq->srq.handle = resp->srq_handle; 751 srq->srq.context = context; 752 srq->srq.srq_context = attr_ex->srq_context; 753 srq->srq.pd = attr_ex->pd; 754 srq->srq.events_completed = 0; 755 756 /* 757 * check that the last field is available. 758 * If it is than all the others exist as well 759 */ 760 if (vext_field_avail(struct verbs_srq, srq_num, vsrq_sz)) { 761 srq->comp_mask = IBV_SRQ_INIT_ATTR_TYPE; 762 srq->srq_type = (attr_ex->comp_mask & IBV_SRQ_INIT_ATTR_TYPE) ? 763 attr_ex->srq_type : IBV_SRQT_BASIC; 764 if (srq->srq_type == IBV_SRQT_XRC) { 765 srq->comp_mask |= VERBS_SRQ_NUM; 766 srq->srq_num = resp->srqn; 767 } 768 if (attr_ex->comp_mask & IBV_SRQ_INIT_ATTR_XRCD) { 769 srq->comp_mask |= VERBS_SRQ_XRCD; 770 srq->xrcd = vxrcd; 771 } 772 if (attr_ex->comp_mask & IBV_SRQ_INIT_ATTR_CQ) { 773 srq->comp_mask |= VERBS_SRQ_CQ; 774 srq->cq = attr_ex->cq; 775 } 776 } 777 778 attr_ex->attr.max_wr = resp->max_wr; 779 attr_ex->attr.max_sge = resp->max_sge; 780 781 return 0; 782 err_cond: 783 pthread_cond_destroy(&srq->srq.cond); 784 err_mutex: 785 pthread_mutex_destroy(&srq->srq.mutex); 786 err: 787 return ret; 788 } 789 790 791 static int ibv_cmd_modify_srq_v3(struct ibv_srq *srq, 792 struct ibv_srq_attr *srq_attr, 793 int srq_attr_mask, 794 struct ibv_modify_srq *new_cmd, 795 size_t new_cmd_size) 796 { 797 struct ibv_modify_srq_v3 *cmd; 798 size_t cmd_size; 799 800 cmd_size = sizeof *cmd + new_cmd_size - sizeof *new_cmd; 801 cmd = alloca(cmd_size); 802 memcpy(cmd->driver_data, new_cmd->driver_data, new_cmd_size - sizeof *new_cmd); 803 804 IBV_INIT_CMD(cmd, cmd_size, MODIFY_SRQ); 805 806 cmd->srq_handle = srq->handle; 807 cmd->attr_mask = srq_attr_mask; 808 cmd->max_wr = srq_attr->max_wr; 809 cmd->srq_limit = srq_attr->srq_limit; 810 cmd->max_sge = 0; 811 cmd->reserved = 0; 812 813 if (write(srq->context->cmd_fd, cmd, cmd_size) != cmd_size) 814 return errno; 815 816 return 0; 817 } 818 819 int ibv_cmd_modify_srq(struct ibv_srq *srq, 820 struct ibv_srq_attr *srq_attr, 821 int srq_attr_mask, 822 struct ibv_modify_srq *cmd, size_t cmd_size) 823 { 824 if (abi_ver == 3) 825 return ibv_cmd_modify_srq_v3(srq, srq_attr, srq_attr_mask, 826 cmd, cmd_size); 827 828 IBV_INIT_CMD(cmd, cmd_size, MODIFY_SRQ); 829 830 cmd->srq_handle = srq->handle; 831 cmd->attr_mask = srq_attr_mask; 832 cmd->max_wr = srq_attr->max_wr; 833 cmd->srq_limit = srq_attr->srq_limit; 834 835 if (write(srq->context->cmd_fd, cmd, cmd_size) != cmd_size) 836 return errno; 837 838 return 0; 839 } 840 841 int ibv_cmd_query_srq(struct ibv_srq *srq, struct ibv_srq_attr *srq_attr, 842 struct ibv_query_srq *cmd, size_t cmd_size) 843 { 844 struct ibv_query_srq_resp resp; 845 846 IBV_INIT_CMD_RESP(cmd, cmd_size, QUERY_SRQ, &resp, sizeof resp); 847 cmd->srq_handle = srq->handle; 848 cmd->reserved = 0; 849 850 if (write(srq->context->cmd_fd, cmd, cmd_size) != cmd_size) 851 return errno; 852 853 (void) VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof resp); 854 855 srq_attr->max_wr = resp.max_wr; 856 srq_attr->max_sge = resp.max_sge; 857 srq_attr->srq_limit = resp.srq_limit; 858 859 return 0; 860 } 861 862 int ibv_cmd_destroy_srq(struct ibv_srq *srq) 863 { 864 struct ibv_destroy_srq cmd; 865 struct ibv_destroy_srq_resp resp; 866 867 IBV_INIT_CMD_RESP(&cmd, sizeof cmd, DESTROY_SRQ, &resp, sizeof resp); 868 cmd.srq_handle = srq->handle; 869 cmd.reserved = 0; 870 871 if (write(srq->context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd) 872 return errno; 873 874 (void) VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof resp); 875 876 pthread_mutex_lock(&srq->mutex); 877 while (srq->events_completed != resp.events_reported) 878 pthread_cond_wait(&srq->cond, &srq->mutex); 879 pthread_mutex_unlock(&srq->mutex); 880 881 pthread_cond_destroy(&srq->cond); 882 pthread_mutex_destroy(&srq->mutex); 883 884 return 0; 885 } 886 887 static int create_qp_ex_common(struct verbs_qp *qp, 888 struct ibv_qp_init_attr_ex *qp_attr, 889 struct verbs_xrcd *vxrcd, 890 struct ibv_create_qp_common *cmd) 891 { 892 cmd->user_handle = (uintptr_t)qp; 893 894 if (qp_attr->comp_mask & IBV_QP_INIT_ATTR_XRCD) { 895 vxrcd = container_of(qp_attr->xrcd, struct verbs_xrcd, xrcd); 896 cmd->pd_handle = vxrcd->handle; 897 } else { 898 if (!(qp_attr->comp_mask & IBV_QP_INIT_ATTR_PD)) 899 return EINVAL; 900 901 cmd->pd_handle = qp_attr->pd->handle; 902 if (qp_attr->comp_mask & IBV_QP_INIT_ATTR_IND_TABLE) { 903 if (cmd->max_recv_wr || cmd->max_recv_sge || 904 cmd->recv_cq_handle || qp_attr->srq) 905 return EINVAL; 906 907 /* send_cq is optinal */ 908 if (qp_attr->cap.max_send_wr) 909 cmd->send_cq_handle = qp_attr->send_cq->handle; 910 } else { 911 cmd->send_cq_handle = qp_attr->send_cq->handle; 912 913 if (qp_attr->qp_type != IBV_QPT_XRC_SEND) { 914 cmd->recv_cq_handle = qp_attr->recv_cq->handle; 915 cmd->srq_handle = qp_attr->srq ? qp_attr->srq->handle : 916 0; 917 } 918 } 919 } 920 921 cmd->max_send_wr = qp_attr->cap.max_send_wr; 922 cmd->max_recv_wr = qp_attr->cap.max_recv_wr; 923 cmd->max_send_sge = qp_attr->cap.max_send_sge; 924 cmd->max_recv_sge = qp_attr->cap.max_recv_sge; 925 cmd->max_inline_data = qp_attr->cap.max_inline_data; 926 cmd->sq_sig_all = qp_attr->sq_sig_all; 927 cmd->qp_type = qp_attr->qp_type; 928 cmd->is_srq = !!qp_attr->srq; 929 cmd->reserved = 0; 930 931 return 0; 932 } 933 934 static void create_qp_handle_resp_common_cleanup(struct verbs_qp *qp) 935 { 936 pthread_cond_destroy(&qp->qp.cond); 937 pthread_mutex_destroy(&qp->qp.mutex); 938 } 939 940 static int create_qp_handle_resp_common_init(struct verbs_qp *qp) 941 { 942 int ret = 0; 943 944 ret = pthread_mutex_init(&qp->qp.mutex, NULL); 945 if (ret) 946 return ret; 947 ret = pthread_cond_init(&qp->qp.cond, NULL); 948 if (ret) 949 goto err; 950 951 return ret; 952 953 err: 954 pthread_mutex_destroy(&qp->qp.mutex); 955 956 return ret; 957 } 958 959 static void create_qp_handle_resp_common(struct ibv_context *context, 960 struct verbs_qp *qp, 961 struct ibv_qp_init_attr_ex *qp_attr, 962 struct ibv_create_qp_resp *resp, 963 struct verbs_xrcd *vxrcd, 964 int vqp_sz) 965 { 966 if (abi_ver > 3) { 967 qp_attr->cap.max_recv_sge = resp->max_recv_sge; 968 qp_attr->cap.max_send_sge = resp->max_send_sge; 969 qp_attr->cap.max_recv_wr = resp->max_recv_wr; 970 qp_attr->cap.max_send_wr = resp->max_send_wr; 971 qp_attr->cap.max_inline_data = resp->max_inline_data; 972 } 973 974 qp->qp.handle = resp->qp_handle; 975 qp->qp.qp_num = resp->qpn; 976 qp->qp.context = context; 977 qp->qp.qp_context = qp_attr->qp_context; 978 qp->qp.pd = qp_attr->pd; 979 qp->qp.send_cq = qp_attr->send_cq; 980 qp->qp.recv_cq = qp_attr->recv_cq; 981 qp->qp.srq = qp_attr->srq; 982 qp->qp.qp_type = qp_attr->qp_type; 983 qp->qp.state = IBV_QPS_RESET; 984 qp->qp.events_completed = 0; 985 986 qp->comp_mask = 0; 987 if (vext_field_avail(struct verbs_qp, xrcd, vqp_sz) && 988 (qp_attr->comp_mask & IBV_QP_INIT_ATTR_XRCD)) { 989 qp->comp_mask |= VERBS_QP_XRCD; 990 qp->xrcd = vxrcd; 991 } 992 } 993 994 enum { 995 CREATE_QP_EX2_SUP_CREATE_FLAGS = IBV_QP_CREATE_BLOCK_SELF_MCAST_LB | 996 IBV_QP_CREATE_SCATTER_FCS | 997 IBV_QP_CREATE_CVLAN_STRIPPING, 998 }; 999 1000 int ibv_cmd_create_qp_ex2(struct ibv_context *context, 1001 struct verbs_qp *qp, int vqp_sz, 1002 struct ibv_qp_init_attr_ex *qp_attr, 1003 struct ibv_create_qp_ex *cmd, 1004 size_t cmd_core_size, 1005 size_t cmd_size, 1006 struct ibv_create_qp_resp_ex *resp, 1007 size_t resp_core_size, 1008 size_t resp_size) 1009 { 1010 struct verbs_xrcd *vxrcd = NULL; 1011 int err; 1012 1013 if (qp_attr->comp_mask >= IBV_QP_INIT_ATTR_RESERVED) 1014 return EINVAL; 1015 1016 if (resp_core_size < 1017 offsetof(struct ibv_create_qp_resp_ex, response_length) + 1018 sizeof(resp->response_length)) 1019 return EINVAL; 1020 1021 memset(cmd, 0, cmd_core_size); 1022 1023 IBV_INIT_CMD_RESP_EX_V(cmd, cmd_core_size, cmd_size, CREATE_QP_EX, resp, 1024 resp_core_size, resp_size); 1025 1026 err = create_qp_ex_common(qp, qp_attr, vxrcd, &cmd->base); 1027 if (err) 1028 return err; 1029 1030 if (qp_attr->comp_mask & IBV_QP_INIT_ATTR_CREATE_FLAGS) { 1031 if (qp_attr->create_flags & ~CREATE_QP_EX2_SUP_CREATE_FLAGS) 1032 return EINVAL; 1033 if (cmd_core_size < offsetof(struct ibv_create_qp_ex, create_flags) + 1034 sizeof(qp_attr->create_flags)) 1035 return EINVAL; 1036 cmd->create_flags = qp_attr->create_flags; 1037 } 1038 1039 if (qp_attr->comp_mask & IBV_QP_INIT_ATTR_IND_TABLE) { 1040 if (cmd_core_size < offsetof(struct ibv_create_qp_ex, ind_tbl_handle) + 1041 sizeof(cmd->ind_tbl_handle)) 1042 return EINVAL; 1043 cmd->ind_tbl_handle = qp_attr->rwq_ind_tbl->ind_tbl_handle; 1044 cmd->comp_mask = IBV_CREATE_QP_EX_KERNEL_MASK_IND_TABLE; 1045 } 1046 1047 err = create_qp_handle_resp_common_init(qp); 1048 if (err) 1049 return err; 1050 1051 err = write(context->cmd_fd, cmd, cmd_size); 1052 if (err != cmd_size) { 1053 err = errno; 1054 goto err; 1055 } 1056 1057 (void)VALGRIND_MAKE_MEM_DEFINED(resp, resp_size); 1058 1059 create_qp_handle_resp_common(context, qp, qp_attr, &resp->base, vxrcd, 1060 vqp_sz); 1061 1062 return 0; 1063 1064 err: 1065 create_qp_handle_resp_common_cleanup(qp); 1066 1067 return err; 1068 } 1069 1070 int ibv_cmd_create_qp_ex(struct ibv_context *context, 1071 struct verbs_qp *qp, int vqp_sz, 1072 struct ibv_qp_init_attr_ex *attr_ex, 1073 struct ibv_create_qp *cmd, size_t cmd_size, 1074 struct ibv_create_qp_resp *resp, size_t resp_size) 1075 { 1076 struct verbs_xrcd *vxrcd = NULL; 1077 int err; 1078 1079 IBV_INIT_CMD_RESP(cmd, cmd_size, CREATE_QP, resp, resp_size); 1080 1081 if (attr_ex->comp_mask > (IBV_QP_INIT_ATTR_XRCD | IBV_QP_INIT_ATTR_PD)) 1082 return ENOSYS; 1083 1084 err = create_qp_ex_common(qp, attr_ex, vxrcd, 1085 (struct ibv_create_qp_common *)&cmd->user_handle); 1086 if (err) 1087 return err; 1088 1089 err = create_qp_handle_resp_common_init(qp); 1090 if (err) 1091 return err; 1092 1093 err = write(context->cmd_fd, cmd, cmd_size); 1094 if (err != cmd_size) { 1095 err = errno; 1096 goto err; 1097 } 1098 1099 (void)VALGRIND_MAKE_MEM_DEFINED(resp, resp_size); 1100 1101 if (abi_ver == 4) { 1102 struct ibv_create_qp_resp_v4 *resp_v4 = 1103 (struct ibv_create_qp_resp_v4 *)resp; 1104 1105 memmove((void *)resp + sizeof *resp, 1106 (void *)resp_v4 + sizeof *resp_v4, 1107 resp_size - sizeof *resp); 1108 } else if (abi_ver <= 3) { 1109 struct ibv_create_qp_resp_v3 *resp_v3 = 1110 (struct ibv_create_qp_resp_v3 *)resp; 1111 1112 memmove((void *)resp + sizeof *resp, 1113 (void *)resp_v3 + sizeof *resp_v3, 1114 resp_size - sizeof *resp); 1115 } 1116 1117 create_qp_handle_resp_common(context, qp, attr_ex, resp, vxrcd, vqp_sz); 1118 1119 return 0; 1120 1121 err: 1122 create_qp_handle_resp_common_cleanup(qp); 1123 1124 return err; 1125 } 1126 1127 int ibv_cmd_create_qp(struct ibv_pd *pd, 1128 struct ibv_qp *qp, struct ibv_qp_init_attr *attr, 1129 struct ibv_create_qp *cmd, size_t cmd_size, 1130 struct ibv_create_qp_resp *resp, size_t resp_size) 1131 { 1132 IBV_INIT_CMD_RESP(cmd, cmd_size, CREATE_QP, resp, resp_size); 1133 1134 cmd->user_handle = (uintptr_t) qp; 1135 cmd->pd_handle = pd->handle; 1136 cmd->send_cq_handle = attr->send_cq->handle; 1137 cmd->recv_cq_handle = attr->recv_cq->handle; 1138 cmd->srq_handle = attr->srq ? attr->srq->handle : 0; 1139 cmd->max_send_wr = attr->cap.max_send_wr; 1140 cmd->max_recv_wr = attr->cap.max_recv_wr; 1141 cmd->max_send_sge = attr->cap.max_send_sge; 1142 cmd->max_recv_sge = attr->cap.max_recv_sge; 1143 cmd->max_inline_data = attr->cap.max_inline_data; 1144 cmd->sq_sig_all = attr->sq_sig_all; 1145 cmd->qp_type = attr->qp_type; 1146 cmd->is_srq = !!attr->srq; 1147 cmd->reserved = 0; 1148 1149 if (write(pd->context->cmd_fd, cmd, cmd_size) != cmd_size) 1150 return errno; 1151 1152 (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size); 1153 1154 qp->handle = resp->qp_handle; 1155 qp->qp_num = resp->qpn; 1156 qp->context = pd->context; 1157 1158 if (abi_ver > 3) { 1159 attr->cap.max_recv_sge = resp->max_recv_sge; 1160 attr->cap.max_send_sge = resp->max_send_sge; 1161 attr->cap.max_recv_wr = resp->max_recv_wr; 1162 attr->cap.max_send_wr = resp->max_send_wr; 1163 attr->cap.max_inline_data = resp->max_inline_data; 1164 } 1165 1166 if (abi_ver == 4) { 1167 struct ibv_create_qp_resp_v4 *resp_v4 = 1168 (struct ibv_create_qp_resp_v4 *) resp; 1169 1170 memmove((void *) resp + sizeof *resp, 1171 (void *) resp_v4 + sizeof *resp_v4, 1172 resp_size - sizeof *resp); 1173 } else if (abi_ver <= 3) { 1174 struct ibv_create_qp_resp_v3 *resp_v3 = 1175 (struct ibv_create_qp_resp_v3 *) resp; 1176 1177 memmove((void *) resp + sizeof *resp, 1178 (void *) resp_v3 + sizeof *resp_v3, 1179 resp_size - sizeof *resp); 1180 } 1181 1182 return 0; 1183 } 1184 1185 int ibv_cmd_open_qp(struct ibv_context *context, struct verbs_qp *qp, 1186 int vqp_sz, 1187 struct ibv_qp_open_attr *attr, 1188 struct ibv_open_qp *cmd, size_t cmd_size, 1189 struct ibv_create_qp_resp *resp, size_t resp_size) 1190 { 1191 int err = 0; 1192 struct verbs_xrcd *xrcd; 1193 IBV_INIT_CMD_RESP(cmd, cmd_size, OPEN_QP, resp, resp_size); 1194 1195 if (attr->comp_mask >= IBV_QP_OPEN_ATTR_RESERVED) 1196 return ENOSYS; 1197 1198 if (!(attr->comp_mask & IBV_QP_OPEN_ATTR_XRCD) || 1199 !(attr->comp_mask & IBV_QP_OPEN_ATTR_NUM) || 1200 !(attr->comp_mask & IBV_QP_OPEN_ATTR_TYPE)) 1201 return EINVAL; 1202 1203 xrcd = container_of(attr->xrcd, struct verbs_xrcd, xrcd); 1204 cmd->user_handle = (uintptr_t) qp; 1205 cmd->pd_handle = xrcd->handle; 1206 cmd->qpn = attr->qp_num; 1207 cmd->qp_type = attr->qp_type; 1208 1209 err = pthread_mutex_init(&qp->qp.mutex, NULL); 1210 if (err) 1211 return err; 1212 err = pthread_cond_init(&qp->qp.cond, NULL); 1213 if (err) 1214 goto err_mutex; 1215 1216 err = write(context->cmd_fd, cmd, cmd_size); 1217 if (err != cmd_size) { 1218 err = errno; 1219 goto err_cond; 1220 } 1221 1222 (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size); 1223 1224 qp->qp.handle = resp->qp_handle; 1225 qp->qp.context = context; 1226 qp->qp.qp_context = attr->qp_context; 1227 qp->qp.pd = NULL; 1228 qp->qp.send_cq = NULL; 1229 qp->qp.recv_cq = NULL; 1230 qp->qp.srq = NULL; 1231 qp->qp.qp_num = attr->qp_num; 1232 qp->qp.qp_type = attr->qp_type; 1233 qp->qp.state = IBV_QPS_UNKNOWN; 1234 qp->qp.events_completed = 0; 1235 qp->comp_mask = 0; 1236 if (vext_field_avail(struct verbs_qp, xrcd, vqp_sz)) { 1237 qp->comp_mask = VERBS_QP_XRCD; 1238 qp->xrcd = xrcd; 1239 } 1240 1241 return 0; 1242 1243 err_cond: 1244 pthread_cond_destroy(&qp->qp.cond); 1245 err_mutex: 1246 pthread_mutex_destroy(&qp->qp.mutex); 1247 1248 return err; 1249 } 1250 1251 int ibv_cmd_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, 1252 int attr_mask, 1253 struct ibv_qp_init_attr *init_attr, 1254 struct ibv_query_qp *cmd, size_t cmd_size) 1255 { 1256 struct ibv_query_qp_resp resp; 1257 1258 /* 1259 * Masks over IBV_QP_DEST_QPN are not supported by 1260 * that not extended command. 1261 */ 1262 if (attr_mask & ~((IBV_QP_DEST_QPN << 1) - 1)) 1263 return EOPNOTSUPP; 1264 1265 IBV_INIT_CMD_RESP(cmd, cmd_size, QUERY_QP, &resp, sizeof resp); 1266 cmd->qp_handle = qp->handle; 1267 cmd->attr_mask = attr_mask; 1268 1269 if (write(qp->context->cmd_fd, cmd, cmd_size) != cmd_size) 1270 return errno; 1271 1272 (void) VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof resp); 1273 1274 attr->qkey = resp.qkey; 1275 attr->rq_psn = resp.rq_psn; 1276 attr->sq_psn = resp.sq_psn; 1277 attr->dest_qp_num = resp.dest_qp_num; 1278 attr->qp_access_flags = resp.qp_access_flags; 1279 attr->pkey_index = resp.pkey_index; 1280 attr->alt_pkey_index = resp.alt_pkey_index; 1281 attr->qp_state = resp.qp_state; 1282 attr->cur_qp_state = resp.cur_qp_state; 1283 attr->path_mtu = resp.path_mtu; 1284 attr->path_mig_state = resp.path_mig_state; 1285 attr->sq_draining = resp.sq_draining; 1286 attr->max_rd_atomic = resp.max_rd_atomic; 1287 attr->max_dest_rd_atomic = resp.max_dest_rd_atomic; 1288 attr->min_rnr_timer = resp.min_rnr_timer; 1289 attr->port_num = resp.port_num; 1290 attr->timeout = resp.timeout; 1291 attr->retry_cnt = resp.retry_cnt; 1292 attr->rnr_retry = resp.rnr_retry; 1293 attr->alt_port_num = resp.alt_port_num; 1294 attr->alt_timeout = resp.alt_timeout; 1295 attr->cap.max_send_wr = resp.max_send_wr; 1296 attr->cap.max_recv_wr = resp.max_recv_wr; 1297 attr->cap.max_send_sge = resp.max_send_sge; 1298 attr->cap.max_recv_sge = resp.max_recv_sge; 1299 attr->cap.max_inline_data = resp.max_inline_data; 1300 1301 memcpy(attr->ah_attr.grh.dgid.raw, resp.dest.dgid, 16); 1302 attr->ah_attr.grh.flow_label = resp.dest.flow_label; 1303 attr->ah_attr.dlid = resp.dest.dlid; 1304 attr->ah_attr.grh.sgid_index = resp.dest.sgid_index; 1305 attr->ah_attr.grh.hop_limit = resp.dest.hop_limit; 1306 attr->ah_attr.grh.traffic_class = resp.dest.traffic_class; 1307 attr->ah_attr.sl = resp.dest.sl; 1308 attr->ah_attr.src_path_bits = resp.dest.src_path_bits; 1309 attr->ah_attr.static_rate = resp.dest.static_rate; 1310 attr->ah_attr.is_global = resp.dest.is_global; 1311 attr->ah_attr.port_num = resp.dest.port_num; 1312 1313 memcpy(attr->alt_ah_attr.grh.dgid.raw, resp.alt_dest.dgid, 16); 1314 attr->alt_ah_attr.grh.flow_label = resp.alt_dest.flow_label; 1315 attr->alt_ah_attr.dlid = resp.alt_dest.dlid; 1316 attr->alt_ah_attr.grh.sgid_index = resp.alt_dest.sgid_index; 1317 attr->alt_ah_attr.grh.hop_limit = resp.alt_dest.hop_limit; 1318 attr->alt_ah_attr.grh.traffic_class = resp.alt_dest.traffic_class; 1319 attr->alt_ah_attr.sl = resp.alt_dest.sl; 1320 attr->alt_ah_attr.src_path_bits = resp.alt_dest.src_path_bits; 1321 attr->alt_ah_attr.static_rate = resp.alt_dest.static_rate; 1322 attr->alt_ah_attr.is_global = resp.alt_dest.is_global; 1323 attr->alt_ah_attr.port_num = resp.alt_dest.port_num; 1324 1325 init_attr->qp_context = qp->qp_context; 1326 init_attr->send_cq = qp->send_cq; 1327 init_attr->recv_cq = qp->recv_cq; 1328 init_attr->srq = qp->srq; 1329 init_attr->qp_type = qp->qp_type; 1330 init_attr->cap.max_send_wr = resp.max_send_wr; 1331 init_attr->cap.max_recv_wr = resp.max_recv_wr; 1332 init_attr->cap.max_send_sge = resp.max_send_sge; 1333 init_attr->cap.max_recv_sge = resp.max_recv_sge; 1334 init_attr->cap.max_inline_data = resp.max_inline_data; 1335 init_attr->sq_sig_all = resp.sq_sig_all; 1336 1337 return 0; 1338 } 1339 1340 static void copy_modify_qp_fields(struct ibv_qp *qp, struct ibv_qp_attr *attr, 1341 int attr_mask, 1342 struct ibv_modify_qp_common *cmd) 1343 { 1344 cmd->qp_handle = qp->handle; 1345 cmd->attr_mask = attr_mask; 1346 1347 if (attr_mask & IBV_QP_STATE) 1348 cmd->qp_state = attr->qp_state; 1349 if (attr_mask & IBV_QP_CUR_STATE) 1350 cmd->cur_qp_state = attr->cur_qp_state; 1351 if (attr_mask & IBV_QP_EN_SQD_ASYNC_NOTIFY) 1352 cmd->en_sqd_async_notify = attr->en_sqd_async_notify; 1353 if (attr_mask & IBV_QP_ACCESS_FLAGS) 1354 cmd->qp_access_flags = attr->qp_access_flags; 1355 if (attr_mask & IBV_QP_PKEY_INDEX) 1356 cmd->pkey_index = attr->pkey_index; 1357 if (attr_mask & IBV_QP_PORT) 1358 cmd->port_num = attr->port_num; 1359 if (attr_mask & IBV_QP_QKEY) 1360 cmd->qkey = attr->qkey; 1361 1362 if (attr_mask & IBV_QP_AV) { 1363 memcpy(cmd->dest.dgid, attr->ah_attr.grh.dgid.raw, 16); 1364 cmd->dest.flow_label = attr->ah_attr.grh.flow_label; 1365 cmd->dest.dlid = attr->ah_attr.dlid; 1366 cmd->dest.reserved = 0; 1367 cmd->dest.sgid_index = attr->ah_attr.grh.sgid_index; 1368 cmd->dest.hop_limit = attr->ah_attr.grh.hop_limit; 1369 cmd->dest.traffic_class = attr->ah_attr.grh.traffic_class; 1370 cmd->dest.sl = attr->ah_attr.sl; 1371 cmd->dest.src_path_bits = attr->ah_attr.src_path_bits; 1372 cmd->dest.static_rate = attr->ah_attr.static_rate; 1373 cmd->dest.is_global = attr->ah_attr.is_global; 1374 cmd->dest.port_num = attr->ah_attr.port_num; 1375 } 1376 1377 if (attr_mask & IBV_QP_PATH_MTU) 1378 cmd->path_mtu = attr->path_mtu; 1379 if (attr_mask & IBV_QP_TIMEOUT) 1380 cmd->timeout = attr->timeout; 1381 if (attr_mask & IBV_QP_RETRY_CNT) 1382 cmd->retry_cnt = attr->retry_cnt; 1383 if (attr_mask & IBV_QP_RNR_RETRY) 1384 cmd->rnr_retry = attr->rnr_retry; 1385 if (attr_mask & IBV_QP_RQ_PSN) 1386 cmd->rq_psn = attr->rq_psn; 1387 if (attr_mask & IBV_QP_MAX_QP_RD_ATOMIC) 1388 cmd->max_rd_atomic = attr->max_rd_atomic; 1389 1390 if (attr_mask & IBV_QP_ALT_PATH) { 1391 cmd->alt_pkey_index = attr->alt_pkey_index; 1392 cmd->alt_port_num = attr->alt_port_num; 1393 cmd->alt_timeout = attr->alt_timeout; 1394 1395 memcpy(cmd->alt_dest.dgid, attr->alt_ah_attr.grh.dgid.raw, 16); 1396 cmd->alt_dest.flow_label = attr->alt_ah_attr.grh.flow_label; 1397 cmd->alt_dest.dlid = attr->alt_ah_attr.dlid; 1398 cmd->alt_dest.reserved = 0; 1399 cmd->alt_dest.sgid_index = attr->alt_ah_attr.grh.sgid_index; 1400 cmd->alt_dest.hop_limit = attr->alt_ah_attr.grh.hop_limit; 1401 cmd->alt_dest.traffic_class = 1402 attr->alt_ah_attr.grh.traffic_class; 1403 cmd->alt_dest.sl = attr->alt_ah_attr.sl; 1404 cmd->alt_dest.src_path_bits = attr->alt_ah_attr.src_path_bits; 1405 cmd->alt_dest.static_rate = attr->alt_ah_attr.static_rate; 1406 cmd->alt_dest.is_global = attr->alt_ah_attr.is_global; 1407 cmd->alt_dest.port_num = attr->alt_ah_attr.port_num; 1408 } 1409 1410 if (attr_mask & IBV_QP_MIN_RNR_TIMER) 1411 cmd->min_rnr_timer = attr->min_rnr_timer; 1412 if (attr_mask & IBV_QP_SQ_PSN) 1413 cmd->sq_psn = attr->sq_psn; 1414 if (attr_mask & IBV_QP_MAX_DEST_RD_ATOMIC) 1415 cmd->max_dest_rd_atomic = attr->max_dest_rd_atomic; 1416 if (attr_mask & IBV_QP_PATH_MIG_STATE) 1417 cmd->path_mig_state = attr->path_mig_state; 1418 if (attr_mask & IBV_QP_DEST_QPN) 1419 cmd->dest_qp_num = attr->dest_qp_num; 1420 1421 cmd->reserved[0] = cmd->reserved[1] = 0; 1422 } 1423 1424 int ibv_cmd_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, 1425 int attr_mask, 1426 struct ibv_modify_qp *cmd, size_t cmd_size) 1427 { 1428 /* 1429 * Masks over IBV_QP_DEST_QPN are only supported by 1430 * ibv_cmd_modify_qp_ex. 1431 */ 1432 if (attr_mask & ~((IBV_QP_DEST_QPN << 1) - 1)) 1433 return EOPNOTSUPP; 1434 1435 IBV_INIT_CMD(cmd, cmd_size, MODIFY_QP); 1436 1437 copy_modify_qp_fields(qp, attr, attr_mask, &cmd->base); 1438 1439 if (write(qp->context->cmd_fd, cmd, cmd_size) != cmd_size) 1440 return errno; 1441 1442 return 0; 1443 } 1444 1445 int ibv_cmd_modify_qp_ex(struct ibv_qp *qp, struct ibv_qp_attr *attr, 1446 int attr_mask, struct ibv_modify_qp_ex *cmd, 1447 size_t cmd_core_size, size_t cmd_size, 1448 struct ibv_modify_qp_resp_ex *resp, 1449 size_t resp_core_size, size_t resp_size) 1450 { 1451 if (resp_core_size < offsetof(struct ibv_modify_qp_resp_ex, 1452 response_length) + sizeof(resp->response_length)) 1453 return EINVAL; 1454 1455 IBV_INIT_CMD_RESP_EX_V(cmd, cmd_core_size, cmd_size, MODIFY_QP_EX, 1456 resp, resp_core_size, resp_size); 1457 1458 copy_modify_qp_fields(qp, attr, attr_mask, &cmd->base); 1459 1460 if (attr_mask & IBV_QP_RATE_LIMIT) { 1461 if (cmd_size >= offsetof(struct ibv_modify_qp_ex, rate_limit) + 1462 sizeof(cmd->rate_limit)) 1463 cmd->rate_limit = attr->rate_limit; 1464 else 1465 return EINVAL; 1466 } 1467 1468 if (write(qp->context->cmd_fd, cmd, cmd_size) != cmd_size) 1469 return errno; 1470 1471 (void)VALGRIND_MAKE_MEM_DEFINED(resp, resp_size); 1472 1473 return 0; 1474 } 1475 1476 int ibv_cmd_post_send(struct ibv_qp *ibqp, struct ibv_send_wr *wr, 1477 struct ibv_send_wr **bad_wr) 1478 { 1479 struct ibv_post_send *cmd; 1480 struct ibv_post_send_resp resp; 1481 struct ibv_send_wr *i; 1482 struct ibv_kern_send_wr *n, *tmp; 1483 struct ibv_sge *s; 1484 unsigned wr_count = 0; 1485 unsigned sge_count = 0; 1486 int cmd_size; 1487 int ret = 0; 1488 1489 for (i = wr; i; i = i->next) { 1490 wr_count++; 1491 sge_count += i->num_sge; 1492 } 1493 1494 cmd_size = sizeof *cmd + wr_count * sizeof *n + sge_count * sizeof *s; 1495 cmd = alloca(cmd_size); 1496 1497 IBV_INIT_CMD_RESP(cmd, cmd_size, POST_SEND, &resp, sizeof resp); 1498 cmd->qp_handle = ibqp->handle; 1499 cmd->wr_count = wr_count; 1500 cmd->sge_count = sge_count; 1501 cmd->wqe_size = sizeof *n; 1502 1503 n = (struct ibv_kern_send_wr *) ((void *) cmd + sizeof *cmd); 1504 s = (struct ibv_sge *) (n + wr_count); 1505 1506 tmp = n; 1507 for (i = wr; i; i = i->next) { 1508 tmp->wr_id = i->wr_id; 1509 tmp->num_sge = i->num_sge; 1510 tmp->opcode = i->opcode; 1511 tmp->send_flags = i->send_flags; 1512 tmp->imm_data = i->imm_data; 1513 if (ibqp->qp_type == IBV_QPT_UD) { 1514 tmp->wr.ud.ah = i->wr.ud.ah->handle; 1515 tmp->wr.ud.remote_qpn = i->wr.ud.remote_qpn; 1516 tmp->wr.ud.remote_qkey = i->wr.ud.remote_qkey; 1517 } else { 1518 switch (i->opcode) { 1519 case IBV_WR_RDMA_WRITE: 1520 case IBV_WR_RDMA_WRITE_WITH_IMM: 1521 case IBV_WR_RDMA_READ: 1522 tmp->wr.rdma.remote_addr = 1523 i->wr.rdma.remote_addr; 1524 tmp->wr.rdma.rkey = i->wr.rdma.rkey; 1525 break; 1526 case IBV_WR_ATOMIC_CMP_AND_SWP: 1527 case IBV_WR_ATOMIC_FETCH_AND_ADD: 1528 tmp->wr.atomic.remote_addr = 1529 i->wr.atomic.remote_addr; 1530 tmp->wr.atomic.compare_add = 1531 i->wr.atomic.compare_add; 1532 tmp->wr.atomic.swap = i->wr.atomic.swap; 1533 tmp->wr.atomic.rkey = i->wr.atomic.rkey; 1534 break; 1535 default: 1536 break; 1537 } 1538 } 1539 1540 if (tmp->num_sge) { 1541 memcpy(s, i->sg_list, tmp->num_sge * sizeof *s); 1542 s += tmp->num_sge; 1543 } 1544 1545 tmp++; 1546 } 1547 1548 resp.bad_wr = 0; 1549 if (write(ibqp->context->cmd_fd, cmd, cmd_size) != cmd_size) 1550 ret = errno; 1551 1552 (void) VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof resp); 1553 1554 wr_count = resp.bad_wr; 1555 if (wr_count) { 1556 i = wr; 1557 while (--wr_count) 1558 i = i->next; 1559 *bad_wr = i; 1560 } else if (ret) 1561 *bad_wr = wr; 1562 1563 return ret; 1564 } 1565 1566 int ibv_cmd_post_recv(struct ibv_qp *ibqp, struct ibv_recv_wr *wr, 1567 struct ibv_recv_wr **bad_wr) 1568 { 1569 struct ibv_post_recv *cmd; 1570 struct ibv_post_recv_resp resp; 1571 struct ibv_recv_wr *i; 1572 struct ibv_kern_recv_wr *n, *tmp; 1573 struct ibv_sge *s; 1574 unsigned wr_count = 0; 1575 unsigned sge_count = 0; 1576 int cmd_size; 1577 int ret = 0; 1578 1579 for (i = wr; i; i = i->next) { 1580 wr_count++; 1581 sge_count += i->num_sge; 1582 } 1583 1584 cmd_size = sizeof *cmd + wr_count * sizeof *n + sge_count * sizeof *s; 1585 cmd = alloca(cmd_size); 1586 1587 IBV_INIT_CMD_RESP(cmd, cmd_size, POST_RECV, &resp, sizeof resp); 1588 cmd->qp_handle = ibqp->handle; 1589 cmd->wr_count = wr_count; 1590 cmd->sge_count = sge_count; 1591 cmd->wqe_size = sizeof *n; 1592 1593 n = (struct ibv_kern_recv_wr *) ((void *) cmd + sizeof *cmd); 1594 s = (struct ibv_sge *) (n + wr_count); 1595 1596 tmp = n; 1597 for (i = wr; i; i = i->next) { 1598 tmp->wr_id = i->wr_id; 1599 tmp->num_sge = i->num_sge; 1600 1601 if (tmp->num_sge) { 1602 memcpy(s, i->sg_list, tmp->num_sge * sizeof *s); 1603 s += tmp->num_sge; 1604 } 1605 1606 tmp++; 1607 } 1608 1609 resp.bad_wr = 0; 1610 if (write(ibqp->context->cmd_fd, cmd, cmd_size) != cmd_size) 1611 ret = errno; 1612 1613 (void) VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof resp); 1614 1615 wr_count = resp.bad_wr; 1616 if (wr_count) { 1617 i = wr; 1618 while (--wr_count) 1619 i = i->next; 1620 *bad_wr = i; 1621 } else if (ret) 1622 *bad_wr = wr; 1623 1624 return ret; 1625 } 1626 1627 int ibv_cmd_post_srq_recv(struct ibv_srq *srq, struct ibv_recv_wr *wr, 1628 struct ibv_recv_wr **bad_wr) 1629 { 1630 struct ibv_post_srq_recv *cmd; 1631 struct ibv_post_srq_recv_resp resp; 1632 struct ibv_recv_wr *i; 1633 struct ibv_kern_recv_wr *n, *tmp; 1634 struct ibv_sge *s; 1635 unsigned wr_count = 0; 1636 unsigned sge_count = 0; 1637 int cmd_size; 1638 int ret = 0; 1639 1640 for (i = wr; i; i = i->next) { 1641 wr_count++; 1642 sge_count += i->num_sge; 1643 } 1644 1645 cmd_size = sizeof *cmd + wr_count * sizeof *n + sge_count * sizeof *s; 1646 cmd = alloca(cmd_size); 1647 1648 IBV_INIT_CMD_RESP(cmd, cmd_size, POST_SRQ_RECV, &resp, sizeof resp); 1649 cmd->srq_handle = srq->handle; 1650 cmd->wr_count = wr_count; 1651 cmd->sge_count = sge_count; 1652 cmd->wqe_size = sizeof *n; 1653 1654 n = (struct ibv_kern_recv_wr *) ((void *) cmd + sizeof *cmd); 1655 s = (struct ibv_sge *) (n + wr_count); 1656 1657 tmp = n; 1658 for (i = wr; i; i = i->next) { 1659 tmp->wr_id = i->wr_id; 1660 tmp->num_sge = i->num_sge; 1661 1662 if (tmp->num_sge) { 1663 memcpy(s, i->sg_list, tmp->num_sge * sizeof *s); 1664 s += tmp->num_sge; 1665 } 1666 1667 tmp++; 1668 } 1669 1670 resp.bad_wr = 0; 1671 if (write(srq->context->cmd_fd, cmd, cmd_size) != cmd_size) 1672 ret = errno; 1673 1674 (void) VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof resp); 1675 1676 wr_count = resp.bad_wr; 1677 if (wr_count) { 1678 i = wr; 1679 while (--wr_count) 1680 i = i->next; 1681 *bad_wr = i; 1682 } else if (ret) 1683 *bad_wr = wr; 1684 1685 return ret; 1686 } 1687 1688 int ibv_cmd_create_ah(struct ibv_pd *pd, struct ibv_ah *ah, 1689 struct ibv_ah_attr *attr, 1690 struct ibv_create_ah_resp *resp, 1691 size_t resp_size) 1692 { 1693 struct ibv_create_ah cmd; 1694 1695 IBV_INIT_CMD_RESP(&cmd, sizeof cmd, CREATE_AH, resp, resp_size); 1696 cmd.user_handle = (uintptr_t) ah; 1697 cmd.pd_handle = pd->handle; 1698 cmd.attr.dlid = attr->dlid; 1699 cmd.attr.sl = attr->sl; 1700 cmd.attr.src_path_bits = attr->src_path_bits; 1701 cmd.attr.static_rate = attr->static_rate; 1702 cmd.attr.is_global = attr->is_global; 1703 cmd.attr.port_num = attr->port_num; 1704 cmd.attr.grh.flow_label = attr->grh.flow_label; 1705 cmd.attr.grh.sgid_index = attr->grh.sgid_index; 1706 cmd.attr.grh.hop_limit = attr->grh.hop_limit; 1707 cmd.attr.grh.traffic_class = attr->grh.traffic_class; 1708 memcpy(cmd.attr.grh.dgid, attr->grh.dgid.raw, 16); 1709 1710 if (write(pd->context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd) 1711 return errno; 1712 1713 (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size); 1714 1715 ah->handle = resp->handle; 1716 ah->context = pd->context; 1717 1718 return 0; 1719 } 1720 1721 int ibv_cmd_destroy_ah(struct ibv_ah *ah) 1722 { 1723 struct ibv_destroy_ah cmd; 1724 1725 IBV_INIT_CMD(&cmd, sizeof cmd, DESTROY_AH); 1726 cmd.ah_handle = ah->handle; 1727 1728 if (write(ah->context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd) 1729 return errno; 1730 1731 return 0; 1732 } 1733 1734 int ibv_cmd_destroy_qp(struct ibv_qp *qp) 1735 { 1736 struct ibv_destroy_qp cmd; 1737 struct ibv_destroy_qp_resp resp; 1738 1739 IBV_INIT_CMD_RESP(&cmd, sizeof cmd, DESTROY_QP, &resp, sizeof resp); 1740 cmd.qp_handle = qp->handle; 1741 cmd.reserved = 0; 1742 1743 if (write(qp->context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd) 1744 return errno; 1745 1746 (void) VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof resp); 1747 1748 pthread_mutex_lock(&qp->mutex); 1749 while (qp->events_completed != resp.events_reported) 1750 pthread_cond_wait(&qp->cond, &qp->mutex); 1751 pthread_mutex_unlock(&qp->mutex); 1752 1753 pthread_cond_destroy(&qp->cond); 1754 pthread_mutex_destroy(&qp->mutex); 1755 1756 return 0; 1757 } 1758 1759 int ibv_cmd_attach_mcast(struct ibv_qp *qp, const union ibv_gid *gid, uint16_t lid) 1760 { 1761 struct ibv_attach_mcast cmd; 1762 1763 IBV_INIT_CMD(&cmd, sizeof cmd, ATTACH_MCAST); 1764 memcpy(cmd.gid, gid->raw, sizeof cmd.gid); 1765 cmd.qp_handle = qp->handle; 1766 cmd.mlid = lid; 1767 cmd.reserved = 0; 1768 1769 if (write(qp->context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd) 1770 return errno; 1771 1772 return 0; 1773 } 1774 1775 int ibv_cmd_detach_mcast(struct ibv_qp *qp, const union ibv_gid *gid, uint16_t lid) 1776 { 1777 struct ibv_detach_mcast cmd; 1778 1779 IBV_INIT_CMD(&cmd, sizeof cmd, DETACH_MCAST); 1780 memcpy(cmd.gid, gid->raw, sizeof cmd.gid); 1781 cmd.qp_handle = qp->handle; 1782 cmd.mlid = lid; 1783 cmd.reserved = 0; 1784 1785 if (write(qp->context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd) 1786 return errno; 1787 1788 return 0; 1789 } 1790 1791 static int buffer_is_zero(char *addr, ssize_t size) 1792 { 1793 return addr[0] == 0 && !memcmp(addr, addr + 1, size - 1); 1794 } 1795 1796 static int get_filters_size(struct ibv_flow_spec *ib_spec, 1797 struct ibv_kern_spec *kern_spec, 1798 int *ib_filter_size, int *kern_filter_size, 1799 enum ibv_flow_spec_type type) 1800 { 1801 void *ib_spec_filter_mask; 1802 int curr_kern_filter_size; 1803 int min_filter_size; 1804 1805 *ib_filter_size = (ib_spec->hdr.size - sizeof(ib_spec->hdr)) / 2; 1806 1807 switch (type) { 1808 case IBV_FLOW_SPEC_IPV4_EXT: 1809 min_filter_size = 1810 offsetof(struct ibv_kern_ipv4_ext_filter, flags) + 1811 sizeof(kern_spec->ipv4_ext.mask.flags); 1812 curr_kern_filter_size = min_filter_size; 1813 ib_spec_filter_mask = (void *)&ib_spec->ipv4_ext.val + 1814 *ib_filter_size; 1815 break; 1816 case IBV_FLOW_SPEC_IPV6: 1817 min_filter_size = 1818 offsetof(struct ibv_kern_ipv6_filter, hop_limit) + 1819 sizeof(kern_spec->ipv6.mask.hop_limit); 1820 curr_kern_filter_size = min_filter_size; 1821 ib_spec_filter_mask = (void *)&ib_spec->ipv6.val + 1822 *ib_filter_size; 1823 break; 1824 case IBV_FLOW_SPEC_VXLAN_TUNNEL: 1825 min_filter_size = 1826 offsetof(struct ibv_kern_tunnel_filter, 1827 tunnel_id) + 1828 sizeof(kern_spec->tunnel.mask.tunnel_id); 1829 curr_kern_filter_size = min_filter_size; 1830 ib_spec_filter_mask = (void *)&ib_spec->tunnel.val + 1831 *ib_filter_size; 1832 break; 1833 default: 1834 return EINVAL; 1835 } 1836 1837 if (*ib_filter_size < min_filter_size) 1838 return EINVAL; 1839 1840 if (*ib_filter_size > curr_kern_filter_size && 1841 !buffer_is_zero(ib_spec_filter_mask + curr_kern_filter_size, 1842 *ib_filter_size - curr_kern_filter_size)) 1843 return EOPNOTSUPP; 1844 1845 *kern_filter_size = min_t(int, curr_kern_filter_size, *ib_filter_size); 1846 1847 return 0; 1848 } 1849 1850 static int ib_spec_to_kern_spec(struct ibv_flow_spec *ib_spec, 1851 struct ibv_kern_spec *kern_spec) 1852 { 1853 int kern_filter_size; 1854 int ib_filter_size; 1855 int ret; 1856 1857 kern_spec->hdr.type = ib_spec->hdr.type; 1858 1859 switch (kern_spec->hdr.type) { 1860 case IBV_FLOW_SPEC_ETH: 1861 case IBV_FLOW_SPEC_ETH | IBV_FLOW_SPEC_INNER: 1862 kern_spec->eth.size = sizeof(struct ibv_kern_spec_eth); 1863 memcpy(&kern_spec->eth.val, &ib_spec->eth.val, 1864 sizeof(struct ibv_flow_eth_filter)); 1865 memcpy(&kern_spec->eth.mask, &ib_spec->eth.mask, 1866 sizeof(struct ibv_flow_eth_filter)); 1867 break; 1868 case IBV_FLOW_SPEC_IPV4: 1869 case IBV_FLOW_SPEC_IPV4 | IBV_FLOW_SPEC_INNER: 1870 kern_spec->ipv4.size = sizeof(struct ibv_kern_spec_ipv4); 1871 memcpy(&kern_spec->ipv4.val, &ib_spec->ipv4.val, 1872 sizeof(struct ibv_flow_ipv4_filter)); 1873 memcpy(&kern_spec->ipv4.mask, &ib_spec->ipv4.mask, 1874 sizeof(struct ibv_flow_ipv4_filter)); 1875 break; 1876 case IBV_FLOW_SPEC_IPV4_EXT: 1877 case IBV_FLOW_SPEC_IPV4_EXT | IBV_FLOW_SPEC_INNER: 1878 ret = get_filters_size(ib_spec, kern_spec, 1879 &ib_filter_size, &kern_filter_size, 1880 IBV_FLOW_SPEC_IPV4_EXT); 1881 if (ret) 1882 return ret; 1883 1884 kern_spec->hdr.type = IBV_FLOW_SPEC_IPV4 | 1885 (IBV_FLOW_SPEC_INNER & ib_spec->hdr.type); 1886 kern_spec->ipv4_ext.size = sizeof(struct 1887 ibv_kern_spec_ipv4_ext); 1888 memcpy(&kern_spec->ipv4_ext.val, &ib_spec->ipv4_ext.val, 1889 kern_filter_size); 1890 memcpy(&kern_spec->ipv4_ext.mask, (void *)&ib_spec->ipv4_ext.val 1891 + ib_filter_size, kern_filter_size); 1892 break; 1893 case IBV_FLOW_SPEC_IPV6: 1894 case IBV_FLOW_SPEC_IPV6 | IBV_FLOW_SPEC_INNER: 1895 ret = get_filters_size(ib_spec, kern_spec, 1896 &ib_filter_size, &kern_filter_size, 1897 IBV_FLOW_SPEC_IPV6); 1898 if (ret) 1899 return ret; 1900 1901 kern_spec->ipv6.size = sizeof(struct ibv_kern_spec_ipv6); 1902 memcpy(&kern_spec->ipv6.val, &ib_spec->ipv6.val, 1903 kern_filter_size); 1904 memcpy(&kern_spec->ipv6.mask, (void *)&ib_spec->ipv6.val 1905 + ib_filter_size, kern_filter_size); 1906 break; 1907 case IBV_FLOW_SPEC_TCP: 1908 case IBV_FLOW_SPEC_UDP: 1909 case IBV_FLOW_SPEC_TCP | IBV_FLOW_SPEC_INNER: 1910 case IBV_FLOW_SPEC_UDP | IBV_FLOW_SPEC_INNER: 1911 kern_spec->tcp_udp.size = sizeof(struct ibv_kern_spec_tcp_udp); 1912 memcpy(&kern_spec->tcp_udp.val, &ib_spec->tcp_udp.val, 1913 sizeof(struct ibv_flow_tcp_udp_filter)); 1914 memcpy(&kern_spec->tcp_udp.mask, &ib_spec->tcp_udp.mask, 1915 sizeof(struct ibv_flow_tcp_udp_filter)); 1916 break; 1917 case IBV_FLOW_SPEC_GRE: 1918 kern_spec->gre.size = sizeof(struct ib_uverbs_flow_spec_gre); 1919 memcpy(&kern_spec->gre.val, &ib_spec->gre.val, 1920 sizeof(struct ibv_flow_gre_filter)); 1921 memcpy(&kern_spec->gre.mask, &ib_spec->gre.mask, 1922 sizeof(struct ibv_flow_gre_filter)); 1923 break; 1924 case IBV_FLOW_SPEC_MPLS: 1925 case IBV_FLOW_SPEC_MPLS | IBV_FLOW_SPEC_INNER: 1926 kern_spec->mpls.size = sizeof(struct ib_uverbs_flow_spec_mpls); 1927 memcpy(&kern_spec->mpls.val, &ib_spec->mpls.val, 1928 sizeof(struct ibv_flow_mpls_filter)); 1929 memcpy(&kern_spec->mpls.mask, &ib_spec->mpls.mask, 1930 sizeof(struct ibv_flow_mpls_filter)); 1931 break; 1932 case IBV_FLOW_SPEC_VXLAN_TUNNEL: 1933 ret = get_filters_size(ib_spec, kern_spec, 1934 &ib_filter_size, &kern_filter_size, 1935 IBV_FLOW_SPEC_VXLAN_TUNNEL); 1936 if (ret) 1937 return ret; 1938 1939 kern_spec->tunnel.size = sizeof(struct ibv_kern_spec_tunnel); 1940 memcpy(&kern_spec->tunnel.val, &ib_spec->tunnel.val, 1941 kern_filter_size); 1942 memcpy(&kern_spec->tunnel.mask, (void *)&ib_spec->tunnel.val 1943 + ib_filter_size, kern_filter_size); 1944 break; 1945 case IBV_FLOW_SPEC_ACTION_TAG: 1946 kern_spec->flow_tag.size = 1947 sizeof(struct ibv_kern_spec_action_tag); 1948 kern_spec->flow_tag.tag_id = ib_spec->flow_tag.tag_id; 1949 break; 1950 case IBV_FLOW_SPEC_ACTION_DROP: 1951 kern_spec->drop.size = sizeof(struct ibv_kern_spec_action_drop); 1952 break; 1953 default: 1954 return EINVAL; 1955 } 1956 return 0; 1957 } 1958 1959 struct ibv_flow *ibv_cmd_create_flow(struct ibv_qp *qp, 1960 struct ibv_flow_attr *flow_attr) 1961 { 1962 struct ibv_create_flow *cmd; 1963 struct ibv_create_flow_resp resp; 1964 struct ibv_flow *flow_id; 1965 size_t cmd_size; 1966 size_t written_size; 1967 int i, err; 1968 void *kern_spec; 1969 void *ib_spec; 1970 1971 cmd_size = sizeof(*cmd) + (flow_attr->num_of_specs * 1972 sizeof(struct ibv_kern_spec)); 1973 cmd = alloca(cmd_size); 1974 flow_id = malloc(sizeof(*flow_id)); 1975 if (!flow_id) 1976 return NULL; 1977 memset(cmd, 0, cmd_size); 1978 1979 cmd->qp_handle = qp->handle; 1980 1981 cmd->flow_attr.type = flow_attr->type; 1982 cmd->flow_attr.priority = flow_attr->priority; 1983 cmd->flow_attr.num_of_specs = flow_attr->num_of_specs; 1984 cmd->flow_attr.port = flow_attr->port; 1985 cmd->flow_attr.flags = flow_attr->flags; 1986 1987 kern_spec = cmd + 1; 1988 ib_spec = flow_attr + 1; 1989 for (i = 0; i < flow_attr->num_of_specs; i++) { 1990 err = ib_spec_to_kern_spec(ib_spec, kern_spec); 1991 if (err) { 1992 errno = err; 1993 goto err; 1994 } 1995 cmd->flow_attr.size += 1996 ((struct ibv_kern_spec *)kern_spec)->hdr.size; 1997 kern_spec += ((struct ibv_kern_spec *)kern_spec)->hdr.size; 1998 ib_spec += ((struct ibv_flow_spec *)ib_spec)->hdr.size; 1999 } 2000 2001 written_size = sizeof(*cmd) + cmd->flow_attr.size; 2002 IBV_INIT_CMD_RESP_EX_VCMD(cmd, written_size, written_size, CREATE_FLOW, 2003 &resp, sizeof(resp)); 2004 if (write(qp->context->cmd_fd, cmd, written_size) != written_size) 2005 goto err; 2006 2007 (void) VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof(resp)); 2008 2009 flow_id->context = qp->context; 2010 flow_id->handle = resp.flow_handle; 2011 return flow_id; 2012 err: 2013 free(flow_id); 2014 return NULL; 2015 } 2016 2017 int ibv_cmd_destroy_flow(struct ibv_flow *flow_id) 2018 { 2019 struct ibv_destroy_flow cmd; 2020 int ret = 0; 2021 2022 memset(&cmd, 0, sizeof(cmd)); 2023 IBV_INIT_CMD_EX(&cmd, sizeof(cmd), DESTROY_FLOW); 2024 cmd.flow_handle = flow_id->handle; 2025 2026 if (write(flow_id->context->cmd_fd, &cmd, sizeof(cmd)) != sizeof(cmd)) 2027 ret = errno; 2028 free(flow_id); 2029 return ret; 2030 } 2031 2032 int ibv_cmd_create_wq(struct ibv_context *context, 2033 struct ibv_wq_init_attr *wq_init_attr, 2034 struct ibv_wq *wq, 2035 struct ibv_create_wq *cmd, 2036 size_t cmd_core_size, 2037 size_t cmd_size, 2038 struct ibv_create_wq_resp *resp, 2039 size_t resp_core_size, 2040 size_t resp_size) 2041 { 2042 int err; 2043 2044 if (wq_init_attr->comp_mask >= IBV_WQ_INIT_ATTR_RESERVED) 2045 return EINVAL; 2046 2047 IBV_INIT_CMD_RESP_EX_V(cmd, cmd_core_size, cmd_size, 2048 CREATE_WQ, resp, 2049 resp_core_size, resp_size); 2050 2051 cmd->user_handle = (uintptr_t)wq; 2052 cmd->pd_handle = wq_init_attr->pd->handle; 2053 cmd->cq_handle = wq_init_attr->cq->handle; 2054 cmd->wq_type = wq_init_attr->wq_type; 2055 cmd->max_sge = wq_init_attr->max_sge; 2056 cmd->max_wr = wq_init_attr->max_wr; 2057 cmd->comp_mask = 0; 2058 2059 if (cmd_core_size >= offsetof(struct ibv_create_wq, create_flags) + 2060 sizeof(cmd->create_flags)) { 2061 if (wq_init_attr->comp_mask & IBV_WQ_INIT_ATTR_FLAGS) { 2062 if (wq_init_attr->create_flags & ~(IBV_WQ_FLAGS_RESERVED - 1)) 2063 return EOPNOTSUPP; 2064 cmd->create_flags = wq_init_attr->create_flags; 2065 } 2066 } 2067 2068 err = write(context->cmd_fd, cmd, cmd_size); 2069 if (err != cmd_size) 2070 return errno; 2071 2072 (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size); 2073 2074 if (resp->response_length < resp_core_size) 2075 return EINVAL; 2076 2077 wq->handle = resp->wq_handle; 2078 wq_init_attr->max_wr = resp->max_wr; 2079 wq_init_attr->max_sge = resp->max_sge; 2080 wq->wq_num = resp->wqn; 2081 wq->context = context; 2082 wq->cq = wq_init_attr->cq; 2083 wq->pd = wq_init_attr->pd; 2084 wq->wq_type = wq_init_attr->wq_type; 2085 2086 return 0; 2087 } 2088 2089 int ibv_cmd_modify_wq(struct ibv_wq *wq, struct ibv_wq_attr *attr, 2090 struct ibv_modify_wq *cmd, size_t cmd_core_size, 2091 size_t cmd_size) 2092 { 2093 if (attr->attr_mask >= IBV_WQ_ATTR_RESERVED) 2094 return EINVAL; 2095 2096 memset(cmd, 0, cmd_core_size); 2097 IBV_INIT_CMD_EX(cmd, cmd_size, MODIFY_WQ); 2098 2099 cmd->curr_wq_state = attr->curr_wq_state; 2100 cmd->wq_state = attr->wq_state; 2101 if (cmd_core_size >= offsetof(struct ibv_modify_wq, flags_mask) + 2102 sizeof(cmd->flags_mask)) { 2103 if (attr->attr_mask & IBV_WQ_ATTR_FLAGS) { 2104 if (attr->flags_mask & ~(IBV_WQ_FLAGS_RESERVED - 1)) 2105 return EOPNOTSUPP; 2106 cmd->flags = attr->flags; 2107 cmd->flags_mask = attr->flags_mask; 2108 } 2109 } 2110 cmd->wq_handle = wq->handle; 2111 cmd->attr_mask = attr->attr_mask; 2112 2113 if (write(wq->context->cmd_fd, cmd, cmd_size) != cmd_size) 2114 return errno; 2115 2116 if (attr->attr_mask & IBV_WQ_ATTR_STATE) 2117 wq->state = attr->wq_state; 2118 2119 return 0; 2120 } 2121 2122 int ibv_cmd_destroy_wq(struct ibv_wq *wq) 2123 { 2124 struct ibv_destroy_wq cmd; 2125 struct ibv_destroy_wq_resp resp; 2126 int ret = 0; 2127 2128 memset(&cmd, 0, sizeof(cmd)); 2129 memset(&resp, 0, sizeof(resp)); 2130 2131 IBV_INIT_CMD_RESP_EX(&cmd, sizeof(cmd), DESTROY_WQ, &resp, sizeof(resp)); 2132 cmd.wq_handle = wq->handle; 2133 2134 if (write(wq->context->cmd_fd, &cmd, sizeof(cmd)) != sizeof(cmd)) 2135 return errno; 2136 2137 if (resp.response_length < sizeof(resp)) 2138 return EINVAL; 2139 2140 pthread_mutex_lock(&wq->mutex); 2141 while (wq->events_completed != resp.events_reported) 2142 pthread_cond_wait(&wq->cond, &wq->mutex); 2143 pthread_mutex_unlock(&wq->mutex); 2144 2145 return ret; 2146 } 2147 2148 int ibv_cmd_create_rwq_ind_table(struct ibv_context *context, 2149 struct ibv_rwq_ind_table_init_attr *init_attr, 2150 struct ibv_rwq_ind_table *rwq_ind_table, 2151 struct ibv_create_rwq_ind_table *cmd, 2152 size_t cmd_core_size, 2153 size_t cmd_size, 2154 struct ibv_create_rwq_ind_table_resp *resp, 2155 size_t resp_core_size, 2156 size_t resp_size) 2157 { 2158 int err, i; 2159 uint32_t required_tbl_size, alloc_tbl_size; 2160 uint32_t *tbl_start; 2161 int num_tbl_entries; 2162 2163 if (init_attr->comp_mask >= IBV_CREATE_IND_TABLE_RESERVED) 2164 return EINVAL; 2165 2166 alloc_tbl_size = cmd_core_size - sizeof(*cmd); 2167 num_tbl_entries = 1 << init_attr->log_ind_tbl_size; 2168 2169 /* Data must be u64 aligned */ 2170 required_tbl_size = (num_tbl_entries * sizeof(uint32_t)) < sizeof(uint64_t) ? 2171 sizeof(uint64_t) : (num_tbl_entries * sizeof(uint32_t)); 2172 2173 if (alloc_tbl_size < required_tbl_size) 2174 return EINVAL; 2175 2176 tbl_start = (uint32_t *)((uint8_t *)cmd + sizeof(*cmd)); 2177 for (i = 0; i < num_tbl_entries; i++) 2178 tbl_start[i] = init_attr->ind_tbl[i]->handle; 2179 2180 IBV_INIT_CMD_RESP_EX_V(cmd, cmd_core_size, cmd_size, 2181 CREATE_RWQ_IND_TBL, resp, 2182 resp_core_size, resp_size); 2183 cmd->log_ind_tbl_size = init_attr->log_ind_tbl_size; 2184 cmd->comp_mask = 0; 2185 2186 err = write(context->cmd_fd, cmd, cmd_size); 2187 if (err != cmd_size) 2188 return errno; 2189 2190 (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size); 2191 2192 if (resp->response_length < resp_core_size) 2193 return EINVAL; 2194 2195 rwq_ind_table->ind_tbl_handle = resp->ind_tbl_handle; 2196 rwq_ind_table->ind_tbl_num = resp->ind_tbl_num; 2197 rwq_ind_table->context = context; 2198 return 0; 2199 } 2200 2201 int ibv_cmd_destroy_rwq_ind_table(struct ibv_rwq_ind_table *rwq_ind_table) 2202 { 2203 struct ibv_destroy_rwq_ind_table cmd; 2204 int ret = 0; 2205 2206 memset(&cmd, 0, sizeof(cmd)); 2207 IBV_INIT_CMD_EX(&cmd, sizeof(cmd), DESTROY_RWQ_IND_TBL); 2208 cmd.ind_tbl_handle = rwq_ind_table->ind_tbl_handle; 2209 2210 if (write(rwq_ind_table->context->cmd_fd, &cmd, sizeof(cmd)) != sizeof(cmd)) 2211 ret = errno; 2212 2213 return ret; 2214 } 2215