1 /* 2 * Copyright (c) 2005 Topspin Communications. All rights reserved. 3 * Copyright (c) 2005 PathScale, Inc. All rights reserved. 4 * Copyright (c) 2006 Cisco Systems, Inc. All rights reserved. 5 * 6 * This software is available to you under a choice of one of two 7 * licenses. You may choose to be licensed under the terms of the GNU 8 * General Public License (GPL) Version 2, available from the file 9 * COPYING in the main directory of this source tree, or the 10 * OpenIB.org BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or 13 * without modification, are permitted provided that the following 14 * conditions are met: 15 * 16 * - Redistributions of source code must retain the above 17 * copyright notice, this list of conditions and the following 18 * disclaimer. 19 * 20 * - Redistributions in binary form must reproduce the above 21 * copyright notice, this list of conditions and the following 22 * disclaimer in the documentation and/or other materials 23 * provided with the distribution. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32 * SOFTWARE. 33 */ 34 35 #include <config.h> 36 37 #include <stdio.h> 38 #include <unistd.h> 39 #include <stdlib.h> 40 #include <errno.h> 41 #include <string.h> 42 43 #include "ibverbs.h" 44 #include <sys/param.h> 45 46 int ibv_cmd_get_context(struct ibv_context *context, struct ibv_get_context *cmd, 47 size_t cmd_size, struct ibv_get_context_resp *resp, 48 size_t resp_size) 49 { 50 if (abi_ver < IB_USER_VERBS_MIN_ABI_VERSION) 51 return ENOSYS; 52 53 IBV_INIT_CMD_RESP(cmd, cmd_size, GET_CONTEXT, resp, resp_size); 54 55 if (write(context->cmd_fd, cmd, cmd_size) != cmd_size) 56 return errno; 57 58 (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size); 59 60 context->async_fd = resp->async_fd; 61 context->num_comp_vectors = resp->num_comp_vectors; 62 63 return 0; 64 } 65 66 static void copy_query_dev_fields(struct ibv_device_attr *device_attr, 67 struct ibv_query_device_resp *resp, 68 uint64_t *raw_fw_ver) 69 { 70 *raw_fw_ver = resp->fw_ver; 71 device_attr->node_guid = resp->node_guid; 72 device_attr->sys_image_guid = resp->sys_image_guid; 73 device_attr->max_mr_size = resp->max_mr_size; 74 device_attr->page_size_cap = resp->page_size_cap; 75 device_attr->vendor_id = resp->vendor_id; 76 device_attr->vendor_part_id = resp->vendor_part_id; 77 device_attr->hw_ver = resp->hw_ver; 78 device_attr->max_qp = resp->max_qp; 79 device_attr->max_qp_wr = resp->max_qp_wr; 80 device_attr->device_cap_flags = resp->device_cap_flags; 81 device_attr->max_sge = resp->max_sge; 82 device_attr->max_sge_rd = resp->max_sge_rd; 83 device_attr->max_cq = resp->max_cq; 84 device_attr->max_cqe = resp->max_cqe; 85 device_attr->max_mr = resp->max_mr; 86 device_attr->max_pd = resp->max_pd; 87 device_attr->max_qp_rd_atom = resp->max_qp_rd_atom; 88 device_attr->max_ee_rd_atom = resp->max_ee_rd_atom; 89 device_attr->max_res_rd_atom = resp->max_res_rd_atom; 90 device_attr->max_qp_init_rd_atom = resp->max_qp_init_rd_atom; 91 device_attr->max_ee_init_rd_atom = resp->max_ee_init_rd_atom; 92 device_attr->atomic_cap = resp->atomic_cap; 93 device_attr->max_ee = resp->max_ee; 94 device_attr->max_rdd = resp->max_rdd; 95 device_attr->max_mw = resp->max_mw; 96 device_attr->max_raw_ipv6_qp = resp->max_raw_ipv6_qp; 97 device_attr->max_raw_ethy_qp = resp->max_raw_ethy_qp; 98 device_attr->max_mcast_grp = resp->max_mcast_grp; 99 device_attr->max_mcast_qp_attach = resp->max_mcast_qp_attach; 100 device_attr->max_total_mcast_qp_attach = resp->max_total_mcast_qp_attach; 101 device_attr->max_ah = resp->max_ah; 102 device_attr->max_fmr = resp->max_fmr; 103 device_attr->max_map_per_fmr = resp->max_map_per_fmr; 104 device_attr->max_srq = resp->max_srq; 105 device_attr->max_srq_wr = resp->max_srq_wr; 106 device_attr->max_srq_sge = resp->max_srq_sge; 107 device_attr->max_pkeys = resp->max_pkeys; 108 device_attr->local_ca_ack_delay = resp->local_ca_ack_delay; 109 device_attr->phys_port_cnt = resp->phys_port_cnt; 110 } 111 112 int ibv_cmd_query_device(struct ibv_context *context, 113 struct ibv_device_attr *device_attr, 114 uint64_t *raw_fw_ver, 115 struct ibv_query_device *cmd, size_t cmd_size) 116 { 117 struct ibv_query_device_resp resp; 118 119 IBV_INIT_CMD_RESP(cmd, cmd_size, QUERY_DEVICE, &resp, sizeof resp); 120 121 if (write(context->cmd_fd, cmd, cmd_size) != cmd_size) 122 return errno; 123 124 (void) VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof resp); 125 126 memset(device_attr->fw_ver, 0, sizeof device_attr->fw_ver); 127 copy_query_dev_fields(device_attr, &resp, raw_fw_ver); 128 129 return 0; 130 } 131 132 int ibv_cmd_query_device_ex(struct ibv_context *context, 133 const struct ibv_query_device_ex_input *input, 134 struct ibv_device_attr_ex *attr, size_t attr_size, 135 uint64_t *raw_fw_ver, 136 struct ibv_query_device_ex *cmd, 137 size_t cmd_core_size, 138 size_t cmd_size, 139 struct ibv_query_device_resp_ex *resp, 140 size_t resp_core_size, 141 size_t resp_size) 142 { 143 int err; 144 145 if (input && input->comp_mask) 146 return EINVAL; 147 148 if (attr_size < offsetof(struct ibv_device_attr_ex, comp_mask) + 149 sizeof(attr->comp_mask)) 150 return EINVAL; 151 152 if (resp_core_size < offsetof(struct ibv_query_device_resp_ex, 153 response_length) + 154 sizeof(resp->response_length)) 155 return EINVAL; 156 157 IBV_INIT_CMD_RESP_EX_V(cmd, cmd_core_size, cmd_size, 158 QUERY_DEVICE_EX, resp, resp_core_size, 159 resp_size); 160 cmd->comp_mask = 0; 161 cmd->reserved = 0; 162 memset(attr->orig_attr.fw_ver, 0, sizeof(attr->orig_attr.fw_ver)); 163 memset(&attr->comp_mask, 0, attr_size - sizeof(attr->orig_attr)); 164 err = write(context->cmd_fd, cmd, cmd_size); 165 if (err != cmd_size) 166 return errno; 167 168 (void)VALGRIND_MAKE_MEM_DEFINED(resp, resp_size); 169 copy_query_dev_fields(&attr->orig_attr, &resp->base, raw_fw_ver); 170 /* Report back supported comp_mask bits. For now no comp_mask bit is 171 * defined */ 172 attr->comp_mask = resp->comp_mask & 0; 173 if (attr_size >= offsetof(struct ibv_device_attr_ex, odp_caps) + 174 sizeof(attr->odp_caps)) { 175 if (resp->response_length >= 176 offsetof(struct ibv_query_device_resp_ex, odp_caps) + 177 sizeof(resp->odp_caps)) { 178 attr->odp_caps.general_caps = resp->odp_caps.general_caps; 179 attr->odp_caps.per_transport_caps.rc_odp_caps = 180 resp->odp_caps.per_transport_caps.rc_odp_caps; 181 attr->odp_caps.per_transport_caps.uc_odp_caps = 182 resp->odp_caps.per_transport_caps.uc_odp_caps; 183 attr->odp_caps.per_transport_caps.ud_odp_caps = 184 resp->odp_caps.per_transport_caps.ud_odp_caps; 185 } 186 } 187 188 if (attr_size >= offsetof(struct ibv_device_attr_ex, 189 completion_timestamp_mask) + 190 sizeof(attr->completion_timestamp_mask)) { 191 if (resp->response_length >= 192 offsetof(struct ibv_query_device_resp_ex, timestamp_mask) + 193 sizeof(resp->timestamp_mask)) 194 attr->completion_timestamp_mask = resp->timestamp_mask; 195 } 196 197 if (attr_size >= offsetof(struct ibv_device_attr_ex, hca_core_clock) + 198 sizeof(attr->hca_core_clock)) { 199 if (resp->response_length >= 200 offsetof(struct ibv_query_device_resp_ex, hca_core_clock) + 201 sizeof(resp->hca_core_clock)) 202 attr->hca_core_clock = resp->hca_core_clock; 203 } 204 205 if (attr_size >= offsetof(struct ibv_device_attr_ex, device_cap_flags_ex) + 206 sizeof(attr->device_cap_flags_ex)) { 207 if (resp->response_length >= 208 offsetof(struct ibv_query_device_resp_ex, device_cap_flags_ex) + 209 sizeof(resp->device_cap_flags_ex)) 210 attr->device_cap_flags_ex = resp->device_cap_flags_ex; 211 } 212 213 if (attr_size >= offsetof(struct ibv_device_attr_ex, rss_caps) + 214 sizeof(attr->rss_caps)) { 215 if (resp->response_length >= 216 offsetof(struct ibv_query_device_resp_ex, rss_caps) + 217 sizeof(resp->rss_caps)) { 218 attr->rss_caps.supported_qpts = resp->rss_caps.supported_qpts; 219 attr->rss_caps.max_rwq_indirection_tables = resp->rss_caps.max_rwq_indirection_tables; 220 attr->rss_caps.max_rwq_indirection_table_size = resp->rss_caps.max_rwq_indirection_table_size; 221 } 222 } 223 224 if (attr_size >= offsetof(struct ibv_device_attr_ex, max_wq_type_rq) + 225 sizeof(attr->max_wq_type_rq)) { 226 if (resp->response_length >= 227 offsetof(struct ibv_query_device_resp_ex, max_wq_type_rq) + 228 sizeof(resp->max_wq_type_rq)) 229 attr->max_wq_type_rq = resp->max_wq_type_rq; 230 } 231 232 if (attr_size >= offsetof(struct ibv_device_attr_ex, raw_packet_caps) + 233 sizeof(attr->raw_packet_caps)) { 234 if (resp->response_length >= 235 offsetof(struct ibv_query_device_resp_ex, raw_packet_caps) + 236 sizeof(resp->raw_packet_caps)) 237 attr->raw_packet_caps = resp->raw_packet_caps; 238 } 239 240 return 0; 241 } 242 243 int ibv_cmd_query_port(struct ibv_context *context, uint8_t port_num, 244 struct ibv_port_attr *port_attr, 245 struct ibv_query_port *cmd, size_t cmd_size) 246 { 247 struct ibv_query_port_resp resp; 248 249 IBV_INIT_CMD_RESP(cmd, cmd_size, QUERY_PORT, &resp, sizeof resp); 250 cmd->port_num = port_num; 251 memset(cmd->reserved, 0, sizeof cmd->reserved); 252 253 if (write(context->cmd_fd, cmd, cmd_size) != cmd_size) 254 return errno; 255 256 (void) VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof resp); 257 258 port_attr->state = resp.state; 259 port_attr->max_mtu = resp.max_mtu; 260 port_attr->active_mtu = resp.active_mtu; 261 port_attr->gid_tbl_len = resp.gid_tbl_len; 262 port_attr->port_cap_flags = resp.port_cap_flags; 263 port_attr->max_msg_sz = resp.max_msg_sz; 264 port_attr->bad_pkey_cntr = resp.bad_pkey_cntr; 265 port_attr->qkey_viol_cntr = resp.qkey_viol_cntr; 266 port_attr->pkey_tbl_len = resp.pkey_tbl_len; 267 port_attr->lid = resp.lid; 268 port_attr->sm_lid = resp.sm_lid; 269 port_attr->lmc = resp.lmc; 270 port_attr->max_vl_num = resp.max_vl_num; 271 port_attr->sm_sl = resp.sm_sl; 272 port_attr->subnet_timeout = resp.subnet_timeout; 273 port_attr->init_type_reply = resp.init_type_reply; 274 port_attr->active_width = resp.active_width; 275 port_attr->active_speed = resp.active_speed; 276 port_attr->phys_state = resp.phys_state; 277 port_attr->link_layer = resp.link_layer; 278 279 return 0; 280 } 281 282 int ibv_cmd_alloc_pd(struct ibv_context *context, struct ibv_pd *pd, 283 struct ibv_alloc_pd *cmd, size_t cmd_size, 284 struct ibv_alloc_pd_resp *resp, size_t resp_size) 285 { 286 IBV_INIT_CMD_RESP(cmd, cmd_size, ALLOC_PD, resp, resp_size); 287 288 if (write(context->cmd_fd, cmd, cmd_size) != cmd_size) 289 return errno; 290 291 (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size); 292 293 pd->handle = resp->pd_handle; 294 pd->context = context; 295 296 return 0; 297 } 298 299 int ibv_cmd_dealloc_pd(struct ibv_pd *pd) 300 { 301 struct ibv_dealloc_pd cmd; 302 303 IBV_INIT_CMD(&cmd, sizeof cmd, DEALLOC_PD); 304 cmd.pd_handle = pd->handle; 305 306 if (write(pd->context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd) 307 return errno; 308 309 return 0; 310 } 311 312 int ibv_cmd_open_xrcd(struct ibv_context *context, struct verbs_xrcd *xrcd, 313 int vxrcd_size, 314 struct ibv_xrcd_init_attr *attr, 315 struct ibv_open_xrcd *cmd, size_t cmd_size, 316 struct ibv_open_xrcd_resp *resp, size_t resp_size) 317 { 318 IBV_INIT_CMD_RESP(cmd, cmd_size, OPEN_XRCD, resp, resp_size); 319 320 if (attr->comp_mask >= IBV_XRCD_INIT_ATTR_RESERVED) 321 return ENOSYS; 322 323 if (!(attr->comp_mask & IBV_XRCD_INIT_ATTR_FD) || 324 !(attr->comp_mask & IBV_XRCD_INIT_ATTR_OFLAGS)) 325 return EINVAL; 326 327 cmd->fd = attr->fd; 328 cmd->oflags = attr->oflags; 329 if (write(context->cmd_fd, cmd, cmd_size) != cmd_size) 330 return errno; 331 332 (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size); 333 334 xrcd->xrcd.context = context; 335 xrcd->comp_mask = 0; 336 if (vext_field_avail(struct verbs_xrcd, handle, vxrcd_size)) { 337 xrcd->comp_mask = VERBS_XRCD_HANDLE; 338 xrcd->handle = resp->xrcd_handle; 339 } 340 341 return 0; 342 } 343 344 int ibv_cmd_close_xrcd(struct verbs_xrcd *xrcd) 345 { 346 struct ibv_close_xrcd cmd; 347 348 IBV_INIT_CMD(&cmd, sizeof cmd, CLOSE_XRCD); 349 cmd.xrcd_handle = xrcd->handle; 350 351 if (write(xrcd->xrcd.context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd) 352 return errno; 353 354 return 0; 355 } 356 357 int ibv_cmd_reg_mr(struct ibv_pd *pd, void *addr, size_t length, 358 uint64_t hca_va, int access, 359 struct ibv_mr *mr, struct ibv_reg_mr *cmd, 360 size_t cmd_size, 361 struct ibv_reg_mr_resp *resp, size_t resp_size) 362 { 363 364 IBV_INIT_CMD_RESP(cmd, cmd_size, REG_MR, resp, resp_size); 365 366 cmd->start = (uintptr_t) addr; 367 cmd->length = length; 368 cmd->hca_va = hca_va; 369 cmd->pd_handle = pd->handle; 370 cmd->access_flags = access; 371 372 if (write(pd->context->cmd_fd, cmd, cmd_size) != cmd_size) 373 return errno; 374 375 (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size); 376 377 mr->handle = resp->mr_handle; 378 mr->lkey = resp->lkey; 379 mr->rkey = resp->rkey; 380 mr->context = pd->context; 381 382 return 0; 383 } 384 385 int ibv_cmd_rereg_mr(struct ibv_mr *mr, uint32_t flags, void *addr, 386 size_t length, uint64_t hca_va, int access, 387 struct ibv_pd *pd, struct ibv_rereg_mr *cmd, 388 size_t cmd_sz, struct ibv_rereg_mr_resp *resp, 389 size_t resp_sz) 390 { 391 IBV_INIT_CMD_RESP(cmd, cmd_sz, REREG_MR, resp, resp_sz); 392 393 cmd->mr_handle = mr->handle; 394 cmd->flags = flags; 395 cmd->start = (uintptr_t)addr; 396 cmd->length = length; 397 cmd->hca_va = hca_va; 398 cmd->pd_handle = (flags & IBV_REREG_MR_CHANGE_PD) ? pd->handle : 0; 399 cmd->access_flags = access; 400 401 if (write(mr->context->cmd_fd, cmd, cmd_sz) != cmd_sz) 402 return errno; 403 404 (void)VALGRIND_MAKE_MEM_DEFINED(resp, resp_sz); 405 406 mr->lkey = resp->lkey; 407 mr->rkey = resp->rkey; 408 if (flags & IBV_REREG_MR_CHANGE_PD) 409 mr->context = pd->context; 410 411 return 0; 412 } 413 414 int ibv_cmd_dereg_mr(struct ibv_mr *mr) 415 { 416 struct ibv_dereg_mr cmd; 417 418 IBV_INIT_CMD(&cmd, sizeof cmd, DEREG_MR); 419 cmd.mr_handle = mr->handle; 420 421 if (write(mr->context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd) 422 return errno; 423 424 return 0; 425 } 426 427 int ibv_cmd_alloc_mw(struct ibv_pd *pd, enum ibv_mw_type type, 428 struct ibv_mw *mw, struct ibv_alloc_mw *cmd, 429 size_t cmd_size, 430 struct ibv_alloc_mw_resp *resp, size_t resp_size) 431 { 432 IBV_INIT_CMD_RESP(cmd, cmd_size, ALLOC_MW, resp, resp_size); 433 cmd->pd_handle = pd->handle; 434 cmd->mw_type = type; 435 memset(cmd->reserved, 0, sizeof(cmd->reserved)); 436 437 if (write(pd->context->cmd_fd, cmd, cmd_size) != cmd_size) 438 return errno; 439 440 (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size); 441 442 mw->context = pd->context; 443 mw->pd = pd; 444 mw->rkey = resp->rkey; 445 mw->handle = resp->mw_handle; 446 mw->type = type; 447 448 return 0; 449 } 450 451 int ibv_cmd_dealloc_mw(struct ibv_mw *mw, 452 struct ibv_dealloc_mw *cmd, size_t cmd_size) 453 { 454 IBV_INIT_CMD(cmd, cmd_size, DEALLOC_MW); 455 cmd->mw_handle = mw->handle; 456 cmd->reserved = 0; 457 458 if (write(mw->context->cmd_fd, cmd, cmd_size) != cmd_size) 459 return errno; 460 461 return 0; 462 } 463 464 int ibv_cmd_create_cq(struct ibv_context *context, int cqe, 465 struct ibv_comp_channel *channel, 466 int comp_vector, struct ibv_cq *cq, 467 struct ibv_create_cq *cmd, size_t cmd_size, 468 struct ibv_create_cq_resp *resp, size_t resp_size) 469 { 470 IBV_INIT_CMD_RESP(cmd, cmd_size, CREATE_CQ, resp, resp_size); 471 cmd->user_handle = (uintptr_t) cq; 472 cmd->cqe = cqe; 473 cmd->comp_vector = comp_vector; 474 cmd->comp_channel = channel ? channel->fd : -1; 475 cmd->reserved = 0; 476 477 if (write(context->cmd_fd, cmd, cmd_size) != cmd_size) 478 return errno; 479 480 (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size); 481 482 cq->handle = resp->cq_handle; 483 cq->cqe = resp->cqe; 484 cq->context = context; 485 486 return 0; 487 } 488 489 int ibv_cmd_create_cq_ex(struct ibv_context *context, 490 struct ibv_cq_init_attr_ex *cq_attr, 491 struct ibv_cq_ex *cq, 492 struct ibv_create_cq_ex *cmd, 493 size_t cmd_core_size, 494 size_t cmd_size, 495 struct ibv_create_cq_resp_ex *resp, 496 size_t resp_core_size, 497 size_t resp_size) 498 { 499 int err; 500 501 memset(cmd, 0, cmd_core_size); 502 IBV_INIT_CMD_RESP_EX_V(cmd, cmd_core_size, cmd_size, CREATE_CQ_EX, resp, 503 resp_core_size, resp_size); 504 505 if (cq_attr->comp_mask & ~(IBV_CQ_INIT_ATTR_MASK_RESERVED - 1)) 506 return EINVAL; 507 508 cmd->user_handle = (uintptr_t)cq; 509 cmd->cqe = cq_attr->cqe; 510 cmd->comp_vector = cq_attr->comp_vector; 511 cmd->comp_channel = cq_attr->channel ? cq_attr->channel->fd : -1; 512 cmd->comp_mask = 0; 513 514 if (cmd_core_size >= offsetof(struct ibv_create_cq_ex, flags) + 515 sizeof(cmd->flags)) { 516 if ((cq_attr->comp_mask & IBV_CQ_INIT_ATTR_MASK_FLAGS) && 517 (cq_attr->flags & ~(IBV_CREATE_CQ_ATTR_RESERVED - 1))) 518 return EOPNOTSUPP; 519 520 if (cq_attr->wc_flags & IBV_WC_EX_WITH_COMPLETION_TIMESTAMP) 521 cmd->flags |= IBV_CREATE_CQ_EX_KERNEL_FLAG_COMPLETION_TIMESTAMP; 522 } 523 524 err = write(context->cmd_fd, cmd, cmd_size); 525 if (err != cmd_size) 526 return errno; 527 528 (void)VALGRIND_MAKE_MEM_DEFINED(resp, resp_size); 529 530 cq->handle = resp->base.cq_handle; 531 cq->cqe = resp->base.cqe; 532 cq->context = context; 533 534 return 0; 535 } 536 537 int ibv_cmd_poll_cq(struct ibv_cq *ibcq, int ne, struct ibv_wc *wc) 538 { 539 struct ibv_poll_cq cmd; 540 struct ibv_poll_cq_resp *resp; 541 int i; 542 int rsize; 543 int ret; 544 545 rsize = sizeof *resp + ne * sizeof(struct ibv_kern_wc); 546 resp = malloc(rsize); 547 if (!resp) 548 return -1; 549 550 IBV_INIT_CMD_RESP(&cmd, sizeof cmd, POLL_CQ, resp, rsize); 551 cmd.cq_handle = ibcq->handle; 552 cmd.ne = ne; 553 554 if (write(ibcq->context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd) { 555 ret = -1; 556 goto out; 557 } 558 559 (void) VALGRIND_MAKE_MEM_DEFINED(resp, rsize); 560 561 for (i = 0; i < resp->count; i++) { 562 wc[i].wr_id = resp->wc[i].wr_id; 563 wc[i].status = resp->wc[i].status; 564 wc[i].opcode = resp->wc[i].opcode; 565 wc[i].vendor_err = resp->wc[i].vendor_err; 566 wc[i].byte_len = resp->wc[i].byte_len; 567 wc[i].imm_data = resp->wc[i].imm_data; 568 wc[i].qp_num = resp->wc[i].qp_num; 569 wc[i].src_qp = resp->wc[i].src_qp; 570 wc[i].wc_flags = resp->wc[i].wc_flags; 571 wc[i].pkey_index = resp->wc[i].pkey_index; 572 wc[i].slid = resp->wc[i].slid; 573 wc[i].sl = resp->wc[i].sl; 574 wc[i].dlid_path_bits = resp->wc[i].dlid_path_bits; 575 } 576 577 ret = resp->count; 578 579 out: 580 free(resp); 581 return ret; 582 } 583 584 int ibv_cmd_req_notify_cq(struct ibv_cq *ibcq, int solicited_only) 585 { 586 struct ibv_req_notify_cq cmd; 587 588 IBV_INIT_CMD(&cmd, sizeof cmd, REQ_NOTIFY_CQ); 589 cmd.cq_handle = ibcq->handle; 590 cmd.solicited = !!solicited_only; 591 592 if (write(ibcq->context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd) 593 return errno; 594 595 return 0; 596 } 597 598 int ibv_cmd_resize_cq(struct ibv_cq *cq, int cqe, 599 struct ibv_resize_cq *cmd, size_t cmd_size, 600 struct ibv_resize_cq_resp *resp, size_t resp_size) 601 { 602 IBV_INIT_CMD_RESP(cmd, cmd_size, RESIZE_CQ, resp, resp_size); 603 cmd->cq_handle = cq->handle; 604 cmd->cqe = cqe; 605 606 if (write(cq->context->cmd_fd, cmd, cmd_size) != cmd_size) 607 return errno; 608 609 (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size); 610 611 cq->cqe = resp->cqe; 612 613 return 0; 614 } 615 616 int ibv_cmd_destroy_cq(struct ibv_cq *cq) 617 { 618 struct ibv_destroy_cq cmd; 619 struct ibv_destroy_cq_resp resp; 620 621 IBV_INIT_CMD_RESP(&cmd, sizeof cmd, DESTROY_CQ, &resp, sizeof resp); 622 cmd.cq_handle = cq->handle; 623 cmd.reserved = 0; 624 625 if (write(cq->context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd) 626 return errno; 627 628 (void) VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof resp); 629 630 pthread_mutex_lock(&cq->mutex); 631 while (cq->comp_events_completed != resp.comp_events_reported || 632 cq->async_events_completed != resp.async_events_reported) 633 pthread_cond_wait(&cq->cond, &cq->mutex); 634 pthread_mutex_unlock(&cq->mutex); 635 636 return 0; 637 } 638 639 int ibv_cmd_create_srq(struct ibv_pd *pd, 640 struct ibv_srq *srq, struct ibv_srq_init_attr *attr, 641 struct ibv_create_srq *cmd, size_t cmd_size, 642 struct ibv_create_srq_resp *resp, size_t resp_size) 643 { 644 IBV_INIT_CMD_RESP(cmd, cmd_size, CREATE_SRQ, resp, resp_size); 645 cmd->user_handle = (uintptr_t) srq; 646 cmd->pd_handle = pd->handle; 647 cmd->max_wr = attr->attr.max_wr; 648 cmd->max_sge = attr->attr.max_sge; 649 cmd->srq_limit = attr->attr.srq_limit; 650 651 if (write(pd->context->cmd_fd, cmd, cmd_size) != cmd_size) 652 return errno; 653 654 (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size); 655 656 srq->handle = resp->srq_handle; 657 srq->context = pd->context; 658 659 if (abi_ver > 5) { 660 attr->attr.max_wr = resp->max_wr; 661 attr->attr.max_sge = resp->max_sge; 662 } else { 663 struct ibv_create_srq_resp_v5 *resp_v5 = 664 (struct ibv_create_srq_resp_v5 *) resp; 665 666 memmove((void *) resp + sizeof *resp, 667 (void *) resp_v5 + sizeof *resp_v5, 668 resp_size - sizeof *resp); 669 } 670 671 return 0; 672 } 673 674 int ibv_cmd_create_srq_ex(struct ibv_context *context, 675 struct verbs_srq *srq, int vsrq_sz, 676 struct ibv_srq_init_attr_ex *attr_ex, 677 struct ibv_create_xsrq *cmd, size_t cmd_size, 678 struct ibv_create_srq_resp *resp, size_t resp_size) 679 { 680 struct verbs_xrcd *vxrcd = NULL; 681 int ret = 0; 682 683 IBV_INIT_CMD_RESP(cmd, cmd_size, CREATE_XSRQ, resp, resp_size); 684 685 if (attr_ex->comp_mask >= IBV_SRQ_INIT_ATTR_RESERVED) 686 return ENOSYS; 687 688 if (!(attr_ex->comp_mask & IBV_SRQ_INIT_ATTR_PD)) 689 return EINVAL; 690 691 cmd->user_handle = (uintptr_t) srq; 692 cmd->pd_handle = attr_ex->pd->handle; 693 cmd->max_wr = attr_ex->attr.max_wr; 694 cmd->max_sge = attr_ex->attr.max_sge; 695 cmd->srq_limit = attr_ex->attr.srq_limit; 696 697 cmd->srq_type = (attr_ex->comp_mask & IBV_SRQ_INIT_ATTR_TYPE) ? 698 attr_ex->srq_type : IBV_SRQT_BASIC; 699 if (attr_ex->comp_mask & IBV_SRQ_INIT_ATTR_XRCD) { 700 if (!(attr_ex->comp_mask & IBV_SRQ_INIT_ATTR_CQ)) 701 return EINVAL; 702 703 vxrcd = container_of(attr_ex->xrcd, struct verbs_xrcd, xrcd); 704 cmd->xrcd_handle = vxrcd->handle; 705 cmd->cq_handle = attr_ex->cq->handle; 706 } 707 708 ret = pthread_mutex_init(&srq->srq.mutex, NULL); 709 if (ret) 710 goto err; 711 ret = pthread_cond_init(&srq->srq.cond, NULL); 712 if (ret) 713 goto err_mutex; 714 715 if (write(context->cmd_fd, cmd, cmd_size) != cmd_size) { 716 ret = errno; 717 goto err_cond; 718 } 719 720 (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size); 721 722 srq->srq.handle = resp->srq_handle; 723 srq->srq.context = context; 724 srq->srq.srq_context = attr_ex->srq_context; 725 srq->srq.pd = attr_ex->pd; 726 srq->srq.events_completed = 0; 727 728 /* 729 * check that the last field is available. 730 * If it is than all the others exist as well 731 */ 732 if (vext_field_avail(struct verbs_srq, srq_num, vsrq_sz)) { 733 srq->comp_mask = IBV_SRQ_INIT_ATTR_TYPE; 734 srq->srq_type = (attr_ex->comp_mask & IBV_SRQ_INIT_ATTR_TYPE) ? 735 attr_ex->srq_type : IBV_SRQT_BASIC; 736 if (srq->srq_type == IBV_SRQT_XRC) { 737 srq->comp_mask |= VERBS_SRQ_NUM; 738 srq->srq_num = resp->srqn; 739 } 740 if (attr_ex->comp_mask & IBV_SRQ_INIT_ATTR_XRCD) { 741 srq->comp_mask |= VERBS_SRQ_XRCD; 742 srq->xrcd = vxrcd; 743 } 744 if (attr_ex->comp_mask & IBV_SRQ_INIT_ATTR_CQ) { 745 srq->comp_mask |= VERBS_SRQ_CQ; 746 srq->cq = attr_ex->cq; 747 } 748 } 749 750 attr_ex->attr.max_wr = resp->max_wr; 751 attr_ex->attr.max_sge = resp->max_sge; 752 753 return 0; 754 err_cond: 755 pthread_cond_destroy(&srq->srq.cond); 756 err_mutex: 757 pthread_mutex_destroy(&srq->srq.mutex); 758 err: 759 return ret; 760 } 761 762 763 static int ibv_cmd_modify_srq_v3(struct ibv_srq *srq, 764 struct ibv_srq_attr *srq_attr, 765 int srq_attr_mask, 766 struct ibv_modify_srq *new_cmd, 767 size_t new_cmd_size) 768 { 769 struct ibv_modify_srq_v3 *cmd; 770 size_t cmd_size; 771 772 cmd_size = sizeof *cmd + new_cmd_size - sizeof *new_cmd; 773 cmd = alloca(cmd_size); 774 memcpy(cmd->driver_data, new_cmd->driver_data, new_cmd_size - sizeof *new_cmd); 775 776 IBV_INIT_CMD(cmd, cmd_size, MODIFY_SRQ); 777 778 cmd->srq_handle = srq->handle; 779 cmd->attr_mask = srq_attr_mask; 780 cmd->max_wr = srq_attr->max_wr; 781 cmd->srq_limit = srq_attr->srq_limit; 782 cmd->max_sge = 0; 783 cmd->reserved = 0; 784 785 if (write(srq->context->cmd_fd, cmd, cmd_size) != cmd_size) 786 return errno; 787 788 return 0; 789 } 790 791 int ibv_cmd_modify_srq(struct ibv_srq *srq, 792 struct ibv_srq_attr *srq_attr, 793 int srq_attr_mask, 794 struct ibv_modify_srq *cmd, size_t cmd_size) 795 { 796 if (abi_ver == 3) 797 return ibv_cmd_modify_srq_v3(srq, srq_attr, srq_attr_mask, 798 cmd, cmd_size); 799 800 IBV_INIT_CMD(cmd, cmd_size, MODIFY_SRQ); 801 802 cmd->srq_handle = srq->handle; 803 cmd->attr_mask = srq_attr_mask; 804 cmd->max_wr = srq_attr->max_wr; 805 cmd->srq_limit = srq_attr->srq_limit; 806 807 if (write(srq->context->cmd_fd, cmd, cmd_size) != cmd_size) 808 return errno; 809 810 return 0; 811 } 812 813 int ibv_cmd_query_srq(struct ibv_srq *srq, struct ibv_srq_attr *srq_attr, 814 struct ibv_query_srq *cmd, size_t cmd_size) 815 { 816 struct ibv_query_srq_resp resp; 817 818 IBV_INIT_CMD_RESP(cmd, cmd_size, QUERY_SRQ, &resp, sizeof resp); 819 cmd->srq_handle = srq->handle; 820 cmd->reserved = 0; 821 822 if (write(srq->context->cmd_fd, cmd, cmd_size) != cmd_size) 823 return errno; 824 825 (void) VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof resp); 826 827 srq_attr->max_wr = resp.max_wr; 828 srq_attr->max_sge = resp.max_sge; 829 srq_attr->srq_limit = resp.srq_limit; 830 831 return 0; 832 } 833 834 int ibv_cmd_destroy_srq(struct ibv_srq *srq) 835 { 836 struct ibv_destroy_srq cmd; 837 struct ibv_destroy_srq_resp resp; 838 839 IBV_INIT_CMD_RESP(&cmd, sizeof cmd, DESTROY_SRQ, &resp, sizeof resp); 840 cmd.srq_handle = srq->handle; 841 cmd.reserved = 0; 842 843 if (write(srq->context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd) 844 return errno; 845 846 (void) VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof resp); 847 848 pthread_mutex_lock(&srq->mutex); 849 while (srq->events_completed != resp.events_reported) 850 pthread_cond_wait(&srq->cond, &srq->mutex); 851 pthread_mutex_unlock(&srq->mutex); 852 853 pthread_cond_destroy(&srq->cond); 854 pthread_mutex_destroy(&srq->mutex); 855 856 return 0; 857 } 858 859 static int create_qp_ex_common(struct verbs_qp *qp, 860 struct ibv_qp_init_attr_ex *qp_attr, 861 struct verbs_xrcd *vxrcd, 862 struct ibv_create_qp_common *cmd) 863 { 864 cmd->user_handle = (uintptr_t)qp; 865 866 if (qp_attr->comp_mask & IBV_QP_INIT_ATTR_XRCD) { 867 vxrcd = container_of(qp_attr->xrcd, struct verbs_xrcd, xrcd); 868 cmd->pd_handle = vxrcd->handle; 869 } else { 870 if (!(qp_attr->comp_mask & IBV_QP_INIT_ATTR_PD)) 871 return EINVAL; 872 873 cmd->pd_handle = qp_attr->pd->handle; 874 if (qp_attr->comp_mask & IBV_QP_INIT_ATTR_IND_TABLE) { 875 if (cmd->max_recv_wr || cmd->max_recv_sge || 876 cmd->recv_cq_handle || qp_attr->srq) 877 return EINVAL; 878 879 /* send_cq is optinal */ 880 if (qp_attr->cap.max_send_wr) 881 cmd->send_cq_handle = qp_attr->send_cq->handle; 882 } else { 883 cmd->send_cq_handle = qp_attr->send_cq->handle; 884 885 if (qp_attr->qp_type != IBV_QPT_XRC_SEND) { 886 cmd->recv_cq_handle = qp_attr->recv_cq->handle; 887 cmd->srq_handle = qp_attr->srq ? qp_attr->srq->handle : 888 0; 889 } 890 } 891 } 892 893 cmd->max_send_wr = qp_attr->cap.max_send_wr; 894 cmd->max_recv_wr = qp_attr->cap.max_recv_wr; 895 cmd->max_send_sge = qp_attr->cap.max_send_sge; 896 cmd->max_recv_sge = qp_attr->cap.max_recv_sge; 897 cmd->max_inline_data = qp_attr->cap.max_inline_data; 898 cmd->sq_sig_all = qp_attr->sq_sig_all; 899 cmd->qp_type = qp_attr->qp_type; 900 cmd->is_srq = !!qp_attr->srq; 901 cmd->reserved = 0; 902 903 return 0; 904 } 905 906 static void create_qp_handle_resp_common_cleanup(struct verbs_qp *qp) 907 { 908 pthread_cond_destroy(&qp->qp.cond); 909 pthread_mutex_destroy(&qp->qp.mutex); 910 } 911 912 static int create_qp_handle_resp_common_init(struct verbs_qp *qp) 913 { 914 int ret = 0; 915 916 ret = pthread_mutex_init(&qp->qp.mutex, NULL); 917 if (ret) 918 return ret; 919 ret = pthread_cond_init(&qp->qp.cond, NULL); 920 if (ret) 921 goto err; 922 923 return ret; 924 925 err: 926 pthread_mutex_destroy(&qp->qp.mutex); 927 928 return ret; 929 } 930 931 static void create_qp_handle_resp_common(struct ibv_context *context, 932 struct verbs_qp *qp, 933 struct ibv_qp_init_attr_ex *qp_attr, 934 struct ibv_create_qp_resp *resp, 935 struct verbs_xrcd *vxrcd, 936 int vqp_sz) 937 { 938 if (abi_ver > 3) { 939 qp_attr->cap.max_recv_sge = resp->max_recv_sge; 940 qp_attr->cap.max_send_sge = resp->max_send_sge; 941 qp_attr->cap.max_recv_wr = resp->max_recv_wr; 942 qp_attr->cap.max_send_wr = resp->max_send_wr; 943 qp_attr->cap.max_inline_data = resp->max_inline_data; 944 } 945 946 qp->qp.handle = resp->qp_handle; 947 qp->qp.qp_num = resp->qpn; 948 qp->qp.context = context; 949 qp->qp.qp_context = qp_attr->qp_context; 950 qp->qp.pd = qp_attr->pd; 951 qp->qp.send_cq = qp_attr->send_cq; 952 qp->qp.recv_cq = qp_attr->recv_cq; 953 qp->qp.srq = qp_attr->srq; 954 qp->qp.qp_type = qp_attr->qp_type; 955 qp->qp.state = IBV_QPS_RESET; 956 qp->qp.events_completed = 0; 957 958 qp->comp_mask = 0; 959 if (vext_field_avail(struct verbs_qp, xrcd, vqp_sz) && 960 (qp_attr->comp_mask & IBV_QP_INIT_ATTR_XRCD)) { 961 qp->comp_mask |= VERBS_QP_XRCD; 962 qp->xrcd = vxrcd; 963 } 964 } 965 966 enum { 967 CREATE_QP_EX2_SUP_CREATE_FLAGS = IBV_QP_CREATE_BLOCK_SELF_MCAST_LB | 968 IBV_QP_CREATE_SCATTER_FCS | 969 IBV_QP_CREATE_CVLAN_STRIPPING, 970 }; 971 972 int ibv_cmd_create_qp_ex2(struct ibv_context *context, 973 struct verbs_qp *qp, int vqp_sz, 974 struct ibv_qp_init_attr_ex *qp_attr, 975 struct ibv_create_qp_ex *cmd, 976 size_t cmd_core_size, 977 size_t cmd_size, 978 struct ibv_create_qp_resp_ex *resp, 979 size_t resp_core_size, 980 size_t resp_size) 981 { 982 struct verbs_xrcd *vxrcd = NULL; 983 int err; 984 985 if (qp_attr->comp_mask >= IBV_QP_INIT_ATTR_RESERVED) 986 return EINVAL; 987 988 if (resp_core_size < 989 offsetof(struct ibv_create_qp_resp_ex, response_length) + 990 sizeof(resp->response_length)) 991 return EINVAL; 992 993 memset(cmd, 0, cmd_core_size); 994 995 IBV_INIT_CMD_RESP_EX_V(cmd, cmd_core_size, cmd_size, CREATE_QP_EX, resp, 996 resp_core_size, resp_size); 997 998 err = create_qp_ex_common(qp, qp_attr, vxrcd, &cmd->base); 999 if (err) 1000 return err; 1001 1002 if (qp_attr->comp_mask & IBV_QP_INIT_ATTR_CREATE_FLAGS) { 1003 if (qp_attr->create_flags & ~CREATE_QP_EX2_SUP_CREATE_FLAGS) 1004 return EINVAL; 1005 if (cmd_core_size < offsetof(struct ibv_create_qp_ex, create_flags) + 1006 sizeof(qp_attr->create_flags)) 1007 return EINVAL; 1008 cmd->create_flags = qp_attr->create_flags; 1009 } 1010 1011 if (qp_attr->comp_mask & IBV_QP_INIT_ATTR_IND_TABLE) { 1012 if (cmd_core_size < offsetof(struct ibv_create_qp_ex, ind_tbl_handle) + 1013 sizeof(cmd->ind_tbl_handle)) 1014 return EINVAL; 1015 cmd->ind_tbl_handle = qp_attr->rwq_ind_tbl->ind_tbl_handle; 1016 cmd->comp_mask = IBV_CREATE_QP_EX_KERNEL_MASK_IND_TABLE; 1017 } 1018 1019 err = create_qp_handle_resp_common_init(qp); 1020 if (err) 1021 return err; 1022 1023 err = write(context->cmd_fd, cmd, cmd_size); 1024 if (err != cmd_size) { 1025 err = errno; 1026 goto err; 1027 } 1028 1029 (void)VALGRIND_MAKE_MEM_DEFINED(resp, resp_size); 1030 1031 create_qp_handle_resp_common(context, qp, qp_attr, &resp->base, vxrcd, 1032 vqp_sz); 1033 1034 return 0; 1035 1036 err: 1037 create_qp_handle_resp_common_cleanup(qp); 1038 1039 return err; 1040 } 1041 1042 int ibv_cmd_create_qp_ex(struct ibv_context *context, 1043 struct verbs_qp *qp, int vqp_sz, 1044 struct ibv_qp_init_attr_ex *attr_ex, 1045 struct ibv_create_qp *cmd, size_t cmd_size, 1046 struct ibv_create_qp_resp *resp, size_t resp_size) 1047 { 1048 struct verbs_xrcd *vxrcd = NULL; 1049 int err; 1050 1051 IBV_INIT_CMD_RESP(cmd, cmd_size, CREATE_QP, resp, resp_size); 1052 1053 if (attr_ex->comp_mask > (IBV_QP_INIT_ATTR_XRCD | IBV_QP_INIT_ATTR_PD)) 1054 return ENOSYS; 1055 1056 err = create_qp_ex_common(qp, attr_ex, vxrcd, 1057 (struct ibv_create_qp_common *)&cmd->user_handle); 1058 if (err) 1059 return err; 1060 1061 err = create_qp_handle_resp_common_init(qp); 1062 if (err) 1063 return err; 1064 1065 err = write(context->cmd_fd, cmd, cmd_size); 1066 if (err != cmd_size) { 1067 err = errno; 1068 goto err; 1069 } 1070 1071 (void)VALGRIND_MAKE_MEM_DEFINED(resp, resp_size); 1072 1073 if (abi_ver == 4) { 1074 struct ibv_create_qp_resp_v4 *resp_v4 = 1075 (struct ibv_create_qp_resp_v4 *)resp; 1076 1077 memmove((void *)resp + sizeof *resp, 1078 (void *)resp_v4 + sizeof *resp_v4, 1079 resp_size - sizeof *resp); 1080 } else if (abi_ver <= 3) { 1081 struct ibv_create_qp_resp_v3 *resp_v3 = 1082 (struct ibv_create_qp_resp_v3 *)resp; 1083 1084 memmove((void *)resp + sizeof *resp, 1085 (void *)resp_v3 + sizeof *resp_v3, 1086 resp_size - sizeof *resp); 1087 } 1088 1089 create_qp_handle_resp_common(context, qp, attr_ex, resp, vxrcd, vqp_sz); 1090 1091 return 0; 1092 1093 err: 1094 create_qp_handle_resp_common_cleanup(qp); 1095 1096 return err; 1097 } 1098 1099 int ibv_cmd_create_qp(struct ibv_pd *pd, 1100 struct ibv_qp *qp, struct ibv_qp_init_attr *attr, 1101 struct ibv_create_qp *cmd, size_t cmd_size, 1102 struct ibv_create_qp_resp *resp, size_t resp_size) 1103 { 1104 IBV_INIT_CMD_RESP(cmd, cmd_size, CREATE_QP, resp, resp_size); 1105 1106 cmd->user_handle = (uintptr_t) qp; 1107 cmd->pd_handle = pd->handle; 1108 cmd->send_cq_handle = attr->send_cq->handle; 1109 cmd->recv_cq_handle = attr->recv_cq->handle; 1110 cmd->srq_handle = attr->srq ? attr->srq->handle : 0; 1111 cmd->max_send_wr = attr->cap.max_send_wr; 1112 cmd->max_recv_wr = attr->cap.max_recv_wr; 1113 cmd->max_send_sge = attr->cap.max_send_sge; 1114 cmd->max_recv_sge = attr->cap.max_recv_sge; 1115 cmd->max_inline_data = attr->cap.max_inline_data; 1116 cmd->sq_sig_all = attr->sq_sig_all; 1117 cmd->qp_type = attr->qp_type; 1118 cmd->is_srq = !!attr->srq; 1119 cmd->reserved = 0; 1120 1121 if (write(pd->context->cmd_fd, cmd, cmd_size) != cmd_size) 1122 return errno; 1123 1124 (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size); 1125 1126 qp->handle = resp->qp_handle; 1127 qp->qp_num = resp->qpn; 1128 qp->context = pd->context; 1129 1130 if (abi_ver > 3) { 1131 attr->cap.max_recv_sge = resp->max_recv_sge; 1132 attr->cap.max_send_sge = resp->max_send_sge; 1133 attr->cap.max_recv_wr = resp->max_recv_wr; 1134 attr->cap.max_send_wr = resp->max_send_wr; 1135 attr->cap.max_inline_data = resp->max_inline_data; 1136 } 1137 1138 if (abi_ver == 4) { 1139 struct ibv_create_qp_resp_v4 *resp_v4 = 1140 (struct ibv_create_qp_resp_v4 *) resp; 1141 1142 memmove((void *) resp + sizeof *resp, 1143 (void *) resp_v4 + sizeof *resp_v4, 1144 resp_size - sizeof *resp); 1145 } else if (abi_ver <= 3) { 1146 struct ibv_create_qp_resp_v3 *resp_v3 = 1147 (struct ibv_create_qp_resp_v3 *) resp; 1148 1149 memmove((void *) resp + sizeof *resp, 1150 (void *) resp_v3 + sizeof *resp_v3, 1151 resp_size - sizeof *resp); 1152 } 1153 1154 return 0; 1155 } 1156 1157 int ibv_cmd_open_qp(struct ibv_context *context, struct verbs_qp *qp, 1158 int vqp_sz, 1159 struct ibv_qp_open_attr *attr, 1160 struct ibv_open_qp *cmd, size_t cmd_size, 1161 struct ibv_create_qp_resp *resp, size_t resp_size) 1162 { 1163 int err = 0; 1164 struct verbs_xrcd *xrcd; 1165 IBV_INIT_CMD_RESP(cmd, cmd_size, OPEN_QP, resp, resp_size); 1166 1167 if (attr->comp_mask >= IBV_QP_OPEN_ATTR_RESERVED) 1168 return ENOSYS; 1169 1170 if (!(attr->comp_mask & IBV_QP_OPEN_ATTR_XRCD) || 1171 !(attr->comp_mask & IBV_QP_OPEN_ATTR_NUM) || 1172 !(attr->comp_mask & IBV_QP_OPEN_ATTR_TYPE)) 1173 return EINVAL; 1174 1175 xrcd = container_of(attr->xrcd, struct verbs_xrcd, xrcd); 1176 cmd->user_handle = (uintptr_t) qp; 1177 cmd->pd_handle = xrcd->handle; 1178 cmd->qpn = attr->qp_num; 1179 cmd->qp_type = attr->qp_type; 1180 1181 err = pthread_mutex_init(&qp->qp.mutex, NULL); 1182 if (err) 1183 return err; 1184 err = pthread_cond_init(&qp->qp.cond, NULL); 1185 if (err) 1186 goto err_mutex; 1187 1188 err = write(context->cmd_fd, cmd, cmd_size); 1189 if (err != cmd_size) { 1190 err = errno; 1191 goto err_cond; 1192 } 1193 1194 (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size); 1195 1196 qp->qp.handle = resp->qp_handle; 1197 qp->qp.context = context; 1198 qp->qp.qp_context = attr->qp_context; 1199 qp->qp.pd = NULL; 1200 qp->qp.send_cq = NULL; 1201 qp->qp.recv_cq = NULL; 1202 qp->qp.srq = NULL; 1203 qp->qp.qp_num = attr->qp_num; 1204 qp->qp.qp_type = attr->qp_type; 1205 qp->qp.state = IBV_QPS_UNKNOWN; 1206 qp->qp.events_completed = 0; 1207 qp->comp_mask = 0; 1208 if (vext_field_avail(struct verbs_qp, xrcd, vqp_sz)) { 1209 qp->comp_mask = VERBS_QP_XRCD; 1210 qp->xrcd = xrcd; 1211 } 1212 1213 return 0; 1214 1215 err_cond: 1216 pthread_cond_destroy(&qp->qp.cond); 1217 err_mutex: 1218 pthread_mutex_destroy(&qp->qp.mutex); 1219 1220 return err; 1221 } 1222 1223 int ibv_cmd_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, 1224 int attr_mask, 1225 struct ibv_qp_init_attr *init_attr, 1226 struct ibv_query_qp *cmd, size_t cmd_size) 1227 { 1228 struct ibv_query_qp_resp resp; 1229 1230 /* 1231 * Masks over IBV_QP_DEST_QPN are not supported by 1232 * that not extended command. 1233 */ 1234 if (attr_mask & ~((IBV_QP_DEST_QPN << 1) - 1)) 1235 return EOPNOTSUPP; 1236 1237 IBV_INIT_CMD_RESP(cmd, cmd_size, QUERY_QP, &resp, sizeof resp); 1238 cmd->qp_handle = qp->handle; 1239 cmd->attr_mask = attr_mask; 1240 1241 if (write(qp->context->cmd_fd, cmd, cmd_size) != cmd_size) 1242 return errno; 1243 1244 (void) VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof resp); 1245 1246 attr->qkey = resp.qkey; 1247 attr->rq_psn = resp.rq_psn; 1248 attr->sq_psn = resp.sq_psn; 1249 attr->dest_qp_num = resp.dest_qp_num; 1250 attr->qp_access_flags = resp.qp_access_flags; 1251 attr->pkey_index = resp.pkey_index; 1252 attr->alt_pkey_index = resp.alt_pkey_index; 1253 attr->qp_state = resp.qp_state; 1254 attr->cur_qp_state = resp.cur_qp_state; 1255 attr->path_mtu = resp.path_mtu; 1256 attr->path_mig_state = resp.path_mig_state; 1257 attr->sq_draining = resp.sq_draining; 1258 attr->max_rd_atomic = resp.max_rd_atomic; 1259 attr->max_dest_rd_atomic = resp.max_dest_rd_atomic; 1260 attr->min_rnr_timer = resp.min_rnr_timer; 1261 attr->port_num = resp.port_num; 1262 attr->timeout = resp.timeout; 1263 attr->retry_cnt = resp.retry_cnt; 1264 attr->rnr_retry = resp.rnr_retry; 1265 attr->alt_port_num = resp.alt_port_num; 1266 attr->alt_timeout = resp.alt_timeout; 1267 attr->cap.max_send_wr = resp.max_send_wr; 1268 attr->cap.max_recv_wr = resp.max_recv_wr; 1269 attr->cap.max_send_sge = resp.max_send_sge; 1270 attr->cap.max_recv_sge = resp.max_recv_sge; 1271 attr->cap.max_inline_data = resp.max_inline_data; 1272 1273 memcpy(attr->ah_attr.grh.dgid.raw, resp.dest.dgid, 16); 1274 attr->ah_attr.grh.flow_label = resp.dest.flow_label; 1275 attr->ah_attr.dlid = resp.dest.dlid; 1276 attr->ah_attr.grh.sgid_index = resp.dest.sgid_index; 1277 attr->ah_attr.grh.hop_limit = resp.dest.hop_limit; 1278 attr->ah_attr.grh.traffic_class = resp.dest.traffic_class; 1279 attr->ah_attr.sl = resp.dest.sl; 1280 attr->ah_attr.src_path_bits = resp.dest.src_path_bits; 1281 attr->ah_attr.static_rate = resp.dest.static_rate; 1282 attr->ah_attr.is_global = resp.dest.is_global; 1283 attr->ah_attr.port_num = resp.dest.port_num; 1284 1285 memcpy(attr->alt_ah_attr.grh.dgid.raw, resp.alt_dest.dgid, 16); 1286 attr->alt_ah_attr.grh.flow_label = resp.alt_dest.flow_label; 1287 attr->alt_ah_attr.dlid = resp.alt_dest.dlid; 1288 attr->alt_ah_attr.grh.sgid_index = resp.alt_dest.sgid_index; 1289 attr->alt_ah_attr.grh.hop_limit = resp.alt_dest.hop_limit; 1290 attr->alt_ah_attr.grh.traffic_class = resp.alt_dest.traffic_class; 1291 attr->alt_ah_attr.sl = resp.alt_dest.sl; 1292 attr->alt_ah_attr.src_path_bits = resp.alt_dest.src_path_bits; 1293 attr->alt_ah_attr.static_rate = resp.alt_dest.static_rate; 1294 attr->alt_ah_attr.is_global = resp.alt_dest.is_global; 1295 attr->alt_ah_attr.port_num = resp.alt_dest.port_num; 1296 1297 init_attr->qp_context = qp->qp_context; 1298 init_attr->send_cq = qp->send_cq; 1299 init_attr->recv_cq = qp->recv_cq; 1300 init_attr->srq = qp->srq; 1301 init_attr->qp_type = qp->qp_type; 1302 init_attr->cap.max_send_wr = resp.max_send_wr; 1303 init_attr->cap.max_recv_wr = resp.max_recv_wr; 1304 init_attr->cap.max_send_sge = resp.max_send_sge; 1305 init_attr->cap.max_recv_sge = resp.max_recv_sge; 1306 init_attr->cap.max_inline_data = resp.max_inline_data; 1307 init_attr->sq_sig_all = resp.sq_sig_all; 1308 1309 return 0; 1310 } 1311 1312 static void copy_modify_qp_fields(struct ibv_qp *qp, struct ibv_qp_attr *attr, 1313 int attr_mask, 1314 struct ibv_modify_qp_common *cmd) 1315 { 1316 cmd->qp_handle = qp->handle; 1317 cmd->attr_mask = attr_mask; 1318 1319 if (attr_mask & IBV_QP_STATE) 1320 cmd->qp_state = attr->qp_state; 1321 if (attr_mask & IBV_QP_CUR_STATE) 1322 cmd->cur_qp_state = attr->cur_qp_state; 1323 if (attr_mask & IBV_QP_EN_SQD_ASYNC_NOTIFY) 1324 cmd->en_sqd_async_notify = attr->en_sqd_async_notify; 1325 if (attr_mask & IBV_QP_ACCESS_FLAGS) 1326 cmd->qp_access_flags = attr->qp_access_flags; 1327 if (attr_mask & IBV_QP_PKEY_INDEX) 1328 cmd->pkey_index = attr->pkey_index; 1329 if (attr_mask & IBV_QP_PORT) 1330 cmd->port_num = attr->port_num; 1331 if (attr_mask & IBV_QP_QKEY) 1332 cmd->qkey = attr->qkey; 1333 1334 if (attr_mask & IBV_QP_AV) { 1335 memcpy(cmd->dest.dgid, attr->ah_attr.grh.dgid.raw, 16); 1336 cmd->dest.flow_label = attr->ah_attr.grh.flow_label; 1337 cmd->dest.dlid = attr->ah_attr.dlid; 1338 cmd->dest.reserved = 0; 1339 cmd->dest.sgid_index = attr->ah_attr.grh.sgid_index; 1340 cmd->dest.hop_limit = attr->ah_attr.grh.hop_limit; 1341 cmd->dest.traffic_class = attr->ah_attr.grh.traffic_class; 1342 cmd->dest.sl = attr->ah_attr.sl; 1343 cmd->dest.src_path_bits = attr->ah_attr.src_path_bits; 1344 cmd->dest.static_rate = attr->ah_attr.static_rate; 1345 cmd->dest.is_global = attr->ah_attr.is_global; 1346 cmd->dest.port_num = attr->ah_attr.port_num; 1347 } 1348 1349 if (attr_mask & IBV_QP_PATH_MTU) 1350 cmd->path_mtu = attr->path_mtu; 1351 if (attr_mask & IBV_QP_TIMEOUT) 1352 cmd->timeout = attr->timeout; 1353 if (attr_mask & IBV_QP_RETRY_CNT) 1354 cmd->retry_cnt = attr->retry_cnt; 1355 if (attr_mask & IBV_QP_RNR_RETRY) 1356 cmd->rnr_retry = attr->rnr_retry; 1357 if (attr_mask & IBV_QP_RQ_PSN) 1358 cmd->rq_psn = attr->rq_psn; 1359 if (attr_mask & IBV_QP_MAX_QP_RD_ATOMIC) 1360 cmd->max_rd_atomic = attr->max_rd_atomic; 1361 1362 if (attr_mask & IBV_QP_ALT_PATH) { 1363 cmd->alt_pkey_index = attr->alt_pkey_index; 1364 cmd->alt_port_num = attr->alt_port_num; 1365 cmd->alt_timeout = attr->alt_timeout; 1366 1367 memcpy(cmd->alt_dest.dgid, attr->alt_ah_attr.grh.dgid.raw, 16); 1368 cmd->alt_dest.flow_label = attr->alt_ah_attr.grh.flow_label; 1369 cmd->alt_dest.dlid = attr->alt_ah_attr.dlid; 1370 cmd->alt_dest.reserved = 0; 1371 cmd->alt_dest.sgid_index = attr->alt_ah_attr.grh.sgid_index; 1372 cmd->alt_dest.hop_limit = attr->alt_ah_attr.grh.hop_limit; 1373 cmd->alt_dest.traffic_class = 1374 attr->alt_ah_attr.grh.traffic_class; 1375 cmd->alt_dest.sl = attr->alt_ah_attr.sl; 1376 cmd->alt_dest.src_path_bits = attr->alt_ah_attr.src_path_bits; 1377 cmd->alt_dest.static_rate = attr->alt_ah_attr.static_rate; 1378 cmd->alt_dest.is_global = attr->alt_ah_attr.is_global; 1379 cmd->alt_dest.port_num = attr->alt_ah_attr.port_num; 1380 } 1381 1382 if (attr_mask & IBV_QP_MIN_RNR_TIMER) 1383 cmd->min_rnr_timer = attr->min_rnr_timer; 1384 if (attr_mask & IBV_QP_SQ_PSN) 1385 cmd->sq_psn = attr->sq_psn; 1386 if (attr_mask & IBV_QP_MAX_DEST_RD_ATOMIC) 1387 cmd->max_dest_rd_atomic = attr->max_dest_rd_atomic; 1388 if (attr_mask & IBV_QP_PATH_MIG_STATE) 1389 cmd->path_mig_state = attr->path_mig_state; 1390 if (attr_mask & IBV_QP_DEST_QPN) 1391 cmd->dest_qp_num = attr->dest_qp_num; 1392 1393 cmd->reserved[0] = cmd->reserved[1] = 0; 1394 } 1395 1396 int ibv_cmd_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, 1397 int attr_mask, 1398 struct ibv_modify_qp *cmd, size_t cmd_size) 1399 { 1400 /* 1401 * Masks over IBV_QP_DEST_QPN are only supported by 1402 * ibv_cmd_modify_qp_ex. 1403 */ 1404 if (attr_mask & ~((IBV_QP_DEST_QPN << 1) - 1)) 1405 return EOPNOTSUPP; 1406 1407 IBV_INIT_CMD(cmd, cmd_size, MODIFY_QP); 1408 1409 copy_modify_qp_fields(qp, attr, attr_mask, &cmd->base); 1410 1411 if (write(qp->context->cmd_fd, cmd, cmd_size) != cmd_size) 1412 return errno; 1413 1414 return 0; 1415 } 1416 1417 int ibv_cmd_modify_qp_ex(struct ibv_qp *qp, struct ibv_qp_attr *attr, 1418 int attr_mask, struct ibv_modify_qp_ex *cmd, 1419 size_t cmd_core_size, size_t cmd_size, 1420 struct ibv_modify_qp_resp_ex *resp, 1421 size_t resp_core_size, size_t resp_size) 1422 { 1423 if (resp_core_size < offsetof(struct ibv_modify_qp_resp_ex, 1424 response_length) + sizeof(resp->response_length)) 1425 return EINVAL; 1426 1427 IBV_INIT_CMD_RESP_EX_V(cmd, cmd_core_size, cmd_size, MODIFY_QP_EX, 1428 resp, resp_core_size, resp_size); 1429 1430 copy_modify_qp_fields(qp, attr, attr_mask, &cmd->base); 1431 1432 if (attr_mask & IBV_QP_RATE_LIMIT) { 1433 if (cmd_size >= offsetof(struct ibv_modify_qp_ex, rate_limit) + 1434 sizeof(cmd->rate_limit)) 1435 cmd->rate_limit = attr->rate_limit; 1436 else 1437 return EINVAL; 1438 } 1439 1440 if (write(qp->context->cmd_fd, cmd, cmd_size) != cmd_size) 1441 return errno; 1442 1443 (void)VALGRIND_MAKE_MEM_DEFINED(resp, resp_size); 1444 1445 return 0; 1446 } 1447 1448 int ibv_cmd_post_send(struct ibv_qp *ibqp, struct ibv_send_wr *wr, 1449 struct ibv_send_wr **bad_wr) 1450 { 1451 struct ibv_post_send *cmd; 1452 struct ibv_post_send_resp resp; 1453 struct ibv_send_wr *i; 1454 struct ibv_kern_send_wr *n, *tmp; 1455 struct ibv_sge *s; 1456 unsigned wr_count = 0; 1457 unsigned sge_count = 0; 1458 int cmd_size; 1459 int ret = 0; 1460 1461 for (i = wr; i; i = i->next) { 1462 wr_count++; 1463 sge_count += i->num_sge; 1464 } 1465 1466 cmd_size = sizeof *cmd + wr_count * sizeof *n + sge_count * sizeof *s; 1467 cmd = alloca(cmd_size); 1468 1469 IBV_INIT_CMD_RESP(cmd, cmd_size, POST_SEND, &resp, sizeof resp); 1470 cmd->qp_handle = ibqp->handle; 1471 cmd->wr_count = wr_count; 1472 cmd->sge_count = sge_count; 1473 cmd->wqe_size = sizeof *n; 1474 1475 n = (struct ibv_kern_send_wr *) ((void *) cmd + sizeof *cmd); 1476 s = (struct ibv_sge *) (n + wr_count); 1477 1478 tmp = n; 1479 for (i = wr; i; i = i->next) { 1480 tmp->wr_id = i->wr_id; 1481 tmp->num_sge = i->num_sge; 1482 tmp->opcode = i->opcode; 1483 tmp->send_flags = i->send_flags; 1484 tmp->imm_data = i->imm_data; 1485 if (ibqp->qp_type == IBV_QPT_UD) { 1486 tmp->wr.ud.ah = i->wr.ud.ah->handle; 1487 tmp->wr.ud.remote_qpn = i->wr.ud.remote_qpn; 1488 tmp->wr.ud.remote_qkey = i->wr.ud.remote_qkey; 1489 } else { 1490 switch (i->opcode) { 1491 case IBV_WR_RDMA_WRITE: 1492 case IBV_WR_RDMA_WRITE_WITH_IMM: 1493 case IBV_WR_RDMA_READ: 1494 tmp->wr.rdma.remote_addr = 1495 i->wr.rdma.remote_addr; 1496 tmp->wr.rdma.rkey = i->wr.rdma.rkey; 1497 break; 1498 case IBV_WR_ATOMIC_CMP_AND_SWP: 1499 case IBV_WR_ATOMIC_FETCH_AND_ADD: 1500 tmp->wr.atomic.remote_addr = 1501 i->wr.atomic.remote_addr; 1502 tmp->wr.atomic.compare_add = 1503 i->wr.atomic.compare_add; 1504 tmp->wr.atomic.swap = i->wr.atomic.swap; 1505 tmp->wr.atomic.rkey = i->wr.atomic.rkey; 1506 break; 1507 default: 1508 break; 1509 } 1510 } 1511 1512 if (tmp->num_sge) { 1513 memcpy(s, i->sg_list, tmp->num_sge * sizeof *s); 1514 s += tmp->num_sge; 1515 } 1516 1517 tmp++; 1518 } 1519 1520 resp.bad_wr = 0; 1521 if (write(ibqp->context->cmd_fd, cmd, cmd_size) != cmd_size) 1522 ret = errno; 1523 1524 (void) VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof resp); 1525 1526 wr_count = resp.bad_wr; 1527 if (wr_count) { 1528 i = wr; 1529 while (--wr_count) 1530 i = i->next; 1531 *bad_wr = i; 1532 } else if (ret) 1533 *bad_wr = wr; 1534 1535 return ret; 1536 } 1537 1538 int ibv_cmd_post_recv(struct ibv_qp *ibqp, struct ibv_recv_wr *wr, 1539 struct ibv_recv_wr **bad_wr) 1540 { 1541 struct ibv_post_recv *cmd; 1542 struct ibv_post_recv_resp resp; 1543 struct ibv_recv_wr *i; 1544 struct ibv_kern_recv_wr *n, *tmp; 1545 struct ibv_sge *s; 1546 unsigned wr_count = 0; 1547 unsigned sge_count = 0; 1548 int cmd_size; 1549 int ret = 0; 1550 1551 for (i = wr; i; i = i->next) { 1552 wr_count++; 1553 sge_count += i->num_sge; 1554 } 1555 1556 cmd_size = sizeof *cmd + wr_count * sizeof *n + sge_count * sizeof *s; 1557 cmd = alloca(cmd_size); 1558 1559 IBV_INIT_CMD_RESP(cmd, cmd_size, POST_RECV, &resp, sizeof resp); 1560 cmd->qp_handle = ibqp->handle; 1561 cmd->wr_count = wr_count; 1562 cmd->sge_count = sge_count; 1563 cmd->wqe_size = sizeof *n; 1564 1565 n = (struct ibv_kern_recv_wr *) ((void *) cmd + sizeof *cmd); 1566 s = (struct ibv_sge *) (n + wr_count); 1567 1568 tmp = n; 1569 for (i = wr; i; i = i->next) { 1570 tmp->wr_id = i->wr_id; 1571 tmp->num_sge = i->num_sge; 1572 1573 if (tmp->num_sge) { 1574 memcpy(s, i->sg_list, tmp->num_sge * sizeof *s); 1575 s += tmp->num_sge; 1576 } 1577 1578 tmp++; 1579 } 1580 1581 resp.bad_wr = 0; 1582 if (write(ibqp->context->cmd_fd, cmd, cmd_size) != cmd_size) 1583 ret = errno; 1584 1585 (void) VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof resp); 1586 1587 wr_count = resp.bad_wr; 1588 if (wr_count) { 1589 i = wr; 1590 while (--wr_count) 1591 i = i->next; 1592 *bad_wr = i; 1593 } else if (ret) 1594 *bad_wr = wr; 1595 1596 return ret; 1597 } 1598 1599 int ibv_cmd_post_srq_recv(struct ibv_srq *srq, struct ibv_recv_wr *wr, 1600 struct ibv_recv_wr **bad_wr) 1601 { 1602 struct ibv_post_srq_recv *cmd; 1603 struct ibv_post_srq_recv_resp resp; 1604 struct ibv_recv_wr *i; 1605 struct ibv_kern_recv_wr *n, *tmp; 1606 struct ibv_sge *s; 1607 unsigned wr_count = 0; 1608 unsigned sge_count = 0; 1609 int cmd_size; 1610 int ret = 0; 1611 1612 for (i = wr; i; i = i->next) { 1613 wr_count++; 1614 sge_count += i->num_sge; 1615 } 1616 1617 cmd_size = sizeof *cmd + wr_count * sizeof *n + sge_count * sizeof *s; 1618 cmd = alloca(cmd_size); 1619 1620 IBV_INIT_CMD_RESP(cmd, cmd_size, POST_SRQ_RECV, &resp, sizeof resp); 1621 cmd->srq_handle = srq->handle; 1622 cmd->wr_count = wr_count; 1623 cmd->sge_count = sge_count; 1624 cmd->wqe_size = sizeof *n; 1625 1626 n = (struct ibv_kern_recv_wr *) ((void *) cmd + sizeof *cmd); 1627 s = (struct ibv_sge *) (n + wr_count); 1628 1629 tmp = n; 1630 for (i = wr; i; i = i->next) { 1631 tmp->wr_id = i->wr_id; 1632 tmp->num_sge = i->num_sge; 1633 1634 if (tmp->num_sge) { 1635 memcpy(s, i->sg_list, tmp->num_sge * sizeof *s); 1636 s += tmp->num_sge; 1637 } 1638 1639 tmp++; 1640 } 1641 1642 resp.bad_wr = 0; 1643 if (write(srq->context->cmd_fd, cmd, cmd_size) != cmd_size) 1644 ret = errno; 1645 1646 (void) VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof resp); 1647 1648 wr_count = resp.bad_wr; 1649 if (wr_count) { 1650 i = wr; 1651 while (--wr_count) 1652 i = i->next; 1653 *bad_wr = i; 1654 } else if (ret) 1655 *bad_wr = wr; 1656 1657 return ret; 1658 } 1659 1660 int ibv_cmd_create_ah(struct ibv_pd *pd, struct ibv_ah *ah, 1661 struct ibv_ah_attr *attr, 1662 struct ibv_create_ah_resp *resp, 1663 size_t resp_size) 1664 { 1665 struct ibv_create_ah cmd; 1666 1667 IBV_INIT_CMD_RESP(&cmd, sizeof cmd, CREATE_AH, resp, resp_size); 1668 cmd.user_handle = (uintptr_t) ah; 1669 cmd.pd_handle = pd->handle; 1670 cmd.attr.dlid = attr->dlid; 1671 cmd.attr.sl = attr->sl; 1672 cmd.attr.src_path_bits = attr->src_path_bits; 1673 cmd.attr.static_rate = attr->static_rate; 1674 cmd.attr.is_global = attr->is_global; 1675 cmd.attr.port_num = attr->port_num; 1676 cmd.attr.grh.flow_label = attr->grh.flow_label; 1677 cmd.attr.grh.sgid_index = attr->grh.sgid_index; 1678 cmd.attr.grh.hop_limit = attr->grh.hop_limit; 1679 cmd.attr.grh.traffic_class = attr->grh.traffic_class; 1680 memcpy(cmd.attr.grh.dgid, attr->grh.dgid.raw, 16); 1681 1682 if (write(pd->context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd) 1683 return errno; 1684 1685 (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size); 1686 1687 ah->handle = resp->handle; 1688 ah->context = pd->context; 1689 1690 return 0; 1691 } 1692 1693 int ibv_cmd_destroy_ah(struct ibv_ah *ah) 1694 { 1695 struct ibv_destroy_ah cmd; 1696 1697 IBV_INIT_CMD(&cmd, sizeof cmd, DESTROY_AH); 1698 cmd.ah_handle = ah->handle; 1699 1700 if (write(ah->context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd) 1701 return errno; 1702 1703 return 0; 1704 } 1705 1706 int ibv_cmd_destroy_qp(struct ibv_qp *qp) 1707 { 1708 struct ibv_destroy_qp cmd; 1709 struct ibv_destroy_qp_resp resp; 1710 1711 IBV_INIT_CMD_RESP(&cmd, sizeof cmd, DESTROY_QP, &resp, sizeof resp); 1712 cmd.qp_handle = qp->handle; 1713 cmd.reserved = 0; 1714 1715 if (write(qp->context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd) 1716 return errno; 1717 1718 (void) VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof resp); 1719 1720 pthread_mutex_lock(&qp->mutex); 1721 while (qp->events_completed != resp.events_reported) 1722 pthread_cond_wait(&qp->cond, &qp->mutex); 1723 pthread_mutex_unlock(&qp->mutex); 1724 1725 pthread_cond_destroy(&qp->cond); 1726 pthread_mutex_destroy(&qp->mutex); 1727 1728 return 0; 1729 } 1730 1731 int ibv_cmd_attach_mcast(struct ibv_qp *qp, const union ibv_gid *gid, uint16_t lid) 1732 { 1733 struct ibv_attach_mcast cmd; 1734 1735 IBV_INIT_CMD(&cmd, sizeof cmd, ATTACH_MCAST); 1736 memcpy(cmd.gid, gid->raw, sizeof cmd.gid); 1737 cmd.qp_handle = qp->handle; 1738 cmd.mlid = lid; 1739 cmd.reserved = 0; 1740 1741 if (write(qp->context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd) 1742 return errno; 1743 1744 return 0; 1745 } 1746 1747 int ibv_cmd_detach_mcast(struct ibv_qp *qp, const union ibv_gid *gid, uint16_t lid) 1748 { 1749 struct ibv_detach_mcast cmd; 1750 1751 IBV_INIT_CMD(&cmd, sizeof cmd, DETACH_MCAST); 1752 memcpy(cmd.gid, gid->raw, sizeof cmd.gid); 1753 cmd.qp_handle = qp->handle; 1754 cmd.mlid = lid; 1755 cmd.reserved = 0; 1756 1757 if (write(qp->context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd) 1758 return errno; 1759 1760 return 0; 1761 } 1762 1763 static int buffer_is_zero(char *addr, ssize_t size) 1764 { 1765 return addr[0] == 0 && !memcmp(addr, addr + 1, size - 1); 1766 } 1767 1768 static int get_filters_size(struct ibv_flow_spec *ib_spec, 1769 struct ibv_kern_spec *kern_spec, 1770 int *ib_filter_size, int *kern_filter_size, 1771 enum ibv_flow_spec_type type) 1772 { 1773 void *ib_spec_filter_mask; 1774 int curr_kern_filter_size; 1775 int min_filter_size; 1776 1777 *ib_filter_size = (ib_spec->hdr.size - sizeof(ib_spec->hdr)) / 2; 1778 1779 switch (type) { 1780 case IBV_FLOW_SPEC_IPV4_EXT: 1781 min_filter_size = 1782 offsetof(struct ibv_kern_ipv4_ext_filter, flags) + 1783 sizeof(kern_spec->ipv4_ext.mask.flags); 1784 curr_kern_filter_size = min_filter_size; 1785 ib_spec_filter_mask = (void *)&ib_spec->ipv4_ext.val + 1786 *ib_filter_size; 1787 break; 1788 case IBV_FLOW_SPEC_IPV6: 1789 min_filter_size = 1790 offsetof(struct ibv_kern_ipv6_filter, hop_limit) + 1791 sizeof(kern_spec->ipv6.mask.hop_limit); 1792 curr_kern_filter_size = min_filter_size; 1793 ib_spec_filter_mask = (void *)&ib_spec->ipv6.val + 1794 *ib_filter_size; 1795 break; 1796 case IBV_FLOW_SPEC_VXLAN_TUNNEL: 1797 min_filter_size = 1798 offsetof(struct ibv_kern_tunnel_filter, 1799 tunnel_id) + 1800 sizeof(kern_spec->tunnel.mask.tunnel_id); 1801 curr_kern_filter_size = min_filter_size; 1802 ib_spec_filter_mask = (void *)&ib_spec->tunnel.val + 1803 *ib_filter_size; 1804 break; 1805 default: 1806 return EINVAL; 1807 } 1808 1809 if (*ib_filter_size < min_filter_size) 1810 return EINVAL; 1811 1812 if (*ib_filter_size > curr_kern_filter_size && 1813 !buffer_is_zero(ib_spec_filter_mask + curr_kern_filter_size, 1814 *ib_filter_size - curr_kern_filter_size)) 1815 return EOPNOTSUPP; 1816 1817 *kern_filter_size = min_t(int, curr_kern_filter_size, *ib_filter_size); 1818 1819 return 0; 1820 } 1821 1822 static int ib_spec_to_kern_spec(struct ibv_flow_spec *ib_spec, 1823 struct ibv_kern_spec *kern_spec) 1824 { 1825 int kern_filter_size; 1826 int ib_filter_size; 1827 int ret; 1828 1829 kern_spec->hdr.type = ib_spec->hdr.type; 1830 1831 switch (kern_spec->hdr.type) { 1832 case IBV_FLOW_SPEC_ETH: 1833 case IBV_FLOW_SPEC_ETH | IBV_FLOW_SPEC_INNER: 1834 kern_spec->eth.size = sizeof(struct ibv_kern_spec_eth); 1835 memcpy(&kern_spec->eth.val, &ib_spec->eth.val, 1836 sizeof(struct ibv_flow_eth_filter)); 1837 memcpy(&kern_spec->eth.mask, &ib_spec->eth.mask, 1838 sizeof(struct ibv_flow_eth_filter)); 1839 break; 1840 case IBV_FLOW_SPEC_IPV4: 1841 case IBV_FLOW_SPEC_IPV4 | IBV_FLOW_SPEC_INNER: 1842 kern_spec->ipv4.size = sizeof(struct ibv_kern_spec_ipv4); 1843 memcpy(&kern_spec->ipv4.val, &ib_spec->ipv4.val, 1844 sizeof(struct ibv_flow_ipv4_filter)); 1845 memcpy(&kern_spec->ipv4.mask, &ib_spec->ipv4.mask, 1846 sizeof(struct ibv_flow_ipv4_filter)); 1847 break; 1848 case IBV_FLOW_SPEC_IPV4_EXT: 1849 case IBV_FLOW_SPEC_IPV4_EXT | IBV_FLOW_SPEC_INNER: 1850 ret = get_filters_size(ib_spec, kern_spec, 1851 &ib_filter_size, &kern_filter_size, 1852 IBV_FLOW_SPEC_IPV4_EXT); 1853 if (ret) 1854 return ret; 1855 1856 kern_spec->hdr.type = IBV_FLOW_SPEC_IPV4 | 1857 (IBV_FLOW_SPEC_INNER & ib_spec->hdr.type); 1858 kern_spec->ipv4_ext.size = sizeof(struct 1859 ibv_kern_spec_ipv4_ext); 1860 memcpy(&kern_spec->ipv4_ext.val, &ib_spec->ipv4_ext.val, 1861 kern_filter_size); 1862 memcpy(&kern_spec->ipv4_ext.mask, (void *)&ib_spec->ipv4_ext.val 1863 + ib_filter_size, kern_filter_size); 1864 break; 1865 case IBV_FLOW_SPEC_IPV6: 1866 case IBV_FLOW_SPEC_IPV6 | IBV_FLOW_SPEC_INNER: 1867 ret = get_filters_size(ib_spec, kern_spec, 1868 &ib_filter_size, &kern_filter_size, 1869 IBV_FLOW_SPEC_IPV6); 1870 if (ret) 1871 return ret; 1872 1873 kern_spec->ipv6.size = sizeof(struct ibv_kern_spec_ipv6); 1874 memcpy(&kern_spec->ipv6.val, &ib_spec->ipv6.val, 1875 kern_filter_size); 1876 memcpy(&kern_spec->ipv6.mask, (void *)&ib_spec->ipv6.val 1877 + ib_filter_size, kern_filter_size); 1878 break; 1879 case IBV_FLOW_SPEC_TCP: 1880 case IBV_FLOW_SPEC_UDP: 1881 case IBV_FLOW_SPEC_TCP | IBV_FLOW_SPEC_INNER: 1882 case IBV_FLOW_SPEC_UDP | IBV_FLOW_SPEC_INNER: 1883 kern_spec->tcp_udp.size = sizeof(struct ibv_kern_spec_tcp_udp); 1884 memcpy(&kern_spec->tcp_udp.val, &ib_spec->tcp_udp.val, 1885 sizeof(struct ibv_flow_ipv4_filter)); 1886 memcpy(&kern_spec->tcp_udp.mask, &ib_spec->tcp_udp.mask, 1887 sizeof(struct ibv_flow_tcp_udp_filter)); 1888 break; 1889 case IBV_FLOW_SPEC_VXLAN_TUNNEL: 1890 ret = get_filters_size(ib_spec, kern_spec, 1891 &ib_filter_size, &kern_filter_size, 1892 IBV_FLOW_SPEC_VXLAN_TUNNEL); 1893 if (ret) 1894 return ret; 1895 1896 kern_spec->tunnel.size = sizeof(struct ibv_kern_spec_tunnel); 1897 memcpy(&kern_spec->tunnel.val, &ib_spec->tunnel.val, 1898 kern_filter_size); 1899 memcpy(&kern_spec->tunnel.mask, (void *)&ib_spec->tunnel.val 1900 + ib_filter_size, kern_filter_size); 1901 break; 1902 case IBV_FLOW_SPEC_ACTION_TAG: 1903 kern_spec->flow_tag.size = 1904 sizeof(struct ibv_kern_spec_action_tag); 1905 kern_spec->flow_tag.tag_id = ib_spec->flow_tag.tag_id; 1906 break; 1907 case IBV_FLOW_SPEC_ACTION_DROP: 1908 kern_spec->drop.size = sizeof(struct ibv_kern_spec_action_drop); 1909 break; 1910 default: 1911 return EINVAL; 1912 } 1913 return 0; 1914 } 1915 1916 struct ibv_flow *ibv_cmd_create_flow(struct ibv_qp *qp, 1917 struct ibv_flow_attr *flow_attr) 1918 { 1919 struct ibv_create_flow *cmd; 1920 struct ibv_create_flow_resp resp; 1921 struct ibv_flow *flow_id; 1922 size_t cmd_size; 1923 size_t written_size; 1924 int i, err; 1925 void *kern_spec; 1926 void *ib_spec; 1927 1928 cmd_size = sizeof(*cmd) + (flow_attr->num_of_specs * 1929 sizeof(struct ibv_kern_spec)); 1930 cmd = alloca(cmd_size); 1931 flow_id = malloc(sizeof(*flow_id)); 1932 if (!flow_id) 1933 return NULL; 1934 memset(cmd, 0, cmd_size); 1935 1936 cmd->qp_handle = qp->handle; 1937 1938 cmd->flow_attr.type = flow_attr->type; 1939 cmd->flow_attr.priority = flow_attr->priority; 1940 cmd->flow_attr.num_of_specs = flow_attr->num_of_specs; 1941 cmd->flow_attr.port = flow_attr->port; 1942 cmd->flow_attr.flags = flow_attr->flags; 1943 1944 kern_spec = cmd + 1; 1945 ib_spec = flow_attr + 1; 1946 for (i = 0; i < flow_attr->num_of_specs; i++) { 1947 err = ib_spec_to_kern_spec(ib_spec, kern_spec); 1948 if (err) { 1949 errno = err; 1950 goto err; 1951 } 1952 cmd->flow_attr.size += 1953 ((struct ibv_kern_spec *)kern_spec)->hdr.size; 1954 kern_spec += ((struct ibv_kern_spec *)kern_spec)->hdr.size; 1955 ib_spec += ((struct ibv_flow_spec *)ib_spec)->hdr.size; 1956 } 1957 1958 written_size = sizeof(*cmd) + cmd->flow_attr.size; 1959 IBV_INIT_CMD_RESP_EX_VCMD(cmd, written_size, written_size, CREATE_FLOW, 1960 &resp, sizeof(resp)); 1961 if (write(qp->context->cmd_fd, cmd, written_size) != written_size) 1962 goto err; 1963 1964 (void) VALGRIND_MAKE_MEM_DEFINED(&resp, sizeof(resp)); 1965 1966 flow_id->context = qp->context; 1967 flow_id->handle = resp.flow_handle; 1968 return flow_id; 1969 err: 1970 free(flow_id); 1971 return NULL; 1972 } 1973 1974 int ibv_cmd_destroy_flow(struct ibv_flow *flow_id) 1975 { 1976 struct ibv_destroy_flow cmd; 1977 int ret = 0; 1978 1979 memset(&cmd, 0, sizeof(cmd)); 1980 IBV_INIT_CMD_EX(&cmd, sizeof(cmd), DESTROY_FLOW); 1981 cmd.flow_handle = flow_id->handle; 1982 1983 if (write(flow_id->context->cmd_fd, &cmd, sizeof(cmd)) != sizeof(cmd)) 1984 ret = errno; 1985 free(flow_id); 1986 return ret; 1987 } 1988 1989 int ibv_cmd_create_wq(struct ibv_context *context, 1990 struct ibv_wq_init_attr *wq_init_attr, 1991 struct ibv_wq *wq, 1992 struct ibv_create_wq *cmd, 1993 size_t cmd_core_size, 1994 size_t cmd_size, 1995 struct ibv_create_wq_resp *resp, 1996 size_t resp_core_size, 1997 size_t resp_size) 1998 { 1999 int err; 2000 2001 if (wq_init_attr->comp_mask >= IBV_WQ_INIT_ATTR_RESERVED) 2002 return EINVAL; 2003 2004 IBV_INIT_CMD_RESP_EX_V(cmd, cmd_core_size, cmd_size, 2005 CREATE_WQ, resp, 2006 resp_core_size, resp_size); 2007 2008 cmd->user_handle = (uintptr_t)wq; 2009 cmd->pd_handle = wq_init_attr->pd->handle; 2010 cmd->cq_handle = wq_init_attr->cq->handle; 2011 cmd->wq_type = wq_init_attr->wq_type; 2012 cmd->max_sge = wq_init_attr->max_sge; 2013 cmd->max_wr = wq_init_attr->max_wr; 2014 cmd->comp_mask = 0; 2015 2016 if (cmd_core_size >= offsetof(struct ibv_create_wq, create_flags) + 2017 sizeof(cmd->create_flags)) { 2018 if (wq_init_attr->comp_mask & IBV_WQ_INIT_ATTR_FLAGS) { 2019 if (wq_init_attr->create_flags & ~(IBV_WQ_FLAGS_RESERVED - 1)) 2020 return EOPNOTSUPP; 2021 cmd->create_flags = wq_init_attr->create_flags; 2022 } 2023 } 2024 2025 err = write(context->cmd_fd, cmd, cmd_size); 2026 if (err != cmd_size) 2027 return errno; 2028 2029 (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size); 2030 2031 if (resp->response_length < resp_core_size) 2032 return EINVAL; 2033 2034 wq->handle = resp->wq_handle; 2035 wq_init_attr->max_wr = resp->max_wr; 2036 wq_init_attr->max_sge = resp->max_sge; 2037 wq->wq_num = resp->wqn; 2038 wq->context = context; 2039 wq->cq = wq_init_attr->cq; 2040 wq->pd = wq_init_attr->pd; 2041 wq->wq_type = wq_init_attr->wq_type; 2042 2043 return 0; 2044 } 2045 2046 int ibv_cmd_modify_wq(struct ibv_wq *wq, struct ibv_wq_attr *attr, 2047 struct ibv_modify_wq *cmd, size_t cmd_core_size, 2048 size_t cmd_size) 2049 { 2050 if (attr->attr_mask >= IBV_WQ_ATTR_RESERVED) 2051 return EINVAL; 2052 2053 memset(cmd, 0, cmd_core_size); 2054 IBV_INIT_CMD_EX(cmd, cmd_size, MODIFY_WQ); 2055 2056 cmd->curr_wq_state = attr->curr_wq_state; 2057 cmd->wq_state = attr->wq_state; 2058 if (cmd_core_size >= offsetof(struct ibv_modify_wq, flags_mask) + 2059 sizeof(cmd->flags_mask)) { 2060 if (attr->attr_mask & IBV_WQ_ATTR_FLAGS) { 2061 if (attr->flags_mask & ~(IBV_WQ_FLAGS_RESERVED - 1)) 2062 return EOPNOTSUPP; 2063 cmd->flags = attr->flags; 2064 cmd->flags_mask = attr->flags_mask; 2065 } 2066 } 2067 cmd->wq_handle = wq->handle; 2068 cmd->attr_mask = attr->attr_mask; 2069 2070 if (write(wq->context->cmd_fd, cmd, cmd_size) != cmd_size) 2071 return errno; 2072 2073 if (attr->attr_mask & IBV_WQ_ATTR_STATE) 2074 wq->state = attr->wq_state; 2075 2076 return 0; 2077 } 2078 2079 int ibv_cmd_destroy_wq(struct ibv_wq *wq) 2080 { 2081 struct ibv_destroy_wq cmd; 2082 struct ibv_destroy_wq_resp resp; 2083 int ret = 0; 2084 2085 memset(&cmd, 0, sizeof(cmd)); 2086 memset(&resp, 0, sizeof(resp)); 2087 2088 IBV_INIT_CMD_RESP_EX(&cmd, sizeof(cmd), DESTROY_WQ, &resp, sizeof(resp)); 2089 cmd.wq_handle = wq->handle; 2090 2091 if (write(wq->context->cmd_fd, &cmd, sizeof(cmd)) != sizeof(cmd)) 2092 return errno; 2093 2094 if (resp.response_length < sizeof(resp)) 2095 return EINVAL; 2096 2097 pthread_mutex_lock(&wq->mutex); 2098 while (wq->events_completed != resp.events_reported) 2099 pthread_cond_wait(&wq->cond, &wq->mutex); 2100 pthread_mutex_unlock(&wq->mutex); 2101 2102 return ret; 2103 } 2104 2105 int ibv_cmd_create_rwq_ind_table(struct ibv_context *context, 2106 struct ibv_rwq_ind_table_init_attr *init_attr, 2107 struct ibv_rwq_ind_table *rwq_ind_table, 2108 struct ibv_create_rwq_ind_table *cmd, 2109 size_t cmd_core_size, 2110 size_t cmd_size, 2111 struct ibv_create_rwq_ind_table_resp *resp, 2112 size_t resp_core_size, 2113 size_t resp_size) 2114 { 2115 int err, i; 2116 uint32_t required_tbl_size, alloc_tbl_size; 2117 uint32_t *tbl_start; 2118 int num_tbl_entries; 2119 2120 if (init_attr->comp_mask >= IBV_CREATE_IND_TABLE_RESERVED) 2121 return EINVAL; 2122 2123 alloc_tbl_size = cmd_core_size - sizeof(*cmd); 2124 num_tbl_entries = 1 << init_attr->log_ind_tbl_size; 2125 2126 /* Data must be u64 aligned */ 2127 required_tbl_size = (num_tbl_entries * sizeof(uint32_t)) < sizeof(uint64_t) ? 2128 sizeof(uint64_t) : (num_tbl_entries * sizeof(uint32_t)); 2129 2130 if (alloc_tbl_size < required_tbl_size) 2131 return EINVAL; 2132 2133 tbl_start = (uint32_t *)((uint8_t *)cmd + sizeof(*cmd)); 2134 for (i = 0; i < num_tbl_entries; i++) 2135 tbl_start[i] = init_attr->ind_tbl[i]->handle; 2136 2137 IBV_INIT_CMD_RESP_EX_V(cmd, cmd_core_size, cmd_size, 2138 CREATE_RWQ_IND_TBL, resp, 2139 resp_core_size, resp_size); 2140 cmd->log_ind_tbl_size = init_attr->log_ind_tbl_size; 2141 cmd->comp_mask = 0; 2142 2143 err = write(context->cmd_fd, cmd, cmd_size); 2144 if (err != cmd_size) 2145 return errno; 2146 2147 (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size); 2148 2149 if (resp->response_length < resp_core_size) 2150 return EINVAL; 2151 2152 rwq_ind_table->ind_tbl_handle = resp->ind_tbl_handle; 2153 rwq_ind_table->ind_tbl_num = resp->ind_tbl_num; 2154 rwq_ind_table->context = context; 2155 return 0; 2156 } 2157 2158 int ibv_cmd_destroy_rwq_ind_table(struct ibv_rwq_ind_table *rwq_ind_table) 2159 { 2160 struct ibv_destroy_rwq_ind_table cmd; 2161 int ret = 0; 2162 2163 memset(&cmd, 0, sizeof(cmd)); 2164 IBV_INIT_CMD_EX(&cmd, sizeof(cmd), DESTROY_RWQ_IND_TBL); 2165 cmd.ind_tbl_handle = rwq_ind_table->ind_tbl_handle; 2166 2167 if (write(rwq_ind_table->context->cmd_fd, &cmd, sizeof(cmd)) != sizeof(cmd)) 2168 ret = errno; 2169 2170 return ret; 2171 } 2172