1 /* 2 * Copyright (c) 2005 Topspin Communications. All rights reserved. 3 * Copyright (c) 2005, 2006, 2007 Cisco Systems. All rights reserved. 4 * Copyright (c) 2005 PathScale, Inc. All rights reserved. 5 * Copyright (c) 2006 Mellanox Technologies. All rights reserved. 6 * 7 * This software is available to you under a choice of one of two 8 * licenses. You may choose to be licensed under the terms of the GNU 9 * General Public License (GPL) Version 2, available from the file 10 * COPYING in the main directory of this source tree, or the 11 * OpenIB.org BSD license below: 12 * 13 * Redistribution and use in source and binary forms, with or 14 * without modification, are permitted provided that the following 15 * conditions are met: 16 * 17 * - Redistributions of source code must retain the above 18 * copyright notice, this list of conditions and the following 19 * disclaimer. 20 * 21 * - Redistributions in binary form must reproduce the above 22 * copyright notice, this list of conditions and the following 23 * disclaimer in the documentation and/or other materials 24 * provided with the distribution. 25 * 26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 27 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 28 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 29 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 30 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 31 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 32 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 33 * SOFTWARE. 34 */ 35 36 #include <linux/file.h> 37 #include <linux/fs.h> 38 #include <linux/slab.h> 39 #include <linux/sched.h> 40 41 #include <linux/uaccess.h> 42 43 #include <rdma/uverbs_types.h> 44 #include <rdma/uverbs_std_types.h> 45 #include "rdma_core.h" 46 47 #include "uverbs.h" 48 #include "core_priv.h" 49 50 static struct ib_uverbs_completion_event_file * 51 _ib_uverbs_lookup_comp_file(s32 fd, struct ib_uverbs_file *ufile) 52 { 53 struct ib_uobject *uobj = ufd_get_read(UVERBS_OBJECT_COMP_CHANNEL, 54 fd, ufile); 55 56 if (IS_ERR(uobj)) 57 return (void *)uobj; 58 59 uverbs_uobject_get(uobj); 60 uobj_put_read(uobj); 61 62 return container_of(uobj, struct ib_uverbs_completion_event_file, 63 uobj); 64 } 65 #define ib_uverbs_lookup_comp_file(_fd, _ufile) \ 66 _ib_uverbs_lookup_comp_file((_fd)*typecheck(s32, _fd), _ufile) 67 68 ssize_t ib_uverbs_get_context(struct ib_uverbs_file *file, 69 const char __user *buf, 70 int in_len, int out_len) 71 { 72 struct ib_uverbs_get_context cmd; 73 struct ib_uverbs_get_context_resp resp; 74 struct ib_udata udata; 75 struct ib_ucontext *ucontext; 76 struct file *filp; 77 struct ib_rdmacg_object cg_obj; 78 struct ib_device *ib_dev; 79 int ret; 80 81 if (out_len < sizeof resp) 82 return -ENOSPC; 83 84 if (copy_from_user(&cmd, buf, sizeof cmd)) 85 return -EFAULT; 86 87 mutex_lock(&file->ucontext_lock); 88 ib_dev = srcu_dereference(file->device->ib_dev, 89 &file->device->disassociate_srcu); 90 if (!ib_dev) { 91 ret = -EIO; 92 goto err; 93 } 94 95 if (file->ucontext) { 96 ret = -EINVAL; 97 goto err; 98 } 99 100 ib_uverbs_init_udata(&udata, buf + sizeof(cmd), 101 u64_to_user_ptr(cmd.response) + sizeof(resp), 102 in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr), 103 out_len - sizeof(resp)); 104 105 ret = ib_rdmacg_try_charge(&cg_obj, ib_dev, RDMACG_RESOURCE_HCA_HANDLE); 106 if (ret) 107 goto err; 108 109 ucontext = ib_dev->alloc_ucontext(ib_dev, &udata); 110 if (IS_ERR(ucontext)) { 111 ret = PTR_ERR(ucontext); 112 goto err_alloc; 113 } 114 115 ucontext->device = ib_dev; 116 ucontext->cg_obj = cg_obj; 117 /* ufile is required when some objects are released */ 118 ucontext->ufile = file; 119 120 rcu_read_lock(); 121 ucontext->tgid = get_task_pid(current->group_leader, PIDTYPE_PID); 122 rcu_read_unlock(); 123 ucontext->closing = 0; 124 ucontext->cleanup_retryable = false; 125 126 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING 127 ucontext->umem_tree = RB_ROOT_CACHED; 128 init_rwsem(&ucontext->umem_rwsem); 129 ucontext->odp_mrs_count = 0; 130 INIT_LIST_HEAD(&ucontext->no_private_counters); 131 132 if (!(ib_dev->attrs.device_cap_flags & IB_DEVICE_ON_DEMAND_PAGING)) 133 ucontext->invalidate_range = NULL; 134 135 #endif 136 137 resp.num_comp_vectors = file->device->num_comp_vectors; 138 139 ret = get_unused_fd_flags(O_CLOEXEC); 140 if (ret < 0) 141 goto err_free; 142 resp.async_fd = ret; 143 144 filp = ib_uverbs_alloc_async_event_file(file, ib_dev); 145 if (IS_ERR(filp)) { 146 ret = PTR_ERR(filp); 147 goto err_fd; 148 } 149 150 if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp)) { 151 ret = -EFAULT; 152 goto err_file; 153 } 154 155 fd_install(resp.async_fd, filp); 156 157 /* 158 * Make sure that ib_uverbs_get_ucontext() sees the pointer update 159 * only after all writes to setup the ucontext have completed 160 */ 161 smp_store_release(&file->ucontext, ucontext); 162 163 mutex_unlock(&file->ucontext_lock); 164 165 return in_len; 166 167 err_file: 168 ib_uverbs_free_async_event_file(file); 169 fput(filp); 170 171 err_fd: 172 put_unused_fd(resp.async_fd); 173 174 err_free: 175 put_pid(ucontext->tgid); 176 ib_dev->dealloc_ucontext(ucontext); 177 178 err_alloc: 179 ib_rdmacg_uncharge(&cg_obj, ib_dev, RDMACG_RESOURCE_HCA_HANDLE); 180 181 err: 182 mutex_unlock(&file->ucontext_lock); 183 return ret; 184 } 185 186 static void copy_query_dev_fields(struct ib_ucontext *ucontext, 187 struct ib_uverbs_query_device_resp *resp, 188 struct ib_device_attr *attr) 189 { 190 struct ib_device *ib_dev = ucontext->device; 191 192 resp->fw_ver = attr->fw_ver; 193 resp->node_guid = ib_dev->node_guid; 194 resp->sys_image_guid = attr->sys_image_guid; 195 resp->max_mr_size = attr->max_mr_size; 196 resp->page_size_cap = attr->page_size_cap; 197 resp->vendor_id = attr->vendor_id; 198 resp->vendor_part_id = attr->vendor_part_id; 199 resp->hw_ver = attr->hw_ver; 200 resp->max_qp = attr->max_qp; 201 resp->max_qp_wr = attr->max_qp_wr; 202 resp->device_cap_flags = lower_32_bits(attr->device_cap_flags); 203 resp->max_sge = min(attr->max_send_sge, attr->max_recv_sge); 204 resp->max_sge_rd = attr->max_sge_rd; 205 resp->max_cq = attr->max_cq; 206 resp->max_cqe = attr->max_cqe; 207 resp->max_mr = attr->max_mr; 208 resp->max_pd = attr->max_pd; 209 resp->max_qp_rd_atom = attr->max_qp_rd_atom; 210 resp->max_ee_rd_atom = attr->max_ee_rd_atom; 211 resp->max_res_rd_atom = attr->max_res_rd_atom; 212 resp->max_qp_init_rd_atom = attr->max_qp_init_rd_atom; 213 resp->max_ee_init_rd_atom = attr->max_ee_init_rd_atom; 214 resp->atomic_cap = attr->atomic_cap; 215 resp->max_ee = attr->max_ee; 216 resp->max_rdd = attr->max_rdd; 217 resp->max_mw = attr->max_mw; 218 resp->max_raw_ipv6_qp = attr->max_raw_ipv6_qp; 219 resp->max_raw_ethy_qp = attr->max_raw_ethy_qp; 220 resp->max_mcast_grp = attr->max_mcast_grp; 221 resp->max_mcast_qp_attach = attr->max_mcast_qp_attach; 222 resp->max_total_mcast_qp_attach = attr->max_total_mcast_qp_attach; 223 resp->max_ah = attr->max_ah; 224 resp->max_fmr = attr->max_fmr; 225 resp->max_map_per_fmr = attr->max_map_per_fmr; 226 resp->max_srq = attr->max_srq; 227 resp->max_srq_wr = attr->max_srq_wr; 228 resp->max_srq_sge = attr->max_srq_sge; 229 resp->max_pkeys = attr->max_pkeys; 230 resp->local_ca_ack_delay = attr->local_ca_ack_delay; 231 resp->phys_port_cnt = ib_dev->phys_port_cnt; 232 } 233 234 ssize_t ib_uverbs_query_device(struct ib_uverbs_file *file, 235 const char __user *buf, 236 int in_len, int out_len) 237 { 238 struct ib_uverbs_query_device cmd; 239 struct ib_uverbs_query_device_resp resp; 240 struct ib_ucontext *ucontext; 241 242 ucontext = ib_uverbs_get_ucontext(file); 243 if (IS_ERR(ucontext)) 244 return PTR_ERR(ucontext); 245 246 if (out_len < sizeof resp) 247 return -ENOSPC; 248 249 if (copy_from_user(&cmd, buf, sizeof cmd)) 250 return -EFAULT; 251 252 memset(&resp, 0, sizeof resp); 253 copy_query_dev_fields(ucontext, &resp, &ucontext->device->attrs); 254 255 if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp)) 256 return -EFAULT; 257 258 return in_len; 259 } 260 261 /* 262 * ib_uverbs_query_port_resp.port_cap_flags started out as just a copy of the 263 * PortInfo CapabilityMask, but was extended with unique bits. 264 */ 265 static u32 make_port_cap_flags(const struct ib_port_attr *attr) 266 { 267 u32 res; 268 269 /* All IBA CapabilityMask bits are passed through here, except bit 26, 270 * which is overridden with IP_BASED_GIDS. This is due to a historical 271 * mistake in the implementation of IP_BASED_GIDS. Otherwise all other 272 * bits match the IBA definition across all kernel versions. 273 */ 274 res = attr->port_cap_flags & ~(u32)IB_UVERBS_PCF_IP_BASED_GIDS; 275 276 if (attr->ip_gids) 277 res |= IB_UVERBS_PCF_IP_BASED_GIDS; 278 279 return res; 280 } 281 282 ssize_t ib_uverbs_query_port(struct ib_uverbs_file *file, 283 const char __user *buf, 284 int in_len, int out_len) 285 { 286 struct ib_uverbs_query_port cmd; 287 struct ib_uverbs_query_port_resp resp; 288 struct ib_port_attr attr; 289 int ret; 290 struct ib_ucontext *ucontext; 291 struct ib_device *ib_dev; 292 293 ucontext = ib_uverbs_get_ucontext(file); 294 if (IS_ERR(ucontext)) 295 return PTR_ERR(ucontext); 296 ib_dev = ucontext->device; 297 298 if (out_len < sizeof resp) 299 return -ENOSPC; 300 301 if (copy_from_user(&cmd, buf, sizeof cmd)) 302 return -EFAULT; 303 304 ret = ib_query_port(ib_dev, cmd.port_num, &attr); 305 if (ret) 306 return ret; 307 308 memset(&resp, 0, sizeof resp); 309 310 resp.state = attr.state; 311 resp.max_mtu = attr.max_mtu; 312 resp.active_mtu = attr.active_mtu; 313 resp.gid_tbl_len = attr.gid_tbl_len; 314 resp.port_cap_flags = make_port_cap_flags(&attr); 315 resp.max_msg_sz = attr.max_msg_sz; 316 resp.bad_pkey_cntr = attr.bad_pkey_cntr; 317 resp.qkey_viol_cntr = attr.qkey_viol_cntr; 318 resp.pkey_tbl_len = attr.pkey_tbl_len; 319 320 if (rdma_is_grh_required(ib_dev, cmd.port_num)) 321 resp.flags |= IB_UVERBS_QPF_GRH_REQUIRED; 322 323 if (rdma_cap_opa_ah(ib_dev, cmd.port_num)) { 324 resp.lid = OPA_TO_IB_UCAST_LID(attr.lid); 325 resp.sm_lid = OPA_TO_IB_UCAST_LID(attr.sm_lid); 326 } else { 327 resp.lid = ib_lid_cpu16(attr.lid); 328 resp.sm_lid = ib_lid_cpu16(attr.sm_lid); 329 } 330 resp.lmc = attr.lmc; 331 resp.max_vl_num = attr.max_vl_num; 332 resp.sm_sl = attr.sm_sl; 333 resp.subnet_timeout = attr.subnet_timeout; 334 resp.init_type_reply = attr.init_type_reply; 335 resp.active_width = attr.active_width; 336 resp.active_speed = attr.active_speed; 337 resp.phys_state = attr.phys_state; 338 resp.link_layer = rdma_port_get_link_layer(ib_dev, 339 cmd.port_num); 340 341 if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp)) 342 return -EFAULT; 343 344 return in_len; 345 } 346 347 ssize_t ib_uverbs_alloc_pd(struct ib_uverbs_file *file, 348 const char __user *buf, 349 int in_len, int out_len) 350 { 351 struct ib_uverbs_alloc_pd cmd; 352 struct ib_uverbs_alloc_pd_resp resp; 353 struct ib_udata udata; 354 struct ib_uobject *uobj; 355 struct ib_pd *pd; 356 int ret; 357 struct ib_device *ib_dev; 358 359 if (out_len < sizeof resp) 360 return -ENOSPC; 361 362 if (copy_from_user(&cmd, buf, sizeof cmd)) 363 return -EFAULT; 364 365 ib_uverbs_init_udata(&udata, buf + sizeof(cmd), 366 u64_to_user_ptr(cmd.response) + sizeof(resp), 367 in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr), 368 out_len - sizeof(resp)); 369 370 uobj = uobj_alloc(UVERBS_OBJECT_PD, file, &ib_dev); 371 if (IS_ERR(uobj)) 372 return PTR_ERR(uobj); 373 374 pd = ib_dev->alloc_pd(ib_dev, uobj->context, &udata); 375 if (IS_ERR(pd)) { 376 ret = PTR_ERR(pd); 377 goto err; 378 } 379 380 pd->device = ib_dev; 381 pd->uobject = uobj; 382 pd->__internal_mr = NULL; 383 atomic_set(&pd->usecnt, 0); 384 385 uobj->object = pd; 386 memset(&resp, 0, sizeof resp); 387 resp.pd_handle = uobj->id; 388 pd->res.type = RDMA_RESTRACK_PD; 389 rdma_restrack_add(&pd->res); 390 391 if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp)) { 392 ret = -EFAULT; 393 goto err_copy; 394 } 395 396 return uobj_alloc_commit(uobj, in_len); 397 398 err_copy: 399 ib_dealloc_pd(pd); 400 401 err: 402 uobj_alloc_abort(uobj); 403 return ret; 404 } 405 406 ssize_t ib_uverbs_dealloc_pd(struct ib_uverbs_file *file, 407 const char __user *buf, 408 int in_len, int out_len) 409 { 410 struct ib_uverbs_dealloc_pd cmd; 411 412 if (copy_from_user(&cmd, buf, sizeof cmd)) 413 return -EFAULT; 414 415 return uobj_perform_destroy(UVERBS_OBJECT_PD, cmd.pd_handle, file, 416 in_len); 417 } 418 419 struct xrcd_table_entry { 420 struct rb_node node; 421 struct ib_xrcd *xrcd; 422 struct inode *inode; 423 }; 424 425 static int xrcd_table_insert(struct ib_uverbs_device *dev, 426 struct inode *inode, 427 struct ib_xrcd *xrcd) 428 { 429 struct xrcd_table_entry *entry, *scan; 430 struct rb_node **p = &dev->xrcd_tree.rb_node; 431 struct rb_node *parent = NULL; 432 433 entry = kmalloc(sizeof *entry, GFP_KERNEL); 434 if (!entry) 435 return -ENOMEM; 436 437 entry->xrcd = xrcd; 438 entry->inode = inode; 439 440 while (*p) { 441 parent = *p; 442 scan = rb_entry(parent, struct xrcd_table_entry, node); 443 444 if (inode < scan->inode) { 445 p = &(*p)->rb_left; 446 } else if (inode > scan->inode) { 447 p = &(*p)->rb_right; 448 } else { 449 kfree(entry); 450 return -EEXIST; 451 } 452 } 453 454 rb_link_node(&entry->node, parent, p); 455 rb_insert_color(&entry->node, &dev->xrcd_tree); 456 igrab(inode); 457 return 0; 458 } 459 460 static struct xrcd_table_entry *xrcd_table_search(struct ib_uverbs_device *dev, 461 struct inode *inode) 462 { 463 struct xrcd_table_entry *entry; 464 struct rb_node *p = dev->xrcd_tree.rb_node; 465 466 while (p) { 467 entry = rb_entry(p, struct xrcd_table_entry, node); 468 469 if (inode < entry->inode) 470 p = p->rb_left; 471 else if (inode > entry->inode) 472 p = p->rb_right; 473 else 474 return entry; 475 } 476 477 return NULL; 478 } 479 480 static struct ib_xrcd *find_xrcd(struct ib_uverbs_device *dev, struct inode *inode) 481 { 482 struct xrcd_table_entry *entry; 483 484 entry = xrcd_table_search(dev, inode); 485 if (!entry) 486 return NULL; 487 488 return entry->xrcd; 489 } 490 491 static void xrcd_table_delete(struct ib_uverbs_device *dev, 492 struct inode *inode) 493 { 494 struct xrcd_table_entry *entry; 495 496 entry = xrcd_table_search(dev, inode); 497 if (entry) { 498 iput(inode); 499 rb_erase(&entry->node, &dev->xrcd_tree); 500 kfree(entry); 501 } 502 } 503 504 ssize_t ib_uverbs_open_xrcd(struct ib_uverbs_file *file, 505 const char __user *buf, int in_len, 506 int out_len) 507 { 508 struct ib_uverbs_open_xrcd cmd; 509 struct ib_uverbs_open_xrcd_resp resp; 510 struct ib_udata udata; 511 struct ib_uxrcd_object *obj; 512 struct ib_xrcd *xrcd = NULL; 513 struct fd f = {NULL, 0}; 514 struct inode *inode = NULL; 515 int ret = 0; 516 int new_xrcd = 0; 517 struct ib_device *ib_dev; 518 519 if (out_len < sizeof resp) 520 return -ENOSPC; 521 522 if (copy_from_user(&cmd, buf, sizeof cmd)) 523 return -EFAULT; 524 525 ib_uverbs_init_udata(&udata, buf + sizeof(cmd), 526 u64_to_user_ptr(cmd.response) + sizeof(resp), 527 in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr), 528 out_len - sizeof(resp)); 529 530 mutex_lock(&file->device->xrcd_tree_mutex); 531 532 if (cmd.fd != -1) { 533 /* search for file descriptor */ 534 f = fdget(cmd.fd); 535 if (!f.file) { 536 ret = -EBADF; 537 goto err_tree_mutex_unlock; 538 } 539 540 inode = file_inode(f.file); 541 xrcd = find_xrcd(file->device, inode); 542 if (!xrcd && !(cmd.oflags & O_CREAT)) { 543 /* no file descriptor. Need CREATE flag */ 544 ret = -EAGAIN; 545 goto err_tree_mutex_unlock; 546 } 547 548 if (xrcd && cmd.oflags & O_EXCL) { 549 ret = -EINVAL; 550 goto err_tree_mutex_unlock; 551 } 552 } 553 554 obj = (struct ib_uxrcd_object *)uobj_alloc(UVERBS_OBJECT_XRCD, file, 555 &ib_dev); 556 if (IS_ERR(obj)) { 557 ret = PTR_ERR(obj); 558 goto err_tree_mutex_unlock; 559 } 560 561 if (!xrcd) { 562 xrcd = ib_dev->alloc_xrcd(ib_dev, obj->uobject.context, &udata); 563 if (IS_ERR(xrcd)) { 564 ret = PTR_ERR(xrcd); 565 goto err; 566 } 567 568 xrcd->inode = inode; 569 xrcd->device = ib_dev; 570 atomic_set(&xrcd->usecnt, 0); 571 mutex_init(&xrcd->tgt_qp_mutex); 572 INIT_LIST_HEAD(&xrcd->tgt_qp_list); 573 new_xrcd = 1; 574 } 575 576 atomic_set(&obj->refcnt, 0); 577 obj->uobject.object = xrcd; 578 memset(&resp, 0, sizeof resp); 579 resp.xrcd_handle = obj->uobject.id; 580 581 if (inode) { 582 if (new_xrcd) { 583 /* create new inode/xrcd table entry */ 584 ret = xrcd_table_insert(file->device, inode, xrcd); 585 if (ret) 586 goto err_dealloc_xrcd; 587 } 588 atomic_inc(&xrcd->usecnt); 589 } 590 591 if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp)) { 592 ret = -EFAULT; 593 goto err_copy; 594 } 595 596 if (f.file) 597 fdput(f); 598 599 mutex_unlock(&file->device->xrcd_tree_mutex); 600 601 return uobj_alloc_commit(&obj->uobject, in_len); 602 603 err_copy: 604 if (inode) { 605 if (new_xrcd) 606 xrcd_table_delete(file->device, inode); 607 atomic_dec(&xrcd->usecnt); 608 } 609 610 err_dealloc_xrcd: 611 ib_dealloc_xrcd(xrcd); 612 613 err: 614 uobj_alloc_abort(&obj->uobject); 615 616 err_tree_mutex_unlock: 617 if (f.file) 618 fdput(f); 619 620 mutex_unlock(&file->device->xrcd_tree_mutex); 621 622 return ret; 623 } 624 625 ssize_t ib_uverbs_close_xrcd(struct ib_uverbs_file *file, 626 const char __user *buf, int in_len, 627 int out_len) 628 { 629 struct ib_uverbs_close_xrcd cmd; 630 631 if (copy_from_user(&cmd, buf, sizeof cmd)) 632 return -EFAULT; 633 634 return uobj_perform_destroy(UVERBS_OBJECT_XRCD, cmd.xrcd_handle, file, 635 in_len); 636 } 637 638 int ib_uverbs_dealloc_xrcd(struct ib_uobject *uobject, 639 struct ib_xrcd *xrcd, 640 enum rdma_remove_reason why) 641 { 642 struct inode *inode; 643 int ret; 644 struct ib_uverbs_device *dev = uobject->context->ufile->device; 645 646 inode = xrcd->inode; 647 if (inode && !atomic_dec_and_test(&xrcd->usecnt)) 648 return 0; 649 650 ret = ib_dealloc_xrcd(xrcd); 651 652 if (ib_is_destroy_retryable(ret, why, uobject)) { 653 atomic_inc(&xrcd->usecnt); 654 return ret; 655 } 656 657 if (inode) 658 xrcd_table_delete(dev, inode); 659 660 return ret; 661 } 662 663 ssize_t ib_uverbs_reg_mr(struct ib_uverbs_file *file, 664 const char __user *buf, int in_len, 665 int out_len) 666 { 667 struct ib_uverbs_reg_mr cmd; 668 struct ib_uverbs_reg_mr_resp resp; 669 struct ib_udata udata; 670 struct ib_uobject *uobj; 671 struct ib_pd *pd; 672 struct ib_mr *mr; 673 int ret; 674 struct ib_device *ib_dev; 675 676 if (out_len < sizeof resp) 677 return -ENOSPC; 678 679 if (copy_from_user(&cmd, buf, sizeof cmd)) 680 return -EFAULT; 681 682 ib_uverbs_init_udata(&udata, buf + sizeof(cmd), 683 u64_to_user_ptr(cmd.response) + sizeof(resp), 684 in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr), 685 out_len - sizeof(resp)); 686 687 if ((cmd.start & ~PAGE_MASK) != (cmd.hca_va & ~PAGE_MASK)) 688 return -EINVAL; 689 690 ret = ib_check_mr_access(cmd.access_flags); 691 if (ret) 692 return ret; 693 694 uobj = uobj_alloc(UVERBS_OBJECT_MR, file, &ib_dev); 695 if (IS_ERR(uobj)) 696 return PTR_ERR(uobj); 697 698 pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, file); 699 if (!pd) { 700 ret = -EINVAL; 701 goto err_free; 702 } 703 704 if (cmd.access_flags & IB_ACCESS_ON_DEMAND) { 705 if (!(pd->device->attrs.device_cap_flags & 706 IB_DEVICE_ON_DEMAND_PAGING)) { 707 pr_debug("ODP support not available\n"); 708 ret = -EINVAL; 709 goto err_put; 710 } 711 } 712 713 mr = pd->device->reg_user_mr(pd, cmd.start, cmd.length, cmd.hca_va, 714 cmd.access_flags, &udata); 715 if (IS_ERR(mr)) { 716 ret = PTR_ERR(mr); 717 goto err_put; 718 } 719 720 mr->device = pd->device; 721 mr->pd = pd; 722 mr->dm = NULL; 723 mr->uobject = uobj; 724 atomic_inc(&pd->usecnt); 725 mr->res.type = RDMA_RESTRACK_MR; 726 rdma_restrack_add(&mr->res); 727 728 uobj->object = mr; 729 730 memset(&resp, 0, sizeof resp); 731 resp.lkey = mr->lkey; 732 resp.rkey = mr->rkey; 733 resp.mr_handle = uobj->id; 734 735 if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp)) { 736 ret = -EFAULT; 737 goto err_copy; 738 } 739 740 uobj_put_obj_read(pd); 741 742 return uobj_alloc_commit(uobj, in_len); 743 744 err_copy: 745 ib_dereg_mr(mr); 746 747 err_put: 748 uobj_put_obj_read(pd); 749 750 err_free: 751 uobj_alloc_abort(uobj); 752 return ret; 753 } 754 755 ssize_t ib_uverbs_rereg_mr(struct ib_uverbs_file *file, 756 const char __user *buf, int in_len, 757 int out_len) 758 { 759 struct ib_uverbs_rereg_mr cmd; 760 struct ib_uverbs_rereg_mr_resp resp; 761 struct ib_udata udata; 762 struct ib_pd *pd = NULL; 763 struct ib_mr *mr; 764 struct ib_pd *old_pd; 765 int ret; 766 struct ib_uobject *uobj; 767 768 if (out_len < sizeof(resp)) 769 return -ENOSPC; 770 771 if (copy_from_user(&cmd, buf, sizeof(cmd))) 772 return -EFAULT; 773 774 ib_uverbs_init_udata(&udata, buf + sizeof(cmd), 775 u64_to_user_ptr(cmd.response) + sizeof(resp), 776 in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr), 777 out_len - sizeof(resp)); 778 779 if (cmd.flags & ~IB_MR_REREG_SUPPORTED || !cmd.flags) 780 return -EINVAL; 781 782 if ((cmd.flags & IB_MR_REREG_TRANS) && 783 (!cmd.start || !cmd.hca_va || 0 >= cmd.length || 784 (cmd.start & ~PAGE_MASK) != (cmd.hca_va & ~PAGE_MASK))) 785 return -EINVAL; 786 787 uobj = uobj_get_write(UVERBS_OBJECT_MR, cmd.mr_handle, file); 788 if (IS_ERR(uobj)) 789 return PTR_ERR(uobj); 790 791 mr = uobj->object; 792 793 if (mr->dm) { 794 ret = -EINVAL; 795 goto put_uobjs; 796 } 797 798 if (cmd.flags & IB_MR_REREG_ACCESS) { 799 ret = ib_check_mr_access(cmd.access_flags); 800 if (ret) 801 goto put_uobjs; 802 } 803 804 if (cmd.flags & IB_MR_REREG_PD) { 805 pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, 806 file); 807 if (!pd) { 808 ret = -EINVAL; 809 goto put_uobjs; 810 } 811 } 812 813 old_pd = mr->pd; 814 ret = mr->device->rereg_user_mr(mr, cmd.flags, cmd.start, 815 cmd.length, cmd.hca_va, 816 cmd.access_flags, pd, &udata); 817 if (!ret) { 818 if (cmd.flags & IB_MR_REREG_PD) { 819 atomic_inc(&pd->usecnt); 820 mr->pd = pd; 821 atomic_dec(&old_pd->usecnt); 822 } 823 } else { 824 goto put_uobj_pd; 825 } 826 827 memset(&resp, 0, sizeof(resp)); 828 resp.lkey = mr->lkey; 829 resp.rkey = mr->rkey; 830 831 if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof(resp))) 832 ret = -EFAULT; 833 else 834 ret = in_len; 835 836 put_uobj_pd: 837 if (cmd.flags & IB_MR_REREG_PD) 838 uobj_put_obj_read(pd); 839 840 put_uobjs: 841 uobj_put_write(uobj); 842 843 return ret; 844 } 845 846 ssize_t ib_uverbs_dereg_mr(struct ib_uverbs_file *file, 847 const char __user *buf, int in_len, 848 int out_len) 849 { 850 struct ib_uverbs_dereg_mr cmd; 851 852 if (copy_from_user(&cmd, buf, sizeof cmd)) 853 return -EFAULT; 854 855 return uobj_perform_destroy(UVERBS_OBJECT_MR, cmd.mr_handle, file, 856 in_len); 857 } 858 859 ssize_t ib_uverbs_alloc_mw(struct ib_uverbs_file *file, 860 const char __user *buf, int in_len, 861 int out_len) 862 { 863 struct ib_uverbs_alloc_mw cmd; 864 struct ib_uverbs_alloc_mw_resp resp; 865 struct ib_uobject *uobj; 866 struct ib_pd *pd; 867 struct ib_mw *mw; 868 struct ib_udata udata; 869 int ret; 870 struct ib_device *ib_dev; 871 872 if (out_len < sizeof(resp)) 873 return -ENOSPC; 874 875 if (copy_from_user(&cmd, buf, sizeof(cmd))) 876 return -EFAULT; 877 878 uobj = uobj_alloc(UVERBS_OBJECT_MW, file, &ib_dev); 879 if (IS_ERR(uobj)) 880 return PTR_ERR(uobj); 881 882 pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, file); 883 if (!pd) { 884 ret = -EINVAL; 885 goto err_free; 886 } 887 888 ib_uverbs_init_udata(&udata, buf + sizeof(cmd), 889 u64_to_user_ptr(cmd.response) + sizeof(resp), 890 in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr), 891 out_len - sizeof(resp)); 892 893 mw = pd->device->alloc_mw(pd, cmd.mw_type, &udata); 894 if (IS_ERR(mw)) { 895 ret = PTR_ERR(mw); 896 goto err_put; 897 } 898 899 mw->device = pd->device; 900 mw->pd = pd; 901 mw->uobject = uobj; 902 atomic_inc(&pd->usecnt); 903 904 uobj->object = mw; 905 906 memset(&resp, 0, sizeof(resp)); 907 resp.rkey = mw->rkey; 908 resp.mw_handle = uobj->id; 909 910 if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof(resp))) { 911 ret = -EFAULT; 912 goto err_copy; 913 } 914 915 uobj_put_obj_read(pd); 916 return uobj_alloc_commit(uobj, in_len); 917 918 err_copy: 919 uverbs_dealloc_mw(mw); 920 err_put: 921 uobj_put_obj_read(pd); 922 err_free: 923 uobj_alloc_abort(uobj); 924 return ret; 925 } 926 927 ssize_t ib_uverbs_dealloc_mw(struct ib_uverbs_file *file, 928 const char __user *buf, int in_len, 929 int out_len) 930 { 931 struct ib_uverbs_dealloc_mw cmd; 932 933 if (copy_from_user(&cmd, buf, sizeof(cmd))) 934 return -EFAULT; 935 936 return uobj_perform_destroy(UVERBS_OBJECT_MW, cmd.mw_handle, file, 937 in_len); 938 } 939 940 ssize_t ib_uverbs_create_comp_channel(struct ib_uverbs_file *file, 941 const char __user *buf, int in_len, 942 int out_len) 943 { 944 struct ib_uverbs_create_comp_channel cmd; 945 struct ib_uverbs_create_comp_channel_resp resp; 946 struct ib_uobject *uobj; 947 struct ib_uverbs_completion_event_file *ev_file; 948 struct ib_device *ib_dev; 949 950 if (out_len < sizeof resp) 951 return -ENOSPC; 952 953 if (copy_from_user(&cmd, buf, sizeof cmd)) 954 return -EFAULT; 955 956 uobj = uobj_alloc(UVERBS_OBJECT_COMP_CHANNEL, file, &ib_dev); 957 if (IS_ERR(uobj)) 958 return PTR_ERR(uobj); 959 960 resp.fd = uobj->id; 961 962 ev_file = container_of(uobj, struct ib_uverbs_completion_event_file, 963 uobj); 964 ib_uverbs_init_event_queue(&ev_file->ev_queue); 965 966 if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp)) { 967 uobj_alloc_abort(uobj); 968 return -EFAULT; 969 } 970 971 return uobj_alloc_commit(uobj, in_len); 972 } 973 974 static struct ib_ucq_object *create_cq(struct ib_uverbs_file *file, 975 struct ib_udata *ucore, 976 struct ib_udata *uhw, 977 struct ib_uverbs_ex_create_cq *cmd, 978 size_t cmd_sz, 979 int (*cb)(struct ib_uverbs_file *file, 980 struct ib_ucq_object *obj, 981 struct ib_uverbs_ex_create_cq_resp *resp, 982 struct ib_udata *udata, 983 void *context), 984 void *context) 985 { 986 struct ib_ucq_object *obj; 987 struct ib_uverbs_completion_event_file *ev_file = NULL; 988 struct ib_cq *cq; 989 int ret; 990 struct ib_uverbs_ex_create_cq_resp resp; 991 struct ib_cq_init_attr attr = {}; 992 struct ib_device *ib_dev; 993 994 if (cmd->comp_vector >= file->device->num_comp_vectors) 995 return ERR_PTR(-EINVAL); 996 997 obj = (struct ib_ucq_object *)uobj_alloc(UVERBS_OBJECT_CQ, file, 998 &ib_dev); 999 if (IS_ERR(obj)) 1000 return obj; 1001 1002 if (!ib_dev->create_cq) { 1003 ret = -EOPNOTSUPP; 1004 goto err; 1005 } 1006 1007 if (cmd->comp_channel >= 0) { 1008 ev_file = ib_uverbs_lookup_comp_file(cmd->comp_channel, file); 1009 if (IS_ERR(ev_file)) { 1010 ret = PTR_ERR(ev_file); 1011 goto err; 1012 } 1013 } 1014 1015 obj->uobject.user_handle = cmd->user_handle; 1016 obj->comp_events_reported = 0; 1017 obj->async_events_reported = 0; 1018 INIT_LIST_HEAD(&obj->comp_list); 1019 INIT_LIST_HEAD(&obj->async_list); 1020 1021 attr.cqe = cmd->cqe; 1022 attr.comp_vector = cmd->comp_vector; 1023 1024 if (cmd_sz > offsetof(typeof(*cmd), flags) + sizeof(cmd->flags)) 1025 attr.flags = cmd->flags; 1026 1027 cq = ib_dev->create_cq(ib_dev, &attr, obj->uobject.context, uhw); 1028 if (IS_ERR(cq)) { 1029 ret = PTR_ERR(cq); 1030 goto err_file; 1031 } 1032 1033 cq->device = ib_dev; 1034 cq->uobject = &obj->uobject; 1035 cq->comp_handler = ib_uverbs_comp_handler; 1036 cq->event_handler = ib_uverbs_cq_event_handler; 1037 cq->cq_context = ev_file ? &ev_file->ev_queue : NULL; 1038 atomic_set(&cq->usecnt, 0); 1039 1040 obj->uobject.object = cq; 1041 memset(&resp, 0, sizeof resp); 1042 resp.base.cq_handle = obj->uobject.id; 1043 resp.base.cqe = cq->cqe; 1044 1045 resp.response_length = offsetof(typeof(resp), response_length) + 1046 sizeof(resp.response_length); 1047 1048 cq->res.type = RDMA_RESTRACK_CQ; 1049 rdma_restrack_add(&cq->res); 1050 1051 ret = cb(file, obj, &resp, ucore, context); 1052 if (ret) 1053 goto err_cb; 1054 1055 ret = uobj_alloc_commit(&obj->uobject, 0); 1056 if (ret) 1057 return ERR_PTR(ret); 1058 return obj; 1059 1060 err_cb: 1061 ib_destroy_cq(cq); 1062 1063 err_file: 1064 if (ev_file) 1065 ib_uverbs_release_ucq(file, ev_file, obj); 1066 1067 err: 1068 uobj_alloc_abort(&obj->uobject); 1069 1070 return ERR_PTR(ret); 1071 } 1072 1073 static int ib_uverbs_create_cq_cb(struct ib_uverbs_file *file, 1074 struct ib_ucq_object *obj, 1075 struct ib_uverbs_ex_create_cq_resp *resp, 1076 struct ib_udata *ucore, void *context) 1077 { 1078 if (ib_copy_to_udata(ucore, &resp->base, sizeof(resp->base))) 1079 return -EFAULT; 1080 1081 return 0; 1082 } 1083 1084 ssize_t ib_uverbs_create_cq(struct ib_uverbs_file *file, 1085 const char __user *buf, int in_len, 1086 int out_len) 1087 { 1088 struct ib_uverbs_create_cq cmd; 1089 struct ib_uverbs_ex_create_cq cmd_ex; 1090 struct ib_uverbs_create_cq_resp resp; 1091 struct ib_udata ucore; 1092 struct ib_udata uhw; 1093 struct ib_ucq_object *obj; 1094 1095 if (out_len < sizeof(resp)) 1096 return -ENOSPC; 1097 1098 if (copy_from_user(&cmd, buf, sizeof(cmd))) 1099 return -EFAULT; 1100 1101 ib_uverbs_init_udata(&ucore, buf, u64_to_user_ptr(cmd.response), 1102 sizeof(cmd), sizeof(resp)); 1103 1104 ib_uverbs_init_udata(&uhw, buf + sizeof(cmd), 1105 u64_to_user_ptr(cmd.response) + sizeof(resp), 1106 in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr), 1107 out_len - sizeof(resp)); 1108 1109 memset(&cmd_ex, 0, sizeof(cmd_ex)); 1110 cmd_ex.user_handle = cmd.user_handle; 1111 cmd_ex.cqe = cmd.cqe; 1112 cmd_ex.comp_vector = cmd.comp_vector; 1113 cmd_ex.comp_channel = cmd.comp_channel; 1114 1115 obj = create_cq(file, &ucore, &uhw, &cmd_ex, 1116 offsetof(typeof(cmd_ex), comp_channel) + 1117 sizeof(cmd.comp_channel), ib_uverbs_create_cq_cb, 1118 NULL); 1119 1120 if (IS_ERR(obj)) 1121 return PTR_ERR(obj); 1122 1123 return in_len; 1124 } 1125 1126 static int ib_uverbs_ex_create_cq_cb(struct ib_uverbs_file *file, 1127 struct ib_ucq_object *obj, 1128 struct ib_uverbs_ex_create_cq_resp *resp, 1129 struct ib_udata *ucore, void *context) 1130 { 1131 if (ib_copy_to_udata(ucore, resp, resp->response_length)) 1132 return -EFAULT; 1133 1134 return 0; 1135 } 1136 1137 int ib_uverbs_ex_create_cq(struct ib_uverbs_file *file, 1138 struct ib_udata *ucore, 1139 struct ib_udata *uhw) 1140 { 1141 struct ib_uverbs_ex_create_cq_resp resp; 1142 struct ib_uverbs_ex_create_cq cmd; 1143 struct ib_ucq_object *obj; 1144 int err; 1145 1146 if (ucore->inlen < sizeof(cmd)) 1147 return -EINVAL; 1148 1149 err = ib_copy_from_udata(&cmd, ucore, sizeof(cmd)); 1150 if (err) 1151 return err; 1152 1153 if (cmd.comp_mask) 1154 return -EINVAL; 1155 1156 if (cmd.reserved) 1157 return -EINVAL; 1158 1159 if (ucore->outlen < (offsetof(typeof(resp), response_length) + 1160 sizeof(resp.response_length))) 1161 return -ENOSPC; 1162 1163 obj = create_cq(file, ucore, uhw, &cmd, 1164 min(ucore->inlen, sizeof(cmd)), 1165 ib_uverbs_ex_create_cq_cb, NULL); 1166 1167 return PTR_ERR_OR_ZERO(obj); 1168 } 1169 1170 ssize_t ib_uverbs_resize_cq(struct ib_uverbs_file *file, 1171 const char __user *buf, int in_len, 1172 int out_len) 1173 { 1174 struct ib_uverbs_resize_cq cmd; 1175 struct ib_uverbs_resize_cq_resp resp = {}; 1176 struct ib_udata udata; 1177 struct ib_cq *cq; 1178 int ret = -EINVAL; 1179 1180 if (copy_from_user(&cmd, buf, sizeof cmd)) 1181 return -EFAULT; 1182 1183 ib_uverbs_init_udata(&udata, buf + sizeof(cmd), 1184 u64_to_user_ptr(cmd.response) + sizeof(resp), 1185 in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr), 1186 out_len - sizeof(resp)); 1187 1188 cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, file); 1189 if (!cq) 1190 return -EINVAL; 1191 1192 ret = cq->device->resize_cq(cq, cmd.cqe, &udata); 1193 if (ret) 1194 goto out; 1195 1196 resp.cqe = cq->cqe; 1197 1198 if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp.cqe)) 1199 ret = -EFAULT; 1200 1201 out: 1202 uobj_put_obj_read(cq); 1203 1204 return ret ? ret : in_len; 1205 } 1206 1207 static int copy_wc_to_user(struct ib_device *ib_dev, void __user *dest, 1208 struct ib_wc *wc) 1209 { 1210 struct ib_uverbs_wc tmp; 1211 1212 tmp.wr_id = wc->wr_id; 1213 tmp.status = wc->status; 1214 tmp.opcode = wc->opcode; 1215 tmp.vendor_err = wc->vendor_err; 1216 tmp.byte_len = wc->byte_len; 1217 tmp.ex.imm_data = wc->ex.imm_data; 1218 tmp.qp_num = wc->qp->qp_num; 1219 tmp.src_qp = wc->src_qp; 1220 tmp.wc_flags = wc->wc_flags; 1221 tmp.pkey_index = wc->pkey_index; 1222 if (rdma_cap_opa_ah(ib_dev, wc->port_num)) 1223 tmp.slid = OPA_TO_IB_UCAST_LID(wc->slid); 1224 else 1225 tmp.slid = ib_lid_cpu16(wc->slid); 1226 tmp.sl = wc->sl; 1227 tmp.dlid_path_bits = wc->dlid_path_bits; 1228 tmp.port_num = wc->port_num; 1229 tmp.reserved = 0; 1230 1231 if (copy_to_user(dest, &tmp, sizeof tmp)) 1232 return -EFAULT; 1233 1234 return 0; 1235 } 1236 1237 ssize_t ib_uverbs_poll_cq(struct ib_uverbs_file *file, 1238 const char __user *buf, int in_len, 1239 int out_len) 1240 { 1241 struct ib_uverbs_poll_cq cmd; 1242 struct ib_uverbs_poll_cq_resp resp; 1243 u8 __user *header_ptr; 1244 u8 __user *data_ptr; 1245 struct ib_cq *cq; 1246 struct ib_wc wc; 1247 int ret; 1248 1249 if (copy_from_user(&cmd, buf, sizeof cmd)) 1250 return -EFAULT; 1251 1252 cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, file); 1253 if (!cq) 1254 return -EINVAL; 1255 1256 /* we copy a struct ib_uverbs_poll_cq_resp to user space */ 1257 header_ptr = u64_to_user_ptr(cmd.response); 1258 data_ptr = header_ptr + sizeof resp; 1259 1260 memset(&resp, 0, sizeof resp); 1261 while (resp.count < cmd.ne) { 1262 ret = ib_poll_cq(cq, 1, &wc); 1263 if (ret < 0) 1264 goto out_put; 1265 if (!ret) 1266 break; 1267 1268 ret = copy_wc_to_user(cq->device, data_ptr, &wc); 1269 if (ret) 1270 goto out_put; 1271 1272 data_ptr += sizeof(struct ib_uverbs_wc); 1273 ++resp.count; 1274 } 1275 1276 if (copy_to_user(header_ptr, &resp, sizeof resp)) { 1277 ret = -EFAULT; 1278 goto out_put; 1279 } 1280 1281 ret = in_len; 1282 1283 out_put: 1284 uobj_put_obj_read(cq); 1285 return ret; 1286 } 1287 1288 ssize_t ib_uverbs_req_notify_cq(struct ib_uverbs_file *file, 1289 const char __user *buf, int in_len, 1290 int out_len) 1291 { 1292 struct ib_uverbs_req_notify_cq cmd; 1293 struct ib_cq *cq; 1294 1295 if (copy_from_user(&cmd, buf, sizeof cmd)) 1296 return -EFAULT; 1297 1298 cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, file); 1299 if (!cq) 1300 return -EINVAL; 1301 1302 ib_req_notify_cq(cq, cmd.solicited_only ? 1303 IB_CQ_SOLICITED : IB_CQ_NEXT_COMP); 1304 1305 uobj_put_obj_read(cq); 1306 1307 return in_len; 1308 } 1309 1310 ssize_t ib_uverbs_destroy_cq(struct ib_uverbs_file *file, 1311 const char __user *buf, int in_len, 1312 int out_len) 1313 { 1314 struct ib_uverbs_destroy_cq cmd; 1315 struct ib_uverbs_destroy_cq_resp resp; 1316 struct ib_uobject *uobj; 1317 struct ib_ucq_object *obj; 1318 1319 if (copy_from_user(&cmd, buf, sizeof cmd)) 1320 return -EFAULT; 1321 1322 uobj = uobj_get_destroy(UVERBS_OBJECT_CQ, cmd.cq_handle, file); 1323 if (IS_ERR(uobj)) 1324 return PTR_ERR(uobj); 1325 1326 obj = container_of(uobj, struct ib_ucq_object, uobject); 1327 memset(&resp, 0, sizeof(resp)); 1328 resp.comp_events_reported = obj->comp_events_reported; 1329 resp.async_events_reported = obj->async_events_reported; 1330 1331 uobj_put_destroy(uobj); 1332 1333 if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp)) 1334 return -EFAULT; 1335 1336 return in_len; 1337 } 1338 1339 static int create_qp(struct ib_uverbs_file *file, 1340 struct ib_udata *ucore, 1341 struct ib_udata *uhw, 1342 struct ib_uverbs_ex_create_qp *cmd, 1343 size_t cmd_sz, 1344 int (*cb)(struct ib_uverbs_file *file, 1345 struct ib_uverbs_ex_create_qp_resp *resp, 1346 struct ib_udata *udata), 1347 void *context) 1348 { 1349 struct ib_uqp_object *obj; 1350 struct ib_device *device; 1351 struct ib_pd *pd = NULL; 1352 struct ib_xrcd *xrcd = NULL; 1353 struct ib_uobject *xrcd_uobj = ERR_PTR(-ENOENT); 1354 struct ib_cq *scq = NULL, *rcq = NULL; 1355 struct ib_srq *srq = NULL; 1356 struct ib_qp *qp; 1357 char *buf; 1358 struct ib_qp_init_attr attr = {}; 1359 struct ib_uverbs_ex_create_qp_resp resp; 1360 int ret; 1361 struct ib_rwq_ind_table *ind_tbl = NULL; 1362 bool has_sq = true; 1363 struct ib_device *ib_dev; 1364 1365 if (cmd->qp_type == IB_QPT_RAW_PACKET && !capable(CAP_NET_RAW)) 1366 return -EPERM; 1367 1368 obj = (struct ib_uqp_object *)uobj_alloc(UVERBS_OBJECT_QP, file, 1369 &ib_dev); 1370 if (IS_ERR(obj)) 1371 return PTR_ERR(obj); 1372 obj->uxrcd = NULL; 1373 obj->uevent.uobject.user_handle = cmd->user_handle; 1374 mutex_init(&obj->mcast_lock); 1375 1376 if (cmd_sz >= offsetof(typeof(*cmd), rwq_ind_tbl_handle) + 1377 sizeof(cmd->rwq_ind_tbl_handle) && 1378 (cmd->comp_mask & IB_UVERBS_CREATE_QP_MASK_IND_TABLE)) { 1379 ind_tbl = uobj_get_obj_read(rwq_ind_table, 1380 UVERBS_OBJECT_RWQ_IND_TBL, 1381 cmd->rwq_ind_tbl_handle, file); 1382 if (!ind_tbl) { 1383 ret = -EINVAL; 1384 goto err_put; 1385 } 1386 1387 attr.rwq_ind_tbl = ind_tbl; 1388 } 1389 1390 if (cmd_sz > sizeof(*cmd) && 1391 !ib_is_udata_cleared(ucore, sizeof(*cmd), 1392 cmd_sz - sizeof(*cmd))) { 1393 ret = -EOPNOTSUPP; 1394 goto err_put; 1395 } 1396 1397 if (ind_tbl && (cmd->max_recv_wr || cmd->max_recv_sge || cmd->is_srq)) { 1398 ret = -EINVAL; 1399 goto err_put; 1400 } 1401 1402 if (ind_tbl && !cmd->max_send_wr) 1403 has_sq = false; 1404 1405 if (cmd->qp_type == IB_QPT_XRC_TGT) { 1406 xrcd_uobj = uobj_get_read(UVERBS_OBJECT_XRCD, cmd->pd_handle, 1407 file); 1408 1409 if (IS_ERR(xrcd_uobj)) { 1410 ret = -EINVAL; 1411 goto err_put; 1412 } 1413 1414 xrcd = (struct ib_xrcd *)xrcd_uobj->object; 1415 if (!xrcd) { 1416 ret = -EINVAL; 1417 goto err_put; 1418 } 1419 device = xrcd->device; 1420 } else { 1421 if (cmd->qp_type == IB_QPT_XRC_INI) { 1422 cmd->max_recv_wr = 0; 1423 cmd->max_recv_sge = 0; 1424 } else { 1425 if (cmd->is_srq) { 1426 srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ, 1427 cmd->srq_handle, file); 1428 if (!srq || srq->srq_type == IB_SRQT_XRC) { 1429 ret = -EINVAL; 1430 goto err_put; 1431 } 1432 } 1433 1434 if (!ind_tbl) { 1435 if (cmd->recv_cq_handle != cmd->send_cq_handle) { 1436 rcq = uobj_get_obj_read( 1437 cq, UVERBS_OBJECT_CQ, 1438 cmd->recv_cq_handle, file); 1439 if (!rcq) { 1440 ret = -EINVAL; 1441 goto err_put; 1442 } 1443 } 1444 } 1445 } 1446 1447 if (has_sq) 1448 scq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, 1449 cmd->send_cq_handle, file); 1450 if (!ind_tbl) 1451 rcq = rcq ?: scq; 1452 pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd->pd_handle, 1453 file); 1454 if (!pd || (!scq && has_sq)) { 1455 ret = -EINVAL; 1456 goto err_put; 1457 } 1458 1459 device = pd->device; 1460 } 1461 1462 attr.event_handler = ib_uverbs_qp_event_handler; 1463 attr.qp_context = file; 1464 attr.send_cq = scq; 1465 attr.recv_cq = rcq; 1466 attr.srq = srq; 1467 attr.xrcd = xrcd; 1468 attr.sq_sig_type = cmd->sq_sig_all ? IB_SIGNAL_ALL_WR : 1469 IB_SIGNAL_REQ_WR; 1470 attr.qp_type = cmd->qp_type; 1471 attr.create_flags = 0; 1472 1473 attr.cap.max_send_wr = cmd->max_send_wr; 1474 attr.cap.max_recv_wr = cmd->max_recv_wr; 1475 attr.cap.max_send_sge = cmd->max_send_sge; 1476 attr.cap.max_recv_sge = cmd->max_recv_sge; 1477 attr.cap.max_inline_data = cmd->max_inline_data; 1478 1479 obj->uevent.events_reported = 0; 1480 INIT_LIST_HEAD(&obj->uevent.event_list); 1481 INIT_LIST_HEAD(&obj->mcast_list); 1482 1483 if (cmd_sz >= offsetof(typeof(*cmd), create_flags) + 1484 sizeof(cmd->create_flags)) 1485 attr.create_flags = cmd->create_flags; 1486 1487 if (attr.create_flags & ~(IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK | 1488 IB_QP_CREATE_CROSS_CHANNEL | 1489 IB_QP_CREATE_MANAGED_SEND | 1490 IB_QP_CREATE_MANAGED_RECV | 1491 IB_QP_CREATE_SCATTER_FCS | 1492 IB_QP_CREATE_CVLAN_STRIPPING | 1493 IB_QP_CREATE_SOURCE_QPN | 1494 IB_QP_CREATE_PCI_WRITE_END_PADDING)) { 1495 ret = -EINVAL; 1496 goto err_put; 1497 } 1498 1499 if (attr.create_flags & IB_QP_CREATE_SOURCE_QPN) { 1500 if (!capable(CAP_NET_RAW)) { 1501 ret = -EPERM; 1502 goto err_put; 1503 } 1504 1505 attr.source_qpn = cmd->source_qpn; 1506 } 1507 1508 buf = (void *)cmd + sizeof(*cmd); 1509 if (cmd_sz > sizeof(*cmd)) 1510 if (!(buf[0] == 0 && !memcmp(buf, buf + 1, 1511 cmd_sz - sizeof(*cmd) - 1))) { 1512 ret = -EINVAL; 1513 goto err_put; 1514 } 1515 1516 if (cmd->qp_type == IB_QPT_XRC_TGT) 1517 qp = ib_create_qp(pd, &attr); 1518 else 1519 qp = _ib_create_qp(device, pd, &attr, uhw, 1520 &obj->uevent.uobject); 1521 1522 if (IS_ERR(qp)) { 1523 ret = PTR_ERR(qp); 1524 goto err_put; 1525 } 1526 1527 if (cmd->qp_type != IB_QPT_XRC_TGT) { 1528 ret = ib_create_qp_security(qp, device); 1529 if (ret) 1530 goto err_cb; 1531 1532 qp->real_qp = qp; 1533 qp->pd = pd; 1534 qp->send_cq = attr.send_cq; 1535 qp->recv_cq = attr.recv_cq; 1536 qp->srq = attr.srq; 1537 qp->rwq_ind_tbl = ind_tbl; 1538 qp->event_handler = attr.event_handler; 1539 qp->qp_context = attr.qp_context; 1540 qp->qp_type = attr.qp_type; 1541 atomic_set(&qp->usecnt, 0); 1542 atomic_inc(&pd->usecnt); 1543 qp->port = 0; 1544 if (attr.send_cq) 1545 atomic_inc(&attr.send_cq->usecnt); 1546 if (attr.recv_cq) 1547 atomic_inc(&attr.recv_cq->usecnt); 1548 if (attr.srq) 1549 atomic_inc(&attr.srq->usecnt); 1550 if (ind_tbl) 1551 atomic_inc(&ind_tbl->usecnt); 1552 } else { 1553 /* It is done in _ib_create_qp for other QP types */ 1554 qp->uobject = &obj->uevent.uobject; 1555 } 1556 1557 obj->uevent.uobject.object = qp; 1558 1559 memset(&resp, 0, sizeof resp); 1560 resp.base.qpn = qp->qp_num; 1561 resp.base.qp_handle = obj->uevent.uobject.id; 1562 resp.base.max_recv_sge = attr.cap.max_recv_sge; 1563 resp.base.max_send_sge = attr.cap.max_send_sge; 1564 resp.base.max_recv_wr = attr.cap.max_recv_wr; 1565 resp.base.max_send_wr = attr.cap.max_send_wr; 1566 resp.base.max_inline_data = attr.cap.max_inline_data; 1567 1568 resp.response_length = offsetof(typeof(resp), response_length) + 1569 sizeof(resp.response_length); 1570 1571 ret = cb(file, &resp, ucore); 1572 if (ret) 1573 goto err_cb; 1574 1575 if (xrcd) { 1576 obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object, 1577 uobject); 1578 atomic_inc(&obj->uxrcd->refcnt); 1579 uobj_put_read(xrcd_uobj); 1580 } 1581 1582 if (pd) 1583 uobj_put_obj_read(pd); 1584 if (scq) 1585 uobj_put_obj_read(scq); 1586 if (rcq && rcq != scq) 1587 uobj_put_obj_read(rcq); 1588 if (srq) 1589 uobj_put_obj_read(srq); 1590 if (ind_tbl) 1591 uobj_put_obj_read(ind_tbl); 1592 1593 return uobj_alloc_commit(&obj->uevent.uobject, 0); 1594 err_cb: 1595 ib_destroy_qp(qp); 1596 1597 err_put: 1598 if (!IS_ERR(xrcd_uobj)) 1599 uobj_put_read(xrcd_uobj); 1600 if (pd) 1601 uobj_put_obj_read(pd); 1602 if (scq) 1603 uobj_put_obj_read(scq); 1604 if (rcq && rcq != scq) 1605 uobj_put_obj_read(rcq); 1606 if (srq) 1607 uobj_put_obj_read(srq); 1608 if (ind_tbl) 1609 uobj_put_obj_read(ind_tbl); 1610 1611 uobj_alloc_abort(&obj->uevent.uobject); 1612 return ret; 1613 } 1614 1615 static int ib_uverbs_create_qp_cb(struct ib_uverbs_file *file, 1616 struct ib_uverbs_ex_create_qp_resp *resp, 1617 struct ib_udata *ucore) 1618 { 1619 if (ib_copy_to_udata(ucore, &resp->base, sizeof(resp->base))) 1620 return -EFAULT; 1621 1622 return 0; 1623 } 1624 1625 ssize_t ib_uverbs_create_qp(struct ib_uverbs_file *file, 1626 const char __user *buf, int in_len, 1627 int out_len) 1628 { 1629 struct ib_uverbs_create_qp cmd; 1630 struct ib_uverbs_ex_create_qp cmd_ex; 1631 struct ib_udata ucore; 1632 struct ib_udata uhw; 1633 ssize_t resp_size = sizeof(struct ib_uverbs_create_qp_resp); 1634 int err; 1635 1636 if (out_len < resp_size) 1637 return -ENOSPC; 1638 1639 if (copy_from_user(&cmd, buf, sizeof(cmd))) 1640 return -EFAULT; 1641 1642 ib_uverbs_init_udata(&ucore, buf, u64_to_user_ptr(cmd.response), 1643 sizeof(cmd), resp_size); 1644 ib_uverbs_init_udata(&uhw, buf + sizeof(cmd), 1645 u64_to_user_ptr(cmd.response) + resp_size, 1646 in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr), 1647 out_len - resp_size); 1648 1649 memset(&cmd_ex, 0, sizeof(cmd_ex)); 1650 cmd_ex.user_handle = cmd.user_handle; 1651 cmd_ex.pd_handle = cmd.pd_handle; 1652 cmd_ex.send_cq_handle = cmd.send_cq_handle; 1653 cmd_ex.recv_cq_handle = cmd.recv_cq_handle; 1654 cmd_ex.srq_handle = cmd.srq_handle; 1655 cmd_ex.max_send_wr = cmd.max_send_wr; 1656 cmd_ex.max_recv_wr = cmd.max_recv_wr; 1657 cmd_ex.max_send_sge = cmd.max_send_sge; 1658 cmd_ex.max_recv_sge = cmd.max_recv_sge; 1659 cmd_ex.max_inline_data = cmd.max_inline_data; 1660 cmd_ex.sq_sig_all = cmd.sq_sig_all; 1661 cmd_ex.qp_type = cmd.qp_type; 1662 cmd_ex.is_srq = cmd.is_srq; 1663 1664 err = create_qp(file, &ucore, &uhw, &cmd_ex, 1665 offsetof(typeof(cmd_ex), is_srq) + 1666 sizeof(cmd.is_srq), ib_uverbs_create_qp_cb, 1667 NULL); 1668 1669 if (err) 1670 return err; 1671 1672 return in_len; 1673 } 1674 1675 static int ib_uverbs_ex_create_qp_cb(struct ib_uverbs_file *file, 1676 struct ib_uverbs_ex_create_qp_resp *resp, 1677 struct ib_udata *ucore) 1678 { 1679 if (ib_copy_to_udata(ucore, resp, resp->response_length)) 1680 return -EFAULT; 1681 1682 return 0; 1683 } 1684 1685 int ib_uverbs_ex_create_qp(struct ib_uverbs_file *file, 1686 struct ib_udata *ucore, 1687 struct ib_udata *uhw) 1688 { 1689 struct ib_uverbs_ex_create_qp_resp resp; 1690 struct ib_uverbs_ex_create_qp cmd = {0}; 1691 int err; 1692 1693 if (ucore->inlen < (offsetof(typeof(cmd), comp_mask) + 1694 sizeof(cmd.comp_mask))) 1695 return -EINVAL; 1696 1697 err = ib_copy_from_udata(&cmd, ucore, min(sizeof(cmd), ucore->inlen)); 1698 if (err) 1699 return err; 1700 1701 if (cmd.comp_mask & ~IB_UVERBS_CREATE_QP_SUP_COMP_MASK) 1702 return -EINVAL; 1703 1704 if (cmd.reserved) 1705 return -EINVAL; 1706 1707 if (ucore->outlen < (offsetof(typeof(resp), response_length) + 1708 sizeof(resp.response_length))) 1709 return -ENOSPC; 1710 1711 err = create_qp(file, ucore, uhw, &cmd, 1712 min(ucore->inlen, sizeof(cmd)), 1713 ib_uverbs_ex_create_qp_cb, NULL); 1714 1715 if (err) 1716 return err; 1717 1718 return 0; 1719 } 1720 1721 ssize_t ib_uverbs_open_qp(struct ib_uverbs_file *file, 1722 const char __user *buf, int in_len, int out_len) 1723 { 1724 struct ib_uverbs_open_qp cmd; 1725 struct ib_uverbs_create_qp_resp resp; 1726 struct ib_udata udata; 1727 struct ib_uqp_object *obj; 1728 struct ib_xrcd *xrcd; 1729 struct ib_uobject *uninitialized_var(xrcd_uobj); 1730 struct ib_qp *qp; 1731 struct ib_qp_open_attr attr; 1732 int ret; 1733 struct ib_device *ib_dev; 1734 1735 if (out_len < sizeof resp) 1736 return -ENOSPC; 1737 1738 if (copy_from_user(&cmd, buf, sizeof cmd)) 1739 return -EFAULT; 1740 1741 ib_uverbs_init_udata(&udata, buf + sizeof(cmd), 1742 u64_to_user_ptr(cmd.response) + sizeof(resp), 1743 in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr), 1744 out_len - sizeof(resp)); 1745 1746 obj = (struct ib_uqp_object *)uobj_alloc(UVERBS_OBJECT_QP, file, 1747 &ib_dev); 1748 if (IS_ERR(obj)) 1749 return PTR_ERR(obj); 1750 1751 xrcd_uobj = uobj_get_read(UVERBS_OBJECT_XRCD, cmd.pd_handle, file); 1752 if (IS_ERR(xrcd_uobj)) { 1753 ret = -EINVAL; 1754 goto err_put; 1755 } 1756 1757 xrcd = (struct ib_xrcd *)xrcd_uobj->object; 1758 if (!xrcd) { 1759 ret = -EINVAL; 1760 goto err_xrcd; 1761 } 1762 1763 attr.event_handler = ib_uverbs_qp_event_handler; 1764 attr.qp_context = file; 1765 attr.qp_num = cmd.qpn; 1766 attr.qp_type = cmd.qp_type; 1767 1768 obj->uevent.events_reported = 0; 1769 INIT_LIST_HEAD(&obj->uevent.event_list); 1770 INIT_LIST_HEAD(&obj->mcast_list); 1771 1772 qp = ib_open_qp(xrcd, &attr); 1773 if (IS_ERR(qp)) { 1774 ret = PTR_ERR(qp); 1775 goto err_xrcd; 1776 } 1777 1778 obj->uevent.uobject.object = qp; 1779 obj->uevent.uobject.user_handle = cmd.user_handle; 1780 1781 memset(&resp, 0, sizeof resp); 1782 resp.qpn = qp->qp_num; 1783 resp.qp_handle = obj->uevent.uobject.id; 1784 1785 if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp)) { 1786 ret = -EFAULT; 1787 goto err_destroy; 1788 } 1789 1790 obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object, uobject); 1791 atomic_inc(&obj->uxrcd->refcnt); 1792 qp->uobject = &obj->uevent.uobject; 1793 uobj_put_read(xrcd_uobj); 1794 1795 return uobj_alloc_commit(&obj->uevent.uobject, in_len); 1796 1797 err_destroy: 1798 ib_destroy_qp(qp); 1799 err_xrcd: 1800 uobj_put_read(xrcd_uobj); 1801 err_put: 1802 uobj_alloc_abort(&obj->uevent.uobject); 1803 return ret; 1804 } 1805 1806 static void copy_ah_attr_to_uverbs(struct ib_uverbs_qp_dest *uverb_attr, 1807 struct rdma_ah_attr *rdma_attr) 1808 { 1809 const struct ib_global_route *grh; 1810 1811 uverb_attr->dlid = rdma_ah_get_dlid(rdma_attr); 1812 uverb_attr->sl = rdma_ah_get_sl(rdma_attr); 1813 uverb_attr->src_path_bits = rdma_ah_get_path_bits(rdma_attr); 1814 uverb_attr->static_rate = rdma_ah_get_static_rate(rdma_attr); 1815 uverb_attr->is_global = !!(rdma_ah_get_ah_flags(rdma_attr) & 1816 IB_AH_GRH); 1817 if (uverb_attr->is_global) { 1818 grh = rdma_ah_read_grh(rdma_attr); 1819 memcpy(uverb_attr->dgid, grh->dgid.raw, 16); 1820 uverb_attr->flow_label = grh->flow_label; 1821 uverb_attr->sgid_index = grh->sgid_index; 1822 uverb_attr->hop_limit = grh->hop_limit; 1823 uverb_attr->traffic_class = grh->traffic_class; 1824 } 1825 uverb_attr->port_num = rdma_ah_get_port_num(rdma_attr); 1826 } 1827 1828 ssize_t ib_uverbs_query_qp(struct ib_uverbs_file *file, 1829 const char __user *buf, int in_len, 1830 int out_len) 1831 { 1832 struct ib_uverbs_query_qp cmd; 1833 struct ib_uverbs_query_qp_resp resp; 1834 struct ib_qp *qp; 1835 struct ib_qp_attr *attr; 1836 struct ib_qp_init_attr *init_attr; 1837 int ret; 1838 1839 if (copy_from_user(&cmd, buf, sizeof cmd)) 1840 return -EFAULT; 1841 1842 attr = kmalloc(sizeof *attr, GFP_KERNEL); 1843 init_attr = kmalloc(sizeof *init_attr, GFP_KERNEL); 1844 if (!attr || !init_attr) { 1845 ret = -ENOMEM; 1846 goto out; 1847 } 1848 1849 qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, file); 1850 if (!qp) { 1851 ret = -EINVAL; 1852 goto out; 1853 } 1854 1855 ret = ib_query_qp(qp, attr, cmd.attr_mask, init_attr); 1856 1857 uobj_put_obj_read(qp); 1858 1859 if (ret) 1860 goto out; 1861 1862 memset(&resp, 0, sizeof resp); 1863 1864 resp.qp_state = attr->qp_state; 1865 resp.cur_qp_state = attr->cur_qp_state; 1866 resp.path_mtu = attr->path_mtu; 1867 resp.path_mig_state = attr->path_mig_state; 1868 resp.qkey = attr->qkey; 1869 resp.rq_psn = attr->rq_psn; 1870 resp.sq_psn = attr->sq_psn; 1871 resp.dest_qp_num = attr->dest_qp_num; 1872 resp.qp_access_flags = attr->qp_access_flags; 1873 resp.pkey_index = attr->pkey_index; 1874 resp.alt_pkey_index = attr->alt_pkey_index; 1875 resp.sq_draining = attr->sq_draining; 1876 resp.max_rd_atomic = attr->max_rd_atomic; 1877 resp.max_dest_rd_atomic = attr->max_dest_rd_atomic; 1878 resp.min_rnr_timer = attr->min_rnr_timer; 1879 resp.port_num = attr->port_num; 1880 resp.timeout = attr->timeout; 1881 resp.retry_cnt = attr->retry_cnt; 1882 resp.rnr_retry = attr->rnr_retry; 1883 resp.alt_port_num = attr->alt_port_num; 1884 resp.alt_timeout = attr->alt_timeout; 1885 1886 copy_ah_attr_to_uverbs(&resp.dest, &attr->ah_attr); 1887 copy_ah_attr_to_uverbs(&resp.alt_dest, &attr->alt_ah_attr); 1888 1889 resp.max_send_wr = init_attr->cap.max_send_wr; 1890 resp.max_recv_wr = init_attr->cap.max_recv_wr; 1891 resp.max_send_sge = init_attr->cap.max_send_sge; 1892 resp.max_recv_sge = init_attr->cap.max_recv_sge; 1893 resp.max_inline_data = init_attr->cap.max_inline_data; 1894 resp.sq_sig_all = init_attr->sq_sig_type == IB_SIGNAL_ALL_WR; 1895 1896 if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp)) 1897 ret = -EFAULT; 1898 1899 out: 1900 kfree(attr); 1901 kfree(init_attr); 1902 1903 return ret ? ret : in_len; 1904 } 1905 1906 /* Remove ignored fields set in the attribute mask */ 1907 static int modify_qp_mask(enum ib_qp_type qp_type, int mask) 1908 { 1909 switch (qp_type) { 1910 case IB_QPT_XRC_INI: 1911 return mask & ~(IB_QP_MAX_DEST_RD_ATOMIC | IB_QP_MIN_RNR_TIMER); 1912 case IB_QPT_XRC_TGT: 1913 return mask & ~(IB_QP_MAX_QP_RD_ATOMIC | IB_QP_RETRY_CNT | 1914 IB_QP_RNR_RETRY); 1915 default: 1916 return mask; 1917 } 1918 } 1919 1920 static void copy_ah_attr_from_uverbs(struct ib_device *dev, 1921 struct rdma_ah_attr *rdma_attr, 1922 struct ib_uverbs_qp_dest *uverb_attr) 1923 { 1924 rdma_attr->type = rdma_ah_find_type(dev, uverb_attr->port_num); 1925 if (uverb_attr->is_global) { 1926 rdma_ah_set_grh(rdma_attr, NULL, 1927 uverb_attr->flow_label, 1928 uverb_attr->sgid_index, 1929 uverb_attr->hop_limit, 1930 uverb_attr->traffic_class); 1931 rdma_ah_set_dgid_raw(rdma_attr, uverb_attr->dgid); 1932 } else { 1933 rdma_ah_set_ah_flags(rdma_attr, 0); 1934 } 1935 rdma_ah_set_dlid(rdma_attr, uverb_attr->dlid); 1936 rdma_ah_set_sl(rdma_attr, uverb_attr->sl); 1937 rdma_ah_set_path_bits(rdma_attr, uverb_attr->src_path_bits); 1938 rdma_ah_set_static_rate(rdma_attr, uverb_attr->static_rate); 1939 rdma_ah_set_port_num(rdma_attr, uverb_attr->port_num); 1940 rdma_ah_set_make_grd(rdma_attr, false); 1941 } 1942 1943 static int modify_qp(struct ib_uverbs_file *file, 1944 struct ib_uverbs_ex_modify_qp *cmd, struct ib_udata *udata) 1945 { 1946 struct ib_qp_attr *attr; 1947 struct ib_qp *qp; 1948 int ret; 1949 1950 attr = kzalloc(sizeof(*attr), GFP_KERNEL); 1951 if (!attr) 1952 return -ENOMEM; 1953 1954 qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd->base.qp_handle, file); 1955 if (!qp) { 1956 ret = -EINVAL; 1957 goto out; 1958 } 1959 1960 if ((cmd->base.attr_mask & IB_QP_PORT) && 1961 !rdma_is_port_valid(qp->device, cmd->base.port_num)) { 1962 ret = -EINVAL; 1963 goto release_qp; 1964 } 1965 1966 if ((cmd->base.attr_mask & IB_QP_AV)) { 1967 if (!rdma_is_port_valid(qp->device, cmd->base.dest.port_num)) { 1968 ret = -EINVAL; 1969 goto release_qp; 1970 } 1971 1972 if (cmd->base.attr_mask & IB_QP_STATE && 1973 cmd->base.qp_state == IB_QPS_RTR) { 1974 /* We are in INIT->RTR TRANSITION (if we are not, 1975 * this transition will be rejected in subsequent checks). 1976 * In the INIT->RTR transition, we cannot have IB_QP_PORT set, 1977 * but the IB_QP_STATE flag is required. 1978 * 1979 * Since kernel 3.14 (commit dbf727de7440), the uverbs driver, 1980 * when IB_QP_AV is set, has required inclusion of a valid 1981 * port number in the primary AV. (AVs are created and handled 1982 * differently for infiniband and ethernet (RoCE) ports). 1983 * 1984 * Check the port number included in the primary AV against 1985 * the port number in the qp struct, which was set (and saved) 1986 * in the RST->INIT transition. 1987 */ 1988 if (cmd->base.dest.port_num != qp->real_qp->port) { 1989 ret = -EINVAL; 1990 goto release_qp; 1991 } 1992 } else { 1993 /* We are in SQD->SQD. (If we are not, this transition will 1994 * be rejected later in the verbs layer checks). 1995 * Check for both IB_QP_PORT and IB_QP_AV, these can be set 1996 * together in the SQD->SQD transition. 1997 * 1998 * If only IP_QP_AV was set, add in IB_QP_PORT as well (the 1999 * verbs layer driver does not track primary port changes 2000 * resulting from path migration. Thus, in SQD, if the primary 2001 * AV is modified, the primary port should also be modified). 2002 * 2003 * Note that in this transition, the IB_QP_STATE flag 2004 * is not allowed. 2005 */ 2006 if (((cmd->base.attr_mask & (IB_QP_AV | IB_QP_PORT)) 2007 == (IB_QP_AV | IB_QP_PORT)) && 2008 cmd->base.port_num != cmd->base.dest.port_num) { 2009 ret = -EINVAL; 2010 goto release_qp; 2011 } 2012 if ((cmd->base.attr_mask & (IB_QP_AV | IB_QP_PORT)) 2013 == IB_QP_AV) { 2014 cmd->base.attr_mask |= IB_QP_PORT; 2015 cmd->base.port_num = cmd->base.dest.port_num; 2016 } 2017 } 2018 } 2019 2020 if ((cmd->base.attr_mask & IB_QP_ALT_PATH) && 2021 (!rdma_is_port_valid(qp->device, cmd->base.alt_port_num) || 2022 !rdma_is_port_valid(qp->device, cmd->base.alt_dest.port_num) || 2023 cmd->base.alt_port_num != cmd->base.alt_dest.port_num)) { 2024 ret = -EINVAL; 2025 goto release_qp; 2026 } 2027 2028 if ((cmd->base.attr_mask & IB_QP_CUR_STATE && 2029 cmd->base.cur_qp_state > IB_QPS_ERR) || 2030 (cmd->base.attr_mask & IB_QP_STATE && 2031 cmd->base.qp_state > IB_QPS_ERR)) { 2032 ret = -EINVAL; 2033 goto release_qp; 2034 } 2035 2036 if (cmd->base.attr_mask & IB_QP_STATE) 2037 attr->qp_state = cmd->base.qp_state; 2038 if (cmd->base.attr_mask & IB_QP_CUR_STATE) 2039 attr->cur_qp_state = cmd->base.cur_qp_state; 2040 if (cmd->base.attr_mask & IB_QP_PATH_MTU) 2041 attr->path_mtu = cmd->base.path_mtu; 2042 if (cmd->base.attr_mask & IB_QP_PATH_MIG_STATE) 2043 attr->path_mig_state = cmd->base.path_mig_state; 2044 if (cmd->base.attr_mask & IB_QP_QKEY) 2045 attr->qkey = cmd->base.qkey; 2046 if (cmd->base.attr_mask & IB_QP_RQ_PSN) 2047 attr->rq_psn = cmd->base.rq_psn; 2048 if (cmd->base.attr_mask & IB_QP_SQ_PSN) 2049 attr->sq_psn = cmd->base.sq_psn; 2050 if (cmd->base.attr_mask & IB_QP_DEST_QPN) 2051 attr->dest_qp_num = cmd->base.dest_qp_num; 2052 if (cmd->base.attr_mask & IB_QP_ACCESS_FLAGS) 2053 attr->qp_access_flags = cmd->base.qp_access_flags; 2054 if (cmd->base.attr_mask & IB_QP_PKEY_INDEX) 2055 attr->pkey_index = cmd->base.pkey_index; 2056 if (cmd->base.attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY) 2057 attr->en_sqd_async_notify = cmd->base.en_sqd_async_notify; 2058 if (cmd->base.attr_mask & IB_QP_MAX_QP_RD_ATOMIC) 2059 attr->max_rd_atomic = cmd->base.max_rd_atomic; 2060 if (cmd->base.attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) 2061 attr->max_dest_rd_atomic = cmd->base.max_dest_rd_atomic; 2062 if (cmd->base.attr_mask & IB_QP_MIN_RNR_TIMER) 2063 attr->min_rnr_timer = cmd->base.min_rnr_timer; 2064 if (cmd->base.attr_mask & IB_QP_PORT) 2065 attr->port_num = cmd->base.port_num; 2066 if (cmd->base.attr_mask & IB_QP_TIMEOUT) 2067 attr->timeout = cmd->base.timeout; 2068 if (cmd->base.attr_mask & IB_QP_RETRY_CNT) 2069 attr->retry_cnt = cmd->base.retry_cnt; 2070 if (cmd->base.attr_mask & IB_QP_RNR_RETRY) 2071 attr->rnr_retry = cmd->base.rnr_retry; 2072 if (cmd->base.attr_mask & IB_QP_ALT_PATH) { 2073 attr->alt_port_num = cmd->base.alt_port_num; 2074 attr->alt_timeout = cmd->base.alt_timeout; 2075 attr->alt_pkey_index = cmd->base.alt_pkey_index; 2076 } 2077 if (cmd->base.attr_mask & IB_QP_RATE_LIMIT) 2078 attr->rate_limit = cmd->rate_limit; 2079 2080 if (cmd->base.attr_mask & IB_QP_AV) 2081 copy_ah_attr_from_uverbs(qp->device, &attr->ah_attr, 2082 &cmd->base.dest); 2083 2084 if (cmd->base.attr_mask & IB_QP_ALT_PATH) 2085 copy_ah_attr_from_uverbs(qp->device, &attr->alt_ah_attr, 2086 &cmd->base.alt_dest); 2087 2088 ret = ib_modify_qp_with_udata(qp, attr, 2089 modify_qp_mask(qp->qp_type, 2090 cmd->base.attr_mask), 2091 udata); 2092 2093 release_qp: 2094 uobj_put_obj_read(qp); 2095 out: 2096 kfree(attr); 2097 2098 return ret; 2099 } 2100 2101 ssize_t ib_uverbs_modify_qp(struct ib_uverbs_file *file, 2102 const char __user *buf, int in_len, 2103 int out_len) 2104 { 2105 struct ib_uverbs_ex_modify_qp cmd = {}; 2106 struct ib_udata udata; 2107 int ret; 2108 2109 if (copy_from_user(&cmd.base, buf, sizeof(cmd.base))) 2110 return -EFAULT; 2111 2112 if (cmd.base.attr_mask & 2113 ~((IB_USER_LEGACY_LAST_QP_ATTR_MASK << 1) - 1)) 2114 return -EOPNOTSUPP; 2115 2116 ib_uverbs_init_udata(&udata, buf + sizeof(cmd.base), NULL, 2117 in_len - sizeof(cmd.base) - sizeof(struct ib_uverbs_cmd_hdr), 2118 out_len); 2119 2120 ret = modify_qp(file, &cmd, &udata); 2121 if (ret) 2122 return ret; 2123 2124 return in_len; 2125 } 2126 2127 int ib_uverbs_ex_modify_qp(struct ib_uverbs_file *file, 2128 struct ib_udata *ucore, 2129 struct ib_udata *uhw) 2130 { 2131 struct ib_uverbs_ex_modify_qp cmd = {}; 2132 int ret; 2133 2134 /* 2135 * Last bit is reserved for extending the attr_mask by 2136 * using another field. 2137 */ 2138 BUILD_BUG_ON(IB_USER_LAST_QP_ATTR_MASK == (1 << 31)); 2139 2140 if (ucore->inlen < sizeof(cmd.base)) 2141 return -EINVAL; 2142 2143 ret = ib_copy_from_udata(&cmd, ucore, min(sizeof(cmd), ucore->inlen)); 2144 if (ret) 2145 return ret; 2146 2147 if (cmd.base.attr_mask & 2148 ~((IB_USER_LAST_QP_ATTR_MASK << 1) - 1)) 2149 return -EOPNOTSUPP; 2150 2151 if (ucore->inlen > sizeof(cmd)) { 2152 if (!ib_is_udata_cleared(ucore, sizeof(cmd), 2153 ucore->inlen - sizeof(cmd))) 2154 return -EOPNOTSUPP; 2155 } 2156 2157 ret = modify_qp(file, &cmd, uhw); 2158 2159 return ret; 2160 } 2161 2162 ssize_t ib_uverbs_destroy_qp(struct ib_uverbs_file *file, 2163 const char __user *buf, int in_len, 2164 int out_len) 2165 { 2166 struct ib_uverbs_destroy_qp cmd; 2167 struct ib_uverbs_destroy_qp_resp resp; 2168 struct ib_uobject *uobj; 2169 struct ib_uqp_object *obj; 2170 2171 if (copy_from_user(&cmd, buf, sizeof cmd)) 2172 return -EFAULT; 2173 2174 uobj = uobj_get_destroy(UVERBS_OBJECT_QP, cmd.qp_handle, file); 2175 if (IS_ERR(uobj)) 2176 return PTR_ERR(uobj); 2177 2178 obj = container_of(uobj, struct ib_uqp_object, uevent.uobject); 2179 memset(&resp, 0, sizeof(resp)); 2180 resp.events_reported = obj->uevent.events_reported; 2181 2182 uobj_put_destroy(uobj); 2183 2184 if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp)) 2185 return -EFAULT; 2186 2187 return in_len; 2188 } 2189 2190 static void *alloc_wr(size_t wr_size, __u32 num_sge) 2191 { 2192 if (num_sge >= (U32_MAX - ALIGN(wr_size, sizeof (struct ib_sge))) / 2193 sizeof (struct ib_sge)) 2194 return NULL; 2195 2196 return kmalloc(ALIGN(wr_size, sizeof (struct ib_sge)) + 2197 num_sge * sizeof (struct ib_sge), GFP_KERNEL); 2198 } 2199 2200 ssize_t ib_uverbs_post_send(struct ib_uverbs_file *file, 2201 const char __user *buf, int in_len, 2202 int out_len) 2203 { 2204 struct ib_uverbs_post_send cmd; 2205 struct ib_uverbs_post_send_resp resp; 2206 struct ib_uverbs_send_wr *user_wr; 2207 struct ib_send_wr *wr = NULL, *last, *next; 2208 const struct ib_send_wr *bad_wr; 2209 struct ib_qp *qp; 2210 int i, sg_ind; 2211 int is_ud; 2212 ssize_t ret = -EINVAL; 2213 size_t next_size; 2214 2215 if (copy_from_user(&cmd, buf, sizeof cmd)) 2216 return -EFAULT; 2217 2218 if (in_len < sizeof cmd + cmd.wqe_size * cmd.wr_count + 2219 cmd.sge_count * sizeof (struct ib_uverbs_sge)) 2220 return -EINVAL; 2221 2222 if (cmd.wqe_size < sizeof (struct ib_uverbs_send_wr)) 2223 return -EINVAL; 2224 2225 user_wr = kmalloc(cmd.wqe_size, GFP_KERNEL); 2226 if (!user_wr) 2227 return -ENOMEM; 2228 2229 qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, file); 2230 if (!qp) 2231 goto out; 2232 2233 is_ud = qp->qp_type == IB_QPT_UD; 2234 sg_ind = 0; 2235 last = NULL; 2236 for (i = 0; i < cmd.wr_count; ++i) { 2237 if (copy_from_user(user_wr, 2238 buf + sizeof cmd + i * cmd.wqe_size, 2239 cmd.wqe_size)) { 2240 ret = -EFAULT; 2241 goto out_put; 2242 } 2243 2244 if (user_wr->num_sge + sg_ind > cmd.sge_count) { 2245 ret = -EINVAL; 2246 goto out_put; 2247 } 2248 2249 if (is_ud) { 2250 struct ib_ud_wr *ud; 2251 2252 if (user_wr->opcode != IB_WR_SEND && 2253 user_wr->opcode != IB_WR_SEND_WITH_IMM) { 2254 ret = -EINVAL; 2255 goto out_put; 2256 } 2257 2258 next_size = sizeof(*ud); 2259 ud = alloc_wr(next_size, user_wr->num_sge); 2260 if (!ud) { 2261 ret = -ENOMEM; 2262 goto out_put; 2263 } 2264 2265 ud->ah = uobj_get_obj_read(ah, UVERBS_OBJECT_AH, 2266 user_wr->wr.ud.ah, file); 2267 if (!ud->ah) { 2268 kfree(ud); 2269 ret = -EINVAL; 2270 goto out_put; 2271 } 2272 ud->remote_qpn = user_wr->wr.ud.remote_qpn; 2273 ud->remote_qkey = user_wr->wr.ud.remote_qkey; 2274 2275 next = &ud->wr; 2276 } else if (user_wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM || 2277 user_wr->opcode == IB_WR_RDMA_WRITE || 2278 user_wr->opcode == IB_WR_RDMA_READ) { 2279 struct ib_rdma_wr *rdma; 2280 2281 next_size = sizeof(*rdma); 2282 rdma = alloc_wr(next_size, user_wr->num_sge); 2283 if (!rdma) { 2284 ret = -ENOMEM; 2285 goto out_put; 2286 } 2287 2288 rdma->remote_addr = user_wr->wr.rdma.remote_addr; 2289 rdma->rkey = user_wr->wr.rdma.rkey; 2290 2291 next = &rdma->wr; 2292 } else if (user_wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP || 2293 user_wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) { 2294 struct ib_atomic_wr *atomic; 2295 2296 next_size = sizeof(*atomic); 2297 atomic = alloc_wr(next_size, user_wr->num_sge); 2298 if (!atomic) { 2299 ret = -ENOMEM; 2300 goto out_put; 2301 } 2302 2303 atomic->remote_addr = user_wr->wr.atomic.remote_addr; 2304 atomic->compare_add = user_wr->wr.atomic.compare_add; 2305 atomic->swap = user_wr->wr.atomic.swap; 2306 atomic->rkey = user_wr->wr.atomic.rkey; 2307 2308 next = &atomic->wr; 2309 } else if (user_wr->opcode == IB_WR_SEND || 2310 user_wr->opcode == IB_WR_SEND_WITH_IMM || 2311 user_wr->opcode == IB_WR_SEND_WITH_INV) { 2312 next_size = sizeof(*next); 2313 next = alloc_wr(next_size, user_wr->num_sge); 2314 if (!next) { 2315 ret = -ENOMEM; 2316 goto out_put; 2317 } 2318 } else { 2319 ret = -EINVAL; 2320 goto out_put; 2321 } 2322 2323 if (user_wr->opcode == IB_WR_SEND_WITH_IMM || 2324 user_wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM) { 2325 next->ex.imm_data = 2326 (__be32 __force) user_wr->ex.imm_data; 2327 } else if (user_wr->opcode == IB_WR_SEND_WITH_INV) { 2328 next->ex.invalidate_rkey = user_wr->ex.invalidate_rkey; 2329 } 2330 2331 if (!last) 2332 wr = next; 2333 else 2334 last->next = next; 2335 last = next; 2336 2337 next->next = NULL; 2338 next->wr_id = user_wr->wr_id; 2339 next->num_sge = user_wr->num_sge; 2340 next->opcode = user_wr->opcode; 2341 next->send_flags = user_wr->send_flags; 2342 2343 if (next->num_sge) { 2344 next->sg_list = (void *) next + 2345 ALIGN(next_size, sizeof(struct ib_sge)); 2346 if (copy_from_user(next->sg_list, 2347 buf + sizeof cmd + 2348 cmd.wr_count * cmd.wqe_size + 2349 sg_ind * sizeof (struct ib_sge), 2350 next->num_sge * sizeof (struct ib_sge))) { 2351 ret = -EFAULT; 2352 goto out_put; 2353 } 2354 sg_ind += next->num_sge; 2355 } else 2356 next->sg_list = NULL; 2357 } 2358 2359 resp.bad_wr = 0; 2360 ret = qp->device->post_send(qp->real_qp, wr, &bad_wr); 2361 if (ret) 2362 for (next = wr; next; next = next->next) { 2363 ++resp.bad_wr; 2364 if (next == bad_wr) 2365 break; 2366 } 2367 2368 if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp)) 2369 ret = -EFAULT; 2370 2371 out_put: 2372 uobj_put_obj_read(qp); 2373 2374 while (wr) { 2375 if (is_ud && ud_wr(wr)->ah) 2376 uobj_put_obj_read(ud_wr(wr)->ah); 2377 next = wr->next; 2378 kfree(wr); 2379 wr = next; 2380 } 2381 2382 out: 2383 kfree(user_wr); 2384 2385 return ret ? ret : in_len; 2386 } 2387 2388 static struct ib_recv_wr *ib_uverbs_unmarshall_recv(const char __user *buf, 2389 int in_len, 2390 u32 wr_count, 2391 u32 sge_count, 2392 u32 wqe_size) 2393 { 2394 struct ib_uverbs_recv_wr *user_wr; 2395 struct ib_recv_wr *wr = NULL, *last, *next; 2396 int sg_ind; 2397 int i; 2398 int ret; 2399 2400 if (in_len < wqe_size * wr_count + 2401 sge_count * sizeof (struct ib_uverbs_sge)) 2402 return ERR_PTR(-EINVAL); 2403 2404 if (wqe_size < sizeof (struct ib_uverbs_recv_wr)) 2405 return ERR_PTR(-EINVAL); 2406 2407 user_wr = kmalloc(wqe_size, GFP_KERNEL); 2408 if (!user_wr) 2409 return ERR_PTR(-ENOMEM); 2410 2411 sg_ind = 0; 2412 last = NULL; 2413 for (i = 0; i < wr_count; ++i) { 2414 if (copy_from_user(user_wr, buf + i * wqe_size, 2415 wqe_size)) { 2416 ret = -EFAULT; 2417 goto err; 2418 } 2419 2420 if (user_wr->num_sge + sg_ind > sge_count) { 2421 ret = -EINVAL; 2422 goto err; 2423 } 2424 2425 if (user_wr->num_sge >= 2426 (U32_MAX - ALIGN(sizeof *next, sizeof (struct ib_sge))) / 2427 sizeof (struct ib_sge)) { 2428 ret = -EINVAL; 2429 goto err; 2430 } 2431 2432 next = kmalloc(ALIGN(sizeof *next, sizeof (struct ib_sge)) + 2433 user_wr->num_sge * sizeof (struct ib_sge), 2434 GFP_KERNEL); 2435 if (!next) { 2436 ret = -ENOMEM; 2437 goto err; 2438 } 2439 2440 if (!last) 2441 wr = next; 2442 else 2443 last->next = next; 2444 last = next; 2445 2446 next->next = NULL; 2447 next->wr_id = user_wr->wr_id; 2448 next->num_sge = user_wr->num_sge; 2449 2450 if (next->num_sge) { 2451 next->sg_list = (void *) next + 2452 ALIGN(sizeof *next, sizeof (struct ib_sge)); 2453 if (copy_from_user(next->sg_list, 2454 buf + wr_count * wqe_size + 2455 sg_ind * sizeof (struct ib_sge), 2456 next->num_sge * sizeof (struct ib_sge))) { 2457 ret = -EFAULT; 2458 goto err; 2459 } 2460 sg_ind += next->num_sge; 2461 } else 2462 next->sg_list = NULL; 2463 } 2464 2465 kfree(user_wr); 2466 return wr; 2467 2468 err: 2469 kfree(user_wr); 2470 2471 while (wr) { 2472 next = wr->next; 2473 kfree(wr); 2474 wr = next; 2475 } 2476 2477 return ERR_PTR(ret); 2478 } 2479 2480 ssize_t ib_uverbs_post_recv(struct ib_uverbs_file *file, 2481 const char __user *buf, int in_len, 2482 int out_len) 2483 { 2484 struct ib_uverbs_post_recv cmd; 2485 struct ib_uverbs_post_recv_resp resp; 2486 struct ib_recv_wr *wr, *next; 2487 const struct ib_recv_wr *bad_wr; 2488 struct ib_qp *qp; 2489 ssize_t ret = -EINVAL; 2490 2491 if (copy_from_user(&cmd, buf, sizeof cmd)) 2492 return -EFAULT; 2493 2494 wr = ib_uverbs_unmarshall_recv(buf + sizeof cmd, 2495 in_len - sizeof cmd, cmd.wr_count, 2496 cmd.sge_count, cmd.wqe_size); 2497 if (IS_ERR(wr)) 2498 return PTR_ERR(wr); 2499 2500 qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, file); 2501 if (!qp) 2502 goto out; 2503 2504 resp.bad_wr = 0; 2505 ret = qp->device->post_recv(qp->real_qp, wr, &bad_wr); 2506 2507 uobj_put_obj_read(qp); 2508 if (ret) { 2509 for (next = wr; next; next = next->next) { 2510 ++resp.bad_wr; 2511 if (next == bad_wr) 2512 break; 2513 } 2514 } 2515 2516 if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp)) 2517 ret = -EFAULT; 2518 2519 out: 2520 while (wr) { 2521 next = wr->next; 2522 kfree(wr); 2523 wr = next; 2524 } 2525 2526 return ret ? ret : in_len; 2527 } 2528 2529 ssize_t ib_uverbs_post_srq_recv(struct ib_uverbs_file *file, 2530 const char __user *buf, int in_len, 2531 int out_len) 2532 { 2533 struct ib_uverbs_post_srq_recv cmd; 2534 struct ib_uverbs_post_srq_recv_resp resp; 2535 struct ib_recv_wr *wr, *next; 2536 const struct ib_recv_wr *bad_wr; 2537 struct ib_srq *srq; 2538 ssize_t ret = -EINVAL; 2539 2540 if (copy_from_user(&cmd, buf, sizeof cmd)) 2541 return -EFAULT; 2542 2543 wr = ib_uverbs_unmarshall_recv(buf + sizeof cmd, 2544 in_len - sizeof cmd, cmd.wr_count, 2545 cmd.sge_count, cmd.wqe_size); 2546 if (IS_ERR(wr)) 2547 return PTR_ERR(wr); 2548 2549 srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ, cmd.srq_handle, file); 2550 if (!srq) 2551 goto out; 2552 2553 resp.bad_wr = 0; 2554 ret = srq->device->post_srq_recv ? 2555 srq->device->post_srq_recv(srq, wr, &bad_wr) : -EOPNOTSUPP; 2556 2557 uobj_put_obj_read(srq); 2558 2559 if (ret) 2560 for (next = wr; next; next = next->next) { 2561 ++resp.bad_wr; 2562 if (next == bad_wr) 2563 break; 2564 } 2565 2566 if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp)) 2567 ret = -EFAULT; 2568 2569 out: 2570 while (wr) { 2571 next = wr->next; 2572 kfree(wr); 2573 wr = next; 2574 } 2575 2576 return ret ? ret : in_len; 2577 } 2578 2579 ssize_t ib_uverbs_create_ah(struct ib_uverbs_file *file, 2580 const char __user *buf, int in_len, 2581 int out_len) 2582 { 2583 struct ib_uverbs_create_ah cmd; 2584 struct ib_uverbs_create_ah_resp resp; 2585 struct ib_uobject *uobj; 2586 struct ib_pd *pd; 2587 struct ib_ah *ah; 2588 struct rdma_ah_attr attr = {}; 2589 int ret; 2590 struct ib_udata udata; 2591 struct ib_device *ib_dev; 2592 2593 if (out_len < sizeof resp) 2594 return -ENOSPC; 2595 2596 if (copy_from_user(&cmd, buf, sizeof cmd)) 2597 return -EFAULT; 2598 2599 ib_uverbs_init_udata(&udata, buf + sizeof(cmd), 2600 u64_to_user_ptr(cmd.response) + sizeof(resp), 2601 in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr), 2602 out_len - sizeof(resp)); 2603 2604 uobj = uobj_alloc(UVERBS_OBJECT_AH, file, &ib_dev); 2605 if (IS_ERR(uobj)) 2606 return PTR_ERR(uobj); 2607 2608 if (!rdma_is_port_valid(ib_dev, cmd.attr.port_num)) { 2609 ret = -EINVAL; 2610 goto err; 2611 } 2612 2613 pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, file); 2614 if (!pd) { 2615 ret = -EINVAL; 2616 goto err; 2617 } 2618 2619 attr.type = rdma_ah_find_type(ib_dev, cmd.attr.port_num); 2620 rdma_ah_set_make_grd(&attr, false); 2621 rdma_ah_set_dlid(&attr, cmd.attr.dlid); 2622 rdma_ah_set_sl(&attr, cmd.attr.sl); 2623 rdma_ah_set_path_bits(&attr, cmd.attr.src_path_bits); 2624 rdma_ah_set_static_rate(&attr, cmd.attr.static_rate); 2625 rdma_ah_set_port_num(&attr, cmd.attr.port_num); 2626 2627 if (cmd.attr.is_global) { 2628 rdma_ah_set_grh(&attr, NULL, cmd.attr.grh.flow_label, 2629 cmd.attr.grh.sgid_index, 2630 cmd.attr.grh.hop_limit, 2631 cmd.attr.grh.traffic_class); 2632 rdma_ah_set_dgid_raw(&attr, cmd.attr.grh.dgid); 2633 } else { 2634 rdma_ah_set_ah_flags(&attr, 0); 2635 } 2636 2637 ah = rdma_create_user_ah(pd, &attr, &udata); 2638 if (IS_ERR(ah)) { 2639 ret = PTR_ERR(ah); 2640 goto err_put; 2641 } 2642 2643 ah->uobject = uobj; 2644 uobj->user_handle = cmd.user_handle; 2645 uobj->object = ah; 2646 2647 resp.ah_handle = uobj->id; 2648 2649 if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp)) { 2650 ret = -EFAULT; 2651 goto err_copy; 2652 } 2653 2654 uobj_put_obj_read(pd); 2655 return uobj_alloc_commit(uobj, in_len); 2656 2657 err_copy: 2658 rdma_destroy_ah(ah); 2659 2660 err_put: 2661 uobj_put_obj_read(pd); 2662 2663 err: 2664 uobj_alloc_abort(uobj); 2665 return ret; 2666 } 2667 2668 ssize_t ib_uverbs_destroy_ah(struct ib_uverbs_file *file, 2669 const char __user *buf, int in_len, int out_len) 2670 { 2671 struct ib_uverbs_destroy_ah cmd; 2672 2673 if (copy_from_user(&cmd, buf, sizeof cmd)) 2674 return -EFAULT; 2675 2676 return uobj_perform_destroy(UVERBS_OBJECT_AH, cmd.ah_handle, file, 2677 in_len); 2678 } 2679 2680 ssize_t ib_uverbs_attach_mcast(struct ib_uverbs_file *file, 2681 const char __user *buf, int in_len, 2682 int out_len) 2683 { 2684 struct ib_uverbs_attach_mcast cmd; 2685 struct ib_qp *qp; 2686 struct ib_uqp_object *obj; 2687 struct ib_uverbs_mcast_entry *mcast; 2688 int ret; 2689 2690 if (copy_from_user(&cmd, buf, sizeof cmd)) 2691 return -EFAULT; 2692 2693 qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, file); 2694 if (!qp) 2695 return -EINVAL; 2696 2697 obj = container_of(qp->uobject, struct ib_uqp_object, uevent.uobject); 2698 2699 mutex_lock(&obj->mcast_lock); 2700 list_for_each_entry(mcast, &obj->mcast_list, list) 2701 if (cmd.mlid == mcast->lid && 2702 !memcmp(cmd.gid, mcast->gid.raw, sizeof mcast->gid.raw)) { 2703 ret = 0; 2704 goto out_put; 2705 } 2706 2707 mcast = kmalloc(sizeof *mcast, GFP_KERNEL); 2708 if (!mcast) { 2709 ret = -ENOMEM; 2710 goto out_put; 2711 } 2712 2713 mcast->lid = cmd.mlid; 2714 memcpy(mcast->gid.raw, cmd.gid, sizeof mcast->gid.raw); 2715 2716 ret = ib_attach_mcast(qp, &mcast->gid, cmd.mlid); 2717 if (!ret) 2718 list_add_tail(&mcast->list, &obj->mcast_list); 2719 else 2720 kfree(mcast); 2721 2722 out_put: 2723 mutex_unlock(&obj->mcast_lock); 2724 uobj_put_obj_read(qp); 2725 2726 return ret ? ret : in_len; 2727 } 2728 2729 ssize_t ib_uverbs_detach_mcast(struct ib_uverbs_file *file, 2730 const char __user *buf, int in_len, 2731 int out_len) 2732 { 2733 struct ib_uverbs_detach_mcast cmd; 2734 struct ib_uqp_object *obj; 2735 struct ib_qp *qp; 2736 struct ib_uverbs_mcast_entry *mcast; 2737 int ret = -EINVAL; 2738 bool found = false; 2739 2740 if (copy_from_user(&cmd, buf, sizeof cmd)) 2741 return -EFAULT; 2742 2743 qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, file); 2744 if (!qp) 2745 return -EINVAL; 2746 2747 obj = container_of(qp->uobject, struct ib_uqp_object, uevent.uobject); 2748 mutex_lock(&obj->mcast_lock); 2749 2750 list_for_each_entry(mcast, &obj->mcast_list, list) 2751 if (cmd.mlid == mcast->lid && 2752 !memcmp(cmd.gid, mcast->gid.raw, sizeof mcast->gid.raw)) { 2753 list_del(&mcast->list); 2754 kfree(mcast); 2755 found = true; 2756 break; 2757 } 2758 2759 if (!found) { 2760 ret = -EINVAL; 2761 goto out_put; 2762 } 2763 2764 ret = ib_detach_mcast(qp, (union ib_gid *)cmd.gid, cmd.mlid); 2765 2766 out_put: 2767 mutex_unlock(&obj->mcast_lock); 2768 uobj_put_obj_read(qp); 2769 return ret ? ret : in_len; 2770 } 2771 2772 struct ib_uflow_resources { 2773 size_t max; 2774 size_t num; 2775 size_t collection_num; 2776 size_t counters_num; 2777 struct ib_counters **counters; 2778 struct ib_flow_action **collection; 2779 }; 2780 2781 static struct ib_uflow_resources *flow_resources_alloc(size_t num_specs) 2782 { 2783 struct ib_uflow_resources *resources; 2784 2785 resources = kzalloc(sizeof(*resources), GFP_KERNEL); 2786 2787 if (!resources) 2788 return NULL; 2789 2790 if (!num_specs) 2791 goto out; 2792 2793 resources->counters = 2794 kcalloc(num_specs, sizeof(*resources->counters), GFP_KERNEL); 2795 resources->collection = 2796 kcalloc(num_specs, sizeof(*resources->collection), GFP_KERNEL); 2797 2798 if (!resources->counters || !resources->collection) 2799 goto err; 2800 2801 out: 2802 resources->max = num_specs; 2803 return resources; 2804 2805 err: 2806 kfree(resources->counters); 2807 kfree(resources); 2808 2809 return NULL; 2810 } 2811 2812 void ib_uverbs_flow_resources_free(struct ib_uflow_resources *uflow_res) 2813 { 2814 unsigned int i; 2815 2816 if (!uflow_res) 2817 return; 2818 2819 for (i = 0; i < uflow_res->collection_num; i++) 2820 atomic_dec(&uflow_res->collection[i]->usecnt); 2821 2822 for (i = 0; i < uflow_res->counters_num; i++) 2823 atomic_dec(&uflow_res->counters[i]->usecnt); 2824 2825 kfree(uflow_res->collection); 2826 kfree(uflow_res->counters); 2827 kfree(uflow_res); 2828 } 2829 2830 static void flow_resources_add(struct ib_uflow_resources *uflow_res, 2831 enum ib_flow_spec_type type, 2832 void *ibobj) 2833 { 2834 WARN_ON(uflow_res->num >= uflow_res->max); 2835 2836 switch (type) { 2837 case IB_FLOW_SPEC_ACTION_HANDLE: 2838 atomic_inc(&((struct ib_flow_action *)ibobj)->usecnt); 2839 uflow_res->collection[uflow_res->collection_num++] = 2840 (struct ib_flow_action *)ibobj; 2841 break; 2842 case IB_FLOW_SPEC_ACTION_COUNT: 2843 atomic_inc(&((struct ib_counters *)ibobj)->usecnt); 2844 uflow_res->counters[uflow_res->counters_num++] = 2845 (struct ib_counters *)ibobj; 2846 break; 2847 default: 2848 WARN_ON(1); 2849 } 2850 2851 uflow_res->num++; 2852 } 2853 2854 static int kern_spec_to_ib_spec_action(struct ib_uverbs_file *ufile, 2855 struct ib_uverbs_flow_spec *kern_spec, 2856 union ib_flow_spec *ib_spec, 2857 struct ib_uflow_resources *uflow_res) 2858 { 2859 ib_spec->type = kern_spec->type; 2860 switch (ib_spec->type) { 2861 case IB_FLOW_SPEC_ACTION_TAG: 2862 if (kern_spec->flow_tag.size != 2863 sizeof(struct ib_uverbs_flow_spec_action_tag)) 2864 return -EINVAL; 2865 2866 ib_spec->flow_tag.size = sizeof(struct ib_flow_spec_action_tag); 2867 ib_spec->flow_tag.tag_id = kern_spec->flow_tag.tag_id; 2868 break; 2869 case IB_FLOW_SPEC_ACTION_DROP: 2870 if (kern_spec->drop.size != 2871 sizeof(struct ib_uverbs_flow_spec_action_drop)) 2872 return -EINVAL; 2873 2874 ib_spec->drop.size = sizeof(struct ib_flow_spec_action_drop); 2875 break; 2876 case IB_FLOW_SPEC_ACTION_HANDLE: 2877 if (kern_spec->action.size != 2878 sizeof(struct ib_uverbs_flow_spec_action_handle)) 2879 return -EOPNOTSUPP; 2880 ib_spec->action.act = uobj_get_obj_read(flow_action, 2881 UVERBS_OBJECT_FLOW_ACTION, 2882 kern_spec->action.handle, 2883 ufile); 2884 if (!ib_spec->action.act) 2885 return -EINVAL; 2886 ib_spec->action.size = 2887 sizeof(struct ib_flow_spec_action_handle); 2888 flow_resources_add(uflow_res, 2889 IB_FLOW_SPEC_ACTION_HANDLE, 2890 ib_spec->action.act); 2891 uobj_put_obj_read(ib_spec->action.act); 2892 break; 2893 case IB_FLOW_SPEC_ACTION_COUNT: 2894 if (kern_spec->flow_count.size != 2895 sizeof(struct ib_uverbs_flow_spec_action_count)) 2896 return -EINVAL; 2897 ib_spec->flow_count.counters = 2898 uobj_get_obj_read(counters, 2899 UVERBS_OBJECT_COUNTERS, 2900 kern_spec->flow_count.handle, 2901 ufile); 2902 if (!ib_spec->flow_count.counters) 2903 return -EINVAL; 2904 ib_spec->flow_count.size = 2905 sizeof(struct ib_flow_spec_action_count); 2906 flow_resources_add(uflow_res, 2907 IB_FLOW_SPEC_ACTION_COUNT, 2908 ib_spec->flow_count.counters); 2909 uobj_put_obj_read(ib_spec->flow_count.counters); 2910 break; 2911 default: 2912 return -EINVAL; 2913 } 2914 return 0; 2915 } 2916 2917 static size_t kern_spec_filter_sz(const struct ib_uverbs_flow_spec_hdr *spec) 2918 { 2919 /* Returns user space filter size, includes padding */ 2920 return (spec->size - sizeof(struct ib_uverbs_flow_spec_hdr)) / 2; 2921 } 2922 2923 static ssize_t spec_filter_size(const void *kern_spec_filter, u16 kern_filter_size, 2924 u16 ib_real_filter_sz) 2925 { 2926 /* 2927 * User space filter structures must be 64 bit aligned, otherwise this 2928 * may pass, but we won't handle additional new attributes. 2929 */ 2930 2931 if (kern_filter_size > ib_real_filter_sz) { 2932 if (memchr_inv(kern_spec_filter + 2933 ib_real_filter_sz, 0, 2934 kern_filter_size - ib_real_filter_sz)) 2935 return -EINVAL; 2936 return ib_real_filter_sz; 2937 } 2938 return kern_filter_size; 2939 } 2940 2941 int ib_uverbs_kern_spec_to_ib_spec_filter(enum ib_flow_spec_type type, 2942 const void *kern_spec_mask, 2943 const void *kern_spec_val, 2944 size_t kern_filter_sz, 2945 union ib_flow_spec *ib_spec) 2946 { 2947 ssize_t actual_filter_sz; 2948 ssize_t ib_filter_sz; 2949 2950 /* User flow spec size must be aligned to 4 bytes */ 2951 if (kern_filter_sz != ALIGN(kern_filter_sz, 4)) 2952 return -EINVAL; 2953 2954 ib_spec->type = type; 2955 2956 if (ib_spec->type == (IB_FLOW_SPEC_INNER | IB_FLOW_SPEC_VXLAN_TUNNEL)) 2957 return -EINVAL; 2958 2959 switch (ib_spec->type & ~IB_FLOW_SPEC_INNER) { 2960 case IB_FLOW_SPEC_ETH: 2961 ib_filter_sz = offsetof(struct ib_flow_eth_filter, real_sz); 2962 actual_filter_sz = spec_filter_size(kern_spec_mask, 2963 kern_filter_sz, 2964 ib_filter_sz); 2965 if (actual_filter_sz <= 0) 2966 return -EINVAL; 2967 ib_spec->size = sizeof(struct ib_flow_spec_eth); 2968 memcpy(&ib_spec->eth.val, kern_spec_val, actual_filter_sz); 2969 memcpy(&ib_spec->eth.mask, kern_spec_mask, actual_filter_sz); 2970 break; 2971 case IB_FLOW_SPEC_IPV4: 2972 ib_filter_sz = offsetof(struct ib_flow_ipv4_filter, real_sz); 2973 actual_filter_sz = spec_filter_size(kern_spec_mask, 2974 kern_filter_sz, 2975 ib_filter_sz); 2976 if (actual_filter_sz <= 0) 2977 return -EINVAL; 2978 ib_spec->size = sizeof(struct ib_flow_spec_ipv4); 2979 memcpy(&ib_spec->ipv4.val, kern_spec_val, actual_filter_sz); 2980 memcpy(&ib_spec->ipv4.mask, kern_spec_mask, actual_filter_sz); 2981 break; 2982 case IB_FLOW_SPEC_IPV6: 2983 ib_filter_sz = offsetof(struct ib_flow_ipv6_filter, real_sz); 2984 actual_filter_sz = spec_filter_size(kern_spec_mask, 2985 kern_filter_sz, 2986 ib_filter_sz); 2987 if (actual_filter_sz <= 0) 2988 return -EINVAL; 2989 ib_spec->size = sizeof(struct ib_flow_spec_ipv6); 2990 memcpy(&ib_spec->ipv6.val, kern_spec_val, actual_filter_sz); 2991 memcpy(&ib_spec->ipv6.mask, kern_spec_mask, actual_filter_sz); 2992 2993 if ((ntohl(ib_spec->ipv6.mask.flow_label)) >= BIT(20) || 2994 (ntohl(ib_spec->ipv6.val.flow_label)) >= BIT(20)) 2995 return -EINVAL; 2996 break; 2997 case IB_FLOW_SPEC_TCP: 2998 case IB_FLOW_SPEC_UDP: 2999 ib_filter_sz = offsetof(struct ib_flow_tcp_udp_filter, real_sz); 3000 actual_filter_sz = spec_filter_size(kern_spec_mask, 3001 kern_filter_sz, 3002 ib_filter_sz); 3003 if (actual_filter_sz <= 0) 3004 return -EINVAL; 3005 ib_spec->size = sizeof(struct ib_flow_spec_tcp_udp); 3006 memcpy(&ib_spec->tcp_udp.val, kern_spec_val, actual_filter_sz); 3007 memcpy(&ib_spec->tcp_udp.mask, kern_spec_mask, actual_filter_sz); 3008 break; 3009 case IB_FLOW_SPEC_VXLAN_TUNNEL: 3010 ib_filter_sz = offsetof(struct ib_flow_tunnel_filter, real_sz); 3011 actual_filter_sz = spec_filter_size(kern_spec_mask, 3012 kern_filter_sz, 3013 ib_filter_sz); 3014 if (actual_filter_sz <= 0) 3015 return -EINVAL; 3016 ib_spec->tunnel.size = sizeof(struct ib_flow_spec_tunnel); 3017 memcpy(&ib_spec->tunnel.val, kern_spec_val, actual_filter_sz); 3018 memcpy(&ib_spec->tunnel.mask, kern_spec_mask, actual_filter_sz); 3019 3020 if ((ntohl(ib_spec->tunnel.mask.tunnel_id)) >= BIT(24) || 3021 (ntohl(ib_spec->tunnel.val.tunnel_id)) >= BIT(24)) 3022 return -EINVAL; 3023 break; 3024 case IB_FLOW_SPEC_ESP: 3025 ib_filter_sz = offsetof(struct ib_flow_esp_filter, real_sz); 3026 actual_filter_sz = spec_filter_size(kern_spec_mask, 3027 kern_filter_sz, 3028 ib_filter_sz); 3029 if (actual_filter_sz <= 0) 3030 return -EINVAL; 3031 ib_spec->esp.size = sizeof(struct ib_flow_spec_esp); 3032 memcpy(&ib_spec->esp.val, kern_spec_val, actual_filter_sz); 3033 memcpy(&ib_spec->esp.mask, kern_spec_mask, actual_filter_sz); 3034 break; 3035 case IB_FLOW_SPEC_GRE: 3036 ib_filter_sz = offsetof(struct ib_flow_gre_filter, real_sz); 3037 actual_filter_sz = spec_filter_size(kern_spec_mask, 3038 kern_filter_sz, 3039 ib_filter_sz); 3040 if (actual_filter_sz <= 0) 3041 return -EINVAL; 3042 ib_spec->gre.size = sizeof(struct ib_flow_spec_gre); 3043 memcpy(&ib_spec->gre.val, kern_spec_val, actual_filter_sz); 3044 memcpy(&ib_spec->gre.mask, kern_spec_mask, actual_filter_sz); 3045 break; 3046 case IB_FLOW_SPEC_MPLS: 3047 ib_filter_sz = offsetof(struct ib_flow_mpls_filter, real_sz); 3048 actual_filter_sz = spec_filter_size(kern_spec_mask, 3049 kern_filter_sz, 3050 ib_filter_sz); 3051 if (actual_filter_sz <= 0) 3052 return -EINVAL; 3053 ib_spec->mpls.size = sizeof(struct ib_flow_spec_mpls); 3054 memcpy(&ib_spec->mpls.val, kern_spec_val, actual_filter_sz); 3055 memcpy(&ib_spec->mpls.mask, kern_spec_mask, actual_filter_sz); 3056 break; 3057 default: 3058 return -EINVAL; 3059 } 3060 return 0; 3061 } 3062 3063 static int kern_spec_to_ib_spec_filter(struct ib_uverbs_flow_spec *kern_spec, 3064 union ib_flow_spec *ib_spec) 3065 { 3066 ssize_t kern_filter_sz; 3067 void *kern_spec_mask; 3068 void *kern_spec_val; 3069 3070 kern_filter_sz = kern_spec_filter_sz(&kern_spec->hdr); 3071 3072 kern_spec_val = (void *)kern_spec + 3073 sizeof(struct ib_uverbs_flow_spec_hdr); 3074 kern_spec_mask = kern_spec_val + kern_filter_sz; 3075 3076 return ib_uverbs_kern_spec_to_ib_spec_filter(kern_spec->type, 3077 kern_spec_mask, 3078 kern_spec_val, 3079 kern_filter_sz, ib_spec); 3080 } 3081 3082 static int kern_spec_to_ib_spec(struct ib_uverbs_file *ufile, 3083 struct ib_uverbs_flow_spec *kern_spec, 3084 union ib_flow_spec *ib_spec, 3085 struct ib_uflow_resources *uflow_res) 3086 { 3087 if (kern_spec->reserved) 3088 return -EINVAL; 3089 3090 if (kern_spec->type >= IB_FLOW_SPEC_ACTION_TAG) 3091 return kern_spec_to_ib_spec_action(ufile, kern_spec, ib_spec, 3092 uflow_res); 3093 else 3094 return kern_spec_to_ib_spec_filter(kern_spec, ib_spec); 3095 } 3096 3097 int ib_uverbs_ex_create_wq(struct ib_uverbs_file *file, 3098 struct ib_udata *ucore, 3099 struct ib_udata *uhw) 3100 { 3101 struct ib_uverbs_ex_create_wq cmd = {}; 3102 struct ib_uverbs_ex_create_wq_resp resp = {}; 3103 struct ib_uwq_object *obj; 3104 int err = 0; 3105 struct ib_cq *cq; 3106 struct ib_pd *pd; 3107 struct ib_wq *wq; 3108 struct ib_wq_init_attr wq_init_attr = {}; 3109 size_t required_cmd_sz; 3110 size_t required_resp_len; 3111 struct ib_device *ib_dev; 3112 3113 required_cmd_sz = offsetof(typeof(cmd), max_sge) + sizeof(cmd.max_sge); 3114 required_resp_len = offsetof(typeof(resp), wqn) + sizeof(resp.wqn); 3115 3116 if (ucore->inlen < required_cmd_sz) 3117 return -EINVAL; 3118 3119 if (ucore->outlen < required_resp_len) 3120 return -ENOSPC; 3121 3122 if (ucore->inlen > sizeof(cmd) && 3123 !ib_is_udata_cleared(ucore, sizeof(cmd), 3124 ucore->inlen - sizeof(cmd))) 3125 return -EOPNOTSUPP; 3126 3127 err = ib_copy_from_udata(&cmd, ucore, min(sizeof(cmd), ucore->inlen)); 3128 if (err) 3129 return err; 3130 3131 if (cmd.comp_mask) 3132 return -EOPNOTSUPP; 3133 3134 obj = (struct ib_uwq_object *)uobj_alloc(UVERBS_OBJECT_WQ, file, 3135 &ib_dev); 3136 if (IS_ERR(obj)) 3137 return PTR_ERR(obj); 3138 3139 pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, file); 3140 if (!pd) { 3141 err = -EINVAL; 3142 goto err_uobj; 3143 } 3144 3145 cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, file); 3146 if (!cq) { 3147 err = -EINVAL; 3148 goto err_put_pd; 3149 } 3150 3151 wq_init_attr.cq = cq; 3152 wq_init_attr.max_sge = cmd.max_sge; 3153 wq_init_attr.max_wr = cmd.max_wr; 3154 wq_init_attr.wq_context = file; 3155 wq_init_attr.wq_type = cmd.wq_type; 3156 wq_init_attr.event_handler = ib_uverbs_wq_event_handler; 3157 if (ucore->inlen >= (offsetof(typeof(cmd), create_flags) + 3158 sizeof(cmd.create_flags))) 3159 wq_init_attr.create_flags = cmd.create_flags; 3160 obj->uevent.events_reported = 0; 3161 INIT_LIST_HEAD(&obj->uevent.event_list); 3162 3163 if (!pd->device->create_wq) { 3164 err = -EOPNOTSUPP; 3165 goto err_put_cq; 3166 } 3167 wq = pd->device->create_wq(pd, &wq_init_attr, uhw); 3168 if (IS_ERR(wq)) { 3169 err = PTR_ERR(wq); 3170 goto err_put_cq; 3171 } 3172 3173 wq->uobject = &obj->uevent.uobject; 3174 obj->uevent.uobject.object = wq; 3175 wq->wq_type = wq_init_attr.wq_type; 3176 wq->cq = cq; 3177 wq->pd = pd; 3178 wq->device = pd->device; 3179 wq->wq_context = wq_init_attr.wq_context; 3180 atomic_set(&wq->usecnt, 0); 3181 atomic_inc(&pd->usecnt); 3182 atomic_inc(&cq->usecnt); 3183 wq->uobject = &obj->uevent.uobject; 3184 obj->uevent.uobject.object = wq; 3185 3186 memset(&resp, 0, sizeof(resp)); 3187 resp.wq_handle = obj->uevent.uobject.id; 3188 resp.max_sge = wq_init_attr.max_sge; 3189 resp.max_wr = wq_init_attr.max_wr; 3190 resp.wqn = wq->wq_num; 3191 resp.response_length = required_resp_len; 3192 err = ib_copy_to_udata(ucore, 3193 &resp, resp.response_length); 3194 if (err) 3195 goto err_copy; 3196 3197 uobj_put_obj_read(pd); 3198 uobj_put_obj_read(cq); 3199 return uobj_alloc_commit(&obj->uevent.uobject, 0); 3200 3201 err_copy: 3202 ib_destroy_wq(wq); 3203 err_put_cq: 3204 uobj_put_obj_read(cq); 3205 err_put_pd: 3206 uobj_put_obj_read(pd); 3207 err_uobj: 3208 uobj_alloc_abort(&obj->uevent.uobject); 3209 3210 return err; 3211 } 3212 3213 int ib_uverbs_ex_destroy_wq(struct ib_uverbs_file *file, 3214 struct ib_udata *ucore, 3215 struct ib_udata *uhw) 3216 { 3217 struct ib_uverbs_ex_destroy_wq cmd = {}; 3218 struct ib_uverbs_ex_destroy_wq_resp resp = {}; 3219 struct ib_uobject *uobj; 3220 struct ib_uwq_object *obj; 3221 size_t required_cmd_sz; 3222 size_t required_resp_len; 3223 int ret; 3224 3225 required_cmd_sz = offsetof(typeof(cmd), wq_handle) + sizeof(cmd.wq_handle); 3226 required_resp_len = offsetof(typeof(resp), reserved) + sizeof(resp.reserved); 3227 3228 if (ucore->inlen < required_cmd_sz) 3229 return -EINVAL; 3230 3231 if (ucore->outlen < required_resp_len) 3232 return -ENOSPC; 3233 3234 if (ucore->inlen > sizeof(cmd) && 3235 !ib_is_udata_cleared(ucore, sizeof(cmd), 3236 ucore->inlen - sizeof(cmd))) 3237 return -EOPNOTSUPP; 3238 3239 ret = ib_copy_from_udata(&cmd, ucore, min(sizeof(cmd), ucore->inlen)); 3240 if (ret) 3241 return ret; 3242 3243 if (cmd.comp_mask) 3244 return -EOPNOTSUPP; 3245 3246 resp.response_length = required_resp_len; 3247 uobj = uobj_get_destroy(UVERBS_OBJECT_WQ, cmd.wq_handle, file); 3248 if (IS_ERR(uobj)) 3249 return PTR_ERR(uobj); 3250 3251 obj = container_of(uobj, struct ib_uwq_object, uevent.uobject); 3252 resp.events_reported = obj->uevent.events_reported; 3253 3254 uobj_put_destroy(uobj); 3255 3256 return ib_copy_to_udata(ucore, &resp, resp.response_length); 3257 } 3258 3259 int ib_uverbs_ex_modify_wq(struct ib_uverbs_file *file, 3260 struct ib_udata *ucore, 3261 struct ib_udata *uhw) 3262 { 3263 struct ib_uverbs_ex_modify_wq cmd = {}; 3264 struct ib_wq *wq; 3265 struct ib_wq_attr wq_attr = {}; 3266 size_t required_cmd_sz; 3267 int ret; 3268 3269 required_cmd_sz = offsetof(typeof(cmd), curr_wq_state) + sizeof(cmd.curr_wq_state); 3270 if (ucore->inlen < required_cmd_sz) 3271 return -EINVAL; 3272 3273 if (ucore->inlen > sizeof(cmd) && 3274 !ib_is_udata_cleared(ucore, sizeof(cmd), 3275 ucore->inlen - sizeof(cmd))) 3276 return -EOPNOTSUPP; 3277 3278 ret = ib_copy_from_udata(&cmd, ucore, min(sizeof(cmd), ucore->inlen)); 3279 if (ret) 3280 return ret; 3281 3282 if (!cmd.attr_mask) 3283 return -EINVAL; 3284 3285 if (cmd.attr_mask > (IB_WQ_STATE | IB_WQ_CUR_STATE | IB_WQ_FLAGS)) 3286 return -EINVAL; 3287 3288 wq = uobj_get_obj_read(wq, UVERBS_OBJECT_WQ, cmd.wq_handle, file); 3289 if (!wq) 3290 return -EINVAL; 3291 3292 wq_attr.curr_wq_state = cmd.curr_wq_state; 3293 wq_attr.wq_state = cmd.wq_state; 3294 if (cmd.attr_mask & IB_WQ_FLAGS) { 3295 wq_attr.flags = cmd.flags; 3296 wq_attr.flags_mask = cmd.flags_mask; 3297 } 3298 if (!wq->device->modify_wq) { 3299 ret = -EOPNOTSUPP; 3300 goto out; 3301 } 3302 ret = wq->device->modify_wq(wq, &wq_attr, cmd.attr_mask, uhw); 3303 out: 3304 uobj_put_obj_read(wq); 3305 return ret; 3306 } 3307 3308 int ib_uverbs_ex_create_rwq_ind_table(struct ib_uverbs_file *file, 3309 struct ib_udata *ucore, 3310 struct ib_udata *uhw) 3311 { 3312 struct ib_uverbs_ex_create_rwq_ind_table cmd = {}; 3313 struct ib_uverbs_ex_create_rwq_ind_table_resp resp = {}; 3314 struct ib_uobject *uobj; 3315 int err = 0; 3316 struct ib_rwq_ind_table_init_attr init_attr = {}; 3317 struct ib_rwq_ind_table *rwq_ind_tbl; 3318 struct ib_wq **wqs = NULL; 3319 u32 *wqs_handles = NULL; 3320 struct ib_wq *wq = NULL; 3321 int i, j, num_read_wqs; 3322 u32 num_wq_handles; 3323 u32 expected_in_size; 3324 size_t required_cmd_sz_header; 3325 size_t required_resp_len; 3326 struct ib_device *ib_dev; 3327 3328 required_cmd_sz_header = offsetof(typeof(cmd), log_ind_tbl_size) + sizeof(cmd.log_ind_tbl_size); 3329 required_resp_len = offsetof(typeof(resp), ind_tbl_num) + sizeof(resp.ind_tbl_num); 3330 3331 if (ucore->inlen < required_cmd_sz_header) 3332 return -EINVAL; 3333 3334 if (ucore->outlen < required_resp_len) 3335 return -ENOSPC; 3336 3337 err = ib_copy_from_udata(&cmd, ucore, required_cmd_sz_header); 3338 if (err) 3339 return err; 3340 3341 ucore->inbuf += required_cmd_sz_header; 3342 ucore->inlen -= required_cmd_sz_header; 3343 3344 if (cmd.comp_mask) 3345 return -EOPNOTSUPP; 3346 3347 if (cmd.log_ind_tbl_size > IB_USER_VERBS_MAX_LOG_IND_TBL_SIZE) 3348 return -EINVAL; 3349 3350 num_wq_handles = 1 << cmd.log_ind_tbl_size; 3351 expected_in_size = num_wq_handles * sizeof(__u32); 3352 if (num_wq_handles == 1) 3353 /* input size for wq handles is u64 aligned */ 3354 expected_in_size += sizeof(__u32); 3355 3356 if (ucore->inlen < expected_in_size) 3357 return -EINVAL; 3358 3359 if (ucore->inlen > expected_in_size && 3360 !ib_is_udata_cleared(ucore, expected_in_size, 3361 ucore->inlen - expected_in_size)) 3362 return -EOPNOTSUPP; 3363 3364 wqs_handles = kcalloc(num_wq_handles, sizeof(*wqs_handles), 3365 GFP_KERNEL); 3366 if (!wqs_handles) 3367 return -ENOMEM; 3368 3369 err = ib_copy_from_udata(wqs_handles, ucore, 3370 num_wq_handles * sizeof(__u32)); 3371 if (err) 3372 goto err_free; 3373 3374 wqs = kcalloc(num_wq_handles, sizeof(*wqs), GFP_KERNEL); 3375 if (!wqs) { 3376 err = -ENOMEM; 3377 goto err_free; 3378 } 3379 3380 for (num_read_wqs = 0; num_read_wqs < num_wq_handles; 3381 num_read_wqs++) { 3382 wq = uobj_get_obj_read(wq, UVERBS_OBJECT_WQ, 3383 wqs_handles[num_read_wqs], file); 3384 if (!wq) { 3385 err = -EINVAL; 3386 goto put_wqs; 3387 } 3388 3389 wqs[num_read_wqs] = wq; 3390 } 3391 3392 uobj = uobj_alloc(UVERBS_OBJECT_RWQ_IND_TBL, file, &ib_dev); 3393 if (IS_ERR(uobj)) { 3394 err = PTR_ERR(uobj); 3395 goto put_wqs; 3396 } 3397 3398 init_attr.log_ind_tbl_size = cmd.log_ind_tbl_size; 3399 init_attr.ind_tbl = wqs; 3400 3401 if (!ib_dev->create_rwq_ind_table) { 3402 err = -EOPNOTSUPP; 3403 goto err_uobj; 3404 } 3405 rwq_ind_tbl = ib_dev->create_rwq_ind_table(ib_dev, &init_attr, uhw); 3406 3407 if (IS_ERR(rwq_ind_tbl)) { 3408 err = PTR_ERR(rwq_ind_tbl); 3409 goto err_uobj; 3410 } 3411 3412 rwq_ind_tbl->ind_tbl = wqs; 3413 rwq_ind_tbl->log_ind_tbl_size = init_attr.log_ind_tbl_size; 3414 rwq_ind_tbl->uobject = uobj; 3415 uobj->object = rwq_ind_tbl; 3416 rwq_ind_tbl->device = ib_dev; 3417 atomic_set(&rwq_ind_tbl->usecnt, 0); 3418 3419 for (i = 0; i < num_wq_handles; i++) 3420 atomic_inc(&wqs[i]->usecnt); 3421 3422 resp.ind_tbl_handle = uobj->id; 3423 resp.ind_tbl_num = rwq_ind_tbl->ind_tbl_num; 3424 resp.response_length = required_resp_len; 3425 3426 err = ib_copy_to_udata(ucore, 3427 &resp, resp.response_length); 3428 if (err) 3429 goto err_copy; 3430 3431 kfree(wqs_handles); 3432 3433 for (j = 0; j < num_read_wqs; j++) 3434 uobj_put_obj_read(wqs[j]); 3435 3436 return uobj_alloc_commit(uobj, 0); 3437 3438 err_copy: 3439 ib_destroy_rwq_ind_table(rwq_ind_tbl); 3440 err_uobj: 3441 uobj_alloc_abort(uobj); 3442 put_wqs: 3443 for (j = 0; j < num_read_wqs; j++) 3444 uobj_put_obj_read(wqs[j]); 3445 err_free: 3446 kfree(wqs_handles); 3447 kfree(wqs); 3448 return err; 3449 } 3450 3451 int ib_uverbs_ex_destroy_rwq_ind_table(struct ib_uverbs_file *file, 3452 struct ib_udata *ucore, 3453 struct ib_udata *uhw) 3454 { 3455 struct ib_uverbs_ex_destroy_rwq_ind_table cmd = {}; 3456 int ret; 3457 size_t required_cmd_sz; 3458 3459 required_cmd_sz = offsetof(typeof(cmd), ind_tbl_handle) + sizeof(cmd.ind_tbl_handle); 3460 3461 if (ucore->inlen < required_cmd_sz) 3462 return -EINVAL; 3463 3464 if (ucore->inlen > sizeof(cmd) && 3465 !ib_is_udata_cleared(ucore, sizeof(cmd), 3466 ucore->inlen - sizeof(cmd))) 3467 return -EOPNOTSUPP; 3468 3469 ret = ib_copy_from_udata(&cmd, ucore, min(sizeof(cmd), ucore->inlen)); 3470 if (ret) 3471 return ret; 3472 3473 if (cmd.comp_mask) 3474 return -EOPNOTSUPP; 3475 3476 return uobj_perform_destroy(UVERBS_OBJECT_RWQ_IND_TBL, 3477 cmd.ind_tbl_handle, file, 0); 3478 } 3479 3480 int ib_uverbs_ex_create_flow(struct ib_uverbs_file *file, 3481 struct ib_udata *ucore, 3482 struct ib_udata *uhw) 3483 { 3484 struct ib_uverbs_create_flow cmd; 3485 struct ib_uverbs_create_flow_resp resp; 3486 struct ib_uobject *uobj; 3487 struct ib_uflow_object *uflow; 3488 struct ib_flow *flow_id; 3489 struct ib_uverbs_flow_attr *kern_flow_attr; 3490 struct ib_flow_attr *flow_attr; 3491 struct ib_qp *qp; 3492 struct ib_uflow_resources *uflow_res; 3493 struct ib_uverbs_flow_spec_hdr *kern_spec; 3494 int err = 0; 3495 void *ib_spec; 3496 int i; 3497 struct ib_device *ib_dev; 3498 3499 if (ucore->inlen < sizeof(cmd)) 3500 return -EINVAL; 3501 3502 if (ucore->outlen < sizeof(resp)) 3503 return -ENOSPC; 3504 3505 err = ib_copy_from_udata(&cmd, ucore, sizeof(cmd)); 3506 if (err) 3507 return err; 3508 3509 ucore->inbuf += sizeof(cmd); 3510 ucore->inlen -= sizeof(cmd); 3511 3512 if (cmd.comp_mask) 3513 return -EINVAL; 3514 3515 if (!capable(CAP_NET_RAW)) 3516 return -EPERM; 3517 3518 if (cmd.flow_attr.flags >= IB_FLOW_ATTR_FLAGS_RESERVED) 3519 return -EINVAL; 3520 3521 if ((cmd.flow_attr.flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP) && 3522 ((cmd.flow_attr.type == IB_FLOW_ATTR_ALL_DEFAULT) || 3523 (cmd.flow_attr.type == IB_FLOW_ATTR_MC_DEFAULT))) 3524 return -EINVAL; 3525 3526 if (cmd.flow_attr.num_of_specs > IB_FLOW_SPEC_SUPPORT_LAYERS) 3527 return -EINVAL; 3528 3529 if (cmd.flow_attr.size > ucore->inlen || 3530 cmd.flow_attr.size > 3531 (cmd.flow_attr.num_of_specs * sizeof(struct ib_uverbs_flow_spec))) 3532 return -EINVAL; 3533 3534 if (cmd.flow_attr.reserved[0] || 3535 cmd.flow_attr.reserved[1]) 3536 return -EINVAL; 3537 3538 if (cmd.flow_attr.num_of_specs) { 3539 kern_flow_attr = kmalloc(sizeof(*kern_flow_attr) + cmd.flow_attr.size, 3540 GFP_KERNEL); 3541 if (!kern_flow_attr) 3542 return -ENOMEM; 3543 3544 *kern_flow_attr = cmd.flow_attr; 3545 err = ib_copy_from_udata(&kern_flow_attr->flow_specs, ucore, 3546 cmd.flow_attr.size); 3547 if (err) 3548 goto err_free_attr; 3549 } else { 3550 kern_flow_attr = &cmd.flow_attr; 3551 } 3552 3553 uobj = uobj_alloc(UVERBS_OBJECT_FLOW, file, &ib_dev); 3554 if (IS_ERR(uobj)) { 3555 err = PTR_ERR(uobj); 3556 goto err_free_attr; 3557 } 3558 3559 qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, file); 3560 if (!qp) { 3561 err = -EINVAL; 3562 goto err_uobj; 3563 } 3564 3565 if (qp->qp_type != IB_QPT_UD && qp->qp_type != IB_QPT_RAW_PACKET) { 3566 err = -EINVAL; 3567 goto err_put; 3568 } 3569 3570 if (!qp->device->create_flow) { 3571 err = -EOPNOTSUPP; 3572 goto err_put; 3573 } 3574 3575 flow_attr = kzalloc(struct_size(flow_attr, flows, 3576 cmd.flow_attr.num_of_specs), GFP_KERNEL); 3577 if (!flow_attr) { 3578 err = -ENOMEM; 3579 goto err_put; 3580 } 3581 uflow_res = flow_resources_alloc(cmd.flow_attr.num_of_specs); 3582 if (!uflow_res) { 3583 err = -ENOMEM; 3584 goto err_free_flow_attr; 3585 } 3586 3587 flow_attr->type = kern_flow_attr->type; 3588 flow_attr->priority = kern_flow_attr->priority; 3589 flow_attr->num_of_specs = kern_flow_attr->num_of_specs; 3590 flow_attr->port = kern_flow_attr->port; 3591 flow_attr->flags = kern_flow_attr->flags; 3592 flow_attr->size = sizeof(*flow_attr); 3593 3594 kern_spec = kern_flow_attr->flow_specs; 3595 ib_spec = flow_attr + 1; 3596 for (i = 0; i < flow_attr->num_of_specs && 3597 cmd.flow_attr.size >= sizeof(*kern_spec) && 3598 cmd.flow_attr.size >= kern_spec->size; 3599 i++) { 3600 err = kern_spec_to_ib_spec( 3601 file, (struct ib_uverbs_flow_spec *)kern_spec, 3602 ib_spec, uflow_res); 3603 if (err) 3604 goto err_free; 3605 3606 flow_attr->size += 3607 ((union ib_flow_spec *) ib_spec)->size; 3608 cmd.flow_attr.size -= kern_spec->size; 3609 kern_spec = ((void *)kern_spec) + kern_spec->size; 3610 ib_spec += ((union ib_flow_spec *) ib_spec)->size; 3611 } 3612 if (cmd.flow_attr.size || (i != flow_attr->num_of_specs)) { 3613 pr_warn("create flow failed, flow %d: %d bytes left from uverb cmd\n", 3614 i, cmd.flow_attr.size); 3615 err = -EINVAL; 3616 goto err_free; 3617 } 3618 3619 flow_id = qp->device->create_flow(qp, flow_attr, 3620 IB_FLOW_DOMAIN_USER, uhw); 3621 3622 if (IS_ERR(flow_id)) { 3623 err = PTR_ERR(flow_id); 3624 goto err_free; 3625 } 3626 atomic_inc(&qp->usecnt); 3627 flow_id->qp = qp; 3628 flow_id->device = qp->device; 3629 flow_id->uobject = uobj; 3630 uobj->object = flow_id; 3631 uflow = container_of(uobj, typeof(*uflow), uobject); 3632 uflow->resources = uflow_res; 3633 3634 memset(&resp, 0, sizeof(resp)); 3635 resp.flow_handle = uobj->id; 3636 3637 err = ib_copy_to_udata(ucore, 3638 &resp, sizeof(resp)); 3639 if (err) 3640 goto err_copy; 3641 3642 uobj_put_obj_read(qp); 3643 kfree(flow_attr); 3644 if (cmd.flow_attr.num_of_specs) 3645 kfree(kern_flow_attr); 3646 return uobj_alloc_commit(uobj, 0); 3647 err_copy: 3648 if (!qp->device->destroy_flow(flow_id)) 3649 atomic_dec(&qp->usecnt); 3650 err_free: 3651 ib_uverbs_flow_resources_free(uflow_res); 3652 err_free_flow_attr: 3653 kfree(flow_attr); 3654 err_put: 3655 uobj_put_obj_read(qp); 3656 err_uobj: 3657 uobj_alloc_abort(uobj); 3658 err_free_attr: 3659 if (cmd.flow_attr.num_of_specs) 3660 kfree(kern_flow_attr); 3661 return err; 3662 } 3663 3664 int ib_uverbs_ex_destroy_flow(struct ib_uverbs_file *file, 3665 struct ib_udata *ucore, 3666 struct ib_udata *uhw) 3667 { 3668 struct ib_uverbs_destroy_flow cmd; 3669 int ret; 3670 3671 if (ucore->inlen < sizeof(cmd)) 3672 return -EINVAL; 3673 3674 ret = ib_copy_from_udata(&cmd, ucore, sizeof(cmd)); 3675 if (ret) 3676 return ret; 3677 3678 if (cmd.comp_mask) 3679 return -EINVAL; 3680 3681 return uobj_perform_destroy(UVERBS_OBJECT_FLOW, cmd.flow_handle, file, 3682 0); 3683 } 3684 3685 static int __uverbs_create_xsrq(struct ib_uverbs_file *file, 3686 struct ib_uverbs_create_xsrq *cmd, 3687 struct ib_udata *udata) 3688 { 3689 struct ib_uverbs_create_srq_resp resp; 3690 struct ib_usrq_object *obj; 3691 struct ib_pd *pd; 3692 struct ib_srq *srq; 3693 struct ib_uobject *uninitialized_var(xrcd_uobj); 3694 struct ib_srq_init_attr attr; 3695 int ret; 3696 struct ib_device *ib_dev; 3697 3698 obj = (struct ib_usrq_object *)uobj_alloc(UVERBS_OBJECT_SRQ, file, 3699 &ib_dev); 3700 if (IS_ERR(obj)) 3701 return PTR_ERR(obj); 3702 3703 if (cmd->srq_type == IB_SRQT_TM) 3704 attr.ext.tag_matching.max_num_tags = cmd->max_num_tags; 3705 3706 if (cmd->srq_type == IB_SRQT_XRC) { 3707 xrcd_uobj = uobj_get_read(UVERBS_OBJECT_XRCD, cmd->xrcd_handle, 3708 file); 3709 if (IS_ERR(xrcd_uobj)) { 3710 ret = -EINVAL; 3711 goto err; 3712 } 3713 3714 attr.ext.xrc.xrcd = (struct ib_xrcd *)xrcd_uobj->object; 3715 if (!attr.ext.xrc.xrcd) { 3716 ret = -EINVAL; 3717 goto err_put_xrcd; 3718 } 3719 3720 obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object, uobject); 3721 atomic_inc(&obj->uxrcd->refcnt); 3722 } 3723 3724 if (ib_srq_has_cq(cmd->srq_type)) { 3725 attr.ext.cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, 3726 cmd->cq_handle, file); 3727 if (!attr.ext.cq) { 3728 ret = -EINVAL; 3729 goto err_put_xrcd; 3730 } 3731 } 3732 3733 pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd->pd_handle, file); 3734 if (!pd) { 3735 ret = -EINVAL; 3736 goto err_put_cq; 3737 } 3738 3739 attr.event_handler = ib_uverbs_srq_event_handler; 3740 attr.srq_context = file; 3741 attr.srq_type = cmd->srq_type; 3742 attr.attr.max_wr = cmd->max_wr; 3743 attr.attr.max_sge = cmd->max_sge; 3744 attr.attr.srq_limit = cmd->srq_limit; 3745 3746 obj->uevent.events_reported = 0; 3747 INIT_LIST_HEAD(&obj->uevent.event_list); 3748 3749 srq = pd->device->create_srq(pd, &attr, udata); 3750 if (IS_ERR(srq)) { 3751 ret = PTR_ERR(srq); 3752 goto err_put; 3753 } 3754 3755 srq->device = pd->device; 3756 srq->pd = pd; 3757 srq->srq_type = cmd->srq_type; 3758 srq->uobject = &obj->uevent.uobject; 3759 srq->event_handler = attr.event_handler; 3760 srq->srq_context = attr.srq_context; 3761 3762 if (ib_srq_has_cq(cmd->srq_type)) { 3763 srq->ext.cq = attr.ext.cq; 3764 atomic_inc(&attr.ext.cq->usecnt); 3765 } 3766 3767 if (cmd->srq_type == IB_SRQT_XRC) { 3768 srq->ext.xrc.xrcd = attr.ext.xrc.xrcd; 3769 atomic_inc(&attr.ext.xrc.xrcd->usecnt); 3770 } 3771 3772 atomic_inc(&pd->usecnt); 3773 atomic_set(&srq->usecnt, 0); 3774 3775 obj->uevent.uobject.object = srq; 3776 obj->uevent.uobject.user_handle = cmd->user_handle; 3777 3778 memset(&resp, 0, sizeof resp); 3779 resp.srq_handle = obj->uevent.uobject.id; 3780 resp.max_wr = attr.attr.max_wr; 3781 resp.max_sge = attr.attr.max_sge; 3782 if (cmd->srq_type == IB_SRQT_XRC) 3783 resp.srqn = srq->ext.xrc.srq_num; 3784 3785 if (copy_to_user(u64_to_user_ptr(cmd->response), 3786 &resp, sizeof resp)) { 3787 ret = -EFAULT; 3788 goto err_copy; 3789 } 3790 3791 if (cmd->srq_type == IB_SRQT_XRC) 3792 uobj_put_read(xrcd_uobj); 3793 3794 if (ib_srq_has_cq(cmd->srq_type)) 3795 uobj_put_obj_read(attr.ext.cq); 3796 3797 uobj_put_obj_read(pd); 3798 return uobj_alloc_commit(&obj->uevent.uobject, 0); 3799 3800 err_copy: 3801 ib_destroy_srq(srq); 3802 3803 err_put: 3804 uobj_put_obj_read(pd); 3805 3806 err_put_cq: 3807 if (ib_srq_has_cq(cmd->srq_type)) 3808 uobj_put_obj_read(attr.ext.cq); 3809 3810 err_put_xrcd: 3811 if (cmd->srq_type == IB_SRQT_XRC) { 3812 atomic_dec(&obj->uxrcd->refcnt); 3813 uobj_put_read(xrcd_uobj); 3814 } 3815 3816 err: 3817 uobj_alloc_abort(&obj->uevent.uobject); 3818 return ret; 3819 } 3820 3821 ssize_t ib_uverbs_create_srq(struct ib_uverbs_file *file, 3822 const char __user *buf, int in_len, 3823 int out_len) 3824 { 3825 struct ib_uverbs_create_srq cmd; 3826 struct ib_uverbs_create_xsrq xcmd; 3827 struct ib_uverbs_create_srq_resp resp; 3828 struct ib_udata udata; 3829 int ret; 3830 3831 if (out_len < sizeof resp) 3832 return -ENOSPC; 3833 3834 if (copy_from_user(&cmd, buf, sizeof cmd)) 3835 return -EFAULT; 3836 3837 memset(&xcmd, 0, sizeof(xcmd)); 3838 xcmd.response = cmd.response; 3839 xcmd.user_handle = cmd.user_handle; 3840 xcmd.srq_type = IB_SRQT_BASIC; 3841 xcmd.pd_handle = cmd.pd_handle; 3842 xcmd.max_wr = cmd.max_wr; 3843 xcmd.max_sge = cmd.max_sge; 3844 xcmd.srq_limit = cmd.srq_limit; 3845 3846 ib_uverbs_init_udata(&udata, buf + sizeof(cmd), 3847 u64_to_user_ptr(cmd.response) + sizeof(resp), 3848 in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr), 3849 out_len - sizeof(resp)); 3850 3851 ret = __uverbs_create_xsrq(file, &xcmd, &udata); 3852 if (ret) 3853 return ret; 3854 3855 return in_len; 3856 } 3857 3858 ssize_t ib_uverbs_create_xsrq(struct ib_uverbs_file *file, 3859 const char __user *buf, int in_len, int out_len) 3860 { 3861 struct ib_uverbs_create_xsrq cmd; 3862 struct ib_uverbs_create_srq_resp resp; 3863 struct ib_udata udata; 3864 int ret; 3865 3866 if (out_len < sizeof resp) 3867 return -ENOSPC; 3868 3869 if (copy_from_user(&cmd, buf, sizeof cmd)) 3870 return -EFAULT; 3871 3872 ib_uverbs_init_udata(&udata, buf + sizeof(cmd), 3873 u64_to_user_ptr(cmd.response) + sizeof(resp), 3874 in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr), 3875 out_len - sizeof(resp)); 3876 3877 ret = __uverbs_create_xsrq(file, &cmd, &udata); 3878 if (ret) 3879 return ret; 3880 3881 return in_len; 3882 } 3883 3884 ssize_t ib_uverbs_modify_srq(struct ib_uverbs_file *file, 3885 const char __user *buf, int in_len, 3886 int out_len) 3887 { 3888 struct ib_uverbs_modify_srq cmd; 3889 struct ib_udata udata; 3890 struct ib_srq *srq; 3891 struct ib_srq_attr attr; 3892 int ret; 3893 3894 if (copy_from_user(&cmd, buf, sizeof cmd)) 3895 return -EFAULT; 3896 3897 ib_uverbs_init_udata(&udata, buf + sizeof cmd, NULL, in_len - sizeof cmd, 3898 out_len); 3899 3900 srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ, cmd.srq_handle, file); 3901 if (!srq) 3902 return -EINVAL; 3903 3904 attr.max_wr = cmd.max_wr; 3905 attr.srq_limit = cmd.srq_limit; 3906 3907 ret = srq->device->modify_srq(srq, &attr, cmd.attr_mask, &udata); 3908 3909 uobj_put_obj_read(srq); 3910 3911 return ret ? ret : in_len; 3912 } 3913 3914 ssize_t ib_uverbs_query_srq(struct ib_uverbs_file *file, 3915 const char __user *buf, 3916 int in_len, int out_len) 3917 { 3918 struct ib_uverbs_query_srq cmd; 3919 struct ib_uverbs_query_srq_resp resp; 3920 struct ib_srq_attr attr; 3921 struct ib_srq *srq; 3922 int ret; 3923 3924 if (out_len < sizeof resp) 3925 return -ENOSPC; 3926 3927 if (copy_from_user(&cmd, buf, sizeof cmd)) 3928 return -EFAULT; 3929 3930 srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ, cmd.srq_handle, file); 3931 if (!srq) 3932 return -EINVAL; 3933 3934 ret = ib_query_srq(srq, &attr); 3935 3936 uobj_put_obj_read(srq); 3937 3938 if (ret) 3939 return ret; 3940 3941 memset(&resp, 0, sizeof resp); 3942 3943 resp.max_wr = attr.max_wr; 3944 resp.max_sge = attr.max_sge; 3945 resp.srq_limit = attr.srq_limit; 3946 3947 if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp)) 3948 return -EFAULT; 3949 3950 return in_len; 3951 } 3952 3953 ssize_t ib_uverbs_destroy_srq(struct ib_uverbs_file *file, 3954 const char __user *buf, int in_len, 3955 int out_len) 3956 { 3957 struct ib_uverbs_destroy_srq cmd; 3958 struct ib_uverbs_destroy_srq_resp resp; 3959 struct ib_uobject *uobj; 3960 struct ib_uevent_object *obj; 3961 3962 if (copy_from_user(&cmd, buf, sizeof cmd)) 3963 return -EFAULT; 3964 3965 uobj = uobj_get_destroy(UVERBS_OBJECT_SRQ, cmd.srq_handle, file); 3966 if (IS_ERR(uobj)) 3967 return PTR_ERR(uobj); 3968 3969 obj = container_of(uobj, struct ib_uevent_object, uobject); 3970 memset(&resp, 0, sizeof(resp)); 3971 resp.events_reported = obj->events_reported; 3972 3973 uobj_put_destroy(uobj); 3974 3975 if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof(resp))) 3976 return -EFAULT; 3977 3978 return in_len; 3979 } 3980 3981 int ib_uverbs_ex_query_device(struct ib_uverbs_file *file, 3982 struct ib_udata *ucore, 3983 struct ib_udata *uhw) 3984 { 3985 struct ib_uverbs_ex_query_device_resp resp = { {0} }; 3986 struct ib_uverbs_ex_query_device cmd; 3987 struct ib_device_attr attr = {0}; 3988 struct ib_ucontext *ucontext; 3989 struct ib_device *ib_dev; 3990 int err; 3991 3992 ucontext = ib_uverbs_get_ucontext(file); 3993 if (IS_ERR(ucontext)) 3994 return PTR_ERR(ucontext); 3995 ib_dev = ucontext->device; 3996 3997 if (!ib_dev->query_device) 3998 return -EOPNOTSUPP; 3999 4000 if (ucore->inlen < sizeof(cmd)) 4001 return -EINVAL; 4002 4003 err = ib_copy_from_udata(&cmd, ucore, sizeof(cmd)); 4004 if (err) 4005 return err; 4006 4007 if (cmd.comp_mask) 4008 return -EINVAL; 4009 4010 if (cmd.reserved) 4011 return -EINVAL; 4012 4013 resp.response_length = offsetof(typeof(resp), odp_caps); 4014 4015 if (ucore->outlen < resp.response_length) 4016 return -ENOSPC; 4017 4018 err = ib_dev->query_device(ib_dev, &attr, uhw); 4019 if (err) 4020 return err; 4021 4022 copy_query_dev_fields(ucontext, &resp.base, &attr); 4023 4024 if (ucore->outlen < resp.response_length + sizeof(resp.odp_caps)) 4025 goto end; 4026 4027 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING 4028 resp.odp_caps.general_caps = attr.odp_caps.general_caps; 4029 resp.odp_caps.per_transport_caps.rc_odp_caps = 4030 attr.odp_caps.per_transport_caps.rc_odp_caps; 4031 resp.odp_caps.per_transport_caps.uc_odp_caps = 4032 attr.odp_caps.per_transport_caps.uc_odp_caps; 4033 resp.odp_caps.per_transport_caps.ud_odp_caps = 4034 attr.odp_caps.per_transport_caps.ud_odp_caps; 4035 #endif 4036 resp.response_length += sizeof(resp.odp_caps); 4037 4038 if (ucore->outlen < resp.response_length + sizeof(resp.timestamp_mask)) 4039 goto end; 4040 4041 resp.timestamp_mask = attr.timestamp_mask; 4042 resp.response_length += sizeof(resp.timestamp_mask); 4043 4044 if (ucore->outlen < resp.response_length + sizeof(resp.hca_core_clock)) 4045 goto end; 4046 4047 resp.hca_core_clock = attr.hca_core_clock; 4048 resp.response_length += sizeof(resp.hca_core_clock); 4049 4050 if (ucore->outlen < resp.response_length + sizeof(resp.device_cap_flags_ex)) 4051 goto end; 4052 4053 resp.device_cap_flags_ex = attr.device_cap_flags; 4054 resp.response_length += sizeof(resp.device_cap_flags_ex); 4055 4056 if (ucore->outlen < resp.response_length + sizeof(resp.rss_caps)) 4057 goto end; 4058 4059 resp.rss_caps.supported_qpts = attr.rss_caps.supported_qpts; 4060 resp.rss_caps.max_rwq_indirection_tables = 4061 attr.rss_caps.max_rwq_indirection_tables; 4062 resp.rss_caps.max_rwq_indirection_table_size = 4063 attr.rss_caps.max_rwq_indirection_table_size; 4064 4065 resp.response_length += sizeof(resp.rss_caps); 4066 4067 if (ucore->outlen < resp.response_length + sizeof(resp.max_wq_type_rq)) 4068 goto end; 4069 4070 resp.max_wq_type_rq = attr.max_wq_type_rq; 4071 resp.response_length += sizeof(resp.max_wq_type_rq); 4072 4073 if (ucore->outlen < resp.response_length + sizeof(resp.raw_packet_caps)) 4074 goto end; 4075 4076 resp.raw_packet_caps = attr.raw_packet_caps; 4077 resp.response_length += sizeof(resp.raw_packet_caps); 4078 4079 if (ucore->outlen < resp.response_length + sizeof(resp.tm_caps)) 4080 goto end; 4081 4082 resp.tm_caps.max_rndv_hdr_size = attr.tm_caps.max_rndv_hdr_size; 4083 resp.tm_caps.max_num_tags = attr.tm_caps.max_num_tags; 4084 resp.tm_caps.max_ops = attr.tm_caps.max_ops; 4085 resp.tm_caps.max_sge = attr.tm_caps.max_sge; 4086 resp.tm_caps.flags = attr.tm_caps.flags; 4087 resp.response_length += sizeof(resp.tm_caps); 4088 4089 if (ucore->outlen < resp.response_length + sizeof(resp.cq_moderation_caps)) 4090 goto end; 4091 4092 resp.cq_moderation_caps.max_cq_moderation_count = 4093 attr.cq_caps.max_cq_moderation_count; 4094 resp.cq_moderation_caps.max_cq_moderation_period = 4095 attr.cq_caps.max_cq_moderation_period; 4096 resp.response_length += sizeof(resp.cq_moderation_caps); 4097 4098 if (ucore->outlen < resp.response_length + sizeof(resp.max_dm_size)) 4099 goto end; 4100 4101 resp.max_dm_size = attr.max_dm_size; 4102 resp.response_length += sizeof(resp.max_dm_size); 4103 end: 4104 err = ib_copy_to_udata(ucore, &resp, resp.response_length); 4105 return err; 4106 } 4107 4108 int ib_uverbs_ex_modify_cq(struct ib_uverbs_file *file, 4109 struct ib_udata *ucore, 4110 struct ib_udata *uhw) 4111 { 4112 struct ib_uverbs_ex_modify_cq cmd = {}; 4113 struct ib_cq *cq; 4114 size_t required_cmd_sz; 4115 int ret; 4116 4117 required_cmd_sz = offsetof(typeof(cmd), reserved) + 4118 sizeof(cmd.reserved); 4119 if (ucore->inlen < required_cmd_sz) 4120 return -EINVAL; 4121 4122 /* sanity checks */ 4123 if (ucore->inlen > sizeof(cmd) && 4124 !ib_is_udata_cleared(ucore, sizeof(cmd), 4125 ucore->inlen - sizeof(cmd))) 4126 return -EOPNOTSUPP; 4127 4128 ret = ib_copy_from_udata(&cmd, ucore, min(sizeof(cmd), ucore->inlen)); 4129 if (ret) 4130 return ret; 4131 4132 if (!cmd.attr_mask || cmd.reserved) 4133 return -EINVAL; 4134 4135 if (cmd.attr_mask > IB_CQ_MODERATE) 4136 return -EOPNOTSUPP; 4137 4138 cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, file); 4139 if (!cq) 4140 return -EINVAL; 4141 4142 ret = rdma_set_cq_moderation(cq, cmd.attr.cq_count, cmd.attr.cq_period); 4143 4144 uobj_put_obj_read(cq); 4145 4146 return ret; 4147 } 4148