1 /* 2 * Copyright (c) 2005 Topspin Communications. All rights reserved. 3 * Copyright (c) 2005, 2006, 2007 Cisco Systems. All rights reserved. 4 * Copyright (c) 2005 PathScale, Inc. All rights reserved. 5 * Copyright (c) 2006 Mellanox Technologies. All rights reserved. 6 * 7 * This software is available to you under a choice of one of two 8 * licenses. You may choose to be licensed under the terms of the GNU 9 * General Public License (GPL) Version 2, available from the file 10 * COPYING in the main directory of this source tree, or the 11 * OpenIB.org BSD license below: 12 * 13 * Redistribution and use in source and binary forms, with or 14 * without modification, are permitted provided that the following 15 * conditions are met: 16 * 17 * - Redistributions of source code must retain the above 18 * copyright notice, this list of conditions and the following 19 * disclaimer. 20 * 21 * - Redistributions in binary form must reproduce the above 22 * copyright notice, this list of conditions and the following 23 * disclaimer in the documentation and/or other materials 24 * provided with the distribution. 25 * 26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 27 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 28 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 29 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 30 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 31 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 32 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 33 * SOFTWARE. 34 */ 35 36 #include <linux/file.h> 37 #include <linux/fs.h> 38 #include <linux/slab.h> 39 #include <linux/sched.h> 40 41 #include <linux/uaccess.h> 42 43 #include <rdma/uverbs_types.h> 44 #include <rdma/uverbs_std_types.h> 45 #include "rdma_core.h" 46 47 #include "uverbs.h" 48 #include "core_priv.h" 49 50 static struct ib_uverbs_completion_event_file * 51 _ib_uverbs_lookup_comp_file(s32 fd, struct ib_uverbs_file *ufile) 52 { 53 struct ib_uobject *uobj = ufd_get_read(UVERBS_OBJECT_COMP_CHANNEL, 54 fd, ufile); 55 56 if (IS_ERR(uobj)) 57 return (void *)uobj; 58 59 uverbs_uobject_get(uobj); 60 uobj_put_read(uobj); 61 62 return container_of(uobj, struct ib_uverbs_completion_event_file, 63 uobj); 64 } 65 #define ib_uverbs_lookup_comp_file(_fd, _ufile) \ 66 _ib_uverbs_lookup_comp_file((_fd)*typecheck(s32, _fd), _ufile) 67 68 ssize_t ib_uverbs_get_context(struct ib_uverbs_file *file, 69 const char __user *buf, 70 int in_len, int out_len) 71 { 72 struct ib_uverbs_get_context cmd; 73 struct ib_uverbs_get_context_resp resp; 74 struct ib_udata udata; 75 struct ib_ucontext *ucontext; 76 struct file *filp; 77 struct ib_rdmacg_object cg_obj; 78 struct ib_device *ib_dev; 79 int ret; 80 81 if (out_len < sizeof resp) 82 return -ENOSPC; 83 84 if (copy_from_user(&cmd, buf, sizeof cmd)) 85 return -EFAULT; 86 87 mutex_lock(&file->ucontext_lock); 88 ib_dev = srcu_dereference(file->device->ib_dev, 89 &file->device->disassociate_srcu); 90 if (!ib_dev) { 91 ret = -EIO; 92 goto err; 93 } 94 95 if (file->ucontext) { 96 ret = -EINVAL; 97 goto err; 98 } 99 100 ib_uverbs_init_udata(&udata, buf + sizeof(cmd), 101 u64_to_user_ptr(cmd.response) + sizeof(resp), 102 in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr), 103 out_len - sizeof(resp)); 104 105 ret = ib_rdmacg_try_charge(&cg_obj, ib_dev, RDMACG_RESOURCE_HCA_HANDLE); 106 if (ret) 107 goto err; 108 109 ucontext = ib_dev->alloc_ucontext(ib_dev, &udata); 110 if (IS_ERR(ucontext)) { 111 ret = PTR_ERR(ucontext); 112 goto err_alloc; 113 } 114 115 ucontext->device = ib_dev; 116 ucontext->cg_obj = cg_obj; 117 /* ufile is required when some objects are released */ 118 ucontext->ufile = file; 119 120 rcu_read_lock(); 121 ucontext->tgid = get_task_pid(current->group_leader, PIDTYPE_PID); 122 rcu_read_unlock(); 123 ucontext->closing = 0; 124 ucontext->cleanup_retryable = false; 125 126 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING 127 ucontext->umem_tree = RB_ROOT_CACHED; 128 init_rwsem(&ucontext->umem_rwsem); 129 ucontext->odp_mrs_count = 0; 130 INIT_LIST_HEAD(&ucontext->no_private_counters); 131 132 if (!(ib_dev->attrs.device_cap_flags & IB_DEVICE_ON_DEMAND_PAGING)) 133 ucontext->invalidate_range = NULL; 134 135 #endif 136 137 resp.num_comp_vectors = file->device->num_comp_vectors; 138 139 ret = get_unused_fd_flags(O_CLOEXEC); 140 if (ret < 0) 141 goto err_free; 142 resp.async_fd = ret; 143 144 filp = ib_uverbs_alloc_async_event_file(file, ib_dev); 145 if (IS_ERR(filp)) { 146 ret = PTR_ERR(filp); 147 goto err_fd; 148 } 149 150 if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp)) { 151 ret = -EFAULT; 152 goto err_file; 153 } 154 155 fd_install(resp.async_fd, filp); 156 157 /* 158 * Make sure that ib_uverbs_get_ucontext() sees the pointer update 159 * only after all writes to setup the ucontext have completed 160 */ 161 smp_store_release(&file->ucontext, ucontext); 162 163 mutex_unlock(&file->ucontext_lock); 164 165 return in_len; 166 167 err_file: 168 ib_uverbs_free_async_event_file(file); 169 fput(filp); 170 171 err_fd: 172 put_unused_fd(resp.async_fd); 173 174 err_free: 175 put_pid(ucontext->tgid); 176 ib_dev->dealloc_ucontext(ucontext); 177 178 err_alloc: 179 ib_rdmacg_uncharge(&cg_obj, ib_dev, RDMACG_RESOURCE_HCA_HANDLE); 180 181 err: 182 mutex_unlock(&file->ucontext_lock); 183 return ret; 184 } 185 186 static void copy_query_dev_fields(struct ib_ucontext *ucontext, 187 struct ib_uverbs_query_device_resp *resp, 188 struct ib_device_attr *attr) 189 { 190 struct ib_device *ib_dev = ucontext->device; 191 192 resp->fw_ver = attr->fw_ver; 193 resp->node_guid = ib_dev->node_guid; 194 resp->sys_image_guid = attr->sys_image_guid; 195 resp->max_mr_size = attr->max_mr_size; 196 resp->page_size_cap = attr->page_size_cap; 197 resp->vendor_id = attr->vendor_id; 198 resp->vendor_part_id = attr->vendor_part_id; 199 resp->hw_ver = attr->hw_ver; 200 resp->max_qp = attr->max_qp; 201 resp->max_qp_wr = attr->max_qp_wr; 202 resp->device_cap_flags = lower_32_bits(attr->device_cap_flags); 203 resp->max_sge = min(attr->max_send_sge, attr->max_recv_sge); 204 resp->max_sge_rd = attr->max_sge_rd; 205 resp->max_cq = attr->max_cq; 206 resp->max_cqe = attr->max_cqe; 207 resp->max_mr = attr->max_mr; 208 resp->max_pd = attr->max_pd; 209 resp->max_qp_rd_atom = attr->max_qp_rd_atom; 210 resp->max_ee_rd_atom = attr->max_ee_rd_atom; 211 resp->max_res_rd_atom = attr->max_res_rd_atom; 212 resp->max_qp_init_rd_atom = attr->max_qp_init_rd_atom; 213 resp->max_ee_init_rd_atom = attr->max_ee_init_rd_atom; 214 resp->atomic_cap = attr->atomic_cap; 215 resp->max_ee = attr->max_ee; 216 resp->max_rdd = attr->max_rdd; 217 resp->max_mw = attr->max_mw; 218 resp->max_raw_ipv6_qp = attr->max_raw_ipv6_qp; 219 resp->max_raw_ethy_qp = attr->max_raw_ethy_qp; 220 resp->max_mcast_grp = attr->max_mcast_grp; 221 resp->max_mcast_qp_attach = attr->max_mcast_qp_attach; 222 resp->max_total_mcast_qp_attach = attr->max_total_mcast_qp_attach; 223 resp->max_ah = attr->max_ah; 224 resp->max_fmr = attr->max_fmr; 225 resp->max_map_per_fmr = attr->max_map_per_fmr; 226 resp->max_srq = attr->max_srq; 227 resp->max_srq_wr = attr->max_srq_wr; 228 resp->max_srq_sge = attr->max_srq_sge; 229 resp->max_pkeys = attr->max_pkeys; 230 resp->local_ca_ack_delay = attr->local_ca_ack_delay; 231 resp->phys_port_cnt = ib_dev->phys_port_cnt; 232 } 233 234 ssize_t ib_uverbs_query_device(struct ib_uverbs_file *file, 235 const char __user *buf, 236 int in_len, int out_len) 237 { 238 struct ib_uverbs_query_device cmd; 239 struct ib_uverbs_query_device_resp resp; 240 struct ib_ucontext *ucontext; 241 242 ucontext = ib_uverbs_get_ucontext(file); 243 if (IS_ERR(ucontext)) 244 return PTR_ERR(ucontext); 245 246 if (out_len < sizeof resp) 247 return -ENOSPC; 248 249 if (copy_from_user(&cmd, buf, sizeof cmd)) 250 return -EFAULT; 251 252 memset(&resp, 0, sizeof resp); 253 copy_query_dev_fields(ucontext, &resp, &ucontext->device->attrs); 254 255 if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp)) 256 return -EFAULT; 257 258 return in_len; 259 } 260 261 /* 262 * ib_uverbs_query_port_resp.port_cap_flags started out as just a copy of the 263 * PortInfo CapabilityMask, but was extended with unique bits. 264 */ 265 static u32 make_port_cap_flags(const struct ib_port_attr *attr) 266 { 267 u32 res; 268 269 /* All IBA CapabilityMask bits are passed through here, except bit 26, 270 * which is overridden with IP_BASED_GIDS. This is due to a historical 271 * mistake in the implementation of IP_BASED_GIDS. Otherwise all other 272 * bits match the IBA definition across all kernel versions. 273 */ 274 res = attr->port_cap_flags & ~(u32)IB_UVERBS_PCF_IP_BASED_GIDS; 275 276 if (attr->ip_gids) 277 res |= IB_UVERBS_PCF_IP_BASED_GIDS; 278 279 return res; 280 } 281 282 ssize_t ib_uverbs_query_port(struct ib_uverbs_file *file, 283 const char __user *buf, 284 int in_len, int out_len) 285 { 286 struct ib_uverbs_query_port cmd; 287 struct ib_uverbs_query_port_resp resp; 288 struct ib_port_attr attr; 289 int ret; 290 struct ib_ucontext *ucontext; 291 struct ib_device *ib_dev; 292 293 ucontext = ib_uverbs_get_ucontext(file); 294 if (IS_ERR(ucontext)) 295 return PTR_ERR(ucontext); 296 ib_dev = ucontext->device; 297 298 if (out_len < sizeof resp) 299 return -ENOSPC; 300 301 if (copy_from_user(&cmd, buf, sizeof cmd)) 302 return -EFAULT; 303 304 ret = ib_query_port(ib_dev, cmd.port_num, &attr); 305 if (ret) 306 return ret; 307 308 memset(&resp, 0, sizeof resp); 309 310 resp.state = attr.state; 311 resp.max_mtu = attr.max_mtu; 312 resp.active_mtu = attr.active_mtu; 313 resp.gid_tbl_len = attr.gid_tbl_len; 314 resp.port_cap_flags = make_port_cap_flags(&attr); 315 resp.max_msg_sz = attr.max_msg_sz; 316 resp.bad_pkey_cntr = attr.bad_pkey_cntr; 317 resp.qkey_viol_cntr = attr.qkey_viol_cntr; 318 resp.pkey_tbl_len = attr.pkey_tbl_len; 319 320 if (rdma_is_grh_required(ib_dev, cmd.port_num)) 321 resp.flags |= IB_UVERBS_QPF_GRH_REQUIRED; 322 323 if (rdma_cap_opa_ah(ib_dev, cmd.port_num)) { 324 resp.lid = OPA_TO_IB_UCAST_LID(attr.lid); 325 resp.sm_lid = OPA_TO_IB_UCAST_LID(attr.sm_lid); 326 } else { 327 resp.lid = ib_lid_cpu16(attr.lid); 328 resp.sm_lid = ib_lid_cpu16(attr.sm_lid); 329 } 330 resp.lmc = attr.lmc; 331 resp.max_vl_num = attr.max_vl_num; 332 resp.sm_sl = attr.sm_sl; 333 resp.subnet_timeout = attr.subnet_timeout; 334 resp.init_type_reply = attr.init_type_reply; 335 resp.active_width = attr.active_width; 336 resp.active_speed = attr.active_speed; 337 resp.phys_state = attr.phys_state; 338 resp.link_layer = rdma_port_get_link_layer(ib_dev, 339 cmd.port_num); 340 341 if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp)) 342 return -EFAULT; 343 344 return in_len; 345 } 346 347 ssize_t ib_uverbs_alloc_pd(struct ib_uverbs_file *file, 348 const char __user *buf, 349 int in_len, int out_len) 350 { 351 struct ib_uverbs_alloc_pd cmd; 352 struct ib_uverbs_alloc_pd_resp resp; 353 struct ib_udata udata; 354 struct ib_uobject *uobj; 355 struct ib_pd *pd; 356 int ret; 357 struct ib_device *ib_dev; 358 359 if (out_len < sizeof resp) 360 return -ENOSPC; 361 362 if (copy_from_user(&cmd, buf, sizeof cmd)) 363 return -EFAULT; 364 365 ib_uverbs_init_udata(&udata, buf + sizeof(cmd), 366 u64_to_user_ptr(cmd.response) + sizeof(resp), 367 in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr), 368 out_len - sizeof(resp)); 369 370 uobj = uobj_alloc(UVERBS_OBJECT_PD, file, &ib_dev); 371 if (IS_ERR(uobj)) 372 return PTR_ERR(uobj); 373 374 pd = ib_dev->alloc_pd(ib_dev, uobj->context, &udata); 375 if (IS_ERR(pd)) { 376 ret = PTR_ERR(pd); 377 goto err; 378 } 379 380 pd->device = ib_dev; 381 pd->uobject = uobj; 382 pd->__internal_mr = NULL; 383 atomic_set(&pd->usecnt, 0); 384 385 uobj->object = pd; 386 memset(&resp, 0, sizeof resp); 387 resp.pd_handle = uobj->id; 388 pd->res.type = RDMA_RESTRACK_PD; 389 rdma_restrack_add(&pd->res); 390 391 if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp)) { 392 ret = -EFAULT; 393 goto err_copy; 394 } 395 396 return uobj_alloc_commit(uobj, in_len); 397 398 err_copy: 399 ib_dealloc_pd(pd); 400 401 err: 402 uobj_alloc_abort(uobj); 403 return ret; 404 } 405 406 ssize_t ib_uverbs_dealloc_pd(struct ib_uverbs_file *file, 407 const char __user *buf, 408 int in_len, int out_len) 409 { 410 struct ib_uverbs_dealloc_pd cmd; 411 412 if (copy_from_user(&cmd, buf, sizeof cmd)) 413 return -EFAULT; 414 415 return uobj_perform_destroy(UVERBS_OBJECT_PD, cmd.pd_handle, file, 416 in_len); 417 } 418 419 struct xrcd_table_entry { 420 struct rb_node node; 421 struct ib_xrcd *xrcd; 422 struct inode *inode; 423 }; 424 425 static int xrcd_table_insert(struct ib_uverbs_device *dev, 426 struct inode *inode, 427 struct ib_xrcd *xrcd) 428 { 429 struct xrcd_table_entry *entry, *scan; 430 struct rb_node **p = &dev->xrcd_tree.rb_node; 431 struct rb_node *parent = NULL; 432 433 entry = kmalloc(sizeof *entry, GFP_KERNEL); 434 if (!entry) 435 return -ENOMEM; 436 437 entry->xrcd = xrcd; 438 entry->inode = inode; 439 440 while (*p) { 441 parent = *p; 442 scan = rb_entry(parent, struct xrcd_table_entry, node); 443 444 if (inode < scan->inode) { 445 p = &(*p)->rb_left; 446 } else if (inode > scan->inode) { 447 p = &(*p)->rb_right; 448 } else { 449 kfree(entry); 450 return -EEXIST; 451 } 452 } 453 454 rb_link_node(&entry->node, parent, p); 455 rb_insert_color(&entry->node, &dev->xrcd_tree); 456 igrab(inode); 457 return 0; 458 } 459 460 static struct xrcd_table_entry *xrcd_table_search(struct ib_uverbs_device *dev, 461 struct inode *inode) 462 { 463 struct xrcd_table_entry *entry; 464 struct rb_node *p = dev->xrcd_tree.rb_node; 465 466 while (p) { 467 entry = rb_entry(p, struct xrcd_table_entry, node); 468 469 if (inode < entry->inode) 470 p = p->rb_left; 471 else if (inode > entry->inode) 472 p = p->rb_right; 473 else 474 return entry; 475 } 476 477 return NULL; 478 } 479 480 static struct ib_xrcd *find_xrcd(struct ib_uverbs_device *dev, struct inode *inode) 481 { 482 struct xrcd_table_entry *entry; 483 484 entry = xrcd_table_search(dev, inode); 485 if (!entry) 486 return NULL; 487 488 return entry->xrcd; 489 } 490 491 static void xrcd_table_delete(struct ib_uverbs_device *dev, 492 struct inode *inode) 493 { 494 struct xrcd_table_entry *entry; 495 496 entry = xrcd_table_search(dev, inode); 497 if (entry) { 498 iput(inode); 499 rb_erase(&entry->node, &dev->xrcd_tree); 500 kfree(entry); 501 } 502 } 503 504 ssize_t ib_uverbs_open_xrcd(struct ib_uverbs_file *file, 505 const char __user *buf, int in_len, 506 int out_len) 507 { 508 struct ib_uverbs_open_xrcd cmd; 509 struct ib_uverbs_open_xrcd_resp resp; 510 struct ib_udata udata; 511 struct ib_uxrcd_object *obj; 512 struct ib_xrcd *xrcd = NULL; 513 struct fd f = {NULL, 0}; 514 struct inode *inode = NULL; 515 int ret = 0; 516 int new_xrcd = 0; 517 struct ib_device *ib_dev; 518 519 if (out_len < sizeof resp) 520 return -ENOSPC; 521 522 if (copy_from_user(&cmd, buf, sizeof cmd)) 523 return -EFAULT; 524 525 ib_uverbs_init_udata(&udata, buf + sizeof(cmd), 526 u64_to_user_ptr(cmd.response) + sizeof(resp), 527 in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr), 528 out_len - sizeof(resp)); 529 530 mutex_lock(&file->device->xrcd_tree_mutex); 531 532 if (cmd.fd != -1) { 533 /* search for file descriptor */ 534 f = fdget(cmd.fd); 535 if (!f.file) { 536 ret = -EBADF; 537 goto err_tree_mutex_unlock; 538 } 539 540 inode = file_inode(f.file); 541 xrcd = find_xrcd(file->device, inode); 542 if (!xrcd && !(cmd.oflags & O_CREAT)) { 543 /* no file descriptor. Need CREATE flag */ 544 ret = -EAGAIN; 545 goto err_tree_mutex_unlock; 546 } 547 548 if (xrcd && cmd.oflags & O_EXCL) { 549 ret = -EINVAL; 550 goto err_tree_mutex_unlock; 551 } 552 } 553 554 obj = (struct ib_uxrcd_object *)uobj_alloc(UVERBS_OBJECT_XRCD, file, 555 &ib_dev); 556 if (IS_ERR(obj)) { 557 ret = PTR_ERR(obj); 558 goto err_tree_mutex_unlock; 559 } 560 561 if (!xrcd) { 562 xrcd = ib_dev->alloc_xrcd(ib_dev, obj->uobject.context, &udata); 563 if (IS_ERR(xrcd)) { 564 ret = PTR_ERR(xrcd); 565 goto err; 566 } 567 568 xrcd->inode = inode; 569 xrcd->device = ib_dev; 570 atomic_set(&xrcd->usecnt, 0); 571 mutex_init(&xrcd->tgt_qp_mutex); 572 INIT_LIST_HEAD(&xrcd->tgt_qp_list); 573 new_xrcd = 1; 574 } 575 576 atomic_set(&obj->refcnt, 0); 577 obj->uobject.object = xrcd; 578 memset(&resp, 0, sizeof resp); 579 resp.xrcd_handle = obj->uobject.id; 580 581 if (inode) { 582 if (new_xrcd) { 583 /* create new inode/xrcd table entry */ 584 ret = xrcd_table_insert(file->device, inode, xrcd); 585 if (ret) 586 goto err_dealloc_xrcd; 587 } 588 atomic_inc(&xrcd->usecnt); 589 } 590 591 if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp)) { 592 ret = -EFAULT; 593 goto err_copy; 594 } 595 596 if (f.file) 597 fdput(f); 598 599 mutex_unlock(&file->device->xrcd_tree_mutex); 600 601 return uobj_alloc_commit(&obj->uobject, in_len); 602 603 err_copy: 604 if (inode) { 605 if (new_xrcd) 606 xrcd_table_delete(file->device, inode); 607 atomic_dec(&xrcd->usecnt); 608 } 609 610 err_dealloc_xrcd: 611 ib_dealloc_xrcd(xrcd); 612 613 err: 614 uobj_alloc_abort(&obj->uobject); 615 616 err_tree_mutex_unlock: 617 if (f.file) 618 fdput(f); 619 620 mutex_unlock(&file->device->xrcd_tree_mutex); 621 622 return ret; 623 } 624 625 ssize_t ib_uverbs_close_xrcd(struct ib_uverbs_file *file, 626 const char __user *buf, int in_len, 627 int out_len) 628 { 629 struct ib_uverbs_close_xrcd cmd; 630 631 if (copy_from_user(&cmd, buf, sizeof cmd)) 632 return -EFAULT; 633 634 return uobj_perform_destroy(UVERBS_OBJECT_XRCD, cmd.xrcd_handle, file, 635 in_len); 636 } 637 638 int ib_uverbs_dealloc_xrcd(struct ib_uobject *uobject, 639 struct ib_xrcd *xrcd, 640 enum rdma_remove_reason why) 641 { 642 struct inode *inode; 643 int ret; 644 struct ib_uverbs_device *dev = uobject->context->ufile->device; 645 646 inode = xrcd->inode; 647 if (inode && !atomic_dec_and_test(&xrcd->usecnt)) 648 return 0; 649 650 ret = ib_dealloc_xrcd(xrcd); 651 652 if (ib_is_destroy_retryable(ret, why, uobject)) { 653 atomic_inc(&xrcd->usecnt); 654 return ret; 655 } 656 657 if (inode) 658 xrcd_table_delete(dev, inode); 659 660 return ret; 661 } 662 663 ssize_t ib_uverbs_reg_mr(struct ib_uverbs_file *file, 664 const char __user *buf, int in_len, 665 int out_len) 666 { 667 struct ib_uverbs_reg_mr cmd; 668 struct ib_uverbs_reg_mr_resp resp; 669 struct ib_udata udata; 670 struct ib_uobject *uobj; 671 struct ib_pd *pd; 672 struct ib_mr *mr; 673 int ret; 674 struct ib_device *ib_dev; 675 676 if (out_len < sizeof resp) 677 return -ENOSPC; 678 679 if (copy_from_user(&cmd, buf, sizeof cmd)) 680 return -EFAULT; 681 682 ib_uverbs_init_udata(&udata, buf + sizeof(cmd), 683 u64_to_user_ptr(cmd.response) + sizeof(resp), 684 in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr), 685 out_len - sizeof(resp)); 686 687 if ((cmd.start & ~PAGE_MASK) != (cmd.hca_va & ~PAGE_MASK)) 688 return -EINVAL; 689 690 ret = ib_check_mr_access(cmd.access_flags); 691 if (ret) 692 return ret; 693 694 uobj = uobj_alloc(UVERBS_OBJECT_MR, file, &ib_dev); 695 if (IS_ERR(uobj)) 696 return PTR_ERR(uobj); 697 698 pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, file); 699 if (!pd) { 700 ret = -EINVAL; 701 goto err_free; 702 } 703 704 if (cmd.access_flags & IB_ACCESS_ON_DEMAND) { 705 if (!(pd->device->attrs.device_cap_flags & 706 IB_DEVICE_ON_DEMAND_PAGING)) { 707 pr_debug("ODP support not available\n"); 708 ret = -EINVAL; 709 goto err_put; 710 } 711 } 712 713 mr = pd->device->reg_user_mr(pd, cmd.start, cmd.length, cmd.hca_va, 714 cmd.access_flags, &udata); 715 if (IS_ERR(mr)) { 716 ret = PTR_ERR(mr); 717 goto err_put; 718 } 719 720 mr->device = pd->device; 721 mr->pd = pd; 722 mr->dm = NULL; 723 mr->uobject = uobj; 724 atomic_inc(&pd->usecnt); 725 mr->res.type = RDMA_RESTRACK_MR; 726 rdma_restrack_add(&mr->res); 727 728 uobj->object = mr; 729 730 memset(&resp, 0, sizeof resp); 731 resp.lkey = mr->lkey; 732 resp.rkey = mr->rkey; 733 resp.mr_handle = uobj->id; 734 735 if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp)) { 736 ret = -EFAULT; 737 goto err_copy; 738 } 739 740 uobj_put_obj_read(pd); 741 742 return uobj_alloc_commit(uobj, in_len); 743 744 err_copy: 745 ib_dereg_mr(mr); 746 747 err_put: 748 uobj_put_obj_read(pd); 749 750 err_free: 751 uobj_alloc_abort(uobj); 752 return ret; 753 } 754 755 ssize_t ib_uverbs_rereg_mr(struct ib_uverbs_file *file, 756 const char __user *buf, int in_len, 757 int out_len) 758 { 759 struct ib_uverbs_rereg_mr cmd; 760 struct ib_uverbs_rereg_mr_resp resp; 761 struct ib_udata udata; 762 struct ib_pd *pd = NULL; 763 struct ib_mr *mr; 764 struct ib_pd *old_pd; 765 int ret; 766 struct ib_uobject *uobj; 767 768 if (out_len < sizeof(resp)) 769 return -ENOSPC; 770 771 if (copy_from_user(&cmd, buf, sizeof(cmd))) 772 return -EFAULT; 773 774 ib_uverbs_init_udata(&udata, buf + sizeof(cmd), 775 u64_to_user_ptr(cmd.response) + sizeof(resp), 776 in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr), 777 out_len - sizeof(resp)); 778 779 if (cmd.flags & ~IB_MR_REREG_SUPPORTED || !cmd.flags) 780 return -EINVAL; 781 782 if ((cmd.flags & IB_MR_REREG_TRANS) && 783 (!cmd.start || !cmd.hca_va || 0 >= cmd.length || 784 (cmd.start & ~PAGE_MASK) != (cmd.hca_va & ~PAGE_MASK))) 785 return -EINVAL; 786 787 uobj = uobj_get_write(UVERBS_OBJECT_MR, cmd.mr_handle, file); 788 if (IS_ERR(uobj)) 789 return PTR_ERR(uobj); 790 791 mr = uobj->object; 792 793 if (mr->dm) { 794 ret = -EINVAL; 795 goto put_uobjs; 796 } 797 798 if (cmd.flags & IB_MR_REREG_ACCESS) { 799 ret = ib_check_mr_access(cmd.access_flags); 800 if (ret) 801 goto put_uobjs; 802 } 803 804 if (cmd.flags & IB_MR_REREG_PD) { 805 pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, 806 file); 807 if (!pd) { 808 ret = -EINVAL; 809 goto put_uobjs; 810 } 811 } 812 813 old_pd = mr->pd; 814 ret = mr->device->rereg_user_mr(mr, cmd.flags, cmd.start, 815 cmd.length, cmd.hca_va, 816 cmd.access_flags, pd, &udata); 817 if (!ret) { 818 if (cmd.flags & IB_MR_REREG_PD) { 819 atomic_inc(&pd->usecnt); 820 mr->pd = pd; 821 atomic_dec(&old_pd->usecnt); 822 } 823 } else { 824 goto put_uobj_pd; 825 } 826 827 memset(&resp, 0, sizeof(resp)); 828 resp.lkey = mr->lkey; 829 resp.rkey = mr->rkey; 830 831 if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof(resp))) 832 ret = -EFAULT; 833 else 834 ret = in_len; 835 836 put_uobj_pd: 837 if (cmd.flags & IB_MR_REREG_PD) 838 uobj_put_obj_read(pd); 839 840 put_uobjs: 841 uobj_put_write(uobj); 842 843 return ret; 844 } 845 846 ssize_t ib_uverbs_dereg_mr(struct ib_uverbs_file *file, 847 const char __user *buf, int in_len, 848 int out_len) 849 { 850 struct ib_uverbs_dereg_mr cmd; 851 852 if (copy_from_user(&cmd, buf, sizeof cmd)) 853 return -EFAULT; 854 855 return uobj_perform_destroy(UVERBS_OBJECT_MR, cmd.mr_handle, file, 856 in_len); 857 } 858 859 ssize_t ib_uverbs_alloc_mw(struct ib_uverbs_file *file, 860 const char __user *buf, int in_len, 861 int out_len) 862 { 863 struct ib_uverbs_alloc_mw cmd; 864 struct ib_uverbs_alloc_mw_resp resp; 865 struct ib_uobject *uobj; 866 struct ib_pd *pd; 867 struct ib_mw *mw; 868 struct ib_udata udata; 869 int ret; 870 struct ib_device *ib_dev; 871 872 if (out_len < sizeof(resp)) 873 return -ENOSPC; 874 875 if (copy_from_user(&cmd, buf, sizeof(cmd))) 876 return -EFAULT; 877 878 uobj = uobj_alloc(UVERBS_OBJECT_MW, file, &ib_dev); 879 if (IS_ERR(uobj)) 880 return PTR_ERR(uobj); 881 882 pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, file); 883 if (!pd) { 884 ret = -EINVAL; 885 goto err_free; 886 } 887 888 ib_uverbs_init_udata(&udata, buf + sizeof(cmd), 889 u64_to_user_ptr(cmd.response) + sizeof(resp), 890 in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr), 891 out_len - sizeof(resp)); 892 893 mw = pd->device->alloc_mw(pd, cmd.mw_type, &udata); 894 if (IS_ERR(mw)) { 895 ret = PTR_ERR(mw); 896 goto err_put; 897 } 898 899 mw->device = pd->device; 900 mw->pd = pd; 901 mw->uobject = uobj; 902 atomic_inc(&pd->usecnt); 903 904 uobj->object = mw; 905 906 memset(&resp, 0, sizeof(resp)); 907 resp.rkey = mw->rkey; 908 resp.mw_handle = uobj->id; 909 910 if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof(resp))) { 911 ret = -EFAULT; 912 goto err_copy; 913 } 914 915 uobj_put_obj_read(pd); 916 return uobj_alloc_commit(uobj, in_len); 917 918 err_copy: 919 uverbs_dealloc_mw(mw); 920 err_put: 921 uobj_put_obj_read(pd); 922 err_free: 923 uobj_alloc_abort(uobj); 924 return ret; 925 } 926 927 ssize_t ib_uverbs_dealloc_mw(struct ib_uverbs_file *file, 928 const char __user *buf, int in_len, 929 int out_len) 930 { 931 struct ib_uverbs_dealloc_mw cmd; 932 933 if (copy_from_user(&cmd, buf, sizeof(cmd))) 934 return -EFAULT; 935 936 return uobj_perform_destroy(UVERBS_OBJECT_MW, cmd.mw_handle, file, 937 in_len); 938 } 939 940 ssize_t ib_uverbs_create_comp_channel(struct ib_uverbs_file *file, 941 const char __user *buf, int in_len, 942 int out_len) 943 { 944 struct ib_uverbs_create_comp_channel cmd; 945 struct ib_uverbs_create_comp_channel_resp resp; 946 struct ib_uobject *uobj; 947 struct ib_uverbs_completion_event_file *ev_file; 948 struct ib_device *ib_dev; 949 950 if (out_len < sizeof resp) 951 return -ENOSPC; 952 953 if (copy_from_user(&cmd, buf, sizeof cmd)) 954 return -EFAULT; 955 956 uobj = uobj_alloc(UVERBS_OBJECT_COMP_CHANNEL, file, &ib_dev); 957 if (IS_ERR(uobj)) 958 return PTR_ERR(uobj); 959 960 resp.fd = uobj->id; 961 962 ev_file = container_of(uobj, struct ib_uverbs_completion_event_file, 963 uobj); 964 ib_uverbs_init_event_queue(&ev_file->ev_queue); 965 966 if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp)) { 967 uobj_alloc_abort(uobj); 968 return -EFAULT; 969 } 970 971 return uobj_alloc_commit(uobj, in_len); 972 } 973 974 static struct ib_ucq_object *create_cq(struct ib_uverbs_file *file, 975 struct ib_udata *ucore, 976 struct ib_udata *uhw, 977 struct ib_uverbs_ex_create_cq *cmd, 978 size_t cmd_sz, 979 int (*cb)(struct ib_uverbs_file *file, 980 struct ib_ucq_object *obj, 981 struct ib_uverbs_ex_create_cq_resp *resp, 982 struct ib_udata *udata, 983 void *context), 984 void *context) 985 { 986 struct ib_ucq_object *obj; 987 struct ib_uverbs_completion_event_file *ev_file = NULL; 988 struct ib_cq *cq; 989 int ret; 990 struct ib_uverbs_ex_create_cq_resp resp; 991 struct ib_cq_init_attr attr = {}; 992 struct ib_device *ib_dev; 993 994 if (cmd->comp_vector >= file->device->num_comp_vectors) 995 return ERR_PTR(-EINVAL); 996 997 obj = (struct ib_ucq_object *)uobj_alloc(UVERBS_OBJECT_CQ, file, 998 &ib_dev); 999 if (IS_ERR(obj)) 1000 return obj; 1001 1002 if (!ib_dev->create_cq) { 1003 ret = -EOPNOTSUPP; 1004 goto err; 1005 } 1006 1007 if (cmd->comp_channel >= 0) { 1008 ev_file = ib_uverbs_lookup_comp_file(cmd->comp_channel, file); 1009 if (IS_ERR(ev_file)) { 1010 ret = PTR_ERR(ev_file); 1011 goto err; 1012 } 1013 } 1014 1015 obj->uobject.user_handle = cmd->user_handle; 1016 obj->comp_events_reported = 0; 1017 obj->async_events_reported = 0; 1018 INIT_LIST_HEAD(&obj->comp_list); 1019 INIT_LIST_HEAD(&obj->async_list); 1020 1021 attr.cqe = cmd->cqe; 1022 attr.comp_vector = cmd->comp_vector; 1023 1024 if (cmd_sz > offsetof(typeof(*cmd), flags) + sizeof(cmd->flags)) 1025 attr.flags = cmd->flags; 1026 1027 cq = ib_dev->create_cq(ib_dev, &attr, obj->uobject.context, uhw); 1028 if (IS_ERR(cq)) { 1029 ret = PTR_ERR(cq); 1030 goto err_file; 1031 } 1032 1033 cq->device = ib_dev; 1034 cq->uobject = &obj->uobject; 1035 cq->comp_handler = ib_uverbs_comp_handler; 1036 cq->event_handler = ib_uverbs_cq_event_handler; 1037 cq->cq_context = ev_file ? &ev_file->ev_queue : NULL; 1038 atomic_set(&cq->usecnt, 0); 1039 1040 obj->uobject.object = cq; 1041 memset(&resp, 0, sizeof resp); 1042 resp.base.cq_handle = obj->uobject.id; 1043 resp.base.cqe = cq->cqe; 1044 1045 resp.response_length = offsetof(typeof(resp), response_length) + 1046 sizeof(resp.response_length); 1047 1048 cq->res.type = RDMA_RESTRACK_CQ; 1049 rdma_restrack_add(&cq->res); 1050 1051 ret = cb(file, obj, &resp, ucore, context); 1052 if (ret) 1053 goto err_cb; 1054 1055 ret = uobj_alloc_commit(&obj->uobject, 0); 1056 if (ret) 1057 return ERR_PTR(ret); 1058 return obj; 1059 1060 err_cb: 1061 ib_destroy_cq(cq); 1062 1063 err_file: 1064 if (ev_file) 1065 ib_uverbs_release_ucq(file, ev_file, obj); 1066 1067 err: 1068 uobj_alloc_abort(&obj->uobject); 1069 1070 return ERR_PTR(ret); 1071 } 1072 1073 static int ib_uverbs_create_cq_cb(struct ib_uverbs_file *file, 1074 struct ib_ucq_object *obj, 1075 struct ib_uverbs_ex_create_cq_resp *resp, 1076 struct ib_udata *ucore, void *context) 1077 { 1078 if (ib_copy_to_udata(ucore, &resp->base, sizeof(resp->base))) 1079 return -EFAULT; 1080 1081 return 0; 1082 } 1083 1084 ssize_t ib_uverbs_create_cq(struct ib_uverbs_file *file, 1085 const char __user *buf, int in_len, 1086 int out_len) 1087 { 1088 struct ib_uverbs_create_cq cmd; 1089 struct ib_uverbs_ex_create_cq cmd_ex; 1090 struct ib_uverbs_create_cq_resp resp; 1091 struct ib_udata ucore; 1092 struct ib_udata uhw; 1093 struct ib_ucq_object *obj; 1094 1095 if (out_len < sizeof(resp)) 1096 return -ENOSPC; 1097 1098 if (copy_from_user(&cmd, buf, sizeof(cmd))) 1099 return -EFAULT; 1100 1101 ib_uverbs_init_udata(&ucore, buf, u64_to_user_ptr(cmd.response), 1102 sizeof(cmd), sizeof(resp)); 1103 1104 ib_uverbs_init_udata(&uhw, buf + sizeof(cmd), 1105 u64_to_user_ptr(cmd.response) + sizeof(resp), 1106 in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr), 1107 out_len - sizeof(resp)); 1108 1109 memset(&cmd_ex, 0, sizeof(cmd_ex)); 1110 cmd_ex.user_handle = cmd.user_handle; 1111 cmd_ex.cqe = cmd.cqe; 1112 cmd_ex.comp_vector = cmd.comp_vector; 1113 cmd_ex.comp_channel = cmd.comp_channel; 1114 1115 obj = create_cq(file, &ucore, &uhw, &cmd_ex, 1116 offsetof(typeof(cmd_ex), comp_channel) + 1117 sizeof(cmd.comp_channel), ib_uverbs_create_cq_cb, 1118 NULL); 1119 1120 if (IS_ERR(obj)) 1121 return PTR_ERR(obj); 1122 1123 return in_len; 1124 } 1125 1126 static int ib_uverbs_ex_create_cq_cb(struct ib_uverbs_file *file, 1127 struct ib_ucq_object *obj, 1128 struct ib_uverbs_ex_create_cq_resp *resp, 1129 struct ib_udata *ucore, void *context) 1130 { 1131 if (ib_copy_to_udata(ucore, resp, resp->response_length)) 1132 return -EFAULT; 1133 1134 return 0; 1135 } 1136 1137 int ib_uverbs_ex_create_cq(struct ib_uverbs_file *file, 1138 struct ib_udata *ucore, 1139 struct ib_udata *uhw) 1140 { 1141 struct ib_uverbs_ex_create_cq_resp resp; 1142 struct ib_uverbs_ex_create_cq cmd; 1143 struct ib_ucq_object *obj; 1144 int err; 1145 1146 if (ucore->inlen < sizeof(cmd)) 1147 return -EINVAL; 1148 1149 err = ib_copy_from_udata(&cmd, ucore, sizeof(cmd)); 1150 if (err) 1151 return err; 1152 1153 if (cmd.comp_mask) 1154 return -EINVAL; 1155 1156 if (cmd.reserved) 1157 return -EINVAL; 1158 1159 if (ucore->outlen < (offsetof(typeof(resp), response_length) + 1160 sizeof(resp.response_length))) 1161 return -ENOSPC; 1162 1163 obj = create_cq(file, ucore, uhw, &cmd, 1164 min(ucore->inlen, sizeof(cmd)), 1165 ib_uverbs_ex_create_cq_cb, NULL); 1166 1167 return PTR_ERR_OR_ZERO(obj); 1168 } 1169 1170 ssize_t ib_uverbs_resize_cq(struct ib_uverbs_file *file, 1171 const char __user *buf, int in_len, 1172 int out_len) 1173 { 1174 struct ib_uverbs_resize_cq cmd; 1175 struct ib_uverbs_resize_cq_resp resp = {}; 1176 struct ib_udata udata; 1177 struct ib_cq *cq; 1178 int ret = -EINVAL; 1179 1180 if (copy_from_user(&cmd, buf, sizeof cmd)) 1181 return -EFAULT; 1182 1183 ib_uverbs_init_udata(&udata, buf + sizeof(cmd), 1184 u64_to_user_ptr(cmd.response) + sizeof(resp), 1185 in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr), 1186 out_len - sizeof(resp)); 1187 1188 cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, file); 1189 if (!cq) 1190 return -EINVAL; 1191 1192 ret = cq->device->resize_cq(cq, cmd.cqe, &udata); 1193 if (ret) 1194 goto out; 1195 1196 resp.cqe = cq->cqe; 1197 1198 if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp.cqe)) 1199 ret = -EFAULT; 1200 1201 out: 1202 uobj_put_obj_read(cq); 1203 1204 return ret ? ret : in_len; 1205 } 1206 1207 static int copy_wc_to_user(struct ib_device *ib_dev, void __user *dest, 1208 struct ib_wc *wc) 1209 { 1210 struct ib_uverbs_wc tmp; 1211 1212 tmp.wr_id = wc->wr_id; 1213 tmp.status = wc->status; 1214 tmp.opcode = wc->opcode; 1215 tmp.vendor_err = wc->vendor_err; 1216 tmp.byte_len = wc->byte_len; 1217 tmp.ex.imm_data = wc->ex.imm_data; 1218 tmp.qp_num = wc->qp->qp_num; 1219 tmp.src_qp = wc->src_qp; 1220 tmp.wc_flags = wc->wc_flags; 1221 tmp.pkey_index = wc->pkey_index; 1222 if (rdma_cap_opa_ah(ib_dev, wc->port_num)) 1223 tmp.slid = OPA_TO_IB_UCAST_LID(wc->slid); 1224 else 1225 tmp.slid = ib_lid_cpu16(wc->slid); 1226 tmp.sl = wc->sl; 1227 tmp.dlid_path_bits = wc->dlid_path_bits; 1228 tmp.port_num = wc->port_num; 1229 tmp.reserved = 0; 1230 1231 if (copy_to_user(dest, &tmp, sizeof tmp)) 1232 return -EFAULT; 1233 1234 return 0; 1235 } 1236 1237 ssize_t ib_uverbs_poll_cq(struct ib_uverbs_file *file, 1238 const char __user *buf, int in_len, 1239 int out_len) 1240 { 1241 struct ib_uverbs_poll_cq cmd; 1242 struct ib_uverbs_poll_cq_resp resp; 1243 u8 __user *header_ptr; 1244 u8 __user *data_ptr; 1245 struct ib_cq *cq; 1246 struct ib_wc wc; 1247 int ret; 1248 1249 if (copy_from_user(&cmd, buf, sizeof cmd)) 1250 return -EFAULT; 1251 1252 cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, file); 1253 if (!cq) 1254 return -EINVAL; 1255 1256 /* we copy a struct ib_uverbs_poll_cq_resp to user space */ 1257 header_ptr = u64_to_user_ptr(cmd.response); 1258 data_ptr = header_ptr + sizeof resp; 1259 1260 memset(&resp, 0, sizeof resp); 1261 while (resp.count < cmd.ne) { 1262 ret = ib_poll_cq(cq, 1, &wc); 1263 if (ret < 0) 1264 goto out_put; 1265 if (!ret) 1266 break; 1267 1268 ret = copy_wc_to_user(cq->device, data_ptr, &wc); 1269 if (ret) 1270 goto out_put; 1271 1272 data_ptr += sizeof(struct ib_uverbs_wc); 1273 ++resp.count; 1274 } 1275 1276 if (copy_to_user(header_ptr, &resp, sizeof resp)) { 1277 ret = -EFAULT; 1278 goto out_put; 1279 } 1280 1281 ret = in_len; 1282 1283 out_put: 1284 uobj_put_obj_read(cq); 1285 return ret; 1286 } 1287 1288 ssize_t ib_uverbs_req_notify_cq(struct ib_uverbs_file *file, 1289 const char __user *buf, int in_len, 1290 int out_len) 1291 { 1292 struct ib_uverbs_req_notify_cq cmd; 1293 struct ib_cq *cq; 1294 1295 if (copy_from_user(&cmd, buf, sizeof cmd)) 1296 return -EFAULT; 1297 1298 cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, file); 1299 if (!cq) 1300 return -EINVAL; 1301 1302 ib_req_notify_cq(cq, cmd.solicited_only ? 1303 IB_CQ_SOLICITED : IB_CQ_NEXT_COMP); 1304 1305 uobj_put_obj_read(cq); 1306 1307 return in_len; 1308 } 1309 1310 ssize_t ib_uverbs_destroy_cq(struct ib_uverbs_file *file, 1311 const char __user *buf, int in_len, 1312 int out_len) 1313 { 1314 struct ib_uverbs_destroy_cq cmd; 1315 struct ib_uverbs_destroy_cq_resp resp; 1316 struct ib_uobject *uobj; 1317 struct ib_ucq_object *obj; 1318 1319 if (copy_from_user(&cmd, buf, sizeof cmd)) 1320 return -EFAULT; 1321 1322 uobj = uobj_get_destroy(UVERBS_OBJECT_CQ, cmd.cq_handle, file); 1323 if (IS_ERR(uobj)) 1324 return PTR_ERR(uobj); 1325 1326 obj = container_of(uobj, struct ib_ucq_object, uobject); 1327 memset(&resp, 0, sizeof(resp)); 1328 resp.comp_events_reported = obj->comp_events_reported; 1329 resp.async_events_reported = obj->async_events_reported; 1330 1331 uobj_put_destroy(uobj); 1332 1333 if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp)) 1334 return -EFAULT; 1335 1336 return in_len; 1337 } 1338 1339 static int create_qp(struct ib_uverbs_file *file, 1340 struct ib_udata *ucore, 1341 struct ib_udata *uhw, 1342 struct ib_uverbs_ex_create_qp *cmd, 1343 size_t cmd_sz, 1344 int (*cb)(struct ib_uverbs_file *file, 1345 struct ib_uverbs_ex_create_qp_resp *resp, 1346 struct ib_udata *udata), 1347 void *context) 1348 { 1349 struct ib_uqp_object *obj; 1350 struct ib_device *device; 1351 struct ib_pd *pd = NULL; 1352 struct ib_xrcd *xrcd = NULL; 1353 struct ib_uobject *xrcd_uobj = ERR_PTR(-ENOENT); 1354 struct ib_cq *scq = NULL, *rcq = NULL; 1355 struct ib_srq *srq = NULL; 1356 struct ib_qp *qp; 1357 char *buf; 1358 struct ib_qp_init_attr attr = {}; 1359 struct ib_uverbs_ex_create_qp_resp resp; 1360 int ret; 1361 struct ib_rwq_ind_table *ind_tbl = NULL; 1362 bool has_sq = true; 1363 struct ib_device *ib_dev; 1364 1365 if (cmd->qp_type == IB_QPT_RAW_PACKET && !capable(CAP_NET_RAW)) 1366 return -EPERM; 1367 1368 obj = (struct ib_uqp_object *)uobj_alloc(UVERBS_OBJECT_QP, file, 1369 &ib_dev); 1370 if (IS_ERR(obj)) 1371 return PTR_ERR(obj); 1372 obj->uxrcd = NULL; 1373 obj->uevent.uobject.user_handle = cmd->user_handle; 1374 mutex_init(&obj->mcast_lock); 1375 1376 if (cmd_sz >= offsetof(typeof(*cmd), rwq_ind_tbl_handle) + 1377 sizeof(cmd->rwq_ind_tbl_handle) && 1378 (cmd->comp_mask & IB_UVERBS_CREATE_QP_MASK_IND_TABLE)) { 1379 ind_tbl = uobj_get_obj_read(rwq_ind_table, 1380 UVERBS_OBJECT_RWQ_IND_TBL, 1381 cmd->rwq_ind_tbl_handle, file); 1382 if (!ind_tbl) { 1383 ret = -EINVAL; 1384 goto err_put; 1385 } 1386 1387 attr.rwq_ind_tbl = ind_tbl; 1388 } 1389 1390 if (cmd_sz > sizeof(*cmd) && 1391 !ib_is_udata_cleared(ucore, sizeof(*cmd), 1392 cmd_sz - sizeof(*cmd))) { 1393 ret = -EOPNOTSUPP; 1394 goto err_put; 1395 } 1396 1397 if (ind_tbl && (cmd->max_recv_wr || cmd->max_recv_sge || cmd->is_srq)) { 1398 ret = -EINVAL; 1399 goto err_put; 1400 } 1401 1402 if (ind_tbl && !cmd->max_send_wr) 1403 has_sq = false; 1404 1405 if (cmd->qp_type == IB_QPT_XRC_TGT) { 1406 xrcd_uobj = uobj_get_read(UVERBS_OBJECT_XRCD, cmd->pd_handle, 1407 file); 1408 1409 if (IS_ERR(xrcd_uobj)) { 1410 ret = -EINVAL; 1411 goto err_put; 1412 } 1413 1414 xrcd = (struct ib_xrcd *)xrcd_uobj->object; 1415 if (!xrcd) { 1416 ret = -EINVAL; 1417 goto err_put; 1418 } 1419 device = xrcd->device; 1420 } else { 1421 if (cmd->qp_type == IB_QPT_XRC_INI) { 1422 cmd->max_recv_wr = 0; 1423 cmd->max_recv_sge = 0; 1424 } else { 1425 if (cmd->is_srq) { 1426 srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ, 1427 cmd->srq_handle, file); 1428 if (!srq || srq->srq_type == IB_SRQT_XRC) { 1429 ret = -EINVAL; 1430 goto err_put; 1431 } 1432 } 1433 1434 if (!ind_tbl) { 1435 if (cmd->recv_cq_handle != cmd->send_cq_handle) { 1436 rcq = uobj_get_obj_read( 1437 cq, UVERBS_OBJECT_CQ, 1438 cmd->recv_cq_handle, file); 1439 if (!rcq) { 1440 ret = -EINVAL; 1441 goto err_put; 1442 } 1443 } 1444 } 1445 } 1446 1447 if (has_sq) 1448 scq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, 1449 cmd->send_cq_handle, file); 1450 if (!ind_tbl) 1451 rcq = rcq ?: scq; 1452 pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd->pd_handle, 1453 file); 1454 if (!pd || (!scq && has_sq)) { 1455 ret = -EINVAL; 1456 goto err_put; 1457 } 1458 1459 device = pd->device; 1460 } 1461 1462 attr.event_handler = ib_uverbs_qp_event_handler; 1463 attr.qp_context = file; 1464 attr.send_cq = scq; 1465 attr.recv_cq = rcq; 1466 attr.srq = srq; 1467 attr.xrcd = xrcd; 1468 attr.sq_sig_type = cmd->sq_sig_all ? IB_SIGNAL_ALL_WR : 1469 IB_SIGNAL_REQ_WR; 1470 attr.qp_type = cmd->qp_type; 1471 attr.create_flags = 0; 1472 1473 attr.cap.max_send_wr = cmd->max_send_wr; 1474 attr.cap.max_recv_wr = cmd->max_recv_wr; 1475 attr.cap.max_send_sge = cmd->max_send_sge; 1476 attr.cap.max_recv_sge = cmd->max_recv_sge; 1477 attr.cap.max_inline_data = cmd->max_inline_data; 1478 1479 obj->uevent.events_reported = 0; 1480 INIT_LIST_HEAD(&obj->uevent.event_list); 1481 INIT_LIST_HEAD(&obj->mcast_list); 1482 1483 if (cmd_sz >= offsetof(typeof(*cmd), create_flags) + 1484 sizeof(cmd->create_flags)) 1485 attr.create_flags = cmd->create_flags; 1486 1487 if (attr.create_flags & ~(IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK | 1488 IB_QP_CREATE_CROSS_CHANNEL | 1489 IB_QP_CREATE_MANAGED_SEND | 1490 IB_QP_CREATE_MANAGED_RECV | 1491 IB_QP_CREATE_SCATTER_FCS | 1492 IB_QP_CREATE_CVLAN_STRIPPING | 1493 IB_QP_CREATE_SOURCE_QPN | 1494 IB_QP_CREATE_PCI_WRITE_END_PADDING)) { 1495 ret = -EINVAL; 1496 goto err_put; 1497 } 1498 1499 if (attr.create_flags & IB_QP_CREATE_SOURCE_QPN) { 1500 if (!capable(CAP_NET_RAW)) { 1501 ret = -EPERM; 1502 goto err_put; 1503 } 1504 1505 attr.source_qpn = cmd->source_qpn; 1506 } 1507 1508 buf = (void *)cmd + sizeof(*cmd); 1509 if (cmd_sz > sizeof(*cmd)) 1510 if (!(buf[0] == 0 && !memcmp(buf, buf + 1, 1511 cmd_sz - sizeof(*cmd) - 1))) { 1512 ret = -EINVAL; 1513 goto err_put; 1514 } 1515 1516 if (cmd->qp_type == IB_QPT_XRC_TGT) 1517 qp = ib_create_qp(pd, &attr); 1518 else 1519 qp = _ib_create_qp(device, pd, &attr, uhw, 1520 &obj->uevent.uobject); 1521 1522 if (IS_ERR(qp)) { 1523 ret = PTR_ERR(qp); 1524 goto err_put; 1525 } 1526 1527 if (cmd->qp_type != IB_QPT_XRC_TGT) { 1528 ret = ib_create_qp_security(qp, device); 1529 if (ret) 1530 goto err_cb; 1531 1532 qp->real_qp = qp; 1533 qp->pd = pd; 1534 qp->send_cq = attr.send_cq; 1535 qp->recv_cq = attr.recv_cq; 1536 qp->srq = attr.srq; 1537 qp->rwq_ind_tbl = ind_tbl; 1538 qp->event_handler = attr.event_handler; 1539 qp->qp_context = attr.qp_context; 1540 qp->qp_type = attr.qp_type; 1541 atomic_set(&qp->usecnt, 0); 1542 atomic_inc(&pd->usecnt); 1543 qp->port = 0; 1544 if (attr.send_cq) 1545 atomic_inc(&attr.send_cq->usecnt); 1546 if (attr.recv_cq) 1547 atomic_inc(&attr.recv_cq->usecnt); 1548 if (attr.srq) 1549 atomic_inc(&attr.srq->usecnt); 1550 if (ind_tbl) 1551 atomic_inc(&ind_tbl->usecnt); 1552 } else { 1553 /* It is done in _ib_create_qp for other QP types */ 1554 qp->uobject = &obj->uevent.uobject; 1555 } 1556 1557 obj->uevent.uobject.object = qp; 1558 1559 memset(&resp, 0, sizeof resp); 1560 resp.base.qpn = qp->qp_num; 1561 resp.base.qp_handle = obj->uevent.uobject.id; 1562 resp.base.max_recv_sge = attr.cap.max_recv_sge; 1563 resp.base.max_send_sge = attr.cap.max_send_sge; 1564 resp.base.max_recv_wr = attr.cap.max_recv_wr; 1565 resp.base.max_send_wr = attr.cap.max_send_wr; 1566 resp.base.max_inline_data = attr.cap.max_inline_data; 1567 1568 resp.response_length = offsetof(typeof(resp), response_length) + 1569 sizeof(resp.response_length); 1570 1571 ret = cb(file, &resp, ucore); 1572 if (ret) 1573 goto err_cb; 1574 1575 if (xrcd) { 1576 obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object, 1577 uobject); 1578 atomic_inc(&obj->uxrcd->refcnt); 1579 uobj_put_read(xrcd_uobj); 1580 } 1581 1582 if (pd) 1583 uobj_put_obj_read(pd); 1584 if (scq) 1585 uobj_put_obj_read(scq); 1586 if (rcq && rcq != scq) 1587 uobj_put_obj_read(rcq); 1588 if (srq) 1589 uobj_put_obj_read(srq); 1590 if (ind_tbl) 1591 uobj_put_obj_read(ind_tbl); 1592 1593 return uobj_alloc_commit(&obj->uevent.uobject, 0); 1594 err_cb: 1595 ib_destroy_qp(qp); 1596 1597 err_put: 1598 if (!IS_ERR(xrcd_uobj)) 1599 uobj_put_read(xrcd_uobj); 1600 if (pd) 1601 uobj_put_obj_read(pd); 1602 if (scq) 1603 uobj_put_obj_read(scq); 1604 if (rcq && rcq != scq) 1605 uobj_put_obj_read(rcq); 1606 if (srq) 1607 uobj_put_obj_read(srq); 1608 if (ind_tbl) 1609 uobj_put_obj_read(ind_tbl); 1610 1611 uobj_alloc_abort(&obj->uevent.uobject); 1612 return ret; 1613 } 1614 1615 static int ib_uverbs_create_qp_cb(struct ib_uverbs_file *file, 1616 struct ib_uverbs_ex_create_qp_resp *resp, 1617 struct ib_udata *ucore) 1618 { 1619 if (ib_copy_to_udata(ucore, &resp->base, sizeof(resp->base))) 1620 return -EFAULT; 1621 1622 return 0; 1623 } 1624 1625 ssize_t ib_uverbs_create_qp(struct ib_uverbs_file *file, 1626 const char __user *buf, int in_len, 1627 int out_len) 1628 { 1629 struct ib_uverbs_create_qp cmd; 1630 struct ib_uverbs_ex_create_qp cmd_ex; 1631 struct ib_udata ucore; 1632 struct ib_udata uhw; 1633 ssize_t resp_size = sizeof(struct ib_uverbs_create_qp_resp); 1634 int err; 1635 1636 if (out_len < resp_size) 1637 return -ENOSPC; 1638 1639 if (copy_from_user(&cmd, buf, sizeof(cmd))) 1640 return -EFAULT; 1641 1642 ib_uverbs_init_udata(&ucore, buf, u64_to_user_ptr(cmd.response), 1643 sizeof(cmd), resp_size); 1644 ib_uverbs_init_udata(&uhw, buf + sizeof(cmd), 1645 u64_to_user_ptr(cmd.response) + resp_size, 1646 in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr), 1647 out_len - resp_size); 1648 1649 memset(&cmd_ex, 0, sizeof(cmd_ex)); 1650 cmd_ex.user_handle = cmd.user_handle; 1651 cmd_ex.pd_handle = cmd.pd_handle; 1652 cmd_ex.send_cq_handle = cmd.send_cq_handle; 1653 cmd_ex.recv_cq_handle = cmd.recv_cq_handle; 1654 cmd_ex.srq_handle = cmd.srq_handle; 1655 cmd_ex.max_send_wr = cmd.max_send_wr; 1656 cmd_ex.max_recv_wr = cmd.max_recv_wr; 1657 cmd_ex.max_send_sge = cmd.max_send_sge; 1658 cmd_ex.max_recv_sge = cmd.max_recv_sge; 1659 cmd_ex.max_inline_data = cmd.max_inline_data; 1660 cmd_ex.sq_sig_all = cmd.sq_sig_all; 1661 cmd_ex.qp_type = cmd.qp_type; 1662 cmd_ex.is_srq = cmd.is_srq; 1663 1664 err = create_qp(file, &ucore, &uhw, &cmd_ex, 1665 offsetof(typeof(cmd_ex), is_srq) + 1666 sizeof(cmd.is_srq), ib_uverbs_create_qp_cb, 1667 NULL); 1668 1669 if (err) 1670 return err; 1671 1672 return in_len; 1673 } 1674 1675 static int ib_uverbs_ex_create_qp_cb(struct ib_uverbs_file *file, 1676 struct ib_uverbs_ex_create_qp_resp *resp, 1677 struct ib_udata *ucore) 1678 { 1679 if (ib_copy_to_udata(ucore, resp, resp->response_length)) 1680 return -EFAULT; 1681 1682 return 0; 1683 } 1684 1685 int ib_uverbs_ex_create_qp(struct ib_uverbs_file *file, 1686 struct ib_udata *ucore, 1687 struct ib_udata *uhw) 1688 { 1689 struct ib_uverbs_ex_create_qp_resp resp; 1690 struct ib_uverbs_ex_create_qp cmd = {0}; 1691 int err; 1692 1693 if (ucore->inlen < (offsetof(typeof(cmd), comp_mask) + 1694 sizeof(cmd.comp_mask))) 1695 return -EINVAL; 1696 1697 err = ib_copy_from_udata(&cmd, ucore, min(sizeof(cmd), ucore->inlen)); 1698 if (err) 1699 return err; 1700 1701 if (cmd.comp_mask & ~IB_UVERBS_CREATE_QP_SUP_COMP_MASK) 1702 return -EINVAL; 1703 1704 if (cmd.reserved) 1705 return -EINVAL; 1706 1707 if (ucore->outlen < (offsetof(typeof(resp), response_length) + 1708 sizeof(resp.response_length))) 1709 return -ENOSPC; 1710 1711 err = create_qp(file, ucore, uhw, &cmd, 1712 min(ucore->inlen, sizeof(cmd)), 1713 ib_uverbs_ex_create_qp_cb, NULL); 1714 1715 if (err) 1716 return err; 1717 1718 return 0; 1719 } 1720 1721 ssize_t ib_uverbs_open_qp(struct ib_uverbs_file *file, 1722 const char __user *buf, int in_len, int out_len) 1723 { 1724 struct ib_uverbs_open_qp cmd; 1725 struct ib_uverbs_create_qp_resp resp; 1726 struct ib_udata udata; 1727 struct ib_uqp_object *obj; 1728 struct ib_xrcd *xrcd; 1729 struct ib_uobject *uninitialized_var(xrcd_uobj); 1730 struct ib_qp *qp; 1731 struct ib_qp_open_attr attr; 1732 int ret; 1733 struct ib_device *ib_dev; 1734 1735 if (out_len < sizeof resp) 1736 return -ENOSPC; 1737 1738 if (copy_from_user(&cmd, buf, sizeof cmd)) 1739 return -EFAULT; 1740 1741 ib_uverbs_init_udata(&udata, buf + sizeof(cmd), 1742 u64_to_user_ptr(cmd.response) + sizeof(resp), 1743 in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr), 1744 out_len - sizeof(resp)); 1745 1746 obj = (struct ib_uqp_object *)uobj_alloc(UVERBS_OBJECT_QP, file, 1747 &ib_dev); 1748 if (IS_ERR(obj)) 1749 return PTR_ERR(obj); 1750 1751 xrcd_uobj = uobj_get_read(UVERBS_OBJECT_XRCD, cmd.pd_handle, file); 1752 if (IS_ERR(xrcd_uobj)) { 1753 ret = -EINVAL; 1754 goto err_put; 1755 } 1756 1757 xrcd = (struct ib_xrcd *)xrcd_uobj->object; 1758 if (!xrcd) { 1759 ret = -EINVAL; 1760 goto err_xrcd; 1761 } 1762 1763 attr.event_handler = ib_uverbs_qp_event_handler; 1764 attr.qp_context = file; 1765 attr.qp_num = cmd.qpn; 1766 attr.qp_type = cmd.qp_type; 1767 1768 obj->uevent.events_reported = 0; 1769 INIT_LIST_HEAD(&obj->uevent.event_list); 1770 INIT_LIST_HEAD(&obj->mcast_list); 1771 1772 qp = ib_open_qp(xrcd, &attr); 1773 if (IS_ERR(qp)) { 1774 ret = PTR_ERR(qp); 1775 goto err_xrcd; 1776 } 1777 1778 obj->uevent.uobject.object = qp; 1779 obj->uevent.uobject.user_handle = cmd.user_handle; 1780 1781 memset(&resp, 0, sizeof resp); 1782 resp.qpn = qp->qp_num; 1783 resp.qp_handle = obj->uevent.uobject.id; 1784 1785 if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp)) { 1786 ret = -EFAULT; 1787 goto err_destroy; 1788 } 1789 1790 obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object, uobject); 1791 atomic_inc(&obj->uxrcd->refcnt); 1792 qp->uobject = &obj->uevent.uobject; 1793 uobj_put_read(xrcd_uobj); 1794 1795 return uobj_alloc_commit(&obj->uevent.uobject, in_len); 1796 1797 err_destroy: 1798 ib_destroy_qp(qp); 1799 err_xrcd: 1800 uobj_put_read(xrcd_uobj); 1801 err_put: 1802 uobj_alloc_abort(&obj->uevent.uobject); 1803 return ret; 1804 } 1805 1806 static void copy_ah_attr_to_uverbs(struct ib_uverbs_qp_dest *uverb_attr, 1807 struct rdma_ah_attr *rdma_attr) 1808 { 1809 const struct ib_global_route *grh; 1810 1811 uverb_attr->dlid = rdma_ah_get_dlid(rdma_attr); 1812 uverb_attr->sl = rdma_ah_get_sl(rdma_attr); 1813 uverb_attr->src_path_bits = rdma_ah_get_path_bits(rdma_attr); 1814 uverb_attr->static_rate = rdma_ah_get_static_rate(rdma_attr); 1815 uverb_attr->is_global = !!(rdma_ah_get_ah_flags(rdma_attr) & 1816 IB_AH_GRH); 1817 if (uverb_attr->is_global) { 1818 grh = rdma_ah_read_grh(rdma_attr); 1819 memcpy(uverb_attr->dgid, grh->dgid.raw, 16); 1820 uverb_attr->flow_label = grh->flow_label; 1821 uverb_attr->sgid_index = grh->sgid_index; 1822 uverb_attr->hop_limit = grh->hop_limit; 1823 uverb_attr->traffic_class = grh->traffic_class; 1824 } 1825 uverb_attr->port_num = rdma_ah_get_port_num(rdma_attr); 1826 } 1827 1828 ssize_t ib_uverbs_query_qp(struct ib_uverbs_file *file, 1829 const char __user *buf, int in_len, 1830 int out_len) 1831 { 1832 struct ib_uverbs_query_qp cmd; 1833 struct ib_uverbs_query_qp_resp resp; 1834 struct ib_qp *qp; 1835 struct ib_qp_attr *attr; 1836 struct ib_qp_init_attr *init_attr; 1837 int ret; 1838 1839 if (copy_from_user(&cmd, buf, sizeof cmd)) 1840 return -EFAULT; 1841 1842 attr = kmalloc(sizeof *attr, GFP_KERNEL); 1843 init_attr = kmalloc(sizeof *init_attr, GFP_KERNEL); 1844 if (!attr || !init_attr) { 1845 ret = -ENOMEM; 1846 goto out; 1847 } 1848 1849 qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, file); 1850 if (!qp) { 1851 ret = -EINVAL; 1852 goto out; 1853 } 1854 1855 ret = ib_query_qp(qp, attr, cmd.attr_mask, init_attr); 1856 1857 uobj_put_obj_read(qp); 1858 1859 if (ret) 1860 goto out; 1861 1862 memset(&resp, 0, sizeof resp); 1863 1864 resp.qp_state = attr->qp_state; 1865 resp.cur_qp_state = attr->cur_qp_state; 1866 resp.path_mtu = attr->path_mtu; 1867 resp.path_mig_state = attr->path_mig_state; 1868 resp.qkey = attr->qkey; 1869 resp.rq_psn = attr->rq_psn; 1870 resp.sq_psn = attr->sq_psn; 1871 resp.dest_qp_num = attr->dest_qp_num; 1872 resp.qp_access_flags = attr->qp_access_flags; 1873 resp.pkey_index = attr->pkey_index; 1874 resp.alt_pkey_index = attr->alt_pkey_index; 1875 resp.sq_draining = attr->sq_draining; 1876 resp.max_rd_atomic = attr->max_rd_atomic; 1877 resp.max_dest_rd_atomic = attr->max_dest_rd_atomic; 1878 resp.min_rnr_timer = attr->min_rnr_timer; 1879 resp.port_num = attr->port_num; 1880 resp.timeout = attr->timeout; 1881 resp.retry_cnt = attr->retry_cnt; 1882 resp.rnr_retry = attr->rnr_retry; 1883 resp.alt_port_num = attr->alt_port_num; 1884 resp.alt_timeout = attr->alt_timeout; 1885 1886 copy_ah_attr_to_uverbs(&resp.dest, &attr->ah_attr); 1887 copy_ah_attr_to_uverbs(&resp.alt_dest, &attr->alt_ah_attr); 1888 1889 resp.max_send_wr = init_attr->cap.max_send_wr; 1890 resp.max_recv_wr = init_attr->cap.max_recv_wr; 1891 resp.max_send_sge = init_attr->cap.max_send_sge; 1892 resp.max_recv_sge = init_attr->cap.max_recv_sge; 1893 resp.max_inline_data = init_attr->cap.max_inline_data; 1894 resp.sq_sig_all = init_attr->sq_sig_type == IB_SIGNAL_ALL_WR; 1895 1896 if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp)) 1897 ret = -EFAULT; 1898 1899 out: 1900 kfree(attr); 1901 kfree(init_attr); 1902 1903 return ret ? ret : in_len; 1904 } 1905 1906 /* Remove ignored fields set in the attribute mask */ 1907 static int modify_qp_mask(enum ib_qp_type qp_type, int mask) 1908 { 1909 switch (qp_type) { 1910 case IB_QPT_XRC_INI: 1911 return mask & ~(IB_QP_MAX_DEST_RD_ATOMIC | IB_QP_MIN_RNR_TIMER); 1912 case IB_QPT_XRC_TGT: 1913 return mask & ~(IB_QP_MAX_QP_RD_ATOMIC | IB_QP_RETRY_CNT | 1914 IB_QP_RNR_RETRY); 1915 default: 1916 return mask; 1917 } 1918 } 1919 1920 static void copy_ah_attr_from_uverbs(struct ib_device *dev, 1921 struct rdma_ah_attr *rdma_attr, 1922 struct ib_uverbs_qp_dest *uverb_attr) 1923 { 1924 rdma_attr->type = rdma_ah_find_type(dev, uverb_attr->port_num); 1925 if (uverb_attr->is_global) { 1926 rdma_ah_set_grh(rdma_attr, NULL, 1927 uverb_attr->flow_label, 1928 uverb_attr->sgid_index, 1929 uverb_attr->hop_limit, 1930 uverb_attr->traffic_class); 1931 rdma_ah_set_dgid_raw(rdma_attr, uverb_attr->dgid); 1932 } else { 1933 rdma_ah_set_ah_flags(rdma_attr, 0); 1934 } 1935 rdma_ah_set_dlid(rdma_attr, uverb_attr->dlid); 1936 rdma_ah_set_sl(rdma_attr, uverb_attr->sl); 1937 rdma_ah_set_path_bits(rdma_attr, uverb_attr->src_path_bits); 1938 rdma_ah_set_static_rate(rdma_attr, uverb_attr->static_rate); 1939 rdma_ah_set_port_num(rdma_attr, uverb_attr->port_num); 1940 rdma_ah_set_make_grd(rdma_attr, false); 1941 } 1942 1943 static int modify_qp(struct ib_uverbs_file *file, 1944 struct ib_uverbs_ex_modify_qp *cmd, struct ib_udata *udata) 1945 { 1946 struct ib_qp_attr *attr; 1947 struct ib_qp *qp; 1948 int ret; 1949 1950 attr = kzalloc(sizeof(*attr), GFP_KERNEL); 1951 if (!attr) 1952 return -ENOMEM; 1953 1954 qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd->base.qp_handle, file); 1955 if (!qp) { 1956 ret = -EINVAL; 1957 goto out; 1958 } 1959 1960 if ((cmd->base.attr_mask & IB_QP_PORT) && 1961 !rdma_is_port_valid(qp->device, cmd->base.port_num)) { 1962 ret = -EINVAL; 1963 goto release_qp; 1964 } 1965 1966 if ((cmd->base.attr_mask & IB_QP_AV)) { 1967 if (!rdma_is_port_valid(qp->device, cmd->base.dest.port_num)) { 1968 ret = -EINVAL; 1969 goto release_qp; 1970 } 1971 1972 if (cmd->base.attr_mask & IB_QP_STATE && 1973 cmd->base.qp_state == IB_QPS_RTR) { 1974 /* We are in INIT->RTR TRANSITION (if we are not, 1975 * this transition will be rejected in subsequent checks). 1976 * In the INIT->RTR transition, we cannot have IB_QP_PORT set, 1977 * but the IB_QP_STATE flag is required. 1978 * 1979 * Since kernel 3.14 (commit dbf727de7440), the uverbs driver, 1980 * when IB_QP_AV is set, has required inclusion of a valid 1981 * port number in the primary AV. (AVs are created and handled 1982 * differently for infiniband and ethernet (RoCE) ports). 1983 * 1984 * Check the port number included in the primary AV against 1985 * the port number in the qp struct, which was set (and saved) 1986 * in the RST->INIT transition. 1987 */ 1988 if (cmd->base.dest.port_num != qp->real_qp->port) { 1989 ret = -EINVAL; 1990 goto release_qp; 1991 } 1992 } else { 1993 /* We are in SQD->SQD. (If we are not, this transition will 1994 * be rejected later in the verbs layer checks). 1995 * Check for both IB_QP_PORT and IB_QP_AV, these can be set 1996 * together in the SQD->SQD transition. 1997 * 1998 * If only IP_QP_AV was set, add in IB_QP_PORT as well (the 1999 * verbs layer driver does not track primary port changes 2000 * resulting from path migration. Thus, in SQD, if the primary 2001 * AV is modified, the primary port should also be modified). 2002 * 2003 * Note that in this transition, the IB_QP_STATE flag 2004 * is not allowed. 2005 */ 2006 if (((cmd->base.attr_mask & (IB_QP_AV | IB_QP_PORT)) 2007 == (IB_QP_AV | IB_QP_PORT)) && 2008 cmd->base.port_num != cmd->base.dest.port_num) { 2009 ret = -EINVAL; 2010 goto release_qp; 2011 } 2012 if ((cmd->base.attr_mask & (IB_QP_AV | IB_QP_PORT)) 2013 == IB_QP_AV) { 2014 cmd->base.attr_mask |= IB_QP_PORT; 2015 cmd->base.port_num = cmd->base.dest.port_num; 2016 } 2017 } 2018 } 2019 2020 if ((cmd->base.attr_mask & IB_QP_ALT_PATH) && 2021 (!rdma_is_port_valid(qp->device, cmd->base.alt_port_num) || 2022 !rdma_is_port_valid(qp->device, cmd->base.alt_dest.port_num) || 2023 cmd->base.alt_port_num != cmd->base.alt_dest.port_num)) { 2024 ret = -EINVAL; 2025 goto release_qp; 2026 } 2027 2028 if ((cmd->base.attr_mask & IB_QP_CUR_STATE && 2029 cmd->base.cur_qp_state > IB_QPS_ERR) || 2030 cmd->base.qp_state > IB_QPS_ERR) { 2031 ret = -EINVAL; 2032 goto release_qp; 2033 } 2034 2035 attr->qp_state = cmd->base.qp_state; 2036 attr->cur_qp_state = cmd->base.cur_qp_state; 2037 attr->path_mtu = cmd->base.path_mtu; 2038 attr->path_mig_state = cmd->base.path_mig_state; 2039 attr->qkey = cmd->base.qkey; 2040 attr->rq_psn = cmd->base.rq_psn; 2041 attr->sq_psn = cmd->base.sq_psn; 2042 attr->dest_qp_num = cmd->base.dest_qp_num; 2043 attr->qp_access_flags = cmd->base.qp_access_flags; 2044 attr->pkey_index = cmd->base.pkey_index; 2045 attr->alt_pkey_index = cmd->base.alt_pkey_index; 2046 attr->en_sqd_async_notify = cmd->base.en_sqd_async_notify; 2047 attr->max_rd_atomic = cmd->base.max_rd_atomic; 2048 attr->max_dest_rd_atomic = cmd->base.max_dest_rd_atomic; 2049 attr->min_rnr_timer = cmd->base.min_rnr_timer; 2050 attr->port_num = cmd->base.port_num; 2051 attr->timeout = cmd->base.timeout; 2052 attr->retry_cnt = cmd->base.retry_cnt; 2053 attr->rnr_retry = cmd->base.rnr_retry; 2054 attr->alt_port_num = cmd->base.alt_port_num; 2055 attr->alt_timeout = cmd->base.alt_timeout; 2056 attr->rate_limit = cmd->rate_limit; 2057 2058 if (cmd->base.attr_mask & IB_QP_AV) 2059 copy_ah_attr_from_uverbs(qp->device, &attr->ah_attr, 2060 &cmd->base.dest); 2061 2062 if (cmd->base.attr_mask & IB_QP_ALT_PATH) 2063 copy_ah_attr_from_uverbs(qp->device, &attr->alt_ah_attr, 2064 &cmd->base.alt_dest); 2065 2066 ret = ib_modify_qp_with_udata(qp, attr, 2067 modify_qp_mask(qp->qp_type, 2068 cmd->base.attr_mask), 2069 udata); 2070 2071 release_qp: 2072 uobj_put_obj_read(qp); 2073 out: 2074 kfree(attr); 2075 2076 return ret; 2077 } 2078 2079 ssize_t ib_uverbs_modify_qp(struct ib_uverbs_file *file, 2080 const char __user *buf, int in_len, 2081 int out_len) 2082 { 2083 struct ib_uverbs_ex_modify_qp cmd = {}; 2084 struct ib_udata udata; 2085 int ret; 2086 2087 if (copy_from_user(&cmd.base, buf, sizeof(cmd.base))) 2088 return -EFAULT; 2089 2090 if (cmd.base.attr_mask & 2091 ~((IB_USER_LEGACY_LAST_QP_ATTR_MASK << 1) - 1)) 2092 return -EOPNOTSUPP; 2093 2094 ib_uverbs_init_udata(&udata, buf + sizeof(cmd.base), NULL, 2095 in_len - sizeof(cmd.base) - sizeof(struct ib_uverbs_cmd_hdr), 2096 out_len); 2097 2098 ret = modify_qp(file, &cmd, &udata); 2099 if (ret) 2100 return ret; 2101 2102 return in_len; 2103 } 2104 2105 int ib_uverbs_ex_modify_qp(struct ib_uverbs_file *file, 2106 struct ib_udata *ucore, 2107 struct ib_udata *uhw) 2108 { 2109 struct ib_uverbs_ex_modify_qp cmd = {}; 2110 int ret; 2111 2112 /* 2113 * Last bit is reserved for extending the attr_mask by 2114 * using another field. 2115 */ 2116 BUILD_BUG_ON(IB_USER_LAST_QP_ATTR_MASK == (1 << 31)); 2117 2118 if (ucore->inlen < sizeof(cmd.base)) 2119 return -EINVAL; 2120 2121 ret = ib_copy_from_udata(&cmd, ucore, min(sizeof(cmd), ucore->inlen)); 2122 if (ret) 2123 return ret; 2124 2125 if (cmd.base.attr_mask & 2126 ~((IB_USER_LAST_QP_ATTR_MASK << 1) - 1)) 2127 return -EOPNOTSUPP; 2128 2129 if (ucore->inlen > sizeof(cmd)) { 2130 if (!ib_is_udata_cleared(ucore, sizeof(cmd), 2131 ucore->inlen - sizeof(cmd))) 2132 return -EOPNOTSUPP; 2133 } 2134 2135 ret = modify_qp(file, &cmd, uhw); 2136 2137 return ret; 2138 } 2139 2140 ssize_t ib_uverbs_destroy_qp(struct ib_uverbs_file *file, 2141 const char __user *buf, int in_len, 2142 int out_len) 2143 { 2144 struct ib_uverbs_destroy_qp cmd; 2145 struct ib_uverbs_destroy_qp_resp resp; 2146 struct ib_uobject *uobj; 2147 struct ib_uqp_object *obj; 2148 2149 if (copy_from_user(&cmd, buf, sizeof cmd)) 2150 return -EFAULT; 2151 2152 uobj = uobj_get_destroy(UVERBS_OBJECT_QP, cmd.qp_handle, file); 2153 if (IS_ERR(uobj)) 2154 return PTR_ERR(uobj); 2155 2156 obj = container_of(uobj, struct ib_uqp_object, uevent.uobject); 2157 memset(&resp, 0, sizeof(resp)); 2158 resp.events_reported = obj->uevent.events_reported; 2159 2160 uobj_put_destroy(uobj); 2161 2162 if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp)) 2163 return -EFAULT; 2164 2165 return in_len; 2166 } 2167 2168 static void *alloc_wr(size_t wr_size, __u32 num_sge) 2169 { 2170 if (num_sge >= (U32_MAX - ALIGN(wr_size, sizeof (struct ib_sge))) / 2171 sizeof (struct ib_sge)) 2172 return NULL; 2173 2174 return kmalloc(ALIGN(wr_size, sizeof (struct ib_sge)) + 2175 num_sge * sizeof (struct ib_sge), GFP_KERNEL); 2176 } 2177 2178 ssize_t ib_uverbs_post_send(struct ib_uverbs_file *file, 2179 const char __user *buf, int in_len, 2180 int out_len) 2181 { 2182 struct ib_uverbs_post_send cmd; 2183 struct ib_uverbs_post_send_resp resp; 2184 struct ib_uverbs_send_wr *user_wr; 2185 struct ib_send_wr *wr = NULL, *last, *next; 2186 const struct ib_send_wr *bad_wr; 2187 struct ib_qp *qp; 2188 int i, sg_ind; 2189 int is_ud; 2190 ssize_t ret = -EINVAL; 2191 size_t next_size; 2192 2193 if (copy_from_user(&cmd, buf, sizeof cmd)) 2194 return -EFAULT; 2195 2196 if (in_len < sizeof cmd + cmd.wqe_size * cmd.wr_count + 2197 cmd.sge_count * sizeof (struct ib_uverbs_sge)) 2198 return -EINVAL; 2199 2200 if (cmd.wqe_size < sizeof (struct ib_uverbs_send_wr)) 2201 return -EINVAL; 2202 2203 user_wr = kmalloc(cmd.wqe_size, GFP_KERNEL); 2204 if (!user_wr) 2205 return -ENOMEM; 2206 2207 qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, file); 2208 if (!qp) 2209 goto out; 2210 2211 is_ud = qp->qp_type == IB_QPT_UD; 2212 sg_ind = 0; 2213 last = NULL; 2214 for (i = 0; i < cmd.wr_count; ++i) { 2215 if (copy_from_user(user_wr, 2216 buf + sizeof cmd + i * cmd.wqe_size, 2217 cmd.wqe_size)) { 2218 ret = -EFAULT; 2219 goto out_put; 2220 } 2221 2222 if (user_wr->num_sge + sg_ind > cmd.sge_count) { 2223 ret = -EINVAL; 2224 goto out_put; 2225 } 2226 2227 if (is_ud) { 2228 struct ib_ud_wr *ud; 2229 2230 if (user_wr->opcode != IB_WR_SEND && 2231 user_wr->opcode != IB_WR_SEND_WITH_IMM) { 2232 ret = -EINVAL; 2233 goto out_put; 2234 } 2235 2236 next_size = sizeof(*ud); 2237 ud = alloc_wr(next_size, user_wr->num_sge); 2238 if (!ud) { 2239 ret = -ENOMEM; 2240 goto out_put; 2241 } 2242 2243 ud->ah = uobj_get_obj_read(ah, UVERBS_OBJECT_AH, 2244 user_wr->wr.ud.ah, file); 2245 if (!ud->ah) { 2246 kfree(ud); 2247 ret = -EINVAL; 2248 goto out_put; 2249 } 2250 ud->remote_qpn = user_wr->wr.ud.remote_qpn; 2251 ud->remote_qkey = user_wr->wr.ud.remote_qkey; 2252 2253 next = &ud->wr; 2254 } else if (user_wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM || 2255 user_wr->opcode == IB_WR_RDMA_WRITE || 2256 user_wr->opcode == IB_WR_RDMA_READ) { 2257 struct ib_rdma_wr *rdma; 2258 2259 next_size = sizeof(*rdma); 2260 rdma = alloc_wr(next_size, user_wr->num_sge); 2261 if (!rdma) { 2262 ret = -ENOMEM; 2263 goto out_put; 2264 } 2265 2266 rdma->remote_addr = user_wr->wr.rdma.remote_addr; 2267 rdma->rkey = user_wr->wr.rdma.rkey; 2268 2269 next = &rdma->wr; 2270 } else if (user_wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP || 2271 user_wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) { 2272 struct ib_atomic_wr *atomic; 2273 2274 next_size = sizeof(*atomic); 2275 atomic = alloc_wr(next_size, user_wr->num_sge); 2276 if (!atomic) { 2277 ret = -ENOMEM; 2278 goto out_put; 2279 } 2280 2281 atomic->remote_addr = user_wr->wr.atomic.remote_addr; 2282 atomic->compare_add = user_wr->wr.atomic.compare_add; 2283 atomic->swap = user_wr->wr.atomic.swap; 2284 atomic->rkey = user_wr->wr.atomic.rkey; 2285 2286 next = &atomic->wr; 2287 } else if (user_wr->opcode == IB_WR_SEND || 2288 user_wr->opcode == IB_WR_SEND_WITH_IMM || 2289 user_wr->opcode == IB_WR_SEND_WITH_INV) { 2290 next_size = sizeof(*next); 2291 next = alloc_wr(next_size, user_wr->num_sge); 2292 if (!next) { 2293 ret = -ENOMEM; 2294 goto out_put; 2295 } 2296 } else { 2297 ret = -EINVAL; 2298 goto out_put; 2299 } 2300 2301 if (user_wr->opcode == IB_WR_SEND_WITH_IMM || 2302 user_wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM) { 2303 next->ex.imm_data = 2304 (__be32 __force) user_wr->ex.imm_data; 2305 } else if (user_wr->opcode == IB_WR_SEND_WITH_INV) { 2306 next->ex.invalidate_rkey = user_wr->ex.invalidate_rkey; 2307 } 2308 2309 if (!last) 2310 wr = next; 2311 else 2312 last->next = next; 2313 last = next; 2314 2315 next->next = NULL; 2316 next->wr_id = user_wr->wr_id; 2317 next->num_sge = user_wr->num_sge; 2318 next->opcode = user_wr->opcode; 2319 next->send_flags = user_wr->send_flags; 2320 2321 if (next->num_sge) { 2322 next->sg_list = (void *) next + 2323 ALIGN(next_size, sizeof(struct ib_sge)); 2324 if (copy_from_user(next->sg_list, 2325 buf + sizeof cmd + 2326 cmd.wr_count * cmd.wqe_size + 2327 sg_ind * sizeof (struct ib_sge), 2328 next->num_sge * sizeof (struct ib_sge))) { 2329 ret = -EFAULT; 2330 goto out_put; 2331 } 2332 sg_ind += next->num_sge; 2333 } else 2334 next->sg_list = NULL; 2335 } 2336 2337 resp.bad_wr = 0; 2338 ret = qp->device->post_send(qp->real_qp, wr, &bad_wr); 2339 if (ret) 2340 for (next = wr; next; next = next->next) { 2341 ++resp.bad_wr; 2342 if (next == bad_wr) 2343 break; 2344 } 2345 2346 if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp)) 2347 ret = -EFAULT; 2348 2349 out_put: 2350 uobj_put_obj_read(qp); 2351 2352 while (wr) { 2353 if (is_ud && ud_wr(wr)->ah) 2354 uobj_put_obj_read(ud_wr(wr)->ah); 2355 next = wr->next; 2356 kfree(wr); 2357 wr = next; 2358 } 2359 2360 out: 2361 kfree(user_wr); 2362 2363 return ret ? ret : in_len; 2364 } 2365 2366 static struct ib_recv_wr *ib_uverbs_unmarshall_recv(const char __user *buf, 2367 int in_len, 2368 u32 wr_count, 2369 u32 sge_count, 2370 u32 wqe_size) 2371 { 2372 struct ib_uverbs_recv_wr *user_wr; 2373 struct ib_recv_wr *wr = NULL, *last, *next; 2374 int sg_ind; 2375 int i; 2376 int ret; 2377 2378 if (in_len < wqe_size * wr_count + 2379 sge_count * sizeof (struct ib_uverbs_sge)) 2380 return ERR_PTR(-EINVAL); 2381 2382 if (wqe_size < sizeof (struct ib_uverbs_recv_wr)) 2383 return ERR_PTR(-EINVAL); 2384 2385 user_wr = kmalloc(wqe_size, GFP_KERNEL); 2386 if (!user_wr) 2387 return ERR_PTR(-ENOMEM); 2388 2389 sg_ind = 0; 2390 last = NULL; 2391 for (i = 0; i < wr_count; ++i) { 2392 if (copy_from_user(user_wr, buf + i * wqe_size, 2393 wqe_size)) { 2394 ret = -EFAULT; 2395 goto err; 2396 } 2397 2398 if (user_wr->num_sge + sg_ind > sge_count) { 2399 ret = -EINVAL; 2400 goto err; 2401 } 2402 2403 if (user_wr->num_sge >= 2404 (U32_MAX - ALIGN(sizeof *next, sizeof (struct ib_sge))) / 2405 sizeof (struct ib_sge)) { 2406 ret = -EINVAL; 2407 goto err; 2408 } 2409 2410 next = kmalloc(ALIGN(sizeof *next, sizeof (struct ib_sge)) + 2411 user_wr->num_sge * sizeof (struct ib_sge), 2412 GFP_KERNEL); 2413 if (!next) { 2414 ret = -ENOMEM; 2415 goto err; 2416 } 2417 2418 if (!last) 2419 wr = next; 2420 else 2421 last->next = next; 2422 last = next; 2423 2424 next->next = NULL; 2425 next->wr_id = user_wr->wr_id; 2426 next->num_sge = user_wr->num_sge; 2427 2428 if (next->num_sge) { 2429 next->sg_list = (void *) next + 2430 ALIGN(sizeof *next, sizeof (struct ib_sge)); 2431 if (copy_from_user(next->sg_list, 2432 buf + wr_count * wqe_size + 2433 sg_ind * sizeof (struct ib_sge), 2434 next->num_sge * sizeof (struct ib_sge))) { 2435 ret = -EFAULT; 2436 goto err; 2437 } 2438 sg_ind += next->num_sge; 2439 } else 2440 next->sg_list = NULL; 2441 } 2442 2443 kfree(user_wr); 2444 return wr; 2445 2446 err: 2447 kfree(user_wr); 2448 2449 while (wr) { 2450 next = wr->next; 2451 kfree(wr); 2452 wr = next; 2453 } 2454 2455 return ERR_PTR(ret); 2456 } 2457 2458 ssize_t ib_uverbs_post_recv(struct ib_uverbs_file *file, 2459 const char __user *buf, int in_len, 2460 int out_len) 2461 { 2462 struct ib_uverbs_post_recv cmd; 2463 struct ib_uverbs_post_recv_resp resp; 2464 struct ib_recv_wr *wr, *next; 2465 const struct ib_recv_wr *bad_wr; 2466 struct ib_qp *qp; 2467 ssize_t ret = -EINVAL; 2468 2469 if (copy_from_user(&cmd, buf, sizeof cmd)) 2470 return -EFAULT; 2471 2472 wr = ib_uverbs_unmarshall_recv(buf + sizeof cmd, 2473 in_len - sizeof cmd, cmd.wr_count, 2474 cmd.sge_count, cmd.wqe_size); 2475 if (IS_ERR(wr)) 2476 return PTR_ERR(wr); 2477 2478 qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, file); 2479 if (!qp) 2480 goto out; 2481 2482 resp.bad_wr = 0; 2483 ret = qp->device->post_recv(qp->real_qp, wr, &bad_wr); 2484 2485 uobj_put_obj_read(qp); 2486 if (ret) { 2487 for (next = wr; next; next = next->next) { 2488 ++resp.bad_wr; 2489 if (next == bad_wr) 2490 break; 2491 } 2492 } 2493 2494 if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp)) 2495 ret = -EFAULT; 2496 2497 out: 2498 while (wr) { 2499 next = wr->next; 2500 kfree(wr); 2501 wr = next; 2502 } 2503 2504 return ret ? ret : in_len; 2505 } 2506 2507 ssize_t ib_uverbs_post_srq_recv(struct ib_uverbs_file *file, 2508 const char __user *buf, int in_len, 2509 int out_len) 2510 { 2511 struct ib_uverbs_post_srq_recv cmd; 2512 struct ib_uverbs_post_srq_recv_resp resp; 2513 struct ib_recv_wr *wr, *next; 2514 const struct ib_recv_wr *bad_wr; 2515 struct ib_srq *srq; 2516 ssize_t ret = -EINVAL; 2517 2518 if (copy_from_user(&cmd, buf, sizeof cmd)) 2519 return -EFAULT; 2520 2521 wr = ib_uverbs_unmarshall_recv(buf + sizeof cmd, 2522 in_len - sizeof cmd, cmd.wr_count, 2523 cmd.sge_count, cmd.wqe_size); 2524 if (IS_ERR(wr)) 2525 return PTR_ERR(wr); 2526 2527 srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ, cmd.srq_handle, file); 2528 if (!srq) 2529 goto out; 2530 2531 resp.bad_wr = 0; 2532 ret = srq->device->post_srq_recv ? 2533 srq->device->post_srq_recv(srq, wr, &bad_wr) : -EOPNOTSUPP; 2534 2535 uobj_put_obj_read(srq); 2536 2537 if (ret) 2538 for (next = wr; next; next = next->next) { 2539 ++resp.bad_wr; 2540 if (next == bad_wr) 2541 break; 2542 } 2543 2544 if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp)) 2545 ret = -EFAULT; 2546 2547 out: 2548 while (wr) { 2549 next = wr->next; 2550 kfree(wr); 2551 wr = next; 2552 } 2553 2554 return ret ? ret : in_len; 2555 } 2556 2557 ssize_t ib_uverbs_create_ah(struct ib_uverbs_file *file, 2558 const char __user *buf, int in_len, 2559 int out_len) 2560 { 2561 struct ib_uverbs_create_ah cmd; 2562 struct ib_uverbs_create_ah_resp resp; 2563 struct ib_uobject *uobj; 2564 struct ib_pd *pd; 2565 struct ib_ah *ah; 2566 struct rdma_ah_attr attr = {}; 2567 int ret; 2568 struct ib_udata udata; 2569 struct ib_device *ib_dev; 2570 2571 if (out_len < sizeof resp) 2572 return -ENOSPC; 2573 2574 if (copy_from_user(&cmd, buf, sizeof cmd)) 2575 return -EFAULT; 2576 2577 ib_uverbs_init_udata(&udata, buf + sizeof(cmd), 2578 u64_to_user_ptr(cmd.response) + sizeof(resp), 2579 in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr), 2580 out_len - sizeof(resp)); 2581 2582 uobj = uobj_alloc(UVERBS_OBJECT_AH, file, &ib_dev); 2583 if (IS_ERR(uobj)) 2584 return PTR_ERR(uobj); 2585 2586 if (!rdma_is_port_valid(ib_dev, cmd.attr.port_num)) { 2587 ret = -EINVAL; 2588 goto err; 2589 } 2590 2591 pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, file); 2592 if (!pd) { 2593 ret = -EINVAL; 2594 goto err; 2595 } 2596 2597 attr.type = rdma_ah_find_type(ib_dev, cmd.attr.port_num); 2598 rdma_ah_set_make_grd(&attr, false); 2599 rdma_ah_set_dlid(&attr, cmd.attr.dlid); 2600 rdma_ah_set_sl(&attr, cmd.attr.sl); 2601 rdma_ah_set_path_bits(&attr, cmd.attr.src_path_bits); 2602 rdma_ah_set_static_rate(&attr, cmd.attr.static_rate); 2603 rdma_ah_set_port_num(&attr, cmd.attr.port_num); 2604 2605 if (cmd.attr.is_global) { 2606 rdma_ah_set_grh(&attr, NULL, cmd.attr.grh.flow_label, 2607 cmd.attr.grh.sgid_index, 2608 cmd.attr.grh.hop_limit, 2609 cmd.attr.grh.traffic_class); 2610 rdma_ah_set_dgid_raw(&attr, cmd.attr.grh.dgid); 2611 } else { 2612 rdma_ah_set_ah_flags(&attr, 0); 2613 } 2614 2615 ah = rdma_create_user_ah(pd, &attr, &udata); 2616 if (IS_ERR(ah)) { 2617 ret = PTR_ERR(ah); 2618 goto err_put; 2619 } 2620 2621 ah->uobject = uobj; 2622 uobj->user_handle = cmd.user_handle; 2623 uobj->object = ah; 2624 2625 resp.ah_handle = uobj->id; 2626 2627 if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp)) { 2628 ret = -EFAULT; 2629 goto err_copy; 2630 } 2631 2632 uobj_put_obj_read(pd); 2633 return uobj_alloc_commit(uobj, in_len); 2634 2635 err_copy: 2636 rdma_destroy_ah(ah); 2637 2638 err_put: 2639 uobj_put_obj_read(pd); 2640 2641 err: 2642 uobj_alloc_abort(uobj); 2643 return ret; 2644 } 2645 2646 ssize_t ib_uverbs_destroy_ah(struct ib_uverbs_file *file, 2647 const char __user *buf, int in_len, int out_len) 2648 { 2649 struct ib_uverbs_destroy_ah cmd; 2650 2651 if (copy_from_user(&cmd, buf, sizeof cmd)) 2652 return -EFAULT; 2653 2654 return uobj_perform_destroy(UVERBS_OBJECT_AH, cmd.ah_handle, file, 2655 in_len); 2656 } 2657 2658 ssize_t ib_uverbs_attach_mcast(struct ib_uverbs_file *file, 2659 const char __user *buf, int in_len, 2660 int out_len) 2661 { 2662 struct ib_uverbs_attach_mcast cmd; 2663 struct ib_qp *qp; 2664 struct ib_uqp_object *obj; 2665 struct ib_uverbs_mcast_entry *mcast; 2666 int ret; 2667 2668 if (copy_from_user(&cmd, buf, sizeof cmd)) 2669 return -EFAULT; 2670 2671 qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, file); 2672 if (!qp) 2673 return -EINVAL; 2674 2675 obj = container_of(qp->uobject, struct ib_uqp_object, uevent.uobject); 2676 2677 mutex_lock(&obj->mcast_lock); 2678 list_for_each_entry(mcast, &obj->mcast_list, list) 2679 if (cmd.mlid == mcast->lid && 2680 !memcmp(cmd.gid, mcast->gid.raw, sizeof mcast->gid.raw)) { 2681 ret = 0; 2682 goto out_put; 2683 } 2684 2685 mcast = kmalloc(sizeof *mcast, GFP_KERNEL); 2686 if (!mcast) { 2687 ret = -ENOMEM; 2688 goto out_put; 2689 } 2690 2691 mcast->lid = cmd.mlid; 2692 memcpy(mcast->gid.raw, cmd.gid, sizeof mcast->gid.raw); 2693 2694 ret = ib_attach_mcast(qp, &mcast->gid, cmd.mlid); 2695 if (!ret) 2696 list_add_tail(&mcast->list, &obj->mcast_list); 2697 else 2698 kfree(mcast); 2699 2700 out_put: 2701 mutex_unlock(&obj->mcast_lock); 2702 uobj_put_obj_read(qp); 2703 2704 return ret ? ret : in_len; 2705 } 2706 2707 ssize_t ib_uverbs_detach_mcast(struct ib_uverbs_file *file, 2708 const char __user *buf, int in_len, 2709 int out_len) 2710 { 2711 struct ib_uverbs_detach_mcast cmd; 2712 struct ib_uqp_object *obj; 2713 struct ib_qp *qp; 2714 struct ib_uverbs_mcast_entry *mcast; 2715 int ret = -EINVAL; 2716 bool found = false; 2717 2718 if (copy_from_user(&cmd, buf, sizeof cmd)) 2719 return -EFAULT; 2720 2721 qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, file); 2722 if (!qp) 2723 return -EINVAL; 2724 2725 obj = container_of(qp->uobject, struct ib_uqp_object, uevent.uobject); 2726 mutex_lock(&obj->mcast_lock); 2727 2728 list_for_each_entry(mcast, &obj->mcast_list, list) 2729 if (cmd.mlid == mcast->lid && 2730 !memcmp(cmd.gid, mcast->gid.raw, sizeof mcast->gid.raw)) { 2731 list_del(&mcast->list); 2732 kfree(mcast); 2733 found = true; 2734 break; 2735 } 2736 2737 if (!found) { 2738 ret = -EINVAL; 2739 goto out_put; 2740 } 2741 2742 ret = ib_detach_mcast(qp, (union ib_gid *)cmd.gid, cmd.mlid); 2743 2744 out_put: 2745 mutex_unlock(&obj->mcast_lock); 2746 uobj_put_obj_read(qp); 2747 return ret ? ret : in_len; 2748 } 2749 2750 struct ib_uflow_resources { 2751 size_t max; 2752 size_t num; 2753 size_t collection_num; 2754 size_t counters_num; 2755 struct ib_counters **counters; 2756 struct ib_flow_action **collection; 2757 }; 2758 2759 static struct ib_uflow_resources *flow_resources_alloc(size_t num_specs) 2760 { 2761 struct ib_uflow_resources *resources; 2762 2763 resources = kzalloc(sizeof(*resources), GFP_KERNEL); 2764 2765 if (!resources) 2766 return NULL; 2767 2768 if (!num_specs) 2769 goto out; 2770 2771 resources->counters = 2772 kcalloc(num_specs, sizeof(*resources->counters), GFP_KERNEL); 2773 resources->collection = 2774 kcalloc(num_specs, sizeof(*resources->collection), GFP_KERNEL); 2775 2776 if (!resources->counters || !resources->collection) 2777 goto err; 2778 2779 out: 2780 resources->max = num_specs; 2781 return resources; 2782 2783 err: 2784 kfree(resources->counters); 2785 kfree(resources); 2786 2787 return NULL; 2788 } 2789 2790 void ib_uverbs_flow_resources_free(struct ib_uflow_resources *uflow_res) 2791 { 2792 unsigned int i; 2793 2794 if (!uflow_res) 2795 return; 2796 2797 for (i = 0; i < uflow_res->collection_num; i++) 2798 atomic_dec(&uflow_res->collection[i]->usecnt); 2799 2800 for (i = 0; i < uflow_res->counters_num; i++) 2801 atomic_dec(&uflow_res->counters[i]->usecnt); 2802 2803 kfree(uflow_res->collection); 2804 kfree(uflow_res->counters); 2805 kfree(uflow_res); 2806 } 2807 2808 static void flow_resources_add(struct ib_uflow_resources *uflow_res, 2809 enum ib_flow_spec_type type, 2810 void *ibobj) 2811 { 2812 WARN_ON(uflow_res->num >= uflow_res->max); 2813 2814 switch (type) { 2815 case IB_FLOW_SPEC_ACTION_HANDLE: 2816 atomic_inc(&((struct ib_flow_action *)ibobj)->usecnt); 2817 uflow_res->collection[uflow_res->collection_num++] = 2818 (struct ib_flow_action *)ibobj; 2819 break; 2820 case IB_FLOW_SPEC_ACTION_COUNT: 2821 atomic_inc(&((struct ib_counters *)ibobj)->usecnt); 2822 uflow_res->counters[uflow_res->counters_num++] = 2823 (struct ib_counters *)ibobj; 2824 break; 2825 default: 2826 WARN_ON(1); 2827 } 2828 2829 uflow_res->num++; 2830 } 2831 2832 static int kern_spec_to_ib_spec_action(struct ib_uverbs_file *ufile, 2833 struct ib_uverbs_flow_spec *kern_spec, 2834 union ib_flow_spec *ib_spec, 2835 struct ib_uflow_resources *uflow_res) 2836 { 2837 ib_spec->type = kern_spec->type; 2838 switch (ib_spec->type) { 2839 case IB_FLOW_SPEC_ACTION_TAG: 2840 if (kern_spec->flow_tag.size != 2841 sizeof(struct ib_uverbs_flow_spec_action_tag)) 2842 return -EINVAL; 2843 2844 ib_spec->flow_tag.size = sizeof(struct ib_flow_spec_action_tag); 2845 ib_spec->flow_tag.tag_id = kern_spec->flow_tag.tag_id; 2846 break; 2847 case IB_FLOW_SPEC_ACTION_DROP: 2848 if (kern_spec->drop.size != 2849 sizeof(struct ib_uverbs_flow_spec_action_drop)) 2850 return -EINVAL; 2851 2852 ib_spec->drop.size = sizeof(struct ib_flow_spec_action_drop); 2853 break; 2854 case IB_FLOW_SPEC_ACTION_HANDLE: 2855 if (kern_spec->action.size != 2856 sizeof(struct ib_uverbs_flow_spec_action_handle)) 2857 return -EOPNOTSUPP; 2858 ib_spec->action.act = uobj_get_obj_read(flow_action, 2859 UVERBS_OBJECT_FLOW_ACTION, 2860 kern_spec->action.handle, 2861 ufile); 2862 if (!ib_spec->action.act) 2863 return -EINVAL; 2864 ib_spec->action.size = 2865 sizeof(struct ib_flow_spec_action_handle); 2866 flow_resources_add(uflow_res, 2867 IB_FLOW_SPEC_ACTION_HANDLE, 2868 ib_spec->action.act); 2869 uobj_put_obj_read(ib_spec->action.act); 2870 break; 2871 case IB_FLOW_SPEC_ACTION_COUNT: 2872 if (kern_spec->flow_count.size != 2873 sizeof(struct ib_uverbs_flow_spec_action_count)) 2874 return -EINVAL; 2875 ib_spec->flow_count.counters = 2876 uobj_get_obj_read(counters, 2877 UVERBS_OBJECT_COUNTERS, 2878 kern_spec->flow_count.handle, 2879 ufile); 2880 if (!ib_spec->flow_count.counters) 2881 return -EINVAL; 2882 ib_spec->flow_count.size = 2883 sizeof(struct ib_flow_spec_action_count); 2884 flow_resources_add(uflow_res, 2885 IB_FLOW_SPEC_ACTION_COUNT, 2886 ib_spec->flow_count.counters); 2887 uobj_put_obj_read(ib_spec->flow_count.counters); 2888 break; 2889 default: 2890 return -EINVAL; 2891 } 2892 return 0; 2893 } 2894 2895 static size_t kern_spec_filter_sz(const struct ib_uverbs_flow_spec_hdr *spec) 2896 { 2897 /* Returns user space filter size, includes padding */ 2898 return (spec->size - sizeof(struct ib_uverbs_flow_spec_hdr)) / 2; 2899 } 2900 2901 static ssize_t spec_filter_size(const void *kern_spec_filter, u16 kern_filter_size, 2902 u16 ib_real_filter_sz) 2903 { 2904 /* 2905 * User space filter structures must be 64 bit aligned, otherwise this 2906 * may pass, but we won't handle additional new attributes. 2907 */ 2908 2909 if (kern_filter_size > ib_real_filter_sz) { 2910 if (memchr_inv(kern_spec_filter + 2911 ib_real_filter_sz, 0, 2912 kern_filter_size - ib_real_filter_sz)) 2913 return -EINVAL; 2914 return ib_real_filter_sz; 2915 } 2916 return kern_filter_size; 2917 } 2918 2919 int ib_uverbs_kern_spec_to_ib_spec_filter(enum ib_flow_spec_type type, 2920 const void *kern_spec_mask, 2921 const void *kern_spec_val, 2922 size_t kern_filter_sz, 2923 union ib_flow_spec *ib_spec) 2924 { 2925 ssize_t actual_filter_sz; 2926 ssize_t ib_filter_sz; 2927 2928 /* User flow spec size must be aligned to 4 bytes */ 2929 if (kern_filter_sz != ALIGN(kern_filter_sz, 4)) 2930 return -EINVAL; 2931 2932 ib_spec->type = type; 2933 2934 if (ib_spec->type == (IB_FLOW_SPEC_INNER | IB_FLOW_SPEC_VXLAN_TUNNEL)) 2935 return -EINVAL; 2936 2937 switch (ib_spec->type & ~IB_FLOW_SPEC_INNER) { 2938 case IB_FLOW_SPEC_ETH: 2939 ib_filter_sz = offsetof(struct ib_flow_eth_filter, real_sz); 2940 actual_filter_sz = spec_filter_size(kern_spec_mask, 2941 kern_filter_sz, 2942 ib_filter_sz); 2943 if (actual_filter_sz <= 0) 2944 return -EINVAL; 2945 ib_spec->size = sizeof(struct ib_flow_spec_eth); 2946 memcpy(&ib_spec->eth.val, kern_spec_val, actual_filter_sz); 2947 memcpy(&ib_spec->eth.mask, kern_spec_mask, actual_filter_sz); 2948 break; 2949 case IB_FLOW_SPEC_IPV4: 2950 ib_filter_sz = offsetof(struct ib_flow_ipv4_filter, real_sz); 2951 actual_filter_sz = spec_filter_size(kern_spec_mask, 2952 kern_filter_sz, 2953 ib_filter_sz); 2954 if (actual_filter_sz <= 0) 2955 return -EINVAL; 2956 ib_spec->size = sizeof(struct ib_flow_spec_ipv4); 2957 memcpy(&ib_spec->ipv4.val, kern_spec_val, actual_filter_sz); 2958 memcpy(&ib_spec->ipv4.mask, kern_spec_mask, actual_filter_sz); 2959 break; 2960 case IB_FLOW_SPEC_IPV6: 2961 ib_filter_sz = offsetof(struct ib_flow_ipv6_filter, real_sz); 2962 actual_filter_sz = spec_filter_size(kern_spec_mask, 2963 kern_filter_sz, 2964 ib_filter_sz); 2965 if (actual_filter_sz <= 0) 2966 return -EINVAL; 2967 ib_spec->size = sizeof(struct ib_flow_spec_ipv6); 2968 memcpy(&ib_spec->ipv6.val, kern_spec_val, actual_filter_sz); 2969 memcpy(&ib_spec->ipv6.mask, kern_spec_mask, actual_filter_sz); 2970 2971 if ((ntohl(ib_spec->ipv6.mask.flow_label)) >= BIT(20) || 2972 (ntohl(ib_spec->ipv6.val.flow_label)) >= BIT(20)) 2973 return -EINVAL; 2974 break; 2975 case IB_FLOW_SPEC_TCP: 2976 case IB_FLOW_SPEC_UDP: 2977 ib_filter_sz = offsetof(struct ib_flow_tcp_udp_filter, real_sz); 2978 actual_filter_sz = spec_filter_size(kern_spec_mask, 2979 kern_filter_sz, 2980 ib_filter_sz); 2981 if (actual_filter_sz <= 0) 2982 return -EINVAL; 2983 ib_spec->size = sizeof(struct ib_flow_spec_tcp_udp); 2984 memcpy(&ib_spec->tcp_udp.val, kern_spec_val, actual_filter_sz); 2985 memcpy(&ib_spec->tcp_udp.mask, kern_spec_mask, actual_filter_sz); 2986 break; 2987 case IB_FLOW_SPEC_VXLAN_TUNNEL: 2988 ib_filter_sz = offsetof(struct ib_flow_tunnel_filter, real_sz); 2989 actual_filter_sz = spec_filter_size(kern_spec_mask, 2990 kern_filter_sz, 2991 ib_filter_sz); 2992 if (actual_filter_sz <= 0) 2993 return -EINVAL; 2994 ib_spec->tunnel.size = sizeof(struct ib_flow_spec_tunnel); 2995 memcpy(&ib_spec->tunnel.val, kern_spec_val, actual_filter_sz); 2996 memcpy(&ib_spec->tunnel.mask, kern_spec_mask, actual_filter_sz); 2997 2998 if ((ntohl(ib_spec->tunnel.mask.tunnel_id)) >= BIT(24) || 2999 (ntohl(ib_spec->tunnel.val.tunnel_id)) >= BIT(24)) 3000 return -EINVAL; 3001 break; 3002 case IB_FLOW_SPEC_ESP: 3003 ib_filter_sz = offsetof(struct ib_flow_esp_filter, real_sz); 3004 actual_filter_sz = spec_filter_size(kern_spec_mask, 3005 kern_filter_sz, 3006 ib_filter_sz); 3007 if (actual_filter_sz <= 0) 3008 return -EINVAL; 3009 ib_spec->esp.size = sizeof(struct ib_flow_spec_esp); 3010 memcpy(&ib_spec->esp.val, kern_spec_val, actual_filter_sz); 3011 memcpy(&ib_spec->esp.mask, kern_spec_mask, actual_filter_sz); 3012 break; 3013 case IB_FLOW_SPEC_GRE: 3014 ib_filter_sz = offsetof(struct ib_flow_gre_filter, real_sz); 3015 actual_filter_sz = spec_filter_size(kern_spec_mask, 3016 kern_filter_sz, 3017 ib_filter_sz); 3018 if (actual_filter_sz <= 0) 3019 return -EINVAL; 3020 ib_spec->gre.size = sizeof(struct ib_flow_spec_gre); 3021 memcpy(&ib_spec->gre.val, kern_spec_val, actual_filter_sz); 3022 memcpy(&ib_spec->gre.mask, kern_spec_mask, actual_filter_sz); 3023 break; 3024 case IB_FLOW_SPEC_MPLS: 3025 ib_filter_sz = offsetof(struct ib_flow_mpls_filter, real_sz); 3026 actual_filter_sz = spec_filter_size(kern_spec_mask, 3027 kern_filter_sz, 3028 ib_filter_sz); 3029 if (actual_filter_sz <= 0) 3030 return -EINVAL; 3031 ib_spec->mpls.size = sizeof(struct ib_flow_spec_mpls); 3032 memcpy(&ib_spec->mpls.val, kern_spec_val, actual_filter_sz); 3033 memcpy(&ib_spec->mpls.mask, kern_spec_mask, actual_filter_sz); 3034 break; 3035 default: 3036 return -EINVAL; 3037 } 3038 return 0; 3039 } 3040 3041 static int kern_spec_to_ib_spec_filter(struct ib_uverbs_flow_spec *kern_spec, 3042 union ib_flow_spec *ib_spec) 3043 { 3044 ssize_t kern_filter_sz; 3045 void *kern_spec_mask; 3046 void *kern_spec_val; 3047 3048 kern_filter_sz = kern_spec_filter_sz(&kern_spec->hdr); 3049 3050 kern_spec_val = (void *)kern_spec + 3051 sizeof(struct ib_uverbs_flow_spec_hdr); 3052 kern_spec_mask = kern_spec_val + kern_filter_sz; 3053 3054 return ib_uverbs_kern_spec_to_ib_spec_filter(kern_spec->type, 3055 kern_spec_mask, 3056 kern_spec_val, 3057 kern_filter_sz, ib_spec); 3058 } 3059 3060 static int kern_spec_to_ib_spec(struct ib_uverbs_file *ufile, 3061 struct ib_uverbs_flow_spec *kern_spec, 3062 union ib_flow_spec *ib_spec, 3063 struct ib_uflow_resources *uflow_res) 3064 { 3065 if (kern_spec->reserved) 3066 return -EINVAL; 3067 3068 if (kern_spec->type >= IB_FLOW_SPEC_ACTION_TAG) 3069 return kern_spec_to_ib_spec_action(ufile, kern_spec, ib_spec, 3070 uflow_res); 3071 else 3072 return kern_spec_to_ib_spec_filter(kern_spec, ib_spec); 3073 } 3074 3075 int ib_uverbs_ex_create_wq(struct ib_uverbs_file *file, 3076 struct ib_udata *ucore, 3077 struct ib_udata *uhw) 3078 { 3079 struct ib_uverbs_ex_create_wq cmd = {}; 3080 struct ib_uverbs_ex_create_wq_resp resp = {}; 3081 struct ib_uwq_object *obj; 3082 int err = 0; 3083 struct ib_cq *cq; 3084 struct ib_pd *pd; 3085 struct ib_wq *wq; 3086 struct ib_wq_init_attr wq_init_attr = {}; 3087 size_t required_cmd_sz; 3088 size_t required_resp_len; 3089 struct ib_device *ib_dev; 3090 3091 required_cmd_sz = offsetof(typeof(cmd), max_sge) + sizeof(cmd.max_sge); 3092 required_resp_len = offsetof(typeof(resp), wqn) + sizeof(resp.wqn); 3093 3094 if (ucore->inlen < required_cmd_sz) 3095 return -EINVAL; 3096 3097 if (ucore->outlen < required_resp_len) 3098 return -ENOSPC; 3099 3100 if (ucore->inlen > sizeof(cmd) && 3101 !ib_is_udata_cleared(ucore, sizeof(cmd), 3102 ucore->inlen - sizeof(cmd))) 3103 return -EOPNOTSUPP; 3104 3105 err = ib_copy_from_udata(&cmd, ucore, min(sizeof(cmd), ucore->inlen)); 3106 if (err) 3107 return err; 3108 3109 if (cmd.comp_mask) 3110 return -EOPNOTSUPP; 3111 3112 obj = (struct ib_uwq_object *)uobj_alloc(UVERBS_OBJECT_WQ, file, 3113 &ib_dev); 3114 if (IS_ERR(obj)) 3115 return PTR_ERR(obj); 3116 3117 pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, file); 3118 if (!pd) { 3119 err = -EINVAL; 3120 goto err_uobj; 3121 } 3122 3123 cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, file); 3124 if (!cq) { 3125 err = -EINVAL; 3126 goto err_put_pd; 3127 } 3128 3129 wq_init_attr.cq = cq; 3130 wq_init_attr.max_sge = cmd.max_sge; 3131 wq_init_attr.max_wr = cmd.max_wr; 3132 wq_init_attr.wq_context = file; 3133 wq_init_attr.wq_type = cmd.wq_type; 3134 wq_init_attr.event_handler = ib_uverbs_wq_event_handler; 3135 if (ucore->inlen >= (offsetof(typeof(cmd), create_flags) + 3136 sizeof(cmd.create_flags))) 3137 wq_init_attr.create_flags = cmd.create_flags; 3138 obj->uevent.events_reported = 0; 3139 INIT_LIST_HEAD(&obj->uevent.event_list); 3140 3141 if (!pd->device->create_wq) { 3142 err = -EOPNOTSUPP; 3143 goto err_put_cq; 3144 } 3145 wq = pd->device->create_wq(pd, &wq_init_attr, uhw); 3146 if (IS_ERR(wq)) { 3147 err = PTR_ERR(wq); 3148 goto err_put_cq; 3149 } 3150 3151 wq->uobject = &obj->uevent.uobject; 3152 obj->uevent.uobject.object = wq; 3153 wq->wq_type = wq_init_attr.wq_type; 3154 wq->cq = cq; 3155 wq->pd = pd; 3156 wq->device = pd->device; 3157 wq->wq_context = wq_init_attr.wq_context; 3158 atomic_set(&wq->usecnt, 0); 3159 atomic_inc(&pd->usecnt); 3160 atomic_inc(&cq->usecnt); 3161 wq->uobject = &obj->uevent.uobject; 3162 obj->uevent.uobject.object = wq; 3163 3164 memset(&resp, 0, sizeof(resp)); 3165 resp.wq_handle = obj->uevent.uobject.id; 3166 resp.max_sge = wq_init_attr.max_sge; 3167 resp.max_wr = wq_init_attr.max_wr; 3168 resp.wqn = wq->wq_num; 3169 resp.response_length = required_resp_len; 3170 err = ib_copy_to_udata(ucore, 3171 &resp, resp.response_length); 3172 if (err) 3173 goto err_copy; 3174 3175 uobj_put_obj_read(pd); 3176 uobj_put_obj_read(cq); 3177 return uobj_alloc_commit(&obj->uevent.uobject, 0); 3178 3179 err_copy: 3180 ib_destroy_wq(wq); 3181 err_put_cq: 3182 uobj_put_obj_read(cq); 3183 err_put_pd: 3184 uobj_put_obj_read(pd); 3185 err_uobj: 3186 uobj_alloc_abort(&obj->uevent.uobject); 3187 3188 return err; 3189 } 3190 3191 int ib_uverbs_ex_destroy_wq(struct ib_uverbs_file *file, 3192 struct ib_udata *ucore, 3193 struct ib_udata *uhw) 3194 { 3195 struct ib_uverbs_ex_destroy_wq cmd = {}; 3196 struct ib_uverbs_ex_destroy_wq_resp resp = {}; 3197 struct ib_uobject *uobj; 3198 struct ib_uwq_object *obj; 3199 size_t required_cmd_sz; 3200 size_t required_resp_len; 3201 int ret; 3202 3203 required_cmd_sz = offsetof(typeof(cmd), wq_handle) + sizeof(cmd.wq_handle); 3204 required_resp_len = offsetof(typeof(resp), reserved) + sizeof(resp.reserved); 3205 3206 if (ucore->inlen < required_cmd_sz) 3207 return -EINVAL; 3208 3209 if (ucore->outlen < required_resp_len) 3210 return -ENOSPC; 3211 3212 if (ucore->inlen > sizeof(cmd) && 3213 !ib_is_udata_cleared(ucore, sizeof(cmd), 3214 ucore->inlen - sizeof(cmd))) 3215 return -EOPNOTSUPP; 3216 3217 ret = ib_copy_from_udata(&cmd, ucore, min(sizeof(cmd), ucore->inlen)); 3218 if (ret) 3219 return ret; 3220 3221 if (cmd.comp_mask) 3222 return -EOPNOTSUPP; 3223 3224 resp.response_length = required_resp_len; 3225 uobj = uobj_get_destroy(UVERBS_OBJECT_WQ, cmd.wq_handle, file); 3226 if (IS_ERR(uobj)) 3227 return PTR_ERR(uobj); 3228 3229 obj = container_of(uobj, struct ib_uwq_object, uevent.uobject); 3230 resp.events_reported = obj->uevent.events_reported; 3231 3232 uobj_put_destroy(uobj); 3233 3234 return ib_copy_to_udata(ucore, &resp, resp.response_length); 3235 } 3236 3237 int ib_uverbs_ex_modify_wq(struct ib_uverbs_file *file, 3238 struct ib_udata *ucore, 3239 struct ib_udata *uhw) 3240 { 3241 struct ib_uverbs_ex_modify_wq cmd = {}; 3242 struct ib_wq *wq; 3243 struct ib_wq_attr wq_attr = {}; 3244 size_t required_cmd_sz; 3245 int ret; 3246 3247 required_cmd_sz = offsetof(typeof(cmd), curr_wq_state) + sizeof(cmd.curr_wq_state); 3248 if (ucore->inlen < required_cmd_sz) 3249 return -EINVAL; 3250 3251 if (ucore->inlen > sizeof(cmd) && 3252 !ib_is_udata_cleared(ucore, sizeof(cmd), 3253 ucore->inlen - sizeof(cmd))) 3254 return -EOPNOTSUPP; 3255 3256 ret = ib_copy_from_udata(&cmd, ucore, min(sizeof(cmd), ucore->inlen)); 3257 if (ret) 3258 return ret; 3259 3260 if (!cmd.attr_mask) 3261 return -EINVAL; 3262 3263 if (cmd.attr_mask > (IB_WQ_STATE | IB_WQ_CUR_STATE | IB_WQ_FLAGS)) 3264 return -EINVAL; 3265 3266 wq = uobj_get_obj_read(wq, UVERBS_OBJECT_WQ, cmd.wq_handle, file); 3267 if (!wq) 3268 return -EINVAL; 3269 3270 wq_attr.curr_wq_state = cmd.curr_wq_state; 3271 wq_attr.wq_state = cmd.wq_state; 3272 if (cmd.attr_mask & IB_WQ_FLAGS) { 3273 wq_attr.flags = cmd.flags; 3274 wq_attr.flags_mask = cmd.flags_mask; 3275 } 3276 if (!wq->device->modify_wq) { 3277 ret = -EOPNOTSUPP; 3278 goto out; 3279 } 3280 ret = wq->device->modify_wq(wq, &wq_attr, cmd.attr_mask, uhw); 3281 out: 3282 uobj_put_obj_read(wq); 3283 return ret; 3284 } 3285 3286 int ib_uverbs_ex_create_rwq_ind_table(struct ib_uverbs_file *file, 3287 struct ib_udata *ucore, 3288 struct ib_udata *uhw) 3289 { 3290 struct ib_uverbs_ex_create_rwq_ind_table cmd = {}; 3291 struct ib_uverbs_ex_create_rwq_ind_table_resp resp = {}; 3292 struct ib_uobject *uobj; 3293 int err = 0; 3294 struct ib_rwq_ind_table_init_attr init_attr = {}; 3295 struct ib_rwq_ind_table *rwq_ind_tbl; 3296 struct ib_wq **wqs = NULL; 3297 u32 *wqs_handles = NULL; 3298 struct ib_wq *wq = NULL; 3299 int i, j, num_read_wqs; 3300 u32 num_wq_handles; 3301 u32 expected_in_size; 3302 size_t required_cmd_sz_header; 3303 size_t required_resp_len; 3304 struct ib_device *ib_dev; 3305 3306 required_cmd_sz_header = offsetof(typeof(cmd), log_ind_tbl_size) + sizeof(cmd.log_ind_tbl_size); 3307 required_resp_len = offsetof(typeof(resp), ind_tbl_num) + sizeof(resp.ind_tbl_num); 3308 3309 if (ucore->inlen < required_cmd_sz_header) 3310 return -EINVAL; 3311 3312 if (ucore->outlen < required_resp_len) 3313 return -ENOSPC; 3314 3315 err = ib_copy_from_udata(&cmd, ucore, required_cmd_sz_header); 3316 if (err) 3317 return err; 3318 3319 ucore->inbuf += required_cmd_sz_header; 3320 ucore->inlen -= required_cmd_sz_header; 3321 3322 if (cmd.comp_mask) 3323 return -EOPNOTSUPP; 3324 3325 if (cmd.log_ind_tbl_size > IB_USER_VERBS_MAX_LOG_IND_TBL_SIZE) 3326 return -EINVAL; 3327 3328 num_wq_handles = 1 << cmd.log_ind_tbl_size; 3329 expected_in_size = num_wq_handles * sizeof(__u32); 3330 if (num_wq_handles == 1) 3331 /* input size for wq handles is u64 aligned */ 3332 expected_in_size += sizeof(__u32); 3333 3334 if (ucore->inlen < expected_in_size) 3335 return -EINVAL; 3336 3337 if (ucore->inlen > expected_in_size && 3338 !ib_is_udata_cleared(ucore, expected_in_size, 3339 ucore->inlen - expected_in_size)) 3340 return -EOPNOTSUPP; 3341 3342 wqs_handles = kcalloc(num_wq_handles, sizeof(*wqs_handles), 3343 GFP_KERNEL); 3344 if (!wqs_handles) 3345 return -ENOMEM; 3346 3347 err = ib_copy_from_udata(wqs_handles, ucore, 3348 num_wq_handles * sizeof(__u32)); 3349 if (err) 3350 goto err_free; 3351 3352 wqs = kcalloc(num_wq_handles, sizeof(*wqs), GFP_KERNEL); 3353 if (!wqs) { 3354 err = -ENOMEM; 3355 goto err_free; 3356 } 3357 3358 for (num_read_wqs = 0; num_read_wqs < num_wq_handles; 3359 num_read_wqs++) { 3360 wq = uobj_get_obj_read(wq, UVERBS_OBJECT_WQ, 3361 wqs_handles[num_read_wqs], file); 3362 if (!wq) { 3363 err = -EINVAL; 3364 goto put_wqs; 3365 } 3366 3367 wqs[num_read_wqs] = wq; 3368 } 3369 3370 uobj = uobj_alloc(UVERBS_OBJECT_RWQ_IND_TBL, file, &ib_dev); 3371 if (IS_ERR(uobj)) { 3372 err = PTR_ERR(uobj); 3373 goto put_wqs; 3374 } 3375 3376 init_attr.log_ind_tbl_size = cmd.log_ind_tbl_size; 3377 init_attr.ind_tbl = wqs; 3378 3379 if (!ib_dev->create_rwq_ind_table) { 3380 err = -EOPNOTSUPP; 3381 goto err_uobj; 3382 } 3383 rwq_ind_tbl = ib_dev->create_rwq_ind_table(ib_dev, &init_attr, uhw); 3384 3385 if (IS_ERR(rwq_ind_tbl)) { 3386 err = PTR_ERR(rwq_ind_tbl); 3387 goto err_uobj; 3388 } 3389 3390 rwq_ind_tbl->ind_tbl = wqs; 3391 rwq_ind_tbl->log_ind_tbl_size = init_attr.log_ind_tbl_size; 3392 rwq_ind_tbl->uobject = uobj; 3393 uobj->object = rwq_ind_tbl; 3394 rwq_ind_tbl->device = ib_dev; 3395 atomic_set(&rwq_ind_tbl->usecnt, 0); 3396 3397 for (i = 0; i < num_wq_handles; i++) 3398 atomic_inc(&wqs[i]->usecnt); 3399 3400 resp.ind_tbl_handle = uobj->id; 3401 resp.ind_tbl_num = rwq_ind_tbl->ind_tbl_num; 3402 resp.response_length = required_resp_len; 3403 3404 err = ib_copy_to_udata(ucore, 3405 &resp, resp.response_length); 3406 if (err) 3407 goto err_copy; 3408 3409 kfree(wqs_handles); 3410 3411 for (j = 0; j < num_read_wqs; j++) 3412 uobj_put_obj_read(wqs[j]); 3413 3414 return uobj_alloc_commit(uobj, 0); 3415 3416 err_copy: 3417 ib_destroy_rwq_ind_table(rwq_ind_tbl); 3418 err_uobj: 3419 uobj_alloc_abort(uobj); 3420 put_wqs: 3421 for (j = 0; j < num_read_wqs; j++) 3422 uobj_put_obj_read(wqs[j]); 3423 err_free: 3424 kfree(wqs_handles); 3425 kfree(wqs); 3426 return err; 3427 } 3428 3429 int ib_uverbs_ex_destroy_rwq_ind_table(struct ib_uverbs_file *file, 3430 struct ib_udata *ucore, 3431 struct ib_udata *uhw) 3432 { 3433 struct ib_uverbs_ex_destroy_rwq_ind_table cmd = {}; 3434 int ret; 3435 size_t required_cmd_sz; 3436 3437 required_cmd_sz = offsetof(typeof(cmd), ind_tbl_handle) + sizeof(cmd.ind_tbl_handle); 3438 3439 if (ucore->inlen < required_cmd_sz) 3440 return -EINVAL; 3441 3442 if (ucore->inlen > sizeof(cmd) && 3443 !ib_is_udata_cleared(ucore, sizeof(cmd), 3444 ucore->inlen - sizeof(cmd))) 3445 return -EOPNOTSUPP; 3446 3447 ret = ib_copy_from_udata(&cmd, ucore, min(sizeof(cmd), ucore->inlen)); 3448 if (ret) 3449 return ret; 3450 3451 if (cmd.comp_mask) 3452 return -EOPNOTSUPP; 3453 3454 return uobj_perform_destroy(UVERBS_OBJECT_RWQ_IND_TBL, 3455 cmd.ind_tbl_handle, file, 0); 3456 } 3457 3458 int ib_uverbs_ex_create_flow(struct ib_uverbs_file *file, 3459 struct ib_udata *ucore, 3460 struct ib_udata *uhw) 3461 { 3462 struct ib_uverbs_create_flow cmd; 3463 struct ib_uverbs_create_flow_resp resp; 3464 struct ib_uobject *uobj; 3465 struct ib_uflow_object *uflow; 3466 struct ib_flow *flow_id; 3467 struct ib_uverbs_flow_attr *kern_flow_attr; 3468 struct ib_flow_attr *flow_attr; 3469 struct ib_qp *qp; 3470 struct ib_uflow_resources *uflow_res; 3471 struct ib_uverbs_flow_spec_hdr *kern_spec; 3472 int err = 0; 3473 void *ib_spec; 3474 int i; 3475 struct ib_device *ib_dev; 3476 3477 if (ucore->inlen < sizeof(cmd)) 3478 return -EINVAL; 3479 3480 if (ucore->outlen < sizeof(resp)) 3481 return -ENOSPC; 3482 3483 err = ib_copy_from_udata(&cmd, ucore, sizeof(cmd)); 3484 if (err) 3485 return err; 3486 3487 ucore->inbuf += sizeof(cmd); 3488 ucore->inlen -= sizeof(cmd); 3489 3490 if (cmd.comp_mask) 3491 return -EINVAL; 3492 3493 if (!capable(CAP_NET_RAW)) 3494 return -EPERM; 3495 3496 if (cmd.flow_attr.flags >= IB_FLOW_ATTR_FLAGS_RESERVED) 3497 return -EINVAL; 3498 3499 if ((cmd.flow_attr.flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP) && 3500 ((cmd.flow_attr.type == IB_FLOW_ATTR_ALL_DEFAULT) || 3501 (cmd.flow_attr.type == IB_FLOW_ATTR_MC_DEFAULT))) 3502 return -EINVAL; 3503 3504 if (cmd.flow_attr.num_of_specs > IB_FLOW_SPEC_SUPPORT_LAYERS) 3505 return -EINVAL; 3506 3507 if (cmd.flow_attr.size > ucore->inlen || 3508 cmd.flow_attr.size > 3509 (cmd.flow_attr.num_of_specs * sizeof(struct ib_uverbs_flow_spec))) 3510 return -EINVAL; 3511 3512 if (cmd.flow_attr.reserved[0] || 3513 cmd.flow_attr.reserved[1]) 3514 return -EINVAL; 3515 3516 if (cmd.flow_attr.num_of_specs) { 3517 kern_flow_attr = kmalloc(sizeof(*kern_flow_attr) + cmd.flow_attr.size, 3518 GFP_KERNEL); 3519 if (!kern_flow_attr) 3520 return -ENOMEM; 3521 3522 *kern_flow_attr = cmd.flow_attr; 3523 err = ib_copy_from_udata(&kern_flow_attr->flow_specs, ucore, 3524 cmd.flow_attr.size); 3525 if (err) 3526 goto err_free_attr; 3527 } else { 3528 kern_flow_attr = &cmd.flow_attr; 3529 } 3530 3531 uobj = uobj_alloc(UVERBS_OBJECT_FLOW, file, &ib_dev); 3532 if (IS_ERR(uobj)) { 3533 err = PTR_ERR(uobj); 3534 goto err_free_attr; 3535 } 3536 3537 qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, file); 3538 if (!qp) { 3539 err = -EINVAL; 3540 goto err_uobj; 3541 } 3542 3543 if (qp->qp_type != IB_QPT_UD && qp->qp_type != IB_QPT_RAW_PACKET) { 3544 err = -EINVAL; 3545 goto err_put; 3546 } 3547 3548 if (!qp->device->create_flow) { 3549 err = -EOPNOTSUPP; 3550 goto err_put; 3551 } 3552 3553 flow_attr = kzalloc(struct_size(flow_attr, flows, 3554 cmd.flow_attr.num_of_specs), GFP_KERNEL); 3555 if (!flow_attr) { 3556 err = -ENOMEM; 3557 goto err_put; 3558 } 3559 uflow_res = flow_resources_alloc(cmd.flow_attr.num_of_specs); 3560 if (!uflow_res) { 3561 err = -ENOMEM; 3562 goto err_free_flow_attr; 3563 } 3564 3565 flow_attr->type = kern_flow_attr->type; 3566 flow_attr->priority = kern_flow_attr->priority; 3567 flow_attr->num_of_specs = kern_flow_attr->num_of_specs; 3568 flow_attr->port = kern_flow_attr->port; 3569 flow_attr->flags = kern_flow_attr->flags; 3570 flow_attr->size = sizeof(*flow_attr); 3571 3572 kern_spec = kern_flow_attr->flow_specs; 3573 ib_spec = flow_attr + 1; 3574 for (i = 0; i < flow_attr->num_of_specs && 3575 cmd.flow_attr.size >= sizeof(*kern_spec) && 3576 cmd.flow_attr.size >= kern_spec->size; 3577 i++) { 3578 err = kern_spec_to_ib_spec( 3579 file, (struct ib_uverbs_flow_spec *)kern_spec, 3580 ib_spec, uflow_res); 3581 if (err) 3582 goto err_free; 3583 3584 flow_attr->size += 3585 ((union ib_flow_spec *) ib_spec)->size; 3586 cmd.flow_attr.size -= kern_spec->size; 3587 kern_spec = ((void *)kern_spec) + kern_spec->size; 3588 ib_spec += ((union ib_flow_spec *) ib_spec)->size; 3589 } 3590 if (cmd.flow_attr.size || (i != flow_attr->num_of_specs)) { 3591 pr_warn("create flow failed, flow %d: %d bytes left from uverb cmd\n", 3592 i, cmd.flow_attr.size); 3593 err = -EINVAL; 3594 goto err_free; 3595 } 3596 3597 flow_id = qp->device->create_flow(qp, flow_attr, 3598 IB_FLOW_DOMAIN_USER, uhw); 3599 3600 if (IS_ERR(flow_id)) { 3601 err = PTR_ERR(flow_id); 3602 goto err_free; 3603 } 3604 atomic_inc(&qp->usecnt); 3605 flow_id->qp = qp; 3606 flow_id->device = qp->device; 3607 flow_id->uobject = uobj; 3608 uobj->object = flow_id; 3609 uflow = container_of(uobj, typeof(*uflow), uobject); 3610 uflow->resources = uflow_res; 3611 3612 memset(&resp, 0, sizeof(resp)); 3613 resp.flow_handle = uobj->id; 3614 3615 err = ib_copy_to_udata(ucore, 3616 &resp, sizeof(resp)); 3617 if (err) 3618 goto err_copy; 3619 3620 uobj_put_obj_read(qp); 3621 kfree(flow_attr); 3622 if (cmd.flow_attr.num_of_specs) 3623 kfree(kern_flow_attr); 3624 return uobj_alloc_commit(uobj, 0); 3625 err_copy: 3626 if (!qp->device->destroy_flow(flow_id)) 3627 atomic_dec(&qp->usecnt); 3628 err_free: 3629 ib_uverbs_flow_resources_free(uflow_res); 3630 err_free_flow_attr: 3631 kfree(flow_attr); 3632 err_put: 3633 uobj_put_obj_read(qp); 3634 err_uobj: 3635 uobj_alloc_abort(uobj); 3636 err_free_attr: 3637 if (cmd.flow_attr.num_of_specs) 3638 kfree(kern_flow_attr); 3639 return err; 3640 } 3641 3642 int ib_uverbs_ex_destroy_flow(struct ib_uverbs_file *file, 3643 struct ib_udata *ucore, 3644 struct ib_udata *uhw) 3645 { 3646 struct ib_uverbs_destroy_flow cmd; 3647 int ret; 3648 3649 if (ucore->inlen < sizeof(cmd)) 3650 return -EINVAL; 3651 3652 ret = ib_copy_from_udata(&cmd, ucore, sizeof(cmd)); 3653 if (ret) 3654 return ret; 3655 3656 if (cmd.comp_mask) 3657 return -EINVAL; 3658 3659 return uobj_perform_destroy(UVERBS_OBJECT_FLOW, cmd.flow_handle, file, 3660 0); 3661 } 3662 3663 static int __uverbs_create_xsrq(struct ib_uverbs_file *file, 3664 struct ib_uverbs_create_xsrq *cmd, 3665 struct ib_udata *udata) 3666 { 3667 struct ib_uverbs_create_srq_resp resp; 3668 struct ib_usrq_object *obj; 3669 struct ib_pd *pd; 3670 struct ib_srq *srq; 3671 struct ib_uobject *uninitialized_var(xrcd_uobj); 3672 struct ib_srq_init_attr attr; 3673 int ret; 3674 struct ib_device *ib_dev; 3675 3676 obj = (struct ib_usrq_object *)uobj_alloc(UVERBS_OBJECT_SRQ, file, 3677 &ib_dev); 3678 if (IS_ERR(obj)) 3679 return PTR_ERR(obj); 3680 3681 if (cmd->srq_type == IB_SRQT_TM) 3682 attr.ext.tag_matching.max_num_tags = cmd->max_num_tags; 3683 3684 if (cmd->srq_type == IB_SRQT_XRC) { 3685 xrcd_uobj = uobj_get_read(UVERBS_OBJECT_XRCD, cmd->xrcd_handle, 3686 file); 3687 if (IS_ERR(xrcd_uobj)) { 3688 ret = -EINVAL; 3689 goto err; 3690 } 3691 3692 attr.ext.xrc.xrcd = (struct ib_xrcd *)xrcd_uobj->object; 3693 if (!attr.ext.xrc.xrcd) { 3694 ret = -EINVAL; 3695 goto err_put_xrcd; 3696 } 3697 3698 obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object, uobject); 3699 atomic_inc(&obj->uxrcd->refcnt); 3700 } 3701 3702 if (ib_srq_has_cq(cmd->srq_type)) { 3703 attr.ext.cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, 3704 cmd->cq_handle, file); 3705 if (!attr.ext.cq) { 3706 ret = -EINVAL; 3707 goto err_put_xrcd; 3708 } 3709 } 3710 3711 pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd->pd_handle, file); 3712 if (!pd) { 3713 ret = -EINVAL; 3714 goto err_put_cq; 3715 } 3716 3717 attr.event_handler = ib_uverbs_srq_event_handler; 3718 attr.srq_context = file; 3719 attr.srq_type = cmd->srq_type; 3720 attr.attr.max_wr = cmd->max_wr; 3721 attr.attr.max_sge = cmd->max_sge; 3722 attr.attr.srq_limit = cmd->srq_limit; 3723 3724 obj->uevent.events_reported = 0; 3725 INIT_LIST_HEAD(&obj->uevent.event_list); 3726 3727 srq = pd->device->create_srq(pd, &attr, udata); 3728 if (IS_ERR(srq)) { 3729 ret = PTR_ERR(srq); 3730 goto err_put; 3731 } 3732 3733 srq->device = pd->device; 3734 srq->pd = pd; 3735 srq->srq_type = cmd->srq_type; 3736 srq->uobject = &obj->uevent.uobject; 3737 srq->event_handler = attr.event_handler; 3738 srq->srq_context = attr.srq_context; 3739 3740 if (ib_srq_has_cq(cmd->srq_type)) { 3741 srq->ext.cq = attr.ext.cq; 3742 atomic_inc(&attr.ext.cq->usecnt); 3743 } 3744 3745 if (cmd->srq_type == IB_SRQT_XRC) { 3746 srq->ext.xrc.xrcd = attr.ext.xrc.xrcd; 3747 atomic_inc(&attr.ext.xrc.xrcd->usecnt); 3748 } 3749 3750 atomic_inc(&pd->usecnt); 3751 atomic_set(&srq->usecnt, 0); 3752 3753 obj->uevent.uobject.object = srq; 3754 obj->uevent.uobject.user_handle = cmd->user_handle; 3755 3756 memset(&resp, 0, sizeof resp); 3757 resp.srq_handle = obj->uevent.uobject.id; 3758 resp.max_wr = attr.attr.max_wr; 3759 resp.max_sge = attr.attr.max_sge; 3760 if (cmd->srq_type == IB_SRQT_XRC) 3761 resp.srqn = srq->ext.xrc.srq_num; 3762 3763 if (copy_to_user(u64_to_user_ptr(cmd->response), 3764 &resp, sizeof resp)) { 3765 ret = -EFAULT; 3766 goto err_copy; 3767 } 3768 3769 if (cmd->srq_type == IB_SRQT_XRC) 3770 uobj_put_read(xrcd_uobj); 3771 3772 if (ib_srq_has_cq(cmd->srq_type)) 3773 uobj_put_obj_read(attr.ext.cq); 3774 3775 uobj_put_obj_read(pd); 3776 return uobj_alloc_commit(&obj->uevent.uobject, 0); 3777 3778 err_copy: 3779 ib_destroy_srq(srq); 3780 3781 err_put: 3782 uobj_put_obj_read(pd); 3783 3784 err_put_cq: 3785 if (ib_srq_has_cq(cmd->srq_type)) 3786 uobj_put_obj_read(attr.ext.cq); 3787 3788 err_put_xrcd: 3789 if (cmd->srq_type == IB_SRQT_XRC) { 3790 atomic_dec(&obj->uxrcd->refcnt); 3791 uobj_put_read(xrcd_uobj); 3792 } 3793 3794 err: 3795 uobj_alloc_abort(&obj->uevent.uobject); 3796 return ret; 3797 } 3798 3799 ssize_t ib_uverbs_create_srq(struct ib_uverbs_file *file, 3800 const char __user *buf, int in_len, 3801 int out_len) 3802 { 3803 struct ib_uverbs_create_srq cmd; 3804 struct ib_uverbs_create_xsrq xcmd; 3805 struct ib_uverbs_create_srq_resp resp; 3806 struct ib_udata udata; 3807 int ret; 3808 3809 if (out_len < sizeof resp) 3810 return -ENOSPC; 3811 3812 if (copy_from_user(&cmd, buf, sizeof cmd)) 3813 return -EFAULT; 3814 3815 memset(&xcmd, 0, sizeof(xcmd)); 3816 xcmd.response = cmd.response; 3817 xcmd.user_handle = cmd.user_handle; 3818 xcmd.srq_type = IB_SRQT_BASIC; 3819 xcmd.pd_handle = cmd.pd_handle; 3820 xcmd.max_wr = cmd.max_wr; 3821 xcmd.max_sge = cmd.max_sge; 3822 xcmd.srq_limit = cmd.srq_limit; 3823 3824 ib_uverbs_init_udata(&udata, buf + sizeof(cmd), 3825 u64_to_user_ptr(cmd.response) + sizeof(resp), 3826 in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr), 3827 out_len - sizeof(resp)); 3828 3829 ret = __uverbs_create_xsrq(file, &xcmd, &udata); 3830 if (ret) 3831 return ret; 3832 3833 return in_len; 3834 } 3835 3836 ssize_t ib_uverbs_create_xsrq(struct ib_uverbs_file *file, 3837 const char __user *buf, int in_len, int out_len) 3838 { 3839 struct ib_uverbs_create_xsrq cmd; 3840 struct ib_uverbs_create_srq_resp resp; 3841 struct ib_udata udata; 3842 int ret; 3843 3844 if (out_len < sizeof resp) 3845 return -ENOSPC; 3846 3847 if (copy_from_user(&cmd, buf, sizeof cmd)) 3848 return -EFAULT; 3849 3850 ib_uverbs_init_udata(&udata, buf + sizeof(cmd), 3851 u64_to_user_ptr(cmd.response) + sizeof(resp), 3852 in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr), 3853 out_len - sizeof(resp)); 3854 3855 ret = __uverbs_create_xsrq(file, &cmd, &udata); 3856 if (ret) 3857 return ret; 3858 3859 return in_len; 3860 } 3861 3862 ssize_t ib_uverbs_modify_srq(struct ib_uverbs_file *file, 3863 const char __user *buf, int in_len, 3864 int out_len) 3865 { 3866 struct ib_uverbs_modify_srq cmd; 3867 struct ib_udata udata; 3868 struct ib_srq *srq; 3869 struct ib_srq_attr attr; 3870 int ret; 3871 3872 if (copy_from_user(&cmd, buf, sizeof cmd)) 3873 return -EFAULT; 3874 3875 ib_uverbs_init_udata(&udata, buf + sizeof cmd, NULL, in_len - sizeof cmd, 3876 out_len); 3877 3878 srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ, cmd.srq_handle, file); 3879 if (!srq) 3880 return -EINVAL; 3881 3882 attr.max_wr = cmd.max_wr; 3883 attr.srq_limit = cmd.srq_limit; 3884 3885 ret = srq->device->modify_srq(srq, &attr, cmd.attr_mask, &udata); 3886 3887 uobj_put_obj_read(srq); 3888 3889 return ret ? ret : in_len; 3890 } 3891 3892 ssize_t ib_uverbs_query_srq(struct ib_uverbs_file *file, 3893 const char __user *buf, 3894 int in_len, int out_len) 3895 { 3896 struct ib_uverbs_query_srq cmd; 3897 struct ib_uverbs_query_srq_resp resp; 3898 struct ib_srq_attr attr; 3899 struct ib_srq *srq; 3900 int ret; 3901 3902 if (out_len < sizeof resp) 3903 return -ENOSPC; 3904 3905 if (copy_from_user(&cmd, buf, sizeof cmd)) 3906 return -EFAULT; 3907 3908 srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ, cmd.srq_handle, file); 3909 if (!srq) 3910 return -EINVAL; 3911 3912 ret = ib_query_srq(srq, &attr); 3913 3914 uobj_put_obj_read(srq); 3915 3916 if (ret) 3917 return ret; 3918 3919 memset(&resp, 0, sizeof resp); 3920 3921 resp.max_wr = attr.max_wr; 3922 resp.max_sge = attr.max_sge; 3923 resp.srq_limit = attr.srq_limit; 3924 3925 if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp)) 3926 return -EFAULT; 3927 3928 return in_len; 3929 } 3930 3931 ssize_t ib_uverbs_destroy_srq(struct ib_uverbs_file *file, 3932 const char __user *buf, int in_len, 3933 int out_len) 3934 { 3935 struct ib_uverbs_destroy_srq cmd; 3936 struct ib_uverbs_destroy_srq_resp resp; 3937 struct ib_uobject *uobj; 3938 struct ib_uevent_object *obj; 3939 3940 if (copy_from_user(&cmd, buf, sizeof cmd)) 3941 return -EFAULT; 3942 3943 uobj = uobj_get_destroy(UVERBS_OBJECT_SRQ, cmd.srq_handle, file); 3944 if (IS_ERR(uobj)) 3945 return PTR_ERR(uobj); 3946 3947 obj = container_of(uobj, struct ib_uevent_object, uobject); 3948 memset(&resp, 0, sizeof(resp)); 3949 resp.events_reported = obj->events_reported; 3950 3951 uobj_put_destroy(uobj); 3952 3953 if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof(resp))) 3954 return -EFAULT; 3955 3956 return in_len; 3957 } 3958 3959 int ib_uverbs_ex_query_device(struct ib_uverbs_file *file, 3960 struct ib_udata *ucore, 3961 struct ib_udata *uhw) 3962 { 3963 struct ib_uverbs_ex_query_device_resp resp = { {0} }; 3964 struct ib_uverbs_ex_query_device cmd; 3965 struct ib_device_attr attr = {0}; 3966 struct ib_ucontext *ucontext; 3967 struct ib_device *ib_dev; 3968 int err; 3969 3970 ucontext = ib_uverbs_get_ucontext(file); 3971 if (IS_ERR(ucontext)) 3972 return PTR_ERR(ucontext); 3973 ib_dev = ucontext->device; 3974 3975 if (!ib_dev->query_device) 3976 return -EOPNOTSUPP; 3977 3978 if (ucore->inlen < sizeof(cmd)) 3979 return -EINVAL; 3980 3981 err = ib_copy_from_udata(&cmd, ucore, sizeof(cmd)); 3982 if (err) 3983 return err; 3984 3985 if (cmd.comp_mask) 3986 return -EINVAL; 3987 3988 if (cmd.reserved) 3989 return -EINVAL; 3990 3991 resp.response_length = offsetof(typeof(resp), odp_caps); 3992 3993 if (ucore->outlen < resp.response_length) 3994 return -ENOSPC; 3995 3996 err = ib_dev->query_device(ib_dev, &attr, uhw); 3997 if (err) 3998 return err; 3999 4000 copy_query_dev_fields(ucontext, &resp.base, &attr); 4001 4002 if (ucore->outlen < resp.response_length + sizeof(resp.odp_caps)) 4003 goto end; 4004 4005 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING 4006 resp.odp_caps.general_caps = attr.odp_caps.general_caps; 4007 resp.odp_caps.per_transport_caps.rc_odp_caps = 4008 attr.odp_caps.per_transport_caps.rc_odp_caps; 4009 resp.odp_caps.per_transport_caps.uc_odp_caps = 4010 attr.odp_caps.per_transport_caps.uc_odp_caps; 4011 resp.odp_caps.per_transport_caps.ud_odp_caps = 4012 attr.odp_caps.per_transport_caps.ud_odp_caps; 4013 #endif 4014 resp.response_length += sizeof(resp.odp_caps); 4015 4016 if (ucore->outlen < resp.response_length + sizeof(resp.timestamp_mask)) 4017 goto end; 4018 4019 resp.timestamp_mask = attr.timestamp_mask; 4020 resp.response_length += sizeof(resp.timestamp_mask); 4021 4022 if (ucore->outlen < resp.response_length + sizeof(resp.hca_core_clock)) 4023 goto end; 4024 4025 resp.hca_core_clock = attr.hca_core_clock; 4026 resp.response_length += sizeof(resp.hca_core_clock); 4027 4028 if (ucore->outlen < resp.response_length + sizeof(resp.device_cap_flags_ex)) 4029 goto end; 4030 4031 resp.device_cap_flags_ex = attr.device_cap_flags; 4032 resp.response_length += sizeof(resp.device_cap_flags_ex); 4033 4034 if (ucore->outlen < resp.response_length + sizeof(resp.rss_caps)) 4035 goto end; 4036 4037 resp.rss_caps.supported_qpts = attr.rss_caps.supported_qpts; 4038 resp.rss_caps.max_rwq_indirection_tables = 4039 attr.rss_caps.max_rwq_indirection_tables; 4040 resp.rss_caps.max_rwq_indirection_table_size = 4041 attr.rss_caps.max_rwq_indirection_table_size; 4042 4043 resp.response_length += sizeof(resp.rss_caps); 4044 4045 if (ucore->outlen < resp.response_length + sizeof(resp.max_wq_type_rq)) 4046 goto end; 4047 4048 resp.max_wq_type_rq = attr.max_wq_type_rq; 4049 resp.response_length += sizeof(resp.max_wq_type_rq); 4050 4051 if (ucore->outlen < resp.response_length + sizeof(resp.raw_packet_caps)) 4052 goto end; 4053 4054 resp.raw_packet_caps = attr.raw_packet_caps; 4055 resp.response_length += sizeof(resp.raw_packet_caps); 4056 4057 if (ucore->outlen < resp.response_length + sizeof(resp.tm_caps)) 4058 goto end; 4059 4060 resp.tm_caps.max_rndv_hdr_size = attr.tm_caps.max_rndv_hdr_size; 4061 resp.tm_caps.max_num_tags = attr.tm_caps.max_num_tags; 4062 resp.tm_caps.max_ops = attr.tm_caps.max_ops; 4063 resp.tm_caps.max_sge = attr.tm_caps.max_sge; 4064 resp.tm_caps.flags = attr.tm_caps.flags; 4065 resp.response_length += sizeof(resp.tm_caps); 4066 4067 if (ucore->outlen < resp.response_length + sizeof(resp.cq_moderation_caps)) 4068 goto end; 4069 4070 resp.cq_moderation_caps.max_cq_moderation_count = 4071 attr.cq_caps.max_cq_moderation_count; 4072 resp.cq_moderation_caps.max_cq_moderation_period = 4073 attr.cq_caps.max_cq_moderation_period; 4074 resp.response_length += sizeof(resp.cq_moderation_caps); 4075 4076 if (ucore->outlen < resp.response_length + sizeof(resp.max_dm_size)) 4077 goto end; 4078 4079 resp.max_dm_size = attr.max_dm_size; 4080 resp.response_length += sizeof(resp.max_dm_size); 4081 end: 4082 err = ib_copy_to_udata(ucore, &resp, resp.response_length); 4083 return err; 4084 } 4085 4086 int ib_uverbs_ex_modify_cq(struct ib_uverbs_file *file, 4087 struct ib_udata *ucore, 4088 struct ib_udata *uhw) 4089 { 4090 struct ib_uverbs_ex_modify_cq cmd = {}; 4091 struct ib_cq *cq; 4092 size_t required_cmd_sz; 4093 int ret; 4094 4095 required_cmd_sz = offsetof(typeof(cmd), reserved) + 4096 sizeof(cmd.reserved); 4097 if (ucore->inlen < required_cmd_sz) 4098 return -EINVAL; 4099 4100 /* sanity checks */ 4101 if (ucore->inlen > sizeof(cmd) && 4102 !ib_is_udata_cleared(ucore, sizeof(cmd), 4103 ucore->inlen - sizeof(cmd))) 4104 return -EOPNOTSUPP; 4105 4106 ret = ib_copy_from_udata(&cmd, ucore, min(sizeof(cmd), ucore->inlen)); 4107 if (ret) 4108 return ret; 4109 4110 if (!cmd.attr_mask || cmd.reserved) 4111 return -EINVAL; 4112 4113 if (cmd.attr_mask > IB_CQ_MODERATE) 4114 return -EOPNOTSUPP; 4115 4116 cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, file); 4117 if (!cq) 4118 return -EINVAL; 4119 4120 ret = rdma_set_cq_moderation(cq, cmd.attr.cq_count, cmd.attr.cq_period); 4121 4122 uobj_put_obj_read(cq); 4123 4124 return ret; 4125 } 4126