1 /* 2 * Copyright (c) 2005 Topspin Communications. All rights reserved. 3 * Copyright (c) 2005, 2006, 2007 Cisco Systems. All rights reserved. 4 * Copyright (c) 2005 PathScale, Inc. All rights reserved. 5 * Copyright (c) 2006 Mellanox Technologies. All rights reserved. 6 * 7 * This software is available to you under a choice of one of two 8 * licenses. You may choose to be licensed under the terms of the GNU 9 * General Public License (GPL) Version 2, available from the file 10 * COPYING in the main directory of this source tree, or the 11 * OpenIB.org BSD license below: 12 * 13 * Redistribution and use in source and binary forms, with or 14 * without modification, are permitted provided that the following 15 * conditions are met: 16 * 17 * - Redistributions of source code must retain the above 18 * copyright notice, this list of conditions and the following 19 * disclaimer. 20 * 21 * - Redistributions in binary form must reproduce the above 22 * copyright notice, this list of conditions and the following 23 * disclaimer in the documentation and/or other materials 24 * provided with the distribution. 25 * 26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 27 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 28 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 29 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 30 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 31 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 32 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 33 * SOFTWARE. 34 */ 35 36 #include <linux/file.h> 37 #include <linux/fs.h> 38 #include <linux/slab.h> 39 #include <linux/sched.h> 40 41 #include <asm/uaccess.h> 42 43 #include "uverbs.h" 44 #include "core_priv.h" 45 46 struct uverbs_lock_class { 47 struct lock_class_key key; 48 char name[16]; 49 }; 50 51 static struct uverbs_lock_class pd_lock_class = { .name = "PD-uobj" }; 52 static struct uverbs_lock_class mr_lock_class = { .name = "MR-uobj" }; 53 static struct uverbs_lock_class mw_lock_class = { .name = "MW-uobj" }; 54 static struct uverbs_lock_class cq_lock_class = { .name = "CQ-uobj" }; 55 static struct uverbs_lock_class qp_lock_class = { .name = "QP-uobj" }; 56 static struct uverbs_lock_class ah_lock_class = { .name = "AH-uobj" }; 57 static struct uverbs_lock_class srq_lock_class = { .name = "SRQ-uobj" }; 58 static struct uverbs_lock_class xrcd_lock_class = { .name = "XRCD-uobj" }; 59 static struct uverbs_lock_class rule_lock_class = { .name = "RULE-uobj" }; 60 static struct uverbs_lock_class wq_lock_class = { .name = "WQ-uobj" }; 61 static struct uverbs_lock_class rwq_ind_table_lock_class = { .name = "IND_TBL-uobj" }; 62 63 /* 64 * The ib_uobject locking scheme is as follows: 65 * 66 * - ib_uverbs_idr_lock protects the uverbs idrs themselves, so it 67 * needs to be held during all idr write operations. When an object is 68 * looked up, a reference must be taken on the object's kref before 69 * dropping this lock. For read operations, the rcu_read_lock() 70 * and rcu_write_lock() but similarly the kref reference is grabbed 71 * before the rcu_read_unlock(). 72 * 73 * - Each object also has an rwsem. This rwsem must be held for 74 * reading while an operation that uses the object is performed. 75 * For example, while registering an MR, the associated PD's 76 * uobject.mutex must be held for reading. The rwsem must be held 77 * for writing while initializing or destroying an object. 78 * 79 * - In addition, each object has a "live" flag. If this flag is not 80 * set, then lookups of the object will fail even if it is found in 81 * the idr. This handles a reader that blocks and does not acquire 82 * the rwsem until after the object is destroyed. The destroy 83 * operation will set the live flag to 0 and then drop the rwsem; 84 * this will allow the reader to acquire the rwsem, see that the 85 * live flag is 0, and then drop the rwsem and its reference to 86 * object. The underlying storage will not be freed until the last 87 * reference to the object is dropped. 88 */ 89 90 static void init_uobj(struct ib_uobject *uobj, u64 user_handle, 91 struct ib_ucontext *context, struct uverbs_lock_class *c) 92 { 93 uobj->user_handle = user_handle; 94 uobj->context = context; 95 kref_init(&uobj->ref); 96 init_rwsem(&uobj->mutex); 97 lockdep_set_class_and_name(&uobj->mutex, &c->key, c->name); 98 uobj->live = 0; 99 } 100 101 static void release_uobj(struct kref *kref) 102 { 103 kfree_rcu(container_of(kref, struct ib_uobject, ref), rcu); 104 } 105 106 static void put_uobj(struct ib_uobject *uobj) 107 { 108 kref_put(&uobj->ref, release_uobj); 109 } 110 111 static void put_uobj_read(struct ib_uobject *uobj) 112 { 113 up_read(&uobj->mutex); 114 put_uobj(uobj); 115 } 116 117 static void put_uobj_write(struct ib_uobject *uobj) 118 { 119 up_write(&uobj->mutex); 120 put_uobj(uobj); 121 } 122 123 static int idr_add_uobj(struct idr *idr, struct ib_uobject *uobj) 124 { 125 int ret; 126 127 idr_preload(GFP_KERNEL); 128 spin_lock(&ib_uverbs_idr_lock); 129 130 ret = idr_alloc(idr, uobj, 0, 0, GFP_NOWAIT); 131 if (ret >= 0) 132 uobj->id = ret; 133 134 spin_unlock(&ib_uverbs_idr_lock); 135 idr_preload_end(); 136 137 return ret < 0 ? ret : 0; 138 } 139 140 void idr_remove_uobj(struct idr *idr, struct ib_uobject *uobj) 141 { 142 spin_lock(&ib_uverbs_idr_lock); 143 idr_remove(idr, uobj->id); 144 spin_unlock(&ib_uverbs_idr_lock); 145 } 146 147 static struct ib_uobject *__idr_get_uobj(struct idr *idr, int id, 148 struct ib_ucontext *context) 149 { 150 struct ib_uobject *uobj; 151 152 rcu_read_lock(); 153 uobj = idr_find(idr, id); 154 if (uobj) { 155 if (uobj->context == context) 156 kref_get(&uobj->ref); 157 else 158 uobj = NULL; 159 } 160 rcu_read_unlock(); 161 162 return uobj; 163 } 164 165 static struct ib_uobject *idr_read_uobj(struct idr *idr, int id, 166 struct ib_ucontext *context, int nested) 167 { 168 struct ib_uobject *uobj; 169 170 uobj = __idr_get_uobj(idr, id, context); 171 if (!uobj) 172 return NULL; 173 174 if (nested) 175 down_read_nested(&uobj->mutex, SINGLE_DEPTH_NESTING); 176 else 177 down_read(&uobj->mutex); 178 if (!uobj->live) { 179 put_uobj_read(uobj); 180 return NULL; 181 } 182 183 return uobj; 184 } 185 186 static struct ib_uobject *idr_write_uobj(struct idr *idr, int id, 187 struct ib_ucontext *context) 188 { 189 struct ib_uobject *uobj; 190 191 uobj = __idr_get_uobj(idr, id, context); 192 if (!uobj) 193 return NULL; 194 195 down_write(&uobj->mutex); 196 if (!uobj->live) { 197 put_uobj_write(uobj); 198 return NULL; 199 } 200 201 return uobj; 202 } 203 204 static void *idr_read_obj(struct idr *idr, int id, struct ib_ucontext *context, 205 int nested) 206 { 207 struct ib_uobject *uobj; 208 209 uobj = idr_read_uobj(idr, id, context, nested); 210 return uobj ? uobj->object : NULL; 211 } 212 213 static struct ib_pd *idr_read_pd(int pd_handle, struct ib_ucontext *context) 214 { 215 return idr_read_obj(&ib_uverbs_pd_idr, pd_handle, context, 0); 216 } 217 218 static void put_pd_read(struct ib_pd *pd) 219 { 220 put_uobj_read(pd->uobject); 221 } 222 223 static struct ib_cq *idr_read_cq(int cq_handle, struct ib_ucontext *context, int nested) 224 { 225 return idr_read_obj(&ib_uverbs_cq_idr, cq_handle, context, nested); 226 } 227 228 static void put_cq_read(struct ib_cq *cq) 229 { 230 put_uobj_read(cq->uobject); 231 } 232 233 static struct ib_ah *idr_read_ah(int ah_handle, struct ib_ucontext *context) 234 { 235 return idr_read_obj(&ib_uverbs_ah_idr, ah_handle, context, 0); 236 } 237 238 static void put_ah_read(struct ib_ah *ah) 239 { 240 put_uobj_read(ah->uobject); 241 } 242 243 static struct ib_qp *idr_read_qp(int qp_handle, struct ib_ucontext *context) 244 { 245 return idr_read_obj(&ib_uverbs_qp_idr, qp_handle, context, 0); 246 } 247 248 static struct ib_wq *idr_read_wq(int wq_handle, struct ib_ucontext *context) 249 { 250 return idr_read_obj(&ib_uverbs_wq_idr, wq_handle, context, 0); 251 } 252 253 static void put_wq_read(struct ib_wq *wq) 254 { 255 put_uobj_read(wq->uobject); 256 } 257 258 static struct ib_rwq_ind_table *idr_read_rwq_indirection_table(int ind_table_handle, 259 struct ib_ucontext *context) 260 { 261 return idr_read_obj(&ib_uverbs_rwq_ind_tbl_idr, ind_table_handle, context, 0); 262 } 263 264 static void put_rwq_indirection_table_read(struct ib_rwq_ind_table *ind_table) 265 { 266 put_uobj_read(ind_table->uobject); 267 } 268 269 static struct ib_qp *idr_write_qp(int qp_handle, struct ib_ucontext *context) 270 { 271 struct ib_uobject *uobj; 272 273 uobj = idr_write_uobj(&ib_uverbs_qp_idr, qp_handle, context); 274 return uobj ? uobj->object : NULL; 275 } 276 277 static void put_qp_read(struct ib_qp *qp) 278 { 279 put_uobj_read(qp->uobject); 280 } 281 282 static void put_qp_write(struct ib_qp *qp) 283 { 284 put_uobj_write(qp->uobject); 285 } 286 287 static struct ib_srq *idr_read_srq(int srq_handle, struct ib_ucontext *context) 288 { 289 return idr_read_obj(&ib_uverbs_srq_idr, srq_handle, context, 0); 290 } 291 292 static void put_srq_read(struct ib_srq *srq) 293 { 294 put_uobj_read(srq->uobject); 295 } 296 297 static struct ib_xrcd *idr_read_xrcd(int xrcd_handle, struct ib_ucontext *context, 298 struct ib_uobject **uobj) 299 { 300 *uobj = idr_read_uobj(&ib_uverbs_xrcd_idr, xrcd_handle, context, 0); 301 return *uobj ? (*uobj)->object : NULL; 302 } 303 304 static void put_xrcd_read(struct ib_uobject *uobj) 305 { 306 put_uobj_read(uobj); 307 } 308 309 ssize_t ib_uverbs_get_context(struct ib_uverbs_file *file, 310 struct ib_device *ib_dev, 311 const char __user *buf, 312 int in_len, int out_len) 313 { 314 struct ib_uverbs_get_context cmd; 315 struct ib_uverbs_get_context_resp resp; 316 struct ib_udata udata; 317 struct ib_ucontext *ucontext; 318 struct file *filp; 319 int ret; 320 321 if (out_len < sizeof resp) 322 return -ENOSPC; 323 324 if (copy_from_user(&cmd, buf, sizeof cmd)) 325 return -EFAULT; 326 327 mutex_lock(&file->mutex); 328 329 if (file->ucontext) { 330 ret = -EINVAL; 331 goto err; 332 } 333 334 INIT_UDATA(&udata, buf + sizeof cmd, 335 (unsigned long) cmd.response + sizeof resp, 336 in_len - sizeof cmd, out_len - sizeof resp); 337 338 ucontext = ib_dev->alloc_ucontext(ib_dev, &udata); 339 if (IS_ERR(ucontext)) { 340 ret = PTR_ERR(ucontext); 341 goto err; 342 } 343 344 ucontext->device = ib_dev; 345 INIT_LIST_HEAD(&ucontext->pd_list); 346 INIT_LIST_HEAD(&ucontext->mr_list); 347 INIT_LIST_HEAD(&ucontext->mw_list); 348 INIT_LIST_HEAD(&ucontext->cq_list); 349 INIT_LIST_HEAD(&ucontext->qp_list); 350 INIT_LIST_HEAD(&ucontext->srq_list); 351 INIT_LIST_HEAD(&ucontext->ah_list); 352 INIT_LIST_HEAD(&ucontext->wq_list); 353 INIT_LIST_HEAD(&ucontext->rwq_ind_tbl_list); 354 INIT_LIST_HEAD(&ucontext->xrcd_list); 355 INIT_LIST_HEAD(&ucontext->rule_list); 356 rcu_read_lock(); 357 ucontext->tgid = get_task_pid(current->group_leader, PIDTYPE_PID); 358 rcu_read_unlock(); 359 ucontext->closing = 0; 360 361 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING 362 ucontext->umem_tree = RB_ROOT; 363 init_rwsem(&ucontext->umem_rwsem); 364 ucontext->odp_mrs_count = 0; 365 INIT_LIST_HEAD(&ucontext->no_private_counters); 366 367 if (!(ib_dev->attrs.device_cap_flags & IB_DEVICE_ON_DEMAND_PAGING)) 368 ucontext->invalidate_range = NULL; 369 370 #endif 371 372 resp.num_comp_vectors = file->device->num_comp_vectors; 373 374 ret = get_unused_fd_flags(O_CLOEXEC); 375 if (ret < 0) 376 goto err_free; 377 resp.async_fd = ret; 378 379 filp = ib_uverbs_alloc_event_file(file, ib_dev, 1); 380 if (IS_ERR(filp)) { 381 ret = PTR_ERR(filp); 382 goto err_fd; 383 } 384 385 if (copy_to_user((void __user *) (unsigned long) cmd.response, 386 &resp, sizeof resp)) { 387 ret = -EFAULT; 388 goto err_file; 389 } 390 391 file->ucontext = ucontext; 392 393 fd_install(resp.async_fd, filp); 394 395 mutex_unlock(&file->mutex); 396 397 return in_len; 398 399 err_file: 400 ib_uverbs_free_async_event_file(file); 401 fput(filp); 402 403 err_fd: 404 put_unused_fd(resp.async_fd); 405 406 err_free: 407 put_pid(ucontext->tgid); 408 ib_dev->dealloc_ucontext(ucontext); 409 410 err: 411 mutex_unlock(&file->mutex); 412 return ret; 413 } 414 415 static void copy_query_dev_fields(struct ib_uverbs_file *file, 416 struct ib_device *ib_dev, 417 struct ib_uverbs_query_device_resp *resp, 418 struct ib_device_attr *attr) 419 { 420 resp->fw_ver = attr->fw_ver; 421 resp->node_guid = ib_dev->node_guid; 422 resp->sys_image_guid = attr->sys_image_guid; 423 resp->max_mr_size = attr->max_mr_size; 424 resp->page_size_cap = attr->page_size_cap; 425 resp->vendor_id = attr->vendor_id; 426 resp->vendor_part_id = attr->vendor_part_id; 427 resp->hw_ver = attr->hw_ver; 428 resp->max_qp = attr->max_qp; 429 resp->max_qp_wr = attr->max_qp_wr; 430 resp->device_cap_flags = lower_32_bits(attr->device_cap_flags); 431 resp->max_sge = attr->max_sge; 432 resp->max_sge_rd = attr->max_sge_rd; 433 resp->max_cq = attr->max_cq; 434 resp->max_cqe = attr->max_cqe; 435 resp->max_mr = attr->max_mr; 436 resp->max_pd = attr->max_pd; 437 resp->max_qp_rd_atom = attr->max_qp_rd_atom; 438 resp->max_ee_rd_atom = attr->max_ee_rd_atom; 439 resp->max_res_rd_atom = attr->max_res_rd_atom; 440 resp->max_qp_init_rd_atom = attr->max_qp_init_rd_atom; 441 resp->max_ee_init_rd_atom = attr->max_ee_init_rd_atom; 442 resp->atomic_cap = attr->atomic_cap; 443 resp->max_ee = attr->max_ee; 444 resp->max_rdd = attr->max_rdd; 445 resp->max_mw = attr->max_mw; 446 resp->max_raw_ipv6_qp = attr->max_raw_ipv6_qp; 447 resp->max_raw_ethy_qp = attr->max_raw_ethy_qp; 448 resp->max_mcast_grp = attr->max_mcast_grp; 449 resp->max_mcast_qp_attach = attr->max_mcast_qp_attach; 450 resp->max_total_mcast_qp_attach = attr->max_total_mcast_qp_attach; 451 resp->max_ah = attr->max_ah; 452 resp->max_fmr = attr->max_fmr; 453 resp->max_map_per_fmr = attr->max_map_per_fmr; 454 resp->max_srq = attr->max_srq; 455 resp->max_srq_wr = attr->max_srq_wr; 456 resp->max_srq_sge = attr->max_srq_sge; 457 resp->max_pkeys = attr->max_pkeys; 458 resp->local_ca_ack_delay = attr->local_ca_ack_delay; 459 resp->phys_port_cnt = ib_dev->phys_port_cnt; 460 } 461 462 ssize_t ib_uverbs_query_device(struct ib_uverbs_file *file, 463 struct ib_device *ib_dev, 464 const char __user *buf, 465 int in_len, int out_len) 466 { 467 struct ib_uverbs_query_device cmd; 468 struct ib_uverbs_query_device_resp resp; 469 470 if (out_len < sizeof resp) 471 return -ENOSPC; 472 473 if (copy_from_user(&cmd, buf, sizeof cmd)) 474 return -EFAULT; 475 476 memset(&resp, 0, sizeof resp); 477 copy_query_dev_fields(file, ib_dev, &resp, &ib_dev->attrs); 478 479 if (copy_to_user((void __user *) (unsigned long) cmd.response, 480 &resp, sizeof resp)) 481 return -EFAULT; 482 483 return in_len; 484 } 485 486 ssize_t ib_uverbs_query_port(struct ib_uverbs_file *file, 487 struct ib_device *ib_dev, 488 const char __user *buf, 489 int in_len, int out_len) 490 { 491 struct ib_uverbs_query_port cmd; 492 struct ib_uverbs_query_port_resp resp; 493 struct ib_port_attr attr; 494 int ret; 495 496 if (out_len < sizeof resp) 497 return -ENOSPC; 498 499 if (copy_from_user(&cmd, buf, sizeof cmd)) 500 return -EFAULT; 501 502 ret = ib_query_port(ib_dev, cmd.port_num, &attr); 503 if (ret) 504 return ret; 505 506 memset(&resp, 0, sizeof resp); 507 508 resp.state = attr.state; 509 resp.max_mtu = attr.max_mtu; 510 resp.active_mtu = attr.active_mtu; 511 resp.gid_tbl_len = attr.gid_tbl_len; 512 resp.port_cap_flags = attr.port_cap_flags; 513 resp.max_msg_sz = attr.max_msg_sz; 514 resp.bad_pkey_cntr = attr.bad_pkey_cntr; 515 resp.qkey_viol_cntr = attr.qkey_viol_cntr; 516 resp.pkey_tbl_len = attr.pkey_tbl_len; 517 resp.lid = attr.lid; 518 resp.sm_lid = attr.sm_lid; 519 resp.lmc = attr.lmc; 520 resp.max_vl_num = attr.max_vl_num; 521 resp.sm_sl = attr.sm_sl; 522 resp.subnet_timeout = attr.subnet_timeout; 523 resp.init_type_reply = attr.init_type_reply; 524 resp.active_width = attr.active_width; 525 resp.active_speed = attr.active_speed; 526 resp.phys_state = attr.phys_state; 527 resp.link_layer = rdma_port_get_link_layer(ib_dev, 528 cmd.port_num); 529 530 if (copy_to_user((void __user *) (unsigned long) cmd.response, 531 &resp, sizeof resp)) 532 return -EFAULT; 533 534 return in_len; 535 } 536 537 ssize_t ib_uverbs_alloc_pd(struct ib_uverbs_file *file, 538 struct ib_device *ib_dev, 539 const char __user *buf, 540 int in_len, int out_len) 541 { 542 struct ib_uverbs_alloc_pd cmd; 543 struct ib_uverbs_alloc_pd_resp resp; 544 struct ib_udata udata; 545 struct ib_uobject *uobj; 546 struct ib_pd *pd; 547 int ret; 548 549 if (out_len < sizeof resp) 550 return -ENOSPC; 551 552 if (copy_from_user(&cmd, buf, sizeof cmd)) 553 return -EFAULT; 554 555 INIT_UDATA(&udata, buf + sizeof cmd, 556 (unsigned long) cmd.response + sizeof resp, 557 in_len - sizeof cmd, out_len - sizeof resp); 558 559 uobj = kmalloc(sizeof *uobj, GFP_KERNEL); 560 if (!uobj) 561 return -ENOMEM; 562 563 init_uobj(uobj, 0, file->ucontext, &pd_lock_class); 564 down_write(&uobj->mutex); 565 566 pd = ib_dev->alloc_pd(ib_dev, file->ucontext, &udata); 567 if (IS_ERR(pd)) { 568 ret = PTR_ERR(pd); 569 goto err; 570 } 571 572 pd->device = ib_dev; 573 pd->uobject = uobj; 574 pd->__internal_mr = NULL; 575 atomic_set(&pd->usecnt, 0); 576 577 uobj->object = pd; 578 ret = idr_add_uobj(&ib_uverbs_pd_idr, uobj); 579 if (ret) 580 goto err_idr; 581 582 memset(&resp, 0, sizeof resp); 583 resp.pd_handle = uobj->id; 584 585 if (copy_to_user((void __user *) (unsigned long) cmd.response, 586 &resp, sizeof resp)) { 587 ret = -EFAULT; 588 goto err_copy; 589 } 590 591 mutex_lock(&file->mutex); 592 list_add_tail(&uobj->list, &file->ucontext->pd_list); 593 mutex_unlock(&file->mutex); 594 595 uobj->live = 1; 596 597 up_write(&uobj->mutex); 598 599 return in_len; 600 601 err_copy: 602 idr_remove_uobj(&ib_uverbs_pd_idr, uobj); 603 604 err_idr: 605 ib_dealloc_pd(pd); 606 607 err: 608 put_uobj_write(uobj); 609 return ret; 610 } 611 612 ssize_t ib_uverbs_dealloc_pd(struct ib_uverbs_file *file, 613 struct ib_device *ib_dev, 614 const char __user *buf, 615 int in_len, int out_len) 616 { 617 struct ib_uverbs_dealloc_pd cmd; 618 struct ib_uobject *uobj; 619 struct ib_pd *pd; 620 int ret; 621 622 if (copy_from_user(&cmd, buf, sizeof cmd)) 623 return -EFAULT; 624 625 uobj = idr_write_uobj(&ib_uverbs_pd_idr, cmd.pd_handle, file->ucontext); 626 if (!uobj) 627 return -EINVAL; 628 pd = uobj->object; 629 630 if (atomic_read(&pd->usecnt)) { 631 ret = -EBUSY; 632 goto err_put; 633 } 634 635 ret = pd->device->dealloc_pd(uobj->object); 636 WARN_ONCE(ret, "Infiniband HW driver failed dealloc_pd"); 637 if (ret) 638 goto err_put; 639 640 uobj->live = 0; 641 put_uobj_write(uobj); 642 643 idr_remove_uobj(&ib_uverbs_pd_idr, uobj); 644 645 mutex_lock(&file->mutex); 646 list_del(&uobj->list); 647 mutex_unlock(&file->mutex); 648 649 put_uobj(uobj); 650 651 return in_len; 652 653 err_put: 654 put_uobj_write(uobj); 655 return ret; 656 } 657 658 struct xrcd_table_entry { 659 struct rb_node node; 660 struct ib_xrcd *xrcd; 661 struct inode *inode; 662 }; 663 664 static int xrcd_table_insert(struct ib_uverbs_device *dev, 665 struct inode *inode, 666 struct ib_xrcd *xrcd) 667 { 668 struct xrcd_table_entry *entry, *scan; 669 struct rb_node **p = &dev->xrcd_tree.rb_node; 670 struct rb_node *parent = NULL; 671 672 entry = kmalloc(sizeof *entry, GFP_KERNEL); 673 if (!entry) 674 return -ENOMEM; 675 676 entry->xrcd = xrcd; 677 entry->inode = inode; 678 679 while (*p) { 680 parent = *p; 681 scan = rb_entry(parent, struct xrcd_table_entry, node); 682 683 if (inode < scan->inode) { 684 p = &(*p)->rb_left; 685 } else if (inode > scan->inode) { 686 p = &(*p)->rb_right; 687 } else { 688 kfree(entry); 689 return -EEXIST; 690 } 691 } 692 693 rb_link_node(&entry->node, parent, p); 694 rb_insert_color(&entry->node, &dev->xrcd_tree); 695 igrab(inode); 696 return 0; 697 } 698 699 static struct xrcd_table_entry *xrcd_table_search(struct ib_uverbs_device *dev, 700 struct inode *inode) 701 { 702 struct xrcd_table_entry *entry; 703 struct rb_node *p = dev->xrcd_tree.rb_node; 704 705 while (p) { 706 entry = rb_entry(p, struct xrcd_table_entry, node); 707 708 if (inode < entry->inode) 709 p = p->rb_left; 710 else if (inode > entry->inode) 711 p = p->rb_right; 712 else 713 return entry; 714 } 715 716 return NULL; 717 } 718 719 static struct ib_xrcd *find_xrcd(struct ib_uverbs_device *dev, struct inode *inode) 720 { 721 struct xrcd_table_entry *entry; 722 723 entry = xrcd_table_search(dev, inode); 724 if (!entry) 725 return NULL; 726 727 return entry->xrcd; 728 } 729 730 static void xrcd_table_delete(struct ib_uverbs_device *dev, 731 struct inode *inode) 732 { 733 struct xrcd_table_entry *entry; 734 735 entry = xrcd_table_search(dev, inode); 736 if (entry) { 737 iput(inode); 738 rb_erase(&entry->node, &dev->xrcd_tree); 739 kfree(entry); 740 } 741 } 742 743 ssize_t ib_uverbs_open_xrcd(struct ib_uverbs_file *file, 744 struct ib_device *ib_dev, 745 const char __user *buf, int in_len, 746 int out_len) 747 { 748 struct ib_uverbs_open_xrcd cmd; 749 struct ib_uverbs_open_xrcd_resp resp; 750 struct ib_udata udata; 751 struct ib_uxrcd_object *obj; 752 struct ib_xrcd *xrcd = NULL; 753 struct fd f = {NULL, 0}; 754 struct inode *inode = NULL; 755 int ret = 0; 756 int new_xrcd = 0; 757 758 if (out_len < sizeof resp) 759 return -ENOSPC; 760 761 if (copy_from_user(&cmd, buf, sizeof cmd)) 762 return -EFAULT; 763 764 INIT_UDATA(&udata, buf + sizeof cmd, 765 (unsigned long) cmd.response + sizeof resp, 766 in_len - sizeof cmd, out_len - sizeof resp); 767 768 mutex_lock(&file->device->xrcd_tree_mutex); 769 770 if (cmd.fd != -1) { 771 /* search for file descriptor */ 772 f = fdget(cmd.fd); 773 if (!f.file) { 774 ret = -EBADF; 775 goto err_tree_mutex_unlock; 776 } 777 778 inode = file_inode(f.file); 779 xrcd = find_xrcd(file->device, inode); 780 if (!xrcd && !(cmd.oflags & O_CREAT)) { 781 /* no file descriptor. Need CREATE flag */ 782 ret = -EAGAIN; 783 goto err_tree_mutex_unlock; 784 } 785 786 if (xrcd && cmd.oflags & O_EXCL) { 787 ret = -EINVAL; 788 goto err_tree_mutex_unlock; 789 } 790 } 791 792 obj = kmalloc(sizeof *obj, GFP_KERNEL); 793 if (!obj) { 794 ret = -ENOMEM; 795 goto err_tree_mutex_unlock; 796 } 797 798 init_uobj(&obj->uobject, 0, file->ucontext, &xrcd_lock_class); 799 800 down_write(&obj->uobject.mutex); 801 802 if (!xrcd) { 803 xrcd = ib_dev->alloc_xrcd(ib_dev, file->ucontext, &udata); 804 if (IS_ERR(xrcd)) { 805 ret = PTR_ERR(xrcd); 806 goto err; 807 } 808 809 xrcd->inode = inode; 810 xrcd->device = ib_dev; 811 atomic_set(&xrcd->usecnt, 0); 812 mutex_init(&xrcd->tgt_qp_mutex); 813 INIT_LIST_HEAD(&xrcd->tgt_qp_list); 814 new_xrcd = 1; 815 } 816 817 atomic_set(&obj->refcnt, 0); 818 obj->uobject.object = xrcd; 819 ret = idr_add_uobj(&ib_uverbs_xrcd_idr, &obj->uobject); 820 if (ret) 821 goto err_idr; 822 823 memset(&resp, 0, sizeof resp); 824 resp.xrcd_handle = obj->uobject.id; 825 826 if (inode) { 827 if (new_xrcd) { 828 /* create new inode/xrcd table entry */ 829 ret = xrcd_table_insert(file->device, inode, xrcd); 830 if (ret) 831 goto err_insert_xrcd; 832 } 833 atomic_inc(&xrcd->usecnt); 834 } 835 836 if (copy_to_user((void __user *) (unsigned long) cmd.response, 837 &resp, sizeof resp)) { 838 ret = -EFAULT; 839 goto err_copy; 840 } 841 842 if (f.file) 843 fdput(f); 844 845 mutex_lock(&file->mutex); 846 list_add_tail(&obj->uobject.list, &file->ucontext->xrcd_list); 847 mutex_unlock(&file->mutex); 848 849 obj->uobject.live = 1; 850 up_write(&obj->uobject.mutex); 851 852 mutex_unlock(&file->device->xrcd_tree_mutex); 853 return in_len; 854 855 err_copy: 856 if (inode) { 857 if (new_xrcd) 858 xrcd_table_delete(file->device, inode); 859 atomic_dec(&xrcd->usecnt); 860 } 861 862 err_insert_xrcd: 863 idr_remove_uobj(&ib_uverbs_xrcd_idr, &obj->uobject); 864 865 err_idr: 866 ib_dealloc_xrcd(xrcd); 867 868 err: 869 put_uobj_write(&obj->uobject); 870 871 err_tree_mutex_unlock: 872 if (f.file) 873 fdput(f); 874 875 mutex_unlock(&file->device->xrcd_tree_mutex); 876 877 return ret; 878 } 879 880 ssize_t ib_uverbs_close_xrcd(struct ib_uverbs_file *file, 881 struct ib_device *ib_dev, 882 const char __user *buf, int in_len, 883 int out_len) 884 { 885 struct ib_uverbs_close_xrcd cmd; 886 struct ib_uobject *uobj; 887 struct ib_xrcd *xrcd = NULL; 888 struct inode *inode = NULL; 889 struct ib_uxrcd_object *obj; 890 int live; 891 int ret = 0; 892 893 if (copy_from_user(&cmd, buf, sizeof cmd)) 894 return -EFAULT; 895 896 mutex_lock(&file->device->xrcd_tree_mutex); 897 uobj = idr_write_uobj(&ib_uverbs_xrcd_idr, cmd.xrcd_handle, file->ucontext); 898 if (!uobj) { 899 ret = -EINVAL; 900 goto out; 901 } 902 903 xrcd = uobj->object; 904 inode = xrcd->inode; 905 obj = container_of(uobj, struct ib_uxrcd_object, uobject); 906 if (atomic_read(&obj->refcnt)) { 907 put_uobj_write(uobj); 908 ret = -EBUSY; 909 goto out; 910 } 911 912 if (!inode || atomic_dec_and_test(&xrcd->usecnt)) { 913 ret = ib_dealloc_xrcd(uobj->object); 914 if (!ret) 915 uobj->live = 0; 916 } 917 918 live = uobj->live; 919 if (inode && ret) 920 atomic_inc(&xrcd->usecnt); 921 922 put_uobj_write(uobj); 923 924 if (ret) 925 goto out; 926 927 if (inode && !live) 928 xrcd_table_delete(file->device, inode); 929 930 idr_remove_uobj(&ib_uverbs_xrcd_idr, uobj); 931 mutex_lock(&file->mutex); 932 list_del(&uobj->list); 933 mutex_unlock(&file->mutex); 934 935 put_uobj(uobj); 936 ret = in_len; 937 938 out: 939 mutex_unlock(&file->device->xrcd_tree_mutex); 940 return ret; 941 } 942 943 void ib_uverbs_dealloc_xrcd(struct ib_uverbs_device *dev, 944 struct ib_xrcd *xrcd) 945 { 946 struct inode *inode; 947 948 inode = xrcd->inode; 949 if (inode && !atomic_dec_and_test(&xrcd->usecnt)) 950 return; 951 952 ib_dealloc_xrcd(xrcd); 953 954 if (inode) 955 xrcd_table_delete(dev, inode); 956 } 957 958 ssize_t ib_uverbs_reg_mr(struct ib_uverbs_file *file, 959 struct ib_device *ib_dev, 960 const char __user *buf, int in_len, 961 int out_len) 962 { 963 struct ib_uverbs_reg_mr cmd; 964 struct ib_uverbs_reg_mr_resp resp; 965 struct ib_udata udata; 966 struct ib_uobject *uobj; 967 struct ib_pd *pd; 968 struct ib_mr *mr; 969 int ret; 970 971 if (out_len < sizeof resp) 972 return -ENOSPC; 973 974 if (copy_from_user(&cmd, buf, sizeof cmd)) 975 return -EFAULT; 976 977 INIT_UDATA(&udata, buf + sizeof cmd, 978 (unsigned long) cmd.response + sizeof resp, 979 in_len - sizeof cmd, out_len - sizeof resp); 980 981 if ((cmd.start & ~PAGE_MASK) != (cmd.hca_va & ~PAGE_MASK)) 982 return -EINVAL; 983 984 ret = ib_check_mr_access(cmd.access_flags); 985 if (ret) 986 return ret; 987 988 uobj = kmalloc(sizeof *uobj, GFP_KERNEL); 989 if (!uobj) 990 return -ENOMEM; 991 992 init_uobj(uobj, 0, file->ucontext, &mr_lock_class); 993 down_write(&uobj->mutex); 994 995 pd = idr_read_pd(cmd.pd_handle, file->ucontext); 996 if (!pd) { 997 ret = -EINVAL; 998 goto err_free; 999 } 1000 1001 if (cmd.access_flags & IB_ACCESS_ON_DEMAND) { 1002 if (!(pd->device->attrs.device_cap_flags & 1003 IB_DEVICE_ON_DEMAND_PAGING)) { 1004 pr_debug("ODP support not available\n"); 1005 ret = -EINVAL; 1006 goto err_put; 1007 } 1008 } 1009 1010 mr = pd->device->reg_user_mr(pd, cmd.start, cmd.length, cmd.hca_va, 1011 cmd.access_flags, &udata); 1012 if (IS_ERR(mr)) { 1013 ret = PTR_ERR(mr); 1014 goto err_put; 1015 } 1016 1017 mr->device = pd->device; 1018 mr->pd = pd; 1019 mr->uobject = uobj; 1020 atomic_inc(&pd->usecnt); 1021 1022 uobj->object = mr; 1023 ret = idr_add_uobj(&ib_uverbs_mr_idr, uobj); 1024 if (ret) 1025 goto err_unreg; 1026 1027 memset(&resp, 0, sizeof resp); 1028 resp.lkey = mr->lkey; 1029 resp.rkey = mr->rkey; 1030 resp.mr_handle = uobj->id; 1031 1032 if (copy_to_user((void __user *) (unsigned long) cmd.response, 1033 &resp, sizeof resp)) { 1034 ret = -EFAULT; 1035 goto err_copy; 1036 } 1037 1038 put_pd_read(pd); 1039 1040 mutex_lock(&file->mutex); 1041 list_add_tail(&uobj->list, &file->ucontext->mr_list); 1042 mutex_unlock(&file->mutex); 1043 1044 uobj->live = 1; 1045 1046 up_write(&uobj->mutex); 1047 1048 return in_len; 1049 1050 err_copy: 1051 idr_remove_uobj(&ib_uverbs_mr_idr, uobj); 1052 1053 err_unreg: 1054 ib_dereg_mr(mr); 1055 1056 err_put: 1057 put_pd_read(pd); 1058 1059 err_free: 1060 put_uobj_write(uobj); 1061 return ret; 1062 } 1063 1064 ssize_t ib_uverbs_rereg_mr(struct ib_uverbs_file *file, 1065 struct ib_device *ib_dev, 1066 const char __user *buf, int in_len, 1067 int out_len) 1068 { 1069 struct ib_uverbs_rereg_mr cmd; 1070 struct ib_uverbs_rereg_mr_resp resp; 1071 struct ib_udata udata; 1072 struct ib_pd *pd = NULL; 1073 struct ib_mr *mr; 1074 struct ib_pd *old_pd; 1075 int ret; 1076 struct ib_uobject *uobj; 1077 1078 if (out_len < sizeof(resp)) 1079 return -ENOSPC; 1080 1081 if (copy_from_user(&cmd, buf, sizeof(cmd))) 1082 return -EFAULT; 1083 1084 INIT_UDATA(&udata, buf + sizeof(cmd), 1085 (unsigned long) cmd.response + sizeof(resp), 1086 in_len - sizeof(cmd), out_len - sizeof(resp)); 1087 1088 if (cmd.flags & ~IB_MR_REREG_SUPPORTED || !cmd.flags) 1089 return -EINVAL; 1090 1091 if ((cmd.flags & IB_MR_REREG_TRANS) && 1092 (!cmd.start || !cmd.hca_va || 0 >= cmd.length || 1093 (cmd.start & ~PAGE_MASK) != (cmd.hca_va & ~PAGE_MASK))) 1094 return -EINVAL; 1095 1096 uobj = idr_write_uobj(&ib_uverbs_mr_idr, cmd.mr_handle, 1097 file->ucontext); 1098 1099 if (!uobj) 1100 return -EINVAL; 1101 1102 mr = uobj->object; 1103 1104 if (cmd.flags & IB_MR_REREG_ACCESS) { 1105 ret = ib_check_mr_access(cmd.access_flags); 1106 if (ret) 1107 goto put_uobjs; 1108 } 1109 1110 if (cmd.flags & IB_MR_REREG_PD) { 1111 pd = idr_read_pd(cmd.pd_handle, file->ucontext); 1112 if (!pd) { 1113 ret = -EINVAL; 1114 goto put_uobjs; 1115 } 1116 } 1117 1118 old_pd = mr->pd; 1119 ret = mr->device->rereg_user_mr(mr, cmd.flags, cmd.start, 1120 cmd.length, cmd.hca_va, 1121 cmd.access_flags, pd, &udata); 1122 if (!ret) { 1123 if (cmd.flags & IB_MR_REREG_PD) { 1124 atomic_inc(&pd->usecnt); 1125 mr->pd = pd; 1126 atomic_dec(&old_pd->usecnt); 1127 } 1128 } else { 1129 goto put_uobj_pd; 1130 } 1131 1132 memset(&resp, 0, sizeof(resp)); 1133 resp.lkey = mr->lkey; 1134 resp.rkey = mr->rkey; 1135 1136 if (copy_to_user((void __user *)(unsigned long)cmd.response, 1137 &resp, sizeof(resp))) 1138 ret = -EFAULT; 1139 else 1140 ret = in_len; 1141 1142 put_uobj_pd: 1143 if (cmd.flags & IB_MR_REREG_PD) 1144 put_pd_read(pd); 1145 1146 put_uobjs: 1147 1148 put_uobj_write(mr->uobject); 1149 1150 return ret; 1151 } 1152 1153 ssize_t ib_uverbs_dereg_mr(struct ib_uverbs_file *file, 1154 struct ib_device *ib_dev, 1155 const char __user *buf, int in_len, 1156 int out_len) 1157 { 1158 struct ib_uverbs_dereg_mr cmd; 1159 struct ib_mr *mr; 1160 struct ib_uobject *uobj; 1161 int ret = -EINVAL; 1162 1163 if (copy_from_user(&cmd, buf, sizeof cmd)) 1164 return -EFAULT; 1165 1166 uobj = idr_write_uobj(&ib_uverbs_mr_idr, cmd.mr_handle, file->ucontext); 1167 if (!uobj) 1168 return -EINVAL; 1169 1170 mr = uobj->object; 1171 1172 ret = ib_dereg_mr(mr); 1173 if (!ret) 1174 uobj->live = 0; 1175 1176 put_uobj_write(uobj); 1177 1178 if (ret) 1179 return ret; 1180 1181 idr_remove_uobj(&ib_uverbs_mr_idr, uobj); 1182 1183 mutex_lock(&file->mutex); 1184 list_del(&uobj->list); 1185 mutex_unlock(&file->mutex); 1186 1187 put_uobj(uobj); 1188 1189 return in_len; 1190 } 1191 1192 ssize_t ib_uverbs_alloc_mw(struct ib_uverbs_file *file, 1193 struct ib_device *ib_dev, 1194 const char __user *buf, int in_len, 1195 int out_len) 1196 { 1197 struct ib_uverbs_alloc_mw cmd; 1198 struct ib_uverbs_alloc_mw_resp resp; 1199 struct ib_uobject *uobj; 1200 struct ib_pd *pd; 1201 struct ib_mw *mw; 1202 struct ib_udata udata; 1203 int ret; 1204 1205 if (out_len < sizeof(resp)) 1206 return -ENOSPC; 1207 1208 if (copy_from_user(&cmd, buf, sizeof(cmd))) 1209 return -EFAULT; 1210 1211 uobj = kmalloc(sizeof(*uobj), GFP_KERNEL); 1212 if (!uobj) 1213 return -ENOMEM; 1214 1215 init_uobj(uobj, 0, file->ucontext, &mw_lock_class); 1216 down_write(&uobj->mutex); 1217 1218 pd = idr_read_pd(cmd.pd_handle, file->ucontext); 1219 if (!pd) { 1220 ret = -EINVAL; 1221 goto err_free; 1222 } 1223 1224 INIT_UDATA(&udata, buf + sizeof(cmd), 1225 (unsigned long)cmd.response + sizeof(resp), 1226 in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr), 1227 out_len - sizeof(resp)); 1228 1229 mw = pd->device->alloc_mw(pd, cmd.mw_type, &udata); 1230 if (IS_ERR(mw)) { 1231 ret = PTR_ERR(mw); 1232 goto err_put; 1233 } 1234 1235 mw->device = pd->device; 1236 mw->pd = pd; 1237 mw->uobject = uobj; 1238 atomic_inc(&pd->usecnt); 1239 1240 uobj->object = mw; 1241 ret = idr_add_uobj(&ib_uverbs_mw_idr, uobj); 1242 if (ret) 1243 goto err_unalloc; 1244 1245 memset(&resp, 0, sizeof(resp)); 1246 resp.rkey = mw->rkey; 1247 resp.mw_handle = uobj->id; 1248 1249 if (copy_to_user((void __user *)(unsigned long)cmd.response, 1250 &resp, sizeof(resp))) { 1251 ret = -EFAULT; 1252 goto err_copy; 1253 } 1254 1255 put_pd_read(pd); 1256 1257 mutex_lock(&file->mutex); 1258 list_add_tail(&uobj->list, &file->ucontext->mw_list); 1259 mutex_unlock(&file->mutex); 1260 1261 uobj->live = 1; 1262 1263 up_write(&uobj->mutex); 1264 1265 return in_len; 1266 1267 err_copy: 1268 idr_remove_uobj(&ib_uverbs_mw_idr, uobj); 1269 1270 err_unalloc: 1271 uverbs_dealloc_mw(mw); 1272 1273 err_put: 1274 put_pd_read(pd); 1275 1276 err_free: 1277 put_uobj_write(uobj); 1278 return ret; 1279 } 1280 1281 ssize_t ib_uverbs_dealloc_mw(struct ib_uverbs_file *file, 1282 struct ib_device *ib_dev, 1283 const char __user *buf, int in_len, 1284 int out_len) 1285 { 1286 struct ib_uverbs_dealloc_mw cmd; 1287 struct ib_mw *mw; 1288 struct ib_uobject *uobj; 1289 int ret = -EINVAL; 1290 1291 if (copy_from_user(&cmd, buf, sizeof(cmd))) 1292 return -EFAULT; 1293 1294 uobj = idr_write_uobj(&ib_uverbs_mw_idr, cmd.mw_handle, file->ucontext); 1295 if (!uobj) 1296 return -EINVAL; 1297 1298 mw = uobj->object; 1299 1300 ret = uverbs_dealloc_mw(mw); 1301 if (!ret) 1302 uobj->live = 0; 1303 1304 put_uobj_write(uobj); 1305 1306 if (ret) 1307 return ret; 1308 1309 idr_remove_uobj(&ib_uverbs_mw_idr, uobj); 1310 1311 mutex_lock(&file->mutex); 1312 list_del(&uobj->list); 1313 mutex_unlock(&file->mutex); 1314 1315 put_uobj(uobj); 1316 1317 return in_len; 1318 } 1319 1320 ssize_t ib_uverbs_create_comp_channel(struct ib_uverbs_file *file, 1321 struct ib_device *ib_dev, 1322 const char __user *buf, int in_len, 1323 int out_len) 1324 { 1325 struct ib_uverbs_create_comp_channel cmd; 1326 struct ib_uverbs_create_comp_channel_resp resp; 1327 struct file *filp; 1328 int ret; 1329 1330 if (out_len < sizeof resp) 1331 return -ENOSPC; 1332 1333 if (copy_from_user(&cmd, buf, sizeof cmd)) 1334 return -EFAULT; 1335 1336 ret = get_unused_fd_flags(O_CLOEXEC); 1337 if (ret < 0) 1338 return ret; 1339 resp.fd = ret; 1340 1341 filp = ib_uverbs_alloc_event_file(file, ib_dev, 0); 1342 if (IS_ERR(filp)) { 1343 put_unused_fd(resp.fd); 1344 return PTR_ERR(filp); 1345 } 1346 1347 if (copy_to_user((void __user *) (unsigned long) cmd.response, 1348 &resp, sizeof resp)) { 1349 put_unused_fd(resp.fd); 1350 fput(filp); 1351 return -EFAULT; 1352 } 1353 1354 fd_install(resp.fd, filp); 1355 return in_len; 1356 } 1357 1358 static struct ib_ucq_object *create_cq(struct ib_uverbs_file *file, 1359 struct ib_device *ib_dev, 1360 struct ib_udata *ucore, 1361 struct ib_udata *uhw, 1362 struct ib_uverbs_ex_create_cq *cmd, 1363 size_t cmd_sz, 1364 int (*cb)(struct ib_uverbs_file *file, 1365 struct ib_ucq_object *obj, 1366 struct ib_uverbs_ex_create_cq_resp *resp, 1367 struct ib_udata *udata, 1368 void *context), 1369 void *context) 1370 { 1371 struct ib_ucq_object *obj; 1372 struct ib_uverbs_event_file *ev_file = NULL; 1373 struct ib_cq *cq; 1374 int ret; 1375 struct ib_uverbs_ex_create_cq_resp resp; 1376 struct ib_cq_init_attr attr = {}; 1377 1378 if (cmd->comp_vector >= file->device->num_comp_vectors) 1379 return ERR_PTR(-EINVAL); 1380 1381 obj = kmalloc(sizeof *obj, GFP_KERNEL); 1382 if (!obj) 1383 return ERR_PTR(-ENOMEM); 1384 1385 init_uobj(&obj->uobject, cmd->user_handle, file->ucontext, &cq_lock_class); 1386 down_write(&obj->uobject.mutex); 1387 1388 if (cmd->comp_channel >= 0) { 1389 ev_file = ib_uverbs_lookup_comp_file(cmd->comp_channel); 1390 if (!ev_file) { 1391 ret = -EINVAL; 1392 goto err; 1393 } 1394 } 1395 1396 obj->uverbs_file = file; 1397 obj->comp_events_reported = 0; 1398 obj->async_events_reported = 0; 1399 INIT_LIST_HEAD(&obj->comp_list); 1400 INIT_LIST_HEAD(&obj->async_list); 1401 1402 attr.cqe = cmd->cqe; 1403 attr.comp_vector = cmd->comp_vector; 1404 1405 if (cmd_sz > offsetof(typeof(*cmd), flags) + sizeof(cmd->flags)) 1406 attr.flags = cmd->flags; 1407 1408 cq = ib_dev->create_cq(ib_dev, &attr, 1409 file->ucontext, uhw); 1410 if (IS_ERR(cq)) { 1411 ret = PTR_ERR(cq); 1412 goto err_file; 1413 } 1414 1415 cq->device = ib_dev; 1416 cq->uobject = &obj->uobject; 1417 cq->comp_handler = ib_uverbs_comp_handler; 1418 cq->event_handler = ib_uverbs_cq_event_handler; 1419 cq->cq_context = ev_file; 1420 atomic_set(&cq->usecnt, 0); 1421 1422 obj->uobject.object = cq; 1423 ret = idr_add_uobj(&ib_uverbs_cq_idr, &obj->uobject); 1424 if (ret) 1425 goto err_free; 1426 1427 memset(&resp, 0, sizeof resp); 1428 resp.base.cq_handle = obj->uobject.id; 1429 resp.base.cqe = cq->cqe; 1430 1431 resp.response_length = offsetof(typeof(resp), response_length) + 1432 sizeof(resp.response_length); 1433 1434 ret = cb(file, obj, &resp, ucore, context); 1435 if (ret) 1436 goto err_cb; 1437 1438 mutex_lock(&file->mutex); 1439 list_add_tail(&obj->uobject.list, &file->ucontext->cq_list); 1440 mutex_unlock(&file->mutex); 1441 1442 obj->uobject.live = 1; 1443 1444 up_write(&obj->uobject.mutex); 1445 1446 return obj; 1447 1448 err_cb: 1449 idr_remove_uobj(&ib_uverbs_cq_idr, &obj->uobject); 1450 1451 err_free: 1452 ib_destroy_cq(cq); 1453 1454 err_file: 1455 if (ev_file) 1456 ib_uverbs_release_ucq(file, ev_file, obj); 1457 1458 err: 1459 put_uobj_write(&obj->uobject); 1460 1461 return ERR_PTR(ret); 1462 } 1463 1464 static int ib_uverbs_create_cq_cb(struct ib_uverbs_file *file, 1465 struct ib_ucq_object *obj, 1466 struct ib_uverbs_ex_create_cq_resp *resp, 1467 struct ib_udata *ucore, void *context) 1468 { 1469 if (ib_copy_to_udata(ucore, &resp->base, sizeof(resp->base))) 1470 return -EFAULT; 1471 1472 return 0; 1473 } 1474 1475 ssize_t ib_uverbs_create_cq(struct ib_uverbs_file *file, 1476 struct ib_device *ib_dev, 1477 const char __user *buf, int in_len, 1478 int out_len) 1479 { 1480 struct ib_uverbs_create_cq cmd; 1481 struct ib_uverbs_ex_create_cq cmd_ex; 1482 struct ib_uverbs_create_cq_resp resp; 1483 struct ib_udata ucore; 1484 struct ib_udata uhw; 1485 struct ib_ucq_object *obj; 1486 1487 if (out_len < sizeof(resp)) 1488 return -ENOSPC; 1489 1490 if (copy_from_user(&cmd, buf, sizeof(cmd))) 1491 return -EFAULT; 1492 1493 INIT_UDATA(&ucore, buf, (unsigned long)cmd.response, sizeof(cmd), sizeof(resp)); 1494 1495 INIT_UDATA(&uhw, buf + sizeof(cmd), 1496 (unsigned long)cmd.response + sizeof(resp), 1497 in_len - sizeof(cmd), out_len - sizeof(resp)); 1498 1499 memset(&cmd_ex, 0, sizeof(cmd_ex)); 1500 cmd_ex.user_handle = cmd.user_handle; 1501 cmd_ex.cqe = cmd.cqe; 1502 cmd_ex.comp_vector = cmd.comp_vector; 1503 cmd_ex.comp_channel = cmd.comp_channel; 1504 1505 obj = create_cq(file, ib_dev, &ucore, &uhw, &cmd_ex, 1506 offsetof(typeof(cmd_ex), comp_channel) + 1507 sizeof(cmd.comp_channel), ib_uverbs_create_cq_cb, 1508 NULL); 1509 1510 if (IS_ERR(obj)) 1511 return PTR_ERR(obj); 1512 1513 return in_len; 1514 } 1515 1516 static int ib_uverbs_ex_create_cq_cb(struct ib_uverbs_file *file, 1517 struct ib_ucq_object *obj, 1518 struct ib_uverbs_ex_create_cq_resp *resp, 1519 struct ib_udata *ucore, void *context) 1520 { 1521 if (ib_copy_to_udata(ucore, resp, resp->response_length)) 1522 return -EFAULT; 1523 1524 return 0; 1525 } 1526 1527 int ib_uverbs_ex_create_cq(struct ib_uverbs_file *file, 1528 struct ib_device *ib_dev, 1529 struct ib_udata *ucore, 1530 struct ib_udata *uhw) 1531 { 1532 struct ib_uverbs_ex_create_cq_resp resp; 1533 struct ib_uverbs_ex_create_cq cmd; 1534 struct ib_ucq_object *obj; 1535 int err; 1536 1537 if (ucore->inlen < sizeof(cmd)) 1538 return -EINVAL; 1539 1540 err = ib_copy_from_udata(&cmd, ucore, sizeof(cmd)); 1541 if (err) 1542 return err; 1543 1544 if (cmd.comp_mask) 1545 return -EINVAL; 1546 1547 if (cmd.reserved) 1548 return -EINVAL; 1549 1550 if (ucore->outlen < (offsetof(typeof(resp), response_length) + 1551 sizeof(resp.response_length))) 1552 return -ENOSPC; 1553 1554 obj = create_cq(file, ib_dev, ucore, uhw, &cmd, 1555 min(ucore->inlen, sizeof(cmd)), 1556 ib_uverbs_ex_create_cq_cb, NULL); 1557 1558 if (IS_ERR(obj)) 1559 return PTR_ERR(obj); 1560 1561 return 0; 1562 } 1563 1564 ssize_t ib_uverbs_resize_cq(struct ib_uverbs_file *file, 1565 struct ib_device *ib_dev, 1566 const char __user *buf, int in_len, 1567 int out_len) 1568 { 1569 struct ib_uverbs_resize_cq cmd; 1570 struct ib_uverbs_resize_cq_resp resp; 1571 struct ib_udata udata; 1572 struct ib_cq *cq; 1573 int ret = -EINVAL; 1574 1575 if (copy_from_user(&cmd, buf, sizeof cmd)) 1576 return -EFAULT; 1577 1578 INIT_UDATA(&udata, buf + sizeof cmd, 1579 (unsigned long) cmd.response + sizeof resp, 1580 in_len - sizeof cmd, out_len - sizeof resp); 1581 1582 cq = idr_read_cq(cmd.cq_handle, file->ucontext, 0); 1583 if (!cq) 1584 return -EINVAL; 1585 1586 ret = cq->device->resize_cq(cq, cmd.cqe, &udata); 1587 if (ret) 1588 goto out; 1589 1590 resp.cqe = cq->cqe; 1591 1592 if (copy_to_user((void __user *) (unsigned long) cmd.response, 1593 &resp, sizeof resp.cqe)) 1594 ret = -EFAULT; 1595 1596 out: 1597 put_cq_read(cq); 1598 1599 return ret ? ret : in_len; 1600 } 1601 1602 static int copy_wc_to_user(void __user *dest, struct ib_wc *wc) 1603 { 1604 struct ib_uverbs_wc tmp; 1605 1606 tmp.wr_id = wc->wr_id; 1607 tmp.status = wc->status; 1608 tmp.opcode = wc->opcode; 1609 tmp.vendor_err = wc->vendor_err; 1610 tmp.byte_len = wc->byte_len; 1611 tmp.ex.imm_data = (__u32 __force) wc->ex.imm_data; 1612 tmp.qp_num = wc->qp->qp_num; 1613 tmp.src_qp = wc->src_qp; 1614 tmp.wc_flags = wc->wc_flags; 1615 tmp.pkey_index = wc->pkey_index; 1616 tmp.slid = wc->slid; 1617 tmp.sl = wc->sl; 1618 tmp.dlid_path_bits = wc->dlid_path_bits; 1619 tmp.port_num = wc->port_num; 1620 tmp.reserved = 0; 1621 1622 if (copy_to_user(dest, &tmp, sizeof tmp)) 1623 return -EFAULT; 1624 1625 return 0; 1626 } 1627 1628 ssize_t ib_uverbs_poll_cq(struct ib_uverbs_file *file, 1629 struct ib_device *ib_dev, 1630 const char __user *buf, int in_len, 1631 int out_len) 1632 { 1633 struct ib_uverbs_poll_cq cmd; 1634 struct ib_uverbs_poll_cq_resp resp; 1635 u8 __user *header_ptr; 1636 u8 __user *data_ptr; 1637 struct ib_cq *cq; 1638 struct ib_wc wc; 1639 int ret; 1640 1641 if (copy_from_user(&cmd, buf, sizeof cmd)) 1642 return -EFAULT; 1643 1644 cq = idr_read_cq(cmd.cq_handle, file->ucontext, 0); 1645 if (!cq) 1646 return -EINVAL; 1647 1648 /* we copy a struct ib_uverbs_poll_cq_resp to user space */ 1649 header_ptr = (void __user *)(unsigned long) cmd.response; 1650 data_ptr = header_ptr + sizeof resp; 1651 1652 memset(&resp, 0, sizeof resp); 1653 while (resp.count < cmd.ne) { 1654 ret = ib_poll_cq(cq, 1, &wc); 1655 if (ret < 0) 1656 goto out_put; 1657 if (!ret) 1658 break; 1659 1660 ret = copy_wc_to_user(data_ptr, &wc); 1661 if (ret) 1662 goto out_put; 1663 1664 data_ptr += sizeof(struct ib_uverbs_wc); 1665 ++resp.count; 1666 } 1667 1668 if (copy_to_user(header_ptr, &resp, sizeof resp)) { 1669 ret = -EFAULT; 1670 goto out_put; 1671 } 1672 1673 ret = in_len; 1674 1675 out_put: 1676 put_cq_read(cq); 1677 return ret; 1678 } 1679 1680 ssize_t ib_uverbs_req_notify_cq(struct ib_uverbs_file *file, 1681 struct ib_device *ib_dev, 1682 const char __user *buf, int in_len, 1683 int out_len) 1684 { 1685 struct ib_uverbs_req_notify_cq cmd; 1686 struct ib_cq *cq; 1687 1688 if (copy_from_user(&cmd, buf, sizeof cmd)) 1689 return -EFAULT; 1690 1691 cq = idr_read_cq(cmd.cq_handle, file->ucontext, 0); 1692 if (!cq) 1693 return -EINVAL; 1694 1695 ib_req_notify_cq(cq, cmd.solicited_only ? 1696 IB_CQ_SOLICITED : IB_CQ_NEXT_COMP); 1697 1698 put_cq_read(cq); 1699 1700 return in_len; 1701 } 1702 1703 ssize_t ib_uverbs_destroy_cq(struct ib_uverbs_file *file, 1704 struct ib_device *ib_dev, 1705 const char __user *buf, int in_len, 1706 int out_len) 1707 { 1708 struct ib_uverbs_destroy_cq cmd; 1709 struct ib_uverbs_destroy_cq_resp resp; 1710 struct ib_uobject *uobj; 1711 struct ib_cq *cq; 1712 struct ib_ucq_object *obj; 1713 struct ib_uverbs_event_file *ev_file; 1714 int ret = -EINVAL; 1715 1716 if (copy_from_user(&cmd, buf, sizeof cmd)) 1717 return -EFAULT; 1718 1719 uobj = idr_write_uobj(&ib_uverbs_cq_idr, cmd.cq_handle, file->ucontext); 1720 if (!uobj) 1721 return -EINVAL; 1722 cq = uobj->object; 1723 ev_file = cq->cq_context; 1724 obj = container_of(cq->uobject, struct ib_ucq_object, uobject); 1725 1726 ret = ib_destroy_cq(cq); 1727 if (!ret) 1728 uobj->live = 0; 1729 1730 put_uobj_write(uobj); 1731 1732 if (ret) 1733 return ret; 1734 1735 idr_remove_uobj(&ib_uverbs_cq_idr, uobj); 1736 1737 mutex_lock(&file->mutex); 1738 list_del(&uobj->list); 1739 mutex_unlock(&file->mutex); 1740 1741 ib_uverbs_release_ucq(file, ev_file, obj); 1742 1743 memset(&resp, 0, sizeof resp); 1744 resp.comp_events_reported = obj->comp_events_reported; 1745 resp.async_events_reported = obj->async_events_reported; 1746 1747 put_uobj(uobj); 1748 1749 if (copy_to_user((void __user *) (unsigned long) cmd.response, 1750 &resp, sizeof resp)) 1751 return -EFAULT; 1752 1753 return in_len; 1754 } 1755 1756 static int create_qp(struct ib_uverbs_file *file, 1757 struct ib_udata *ucore, 1758 struct ib_udata *uhw, 1759 struct ib_uverbs_ex_create_qp *cmd, 1760 size_t cmd_sz, 1761 int (*cb)(struct ib_uverbs_file *file, 1762 struct ib_uverbs_ex_create_qp_resp *resp, 1763 struct ib_udata *udata), 1764 void *context) 1765 { 1766 struct ib_uqp_object *obj; 1767 struct ib_device *device; 1768 struct ib_pd *pd = NULL; 1769 struct ib_xrcd *xrcd = NULL; 1770 struct ib_uobject *uninitialized_var(xrcd_uobj); 1771 struct ib_cq *scq = NULL, *rcq = NULL; 1772 struct ib_srq *srq = NULL; 1773 struct ib_qp *qp; 1774 char *buf; 1775 struct ib_qp_init_attr attr = {}; 1776 struct ib_uverbs_ex_create_qp_resp resp; 1777 int ret; 1778 struct ib_rwq_ind_table *ind_tbl = NULL; 1779 bool has_sq = true; 1780 1781 if (cmd->qp_type == IB_QPT_RAW_PACKET && !capable(CAP_NET_RAW)) 1782 return -EPERM; 1783 1784 obj = kzalloc(sizeof *obj, GFP_KERNEL); 1785 if (!obj) 1786 return -ENOMEM; 1787 1788 init_uobj(&obj->uevent.uobject, cmd->user_handle, file->ucontext, 1789 &qp_lock_class); 1790 down_write(&obj->uevent.uobject.mutex); 1791 if (cmd_sz >= offsetof(typeof(*cmd), rwq_ind_tbl_handle) + 1792 sizeof(cmd->rwq_ind_tbl_handle) && 1793 (cmd->comp_mask & IB_UVERBS_CREATE_QP_MASK_IND_TABLE)) { 1794 ind_tbl = idr_read_rwq_indirection_table(cmd->rwq_ind_tbl_handle, 1795 file->ucontext); 1796 if (!ind_tbl) { 1797 ret = -EINVAL; 1798 goto err_put; 1799 } 1800 1801 attr.rwq_ind_tbl = ind_tbl; 1802 } 1803 1804 if ((cmd_sz >= offsetof(typeof(*cmd), reserved1) + 1805 sizeof(cmd->reserved1)) && cmd->reserved1) { 1806 ret = -EOPNOTSUPP; 1807 goto err_put; 1808 } 1809 1810 if (ind_tbl && (cmd->max_recv_wr || cmd->max_recv_sge || cmd->is_srq)) { 1811 ret = -EINVAL; 1812 goto err_put; 1813 } 1814 1815 if (ind_tbl && !cmd->max_send_wr) 1816 has_sq = false; 1817 1818 if (cmd->qp_type == IB_QPT_XRC_TGT) { 1819 xrcd = idr_read_xrcd(cmd->pd_handle, file->ucontext, 1820 &xrcd_uobj); 1821 if (!xrcd) { 1822 ret = -EINVAL; 1823 goto err_put; 1824 } 1825 device = xrcd->device; 1826 } else { 1827 if (cmd->qp_type == IB_QPT_XRC_INI) { 1828 cmd->max_recv_wr = 0; 1829 cmd->max_recv_sge = 0; 1830 } else { 1831 if (cmd->is_srq) { 1832 srq = idr_read_srq(cmd->srq_handle, 1833 file->ucontext); 1834 if (!srq || srq->srq_type != IB_SRQT_BASIC) { 1835 ret = -EINVAL; 1836 goto err_put; 1837 } 1838 } 1839 1840 if (!ind_tbl) { 1841 if (cmd->recv_cq_handle != cmd->send_cq_handle) { 1842 rcq = idr_read_cq(cmd->recv_cq_handle, 1843 file->ucontext, 0); 1844 if (!rcq) { 1845 ret = -EINVAL; 1846 goto err_put; 1847 } 1848 } 1849 } 1850 } 1851 1852 if (has_sq) 1853 scq = idr_read_cq(cmd->send_cq_handle, file->ucontext, !!rcq); 1854 if (!ind_tbl) 1855 rcq = rcq ?: scq; 1856 pd = idr_read_pd(cmd->pd_handle, file->ucontext); 1857 if (!pd || (!scq && has_sq)) { 1858 ret = -EINVAL; 1859 goto err_put; 1860 } 1861 1862 device = pd->device; 1863 } 1864 1865 attr.event_handler = ib_uverbs_qp_event_handler; 1866 attr.qp_context = file; 1867 attr.send_cq = scq; 1868 attr.recv_cq = rcq; 1869 attr.srq = srq; 1870 attr.xrcd = xrcd; 1871 attr.sq_sig_type = cmd->sq_sig_all ? IB_SIGNAL_ALL_WR : 1872 IB_SIGNAL_REQ_WR; 1873 attr.qp_type = cmd->qp_type; 1874 attr.create_flags = 0; 1875 1876 attr.cap.max_send_wr = cmd->max_send_wr; 1877 attr.cap.max_recv_wr = cmd->max_recv_wr; 1878 attr.cap.max_send_sge = cmd->max_send_sge; 1879 attr.cap.max_recv_sge = cmd->max_recv_sge; 1880 attr.cap.max_inline_data = cmd->max_inline_data; 1881 1882 obj->uevent.events_reported = 0; 1883 INIT_LIST_HEAD(&obj->uevent.event_list); 1884 INIT_LIST_HEAD(&obj->mcast_list); 1885 1886 if (cmd_sz >= offsetof(typeof(*cmd), create_flags) + 1887 sizeof(cmd->create_flags)) 1888 attr.create_flags = cmd->create_flags; 1889 1890 if (attr.create_flags & ~(IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK | 1891 IB_QP_CREATE_CROSS_CHANNEL | 1892 IB_QP_CREATE_MANAGED_SEND | 1893 IB_QP_CREATE_MANAGED_RECV | 1894 IB_QP_CREATE_SCATTER_FCS)) { 1895 ret = -EINVAL; 1896 goto err_put; 1897 } 1898 1899 buf = (void *)cmd + sizeof(*cmd); 1900 if (cmd_sz > sizeof(*cmd)) 1901 if (!(buf[0] == 0 && !memcmp(buf, buf + 1, 1902 cmd_sz - sizeof(*cmd) - 1))) { 1903 ret = -EINVAL; 1904 goto err_put; 1905 } 1906 1907 if (cmd->qp_type == IB_QPT_XRC_TGT) 1908 qp = ib_create_qp(pd, &attr); 1909 else 1910 qp = device->create_qp(pd, &attr, uhw); 1911 1912 if (IS_ERR(qp)) { 1913 ret = PTR_ERR(qp); 1914 goto err_put; 1915 } 1916 1917 if (cmd->qp_type != IB_QPT_XRC_TGT) { 1918 qp->real_qp = qp; 1919 qp->device = device; 1920 qp->pd = pd; 1921 qp->send_cq = attr.send_cq; 1922 qp->recv_cq = attr.recv_cq; 1923 qp->srq = attr.srq; 1924 qp->rwq_ind_tbl = ind_tbl; 1925 qp->event_handler = attr.event_handler; 1926 qp->qp_context = attr.qp_context; 1927 qp->qp_type = attr.qp_type; 1928 atomic_set(&qp->usecnt, 0); 1929 atomic_inc(&pd->usecnt); 1930 if (attr.send_cq) 1931 atomic_inc(&attr.send_cq->usecnt); 1932 if (attr.recv_cq) 1933 atomic_inc(&attr.recv_cq->usecnt); 1934 if (attr.srq) 1935 atomic_inc(&attr.srq->usecnt); 1936 if (ind_tbl) 1937 atomic_inc(&ind_tbl->usecnt); 1938 } 1939 qp->uobject = &obj->uevent.uobject; 1940 1941 obj->uevent.uobject.object = qp; 1942 ret = idr_add_uobj(&ib_uverbs_qp_idr, &obj->uevent.uobject); 1943 if (ret) 1944 goto err_destroy; 1945 1946 memset(&resp, 0, sizeof resp); 1947 resp.base.qpn = qp->qp_num; 1948 resp.base.qp_handle = obj->uevent.uobject.id; 1949 resp.base.max_recv_sge = attr.cap.max_recv_sge; 1950 resp.base.max_send_sge = attr.cap.max_send_sge; 1951 resp.base.max_recv_wr = attr.cap.max_recv_wr; 1952 resp.base.max_send_wr = attr.cap.max_send_wr; 1953 resp.base.max_inline_data = attr.cap.max_inline_data; 1954 1955 resp.response_length = offsetof(typeof(resp), response_length) + 1956 sizeof(resp.response_length); 1957 1958 ret = cb(file, &resp, ucore); 1959 if (ret) 1960 goto err_cb; 1961 1962 if (xrcd) { 1963 obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object, 1964 uobject); 1965 atomic_inc(&obj->uxrcd->refcnt); 1966 put_xrcd_read(xrcd_uobj); 1967 } 1968 1969 if (pd) 1970 put_pd_read(pd); 1971 if (scq) 1972 put_cq_read(scq); 1973 if (rcq && rcq != scq) 1974 put_cq_read(rcq); 1975 if (srq) 1976 put_srq_read(srq); 1977 if (ind_tbl) 1978 put_rwq_indirection_table_read(ind_tbl); 1979 1980 mutex_lock(&file->mutex); 1981 list_add_tail(&obj->uevent.uobject.list, &file->ucontext->qp_list); 1982 mutex_unlock(&file->mutex); 1983 1984 obj->uevent.uobject.live = 1; 1985 1986 up_write(&obj->uevent.uobject.mutex); 1987 1988 return 0; 1989 err_cb: 1990 idr_remove_uobj(&ib_uverbs_qp_idr, &obj->uevent.uobject); 1991 1992 err_destroy: 1993 ib_destroy_qp(qp); 1994 1995 err_put: 1996 if (xrcd) 1997 put_xrcd_read(xrcd_uobj); 1998 if (pd) 1999 put_pd_read(pd); 2000 if (scq) 2001 put_cq_read(scq); 2002 if (rcq && rcq != scq) 2003 put_cq_read(rcq); 2004 if (srq) 2005 put_srq_read(srq); 2006 if (ind_tbl) 2007 put_rwq_indirection_table_read(ind_tbl); 2008 2009 put_uobj_write(&obj->uevent.uobject); 2010 return ret; 2011 } 2012 2013 static int ib_uverbs_create_qp_cb(struct ib_uverbs_file *file, 2014 struct ib_uverbs_ex_create_qp_resp *resp, 2015 struct ib_udata *ucore) 2016 { 2017 if (ib_copy_to_udata(ucore, &resp->base, sizeof(resp->base))) 2018 return -EFAULT; 2019 2020 return 0; 2021 } 2022 2023 ssize_t ib_uverbs_create_qp(struct ib_uverbs_file *file, 2024 struct ib_device *ib_dev, 2025 const char __user *buf, int in_len, 2026 int out_len) 2027 { 2028 struct ib_uverbs_create_qp cmd; 2029 struct ib_uverbs_ex_create_qp cmd_ex; 2030 struct ib_udata ucore; 2031 struct ib_udata uhw; 2032 ssize_t resp_size = sizeof(struct ib_uverbs_create_qp_resp); 2033 int err; 2034 2035 if (out_len < resp_size) 2036 return -ENOSPC; 2037 2038 if (copy_from_user(&cmd, buf, sizeof(cmd))) 2039 return -EFAULT; 2040 2041 INIT_UDATA(&ucore, buf, (unsigned long)cmd.response, sizeof(cmd), 2042 resp_size); 2043 INIT_UDATA(&uhw, buf + sizeof(cmd), 2044 (unsigned long)cmd.response + resp_size, 2045 in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr), 2046 out_len - resp_size); 2047 2048 memset(&cmd_ex, 0, sizeof(cmd_ex)); 2049 cmd_ex.user_handle = cmd.user_handle; 2050 cmd_ex.pd_handle = cmd.pd_handle; 2051 cmd_ex.send_cq_handle = cmd.send_cq_handle; 2052 cmd_ex.recv_cq_handle = cmd.recv_cq_handle; 2053 cmd_ex.srq_handle = cmd.srq_handle; 2054 cmd_ex.max_send_wr = cmd.max_send_wr; 2055 cmd_ex.max_recv_wr = cmd.max_recv_wr; 2056 cmd_ex.max_send_sge = cmd.max_send_sge; 2057 cmd_ex.max_recv_sge = cmd.max_recv_sge; 2058 cmd_ex.max_inline_data = cmd.max_inline_data; 2059 cmd_ex.sq_sig_all = cmd.sq_sig_all; 2060 cmd_ex.qp_type = cmd.qp_type; 2061 cmd_ex.is_srq = cmd.is_srq; 2062 2063 err = create_qp(file, &ucore, &uhw, &cmd_ex, 2064 offsetof(typeof(cmd_ex), is_srq) + 2065 sizeof(cmd.is_srq), ib_uverbs_create_qp_cb, 2066 NULL); 2067 2068 if (err) 2069 return err; 2070 2071 return in_len; 2072 } 2073 2074 static int ib_uverbs_ex_create_qp_cb(struct ib_uverbs_file *file, 2075 struct ib_uverbs_ex_create_qp_resp *resp, 2076 struct ib_udata *ucore) 2077 { 2078 if (ib_copy_to_udata(ucore, resp, resp->response_length)) 2079 return -EFAULT; 2080 2081 return 0; 2082 } 2083 2084 int ib_uverbs_ex_create_qp(struct ib_uverbs_file *file, 2085 struct ib_device *ib_dev, 2086 struct ib_udata *ucore, 2087 struct ib_udata *uhw) 2088 { 2089 struct ib_uverbs_ex_create_qp_resp resp; 2090 struct ib_uverbs_ex_create_qp cmd = {0}; 2091 int err; 2092 2093 if (ucore->inlen < (offsetof(typeof(cmd), comp_mask) + 2094 sizeof(cmd.comp_mask))) 2095 return -EINVAL; 2096 2097 err = ib_copy_from_udata(&cmd, ucore, min(sizeof(cmd), ucore->inlen)); 2098 if (err) 2099 return err; 2100 2101 if (cmd.comp_mask & ~IB_UVERBS_CREATE_QP_SUP_COMP_MASK) 2102 return -EINVAL; 2103 2104 if (cmd.reserved) 2105 return -EINVAL; 2106 2107 if (ucore->outlen < (offsetof(typeof(resp), response_length) + 2108 sizeof(resp.response_length))) 2109 return -ENOSPC; 2110 2111 err = create_qp(file, ucore, uhw, &cmd, 2112 min(ucore->inlen, sizeof(cmd)), 2113 ib_uverbs_ex_create_qp_cb, NULL); 2114 2115 if (err) 2116 return err; 2117 2118 return 0; 2119 } 2120 2121 ssize_t ib_uverbs_open_qp(struct ib_uverbs_file *file, 2122 struct ib_device *ib_dev, 2123 const char __user *buf, int in_len, int out_len) 2124 { 2125 struct ib_uverbs_open_qp cmd; 2126 struct ib_uverbs_create_qp_resp resp; 2127 struct ib_udata udata; 2128 struct ib_uqp_object *obj; 2129 struct ib_xrcd *xrcd; 2130 struct ib_uobject *uninitialized_var(xrcd_uobj); 2131 struct ib_qp *qp; 2132 struct ib_qp_open_attr attr; 2133 int ret; 2134 2135 if (out_len < sizeof resp) 2136 return -ENOSPC; 2137 2138 if (copy_from_user(&cmd, buf, sizeof cmd)) 2139 return -EFAULT; 2140 2141 INIT_UDATA(&udata, buf + sizeof cmd, 2142 (unsigned long) cmd.response + sizeof resp, 2143 in_len - sizeof cmd, out_len - sizeof resp); 2144 2145 obj = kmalloc(sizeof *obj, GFP_KERNEL); 2146 if (!obj) 2147 return -ENOMEM; 2148 2149 init_uobj(&obj->uevent.uobject, cmd.user_handle, file->ucontext, &qp_lock_class); 2150 down_write(&obj->uevent.uobject.mutex); 2151 2152 xrcd = idr_read_xrcd(cmd.pd_handle, file->ucontext, &xrcd_uobj); 2153 if (!xrcd) { 2154 ret = -EINVAL; 2155 goto err_put; 2156 } 2157 2158 attr.event_handler = ib_uverbs_qp_event_handler; 2159 attr.qp_context = file; 2160 attr.qp_num = cmd.qpn; 2161 attr.qp_type = cmd.qp_type; 2162 2163 obj->uevent.events_reported = 0; 2164 INIT_LIST_HEAD(&obj->uevent.event_list); 2165 INIT_LIST_HEAD(&obj->mcast_list); 2166 2167 qp = ib_open_qp(xrcd, &attr); 2168 if (IS_ERR(qp)) { 2169 ret = PTR_ERR(qp); 2170 goto err_put; 2171 } 2172 2173 qp->uobject = &obj->uevent.uobject; 2174 2175 obj->uevent.uobject.object = qp; 2176 ret = idr_add_uobj(&ib_uverbs_qp_idr, &obj->uevent.uobject); 2177 if (ret) 2178 goto err_destroy; 2179 2180 memset(&resp, 0, sizeof resp); 2181 resp.qpn = qp->qp_num; 2182 resp.qp_handle = obj->uevent.uobject.id; 2183 2184 if (copy_to_user((void __user *) (unsigned long) cmd.response, 2185 &resp, sizeof resp)) { 2186 ret = -EFAULT; 2187 goto err_remove; 2188 } 2189 2190 obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object, uobject); 2191 atomic_inc(&obj->uxrcd->refcnt); 2192 put_xrcd_read(xrcd_uobj); 2193 2194 mutex_lock(&file->mutex); 2195 list_add_tail(&obj->uevent.uobject.list, &file->ucontext->qp_list); 2196 mutex_unlock(&file->mutex); 2197 2198 obj->uevent.uobject.live = 1; 2199 2200 up_write(&obj->uevent.uobject.mutex); 2201 2202 return in_len; 2203 2204 err_remove: 2205 idr_remove_uobj(&ib_uverbs_qp_idr, &obj->uevent.uobject); 2206 2207 err_destroy: 2208 ib_destroy_qp(qp); 2209 2210 err_put: 2211 put_xrcd_read(xrcd_uobj); 2212 put_uobj_write(&obj->uevent.uobject); 2213 return ret; 2214 } 2215 2216 ssize_t ib_uverbs_query_qp(struct ib_uverbs_file *file, 2217 struct ib_device *ib_dev, 2218 const char __user *buf, int in_len, 2219 int out_len) 2220 { 2221 struct ib_uverbs_query_qp cmd; 2222 struct ib_uverbs_query_qp_resp resp; 2223 struct ib_qp *qp; 2224 struct ib_qp_attr *attr; 2225 struct ib_qp_init_attr *init_attr; 2226 int ret; 2227 2228 if (copy_from_user(&cmd, buf, sizeof cmd)) 2229 return -EFAULT; 2230 2231 attr = kmalloc(sizeof *attr, GFP_KERNEL); 2232 init_attr = kmalloc(sizeof *init_attr, GFP_KERNEL); 2233 if (!attr || !init_attr) { 2234 ret = -ENOMEM; 2235 goto out; 2236 } 2237 2238 qp = idr_read_qp(cmd.qp_handle, file->ucontext); 2239 if (!qp) { 2240 ret = -EINVAL; 2241 goto out; 2242 } 2243 2244 ret = ib_query_qp(qp, attr, cmd.attr_mask, init_attr); 2245 2246 put_qp_read(qp); 2247 2248 if (ret) 2249 goto out; 2250 2251 memset(&resp, 0, sizeof resp); 2252 2253 resp.qp_state = attr->qp_state; 2254 resp.cur_qp_state = attr->cur_qp_state; 2255 resp.path_mtu = attr->path_mtu; 2256 resp.path_mig_state = attr->path_mig_state; 2257 resp.qkey = attr->qkey; 2258 resp.rq_psn = attr->rq_psn; 2259 resp.sq_psn = attr->sq_psn; 2260 resp.dest_qp_num = attr->dest_qp_num; 2261 resp.qp_access_flags = attr->qp_access_flags; 2262 resp.pkey_index = attr->pkey_index; 2263 resp.alt_pkey_index = attr->alt_pkey_index; 2264 resp.sq_draining = attr->sq_draining; 2265 resp.max_rd_atomic = attr->max_rd_atomic; 2266 resp.max_dest_rd_atomic = attr->max_dest_rd_atomic; 2267 resp.min_rnr_timer = attr->min_rnr_timer; 2268 resp.port_num = attr->port_num; 2269 resp.timeout = attr->timeout; 2270 resp.retry_cnt = attr->retry_cnt; 2271 resp.rnr_retry = attr->rnr_retry; 2272 resp.alt_port_num = attr->alt_port_num; 2273 resp.alt_timeout = attr->alt_timeout; 2274 2275 memcpy(resp.dest.dgid, attr->ah_attr.grh.dgid.raw, 16); 2276 resp.dest.flow_label = attr->ah_attr.grh.flow_label; 2277 resp.dest.sgid_index = attr->ah_attr.grh.sgid_index; 2278 resp.dest.hop_limit = attr->ah_attr.grh.hop_limit; 2279 resp.dest.traffic_class = attr->ah_attr.grh.traffic_class; 2280 resp.dest.dlid = attr->ah_attr.dlid; 2281 resp.dest.sl = attr->ah_attr.sl; 2282 resp.dest.src_path_bits = attr->ah_attr.src_path_bits; 2283 resp.dest.static_rate = attr->ah_attr.static_rate; 2284 resp.dest.is_global = !!(attr->ah_attr.ah_flags & IB_AH_GRH); 2285 resp.dest.port_num = attr->ah_attr.port_num; 2286 2287 memcpy(resp.alt_dest.dgid, attr->alt_ah_attr.grh.dgid.raw, 16); 2288 resp.alt_dest.flow_label = attr->alt_ah_attr.grh.flow_label; 2289 resp.alt_dest.sgid_index = attr->alt_ah_attr.grh.sgid_index; 2290 resp.alt_dest.hop_limit = attr->alt_ah_attr.grh.hop_limit; 2291 resp.alt_dest.traffic_class = attr->alt_ah_attr.grh.traffic_class; 2292 resp.alt_dest.dlid = attr->alt_ah_attr.dlid; 2293 resp.alt_dest.sl = attr->alt_ah_attr.sl; 2294 resp.alt_dest.src_path_bits = attr->alt_ah_attr.src_path_bits; 2295 resp.alt_dest.static_rate = attr->alt_ah_attr.static_rate; 2296 resp.alt_dest.is_global = !!(attr->alt_ah_attr.ah_flags & IB_AH_GRH); 2297 resp.alt_dest.port_num = attr->alt_ah_attr.port_num; 2298 2299 resp.max_send_wr = init_attr->cap.max_send_wr; 2300 resp.max_recv_wr = init_attr->cap.max_recv_wr; 2301 resp.max_send_sge = init_attr->cap.max_send_sge; 2302 resp.max_recv_sge = init_attr->cap.max_recv_sge; 2303 resp.max_inline_data = init_attr->cap.max_inline_data; 2304 resp.sq_sig_all = init_attr->sq_sig_type == IB_SIGNAL_ALL_WR; 2305 2306 if (copy_to_user((void __user *) (unsigned long) cmd.response, 2307 &resp, sizeof resp)) 2308 ret = -EFAULT; 2309 2310 out: 2311 kfree(attr); 2312 kfree(init_attr); 2313 2314 return ret ? ret : in_len; 2315 } 2316 2317 /* Remove ignored fields set in the attribute mask */ 2318 static int modify_qp_mask(enum ib_qp_type qp_type, int mask) 2319 { 2320 switch (qp_type) { 2321 case IB_QPT_XRC_INI: 2322 return mask & ~(IB_QP_MAX_DEST_RD_ATOMIC | IB_QP_MIN_RNR_TIMER); 2323 case IB_QPT_XRC_TGT: 2324 return mask & ~(IB_QP_MAX_QP_RD_ATOMIC | IB_QP_RETRY_CNT | 2325 IB_QP_RNR_RETRY); 2326 default: 2327 return mask; 2328 } 2329 } 2330 2331 static int modify_qp(struct ib_uverbs_file *file, 2332 struct ib_uverbs_ex_modify_qp *cmd, struct ib_udata *udata) 2333 { 2334 struct ib_qp_attr *attr; 2335 struct ib_qp *qp; 2336 int ret; 2337 2338 attr = kmalloc(sizeof *attr, GFP_KERNEL); 2339 if (!attr) 2340 return -ENOMEM; 2341 2342 qp = idr_read_qp(cmd->base.qp_handle, file->ucontext); 2343 if (!qp) { 2344 ret = -EINVAL; 2345 goto out; 2346 } 2347 2348 attr->qp_state = cmd->base.qp_state; 2349 attr->cur_qp_state = cmd->base.cur_qp_state; 2350 attr->path_mtu = cmd->base.path_mtu; 2351 attr->path_mig_state = cmd->base.path_mig_state; 2352 attr->qkey = cmd->base.qkey; 2353 attr->rq_psn = cmd->base.rq_psn; 2354 attr->sq_psn = cmd->base.sq_psn; 2355 attr->dest_qp_num = cmd->base.dest_qp_num; 2356 attr->qp_access_flags = cmd->base.qp_access_flags; 2357 attr->pkey_index = cmd->base.pkey_index; 2358 attr->alt_pkey_index = cmd->base.alt_pkey_index; 2359 attr->en_sqd_async_notify = cmd->base.en_sqd_async_notify; 2360 attr->max_rd_atomic = cmd->base.max_rd_atomic; 2361 attr->max_dest_rd_atomic = cmd->base.max_dest_rd_atomic; 2362 attr->min_rnr_timer = cmd->base.min_rnr_timer; 2363 attr->port_num = cmd->base.port_num; 2364 attr->timeout = cmd->base.timeout; 2365 attr->retry_cnt = cmd->base.retry_cnt; 2366 attr->rnr_retry = cmd->base.rnr_retry; 2367 attr->alt_port_num = cmd->base.alt_port_num; 2368 attr->alt_timeout = cmd->base.alt_timeout; 2369 attr->rate_limit = cmd->rate_limit; 2370 2371 memcpy(attr->ah_attr.grh.dgid.raw, cmd->base.dest.dgid, 16); 2372 attr->ah_attr.grh.flow_label = cmd->base.dest.flow_label; 2373 attr->ah_attr.grh.sgid_index = cmd->base.dest.sgid_index; 2374 attr->ah_attr.grh.hop_limit = cmd->base.dest.hop_limit; 2375 attr->ah_attr.grh.traffic_class = cmd->base.dest.traffic_class; 2376 attr->ah_attr.dlid = cmd->base.dest.dlid; 2377 attr->ah_attr.sl = cmd->base.dest.sl; 2378 attr->ah_attr.src_path_bits = cmd->base.dest.src_path_bits; 2379 attr->ah_attr.static_rate = cmd->base.dest.static_rate; 2380 attr->ah_attr.ah_flags = cmd->base.dest.is_global ? 2381 IB_AH_GRH : 0; 2382 attr->ah_attr.port_num = cmd->base.dest.port_num; 2383 2384 memcpy(attr->alt_ah_attr.grh.dgid.raw, cmd->base.alt_dest.dgid, 16); 2385 attr->alt_ah_attr.grh.flow_label = cmd->base.alt_dest.flow_label; 2386 attr->alt_ah_attr.grh.sgid_index = cmd->base.alt_dest.sgid_index; 2387 attr->alt_ah_attr.grh.hop_limit = cmd->base.alt_dest.hop_limit; 2388 attr->alt_ah_attr.grh.traffic_class = cmd->base.alt_dest.traffic_class; 2389 attr->alt_ah_attr.dlid = cmd->base.alt_dest.dlid; 2390 attr->alt_ah_attr.sl = cmd->base.alt_dest.sl; 2391 attr->alt_ah_attr.src_path_bits = cmd->base.alt_dest.src_path_bits; 2392 attr->alt_ah_attr.static_rate = cmd->base.alt_dest.static_rate; 2393 attr->alt_ah_attr.ah_flags = cmd->base.alt_dest.is_global ? 2394 IB_AH_GRH : 0; 2395 attr->alt_ah_attr.port_num = cmd->base.alt_dest.port_num; 2396 2397 if (qp->real_qp == qp) { 2398 if (cmd->base.attr_mask & IB_QP_AV) { 2399 ret = ib_resolve_eth_dmac(qp->device, &attr->ah_attr); 2400 if (ret) 2401 goto release_qp; 2402 } 2403 ret = qp->device->modify_qp(qp, attr, 2404 modify_qp_mask(qp->qp_type, 2405 cmd->base.attr_mask), 2406 udata); 2407 } else { 2408 ret = ib_modify_qp(qp, attr, 2409 modify_qp_mask(qp->qp_type, 2410 cmd->base.attr_mask)); 2411 } 2412 2413 release_qp: 2414 put_qp_read(qp); 2415 2416 out: 2417 kfree(attr); 2418 2419 return ret; 2420 } 2421 2422 ssize_t ib_uverbs_modify_qp(struct ib_uverbs_file *file, 2423 struct ib_device *ib_dev, 2424 const char __user *buf, int in_len, 2425 int out_len) 2426 { 2427 struct ib_uverbs_ex_modify_qp cmd = {}; 2428 struct ib_udata udata; 2429 int ret; 2430 2431 if (copy_from_user(&cmd.base, buf, sizeof(cmd.base))) 2432 return -EFAULT; 2433 2434 if (cmd.base.attr_mask & 2435 ~((IB_USER_LEGACY_LAST_QP_ATTR_MASK << 1) - 1)) 2436 return -EOPNOTSUPP; 2437 2438 INIT_UDATA(&udata, buf + sizeof(cmd.base), NULL, 2439 in_len - sizeof(cmd.base), out_len); 2440 2441 ret = modify_qp(file, &cmd, &udata); 2442 if (ret) 2443 return ret; 2444 2445 return in_len; 2446 } 2447 2448 int ib_uverbs_ex_modify_qp(struct ib_uverbs_file *file, 2449 struct ib_device *ib_dev, 2450 struct ib_udata *ucore, 2451 struct ib_udata *uhw) 2452 { 2453 struct ib_uverbs_ex_modify_qp cmd = {}; 2454 int ret; 2455 2456 /* 2457 * Last bit is reserved for extending the attr_mask by 2458 * using another field. 2459 */ 2460 BUILD_BUG_ON(IB_USER_LAST_QP_ATTR_MASK == (1 << 31)); 2461 2462 if (ucore->inlen < sizeof(cmd.base)) 2463 return -EINVAL; 2464 2465 ret = ib_copy_from_udata(&cmd, ucore, min(sizeof(cmd), ucore->inlen)); 2466 if (ret) 2467 return ret; 2468 2469 if (cmd.base.attr_mask & 2470 ~((IB_USER_LAST_QP_ATTR_MASK << 1) - 1)) 2471 return -EOPNOTSUPP; 2472 2473 if (ucore->inlen > sizeof(cmd)) { 2474 if (ib_is_udata_cleared(ucore, sizeof(cmd), 2475 ucore->inlen - sizeof(cmd))) 2476 return -EOPNOTSUPP; 2477 } 2478 2479 ret = modify_qp(file, &cmd, uhw); 2480 2481 return ret; 2482 } 2483 2484 ssize_t ib_uverbs_destroy_qp(struct ib_uverbs_file *file, 2485 struct ib_device *ib_dev, 2486 const char __user *buf, int in_len, 2487 int out_len) 2488 { 2489 struct ib_uverbs_destroy_qp cmd; 2490 struct ib_uverbs_destroy_qp_resp resp; 2491 struct ib_uobject *uobj; 2492 struct ib_qp *qp; 2493 struct ib_uqp_object *obj; 2494 int ret = -EINVAL; 2495 2496 if (copy_from_user(&cmd, buf, sizeof cmd)) 2497 return -EFAULT; 2498 2499 memset(&resp, 0, sizeof resp); 2500 2501 uobj = idr_write_uobj(&ib_uverbs_qp_idr, cmd.qp_handle, file->ucontext); 2502 if (!uobj) 2503 return -EINVAL; 2504 qp = uobj->object; 2505 obj = container_of(uobj, struct ib_uqp_object, uevent.uobject); 2506 2507 if (!list_empty(&obj->mcast_list)) { 2508 put_uobj_write(uobj); 2509 return -EBUSY; 2510 } 2511 2512 ret = ib_destroy_qp(qp); 2513 if (!ret) 2514 uobj->live = 0; 2515 2516 put_uobj_write(uobj); 2517 2518 if (ret) 2519 return ret; 2520 2521 if (obj->uxrcd) 2522 atomic_dec(&obj->uxrcd->refcnt); 2523 2524 idr_remove_uobj(&ib_uverbs_qp_idr, uobj); 2525 2526 mutex_lock(&file->mutex); 2527 list_del(&uobj->list); 2528 mutex_unlock(&file->mutex); 2529 2530 ib_uverbs_release_uevent(file, &obj->uevent); 2531 2532 resp.events_reported = obj->uevent.events_reported; 2533 2534 put_uobj(uobj); 2535 2536 if (copy_to_user((void __user *) (unsigned long) cmd.response, 2537 &resp, sizeof resp)) 2538 return -EFAULT; 2539 2540 return in_len; 2541 } 2542 2543 static void *alloc_wr(size_t wr_size, __u32 num_sge) 2544 { 2545 return kmalloc(ALIGN(wr_size, sizeof (struct ib_sge)) + 2546 num_sge * sizeof (struct ib_sge), GFP_KERNEL); 2547 }; 2548 2549 ssize_t ib_uverbs_post_send(struct ib_uverbs_file *file, 2550 struct ib_device *ib_dev, 2551 const char __user *buf, int in_len, 2552 int out_len) 2553 { 2554 struct ib_uverbs_post_send cmd; 2555 struct ib_uverbs_post_send_resp resp; 2556 struct ib_uverbs_send_wr *user_wr; 2557 struct ib_send_wr *wr = NULL, *last, *next, *bad_wr; 2558 struct ib_qp *qp; 2559 int i, sg_ind; 2560 int is_ud; 2561 ssize_t ret = -EINVAL; 2562 size_t next_size; 2563 2564 if (copy_from_user(&cmd, buf, sizeof cmd)) 2565 return -EFAULT; 2566 2567 if (in_len < sizeof cmd + cmd.wqe_size * cmd.wr_count + 2568 cmd.sge_count * sizeof (struct ib_uverbs_sge)) 2569 return -EINVAL; 2570 2571 if (cmd.wqe_size < sizeof (struct ib_uverbs_send_wr)) 2572 return -EINVAL; 2573 2574 user_wr = kmalloc(cmd.wqe_size, GFP_KERNEL); 2575 if (!user_wr) 2576 return -ENOMEM; 2577 2578 qp = idr_read_qp(cmd.qp_handle, file->ucontext); 2579 if (!qp) 2580 goto out; 2581 2582 is_ud = qp->qp_type == IB_QPT_UD; 2583 sg_ind = 0; 2584 last = NULL; 2585 for (i = 0; i < cmd.wr_count; ++i) { 2586 if (copy_from_user(user_wr, 2587 buf + sizeof cmd + i * cmd.wqe_size, 2588 cmd.wqe_size)) { 2589 ret = -EFAULT; 2590 goto out_put; 2591 } 2592 2593 if (user_wr->num_sge + sg_ind > cmd.sge_count) { 2594 ret = -EINVAL; 2595 goto out_put; 2596 } 2597 2598 if (is_ud) { 2599 struct ib_ud_wr *ud; 2600 2601 if (user_wr->opcode != IB_WR_SEND && 2602 user_wr->opcode != IB_WR_SEND_WITH_IMM) { 2603 ret = -EINVAL; 2604 goto out_put; 2605 } 2606 2607 next_size = sizeof(*ud); 2608 ud = alloc_wr(next_size, user_wr->num_sge); 2609 if (!ud) { 2610 ret = -ENOMEM; 2611 goto out_put; 2612 } 2613 2614 ud->ah = idr_read_ah(user_wr->wr.ud.ah, file->ucontext); 2615 if (!ud->ah) { 2616 kfree(ud); 2617 ret = -EINVAL; 2618 goto out_put; 2619 } 2620 ud->remote_qpn = user_wr->wr.ud.remote_qpn; 2621 ud->remote_qkey = user_wr->wr.ud.remote_qkey; 2622 2623 next = &ud->wr; 2624 } else if (user_wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM || 2625 user_wr->opcode == IB_WR_RDMA_WRITE || 2626 user_wr->opcode == IB_WR_RDMA_READ) { 2627 struct ib_rdma_wr *rdma; 2628 2629 next_size = sizeof(*rdma); 2630 rdma = alloc_wr(next_size, user_wr->num_sge); 2631 if (!rdma) { 2632 ret = -ENOMEM; 2633 goto out_put; 2634 } 2635 2636 rdma->remote_addr = user_wr->wr.rdma.remote_addr; 2637 rdma->rkey = user_wr->wr.rdma.rkey; 2638 2639 next = &rdma->wr; 2640 } else if (user_wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP || 2641 user_wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) { 2642 struct ib_atomic_wr *atomic; 2643 2644 next_size = sizeof(*atomic); 2645 atomic = alloc_wr(next_size, user_wr->num_sge); 2646 if (!atomic) { 2647 ret = -ENOMEM; 2648 goto out_put; 2649 } 2650 2651 atomic->remote_addr = user_wr->wr.atomic.remote_addr; 2652 atomic->compare_add = user_wr->wr.atomic.compare_add; 2653 atomic->swap = user_wr->wr.atomic.swap; 2654 atomic->rkey = user_wr->wr.atomic.rkey; 2655 2656 next = &atomic->wr; 2657 } else if (user_wr->opcode == IB_WR_SEND || 2658 user_wr->opcode == IB_WR_SEND_WITH_IMM || 2659 user_wr->opcode == IB_WR_SEND_WITH_INV) { 2660 next_size = sizeof(*next); 2661 next = alloc_wr(next_size, user_wr->num_sge); 2662 if (!next) { 2663 ret = -ENOMEM; 2664 goto out_put; 2665 } 2666 } else { 2667 ret = -EINVAL; 2668 goto out_put; 2669 } 2670 2671 if (user_wr->opcode == IB_WR_SEND_WITH_IMM || 2672 user_wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM) { 2673 next->ex.imm_data = 2674 (__be32 __force) user_wr->ex.imm_data; 2675 } else if (user_wr->opcode == IB_WR_SEND_WITH_INV) { 2676 next->ex.invalidate_rkey = user_wr->ex.invalidate_rkey; 2677 } 2678 2679 if (!last) 2680 wr = next; 2681 else 2682 last->next = next; 2683 last = next; 2684 2685 next->next = NULL; 2686 next->wr_id = user_wr->wr_id; 2687 next->num_sge = user_wr->num_sge; 2688 next->opcode = user_wr->opcode; 2689 next->send_flags = user_wr->send_flags; 2690 2691 if (next->num_sge) { 2692 next->sg_list = (void *) next + 2693 ALIGN(next_size, sizeof(struct ib_sge)); 2694 if (copy_from_user(next->sg_list, 2695 buf + sizeof cmd + 2696 cmd.wr_count * cmd.wqe_size + 2697 sg_ind * sizeof (struct ib_sge), 2698 next->num_sge * sizeof (struct ib_sge))) { 2699 ret = -EFAULT; 2700 goto out_put; 2701 } 2702 sg_ind += next->num_sge; 2703 } else 2704 next->sg_list = NULL; 2705 } 2706 2707 resp.bad_wr = 0; 2708 ret = qp->device->post_send(qp->real_qp, wr, &bad_wr); 2709 if (ret) 2710 for (next = wr; next; next = next->next) { 2711 ++resp.bad_wr; 2712 if (next == bad_wr) 2713 break; 2714 } 2715 2716 if (copy_to_user((void __user *) (unsigned long) cmd.response, 2717 &resp, sizeof resp)) 2718 ret = -EFAULT; 2719 2720 out_put: 2721 put_qp_read(qp); 2722 2723 while (wr) { 2724 if (is_ud && ud_wr(wr)->ah) 2725 put_ah_read(ud_wr(wr)->ah); 2726 next = wr->next; 2727 kfree(wr); 2728 wr = next; 2729 } 2730 2731 out: 2732 kfree(user_wr); 2733 2734 return ret ? ret : in_len; 2735 } 2736 2737 static struct ib_recv_wr *ib_uverbs_unmarshall_recv(const char __user *buf, 2738 int in_len, 2739 u32 wr_count, 2740 u32 sge_count, 2741 u32 wqe_size) 2742 { 2743 struct ib_uverbs_recv_wr *user_wr; 2744 struct ib_recv_wr *wr = NULL, *last, *next; 2745 int sg_ind; 2746 int i; 2747 int ret; 2748 2749 if (in_len < wqe_size * wr_count + 2750 sge_count * sizeof (struct ib_uverbs_sge)) 2751 return ERR_PTR(-EINVAL); 2752 2753 if (wqe_size < sizeof (struct ib_uverbs_recv_wr)) 2754 return ERR_PTR(-EINVAL); 2755 2756 user_wr = kmalloc(wqe_size, GFP_KERNEL); 2757 if (!user_wr) 2758 return ERR_PTR(-ENOMEM); 2759 2760 sg_ind = 0; 2761 last = NULL; 2762 for (i = 0; i < wr_count; ++i) { 2763 if (copy_from_user(user_wr, buf + i * wqe_size, 2764 wqe_size)) { 2765 ret = -EFAULT; 2766 goto err; 2767 } 2768 2769 if (user_wr->num_sge + sg_ind > sge_count) { 2770 ret = -EINVAL; 2771 goto err; 2772 } 2773 2774 next = kmalloc(ALIGN(sizeof *next, sizeof (struct ib_sge)) + 2775 user_wr->num_sge * sizeof (struct ib_sge), 2776 GFP_KERNEL); 2777 if (!next) { 2778 ret = -ENOMEM; 2779 goto err; 2780 } 2781 2782 if (!last) 2783 wr = next; 2784 else 2785 last->next = next; 2786 last = next; 2787 2788 next->next = NULL; 2789 next->wr_id = user_wr->wr_id; 2790 next->num_sge = user_wr->num_sge; 2791 2792 if (next->num_sge) { 2793 next->sg_list = (void *) next + 2794 ALIGN(sizeof *next, sizeof (struct ib_sge)); 2795 if (copy_from_user(next->sg_list, 2796 buf + wr_count * wqe_size + 2797 sg_ind * sizeof (struct ib_sge), 2798 next->num_sge * sizeof (struct ib_sge))) { 2799 ret = -EFAULT; 2800 goto err; 2801 } 2802 sg_ind += next->num_sge; 2803 } else 2804 next->sg_list = NULL; 2805 } 2806 2807 kfree(user_wr); 2808 return wr; 2809 2810 err: 2811 kfree(user_wr); 2812 2813 while (wr) { 2814 next = wr->next; 2815 kfree(wr); 2816 wr = next; 2817 } 2818 2819 return ERR_PTR(ret); 2820 } 2821 2822 ssize_t ib_uverbs_post_recv(struct ib_uverbs_file *file, 2823 struct ib_device *ib_dev, 2824 const char __user *buf, int in_len, 2825 int out_len) 2826 { 2827 struct ib_uverbs_post_recv cmd; 2828 struct ib_uverbs_post_recv_resp resp; 2829 struct ib_recv_wr *wr, *next, *bad_wr; 2830 struct ib_qp *qp; 2831 ssize_t ret = -EINVAL; 2832 2833 if (copy_from_user(&cmd, buf, sizeof cmd)) 2834 return -EFAULT; 2835 2836 wr = ib_uverbs_unmarshall_recv(buf + sizeof cmd, 2837 in_len - sizeof cmd, cmd.wr_count, 2838 cmd.sge_count, cmd.wqe_size); 2839 if (IS_ERR(wr)) 2840 return PTR_ERR(wr); 2841 2842 qp = idr_read_qp(cmd.qp_handle, file->ucontext); 2843 if (!qp) 2844 goto out; 2845 2846 resp.bad_wr = 0; 2847 ret = qp->device->post_recv(qp->real_qp, wr, &bad_wr); 2848 2849 put_qp_read(qp); 2850 2851 if (ret) 2852 for (next = wr; next; next = next->next) { 2853 ++resp.bad_wr; 2854 if (next == bad_wr) 2855 break; 2856 } 2857 2858 if (copy_to_user((void __user *) (unsigned long) cmd.response, 2859 &resp, sizeof resp)) 2860 ret = -EFAULT; 2861 2862 out: 2863 while (wr) { 2864 next = wr->next; 2865 kfree(wr); 2866 wr = next; 2867 } 2868 2869 return ret ? ret : in_len; 2870 } 2871 2872 ssize_t ib_uverbs_post_srq_recv(struct ib_uverbs_file *file, 2873 struct ib_device *ib_dev, 2874 const char __user *buf, int in_len, 2875 int out_len) 2876 { 2877 struct ib_uverbs_post_srq_recv cmd; 2878 struct ib_uverbs_post_srq_recv_resp resp; 2879 struct ib_recv_wr *wr, *next, *bad_wr; 2880 struct ib_srq *srq; 2881 ssize_t ret = -EINVAL; 2882 2883 if (copy_from_user(&cmd, buf, sizeof cmd)) 2884 return -EFAULT; 2885 2886 wr = ib_uverbs_unmarshall_recv(buf + sizeof cmd, 2887 in_len - sizeof cmd, cmd.wr_count, 2888 cmd.sge_count, cmd.wqe_size); 2889 if (IS_ERR(wr)) 2890 return PTR_ERR(wr); 2891 2892 srq = idr_read_srq(cmd.srq_handle, file->ucontext); 2893 if (!srq) 2894 goto out; 2895 2896 resp.bad_wr = 0; 2897 ret = srq->device->post_srq_recv(srq, wr, &bad_wr); 2898 2899 put_srq_read(srq); 2900 2901 if (ret) 2902 for (next = wr; next; next = next->next) { 2903 ++resp.bad_wr; 2904 if (next == bad_wr) 2905 break; 2906 } 2907 2908 if (copy_to_user((void __user *) (unsigned long) cmd.response, 2909 &resp, sizeof resp)) 2910 ret = -EFAULT; 2911 2912 out: 2913 while (wr) { 2914 next = wr->next; 2915 kfree(wr); 2916 wr = next; 2917 } 2918 2919 return ret ? ret : in_len; 2920 } 2921 2922 ssize_t ib_uverbs_create_ah(struct ib_uverbs_file *file, 2923 struct ib_device *ib_dev, 2924 const char __user *buf, int in_len, 2925 int out_len) 2926 { 2927 struct ib_uverbs_create_ah cmd; 2928 struct ib_uverbs_create_ah_resp resp; 2929 struct ib_uobject *uobj; 2930 struct ib_pd *pd; 2931 struct ib_ah *ah; 2932 struct ib_ah_attr attr; 2933 int ret; 2934 struct ib_udata udata; 2935 2936 if (out_len < sizeof resp) 2937 return -ENOSPC; 2938 2939 if (copy_from_user(&cmd, buf, sizeof cmd)) 2940 return -EFAULT; 2941 2942 INIT_UDATA(&udata, buf + sizeof(cmd), 2943 (unsigned long)cmd.response + sizeof(resp), 2944 in_len - sizeof(cmd), out_len - sizeof(resp)); 2945 2946 uobj = kmalloc(sizeof *uobj, GFP_KERNEL); 2947 if (!uobj) 2948 return -ENOMEM; 2949 2950 init_uobj(uobj, cmd.user_handle, file->ucontext, &ah_lock_class); 2951 down_write(&uobj->mutex); 2952 2953 pd = idr_read_pd(cmd.pd_handle, file->ucontext); 2954 if (!pd) { 2955 ret = -EINVAL; 2956 goto err; 2957 } 2958 2959 attr.dlid = cmd.attr.dlid; 2960 attr.sl = cmd.attr.sl; 2961 attr.src_path_bits = cmd.attr.src_path_bits; 2962 attr.static_rate = cmd.attr.static_rate; 2963 attr.ah_flags = cmd.attr.is_global ? IB_AH_GRH : 0; 2964 attr.port_num = cmd.attr.port_num; 2965 attr.grh.flow_label = cmd.attr.grh.flow_label; 2966 attr.grh.sgid_index = cmd.attr.grh.sgid_index; 2967 attr.grh.hop_limit = cmd.attr.grh.hop_limit; 2968 attr.grh.traffic_class = cmd.attr.grh.traffic_class; 2969 memset(&attr.dmac, 0, sizeof(attr.dmac)); 2970 memcpy(attr.grh.dgid.raw, cmd.attr.grh.dgid, 16); 2971 2972 ah = pd->device->create_ah(pd, &attr, &udata); 2973 2974 if (IS_ERR(ah)) { 2975 ret = PTR_ERR(ah); 2976 goto err_put; 2977 } 2978 2979 ah->device = pd->device; 2980 ah->pd = pd; 2981 atomic_inc(&pd->usecnt); 2982 ah->uobject = uobj; 2983 uobj->object = ah; 2984 2985 ret = idr_add_uobj(&ib_uverbs_ah_idr, uobj); 2986 if (ret) 2987 goto err_destroy; 2988 2989 resp.ah_handle = uobj->id; 2990 2991 if (copy_to_user((void __user *) (unsigned long) cmd.response, 2992 &resp, sizeof resp)) { 2993 ret = -EFAULT; 2994 goto err_copy; 2995 } 2996 2997 put_pd_read(pd); 2998 2999 mutex_lock(&file->mutex); 3000 list_add_tail(&uobj->list, &file->ucontext->ah_list); 3001 mutex_unlock(&file->mutex); 3002 3003 uobj->live = 1; 3004 3005 up_write(&uobj->mutex); 3006 3007 return in_len; 3008 3009 err_copy: 3010 idr_remove_uobj(&ib_uverbs_ah_idr, uobj); 3011 3012 err_destroy: 3013 ib_destroy_ah(ah); 3014 3015 err_put: 3016 put_pd_read(pd); 3017 3018 err: 3019 put_uobj_write(uobj); 3020 return ret; 3021 } 3022 3023 ssize_t ib_uverbs_destroy_ah(struct ib_uverbs_file *file, 3024 struct ib_device *ib_dev, 3025 const char __user *buf, int in_len, int out_len) 3026 { 3027 struct ib_uverbs_destroy_ah cmd; 3028 struct ib_ah *ah; 3029 struct ib_uobject *uobj; 3030 int ret; 3031 3032 if (copy_from_user(&cmd, buf, sizeof cmd)) 3033 return -EFAULT; 3034 3035 uobj = idr_write_uobj(&ib_uverbs_ah_idr, cmd.ah_handle, file->ucontext); 3036 if (!uobj) 3037 return -EINVAL; 3038 ah = uobj->object; 3039 3040 ret = ib_destroy_ah(ah); 3041 if (!ret) 3042 uobj->live = 0; 3043 3044 put_uobj_write(uobj); 3045 3046 if (ret) 3047 return ret; 3048 3049 idr_remove_uobj(&ib_uverbs_ah_idr, uobj); 3050 3051 mutex_lock(&file->mutex); 3052 list_del(&uobj->list); 3053 mutex_unlock(&file->mutex); 3054 3055 put_uobj(uobj); 3056 3057 return in_len; 3058 } 3059 3060 ssize_t ib_uverbs_attach_mcast(struct ib_uverbs_file *file, 3061 struct ib_device *ib_dev, 3062 const char __user *buf, int in_len, 3063 int out_len) 3064 { 3065 struct ib_uverbs_attach_mcast cmd; 3066 struct ib_qp *qp; 3067 struct ib_uqp_object *obj; 3068 struct ib_uverbs_mcast_entry *mcast; 3069 int ret; 3070 3071 if (copy_from_user(&cmd, buf, sizeof cmd)) 3072 return -EFAULT; 3073 3074 qp = idr_write_qp(cmd.qp_handle, file->ucontext); 3075 if (!qp) 3076 return -EINVAL; 3077 3078 obj = container_of(qp->uobject, struct ib_uqp_object, uevent.uobject); 3079 3080 list_for_each_entry(mcast, &obj->mcast_list, list) 3081 if (cmd.mlid == mcast->lid && 3082 !memcmp(cmd.gid, mcast->gid.raw, sizeof mcast->gid.raw)) { 3083 ret = 0; 3084 goto out_put; 3085 } 3086 3087 mcast = kmalloc(sizeof *mcast, GFP_KERNEL); 3088 if (!mcast) { 3089 ret = -ENOMEM; 3090 goto out_put; 3091 } 3092 3093 mcast->lid = cmd.mlid; 3094 memcpy(mcast->gid.raw, cmd.gid, sizeof mcast->gid.raw); 3095 3096 ret = ib_attach_mcast(qp, &mcast->gid, cmd.mlid); 3097 if (!ret) 3098 list_add_tail(&mcast->list, &obj->mcast_list); 3099 else 3100 kfree(mcast); 3101 3102 out_put: 3103 put_qp_write(qp); 3104 3105 return ret ? ret : in_len; 3106 } 3107 3108 ssize_t ib_uverbs_detach_mcast(struct ib_uverbs_file *file, 3109 struct ib_device *ib_dev, 3110 const char __user *buf, int in_len, 3111 int out_len) 3112 { 3113 struct ib_uverbs_detach_mcast cmd; 3114 struct ib_uqp_object *obj; 3115 struct ib_qp *qp; 3116 struct ib_uverbs_mcast_entry *mcast; 3117 int ret = -EINVAL; 3118 3119 if (copy_from_user(&cmd, buf, sizeof cmd)) 3120 return -EFAULT; 3121 3122 qp = idr_write_qp(cmd.qp_handle, file->ucontext); 3123 if (!qp) 3124 return -EINVAL; 3125 3126 ret = ib_detach_mcast(qp, (union ib_gid *) cmd.gid, cmd.mlid); 3127 if (ret) 3128 goto out_put; 3129 3130 obj = container_of(qp->uobject, struct ib_uqp_object, uevent.uobject); 3131 3132 list_for_each_entry(mcast, &obj->mcast_list, list) 3133 if (cmd.mlid == mcast->lid && 3134 !memcmp(cmd.gid, mcast->gid.raw, sizeof mcast->gid.raw)) { 3135 list_del(&mcast->list); 3136 kfree(mcast); 3137 break; 3138 } 3139 3140 out_put: 3141 put_qp_write(qp); 3142 3143 return ret ? ret : in_len; 3144 } 3145 3146 static size_t kern_spec_filter_sz(struct ib_uverbs_flow_spec_hdr *spec) 3147 { 3148 /* Returns user space filter size, includes padding */ 3149 return (spec->size - sizeof(struct ib_uverbs_flow_spec_hdr)) / 2; 3150 } 3151 3152 static ssize_t spec_filter_size(void *kern_spec_filter, u16 kern_filter_size, 3153 u16 ib_real_filter_sz) 3154 { 3155 /* 3156 * User space filter structures must be 64 bit aligned, otherwise this 3157 * may pass, but we won't handle additional new attributes. 3158 */ 3159 3160 if (kern_filter_size > ib_real_filter_sz) { 3161 if (memchr_inv(kern_spec_filter + 3162 ib_real_filter_sz, 0, 3163 kern_filter_size - ib_real_filter_sz)) 3164 return -EINVAL; 3165 return ib_real_filter_sz; 3166 } 3167 return kern_filter_size; 3168 } 3169 3170 static int kern_spec_to_ib_spec(struct ib_uverbs_flow_spec *kern_spec, 3171 union ib_flow_spec *ib_spec) 3172 { 3173 ssize_t actual_filter_sz; 3174 ssize_t kern_filter_sz; 3175 ssize_t ib_filter_sz; 3176 void *kern_spec_mask; 3177 void *kern_spec_val; 3178 3179 if (kern_spec->reserved) 3180 return -EINVAL; 3181 3182 ib_spec->type = kern_spec->type; 3183 3184 kern_filter_sz = kern_spec_filter_sz(&kern_spec->hdr); 3185 /* User flow spec size must be aligned to 4 bytes */ 3186 if (kern_filter_sz != ALIGN(kern_filter_sz, 4)) 3187 return -EINVAL; 3188 3189 kern_spec_val = (void *)kern_spec + 3190 sizeof(struct ib_uverbs_flow_spec_hdr); 3191 kern_spec_mask = kern_spec_val + kern_filter_sz; 3192 if (ib_spec->type == (IB_FLOW_SPEC_INNER | IB_FLOW_SPEC_VXLAN_TUNNEL)) 3193 return -EINVAL; 3194 3195 switch (ib_spec->type & ~IB_FLOW_SPEC_INNER) { 3196 case IB_FLOW_SPEC_ETH: 3197 ib_filter_sz = offsetof(struct ib_flow_eth_filter, real_sz); 3198 actual_filter_sz = spec_filter_size(kern_spec_mask, 3199 kern_filter_sz, 3200 ib_filter_sz); 3201 if (actual_filter_sz <= 0) 3202 return -EINVAL; 3203 ib_spec->size = sizeof(struct ib_flow_spec_eth); 3204 memcpy(&ib_spec->eth.val, kern_spec_val, actual_filter_sz); 3205 memcpy(&ib_spec->eth.mask, kern_spec_mask, actual_filter_sz); 3206 break; 3207 case IB_FLOW_SPEC_IPV4: 3208 ib_filter_sz = offsetof(struct ib_flow_ipv4_filter, real_sz); 3209 actual_filter_sz = spec_filter_size(kern_spec_mask, 3210 kern_filter_sz, 3211 ib_filter_sz); 3212 if (actual_filter_sz <= 0) 3213 return -EINVAL; 3214 ib_spec->size = sizeof(struct ib_flow_spec_ipv4); 3215 memcpy(&ib_spec->ipv4.val, kern_spec_val, actual_filter_sz); 3216 memcpy(&ib_spec->ipv4.mask, kern_spec_mask, actual_filter_sz); 3217 break; 3218 case IB_FLOW_SPEC_IPV6: 3219 ib_filter_sz = offsetof(struct ib_flow_ipv6_filter, real_sz); 3220 actual_filter_sz = spec_filter_size(kern_spec_mask, 3221 kern_filter_sz, 3222 ib_filter_sz); 3223 if (actual_filter_sz <= 0) 3224 return -EINVAL; 3225 ib_spec->size = sizeof(struct ib_flow_spec_ipv6); 3226 memcpy(&ib_spec->ipv6.val, kern_spec_val, actual_filter_sz); 3227 memcpy(&ib_spec->ipv6.mask, kern_spec_mask, actual_filter_sz); 3228 3229 if ((ntohl(ib_spec->ipv6.mask.flow_label)) >= BIT(20) || 3230 (ntohl(ib_spec->ipv6.val.flow_label)) >= BIT(20)) 3231 return -EINVAL; 3232 break; 3233 case IB_FLOW_SPEC_TCP: 3234 case IB_FLOW_SPEC_UDP: 3235 ib_filter_sz = offsetof(struct ib_flow_tcp_udp_filter, real_sz); 3236 actual_filter_sz = spec_filter_size(kern_spec_mask, 3237 kern_filter_sz, 3238 ib_filter_sz); 3239 if (actual_filter_sz <= 0) 3240 return -EINVAL; 3241 ib_spec->size = sizeof(struct ib_flow_spec_tcp_udp); 3242 memcpy(&ib_spec->tcp_udp.val, kern_spec_val, actual_filter_sz); 3243 memcpy(&ib_spec->tcp_udp.mask, kern_spec_mask, actual_filter_sz); 3244 break; 3245 case IB_FLOW_SPEC_VXLAN_TUNNEL: 3246 ib_filter_sz = offsetof(struct ib_flow_tunnel_filter, real_sz); 3247 actual_filter_sz = spec_filter_size(kern_spec_mask, 3248 kern_filter_sz, 3249 ib_filter_sz); 3250 if (actual_filter_sz <= 0) 3251 return -EINVAL; 3252 ib_spec->tunnel.size = sizeof(struct ib_flow_spec_tunnel); 3253 memcpy(&ib_spec->tunnel.val, kern_spec_val, actual_filter_sz); 3254 memcpy(&ib_spec->tunnel.mask, kern_spec_mask, actual_filter_sz); 3255 3256 if ((ntohl(ib_spec->tunnel.mask.tunnel_id)) >= BIT(24) || 3257 (ntohl(ib_spec->tunnel.val.tunnel_id)) >= BIT(24)) 3258 return -EINVAL; 3259 break; 3260 default: 3261 return -EINVAL; 3262 } 3263 return 0; 3264 } 3265 3266 int ib_uverbs_ex_create_wq(struct ib_uverbs_file *file, 3267 struct ib_device *ib_dev, 3268 struct ib_udata *ucore, 3269 struct ib_udata *uhw) 3270 { 3271 struct ib_uverbs_ex_create_wq cmd = {}; 3272 struct ib_uverbs_ex_create_wq_resp resp = {}; 3273 struct ib_uwq_object *obj; 3274 int err = 0; 3275 struct ib_cq *cq; 3276 struct ib_pd *pd; 3277 struct ib_wq *wq; 3278 struct ib_wq_init_attr wq_init_attr = {}; 3279 size_t required_cmd_sz; 3280 size_t required_resp_len; 3281 3282 required_cmd_sz = offsetof(typeof(cmd), max_sge) + sizeof(cmd.max_sge); 3283 required_resp_len = offsetof(typeof(resp), wqn) + sizeof(resp.wqn); 3284 3285 if (ucore->inlen < required_cmd_sz) 3286 return -EINVAL; 3287 3288 if (ucore->outlen < required_resp_len) 3289 return -ENOSPC; 3290 3291 if (ucore->inlen > sizeof(cmd) && 3292 !ib_is_udata_cleared(ucore, sizeof(cmd), 3293 ucore->inlen - sizeof(cmd))) 3294 return -EOPNOTSUPP; 3295 3296 err = ib_copy_from_udata(&cmd, ucore, min(sizeof(cmd), ucore->inlen)); 3297 if (err) 3298 return err; 3299 3300 if (cmd.comp_mask) 3301 return -EOPNOTSUPP; 3302 3303 obj = kmalloc(sizeof(*obj), GFP_KERNEL); 3304 if (!obj) 3305 return -ENOMEM; 3306 3307 init_uobj(&obj->uevent.uobject, cmd.user_handle, file->ucontext, 3308 &wq_lock_class); 3309 down_write(&obj->uevent.uobject.mutex); 3310 pd = idr_read_pd(cmd.pd_handle, file->ucontext); 3311 if (!pd) { 3312 err = -EINVAL; 3313 goto err_uobj; 3314 } 3315 3316 cq = idr_read_cq(cmd.cq_handle, file->ucontext, 0); 3317 if (!cq) { 3318 err = -EINVAL; 3319 goto err_put_pd; 3320 } 3321 3322 wq_init_attr.cq = cq; 3323 wq_init_attr.max_sge = cmd.max_sge; 3324 wq_init_attr.max_wr = cmd.max_wr; 3325 wq_init_attr.wq_context = file; 3326 wq_init_attr.wq_type = cmd.wq_type; 3327 wq_init_attr.event_handler = ib_uverbs_wq_event_handler; 3328 obj->uevent.events_reported = 0; 3329 INIT_LIST_HEAD(&obj->uevent.event_list); 3330 wq = pd->device->create_wq(pd, &wq_init_attr, uhw); 3331 if (IS_ERR(wq)) { 3332 err = PTR_ERR(wq); 3333 goto err_put_cq; 3334 } 3335 3336 wq->uobject = &obj->uevent.uobject; 3337 obj->uevent.uobject.object = wq; 3338 wq->wq_type = wq_init_attr.wq_type; 3339 wq->cq = cq; 3340 wq->pd = pd; 3341 wq->device = pd->device; 3342 wq->wq_context = wq_init_attr.wq_context; 3343 atomic_set(&wq->usecnt, 0); 3344 atomic_inc(&pd->usecnt); 3345 atomic_inc(&cq->usecnt); 3346 wq->uobject = &obj->uevent.uobject; 3347 obj->uevent.uobject.object = wq; 3348 err = idr_add_uobj(&ib_uverbs_wq_idr, &obj->uevent.uobject); 3349 if (err) 3350 goto destroy_wq; 3351 3352 memset(&resp, 0, sizeof(resp)); 3353 resp.wq_handle = obj->uevent.uobject.id; 3354 resp.max_sge = wq_init_attr.max_sge; 3355 resp.max_wr = wq_init_attr.max_wr; 3356 resp.wqn = wq->wq_num; 3357 resp.response_length = required_resp_len; 3358 err = ib_copy_to_udata(ucore, 3359 &resp, resp.response_length); 3360 if (err) 3361 goto err_copy; 3362 3363 put_pd_read(pd); 3364 put_cq_read(cq); 3365 3366 mutex_lock(&file->mutex); 3367 list_add_tail(&obj->uevent.uobject.list, &file->ucontext->wq_list); 3368 mutex_unlock(&file->mutex); 3369 3370 obj->uevent.uobject.live = 1; 3371 up_write(&obj->uevent.uobject.mutex); 3372 return 0; 3373 3374 err_copy: 3375 idr_remove_uobj(&ib_uverbs_wq_idr, &obj->uevent.uobject); 3376 destroy_wq: 3377 ib_destroy_wq(wq); 3378 err_put_cq: 3379 put_cq_read(cq); 3380 err_put_pd: 3381 put_pd_read(pd); 3382 err_uobj: 3383 put_uobj_write(&obj->uevent.uobject); 3384 3385 return err; 3386 } 3387 3388 int ib_uverbs_ex_destroy_wq(struct ib_uverbs_file *file, 3389 struct ib_device *ib_dev, 3390 struct ib_udata *ucore, 3391 struct ib_udata *uhw) 3392 { 3393 struct ib_uverbs_ex_destroy_wq cmd = {}; 3394 struct ib_uverbs_ex_destroy_wq_resp resp = {}; 3395 struct ib_wq *wq; 3396 struct ib_uobject *uobj; 3397 struct ib_uwq_object *obj; 3398 size_t required_cmd_sz; 3399 size_t required_resp_len; 3400 int ret; 3401 3402 required_cmd_sz = offsetof(typeof(cmd), wq_handle) + sizeof(cmd.wq_handle); 3403 required_resp_len = offsetof(typeof(resp), reserved) + sizeof(resp.reserved); 3404 3405 if (ucore->inlen < required_cmd_sz) 3406 return -EINVAL; 3407 3408 if (ucore->outlen < required_resp_len) 3409 return -ENOSPC; 3410 3411 if (ucore->inlen > sizeof(cmd) && 3412 !ib_is_udata_cleared(ucore, sizeof(cmd), 3413 ucore->inlen - sizeof(cmd))) 3414 return -EOPNOTSUPP; 3415 3416 ret = ib_copy_from_udata(&cmd, ucore, min(sizeof(cmd), ucore->inlen)); 3417 if (ret) 3418 return ret; 3419 3420 if (cmd.comp_mask) 3421 return -EOPNOTSUPP; 3422 3423 resp.response_length = required_resp_len; 3424 uobj = idr_write_uobj(&ib_uverbs_wq_idr, cmd.wq_handle, 3425 file->ucontext); 3426 if (!uobj) 3427 return -EINVAL; 3428 3429 wq = uobj->object; 3430 obj = container_of(uobj, struct ib_uwq_object, uevent.uobject); 3431 ret = ib_destroy_wq(wq); 3432 if (!ret) 3433 uobj->live = 0; 3434 3435 put_uobj_write(uobj); 3436 if (ret) 3437 return ret; 3438 3439 idr_remove_uobj(&ib_uverbs_wq_idr, uobj); 3440 3441 mutex_lock(&file->mutex); 3442 list_del(&uobj->list); 3443 mutex_unlock(&file->mutex); 3444 3445 ib_uverbs_release_uevent(file, &obj->uevent); 3446 resp.events_reported = obj->uevent.events_reported; 3447 put_uobj(uobj); 3448 3449 ret = ib_copy_to_udata(ucore, &resp, resp.response_length); 3450 if (ret) 3451 return ret; 3452 3453 return 0; 3454 } 3455 3456 int ib_uverbs_ex_modify_wq(struct ib_uverbs_file *file, 3457 struct ib_device *ib_dev, 3458 struct ib_udata *ucore, 3459 struct ib_udata *uhw) 3460 { 3461 struct ib_uverbs_ex_modify_wq cmd = {}; 3462 struct ib_wq *wq; 3463 struct ib_wq_attr wq_attr = {}; 3464 size_t required_cmd_sz; 3465 int ret; 3466 3467 required_cmd_sz = offsetof(typeof(cmd), curr_wq_state) + sizeof(cmd.curr_wq_state); 3468 if (ucore->inlen < required_cmd_sz) 3469 return -EINVAL; 3470 3471 if (ucore->inlen > sizeof(cmd) && 3472 !ib_is_udata_cleared(ucore, sizeof(cmd), 3473 ucore->inlen - sizeof(cmd))) 3474 return -EOPNOTSUPP; 3475 3476 ret = ib_copy_from_udata(&cmd, ucore, min(sizeof(cmd), ucore->inlen)); 3477 if (ret) 3478 return ret; 3479 3480 if (!cmd.attr_mask) 3481 return -EINVAL; 3482 3483 if (cmd.attr_mask > (IB_WQ_STATE | IB_WQ_CUR_STATE)) 3484 return -EINVAL; 3485 3486 wq = idr_read_wq(cmd.wq_handle, file->ucontext); 3487 if (!wq) 3488 return -EINVAL; 3489 3490 wq_attr.curr_wq_state = cmd.curr_wq_state; 3491 wq_attr.wq_state = cmd.wq_state; 3492 ret = wq->device->modify_wq(wq, &wq_attr, cmd.attr_mask, uhw); 3493 put_wq_read(wq); 3494 return ret; 3495 } 3496 3497 int ib_uverbs_ex_create_rwq_ind_table(struct ib_uverbs_file *file, 3498 struct ib_device *ib_dev, 3499 struct ib_udata *ucore, 3500 struct ib_udata *uhw) 3501 { 3502 struct ib_uverbs_ex_create_rwq_ind_table cmd = {}; 3503 struct ib_uverbs_ex_create_rwq_ind_table_resp resp = {}; 3504 struct ib_uobject *uobj; 3505 int err = 0; 3506 struct ib_rwq_ind_table_init_attr init_attr = {}; 3507 struct ib_rwq_ind_table *rwq_ind_tbl; 3508 struct ib_wq **wqs = NULL; 3509 u32 *wqs_handles = NULL; 3510 struct ib_wq *wq = NULL; 3511 int i, j, num_read_wqs; 3512 u32 num_wq_handles; 3513 u32 expected_in_size; 3514 size_t required_cmd_sz_header; 3515 size_t required_resp_len; 3516 3517 required_cmd_sz_header = offsetof(typeof(cmd), log_ind_tbl_size) + sizeof(cmd.log_ind_tbl_size); 3518 required_resp_len = offsetof(typeof(resp), ind_tbl_num) + sizeof(resp.ind_tbl_num); 3519 3520 if (ucore->inlen < required_cmd_sz_header) 3521 return -EINVAL; 3522 3523 if (ucore->outlen < required_resp_len) 3524 return -ENOSPC; 3525 3526 err = ib_copy_from_udata(&cmd, ucore, required_cmd_sz_header); 3527 if (err) 3528 return err; 3529 3530 ucore->inbuf += required_cmd_sz_header; 3531 ucore->inlen -= required_cmd_sz_header; 3532 3533 if (cmd.comp_mask) 3534 return -EOPNOTSUPP; 3535 3536 if (cmd.log_ind_tbl_size > IB_USER_VERBS_MAX_LOG_IND_TBL_SIZE) 3537 return -EINVAL; 3538 3539 num_wq_handles = 1 << cmd.log_ind_tbl_size; 3540 expected_in_size = num_wq_handles * sizeof(__u32); 3541 if (num_wq_handles == 1) 3542 /* input size for wq handles is u64 aligned */ 3543 expected_in_size += sizeof(__u32); 3544 3545 if (ucore->inlen < expected_in_size) 3546 return -EINVAL; 3547 3548 if (ucore->inlen > expected_in_size && 3549 !ib_is_udata_cleared(ucore, expected_in_size, 3550 ucore->inlen - expected_in_size)) 3551 return -EOPNOTSUPP; 3552 3553 wqs_handles = kcalloc(num_wq_handles, sizeof(*wqs_handles), 3554 GFP_KERNEL); 3555 if (!wqs_handles) 3556 return -ENOMEM; 3557 3558 err = ib_copy_from_udata(wqs_handles, ucore, 3559 num_wq_handles * sizeof(__u32)); 3560 if (err) 3561 goto err_free; 3562 3563 wqs = kcalloc(num_wq_handles, sizeof(*wqs), GFP_KERNEL); 3564 if (!wqs) { 3565 err = -ENOMEM; 3566 goto err_free; 3567 } 3568 3569 for (num_read_wqs = 0; num_read_wqs < num_wq_handles; 3570 num_read_wqs++) { 3571 wq = idr_read_wq(wqs_handles[num_read_wqs], file->ucontext); 3572 if (!wq) { 3573 err = -EINVAL; 3574 goto put_wqs; 3575 } 3576 3577 wqs[num_read_wqs] = wq; 3578 } 3579 3580 uobj = kmalloc(sizeof(*uobj), GFP_KERNEL); 3581 if (!uobj) { 3582 err = -ENOMEM; 3583 goto put_wqs; 3584 } 3585 3586 init_uobj(uobj, 0, file->ucontext, &rwq_ind_table_lock_class); 3587 down_write(&uobj->mutex); 3588 init_attr.log_ind_tbl_size = cmd.log_ind_tbl_size; 3589 init_attr.ind_tbl = wqs; 3590 rwq_ind_tbl = ib_dev->create_rwq_ind_table(ib_dev, &init_attr, uhw); 3591 3592 if (IS_ERR(rwq_ind_tbl)) { 3593 err = PTR_ERR(rwq_ind_tbl); 3594 goto err_uobj; 3595 } 3596 3597 rwq_ind_tbl->ind_tbl = wqs; 3598 rwq_ind_tbl->log_ind_tbl_size = init_attr.log_ind_tbl_size; 3599 rwq_ind_tbl->uobject = uobj; 3600 uobj->object = rwq_ind_tbl; 3601 rwq_ind_tbl->device = ib_dev; 3602 atomic_set(&rwq_ind_tbl->usecnt, 0); 3603 3604 for (i = 0; i < num_wq_handles; i++) 3605 atomic_inc(&wqs[i]->usecnt); 3606 3607 err = idr_add_uobj(&ib_uverbs_rwq_ind_tbl_idr, uobj); 3608 if (err) 3609 goto destroy_ind_tbl; 3610 3611 resp.ind_tbl_handle = uobj->id; 3612 resp.ind_tbl_num = rwq_ind_tbl->ind_tbl_num; 3613 resp.response_length = required_resp_len; 3614 3615 err = ib_copy_to_udata(ucore, 3616 &resp, resp.response_length); 3617 if (err) 3618 goto err_copy; 3619 3620 kfree(wqs_handles); 3621 3622 for (j = 0; j < num_read_wqs; j++) 3623 put_wq_read(wqs[j]); 3624 3625 mutex_lock(&file->mutex); 3626 list_add_tail(&uobj->list, &file->ucontext->rwq_ind_tbl_list); 3627 mutex_unlock(&file->mutex); 3628 3629 uobj->live = 1; 3630 3631 up_write(&uobj->mutex); 3632 return 0; 3633 3634 err_copy: 3635 idr_remove_uobj(&ib_uverbs_rwq_ind_tbl_idr, uobj); 3636 destroy_ind_tbl: 3637 ib_destroy_rwq_ind_table(rwq_ind_tbl); 3638 err_uobj: 3639 put_uobj_write(uobj); 3640 put_wqs: 3641 for (j = 0; j < num_read_wqs; j++) 3642 put_wq_read(wqs[j]); 3643 err_free: 3644 kfree(wqs_handles); 3645 kfree(wqs); 3646 return err; 3647 } 3648 3649 int ib_uverbs_ex_destroy_rwq_ind_table(struct ib_uverbs_file *file, 3650 struct ib_device *ib_dev, 3651 struct ib_udata *ucore, 3652 struct ib_udata *uhw) 3653 { 3654 struct ib_uverbs_ex_destroy_rwq_ind_table cmd = {}; 3655 struct ib_rwq_ind_table *rwq_ind_tbl; 3656 struct ib_uobject *uobj; 3657 int ret; 3658 struct ib_wq **ind_tbl; 3659 size_t required_cmd_sz; 3660 3661 required_cmd_sz = offsetof(typeof(cmd), ind_tbl_handle) + sizeof(cmd.ind_tbl_handle); 3662 3663 if (ucore->inlen < required_cmd_sz) 3664 return -EINVAL; 3665 3666 if (ucore->inlen > sizeof(cmd) && 3667 !ib_is_udata_cleared(ucore, sizeof(cmd), 3668 ucore->inlen - sizeof(cmd))) 3669 return -EOPNOTSUPP; 3670 3671 ret = ib_copy_from_udata(&cmd, ucore, min(sizeof(cmd), ucore->inlen)); 3672 if (ret) 3673 return ret; 3674 3675 if (cmd.comp_mask) 3676 return -EOPNOTSUPP; 3677 3678 uobj = idr_write_uobj(&ib_uverbs_rwq_ind_tbl_idr, cmd.ind_tbl_handle, 3679 file->ucontext); 3680 if (!uobj) 3681 return -EINVAL; 3682 rwq_ind_tbl = uobj->object; 3683 ind_tbl = rwq_ind_tbl->ind_tbl; 3684 3685 ret = ib_destroy_rwq_ind_table(rwq_ind_tbl); 3686 if (!ret) 3687 uobj->live = 0; 3688 3689 put_uobj_write(uobj); 3690 3691 if (ret) 3692 return ret; 3693 3694 idr_remove_uobj(&ib_uverbs_rwq_ind_tbl_idr, uobj); 3695 3696 mutex_lock(&file->mutex); 3697 list_del(&uobj->list); 3698 mutex_unlock(&file->mutex); 3699 3700 put_uobj(uobj); 3701 kfree(ind_tbl); 3702 return ret; 3703 } 3704 3705 int ib_uverbs_ex_create_flow(struct ib_uverbs_file *file, 3706 struct ib_device *ib_dev, 3707 struct ib_udata *ucore, 3708 struct ib_udata *uhw) 3709 { 3710 struct ib_uverbs_create_flow cmd; 3711 struct ib_uverbs_create_flow_resp resp; 3712 struct ib_uobject *uobj; 3713 struct ib_flow *flow_id; 3714 struct ib_uverbs_flow_attr *kern_flow_attr; 3715 struct ib_flow_attr *flow_attr; 3716 struct ib_qp *qp; 3717 int err = 0; 3718 void *kern_spec; 3719 void *ib_spec; 3720 int i; 3721 3722 if (ucore->inlen < sizeof(cmd)) 3723 return -EINVAL; 3724 3725 if (ucore->outlen < sizeof(resp)) 3726 return -ENOSPC; 3727 3728 err = ib_copy_from_udata(&cmd, ucore, sizeof(cmd)); 3729 if (err) 3730 return err; 3731 3732 ucore->inbuf += sizeof(cmd); 3733 ucore->inlen -= sizeof(cmd); 3734 3735 if (cmd.comp_mask) 3736 return -EINVAL; 3737 3738 if (!capable(CAP_NET_RAW)) 3739 return -EPERM; 3740 3741 if (cmd.flow_attr.flags >= IB_FLOW_ATTR_FLAGS_RESERVED) 3742 return -EINVAL; 3743 3744 if ((cmd.flow_attr.flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP) && 3745 ((cmd.flow_attr.type == IB_FLOW_ATTR_ALL_DEFAULT) || 3746 (cmd.flow_attr.type == IB_FLOW_ATTR_MC_DEFAULT))) 3747 return -EINVAL; 3748 3749 if (cmd.flow_attr.num_of_specs > IB_FLOW_SPEC_SUPPORT_LAYERS) 3750 return -EINVAL; 3751 3752 if (cmd.flow_attr.size > ucore->inlen || 3753 cmd.flow_attr.size > 3754 (cmd.flow_attr.num_of_specs * sizeof(struct ib_uverbs_flow_spec))) 3755 return -EINVAL; 3756 3757 if (cmd.flow_attr.reserved[0] || 3758 cmd.flow_attr.reserved[1]) 3759 return -EINVAL; 3760 3761 if (cmd.flow_attr.num_of_specs) { 3762 kern_flow_attr = kmalloc(sizeof(*kern_flow_attr) + cmd.flow_attr.size, 3763 GFP_KERNEL); 3764 if (!kern_flow_attr) 3765 return -ENOMEM; 3766 3767 memcpy(kern_flow_attr, &cmd.flow_attr, sizeof(*kern_flow_attr)); 3768 err = ib_copy_from_udata(kern_flow_attr + 1, ucore, 3769 cmd.flow_attr.size); 3770 if (err) 3771 goto err_free_attr; 3772 } else { 3773 kern_flow_attr = &cmd.flow_attr; 3774 } 3775 3776 uobj = kmalloc(sizeof(*uobj), GFP_KERNEL); 3777 if (!uobj) { 3778 err = -ENOMEM; 3779 goto err_free_attr; 3780 } 3781 init_uobj(uobj, 0, file->ucontext, &rule_lock_class); 3782 down_write(&uobj->mutex); 3783 3784 qp = idr_read_qp(cmd.qp_handle, file->ucontext); 3785 if (!qp) { 3786 err = -EINVAL; 3787 goto err_uobj; 3788 } 3789 3790 flow_attr = kzalloc(sizeof(*flow_attr) + cmd.flow_attr.num_of_specs * 3791 sizeof(union ib_flow_spec), GFP_KERNEL); 3792 if (!flow_attr) { 3793 err = -ENOMEM; 3794 goto err_put; 3795 } 3796 3797 flow_attr->type = kern_flow_attr->type; 3798 flow_attr->priority = kern_flow_attr->priority; 3799 flow_attr->num_of_specs = kern_flow_attr->num_of_specs; 3800 flow_attr->port = kern_flow_attr->port; 3801 flow_attr->flags = kern_flow_attr->flags; 3802 flow_attr->size = sizeof(*flow_attr); 3803 3804 kern_spec = kern_flow_attr + 1; 3805 ib_spec = flow_attr + 1; 3806 for (i = 0; i < flow_attr->num_of_specs && 3807 cmd.flow_attr.size > offsetof(struct ib_uverbs_flow_spec, reserved) && 3808 cmd.flow_attr.size >= 3809 ((struct ib_uverbs_flow_spec *)kern_spec)->size; i++) { 3810 err = kern_spec_to_ib_spec(kern_spec, ib_spec); 3811 if (err) 3812 goto err_free; 3813 flow_attr->size += 3814 ((union ib_flow_spec *) ib_spec)->size; 3815 cmd.flow_attr.size -= ((struct ib_uverbs_flow_spec *)kern_spec)->size; 3816 kern_spec += ((struct ib_uverbs_flow_spec *) kern_spec)->size; 3817 ib_spec += ((union ib_flow_spec *) ib_spec)->size; 3818 } 3819 if (cmd.flow_attr.size || (i != flow_attr->num_of_specs)) { 3820 pr_warn("create flow failed, flow %d: %d bytes left from uverb cmd\n", 3821 i, cmd.flow_attr.size); 3822 err = -EINVAL; 3823 goto err_free; 3824 } 3825 flow_id = ib_create_flow(qp, flow_attr, IB_FLOW_DOMAIN_USER); 3826 if (IS_ERR(flow_id)) { 3827 err = PTR_ERR(flow_id); 3828 goto err_free; 3829 } 3830 flow_id->uobject = uobj; 3831 uobj->object = flow_id; 3832 3833 err = idr_add_uobj(&ib_uverbs_rule_idr, uobj); 3834 if (err) 3835 goto destroy_flow; 3836 3837 memset(&resp, 0, sizeof(resp)); 3838 resp.flow_handle = uobj->id; 3839 3840 err = ib_copy_to_udata(ucore, 3841 &resp, sizeof(resp)); 3842 if (err) 3843 goto err_copy; 3844 3845 put_qp_read(qp); 3846 mutex_lock(&file->mutex); 3847 list_add_tail(&uobj->list, &file->ucontext->rule_list); 3848 mutex_unlock(&file->mutex); 3849 3850 uobj->live = 1; 3851 3852 up_write(&uobj->mutex); 3853 kfree(flow_attr); 3854 if (cmd.flow_attr.num_of_specs) 3855 kfree(kern_flow_attr); 3856 return 0; 3857 err_copy: 3858 idr_remove_uobj(&ib_uverbs_rule_idr, uobj); 3859 destroy_flow: 3860 ib_destroy_flow(flow_id); 3861 err_free: 3862 kfree(flow_attr); 3863 err_put: 3864 put_qp_read(qp); 3865 err_uobj: 3866 put_uobj_write(uobj); 3867 err_free_attr: 3868 if (cmd.flow_attr.num_of_specs) 3869 kfree(kern_flow_attr); 3870 return err; 3871 } 3872 3873 int ib_uverbs_ex_destroy_flow(struct ib_uverbs_file *file, 3874 struct ib_device *ib_dev, 3875 struct ib_udata *ucore, 3876 struct ib_udata *uhw) 3877 { 3878 struct ib_uverbs_destroy_flow cmd; 3879 struct ib_flow *flow_id; 3880 struct ib_uobject *uobj; 3881 int ret; 3882 3883 if (ucore->inlen < sizeof(cmd)) 3884 return -EINVAL; 3885 3886 ret = ib_copy_from_udata(&cmd, ucore, sizeof(cmd)); 3887 if (ret) 3888 return ret; 3889 3890 if (cmd.comp_mask) 3891 return -EINVAL; 3892 3893 uobj = idr_write_uobj(&ib_uverbs_rule_idr, cmd.flow_handle, 3894 file->ucontext); 3895 if (!uobj) 3896 return -EINVAL; 3897 flow_id = uobj->object; 3898 3899 ret = ib_destroy_flow(flow_id); 3900 if (!ret) 3901 uobj->live = 0; 3902 3903 put_uobj_write(uobj); 3904 3905 idr_remove_uobj(&ib_uverbs_rule_idr, uobj); 3906 3907 mutex_lock(&file->mutex); 3908 list_del(&uobj->list); 3909 mutex_unlock(&file->mutex); 3910 3911 put_uobj(uobj); 3912 3913 return ret; 3914 } 3915 3916 static int __uverbs_create_xsrq(struct ib_uverbs_file *file, 3917 struct ib_device *ib_dev, 3918 struct ib_uverbs_create_xsrq *cmd, 3919 struct ib_udata *udata) 3920 { 3921 struct ib_uverbs_create_srq_resp resp; 3922 struct ib_usrq_object *obj; 3923 struct ib_pd *pd; 3924 struct ib_srq *srq; 3925 struct ib_uobject *uninitialized_var(xrcd_uobj); 3926 struct ib_srq_init_attr attr; 3927 int ret; 3928 3929 obj = kmalloc(sizeof *obj, GFP_KERNEL); 3930 if (!obj) 3931 return -ENOMEM; 3932 3933 init_uobj(&obj->uevent.uobject, cmd->user_handle, file->ucontext, &srq_lock_class); 3934 down_write(&obj->uevent.uobject.mutex); 3935 3936 if (cmd->srq_type == IB_SRQT_XRC) { 3937 attr.ext.xrc.xrcd = idr_read_xrcd(cmd->xrcd_handle, file->ucontext, &xrcd_uobj); 3938 if (!attr.ext.xrc.xrcd) { 3939 ret = -EINVAL; 3940 goto err; 3941 } 3942 3943 obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object, uobject); 3944 atomic_inc(&obj->uxrcd->refcnt); 3945 3946 attr.ext.xrc.cq = idr_read_cq(cmd->cq_handle, file->ucontext, 0); 3947 if (!attr.ext.xrc.cq) { 3948 ret = -EINVAL; 3949 goto err_put_xrcd; 3950 } 3951 } 3952 3953 pd = idr_read_pd(cmd->pd_handle, file->ucontext); 3954 if (!pd) { 3955 ret = -EINVAL; 3956 goto err_put_cq; 3957 } 3958 3959 attr.event_handler = ib_uverbs_srq_event_handler; 3960 attr.srq_context = file; 3961 attr.srq_type = cmd->srq_type; 3962 attr.attr.max_wr = cmd->max_wr; 3963 attr.attr.max_sge = cmd->max_sge; 3964 attr.attr.srq_limit = cmd->srq_limit; 3965 3966 obj->uevent.events_reported = 0; 3967 INIT_LIST_HEAD(&obj->uevent.event_list); 3968 3969 srq = pd->device->create_srq(pd, &attr, udata); 3970 if (IS_ERR(srq)) { 3971 ret = PTR_ERR(srq); 3972 goto err_put; 3973 } 3974 3975 srq->device = pd->device; 3976 srq->pd = pd; 3977 srq->srq_type = cmd->srq_type; 3978 srq->uobject = &obj->uevent.uobject; 3979 srq->event_handler = attr.event_handler; 3980 srq->srq_context = attr.srq_context; 3981 3982 if (cmd->srq_type == IB_SRQT_XRC) { 3983 srq->ext.xrc.cq = attr.ext.xrc.cq; 3984 srq->ext.xrc.xrcd = attr.ext.xrc.xrcd; 3985 atomic_inc(&attr.ext.xrc.cq->usecnt); 3986 atomic_inc(&attr.ext.xrc.xrcd->usecnt); 3987 } 3988 3989 atomic_inc(&pd->usecnt); 3990 atomic_set(&srq->usecnt, 0); 3991 3992 obj->uevent.uobject.object = srq; 3993 ret = idr_add_uobj(&ib_uverbs_srq_idr, &obj->uevent.uobject); 3994 if (ret) 3995 goto err_destroy; 3996 3997 memset(&resp, 0, sizeof resp); 3998 resp.srq_handle = obj->uevent.uobject.id; 3999 resp.max_wr = attr.attr.max_wr; 4000 resp.max_sge = attr.attr.max_sge; 4001 if (cmd->srq_type == IB_SRQT_XRC) 4002 resp.srqn = srq->ext.xrc.srq_num; 4003 4004 if (copy_to_user((void __user *) (unsigned long) cmd->response, 4005 &resp, sizeof resp)) { 4006 ret = -EFAULT; 4007 goto err_copy; 4008 } 4009 4010 if (cmd->srq_type == IB_SRQT_XRC) { 4011 put_uobj_read(xrcd_uobj); 4012 put_cq_read(attr.ext.xrc.cq); 4013 } 4014 put_pd_read(pd); 4015 4016 mutex_lock(&file->mutex); 4017 list_add_tail(&obj->uevent.uobject.list, &file->ucontext->srq_list); 4018 mutex_unlock(&file->mutex); 4019 4020 obj->uevent.uobject.live = 1; 4021 4022 up_write(&obj->uevent.uobject.mutex); 4023 4024 return 0; 4025 4026 err_copy: 4027 idr_remove_uobj(&ib_uverbs_srq_idr, &obj->uevent.uobject); 4028 4029 err_destroy: 4030 ib_destroy_srq(srq); 4031 4032 err_put: 4033 put_pd_read(pd); 4034 4035 err_put_cq: 4036 if (cmd->srq_type == IB_SRQT_XRC) 4037 put_cq_read(attr.ext.xrc.cq); 4038 4039 err_put_xrcd: 4040 if (cmd->srq_type == IB_SRQT_XRC) { 4041 atomic_dec(&obj->uxrcd->refcnt); 4042 put_uobj_read(xrcd_uobj); 4043 } 4044 4045 err: 4046 put_uobj_write(&obj->uevent.uobject); 4047 return ret; 4048 } 4049 4050 ssize_t ib_uverbs_create_srq(struct ib_uverbs_file *file, 4051 struct ib_device *ib_dev, 4052 const char __user *buf, int in_len, 4053 int out_len) 4054 { 4055 struct ib_uverbs_create_srq cmd; 4056 struct ib_uverbs_create_xsrq xcmd; 4057 struct ib_uverbs_create_srq_resp resp; 4058 struct ib_udata udata; 4059 int ret; 4060 4061 if (out_len < sizeof resp) 4062 return -ENOSPC; 4063 4064 if (copy_from_user(&cmd, buf, sizeof cmd)) 4065 return -EFAULT; 4066 4067 xcmd.response = cmd.response; 4068 xcmd.user_handle = cmd.user_handle; 4069 xcmd.srq_type = IB_SRQT_BASIC; 4070 xcmd.pd_handle = cmd.pd_handle; 4071 xcmd.max_wr = cmd.max_wr; 4072 xcmd.max_sge = cmd.max_sge; 4073 xcmd.srq_limit = cmd.srq_limit; 4074 4075 INIT_UDATA(&udata, buf + sizeof cmd, 4076 (unsigned long) cmd.response + sizeof resp, 4077 in_len - sizeof cmd - sizeof(struct ib_uverbs_cmd_hdr), 4078 out_len - sizeof resp); 4079 4080 ret = __uverbs_create_xsrq(file, ib_dev, &xcmd, &udata); 4081 if (ret) 4082 return ret; 4083 4084 return in_len; 4085 } 4086 4087 ssize_t ib_uverbs_create_xsrq(struct ib_uverbs_file *file, 4088 struct ib_device *ib_dev, 4089 const char __user *buf, int in_len, int out_len) 4090 { 4091 struct ib_uverbs_create_xsrq cmd; 4092 struct ib_uverbs_create_srq_resp resp; 4093 struct ib_udata udata; 4094 int ret; 4095 4096 if (out_len < sizeof resp) 4097 return -ENOSPC; 4098 4099 if (copy_from_user(&cmd, buf, sizeof cmd)) 4100 return -EFAULT; 4101 4102 INIT_UDATA(&udata, buf + sizeof cmd, 4103 (unsigned long) cmd.response + sizeof resp, 4104 in_len - sizeof cmd - sizeof(struct ib_uverbs_cmd_hdr), 4105 out_len - sizeof resp); 4106 4107 ret = __uverbs_create_xsrq(file, ib_dev, &cmd, &udata); 4108 if (ret) 4109 return ret; 4110 4111 return in_len; 4112 } 4113 4114 ssize_t ib_uverbs_modify_srq(struct ib_uverbs_file *file, 4115 struct ib_device *ib_dev, 4116 const char __user *buf, int in_len, 4117 int out_len) 4118 { 4119 struct ib_uverbs_modify_srq cmd; 4120 struct ib_udata udata; 4121 struct ib_srq *srq; 4122 struct ib_srq_attr attr; 4123 int ret; 4124 4125 if (copy_from_user(&cmd, buf, sizeof cmd)) 4126 return -EFAULT; 4127 4128 INIT_UDATA(&udata, buf + sizeof cmd, NULL, in_len - sizeof cmd, 4129 out_len); 4130 4131 srq = idr_read_srq(cmd.srq_handle, file->ucontext); 4132 if (!srq) 4133 return -EINVAL; 4134 4135 attr.max_wr = cmd.max_wr; 4136 attr.srq_limit = cmd.srq_limit; 4137 4138 ret = srq->device->modify_srq(srq, &attr, cmd.attr_mask, &udata); 4139 4140 put_srq_read(srq); 4141 4142 return ret ? ret : in_len; 4143 } 4144 4145 ssize_t ib_uverbs_query_srq(struct ib_uverbs_file *file, 4146 struct ib_device *ib_dev, 4147 const char __user *buf, 4148 int in_len, int out_len) 4149 { 4150 struct ib_uverbs_query_srq cmd; 4151 struct ib_uverbs_query_srq_resp resp; 4152 struct ib_srq_attr attr; 4153 struct ib_srq *srq; 4154 int ret; 4155 4156 if (out_len < sizeof resp) 4157 return -ENOSPC; 4158 4159 if (copy_from_user(&cmd, buf, sizeof cmd)) 4160 return -EFAULT; 4161 4162 srq = idr_read_srq(cmd.srq_handle, file->ucontext); 4163 if (!srq) 4164 return -EINVAL; 4165 4166 ret = ib_query_srq(srq, &attr); 4167 4168 put_srq_read(srq); 4169 4170 if (ret) 4171 return ret; 4172 4173 memset(&resp, 0, sizeof resp); 4174 4175 resp.max_wr = attr.max_wr; 4176 resp.max_sge = attr.max_sge; 4177 resp.srq_limit = attr.srq_limit; 4178 4179 if (copy_to_user((void __user *) (unsigned long) cmd.response, 4180 &resp, sizeof resp)) 4181 return -EFAULT; 4182 4183 return in_len; 4184 } 4185 4186 ssize_t ib_uverbs_destroy_srq(struct ib_uverbs_file *file, 4187 struct ib_device *ib_dev, 4188 const char __user *buf, int in_len, 4189 int out_len) 4190 { 4191 struct ib_uverbs_destroy_srq cmd; 4192 struct ib_uverbs_destroy_srq_resp resp; 4193 struct ib_uobject *uobj; 4194 struct ib_srq *srq; 4195 struct ib_uevent_object *obj; 4196 int ret = -EINVAL; 4197 struct ib_usrq_object *us; 4198 enum ib_srq_type srq_type; 4199 4200 if (copy_from_user(&cmd, buf, sizeof cmd)) 4201 return -EFAULT; 4202 4203 uobj = idr_write_uobj(&ib_uverbs_srq_idr, cmd.srq_handle, file->ucontext); 4204 if (!uobj) 4205 return -EINVAL; 4206 srq = uobj->object; 4207 obj = container_of(uobj, struct ib_uevent_object, uobject); 4208 srq_type = srq->srq_type; 4209 4210 ret = ib_destroy_srq(srq); 4211 if (!ret) 4212 uobj->live = 0; 4213 4214 put_uobj_write(uobj); 4215 4216 if (ret) 4217 return ret; 4218 4219 if (srq_type == IB_SRQT_XRC) { 4220 us = container_of(obj, struct ib_usrq_object, uevent); 4221 atomic_dec(&us->uxrcd->refcnt); 4222 } 4223 4224 idr_remove_uobj(&ib_uverbs_srq_idr, uobj); 4225 4226 mutex_lock(&file->mutex); 4227 list_del(&uobj->list); 4228 mutex_unlock(&file->mutex); 4229 4230 ib_uverbs_release_uevent(file, obj); 4231 4232 memset(&resp, 0, sizeof resp); 4233 resp.events_reported = obj->events_reported; 4234 4235 put_uobj(uobj); 4236 4237 if (copy_to_user((void __user *) (unsigned long) cmd.response, 4238 &resp, sizeof resp)) 4239 ret = -EFAULT; 4240 4241 return ret ? ret : in_len; 4242 } 4243 4244 int ib_uverbs_ex_query_device(struct ib_uverbs_file *file, 4245 struct ib_device *ib_dev, 4246 struct ib_udata *ucore, 4247 struct ib_udata *uhw) 4248 { 4249 struct ib_uverbs_ex_query_device_resp resp = { {0} }; 4250 struct ib_uverbs_ex_query_device cmd; 4251 struct ib_device_attr attr = {0}; 4252 int err; 4253 4254 if (ucore->inlen < sizeof(cmd)) 4255 return -EINVAL; 4256 4257 err = ib_copy_from_udata(&cmd, ucore, sizeof(cmd)); 4258 if (err) 4259 return err; 4260 4261 if (cmd.comp_mask) 4262 return -EINVAL; 4263 4264 if (cmd.reserved) 4265 return -EINVAL; 4266 4267 resp.response_length = offsetof(typeof(resp), odp_caps); 4268 4269 if (ucore->outlen < resp.response_length) 4270 return -ENOSPC; 4271 4272 err = ib_dev->query_device(ib_dev, &attr, uhw); 4273 if (err) 4274 return err; 4275 4276 copy_query_dev_fields(file, ib_dev, &resp.base, &attr); 4277 4278 if (ucore->outlen < resp.response_length + sizeof(resp.odp_caps)) 4279 goto end; 4280 4281 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING 4282 resp.odp_caps.general_caps = attr.odp_caps.general_caps; 4283 resp.odp_caps.per_transport_caps.rc_odp_caps = 4284 attr.odp_caps.per_transport_caps.rc_odp_caps; 4285 resp.odp_caps.per_transport_caps.uc_odp_caps = 4286 attr.odp_caps.per_transport_caps.uc_odp_caps; 4287 resp.odp_caps.per_transport_caps.ud_odp_caps = 4288 attr.odp_caps.per_transport_caps.ud_odp_caps; 4289 #endif 4290 resp.response_length += sizeof(resp.odp_caps); 4291 4292 if (ucore->outlen < resp.response_length + sizeof(resp.timestamp_mask)) 4293 goto end; 4294 4295 resp.timestamp_mask = attr.timestamp_mask; 4296 resp.response_length += sizeof(resp.timestamp_mask); 4297 4298 if (ucore->outlen < resp.response_length + sizeof(resp.hca_core_clock)) 4299 goto end; 4300 4301 resp.hca_core_clock = attr.hca_core_clock; 4302 resp.response_length += sizeof(resp.hca_core_clock); 4303 4304 if (ucore->outlen < resp.response_length + sizeof(resp.device_cap_flags_ex)) 4305 goto end; 4306 4307 resp.device_cap_flags_ex = attr.device_cap_flags; 4308 resp.response_length += sizeof(resp.device_cap_flags_ex); 4309 4310 if (ucore->outlen < resp.response_length + sizeof(resp.rss_caps)) 4311 goto end; 4312 4313 resp.rss_caps.supported_qpts = attr.rss_caps.supported_qpts; 4314 resp.rss_caps.max_rwq_indirection_tables = 4315 attr.rss_caps.max_rwq_indirection_tables; 4316 resp.rss_caps.max_rwq_indirection_table_size = 4317 attr.rss_caps.max_rwq_indirection_table_size; 4318 4319 resp.response_length += sizeof(resp.rss_caps); 4320 4321 if (ucore->outlen < resp.response_length + sizeof(resp.max_wq_type_rq)) 4322 goto end; 4323 4324 resp.max_wq_type_rq = attr.max_wq_type_rq; 4325 resp.response_length += sizeof(resp.max_wq_type_rq); 4326 end: 4327 err = ib_copy_to_udata(ucore, &resp, resp.response_length); 4328 return err; 4329 } 4330