1 /* 2 * Copyright (c) 2005 Topspin Communications. All rights reserved. 3 * Copyright (c) 2005, 2006, 2007 Cisco Systems. All rights reserved. 4 * Copyright (c) 2005 PathScale, Inc. All rights reserved. 5 * Copyright (c) 2006 Mellanox Technologies. All rights reserved. 6 * 7 * This software is available to you under a choice of one of two 8 * licenses. You may choose to be licensed under the terms of the GNU 9 * General Public License (GPL) Version 2, available from the file 10 * COPYING in the main directory of this source tree, or the 11 * OpenIB.org BSD license below: 12 * 13 * Redistribution and use in source and binary forms, with or 14 * without modification, are permitted provided that the following 15 * conditions are met: 16 * 17 * - Redistributions of source code must retain the above 18 * copyright notice, this list of conditions and the following 19 * disclaimer. 20 * 21 * - Redistributions in binary form must reproduce the above 22 * copyright notice, this list of conditions and the following 23 * disclaimer in the documentation and/or other materials 24 * provided with the distribution. 25 * 26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 27 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 28 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 29 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 30 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 31 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 32 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 33 * SOFTWARE. 34 */ 35 36 #include <linux/file.h> 37 #include <linux/fs.h> 38 #include <linux/slab.h> 39 #include <linux/sched.h> 40 41 #include <asm/uaccess.h> 42 43 #include "uverbs.h" 44 #include "core_priv.h" 45 46 struct uverbs_lock_class { 47 struct lock_class_key key; 48 char name[16]; 49 }; 50 51 static struct uverbs_lock_class pd_lock_class = { .name = "PD-uobj" }; 52 static struct uverbs_lock_class mr_lock_class = { .name = "MR-uobj" }; 53 static struct uverbs_lock_class mw_lock_class = { .name = "MW-uobj" }; 54 static struct uverbs_lock_class cq_lock_class = { .name = "CQ-uobj" }; 55 static struct uverbs_lock_class qp_lock_class = { .name = "QP-uobj" }; 56 static struct uverbs_lock_class ah_lock_class = { .name = "AH-uobj" }; 57 static struct uverbs_lock_class srq_lock_class = { .name = "SRQ-uobj" }; 58 static struct uverbs_lock_class xrcd_lock_class = { .name = "XRCD-uobj" }; 59 static struct uverbs_lock_class rule_lock_class = { .name = "RULE-uobj" }; 60 61 /* 62 * The ib_uobject locking scheme is as follows: 63 * 64 * - ib_uverbs_idr_lock protects the uverbs idrs themselves, so it 65 * needs to be held during all idr operations. When an object is 66 * looked up, a reference must be taken on the object's kref before 67 * dropping this lock. 68 * 69 * - Each object also has an rwsem. This rwsem must be held for 70 * reading while an operation that uses the object is performed. 71 * For example, while registering an MR, the associated PD's 72 * uobject.mutex must be held for reading. The rwsem must be held 73 * for writing while initializing or destroying an object. 74 * 75 * - In addition, each object has a "live" flag. If this flag is not 76 * set, then lookups of the object will fail even if it is found in 77 * the idr. This handles a reader that blocks and does not acquire 78 * the rwsem until after the object is destroyed. The destroy 79 * operation will set the live flag to 0 and then drop the rwsem; 80 * this will allow the reader to acquire the rwsem, see that the 81 * live flag is 0, and then drop the rwsem and its reference to 82 * object. The underlying storage will not be freed until the last 83 * reference to the object is dropped. 84 */ 85 86 static void init_uobj(struct ib_uobject *uobj, u64 user_handle, 87 struct ib_ucontext *context, struct uverbs_lock_class *c) 88 { 89 uobj->user_handle = user_handle; 90 uobj->context = context; 91 kref_init(&uobj->ref); 92 init_rwsem(&uobj->mutex); 93 lockdep_set_class_and_name(&uobj->mutex, &c->key, c->name); 94 uobj->live = 0; 95 } 96 97 static void release_uobj(struct kref *kref) 98 { 99 kfree(container_of(kref, struct ib_uobject, ref)); 100 } 101 102 static void put_uobj(struct ib_uobject *uobj) 103 { 104 kref_put(&uobj->ref, release_uobj); 105 } 106 107 static void put_uobj_read(struct ib_uobject *uobj) 108 { 109 up_read(&uobj->mutex); 110 put_uobj(uobj); 111 } 112 113 static void put_uobj_write(struct ib_uobject *uobj) 114 { 115 up_write(&uobj->mutex); 116 put_uobj(uobj); 117 } 118 119 static int idr_add_uobj(struct idr *idr, struct ib_uobject *uobj) 120 { 121 int ret; 122 123 idr_preload(GFP_KERNEL); 124 spin_lock(&ib_uverbs_idr_lock); 125 126 ret = idr_alloc(idr, uobj, 0, 0, GFP_NOWAIT); 127 if (ret >= 0) 128 uobj->id = ret; 129 130 spin_unlock(&ib_uverbs_idr_lock); 131 idr_preload_end(); 132 133 return ret < 0 ? ret : 0; 134 } 135 136 void idr_remove_uobj(struct idr *idr, struct ib_uobject *uobj) 137 { 138 spin_lock(&ib_uverbs_idr_lock); 139 idr_remove(idr, uobj->id); 140 spin_unlock(&ib_uverbs_idr_lock); 141 } 142 143 static struct ib_uobject *__idr_get_uobj(struct idr *idr, int id, 144 struct ib_ucontext *context) 145 { 146 struct ib_uobject *uobj; 147 148 spin_lock(&ib_uverbs_idr_lock); 149 uobj = idr_find(idr, id); 150 if (uobj) { 151 if (uobj->context == context) 152 kref_get(&uobj->ref); 153 else 154 uobj = NULL; 155 } 156 spin_unlock(&ib_uverbs_idr_lock); 157 158 return uobj; 159 } 160 161 static struct ib_uobject *idr_read_uobj(struct idr *idr, int id, 162 struct ib_ucontext *context, int nested) 163 { 164 struct ib_uobject *uobj; 165 166 uobj = __idr_get_uobj(idr, id, context); 167 if (!uobj) 168 return NULL; 169 170 if (nested) 171 down_read_nested(&uobj->mutex, SINGLE_DEPTH_NESTING); 172 else 173 down_read(&uobj->mutex); 174 if (!uobj->live) { 175 put_uobj_read(uobj); 176 return NULL; 177 } 178 179 return uobj; 180 } 181 182 static struct ib_uobject *idr_write_uobj(struct idr *idr, int id, 183 struct ib_ucontext *context) 184 { 185 struct ib_uobject *uobj; 186 187 uobj = __idr_get_uobj(idr, id, context); 188 if (!uobj) 189 return NULL; 190 191 down_write(&uobj->mutex); 192 if (!uobj->live) { 193 put_uobj_write(uobj); 194 return NULL; 195 } 196 197 return uobj; 198 } 199 200 static void *idr_read_obj(struct idr *idr, int id, struct ib_ucontext *context, 201 int nested) 202 { 203 struct ib_uobject *uobj; 204 205 uobj = idr_read_uobj(idr, id, context, nested); 206 return uobj ? uobj->object : NULL; 207 } 208 209 static struct ib_pd *idr_read_pd(int pd_handle, struct ib_ucontext *context) 210 { 211 return idr_read_obj(&ib_uverbs_pd_idr, pd_handle, context, 0); 212 } 213 214 static void put_pd_read(struct ib_pd *pd) 215 { 216 put_uobj_read(pd->uobject); 217 } 218 219 static struct ib_cq *idr_read_cq(int cq_handle, struct ib_ucontext *context, int nested) 220 { 221 return idr_read_obj(&ib_uverbs_cq_idr, cq_handle, context, nested); 222 } 223 224 static void put_cq_read(struct ib_cq *cq) 225 { 226 put_uobj_read(cq->uobject); 227 } 228 229 static struct ib_ah *idr_read_ah(int ah_handle, struct ib_ucontext *context) 230 { 231 return idr_read_obj(&ib_uverbs_ah_idr, ah_handle, context, 0); 232 } 233 234 static void put_ah_read(struct ib_ah *ah) 235 { 236 put_uobj_read(ah->uobject); 237 } 238 239 static struct ib_qp *idr_read_qp(int qp_handle, struct ib_ucontext *context) 240 { 241 return idr_read_obj(&ib_uverbs_qp_idr, qp_handle, context, 0); 242 } 243 244 static struct ib_qp *idr_write_qp(int qp_handle, struct ib_ucontext *context) 245 { 246 struct ib_uobject *uobj; 247 248 uobj = idr_write_uobj(&ib_uverbs_qp_idr, qp_handle, context); 249 return uobj ? uobj->object : NULL; 250 } 251 252 static void put_qp_read(struct ib_qp *qp) 253 { 254 put_uobj_read(qp->uobject); 255 } 256 257 static void put_qp_write(struct ib_qp *qp) 258 { 259 put_uobj_write(qp->uobject); 260 } 261 262 static struct ib_srq *idr_read_srq(int srq_handle, struct ib_ucontext *context) 263 { 264 return idr_read_obj(&ib_uverbs_srq_idr, srq_handle, context, 0); 265 } 266 267 static void put_srq_read(struct ib_srq *srq) 268 { 269 put_uobj_read(srq->uobject); 270 } 271 272 static struct ib_xrcd *idr_read_xrcd(int xrcd_handle, struct ib_ucontext *context, 273 struct ib_uobject **uobj) 274 { 275 *uobj = idr_read_uobj(&ib_uverbs_xrcd_idr, xrcd_handle, context, 0); 276 return *uobj ? (*uobj)->object : NULL; 277 } 278 279 static void put_xrcd_read(struct ib_uobject *uobj) 280 { 281 put_uobj_read(uobj); 282 } 283 284 ssize_t ib_uverbs_get_context(struct ib_uverbs_file *file, 285 const char __user *buf, 286 int in_len, int out_len) 287 { 288 struct ib_uverbs_get_context cmd; 289 struct ib_uverbs_get_context_resp resp; 290 struct ib_udata udata; 291 struct ib_device *ibdev = file->device->ib_dev; 292 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING 293 struct ib_device_attr dev_attr; 294 #endif 295 struct ib_ucontext *ucontext; 296 struct file *filp; 297 int ret; 298 299 if (out_len < sizeof resp) 300 return -ENOSPC; 301 302 if (copy_from_user(&cmd, buf, sizeof cmd)) 303 return -EFAULT; 304 305 mutex_lock(&file->mutex); 306 307 if (file->ucontext) { 308 ret = -EINVAL; 309 goto err; 310 } 311 312 INIT_UDATA(&udata, buf + sizeof cmd, 313 (unsigned long) cmd.response + sizeof resp, 314 in_len - sizeof cmd, out_len - sizeof resp); 315 316 ucontext = ibdev->alloc_ucontext(ibdev, &udata); 317 if (IS_ERR(ucontext)) { 318 ret = PTR_ERR(ucontext); 319 goto err; 320 } 321 322 ucontext->device = ibdev; 323 INIT_LIST_HEAD(&ucontext->pd_list); 324 INIT_LIST_HEAD(&ucontext->mr_list); 325 INIT_LIST_HEAD(&ucontext->mw_list); 326 INIT_LIST_HEAD(&ucontext->cq_list); 327 INIT_LIST_HEAD(&ucontext->qp_list); 328 INIT_LIST_HEAD(&ucontext->srq_list); 329 INIT_LIST_HEAD(&ucontext->ah_list); 330 INIT_LIST_HEAD(&ucontext->xrcd_list); 331 INIT_LIST_HEAD(&ucontext->rule_list); 332 rcu_read_lock(); 333 ucontext->tgid = get_task_pid(current->group_leader, PIDTYPE_PID); 334 rcu_read_unlock(); 335 ucontext->closing = 0; 336 337 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING 338 ucontext->umem_tree = RB_ROOT; 339 init_rwsem(&ucontext->umem_rwsem); 340 ucontext->odp_mrs_count = 0; 341 INIT_LIST_HEAD(&ucontext->no_private_counters); 342 343 ret = ib_query_device(ibdev, &dev_attr); 344 if (ret) 345 goto err_free; 346 if (!(dev_attr.device_cap_flags & IB_DEVICE_ON_DEMAND_PAGING)) 347 ucontext->invalidate_range = NULL; 348 349 #endif 350 351 resp.num_comp_vectors = file->device->num_comp_vectors; 352 353 ret = get_unused_fd_flags(O_CLOEXEC); 354 if (ret < 0) 355 goto err_free; 356 resp.async_fd = ret; 357 358 filp = ib_uverbs_alloc_event_file(file, 1); 359 if (IS_ERR(filp)) { 360 ret = PTR_ERR(filp); 361 goto err_fd; 362 } 363 364 if (copy_to_user((void __user *) (unsigned long) cmd.response, 365 &resp, sizeof resp)) { 366 ret = -EFAULT; 367 goto err_file; 368 } 369 370 file->async_file = filp->private_data; 371 372 INIT_IB_EVENT_HANDLER(&file->event_handler, file->device->ib_dev, 373 ib_uverbs_event_handler); 374 ret = ib_register_event_handler(&file->event_handler); 375 if (ret) 376 goto err_file; 377 378 kref_get(&file->async_file->ref); 379 kref_get(&file->ref); 380 file->ucontext = ucontext; 381 382 fd_install(resp.async_fd, filp); 383 384 mutex_unlock(&file->mutex); 385 386 return in_len; 387 388 err_file: 389 fput(filp); 390 391 err_fd: 392 put_unused_fd(resp.async_fd); 393 394 err_free: 395 put_pid(ucontext->tgid); 396 ibdev->dealloc_ucontext(ucontext); 397 398 err: 399 mutex_unlock(&file->mutex); 400 return ret; 401 } 402 403 static void copy_query_dev_fields(struct ib_uverbs_file *file, 404 struct ib_uverbs_query_device_resp *resp, 405 struct ib_device_attr *attr) 406 { 407 resp->fw_ver = attr->fw_ver; 408 resp->node_guid = file->device->ib_dev->node_guid; 409 resp->sys_image_guid = attr->sys_image_guid; 410 resp->max_mr_size = attr->max_mr_size; 411 resp->page_size_cap = attr->page_size_cap; 412 resp->vendor_id = attr->vendor_id; 413 resp->vendor_part_id = attr->vendor_part_id; 414 resp->hw_ver = attr->hw_ver; 415 resp->max_qp = attr->max_qp; 416 resp->max_qp_wr = attr->max_qp_wr; 417 resp->device_cap_flags = attr->device_cap_flags; 418 resp->max_sge = attr->max_sge; 419 resp->max_sge_rd = attr->max_sge_rd; 420 resp->max_cq = attr->max_cq; 421 resp->max_cqe = attr->max_cqe; 422 resp->max_mr = attr->max_mr; 423 resp->max_pd = attr->max_pd; 424 resp->max_qp_rd_atom = attr->max_qp_rd_atom; 425 resp->max_ee_rd_atom = attr->max_ee_rd_atom; 426 resp->max_res_rd_atom = attr->max_res_rd_atom; 427 resp->max_qp_init_rd_atom = attr->max_qp_init_rd_atom; 428 resp->max_ee_init_rd_atom = attr->max_ee_init_rd_atom; 429 resp->atomic_cap = attr->atomic_cap; 430 resp->max_ee = attr->max_ee; 431 resp->max_rdd = attr->max_rdd; 432 resp->max_mw = attr->max_mw; 433 resp->max_raw_ipv6_qp = attr->max_raw_ipv6_qp; 434 resp->max_raw_ethy_qp = attr->max_raw_ethy_qp; 435 resp->max_mcast_grp = attr->max_mcast_grp; 436 resp->max_mcast_qp_attach = attr->max_mcast_qp_attach; 437 resp->max_total_mcast_qp_attach = attr->max_total_mcast_qp_attach; 438 resp->max_ah = attr->max_ah; 439 resp->max_fmr = attr->max_fmr; 440 resp->max_map_per_fmr = attr->max_map_per_fmr; 441 resp->max_srq = attr->max_srq; 442 resp->max_srq_wr = attr->max_srq_wr; 443 resp->max_srq_sge = attr->max_srq_sge; 444 resp->max_pkeys = attr->max_pkeys; 445 resp->local_ca_ack_delay = attr->local_ca_ack_delay; 446 resp->phys_port_cnt = file->device->ib_dev->phys_port_cnt; 447 } 448 449 ssize_t ib_uverbs_query_device(struct ib_uverbs_file *file, 450 const char __user *buf, 451 int in_len, int out_len) 452 { 453 struct ib_uverbs_query_device cmd; 454 struct ib_uverbs_query_device_resp resp; 455 struct ib_device_attr attr; 456 int ret; 457 458 if (out_len < sizeof resp) 459 return -ENOSPC; 460 461 if (copy_from_user(&cmd, buf, sizeof cmd)) 462 return -EFAULT; 463 464 ret = ib_query_device(file->device->ib_dev, &attr); 465 if (ret) 466 return ret; 467 468 memset(&resp, 0, sizeof resp); 469 copy_query_dev_fields(file, &resp, &attr); 470 471 if (copy_to_user((void __user *) (unsigned long) cmd.response, 472 &resp, sizeof resp)) 473 return -EFAULT; 474 475 return in_len; 476 } 477 478 ssize_t ib_uverbs_query_port(struct ib_uverbs_file *file, 479 const char __user *buf, 480 int in_len, int out_len) 481 { 482 struct ib_uverbs_query_port cmd; 483 struct ib_uverbs_query_port_resp resp; 484 struct ib_port_attr attr; 485 int ret; 486 487 if (out_len < sizeof resp) 488 return -ENOSPC; 489 490 if (copy_from_user(&cmd, buf, sizeof cmd)) 491 return -EFAULT; 492 493 ret = ib_query_port(file->device->ib_dev, cmd.port_num, &attr); 494 if (ret) 495 return ret; 496 497 memset(&resp, 0, sizeof resp); 498 499 resp.state = attr.state; 500 resp.max_mtu = attr.max_mtu; 501 resp.active_mtu = attr.active_mtu; 502 resp.gid_tbl_len = attr.gid_tbl_len; 503 resp.port_cap_flags = attr.port_cap_flags; 504 resp.max_msg_sz = attr.max_msg_sz; 505 resp.bad_pkey_cntr = attr.bad_pkey_cntr; 506 resp.qkey_viol_cntr = attr.qkey_viol_cntr; 507 resp.pkey_tbl_len = attr.pkey_tbl_len; 508 resp.lid = attr.lid; 509 resp.sm_lid = attr.sm_lid; 510 resp.lmc = attr.lmc; 511 resp.max_vl_num = attr.max_vl_num; 512 resp.sm_sl = attr.sm_sl; 513 resp.subnet_timeout = attr.subnet_timeout; 514 resp.init_type_reply = attr.init_type_reply; 515 resp.active_width = attr.active_width; 516 resp.active_speed = attr.active_speed; 517 resp.phys_state = attr.phys_state; 518 resp.link_layer = rdma_port_get_link_layer(file->device->ib_dev, 519 cmd.port_num); 520 521 if (copy_to_user((void __user *) (unsigned long) cmd.response, 522 &resp, sizeof resp)) 523 return -EFAULT; 524 525 return in_len; 526 } 527 528 ssize_t ib_uverbs_alloc_pd(struct ib_uverbs_file *file, 529 const char __user *buf, 530 int in_len, int out_len) 531 { 532 struct ib_uverbs_alloc_pd cmd; 533 struct ib_uverbs_alloc_pd_resp resp; 534 struct ib_udata udata; 535 struct ib_uobject *uobj; 536 struct ib_pd *pd; 537 int ret; 538 539 if (out_len < sizeof resp) 540 return -ENOSPC; 541 542 if (copy_from_user(&cmd, buf, sizeof cmd)) 543 return -EFAULT; 544 545 INIT_UDATA(&udata, buf + sizeof cmd, 546 (unsigned long) cmd.response + sizeof resp, 547 in_len - sizeof cmd, out_len - sizeof resp); 548 549 uobj = kmalloc(sizeof *uobj, GFP_KERNEL); 550 if (!uobj) 551 return -ENOMEM; 552 553 init_uobj(uobj, 0, file->ucontext, &pd_lock_class); 554 down_write(&uobj->mutex); 555 556 pd = file->device->ib_dev->alloc_pd(file->device->ib_dev, 557 file->ucontext, &udata); 558 if (IS_ERR(pd)) { 559 ret = PTR_ERR(pd); 560 goto err; 561 } 562 563 pd->device = file->device->ib_dev; 564 pd->uobject = uobj; 565 atomic_set(&pd->usecnt, 0); 566 567 uobj->object = pd; 568 ret = idr_add_uobj(&ib_uverbs_pd_idr, uobj); 569 if (ret) 570 goto err_idr; 571 572 memset(&resp, 0, sizeof resp); 573 resp.pd_handle = uobj->id; 574 575 if (copy_to_user((void __user *) (unsigned long) cmd.response, 576 &resp, sizeof resp)) { 577 ret = -EFAULT; 578 goto err_copy; 579 } 580 581 mutex_lock(&file->mutex); 582 list_add_tail(&uobj->list, &file->ucontext->pd_list); 583 mutex_unlock(&file->mutex); 584 585 uobj->live = 1; 586 587 up_write(&uobj->mutex); 588 589 return in_len; 590 591 err_copy: 592 idr_remove_uobj(&ib_uverbs_pd_idr, uobj); 593 594 err_idr: 595 ib_dealloc_pd(pd); 596 597 err: 598 put_uobj_write(uobj); 599 return ret; 600 } 601 602 ssize_t ib_uverbs_dealloc_pd(struct ib_uverbs_file *file, 603 const char __user *buf, 604 int in_len, int out_len) 605 { 606 struct ib_uverbs_dealloc_pd cmd; 607 struct ib_uobject *uobj; 608 int ret; 609 610 if (copy_from_user(&cmd, buf, sizeof cmd)) 611 return -EFAULT; 612 613 uobj = idr_write_uobj(&ib_uverbs_pd_idr, cmd.pd_handle, file->ucontext); 614 if (!uobj) 615 return -EINVAL; 616 617 ret = ib_dealloc_pd(uobj->object); 618 if (!ret) 619 uobj->live = 0; 620 621 put_uobj_write(uobj); 622 623 if (ret) 624 return ret; 625 626 idr_remove_uobj(&ib_uverbs_pd_idr, uobj); 627 628 mutex_lock(&file->mutex); 629 list_del(&uobj->list); 630 mutex_unlock(&file->mutex); 631 632 put_uobj(uobj); 633 634 return in_len; 635 } 636 637 struct xrcd_table_entry { 638 struct rb_node node; 639 struct ib_xrcd *xrcd; 640 struct inode *inode; 641 }; 642 643 static int xrcd_table_insert(struct ib_uverbs_device *dev, 644 struct inode *inode, 645 struct ib_xrcd *xrcd) 646 { 647 struct xrcd_table_entry *entry, *scan; 648 struct rb_node **p = &dev->xrcd_tree.rb_node; 649 struct rb_node *parent = NULL; 650 651 entry = kmalloc(sizeof *entry, GFP_KERNEL); 652 if (!entry) 653 return -ENOMEM; 654 655 entry->xrcd = xrcd; 656 entry->inode = inode; 657 658 while (*p) { 659 parent = *p; 660 scan = rb_entry(parent, struct xrcd_table_entry, node); 661 662 if (inode < scan->inode) { 663 p = &(*p)->rb_left; 664 } else if (inode > scan->inode) { 665 p = &(*p)->rb_right; 666 } else { 667 kfree(entry); 668 return -EEXIST; 669 } 670 } 671 672 rb_link_node(&entry->node, parent, p); 673 rb_insert_color(&entry->node, &dev->xrcd_tree); 674 igrab(inode); 675 return 0; 676 } 677 678 static struct xrcd_table_entry *xrcd_table_search(struct ib_uverbs_device *dev, 679 struct inode *inode) 680 { 681 struct xrcd_table_entry *entry; 682 struct rb_node *p = dev->xrcd_tree.rb_node; 683 684 while (p) { 685 entry = rb_entry(p, struct xrcd_table_entry, node); 686 687 if (inode < entry->inode) 688 p = p->rb_left; 689 else if (inode > entry->inode) 690 p = p->rb_right; 691 else 692 return entry; 693 } 694 695 return NULL; 696 } 697 698 static struct ib_xrcd *find_xrcd(struct ib_uverbs_device *dev, struct inode *inode) 699 { 700 struct xrcd_table_entry *entry; 701 702 entry = xrcd_table_search(dev, inode); 703 if (!entry) 704 return NULL; 705 706 return entry->xrcd; 707 } 708 709 static void xrcd_table_delete(struct ib_uverbs_device *dev, 710 struct inode *inode) 711 { 712 struct xrcd_table_entry *entry; 713 714 entry = xrcd_table_search(dev, inode); 715 if (entry) { 716 iput(inode); 717 rb_erase(&entry->node, &dev->xrcd_tree); 718 kfree(entry); 719 } 720 } 721 722 ssize_t ib_uverbs_open_xrcd(struct ib_uverbs_file *file, 723 const char __user *buf, int in_len, 724 int out_len) 725 { 726 struct ib_uverbs_open_xrcd cmd; 727 struct ib_uverbs_open_xrcd_resp resp; 728 struct ib_udata udata; 729 struct ib_uxrcd_object *obj; 730 struct ib_xrcd *xrcd = NULL; 731 struct fd f = {NULL, 0}; 732 struct inode *inode = NULL; 733 int ret = 0; 734 int new_xrcd = 0; 735 736 if (out_len < sizeof resp) 737 return -ENOSPC; 738 739 if (copy_from_user(&cmd, buf, sizeof cmd)) 740 return -EFAULT; 741 742 INIT_UDATA(&udata, buf + sizeof cmd, 743 (unsigned long) cmd.response + sizeof resp, 744 in_len - sizeof cmd, out_len - sizeof resp); 745 746 mutex_lock(&file->device->xrcd_tree_mutex); 747 748 if (cmd.fd != -1) { 749 /* search for file descriptor */ 750 f = fdget(cmd.fd); 751 if (!f.file) { 752 ret = -EBADF; 753 goto err_tree_mutex_unlock; 754 } 755 756 inode = file_inode(f.file); 757 xrcd = find_xrcd(file->device, inode); 758 if (!xrcd && !(cmd.oflags & O_CREAT)) { 759 /* no file descriptor. Need CREATE flag */ 760 ret = -EAGAIN; 761 goto err_tree_mutex_unlock; 762 } 763 764 if (xrcd && cmd.oflags & O_EXCL) { 765 ret = -EINVAL; 766 goto err_tree_mutex_unlock; 767 } 768 } 769 770 obj = kmalloc(sizeof *obj, GFP_KERNEL); 771 if (!obj) { 772 ret = -ENOMEM; 773 goto err_tree_mutex_unlock; 774 } 775 776 init_uobj(&obj->uobject, 0, file->ucontext, &xrcd_lock_class); 777 778 down_write(&obj->uobject.mutex); 779 780 if (!xrcd) { 781 xrcd = file->device->ib_dev->alloc_xrcd(file->device->ib_dev, 782 file->ucontext, &udata); 783 if (IS_ERR(xrcd)) { 784 ret = PTR_ERR(xrcd); 785 goto err; 786 } 787 788 xrcd->inode = inode; 789 xrcd->device = file->device->ib_dev; 790 atomic_set(&xrcd->usecnt, 0); 791 mutex_init(&xrcd->tgt_qp_mutex); 792 INIT_LIST_HEAD(&xrcd->tgt_qp_list); 793 new_xrcd = 1; 794 } 795 796 atomic_set(&obj->refcnt, 0); 797 obj->uobject.object = xrcd; 798 ret = idr_add_uobj(&ib_uverbs_xrcd_idr, &obj->uobject); 799 if (ret) 800 goto err_idr; 801 802 memset(&resp, 0, sizeof resp); 803 resp.xrcd_handle = obj->uobject.id; 804 805 if (inode) { 806 if (new_xrcd) { 807 /* create new inode/xrcd table entry */ 808 ret = xrcd_table_insert(file->device, inode, xrcd); 809 if (ret) 810 goto err_insert_xrcd; 811 } 812 atomic_inc(&xrcd->usecnt); 813 } 814 815 if (copy_to_user((void __user *) (unsigned long) cmd.response, 816 &resp, sizeof resp)) { 817 ret = -EFAULT; 818 goto err_copy; 819 } 820 821 if (f.file) 822 fdput(f); 823 824 mutex_lock(&file->mutex); 825 list_add_tail(&obj->uobject.list, &file->ucontext->xrcd_list); 826 mutex_unlock(&file->mutex); 827 828 obj->uobject.live = 1; 829 up_write(&obj->uobject.mutex); 830 831 mutex_unlock(&file->device->xrcd_tree_mutex); 832 return in_len; 833 834 err_copy: 835 if (inode) { 836 if (new_xrcd) 837 xrcd_table_delete(file->device, inode); 838 atomic_dec(&xrcd->usecnt); 839 } 840 841 err_insert_xrcd: 842 idr_remove_uobj(&ib_uverbs_xrcd_idr, &obj->uobject); 843 844 err_idr: 845 ib_dealloc_xrcd(xrcd); 846 847 err: 848 put_uobj_write(&obj->uobject); 849 850 err_tree_mutex_unlock: 851 if (f.file) 852 fdput(f); 853 854 mutex_unlock(&file->device->xrcd_tree_mutex); 855 856 return ret; 857 } 858 859 ssize_t ib_uverbs_close_xrcd(struct ib_uverbs_file *file, 860 const char __user *buf, int in_len, 861 int out_len) 862 { 863 struct ib_uverbs_close_xrcd cmd; 864 struct ib_uobject *uobj; 865 struct ib_xrcd *xrcd = NULL; 866 struct inode *inode = NULL; 867 struct ib_uxrcd_object *obj; 868 int live; 869 int ret = 0; 870 871 if (copy_from_user(&cmd, buf, sizeof cmd)) 872 return -EFAULT; 873 874 mutex_lock(&file->device->xrcd_tree_mutex); 875 uobj = idr_write_uobj(&ib_uverbs_xrcd_idr, cmd.xrcd_handle, file->ucontext); 876 if (!uobj) { 877 ret = -EINVAL; 878 goto out; 879 } 880 881 xrcd = uobj->object; 882 inode = xrcd->inode; 883 obj = container_of(uobj, struct ib_uxrcd_object, uobject); 884 if (atomic_read(&obj->refcnt)) { 885 put_uobj_write(uobj); 886 ret = -EBUSY; 887 goto out; 888 } 889 890 if (!inode || atomic_dec_and_test(&xrcd->usecnt)) { 891 ret = ib_dealloc_xrcd(uobj->object); 892 if (!ret) 893 uobj->live = 0; 894 } 895 896 live = uobj->live; 897 if (inode && ret) 898 atomic_inc(&xrcd->usecnt); 899 900 put_uobj_write(uobj); 901 902 if (ret) 903 goto out; 904 905 if (inode && !live) 906 xrcd_table_delete(file->device, inode); 907 908 idr_remove_uobj(&ib_uverbs_xrcd_idr, uobj); 909 mutex_lock(&file->mutex); 910 list_del(&uobj->list); 911 mutex_unlock(&file->mutex); 912 913 put_uobj(uobj); 914 ret = in_len; 915 916 out: 917 mutex_unlock(&file->device->xrcd_tree_mutex); 918 return ret; 919 } 920 921 void ib_uverbs_dealloc_xrcd(struct ib_uverbs_device *dev, 922 struct ib_xrcd *xrcd) 923 { 924 struct inode *inode; 925 926 inode = xrcd->inode; 927 if (inode && !atomic_dec_and_test(&xrcd->usecnt)) 928 return; 929 930 ib_dealloc_xrcd(xrcd); 931 932 if (inode) 933 xrcd_table_delete(dev, inode); 934 } 935 936 ssize_t ib_uverbs_reg_mr(struct ib_uverbs_file *file, 937 const char __user *buf, int in_len, 938 int out_len) 939 { 940 struct ib_uverbs_reg_mr cmd; 941 struct ib_uverbs_reg_mr_resp resp; 942 struct ib_udata udata; 943 struct ib_uobject *uobj; 944 struct ib_pd *pd; 945 struct ib_mr *mr; 946 int ret; 947 948 if (out_len < sizeof resp) 949 return -ENOSPC; 950 951 if (copy_from_user(&cmd, buf, sizeof cmd)) 952 return -EFAULT; 953 954 INIT_UDATA(&udata, buf + sizeof cmd, 955 (unsigned long) cmd.response + sizeof resp, 956 in_len - sizeof cmd, out_len - sizeof resp); 957 958 if ((cmd.start & ~PAGE_MASK) != (cmd.hca_va & ~PAGE_MASK)) 959 return -EINVAL; 960 961 ret = ib_check_mr_access(cmd.access_flags); 962 if (ret) 963 return ret; 964 965 uobj = kmalloc(sizeof *uobj, GFP_KERNEL); 966 if (!uobj) 967 return -ENOMEM; 968 969 init_uobj(uobj, 0, file->ucontext, &mr_lock_class); 970 down_write(&uobj->mutex); 971 972 pd = idr_read_pd(cmd.pd_handle, file->ucontext); 973 if (!pd) { 974 ret = -EINVAL; 975 goto err_free; 976 } 977 978 if (cmd.access_flags & IB_ACCESS_ON_DEMAND) { 979 struct ib_device_attr attr; 980 981 ret = ib_query_device(pd->device, &attr); 982 if (ret || !(attr.device_cap_flags & 983 IB_DEVICE_ON_DEMAND_PAGING)) { 984 pr_debug("ODP support not available\n"); 985 ret = -EINVAL; 986 goto err_put; 987 } 988 } 989 990 mr = pd->device->reg_user_mr(pd, cmd.start, cmd.length, cmd.hca_va, 991 cmd.access_flags, &udata); 992 if (IS_ERR(mr)) { 993 ret = PTR_ERR(mr); 994 goto err_put; 995 } 996 997 mr->device = pd->device; 998 mr->pd = pd; 999 mr->uobject = uobj; 1000 atomic_inc(&pd->usecnt); 1001 atomic_set(&mr->usecnt, 0); 1002 1003 uobj->object = mr; 1004 ret = idr_add_uobj(&ib_uverbs_mr_idr, uobj); 1005 if (ret) 1006 goto err_unreg; 1007 1008 memset(&resp, 0, sizeof resp); 1009 resp.lkey = mr->lkey; 1010 resp.rkey = mr->rkey; 1011 resp.mr_handle = uobj->id; 1012 1013 if (copy_to_user((void __user *) (unsigned long) cmd.response, 1014 &resp, sizeof resp)) { 1015 ret = -EFAULT; 1016 goto err_copy; 1017 } 1018 1019 put_pd_read(pd); 1020 1021 mutex_lock(&file->mutex); 1022 list_add_tail(&uobj->list, &file->ucontext->mr_list); 1023 mutex_unlock(&file->mutex); 1024 1025 uobj->live = 1; 1026 1027 up_write(&uobj->mutex); 1028 1029 return in_len; 1030 1031 err_copy: 1032 idr_remove_uobj(&ib_uverbs_mr_idr, uobj); 1033 1034 err_unreg: 1035 ib_dereg_mr(mr); 1036 1037 err_put: 1038 put_pd_read(pd); 1039 1040 err_free: 1041 put_uobj_write(uobj); 1042 return ret; 1043 } 1044 1045 ssize_t ib_uverbs_rereg_mr(struct ib_uverbs_file *file, 1046 const char __user *buf, int in_len, 1047 int out_len) 1048 { 1049 struct ib_uverbs_rereg_mr cmd; 1050 struct ib_uverbs_rereg_mr_resp resp; 1051 struct ib_udata udata; 1052 struct ib_pd *pd = NULL; 1053 struct ib_mr *mr; 1054 struct ib_pd *old_pd; 1055 int ret; 1056 struct ib_uobject *uobj; 1057 1058 if (out_len < sizeof(resp)) 1059 return -ENOSPC; 1060 1061 if (copy_from_user(&cmd, buf, sizeof(cmd))) 1062 return -EFAULT; 1063 1064 INIT_UDATA(&udata, buf + sizeof(cmd), 1065 (unsigned long) cmd.response + sizeof(resp), 1066 in_len - sizeof(cmd), out_len - sizeof(resp)); 1067 1068 if (cmd.flags & ~IB_MR_REREG_SUPPORTED || !cmd.flags) 1069 return -EINVAL; 1070 1071 if ((cmd.flags & IB_MR_REREG_TRANS) && 1072 (!cmd.start || !cmd.hca_va || 0 >= cmd.length || 1073 (cmd.start & ~PAGE_MASK) != (cmd.hca_va & ~PAGE_MASK))) 1074 return -EINVAL; 1075 1076 uobj = idr_write_uobj(&ib_uverbs_mr_idr, cmd.mr_handle, 1077 file->ucontext); 1078 1079 if (!uobj) 1080 return -EINVAL; 1081 1082 mr = uobj->object; 1083 1084 if (cmd.flags & IB_MR_REREG_ACCESS) { 1085 ret = ib_check_mr_access(cmd.access_flags); 1086 if (ret) 1087 goto put_uobjs; 1088 } 1089 1090 if (cmd.flags & IB_MR_REREG_PD) { 1091 pd = idr_read_pd(cmd.pd_handle, file->ucontext); 1092 if (!pd) { 1093 ret = -EINVAL; 1094 goto put_uobjs; 1095 } 1096 } 1097 1098 if (atomic_read(&mr->usecnt)) { 1099 ret = -EBUSY; 1100 goto put_uobj_pd; 1101 } 1102 1103 old_pd = mr->pd; 1104 ret = mr->device->rereg_user_mr(mr, cmd.flags, cmd.start, 1105 cmd.length, cmd.hca_va, 1106 cmd.access_flags, pd, &udata); 1107 if (!ret) { 1108 if (cmd.flags & IB_MR_REREG_PD) { 1109 atomic_inc(&pd->usecnt); 1110 mr->pd = pd; 1111 atomic_dec(&old_pd->usecnt); 1112 } 1113 } else { 1114 goto put_uobj_pd; 1115 } 1116 1117 memset(&resp, 0, sizeof(resp)); 1118 resp.lkey = mr->lkey; 1119 resp.rkey = mr->rkey; 1120 1121 if (copy_to_user((void __user *)(unsigned long)cmd.response, 1122 &resp, sizeof(resp))) 1123 ret = -EFAULT; 1124 else 1125 ret = in_len; 1126 1127 put_uobj_pd: 1128 if (cmd.flags & IB_MR_REREG_PD) 1129 put_pd_read(pd); 1130 1131 put_uobjs: 1132 1133 put_uobj_write(mr->uobject); 1134 1135 return ret; 1136 } 1137 1138 ssize_t ib_uverbs_dereg_mr(struct ib_uverbs_file *file, 1139 const char __user *buf, int in_len, 1140 int out_len) 1141 { 1142 struct ib_uverbs_dereg_mr cmd; 1143 struct ib_mr *mr; 1144 struct ib_uobject *uobj; 1145 int ret = -EINVAL; 1146 1147 if (copy_from_user(&cmd, buf, sizeof cmd)) 1148 return -EFAULT; 1149 1150 uobj = idr_write_uobj(&ib_uverbs_mr_idr, cmd.mr_handle, file->ucontext); 1151 if (!uobj) 1152 return -EINVAL; 1153 1154 mr = uobj->object; 1155 1156 ret = ib_dereg_mr(mr); 1157 if (!ret) 1158 uobj->live = 0; 1159 1160 put_uobj_write(uobj); 1161 1162 if (ret) 1163 return ret; 1164 1165 idr_remove_uobj(&ib_uverbs_mr_idr, uobj); 1166 1167 mutex_lock(&file->mutex); 1168 list_del(&uobj->list); 1169 mutex_unlock(&file->mutex); 1170 1171 put_uobj(uobj); 1172 1173 return in_len; 1174 } 1175 1176 ssize_t ib_uverbs_alloc_mw(struct ib_uverbs_file *file, 1177 const char __user *buf, int in_len, 1178 int out_len) 1179 { 1180 struct ib_uverbs_alloc_mw cmd; 1181 struct ib_uverbs_alloc_mw_resp resp; 1182 struct ib_uobject *uobj; 1183 struct ib_pd *pd; 1184 struct ib_mw *mw; 1185 int ret; 1186 1187 if (out_len < sizeof(resp)) 1188 return -ENOSPC; 1189 1190 if (copy_from_user(&cmd, buf, sizeof(cmd))) 1191 return -EFAULT; 1192 1193 uobj = kmalloc(sizeof(*uobj), GFP_KERNEL); 1194 if (!uobj) 1195 return -ENOMEM; 1196 1197 init_uobj(uobj, 0, file->ucontext, &mw_lock_class); 1198 down_write(&uobj->mutex); 1199 1200 pd = idr_read_pd(cmd.pd_handle, file->ucontext); 1201 if (!pd) { 1202 ret = -EINVAL; 1203 goto err_free; 1204 } 1205 1206 mw = pd->device->alloc_mw(pd, cmd.mw_type); 1207 if (IS_ERR(mw)) { 1208 ret = PTR_ERR(mw); 1209 goto err_put; 1210 } 1211 1212 mw->device = pd->device; 1213 mw->pd = pd; 1214 mw->uobject = uobj; 1215 atomic_inc(&pd->usecnt); 1216 1217 uobj->object = mw; 1218 ret = idr_add_uobj(&ib_uverbs_mw_idr, uobj); 1219 if (ret) 1220 goto err_unalloc; 1221 1222 memset(&resp, 0, sizeof(resp)); 1223 resp.rkey = mw->rkey; 1224 resp.mw_handle = uobj->id; 1225 1226 if (copy_to_user((void __user *)(unsigned long)cmd.response, 1227 &resp, sizeof(resp))) { 1228 ret = -EFAULT; 1229 goto err_copy; 1230 } 1231 1232 put_pd_read(pd); 1233 1234 mutex_lock(&file->mutex); 1235 list_add_tail(&uobj->list, &file->ucontext->mw_list); 1236 mutex_unlock(&file->mutex); 1237 1238 uobj->live = 1; 1239 1240 up_write(&uobj->mutex); 1241 1242 return in_len; 1243 1244 err_copy: 1245 idr_remove_uobj(&ib_uverbs_mw_idr, uobj); 1246 1247 err_unalloc: 1248 ib_dealloc_mw(mw); 1249 1250 err_put: 1251 put_pd_read(pd); 1252 1253 err_free: 1254 put_uobj_write(uobj); 1255 return ret; 1256 } 1257 1258 ssize_t ib_uverbs_dealloc_mw(struct ib_uverbs_file *file, 1259 const char __user *buf, int in_len, 1260 int out_len) 1261 { 1262 struct ib_uverbs_dealloc_mw cmd; 1263 struct ib_mw *mw; 1264 struct ib_uobject *uobj; 1265 int ret = -EINVAL; 1266 1267 if (copy_from_user(&cmd, buf, sizeof(cmd))) 1268 return -EFAULT; 1269 1270 uobj = idr_write_uobj(&ib_uverbs_mw_idr, cmd.mw_handle, file->ucontext); 1271 if (!uobj) 1272 return -EINVAL; 1273 1274 mw = uobj->object; 1275 1276 ret = ib_dealloc_mw(mw); 1277 if (!ret) 1278 uobj->live = 0; 1279 1280 put_uobj_write(uobj); 1281 1282 if (ret) 1283 return ret; 1284 1285 idr_remove_uobj(&ib_uverbs_mw_idr, uobj); 1286 1287 mutex_lock(&file->mutex); 1288 list_del(&uobj->list); 1289 mutex_unlock(&file->mutex); 1290 1291 put_uobj(uobj); 1292 1293 return in_len; 1294 } 1295 1296 ssize_t ib_uverbs_create_comp_channel(struct ib_uverbs_file *file, 1297 const char __user *buf, int in_len, 1298 int out_len) 1299 { 1300 struct ib_uverbs_create_comp_channel cmd; 1301 struct ib_uverbs_create_comp_channel_resp resp; 1302 struct file *filp; 1303 int ret; 1304 1305 if (out_len < sizeof resp) 1306 return -ENOSPC; 1307 1308 if (copy_from_user(&cmd, buf, sizeof cmd)) 1309 return -EFAULT; 1310 1311 ret = get_unused_fd_flags(O_CLOEXEC); 1312 if (ret < 0) 1313 return ret; 1314 resp.fd = ret; 1315 1316 filp = ib_uverbs_alloc_event_file(file, 0); 1317 if (IS_ERR(filp)) { 1318 put_unused_fd(resp.fd); 1319 return PTR_ERR(filp); 1320 } 1321 1322 if (copy_to_user((void __user *) (unsigned long) cmd.response, 1323 &resp, sizeof resp)) { 1324 put_unused_fd(resp.fd); 1325 fput(filp); 1326 return -EFAULT; 1327 } 1328 1329 fd_install(resp.fd, filp); 1330 return in_len; 1331 } 1332 1333 ssize_t ib_uverbs_create_cq(struct ib_uverbs_file *file, 1334 const char __user *buf, int in_len, 1335 int out_len) 1336 { 1337 struct ib_uverbs_create_cq cmd; 1338 struct ib_uverbs_create_cq_resp resp; 1339 struct ib_udata udata; 1340 struct ib_ucq_object *obj; 1341 struct ib_uverbs_event_file *ev_file = NULL; 1342 struct ib_cq *cq; 1343 int ret; 1344 1345 if (out_len < sizeof resp) 1346 return -ENOSPC; 1347 1348 if (copy_from_user(&cmd, buf, sizeof cmd)) 1349 return -EFAULT; 1350 1351 INIT_UDATA(&udata, buf + sizeof cmd, 1352 (unsigned long) cmd.response + sizeof resp, 1353 in_len - sizeof cmd, out_len - sizeof resp); 1354 1355 if (cmd.comp_vector >= file->device->num_comp_vectors) 1356 return -EINVAL; 1357 1358 obj = kmalloc(sizeof *obj, GFP_KERNEL); 1359 if (!obj) 1360 return -ENOMEM; 1361 1362 init_uobj(&obj->uobject, cmd.user_handle, file->ucontext, &cq_lock_class); 1363 down_write(&obj->uobject.mutex); 1364 1365 if (cmd.comp_channel >= 0) { 1366 ev_file = ib_uverbs_lookup_comp_file(cmd.comp_channel); 1367 if (!ev_file) { 1368 ret = -EINVAL; 1369 goto err; 1370 } 1371 } 1372 1373 obj->uverbs_file = file; 1374 obj->comp_events_reported = 0; 1375 obj->async_events_reported = 0; 1376 INIT_LIST_HEAD(&obj->comp_list); 1377 INIT_LIST_HEAD(&obj->async_list); 1378 1379 cq = file->device->ib_dev->create_cq(file->device->ib_dev, cmd.cqe, 1380 cmd.comp_vector, 1381 file->ucontext, &udata); 1382 if (IS_ERR(cq)) { 1383 ret = PTR_ERR(cq); 1384 goto err_file; 1385 } 1386 1387 cq->device = file->device->ib_dev; 1388 cq->uobject = &obj->uobject; 1389 cq->comp_handler = ib_uverbs_comp_handler; 1390 cq->event_handler = ib_uverbs_cq_event_handler; 1391 cq->cq_context = ev_file; 1392 atomic_set(&cq->usecnt, 0); 1393 1394 obj->uobject.object = cq; 1395 ret = idr_add_uobj(&ib_uverbs_cq_idr, &obj->uobject); 1396 if (ret) 1397 goto err_free; 1398 1399 memset(&resp, 0, sizeof resp); 1400 resp.cq_handle = obj->uobject.id; 1401 resp.cqe = cq->cqe; 1402 1403 if (copy_to_user((void __user *) (unsigned long) cmd.response, 1404 &resp, sizeof resp)) { 1405 ret = -EFAULT; 1406 goto err_copy; 1407 } 1408 1409 mutex_lock(&file->mutex); 1410 list_add_tail(&obj->uobject.list, &file->ucontext->cq_list); 1411 mutex_unlock(&file->mutex); 1412 1413 obj->uobject.live = 1; 1414 1415 up_write(&obj->uobject.mutex); 1416 1417 return in_len; 1418 1419 err_copy: 1420 idr_remove_uobj(&ib_uverbs_cq_idr, &obj->uobject); 1421 1422 err_free: 1423 ib_destroy_cq(cq); 1424 1425 err_file: 1426 if (ev_file) 1427 ib_uverbs_release_ucq(file, ev_file, obj); 1428 1429 err: 1430 put_uobj_write(&obj->uobject); 1431 return ret; 1432 } 1433 1434 ssize_t ib_uverbs_resize_cq(struct ib_uverbs_file *file, 1435 const char __user *buf, int in_len, 1436 int out_len) 1437 { 1438 struct ib_uverbs_resize_cq cmd; 1439 struct ib_uverbs_resize_cq_resp resp; 1440 struct ib_udata udata; 1441 struct ib_cq *cq; 1442 int ret = -EINVAL; 1443 1444 if (copy_from_user(&cmd, buf, sizeof cmd)) 1445 return -EFAULT; 1446 1447 INIT_UDATA(&udata, buf + sizeof cmd, 1448 (unsigned long) cmd.response + sizeof resp, 1449 in_len - sizeof cmd, out_len - sizeof resp); 1450 1451 cq = idr_read_cq(cmd.cq_handle, file->ucontext, 0); 1452 if (!cq) 1453 return -EINVAL; 1454 1455 ret = cq->device->resize_cq(cq, cmd.cqe, &udata); 1456 if (ret) 1457 goto out; 1458 1459 resp.cqe = cq->cqe; 1460 1461 if (copy_to_user((void __user *) (unsigned long) cmd.response, 1462 &resp, sizeof resp.cqe)) 1463 ret = -EFAULT; 1464 1465 out: 1466 put_cq_read(cq); 1467 1468 return ret ? ret : in_len; 1469 } 1470 1471 static int copy_wc_to_user(void __user *dest, struct ib_wc *wc) 1472 { 1473 struct ib_uverbs_wc tmp; 1474 1475 tmp.wr_id = wc->wr_id; 1476 tmp.status = wc->status; 1477 tmp.opcode = wc->opcode; 1478 tmp.vendor_err = wc->vendor_err; 1479 tmp.byte_len = wc->byte_len; 1480 tmp.ex.imm_data = (__u32 __force) wc->ex.imm_data; 1481 tmp.qp_num = wc->qp->qp_num; 1482 tmp.src_qp = wc->src_qp; 1483 tmp.wc_flags = wc->wc_flags; 1484 tmp.pkey_index = wc->pkey_index; 1485 tmp.slid = wc->slid; 1486 tmp.sl = wc->sl; 1487 tmp.dlid_path_bits = wc->dlid_path_bits; 1488 tmp.port_num = wc->port_num; 1489 tmp.reserved = 0; 1490 1491 if (copy_to_user(dest, &tmp, sizeof tmp)) 1492 return -EFAULT; 1493 1494 return 0; 1495 } 1496 1497 ssize_t ib_uverbs_poll_cq(struct ib_uverbs_file *file, 1498 const char __user *buf, int in_len, 1499 int out_len) 1500 { 1501 struct ib_uverbs_poll_cq cmd; 1502 struct ib_uverbs_poll_cq_resp resp; 1503 u8 __user *header_ptr; 1504 u8 __user *data_ptr; 1505 struct ib_cq *cq; 1506 struct ib_wc wc; 1507 int ret; 1508 1509 if (copy_from_user(&cmd, buf, sizeof cmd)) 1510 return -EFAULT; 1511 1512 cq = idr_read_cq(cmd.cq_handle, file->ucontext, 0); 1513 if (!cq) 1514 return -EINVAL; 1515 1516 /* we copy a struct ib_uverbs_poll_cq_resp to user space */ 1517 header_ptr = (void __user *)(unsigned long) cmd.response; 1518 data_ptr = header_ptr + sizeof resp; 1519 1520 memset(&resp, 0, sizeof resp); 1521 while (resp.count < cmd.ne) { 1522 ret = ib_poll_cq(cq, 1, &wc); 1523 if (ret < 0) 1524 goto out_put; 1525 if (!ret) 1526 break; 1527 1528 ret = copy_wc_to_user(data_ptr, &wc); 1529 if (ret) 1530 goto out_put; 1531 1532 data_ptr += sizeof(struct ib_uverbs_wc); 1533 ++resp.count; 1534 } 1535 1536 if (copy_to_user(header_ptr, &resp, sizeof resp)) { 1537 ret = -EFAULT; 1538 goto out_put; 1539 } 1540 1541 ret = in_len; 1542 1543 out_put: 1544 put_cq_read(cq); 1545 return ret; 1546 } 1547 1548 ssize_t ib_uverbs_req_notify_cq(struct ib_uverbs_file *file, 1549 const char __user *buf, int in_len, 1550 int out_len) 1551 { 1552 struct ib_uverbs_req_notify_cq cmd; 1553 struct ib_cq *cq; 1554 1555 if (copy_from_user(&cmd, buf, sizeof cmd)) 1556 return -EFAULT; 1557 1558 cq = idr_read_cq(cmd.cq_handle, file->ucontext, 0); 1559 if (!cq) 1560 return -EINVAL; 1561 1562 ib_req_notify_cq(cq, cmd.solicited_only ? 1563 IB_CQ_SOLICITED : IB_CQ_NEXT_COMP); 1564 1565 put_cq_read(cq); 1566 1567 return in_len; 1568 } 1569 1570 ssize_t ib_uverbs_destroy_cq(struct ib_uverbs_file *file, 1571 const char __user *buf, int in_len, 1572 int out_len) 1573 { 1574 struct ib_uverbs_destroy_cq cmd; 1575 struct ib_uverbs_destroy_cq_resp resp; 1576 struct ib_uobject *uobj; 1577 struct ib_cq *cq; 1578 struct ib_ucq_object *obj; 1579 struct ib_uverbs_event_file *ev_file; 1580 int ret = -EINVAL; 1581 1582 if (copy_from_user(&cmd, buf, sizeof cmd)) 1583 return -EFAULT; 1584 1585 uobj = idr_write_uobj(&ib_uverbs_cq_idr, cmd.cq_handle, file->ucontext); 1586 if (!uobj) 1587 return -EINVAL; 1588 cq = uobj->object; 1589 ev_file = cq->cq_context; 1590 obj = container_of(cq->uobject, struct ib_ucq_object, uobject); 1591 1592 ret = ib_destroy_cq(cq); 1593 if (!ret) 1594 uobj->live = 0; 1595 1596 put_uobj_write(uobj); 1597 1598 if (ret) 1599 return ret; 1600 1601 idr_remove_uobj(&ib_uverbs_cq_idr, uobj); 1602 1603 mutex_lock(&file->mutex); 1604 list_del(&uobj->list); 1605 mutex_unlock(&file->mutex); 1606 1607 ib_uverbs_release_ucq(file, ev_file, obj); 1608 1609 memset(&resp, 0, sizeof resp); 1610 resp.comp_events_reported = obj->comp_events_reported; 1611 resp.async_events_reported = obj->async_events_reported; 1612 1613 put_uobj(uobj); 1614 1615 if (copy_to_user((void __user *) (unsigned long) cmd.response, 1616 &resp, sizeof resp)) 1617 return -EFAULT; 1618 1619 return in_len; 1620 } 1621 1622 ssize_t ib_uverbs_create_qp(struct ib_uverbs_file *file, 1623 const char __user *buf, int in_len, 1624 int out_len) 1625 { 1626 struct ib_uverbs_create_qp cmd; 1627 struct ib_uverbs_create_qp_resp resp; 1628 struct ib_udata udata; 1629 struct ib_uqp_object *obj; 1630 struct ib_device *device; 1631 struct ib_pd *pd = NULL; 1632 struct ib_xrcd *xrcd = NULL; 1633 struct ib_uobject *uninitialized_var(xrcd_uobj); 1634 struct ib_cq *scq = NULL, *rcq = NULL; 1635 struct ib_srq *srq = NULL; 1636 struct ib_qp *qp; 1637 struct ib_qp_init_attr attr; 1638 int ret; 1639 1640 if (out_len < sizeof resp) 1641 return -ENOSPC; 1642 1643 if (copy_from_user(&cmd, buf, sizeof cmd)) 1644 return -EFAULT; 1645 1646 if (cmd.qp_type == IB_QPT_RAW_PACKET && !capable(CAP_NET_RAW)) 1647 return -EPERM; 1648 1649 INIT_UDATA(&udata, buf + sizeof cmd, 1650 (unsigned long) cmd.response + sizeof resp, 1651 in_len - sizeof cmd, out_len - sizeof resp); 1652 1653 obj = kzalloc(sizeof *obj, GFP_KERNEL); 1654 if (!obj) 1655 return -ENOMEM; 1656 1657 init_uobj(&obj->uevent.uobject, cmd.user_handle, file->ucontext, &qp_lock_class); 1658 down_write(&obj->uevent.uobject.mutex); 1659 1660 if (cmd.qp_type == IB_QPT_XRC_TGT) { 1661 xrcd = idr_read_xrcd(cmd.pd_handle, file->ucontext, &xrcd_uobj); 1662 if (!xrcd) { 1663 ret = -EINVAL; 1664 goto err_put; 1665 } 1666 device = xrcd->device; 1667 } else { 1668 if (cmd.qp_type == IB_QPT_XRC_INI) { 1669 cmd.max_recv_wr = cmd.max_recv_sge = 0; 1670 } else { 1671 if (cmd.is_srq) { 1672 srq = idr_read_srq(cmd.srq_handle, file->ucontext); 1673 if (!srq || srq->srq_type != IB_SRQT_BASIC) { 1674 ret = -EINVAL; 1675 goto err_put; 1676 } 1677 } 1678 1679 if (cmd.recv_cq_handle != cmd.send_cq_handle) { 1680 rcq = idr_read_cq(cmd.recv_cq_handle, file->ucontext, 0); 1681 if (!rcq) { 1682 ret = -EINVAL; 1683 goto err_put; 1684 } 1685 } 1686 } 1687 1688 scq = idr_read_cq(cmd.send_cq_handle, file->ucontext, !!rcq); 1689 rcq = rcq ?: scq; 1690 pd = idr_read_pd(cmd.pd_handle, file->ucontext); 1691 if (!pd || !scq) { 1692 ret = -EINVAL; 1693 goto err_put; 1694 } 1695 1696 device = pd->device; 1697 } 1698 1699 attr.event_handler = ib_uverbs_qp_event_handler; 1700 attr.qp_context = file; 1701 attr.send_cq = scq; 1702 attr.recv_cq = rcq; 1703 attr.srq = srq; 1704 attr.xrcd = xrcd; 1705 attr.sq_sig_type = cmd.sq_sig_all ? IB_SIGNAL_ALL_WR : IB_SIGNAL_REQ_WR; 1706 attr.qp_type = cmd.qp_type; 1707 attr.create_flags = 0; 1708 1709 attr.cap.max_send_wr = cmd.max_send_wr; 1710 attr.cap.max_recv_wr = cmd.max_recv_wr; 1711 attr.cap.max_send_sge = cmd.max_send_sge; 1712 attr.cap.max_recv_sge = cmd.max_recv_sge; 1713 attr.cap.max_inline_data = cmd.max_inline_data; 1714 1715 obj->uevent.events_reported = 0; 1716 INIT_LIST_HEAD(&obj->uevent.event_list); 1717 INIT_LIST_HEAD(&obj->mcast_list); 1718 1719 if (cmd.qp_type == IB_QPT_XRC_TGT) 1720 qp = ib_create_qp(pd, &attr); 1721 else 1722 qp = device->create_qp(pd, &attr, &udata); 1723 1724 if (IS_ERR(qp)) { 1725 ret = PTR_ERR(qp); 1726 goto err_put; 1727 } 1728 1729 if (cmd.qp_type != IB_QPT_XRC_TGT) { 1730 qp->real_qp = qp; 1731 qp->device = device; 1732 qp->pd = pd; 1733 qp->send_cq = attr.send_cq; 1734 qp->recv_cq = attr.recv_cq; 1735 qp->srq = attr.srq; 1736 qp->event_handler = attr.event_handler; 1737 qp->qp_context = attr.qp_context; 1738 qp->qp_type = attr.qp_type; 1739 atomic_set(&qp->usecnt, 0); 1740 atomic_inc(&pd->usecnt); 1741 atomic_inc(&attr.send_cq->usecnt); 1742 if (attr.recv_cq) 1743 atomic_inc(&attr.recv_cq->usecnt); 1744 if (attr.srq) 1745 atomic_inc(&attr.srq->usecnt); 1746 } 1747 qp->uobject = &obj->uevent.uobject; 1748 1749 obj->uevent.uobject.object = qp; 1750 ret = idr_add_uobj(&ib_uverbs_qp_idr, &obj->uevent.uobject); 1751 if (ret) 1752 goto err_destroy; 1753 1754 memset(&resp, 0, sizeof resp); 1755 resp.qpn = qp->qp_num; 1756 resp.qp_handle = obj->uevent.uobject.id; 1757 resp.max_recv_sge = attr.cap.max_recv_sge; 1758 resp.max_send_sge = attr.cap.max_send_sge; 1759 resp.max_recv_wr = attr.cap.max_recv_wr; 1760 resp.max_send_wr = attr.cap.max_send_wr; 1761 resp.max_inline_data = attr.cap.max_inline_data; 1762 1763 if (copy_to_user((void __user *) (unsigned long) cmd.response, 1764 &resp, sizeof resp)) { 1765 ret = -EFAULT; 1766 goto err_copy; 1767 } 1768 1769 if (xrcd) { 1770 obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object, 1771 uobject); 1772 atomic_inc(&obj->uxrcd->refcnt); 1773 put_xrcd_read(xrcd_uobj); 1774 } 1775 1776 if (pd) 1777 put_pd_read(pd); 1778 if (scq) 1779 put_cq_read(scq); 1780 if (rcq && rcq != scq) 1781 put_cq_read(rcq); 1782 if (srq) 1783 put_srq_read(srq); 1784 1785 mutex_lock(&file->mutex); 1786 list_add_tail(&obj->uevent.uobject.list, &file->ucontext->qp_list); 1787 mutex_unlock(&file->mutex); 1788 1789 obj->uevent.uobject.live = 1; 1790 1791 up_write(&obj->uevent.uobject.mutex); 1792 1793 return in_len; 1794 1795 err_copy: 1796 idr_remove_uobj(&ib_uverbs_qp_idr, &obj->uevent.uobject); 1797 1798 err_destroy: 1799 ib_destroy_qp(qp); 1800 1801 err_put: 1802 if (xrcd) 1803 put_xrcd_read(xrcd_uobj); 1804 if (pd) 1805 put_pd_read(pd); 1806 if (scq) 1807 put_cq_read(scq); 1808 if (rcq && rcq != scq) 1809 put_cq_read(rcq); 1810 if (srq) 1811 put_srq_read(srq); 1812 1813 put_uobj_write(&obj->uevent.uobject); 1814 return ret; 1815 } 1816 1817 ssize_t ib_uverbs_open_qp(struct ib_uverbs_file *file, 1818 const char __user *buf, int in_len, int out_len) 1819 { 1820 struct ib_uverbs_open_qp cmd; 1821 struct ib_uverbs_create_qp_resp resp; 1822 struct ib_udata udata; 1823 struct ib_uqp_object *obj; 1824 struct ib_xrcd *xrcd; 1825 struct ib_uobject *uninitialized_var(xrcd_uobj); 1826 struct ib_qp *qp; 1827 struct ib_qp_open_attr attr; 1828 int ret; 1829 1830 if (out_len < sizeof resp) 1831 return -ENOSPC; 1832 1833 if (copy_from_user(&cmd, buf, sizeof cmd)) 1834 return -EFAULT; 1835 1836 INIT_UDATA(&udata, buf + sizeof cmd, 1837 (unsigned long) cmd.response + sizeof resp, 1838 in_len - sizeof cmd, out_len - sizeof resp); 1839 1840 obj = kmalloc(sizeof *obj, GFP_KERNEL); 1841 if (!obj) 1842 return -ENOMEM; 1843 1844 init_uobj(&obj->uevent.uobject, cmd.user_handle, file->ucontext, &qp_lock_class); 1845 down_write(&obj->uevent.uobject.mutex); 1846 1847 xrcd = idr_read_xrcd(cmd.pd_handle, file->ucontext, &xrcd_uobj); 1848 if (!xrcd) { 1849 ret = -EINVAL; 1850 goto err_put; 1851 } 1852 1853 attr.event_handler = ib_uverbs_qp_event_handler; 1854 attr.qp_context = file; 1855 attr.qp_num = cmd.qpn; 1856 attr.qp_type = cmd.qp_type; 1857 1858 obj->uevent.events_reported = 0; 1859 INIT_LIST_HEAD(&obj->uevent.event_list); 1860 INIT_LIST_HEAD(&obj->mcast_list); 1861 1862 qp = ib_open_qp(xrcd, &attr); 1863 if (IS_ERR(qp)) { 1864 ret = PTR_ERR(qp); 1865 goto err_put; 1866 } 1867 1868 qp->uobject = &obj->uevent.uobject; 1869 1870 obj->uevent.uobject.object = qp; 1871 ret = idr_add_uobj(&ib_uverbs_qp_idr, &obj->uevent.uobject); 1872 if (ret) 1873 goto err_destroy; 1874 1875 memset(&resp, 0, sizeof resp); 1876 resp.qpn = qp->qp_num; 1877 resp.qp_handle = obj->uevent.uobject.id; 1878 1879 if (copy_to_user((void __user *) (unsigned long) cmd.response, 1880 &resp, sizeof resp)) { 1881 ret = -EFAULT; 1882 goto err_remove; 1883 } 1884 1885 obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object, uobject); 1886 atomic_inc(&obj->uxrcd->refcnt); 1887 put_xrcd_read(xrcd_uobj); 1888 1889 mutex_lock(&file->mutex); 1890 list_add_tail(&obj->uevent.uobject.list, &file->ucontext->qp_list); 1891 mutex_unlock(&file->mutex); 1892 1893 obj->uevent.uobject.live = 1; 1894 1895 up_write(&obj->uevent.uobject.mutex); 1896 1897 return in_len; 1898 1899 err_remove: 1900 idr_remove_uobj(&ib_uverbs_qp_idr, &obj->uevent.uobject); 1901 1902 err_destroy: 1903 ib_destroy_qp(qp); 1904 1905 err_put: 1906 put_xrcd_read(xrcd_uobj); 1907 put_uobj_write(&obj->uevent.uobject); 1908 return ret; 1909 } 1910 1911 ssize_t ib_uverbs_query_qp(struct ib_uverbs_file *file, 1912 const char __user *buf, int in_len, 1913 int out_len) 1914 { 1915 struct ib_uverbs_query_qp cmd; 1916 struct ib_uverbs_query_qp_resp resp; 1917 struct ib_qp *qp; 1918 struct ib_qp_attr *attr; 1919 struct ib_qp_init_attr *init_attr; 1920 int ret; 1921 1922 if (copy_from_user(&cmd, buf, sizeof cmd)) 1923 return -EFAULT; 1924 1925 attr = kmalloc(sizeof *attr, GFP_KERNEL); 1926 init_attr = kmalloc(sizeof *init_attr, GFP_KERNEL); 1927 if (!attr || !init_attr) { 1928 ret = -ENOMEM; 1929 goto out; 1930 } 1931 1932 qp = idr_read_qp(cmd.qp_handle, file->ucontext); 1933 if (!qp) { 1934 ret = -EINVAL; 1935 goto out; 1936 } 1937 1938 ret = ib_query_qp(qp, attr, cmd.attr_mask, init_attr); 1939 1940 put_qp_read(qp); 1941 1942 if (ret) 1943 goto out; 1944 1945 memset(&resp, 0, sizeof resp); 1946 1947 resp.qp_state = attr->qp_state; 1948 resp.cur_qp_state = attr->cur_qp_state; 1949 resp.path_mtu = attr->path_mtu; 1950 resp.path_mig_state = attr->path_mig_state; 1951 resp.qkey = attr->qkey; 1952 resp.rq_psn = attr->rq_psn; 1953 resp.sq_psn = attr->sq_psn; 1954 resp.dest_qp_num = attr->dest_qp_num; 1955 resp.qp_access_flags = attr->qp_access_flags; 1956 resp.pkey_index = attr->pkey_index; 1957 resp.alt_pkey_index = attr->alt_pkey_index; 1958 resp.sq_draining = attr->sq_draining; 1959 resp.max_rd_atomic = attr->max_rd_atomic; 1960 resp.max_dest_rd_atomic = attr->max_dest_rd_atomic; 1961 resp.min_rnr_timer = attr->min_rnr_timer; 1962 resp.port_num = attr->port_num; 1963 resp.timeout = attr->timeout; 1964 resp.retry_cnt = attr->retry_cnt; 1965 resp.rnr_retry = attr->rnr_retry; 1966 resp.alt_port_num = attr->alt_port_num; 1967 resp.alt_timeout = attr->alt_timeout; 1968 1969 memcpy(resp.dest.dgid, attr->ah_attr.grh.dgid.raw, 16); 1970 resp.dest.flow_label = attr->ah_attr.grh.flow_label; 1971 resp.dest.sgid_index = attr->ah_attr.grh.sgid_index; 1972 resp.dest.hop_limit = attr->ah_attr.grh.hop_limit; 1973 resp.dest.traffic_class = attr->ah_attr.grh.traffic_class; 1974 resp.dest.dlid = attr->ah_attr.dlid; 1975 resp.dest.sl = attr->ah_attr.sl; 1976 resp.dest.src_path_bits = attr->ah_attr.src_path_bits; 1977 resp.dest.static_rate = attr->ah_attr.static_rate; 1978 resp.dest.is_global = !!(attr->ah_attr.ah_flags & IB_AH_GRH); 1979 resp.dest.port_num = attr->ah_attr.port_num; 1980 1981 memcpy(resp.alt_dest.dgid, attr->alt_ah_attr.grh.dgid.raw, 16); 1982 resp.alt_dest.flow_label = attr->alt_ah_attr.grh.flow_label; 1983 resp.alt_dest.sgid_index = attr->alt_ah_attr.grh.sgid_index; 1984 resp.alt_dest.hop_limit = attr->alt_ah_attr.grh.hop_limit; 1985 resp.alt_dest.traffic_class = attr->alt_ah_attr.grh.traffic_class; 1986 resp.alt_dest.dlid = attr->alt_ah_attr.dlid; 1987 resp.alt_dest.sl = attr->alt_ah_attr.sl; 1988 resp.alt_dest.src_path_bits = attr->alt_ah_attr.src_path_bits; 1989 resp.alt_dest.static_rate = attr->alt_ah_attr.static_rate; 1990 resp.alt_dest.is_global = !!(attr->alt_ah_attr.ah_flags & IB_AH_GRH); 1991 resp.alt_dest.port_num = attr->alt_ah_attr.port_num; 1992 1993 resp.max_send_wr = init_attr->cap.max_send_wr; 1994 resp.max_recv_wr = init_attr->cap.max_recv_wr; 1995 resp.max_send_sge = init_attr->cap.max_send_sge; 1996 resp.max_recv_sge = init_attr->cap.max_recv_sge; 1997 resp.max_inline_data = init_attr->cap.max_inline_data; 1998 resp.sq_sig_all = init_attr->sq_sig_type == IB_SIGNAL_ALL_WR; 1999 2000 if (copy_to_user((void __user *) (unsigned long) cmd.response, 2001 &resp, sizeof resp)) 2002 ret = -EFAULT; 2003 2004 out: 2005 kfree(attr); 2006 kfree(init_attr); 2007 2008 return ret ? ret : in_len; 2009 } 2010 2011 /* Remove ignored fields set in the attribute mask */ 2012 static int modify_qp_mask(enum ib_qp_type qp_type, int mask) 2013 { 2014 switch (qp_type) { 2015 case IB_QPT_XRC_INI: 2016 return mask & ~(IB_QP_MAX_DEST_RD_ATOMIC | IB_QP_MIN_RNR_TIMER); 2017 case IB_QPT_XRC_TGT: 2018 return mask & ~(IB_QP_MAX_QP_RD_ATOMIC | IB_QP_RETRY_CNT | 2019 IB_QP_RNR_RETRY); 2020 default: 2021 return mask; 2022 } 2023 } 2024 2025 ssize_t ib_uverbs_modify_qp(struct ib_uverbs_file *file, 2026 const char __user *buf, int in_len, 2027 int out_len) 2028 { 2029 struct ib_uverbs_modify_qp cmd; 2030 struct ib_udata udata; 2031 struct ib_qp *qp; 2032 struct ib_qp_attr *attr; 2033 int ret; 2034 2035 if (copy_from_user(&cmd, buf, sizeof cmd)) 2036 return -EFAULT; 2037 2038 INIT_UDATA(&udata, buf + sizeof cmd, NULL, in_len - sizeof cmd, 2039 out_len); 2040 2041 attr = kmalloc(sizeof *attr, GFP_KERNEL); 2042 if (!attr) 2043 return -ENOMEM; 2044 2045 qp = idr_read_qp(cmd.qp_handle, file->ucontext); 2046 if (!qp) { 2047 ret = -EINVAL; 2048 goto out; 2049 } 2050 2051 attr->qp_state = cmd.qp_state; 2052 attr->cur_qp_state = cmd.cur_qp_state; 2053 attr->path_mtu = cmd.path_mtu; 2054 attr->path_mig_state = cmd.path_mig_state; 2055 attr->qkey = cmd.qkey; 2056 attr->rq_psn = cmd.rq_psn; 2057 attr->sq_psn = cmd.sq_psn; 2058 attr->dest_qp_num = cmd.dest_qp_num; 2059 attr->qp_access_flags = cmd.qp_access_flags; 2060 attr->pkey_index = cmd.pkey_index; 2061 attr->alt_pkey_index = cmd.alt_pkey_index; 2062 attr->en_sqd_async_notify = cmd.en_sqd_async_notify; 2063 attr->max_rd_atomic = cmd.max_rd_atomic; 2064 attr->max_dest_rd_atomic = cmd.max_dest_rd_atomic; 2065 attr->min_rnr_timer = cmd.min_rnr_timer; 2066 attr->port_num = cmd.port_num; 2067 attr->timeout = cmd.timeout; 2068 attr->retry_cnt = cmd.retry_cnt; 2069 attr->rnr_retry = cmd.rnr_retry; 2070 attr->alt_port_num = cmd.alt_port_num; 2071 attr->alt_timeout = cmd.alt_timeout; 2072 2073 memcpy(attr->ah_attr.grh.dgid.raw, cmd.dest.dgid, 16); 2074 attr->ah_attr.grh.flow_label = cmd.dest.flow_label; 2075 attr->ah_attr.grh.sgid_index = cmd.dest.sgid_index; 2076 attr->ah_attr.grh.hop_limit = cmd.dest.hop_limit; 2077 attr->ah_attr.grh.traffic_class = cmd.dest.traffic_class; 2078 attr->ah_attr.dlid = cmd.dest.dlid; 2079 attr->ah_attr.sl = cmd.dest.sl; 2080 attr->ah_attr.src_path_bits = cmd.dest.src_path_bits; 2081 attr->ah_attr.static_rate = cmd.dest.static_rate; 2082 attr->ah_attr.ah_flags = cmd.dest.is_global ? IB_AH_GRH : 0; 2083 attr->ah_attr.port_num = cmd.dest.port_num; 2084 2085 memcpy(attr->alt_ah_attr.grh.dgid.raw, cmd.alt_dest.dgid, 16); 2086 attr->alt_ah_attr.grh.flow_label = cmd.alt_dest.flow_label; 2087 attr->alt_ah_attr.grh.sgid_index = cmd.alt_dest.sgid_index; 2088 attr->alt_ah_attr.grh.hop_limit = cmd.alt_dest.hop_limit; 2089 attr->alt_ah_attr.grh.traffic_class = cmd.alt_dest.traffic_class; 2090 attr->alt_ah_attr.dlid = cmd.alt_dest.dlid; 2091 attr->alt_ah_attr.sl = cmd.alt_dest.sl; 2092 attr->alt_ah_attr.src_path_bits = cmd.alt_dest.src_path_bits; 2093 attr->alt_ah_attr.static_rate = cmd.alt_dest.static_rate; 2094 attr->alt_ah_attr.ah_flags = cmd.alt_dest.is_global ? IB_AH_GRH : 0; 2095 attr->alt_ah_attr.port_num = cmd.alt_dest.port_num; 2096 2097 if (qp->real_qp == qp) { 2098 ret = ib_resolve_eth_l2_attrs(qp, attr, &cmd.attr_mask); 2099 if (ret) 2100 goto release_qp; 2101 ret = qp->device->modify_qp(qp, attr, 2102 modify_qp_mask(qp->qp_type, cmd.attr_mask), &udata); 2103 } else { 2104 ret = ib_modify_qp(qp, attr, modify_qp_mask(qp->qp_type, cmd.attr_mask)); 2105 } 2106 2107 if (ret) 2108 goto release_qp; 2109 2110 ret = in_len; 2111 2112 release_qp: 2113 put_qp_read(qp); 2114 2115 out: 2116 kfree(attr); 2117 2118 return ret; 2119 } 2120 2121 ssize_t ib_uverbs_destroy_qp(struct ib_uverbs_file *file, 2122 const char __user *buf, int in_len, 2123 int out_len) 2124 { 2125 struct ib_uverbs_destroy_qp cmd; 2126 struct ib_uverbs_destroy_qp_resp resp; 2127 struct ib_uobject *uobj; 2128 struct ib_qp *qp; 2129 struct ib_uqp_object *obj; 2130 int ret = -EINVAL; 2131 2132 if (copy_from_user(&cmd, buf, sizeof cmd)) 2133 return -EFAULT; 2134 2135 memset(&resp, 0, sizeof resp); 2136 2137 uobj = idr_write_uobj(&ib_uverbs_qp_idr, cmd.qp_handle, file->ucontext); 2138 if (!uobj) 2139 return -EINVAL; 2140 qp = uobj->object; 2141 obj = container_of(uobj, struct ib_uqp_object, uevent.uobject); 2142 2143 if (!list_empty(&obj->mcast_list)) { 2144 put_uobj_write(uobj); 2145 return -EBUSY; 2146 } 2147 2148 ret = ib_destroy_qp(qp); 2149 if (!ret) 2150 uobj->live = 0; 2151 2152 put_uobj_write(uobj); 2153 2154 if (ret) 2155 return ret; 2156 2157 if (obj->uxrcd) 2158 atomic_dec(&obj->uxrcd->refcnt); 2159 2160 idr_remove_uobj(&ib_uverbs_qp_idr, uobj); 2161 2162 mutex_lock(&file->mutex); 2163 list_del(&uobj->list); 2164 mutex_unlock(&file->mutex); 2165 2166 ib_uverbs_release_uevent(file, &obj->uevent); 2167 2168 resp.events_reported = obj->uevent.events_reported; 2169 2170 put_uobj(uobj); 2171 2172 if (copy_to_user((void __user *) (unsigned long) cmd.response, 2173 &resp, sizeof resp)) 2174 return -EFAULT; 2175 2176 return in_len; 2177 } 2178 2179 ssize_t ib_uverbs_post_send(struct ib_uverbs_file *file, 2180 const char __user *buf, int in_len, 2181 int out_len) 2182 { 2183 struct ib_uverbs_post_send cmd; 2184 struct ib_uverbs_post_send_resp resp; 2185 struct ib_uverbs_send_wr *user_wr; 2186 struct ib_send_wr *wr = NULL, *last, *next, *bad_wr; 2187 struct ib_qp *qp; 2188 int i, sg_ind; 2189 int is_ud; 2190 ssize_t ret = -EINVAL; 2191 2192 if (copy_from_user(&cmd, buf, sizeof cmd)) 2193 return -EFAULT; 2194 2195 if (in_len < sizeof cmd + cmd.wqe_size * cmd.wr_count + 2196 cmd.sge_count * sizeof (struct ib_uverbs_sge)) 2197 return -EINVAL; 2198 2199 if (cmd.wqe_size < sizeof (struct ib_uverbs_send_wr)) 2200 return -EINVAL; 2201 2202 user_wr = kmalloc(cmd.wqe_size, GFP_KERNEL); 2203 if (!user_wr) 2204 return -ENOMEM; 2205 2206 qp = idr_read_qp(cmd.qp_handle, file->ucontext); 2207 if (!qp) 2208 goto out; 2209 2210 is_ud = qp->qp_type == IB_QPT_UD; 2211 sg_ind = 0; 2212 last = NULL; 2213 for (i = 0; i < cmd.wr_count; ++i) { 2214 if (copy_from_user(user_wr, 2215 buf + sizeof cmd + i * cmd.wqe_size, 2216 cmd.wqe_size)) { 2217 ret = -EFAULT; 2218 goto out_put; 2219 } 2220 2221 if (user_wr->num_sge + sg_ind > cmd.sge_count) { 2222 ret = -EINVAL; 2223 goto out_put; 2224 } 2225 2226 next = kmalloc(ALIGN(sizeof *next, sizeof (struct ib_sge)) + 2227 user_wr->num_sge * sizeof (struct ib_sge), 2228 GFP_KERNEL); 2229 if (!next) { 2230 ret = -ENOMEM; 2231 goto out_put; 2232 } 2233 2234 if (!last) 2235 wr = next; 2236 else 2237 last->next = next; 2238 last = next; 2239 2240 next->next = NULL; 2241 next->wr_id = user_wr->wr_id; 2242 next->num_sge = user_wr->num_sge; 2243 next->opcode = user_wr->opcode; 2244 next->send_flags = user_wr->send_flags; 2245 2246 if (is_ud) { 2247 next->wr.ud.ah = idr_read_ah(user_wr->wr.ud.ah, 2248 file->ucontext); 2249 if (!next->wr.ud.ah) { 2250 ret = -EINVAL; 2251 goto out_put; 2252 } 2253 next->wr.ud.remote_qpn = user_wr->wr.ud.remote_qpn; 2254 next->wr.ud.remote_qkey = user_wr->wr.ud.remote_qkey; 2255 if (next->opcode == IB_WR_SEND_WITH_IMM) 2256 next->ex.imm_data = 2257 (__be32 __force) user_wr->ex.imm_data; 2258 } else { 2259 switch (next->opcode) { 2260 case IB_WR_RDMA_WRITE_WITH_IMM: 2261 next->ex.imm_data = 2262 (__be32 __force) user_wr->ex.imm_data; 2263 case IB_WR_RDMA_WRITE: 2264 case IB_WR_RDMA_READ: 2265 next->wr.rdma.remote_addr = 2266 user_wr->wr.rdma.remote_addr; 2267 next->wr.rdma.rkey = 2268 user_wr->wr.rdma.rkey; 2269 break; 2270 case IB_WR_SEND_WITH_IMM: 2271 next->ex.imm_data = 2272 (__be32 __force) user_wr->ex.imm_data; 2273 break; 2274 case IB_WR_SEND_WITH_INV: 2275 next->ex.invalidate_rkey = 2276 user_wr->ex.invalidate_rkey; 2277 break; 2278 case IB_WR_ATOMIC_CMP_AND_SWP: 2279 case IB_WR_ATOMIC_FETCH_AND_ADD: 2280 next->wr.atomic.remote_addr = 2281 user_wr->wr.atomic.remote_addr; 2282 next->wr.atomic.compare_add = 2283 user_wr->wr.atomic.compare_add; 2284 next->wr.atomic.swap = user_wr->wr.atomic.swap; 2285 next->wr.atomic.rkey = user_wr->wr.atomic.rkey; 2286 break; 2287 default: 2288 break; 2289 } 2290 } 2291 2292 if (next->num_sge) { 2293 next->sg_list = (void *) next + 2294 ALIGN(sizeof *next, sizeof (struct ib_sge)); 2295 if (copy_from_user(next->sg_list, 2296 buf + sizeof cmd + 2297 cmd.wr_count * cmd.wqe_size + 2298 sg_ind * sizeof (struct ib_sge), 2299 next->num_sge * sizeof (struct ib_sge))) { 2300 ret = -EFAULT; 2301 goto out_put; 2302 } 2303 sg_ind += next->num_sge; 2304 } else 2305 next->sg_list = NULL; 2306 } 2307 2308 resp.bad_wr = 0; 2309 ret = qp->device->post_send(qp->real_qp, wr, &bad_wr); 2310 if (ret) 2311 for (next = wr; next; next = next->next) { 2312 ++resp.bad_wr; 2313 if (next == bad_wr) 2314 break; 2315 } 2316 2317 if (copy_to_user((void __user *) (unsigned long) cmd.response, 2318 &resp, sizeof resp)) 2319 ret = -EFAULT; 2320 2321 out_put: 2322 put_qp_read(qp); 2323 2324 while (wr) { 2325 if (is_ud && wr->wr.ud.ah) 2326 put_ah_read(wr->wr.ud.ah); 2327 next = wr->next; 2328 kfree(wr); 2329 wr = next; 2330 } 2331 2332 out: 2333 kfree(user_wr); 2334 2335 return ret ? ret : in_len; 2336 } 2337 2338 static struct ib_recv_wr *ib_uverbs_unmarshall_recv(const char __user *buf, 2339 int in_len, 2340 u32 wr_count, 2341 u32 sge_count, 2342 u32 wqe_size) 2343 { 2344 struct ib_uverbs_recv_wr *user_wr; 2345 struct ib_recv_wr *wr = NULL, *last, *next; 2346 int sg_ind; 2347 int i; 2348 int ret; 2349 2350 if (in_len < wqe_size * wr_count + 2351 sge_count * sizeof (struct ib_uverbs_sge)) 2352 return ERR_PTR(-EINVAL); 2353 2354 if (wqe_size < sizeof (struct ib_uverbs_recv_wr)) 2355 return ERR_PTR(-EINVAL); 2356 2357 user_wr = kmalloc(wqe_size, GFP_KERNEL); 2358 if (!user_wr) 2359 return ERR_PTR(-ENOMEM); 2360 2361 sg_ind = 0; 2362 last = NULL; 2363 for (i = 0; i < wr_count; ++i) { 2364 if (copy_from_user(user_wr, buf + i * wqe_size, 2365 wqe_size)) { 2366 ret = -EFAULT; 2367 goto err; 2368 } 2369 2370 if (user_wr->num_sge + sg_ind > sge_count) { 2371 ret = -EINVAL; 2372 goto err; 2373 } 2374 2375 next = kmalloc(ALIGN(sizeof *next, sizeof (struct ib_sge)) + 2376 user_wr->num_sge * sizeof (struct ib_sge), 2377 GFP_KERNEL); 2378 if (!next) { 2379 ret = -ENOMEM; 2380 goto err; 2381 } 2382 2383 if (!last) 2384 wr = next; 2385 else 2386 last->next = next; 2387 last = next; 2388 2389 next->next = NULL; 2390 next->wr_id = user_wr->wr_id; 2391 next->num_sge = user_wr->num_sge; 2392 2393 if (next->num_sge) { 2394 next->sg_list = (void *) next + 2395 ALIGN(sizeof *next, sizeof (struct ib_sge)); 2396 if (copy_from_user(next->sg_list, 2397 buf + wr_count * wqe_size + 2398 sg_ind * sizeof (struct ib_sge), 2399 next->num_sge * sizeof (struct ib_sge))) { 2400 ret = -EFAULT; 2401 goto err; 2402 } 2403 sg_ind += next->num_sge; 2404 } else 2405 next->sg_list = NULL; 2406 } 2407 2408 kfree(user_wr); 2409 return wr; 2410 2411 err: 2412 kfree(user_wr); 2413 2414 while (wr) { 2415 next = wr->next; 2416 kfree(wr); 2417 wr = next; 2418 } 2419 2420 return ERR_PTR(ret); 2421 } 2422 2423 ssize_t ib_uverbs_post_recv(struct ib_uverbs_file *file, 2424 const char __user *buf, int in_len, 2425 int out_len) 2426 { 2427 struct ib_uverbs_post_recv cmd; 2428 struct ib_uverbs_post_recv_resp resp; 2429 struct ib_recv_wr *wr, *next, *bad_wr; 2430 struct ib_qp *qp; 2431 ssize_t ret = -EINVAL; 2432 2433 if (copy_from_user(&cmd, buf, sizeof cmd)) 2434 return -EFAULT; 2435 2436 wr = ib_uverbs_unmarshall_recv(buf + sizeof cmd, 2437 in_len - sizeof cmd, cmd.wr_count, 2438 cmd.sge_count, cmd.wqe_size); 2439 if (IS_ERR(wr)) 2440 return PTR_ERR(wr); 2441 2442 qp = idr_read_qp(cmd.qp_handle, file->ucontext); 2443 if (!qp) 2444 goto out; 2445 2446 resp.bad_wr = 0; 2447 ret = qp->device->post_recv(qp->real_qp, wr, &bad_wr); 2448 2449 put_qp_read(qp); 2450 2451 if (ret) 2452 for (next = wr; next; next = next->next) { 2453 ++resp.bad_wr; 2454 if (next == bad_wr) 2455 break; 2456 } 2457 2458 if (copy_to_user((void __user *) (unsigned long) cmd.response, 2459 &resp, sizeof resp)) 2460 ret = -EFAULT; 2461 2462 out: 2463 while (wr) { 2464 next = wr->next; 2465 kfree(wr); 2466 wr = next; 2467 } 2468 2469 return ret ? ret : in_len; 2470 } 2471 2472 ssize_t ib_uverbs_post_srq_recv(struct ib_uverbs_file *file, 2473 const char __user *buf, int in_len, 2474 int out_len) 2475 { 2476 struct ib_uverbs_post_srq_recv cmd; 2477 struct ib_uverbs_post_srq_recv_resp resp; 2478 struct ib_recv_wr *wr, *next, *bad_wr; 2479 struct ib_srq *srq; 2480 ssize_t ret = -EINVAL; 2481 2482 if (copy_from_user(&cmd, buf, sizeof cmd)) 2483 return -EFAULT; 2484 2485 wr = ib_uverbs_unmarshall_recv(buf + sizeof cmd, 2486 in_len - sizeof cmd, cmd.wr_count, 2487 cmd.sge_count, cmd.wqe_size); 2488 if (IS_ERR(wr)) 2489 return PTR_ERR(wr); 2490 2491 srq = idr_read_srq(cmd.srq_handle, file->ucontext); 2492 if (!srq) 2493 goto out; 2494 2495 resp.bad_wr = 0; 2496 ret = srq->device->post_srq_recv(srq, wr, &bad_wr); 2497 2498 put_srq_read(srq); 2499 2500 if (ret) 2501 for (next = wr; next; next = next->next) { 2502 ++resp.bad_wr; 2503 if (next == bad_wr) 2504 break; 2505 } 2506 2507 if (copy_to_user((void __user *) (unsigned long) cmd.response, 2508 &resp, sizeof resp)) 2509 ret = -EFAULT; 2510 2511 out: 2512 while (wr) { 2513 next = wr->next; 2514 kfree(wr); 2515 wr = next; 2516 } 2517 2518 return ret ? ret : in_len; 2519 } 2520 2521 ssize_t ib_uverbs_create_ah(struct ib_uverbs_file *file, 2522 const char __user *buf, int in_len, 2523 int out_len) 2524 { 2525 struct ib_uverbs_create_ah cmd; 2526 struct ib_uverbs_create_ah_resp resp; 2527 struct ib_uobject *uobj; 2528 struct ib_pd *pd; 2529 struct ib_ah *ah; 2530 struct ib_ah_attr attr; 2531 int ret; 2532 2533 if (out_len < sizeof resp) 2534 return -ENOSPC; 2535 2536 if (copy_from_user(&cmd, buf, sizeof cmd)) 2537 return -EFAULT; 2538 2539 uobj = kmalloc(sizeof *uobj, GFP_KERNEL); 2540 if (!uobj) 2541 return -ENOMEM; 2542 2543 init_uobj(uobj, cmd.user_handle, file->ucontext, &ah_lock_class); 2544 down_write(&uobj->mutex); 2545 2546 pd = idr_read_pd(cmd.pd_handle, file->ucontext); 2547 if (!pd) { 2548 ret = -EINVAL; 2549 goto err; 2550 } 2551 2552 attr.dlid = cmd.attr.dlid; 2553 attr.sl = cmd.attr.sl; 2554 attr.src_path_bits = cmd.attr.src_path_bits; 2555 attr.static_rate = cmd.attr.static_rate; 2556 attr.ah_flags = cmd.attr.is_global ? IB_AH_GRH : 0; 2557 attr.port_num = cmd.attr.port_num; 2558 attr.grh.flow_label = cmd.attr.grh.flow_label; 2559 attr.grh.sgid_index = cmd.attr.grh.sgid_index; 2560 attr.grh.hop_limit = cmd.attr.grh.hop_limit; 2561 attr.grh.traffic_class = cmd.attr.grh.traffic_class; 2562 attr.vlan_id = 0; 2563 memset(&attr.dmac, 0, sizeof(attr.dmac)); 2564 memcpy(attr.grh.dgid.raw, cmd.attr.grh.dgid, 16); 2565 2566 ah = ib_create_ah(pd, &attr); 2567 if (IS_ERR(ah)) { 2568 ret = PTR_ERR(ah); 2569 goto err_put; 2570 } 2571 2572 ah->uobject = uobj; 2573 uobj->object = ah; 2574 2575 ret = idr_add_uobj(&ib_uverbs_ah_idr, uobj); 2576 if (ret) 2577 goto err_destroy; 2578 2579 resp.ah_handle = uobj->id; 2580 2581 if (copy_to_user((void __user *) (unsigned long) cmd.response, 2582 &resp, sizeof resp)) { 2583 ret = -EFAULT; 2584 goto err_copy; 2585 } 2586 2587 put_pd_read(pd); 2588 2589 mutex_lock(&file->mutex); 2590 list_add_tail(&uobj->list, &file->ucontext->ah_list); 2591 mutex_unlock(&file->mutex); 2592 2593 uobj->live = 1; 2594 2595 up_write(&uobj->mutex); 2596 2597 return in_len; 2598 2599 err_copy: 2600 idr_remove_uobj(&ib_uverbs_ah_idr, uobj); 2601 2602 err_destroy: 2603 ib_destroy_ah(ah); 2604 2605 err_put: 2606 put_pd_read(pd); 2607 2608 err: 2609 put_uobj_write(uobj); 2610 return ret; 2611 } 2612 2613 ssize_t ib_uverbs_destroy_ah(struct ib_uverbs_file *file, 2614 const char __user *buf, int in_len, int out_len) 2615 { 2616 struct ib_uverbs_destroy_ah cmd; 2617 struct ib_ah *ah; 2618 struct ib_uobject *uobj; 2619 int ret; 2620 2621 if (copy_from_user(&cmd, buf, sizeof cmd)) 2622 return -EFAULT; 2623 2624 uobj = idr_write_uobj(&ib_uverbs_ah_idr, cmd.ah_handle, file->ucontext); 2625 if (!uobj) 2626 return -EINVAL; 2627 ah = uobj->object; 2628 2629 ret = ib_destroy_ah(ah); 2630 if (!ret) 2631 uobj->live = 0; 2632 2633 put_uobj_write(uobj); 2634 2635 if (ret) 2636 return ret; 2637 2638 idr_remove_uobj(&ib_uverbs_ah_idr, uobj); 2639 2640 mutex_lock(&file->mutex); 2641 list_del(&uobj->list); 2642 mutex_unlock(&file->mutex); 2643 2644 put_uobj(uobj); 2645 2646 return in_len; 2647 } 2648 2649 ssize_t ib_uverbs_attach_mcast(struct ib_uverbs_file *file, 2650 const char __user *buf, int in_len, 2651 int out_len) 2652 { 2653 struct ib_uverbs_attach_mcast cmd; 2654 struct ib_qp *qp; 2655 struct ib_uqp_object *obj; 2656 struct ib_uverbs_mcast_entry *mcast; 2657 int ret; 2658 2659 if (copy_from_user(&cmd, buf, sizeof cmd)) 2660 return -EFAULT; 2661 2662 qp = idr_write_qp(cmd.qp_handle, file->ucontext); 2663 if (!qp) 2664 return -EINVAL; 2665 2666 obj = container_of(qp->uobject, struct ib_uqp_object, uevent.uobject); 2667 2668 list_for_each_entry(mcast, &obj->mcast_list, list) 2669 if (cmd.mlid == mcast->lid && 2670 !memcmp(cmd.gid, mcast->gid.raw, sizeof mcast->gid.raw)) { 2671 ret = 0; 2672 goto out_put; 2673 } 2674 2675 mcast = kmalloc(sizeof *mcast, GFP_KERNEL); 2676 if (!mcast) { 2677 ret = -ENOMEM; 2678 goto out_put; 2679 } 2680 2681 mcast->lid = cmd.mlid; 2682 memcpy(mcast->gid.raw, cmd.gid, sizeof mcast->gid.raw); 2683 2684 ret = ib_attach_mcast(qp, &mcast->gid, cmd.mlid); 2685 if (!ret) 2686 list_add_tail(&mcast->list, &obj->mcast_list); 2687 else 2688 kfree(mcast); 2689 2690 out_put: 2691 put_qp_write(qp); 2692 2693 return ret ? ret : in_len; 2694 } 2695 2696 ssize_t ib_uverbs_detach_mcast(struct ib_uverbs_file *file, 2697 const char __user *buf, int in_len, 2698 int out_len) 2699 { 2700 struct ib_uverbs_detach_mcast cmd; 2701 struct ib_uqp_object *obj; 2702 struct ib_qp *qp; 2703 struct ib_uverbs_mcast_entry *mcast; 2704 int ret = -EINVAL; 2705 2706 if (copy_from_user(&cmd, buf, sizeof cmd)) 2707 return -EFAULT; 2708 2709 qp = idr_write_qp(cmd.qp_handle, file->ucontext); 2710 if (!qp) 2711 return -EINVAL; 2712 2713 ret = ib_detach_mcast(qp, (union ib_gid *) cmd.gid, cmd.mlid); 2714 if (ret) 2715 goto out_put; 2716 2717 obj = container_of(qp->uobject, struct ib_uqp_object, uevent.uobject); 2718 2719 list_for_each_entry(mcast, &obj->mcast_list, list) 2720 if (cmd.mlid == mcast->lid && 2721 !memcmp(cmd.gid, mcast->gid.raw, sizeof mcast->gid.raw)) { 2722 list_del(&mcast->list); 2723 kfree(mcast); 2724 break; 2725 } 2726 2727 out_put: 2728 put_qp_write(qp); 2729 2730 return ret ? ret : in_len; 2731 } 2732 2733 static int kern_spec_to_ib_spec(struct ib_uverbs_flow_spec *kern_spec, 2734 union ib_flow_spec *ib_spec) 2735 { 2736 if (kern_spec->reserved) 2737 return -EINVAL; 2738 2739 ib_spec->type = kern_spec->type; 2740 2741 switch (ib_spec->type) { 2742 case IB_FLOW_SPEC_ETH: 2743 ib_spec->eth.size = sizeof(struct ib_flow_spec_eth); 2744 if (ib_spec->eth.size != kern_spec->eth.size) 2745 return -EINVAL; 2746 memcpy(&ib_spec->eth.val, &kern_spec->eth.val, 2747 sizeof(struct ib_flow_eth_filter)); 2748 memcpy(&ib_spec->eth.mask, &kern_spec->eth.mask, 2749 sizeof(struct ib_flow_eth_filter)); 2750 break; 2751 case IB_FLOW_SPEC_IPV4: 2752 ib_spec->ipv4.size = sizeof(struct ib_flow_spec_ipv4); 2753 if (ib_spec->ipv4.size != kern_spec->ipv4.size) 2754 return -EINVAL; 2755 memcpy(&ib_spec->ipv4.val, &kern_spec->ipv4.val, 2756 sizeof(struct ib_flow_ipv4_filter)); 2757 memcpy(&ib_spec->ipv4.mask, &kern_spec->ipv4.mask, 2758 sizeof(struct ib_flow_ipv4_filter)); 2759 break; 2760 case IB_FLOW_SPEC_TCP: 2761 case IB_FLOW_SPEC_UDP: 2762 ib_spec->tcp_udp.size = sizeof(struct ib_flow_spec_tcp_udp); 2763 if (ib_spec->tcp_udp.size != kern_spec->tcp_udp.size) 2764 return -EINVAL; 2765 memcpy(&ib_spec->tcp_udp.val, &kern_spec->tcp_udp.val, 2766 sizeof(struct ib_flow_tcp_udp_filter)); 2767 memcpy(&ib_spec->tcp_udp.mask, &kern_spec->tcp_udp.mask, 2768 sizeof(struct ib_flow_tcp_udp_filter)); 2769 break; 2770 default: 2771 return -EINVAL; 2772 } 2773 return 0; 2774 } 2775 2776 int ib_uverbs_ex_create_flow(struct ib_uverbs_file *file, 2777 struct ib_udata *ucore, 2778 struct ib_udata *uhw) 2779 { 2780 struct ib_uverbs_create_flow cmd; 2781 struct ib_uverbs_create_flow_resp resp; 2782 struct ib_uobject *uobj; 2783 struct ib_flow *flow_id; 2784 struct ib_uverbs_flow_attr *kern_flow_attr; 2785 struct ib_flow_attr *flow_attr; 2786 struct ib_qp *qp; 2787 int err = 0; 2788 void *kern_spec; 2789 void *ib_spec; 2790 int i; 2791 2792 if (ucore->inlen < sizeof(cmd)) 2793 return -EINVAL; 2794 2795 if (ucore->outlen < sizeof(resp)) 2796 return -ENOSPC; 2797 2798 err = ib_copy_from_udata(&cmd, ucore, sizeof(cmd)); 2799 if (err) 2800 return err; 2801 2802 ucore->inbuf += sizeof(cmd); 2803 ucore->inlen -= sizeof(cmd); 2804 2805 if (cmd.comp_mask) 2806 return -EINVAL; 2807 2808 if ((cmd.flow_attr.type == IB_FLOW_ATTR_SNIFFER && 2809 !capable(CAP_NET_ADMIN)) || !capable(CAP_NET_RAW)) 2810 return -EPERM; 2811 2812 if (cmd.flow_attr.num_of_specs > IB_FLOW_SPEC_SUPPORT_LAYERS) 2813 return -EINVAL; 2814 2815 if (cmd.flow_attr.size > ucore->inlen || 2816 cmd.flow_attr.size > 2817 (cmd.flow_attr.num_of_specs * sizeof(struct ib_uverbs_flow_spec))) 2818 return -EINVAL; 2819 2820 if (cmd.flow_attr.reserved[0] || 2821 cmd.flow_attr.reserved[1]) 2822 return -EINVAL; 2823 2824 if (cmd.flow_attr.num_of_specs) { 2825 kern_flow_attr = kmalloc(sizeof(*kern_flow_attr) + cmd.flow_attr.size, 2826 GFP_KERNEL); 2827 if (!kern_flow_attr) 2828 return -ENOMEM; 2829 2830 memcpy(kern_flow_attr, &cmd.flow_attr, sizeof(*kern_flow_attr)); 2831 err = ib_copy_from_udata(kern_flow_attr + 1, ucore, 2832 cmd.flow_attr.size); 2833 if (err) 2834 goto err_free_attr; 2835 } else { 2836 kern_flow_attr = &cmd.flow_attr; 2837 } 2838 2839 uobj = kmalloc(sizeof(*uobj), GFP_KERNEL); 2840 if (!uobj) { 2841 err = -ENOMEM; 2842 goto err_free_attr; 2843 } 2844 init_uobj(uobj, 0, file->ucontext, &rule_lock_class); 2845 down_write(&uobj->mutex); 2846 2847 qp = idr_read_qp(cmd.qp_handle, file->ucontext); 2848 if (!qp) { 2849 err = -EINVAL; 2850 goto err_uobj; 2851 } 2852 2853 flow_attr = kmalloc(sizeof(*flow_attr) + cmd.flow_attr.size, GFP_KERNEL); 2854 if (!flow_attr) { 2855 err = -ENOMEM; 2856 goto err_put; 2857 } 2858 2859 flow_attr->type = kern_flow_attr->type; 2860 flow_attr->priority = kern_flow_attr->priority; 2861 flow_attr->num_of_specs = kern_flow_attr->num_of_specs; 2862 flow_attr->port = kern_flow_attr->port; 2863 flow_attr->flags = kern_flow_attr->flags; 2864 flow_attr->size = sizeof(*flow_attr); 2865 2866 kern_spec = kern_flow_attr + 1; 2867 ib_spec = flow_attr + 1; 2868 for (i = 0; i < flow_attr->num_of_specs && 2869 cmd.flow_attr.size > offsetof(struct ib_uverbs_flow_spec, reserved) && 2870 cmd.flow_attr.size >= 2871 ((struct ib_uverbs_flow_spec *)kern_spec)->size; i++) { 2872 err = kern_spec_to_ib_spec(kern_spec, ib_spec); 2873 if (err) 2874 goto err_free; 2875 flow_attr->size += 2876 ((union ib_flow_spec *) ib_spec)->size; 2877 cmd.flow_attr.size -= ((struct ib_uverbs_flow_spec *)kern_spec)->size; 2878 kern_spec += ((struct ib_uverbs_flow_spec *) kern_spec)->size; 2879 ib_spec += ((union ib_flow_spec *) ib_spec)->size; 2880 } 2881 if (cmd.flow_attr.size || (i != flow_attr->num_of_specs)) { 2882 pr_warn("create flow failed, flow %d: %d bytes left from uverb cmd\n", 2883 i, cmd.flow_attr.size); 2884 err = -EINVAL; 2885 goto err_free; 2886 } 2887 flow_id = ib_create_flow(qp, flow_attr, IB_FLOW_DOMAIN_USER); 2888 if (IS_ERR(flow_id)) { 2889 err = PTR_ERR(flow_id); 2890 goto err_free; 2891 } 2892 flow_id->qp = qp; 2893 flow_id->uobject = uobj; 2894 uobj->object = flow_id; 2895 2896 err = idr_add_uobj(&ib_uverbs_rule_idr, uobj); 2897 if (err) 2898 goto destroy_flow; 2899 2900 memset(&resp, 0, sizeof(resp)); 2901 resp.flow_handle = uobj->id; 2902 2903 err = ib_copy_to_udata(ucore, 2904 &resp, sizeof(resp)); 2905 if (err) 2906 goto err_copy; 2907 2908 put_qp_read(qp); 2909 mutex_lock(&file->mutex); 2910 list_add_tail(&uobj->list, &file->ucontext->rule_list); 2911 mutex_unlock(&file->mutex); 2912 2913 uobj->live = 1; 2914 2915 up_write(&uobj->mutex); 2916 kfree(flow_attr); 2917 if (cmd.flow_attr.num_of_specs) 2918 kfree(kern_flow_attr); 2919 return 0; 2920 err_copy: 2921 idr_remove_uobj(&ib_uverbs_rule_idr, uobj); 2922 destroy_flow: 2923 ib_destroy_flow(flow_id); 2924 err_free: 2925 kfree(flow_attr); 2926 err_put: 2927 put_qp_read(qp); 2928 err_uobj: 2929 put_uobj_write(uobj); 2930 err_free_attr: 2931 if (cmd.flow_attr.num_of_specs) 2932 kfree(kern_flow_attr); 2933 return err; 2934 } 2935 2936 int ib_uverbs_ex_destroy_flow(struct ib_uverbs_file *file, 2937 struct ib_udata *ucore, 2938 struct ib_udata *uhw) 2939 { 2940 struct ib_uverbs_destroy_flow cmd; 2941 struct ib_flow *flow_id; 2942 struct ib_uobject *uobj; 2943 int ret; 2944 2945 if (ucore->inlen < sizeof(cmd)) 2946 return -EINVAL; 2947 2948 ret = ib_copy_from_udata(&cmd, ucore, sizeof(cmd)); 2949 if (ret) 2950 return ret; 2951 2952 if (cmd.comp_mask) 2953 return -EINVAL; 2954 2955 uobj = idr_write_uobj(&ib_uverbs_rule_idr, cmd.flow_handle, 2956 file->ucontext); 2957 if (!uobj) 2958 return -EINVAL; 2959 flow_id = uobj->object; 2960 2961 ret = ib_destroy_flow(flow_id); 2962 if (!ret) 2963 uobj->live = 0; 2964 2965 put_uobj_write(uobj); 2966 2967 idr_remove_uobj(&ib_uverbs_rule_idr, uobj); 2968 2969 mutex_lock(&file->mutex); 2970 list_del(&uobj->list); 2971 mutex_unlock(&file->mutex); 2972 2973 put_uobj(uobj); 2974 2975 return ret; 2976 } 2977 2978 static int __uverbs_create_xsrq(struct ib_uverbs_file *file, 2979 struct ib_uverbs_create_xsrq *cmd, 2980 struct ib_udata *udata) 2981 { 2982 struct ib_uverbs_create_srq_resp resp; 2983 struct ib_usrq_object *obj; 2984 struct ib_pd *pd; 2985 struct ib_srq *srq; 2986 struct ib_uobject *uninitialized_var(xrcd_uobj); 2987 struct ib_srq_init_attr attr; 2988 int ret; 2989 2990 obj = kmalloc(sizeof *obj, GFP_KERNEL); 2991 if (!obj) 2992 return -ENOMEM; 2993 2994 init_uobj(&obj->uevent.uobject, cmd->user_handle, file->ucontext, &srq_lock_class); 2995 down_write(&obj->uevent.uobject.mutex); 2996 2997 if (cmd->srq_type == IB_SRQT_XRC) { 2998 attr.ext.xrc.xrcd = idr_read_xrcd(cmd->xrcd_handle, file->ucontext, &xrcd_uobj); 2999 if (!attr.ext.xrc.xrcd) { 3000 ret = -EINVAL; 3001 goto err; 3002 } 3003 3004 obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object, uobject); 3005 atomic_inc(&obj->uxrcd->refcnt); 3006 3007 attr.ext.xrc.cq = idr_read_cq(cmd->cq_handle, file->ucontext, 0); 3008 if (!attr.ext.xrc.cq) { 3009 ret = -EINVAL; 3010 goto err_put_xrcd; 3011 } 3012 } 3013 3014 pd = idr_read_pd(cmd->pd_handle, file->ucontext); 3015 if (!pd) { 3016 ret = -EINVAL; 3017 goto err_put_cq; 3018 } 3019 3020 attr.event_handler = ib_uverbs_srq_event_handler; 3021 attr.srq_context = file; 3022 attr.srq_type = cmd->srq_type; 3023 attr.attr.max_wr = cmd->max_wr; 3024 attr.attr.max_sge = cmd->max_sge; 3025 attr.attr.srq_limit = cmd->srq_limit; 3026 3027 obj->uevent.events_reported = 0; 3028 INIT_LIST_HEAD(&obj->uevent.event_list); 3029 3030 srq = pd->device->create_srq(pd, &attr, udata); 3031 if (IS_ERR(srq)) { 3032 ret = PTR_ERR(srq); 3033 goto err_put; 3034 } 3035 3036 srq->device = pd->device; 3037 srq->pd = pd; 3038 srq->srq_type = cmd->srq_type; 3039 srq->uobject = &obj->uevent.uobject; 3040 srq->event_handler = attr.event_handler; 3041 srq->srq_context = attr.srq_context; 3042 3043 if (cmd->srq_type == IB_SRQT_XRC) { 3044 srq->ext.xrc.cq = attr.ext.xrc.cq; 3045 srq->ext.xrc.xrcd = attr.ext.xrc.xrcd; 3046 atomic_inc(&attr.ext.xrc.cq->usecnt); 3047 atomic_inc(&attr.ext.xrc.xrcd->usecnt); 3048 } 3049 3050 atomic_inc(&pd->usecnt); 3051 atomic_set(&srq->usecnt, 0); 3052 3053 obj->uevent.uobject.object = srq; 3054 ret = idr_add_uobj(&ib_uverbs_srq_idr, &obj->uevent.uobject); 3055 if (ret) 3056 goto err_destroy; 3057 3058 memset(&resp, 0, sizeof resp); 3059 resp.srq_handle = obj->uevent.uobject.id; 3060 resp.max_wr = attr.attr.max_wr; 3061 resp.max_sge = attr.attr.max_sge; 3062 if (cmd->srq_type == IB_SRQT_XRC) 3063 resp.srqn = srq->ext.xrc.srq_num; 3064 3065 if (copy_to_user((void __user *) (unsigned long) cmd->response, 3066 &resp, sizeof resp)) { 3067 ret = -EFAULT; 3068 goto err_copy; 3069 } 3070 3071 if (cmd->srq_type == IB_SRQT_XRC) { 3072 put_uobj_read(xrcd_uobj); 3073 put_cq_read(attr.ext.xrc.cq); 3074 } 3075 put_pd_read(pd); 3076 3077 mutex_lock(&file->mutex); 3078 list_add_tail(&obj->uevent.uobject.list, &file->ucontext->srq_list); 3079 mutex_unlock(&file->mutex); 3080 3081 obj->uevent.uobject.live = 1; 3082 3083 up_write(&obj->uevent.uobject.mutex); 3084 3085 return 0; 3086 3087 err_copy: 3088 idr_remove_uobj(&ib_uverbs_srq_idr, &obj->uevent.uobject); 3089 3090 err_destroy: 3091 ib_destroy_srq(srq); 3092 3093 err_put: 3094 put_pd_read(pd); 3095 3096 err_put_cq: 3097 if (cmd->srq_type == IB_SRQT_XRC) 3098 put_cq_read(attr.ext.xrc.cq); 3099 3100 err_put_xrcd: 3101 if (cmd->srq_type == IB_SRQT_XRC) { 3102 atomic_dec(&obj->uxrcd->refcnt); 3103 put_uobj_read(xrcd_uobj); 3104 } 3105 3106 err: 3107 put_uobj_write(&obj->uevent.uobject); 3108 return ret; 3109 } 3110 3111 ssize_t ib_uverbs_create_srq(struct ib_uverbs_file *file, 3112 const char __user *buf, int in_len, 3113 int out_len) 3114 { 3115 struct ib_uverbs_create_srq cmd; 3116 struct ib_uverbs_create_xsrq xcmd; 3117 struct ib_uverbs_create_srq_resp resp; 3118 struct ib_udata udata; 3119 int ret; 3120 3121 if (out_len < sizeof resp) 3122 return -ENOSPC; 3123 3124 if (copy_from_user(&cmd, buf, sizeof cmd)) 3125 return -EFAULT; 3126 3127 xcmd.response = cmd.response; 3128 xcmd.user_handle = cmd.user_handle; 3129 xcmd.srq_type = IB_SRQT_BASIC; 3130 xcmd.pd_handle = cmd.pd_handle; 3131 xcmd.max_wr = cmd.max_wr; 3132 xcmd.max_sge = cmd.max_sge; 3133 xcmd.srq_limit = cmd.srq_limit; 3134 3135 INIT_UDATA(&udata, buf + sizeof cmd, 3136 (unsigned long) cmd.response + sizeof resp, 3137 in_len - sizeof cmd, out_len - sizeof resp); 3138 3139 ret = __uverbs_create_xsrq(file, &xcmd, &udata); 3140 if (ret) 3141 return ret; 3142 3143 return in_len; 3144 } 3145 3146 ssize_t ib_uverbs_create_xsrq(struct ib_uverbs_file *file, 3147 const char __user *buf, int in_len, int out_len) 3148 { 3149 struct ib_uverbs_create_xsrq cmd; 3150 struct ib_uverbs_create_srq_resp resp; 3151 struct ib_udata udata; 3152 int ret; 3153 3154 if (out_len < sizeof resp) 3155 return -ENOSPC; 3156 3157 if (copy_from_user(&cmd, buf, sizeof cmd)) 3158 return -EFAULT; 3159 3160 INIT_UDATA(&udata, buf + sizeof cmd, 3161 (unsigned long) cmd.response + sizeof resp, 3162 in_len - sizeof cmd, out_len - sizeof resp); 3163 3164 ret = __uverbs_create_xsrq(file, &cmd, &udata); 3165 if (ret) 3166 return ret; 3167 3168 return in_len; 3169 } 3170 3171 ssize_t ib_uverbs_modify_srq(struct ib_uverbs_file *file, 3172 const char __user *buf, int in_len, 3173 int out_len) 3174 { 3175 struct ib_uverbs_modify_srq cmd; 3176 struct ib_udata udata; 3177 struct ib_srq *srq; 3178 struct ib_srq_attr attr; 3179 int ret; 3180 3181 if (copy_from_user(&cmd, buf, sizeof cmd)) 3182 return -EFAULT; 3183 3184 INIT_UDATA(&udata, buf + sizeof cmd, NULL, in_len - sizeof cmd, 3185 out_len); 3186 3187 srq = idr_read_srq(cmd.srq_handle, file->ucontext); 3188 if (!srq) 3189 return -EINVAL; 3190 3191 attr.max_wr = cmd.max_wr; 3192 attr.srq_limit = cmd.srq_limit; 3193 3194 ret = srq->device->modify_srq(srq, &attr, cmd.attr_mask, &udata); 3195 3196 put_srq_read(srq); 3197 3198 return ret ? ret : in_len; 3199 } 3200 3201 ssize_t ib_uverbs_query_srq(struct ib_uverbs_file *file, 3202 const char __user *buf, 3203 int in_len, int out_len) 3204 { 3205 struct ib_uverbs_query_srq cmd; 3206 struct ib_uverbs_query_srq_resp resp; 3207 struct ib_srq_attr attr; 3208 struct ib_srq *srq; 3209 int ret; 3210 3211 if (out_len < sizeof resp) 3212 return -ENOSPC; 3213 3214 if (copy_from_user(&cmd, buf, sizeof cmd)) 3215 return -EFAULT; 3216 3217 srq = idr_read_srq(cmd.srq_handle, file->ucontext); 3218 if (!srq) 3219 return -EINVAL; 3220 3221 ret = ib_query_srq(srq, &attr); 3222 3223 put_srq_read(srq); 3224 3225 if (ret) 3226 return ret; 3227 3228 memset(&resp, 0, sizeof resp); 3229 3230 resp.max_wr = attr.max_wr; 3231 resp.max_sge = attr.max_sge; 3232 resp.srq_limit = attr.srq_limit; 3233 3234 if (copy_to_user((void __user *) (unsigned long) cmd.response, 3235 &resp, sizeof resp)) 3236 return -EFAULT; 3237 3238 return in_len; 3239 } 3240 3241 ssize_t ib_uverbs_destroy_srq(struct ib_uverbs_file *file, 3242 const char __user *buf, int in_len, 3243 int out_len) 3244 { 3245 struct ib_uverbs_destroy_srq cmd; 3246 struct ib_uverbs_destroy_srq_resp resp; 3247 struct ib_uobject *uobj; 3248 struct ib_srq *srq; 3249 struct ib_uevent_object *obj; 3250 int ret = -EINVAL; 3251 struct ib_usrq_object *us; 3252 enum ib_srq_type srq_type; 3253 3254 if (copy_from_user(&cmd, buf, sizeof cmd)) 3255 return -EFAULT; 3256 3257 uobj = idr_write_uobj(&ib_uverbs_srq_idr, cmd.srq_handle, file->ucontext); 3258 if (!uobj) 3259 return -EINVAL; 3260 srq = uobj->object; 3261 obj = container_of(uobj, struct ib_uevent_object, uobject); 3262 srq_type = srq->srq_type; 3263 3264 ret = ib_destroy_srq(srq); 3265 if (!ret) 3266 uobj->live = 0; 3267 3268 put_uobj_write(uobj); 3269 3270 if (ret) 3271 return ret; 3272 3273 if (srq_type == IB_SRQT_XRC) { 3274 us = container_of(obj, struct ib_usrq_object, uevent); 3275 atomic_dec(&us->uxrcd->refcnt); 3276 } 3277 3278 idr_remove_uobj(&ib_uverbs_srq_idr, uobj); 3279 3280 mutex_lock(&file->mutex); 3281 list_del(&uobj->list); 3282 mutex_unlock(&file->mutex); 3283 3284 ib_uverbs_release_uevent(file, obj); 3285 3286 memset(&resp, 0, sizeof resp); 3287 resp.events_reported = obj->events_reported; 3288 3289 put_uobj(uobj); 3290 3291 if (copy_to_user((void __user *) (unsigned long) cmd.response, 3292 &resp, sizeof resp)) 3293 ret = -EFAULT; 3294 3295 return ret ? ret : in_len; 3296 } 3297 3298 int ib_uverbs_ex_query_device(struct ib_uverbs_file *file, 3299 struct ib_udata *ucore, 3300 struct ib_udata *uhw) 3301 { 3302 struct ib_uverbs_ex_query_device_resp resp; 3303 struct ib_uverbs_ex_query_device cmd; 3304 struct ib_device_attr attr; 3305 struct ib_device *device; 3306 int err; 3307 3308 device = file->device->ib_dev; 3309 if (ucore->inlen < sizeof(cmd)) 3310 return -EINVAL; 3311 3312 err = ib_copy_from_udata(&cmd, ucore, sizeof(cmd)); 3313 if (err) 3314 return err; 3315 3316 if (cmd.comp_mask) 3317 return -EINVAL; 3318 3319 if (cmd.reserved) 3320 return -EINVAL; 3321 3322 resp.response_length = offsetof(typeof(resp), odp_caps); 3323 3324 if (ucore->outlen < resp.response_length) 3325 return -ENOSPC; 3326 3327 err = device->query_device(device, &attr); 3328 if (err) 3329 return err; 3330 3331 copy_query_dev_fields(file, &resp.base, &attr); 3332 resp.comp_mask = 0; 3333 3334 if (ucore->outlen < resp.response_length + sizeof(resp.odp_caps)) 3335 goto end; 3336 3337 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING 3338 resp.odp_caps.general_caps = attr.odp_caps.general_caps; 3339 resp.odp_caps.per_transport_caps.rc_odp_caps = 3340 attr.odp_caps.per_transport_caps.rc_odp_caps; 3341 resp.odp_caps.per_transport_caps.uc_odp_caps = 3342 attr.odp_caps.per_transport_caps.uc_odp_caps; 3343 resp.odp_caps.per_transport_caps.ud_odp_caps = 3344 attr.odp_caps.per_transport_caps.ud_odp_caps; 3345 resp.odp_caps.reserved = 0; 3346 #else 3347 memset(&resp.odp_caps, 0, sizeof(resp.odp_caps)); 3348 #endif 3349 resp.response_length += sizeof(resp.odp_caps); 3350 3351 end: 3352 err = ib_copy_to_udata(ucore, &resp, resp.response_length); 3353 if (err) 3354 return err; 3355 3356 return 0; 3357 } 3358