1 /* 2 * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. 3 * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved. 4 * Copyright (c) 2005, 2006 Cisco Systems. All rights reserved. 5 * Copyright (c) 2005 Mellanox Technologies. All rights reserved. 6 * Copyright (c) 2004 Voltaire, Inc. All rights reserved. 7 * 8 * This software is available to you under a choice of one of two 9 * licenses. You may choose to be licensed under the terms of the GNU 10 * General Public License (GPL) Version 2, available from the file 11 * COPYING in the main directory of this source tree, or the 12 * OpenIB.org BSD license below: 13 * 14 * Redistribution and use in source and binary forms, with or 15 * without modification, are permitted provided that the following 16 * conditions are met: 17 * 18 * - Redistributions of source code must retain the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer. 21 * 22 * - Redistributions in binary form must reproduce the above 23 * copyright notice, this list of conditions and the following 24 * disclaimer in the documentation and/or other materials 25 * provided with the distribution. 26 * 27 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 28 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 29 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 30 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 31 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 32 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 33 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 34 * SOFTWARE. 35 */ 36 37 #include <rdma/ib_smi.h> 38 #include <rdma/ib_user_verbs.h> 39 #include <rdma/iter.h> 40 #include <rdma/uverbs_ioctl.h> 41 42 #include <linux/sched.h> 43 #include <linux/slab.h> 44 #include <linux/stat.h> 45 #include <linux/mm.h> 46 #include <linux/export.h> 47 48 #include "mthca_dev.h" 49 #include "mthca_cmd.h" 50 #include <rdma/mthca-abi.h> 51 #include "mthca_memfree.h" 52 53 static int mthca_query_device(struct ib_device *ibdev, struct ib_device_attr *props, 54 struct ib_udata *uhw) 55 { 56 struct ib_smp *in_mad; 57 struct ib_smp *out_mad; 58 int err; 59 struct mthca_dev *mdev = to_mdev(ibdev); 60 61 err = ib_no_udata_io(uhw); 62 if (err) 63 return err; 64 65 in_mad = kzalloc_obj(*in_mad); 66 out_mad = kmalloc_obj(*out_mad); 67 if (!in_mad || !out_mad) { 68 err = -ENOMEM; 69 goto out; 70 } 71 72 props->fw_ver = mdev->fw_ver; 73 74 ib_init_query_mad(in_mad); 75 in_mad->attr_id = IB_SMP_ATTR_NODE_INFO; 76 77 err = mthca_MAD_IFC(mdev, 1, 1, 78 1, NULL, NULL, in_mad, out_mad); 79 if (err) 80 goto out; 81 82 props->device_cap_flags = mdev->device_cap_flags; 83 props->vendor_id = be32_to_cpup((__be32 *) (out_mad->data + 36)) & 84 0xffffff; 85 props->vendor_part_id = be16_to_cpup((__be16 *) (out_mad->data + 30)); 86 props->hw_ver = be32_to_cpup((__be32 *) (out_mad->data + 32)); 87 memcpy(&props->sys_image_guid, out_mad->data + 4, 8); 88 89 props->max_mr_size = ~0ull; 90 props->page_size_cap = mdev->limits.page_size_cap; 91 props->max_qp = mdev->limits.num_qps - mdev->limits.reserved_qps; 92 props->max_qp_wr = mdev->limits.max_wqes; 93 props->max_send_sge = mdev->limits.max_sg; 94 props->max_recv_sge = mdev->limits.max_sg; 95 props->max_sge_rd = mdev->limits.max_sg; 96 props->max_cq = mdev->limits.num_cqs - mdev->limits.reserved_cqs; 97 props->max_cqe = mdev->limits.max_cqes; 98 props->max_mr = mdev->limits.num_mpts - mdev->limits.reserved_mrws; 99 props->max_pd = mdev->limits.num_pds - mdev->limits.reserved_pds; 100 props->max_qp_rd_atom = 1 << mdev->qp_table.rdb_shift; 101 props->max_qp_init_rd_atom = mdev->limits.max_qp_init_rdma; 102 props->max_res_rd_atom = props->max_qp_rd_atom * props->max_qp; 103 props->max_srq = mdev->limits.num_srqs - mdev->limits.reserved_srqs; 104 props->max_srq_wr = mdev->limits.max_srq_wqes; 105 props->max_srq_sge = mdev->limits.max_srq_sge; 106 props->local_ca_ack_delay = mdev->limits.local_ca_ack_delay; 107 props->atomic_cap = mdev->limits.flags & DEV_LIM_FLAG_ATOMIC ? 108 IB_ATOMIC_HCA : IB_ATOMIC_NONE; 109 props->max_pkeys = mdev->limits.pkey_table_len; 110 props->max_mcast_grp = mdev->limits.num_mgms + mdev->limits.num_amgms; 111 props->max_mcast_qp_attach = MTHCA_QP_PER_MGM; 112 props->max_total_mcast_qp_attach = props->max_mcast_qp_attach * 113 props->max_mcast_grp; 114 115 out: 116 kfree(in_mad); 117 kfree(out_mad); 118 return err; 119 } 120 121 static int mthca_query_port(struct ib_device *ibdev, 122 u32 port, struct ib_port_attr *props) 123 { 124 struct ib_smp *in_mad; 125 struct ib_smp *out_mad; 126 int err = -ENOMEM; 127 128 in_mad = kzalloc_obj(*in_mad); 129 out_mad = kmalloc_obj(*out_mad); 130 if (!in_mad || !out_mad) 131 goto out; 132 133 /* props being zeroed by the caller, avoid zeroing it here */ 134 135 ib_init_query_mad(in_mad); 136 in_mad->attr_id = IB_SMP_ATTR_PORT_INFO; 137 in_mad->attr_mod = cpu_to_be32(port); 138 139 err = mthca_MAD_IFC(to_mdev(ibdev), 1, 1, 140 port, NULL, NULL, in_mad, out_mad); 141 if (err) 142 goto out; 143 144 props->lid = be16_to_cpup((__be16 *) (out_mad->data + 16)); 145 props->lmc = out_mad->data[34] & 0x7; 146 props->sm_lid = be16_to_cpup((__be16 *) (out_mad->data + 18)); 147 props->sm_sl = out_mad->data[36] & 0xf; 148 props->state = out_mad->data[32] & 0xf; 149 props->phys_state = out_mad->data[33] >> 4; 150 props->port_cap_flags = be32_to_cpup((__be32 *) (out_mad->data + 20)); 151 props->gid_tbl_len = to_mdev(ibdev)->limits.gid_table_len; 152 props->max_msg_sz = 0x80000000; 153 props->pkey_tbl_len = to_mdev(ibdev)->limits.pkey_table_len; 154 props->bad_pkey_cntr = be16_to_cpup((__be16 *) (out_mad->data + 46)); 155 props->qkey_viol_cntr = be16_to_cpup((__be16 *) (out_mad->data + 48)); 156 props->active_width = out_mad->data[31] & 0xf; 157 props->active_speed = out_mad->data[35] >> 4; 158 props->max_mtu = out_mad->data[41] & 0xf; 159 props->active_mtu = out_mad->data[36] >> 4; 160 props->subnet_timeout = out_mad->data[51] & 0x1f; 161 props->max_vl_num = out_mad->data[37] >> 4; 162 props->init_type_reply = out_mad->data[41] >> 4; 163 164 out: 165 kfree(in_mad); 166 kfree(out_mad); 167 return err; 168 } 169 170 static int mthca_modify_device(struct ib_device *ibdev, 171 int mask, 172 struct ib_device_modify *props) 173 { 174 if (mask & ~IB_DEVICE_MODIFY_NODE_DESC) 175 return -EOPNOTSUPP; 176 177 if (mask & IB_DEVICE_MODIFY_NODE_DESC) { 178 if (mutex_lock_interruptible(&to_mdev(ibdev)->cap_mask_mutex)) 179 return -ERESTARTSYS; 180 memcpy(ibdev->node_desc, props->node_desc, 181 IB_DEVICE_NODE_DESC_MAX); 182 mutex_unlock(&to_mdev(ibdev)->cap_mask_mutex); 183 } 184 185 return 0; 186 } 187 188 static int mthca_modify_port(struct ib_device *ibdev, 189 u32 port, int port_modify_mask, 190 struct ib_port_modify *props) 191 { 192 struct mthca_set_ib_param set_ib; 193 struct ib_port_attr attr; 194 int err; 195 196 if (mutex_lock_interruptible(&to_mdev(ibdev)->cap_mask_mutex)) 197 return -ERESTARTSYS; 198 199 err = ib_query_port(ibdev, port, &attr); 200 if (err) 201 goto out; 202 203 set_ib.set_si_guid = 0; 204 set_ib.reset_qkey_viol = !!(port_modify_mask & IB_PORT_RESET_QKEY_CNTR); 205 206 set_ib.cap_mask = (attr.port_cap_flags | props->set_port_cap_mask) & 207 ~props->clr_port_cap_mask; 208 209 err = mthca_SET_IB(to_mdev(ibdev), &set_ib, port); 210 if (err) 211 goto out; 212 out: 213 mutex_unlock(&to_mdev(ibdev)->cap_mask_mutex); 214 return err; 215 } 216 217 static int mthca_query_pkey(struct ib_device *ibdev, 218 u32 port, u16 index, u16 *pkey) 219 { 220 struct ib_smp *in_mad; 221 struct ib_smp *out_mad; 222 int err = -ENOMEM; 223 224 in_mad = kzalloc_obj(*in_mad); 225 out_mad = kmalloc_obj(*out_mad); 226 if (!in_mad || !out_mad) 227 goto out; 228 229 ib_init_query_mad(in_mad); 230 in_mad->attr_id = IB_SMP_ATTR_PKEY_TABLE; 231 in_mad->attr_mod = cpu_to_be32(index / 32); 232 233 err = mthca_MAD_IFC(to_mdev(ibdev), 1, 1, 234 port, NULL, NULL, in_mad, out_mad); 235 if (err) 236 goto out; 237 238 *pkey = be16_to_cpu(((__be16 *) out_mad->data)[index % 32]); 239 240 out: 241 kfree(in_mad); 242 kfree(out_mad); 243 return err; 244 } 245 246 static int mthca_query_gid(struct ib_device *ibdev, u32 port, 247 int index, union ib_gid *gid) 248 { 249 struct ib_smp *in_mad; 250 struct ib_smp *out_mad; 251 int err = -ENOMEM; 252 253 in_mad = kzalloc_obj(*in_mad); 254 out_mad = kmalloc_obj(*out_mad); 255 if (!in_mad || !out_mad) 256 goto out; 257 258 ib_init_query_mad(in_mad); 259 in_mad->attr_id = IB_SMP_ATTR_PORT_INFO; 260 in_mad->attr_mod = cpu_to_be32(port); 261 262 err = mthca_MAD_IFC(to_mdev(ibdev), 1, 1, 263 port, NULL, NULL, in_mad, out_mad); 264 if (err) 265 goto out; 266 267 memcpy(gid->raw, out_mad->data + 8, 8); 268 269 ib_init_query_mad(in_mad); 270 in_mad->attr_id = IB_SMP_ATTR_GUID_INFO; 271 in_mad->attr_mod = cpu_to_be32(index / 8); 272 273 err = mthca_MAD_IFC(to_mdev(ibdev), 1, 1, 274 port, NULL, NULL, in_mad, out_mad); 275 if (err) 276 goto out; 277 278 memcpy(gid->raw + 8, out_mad->data + (index % 8) * 8, 8); 279 280 out: 281 kfree(in_mad); 282 kfree(out_mad); 283 return err; 284 } 285 286 static int mthca_alloc_ucontext(struct ib_ucontext *uctx, 287 struct ib_udata *udata) 288 { 289 struct ib_device *ibdev = uctx->device; 290 struct mthca_alloc_ucontext_resp uresp = {}; 291 struct mthca_ucontext *context = to_mucontext(uctx); 292 int err; 293 294 if (!(to_mdev(ibdev)->active)) 295 return -EAGAIN; 296 297 uresp.qp_tab_size = to_mdev(ibdev)->limits.num_qps; 298 if (mthca_is_memfree(to_mdev(ibdev))) 299 uresp.uarc_size = to_mdev(ibdev)->uar_table.uarc_size; 300 else 301 uresp.uarc_size = 0; 302 303 err = mthca_uar_alloc(to_mdev(ibdev), &context->uar); 304 if (err) 305 return err; 306 307 context->db_tab = mthca_init_user_db_tab(to_mdev(ibdev)); 308 if (IS_ERR(context->db_tab)) { 309 err = PTR_ERR(context->db_tab); 310 mthca_uar_free(to_mdev(ibdev), &context->uar); 311 return err; 312 } 313 314 err = ib_respond_udata(udata, uresp); 315 if (err) { 316 mthca_cleanup_user_db_tab(to_mdev(ibdev), &context->uar, context->db_tab); 317 mthca_uar_free(to_mdev(ibdev), &context->uar); 318 return err; 319 } 320 321 context->reg_mr_warned = 0; 322 323 return 0; 324 } 325 326 static void mthca_dealloc_ucontext(struct ib_ucontext *context) 327 { 328 mthca_cleanup_user_db_tab(to_mdev(context->device), &to_mucontext(context)->uar, 329 to_mucontext(context)->db_tab); 330 mthca_uar_free(to_mdev(context->device), &to_mucontext(context)->uar); 331 } 332 333 static int mthca_mmap_uar(struct ib_ucontext *context, 334 struct vm_area_struct *vma) 335 { 336 if (vma->vm_end - vma->vm_start != PAGE_SIZE) 337 return -EINVAL; 338 339 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); 340 341 if (io_remap_pfn_range(vma, vma->vm_start, 342 to_mucontext(context)->uar.pfn, 343 PAGE_SIZE, vma->vm_page_prot)) 344 return -EAGAIN; 345 346 return 0; 347 } 348 349 static int mthca_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 350 { 351 struct ib_device *ibdev = ibpd->device; 352 struct mthca_pd *pd = to_mpd(ibpd); 353 int err; 354 355 err = mthca_pd_alloc(to_mdev(ibdev), !udata, pd); 356 if (err) 357 return err; 358 359 if (udata) { 360 struct mthca_alloc_pd_resp uresp = { .pdn = pd->pd_num }; 361 362 err = ib_respond_udata(udata, uresp); 363 if (err) { 364 mthca_pd_free(to_mdev(ibdev), pd); 365 return err; 366 } 367 } 368 369 return 0; 370 } 371 372 static int mthca_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata) 373 { 374 mthca_pd_free(to_mdev(pd->device), to_mpd(pd)); 375 return 0; 376 } 377 378 static int mthca_ah_create(struct ib_ah *ibah, 379 struct rdma_ah_init_attr *init_attr, 380 struct ib_udata *udata) 381 382 { 383 struct mthca_ah *ah = to_mah(ibah); 384 385 return mthca_create_ah(to_mdev(ibah->device), to_mpd(ibah->pd), 386 init_attr->ah_attr, ah); 387 } 388 389 static int mthca_ah_destroy(struct ib_ah *ah, u32 flags) 390 { 391 mthca_destroy_ah(to_mdev(ah->device), to_mah(ah)); 392 return 0; 393 } 394 395 static int mthca_create_srq(struct ib_srq *ibsrq, 396 struct ib_srq_init_attr *init_attr, 397 struct ib_udata *udata) 398 { 399 struct mthca_create_srq ucmd; 400 struct mthca_ucontext *context = rdma_udata_to_drv_context( 401 udata, struct mthca_ucontext, ibucontext); 402 struct mthca_srq *srq = to_msrq(ibsrq); 403 int err; 404 405 if (init_attr->srq_type != IB_SRQT_BASIC) 406 return -EOPNOTSUPP; 407 408 if (udata) { 409 err = ib_copy_validate_udata_in(udata, ucmd, db_page); 410 if (err) 411 return err; 412 413 err = mthca_map_user_db(to_mdev(ibsrq->device), &context->uar, 414 context->db_tab, ucmd.db_index, 415 ucmd.db_page); 416 417 if (err) 418 return err; 419 420 srq->mr.ibmr.lkey = ucmd.lkey; 421 srq->db_index = ucmd.db_index; 422 } 423 424 err = mthca_alloc_srq(to_mdev(ibsrq->device), to_mpd(ibsrq->pd), 425 &init_attr->attr, srq, udata); 426 427 if (err && udata) 428 mthca_unmap_user_db(to_mdev(ibsrq->device), &context->uar, 429 context->db_tab, ucmd.db_index); 430 431 if (err) 432 return err; 433 434 if (context) { 435 struct mthca_create_srq_resp uresp = { .srqn = srq->srqn }; 436 437 err = ib_respond_udata(udata, uresp); 438 if (err) { 439 mthca_free_srq(to_mdev(ibsrq->device), srq); 440 mthca_unmap_user_db(to_mdev(ibsrq->device), 441 &context->uar, context->db_tab, 442 ucmd.db_index); 443 return err; 444 } 445 } 446 447 return 0; 448 } 449 450 static int mthca_destroy_srq(struct ib_srq *srq, struct ib_udata *udata) 451 { 452 mthca_free_srq(to_mdev(srq->device), to_msrq(srq)); 453 if (udata) { 454 struct mthca_ucontext *context = 455 rdma_udata_to_drv_context( 456 udata, 457 struct mthca_ucontext, 458 ibucontext); 459 460 mthca_unmap_user_db(to_mdev(srq->device), &context->uar, 461 context->db_tab, to_msrq(srq)->db_index); 462 } 463 return 0; 464 } 465 466 static int mthca_create_qp(struct ib_qp *ibqp, 467 struct ib_qp_init_attr *init_attr, 468 struct ib_udata *udata) 469 { 470 struct mthca_ucontext *context = rdma_udata_to_drv_context( 471 udata, struct mthca_ucontext, ibucontext); 472 struct mthca_create_qp ucmd; 473 struct mthca_qp *qp = to_mqp(ibqp); 474 struct mthca_dev *dev = to_mdev(ibqp->device); 475 int err; 476 477 if (init_attr->create_flags) 478 return -EOPNOTSUPP; 479 480 switch (init_attr->qp_type) { 481 case IB_QPT_RC: 482 case IB_QPT_UC: 483 case IB_QPT_UD: 484 { 485 if (udata) { 486 err = ib_copy_validate_udata_in(udata, ucmd, rq_db_index); 487 if (err) 488 return err; 489 490 err = mthca_map_user_db(dev, &context->uar, 491 context->db_tab, 492 ucmd.sq_db_index, 493 ucmd.sq_db_page); 494 if (err) 495 return err; 496 497 err = mthca_map_user_db(dev, &context->uar, 498 context->db_tab, 499 ucmd.rq_db_index, 500 ucmd.rq_db_page); 501 if (err) { 502 mthca_unmap_user_db(dev, &context->uar, 503 context->db_tab, 504 ucmd.sq_db_index); 505 return err; 506 } 507 508 qp->mr.ibmr.lkey = ucmd.lkey; 509 qp->sq.db_index = ucmd.sq_db_index; 510 qp->rq.db_index = ucmd.rq_db_index; 511 } 512 513 err = mthca_alloc_qp(dev, to_mpd(ibqp->pd), 514 to_mcq(init_attr->send_cq), 515 to_mcq(init_attr->recv_cq), 516 init_attr->qp_type, init_attr->sq_sig_type, 517 &init_attr->cap, qp, udata); 518 519 if (err && udata) { 520 mthca_unmap_user_db(dev, &context->uar, context->db_tab, 521 ucmd.sq_db_index); 522 mthca_unmap_user_db(dev, &context->uar, context->db_tab, 523 ucmd.rq_db_index); 524 } 525 526 qp->ibqp.qp_num = qp->qpn; 527 break; 528 } 529 case IB_QPT_SMI: 530 case IB_QPT_GSI: 531 { 532 qp->sqp = kzalloc_obj(struct mthca_sqp); 533 if (!qp->sqp) 534 return -ENOMEM; 535 536 qp->ibqp.qp_num = init_attr->qp_type == IB_QPT_SMI ? 0 : 1; 537 538 err = mthca_alloc_sqp(dev, to_mpd(ibqp->pd), 539 to_mcq(init_attr->send_cq), 540 to_mcq(init_attr->recv_cq), 541 init_attr->sq_sig_type, &init_attr->cap, 542 qp->ibqp.qp_num, init_attr->port_num, qp, 543 udata); 544 break; 545 } 546 default: 547 /* Don't support raw QPs */ 548 return -EOPNOTSUPP; 549 } 550 551 if (err) { 552 kfree(qp->sqp); 553 return err; 554 } 555 556 init_attr->cap.max_send_wr = qp->sq.max; 557 init_attr->cap.max_recv_wr = qp->rq.max; 558 init_attr->cap.max_send_sge = qp->sq.max_gs; 559 init_attr->cap.max_recv_sge = qp->rq.max_gs; 560 init_attr->cap.max_inline_data = qp->max_inline_data; 561 562 return 0; 563 } 564 565 static int mthca_destroy_qp(struct ib_qp *qp, struct ib_udata *udata) 566 { 567 if (udata) { 568 struct mthca_ucontext *context = 569 rdma_udata_to_drv_context( 570 udata, 571 struct mthca_ucontext, 572 ibucontext); 573 574 mthca_unmap_user_db(to_mdev(qp->device), 575 &context->uar, 576 context->db_tab, 577 to_mqp(qp)->sq.db_index); 578 mthca_unmap_user_db(to_mdev(qp->device), 579 &context->uar, 580 context->db_tab, 581 to_mqp(qp)->rq.db_index); 582 } 583 mthca_free_qp(to_mdev(qp->device), to_mqp(qp)); 584 kfree(to_mqp(qp)->sqp); 585 return 0; 586 } 587 588 static int mthca_create_cq(struct ib_cq *ibcq, 589 const struct ib_cq_init_attr *attr, 590 struct uverbs_attr_bundle *attrs) 591 { 592 struct ib_udata *udata = &attrs->driver_udata; 593 struct ib_device *ibdev = ibcq->device; 594 int entries = attr->cqe; 595 struct mthca_create_cq ucmd; 596 struct mthca_cq *cq; 597 int nent; 598 int err; 599 struct mthca_ucontext *context = rdma_udata_to_drv_context( 600 udata, struct mthca_ucontext, ibucontext); 601 602 if (attr->flags) 603 return -EOPNOTSUPP; 604 605 if (entries < 1 || entries > to_mdev(ibdev)->limits.max_cqes) 606 return -EINVAL; 607 608 if (udata) { 609 err = ib_copy_validate_udata_in(udata, ucmd, set_db_index); 610 if (err) 611 return err; 612 613 err = mthca_map_user_db(to_mdev(ibdev), &context->uar, 614 context->db_tab, ucmd.set_db_index, 615 ucmd.set_db_page); 616 if (err) 617 return err; 618 619 err = mthca_map_user_db(to_mdev(ibdev), &context->uar, 620 context->db_tab, ucmd.arm_db_index, 621 ucmd.arm_db_page); 622 if (err) 623 goto err_unmap_set; 624 } 625 626 cq = to_mcq(ibcq); 627 628 if (udata) { 629 cq->buf.mr.ibmr.lkey = ucmd.lkey; 630 cq->set_ci_db_index = ucmd.set_db_index; 631 cq->arm_db_index = ucmd.arm_db_index; 632 } 633 634 for (nent = 1; nent <= entries; nent <<= 1) 635 ; /* nothing */ 636 637 err = mthca_init_cq(to_mdev(ibdev), nent, context, 638 udata ? ucmd.pdn : to_mdev(ibdev)->driver_pd.pd_num, 639 cq); 640 if (err) 641 goto err_unmap_arm; 642 643 if (udata) { 644 struct mthca_create_cq_resp uresp = { .cqn = cq->cqn }; 645 646 err = ib_respond_udata(udata, uresp); 647 if (err) { 648 mthca_free_cq(to_mdev(ibdev), cq); 649 goto err_unmap_arm; 650 } 651 } 652 653 cq->resize_buf = NULL; 654 655 return 0; 656 657 err_unmap_arm: 658 if (udata) 659 mthca_unmap_user_db(to_mdev(ibdev), &context->uar, 660 context->db_tab, ucmd.arm_db_index); 661 662 err_unmap_set: 663 if (udata) 664 mthca_unmap_user_db(to_mdev(ibdev), &context->uar, 665 context->db_tab, ucmd.set_db_index); 666 667 return err; 668 } 669 670 static int mthca_alloc_resize_buf(struct mthca_dev *dev, struct mthca_cq *cq, 671 int entries) 672 { 673 int ret; 674 675 spin_lock_irq(&cq->lock); 676 if (cq->resize_buf) { 677 ret = -EBUSY; 678 goto unlock; 679 } 680 681 cq->resize_buf = kmalloc_obj(*cq->resize_buf, GFP_ATOMIC); 682 if (!cq->resize_buf) { 683 ret = -ENOMEM; 684 goto unlock; 685 } 686 687 cq->resize_buf->state = CQ_RESIZE_ALLOC; 688 689 ret = 0; 690 691 unlock: 692 spin_unlock_irq(&cq->lock); 693 694 if (ret) 695 return ret; 696 697 ret = mthca_alloc_cq_buf(dev, &cq->resize_buf->buf, entries); 698 if (ret) { 699 spin_lock_irq(&cq->lock); 700 kfree(cq->resize_buf); 701 cq->resize_buf = NULL; 702 spin_unlock_irq(&cq->lock); 703 return ret; 704 } 705 706 cq->resize_buf->cqe = entries - 1; 707 708 spin_lock_irq(&cq->lock); 709 cq->resize_buf->state = CQ_RESIZE_READY; 710 spin_unlock_irq(&cq->lock); 711 712 return 0; 713 } 714 715 static int mthca_resize_cq(struct ib_cq *ibcq, unsigned int entries, 716 struct ib_udata *udata) 717 { 718 struct mthca_dev *dev = to_mdev(ibcq->device); 719 struct mthca_cq *cq = to_mcq(ibcq); 720 struct mthca_resize_cq ucmd; 721 u32 lkey; 722 int ret; 723 724 if (entries > dev->limits.max_cqes) 725 return -EINVAL; 726 727 mutex_lock(&cq->mutex); 728 729 entries = roundup_pow_of_two(entries + 1); 730 if (entries == ibcq->cqe + 1) { 731 ret = 0; 732 goto out; 733 } 734 735 if (cq->is_kernel) { 736 ret = mthca_alloc_resize_buf(dev, cq, entries); 737 if (ret) 738 goto out; 739 lkey = cq->resize_buf->buf.mr.ibmr.lkey; 740 } else { 741 ret = ib_copy_validate_udata_in(udata, ucmd, reserved); 742 if (ret) 743 goto out; 744 lkey = ucmd.lkey; 745 } 746 747 ret = mthca_RESIZE_CQ(dev, cq->cqn, lkey, ilog2(entries)); 748 749 if (ret) { 750 if (cq->resize_buf) { 751 mthca_free_cq_buf(dev, &cq->resize_buf->buf, 752 cq->resize_buf->cqe); 753 kfree(cq->resize_buf); 754 spin_lock_irq(&cq->lock); 755 cq->resize_buf = NULL; 756 spin_unlock_irq(&cq->lock); 757 } 758 goto out; 759 } 760 761 if (cq->is_kernel) { 762 struct mthca_cq_buf tbuf; 763 int tcqe; 764 765 spin_lock_irq(&cq->lock); 766 if (cq->resize_buf->state == CQ_RESIZE_READY) { 767 mthca_cq_resize_copy_cqes(cq); 768 tbuf = cq->buf; 769 tcqe = cq->ibcq.cqe; 770 cq->buf = cq->resize_buf->buf; 771 cq->ibcq.cqe = cq->resize_buf->cqe; 772 } else { 773 tbuf = cq->resize_buf->buf; 774 tcqe = cq->resize_buf->cqe; 775 } 776 777 kfree(cq->resize_buf); 778 cq->resize_buf = NULL; 779 spin_unlock_irq(&cq->lock); 780 781 mthca_free_cq_buf(dev, &tbuf, tcqe); 782 } else 783 ibcq->cqe = entries - 1; 784 785 out: 786 mutex_unlock(&cq->mutex); 787 788 return ret; 789 } 790 791 static int mthca_destroy_cq(struct ib_cq *cq, struct ib_udata *udata) 792 { 793 if (udata) { 794 struct mthca_ucontext *context = 795 rdma_udata_to_drv_context( 796 udata, 797 struct mthca_ucontext, 798 ibucontext); 799 800 mthca_unmap_user_db(to_mdev(cq->device), 801 &context->uar, 802 context->db_tab, 803 to_mcq(cq)->arm_db_index); 804 mthca_unmap_user_db(to_mdev(cq->device), 805 &context->uar, 806 context->db_tab, 807 to_mcq(cq)->set_ci_db_index); 808 } 809 mthca_free_cq(to_mdev(cq->device), to_mcq(cq)); 810 return 0; 811 } 812 813 static inline u32 convert_access(int acc) 814 { 815 return (acc & IB_ACCESS_REMOTE_ATOMIC ? MTHCA_MPT_FLAG_ATOMIC : 0) | 816 (acc & IB_ACCESS_REMOTE_WRITE ? MTHCA_MPT_FLAG_REMOTE_WRITE : 0) | 817 (acc & IB_ACCESS_REMOTE_READ ? MTHCA_MPT_FLAG_REMOTE_READ : 0) | 818 (acc & IB_ACCESS_LOCAL_WRITE ? MTHCA_MPT_FLAG_LOCAL_WRITE : 0) | 819 MTHCA_MPT_FLAG_LOCAL_READ; 820 } 821 822 static struct ib_mr *mthca_get_dma_mr(struct ib_pd *pd, int acc) 823 { 824 struct mthca_mr *mr; 825 int err; 826 827 mr = kmalloc_obj(*mr); 828 if (!mr) 829 return ERR_PTR(-ENOMEM); 830 831 err = mthca_mr_alloc_notrans(to_mdev(pd->device), 832 to_mpd(pd)->pd_num, 833 convert_access(acc), mr); 834 835 if (err) { 836 kfree(mr); 837 return ERR_PTR(err); 838 } 839 840 mr->umem = NULL; 841 842 return &mr->ibmr; 843 } 844 845 static struct ib_mr *mthca_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, 846 u64 virt, int acc, struct ib_dmah *dmah, 847 struct ib_udata *udata) 848 { 849 struct mthca_dev *dev = to_mdev(pd->device); 850 struct ib_block_iter biter; 851 struct mthca_ucontext *context = rdma_udata_to_drv_context( 852 udata, struct mthca_ucontext, ibucontext); 853 struct mthca_mr *mr; 854 struct mthca_reg_mr ucmd; 855 u64 *pages; 856 int n, i; 857 int err = 0; 858 int write_mtt_size; 859 860 if (dmah) 861 return ERR_PTR(-EOPNOTSUPP); 862 863 if (udata->inlen < sizeof ucmd) { 864 if (!context->reg_mr_warned) { 865 mthca_warn(dev, "Process '%s' did not pass in MR attrs.\n", 866 current->comm); 867 mthca_warn(dev, " Update libmthca to fix this.\n"); 868 } 869 ++context->reg_mr_warned; 870 ucmd.mr_attrs = 0; 871 } else { 872 err = ib_copy_validate_udata_in(udata, ucmd, reserved); 873 if (err) 874 return ERR_PTR(err); 875 } 876 877 mr = kmalloc_obj(*mr); 878 if (!mr) 879 return ERR_PTR(-ENOMEM); 880 881 mr->umem = ib_umem_get_va(pd->device, start, length, acc); 882 if (IS_ERR(mr->umem)) { 883 err = PTR_ERR(mr->umem); 884 goto err; 885 } 886 887 n = ib_umem_num_dma_blocks(mr->umem, PAGE_SIZE); 888 889 mr->mtt = mthca_alloc_mtt(dev, n); 890 if (IS_ERR(mr->mtt)) { 891 err = PTR_ERR(mr->mtt); 892 goto err_umem; 893 } 894 895 /* TODO: switch to "fast and as large as possible" allocation helper */ 896 pages = kmalloc(PAGE_SIZE, GFP_KERNEL); 897 if (!pages) { 898 err = -ENOMEM; 899 goto err_mtt; 900 } 901 902 i = n = 0; 903 904 write_mtt_size = min(mthca_write_mtt_size(dev), (int) (PAGE_SIZE / sizeof *pages)); 905 906 rdma_umem_for_each_dma_block(mr->umem, &biter, PAGE_SIZE) { 907 pages[i++] = rdma_block_iter_dma_address(&biter); 908 909 /* 910 * Be friendly to write_mtt and pass it chunks 911 * of appropriate size. 912 */ 913 if (i == write_mtt_size) { 914 err = mthca_write_mtt(dev, mr->mtt, n, pages, i); 915 if (err) 916 goto mtt_done; 917 n += i; 918 i = 0; 919 } 920 } 921 922 if (i) 923 err = mthca_write_mtt(dev, mr->mtt, n, pages, i); 924 mtt_done: 925 kfree(pages); 926 if (err) 927 goto err_mtt; 928 929 err = mthca_mr_alloc(dev, to_mpd(pd)->pd_num, PAGE_SHIFT, virt, length, 930 convert_access(acc), mr); 931 932 if (err) 933 goto err_mtt; 934 935 return &mr->ibmr; 936 937 err_mtt: 938 mthca_free_mtt(dev, mr->mtt); 939 940 err_umem: 941 ib_umem_release(mr->umem); 942 943 err: 944 kfree(mr); 945 return ERR_PTR(err); 946 } 947 948 static int mthca_dereg_mr(struct ib_mr *mr, struct ib_udata *udata) 949 { 950 struct mthca_mr *mmr = to_mmr(mr); 951 952 mthca_free_mr(to_mdev(mr->device), mmr); 953 ib_umem_release(mmr->umem); 954 kfree(mmr); 955 956 return 0; 957 } 958 959 static ssize_t hw_rev_show(struct device *device, 960 struct device_attribute *attr, char *buf) 961 { 962 struct mthca_dev *dev = 963 rdma_device_to_drv_device(device, struct mthca_dev, ib_dev); 964 965 return sysfs_emit(buf, "%x\n", dev->rev_id); 966 } 967 static DEVICE_ATTR_RO(hw_rev); 968 969 static const char *hca_type_string(int hca_type) 970 { 971 switch (hca_type) { 972 case PCI_DEVICE_ID_MELLANOX_TAVOR: 973 return "MT23108"; 974 case PCI_DEVICE_ID_MELLANOX_ARBEL_COMPAT: 975 return "MT25208 (MT23108 compat mode)"; 976 case PCI_DEVICE_ID_MELLANOX_ARBEL: 977 return "MT25208"; 978 case PCI_DEVICE_ID_MELLANOX_SINAI: 979 case PCI_DEVICE_ID_MELLANOX_SINAI_OLD: 980 return "MT25204"; 981 } 982 983 return "unknown"; 984 } 985 986 static ssize_t hca_type_show(struct device *device, 987 struct device_attribute *attr, char *buf) 988 { 989 struct mthca_dev *dev = 990 rdma_device_to_drv_device(device, struct mthca_dev, ib_dev); 991 992 return sysfs_emit(buf, "%s\n", hca_type_string(dev->pdev->device)); 993 } 994 static DEVICE_ATTR_RO(hca_type); 995 996 static ssize_t board_id_show(struct device *device, 997 struct device_attribute *attr, char *buf) 998 { 999 struct mthca_dev *dev = 1000 rdma_device_to_drv_device(device, struct mthca_dev, ib_dev); 1001 1002 return sysfs_emit(buf, "%.*s\n", MTHCA_BOARD_ID_LEN, dev->board_id); 1003 } 1004 static DEVICE_ATTR_RO(board_id); 1005 1006 static struct attribute *mthca_dev_attributes[] = { 1007 &dev_attr_hw_rev.attr, 1008 &dev_attr_hca_type.attr, 1009 &dev_attr_board_id.attr, 1010 NULL 1011 }; 1012 1013 static const struct attribute_group mthca_attr_group = { 1014 .attrs = mthca_dev_attributes, 1015 }; 1016 1017 static int mthca_init_node_data(struct mthca_dev *dev) 1018 { 1019 struct ib_smp *in_mad; 1020 struct ib_smp *out_mad; 1021 int err = -ENOMEM; 1022 1023 in_mad = kzalloc_obj(*in_mad); 1024 out_mad = kmalloc_obj(*out_mad); 1025 if (!in_mad || !out_mad) 1026 goto out; 1027 1028 ib_init_query_mad(in_mad); 1029 in_mad->attr_id = IB_SMP_ATTR_NODE_DESC; 1030 1031 err = mthca_MAD_IFC(dev, 1, 1, 1032 1, NULL, NULL, in_mad, out_mad); 1033 if (err) 1034 goto out; 1035 1036 memcpy(dev->ib_dev.node_desc, out_mad->data, IB_DEVICE_NODE_DESC_MAX); 1037 1038 in_mad->attr_id = IB_SMP_ATTR_NODE_INFO; 1039 1040 err = mthca_MAD_IFC(dev, 1, 1, 1041 1, NULL, NULL, in_mad, out_mad); 1042 if (err) 1043 goto out; 1044 1045 if (mthca_is_memfree(dev)) 1046 dev->rev_id = be32_to_cpup((__be32 *) (out_mad->data + 32)); 1047 memcpy(&dev->ib_dev.node_guid, out_mad->data + 12, 8); 1048 1049 out: 1050 kfree(in_mad); 1051 kfree(out_mad); 1052 return err; 1053 } 1054 1055 static int mthca_port_immutable(struct ib_device *ibdev, u32 port_num, 1056 struct ib_port_immutable *immutable) 1057 { 1058 struct ib_port_attr attr; 1059 int err; 1060 1061 immutable->core_cap_flags = RDMA_CORE_PORT_IBA_IB; 1062 1063 err = ib_query_port(ibdev, port_num, &attr); 1064 if (err) 1065 return err; 1066 1067 immutable->pkey_tbl_len = attr.pkey_tbl_len; 1068 immutable->gid_tbl_len = attr.gid_tbl_len; 1069 immutable->max_mad_size = IB_MGMT_MAD_SIZE; 1070 1071 return 0; 1072 } 1073 1074 static void get_dev_fw_str(struct ib_device *device, char *str) 1075 { 1076 struct mthca_dev *dev = 1077 container_of(device, struct mthca_dev, ib_dev); 1078 snprintf(str, IB_FW_VERSION_NAME_MAX, "%d.%d.%d", 1079 (int) (dev->fw_ver >> 32), 1080 (int) (dev->fw_ver >> 16) & 0xffff, 1081 (int) dev->fw_ver & 0xffff); 1082 } 1083 1084 static const struct ib_device_ops mthca_dev_ops = { 1085 .owner = THIS_MODULE, 1086 .driver_id = RDMA_DRIVER_MTHCA, 1087 .uverbs_abi_ver = MTHCA_UVERBS_ABI_VERSION, 1088 .uverbs_no_driver_id_binding = 1, 1089 1090 .alloc_pd = mthca_alloc_pd, 1091 .alloc_ucontext = mthca_alloc_ucontext, 1092 .attach_mcast = mthca_multicast_attach, 1093 .create_ah = mthca_ah_create, 1094 .create_cq = mthca_create_cq, 1095 .create_qp = mthca_create_qp, 1096 .dealloc_pd = mthca_dealloc_pd, 1097 .dealloc_ucontext = mthca_dealloc_ucontext, 1098 .dereg_mr = mthca_dereg_mr, 1099 .destroy_ah = mthca_ah_destroy, 1100 .destroy_cq = mthca_destroy_cq, 1101 .destroy_qp = mthca_destroy_qp, 1102 .detach_mcast = mthca_multicast_detach, 1103 .device_group = &mthca_attr_group, 1104 .get_dev_fw_str = get_dev_fw_str, 1105 .get_dma_mr = mthca_get_dma_mr, 1106 .get_port_immutable = mthca_port_immutable, 1107 .mmap = mthca_mmap_uar, 1108 .modify_device = mthca_modify_device, 1109 .modify_port = mthca_modify_port, 1110 .modify_qp = mthca_modify_qp, 1111 .poll_cq = mthca_poll_cq, 1112 .process_mad = mthca_process_mad, 1113 .query_ah = mthca_ah_query, 1114 .query_device = mthca_query_device, 1115 .query_gid = mthca_query_gid, 1116 .query_pkey = mthca_query_pkey, 1117 .query_port = mthca_query_port, 1118 .query_qp = mthca_query_qp, 1119 .reg_user_mr = mthca_reg_user_mr, 1120 .resize_user_cq = mthca_resize_cq, 1121 1122 INIT_RDMA_OBJ_SIZE(ib_ah, mthca_ah, ibah), 1123 INIT_RDMA_OBJ_SIZE(ib_cq, mthca_cq, ibcq), 1124 INIT_RDMA_OBJ_SIZE(ib_pd, mthca_pd, ibpd), 1125 INIT_RDMA_OBJ_SIZE(ib_qp, mthca_qp, ibqp), 1126 INIT_RDMA_OBJ_SIZE(ib_ucontext, mthca_ucontext, ibucontext), 1127 }; 1128 1129 static const struct ib_device_ops mthca_dev_arbel_srq_ops = { 1130 .create_srq = mthca_create_srq, 1131 .destroy_srq = mthca_destroy_srq, 1132 .modify_srq = mthca_modify_srq, 1133 .post_srq_recv = mthca_arbel_post_srq_recv, 1134 .query_srq = mthca_query_srq, 1135 1136 INIT_RDMA_OBJ_SIZE(ib_srq, mthca_srq, ibsrq), 1137 }; 1138 1139 static const struct ib_device_ops mthca_dev_tavor_srq_ops = { 1140 .create_srq = mthca_create_srq, 1141 .destroy_srq = mthca_destroy_srq, 1142 .modify_srq = mthca_modify_srq, 1143 .post_srq_recv = mthca_tavor_post_srq_recv, 1144 .query_srq = mthca_query_srq, 1145 1146 INIT_RDMA_OBJ_SIZE(ib_srq, mthca_srq, ibsrq), 1147 }; 1148 1149 static const struct ib_device_ops mthca_dev_arbel_ops = { 1150 .post_recv = mthca_arbel_post_receive, 1151 .post_send = mthca_arbel_post_send, 1152 .req_notify_cq = mthca_arbel_arm_cq, 1153 }; 1154 1155 static const struct ib_device_ops mthca_dev_tavor_ops = { 1156 .post_recv = mthca_tavor_post_receive, 1157 .post_send = mthca_tavor_post_send, 1158 .req_notify_cq = mthca_tavor_arm_cq, 1159 }; 1160 1161 int mthca_register_device(struct mthca_dev *dev) 1162 { 1163 int ret; 1164 1165 ret = mthca_init_node_data(dev); 1166 if (ret) 1167 return ret; 1168 1169 dev->ib_dev.node_type = RDMA_NODE_IB_CA; 1170 dev->ib_dev.phys_port_cnt = dev->limits.num_ports; 1171 dev->ib_dev.num_comp_vectors = 1; 1172 dev->ib_dev.dev.parent = &dev->pdev->dev; 1173 1174 if (dev->mthca_flags & MTHCA_FLAG_SRQ) { 1175 if (mthca_is_memfree(dev)) 1176 ib_set_device_ops(&dev->ib_dev, 1177 &mthca_dev_arbel_srq_ops); 1178 else 1179 ib_set_device_ops(&dev->ib_dev, 1180 &mthca_dev_tavor_srq_ops); 1181 } 1182 1183 ib_set_device_ops(&dev->ib_dev, &mthca_dev_ops); 1184 1185 if (mthca_is_memfree(dev)) 1186 ib_set_device_ops(&dev->ib_dev, &mthca_dev_arbel_ops); 1187 else 1188 ib_set_device_ops(&dev->ib_dev, &mthca_dev_tavor_ops); 1189 1190 mutex_init(&dev->cap_mask_mutex); 1191 1192 ret = ib_register_device(&dev->ib_dev, "mthca%d", &dev->pdev->dev); 1193 if (ret) 1194 return ret; 1195 1196 mthca_start_catas_poll(dev); 1197 1198 return 0; 1199 } 1200 1201 void mthca_unregister_device(struct mthca_dev *dev) 1202 { 1203 mthca_stop_catas_poll(dev); 1204 ib_unregister_device(&dev->ib_dev); 1205 } 1206