1 /* 2 * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. 3 * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved. 4 * Copyright (c) 2005, 2006 Cisco Systems. All rights reserved. 5 * Copyright (c) 2005 Mellanox Technologies. All rights reserved. 6 * Copyright (c) 2004 Voltaire, Inc. All rights reserved. 7 * 8 * This software is available to you under a choice of one of two 9 * licenses. You may choose to be licensed under the terms of the GNU 10 * General Public License (GPL) Version 2, available from the file 11 * COPYING in the main directory of this source tree, or the 12 * OpenIB.org BSD license below: 13 * 14 * Redistribution and use in source and binary forms, with or 15 * without modification, are permitted provided that the following 16 * conditions are met: 17 * 18 * - Redistributions of source code must retain the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer. 21 * 22 * - Redistributions in binary form must reproduce the above 23 * copyright notice, this list of conditions and the following 24 * disclaimer in the documentation and/or other materials 25 * provided with the distribution. 26 * 27 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 28 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 29 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 30 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 31 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 32 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 33 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 34 * SOFTWARE. 35 */ 36 37 #include <rdma/ib_smi.h> 38 #include <rdma/ib_user_verbs.h> 39 #include <rdma/iter.h> 40 #include <rdma/uverbs_ioctl.h> 41 42 #include <linux/sched.h> 43 #include <linux/slab.h> 44 #include <linux/stat.h> 45 #include <linux/mm.h> 46 #include <linux/export.h> 47 48 #include "mthca_dev.h" 49 #include "mthca_cmd.h" 50 #include <rdma/mthca-abi.h> 51 #include "mthca_memfree.h" 52 53 static int mthca_query_device(struct ib_device *ibdev, struct ib_device_attr *props, 54 struct ib_udata *uhw) 55 { 56 struct ib_smp *in_mad; 57 struct ib_smp *out_mad; 58 int err; 59 struct mthca_dev *mdev = to_mdev(ibdev); 60 61 err = ib_is_udata_in_empty(uhw); 62 if (err) 63 return err; 64 65 in_mad = kzalloc_obj(*in_mad); 66 out_mad = kmalloc_obj(*out_mad); 67 if (!in_mad || !out_mad) { 68 err = -ENOMEM; 69 goto out; 70 } 71 72 memset(props, 0, sizeof *props); 73 74 props->fw_ver = mdev->fw_ver; 75 76 ib_init_query_mad(in_mad); 77 in_mad->attr_id = IB_SMP_ATTR_NODE_INFO; 78 79 err = mthca_MAD_IFC(mdev, 1, 1, 80 1, NULL, NULL, in_mad, out_mad); 81 if (err) 82 goto out; 83 84 props->device_cap_flags = mdev->device_cap_flags; 85 props->vendor_id = be32_to_cpup((__be32 *) (out_mad->data + 36)) & 86 0xffffff; 87 props->vendor_part_id = be16_to_cpup((__be16 *) (out_mad->data + 30)); 88 props->hw_ver = be32_to_cpup((__be32 *) (out_mad->data + 32)); 89 memcpy(&props->sys_image_guid, out_mad->data + 4, 8); 90 91 props->max_mr_size = ~0ull; 92 props->page_size_cap = mdev->limits.page_size_cap; 93 props->max_qp = mdev->limits.num_qps - mdev->limits.reserved_qps; 94 props->max_qp_wr = mdev->limits.max_wqes; 95 props->max_send_sge = mdev->limits.max_sg; 96 props->max_recv_sge = mdev->limits.max_sg; 97 props->max_sge_rd = mdev->limits.max_sg; 98 props->max_cq = mdev->limits.num_cqs - mdev->limits.reserved_cqs; 99 props->max_cqe = mdev->limits.max_cqes; 100 props->max_mr = mdev->limits.num_mpts - mdev->limits.reserved_mrws; 101 props->max_pd = mdev->limits.num_pds - mdev->limits.reserved_pds; 102 props->max_qp_rd_atom = 1 << mdev->qp_table.rdb_shift; 103 props->max_qp_init_rd_atom = mdev->limits.max_qp_init_rdma; 104 props->max_res_rd_atom = props->max_qp_rd_atom * props->max_qp; 105 props->max_srq = mdev->limits.num_srqs - mdev->limits.reserved_srqs; 106 props->max_srq_wr = mdev->limits.max_srq_wqes; 107 props->max_srq_sge = mdev->limits.max_srq_sge; 108 props->local_ca_ack_delay = mdev->limits.local_ca_ack_delay; 109 props->atomic_cap = mdev->limits.flags & DEV_LIM_FLAG_ATOMIC ? 110 IB_ATOMIC_HCA : IB_ATOMIC_NONE; 111 props->max_pkeys = mdev->limits.pkey_table_len; 112 props->max_mcast_grp = mdev->limits.num_mgms + mdev->limits.num_amgms; 113 props->max_mcast_qp_attach = MTHCA_QP_PER_MGM; 114 props->max_total_mcast_qp_attach = props->max_mcast_qp_attach * 115 props->max_mcast_grp; 116 117 err = ib_respond_empty_udata(uhw); 118 out: 119 kfree(in_mad); 120 kfree(out_mad); 121 return err; 122 } 123 124 static int mthca_query_port(struct ib_device *ibdev, 125 u32 port, struct ib_port_attr *props) 126 { 127 struct ib_smp *in_mad; 128 struct ib_smp *out_mad; 129 int err = -ENOMEM; 130 131 in_mad = kzalloc_obj(*in_mad); 132 out_mad = kmalloc_obj(*out_mad); 133 if (!in_mad || !out_mad) 134 goto out; 135 136 /* props being zeroed by the caller, avoid zeroing it here */ 137 138 ib_init_query_mad(in_mad); 139 in_mad->attr_id = IB_SMP_ATTR_PORT_INFO; 140 in_mad->attr_mod = cpu_to_be32(port); 141 142 err = mthca_MAD_IFC(to_mdev(ibdev), 1, 1, 143 port, NULL, NULL, in_mad, out_mad); 144 if (err) 145 goto out; 146 147 props->lid = be16_to_cpup((__be16 *) (out_mad->data + 16)); 148 props->lmc = out_mad->data[34] & 0x7; 149 props->sm_lid = be16_to_cpup((__be16 *) (out_mad->data + 18)); 150 props->sm_sl = out_mad->data[36] & 0xf; 151 props->state = out_mad->data[32] & 0xf; 152 props->phys_state = out_mad->data[33] >> 4; 153 props->port_cap_flags = be32_to_cpup((__be32 *) (out_mad->data + 20)); 154 props->gid_tbl_len = to_mdev(ibdev)->limits.gid_table_len; 155 props->max_msg_sz = 0x80000000; 156 props->pkey_tbl_len = to_mdev(ibdev)->limits.pkey_table_len; 157 props->bad_pkey_cntr = be16_to_cpup((__be16 *) (out_mad->data + 46)); 158 props->qkey_viol_cntr = be16_to_cpup((__be16 *) (out_mad->data + 48)); 159 props->active_width = out_mad->data[31] & 0xf; 160 props->active_speed = out_mad->data[35] >> 4; 161 props->max_mtu = out_mad->data[41] & 0xf; 162 props->active_mtu = out_mad->data[36] >> 4; 163 props->subnet_timeout = out_mad->data[51] & 0x1f; 164 props->max_vl_num = out_mad->data[37] >> 4; 165 props->init_type_reply = out_mad->data[41] >> 4; 166 167 out: 168 kfree(in_mad); 169 kfree(out_mad); 170 return err; 171 } 172 173 static int mthca_modify_device(struct ib_device *ibdev, 174 int mask, 175 struct ib_device_modify *props) 176 { 177 if (mask & ~IB_DEVICE_MODIFY_NODE_DESC) 178 return -EOPNOTSUPP; 179 180 if (mask & IB_DEVICE_MODIFY_NODE_DESC) { 181 if (mutex_lock_interruptible(&to_mdev(ibdev)->cap_mask_mutex)) 182 return -ERESTARTSYS; 183 memcpy(ibdev->node_desc, props->node_desc, 184 IB_DEVICE_NODE_DESC_MAX); 185 mutex_unlock(&to_mdev(ibdev)->cap_mask_mutex); 186 } 187 188 return 0; 189 } 190 191 static int mthca_modify_port(struct ib_device *ibdev, 192 u32 port, int port_modify_mask, 193 struct ib_port_modify *props) 194 { 195 struct mthca_set_ib_param set_ib; 196 struct ib_port_attr attr; 197 int err; 198 199 if (mutex_lock_interruptible(&to_mdev(ibdev)->cap_mask_mutex)) 200 return -ERESTARTSYS; 201 202 err = ib_query_port(ibdev, port, &attr); 203 if (err) 204 goto out; 205 206 set_ib.set_si_guid = 0; 207 set_ib.reset_qkey_viol = !!(port_modify_mask & IB_PORT_RESET_QKEY_CNTR); 208 209 set_ib.cap_mask = (attr.port_cap_flags | props->set_port_cap_mask) & 210 ~props->clr_port_cap_mask; 211 212 err = mthca_SET_IB(to_mdev(ibdev), &set_ib, port); 213 if (err) 214 goto out; 215 out: 216 mutex_unlock(&to_mdev(ibdev)->cap_mask_mutex); 217 return err; 218 } 219 220 static int mthca_query_pkey(struct ib_device *ibdev, 221 u32 port, u16 index, u16 *pkey) 222 { 223 struct ib_smp *in_mad; 224 struct ib_smp *out_mad; 225 int err = -ENOMEM; 226 227 in_mad = kzalloc_obj(*in_mad); 228 out_mad = kmalloc_obj(*out_mad); 229 if (!in_mad || !out_mad) 230 goto out; 231 232 ib_init_query_mad(in_mad); 233 in_mad->attr_id = IB_SMP_ATTR_PKEY_TABLE; 234 in_mad->attr_mod = cpu_to_be32(index / 32); 235 236 err = mthca_MAD_IFC(to_mdev(ibdev), 1, 1, 237 port, NULL, NULL, in_mad, out_mad); 238 if (err) 239 goto out; 240 241 *pkey = be16_to_cpu(((__be16 *) out_mad->data)[index % 32]); 242 243 out: 244 kfree(in_mad); 245 kfree(out_mad); 246 return err; 247 } 248 249 static int mthca_query_gid(struct ib_device *ibdev, u32 port, 250 int index, union ib_gid *gid) 251 { 252 struct ib_smp *in_mad; 253 struct ib_smp *out_mad; 254 int err = -ENOMEM; 255 256 in_mad = kzalloc_obj(*in_mad); 257 out_mad = kmalloc_obj(*out_mad); 258 if (!in_mad || !out_mad) 259 goto out; 260 261 ib_init_query_mad(in_mad); 262 in_mad->attr_id = IB_SMP_ATTR_PORT_INFO; 263 in_mad->attr_mod = cpu_to_be32(port); 264 265 err = mthca_MAD_IFC(to_mdev(ibdev), 1, 1, 266 port, NULL, NULL, in_mad, out_mad); 267 if (err) 268 goto out; 269 270 memcpy(gid->raw, out_mad->data + 8, 8); 271 272 ib_init_query_mad(in_mad); 273 in_mad->attr_id = IB_SMP_ATTR_GUID_INFO; 274 in_mad->attr_mod = cpu_to_be32(index / 8); 275 276 err = mthca_MAD_IFC(to_mdev(ibdev), 1, 1, 277 port, NULL, NULL, in_mad, out_mad); 278 if (err) 279 goto out; 280 281 memcpy(gid->raw + 8, out_mad->data + (index % 8) * 8, 8); 282 283 out: 284 kfree(in_mad); 285 kfree(out_mad); 286 return err; 287 } 288 289 static int mthca_alloc_ucontext(struct ib_ucontext *uctx, 290 struct ib_udata *udata) 291 { 292 struct ib_device *ibdev = uctx->device; 293 struct mthca_alloc_ucontext_resp uresp = {}; 294 struct mthca_ucontext *context = to_mucontext(uctx); 295 int err; 296 297 if (!(to_mdev(ibdev)->active)) 298 return -EAGAIN; 299 300 uresp.qp_tab_size = to_mdev(ibdev)->limits.num_qps; 301 if (mthca_is_memfree(to_mdev(ibdev))) 302 uresp.uarc_size = to_mdev(ibdev)->uar_table.uarc_size; 303 else 304 uresp.uarc_size = 0; 305 306 err = mthca_uar_alloc(to_mdev(ibdev), &context->uar); 307 if (err) 308 return err; 309 310 context->db_tab = mthca_init_user_db_tab(to_mdev(ibdev)); 311 if (IS_ERR(context->db_tab)) { 312 err = PTR_ERR(context->db_tab); 313 mthca_uar_free(to_mdev(ibdev), &context->uar); 314 return err; 315 } 316 317 err = ib_respond_udata(udata, uresp); 318 if (err) { 319 mthca_cleanup_user_db_tab(to_mdev(ibdev), &context->uar, context->db_tab); 320 mthca_uar_free(to_mdev(ibdev), &context->uar); 321 return err; 322 } 323 324 context->reg_mr_warned = 0; 325 326 return 0; 327 } 328 329 static void mthca_dealloc_ucontext(struct ib_ucontext *context) 330 { 331 mthca_cleanup_user_db_tab(to_mdev(context->device), &to_mucontext(context)->uar, 332 to_mucontext(context)->db_tab); 333 mthca_uar_free(to_mdev(context->device), &to_mucontext(context)->uar); 334 } 335 336 static int mthca_mmap_uar(struct ib_ucontext *context, 337 struct vm_area_struct *vma) 338 { 339 if (vma->vm_end - vma->vm_start != PAGE_SIZE) 340 return -EINVAL; 341 342 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); 343 344 if (io_remap_pfn_range(vma, vma->vm_start, 345 to_mucontext(context)->uar.pfn, 346 PAGE_SIZE, vma->vm_page_prot)) 347 return -EAGAIN; 348 349 return 0; 350 } 351 352 static int mthca_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 353 { 354 struct ib_device *ibdev = ibpd->device; 355 struct mthca_pd *pd = to_mpd(ibpd); 356 int err; 357 358 err = mthca_pd_alloc(to_mdev(ibdev), !udata, pd); 359 if (err) 360 return err; 361 362 if (udata) { 363 struct mthca_alloc_pd_resp uresp = { .pdn = pd->pd_num }; 364 365 err = ib_respond_udata(udata, uresp); 366 if (err) { 367 mthca_pd_free(to_mdev(ibdev), pd); 368 return err; 369 } 370 } 371 372 return 0; 373 } 374 375 static int mthca_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata) 376 { 377 mthca_pd_free(to_mdev(pd->device), to_mpd(pd)); 378 return 0; 379 } 380 381 static int mthca_ah_create(struct ib_ah *ibah, 382 struct rdma_ah_init_attr *init_attr, 383 struct ib_udata *udata) 384 385 { 386 struct mthca_ah *ah = to_mah(ibah); 387 388 return mthca_create_ah(to_mdev(ibah->device), to_mpd(ibah->pd), 389 init_attr->ah_attr, ah); 390 } 391 392 static int mthca_ah_destroy(struct ib_ah *ah, u32 flags) 393 { 394 mthca_destroy_ah(to_mdev(ah->device), to_mah(ah)); 395 return 0; 396 } 397 398 static int mthca_create_srq(struct ib_srq *ibsrq, 399 struct ib_srq_init_attr *init_attr, 400 struct ib_udata *udata) 401 { 402 struct mthca_create_srq ucmd; 403 struct mthca_ucontext *context = rdma_udata_to_drv_context( 404 udata, struct mthca_ucontext, ibucontext); 405 struct mthca_srq *srq = to_msrq(ibsrq); 406 int err; 407 408 if (init_attr->srq_type != IB_SRQT_BASIC) 409 return -EOPNOTSUPP; 410 411 if (udata) { 412 err = ib_copy_validate_udata_in(udata, ucmd, db_page); 413 if (err) 414 return err; 415 416 err = mthca_map_user_db(to_mdev(ibsrq->device), &context->uar, 417 context->db_tab, ucmd.db_index, 418 ucmd.db_page); 419 420 if (err) 421 return err; 422 423 srq->mr.ibmr.lkey = ucmd.lkey; 424 srq->db_index = ucmd.db_index; 425 } 426 427 err = mthca_alloc_srq(to_mdev(ibsrq->device), to_mpd(ibsrq->pd), 428 &init_attr->attr, srq, udata); 429 430 if (err && udata) 431 mthca_unmap_user_db(to_mdev(ibsrq->device), &context->uar, 432 context->db_tab, ucmd.db_index); 433 434 if (err) 435 return err; 436 437 if (context) { 438 struct mthca_create_srq_resp uresp = { .srqn = srq->srqn }; 439 440 err = ib_respond_udata(udata, uresp); 441 if (err) { 442 mthca_free_srq(to_mdev(ibsrq->device), srq); 443 mthca_unmap_user_db(to_mdev(ibsrq->device), 444 &context->uar, context->db_tab, 445 ucmd.db_index); 446 return err; 447 } 448 } 449 450 return 0; 451 } 452 453 static int mthca_destroy_srq(struct ib_srq *srq, struct ib_udata *udata) 454 { 455 mthca_free_srq(to_mdev(srq->device), to_msrq(srq)); 456 if (udata) { 457 struct mthca_ucontext *context = 458 rdma_udata_to_drv_context( 459 udata, 460 struct mthca_ucontext, 461 ibucontext); 462 463 mthca_unmap_user_db(to_mdev(srq->device), &context->uar, 464 context->db_tab, to_msrq(srq)->db_index); 465 } 466 return 0; 467 } 468 469 static int mthca_create_qp(struct ib_qp *ibqp, 470 struct ib_qp_init_attr *init_attr, 471 struct ib_udata *udata) 472 { 473 struct mthca_ucontext *context = rdma_udata_to_drv_context( 474 udata, struct mthca_ucontext, ibucontext); 475 struct mthca_create_qp ucmd; 476 struct mthca_qp *qp = to_mqp(ibqp); 477 struct mthca_dev *dev = to_mdev(ibqp->device); 478 int err; 479 480 if (init_attr->create_flags) 481 return -EOPNOTSUPP; 482 483 switch (init_attr->qp_type) { 484 case IB_QPT_RC: 485 case IB_QPT_UC: 486 case IB_QPT_UD: 487 { 488 if (udata) { 489 err = ib_copy_validate_udata_in(udata, ucmd, rq_db_index); 490 if (err) 491 return err; 492 493 err = mthca_map_user_db(dev, &context->uar, 494 context->db_tab, 495 ucmd.sq_db_index, 496 ucmd.sq_db_page); 497 if (err) 498 return err; 499 500 err = mthca_map_user_db(dev, &context->uar, 501 context->db_tab, 502 ucmd.rq_db_index, 503 ucmd.rq_db_page); 504 if (err) { 505 mthca_unmap_user_db(dev, &context->uar, 506 context->db_tab, 507 ucmd.sq_db_index); 508 return err; 509 } 510 511 qp->mr.ibmr.lkey = ucmd.lkey; 512 qp->sq.db_index = ucmd.sq_db_index; 513 qp->rq.db_index = ucmd.rq_db_index; 514 } 515 516 err = mthca_alloc_qp(dev, to_mpd(ibqp->pd), 517 to_mcq(init_attr->send_cq), 518 to_mcq(init_attr->recv_cq), 519 init_attr->qp_type, init_attr->sq_sig_type, 520 &init_attr->cap, qp, udata); 521 522 if (err && udata) { 523 mthca_unmap_user_db(dev, &context->uar, context->db_tab, 524 ucmd.sq_db_index); 525 mthca_unmap_user_db(dev, &context->uar, context->db_tab, 526 ucmd.rq_db_index); 527 } 528 529 qp->ibqp.qp_num = qp->qpn; 530 break; 531 } 532 case IB_QPT_SMI: 533 case IB_QPT_GSI: 534 { 535 qp->sqp = kzalloc_obj(struct mthca_sqp); 536 if (!qp->sqp) 537 return -ENOMEM; 538 539 qp->ibqp.qp_num = init_attr->qp_type == IB_QPT_SMI ? 0 : 1; 540 541 err = mthca_alloc_sqp(dev, to_mpd(ibqp->pd), 542 to_mcq(init_attr->send_cq), 543 to_mcq(init_attr->recv_cq), 544 init_attr->sq_sig_type, &init_attr->cap, 545 qp->ibqp.qp_num, init_attr->port_num, qp, 546 udata); 547 break; 548 } 549 default: 550 /* Don't support raw QPs */ 551 return -EOPNOTSUPP; 552 } 553 554 if (err) { 555 kfree(qp->sqp); 556 return err; 557 } 558 559 init_attr->cap.max_send_wr = qp->sq.max; 560 init_attr->cap.max_recv_wr = qp->rq.max; 561 init_attr->cap.max_send_sge = qp->sq.max_gs; 562 init_attr->cap.max_recv_sge = qp->rq.max_gs; 563 init_attr->cap.max_inline_data = qp->max_inline_data; 564 565 return 0; 566 } 567 568 static int mthca_destroy_qp(struct ib_qp *qp, struct ib_udata *udata) 569 { 570 if (udata) { 571 struct mthca_ucontext *context = 572 rdma_udata_to_drv_context( 573 udata, 574 struct mthca_ucontext, 575 ibucontext); 576 577 mthca_unmap_user_db(to_mdev(qp->device), 578 &context->uar, 579 context->db_tab, 580 to_mqp(qp)->sq.db_index); 581 mthca_unmap_user_db(to_mdev(qp->device), 582 &context->uar, 583 context->db_tab, 584 to_mqp(qp)->rq.db_index); 585 } 586 mthca_free_qp(to_mdev(qp->device), to_mqp(qp)); 587 kfree(to_mqp(qp)->sqp); 588 return 0; 589 } 590 591 static int mthca_create_cq(struct ib_cq *ibcq, 592 const struct ib_cq_init_attr *attr, 593 struct uverbs_attr_bundle *attrs) 594 { 595 struct ib_udata *udata = &attrs->driver_udata; 596 struct ib_device *ibdev = ibcq->device; 597 int entries = attr->cqe; 598 struct mthca_create_cq ucmd; 599 struct mthca_cq *cq; 600 int nent; 601 int err; 602 struct mthca_ucontext *context = rdma_udata_to_drv_context( 603 udata, struct mthca_ucontext, ibucontext); 604 605 if (attr->flags) 606 return -EOPNOTSUPP; 607 608 if (entries < 1 || entries > to_mdev(ibdev)->limits.max_cqes) 609 return -EINVAL; 610 611 if (udata) { 612 err = ib_copy_validate_udata_in(udata, ucmd, set_db_index); 613 if (err) 614 return err; 615 616 err = mthca_map_user_db(to_mdev(ibdev), &context->uar, 617 context->db_tab, ucmd.set_db_index, 618 ucmd.set_db_page); 619 if (err) 620 return err; 621 622 err = mthca_map_user_db(to_mdev(ibdev), &context->uar, 623 context->db_tab, ucmd.arm_db_index, 624 ucmd.arm_db_page); 625 if (err) 626 goto err_unmap_set; 627 } 628 629 cq = to_mcq(ibcq); 630 631 if (udata) { 632 cq->buf.mr.ibmr.lkey = ucmd.lkey; 633 cq->set_ci_db_index = ucmd.set_db_index; 634 cq->arm_db_index = ucmd.arm_db_index; 635 } 636 637 for (nent = 1; nent <= entries; nent <<= 1) 638 ; /* nothing */ 639 640 err = mthca_init_cq(to_mdev(ibdev), nent, context, 641 udata ? ucmd.pdn : to_mdev(ibdev)->driver_pd.pd_num, 642 cq); 643 if (err) 644 goto err_unmap_arm; 645 646 if (udata) { 647 struct mthca_create_cq_resp uresp = { .cqn = cq->cqn }; 648 649 err = ib_respond_udata(udata, uresp); 650 if (err) { 651 mthca_free_cq(to_mdev(ibdev), cq); 652 goto err_unmap_arm; 653 } 654 } 655 656 cq->resize_buf = NULL; 657 658 return 0; 659 660 err_unmap_arm: 661 if (udata) 662 mthca_unmap_user_db(to_mdev(ibdev), &context->uar, 663 context->db_tab, ucmd.arm_db_index); 664 665 err_unmap_set: 666 if (udata) 667 mthca_unmap_user_db(to_mdev(ibdev), &context->uar, 668 context->db_tab, ucmd.set_db_index); 669 670 return err; 671 } 672 673 static int mthca_alloc_resize_buf(struct mthca_dev *dev, struct mthca_cq *cq, 674 int entries) 675 { 676 int ret; 677 678 spin_lock_irq(&cq->lock); 679 if (cq->resize_buf) { 680 ret = -EBUSY; 681 goto unlock; 682 } 683 684 cq->resize_buf = kmalloc_obj(*cq->resize_buf, GFP_ATOMIC); 685 if (!cq->resize_buf) { 686 ret = -ENOMEM; 687 goto unlock; 688 } 689 690 cq->resize_buf->state = CQ_RESIZE_ALLOC; 691 692 ret = 0; 693 694 unlock: 695 spin_unlock_irq(&cq->lock); 696 697 if (ret) 698 return ret; 699 700 ret = mthca_alloc_cq_buf(dev, &cq->resize_buf->buf, entries); 701 if (ret) { 702 spin_lock_irq(&cq->lock); 703 kfree(cq->resize_buf); 704 cq->resize_buf = NULL; 705 spin_unlock_irq(&cq->lock); 706 return ret; 707 } 708 709 cq->resize_buf->cqe = entries - 1; 710 711 spin_lock_irq(&cq->lock); 712 cq->resize_buf->state = CQ_RESIZE_READY; 713 spin_unlock_irq(&cq->lock); 714 715 return 0; 716 } 717 718 static int mthca_resize_cq(struct ib_cq *ibcq, unsigned int entries, 719 struct ib_udata *udata) 720 { 721 struct mthca_dev *dev = to_mdev(ibcq->device); 722 struct mthca_cq *cq = to_mcq(ibcq); 723 struct mthca_resize_cq ucmd; 724 u32 lkey; 725 int ret; 726 727 if (entries > dev->limits.max_cqes) 728 return -EINVAL; 729 730 mutex_lock(&cq->mutex); 731 732 entries = roundup_pow_of_two(entries + 1); 733 if (entries == ibcq->cqe + 1) { 734 ret = 0; 735 goto out; 736 } 737 738 if (cq->is_kernel) { 739 ret = mthca_alloc_resize_buf(dev, cq, entries); 740 if (ret) 741 goto out; 742 lkey = cq->resize_buf->buf.mr.ibmr.lkey; 743 } else { 744 ret = ib_copy_validate_udata_in(udata, ucmd, reserved); 745 if (ret) 746 goto out; 747 lkey = ucmd.lkey; 748 } 749 750 ret = mthca_RESIZE_CQ(dev, cq->cqn, lkey, ilog2(entries)); 751 752 if (ret) { 753 if (cq->resize_buf) { 754 mthca_free_cq_buf(dev, &cq->resize_buf->buf, 755 cq->resize_buf->cqe); 756 kfree(cq->resize_buf); 757 spin_lock_irq(&cq->lock); 758 cq->resize_buf = NULL; 759 spin_unlock_irq(&cq->lock); 760 } 761 goto out; 762 } 763 764 if (cq->is_kernel) { 765 struct mthca_cq_buf tbuf; 766 int tcqe; 767 768 spin_lock_irq(&cq->lock); 769 if (cq->resize_buf->state == CQ_RESIZE_READY) { 770 mthca_cq_resize_copy_cqes(cq); 771 tbuf = cq->buf; 772 tcqe = cq->ibcq.cqe; 773 cq->buf = cq->resize_buf->buf; 774 cq->ibcq.cqe = cq->resize_buf->cqe; 775 } else { 776 tbuf = cq->resize_buf->buf; 777 tcqe = cq->resize_buf->cqe; 778 } 779 780 kfree(cq->resize_buf); 781 cq->resize_buf = NULL; 782 spin_unlock_irq(&cq->lock); 783 784 mthca_free_cq_buf(dev, &tbuf, tcqe); 785 } else 786 ibcq->cqe = entries - 1; 787 788 out: 789 mutex_unlock(&cq->mutex); 790 791 return ret; 792 } 793 794 static int mthca_destroy_cq(struct ib_cq *cq, struct ib_udata *udata) 795 { 796 if (udata) { 797 struct mthca_ucontext *context = 798 rdma_udata_to_drv_context( 799 udata, 800 struct mthca_ucontext, 801 ibucontext); 802 803 mthca_unmap_user_db(to_mdev(cq->device), 804 &context->uar, 805 context->db_tab, 806 to_mcq(cq)->arm_db_index); 807 mthca_unmap_user_db(to_mdev(cq->device), 808 &context->uar, 809 context->db_tab, 810 to_mcq(cq)->set_ci_db_index); 811 } 812 mthca_free_cq(to_mdev(cq->device), to_mcq(cq)); 813 return 0; 814 } 815 816 static inline u32 convert_access(int acc) 817 { 818 return (acc & IB_ACCESS_REMOTE_ATOMIC ? MTHCA_MPT_FLAG_ATOMIC : 0) | 819 (acc & IB_ACCESS_REMOTE_WRITE ? MTHCA_MPT_FLAG_REMOTE_WRITE : 0) | 820 (acc & IB_ACCESS_REMOTE_READ ? MTHCA_MPT_FLAG_REMOTE_READ : 0) | 821 (acc & IB_ACCESS_LOCAL_WRITE ? MTHCA_MPT_FLAG_LOCAL_WRITE : 0) | 822 MTHCA_MPT_FLAG_LOCAL_READ; 823 } 824 825 static struct ib_mr *mthca_get_dma_mr(struct ib_pd *pd, int acc) 826 { 827 struct mthca_mr *mr; 828 int err; 829 830 mr = kmalloc_obj(*mr); 831 if (!mr) 832 return ERR_PTR(-ENOMEM); 833 834 err = mthca_mr_alloc_notrans(to_mdev(pd->device), 835 to_mpd(pd)->pd_num, 836 convert_access(acc), mr); 837 838 if (err) { 839 kfree(mr); 840 return ERR_PTR(err); 841 } 842 843 mr->umem = NULL; 844 845 return &mr->ibmr; 846 } 847 848 static struct ib_mr *mthca_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, 849 u64 virt, int acc, struct ib_dmah *dmah, 850 struct ib_udata *udata) 851 { 852 struct mthca_dev *dev = to_mdev(pd->device); 853 struct ib_block_iter biter; 854 struct mthca_ucontext *context = rdma_udata_to_drv_context( 855 udata, struct mthca_ucontext, ibucontext); 856 struct mthca_mr *mr; 857 struct mthca_reg_mr ucmd; 858 u64 *pages; 859 int n, i; 860 int err = 0; 861 int write_mtt_size; 862 863 if (dmah) 864 return ERR_PTR(-EOPNOTSUPP); 865 866 if (udata->inlen < sizeof ucmd) { 867 if (!context->reg_mr_warned) { 868 mthca_warn(dev, "Process '%s' did not pass in MR attrs.\n", 869 current->comm); 870 mthca_warn(dev, " Update libmthca to fix this.\n"); 871 } 872 ++context->reg_mr_warned; 873 ucmd.mr_attrs = 0; 874 } else { 875 err = ib_copy_validate_udata_in(udata, ucmd, reserved); 876 if (err) 877 return ERR_PTR(err); 878 } 879 880 mr = kmalloc_obj(*mr); 881 if (!mr) 882 return ERR_PTR(-ENOMEM); 883 884 mr->umem = ib_umem_get_va(pd->device, start, length, acc); 885 if (IS_ERR(mr->umem)) { 886 err = PTR_ERR(mr->umem); 887 goto err; 888 } 889 890 n = ib_umem_num_dma_blocks(mr->umem, PAGE_SIZE); 891 892 mr->mtt = mthca_alloc_mtt(dev, n); 893 if (IS_ERR(mr->mtt)) { 894 err = PTR_ERR(mr->mtt); 895 goto err_umem; 896 } 897 898 pages = (u64 *) __get_free_page(GFP_KERNEL); 899 if (!pages) { 900 err = -ENOMEM; 901 goto err_mtt; 902 } 903 904 i = n = 0; 905 906 write_mtt_size = min(mthca_write_mtt_size(dev), (int) (PAGE_SIZE / sizeof *pages)); 907 908 rdma_umem_for_each_dma_block(mr->umem, &biter, PAGE_SIZE) { 909 pages[i++] = rdma_block_iter_dma_address(&biter); 910 911 /* 912 * Be friendly to write_mtt and pass it chunks 913 * of appropriate size. 914 */ 915 if (i == write_mtt_size) { 916 err = mthca_write_mtt(dev, mr->mtt, n, pages, i); 917 if (err) 918 goto mtt_done; 919 n += i; 920 i = 0; 921 } 922 } 923 924 if (i) 925 err = mthca_write_mtt(dev, mr->mtt, n, pages, i); 926 mtt_done: 927 free_page((unsigned long) pages); 928 if (err) 929 goto err_mtt; 930 931 err = mthca_mr_alloc(dev, to_mpd(pd)->pd_num, PAGE_SHIFT, virt, length, 932 convert_access(acc), mr); 933 934 if (err) 935 goto err_mtt; 936 937 return &mr->ibmr; 938 939 err_mtt: 940 mthca_free_mtt(dev, mr->mtt); 941 942 err_umem: 943 ib_umem_release(mr->umem); 944 945 err: 946 kfree(mr); 947 return ERR_PTR(err); 948 } 949 950 static int mthca_dereg_mr(struct ib_mr *mr, struct ib_udata *udata) 951 { 952 struct mthca_mr *mmr = to_mmr(mr); 953 954 mthca_free_mr(to_mdev(mr->device), mmr); 955 ib_umem_release(mmr->umem); 956 kfree(mmr); 957 958 return 0; 959 } 960 961 static ssize_t hw_rev_show(struct device *device, 962 struct device_attribute *attr, char *buf) 963 { 964 struct mthca_dev *dev = 965 rdma_device_to_drv_device(device, struct mthca_dev, ib_dev); 966 967 return sysfs_emit(buf, "%x\n", dev->rev_id); 968 } 969 static DEVICE_ATTR_RO(hw_rev); 970 971 static const char *hca_type_string(int hca_type) 972 { 973 switch (hca_type) { 974 case PCI_DEVICE_ID_MELLANOX_TAVOR: 975 return "MT23108"; 976 case PCI_DEVICE_ID_MELLANOX_ARBEL_COMPAT: 977 return "MT25208 (MT23108 compat mode)"; 978 case PCI_DEVICE_ID_MELLANOX_ARBEL: 979 return "MT25208"; 980 case PCI_DEVICE_ID_MELLANOX_SINAI: 981 case PCI_DEVICE_ID_MELLANOX_SINAI_OLD: 982 return "MT25204"; 983 } 984 985 return "unknown"; 986 } 987 988 static ssize_t hca_type_show(struct device *device, 989 struct device_attribute *attr, char *buf) 990 { 991 struct mthca_dev *dev = 992 rdma_device_to_drv_device(device, struct mthca_dev, ib_dev); 993 994 return sysfs_emit(buf, "%s\n", hca_type_string(dev->pdev->device)); 995 } 996 static DEVICE_ATTR_RO(hca_type); 997 998 static ssize_t board_id_show(struct device *device, 999 struct device_attribute *attr, char *buf) 1000 { 1001 struct mthca_dev *dev = 1002 rdma_device_to_drv_device(device, struct mthca_dev, ib_dev); 1003 1004 return sysfs_emit(buf, "%.*s\n", MTHCA_BOARD_ID_LEN, dev->board_id); 1005 } 1006 static DEVICE_ATTR_RO(board_id); 1007 1008 static struct attribute *mthca_dev_attributes[] = { 1009 &dev_attr_hw_rev.attr, 1010 &dev_attr_hca_type.attr, 1011 &dev_attr_board_id.attr, 1012 NULL 1013 }; 1014 1015 static const struct attribute_group mthca_attr_group = { 1016 .attrs = mthca_dev_attributes, 1017 }; 1018 1019 static int mthca_init_node_data(struct mthca_dev *dev) 1020 { 1021 struct ib_smp *in_mad; 1022 struct ib_smp *out_mad; 1023 int err = -ENOMEM; 1024 1025 in_mad = kzalloc_obj(*in_mad); 1026 out_mad = kmalloc_obj(*out_mad); 1027 if (!in_mad || !out_mad) 1028 goto out; 1029 1030 ib_init_query_mad(in_mad); 1031 in_mad->attr_id = IB_SMP_ATTR_NODE_DESC; 1032 1033 err = mthca_MAD_IFC(dev, 1, 1, 1034 1, NULL, NULL, in_mad, out_mad); 1035 if (err) 1036 goto out; 1037 1038 memcpy(dev->ib_dev.node_desc, out_mad->data, IB_DEVICE_NODE_DESC_MAX); 1039 1040 in_mad->attr_id = IB_SMP_ATTR_NODE_INFO; 1041 1042 err = mthca_MAD_IFC(dev, 1, 1, 1043 1, NULL, NULL, in_mad, out_mad); 1044 if (err) 1045 goto out; 1046 1047 if (mthca_is_memfree(dev)) 1048 dev->rev_id = be32_to_cpup((__be32 *) (out_mad->data + 32)); 1049 memcpy(&dev->ib_dev.node_guid, out_mad->data + 12, 8); 1050 1051 out: 1052 kfree(in_mad); 1053 kfree(out_mad); 1054 return err; 1055 } 1056 1057 static int mthca_port_immutable(struct ib_device *ibdev, u32 port_num, 1058 struct ib_port_immutable *immutable) 1059 { 1060 struct ib_port_attr attr; 1061 int err; 1062 1063 immutable->core_cap_flags = RDMA_CORE_PORT_IBA_IB; 1064 1065 err = ib_query_port(ibdev, port_num, &attr); 1066 if (err) 1067 return err; 1068 1069 immutable->pkey_tbl_len = attr.pkey_tbl_len; 1070 immutable->gid_tbl_len = attr.gid_tbl_len; 1071 immutable->max_mad_size = IB_MGMT_MAD_SIZE; 1072 1073 return 0; 1074 } 1075 1076 static void get_dev_fw_str(struct ib_device *device, char *str) 1077 { 1078 struct mthca_dev *dev = 1079 container_of(device, struct mthca_dev, ib_dev); 1080 snprintf(str, IB_FW_VERSION_NAME_MAX, "%d.%d.%d", 1081 (int) (dev->fw_ver >> 32), 1082 (int) (dev->fw_ver >> 16) & 0xffff, 1083 (int) dev->fw_ver & 0xffff); 1084 } 1085 1086 static const struct ib_device_ops mthca_dev_ops = { 1087 .owner = THIS_MODULE, 1088 .driver_id = RDMA_DRIVER_MTHCA, 1089 .uverbs_abi_ver = MTHCA_UVERBS_ABI_VERSION, 1090 .uverbs_no_driver_id_binding = 1, 1091 1092 .alloc_pd = mthca_alloc_pd, 1093 .alloc_ucontext = mthca_alloc_ucontext, 1094 .attach_mcast = mthca_multicast_attach, 1095 .create_ah = mthca_ah_create, 1096 .create_cq = mthca_create_cq, 1097 .create_qp = mthca_create_qp, 1098 .dealloc_pd = mthca_dealloc_pd, 1099 .dealloc_ucontext = mthca_dealloc_ucontext, 1100 .dereg_mr = mthca_dereg_mr, 1101 .destroy_ah = mthca_ah_destroy, 1102 .destroy_cq = mthca_destroy_cq, 1103 .destroy_qp = mthca_destroy_qp, 1104 .detach_mcast = mthca_multicast_detach, 1105 .device_group = &mthca_attr_group, 1106 .get_dev_fw_str = get_dev_fw_str, 1107 .get_dma_mr = mthca_get_dma_mr, 1108 .get_port_immutable = mthca_port_immutable, 1109 .mmap = mthca_mmap_uar, 1110 .modify_device = mthca_modify_device, 1111 .modify_port = mthca_modify_port, 1112 .modify_qp = mthca_modify_qp, 1113 .poll_cq = mthca_poll_cq, 1114 .process_mad = mthca_process_mad, 1115 .query_ah = mthca_ah_query, 1116 .query_device = mthca_query_device, 1117 .query_gid = mthca_query_gid, 1118 .query_pkey = mthca_query_pkey, 1119 .query_port = mthca_query_port, 1120 .query_qp = mthca_query_qp, 1121 .reg_user_mr = mthca_reg_user_mr, 1122 .resize_user_cq = mthca_resize_cq, 1123 1124 INIT_RDMA_OBJ_SIZE(ib_ah, mthca_ah, ibah), 1125 INIT_RDMA_OBJ_SIZE(ib_cq, mthca_cq, ibcq), 1126 INIT_RDMA_OBJ_SIZE(ib_pd, mthca_pd, ibpd), 1127 INIT_RDMA_OBJ_SIZE(ib_qp, mthca_qp, ibqp), 1128 INIT_RDMA_OBJ_SIZE(ib_ucontext, mthca_ucontext, ibucontext), 1129 }; 1130 1131 static const struct ib_device_ops mthca_dev_arbel_srq_ops = { 1132 .create_srq = mthca_create_srq, 1133 .destroy_srq = mthca_destroy_srq, 1134 .modify_srq = mthca_modify_srq, 1135 .post_srq_recv = mthca_arbel_post_srq_recv, 1136 .query_srq = mthca_query_srq, 1137 1138 INIT_RDMA_OBJ_SIZE(ib_srq, mthca_srq, ibsrq), 1139 }; 1140 1141 static const struct ib_device_ops mthca_dev_tavor_srq_ops = { 1142 .create_srq = mthca_create_srq, 1143 .destroy_srq = mthca_destroy_srq, 1144 .modify_srq = mthca_modify_srq, 1145 .post_srq_recv = mthca_tavor_post_srq_recv, 1146 .query_srq = mthca_query_srq, 1147 1148 INIT_RDMA_OBJ_SIZE(ib_srq, mthca_srq, ibsrq), 1149 }; 1150 1151 static const struct ib_device_ops mthca_dev_arbel_ops = { 1152 .post_recv = mthca_arbel_post_receive, 1153 .post_send = mthca_arbel_post_send, 1154 .req_notify_cq = mthca_arbel_arm_cq, 1155 }; 1156 1157 static const struct ib_device_ops mthca_dev_tavor_ops = { 1158 .post_recv = mthca_tavor_post_receive, 1159 .post_send = mthca_tavor_post_send, 1160 .req_notify_cq = mthca_tavor_arm_cq, 1161 }; 1162 1163 int mthca_register_device(struct mthca_dev *dev) 1164 { 1165 int ret; 1166 1167 ret = mthca_init_node_data(dev); 1168 if (ret) 1169 return ret; 1170 1171 dev->ib_dev.node_type = RDMA_NODE_IB_CA; 1172 dev->ib_dev.phys_port_cnt = dev->limits.num_ports; 1173 dev->ib_dev.num_comp_vectors = 1; 1174 dev->ib_dev.dev.parent = &dev->pdev->dev; 1175 1176 if (dev->mthca_flags & MTHCA_FLAG_SRQ) { 1177 if (mthca_is_memfree(dev)) 1178 ib_set_device_ops(&dev->ib_dev, 1179 &mthca_dev_arbel_srq_ops); 1180 else 1181 ib_set_device_ops(&dev->ib_dev, 1182 &mthca_dev_tavor_srq_ops); 1183 } 1184 1185 ib_set_device_ops(&dev->ib_dev, &mthca_dev_ops); 1186 1187 if (mthca_is_memfree(dev)) 1188 ib_set_device_ops(&dev->ib_dev, &mthca_dev_arbel_ops); 1189 else 1190 ib_set_device_ops(&dev->ib_dev, &mthca_dev_tavor_ops); 1191 1192 mutex_init(&dev->cap_mask_mutex); 1193 1194 ret = ib_register_device(&dev->ib_dev, "mthca%d", &dev->pdev->dev); 1195 if (ret) 1196 return ret; 1197 1198 mthca_start_catas_poll(dev); 1199 1200 return 0; 1201 } 1202 1203 void mthca_unregister_device(struct mthca_dev *dev) 1204 { 1205 mthca_stop_catas_poll(dev); 1206 ib_unregister_device(&dev->ib_dev); 1207 } 1208