1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2022, Microsoft Corporation. All rights reserved. 4 */ 5 6 #include "mana_ib.h" 7 #include "linux/pci.h" 8 9 void mana_ib_uncfg_vport(struct mana_ib_dev *dev, struct mana_ib_pd *pd, 10 u32 port) 11 { 12 struct mana_port_context *mpc; 13 struct net_device *ndev; 14 15 ndev = mana_ib_get_netdev(&dev->ib_dev, port); 16 mpc = netdev_priv(ndev); 17 18 mutex_lock(&pd->vport_mutex); 19 20 pd->vport_use_count--; 21 WARN_ON(pd->vport_use_count < 0); 22 23 if (!pd->vport_use_count) 24 mana_uncfg_vport(mpc); 25 26 mutex_unlock(&pd->vport_mutex); 27 } 28 29 int mana_ib_cfg_vport(struct mana_ib_dev *dev, u32 port, struct mana_ib_pd *pd, 30 u32 doorbell_id) 31 { 32 struct mana_port_context *mpc; 33 struct net_device *ndev; 34 int err; 35 36 ndev = mana_ib_get_netdev(&dev->ib_dev, port); 37 mpc = netdev_priv(ndev); 38 39 mutex_lock(&pd->vport_mutex); 40 41 pd->vport_use_count++; 42 if (pd->vport_use_count > 1) { 43 ibdev_dbg(&dev->ib_dev, 44 "Skip as this PD is already configured vport\n"); 45 mutex_unlock(&pd->vport_mutex); 46 return 0; 47 } 48 49 err = mana_cfg_vport(mpc, pd->pdn, doorbell_id); 50 if (err) { 51 pd->vport_use_count--; 52 mutex_unlock(&pd->vport_mutex); 53 54 ibdev_dbg(&dev->ib_dev, "Failed to configure vPort %d\n", err); 55 return err; 56 } 57 58 mutex_unlock(&pd->vport_mutex); 59 60 pd->tx_shortform_allowed = mpc->tx_shortform_allowed; 61 pd->tx_vp_offset = mpc->tx_vp_offset; 62 63 ibdev_dbg(&dev->ib_dev, "vport handle %llx pdid %x doorbell_id %x\n", 64 mpc->port_handle, pd->pdn, doorbell_id); 65 66 return 0; 67 } 68 69 int mana_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 70 { 71 struct mana_ib_pd *pd = container_of(ibpd, struct mana_ib_pd, ibpd); 72 struct ib_device *ibdev = ibpd->device; 73 struct gdma_create_pd_resp resp = {}; 74 struct gdma_create_pd_req req = {}; 75 enum gdma_pd_flags flags = 0; 76 struct mana_ib_dev *dev; 77 struct gdma_context *gc; 78 int err; 79 80 dev = container_of(ibdev, struct mana_ib_dev, ib_dev); 81 gc = mdev_to_gc(dev); 82 83 mana_gd_init_req_hdr(&req.hdr, GDMA_CREATE_PD, sizeof(req), 84 sizeof(resp)); 85 86 if (!udata) 87 flags |= GDMA_PD_FLAG_ALLOW_GPA_MR; 88 89 req.flags = flags; 90 err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); 91 if (err) 92 return err; 93 94 pd->pd_handle = resp.pd_handle; 95 pd->pdn = resp.pd_id; 96 ibdev_dbg(&dev->ib_dev, "pd_handle 0x%llx pd_id %d\n", 97 pd->pd_handle, pd->pdn); 98 99 mutex_init(&pd->vport_mutex); 100 pd->vport_use_count = 0; 101 return 0; 102 } 103 104 int mana_ib_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 105 { 106 struct mana_ib_pd *pd = container_of(ibpd, struct mana_ib_pd, ibpd); 107 struct ib_device *ibdev = ibpd->device; 108 struct gdma_destory_pd_resp resp = {}; 109 struct gdma_destroy_pd_req req = {}; 110 struct mana_ib_dev *dev; 111 struct gdma_context *gc; 112 113 dev = container_of(ibdev, struct mana_ib_dev, ib_dev); 114 gc = mdev_to_gc(dev); 115 116 mana_gd_init_req_hdr(&req.hdr, GDMA_DESTROY_PD, sizeof(req), 117 sizeof(resp)); 118 119 req.pd_handle = pd->pd_handle; 120 121 return mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); 122 } 123 124 static int mana_gd_destroy_doorbell_page(struct gdma_context *gc, 125 int doorbell_page) 126 { 127 struct gdma_destroy_resource_range_req req = {}; 128 struct gdma_resp_hdr resp = {}; 129 130 mana_gd_init_req_hdr(&req.hdr, GDMA_DESTROY_RESOURCE_RANGE, 131 sizeof(req), sizeof(resp)); 132 133 req.resource_type = GDMA_RESOURCE_DOORBELL_PAGE; 134 req.num_resources = 1; 135 req.allocated_resources = doorbell_page; 136 137 return mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); 138 } 139 140 static int mana_gd_allocate_doorbell_page(struct gdma_context *gc, 141 int *doorbell_page) 142 { 143 struct gdma_allocate_resource_range_req req = {}; 144 struct gdma_allocate_resource_range_resp resp = {}; 145 int err; 146 147 mana_gd_init_req_hdr(&req.hdr, GDMA_ALLOCATE_RESOURCE_RANGE, 148 sizeof(req), sizeof(resp)); 149 150 req.resource_type = GDMA_RESOURCE_DOORBELL_PAGE; 151 req.num_resources = 1; 152 req.alignment = PAGE_SIZE / MANA_PAGE_SIZE; 153 154 /* Have GDMA start searching from 0 */ 155 req.allocated_resources = 0; 156 157 err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); 158 if (err) 159 return err; 160 161 *doorbell_page = resp.allocated_resources; 162 163 return 0; 164 } 165 166 int mana_ib_alloc_ucontext(struct ib_ucontext *ibcontext, 167 struct ib_udata *udata) 168 { 169 struct mana_ib_ucontext *ucontext = 170 container_of(ibcontext, struct mana_ib_ucontext, ibucontext); 171 struct ib_device *ibdev = ibcontext->device; 172 struct mana_ib_dev *mdev; 173 struct gdma_context *gc; 174 int doorbell_page; 175 int ret; 176 177 mdev = container_of(ibdev, struct mana_ib_dev, ib_dev); 178 gc = mdev_to_gc(mdev); 179 180 /* Allocate a doorbell page index */ 181 ret = mana_gd_allocate_doorbell_page(gc, &doorbell_page); 182 if (ret) { 183 ibdev_dbg(ibdev, "Failed to allocate doorbell page %d\n", ret); 184 return ret; 185 } 186 187 ibdev_dbg(ibdev, "Doorbell page allocated %d\n", doorbell_page); 188 189 ucontext->doorbell = doorbell_page; 190 191 return 0; 192 } 193 194 void mana_ib_dealloc_ucontext(struct ib_ucontext *ibcontext) 195 { 196 struct mana_ib_ucontext *mana_ucontext = 197 container_of(ibcontext, struct mana_ib_ucontext, ibucontext); 198 struct ib_device *ibdev = ibcontext->device; 199 struct mana_ib_dev *mdev; 200 struct gdma_context *gc; 201 int ret; 202 203 mdev = container_of(ibdev, struct mana_ib_dev, ib_dev); 204 gc = mdev_to_gc(mdev); 205 206 ret = mana_gd_destroy_doorbell_page(gc, mana_ucontext->doorbell); 207 if (ret) 208 ibdev_dbg(ibdev, "Failed to destroy doorbell page %d\n", ret); 209 } 210 211 int mana_ib_create_kernel_queue(struct mana_ib_dev *mdev, u32 size, enum gdma_queue_type type, 212 struct mana_ib_queue *queue) 213 { 214 struct gdma_queue_spec spec = {}; 215 int err; 216 217 queue->id = INVALID_QUEUE_ID; 218 queue->gdma_region = GDMA_INVALID_DMA_REGION; 219 spec.type = type; 220 spec.monitor_avl_buf = false; 221 spec.queue_size = size; 222 err = mana_gd_create_mana_wq_cq(mdev->gdma_dev, &spec, &queue->kmem); 223 if (err) 224 return err; 225 /* take ownership into mana_ib from mana */ 226 queue->gdma_region = queue->kmem->mem_info.dma_region_handle; 227 queue->kmem->mem_info.dma_region_handle = GDMA_INVALID_DMA_REGION; 228 return 0; 229 } 230 231 int mana_ib_create_queue(struct mana_ib_dev *mdev, u64 addr, u32 size, 232 struct mana_ib_queue *queue) 233 { 234 struct ib_umem *umem; 235 int err; 236 237 queue->umem = NULL; 238 queue->id = INVALID_QUEUE_ID; 239 queue->gdma_region = GDMA_INVALID_DMA_REGION; 240 241 umem = ib_umem_get(&mdev->ib_dev, addr, size, IB_ACCESS_LOCAL_WRITE); 242 if (IS_ERR(umem)) { 243 ibdev_dbg(&mdev->ib_dev, "Failed to get umem, %pe\n", umem); 244 return PTR_ERR(umem); 245 } 246 247 err = mana_ib_create_zero_offset_dma_region(mdev, umem, &queue->gdma_region); 248 if (err) { 249 ibdev_dbg(&mdev->ib_dev, "Failed to create dma region, %d\n", err); 250 goto free_umem; 251 } 252 queue->umem = umem; 253 254 ibdev_dbg(&mdev->ib_dev, "created dma region 0x%llx\n", queue->gdma_region); 255 256 return 0; 257 free_umem: 258 ib_umem_release(umem); 259 return err; 260 } 261 262 void mana_ib_destroy_queue(struct mana_ib_dev *mdev, struct mana_ib_queue *queue) 263 { 264 /* Ignore return code as there is not much we can do about it. 265 * The error message is printed inside. 266 */ 267 mana_ib_gd_destroy_dma_region(mdev, queue->gdma_region); 268 ib_umem_release(queue->umem); 269 if (queue->kmem) 270 mana_gd_destroy_queue(mdev_to_gc(mdev), queue->kmem); 271 } 272 273 static int 274 mana_ib_gd_first_dma_region(struct mana_ib_dev *dev, 275 struct gdma_context *gc, 276 struct gdma_create_dma_region_req *create_req, 277 size_t num_pages, mana_handle_t *gdma_region, 278 u32 expected_status) 279 { 280 struct gdma_create_dma_region_resp create_resp = {}; 281 unsigned int create_req_msg_size; 282 int err; 283 284 create_req_msg_size = 285 struct_size(create_req, page_addr_list, num_pages); 286 create_req->page_addr_list_len = num_pages; 287 288 err = mana_gd_send_request(gc, create_req_msg_size, create_req, 289 sizeof(create_resp), &create_resp); 290 if (err || create_resp.hdr.status != expected_status) { 291 ibdev_dbg(&dev->ib_dev, 292 "Failed to create DMA region: %d, 0x%x\n", 293 err, create_resp.hdr.status); 294 if (!err) 295 err = -EPROTO; 296 297 return err; 298 } 299 300 *gdma_region = create_resp.dma_region_handle; 301 ibdev_dbg(&dev->ib_dev, "Created DMA region handle 0x%llx\n", 302 *gdma_region); 303 304 return 0; 305 } 306 307 static int 308 mana_ib_gd_add_dma_region(struct mana_ib_dev *dev, struct gdma_context *gc, 309 struct gdma_dma_region_add_pages_req *add_req, 310 unsigned int num_pages, u32 expected_status) 311 { 312 unsigned int add_req_msg_size = 313 struct_size(add_req, page_addr_list, num_pages); 314 struct gdma_general_resp add_resp = {}; 315 int err; 316 317 mana_gd_init_req_hdr(&add_req->hdr, GDMA_DMA_REGION_ADD_PAGES, 318 add_req_msg_size, sizeof(add_resp)); 319 add_req->page_addr_list_len = num_pages; 320 321 err = mana_gd_send_request(gc, add_req_msg_size, add_req, 322 sizeof(add_resp), &add_resp); 323 if (err || add_resp.hdr.status != expected_status) { 324 ibdev_dbg(&dev->ib_dev, 325 "Failed to create DMA region: %d, 0x%x\n", 326 err, add_resp.hdr.status); 327 328 if (!err) 329 err = -EPROTO; 330 331 return err; 332 } 333 334 return 0; 335 } 336 337 static int mana_ib_gd_create_dma_region(struct mana_ib_dev *dev, struct ib_umem *umem, 338 mana_handle_t *gdma_region, unsigned long page_sz) 339 { 340 struct gdma_dma_region_add_pages_req *add_req = NULL; 341 size_t num_pages_processed = 0, num_pages_to_handle; 342 struct gdma_create_dma_region_req *create_req; 343 unsigned int create_req_msg_size; 344 struct hw_channel_context *hwc; 345 struct ib_block_iter biter; 346 size_t max_pgs_add_cmd = 0; 347 size_t max_pgs_create_cmd; 348 struct gdma_context *gc; 349 size_t num_pages_total; 350 unsigned int tail = 0; 351 u64 *page_addr_list; 352 void *request_buf; 353 int err = 0; 354 355 gc = mdev_to_gc(dev); 356 hwc = gc->hwc.driver_data; 357 358 num_pages_total = ib_umem_num_dma_blocks(umem, page_sz); 359 360 max_pgs_create_cmd = 361 (hwc->max_req_msg_size - sizeof(*create_req)) / sizeof(u64); 362 num_pages_to_handle = 363 min_t(size_t, num_pages_total, max_pgs_create_cmd); 364 create_req_msg_size = 365 struct_size(create_req, page_addr_list, num_pages_to_handle); 366 367 request_buf = kzalloc(hwc->max_req_msg_size, GFP_KERNEL); 368 if (!request_buf) 369 return -ENOMEM; 370 371 create_req = request_buf; 372 mana_gd_init_req_hdr(&create_req->hdr, GDMA_CREATE_DMA_REGION, 373 create_req_msg_size, 374 sizeof(struct gdma_create_dma_region_resp)); 375 376 create_req->length = umem->length; 377 create_req->offset_in_page = ib_umem_dma_offset(umem, page_sz); 378 create_req->gdma_page_type = order_base_2(page_sz) - MANA_PAGE_SHIFT; 379 create_req->page_count = num_pages_total; 380 381 ibdev_dbg(&dev->ib_dev, "size_dma_region %lu num_pages_total %lu\n", 382 umem->length, num_pages_total); 383 384 ibdev_dbg(&dev->ib_dev, "page_sz %lu offset_in_page %u\n", 385 page_sz, create_req->offset_in_page); 386 387 ibdev_dbg(&dev->ib_dev, "num_pages_to_handle %lu, gdma_page_type %u", 388 num_pages_to_handle, create_req->gdma_page_type); 389 390 page_addr_list = create_req->page_addr_list; 391 rdma_umem_for_each_dma_block(umem, &biter, page_sz) { 392 u32 expected_status = 0; 393 394 page_addr_list[tail++] = rdma_block_iter_dma_address(&biter); 395 if (tail < num_pages_to_handle) 396 continue; 397 398 if (num_pages_processed + num_pages_to_handle < 399 num_pages_total) 400 expected_status = GDMA_STATUS_MORE_ENTRIES; 401 402 if (!num_pages_processed) { 403 /* First create message */ 404 err = mana_ib_gd_first_dma_region(dev, gc, create_req, 405 tail, gdma_region, 406 expected_status); 407 if (err) 408 goto out; 409 410 max_pgs_add_cmd = (hwc->max_req_msg_size - 411 sizeof(*add_req)) / sizeof(u64); 412 413 add_req = request_buf; 414 add_req->dma_region_handle = *gdma_region; 415 add_req->reserved3 = 0; 416 page_addr_list = add_req->page_addr_list; 417 } else { 418 /* Subsequent create messages */ 419 err = mana_ib_gd_add_dma_region(dev, gc, add_req, tail, 420 expected_status); 421 if (err) 422 break; 423 } 424 425 num_pages_processed += tail; 426 tail = 0; 427 428 /* The remaining pages to create */ 429 num_pages_to_handle = 430 min_t(size_t, 431 num_pages_total - num_pages_processed, 432 max_pgs_add_cmd); 433 } 434 435 if (err) 436 mana_ib_gd_destroy_dma_region(dev, *gdma_region); 437 438 out: 439 kfree(request_buf); 440 return err; 441 } 442 443 int mana_ib_create_dma_region(struct mana_ib_dev *dev, struct ib_umem *umem, 444 mana_handle_t *gdma_region, u64 virt) 445 { 446 unsigned long page_sz; 447 448 page_sz = ib_umem_find_best_pgsz(umem, dev->adapter_caps.page_size_cap, virt); 449 if (!page_sz) { 450 ibdev_dbg(&dev->ib_dev, "Failed to find page size.\n"); 451 return -EINVAL; 452 } 453 454 return mana_ib_gd_create_dma_region(dev, umem, gdma_region, page_sz); 455 } 456 457 int mana_ib_create_zero_offset_dma_region(struct mana_ib_dev *dev, struct ib_umem *umem, 458 mana_handle_t *gdma_region) 459 { 460 unsigned long page_sz; 461 462 /* Hardware requires dma region to align to chosen page size */ 463 page_sz = ib_umem_find_best_pgoff(umem, dev->adapter_caps.page_size_cap, 0); 464 if (!page_sz) { 465 ibdev_dbg(&dev->ib_dev, "Failed to find page size.\n"); 466 return -EINVAL; 467 } 468 469 return mana_ib_gd_create_dma_region(dev, umem, gdma_region, page_sz); 470 } 471 472 int mana_ib_gd_destroy_dma_region(struct mana_ib_dev *dev, u64 gdma_region) 473 { 474 struct gdma_context *gc = mdev_to_gc(dev); 475 476 ibdev_dbg(&dev->ib_dev, "destroy dma region 0x%llx\n", gdma_region); 477 478 return mana_gd_destroy_dma_region(gc, gdma_region); 479 } 480 481 int mana_ib_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vma) 482 { 483 struct mana_ib_ucontext *mana_ucontext = 484 container_of(ibcontext, struct mana_ib_ucontext, ibucontext); 485 struct ib_device *ibdev = ibcontext->device; 486 struct mana_ib_dev *mdev; 487 struct gdma_context *gc; 488 phys_addr_t pfn; 489 pgprot_t prot; 490 int ret; 491 492 mdev = container_of(ibdev, struct mana_ib_dev, ib_dev); 493 gc = mdev_to_gc(mdev); 494 495 if (vma->vm_pgoff != 0) { 496 ibdev_dbg(ibdev, "Unexpected vm_pgoff %lu\n", vma->vm_pgoff); 497 return -EINVAL; 498 } 499 500 /* Map to the page indexed by ucontext->doorbell */ 501 pfn = (gc->phys_db_page_base + 502 gc->db_page_size * mana_ucontext->doorbell) >> 503 PAGE_SHIFT; 504 prot = pgprot_writecombine(vma->vm_page_prot); 505 506 ret = rdma_user_mmap_io(ibcontext, vma, pfn, PAGE_SIZE, prot, 507 NULL); 508 if (ret) 509 ibdev_dbg(ibdev, "can't rdma_user_mmap_io ret %d\n", ret); 510 else 511 ibdev_dbg(ibdev, "mapped I/O pfn 0x%llx page_size %lu, ret %d\n", 512 pfn, PAGE_SIZE, ret); 513 514 return ret; 515 } 516 517 int mana_ib_get_port_immutable(struct ib_device *ibdev, u32 port_num, 518 struct ib_port_immutable *immutable) 519 { 520 struct mana_ib_dev *dev = container_of(ibdev, struct mana_ib_dev, ib_dev); 521 struct ib_port_attr attr; 522 int err; 523 524 err = ib_query_port(ibdev, port_num, &attr); 525 if (err) 526 return err; 527 528 immutable->pkey_tbl_len = attr.pkey_tbl_len; 529 immutable->gid_tbl_len = attr.gid_tbl_len; 530 531 if (mana_ib_is_rnic(dev)) { 532 if (port_num == 1) { 533 immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP; 534 immutable->max_mad_size = IB_MGMT_MAD_SIZE; 535 } else { 536 immutable->core_cap_flags = RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP 537 | RDMA_CORE_CAP_ETH_AH; 538 immutable->max_mad_size = 0; 539 } 540 } else { 541 immutable->core_cap_flags = RDMA_CORE_PORT_RAW_PACKET; 542 } 543 544 return 0; 545 } 546 547 int mana_ib_query_device(struct ib_device *ibdev, struct ib_device_attr *props, 548 struct ib_udata *uhw) 549 { 550 struct mana_ib_dev *dev = container_of(ibdev, struct mana_ib_dev, ib_dev); 551 struct pci_dev *pdev = to_pci_dev(mdev_to_gc(dev)->dev); 552 553 memset(props, 0, sizeof(*props)); 554 props->vendor_id = pdev->vendor; 555 props->vendor_part_id = dev->gdma_dev->dev_id.type; 556 props->max_mr_size = MANA_IB_MAX_MR_SIZE; 557 props->page_size_cap = dev->adapter_caps.page_size_cap; 558 props->max_qp = dev->adapter_caps.max_qp_count; 559 props->max_qp_wr = dev->adapter_caps.max_qp_wr; 560 props->device_cap_flags = IB_DEVICE_RC_RNR_NAK_GEN; 561 props->max_send_sge = dev->adapter_caps.max_send_sge_count; 562 props->max_recv_sge = dev->adapter_caps.max_recv_sge_count; 563 props->max_sge_rd = dev->adapter_caps.max_recv_sge_count; 564 props->max_cq = dev->adapter_caps.max_cq_count; 565 props->max_cqe = dev->adapter_caps.max_qp_wr; 566 props->max_mr = dev->adapter_caps.max_mr_count; 567 props->max_pd = dev->adapter_caps.max_pd_count; 568 props->max_qp_rd_atom = dev->adapter_caps.max_inbound_read_limit; 569 props->max_res_rd_atom = props->max_qp_rd_atom * props->max_qp; 570 props->max_qp_init_rd_atom = dev->adapter_caps.max_outbound_read_limit; 571 props->atomic_cap = IB_ATOMIC_NONE; 572 props->masked_atomic_cap = IB_ATOMIC_NONE; 573 props->max_ah = INT_MAX; 574 props->max_pkeys = 1; 575 props->local_ca_ack_delay = MANA_CA_ACK_DELAY; 576 if (!mana_ib_is_rnic(dev)) 577 props->raw_packet_caps = IB_RAW_PACKET_CAP_IP_CSUM; 578 579 return 0; 580 } 581 582 int mana_ib_query_port(struct ib_device *ibdev, u32 port, 583 struct ib_port_attr *props) 584 { 585 struct mana_ib_dev *dev = container_of(ibdev, struct mana_ib_dev, ib_dev); 586 struct net_device *ndev = mana_ib_get_netdev(ibdev, port); 587 588 if (!ndev) 589 return -EINVAL; 590 591 memset(props, 0, sizeof(*props)); 592 props->max_mtu = IB_MTU_4096; 593 props->active_mtu = ib_mtu_int_to_enum(ndev->mtu); 594 595 if (netif_carrier_ok(ndev) && netif_running(ndev)) { 596 props->state = IB_PORT_ACTIVE; 597 props->phys_state = IB_PORT_PHYS_STATE_LINK_UP; 598 } else { 599 props->state = IB_PORT_DOWN; 600 props->phys_state = IB_PORT_PHYS_STATE_DISABLED; 601 } 602 603 props->active_width = IB_WIDTH_4X; 604 props->active_speed = IB_SPEED_EDR; 605 props->pkey_tbl_len = 1; 606 if (mana_ib_is_rnic(dev)) { 607 props->gid_tbl_len = 16; 608 props->ip_gids = true; 609 if (port == 1) 610 props->port_cap_flags = IB_PORT_CM_SUP; 611 } 612 613 return 0; 614 } 615 616 enum rdma_link_layer mana_ib_get_link_layer(struct ib_device *device, u32 port_num) 617 { 618 return IB_LINK_LAYER_ETHERNET; 619 } 620 621 int mana_ib_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey) 622 { 623 if (index != 0) 624 return -EINVAL; 625 *pkey = IB_DEFAULT_PKEY_FULL; 626 return 0; 627 } 628 629 int mana_ib_query_gid(struct ib_device *ibdev, u32 port, int index, 630 union ib_gid *gid) 631 { 632 /* This version doesn't return GID properties */ 633 return 0; 634 } 635 636 void mana_ib_disassociate_ucontext(struct ib_ucontext *ibcontext) 637 { 638 } 639 640 int mana_ib_gd_query_adapter_caps(struct mana_ib_dev *dev) 641 { 642 struct mana_ib_adapter_caps *caps = &dev->adapter_caps; 643 struct mana_ib_query_adapter_caps_resp resp = {}; 644 struct mana_ib_query_adapter_caps_req req = {}; 645 int err; 646 647 mana_gd_init_req_hdr(&req.hdr, MANA_IB_GET_ADAPTER_CAP, sizeof(req), 648 sizeof(resp)); 649 req.hdr.resp.msg_version = GDMA_MESSAGE_V4; 650 req.hdr.dev_id = dev->gdma_dev->dev_id; 651 652 err = mana_gd_send_request(mdev_to_gc(dev), sizeof(req), &req, 653 sizeof(resp), &resp); 654 if (err) 655 return err; 656 657 caps->max_sq_id = resp.max_sq_id; 658 caps->max_rq_id = resp.max_rq_id; 659 caps->max_cq_id = resp.max_cq_id; 660 caps->max_qp_count = resp.max_qp_count; 661 caps->max_cq_count = resp.max_cq_count; 662 caps->max_mr_count = resp.max_mr_count; 663 caps->max_pd_count = resp.max_pd_count; 664 caps->max_inbound_read_limit = resp.max_inbound_read_limit; 665 caps->max_outbound_read_limit = resp.max_outbound_read_limit; 666 caps->mw_count = resp.mw_count; 667 caps->max_srq_count = resp.max_srq_count; 668 caps->max_qp_wr = min_t(u32, 669 resp.max_requester_sq_size / GDMA_MAX_SQE_SIZE, 670 resp.max_requester_rq_size / GDMA_MAX_RQE_SIZE); 671 caps->max_inline_data_size = resp.max_inline_data_size; 672 caps->max_send_sge_count = resp.max_send_sge_count; 673 caps->max_recv_sge_count = resp.max_recv_sge_count; 674 caps->feature_flags = resp.feature_flags; 675 676 caps->page_size_cap = PAGE_SZ_BM; 677 if (mdev_to_gc(dev)->pf_cap_flags1 & GDMA_DRV_CAP_FLAG_1_GDMA_PAGES_4MB_1GB_2GB) 678 caps->page_size_cap |= (SZ_4M | SZ_1G | SZ_2G); 679 680 return 0; 681 } 682 683 int mana_eth_query_adapter_caps(struct mana_ib_dev *dev) 684 { 685 struct mana_ib_adapter_caps *caps = &dev->adapter_caps; 686 struct gdma_query_max_resources_resp resp = {}; 687 struct gdma_general_req req = {}; 688 int err; 689 690 mana_gd_init_req_hdr(&req.hdr, GDMA_QUERY_MAX_RESOURCES, 691 sizeof(req), sizeof(resp)); 692 693 err = mana_gd_send_request(mdev_to_gc(dev), sizeof(req), &req, 694 sizeof(resp), &resp); 695 if (err) 696 return err; 697 698 caps->max_qp_count = min_t(u32, resp.max_sq, resp.max_rq); 699 caps->max_cq_count = resp.max_cq; 700 caps->max_mr_count = resp.max_mst; 701 caps->max_pd_count = 0x6000; 702 caps->max_qp_wr = min_t(u32, 703 0x100000 / GDMA_MAX_SQE_SIZE, 704 0x100000 / GDMA_MAX_RQE_SIZE); 705 caps->max_send_sge_count = 30; 706 caps->max_recv_sge_count = 15; 707 caps->page_size_cap = PAGE_SZ_BM; 708 709 return 0; 710 } 711 712 static void 713 mana_ib_event_handler(void *ctx, struct gdma_queue *q, struct gdma_event *event) 714 { 715 struct mana_ib_dev *mdev = (struct mana_ib_dev *)ctx; 716 struct mana_ib_qp *qp; 717 struct ib_event ev; 718 u32 qpn; 719 720 switch (event->type) { 721 case GDMA_EQE_RNIC_QP_FATAL: 722 qpn = event->details[0]; 723 qp = mana_get_qp_ref(mdev, qpn, false); 724 if (!qp) 725 break; 726 if (qp->ibqp.event_handler) { 727 ev.device = qp->ibqp.device; 728 ev.element.qp = &qp->ibqp; 729 ev.event = IB_EVENT_QP_FATAL; 730 qp->ibqp.event_handler(&ev, qp->ibqp.qp_context); 731 } 732 mana_put_qp_ref(qp); 733 break; 734 default: 735 break; 736 } 737 } 738 739 int mana_ib_create_eqs(struct mana_ib_dev *mdev) 740 { 741 struct gdma_context *gc = mdev_to_gc(mdev); 742 struct gdma_queue_spec spec = {}; 743 int err, i; 744 745 spec.type = GDMA_EQ; 746 spec.monitor_avl_buf = false; 747 spec.queue_size = EQ_SIZE; 748 spec.eq.callback = mana_ib_event_handler; 749 spec.eq.context = mdev; 750 spec.eq.log2_throttle_limit = LOG2_EQ_THROTTLE; 751 spec.eq.msix_index = 0; 752 753 err = mana_gd_create_mana_eq(mdev->gdma_dev, &spec, &mdev->fatal_err_eq); 754 if (err) 755 return err; 756 757 mdev->eqs = kzalloc_objs(struct gdma_queue *, 758 mdev->ib_dev.num_comp_vectors); 759 if (!mdev->eqs) { 760 err = -ENOMEM; 761 goto destroy_fatal_eq; 762 } 763 spec.eq.callback = NULL; 764 for (i = 0; i < mdev->ib_dev.num_comp_vectors; i++) { 765 spec.eq.msix_index = (i + 1) % gc->num_msix_usable; 766 err = mana_gd_create_mana_eq(mdev->gdma_dev, &spec, &mdev->eqs[i]); 767 if (err) 768 goto destroy_eqs; 769 } 770 771 return 0; 772 773 destroy_eqs: 774 while (i-- > 0) 775 mana_gd_destroy_queue(gc, mdev->eqs[i]); 776 kfree(mdev->eqs); 777 destroy_fatal_eq: 778 mana_gd_destroy_queue(gc, mdev->fatal_err_eq); 779 return err; 780 } 781 782 void mana_ib_destroy_eqs(struct mana_ib_dev *mdev) 783 { 784 struct gdma_context *gc = mdev_to_gc(mdev); 785 int i; 786 787 mana_gd_destroy_queue(gc, mdev->fatal_err_eq); 788 789 for (i = 0; i < mdev->ib_dev.num_comp_vectors; i++) 790 mana_gd_destroy_queue(gc, mdev->eqs[i]); 791 792 kfree(mdev->eqs); 793 } 794 795 int mana_ib_gd_create_rnic_adapter(struct mana_ib_dev *mdev) 796 { 797 struct mana_rnic_create_adapter_resp resp = {}; 798 struct mana_rnic_create_adapter_req req = {}; 799 struct gdma_context *gc = mdev_to_gc(mdev); 800 int err; 801 802 mana_gd_init_req_hdr(&req.hdr, MANA_IB_CREATE_ADAPTER, sizeof(req), sizeof(resp)); 803 req.hdr.req.msg_version = GDMA_MESSAGE_V2; 804 req.hdr.dev_id = mdev->gdma_dev->dev_id; 805 req.notify_eq_id = mdev->fatal_err_eq->id; 806 807 if (mdev->adapter_caps.feature_flags & MANA_IB_FEATURE_CLIENT_ERROR_CQE_SUPPORT) 808 req.feature_flags |= MANA_IB_FEATURE_CLIENT_ERROR_CQE_REQUEST; 809 810 err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); 811 if (err) 812 return err; 813 mdev->adapter_handle = resp.adapter; 814 815 return 0; 816 } 817 818 int mana_ib_gd_destroy_rnic_adapter(struct mana_ib_dev *mdev) 819 { 820 struct mana_rnic_destroy_adapter_resp resp = {}; 821 struct mana_rnic_destroy_adapter_req req = {}; 822 struct gdma_context *gc; 823 824 gc = mdev_to_gc(mdev); 825 mana_gd_init_req_hdr(&req.hdr, MANA_IB_DESTROY_ADAPTER, sizeof(req), sizeof(resp)); 826 req.hdr.dev_id = mdev->gdma_dev->dev_id; 827 req.adapter = mdev->adapter_handle; 828 829 return mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); 830 } 831 832 int mana_ib_gd_add_gid(const struct ib_gid_attr *attr, void **context) 833 { 834 struct mana_ib_dev *mdev = container_of(attr->device, struct mana_ib_dev, ib_dev); 835 enum rdma_network_type ntype = rdma_gid_attr_network_type(attr); 836 struct mana_rnic_config_addr_resp resp = {}; 837 struct gdma_context *gc = mdev_to_gc(mdev); 838 struct mana_rnic_config_addr_req req = {}; 839 840 if (ntype != RDMA_NETWORK_IPV4 && ntype != RDMA_NETWORK_IPV6) { 841 ibdev_dbg(&mdev->ib_dev, "Unsupported rdma network type %d", ntype); 842 return -EINVAL; 843 } 844 845 mana_gd_init_req_hdr(&req.hdr, MANA_IB_CONFIG_IP_ADDR, sizeof(req), sizeof(resp)); 846 req.hdr.dev_id = mdev->gdma_dev->dev_id; 847 req.adapter = mdev->adapter_handle; 848 req.op = ADDR_OP_ADD; 849 req.sgid_type = (ntype == RDMA_NETWORK_IPV6) ? SGID_TYPE_IPV6 : SGID_TYPE_IPV4; 850 copy_in_reverse(req.ip_addr, attr->gid.raw, sizeof(union ib_gid)); 851 852 return mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); 853 } 854 855 int mana_ib_gd_del_gid(const struct ib_gid_attr *attr, void **context) 856 { 857 struct mana_ib_dev *mdev = container_of(attr->device, struct mana_ib_dev, ib_dev); 858 enum rdma_network_type ntype = rdma_gid_attr_network_type(attr); 859 struct mana_rnic_config_addr_resp resp = {}; 860 struct gdma_context *gc = mdev_to_gc(mdev); 861 struct mana_rnic_config_addr_req req = {}; 862 863 if (ntype != RDMA_NETWORK_IPV4 && ntype != RDMA_NETWORK_IPV6) { 864 ibdev_dbg(&mdev->ib_dev, "Unsupported rdma network type %d", ntype); 865 return -EINVAL; 866 } 867 868 mana_gd_init_req_hdr(&req.hdr, MANA_IB_CONFIG_IP_ADDR, sizeof(req), sizeof(resp)); 869 req.hdr.dev_id = mdev->gdma_dev->dev_id; 870 req.adapter = mdev->adapter_handle; 871 req.op = ADDR_OP_REMOVE; 872 req.sgid_type = (ntype == RDMA_NETWORK_IPV6) ? SGID_TYPE_IPV6 : SGID_TYPE_IPV4; 873 copy_in_reverse(req.ip_addr, attr->gid.raw, sizeof(union ib_gid)); 874 875 return mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); 876 } 877 878 int mana_ib_gd_config_mac(struct mana_ib_dev *mdev, enum mana_ib_addr_op op, u8 *mac) 879 { 880 struct mana_rnic_config_mac_addr_resp resp = {}; 881 struct mana_rnic_config_mac_addr_req req = {}; 882 struct gdma_context *gc = mdev_to_gc(mdev); 883 884 mana_gd_init_req_hdr(&req.hdr, MANA_IB_CONFIG_MAC_ADDR, sizeof(req), sizeof(resp)); 885 req.hdr.dev_id = mdev->gdma_dev->dev_id; 886 req.adapter = mdev->adapter_handle; 887 req.op = op; 888 copy_in_reverse(req.mac_addr, mac, ETH_ALEN); 889 890 return mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); 891 } 892 893 int mana_ib_gd_create_cq(struct mana_ib_dev *mdev, struct mana_ib_cq *cq, u32 doorbell) 894 { 895 struct gdma_context *gc = mdev_to_gc(mdev); 896 struct mana_rnic_create_cq_resp resp = {}; 897 struct mana_rnic_create_cq_req req = {}; 898 int err; 899 900 if (!mdev->eqs) 901 return -EINVAL; 902 903 mana_gd_init_req_hdr(&req.hdr, MANA_IB_CREATE_CQ, sizeof(req), sizeof(resp)); 904 req.hdr.dev_id = mdev->gdma_dev->dev_id; 905 req.adapter = mdev->adapter_handle; 906 req.gdma_region = cq->queue.gdma_region; 907 req.eq_id = mdev->eqs[cq->comp_vector]->id; 908 req.doorbell_page = doorbell; 909 910 err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); 911 912 if (err) { 913 ibdev_err(&mdev->ib_dev, "Failed to create cq err %d", err); 914 return err; 915 } 916 917 cq->queue.id = resp.cq_id; 918 cq->cq_handle = resp.cq_handle; 919 /* The GDMA region is now owned by the CQ handle */ 920 cq->queue.gdma_region = GDMA_INVALID_DMA_REGION; 921 922 return 0; 923 } 924 925 int mana_ib_gd_destroy_cq(struct mana_ib_dev *mdev, struct mana_ib_cq *cq) 926 { 927 struct gdma_context *gc = mdev_to_gc(mdev); 928 struct mana_rnic_destroy_cq_resp resp = {}; 929 struct mana_rnic_destroy_cq_req req = {}; 930 931 if (cq->cq_handle == INVALID_MANA_HANDLE) 932 return 0; 933 934 mana_gd_init_req_hdr(&req.hdr, MANA_IB_DESTROY_CQ, sizeof(req), sizeof(resp)); 935 req.hdr.dev_id = mdev->gdma_dev->dev_id; 936 req.adapter = mdev->adapter_handle; 937 req.cq_handle = cq->cq_handle; 938 939 return mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); 940 } 941 942 int mana_ib_gd_create_rc_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp, 943 struct ib_qp_init_attr *attr, u32 doorbell, u64 flags) 944 { 945 struct mana_ib_cq *send_cq = container_of(qp->ibqp.send_cq, struct mana_ib_cq, ibcq); 946 struct mana_ib_cq *recv_cq = container_of(qp->ibqp.recv_cq, struct mana_ib_cq, ibcq); 947 struct mana_ib_pd *pd = container_of(qp->ibqp.pd, struct mana_ib_pd, ibpd); 948 struct gdma_context *gc = mdev_to_gc(mdev); 949 struct mana_rnic_create_qp_resp resp = {}; 950 struct mana_rnic_create_qp_req req = {}; 951 int err, i; 952 953 mana_gd_init_req_hdr(&req.hdr, MANA_IB_CREATE_RC_QP, sizeof(req), sizeof(resp)); 954 req.hdr.dev_id = mdev->gdma_dev->dev_id; 955 req.adapter = mdev->adapter_handle; 956 req.pd_handle = pd->pd_handle; 957 req.send_cq_handle = send_cq->cq_handle; 958 req.recv_cq_handle = recv_cq->cq_handle; 959 for (i = 0; i < MANA_RC_QUEUE_TYPE_MAX; i++) 960 req.dma_region[i] = qp->rc_qp.queues[i].gdma_region; 961 req.doorbell_page = doorbell; 962 req.max_send_wr = attr->cap.max_send_wr; 963 req.max_recv_wr = attr->cap.max_recv_wr; 964 req.max_send_sge = attr->cap.max_send_sge; 965 req.max_recv_sge = attr->cap.max_recv_sge; 966 req.flags = flags; 967 968 err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); 969 if (err) 970 return err; 971 972 qp->qp_handle = resp.rc_qp_handle; 973 for (i = 0; i < MANA_RC_QUEUE_TYPE_MAX; i++) { 974 qp->rc_qp.queues[i].id = resp.queue_ids[i]; 975 /* The GDMA regions are now owned by the RNIC QP handle */ 976 qp->rc_qp.queues[i].gdma_region = GDMA_INVALID_DMA_REGION; 977 } 978 return 0; 979 } 980 981 int mana_ib_gd_destroy_rc_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp) 982 { 983 struct mana_rnic_destroy_rc_qp_resp resp = {0}; 984 struct mana_rnic_destroy_rc_qp_req req = {0}; 985 struct gdma_context *gc = mdev_to_gc(mdev); 986 987 mana_gd_init_req_hdr(&req.hdr, MANA_IB_DESTROY_RC_QP, sizeof(req), sizeof(resp)); 988 req.hdr.dev_id = mdev->gdma_dev->dev_id; 989 req.adapter = mdev->adapter_handle; 990 req.rc_qp_handle = qp->qp_handle; 991 992 return mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); 993 } 994 995 int mana_ib_gd_create_ud_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp, 996 struct ib_qp_init_attr *attr, u32 doorbell, u32 type) 997 { 998 struct mana_ib_cq *send_cq = container_of(qp->ibqp.send_cq, struct mana_ib_cq, ibcq); 999 struct mana_ib_cq *recv_cq = container_of(qp->ibqp.recv_cq, struct mana_ib_cq, ibcq); 1000 struct mana_ib_pd *pd = container_of(qp->ibqp.pd, struct mana_ib_pd, ibpd); 1001 struct gdma_context *gc = mdev_to_gc(mdev); 1002 struct mana_rnic_create_udqp_resp resp = {}; 1003 struct mana_rnic_create_udqp_req req = {}; 1004 int err, i; 1005 1006 mana_gd_init_req_hdr(&req.hdr, MANA_IB_CREATE_UD_QP, sizeof(req), sizeof(resp)); 1007 req.hdr.dev_id = mdev->gdma_dev->dev_id; 1008 req.adapter = mdev->adapter_handle; 1009 req.pd_handle = pd->pd_handle; 1010 req.send_cq_handle = send_cq->cq_handle; 1011 req.recv_cq_handle = recv_cq->cq_handle; 1012 for (i = 0; i < MANA_UD_QUEUE_TYPE_MAX; i++) 1013 req.dma_region[i] = qp->ud_qp.queues[i].gdma_region; 1014 req.doorbell_page = doorbell; 1015 req.max_send_wr = attr->cap.max_send_wr; 1016 req.max_recv_wr = attr->cap.max_recv_wr; 1017 req.max_send_sge = attr->cap.max_send_sge; 1018 req.max_recv_sge = attr->cap.max_recv_sge; 1019 req.qp_type = type; 1020 err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); 1021 if (err) 1022 return err; 1023 1024 qp->qp_handle = resp.qp_handle; 1025 for (i = 0; i < MANA_UD_QUEUE_TYPE_MAX; i++) { 1026 qp->ud_qp.queues[i].id = resp.queue_ids[i]; 1027 /* The GDMA regions are now owned by the RNIC QP handle */ 1028 qp->ud_qp.queues[i].gdma_region = GDMA_INVALID_DMA_REGION; 1029 } 1030 return 0; 1031 } 1032 1033 int mana_ib_gd_destroy_ud_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp) 1034 { 1035 struct mana_rnic_destroy_udqp_resp resp = {0}; 1036 struct mana_rnic_destroy_udqp_req req = {0}; 1037 struct gdma_context *gc = mdev_to_gc(mdev); 1038 1039 mana_gd_init_req_hdr(&req.hdr, MANA_IB_DESTROY_UD_QP, sizeof(req), sizeof(resp)); 1040 req.hdr.dev_id = mdev->gdma_dev->dev_id; 1041 req.adapter = mdev->adapter_handle; 1042 req.qp_handle = qp->qp_handle; 1043 1044 return mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); 1045 } 1046