1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* 3 * Copyright 2018-2026 Amazon.com, Inc. or its affiliates. All rights reserved. 4 */ 5 6 #include <linux/dma-buf.h> 7 #include <linux/dma-resv.h> 8 #include <linux/vmalloc.h> 9 #include <linux/log2.h> 10 11 #include <rdma/ib_addr.h> 12 #include <rdma/ib_user_verbs.h> 13 #include <rdma/ib_verbs.h> 14 #include <rdma/iter.h> 15 #include <rdma/uverbs_ioctl.h> 16 #define UVERBS_MODULE_NAME efa_ib 17 #include <rdma/uverbs_named_ioctl.h> 18 #include <rdma/ib_user_ioctl_cmds.h> 19 20 #include "efa.h" 21 #include "efa_io_defs.h" 22 23 enum { 24 EFA_MMAP_DMA_PAGE = 0, 25 EFA_MMAP_IO_WC, 26 EFA_MMAP_IO_NC, 27 }; 28 29 struct efa_user_mmap_entry { 30 struct rdma_user_mmap_entry rdma_entry; 31 u64 address; 32 u8 mmap_flag; 33 }; 34 35 #define EFA_DEFINE_DEVICE_STATS(op) \ 36 op(EFA_SUBMITTED_CMDS, "submitted_cmds") \ 37 op(EFA_COMPLETED_CMDS, "completed_cmds") \ 38 op(EFA_CMDS_ERR, "cmds_err") \ 39 op(EFA_NO_COMPLETION_CMDS, "no_completion_cmds") \ 40 op(EFA_KEEP_ALIVE_RCVD, "keep_alive_rcvd") \ 41 op(EFA_ALLOC_PD_ERR, "alloc_pd_err") \ 42 op(EFA_CREATE_QP_ERR, "create_qp_err") \ 43 op(EFA_CREATE_CQ_ERR, "create_cq_err") \ 44 op(EFA_REG_MR_ERR, "reg_mr_err") \ 45 op(EFA_ALLOC_UCONTEXT_ERR, "alloc_ucontext_err") \ 46 op(EFA_CREATE_AH_ERR, "create_ah_err") \ 47 op(EFA_MMAP_ERR, "mmap_err") 48 49 #define EFA_DEFINE_PORT_STATS(op) \ 50 op(EFA_TX_BYTES, "tx_bytes") \ 51 op(EFA_TX_PKTS, "tx_pkts") \ 52 op(EFA_RX_BYTES, "rx_bytes") \ 53 op(EFA_RX_PKTS, "rx_pkts") \ 54 op(EFA_RX_DROPS, "rx_drops") \ 55 op(EFA_SEND_BYTES, "send_bytes") \ 56 op(EFA_SEND_WRS, "send_wrs") \ 57 op(EFA_RECV_BYTES, "recv_bytes") \ 58 op(EFA_RECV_WRS, "recv_wrs") \ 59 op(EFA_RDMA_READ_WRS, "rdma_read_wrs") \ 60 op(EFA_RDMA_READ_BYTES, "rdma_read_bytes") \ 61 op(EFA_RDMA_READ_WR_ERR, "rdma_read_wr_err") \ 62 op(EFA_RDMA_READ_RESP_BYTES, "rdma_read_resp_bytes") \ 63 op(EFA_RDMA_WRITE_WRS, "rdma_write_wrs") \ 64 op(EFA_RDMA_WRITE_BYTES, "rdma_write_bytes") \ 65 op(EFA_RDMA_WRITE_WR_ERR, "rdma_write_wr_err") \ 66 op(EFA_RDMA_WRITE_RECV_BYTES, "rdma_write_recv_bytes") \ 67 op(EFA_RETRANS_BYTES, "retrans_bytes") \ 68 op(EFA_RETRANS_PKTS, "retrans_pkts") \ 69 op(EFA_RETRANS_TIMEOUT_EVENS, "retrans_timeout_events") \ 70 op(EFA_UNRESPONSIVE_REMOTE_EVENTS, "unresponsive_remote_events") \ 71 op(EFA_IMPAIRED_REMOTE_CONN_EVENTS, "impaired_remote_conn_events") \ 72 73 #define EFA_STATS_ENUM(ename, name) ename, 74 #define EFA_STATS_STR(ename, nam) \ 75 [ename].name = nam, 76 77 enum efa_hw_device_stats { 78 EFA_DEFINE_DEVICE_STATS(EFA_STATS_ENUM) 79 }; 80 81 static const struct rdma_stat_desc efa_device_stats_descs[] = { 82 EFA_DEFINE_DEVICE_STATS(EFA_STATS_STR) 83 }; 84 85 enum efa_hw_port_stats { 86 EFA_DEFINE_PORT_STATS(EFA_STATS_ENUM) 87 }; 88 89 static const struct rdma_stat_desc efa_port_stats_descs[] = { 90 EFA_DEFINE_PORT_STATS(EFA_STATS_STR) 91 }; 92 93 #define EFA_CHUNK_PAYLOAD_SHIFT 12 94 #define EFA_CHUNK_PAYLOAD_SIZE BIT(EFA_CHUNK_PAYLOAD_SHIFT) 95 #define EFA_CHUNK_PAYLOAD_PTR_SIZE 8 96 97 #define EFA_CHUNK_SHIFT 12 98 #define EFA_CHUNK_SIZE BIT(EFA_CHUNK_SHIFT) 99 #define EFA_CHUNK_PTR_SIZE sizeof(struct efa_com_ctrl_buff_info) 100 101 #define EFA_PTRS_PER_CHUNK \ 102 ((EFA_CHUNK_SIZE - EFA_CHUNK_PTR_SIZE) / EFA_CHUNK_PAYLOAD_PTR_SIZE) 103 104 #define EFA_CHUNK_USED_SIZE \ 105 ((EFA_PTRS_PER_CHUNK * EFA_CHUNK_PAYLOAD_PTR_SIZE) + EFA_CHUNK_PTR_SIZE) 106 107 struct pbl_chunk { 108 dma_addr_t dma_addr; 109 u64 *buf; 110 u32 length; 111 }; 112 113 struct pbl_chunk_list { 114 struct pbl_chunk *chunks; 115 unsigned int size; 116 }; 117 118 struct pbl_context { 119 union { 120 struct { 121 dma_addr_t dma_addr; 122 } continuous; 123 struct { 124 u32 pbl_buf_size_in_pages; 125 struct scatterlist *sgl; 126 int sg_dma_cnt; 127 struct pbl_chunk_list chunk_list; 128 } indirect; 129 } phys; 130 u64 *pbl_buf; 131 u32 pbl_buf_size_in_bytes; 132 u8 physically_continuous; 133 }; 134 135 static inline struct efa_dev *to_edev(struct ib_device *ibdev) 136 { 137 return container_of(ibdev, struct efa_dev, ibdev); 138 } 139 140 static inline struct efa_ucontext *to_eucontext(struct ib_ucontext *ibucontext) 141 { 142 return container_of(ibucontext, struct efa_ucontext, ibucontext); 143 } 144 145 static inline struct efa_pd *to_epd(struct ib_pd *ibpd) 146 { 147 return container_of(ibpd, struct efa_pd, ibpd); 148 } 149 150 static inline struct efa_mr *to_emr(struct ib_mr *ibmr) 151 { 152 return container_of(ibmr, struct efa_mr, ibmr); 153 } 154 155 static inline struct efa_qp *to_eqp(struct ib_qp *ibqp) 156 { 157 return container_of(ibqp, struct efa_qp, ibqp); 158 } 159 160 static inline struct efa_cq *to_ecq(struct ib_cq *ibcq) 161 { 162 return container_of(ibcq, struct efa_cq, ibcq); 163 } 164 165 static inline struct efa_ah *to_eah(struct ib_ah *ibah) 166 { 167 return container_of(ibah, struct efa_ah, ibah); 168 } 169 170 static inline struct efa_comp_cntr *to_ecc(struct ib_comp_cntr *ibcc) 171 { 172 return container_of(ibcc, struct efa_comp_cntr, ibcc); 173 } 174 175 static inline struct efa_user_mmap_entry * 176 to_emmap(struct rdma_user_mmap_entry *rdma_entry) 177 { 178 return container_of(rdma_entry, struct efa_user_mmap_entry, rdma_entry); 179 } 180 181 #define EFA_DEV_CAP(dev, cap) \ 182 ((dev)->dev_attr.device_caps & \ 183 EFA_ADMIN_FEATURE_DEVICE_ATTR_DESC_##cap##_MASK) 184 185 #define is_reserved_cleared(reserved) \ 186 !memchr_inv(reserved, 0, sizeof(reserved)) 187 188 static void *efa_zalloc_mapped(struct efa_dev *dev, dma_addr_t *dma_addr, 189 size_t size, enum dma_data_direction dir) 190 { 191 void *addr; 192 193 addr = alloc_pages_exact(size, GFP_KERNEL | __GFP_ZERO); 194 if (!addr) 195 return NULL; 196 197 *dma_addr = dma_map_single(&dev->pdev->dev, addr, size, dir); 198 if (dma_mapping_error(&dev->pdev->dev, *dma_addr)) { 199 ibdev_err(&dev->ibdev, "Failed to map DMA address\n"); 200 free_pages_exact(addr, size); 201 return NULL; 202 } 203 204 return addr; 205 } 206 207 static void efa_free_mapped(struct efa_dev *dev, void *cpu_addr, 208 dma_addr_t dma_addr, 209 size_t size, enum dma_data_direction dir) 210 { 211 dma_unmap_single(&dev->pdev->dev, dma_addr, size, dir); 212 free_pages_exact(cpu_addr, size); 213 } 214 215 int efa_query_device(struct ib_device *ibdev, 216 struct ib_device_attr *props, 217 struct ib_udata *udata) 218 { 219 struct efa_com_get_device_attr_result *dev_attr; 220 struct efa_ibv_ex_query_device_resp resp = {}; 221 struct efa_dev *dev = to_edev(ibdev); 222 int err; 223 224 err = ib_is_udata_in_empty(udata); 225 if (err) 226 return err; 227 228 dev_attr = &dev->dev_attr; 229 230 props->max_mr_size = dev_attr->max_mr_pages * PAGE_SIZE; 231 props->page_size_cap = dev_attr->page_size_cap; 232 props->vendor_id = dev->pdev->vendor; 233 props->vendor_part_id = dev->pdev->device; 234 props->hw_ver = dev->pdev->subsystem_device; 235 props->max_qp = dev_attr->max_qp; 236 props->max_cq = dev_attr->max_cq; 237 props->max_pd = dev_attr->max_pd; 238 props->max_mr = dev_attr->max_mr; 239 props->max_ah = dev_attr->max_ah; 240 props->max_cqe = dev_attr->max_cq_depth; 241 props->max_qp_wr = min_t(u32, dev_attr->max_sq_depth, 242 dev_attr->max_rq_depth); 243 props->max_send_sge = dev_attr->max_sq_sge; 244 props->max_recv_sge = dev_attr->max_rq_sge; 245 props->max_sge_rd = dev_attr->max_wr_rdma_sge; 246 props->max_pkeys = 1; 247 248 if (udata && udata->outlen) { 249 resp.max_sq_sge = dev_attr->max_sq_sge; 250 resp.max_rq_sge = dev_attr->max_rq_sge; 251 resp.max_sq_wr = dev_attr->max_sq_depth; 252 resp.max_rq_wr = dev_attr->max_rq_depth; 253 resp.max_rdma_size = dev_attr->max_rdma_size; 254 255 resp.device_caps |= EFA_QUERY_DEVICE_CAPS_CQ_WITH_SGID; 256 resp.device_caps |= EFA_QUERY_DEVICE_CAPS_CQ_WITH_EXT_MEM; 257 if (EFA_DEV_CAP(dev, RDMA_READ)) 258 resp.device_caps |= EFA_QUERY_DEVICE_CAPS_RDMA_READ; 259 260 if (EFA_DEV_CAP(dev, RNR_RETRY)) 261 resp.device_caps |= EFA_QUERY_DEVICE_CAPS_RNR_RETRY; 262 263 if (EFA_DEV_CAP(dev, DATA_POLLING_128)) 264 resp.device_caps |= EFA_QUERY_DEVICE_CAPS_DATA_POLLING_128; 265 266 if (EFA_DEV_CAP(dev, RDMA_WRITE)) 267 resp.device_caps |= EFA_QUERY_DEVICE_CAPS_RDMA_WRITE; 268 269 if (EFA_DEV_CAP(dev, UNSOLICITED_WRITE_RECV)) 270 resp.device_caps |= EFA_QUERY_DEVICE_CAPS_UNSOLICITED_WRITE_RECV; 271 272 if (EFA_DEV_CAP(dev, EVENT_COUNTERS)) 273 resp.device_caps |= EFA_QUERY_DEVICE_CAPS_COMP_CNTR; 274 275 if (EFA_DEV_CAP(dev, SQ_64_BIT_REQ_ID)) 276 resp.device_caps |= EFA_QUERY_DEVICE_CAPS_SQ_64_BIT_REQ_ID; 277 278 if (dev->neqs) 279 resp.device_caps |= EFA_QUERY_DEVICE_CAPS_CQ_NOTIFICATIONS; 280 281 err = ib_respond_udata(udata, resp); 282 if (err) 283 return err; 284 } 285 286 return 0; 287 } 288 289 static void efa_link_gbps_to_speed_and_width(u16 gbps, 290 enum ib_port_speed *speed, 291 enum ib_port_width *width) 292 { 293 if (gbps >= 1600) { 294 *width = IB_WIDTH_8X; 295 *speed = IB_SPEED_XDR; 296 } else if (gbps >= 800) { 297 *width = IB_WIDTH_8X; 298 *speed = IB_SPEED_NDR; 299 } else if (gbps >= 400) { 300 *width = IB_WIDTH_8X; 301 *speed = IB_SPEED_HDR; 302 } else if (gbps >= 200) { 303 *width = IB_WIDTH_4X; 304 *speed = IB_SPEED_HDR; 305 } else if (gbps >= 120) { 306 *width = IB_WIDTH_12X; 307 *speed = IB_SPEED_FDR10; 308 } else if (gbps >= 100) { 309 *width = IB_WIDTH_4X; 310 *speed = IB_SPEED_EDR; 311 } else if (gbps >= 60) { 312 *width = IB_WIDTH_12X; 313 *speed = IB_SPEED_DDR; 314 } else if (gbps >= 50) { 315 *width = IB_WIDTH_1X; 316 *speed = IB_SPEED_HDR; 317 } else if (gbps >= 40) { 318 *width = IB_WIDTH_4X; 319 *speed = IB_SPEED_FDR10; 320 } else if (gbps >= 30) { 321 *width = IB_WIDTH_12X; 322 *speed = IB_SPEED_SDR; 323 } else { 324 *width = IB_WIDTH_1X; 325 *speed = IB_SPEED_EDR; 326 } 327 } 328 329 int efa_query_port(struct ib_device *ibdev, u32 port, 330 struct ib_port_attr *props) 331 { 332 struct efa_dev *dev = to_edev(ibdev); 333 enum ib_port_speed link_speed; 334 enum ib_port_width link_width; 335 u16 link_gbps; 336 337 props->lmc = 1; 338 339 props->state = IB_PORT_ACTIVE; 340 props->phys_state = IB_PORT_PHYS_STATE_LINK_UP; 341 props->gid_tbl_len = 1; 342 props->pkey_tbl_len = 1; 343 link_gbps = dev->dev_attr.max_link_speed_gbps; 344 efa_link_gbps_to_speed_and_width(link_gbps, &link_speed, &link_width); 345 props->active_speed = link_speed; 346 props->active_width = link_width; 347 props->max_mtu = ib_mtu_int_to_enum(dev->dev_attr.mtu); 348 props->active_mtu = ib_mtu_int_to_enum(dev->dev_attr.mtu); 349 props->max_msg_sz = dev->dev_attr.mtu; 350 props->max_vl_num = 1; 351 352 return 0; 353 } 354 355 int efa_query_port_speed(struct ib_device *ibdev, u32 port_num, u64 *speed) 356 { 357 struct efa_dev *dev = to_edev(ibdev); 358 359 *speed = dev->dev_attr.max_link_speed_gbps * 10; 360 361 return 0; 362 } 363 364 int efa_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, 365 int qp_attr_mask, 366 struct ib_qp_init_attr *qp_init_attr) 367 { 368 struct efa_dev *dev = to_edev(ibqp->device); 369 struct efa_com_query_qp_params params = {}; 370 struct efa_com_query_qp_result result; 371 struct efa_qp *qp = to_eqp(ibqp); 372 int err; 373 374 #define EFA_QUERY_QP_SUPP_MASK \ 375 (IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_PORT | \ 376 IB_QP_QKEY | IB_QP_SQ_PSN | IB_QP_CAP | IB_QP_RNR_RETRY) 377 378 if (qp_attr_mask & ~EFA_QUERY_QP_SUPP_MASK) { 379 ibdev_dbg(&dev->ibdev, 380 "Unsupported qp_attr_mask[%#x] supported[%#x]\n", 381 qp_attr_mask, EFA_QUERY_QP_SUPP_MASK); 382 return -EOPNOTSUPP; 383 } 384 385 memset(qp_attr, 0, sizeof(*qp_attr)); 386 memset(qp_init_attr, 0, sizeof(*qp_init_attr)); 387 388 params.qp_handle = qp->qp_handle; 389 err = efa_com_query_qp(&dev->edev, ¶ms, &result); 390 if (err) 391 return err; 392 393 qp_attr->qp_state = result.qp_state; 394 qp_attr->qkey = result.qkey; 395 qp_attr->sq_psn = result.sq_psn; 396 qp_attr->sq_draining = result.sq_draining; 397 qp_attr->port_num = 1; 398 qp_attr->rnr_retry = result.rnr_retry; 399 400 qp_attr->cap.max_send_wr = qp->max_send_wr; 401 qp_attr->cap.max_recv_wr = qp->max_recv_wr; 402 qp_attr->cap.max_send_sge = qp->max_send_sge; 403 qp_attr->cap.max_recv_sge = qp->max_recv_sge; 404 qp_attr->cap.max_inline_data = qp->max_inline_data; 405 406 qp_init_attr->qp_type = ibqp->qp_type; 407 qp_init_attr->recv_cq = ibqp->recv_cq; 408 qp_init_attr->send_cq = ibqp->send_cq; 409 qp_init_attr->qp_context = ibqp->qp_context; 410 qp_init_attr->cap = qp_attr->cap; 411 412 return 0; 413 } 414 415 int efa_query_gid(struct ib_device *ibdev, u32 port, int index, 416 union ib_gid *gid) 417 { 418 struct efa_dev *dev = to_edev(ibdev); 419 420 memcpy(gid->raw, dev->dev_attr.addr, sizeof(dev->dev_attr.addr)); 421 422 return 0; 423 } 424 425 int efa_query_pkey(struct ib_device *ibdev, u32 port, u16 index, 426 u16 *pkey) 427 { 428 if (index > 0) 429 return -EINVAL; 430 431 *pkey = 0xffff; 432 return 0; 433 } 434 435 static int efa_pd_dealloc(struct efa_dev *dev, u16 pdn) 436 { 437 struct efa_com_dealloc_pd_params params = { 438 .pdn = pdn, 439 }; 440 441 return efa_com_dealloc_pd(&dev->edev, ¶ms); 442 } 443 444 int efa_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 445 { 446 struct efa_dev *dev = to_edev(ibpd->device); 447 struct efa_ibv_alloc_pd_resp resp = {}; 448 struct efa_com_alloc_pd_result result; 449 struct efa_pd *pd = to_epd(ibpd); 450 int err; 451 452 err = ib_is_udata_in_empty(udata); 453 if (err) 454 goto err_out; 455 456 err = efa_com_alloc_pd(&dev->edev, &result); 457 if (err) 458 goto err_out; 459 460 pd->pdn = result.pdn; 461 resp.pdn = result.pdn; 462 463 if (udata->outlen) { 464 err = ib_respond_udata(udata, resp); 465 if (err) 466 goto err_dealloc_pd; 467 } 468 469 ibdev_dbg(&dev->ibdev, "Allocated pd[%d]\n", pd->pdn); 470 471 return 0; 472 473 err_dealloc_pd: 474 efa_pd_dealloc(dev, result.pdn); 475 err_out: 476 atomic64_inc(&dev->stats.alloc_pd_err); 477 return err; 478 } 479 480 int efa_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 481 { 482 struct efa_dev *dev = to_edev(ibpd->device); 483 struct efa_pd *pd = to_epd(ibpd); 484 485 ibdev_dbg(&dev->ibdev, "Dealloc pd[%d]\n", pd->pdn); 486 efa_pd_dealloc(dev, pd->pdn); 487 return 0; 488 } 489 490 static int efa_destroy_qp_handle(struct efa_dev *dev, u32 qp_handle) 491 { 492 struct efa_com_destroy_qp_params params = { .qp_handle = qp_handle }; 493 494 return efa_com_destroy_qp(&dev->edev, ¶ms); 495 } 496 497 static void efa_qp_user_mmap_entries_remove(struct efa_qp *qp) 498 { 499 rdma_user_mmap_entry_remove(qp->rq_mmap_entry); 500 rdma_user_mmap_entry_remove(qp->rq_db_mmap_entry); 501 rdma_user_mmap_entry_remove(qp->llq_desc_mmap_entry); 502 rdma_user_mmap_entry_remove(qp->sq_db_mmap_entry); 503 } 504 505 int efa_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) 506 { 507 struct efa_dev *dev = to_edev(ibqp->pd->device); 508 struct efa_qp *qp = to_eqp(ibqp); 509 int err; 510 511 ibdev_dbg(&dev->ibdev, "Destroy qp[%u]\n", ibqp->qp_num); 512 513 err = efa_destroy_qp_handle(dev, qp->qp_handle); 514 if (err) 515 return err; 516 517 efa_qp_user_mmap_entries_remove(qp); 518 519 if (qp->rq_cpu_addr) { 520 ibdev_dbg(&dev->ibdev, 521 "qp->cpu_addr[0x%p] freed: size[%lu], dma[%pad]\n", 522 qp->rq_cpu_addr, qp->rq_size, 523 &qp->rq_dma_addr); 524 efa_free_mapped(dev, qp->rq_cpu_addr, qp->rq_dma_addr, 525 qp->rq_size, DMA_TO_DEVICE); 526 } 527 528 return 0; 529 } 530 531 static struct rdma_user_mmap_entry* 532 efa_user_mmap_entry_insert(struct ib_ucontext *ucontext, 533 u64 address, size_t length, 534 u8 mmap_flag, u64 *offset) 535 { 536 struct efa_user_mmap_entry *entry = kzalloc_obj(*entry); 537 int err; 538 539 if (!entry) 540 return NULL; 541 542 entry->address = address; 543 entry->mmap_flag = mmap_flag; 544 545 err = rdma_user_mmap_entry_insert(ucontext, &entry->rdma_entry, 546 length); 547 if (err) { 548 kfree(entry); 549 return NULL; 550 } 551 *offset = rdma_user_mmap_get_offset(&entry->rdma_entry); 552 553 return &entry->rdma_entry; 554 } 555 556 static int qp_mmap_entries_setup(struct efa_qp *qp, 557 struct efa_dev *dev, 558 struct efa_ucontext *ucontext, 559 struct efa_com_create_qp_params *params, 560 struct efa_ibv_create_qp_resp *resp) 561 { 562 size_t length; 563 u64 address; 564 565 address = dev->db_bar_addr + resp->sq_db_offset; 566 qp->sq_db_mmap_entry = 567 efa_user_mmap_entry_insert(&ucontext->ibucontext, 568 address, 569 PAGE_SIZE, EFA_MMAP_IO_NC, 570 &resp->sq_db_mmap_key); 571 if (!qp->sq_db_mmap_entry) 572 return -ENOMEM; 573 574 resp->sq_db_offset &= ~PAGE_MASK; 575 576 address = dev->mem_bar_addr + resp->llq_desc_offset; 577 length = PAGE_ALIGN(params->sq_ring_size_in_bytes + 578 offset_in_page(resp->llq_desc_offset)); 579 580 qp->llq_desc_mmap_entry = 581 efa_user_mmap_entry_insert(&ucontext->ibucontext, 582 address, length, 583 EFA_MMAP_IO_WC, 584 &resp->llq_desc_mmap_key); 585 if (!qp->llq_desc_mmap_entry) 586 goto err_remove_mmap; 587 588 resp->llq_desc_offset &= ~PAGE_MASK; 589 590 if (qp->rq_cpu_addr) { 591 address = dev->db_bar_addr + resp->rq_db_offset; 592 593 qp->rq_db_mmap_entry = 594 efa_user_mmap_entry_insert(&ucontext->ibucontext, 595 address, PAGE_SIZE, 596 EFA_MMAP_IO_NC, 597 &resp->rq_db_mmap_key); 598 if (!qp->rq_db_mmap_entry) 599 goto err_remove_mmap; 600 601 resp->rq_db_offset &= ~PAGE_MASK; 602 603 address = virt_to_phys(qp->rq_cpu_addr); 604 qp->rq_mmap_entry = 605 efa_user_mmap_entry_insert(&ucontext->ibucontext, 606 address, qp->rq_size, 607 EFA_MMAP_DMA_PAGE, 608 &resp->rq_mmap_key); 609 if (!qp->rq_mmap_entry) 610 goto err_remove_mmap; 611 612 resp->rq_mmap_size = qp->rq_size; 613 } 614 615 return 0; 616 617 err_remove_mmap: 618 efa_qp_user_mmap_entries_remove(qp); 619 620 return -ENOMEM; 621 } 622 623 static int efa_qp_validate_cap(struct efa_dev *dev, 624 struct ib_qp_init_attr *init_attr, 625 u32 sq_ring_size) 626 { 627 if (init_attr->cap.max_send_wr > dev->dev_attr.max_sq_depth) { 628 ibdev_dbg(&dev->ibdev, 629 "qp: requested send wr[%u] exceeds the max[%u]\n", 630 init_attr->cap.max_send_wr, 631 dev->dev_attr.max_sq_depth); 632 return -EINVAL; 633 } 634 635 if (sq_ring_size > dev->dev_attr.max_llq_size) { 636 ibdev_dbg(&dev->ibdev, 637 "qp: requested sq ring size[%u] exceeds the max[%u]\n", 638 sq_ring_size, dev->dev_attr.max_llq_size); 639 return -EINVAL; 640 } 641 642 if (init_attr->cap.max_recv_wr > dev->dev_attr.max_rq_depth) { 643 ibdev_dbg(&dev->ibdev, 644 "qp: requested receive wr[%u] exceeds the max[%u]\n", 645 init_attr->cap.max_recv_wr, 646 dev->dev_attr.max_rq_depth); 647 return -EINVAL; 648 } 649 if (init_attr->cap.max_send_sge > dev->dev_attr.max_sq_sge) { 650 ibdev_dbg(&dev->ibdev, 651 "qp: requested sge send[%u] exceeds the max[%u]\n", 652 init_attr->cap.max_send_sge, dev->dev_attr.max_sq_sge); 653 return -EINVAL; 654 } 655 if (init_attr->cap.max_recv_sge > dev->dev_attr.max_rq_sge) { 656 ibdev_dbg(&dev->ibdev, 657 "qp: requested sge recv[%u] exceeds the max[%u]\n", 658 init_attr->cap.max_recv_sge, dev->dev_attr.max_rq_sge); 659 return -EINVAL; 660 } 661 if (init_attr->cap.max_inline_data > dev->dev_attr.inline_buf_size_ex) { 662 ibdev_dbg(&dev->ibdev, 663 "qp: requested inline data[%u] exceeds the max[%u]\n", 664 init_attr->cap.max_inline_data, 665 dev->dev_attr.inline_buf_size_ex); 666 return -EINVAL; 667 } 668 669 return 0; 670 } 671 672 static int efa_qp_validate_attr(struct efa_dev *dev, 673 struct ib_qp_init_attr *init_attr) 674 { 675 if (init_attr->qp_type != IB_QPT_DRIVER && 676 init_attr->qp_type != IB_QPT_UD) { 677 ibdev_dbg(&dev->ibdev, 678 "Unsupported qp type %d\n", init_attr->qp_type); 679 return -EOPNOTSUPP; 680 } 681 682 if (init_attr->srq) { 683 ibdev_dbg(&dev->ibdev, "SRQ is not supported\n"); 684 return -EOPNOTSUPP; 685 } 686 687 if (init_attr->create_flags) { 688 ibdev_dbg(&dev->ibdev, "Unsupported create flags\n"); 689 return -EOPNOTSUPP; 690 } 691 692 return 0; 693 } 694 695 int efa_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *init_attr, 696 struct ib_udata *udata) 697 { 698 struct efa_com_create_qp_params create_qp_params = {}; 699 struct efa_com_create_qp_result create_qp_resp; 700 struct efa_dev *dev = to_edev(ibqp->device); 701 struct efa_ibv_create_qp_resp resp = {}; 702 struct efa_ibv_create_qp cmd; 703 struct efa_qp *qp = to_eqp(ibqp); 704 struct efa_ucontext *ucontext; 705 u16 supported_efa_flags = 0; 706 int err; 707 708 ucontext = rdma_udata_to_drv_context(udata, struct efa_ucontext, 709 ibucontext); 710 711 err = ib_copy_validate_udata_in_cm(udata, cmd, driver_qp_type, 0); 712 if (err) 713 goto err_out; 714 715 if (!is_reserved_cleared(cmd.reserved_98)) { 716 ibdev_dbg(&dev->ibdev, 717 "Incompatible ABI params, unknown fields in udata\n"); 718 err = -EINVAL; 719 goto err_out; 720 } 721 722 if (EFA_DEV_CAP(dev, UNSOLICITED_WRITE_RECV)) 723 supported_efa_flags |= EFA_CREATE_QP_WITH_UNSOLICITED_WRITE_RECV; 724 725 if (EFA_DEV_CAP(dev, SQ_64_BIT_REQ_ID)) 726 supported_efa_flags |= EFA_CREATE_QP_WITH_SQ_64_BIT_REQ_ID; 727 728 if (cmd.flags & ~supported_efa_flags) { 729 ibdev_dbg(&dev->ibdev, "Unsupported EFA QP create flags[%#x], supported[%#x]\n", 730 cmd.flags, supported_efa_flags); 731 err = -EOPNOTSUPP; 732 goto err_out; 733 } 734 735 err = efa_qp_validate_cap(dev, init_attr, cmd.sq_ring_size); 736 if (err) 737 goto err_out; 738 739 err = efa_qp_validate_attr(dev, init_attr); 740 if (err) 741 goto err_out; 742 743 create_qp_params.uarn = ucontext->uarn; 744 create_qp_params.pd = to_epd(ibqp->pd)->pdn; 745 746 if (init_attr->qp_type == IB_QPT_UD) { 747 create_qp_params.qp_type = EFA_ADMIN_QP_TYPE_UD; 748 } else if (cmd.driver_qp_type == EFA_QP_DRIVER_TYPE_SRD) { 749 create_qp_params.qp_type = EFA_ADMIN_QP_TYPE_SRD; 750 } else { 751 ibdev_dbg(&dev->ibdev, 752 "Unsupported qp type %d driver qp type %d\n", 753 init_attr->qp_type, cmd.driver_qp_type); 754 err = -EOPNOTSUPP; 755 goto err_out; 756 } 757 758 ibdev_dbg(&dev->ibdev, "Create QP: qp type %d driver qp type %#x\n", 759 init_attr->qp_type, cmd.driver_qp_type); 760 create_qp_params.send_cq_idx = to_ecq(init_attr->send_cq)->cq_idx; 761 create_qp_params.recv_cq_idx = to_ecq(init_attr->recv_cq)->cq_idx; 762 create_qp_params.sq_depth = init_attr->cap.max_send_wr; 763 create_qp_params.sq_ring_size_in_bytes = cmd.sq_ring_size; 764 765 create_qp_params.rq_depth = init_attr->cap.max_recv_wr; 766 create_qp_params.rq_ring_size_in_bytes = cmd.rq_ring_size; 767 qp->rq_size = PAGE_ALIGN(create_qp_params.rq_ring_size_in_bytes); 768 if (qp->rq_size) { 769 qp->rq_cpu_addr = efa_zalloc_mapped(dev, &qp->rq_dma_addr, 770 qp->rq_size, DMA_TO_DEVICE); 771 if (!qp->rq_cpu_addr) { 772 err = -ENOMEM; 773 goto err_out; 774 } 775 776 ibdev_dbg(&dev->ibdev, 777 "qp->cpu_addr[0x%p] allocated: size[%lu], dma[%pad]\n", 778 qp->rq_cpu_addr, qp->rq_size, &qp->rq_dma_addr); 779 create_qp_params.rq_base_addr = qp->rq_dma_addr; 780 } 781 782 create_qp_params.sl = cmd.sl; 783 784 if (cmd.flags & EFA_CREATE_QP_WITH_UNSOLICITED_WRITE_RECV) 785 create_qp_params.unsolicited_write_recv = true; 786 787 if (cmd.flags & EFA_CREATE_QP_WITH_SQ_64_BIT_REQ_ID) 788 create_qp_params.sq_64_bit_req_id = true; 789 790 err = efa_com_create_qp(&dev->edev, &create_qp_params, 791 &create_qp_resp); 792 if (err) 793 goto err_free_mapped; 794 795 resp.sq_db_offset = create_qp_resp.sq_db_offset; 796 resp.rq_db_offset = create_qp_resp.rq_db_offset; 797 resp.llq_desc_offset = create_qp_resp.llq_descriptors_offset; 798 resp.send_sub_cq_idx = create_qp_resp.send_sub_cq_idx; 799 resp.recv_sub_cq_idx = create_qp_resp.recv_sub_cq_idx; 800 801 err = qp_mmap_entries_setup(qp, dev, ucontext, &create_qp_params, 802 &resp); 803 if (err) 804 goto err_destroy_qp; 805 806 qp->qp_handle = create_qp_resp.qp_handle; 807 qp->ibqp.qp_num = create_qp_resp.qp_num; 808 qp->max_send_wr = init_attr->cap.max_send_wr; 809 qp->max_recv_wr = init_attr->cap.max_recv_wr; 810 qp->max_send_sge = init_attr->cap.max_send_sge; 811 qp->max_recv_sge = init_attr->cap.max_recv_sge; 812 qp->max_inline_data = init_attr->cap.max_inline_data; 813 814 if (udata->outlen) { 815 err = ib_respond_udata(udata, resp); 816 if (err) 817 goto err_remove_mmap_entries; 818 } 819 820 ibdev_dbg(&dev->ibdev, "Created qp[%d]\n", qp->ibqp.qp_num); 821 822 return 0; 823 824 err_remove_mmap_entries: 825 efa_qp_user_mmap_entries_remove(qp); 826 err_destroy_qp: 827 efa_destroy_qp_handle(dev, create_qp_resp.qp_handle); 828 err_free_mapped: 829 if (qp->rq_cpu_addr) 830 efa_free_mapped(dev, qp->rq_cpu_addr, qp->rq_dma_addr, 831 qp->rq_size, DMA_TO_DEVICE); 832 err_out: 833 atomic64_inc(&dev->stats.create_qp_err); 834 return err; 835 } 836 837 static const struct { 838 int valid; 839 enum ib_qp_attr_mask req_param; 840 enum ib_qp_attr_mask opt_param; 841 } srd_qp_state_table[IB_QPS_ERR + 1][IB_QPS_ERR + 1] = { 842 [IB_QPS_RESET] = { 843 [IB_QPS_RESET] = { .valid = 1 }, 844 [IB_QPS_INIT] = { 845 .valid = 1, 846 .req_param = IB_QP_PKEY_INDEX | 847 IB_QP_PORT | 848 IB_QP_QKEY, 849 }, 850 }, 851 [IB_QPS_INIT] = { 852 [IB_QPS_RESET] = { .valid = 1 }, 853 [IB_QPS_ERR] = { .valid = 1 }, 854 [IB_QPS_INIT] = { 855 .valid = 1, 856 .opt_param = IB_QP_PKEY_INDEX | 857 IB_QP_PORT | 858 IB_QP_QKEY, 859 }, 860 [IB_QPS_RTR] = { 861 .valid = 1, 862 .opt_param = IB_QP_PKEY_INDEX | 863 IB_QP_QKEY, 864 }, 865 }, 866 [IB_QPS_RTR] = { 867 [IB_QPS_RESET] = { .valid = 1 }, 868 [IB_QPS_ERR] = { .valid = 1 }, 869 [IB_QPS_RTS] = { 870 .valid = 1, 871 .req_param = IB_QP_SQ_PSN, 872 .opt_param = IB_QP_CUR_STATE | 873 IB_QP_QKEY | 874 IB_QP_RNR_RETRY, 875 876 } 877 }, 878 [IB_QPS_RTS] = { 879 [IB_QPS_RESET] = { .valid = 1 }, 880 [IB_QPS_ERR] = { .valid = 1 }, 881 [IB_QPS_RTS] = { 882 .valid = 1, 883 .opt_param = IB_QP_CUR_STATE | 884 IB_QP_QKEY, 885 }, 886 [IB_QPS_SQD] = { 887 .valid = 1, 888 .opt_param = IB_QP_EN_SQD_ASYNC_NOTIFY, 889 }, 890 }, 891 [IB_QPS_SQD] = { 892 [IB_QPS_RESET] = { .valid = 1 }, 893 [IB_QPS_ERR] = { .valid = 1 }, 894 [IB_QPS_RTS] = { 895 .valid = 1, 896 .opt_param = IB_QP_CUR_STATE | 897 IB_QP_QKEY, 898 }, 899 [IB_QPS_SQD] = { 900 .valid = 1, 901 .opt_param = IB_QP_PKEY_INDEX | 902 IB_QP_QKEY, 903 } 904 }, 905 [IB_QPS_SQE] = { 906 [IB_QPS_RESET] = { .valid = 1 }, 907 [IB_QPS_ERR] = { .valid = 1 }, 908 [IB_QPS_RTS] = { 909 .valid = 1, 910 .opt_param = IB_QP_CUR_STATE | 911 IB_QP_QKEY, 912 } 913 }, 914 [IB_QPS_ERR] = { 915 [IB_QPS_RESET] = { .valid = 1 }, 916 [IB_QPS_ERR] = { .valid = 1 }, 917 } 918 }; 919 920 static bool efa_modify_srd_qp_is_ok(enum ib_qp_state cur_state, 921 enum ib_qp_state next_state, 922 enum ib_qp_attr_mask mask) 923 { 924 enum ib_qp_attr_mask req_param, opt_param; 925 926 if (mask & IB_QP_CUR_STATE && 927 cur_state != IB_QPS_RTR && cur_state != IB_QPS_RTS && 928 cur_state != IB_QPS_SQD && cur_state != IB_QPS_SQE) 929 return false; 930 931 if (!srd_qp_state_table[cur_state][next_state].valid) 932 return false; 933 934 req_param = srd_qp_state_table[cur_state][next_state].req_param; 935 opt_param = srd_qp_state_table[cur_state][next_state].opt_param; 936 937 if ((mask & req_param) != req_param) 938 return false; 939 940 if (mask & ~(req_param | opt_param | IB_QP_STATE)) 941 return false; 942 943 return true; 944 } 945 946 static int efa_modify_qp_validate(struct efa_dev *dev, struct efa_qp *qp, 947 struct ib_qp_attr *qp_attr, int qp_attr_mask, 948 enum ib_qp_state cur_state, 949 enum ib_qp_state new_state) 950 { 951 int err; 952 953 #define EFA_MODIFY_QP_SUPP_MASK \ 954 (IB_QP_STATE | IB_QP_CUR_STATE | IB_QP_EN_SQD_ASYNC_NOTIFY | \ 955 IB_QP_PKEY_INDEX | IB_QP_PORT | IB_QP_QKEY | IB_QP_SQ_PSN | \ 956 IB_QP_RNR_RETRY) 957 958 if (qp_attr_mask & ~EFA_MODIFY_QP_SUPP_MASK) { 959 ibdev_dbg(&dev->ibdev, 960 "Unsupported qp_attr_mask[%#x] supported[%#x]\n", 961 qp_attr_mask, EFA_MODIFY_QP_SUPP_MASK); 962 return -EOPNOTSUPP; 963 } 964 965 if (qp->ibqp.qp_type == IB_QPT_DRIVER) 966 err = !efa_modify_srd_qp_is_ok(cur_state, new_state, 967 qp_attr_mask); 968 else 969 err = !ib_modify_qp_is_ok(cur_state, new_state, IB_QPT_UD, 970 qp_attr_mask); 971 972 if (err) { 973 ibdev_dbg(&dev->ibdev, "Invalid modify QP parameters\n"); 974 return -EINVAL; 975 } 976 977 if ((qp_attr_mask & IB_QP_PORT) && qp_attr->port_num != 1) { 978 ibdev_dbg(&dev->ibdev, "Can't change port num\n"); 979 return -EOPNOTSUPP; 980 } 981 982 if ((qp_attr_mask & IB_QP_PKEY_INDEX) && qp_attr->pkey_index) { 983 ibdev_dbg(&dev->ibdev, "Can't change pkey index\n"); 984 return -EOPNOTSUPP; 985 } 986 987 return 0; 988 } 989 990 int efa_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, 991 int qp_attr_mask, struct ib_udata *udata) 992 { 993 struct efa_dev *dev = to_edev(ibqp->device); 994 struct efa_com_modify_qp_params params = {}; 995 struct efa_qp *qp = to_eqp(ibqp); 996 enum ib_qp_state cur_state; 997 enum ib_qp_state new_state; 998 int err; 999 1000 if (qp_attr_mask & ~IB_QP_ATTR_STANDARD_BITS) 1001 return -EOPNOTSUPP; 1002 1003 err = ib_is_udata_in_empty(udata); 1004 if (err) 1005 return err; 1006 1007 cur_state = qp_attr_mask & IB_QP_CUR_STATE ? qp_attr->cur_qp_state : 1008 qp->state; 1009 new_state = qp_attr_mask & IB_QP_STATE ? qp_attr->qp_state : cur_state; 1010 1011 err = efa_modify_qp_validate(dev, qp, qp_attr, qp_attr_mask, cur_state, 1012 new_state); 1013 if (err) 1014 return err; 1015 1016 params.qp_handle = qp->qp_handle; 1017 1018 if (qp_attr_mask & IB_QP_STATE) { 1019 EFA_SET(¶ms.modify_mask, EFA_ADMIN_MODIFY_QP_CMD_QP_STATE, 1020 1); 1021 EFA_SET(¶ms.modify_mask, 1022 EFA_ADMIN_MODIFY_QP_CMD_CUR_QP_STATE, 1); 1023 params.cur_qp_state = cur_state; 1024 params.qp_state = new_state; 1025 } 1026 1027 if (qp_attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY) { 1028 EFA_SET(¶ms.modify_mask, 1029 EFA_ADMIN_MODIFY_QP_CMD_SQ_DRAINED_ASYNC_NOTIFY, 1); 1030 params.sq_drained_async_notify = qp_attr->en_sqd_async_notify; 1031 } 1032 1033 if (qp_attr_mask & IB_QP_QKEY) { 1034 EFA_SET(¶ms.modify_mask, EFA_ADMIN_MODIFY_QP_CMD_QKEY, 1); 1035 params.qkey = qp_attr->qkey; 1036 } 1037 1038 if (qp_attr_mask & IB_QP_SQ_PSN) { 1039 EFA_SET(¶ms.modify_mask, EFA_ADMIN_MODIFY_QP_CMD_SQ_PSN, 1); 1040 params.sq_psn = qp_attr->sq_psn; 1041 } 1042 1043 if (qp_attr_mask & IB_QP_RNR_RETRY) { 1044 EFA_SET(¶ms.modify_mask, EFA_ADMIN_MODIFY_QP_CMD_RNR_RETRY, 1045 1); 1046 params.rnr_retry = qp_attr->rnr_retry; 1047 } 1048 1049 err = efa_com_modify_qp(&dev->edev, ¶ms); 1050 if (err) 1051 return err; 1052 1053 qp->state = new_state; 1054 1055 return 0; 1056 } 1057 1058 static int efa_destroy_cq_idx(struct efa_dev *dev, int cq_idx) 1059 { 1060 struct efa_com_destroy_cq_params params = { .cq_idx = cq_idx }; 1061 1062 return efa_com_destroy_cq(&dev->edev, ¶ms); 1063 } 1064 1065 static void efa_cq_user_mmap_entries_remove(struct efa_cq *cq) 1066 { 1067 rdma_user_mmap_entry_remove(cq->db_mmap_entry); 1068 rdma_user_mmap_entry_remove(cq->mmap_entry); 1069 } 1070 1071 int efa_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) 1072 { 1073 struct efa_dev *dev = to_edev(ibcq->device); 1074 struct efa_cq *cq = to_ecq(ibcq); 1075 1076 ibdev_dbg(&dev->ibdev, 1077 "Destroy cq[%d] virt[0x%p] freed: size[%lu], dma[%pad]\n", 1078 cq->cq_idx, cq->cpu_addr, cq->size, &cq->dma_addr); 1079 1080 efa_destroy_cq_idx(dev, cq->cq_idx); 1081 if (cq->cpu_addr) 1082 efa_cq_user_mmap_entries_remove(cq); 1083 if (cq->eq) { 1084 xa_erase(&dev->cqs_xa, cq->cq_idx); 1085 synchronize_irq(cq->eq->irq.irqn); 1086 } 1087 1088 if (cq->cpu_addr) 1089 efa_free_mapped(dev, cq->cpu_addr, cq->dma_addr, cq->size, DMA_FROM_DEVICE); 1090 ib_umem_release(cq->umem); 1091 return 0; 1092 } 1093 1094 static struct efa_eq *efa_vec2eq(struct efa_dev *dev, int vec) 1095 { 1096 return &dev->eqs[vec]; 1097 } 1098 1099 static int cq_mmap_entries_setup(struct efa_dev *dev, struct efa_cq *cq, 1100 struct efa_ibv_create_cq_resp *resp, 1101 bool db_valid) 1102 { 1103 resp->q_mmap_size = cq->size; 1104 cq->mmap_entry = efa_user_mmap_entry_insert(&cq->ucontext->ibucontext, 1105 virt_to_phys(cq->cpu_addr), 1106 cq->size, EFA_MMAP_DMA_PAGE, 1107 &resp->q_mmap_key); 1108 if (!cq->mmap_entry) 1109 return -ENOMEM; 1110 1111 if (db_valid) { 1112 cq->db_mmap_entry = 1113 efa_user_mmap_entry_insert(&cq->ucontext->ibucontext, 1114 dev->db_bar_addr + resp->db_off, 1115 PAGE_SIZE, EFA_MMAP_IO_NC, 1116 &resp->db_mmap_key); 1117 if (!cq->db_mmap_entry) { 1118 rdma_user_mmap_entry_remove(cq->mmap_entry); 1119 return -ENOMEM; 1120 } 1121 1122 resp->db_off &= ~PAGE_MASK; 1123 resp->comp_mask |= EFA_CREATE_CQ_RESP_DB_OFF; 1124 } 1125 1126 return 0; 1127 } 1128 1129 int efa_create_user_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, 1130 struct uverbs_attr_bundle *attrs) 1131 { 1132 struct ib_udata *udata = &attrs->driver_udata; 1133 struct efa_ucontext *ucontext = rdma_udata_to_drv_context( 1134 udata, struct efa_ucontext, ibucontext); 1135 struct efa_com_create_cq_params params = {}; 1136 struct efa_ibv_create_cq_resp resp = {}; 1137 struct efa_com_create_cq_result result; 1138 struct ib_device *ibdev = ibcq->device; 1139 struct efa_dev *dev = to_edev(ibdev); 1140 struct efa_ibv_create_cq cmd; 1141 struct efa_cq *cq = to_ecq(ibcq); 1142 int entries = attr->cqe; 1143 struct ib_umem *umem; 1144 bool set_src_addr; 1145 int err; 1146 1147 ibdev_dbg(ibdev, "create_cq entries %d\n", entries); 1148 1149 if (attr->flags) 1150 return -EOPNOTSUPP; 1151 1152 if (entries > dev->dev_attr.max_cq_depth) { 1153 ibdev_dbg(ibdev, 1154 "cq: requested entries[%u] greater than max[%u]\n", 1155 entries, dev->dev_attr.max_cq_depth); 1156 err = -EINVAL; 1157 goto err_out; 1158 } 1159 1160 err = ib_copy_validate_udata_in_cm(udata, cmd, num_sub_cqs, 0); 1161 if (err) 1162 goto err_out; 1163 1164 if (!is_reserved_cleared(cmd.reserved_58)) { 1165 ibdev_dbg(ibdev, 1166 "Incompatible ABI params, unknown fields in udata\n"); 1167 err = -EINVAL; 1168 goto err_out; 1169 } 1170 1171 set_src_addr = !!(cmd.flags & EFA_CREATE_CQ_WITH_SGID); 1172 if ((cmd.cq_entry_size != sizeof(struct efa_io_rx_cdesc_ex)) && 1173 (set_src_addr || 1174 cmd.cq_entry_size != sizeof(struct efa_io_rx_cdesc))) { 1175 ibdev_dbg(ibdev, 1176 "Invalid entry size [%u]\n", cmd.cq_entry_size); 1177 err = -EINVAL; 1178 goto err_out; 1179 } 1180 1181 if (cmd.num_sub_cqs != dev->dev_attr.sub_cqs_per_cq) { 1182 ibdev_dbg(ibdev, 1183 "Invalid number of sub cqs[%u] expected[%u]\n", 1184 cmd.num_sub_cqs, dev->dev_attr.sub_cqs_per_cq); 1185 err = -EINVAL; 1186 goto err_out; 1187 } 1188 1189 cq->ucontext = ucontext; 1190 cq->size = PAGE_ALIGN(cmd.cq_entry_size * entries * cmd.num_sub_cqs); 1191 1192 umem = ib_umem_get_cq_buf(ibcq->device, attrs, cq->size, 1193 IB_ACCESS_LOCAL_WRITE); 1194 if (IS_ERR(umem)) { 1195 err = PTR_ERR(umem); 1196 goto err_out; 1197 } 1198 1199 cq->umem = umem; 1200 1201 if (umem) { 1202 if (!ib_umem_is_contiguous(umem)) { 1203 ibdev_dbg(&dev->ibdev, "Non contiguous CQ unsupported\n"); 1204 err = -EINVAL; 1205 goto err_release_umem; 1206 } 1207 1208 cq->dma_addr = ib_umem_start_dma_addr(umem); 1209 } else { 1210 cq->cpu_addr = efa_zalloc_mapped(dev, &cq->dma_addr, cq->size, 1211 DMA_FROM_DEVICE); 1212 if (!cq->cpu_addr) { 1213 err = -ENOMEM; 1214 goto err_release_umem; 1215 } 1216 } 1217 1218 params.uarn = cq->ucontext->uarn; 1219 params.sub_cq_depth = entries; 1220 params.dma_addr = cq->dma_addr; 1221 params.entry_size_in_bytes = cmd.cq_entry_size; 1222 params.num_sub_cqs = cmd.num_sub_cqs; 1223 params.set_src_addr = set_src_addr; 1224 params.sq_comp_64_bit_req_id = !!(cmd.flags & EFA_CREATE_CQ_WITH_SQ_COMP_64_BIT_REQ_ID); 1225 if (cmd.flags & EFA_CREATE_CQ_WITH_COMPLETION_CHANNEL) { 1226 cq->eq = efa_vec2eq(dev, attr->comp_vector); 1227 params.eqn = cq->eq->eeq.eqn; 1228 params.interrupt_mode_enabled = true; 1229 } 1230 1231 err = efa_com_create_cq(&dev->edev, ¶ms, &result); 1232 if (err) 1233 goto err_free_mapped; 1234 1235 resp.db_off = result.db_off; 1236 resp.cq_idx = result.cq_idx; 1237 cq->cq_idx = result.cq_idx; 1238 cq->ibcq.cqe = result.actual_depth; 1239 WARN_ON_ONCE(entries != result.actual_depth); 1240 1241 if (cq->cpu_addr) 1242 err = cq_mmap_entries_setup(dev, cq, &resp, result.db_valid); 1243 1244 if (err) { 1245 ibdev_dbg(ibdev, "Could not setup cq[%u] mmap entries\n", 1246 cq->cq_idx); 1247 goto err_destroy_cq; 1248 } 1249 1250 if (cq->eq) { 1251 err = xa_err(xa_store(&dev->cqs_xa, cq->cq_idx, cq, GFP_KERNEL)); 1252 if (err) { 1253 ibdev_dbg(ibdev, "Failed to store cq[%u] in xarray\n", 1254 cq->cq_idx); 1255 goto err_remove_mmap; 1256 } 1257 } 1258 1259 if (udata->outlen) { 1260 err = ib_respond_udata(udata, resp); 1261 if (err) 1262 goto err_xa_erase; 1263 } 1264 1265 ibdev_dbg(ibdev, "Created cq[%d], cq depth[%u]. dma[%pad] virt[0x%p]\n", 1266 cq->cq_idx, result.actual_depth, &cq->dma_addr, cq->cpu_addr); 1267 1268 return 0; 1269 1270 err_xa_erase: 1271 if (cq->eq) 1272 xa_erase(&dev->cqs_xa, cq->cq_idx); 1273 err_remove_mmap: 1274 if (cq->cpu_addr) 1275 efa_cq_user_mmap_entries_remove(cq); 1276 err_destroy_cq: 1277 efa_destroy_cq_idx(dev, cq->cq_idx); 1278 err_free_mapped: 1279 if (cq->cpu_addr) 1280 efa_free_mapped(dev, cq->cpu_addr, cq->dma_addr, cq->size, 1281 DMA_FROM_DEVICE); 1282 err_release_umem: 1283 ib_umem_release(cq->umem); 1284 err_out: 1285 atomic64_inc(&dev->stats.create_cq_err); 1286 return err; 1287 } 1288 1289 static int umem_to_page_list(struct efa_dev *dev, 1290 struct ib_umem *umem, 1291 u64 *page_list, 1292 u32 hp_cnt, 1293 u8 hp_shift) 1294 { 1295 struct ib_block_iter biter; 1296 unsigned int hp_idx = 0; 1297 1298 rdma_umem_for_each_dma_block(umem, &biter, BIT(hp_shift)) 1299 page_list[hp_idx++] = rdma_block_iter_dma_address(&biter); 1300 1301 return 0; 1302 } 1303 1304 static struct scatterlist *efa_vmalloc_buf_to_sg(u64 *buf, int page_cnt) 1305 { 1306 struct scatterlist *sglist; 1307 struct page *pg; 1308 int i; 1309 1310 sglist = kmalloc_objs(*sglist, page_cnt); 1311 if (!sglist) 1312 return NULL; 1313 sg_init_table(sglist, page_cnt); 1314 for (i = 0; i < page_cnt; i++) { 1315 pg = vmalloc_to_page(buf); 1316 if (!pg) 1317 goto err; 1318 sg_set_page(&sglist[i], pg, PAGE_SIZE, 0); 1319 buf += PAGE_SIZE / sizeof(*buf); 1320 } 1321 return sglist; 1322 1323 err: 1324 kfree(sglist); 1325 return NULL; 1326 } 1327 1328 /* 1329 * create a chunk list of physical pages dma addresses from the supplied 1330 * scatter gather list 1331 */ 1332 static int pbl_chunk_list_create(struct efa_dev *dev, struct pbl_context *pbl) 1333 { 1334 struct pbl_chunk_list *chunk_list = &pbl->phys.indirect.chunk_list; 1335 int page_cnt = pbl->phys.indirect.pbl_buf_size_in_pages; 1336 struct scatterlist *pages_sgl = pbl->phys.indirect.sgl; 1337 unsigned int chunk_list_size, chunk_idx, payload_idx; 1338 int sg_dma_cnt = pbl->phys.indirect.sg_dma_cnt; 1339 struct efa_com_ctrl_buff_info *ctrl_buf; 1340 u64 *cur_chunk_buf, *prev_chunk_buf; 1341 struct ib_block_iter biter; 1342 dma_addr_t dma_addr; 1343 int i; 1344 1345 /* allocate a chunk list that consists of 4KB chunks */ 1346 chunk_list_size = DIV_ROUND_UP(page_cnt, EFA_PTRS_PER_CHUNK); 1347 1348 chunk_list->size = chunk_list_size; 1349 chunk_list->chunks = kzalloc_objs(*chunk_list->chunks, chunk_list_size); 1350 if (!chunk_list->chunks) 1351 return -ENOMEM; 1352 1353 ibdev_dbg(&dev->ibdev, 1354 "chunk_list_size[%u] - pages[%u]\n", chunk_list_size, 1355 page_cnt); 1356 1357 /* allocate chunk buffers: */ 1358 for (i = 0; i < chunk_list_size; i++) { 1359 chunk_list->chunks[i].buf = kzalloc(EFA_CHUNK_SIZE, GFP_KERNEL); 1360 if (!chunk_list->chunks[i].buf) 1361 goto chunk_list_dealloc; 1362 1363 chunk_list->chunks[i].length = EFA_CHUNK_USED_SIZE; 1364 } 1365 1366 if (page_cnt % EFA_PTRS_PER_CHUNK != 0) 1367 chunk_list->chunks[chunk_list_size - 1].length = 1368 ((page_cnt % EFA_PTRS_PER_CHUNK) * EFA_CHUNK_PAYLOAD_PTR_SIZE) + 1369 EFA_CHUNK_PTR_SIZE; 1370 1371 /* fill the dma addresses of sg list pages to chunks: */ 1372 chunk_idx = 0; 1373 payload_idx = 0; 1374 cur_chunk_buf = chunk_list->chunks[0].buf; 1375 rdma_for_each_block(pages_sgl, &biter, sg_dma_cnt, 1376 EFA_CHUNK_PAYLOAD_SIZE) { 1377 cur_chunk_buf[payload_idx++] = 1378 rdma_block_iter_dma_address(&biter); 1379 1380 if (payload_idx == EFA_PTRS_PER_CHUNK) { 1381 payload_idx = 0; 1382 chunk_idx++; 1383 if (chunk_idx >= chunk_list_size) 1384 break; 1385 1386 cur_chunk_buf = chunk_list->chunks[chunk_idx].buf; 1387 } 1388 } 1389 1390 /* map chunks to dma and fill chunks next ptrs */ 1391 for (i = chunk_list_size - 1; i >= 0; i--) { 1392 dma_addr = dma_map_single(&dev->pdev->dev, 1393 chunk_list->chunks[i].buf, 1394 chunk_list->chunks[i].length, 1395 DMA_TO_DEVICE); 1396 if (dma_mapping_error(&dev->pdev->dev, dma_addr)) { 1397 ibdev_err(&dev->ibdev, 1398 "chunk[%u] dma_map_failed\n", i); 1399 goto chunk_list_unmap; 1400 } 1401 1402 chunk_list->chunks[i].dma_addr = dma_addr; 1403 ibdev_dbg(&dev->ibdev, 1404 "chunk[%u] mapped at [%pad]\n", i, &dma_addr); 1405 1406 if (!i) 1407 break; 1408 1409 prev_chunk_buf = chunk_list->chunks[i - 1].buf; 1410 1411 ctrl_buf = (struct efa_com_ctrl_buff_info *) 1412 &prev_chunk_buf[EFA_PTRS_PER_CHUNK]; 1413 ctrl_buf->length = chunk_list->chunks[i].length; 1414 1415 efa_com_set_dma_addr(dma_addr, 1416 &ctrl_buf->address.mem_addr_high, 1417 &ctrl_buf->address.mem_addr_low); 1418 } 1419 1420 return 0; 1421 1422 chunk_list_unmap: 1423 for (; i < chunk_list_size; i++) { 1424 dma_unmap_single(&dev->pdev->dev, chunk_list->chunks[i].dma_addr, 1425 chunk_list->chunks[i].length, DMA_TO_DEVICE); 1426 } 1427 chunk_list_dealloc: 1428 for (i = 0; i < chunk_list_size; i++) 1429 kfree(chunk_list->chunks[i].buf); 1430 1431 kfree(chunk_list->chunks); 1432 return -ENOMEM; 1433 } 1434 1435 static void pbl_chunk_list_destroy(struct efa_dev *dev, struct pbl_context *pbl) 1436 { 1437 struct pbl_chunk_list *chunk_list = &pbl->phys.indirect.chunk_list; 1438 int i; 1439 1440 for (i = 0; i < chunk_list->size; i++) { 1441 dma_unmap_single(&dev->pdev->dev, chunk_list->chunks[i].dma_addr, 1442 chunk_list->chunks[i].length, DMA_TO_DEVICE); 1443 kfree(chunk_list->chunks[i].buf); 1444 } 1445 1446 kfree(chunk_list->chunks); 1447 } 1448 1449 /* initialize pbl continuous mode: map pbl buffer to a dma address. */ 1450 static int pbl_continuous_initialize(struct efa_dev *dev, 1451 struct pbl_context *pbl) 1452 { 1453 dma_addr_t dma_addr; 1454 1455 dma_addr = dma_map_single(&dev->pdev->dev, pbl->pbl_buf, 1456 pbl->pbl_buf_size_in_bytes, DMA_TO_DEVICE); 1457 if (dma_mapping_error(&dev->pdev->dev, dma_addr)) { 1458 ibdev_err(&dev->ibdev, "Unable to map pbl to DMA address\n"); 1459 return -ENOMEM; 1460 } 1461 1462 pbl->phys.continuous.dma_addr = dma_addr; 1463 ibdev_dbg(&dev->ibdev, 1464 "pbl continuous - dma_addr = %pad, size[%u]\n", 1465 &dma_addr, pbl->pbl_buf_size_in_bytes); 1466 1467 return 0; 1468 } 1469 1470 /* 1471 * initialize pbl indirect mode: 1472 * create a chunk list out of the dma addresses of the physical pages of 1473 * pbl buffer. 1474 */ 1475 static int pbl_indirect_initialize(struct efa_dev *dev, struct pbl_context *pbl) 1476 { 1477 u32 size_in_pages = DIV_ROUND_UP(pbl->pbl_buf_size_in_bytes, EFA_CHUNK_PAYLOAD_SIZE); 1478 struct scatterlist *sgl; 1479 int sg_dma_cnt, err; 1480 1481 BUILD_BUG_ON(EFA_CHUNK_PAYLOAD_SIZE > PAGE_SIZE); 1482 sgl = efa_vmalloc_buf_to_sg(pbl->pbl_buf, size_in_pages); 1483 if (!sgl) 1484 return -ENOMEM; 1485 1486 sg_dma_cnt = dma_map_sg(&dev->pdev->dev, sgl, size_in_pages, DMA_TO_DEVICE); 1487 if (!sg_dma_cnt) { 1488 err = -EINVAL; 1489 goto err_map; 1490 } 1491 1492 pbl->phys.indirect.pbl_buf_size_in_pages = size_in_pages; 1493 pbl->phys.indirect.sgl = sgl; 1494 pbl->phys.indirect.sg_dma_cnt = sg_dma_cnt; 1495 err = pbl_chunk_list_create(dev, pbl); 1496 if (err) { 1497 ibdev_dbg(&dev->ibdev, 1498 "chunk_list creation failed[%d]\n", err); 1499 goto err_chunk; 1500 } 1501 1502 ibdev_dbg(&dev->ibdev, 1503 "pbl indirect - size[%u], chunks[%u]\n", 1504 pbl->pbl_buf_size_in_bytes, 1505 pbl->phys.indirect.chunk_list.size); 1506 1507 return 0; 1508 1509 err_chunk: 1510 dma_unmap_sg(&dev->pdev->dev, sgl, size_in_pages, DMA_TO_DEVICE); 1511 err_map: 1512 kfree(sgl); 1513 return err; 1514 } 1515 1516 static void pbl_indirect_terminate(struct efa_dev *dev, struct pbl_context *pbl) 1517 { 1518 pbl_chunk_list_destroy(dev, pbl); 1519 dma_unmap_sg(&dev->pdev->dev, pbl->phys.indirect.sgl, 1520 pbl->phys.indirect.pbl_buf_size_in_pages, DMA_TO_DEVICE); 1521 kfree(pbl->phys.indirect.sgl); 1522 } 1523 1524 /* create a page buffer list from a mapped user memory region */ 1525 static int pbl_create(struct efa_dev *dev, 1526 struct pbl_context *pbl, 1527 struct ib_umem *umem, 1528 int hp_cnt, 1529 u8 hp_shift) 1530 { 1531 int err; 1532 1533 pbl->pbl_buf_size_in_bytes = hp_cnt * EFA_CHUNK_PAYLOAD_PTR_SIZE; 1534 pbl->pbl_buf = kvzalloc(pbl->pbl_buf_size_in_bytes, GFP_KERNEL); 1535 if (!pbl->pbl_buf) 1536 return -ENOMEM; 1537 1538 if (is_vmalloc_addr(pbl->pbl_buf)) { 1539 pbl->physically_continuous = 0; 1540 err = umem_to_page_list(dev, umem, pbl->pbl_buf, hp_cnt, 1541 hp_shift); 1542 if (err) 1543 goto err_free; 1544 1545 err = pbl_indirect_initialize(dev, pbl); 1546 if (err) 1547 goto err_free; 1548 } else { 1549 pbl->physically_continuous = 1; 1550 err = umem_to_page_list(dev, umem, pbl->pbl_buf, hp_cnt, 1551 hp_shift); 1552 if (err) 1553 goto err_free; 1554 1555 err = pbl_continuous_initialize(dev, pbl); 1556 if (err) 1557 goto err_free; 1558 } 1559 1560 ibdev_dbg(&dev->ibdev, 1561 "user_pbl_created: user_pages[%u], continuous[%u]\n", 1562 hp_cnt, pbl->physically_continuous); 1563 1564 return 0; 1565 1566 err_free: 1567 kvfree(pbl->pbl_buf); 1568 return err; 1569 } 1570 1571 static void pbl_destroy(struct efa_dev *dev, struct pbl_context *pbl) 1572 { 1573 if (pbl->physically_continuous) 1574 dma_unmap_single(&dev->pdev->dev, pbl->phys.continuous.dma_addr, 1575 pbl->pbl_buf_size_in_bytes, DMA_TO_DEVICE); 1576 else 1577 pbl_indirect_terminate(dev, pbl); 1578 1579 kvfree(pbl->pbl_buf); 1580 } 1581 1582 static int efa_create_inline_pbl(struct efa_dev *dev, struct efa_mr *mr, 1583 struct efa_com_reg_mr_params *params) 1584 { 1585 int err; 1586 1587 params->inline_pbl = 1; 1588 err = umem_to_page_list(dev, mr->umem, params->pbl.inline_pbl_array, 1589 params->page_num, params->page_shift); 1590 if (err) 1591 return err; 1592 1593 ibdev_dbg(&dev->ibdev, 1594 "inline_pbl_array - pages[%u]\n", params->page_num); 1595 1596 return 0; 1597 } 1598 1599 static int efa_create_pbl(struct efa_dev *dev, 1600 struct pbl_context *pbl, 1601 struct efa_mr *mr, 1602 struct efa_com_reg_mr_params *params) 1603 { 1604 int err; 1605 1606 err = pbl_create(dev, pbl, mr->umem, params->page_num, 1607 params->page_shift); 1608 if (err) { 1609 ibdev_dbg(&dev->ibdev, "Failed to create pbl[%d]\n", err); 1610 return err; 1611 } 1612 1613 params->inline_pbl = 0; 1614 params->indirect = !pbl->physically_continuous; 1615 if (pbl->physically_continuous) { 1616 params->pbl.pbl.length = pbl->pbl_buf_size_in_bytes; 1617 1618 efa_com_set_dma_addr(pbl->phys.continuous.dma_addr, 1619 ¶ms->pbl.pbl.address.mem_addr_high, 1620 ¶ms->pbl.pbl.address.mem_addr_low); 1621 } else { 1622 params->pbl.pbl.length = 1623 pbl->phys.indirect.chunk_list.chunks[0].length; 1624 1625 efa_com_set_dma_addr(pbl->phys.indirect.chunk_list.chunks[0].dma_addr, 1626 ¶ms->pbl.pbl.address.mem_addr_high, 1627 ¶ms->pbl.pbl.address.mem_addr_low); 1628 } 1629 1630 return 0; 1631 } 1632 1633 static struct efa_mr *efa_alloc_mr(struct ib_pd *ibpd, int access_flags, 1634 struct ib_udata *udata) 1635 { 1636 struct efa_dev *dev = to_edev(ibpd->device); 1637 int supp_access_flags; 1638 struct efa_mr *mr; 1639 int ret; 1640 1641 ret = ib_is_udata_in_empty(udata); 1642 if (ret) 1643 return ERR_PTR(ret); 1644 1645 supp_access_flags = 1646 IB_ACCESS_LOCAL_WRITE | 1647 (EFA_DEV_CAP(dev, RDMA_READ) ? IB_ACCESS_REMOTE_READ : 0) | 1648 (EFA_DEV_CAP(dev, RDMA_WRITE) ? IB_ACCESS_REMOTE_WRITE : 0); 1649 1650 access_flags &= ~IB_ACCESS_OPTIONAL; 1651 if (access_flags & ~supp_access_flags) { 1652 ibdev_dbg(&dev->ibdev, 1653 "Unsupported access flags[%#x], supported[%#x]\n", 1654 access_flags, supp_access_flags); 1655 return ERR_PTR(-EOPNOTSUPP); 1656 } 1657 1658 mr = kzalloc_obj(*mr); 1659 if (!mr) 1660 return ERR_PTR(-ENOMEM); 1661 1662 return mr; 1663 } 1664 1665 static int efa_register_mr(struct ib_pd *ibpd, struct efa_mr *mr, u64 start, 1666 u64 length, u64 virt_addr, int access_flags) 1667 { 1668 struct efa_dev *dev = to_edev(ibpd->device); 1669 struct efa_com_reg_mr_params params = {}; 1670 struct efa_com_reg_mr_result result = {}; 1671 struct pbl_context pbl; 1672 unsigned int pg_sz; 1673 int inline_size; 1674 int err; 1675 1676 params.pd = to_epd(ibpd)->pdn; 1677 params.iova = virt_addr; 1678 params.mr_length_in_bytes = length; 1679 params.permissions = access_flags; 1680 1681 pg_sz = ib_umem_find_best_pgsz(mr->umem, 1682 dev->dev_attr.page_size_cap, 1683 virt_addr); 1684 if (!pg_sz) { 1685 ibdev_dbg(&dev->ibdev, "Failed to find a suitable page size in page_size_cap %#llx\n", 1686 dev->dev_attr.page_size_cap); 1687 return -EOPNOTSUPP; 1688 } 1689 1690 params.page_shift = order_base_2(pg_sz); 1691 params.page_num = ib_umem_num_dma_blocks(mr->umem, pg_sz); 1692 1693 ibdev_dbg(&dev->ibdev, 1694 "start %#llx length %#llx params.page_shift %u params.page_num %u\n", 1695 start, length, params.page_shift, params.page_num); 1696 1697 inline_size = ARRAY_SIZE(params.pbl.inline_pbl_array); 1698 if (params.page_num <= inline_size) { 1699 err = efa_create_inline_pbl(dev, mr, ¶ms); 1700 if (err) 1701 return err; 1702 1703 err = efa_com_register_mr(&dev->edev, ¶ms, &result); 1704 if (err) 1705 return err; 1706 } else { 1707 err = efa_create_pbl(dev, &pbl, mr, ¶ms); 1708 if (err) 1709 return err; 1710 1711 err = efa_com_register_mr(&dev->edev, ¶ms, &result); 1712 pbl_destroy(dev, &pbl); 1713 1714 if (err) 1715 return err; 1716 } 1717 1718 mr->ibmr.lkey = result.l_key; 1719 mr->ibmr.rkey = result.r_key; 1720 mr->ibmr.length = length; 1721 mr->ic_info.recv_ic_id = result.ic_info.recv_ic_id; 1722 mr->ic_info.rdma_read_ic_id = result.ic_info.rdma_read_ic_id; 1723 mr->ic_info.rdma_recv_ic_id = result.ic_info.rdma_recv_ic_id; 1724 mr->ic_info.recv_ic_id_valid = result.ic_info.recv_ic_id_valid; 1725 mr->ic_info.rdma_read_ic_id_valid = result.ic_info.rdma_read_ic_id_valid; 1726 mr->ic_info.rdma_recv_ic_id_valid = result.ic_info.rdma_recv_ic_id_valid; 1727 ibdev_dbg(&dev->ibdev, "Registered mr[%d]\n", mr->ibmr.lkey); 1728 1729 return 0; 1730 } 1731 1732 struct ib_mr *efa_reg_user_mr_dmabuf(struct ib_pd *ibpd, u64 start, 1733 u64 length, u64 virt_addr, 1734 int fd, int access_flags, 1735 struct ib_dmah *dmah, 1736 struct uverbs_attr_bundle *attrs) 1737 { 1738 struct efa_dev *dev = to_edev(ibpd->device); 1739 struct ib_umem_dmabuf *umem_dmabuf; 1740 struct efa_mr *mr; 1741 int err; 1742 1743 if (dmah) { 1744 err = -EOPNOTSUPP; 1745 goto err_out; 1746 } 1747 1748 mr = efa_alloc_mr(ibpd, access_flags, &attrs->driver_udata); 1749 if (IS_ERR(mr)) { 1750 err = PTR_ERR(mr); 1751 goto err_out; 1752 } 1753 1754 umem_dmabuf = ib_umem_dmabuf_get_pinned(ibpd->device, start, length, fd, 1755 access_flags); 1756 if (IS_ERR(umem_dmabuf)) { 1757 err = PTR_ERR(umem_dmabuf); 1758 ibdev_dbg(&dev->ibdev, "Failed to get dmabuf umem[%pe]\n", 1759 umem_dmabuf); 1760 goto err_free; 1761 } 1762 1763 mr->umem = &umem_dmabuf->umem; 1764 err = efa_register_mr(ibpd, mr, start, length, virt_addr, access_flags); 1765 if (err) 1766 goto err_release; 1767 1768 return &mr->ibmr; 1769 1770 err_release: 1771 ib_umem_release(mr->umem); 1772 err_free: 1773 kfree(mr); 1774 err_out: 1775 atomic64_inc(&dev->stats.reg_mr_err); 1776 return ERR_PTR(err); 1777 } 1778 1779 struct ib_mr *efa_reg_mr(struct ib_pd *ibpd, u64 start, u64 length, 1780 u64 virt_addr, int access_flags, 1781 struct ib_dmah *dmah, 1782 struct ib_udata *udata) 1783 { 1784 struct efa_dev *dev = to_edev(ibpd->device); 1785 struct efa_mr *mr; 1786 int err; 1787 1788 if (dmah) { 1789 err = -EOPNOTSUPP; 1790 goto err_out; 1791 } 1792 1793 mr = efa_alloc_mr(ibpd, access_flags, udata); 1794 if (IS_ERR(mr)) { 1795 err = PTR_ERR(mr); 1796 goto err_out; 1797 } 1798 1799 mr->umem = ib_umem_get_va(ibpd->device, start, length, access_flags); 1800 if (IS_ERR(mr->umem)) { 1801 err = PTR_ERR(mr->umem); 1802 ibdev_dbg(&dev->ibdev, 1803 "Failed to pin and map user space memory[%pe]\n", 1804 mr->umem); 1805 goto err_free; 1806 } 1807 1808 err = efa_register_mr(ibpd, mr, start, length, virt_addr, access_flags); 1809 if (err) 1810 goto err_release; 1811 1812 return &mr->ibmr; 1813 1814 err_release: 1815 ib_umem_release(mr->umem); 1816 err_free: 1817 kfree(mr); 1818 err_out: 1819 atomic64_inc(&dev->stats.reg_mr_err); 1820 return ERR_PTR(err); 1821 } 1822 1823 static int UVERBS_HANDLER(EFA_IB_METHOD_MR_QUERY)(struct uverbs_attr_bundle *attrs) 1824 { 1825 struct ib_mr *ibmr = uverbs_attr_get_obj(attrs, EFA_IB_ATTR_QUERY_MR_HANDLE); 1826 struct efa_mr *mr = to_emr(ibmr); 1827 u16 ic_id_validity = 0; 1828 int ret; 1829 1830 ret = uverbs_copy_to(attrs, EFA_IB_ATTR_QUERY_MR_RESP_RECV_IC_ID, 1831 &mr->ic_info.recv_ic_id, sizeof(mr->ic_info.recv_ic_id)); 1832 if (ret) 1833 return ret; 1834 1835 ret = uverbs_copy_to(attrs, EFA_IB_ATTR_QUERY_MR_RESP_RDMA_READ_IC_ID, 1836 &mr->ic_info.rdma_read_ic_id, sizeof(mr->ic_info.rdma_read_ic_id)); 1837 if (ret) 1838 return ret; 1839 1840 ret = uverbs_copy_to(attrs, EFA_IB_ATTR_QUERY_MR_RESP_RDMA_RECV_IC_ID, 1841 &mr->ic_info.rdma_recv_ic_id, sizeof(mr->ic_info.rdma_recv_ic_id)); 1842 if (ret) 1843 return ret; 1844 1845 if (mr->ic_info.recv_ic_id_valid) 1846 ic_id_validity |= EFA_QUERY_MR_VALIDITY_RECV_IC_ID; 1847 if (mr->ic_info.rdma_read_ic_id_valid) 1848 ic_id_validity |= EFA_QUERY_MR_VALIDITY_RDMA_READ_IC_ID; 1849 if (mr->ic_info.rdma_recv_ic_id_valid) 1850 ic_id_validity |= EFA_QUERY_MR_VALIDITY_RDMA_RECV_IC_ID; 1851 1852 return uverbs_copy_to(attrs, EFA_IB_ATTR_QUERY_MR_RESP_IC_ID_VALIDITY, 1853 &ic_id_validity, sizeof(ic_id_validity)); 1854 } 1855 1856 int efa_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata) 1857 { 1858 struct efa_dev *dev = to_edev(ibmr->device); 1859 struct efa_com_dereg_mr_params params; 1860 struct efa_mr *mr = to_emr(ibmr); 1861 int err; 1862 1863 ibdev_dbg(&dev->ibdev, "Deregister mr[%d]\n", ibmr->lkey); 1864 1865 params.l_key = mr->ibmr.lkey; 1866 err = efa_com_dereg_mr(&dev->edev, ¶ms); 1867 if (err) 1868 return err; 1869 1870 ib_umem_release(mr->umem); 1871 kfree(mr); 1872 1873 return 0; 1874 } 1875 1876 int efa_get_port_immutable(struct ib_device *ibdev, u32 port_num, 1877 struct ib_port_immutable *immutable) 1878 { 1879 struct ib_port_attr attr; 1880 int err; 1881 1882 err = ib_query_port(ibdev, port_num, &attr); 1883 if (err) { 1884 ibdev_dbg(ibdev, "Couldn't query port err[%d]\n", err); 1885 return err; 1886 } 1887 1888 immutable->pkey_tbl_len = attr.pkey_tbl_len; 1889 immutable->gid_tbl_len = attr.gid_tbl_len; 1890 1891 return 0; 1892 } 1893 1894 static int efa_dealloc_uar(struct efa_dev *dev, u16 uarn) 1895 { 1896 struct efa_com_dealloc_uar_params params = { 1897 .uarn = uarn, 1898 }; 1899 1900 return efa_com_dealloc_uar(&dev->edev, ¶ms); 1901 } 1902 1903 #define EFA_CHECK_USER_SUPP(_dev, _supported_caps, _attr, _mask, _attr_str) \ 1904 (_attr_str = (!(_dev)->dev_attr._attr || ((_supported_caps) & (_mask))) ? \ 1905 NULL : #_attr) 1906 1907 static int efa_user_supp_handshake(const struct ib_ucontext *ibucontext, 1908 const struct efa_ibv_alloc_ucontext_cmd *cmd) 1909 { 1910 struct efa_dev *dev = to_edev(ibucontext->device); 1911 char *attr_str; 1912 1913 if (EFA_CHECK_USER_SUPP(dev, cmd->supported_caps, max_tx_batch, 1914 EFA_ALLOC_UCONTEXT_CMD_SUPP_CAPS_TX_BATCH, 1915 attr_str)) 1916 goto err; 1917 1918 if (EFA_CHECK_USER_SUPP(dev, cmd->supported_caps, min_sq_depth, 1919 EFA_ALLOC_UCONTEXT_CMD_SUPP_CAPS_MIN_SQ_WR, 1920 attr_str)) 1921 goto err; 1922 1923 return 0; 1924 1925 err: 1926 ibdev_dbg(&dev->ibdev, "Userspace handshake failed for %s attribute\n", 1927 attr_str); 1928 return -EOPNOTSUPP; 1929 } 1930 1931 int efa_alloc_ucontext(struct ib_ucontext *ibucontext, struct ib_udata *udata) 1932 { 1933 struct efa_ucontext *ucontext = to_eucontext(ibucontext); 1934 struct efa_dev *dev = to_edev(ibucontext->device); 1935 struct efa_ibv_alloc_ucontext_resp resp = {}; 1936 struct efa_ibv_alloc_ucontext_cmd cmd = {}; 1937 struct efa_com_alloc_uar_result result; 1938 int err; 1939 1940 /* 1941 * it's fine if the driver does not know all request fields, 1942 * we will ack input fields in our response. 1943 */ 1944 1945 err = ib_copy_from_udata(&cmd, udata, 1946 min(sizeof(cmd), udata->inlen)); 1947 if (err) { 1948 ibdev_dbg(&dev->ibdev, 1949 "Cannot copy udata for alloc_ucontext\n"); 1950 goto err_out; 1951 } 1952 1953 err = efa_user_supp_handshake(ibucontext, &cmd); 1954 if (err) 1955 goto err_out; 1956 1957 err = efa_com_alloc_uar(&dev->edev, &result); 1958 if (err) 1959 goto err_out; 1960 1961 ucontext->uarn = result.uarn; 1962 1963 resp.cmds_supp_udata_mask |= EFA_USER_CMDS_SUPP_UDATA_QUERY_DEVICE; 1964 resp.cmds_supp_udata_mask |= EFA_USER_CMDS_SUPP_UDATA_CREATE_AH; 1965 resp.sub_cqs_per_cq = dev->dev_attr.sub_cqs_per_cq; 1966 resp.inline_buf_size = dev->dev_attr.inline_buf_size; 1967 resp.inline_buf_size_ex = dev->dev_attr.inline_buf_size_ex; 1968 resp.max_llq_size = dev->dev_attr.max_llq_size; 1969 resp.max_tx_batch = dev->dev_attr.max_tx_batch; 1970 resp.min_sq_wr = dev->dev_attr.min_sq_depth; 1971 1972 err = ib_respond_udata(udata, resp); 1973 if (err) 1974 goto err_dealloc_uar; 1975 1976 return 0; 1977 1978 err_dealloc_uar: 1979 efa_dealloc_uar(dev, result.uarn); 1980 err_out: 1981 atomic64_inc(&dev->stats.alloc_ucontext_err); 1982 return err; 1983 } 1984 1985 void efa_dealloc_ucontext(struct ib_ucontext *ibucontext) 1986 { 1987 struct efa_ucontext *ucontext = to_eucontext(ibucontext); 1988 struct efa_dev *dev = to_edev(ibucontext->device); 1989 1990 efa_dealloc_uar(dev, ucontext->uarn); 1991 } 1992 1993 void efa_mmap_free(struct rdma_user_mmap_entry *rdma_entry) 1994 { 1995 struct efa_user_mmap_entry *entry = to_emmap(rdma_entry); 1996 1997 kfree(entry); 1998 } 1999 2000 static int __efa_mmap(struct efa_dev *dev, struct efa_ucontext *ucontext, 2001 struct vm_area_struct *vma) 2002 { 2003 struct rdma_user_mmap_entry *rdma_entry; 2004 struct efa_user_mmap_entry *entry; 2005 unsigned long va; 2006 int err = 0; 2007 u64 pfn; 2008 2009 rdma_entry = rdma_user_mmap_entry_get(&ucontext->ibucontext, vma); 2010 if (!rdma_entry) { 2011 ibdev_dbg(&dev->ibdev, 2012 "pgoff[%#lx] does not have valid entry\n", 2013 vma->vm_pgoff); 2014 atomic64_inc(&dev->stats.mmap_err); 2015 return -EINVAL; 2016 } 2017 entry = to_emmap(rdma_entry); 2018 2019 ibdev_dbg(&dev->ibdev, 2020 "Mapping address[%#llx], length[%#zx], mmap_flag[%d]\n", 2021 entry->address, rdma_entry->npages * PAGE_SIZE, 2022 entry->mmap_flag); 2023 2024 pfn = entry->address >> PAGE_SHIFT; 2025 switch (entry->mmap_flag) { 2026 case EFA_MMAP_IO_NC: 2027 err = rdma_user_mmap_io(&ucontext->ibucontext, vma, pfn, 2028 entry->rdma_entry.npages * PAGE_SIZE, 2029 pgprot_noncached(vma->vm_page_prot), 2030 rdma_entry); 2031 break; 2032 case EFA_MMAP_IO_WC: 2033 err = rdma_user_mmap_io(&ucontext->ibucontext, vma, pfn, 2034 entry->rdma_entry.npages * PAGE_SIZE, 2035 pgprot_writecombine(vma->vm_page_prot), 2036 rdma_entry); 2037 break; 2038 case EFA_MMAP_DMA_PAGE: 2039 for (va = vma->vm_start; va < vma->vm_end; 2040 va += PAGE_SIZE, pfn++) { 2041 err = vm_insert_page(vma, va, pfn_to_page(pfn)); 2042 if (err) 2043 break; 2044 } 2045 break; 2046 default: 2047 err = -EINVAL; 2048 } 2049 2050 if (err) { 2051 ibdev_dbg( 2052 &dev->ibdev, 2053 "Couldn't mmap address[%#llx] length[%#zx] mmap_flag[%d] err[%d]\n", 2054 entry->address, rdma_entry->npages * PAGE_SIZE, 2055 entry->mmap_flag, err); 2056 atomic64_inc(&dev->stats.mmap_err); 2057 } 2058 2059 rdma_user_mmap_entry_put(rdma_entry); 2060 return err; 2061 } 2062 2063 int efa_mmap(struct ib_ucontext *ibucontext, 2064 struct vm_area_struct *vma) 2065 { 2066 struct efa_ucontext *ucontext = to_eucontext(ibucontext); 2067 struct efa_dev *dev = to_edev(ibucontext->device); 2068 size_t length = vma->vm_end - vma->vm_start; 2069 2070 ibdev_dbg(&dev->ibdev, 2071 "start %#lx, end %#lx, length = %#zx, pgoff = %#lx\n", 2072 vma->vm_start, vma->vm_end, length, vma->vm_pgoff); 2073 2074 return __efa_mmap(dev, ucontext, vma); 2075 } 2076 2077 static int efa_ah_destroy(struct efa_dev *dev, struct efa_ah *ah) 2078 { 2079 struct efa_com_destroy_ah_params params = {}; 2080 2081 params.ah = ah->ah; 2082 memcpy(params.gid, ah->id, sizeof(params.gid)); 2083 params.pdn = to_epd(ah->ibah.pd)->pdn; 2084 2085 return efa_com_destroy_ah(&dev->edev, ¶ms); 2086 } 2087 2088 int efa_create_ah(struct ib_ah *ibah, 2089 struct rdma_ah_init_attr *init_attr, 2090 struct ib_udata *udata) 2091 { 2092 struct rdma_ah_attr *ah_attr = init_attr->ah_attr; 2093 struct efa_dev *dev = to_edev(ibah->device); 2094 struct efa_com_create_ah_params params = {}; 2095 struct efa_ibv_create_ah_resp resp = {}; 2096 struct efa_com_create_ah_result result; 2097 struct efa_ah *ah = to_eah(ibah); 2098 int err; 2099 2100 if (!(init_attr->flags & RDMA_CREATE_AH_SLEEPABLE)) { 2101 ibdev_dbg(&dev->ibdev, 2102 "Create address handle is not supported in atomic context\n"); 2103 err = -EOPNOTSUPP; 2104 goto err_out; 2105 } 2106 2107 err = ib_is_udata_in_empty(udata); 2108 if (err) 2109 goto err_out; 2110 2111 memcpy(params.dest_addr, ah_attr->grh.dgid.raw, 2112 sizeof(params.dest_addr)); 2113 params.pdn = to_epd(ibah->pd)->pdn; 2114 err = efa_com_create_ah(&dev->edev, ¶ms, &result); 2115 if (err) 2116 goto err_out; 2117 2118 memcpy(ah->id, ah_attr->grh.dgid.raw, sizeof(ah->id)); 2119 ah->ah = result.ah; 2120 2121 resp.efa_address_handle = result.ah; 2122 2123 if (udata->outlen) { 2124 err = ib_respond_udata(udata, resp); 2125 if (err) 2126 goto err_destroy_ah; 2127 } 2128 ibdev_dbg(&dev->ibdev, "Created ah[%d]\n", ah->ah); 2129 2130 return 0; 2131 2132 err_destroy_ah: 2133 efa_ah_destroy(dev, ah); 2134 err_out: 2135 atomic64_inc(&dev->stats.create_ah_err); 2136 return err; 2137 } 2138 2139 int efa_destroy_ah(struct ib_ah *ibah, u32 flags) 2140 { 2141 struct efa_dev *dev = to_edev(ibah->pd->device); 2142 struct efa_ah *ah = to_eah(ibah); 2143 2144 ibdev_dbg(&dev->ibdev, "Destroy ah[%d]\n", ah->ah); 2145 2146 if (!(flags & RDMA_DESTROY_AH_SLEEPABLE)) { 2147 ibdev_dbg(&dev->ibdev, 2148 "Destroy address handle is not supported in atomic context\n"); 2149 return -EOPNOTSUPP; 2150 } 2151 2152 efa_ah_destroy(dev, ah); 2153 return 0; 2154 } 2155 2156 struct rdma_hw_stats *efa_alloc_hw_port_stats(struct ib_device *ibdev, 2157 u32 port_num) 2158 { 2159 return rdma_alloc_hw_stats_struct(efa_port_stats_descs, 2160 ARRAY_SIZE(efa_port_stats_descs), 2161 RDMA_HW_STATS_DEFAULT_LIFESPAN); 2162 } 2163 2164 struct rdma_hw_stats *efa_alloc_hw_device_stats(struct ib_device *ibdev) 2165 { 2166 return rdma_alloc_hw_stats_struct(efa_device_stats_descs, 2167 ARRAY_SIZE(efa_device_stats_descs), 2168 RDMA_HW_STATS_DEFAULT_LIFESPAN); 2169 } 2170 2171 static int efa_fill_device_stats(struct efa_dev *dev, 2172 struct rdma_hw_stats *stats) 2173 { 2174 struct efa_com_stats_admin *as = &dev->edev.aq.stats; 2175 struct efa_stats *s = &dev->stats; 2176 2177 stats->value[EFA_SUBMITTED_CMDS] = atomic64_read(&as->submitted_cmd); 2178 stats->value[EFA_COMPLETED_CMDS] = atomic64_read(&as->completed_cmd); 2179 stats->value[EFA_CMDS_ERR] = atomic64_read(&as->cmd_err); 2180 stats->value[EFA_NO_COMPLETION_CMDS] = atomic64_read(&as->no_completion); 2181 2182 stats->value[EFA_KEEP_ALIVE_RCVD] = atomic64_read(&s->keep_alive_rcvd); 2183 stats->value[EFA_ALLOC_PD_ERR] = atomic64_read(&s->alloc_pd_err); 2184 stats->value[EFA_CREATE_QP_ERR] = atomic64_read(&s->create_qp_err); 2185 stats->value[EFA_CREATE_CQ_ERR] = atomic64_read(&s->create_cq_err); 2186 stats->value[EFA_REG_MR_ERR] = atomic64_read(&s->reg_mr_err); 2187 stats->value[EFA_ALLOC_UCONTEXT_ERR] = 2188 atomic64_read(&s->alloc_ucontext_err); 2189 stats->value[EFA_CREATE_AH_ERR] = atomic64_read(&s->create_ah_err); 2190 stats->value[EFA_MMAP_ERR] = atomic64_read(&s->mmap_err); 2191 2192 return ARRAY_SIZE(efa_device_stats_descs); 2193 } 2194 2195 static int efa_fill_port_stats(struct efa_dev *dev, struct rdma_hw_stats *stats, 2196 u32 port_num) 2197 { 2198 struct efa_com_get_stats_params params = {}; 2199 union efa_com_get_stats_result result; 2200 struct efa_com_rdma_write_stats *rws; 2201 struct efa_com_rdma_read_stats *rrs; 2202 struct efa_com_messages_stats *ms; 2203 struct efa_com_network_stats *ns; 2204 struct efa_com_basic_stats *bs; 2205 int err; 2206 2207 params.scope = EFA_ADMIN_GET_STATS_SCOPE_ALL; 2208 params.type = EFA_ADMIN_GET_STATS_TYPE_BASIC; 2209 2210 err = efa_com_get_stats(&dev->edev, ¶ms, &result); 2211 if (err) 2212 return err; 2213 2214 bs = &result.basic_stats; 2215 stats->value[EFA_TX_BYTES] = bs->tx_bytes; 2216 stats->value[EFA_TX_PKTS] = bs->tx_pkts; 2217 stats->value[EFA_RX_BYTES] = bs->rx_bytes; 2218 stats->value[EFA_RX_PKTS] = bs->rx_pkts; 2219 stats->value[EFA_RX_DROPS] = bs->rx_drops; 2220 2221 params.type = EFA_ADMIN_GET_STATS_TYPE_MESSAGES; 2222 err = efa_com_get_stats(&dev->edev, ¶ms, &result); 2223 if (err) 2224 return err; 2225 2226 ms = &result.messages_stats; 2227 stats->value[EFA_SEND_BYTES] = ms->send_bytes; 2228 stats->value[EFA_SEND_WRS] = ms->send_wrs; 2229 stats->value[EFA_RECV_BYTES] = ms->recv_bytes; 2230 stats->value[EFA_RECV_WRS] = ms->recv_wrs; 2231 2232 params.type = EFA_ADMIN_GET_STATS_TYPE_RDMA_READ; 2233 err = efa_com_get_stats(&dev->edev, ¶ms, &result); 2234 if (err) 2235 return err; 2236 2237 rrs = &result.rdma_read_stats; 2238 stats->value[EFA_RDMA_READ_WRS] = rrs->read_wrs; 2239 stats->value[EFA_RDMA_READ_BYTES] = rrs->read_bytes; 2240 stats->value[EFA_RDMA_READ_WR_ERR] = rrs->read_wr_err; 2241 stats->value[EFA_RDMA_READ_RESP_BYTES] = rrs->read_resp_bytes; 2242 2243 if (EFA_DEV_CAP(dev, RDMA_WRITE)) { 2244 params.type = EFA_ADMIN_GET_STATS_TYPE_RDMA_WRITE; 2245 err = efa_com_get_stats(&dev->edev, ¶ms, &result); 2246 if (err) 2247 return err; 2248 2249 rws = &result.rdma_write_stats; 2250 stats->value[EFA_RDMA_WRITE_WRS] = rws->write_wrs; 2251 stats->value[EFA_RDMA_WRITE_BYTES] = rws->write_bytes; 2252 stats->value[EFA_RDMA_WRITE_WR_ERR] = rws->write_wr_err; 2253 stats->value[EFA_RDMA_WRITE_RECV_BYTES] = rws->write_recv_bytes; 2254 } 2255 2256 params.type = EFA_ADMIN_GET_STATS_TYPE_NETWORK; 2257 err = efa_com_get_stats(&dev->edev, ¶ms, &result); 2258 if (err) 2259 return err; 2260 2261 ns = &result.network_stats; 2262 stats->value[EFA_RETRANS_BYTES] = ns->retrans_bytes; 2263 stats->value[EFA_RETRANS_PKTS] = ns->retrans_pkts; 2264 stats->value[EFA_RETRANS_TIMEOUT_EVENS] = ns->retrans_timeout_events; 2265 stats->value[EFA_UNRESPONSIVE_REMOTE_EVENTS] = ns->unresponsive_remote_events; 2266 stats->value[EFA_IMPAIRED_REMOTE_CONN_EVENTS] = ns->impaired_remote_conn_events; 2267 2268 return ARRAY_SIZE(efa_port_stats_descs); 2269 } 2270 2271 int efa_get_hw_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats, 2272 u32 port_num, int index) 2273 { 2274 if (port_num) 2275 return efa_fill_port_stats(to_edev(ibdev), stats, port_num); 2276 else 2277 return efa_fill_device_stats(to_edev(ibdev), stats); 2278 } 2279 2280 enum rdma_link_layer efa_port_link_layer(struct ib_device *ibdev, 2281 u32 port_num) 2282 { 2283 return IB_LINK_LAYER_UNSPECIFIED; 2284 } 2285 2286 int efa_query_comp_cntr_caps(struct ib_device *ibdev, 2287 struct ib_comp_cntr_caps *caps, 2288 struct uverbs_attr_bundle *attrs) 2289 { 2290 struct efa_dev *dev = to_edev(ibdev); 2291 u32 dev_ops = dev->dev_attr.supported_event_counter_qp_events; 2292 2293 caps->max_counters = dev->dev_attr.max_event_counters / 2; 2294 caps->max_value = dev->dev_attr.event_counter_max_val; 2295 2296 caps->supported_qp_attach_ops = 0; 2297 if (EFA_GET(&dev_ops, EFA_ADMIN_EVENT_COUNTER_ATTACH_QP_EVENTS_SEND_COMP) && 2298 EFA_GET(&dev_ops, EFA_ADMIN_EVENT_COUNTER_ATTACH_QP_EVENTS_SEND_COMP_ERR)) 2299 caps->supported_qp_attach_ops |= IB_QP_ATTACH_COMP_CNTR_OP_SEND; 2300 if (EFA_GET(&dev_ops, EFA_ADMIN_EVENT_COUNTER_ATTACH_QP_EVENTS_RECV_COMP) && 2301 EFA_GET(&dev_ops, EFA_ADMIN_EVENT_COUNTER_ATTACH_QP_EVENTS_RECV_COMP_ERR)) 2302 caps->supported_qp_attach_ops |= IB_QP_ATTACH_COMP_CNTR_OP_RECV; 2303 if (EFA_GET(&dev_ops, EFA_ADMIN_EVENT_COUNTER_ATTACH_QP_EVENTS_READ_COMP) && 2304 EFA_GET(&dev_ops, EFA_ADMIN_EVENT_COUNTER_ATTACH_QP_EVENTS_READ_COMP_ERR)) 2305 caps->supported_qp_attach_ops |= IB_QP_ATTACH_COMP_CNTR_OP_RDMA_READ; 2306 if (EFA_GET(&dev_ops, EFA_ADMIN_EVENT_COUNTER_ATTACH_QP_EVENTS_WRITE_COMP) && 2307 EFA_GET(&dev_ops, EFA_ADMIN_EVENT_COUNTER_ATTACH_QP_EVENTS_WRITE_COMP_ERR)) 2308 caps->supported_qp_attach_ops |= IB_QP_ATTACH_COMP_CNTR_OP_RDMA_WRITE; 2309 if (EFA_GET(&dev_ops, EFA_ADMIN_EVENT_COUNTER_ATTACH_QP_EVENTS_REMOTE_READ_COMP)) 2310 caps->supported_qp_attach_ops |= IB_QP_ATTACH_COMP_CNTR_OP_REMOTE_RDMA_READ; 2311 if (EFA_GET(&dev_ops, EFA_ADMIN_EVENT_COUNTER_ATTACH_QP_EVENTS_REMOTE_WRITE_COMP)) 2312 caps->supported_qp_attach_ops |= IB_QP_ATTACH_COMP_CNTR_OP_REMOTE_RDMA_WRITE; 2313 2314 return 0; 2315 } 2316 2317 static int efa_create_event_counter(struct efa_dev *dev, u16 uarn, dma_addr_t addr, u32 *handle) 2318 { 2319 struct efa_com_create_event_counter_params params = {}; 2320 struct efa_com_create_event_counter_result result; 2321 int err; 2322 2323 params.uarn = uarn; 2324 params.dma_addr = addr; 2325 2326 err = efa_com_create_event_counter(&dev->edev, ¶ms, &result); 2327 if (err) 2328 return err; 2329 2330 *handle = result.cntr_handle; 2331 return 0; 2332 } 2333 2334 static int efa_destroy_event_counter(struct efa_dev *dev, u32 handle) 2335 { 2336 struct efa_com_destroy_event_counter_params params = { 2337 .cntr_handle = handle, 2338 }; 2339 2340 return efa_com_destroy_event_counter(&dev->edev, ¶ms); 2341 } 2342 2343 int efa_create_comp_cntr(struct ib_comp_cntr *ibcc, struct uverbs_attr_bundle *attrs) 2344 { 2345 struct efa_dev *dev = to_edev(ibcc->device); 2346 struct efa_comp_cntr *cc = to_ecc(ibcc); 2347 struct efa_ucontext *ucontext; 2348 struct ib_umem *comp_umem; 2349 struct ib_umem *err_umem; 2350 dma_addr_t comp_addr; 2351 dma_addr_t err_addr; 2352 int err; 2353 2354 ucontext = rdma_udata_to_drv_context(&attrs->driver_udata, struct efa_ucontext, 2355 ibucontext); 2356 2357 comp_umem = ib_umem_get_attr(ibcc->device, attrs, EFA_IB_ATTR_CREATE_COMP_CNTR_COMP_BUFFER, 2358 sizeof(u64), IB_ACCESS_LOCAL_WRITE); 2359 if (IS_ERR(comp_umem)) 2360 return PTR_ERR(comp_umem); 2361 2362 err_umem = ib_umem_get_attr(ibcc->device, attrs, EFA_IB_ATTR_CREATE_COMP_CNTR_ERR_BUFFER, 2363 sizeof(u64), IB_ACCESS_LOCAL_WRITE); 2364 if (IS_ERR(err_umem)) { 2365 err = PTR_ERR(err_umem); 2366 goto err_comp_umem; 2367 } 2368 2369 comp_addr = ib_umem_start_dma_addr(comp_umem); 2370 err_addr = ib_umem_start_dma_addr(err_umem); 2371 2372 if (!IS_ALIGNED(comp_addr, sizeof(u64)) || !IS_ALIGNED(err_addr, sizeof(u64))) { 2373 ibdev_dbg(&dev->ibdev, "Completion Counter memory is unaligned\n"); 2374 err = -EINVAL; 2375 goto err_err_umem; 2376 } 2377 2378 err = efa_create_event_counter(dev, ucontext->uarn, comp_addr, &cc->comp_handle); 2379 if (err) { 2380 ibdev_dbg(&dev->ibdev, "Failed to create comp event counter [%d]\n", err); 2381 goto err_err_umem; 2382 } 2383 2384 err = efa_create_event_counter(dev, ucontext->uarn, err_addr, &cc->err_handle); 2385 if (err) { 2386 ibdev_dbg(&dev->ibdev, "Failed to create err event counter [%d]\n", err); 2387 goto err_destroy_comp_event_cntr; 2388 } 2389 2390 cc->comp_umem = comp_umem; 2391 cc->err_umem = err_umem; 2392 2393 return 0; 2394 2395 err_destroy_comp_event_cntr: 2396 efa_destroy_event_counter(dev, cc->comp_handle); 2397 err_err_umem: 2398 ib_umem_release(err_umem); 2399 err_comp_umem: 2400 ib_umem_release(comp_umem); 2401 return err; 2402 } 2403 2404 int efa_destroy_comp_cntr(struct ib_comp_cntr *ibcc) 2405 { 2406 struct efa_dev *dev = to_edev(ibcc->device); 2407 struct efa_comp_cntr *cc = to_ecc(ibcc); 2408 2409 efa_destroy_event_counter(dev, cc->comp_handle); 2410 efa_destroy_event_counter(dev, cc->err_handle); 2411 2412 ib_umem_release(cc->comp_umem); 2413 ib_umem_release(cc->err_umem); 2414 return 0; 2415 } 2416 2417 int efa_modify_comp_cntr(struct ib_comp_cntr *ibcc, enum ib_comp_cntr_entry entry, 2418 enum ib_comp_cntr_modify_op op, u64 value) 2419 { 2420 struct efa_com_modify_event_counter_params params = {}; 2421 struct efa_comp_cntr *cc = to_ecc(ibcc); 2422 2423 params.cntr_handle = entry == IB_COMP_CNTR_ENTRY_ERR ? cc->err_handle : cc->comp_handle; 2424 params.operation = op == IB_COMP_CNTR_MODIFY_OP_SET ? 2425 EFA_ADMIN_EVENT_COUNTER_MODIFY_SET : EFA_ADMIN_EVENT_COUNTER_MODIFY_ADD; 2426 params.value = value; 2427 2428 return efa_com_modify_event_counter(&to_edev(ibcc->device)->edev, ¶ms); 2429 } 2430 2431 static u32 efa_comp_cntr_op_to_comp_events(u32 op_mask) 2432 { 2433 u32 events = 0; 2434 2435 if (op_mask & IB_QP_ATTACH_COMP_CNTR_OP_SEND) 2436 EFA_SET(&events, EFA_ADMIN_EVENT_COUNTER_ATTACH_QP_EVENTS_SEND_COMP, 1); 2437 if (op_mask & IB_QP_ATTACH_COMP_CNTR_OP_RECV) 2438 EFA_SET(&events, EFA_ADMIN_EVENT_COUNTER_ATTACH_QP_EVENTS_RECV_COMP, 1); 2439 if (op_mask & IB_QP_ATTACH_COMP_CNTR_OP_RDMA_READ) 2440 EFA_SET(&events, EFA_ADMIN_EVENT_COUNTER_ATTACH_QP_EVENTS_READ_COMP, 1); 2441 if (op_mask & IB_QP_ATTACH_COMP_CNTR_OP_REMOTE_RDMA_READ) 2442 EFA_SET(&events, EFA_ADMIN_EVENT_COUNTER_ATTACH_QP_EVENTS_REMOTE_READ_COMP, 1); 2443 if (op_mask & IB_QP_ATTACH_COMP_CNTR_OP_RDMA_WRITE) 2444 EFA_SET(&events, EFA_ADMIN_EVENT_COUNTER_ATTACH_QP_EVENTS_WRITE_COMP, 1); 2445 if (op_mask & IB_QP_ATTACH_COMP_CNTR_OP_REMOTE_RDMA_WRITE) 2446 EFA_SET(&events, EFA_ADMIN_EVENT_COUNTER_ATTACH_QP_EVENTS_REMOTE_WRITE_COMP, 1); 2447 2448 return events; 2449 } 2450 2451 static u32 efa_comp_cntr_op_to_err_events(u32 op_mask) 2452 { 2453 u32 events = 0; 2454 2455 if (op_mask & IB_QP_ATTACH_COMP_CNTR_OP_SEND) 2456 EFA_SET(&events, EFA_ADMIN_EVENT_COUNTER_ATTACH_QP_EVENTS_SEND_COMP_ERR, 1); 2457 if (op_mask & IB_QP_ATTACH_COMP_CNTR_OP_RECV) 2458 EFA_SET(&events, EFA_ADMIN_EVENT_COUNTER_ATTACH_QP_EVENTS_RECV_COMP_ERR, 1); 2459 if (op_mask & IB_QP_ATTACH_COMP_CNTR_OP_RDMA_READ) 2460 EFA_SET(&events, EFA_ADMIN_EVENT_COUNTER_ATTACH_QP_EVENTS_READ_COMP_ERR, 1); 2461 if (op_mask & IB_QP_ATTACH_COMP_CNTR_OP_RDMA_WRITE) 2462 EFA_SET(&events, EFA_ADMIN_EVENT_COUNTER_ATTACH_QP_EVENTS_WRITE_COMP_ERR, 1); 2463 2464 return events; 2465 } 2466 2467 int efa_qp_attach_comp_cntr(struct ib_qp *ibqp, struct ib_comp_cntr *ibcc, 2468 struct ib_qp_attach_comp_cntr_attr *attr) 2469 { 2470 struct efa_com_detach_event_counter_params detach_params = {}; 2471 struct efa_com_attach_event_counter_params params = {}; 2472 struct efa_dev *dev = to_edev(ibqp->device); 2473 struct efa_comp_cntr *cc = to_ecc(ibcc); 2474 struct efa_qp *qp = to_eqp(ibqp); 2475 int err; 2476 2477 params.cntr_handle = cc->comp_handle; 2478 params.qp_handle = qp->qp_handle; 2479 params.events = efa_comp_cntr_op_to_comp_events(attr->op_mask); 2480 2481 err = efa_com_attach_event_counter(&dev->edev, ¶ms); 2482 if (err) 2483 return err; 2484 2485 params.cntr_handle = cc->err_handle; 2486 params.events = efa_comp_cntr_op_to_err_events(attr->op_mask); 2487 2488 err = efa_com_attach_event_counter(&dev->edev, ¶ms); 2489 if (err) { 2490 detach_params.cntr_handle = cc->comp_handle; 2491 detach_params.qp_handle = qp->qp_handle; 2492 detach_params.events = efa_comp_cntr_op_to_comp_events(attr->op_mask); 2493 efa_com_detach_event_counter(&dev->edev, &detach_params); 2494 return err; 2495 } 2496 2497 return 0; 2498 } 2499 2500 DECLARE_UVERBS_NAMED_METHOD(EFA_IB_METHOD_MR_QUERY, 2501 UVERBS_ATTR_IDR(EFA_IB_ATTR_QUERY_MR_HANDLE, 2502 UVERBS_OBJECT_MR, 2503 UVERBS_ACCESS_READ, 2504 UA_MANDATORY), 2505 UVERBS_ATTR_PTR_OUT(EFA_IB_ATTR_QUERY_MR_RESP_IC_ID_VALIDITY, 2506 UVERBS_ATTR_TYPE(u16), 2507 UA_MANDATORY), 2508 UVERBS_ATTR_PTR_OUT(EFA_IB_ATTR_QUERY_MR_RESP_RECV_IC_ID, 2509 UVERBS_ATTR_TYPE(u16), 2510 UA_MANDATORY), 2511 UVERBS_ATTR_PTR_OUT(EFA_IB_ATTR_QUERY_MR_RESP_RDMA_READ_IC_ID, 2512 UVERBS_ATTR_TYPE(u16), 2513 UA_MANDATORY), 2514 UVERBS_ATTR_PTR_OUT(EFA_IB_ATTR_QUERY_MR_RESP_RDMA_RECV_IC_ID, 2515 UVERBS_ATTR_TYPE(u16), 2516 UA_MANDATORY)); 2517 2518 ADD_UVERBS_METHODS(efa_mr, 2519 UVERBS_OBJECT_MR, 2520 &UVERBS_METHOD(EFA_IB_METHOD_MR_QUERY)); 2521 2522 ADD_UVERBS_ATTRIBUTES_SIMPLE( 2523 efa_comp_cntr_create, 2524 UVERBS_OBJECT_COMP_CNTR, 2525 UVERBS_METHOD_COMP_CNTR_CREATE, 2526 UVERBS_ATTR_PTR_IN( 2527 EFA_IB_ATTR_CREATE_COMP_CNTR_COMP_BUFFER, 2528 UVERBS_ATTR_STRUCT(struct ib_uverbs_buffer_desc, length), 2529 UA_MANDATORY), 2530 UVERBS_ATTR_PTR_IN( 2531 EFA_IB_ATTR_CREATE_COMP_CNTR_ERR_BUFFER, 2532 UVERBS_ATTR_STRUCT(struct ib_uverbs_buffer_desc, length), 2533 UA_MANDATORY)); 2534 2535 const struct uapi_definition efa_uapi_defs[] = { 2536 UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_MR, 2537 &efa_mr), 2538 UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_COMP_CNTR, 2539 &efa_comp_cntr_create), 2540 {}, 2541 }; 2542