1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* 3 * Copyright 2018-2026 Amazon.com, Inc. or its affiliates. All rights reserved. 4 */ 5 6 #include <linux/dma-buf.h> 7 #include <linux/dma-resv.h> 8 #include <linux/vmalloc.h> 9 #include <linux/log2.h> 10 11 #include <rdma/ib_addr.h> 12 #include <rdma/ib_user_verbs.h> 13 #include <rdma/ib_verbs.h> 14 #include <rdma/iter.h> 15 #include <rdma/uverbs_ioctl.h> 16 #define UVERBS_MODULE_NAME efa_ib 17 #include <rdma/uverbs_named_ioctl.h> 18 #include <rdma/ib_user_ioctl_cmds.h> 19 20 #include "efa.h" 21 #include "efa_io_defs.h" 22 23 enum { 24 EFA_MMAP_DMA_PAGE = 0, 25 EFA_MMAP_IO_WC, 26 EFA_MMAP_IO_NC, 27 }; 28 29 struct efa_user_mmap_entry { 30 struct rdma_user_mmap_entry rdma_entry; 31 u64 address; 32 u8 mmap_flag; 33 }; 34 35 #define EFA_DEFINE_DEVICE_STATS(op) \ 36 op(EFA_SUBMITTED_CMDS, "submitted_cmds") \ 37 op(EFA_COMPLETED_CMDS, "completed_cmds") \ 38 op(EFA_CMDS_ERR, "cmds_err") \ 39 op(EFA_NO_COMPLETION_CMDS, "no_completion_cmds") \ 40 op(EFA_KEEP_ALIVE_RCVD, "keep_alive_rcvd") \ 41 op(EFA_ALLOC_PD_ERR, "alloc_pd_err") \ 42 op(EFA_CREATE_QP_ERR, "create_qp_err") \ 43 op(EFA_CREATE_CQ_ERR, "create_cq_err") \ 44 op(EFA_REG_MR_ERR, "reg_mr_err") \ 45 op(EFA_ALLOC_UCONTEXT_ERR, "alloc_ucontext_err") \ 46 op(EFA_CREATE_AH_ERR, "create_ah_err") \ 47 op(EFA_MMAP_ERR, "mmap_err") 48 49 #define EFA_DEFINE_PORT_STATS(op) \ 50 op(EFA_TX_BYTES, "tx_bytes") \ 51 op(EFA_TX_PKTS, "tx_pkts") \ 52 op(EFA_RX_BYTES, "rx_bytes") \ 53 op(EFA_RX_PKTS, "rx_pkts") \ 54 op(EFA_RX_DROPS, "rx_drops") \ 55 op(EFA_SEND_BYTES, "send_bytes") \ 56 op(EFA_SEND_WRS, "send_wrs") \ 57 op(EFA_RECV_BYTES, "recv_bytes") \ 58 op(EFA_RECV_WRS, "recv_wrs") \ 59 op(EFA_RDMA_READ_WRS, "rdma_read_wrs") \ 60 op(EFA_RDMA_READ_BYTES, "rdma_read_bytes") \ 61 op(EFA_RDMA_READ_WR_ERR, "rdma_read_wr_err") \ 62 op(EFA_RDMA_READ_RESP_BYTES, "rdma_read_resp_bytes") \ 63 op(EFA_RDMA_WRITE_WRS, "rdma_write_wrs") \ 64 op(EFA_RDMA_WRITE_BYTES, "rdma_write_bytes") \ 65 op(EFA_RDMA_WRITE_WR_ERR, "rdma_write_wr_err") \ 66 op(EFA_RDMA_WRITE_RECV_BYTES, "rdma_write_recv_bytes") \ 67 op(EFA_RETRANS_BYTES, "retrans_bytes") \ 68 op(EFA_RETRANS_PKTS, "retrans_pkts") \ 69 op(EFA_RETRANS_TIMEOUT_EVENS, "retrans_timeout_events") \ 70 op(EFA_UNRESPONSIVE_REMOTE_EVENTS, "unresponsive_remote_events") \ 71 op(EFA_IMPAIRED_REMOTE_CONN_EVENTS, "impaired_remote_conn_events") \ 72 73 #define EFA_STATS_ENUM(ename, name) ename, 74 #define EFA_STATS_STR(ename, nam) \ 75 [ename].name = nam, 76 77 enum efa_hw_device_stats { 78 EFA_DEFINE_DEVICE_STATS(EFA_STATS_ENUM) 79 }; 80 81 static const struct rdma_stat_desc efa_device_stats_descs[] = { 82 EFA_DEFINE_DEVICE_STATS(EFA_STATS_STR) 83 }; 84 85 enum efa_hw_port_stats { 86 EFA_DEFINE_PORT_STATS(EFA_STATS_ENUM) 87 }; 88 89 static const struct rdma_stat_desc efa_port_stats_descs[] = { 90 EFA_DEFINE_PORT_STATS(EFA_STATS_STR) 91 }; 92 93 #define EFA_CHUNK_PAYLOAD_SHIFT 12 94 #define EFA_CHUNK_PAYLOAD_SIZE BIT(EFA_CHUNK_PAYLOAD_SHIFT) 95 #define EFA_CHUNK_PAYLOAD_PTR_SIZE 8 96 97 #define EFA_CHUNK_SHIFT 12 98 #define EFA_CHUNK_SIZE BIT(EFA_CHUNK_SHIFT) 99 #define EFA_CHUNK_PTR_SIZE sizeof(struct efa_com_ctrl_buff_info) 100 101 #define EFA_PTRS_PER_CHUNK \ 102 ((EFA_CHUNK_SIZE - EFA_CHUNK_PTR_SIZE) / EFA_CHUNK_PAYLOAD_PTR_SIZE) 103 104 #define EFA_CHUNK_USED_SIZE \ 105 ((EFA_PTRS_PER_CHUNK * EFA_CHUNK_PAYLOAD_PTR_SIZE) + EFA_CHUNK_PTR_SIZE) 106 107 struct pbl_chunk { 108 dma_addr_t dma_addr; 109 u64 *buf; 110 u32 length; 111 }; 112 113 struct pbl_chunk_list { 114 struct pbl_chunk *chunks; 115 unsigned int size; 116 }; 117 118 struct pbl_context { 119 union { 120 struct { 121 dma_addr_t dma_addr; 122 } continuous; 123 struct { 124 u32 pbl_buf_size_in_pages; 125 struct scatterlist *sgl; 126 int sg_dma_cnt; 127 struct pbl_chunk_list chunk_list; 128 } indirect; 129 } phys; 130 u64 *pbl_buf; 131 u32 pbl_buf_size_in_bytes; 132 u8 physically_continuous; 133 }; 134 135 static inline struct efa_dev *to_edev(struct ib_device *ibdev) 136 { 137 return container_of(ibdev, struct efa_dev, ibdev); 138 } 139 140 static inline struct efa_ucontext *to_eucontext(struct ib_ucontext *ibucontext) 141 { 142 return container_of(ibucontext, struct efa_ucontext, ibucontext); 143 } 144 145 static inline struct efa_pd *to_epd(struct ib_pd *ibpd) 146 { 147 return container_of(ibpd, struct efa_pd, ibpd); 148 } 149 150 static inline struct efa_mr *to_emr(struct ib_mr *ibmr) 151 { 152 return container_of(ibmr, struct efa_mr, ibmr); 153 } 154 155 static inline struct efa_qp *to_eqp(struct ib_qp *ibqp) 156 { 157 return container_of(ibqp, struct efa_qp, ibqp); 158 } 159 160 static inline struct efa_cq *to_ecq(struct ib_cq *ibcq) 161 { 162 return container_of(ibcq, struct efa_cq, ibcq); 163 } 164 165 static inline struct efa_ah *to_eah(struct ib_ah *ibah) 166 { 167 return container_of(ibah, struct efa_ah, ibah); 168 } 169 170 static inline struct efa_user_mmap_entry * 171 to_emmap(struct rdma_user_mmap_entry *rdma_entry) 172 { 173 return container_of(rdma_entry, struct efa_user_mmap_entry, rdma_entry); 174 } 175 176 #define EFA_DEV_CAP(dev, cap) \ 177 ((dev)->dev_attr.device_caps & \ 178 EFA_ADMIN_FEATURE_DEVICE_ATTR_DESC_##cap##_MASK) 179 180 #define is_reserved_cleared(reserved) \ 181 !memchr_inv(reserved, 0, sizeof(reserved)) 182 183 static void *efa_zalloc_mapped(struct efa_dev *dev, dma_addr_t *dma_addr, 184 size_t size, enum dma_data_direction dir) 185 { 186 void *addr; 187 188 addr = alloc_pages_exact(size, GFP_KERNEL | __GFP_ZERO); 189 if (!addr) 190 return NULL; 191 192 *dma_addr = dma_map_single(&dev->pdev->dev, addr, size, dir); 193 if (dma_mapping_error(&dev->pdev->dev, *dma_addr)) { 194 ibdev_err(&dev->ibdev, "Failed to map DMA address\n"); 195 free_pages_exact(addr, size); 196 return NULL; 197 } 198 199 return addr; 200 } 201 202 static void efa_free_mapped(struct efa_dev *dev, void *cpu_addr, 203 dma_addr_t dma_addr, 204 size_t size, enum dma_data_direction dir) 205 { 206 dma_unmap_single(&dev->pdev->dev, dma_addr, size, dir); 207 free_pages_exact(cpu_addr, size); 208 } 209 210 int efa_query_device(struct ib_device *ibdev, 211 struct ib_device_attr *props, 212 struct ib_udata *udata) 213 { 214 struct efa_com_get_device_attr_result *dev_attr; 215 struct efa_ibv_ex_query_device_resp resp = {}; 216 struct efa_dev *dev = to_edev(ibdev); 217 int err; 218 219 err = ib_is_udata_in_empty(udata); 220 if (err) 221 return err; 222 223 dev_attr = &dev->dev_attr; 224 225 memset(props, 0, sizeof(*props)); 226 props->max_mr_size = dev_attr->max_mr_pages * PAGE_SIZE; 227 props->page_size_cap = dev_attr->page_size_cap; 228 props->vendor_id = dev->pdev->vendor; 229 props->vendor_part_id = dev->pdev->device; 230 props->hw_ver = dev->pdev->subsystem_device; 231 props->max_qp = dev_attr->max_qp; 232 props->max_cq = dev_attr->max_cq; 233 props->max_pd = dev_attr->max_pd; 234 props->max_mr = dev_attr->max_mr; 235 props->max_ah = dev_attr->max_ah; 236 props->max_cqe = dev_attr->max_cq_depth; 237 props->max_qp_wr = min_t(u32, dev_attr->max_sq_depth, 238 dev_attr->max_rq_depth); 239 props->max_send_sge = dev_attr->max_sq_sge; 240 props->max_recv_sge = dev_attr->max_rq_sge; 241 props->max_sge_rd = dev_attr->max_wr_rdma_sge; 242 props->max_pkeys = 1; 243 244 if (udata && udata->outlen) { 245 resp.max_sq_sge = dev_attr->max_sq_sge; 246 resp.max_rq_sge = dev_attr->max_rq_sge; 247 resp.max_sq_wr = dev_attr->max_sq_depth; 248 resp.max_rq_wr = dev_attr->max_rq_depth; 249 resp.max_rdma_size = dev_attr->max_rdma_size; 250 251 resp.device_caps |= EFA_QUERY_DEVICE_CAPS_CQ_WITH_SGID; 252 resp.device_caps |= EFA_QUERY_DEVICE_CAPS_CQ_WITH_EXT_MEM; 253 if (EFA_DEV_CAP(dev, RDMA_READ)) 254 resp.device_caps |= EFA_QUERY_DEVICE_CAPS_RDMA_READ; 255 256 if (EFA_DEV_CAP(dev, RNR_RETRY)) 257 resp.device_caps |= EFA_QUERY_DEVICE_CAPS_RNR_RETRY; 258 259 if (EFA_DEV_CAP(dev, DATA_POLLING_128)) 260 resp.device_caps |= EFA_QUERY_DEVICE_CAPS_DATA_POLLING_128; 261 262 if (EFA_DEV_CAP(dev, RDMA_WRITE)) 263 resp.device_caps |= EFA_QUERY_DEVICE_CAPS_RDMA_WRITE; 264 265 if (EFA_DEV_CAP(dev, UNSOLICITED_WRITE_RECV)) 266 resp.device_caps |= EFA_QUERY_DEVICE_CAPS_UNSOLICITED_WRITE_RECV; 267 268 if (dev->neqs) 269 resp.device_caps |= EFA_QUERY_DEVICE_CAPS_CQ_NOTIFICATIONS; 270 271 err = ib_respond_udata(udata, resp); 272 if (err) 273 return err; 274 } 275 276 return 0; 277 } 278 279 static void efa_link_gbps_to_speed_and_width(u16 gbps, 280 enum ib_port_speed *speed, 281 enum ib_port_width *width) 282 { 283 if (gbps >= 1600) { 284 *width = IB_WIDTH_8X; 285 *speed = IB_SPEED_XDR; 286 } else if (gbps >= 800) { 287 *width = IB_WIDTH_8X; 288 *speed = IB_SPEED_NDR; 289 } else if (gbps >= 400) { 290 *width = IB_WIDTH_8X; 291 *speed = IB_SPEED_HDR; 292 } else if (gbps >= 200) { 293 *width = IB_WIDTH_4X; 294 *speed = IB_SPEED_HDR; 295 } else if (gbps >= 120) { 296 *width = IB_WIDTH_12X; 297 *speed = IB_SPEED_FDR10; 298 } else if (gbps >= 100) { 299 *width = IB_WIDTH_4X; 300 *speed = IB_SPEED_EDR; 301 } else if (gbps >= 60) { 302 *width = IB_WIDTH_12X; 303 *speed = IB_SPEED_DDR; 304 } else if (gbps >= 50) { 305 *width = IB_WIDTH_1X; 306 *speed = IB_SPEED_HDR; 307 } else if (gbps >= 40) { 308 *width = IB_WIDTH_4X; 309 *speed = IB_SPEED_FDR10; 310 } else if (gbps >= 30) { 311 *width = IB_WIDTH_12X; 312 *speed = IB_SPEED_SDR; 313 } else { 314 *width = IB_WIDTH_1X; 315 *speed = IB_SPEED_EDR; 316 } 317 } 318 319 int efa_query_port(struct ib_device *ibdev, u32 port, 320 struct ib_port_attr *props) 321 { 322 struct efa_dev *dev = to_edev(ibdev); 323 enum ib_port_speed link_speed; 324 enum ib_port_width link_width; 325 u16 link_gbps; 326 327 props->lmc = 1; 328 329 props->state = IB_PORT_ACTIVE; 330 props->phys_state = IB_PORT_PHYS_STATE_LINK_UP; 331 props->gid_tbl_len = 1; 332 props->pkey_tbl_len = 1; 333 link_gbps = dev->dev_attr.max_link_speed_gbps; 334 efa_link_gbps_to_speed_and_width(link_gbps, &link_speed, &link_width); 335 props->active_speed = link_speed; 336 props->active_width = link_width; 337 props->max_mtu = ib_mtu_int_to_enum(dev->dev_attr.mtu); 338 props->active_mtu = ib_mtu_int_to_enum(dev->dev_attr.mtu); 339 props->max_msg_sz = dev->dev_attr.mtu; 340 props->max_vl_num = 1; 341 342 return 0; 343 } 344 345 int efa_query_port_speed(struct ib_device *ibdev, u32 port_num, u64 *speed) 346 { 347 struct efa_dev *dev = to_edev(ibdev); 348 349 *speed = dev->dev_attr.max_link_speed_gbps * 10; 350 351 return 0; 352 } 353 354 int efa_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, 355 int qp_attr_mask, 356 struct ib_qp_init_attr *qp_init_attr) 357 { 358 struct efa_dev *dev = to_edev(ibqp->device); 359 struct efa_com_query_qp_params params = {}; 360 struct efa_com_query_qp_result result; 361 struct efa_qp *qp = to_eqp(ibqp); 362 int err; 363 364 #define EFA_QUERY_QP_SUPP_MASK \ 365 (IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_PORT | \ 366 IB_QP_QKEY | IB_QP_SQ_PSN | IB_QP_CAP | IB_QP_RNR_RETRY) 367 368 if (qp_attr_mask & ~EFA_QUERY_QP_SUPP_MASK) { 369 ibdev_dbg(&dev->ibdev, 370 "Unsupported qp_attr_mask[%#x] supported[%#x]\n", 371 qp_attr_mask, EFA_QUERY_QP_SUPP_MASK); 372 return -EOPNOTSUPP; 373 } 374 375 memset(qp_attr, 0, sizeof(*qp_attr)); 376 memset(qp_init_attr, 0, sizeof(*qp_init_attr)); 377 378 params.qp_handle = qp->qp_handle; 379 err = efa_com_query_qp(&dev->edev, ¶ms, &result); 380 if (err) 381 return err; 382 383 qp_attr->qp_state = result.qp_state; 384 qp_attr->qkey = result.qkey; 385 qp_attr->sq_psn = result.sq_psn; 386 qp_attr->sq_draining = result.sq_draining; 387 qp_attr->port_num = 1; 388 qp_attr->rnr_retry = result.rnr_retry; 389 390 qp_attr->cap.max_send_wr = qp->max_send_wr; 391 qp_attr->cap.max_recv_wr = qp->max_recv_wr; 392 qp_attr->cap.max_send_sge = qp->max_send_sge; 393 qp_attr->cap.max_recv_sge = qp->max_recv_sge; 394 qp_attr->cap.max_inline_data = qp->max_inline_data; 395 396 qp_init_attr->qp_type = ibqp->qp_type; 397 qp_init_attr->recv_cq = ibqp->recv_cq; 398 qp_init_attr->send_cq = ibqp->send_cq; 399 qp_init_attr->qp_context = ibqp->qp_context; 400 qp_init_attr->cap = qp_attr->cap; 401 402 return 0; 403 } 404 405 int efa_query_gid(struct ib_device *ibdev, u32 port, int index, 406 union ib_gid *gid) 407 { 408 struct efa_dev *dev = to_edev(ibdev); 409 410 memcpy(gid->raw, dev->dev_attr.addr, sizeof(dev->dev_attr.addr)); 411 412 return 0; 413 } 414 415 int efa_query_pkey(struct ib_device *ibdev, u32 port, u16 index, 416 u16 *pkey) 417 { 418 if (index > 0) 419 return -EINVAL; 420 421 *pkey = 0xffff; 422 return 0; 423 } 424 425 static int efa_pd_dealloc(struct efa_dev *dev, u16 pdn) 426 { 427 struct efa_com_dealloc_pd_params params = { 428 .pdn = pdn, 429 }; 430 431 return efa_com_dealloc_pd(&dev->edev, ¶ms); 432 } 433 434 int efa_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 435 { 436 struct efa_dev *dev = to_edev(ibpd->device); 437 struct efa_ibv_alloc_pd_resp resp = {}; 438 struct efa_com_alloc_pd_result result; 439 struct efa_pd *pd = to_epd(ibpd); 440 int err; 441 442 err = ib_is_udata_in_empty(udata); 443 if (err) 444 goto err_out; 445 446 err = efa_com_alloc_pd(&dev->edev, &result); 447 if (err) 448 goto err_out; 449 450 pd->pdn = result.pdn; 451 resp.pdn = result.pdn; 452 453 if (udata->outlen) { 454 err = ib_respond_udata(udata, resp); 455 if (err) 456 goto err_dealloc_pd; 457 } 458 459 ibdev_dbg(&dev->ibdev, "Allocated pd[%d]\n", pd->pdn); 460 461 return 0; 462 463 err_dealloc_pd: 464 efa_pd_dealloc(dev, result.pdn); 465 err_out: 466 atomic64_inc(&dev->stats.alloc_pd_err); 467 return err; 468 } 469 470 int efa_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 471 { 472 struct efa_dev *dev = to_edev(ibpd->device); 473 struct efa_pd *pd = to_epd(ibpd); 474 475 ibdev_dbg(&dev->ibdev, "Dealloc pd[%d]\n", pd->pdn); 476 efa_pd_dealloc(dev, pd->pdn); 477 return 0; 478 } 479 480 static int efa_destroy_qp_handle(struct efa_dev *dev, u32 qp_handle) 481 { 482 struct efa_com_destroy_qp_params params = { .qp_handle = qp_handle }; 483 484 return efa_com_destroy_qp(&dev->edev, ¶ms); 485 } 486 487 static void efa_qp_user_mmap_entries_remove(struct efa_qp *qp) 488 { 489 rdma_user_mmap_entry_remove(qp->rq_mmap_entry); 490 rdma_user_mmap_entry_remove(qp->rq_db_mmap_entry); 491 rdma_user_mmap_entry_remove(qp->llq_desc_mmap_entry); 492 rdma_user_mmap_entry_remove(qp->sq_db_mmap_entry); 493 } 494 495 int efa_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) 496 { 497 struct efa_dev *dev = to_edev(ibqp->pd->device); 498 struct efa_qp *qp = to_eqp(ibqp); 499 int err; 500 501 ibdev_dbg(&dev->ibdev, "Destroy qp[%u]\n", ibqp->qp_num); 502 503 err = efa_destroy_qp_handle(dev, qp->qp_handle); 504 if (err) 505 return err; 506 507 efa_qp_user_mmap_entries_remove(qp); 508 509 if (qp->rq_cpu_addr) { 510 ibdev_dbg(&dev->ibdev, 511 "qp->cpu_addr[0x%p] freed: size[%lu], dma[%pad]\n", 512 qp->rq_cpu_addr, qp->rq_size, 513 &qp->rq_dma_addr); 514 efa_free_mapped(dev, qp->rq_cpu_addr, qp->rq_dma_addr, 515 qp->rq_size, DMA_TO_DEVICE); 516 } 517 518 return 0; 519 } 520 521 static struct rdma_user_mmap_entry* 522 efa_user_mmap_entry_insert(struct ib_ucontext *ucontext, 523 u64 address, size_t length, 524 u8 mmap_flag, u64 *offset) 525 { 526 struct efa_user_mmap_entry *entry = kzalloc_obj(*entry); 527 int err; 528 529 if (!entry) 530 return NULL; 531 532 entry->address = address; 533 entry->mmap_flag = mmap_flag; 534 535 err = rdma_user_mmap_entry_insert(ucontext, &entry->rdma_entry, 536 length); 537 if (err) { 538 kfree(entry); 539 return NULL; 540 } 541 *offset = rdma_user_mmap_get_offset(&entry->rdma_entry); 542 543 return &entry->rdma_entry; 544 } 545 546 static int qp_mmap_entries_setup(struct efa_qp *qp, 547 struct efa_dev *dev, 548 struct efa_ucontext *ucontext, 549 struct efa_com_create_qp_params *params, 550 struct efa_ibv_create_qp_resp *resp) 551 { 552 size_t length; 553 u64 address; 554 555 address = dev->db_bar_addr + resp->sq_db_offset; 556 qp->sq_db_mmap_entry = 557 efa_user_mmap_entry_insert(&ucontext->ibucontext, 558 address, 559 PAGE_SIZE, EFA_MMAP_IO_NC, 560 &resp->sq_db_mmap_key); 561 if (!qp->sq_db_mmap_entry) 562 return -ENOMEM; 563 564 resp->sq_db_offset &= ~PAGE_MASK; 565 566 address = dev->mem_bar_addr + resp->llq_desc_offset; 567 length = PAGE_ALIGN(params->sq_ring_size_in_bytes + 568 offset_in_page(resp->llq_desc_offset)); 569 570 qp->llq_desc_mmap_entry = 571 efa_user_mmap_entry_insert(&ucontext->ibucontext, 572 address, length, 573 EFA_MMAP_IO_WC, 574 &resp->llq_desc_mmap_key); 575 if (!qp->llq_desc_mmap_entry) 576 goto err_remove_mmap; 577 578 resp->llq_desc_offset &= ~PAGE_MASK; 579 580 if (qp->rq_cpu_addr) { 581 address = dev->db_bar_addr + resp->rq_db_offset; 582 583 qp->rq_db_mmap_entry = 584 efa_user_mmap_entry_insert(&ucontext->ibucontext, 585 address, PAGE_SIZE, 586 EFA_MMAP_IO_NC, 587 &resp->rq_db_mmap_key); 588 if (!qp->rq_db_mmap_entry) 589 goto err_remove_mmap; 590 591 resp->rq_db_offset &= ~PAGE_MASK; 592 593 address = virt_to_phys(qp->rq_cpu_addr); 594 qp->rq_mmap_entry = 595 efa_user_mmap_entry_insert(&ucontext->ibucontext, 596 address, qp->rq_size, 597 EFA_MMAP_DMA_PAGE, 598 &resp->rq_mmap_key); 599 if (!qp->rq_mmap_entry) 600 goto err_remove_mmap; 601 602 resp->rq_mmap_size = qp->rq_size; 603 } 604 605 return 0; 606 607 err_remove_mmap: 608 efa_qp_user_mmap_entries_remove(qp); 609 610 return -ENOMEM; 611 } 612 613 static int efa_qp_validate_cap(struct efa_dev *dev, 614 struct ib_qp_init_attr *init_attr, 615 u32 sq_ring_size) 616 { 617 if (init_attr->cap.max_send_wr > dev->dev_attr.max_sq_depth) { 618 ibdev_dbg(&dev->ibdev, 619 "qp: requested send wr[%u] exceeds the max[%u]\n", 620 init_attr->cap.max_send_wr, 621 dev->dev_attr.max_sq_depth); 622 return -EINVAL; 623 } 624 625 if (sq_ring_size > dev->dev_attr.max_llq_size) { 626 ibdev_dbg(&dev->ibdev, 627 "qp: requested sq ring size[%u] exceeds the max[%u]\n", 628 sq_ring_size, dev->dev_attr.max_llq_size); 629 return -EINVAL; 630 } 631 632 if (init_attr->cap.max_recv_wr > dev->dev_attr.max_rq_depth) { 633 ibdev_dbg(&dev->ibdev, 634 "qp: requested receive wr[%u] exceeds the max[%u]\n", 635 init_attr->cap.max_recv_wr, 636 dev->dev_attr.max_rq_depth); 637 return -EINVAL; 638 } 639 if (init_attr->cap.max_send_sge > dev->dev_attr.max_sq_sge) { 640 ibdev_dbg(&dev->ibdev, 641 "qp: requested sge send[%u] exceeds the max[%u]\n", 642 init_attr->cap.max_send_sge, dev->dev_attr.max_sq_sge); 643 return -EINVAL; 644 } 645 if (init_attr->cap.max_recv_sge > dev->dev_attr.max_rq_sge) { 646 ibdev_dbg(&dev->ibdev, 647 "qp: requested sge recv[%u] exceeds the max[%u]\n", 648 init_attr->cap.max_recv_sge, dev->dev_attr.max_rq_sge); 649 return -EINVAL; 650 } 651 if (init_attr->cap.max_inline_data > dev->dev_attr.inline_buf_size_ex) { 652 ibdev_dbg(&dev->ibdev, 653 "qp: requested inline data[%u] exceeds the max[%u]\n", 654 init_attr->cap.max_inline_data, 655 dev->dev_attr.inline_buf_size_ex); 656 return -EINVAL; 657 } 658 659 return 0; 660 } 661 662 static int efa_qp_validate_attr(struct efa_dev *dev, 663 struct ib_qp_init_attr *init_attr) 664 { 665 if (init_attr->qp_type != IB_QPT_DRIVER && 666 init_attr->qp_type != IB_QPT_UD) { 667 ibdev_dbg(&dev->ibdev, 668 "Unsupported qp type %d\n", init_attr->qp_type); 669 return -EOPNOTSUPP; 670 } 671 672 if (init_attr->srq) { 673 ibdev_dbg(&dev->ibdev, "SRQ is not supported\n"); 674 return -EOPNOTSUPP; 675 } 676 677 if (init_attr->create_flags) { 678 ibdev_dbg(&dev->ibdev, "Unsupported create flags\n"); 679 return -EOPNOTSUPP; 680 } 681 682 return 0; 683 } 684 685 int efa_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *init_attr, 686 struct ib_udata *udata) 687 { 688 struct efa_com_create_qp_params create_qp_params = {}; 689 struct efa_com_create_qp_result create_qp_resp; 690 struct efa_dev *dev = to_edev(ibqp->device); 691 struct efa_ibv_create_qp_resp resp = {}; 692 struct efa_ibv_create_qp cmd; 693 struct efa_qp *qp = to_eqp(ibqp); 694 struct efa_ucontext *ucontext; 695 u16 supported_efa_flags = 0; 696 int err; 697 698 ucontext = rdma_udata_to_drv_context(udata, struct efa_ucontext, 699 ibucontext); 700 701 err = ib_copy_validate_udata_in_cm(udata, cmd, driver_qp_type, 0); 702 if (err) 703 goto err_out; 704 705 if (!is_reserved_cleared(cmd.reserved_98)) { 706 ibdev_dbg(&dev->ibdev, 707 "Incompatible ABI params, unknown fields in udata\n"); 708 err = -EINVAL; 709 goto err_out; 710 } 711 712 if (EFA_DEV_CAP(dev, UNSOLICITED_WRITE_RECV)) 713 supported_efa_flags |= EFA_CREATE_QP_WITH_UNSOLICITED_WRITE_RECV; 714 715 if (cmd.flags & ~supported_efa_flags) { 716 ibdev_dbg(&dev->ibdev, "Unsupported EFA QP create flags[%#x], supported[%#x]\n", 717 cmd.flags, supported_efa_flags); 718 err = -EOPNOTSUPP; 719 goto err_out; 720 } 721 722 err = efa_qp_validate_cap(dev, init_attr, cmd.sq_ring_size); 723 if (err) 724 goto err_out; 725 726 err = efa_qp_validate_attr(dev, init_attr); 727 if (err) 728 goto err_out; 729 730 create_qp_params.uarn = ucontext->uarn; 731 create_qp_params.pd = to_epd(ibqp->pd)->pdn; 732 733 if (init_attr->qp_type == IB_QPT_UD) { 734 create_qp_params.qp_type = EFA_ADMIN_QP_TYPE_UD; 735 } else if (cmd.driver_qp_type == EFA_QP_DRIVER_TYPE_SRD) { 736 create_qp_params.qp_type = EFA_ADMIN_QP_TYPE_SRD; 737 } else { 738 ibdev_dbg(&dev->ibdev, 739 "Unsupported qp type %d driver qp type %d\n", 740 init_attr->qp_type, cmd.driver_qp_type); 741 err = -EOPNOTSUPP; 742 goto err_out; 743 } 744 745 ibdev_dbg(&dev->ibdev, "Create QP: qp type %d driver qp type %#x\n", 746 init_attr->qp_type, cmd.driver_qp_type); 747 create_qp_params.send_cq_idx = to_ecq(init_attr->send_cq)->cq_idx; 748 create_qp_params.recv_cq_idx = to_ecq(init_attr->recv_cq)->cq_idx; 749 create_qp_params.sq_depth = init_attr->cap.max_send_wr; 750 create_qp_params.sq_ring_size_in_bytes = cmd.sq_ring_size; 751 752 create_qp_params.rq_depth = init_attr->cap.max_recv_wr; 753 create_qp_params.rq_ring_size_in_bytes = cmd.rq_ring_size; 754 qp->rq_size = PAGE_ALIGN(create_qp_params.rq_ring_size_in_bytes); 755 if (qp->rq_size) { 756 qp->rq_cpu_addr = efa_zalloc_mapped(dev, &qp->rq_dma_addr, 757 qp->rq_size, DMA_TO_DEVICE); 758 if (!qp->rq_cpu_addr) { 759 err = -ENOMEM; 760 goto err_out; 761 } 762 763 ibdev_dbg(&dev->ibdev, 764 "qp->cpu_addr[0x%p] allocated: size[%lu], dma[%pad]\n", 765 qp->rq_cpu_addr, qp->rq_size, &qp->rq_dma_addr); 766 create_qp_params.rq_base_addr = qp->rq_dma_addr; 767 } 768 769 create_qp_params.sl = cmd.sl; 770 771 if (cmd.flags & EFA_CREATE_QP_WITH_UNSOLICITED_WRITE_RECV) 772 create_qp_params.unsolicited_write_recv = true; 773 774 err = efa_com_create_qp(&dev->edev, &create_qp_params, 775 &create_qp_resp); 776 if (err) 777 goto err_free_mapped; 778 779 resp.sq_db_offset = create_qp_resp.sq_db_offset; 780 resp.rq_db_offset = create_qp_resp.rq_db_offset; 781 resp.llq_desc_offset = create_qp_resp.llq_descriptors_offset; 782 resp.send_sub_cq_idx = create_qp_resp.send_sub_cq_idx; 783 resp.recv_sub_cq_idx = create_qp_resp.recv_sub_cq_idx; 784 785 err = qp_mmap_entries_setup(qp, dev, ucontext, &create_qp_params, 786 &resp); 787 if (err) 788 goto err_destroy_qp; 789 790 qp->qp_handle = create_qp_resp.qp_handle; 791 qp->ibqp.qp_num = create_qp_resp.qp_num; 792 qp->max_send_wr = init_attr->cap.max_send_wr; 793 qp->max_recv_wr = init_attr->cap.max_recv_wr; 794 qp->max_send_sge = init_attr->cap.max_send_sge; 795 qp->max_recv_sge = init_attr->cap.max_recv_sge; 796 qp->max_inline_data = init_attr->cap.max_inline_data; 797 798 if (udata->outlen) { 799 err = ib_respond_udata(udata, resp); 800 if (err) 801 goto err_remove_mmap_entries; 802 } 803 804 ibdev_dbg(&dev->ibdev, "Created qp[%d]\n", qp->ibqp.qp_num); 805 806 return 0; 807 808 err_remove_mmap_entries: 809 efa_qp_user_mmap_entries_remove(qp); 810 err_destroy_qp: 811 efa_destroy_qp_handle(dev, create_qp_resp.qp_handle); 812 err_free_mapped: 813 if (qp->rq_cpu_addr) 814 efa_free_mapped(dev, qp->rq_cpu_addr, qp->rq_dma_addr, 815 qp->rq_size, DMA_TO_DEVICE); 816 err_out: 817 atomic64_inc(&dev->stats.create_qp_err); 818 return err; 819 } 820 821 static const struct { 822 int valid; 823 enum ib_qp_attr_mask req_param; 824 enum ib_qp_attr_mask opt_param; 825 } srd_qp_state_table[IB_QPS_ERR + 1][IB_QPS_ERR + 1] = { 826 [IB_QPS_RESET] = { 827 [IB_QPS_RESET] = { .valid = 1 }, 828 [IB_QPS_INIT] = { 829 .valid = 1, 830 .req_param = IB_QP_PKEY_INDEX | 831 IB_QP_PORT | 832 IB_QP_QKEY, 833 }, 834 }, 835 [IB_QPS_INIT] = { 836 [IB_QPS_RESET] = { .valid = 1 }, 837 [IB_QPS_ERR] = { .valid = 1 }, 838 [IB_QPS_INIT] = { 839 .valid = 1, 840 .opt_param = IB_QP_PKEY_INDEX | 841 IB_QP_PORT | 842 IB_QP_QKEY, 843 }, 844 [IB_QPS_RTR] = { 845 .valid = 1, 846 .opt_param = IB_QP_PKEY_INDEX | 847 IB_QP_QKEY, 848 }, 849 }, 850 [IB_QPS_RTR] = { 851 [IB_QPS_RESET] = { .valid = 1 }, 852 [IB_QPS_ERR] = { .valid = 1 }, 853 [IB_QPS_RTS] = { 854 .valid = 1, 855 .req_param = IB_QP_SQ_PSN, 856 .opt_param = IB_QP_CUR_STATE | 857 IB_QP_QKEY | 858 IB_QP_RNR_RETRY, 859 860 } 861 }, 862 [IB_QPS_RTS] = { 863 [IB_QPS_RESET] = { .valid = 1 }, 864 [IB_QPS_ERR] = { .valid = 1 }, 865 [IB_QPS_RTS] = { 866 .valid = 1, 867 .opt_param = IB_QP_CUR_STATE | 868 IB_QP_QKEY, 869 }, 870 [IB_QPS_SQD] = { 871 .valid = 1, 872 .opt_param = IB_QP_EN_SQD_ASYNC_NOTIFY, 873 }, 874 }, 875 [IB_QPS_SQD] = { 876 [IB_QPS_RESET] = { .valid = 1 }, 877 [IB_QPS_ERR] = { .valid = 1 }, 878 [IB_QPS_RTS] = { 879 .valid = 1, 880 .opt_param = IB_QP_CUR_STATE | 881 IB_QP_QKEY, 882 }, 883 [IB_QPS_SQD] = { 884 .valid = 1, 885 .opt_param = IB_QP_PKEY_INDEX | 886 IB_QP_QKEY, 887 } 888 }, 889 [IB_QPS_SQE] = { 890 [IB_QPS_RESET] = { .valid = 1 }, 891 [IB_QPS_ERR] = { .valid = 1 }, 892 [IB_QPS_RTS] = { 893 .valid = 1, 894 .opt_param = IB_QP_CUR_STATE | 895 IB_QP_QKEY, 896 } 897 }, 898 [IB_QPS_ERR] = { 899 [IB_QPS_RESET] = { .valid = 1 }, 900 [IB_QPS_ERR] = { .valid = 1 }, 901 } 902 }; 903 904 static bool efa_modify_srd_qp_is_ok(enum ib_qp_state cur_state, 905 enum ib_qp_state next_state, 906 enum ib_qp_attr_mask mask) 907 { 908 enum ib_qp_attr_mask req_param, opt_param; 909 910 if (mask & IB_QP_CUR_STATE && 911 cur_state != IB_QPS_RTR && cur_state != IB_QPS_RTS && 912 cur_state != IB_QPS_SQD && cur_state != IB_QPS_SQE) 913 return false; 914 915 if (!srd_qp_state_table[cur_state][next_state].valid) 916 return false; 917 918 req_param = srd_qp_state_table[cur_state][next_state].req_param; 919 opt_param = srd_qp_state_table[cur_state][next_state].opt_param; 920 921 if ((mask & req_param) != req_param) 922 return false; 923 924 if (mask & ~(req_param | opt_param | IB_QP_STATE)) 925 return false; 926 927 return true; 928 } 929 930 static int efa_modify_qp_validate(struct efa_dev *dev, struct efa_qp *qp, 931 struct ib_qp_attr *qp_attr, int qp_attr_mask, 932 enum ib_qp_state cur_state, 933 enum ib_qp_state new_state) 934 { 935 int err; 936 937 #define EFA_MODIFY_QP_SUPP_MASK \ 938 (IB_QP_STATE | IB_QP_CUR_STATE | IB_QP_EN_SQD_ASYNC_NOTIFY | \ 939 IB_QP_PKEY_INDEX | IB_QP_PORT | IB_QP_QKEY | IB_QP_SQ_PSN | \ 940 IB_QP_RNR_RETRY) 941 942 if (qp_attr_mask & ~EFA_MODIFY_QP_SUPP_MASK) { 943 ibdev_dbg(&dev->ibdev, 944 "Unsupported qp_attr_mask[%#x] supported[%#x]\n", 945 qp_attr_mask, EFA_MODIFY_QP_SUPP_MASK); 946 return -EOPNOTSUPP; 947 } 948 949 if (qp->ibqp.qp_type == IB_QPT_DRIVER) 950 err = !efa_modify_srd_qp_is_ok(cur_state, new_state, 951 qp_attr_mask); 952 else 953 err = !ib_modify_qp_is_ok(cur_state, new_state, IB_QPT_UD, 954 qp_attr_mask); 955 956 if (err) { 957 ibdev_dbg(&dev->ibdev, "Invalid modify QP parameters\n"); 958 return -EINVAL; 959 } 960 961 if ((qp_attr_mask & IB_QP_PORT) && qp_attr->port_num != 1) { 962 ibdev_dbg(&dev->ibdev, "Can't change port num\n"); 963 return -EOPNOTSUPP; 964 } 965 966 if ((qp_attr_mask & IB_QP_PKEY_INDEX) && qp_attr->pkey_index) { 967 ibdev_dbg(&dev->ibdev, "Can't change pkey index\n"); 968 return -EOPNOTSUPP; 969 } 970 971 return 0; 972 } 973 974 int efa_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, 975 int qp_attr_mask, struct ib_udata *udata) 976 { 977 struct efa_dev *dev = to_edev(ibqp->device); 978 struct efa_com_modify_qp_params params = {}; 979 struct efa_qp *qp = to_eqp(ibqp); 980 enum ib_qp_state cur_state; 981 enum ib_qp_state new_state; 982 int err; 983 984 if (qp_attr_mask & ~IB_QP_ATTR_STANDARD_BITS) 985 return -EOPNOTSUPP; 986 987 err = ib_is_udata_in_empty(udata); 988 if (err) 989 return err; 990 991 cur_state = qp_attr_mask & IB_QP_CUR_STATE ? qp_attr->cur_qp_state : 992 qp->state; 993 new_state = qp_attr_mask & IB_QP_STATE ? qp_attr->qp_state : cur_state; 994 995 err = efa_modify_qp_validate(dev, qp, qp_attr, qp_attr_mask, cur_state, 996 new_state); 997 if (err) 998 return err; 999 1000 params.qp_handle = qp->qp_handle; 1001 1002 if (qp_attr_mask & IB_QP_STATE) { 1003 EFA_SET(¶ms.modify_mask, EFA_ADMIN_MODIFY_QP_CMD_QP_STATE, 1004 1); 1005 EFA_SET(¶ms.modify_mask, 1006 EFA_ADMIN_MODIFY_QP_CMD_CUR_QP_STATE, 1); 1007 params.cur_qp_state = cur_state; 1008 params.qp_state = new_state; 1009 } 1010 1011 if (qp_attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY) { 1012 EFA_SET(¶ms.modify_mask, 1013 EFA_ADMIN_MODIFY_QP_CMD_SQ_DRAINED_ASYNC_NOTIFY, 1); 1014 params.sq_drained_async_notify = qp_attr->en_sqd_async_notify; 1015 } 1016 1017 if (qp_attr_mask & IB_QP_QKEY) { 1018 EFA_SET(¶ms.modify_mask, EFA_ADMIN_MODIFY_QP_CMD_QKEY, 1); 1019 params.qkey = qp_attr->qkey; 1020 } 1021 1022 if (qp_attr_mask & IB_QP_SQ_PSN) { 1023 EFA_SET(¶ms.modify_mask, EFA_ADMIN_MODIFY_QP_CMD_SQ_PSN, 1); 1024 params.sq_psn = qp_attr->sq_psn; 1025 } 1026 1027 if (qp_attr_mask & IB_QP_RNR_RETRY) { 1028 EFA_SET(¶ms.modify_mask, EFA_ADMIN_MODIFY_QP_CMD_RNR_RETRY, 1029 1); 1030 params.rnr_retry = qp_attr->rnr_retry; 1031 } 1032 1033 err = efa_com_modify_qp(&dev->edev, ¶ms); 1034 if (err) 1035 return err; 1036 1037 qp->state = new_state; 1038 1039 return 0; 1040 } 1041 1042 static int efa_destroy_cq_idx(struct efa_dev *dev, int cq_idx) 1043 { 1044 struct efa_com_destroy_cq_params params = { .cq_idx = cq_idx }; 1045 1046 return efa_com_destroy_cq(&dev->edev, ¶ms); 1047 } 1048 1049 static void efa_cq_user_mmap_entries_remove(struct efa_cq *cq) 1050 { 1051 rdma_user_mmap_entry_remove(cq->db_mmap_entry); 1052 rdma_user_mmap_entry_remove(cq->mmap_entry); 1053 } 1054 1055 int efa_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) 1056 { 1057 struct efa_dev *dev = to_edev(ibcq->device); 1058 struct efa_cq *cq = to_ecq(ibcq); 1059 1060 ibdev_dbg(&dev->ibdev, 1061 "Destroy cq[%d] virt[0x%p] freed: size[%lu], dma[%pad]\n", 1062 cq->cq_idx, cq->cpu_addr, cq->size, &cq->dma_addr); 1063 1064 efa_destroy_cq_idx(dev, cq->cq_idx); 1065 if (cq->cpu_addr) 1066 efa_cq_user_mmap_entries_remove(cq); 1067 if (cq->eq) { 1068 xa_erase(&dev->cqs_xa, cq->cq_idx); 1069 synchronize_irq(cq->eq->irq.irqn); 1070 } 1071 1072 if (cq->cpu_addr) 1073 efa_free_mapped(dev, cq->cpu_addr, cq->dma_addr, cq->size, DMA_FROM_DEVICE); 1074 ib_umem_release(cq->umem); 1075 return 0; 1076 } 1077 1078 static struct efa_eq *efa_vec2eq(struct efa_dev *dev, int vec) 1079 { 1080 return &dev->eqs[vec]; 1081 } 1082 1083 static int cq_mmap_entries_setup(struct efa_dev *dev, struct efa_cq *cq, 1084 struct efa_ibv_create_cq_resp *resp, 1085 bool db_valid) 1086 { 1087 resp->q_mmap_size = cq->size; 1088 cq->mmap_entry = efa_user_mmap_entry_insert(&cq->ucontext->ibucontext, 1089 virt_to_phys(cq->cpu_addr), 1090 cq->size, EFA_MMAP_DMA_PAGE, 1091 &resp->q_mmap_key); 1092 if (!cq->mmap_entry) 1093 return -ENOMEM; 1094 1095 if (db_valid) { 1096 cq->db_mmap_entry = 1097 efa_user_mmap_entry_insert(&cq->ucontext->ibucontext, 1098 dev->db_bar_addr + resp->db_off, 1099 PAGE_SIZE, EFA_MMAP_IO_NC, 1100 &resp->db_mmap_key); 1101 if (!cq->db_mmap_entry) { 1102 rdma_user_mmap_entry_remove(cq->mmap_entry); 1103 return -ENOMEM; 1104 } 1105 1106 resp->db_off &= ~PAGE_MASK; 1107 resp->comp_mask |= EFA_CREATE_CQ_RESP_DB_OFF; 1108 } 1109 1110 return 0; 1111 } 1112 1113 int efa_create_user_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, 1114 struct uverbs_attr_bundle *attrs) 1115 { 1116 struct ib_udata *udata = &attrs->driver_udata; 1117 struct efa_ucontext *ucontext = rdma_udata_to_drv_context( 1118 udata, struct efa_ucontext, ibucontext); 1119 struct efa_com_create_cq_params params = {}; 1120 struct efa_ibv_create_cq_resp resp = {}; 1121 struct efa_com_create_cq_result result; 1122 struct ib_device *ibdev = ibcq->device; 1123 struct efa_dev *dev = to_edev(ibdev); 1124 struct efa_ibv_create_cq cmd; 1125 struct efa_cq *cq = to_ecq(ibcq); 1126 int entries = attr->cqe; 1127 struct ib_umem *umem; 1128 bool set_src_addr; 1129 int err; 1130 1131 ibdev_dbg(ibdev, "create_cq entries %d\n", entries); 1132 1133 if (attr->flags) 1134 return -EOPNOTSUPP; 1135 1136 if (entries > dev->dev_attr.max_cq_depth) { 1137 ibdev_dbg(ibdev, 1138 "cq: requested entries[%u] greater than max[%u]\n", 1139 entries, dev->dev_attr.max_cq_depth); 1140 err = -EINVAL; 1141 goto err_out; 1142 } 1143 1144 err = ib_copy_validate_udata_in_cm(udata, cmd, num_sub_cqs, 0); 1145 if (err) 1146 goto err_out; 1147 1148 if (!is_reserved_cleared(cmd.reserved_58)) { 1149 ibdev_dbg(ibdev, 1150 "Incompatible ABI params, unknown fields in udata\n"); 1151 err = -EINVAL; 1152 goto err_out; 1153 } 1154 1155 set_src_addr = !!(cmd.flags & EFA_CREATE_CQ_WITH_SGID); 1156 if ((cmd.cq_entry_size != sizeof(struct efa_io_rx_cdesc_ex)) && 1157 (set_src_addr || 1158 cmd.cq_entry_size != sizeof(struct efa_io_rx_cdesc))) { 1159 ibdev_dbg(ibdev, 1160 "Invalid entry size [%u]\n", cmd.cq_entry_size); 1161 err = -EINVAL; 1162 goto err_out; 1163 } 1164 1165 if (cmd.num_sub_cqs != dev->dev_attr.sub_cqs_per_cq) { 1166 ibdev_dbg(ibdev, 1167 "Invalid number of sub cqs[%u] expected[%u]\n", 1168 cmd.num_sub_cqs, dev->dev_attr.sub_cqs_per_cq); 1169 err = -EINVAL; 1170 goto err_out; 1171 } 1172 1173 cq->ucontext = ucontext; 1174 cq->size = PAGE_ALIGN(cmd.cq_entry_size * entries * cmd.num_sub_cqs); 1175 1176 umem = ib_umem_get_cq_buf(ibcq->device, attrs, cq->size, 1177 IB_ACCESS_LOCAL_WRITE); 1178 if (IS_ERR(umem)) { 1179 err = PTR_ERR(umem); 1180 goto err_out; 1181 } 1182 1183 cq->umem = umem; 1184 1185 if (umem) { 1186 if (!ib_umem_is_contiguous(umem)) { 1187 ibdev_dbg(&dev->ibdev, "Non contiguous CQ unsupported\n"); 1188 err = -EINVAL; 1189 goto err_release_umem; 1190 } 1191 1192 cq->dma_addr = ib_umem_start_dma_addr(umem); 1193 } else { 1194 cq->cpu_addr = efa_zalloc_mapped(dev, &cq->dma_addr, cq->size, 1195 DMA_FROM_DEVICE); 1196 if (!cq->cpu_addr) { 1197 err = -ENOMEM; 1198 goto err_release_umem; 1199 } 1200 } 1201 1202 params.uarn = cq->ucontext->uarn; 1203 params.sub_cq_depth = entries; 1204 params.dma_addr = cq->dma_addr; 1205 params.entry_size_in_bytes = cmd.cq_entry_size; 1206 params.num_sub_cqs = cmd.num_sub_cqs; 1207 params.set_src_addr = set_src_addr; 1208 if (cmd.flags & EFA_CREATE_CQ_WITH_COMPLETION_CHANNEL) { 1209 cq->eq = efa_vec2eq(dev, attr->comp_vector); 1210 params.eqn = cq->eq->eeq.eqn; 1211 params.interrupt_mode_enabled = true; 1212 } 1213 1214 err = efa_com_create_cq(&dev->edev, ¶ms, &result); 1215 if (err) 1216 goto err_free_mapped; 1217 1218 resp.db_off = result.db_off; 1219 resp.cq_idx = result.cq_idx; 1220 cq->cq_idx = result.cq_idx; 1221 cq->ibcq.cqe = result.actual_depth; 1222 WARN_ON_ONCE(entries != result.actual_depth); 1223 1224 if (cq->cpu_addr) 1225 err = cq_mmap_entries_setup(dev, cq, &resp, result.db_valid); 1226 1227 if (err) { 1228 ibdev_dbg(ibdev, "Could not setup cq[%u] mmap entries\n", 1229 cq->cq_idx); 1230 goto err_destroy_cq; 1231 } 1232 1233 if (cq->eq) { 1234 err = xa_err(xa_store(&dev->cqs_xa, cq->cq_idx, cq, GFP_KERNEL)); 1235 if (err) { 1236 ibdev_dbg(ibdev, "Failed to store cq[%u] in xarray\n", 1237 cq->cq_idx); 1238 goto err_remove_mmap; 1239 } 1240 } 1241 1242 if (udata->outlen) { 1243 err = ib_respond_udata(udata, resp); 1244 if (err) 1245 goto err_xa_erase; 1246 } 1247 1248 ibdev_dbg(ibdev, "Created cq[%d], cq depth[%u]. dma[%pad] virt[0x%p]\n", 1249 cq->cq_idx, result.actual_depth, &cq->dma_addr, cq->cpu_addr); 1250 1251 return 0; 1252 1253 err_xa_erase: 1254 if (cq->eq) 1255 xa_erase(&dev->cqs_xa, cq->cq_idx); 1256 err_remove_mmap: 1257 if (cq->cpu_addr) 1258 efa_cq_user_mmap_entries_remove(cq); 1259 err_destroy_cq: 1260 efa_destroy_cq_idx(dev, cq->cq_idx); 1261 err_free_mapped: 1262 if (cq->cpu_addr) 1263 efa_free_mapped(dev, cq->cpu_addr, cq->dma_addr, cq->size, 1264 DMA_FROM_DEVICE); 1265 err_release_umem: 1266 ib_umem_release(cq->umem); 1267 err_out: 1268 atomic64_inc(&dev->stats.create_cq_err); 1269 return err; 1270 } 1271 1272 static int umem_to_page_list(struct efa_dev *dev, 1273 struct ib_umem *umem, 1274 u64 *page_list, 1275 u32 hp_cnt, 1276 u8 hp_shift) 1277 { 1278 struct ib_block_iter biter; 1279 unsigned int hp_idx = 0; 1280 1281 rdma_umem_for_each_dma_block(umem, &biter, BIT(hp_shift)) 1282 page_list[hp_idx++] = rdma_block_iter_dma_address(&biter); 1283 1284 return 0; 1285 } 1286 1287 static struct scatterlist *efa_vmalloc_buf_to_sg(u64 *buf, int page_cnt) 1288 { 1289 struct scatterlist *sglist; 1290 struct page *pg; 1291 int i; 1292 1293 sglist = kmalloc_objs(*sglist, page_cnt); 1294 if (!sglist) 1295 return NULL; 1296 sg_init_table(sglist, page_cnt); 1297 for (i = 0; i < page_cnt; i++) { 1298 pg = vmalloc_to_page(buf); 1299 if (!pg) 1300 goto err; 1301 sg_set_page(&sglist[i], pg, PAGE_SIZE, 0); 1302 buf += PAGE_SIZE / sizeof(*buf); 1303 } 1304 return sglist; 1305 1306 err: 1307 kfree(sglist); 1308 return NULL; 1309 } 1310 1311 /* 1312 * create a chunk list of physical pages dma addresses from the supplied 1313 * scatter gather list 1314 */ 1315 static int pbl_chunk_list_create(struct efa_dev *dev, struct pbl_context *pbl) 1316 { 1317 struct pbl_chunk_list *chunk_list = &pbl->phys.indirect.chunk_list; 1318 int page_cnt = pbl->phys.indirect.pbl_buf_size_in_pages; 1319 struct scatterlist *pages_sgl = pbl->phys.indirect.sgl; 1320 unsigned int chunk_list_size, chunk_idx, payload_idx; 1321 int sg_dma_cnt = pbl->phys.indirect.sg_dma_cnt; 1322 struct efa_com_ctrl_buff_info *ctrl_buf; 1323 u64 *cur_chunk_buf, *prev_chunk_buf; 1324 struct ib_block_iter biter; 1325 dma_addr_t dma_addr; 1326 int i; 1327 1328 /* allocate a chunk list that consists of 4KB chunks */ 1329 chunk_list_size = DIV_ROUND_UP(page_cnt, EFA_PTRS_PER_CHUNK); 1330 1331 chunk_list->size = chunk_list_size; 1332 chunk_list->chunks = kzalloc_objs(*chunk_list->chunks, chunk_list_size); 1333 if (!chunk_list->chunks) 1334 return -ENOMEM; 1335 1336 ibdev_dbg(&dev->ibdev, 1337 "chunk_list_size[%u] - pages[%u]\n", chunk_list_size, 1338 page_cnt); 1339 1340 /* allocate chunk buffers: */ 1341 for (i = 0; i < chunk_list_size; i++) { 1342 chunk_list->chunks[i].buf = kzalloc(EFA_CHUNK_SIZE, GFP_KERNEL); 1343 if (!chunk_list->chunks[i].buf) 1344 goto chunk_list_dealloc; 1345 1346 chunk_list->chunks[i].length = EFA_CHUNK_USED_SIZE; 1347 } 1348 chunk_list->chunks[chunk_list_size - 1].length = 1349 ((page_cnt % EFA_PTRS_PER_CHUNK) * EFA_CHUNK_PAYLOAD_PTR_SIZE) + 1350 EFA_CHUNK_PTR_SIZE; 1351 1352 /* fill the dma addresses of sg list pages to chunks: */ 1353 chunk_idx = 0; 1354 payload_idx = 0; 1355 cur_chunk_buf = chunk_list->chunks[0].buf; 1356 rdma_for_each_block(pages_sgl, &biter, sg_dma_cnt, 1357 EFA_CHUNK_PAYLOAD_SIZE) { 1358 cur_chunk_buf[payload_idx++] = 1359 rdma_block_iter_dma_address(&biter); 1360 1361 if (payload_idx == EFA_PTRS_PER_CHUNK) { 1362 chunk_idx++; 1363 cur_chunk_buf = chunk_list->chunks[chunk_idx].buf; 1364 payload_idx = 0; 1365 } 1366 } 1367 1368 /* map chunks to dma and fill chunks next ptrs */ 1369 for (i = chunk_list_size - 1; i >= 0; i--) { 1370 dma_addr = dma_map_single(&dev->pdev->dev, 1371 chunk_list->chunks[i].buf, 1372 chunk_list->chunks[i].length, 1373 DMA_TO_DEVICE); 1374 if (dma_mapping_error(&dev->pdev->dev, dma_addr)) { 1375 ibdev_err(&dev->ibdev, 1376 "chunk[%u] dma_map_failed\n", i); 1377 goto chunk_list_unmap; 1378 } 1379 1380 chunk_list->chunks[i].dma_addr = dma_addr; 1381 ibdev_dbg(&dev->ibdev, 1382 "chunk[%u] mapped at [%pad]\n", i, &dma_addr); 1383 1384 if (!i) 1385 break; 1386 1387 prev_chunk_buf = chunk_list->chunks[i - 1].buf; 1388 1389 ctrl_buf = (struct efa_com_ctrl_buff_info *) 1390 &prev_chunk_buf[EFA_PTRS_PER_CHUNK]; 1391 ctrl_buf->length = chunk_list->chunks[i].length; 1392 1393 efa_com_set_dma_addr(dma_addr, 1394 &ctrl_buf->address.mem_addr_high, 1395 &ctrl_buf->address.mem_addr_low); 1396 } 1397 1398 return 0; 1399 1400 chunk_list_unmap: 1401 for (; i < chunk_list_size; i++) { 1402 dma_unmap_single(&dev->pdev->dev, chunk_list->chunks[i].dma_addr, 1403 chunk_list->chunks[i].length, DMA_TO_DEVICE); 1404 } 1405 chunk_list_dealloc: 1406 for (i = 0; i < chunk_list_size; i++) 1407 kfree(chunk_list->chunks[i].buf); 1408 1409 kfree(chunk_list->chunks); 1410 return -ENOMEM; 1411 } 1412 1413 static void pbl_chunk_list_destroy(struct efa_dev *dev, struct pbl_context *pbl) 1414 { 1415 struct pbl_chunk_list *chunk_list = &pbl->phys.indirect.chunk_list; 1416 int i; 1417 1418 for (i = 0; i < chunk_list->size; i++) { 1419 dma_unmap_single(&dev->pdev->dev, chunk_list->chunks[i].dma_addr, 1420 chunk_list->chunks[i].length, DMA_TO_DEVICE); 1421 kfree(chunk_list->chunks[i].buf); 1422 } 1423 1424 kfree(chunk_list->chunks); 1425 } 1426 1427 /* initialize pbl continuous mode: map pbl buffer to a dma address. */ 1428 static int pbl_continuous_initialize(struct efa_dev *dev, 1429 struct pbl_context *pbl) 1430 { 1431 dma_addr_t dma_addr; 1432 1433 dma_addr = dma_map_single(&dev->pdev->dev, pbl->pbl_buf, 1434 pbl->pbl_buf_size_in_bytes, DMA_TO_DEVICE); 1435 if (dma_mapping_error(&dev->pdev->dev, dma_addr)) { 1436 ibdev_err(&dev->ibdev, "Unable to map pbl to DMA address\n"); 1437 return -ENOMEM; 1438 } 1439 1440 pbl->phys.continuous.dma_addr = dma_addr; 1441 ibdev_dbg(&dev->ibdev, 1442 "pbl continuous - dma_addr = %pad, size[%u]\n", 1443 &dma_addr, pbl->pbl_buf_size_in_bytes); 1444 1445 return 0; 1446 } 1447 1448 /* 1449 * initialize pbl indirect mode: 1450 * create a chunk list out of the dma addresses of the physical pages of 1451 * pbl buffer. 1452 */ 1453 static int pbl_indirect_initialize(struct efa_dev *dev, struct pbl_context *pbl) 1454 { 1455 u32 size_in_pages = DIV_ROUND_UP(pbl->pbl_buf_size_in_bytes, EFA_CHUNK_PAYLOAD_SIZE); 1456 struct scatterlist *sgl; 1457 int sg_dma_cnt, err; 1458 1459 BUILD_BUG_ON(EFA_CHUNK_PAYLOAD_SIZE > PAGE_SIZE); 1460 sgl = efa_vmalloc_buf_to_sg(pbl->pbl_buf, size_in_pages); 1461 if (!sgl) 1462 return -ENOMEM; 1463 1464 sg_dma_cnt = dma_map_sg(&dev->pdev->dev, sgl, size_in_pages, DMA_TO_DEVICE); 1465 if (!sg_dma_cnt) { 1466 err = -EINVAL; 1467 goto err_map; 1468 } 1469 1470 pbl->phys.indirect.pbl_buf_size_in_pages = size_in_pages; 1471 pbl->phys.indirect.sgl = sgl; 1472 pbl->phys.indirect.sg_dma_cnt = sg_dma_cnt; 1473 err = pbl_chunk_list_create(dev, pbl); 1474 if (err) { 1475 ibdev_dbg(&dev->ibdev, 1476 "chunk_list creation failed[%d]\n", err); 1477 goto err_chunk; 1478 } 1479 1480 ibdev_dbg(&dev->ibdev, 1481 "pbl indirect - size[%u], chunks[%u]\n", 1482 pbl->pbl_buf_size_in_bytes, 1483 pbl->phys.indirect.chunk_list.size); 1484 1485 return 0; 1486 1487 err_chunk: 1488 dma_unmap_sg(&dev->pdev->dev, sgl, size_in_pages, DMA_TO_DEVICE); 1489 err_map: 1490 kfree(sgl); 1491 return err; 1492 } 1493 1494 static void pbl_indirect_terminate(struct efa_dev *dev, struct pbl_context *pbl) 1495 { 1496 pbl_chunk_list_destroy(dev, pbl); 1497 dma_unmap_sg(&dev->pdev->dev, pbl->phys.indirect.sgl, 1498 pbl->phys.indirect.pbl_buf_size_in_pages, DMA_TO_DEVICE); 1499 kfree(pbl->phys.indirect.sgl); 1500 } 1501 1502 /* create a page buffer list from a mapped user memory region */ 1503 static int pbl_create(struct efa_dev *dev, 1504 struct pbl_context *pbl, 1505 struct ib_umem *umem, 1506 int hp_cnt, 1507 u8 hp_shift) 1508 { 1509 int err; 1510 1511 pbl->pbl_buf_size_in_bytes = hp_cnt * EFA_CHUNK_PAYLOAD_PTR_SIZE; 1512 pbl->pbl_buf = kvzalloc(pbl->pbl_buf_size_in_bytes, GFP_KERNEL); 1513 if (!pbl->pbl_buf) 1514 return -ENOMEM; 1515 1516 if (is_vmalloc_addr(pbl->pbl_buf)) { 1517 pbl->physically_continuous = 0; 1518 err = umem_to_page_list(dev, umem, pbl->pbl_buf, hp_cnt, 1519 hp_shift); 1520 if (err) 1521 goto err_free; 1522 1523 err = pbl_indirect_initialize(dev, pbl); 1524 if (err) 1525 goto err_free; 1526 } else { 1527 pbl->physically_continuous = 1; 1528 err = umem_to_page_list(dev, umem, pbl->pbl_buf, hp_cnt, 1529 hp_shift); 1530 if (err) 1531 goto err_free; 1532 1533 err = pbl_continuous_initialize(dev, pbl); 1534 if (err) 1535 goto err_free; 1536 } 1537 1538 ibdev_dbg(&dev->ibdev, 1539 "user_pbl_created: user_pages[%u], continuous[%u]\n", 1540 hp_cnt, pbl->physically_continuous); 1541 1542 return 0; 1543 1544 err_free: 1545 kvfree(pbl->pbl_buf); 1546 return err; 1547 } 1548 1549 static void pbl_destroy(struct efa_dev *dev, struct pbl_context *pbl) 1550 { 1551 if (pbl->physically_continuous) 1552 dma_unmap_single(&dev->pdev->dev, pbl->phys.continuous.dma_addr, 1553 pbl->pbl_buf_size_in_bytes, DMA_TO_DEVICE); 1554 else 1555 pbl_indirect_terminate(dev, pbl); 1556 1557 kvfree(pbl->pbl_buf); 1558 } 1559 1560 static int efa_create_inline_pbl(struct efa_dev *dev, struct efa_mr *mr, 1561 struct efa_com_reg_mr_params *params) 1562 { 1563 int err; 1564 1565 params->inline_pbl = 1; 1566 err = umem_to_page_list(dev, mr->umem, params->pbl.inline_pbl_array, 1567 params->page_num, params->page_shift); 1568 if (err) 1569 return err; 1570 1571 ibdev_dbg(&dev->ibdev, 1572 "inline_pbl_array - pages[%u]\n", params->page_num); 1573 1574 return 0; 1575 } 1576 1577 static int efa_create_pbl(struct efa_dev *dev, 1578 struct pbl_context *pbl, 1579 struct efa_mr *mr, 1580 struct efa_com_reg_mr_params *params) 1581 { 1582 int err; 1583 1584 err = pbl_create(dev, pbl, mr->umem, params->page_num, 1585 params->page_shift); 1586 if (err) { 1587 ibdev_dbg(&dev->ibdev, "Failed to create pbl[%d]\n", err); 1588 return err; 1589 } 1590 1591 params->inline_pbl = 0; 1592 params->indirect = !pbl->physically_continuous; 1593 if (pbl->physically_continuous) { 1594 params->pbl.pbl.length = pbl->pbl_buf_size_in_bytes; 1595 1596 efa_com_set_dma_addr(pbl->phys.continuous.dma_addr, 1597 ¶ms->pbl.pbl.address.mem_addr_high, 1598 ¶ms->pbl.pbl.address.mem_addr_low); 1599 } else { 1600 params->pbl.pbl.length = 1601 pbl->phys.indirect.chunk_list.chunks[0].length; 1602 1603 efa_com_set_dma_addr(pbl->phys.indirect.chunk_list.chunks[0].dma_addr, 1604 ¶ms->pbl.pbl.address.mem_addr_high, 1605 ¶ms->pbl.pbl.address.mem_addr_low); 1606 } 1607 1608 return 0; 1609 } 1610 1611 static struct efa_mr *efa_alloc_mr(struct ib_pd *ibpd, int access_flags, 1612 struct ib_udata *udata) 1613 { 1614 struct efa_dev *dev = to_edev(ibpd->device); 1615 int supp_access_flags; 1616 struct efa_mr *mr; 1617 int ret; 1618 1619 ret = ib_is_udata_in_empty(udata); 1620 if (ret) 1621 return ERR_PTR(ret); 1622 1623 supp_access_flags = 1624 IB_ACCESS_LOCAL_WRITE | 1625 (EFA_DEV_CAP(dev, RDMA_READ) ? IB_ACCESS_REMOTE_READ : 0) | 1626 (EFA_DEV_CAP(dev, RDMA_WRITE) ? IB_ACCESS_REMOTE_WRITE : 0); 1627 1628 access_flags &= ~IB_ACCESS_OPTIONAL; 1629 if (access_flags & ~supp_access_flags) { 1630 ibdev_dbg(&dev->ibdev, 1631 "Unsupported access flags[%#x], supported[%#x]\n", 1632 access_flags, supp_access_flags); 1633 return ERR_PTR(-EOPNOTSUPP); 1634 } 1635 1636 mr = kzalloc_obj(*mr); 1637 if (!mr) 1638 return ERR_PTR(-ENOMEM); 1639 1640 return mr; 1641 } 1642 1643 static int efa_register_mr(struct ib_pd *ibpd, struct efa_mr *mr, u64 start, 1644 u64 length, u64 virt_addr, int access_flags) 1645 { 1646 struct efa_dev *dev = to_edev(ibpd->device); 1647 struct efa_com_reg_mr_params params = {}; 1648 struct efa_com_reg_mr_result result = {}; 1649 struct pbl_context pbl; 1650 unsigned int pg_sz; 1651 int inline_size; 1652 int err; 1653 1654 params.pd = to_epd(ibpd)->pdn; 1655 params.iova = virt_addr; 1656 params.mr_length_in_bytes = length; 1657 params.permissions = access_flags; 1658 1659 pg_sz = ib_umem_find_best_pgsz(mr->umem, 1660 dev->dev_attr.page_size_cap, 1661 virt_addr); 1662 if (!pg_sz) { 1663 ibdev_dbg(&dev->ibdev, "Failed to find a suitable page size in page_size_cap %#llx\n", 1664 dev->dev_attr.page_size_cap); 1665 return -EOPNOTSUPP; 1666 } 1667 1668 params.page_shift = order_base_2(pg_sz); 1669 params.page_num = ib_umem_num_dma_blocks(mr->umem, pg_sz); 1670 1671 ibdev_dbg(&dev->ibdev, 1672 "start %#llx length %#llx params.page_shift %u params.page_num %u\n", 1673 start, length, params.page_shift, params.page_num); 1674 1675 inline_size = ARRAY_SIZE(params.pbl.inline_pbl_array); 1676 if (params.page_num <= inline_size) { 1677 err = efa_create_inline_pbl(dev, mr, ¶ms); 1678 if (err) 1679 return err; 1680 1681 err = efa_com_register_mr(&dev->edev, ¶ms, &result); 1682 if (err) 1683 return err; 1684 } else { 1685 err = efa_create_pbl(dev, &pbl, mr, ¶ms); 1686 if (err) 1687 return err; 1688 1689 err = efa_com_register_mr(&dev->edev, ¶ms, &result); 1690 pbl_destroy(dev, &pbl); 1691 1692 if (err) 1693 return err; 1694 } 1695 1696 mr->ibmr.lkey = result.l_key; 1697 mr->ibmr.rkey = result.r_key; 1698 mr->ibmr.length = length; 1699 mr->ic_info.recv_ic_id = result.ic_info.recv_ic_id; 1700 mr->ic_info.rdma_read_ic_id = result.ic_info.rdma_read_ic_id; 1701 mr->ic_info.rdma_recv_ic_id = result.ic_info.rdma_recv_ic_id; 1702 mr->ic_info.recv_ic_id_valid = result.ic_info.recv_ic_id_valid; 1703 mr->ic_info.rdma_read_ic_id_valid = result.ic_info.rdma_read_ic_id_valid; 1704 mr->ic_info.rdma_recv_ic_id_valid = result.ic_info.rdma_recv_ic_id_valid; 1705 ibdev_dbg(&dev->ibdev, "Registered mr[%d]\n", mr->ibmr.lkey); 1706 1707 return 0; 1708 } 1709 1710 struct ib_mr *efa_reg_user_mr_dmabuf(struct ib_pd *ibpd, u64 start, 1711 u64 length, u64 virt_addr, 1712 int fd, int access_flags, 1713 struct ib_dmah *dmah, 1714 struct uverbs_attr_bundle *attrs) 1715 { 1716 struct efa_dev *dev = to_edev(ibpd->device); 1717 struct ib_umem_dmabuf *umem_dmabuf; 1718 struct efa_mr *mr; 1719 int err; 1720 1721 if (dmah) { 1722 err = -EOPNOTSUPP; 1723 goto err_out; 1724 } 1725 1726 mr = efa_alloc_mr(ibpd, access_flags, &attrs->driver_udata); 1727 if (IS_ERR(mr)) { 1728 err = PTR_ERR(mr); 1729 goto err_out; 1730 } 1731 1732 umem_dmabuf = ib_umem_dmabuf_get_pinned(ibpd->device, start, length, fd, 1733 access_flags); 1734 if (IS_ERR(umem_dmabuf)) { 1735 err = PTR_ERR(umem_dmabuf); 1736 ibdev_dbg(&dev->ibdev, "Failed to get dmabuf umem[%pe]\n", 1737 umem_dmabuf); 1738 goto err_free; 1739 } 1740 1741 mr->umem = &umem_dmabuf->umem; 1742 err = efa_register_mr(ibpd, mr, start, length, virt_addr, access_flags); 1743 if (err) 1744 goto err_release; 1745 1746 return &mr->ibmr; 1747 1748 err_release: 1749 ib_umem_release(mr->umem); 1750 err_free: 1751 kfree(mr); 1752 err_out: 1753 atomic64_inc(&dev->stats.reg_mr_err); 1754 return ERR_PTR(err); 1755 } 1756 1757 struct ib_mr *efa_reg_mr(struct ib_pd *ibpd, u64 start, u64 length, 1758 u64 virt_addr, int access_flags, 1759 struct ib_dmah *dmah, 1760 struct ib_udata *udata) 1761 { 1762 struct efa_dev *dev = to_edev(ibpd->device); 1763 struct efa_mr *mr; 1764 int err; 1765 1766 if (dmah) { 1767 err = -EOPNOTSUPP; 1768 goto err_out; 1769 } 1770 1771 mr = efa_alloc_mr(ibpd, access_flags, udata); 1772 if (IS_ERR(mr)) { 1773 err = PTR_ERR(mr); 1774 goto err_out; 1775 } 1776 1777 mr->umem = ib_umem_get_va(ibpd->device, start, length, access_flags); 1778 if (IS_ERR(mr->umem)) { 1779 err = PTR_ERR(mr->umem); 1780 ibdev_dbg(&dev->ibdev, 1781 "Failed to pin and map user space memory[%pe]\n", 1782 mr->umem); 1783 goto err_free; 1784 } 1785 1786 err = efa_register_mr(ibpd, mr, start, length, virt_addr, access_flags); 1787 if (err) 1788 goto err_release; 1789 1790 return &mr->ibmr; 1791 1792 err_release: 1793 ib_umem_release(mr->umem); 1794 err_free: 1795 kfree(mr); 1796 err_out: 1797 atomic64_inc(&dev->stats.reg_mr_err); 1798 return ERR_PTR(err); 1799 } 1800 1801 static int UVERBS_HANDLER(EFA_IB_METHOD_MR_QUERY)(struct uverbs_attr_bundle *attrs) 1802 { 1803 struct ib_mr *ibmr = uverbs_attr_get_obj(attrs, EFA_IB_ATTR_QUERY_MR_HANDLE); 1804 struct efa_mr *mr = to_emr(ibmr); 1805 u16 ic_id_validity = 0; 1806 int ret; 1807 1808 ret = uverbs_copy_to(attrs, EFA_IB_ATTR_QUERY_MR_RESP_RECV_IC_ID, 1809 &mr->ic_info.recv_ic_id, sizeof(mr->ic_info.recv_ic_id)); 1810 if (ret) 1811 return ret; 1812 1813 ret = uverbs_copy_to(attrs, EFA_IB_ATTR_QUERY_MR_RESP_RDMA_READ_IC_ID, 1814 &mr->ic_info.rdma_read_ic_id, sizeof(mr->ic_info.rdma_read_ic_id)); 1815 if (ret) 1816 return ret; 1817 1818 ret = uverbs_copy_to(attrs, EFA_IB_ATTR_QUERY_MR_RESP_RDMA_RECV_IC_ID, 1819 &mr->ic_info.rdma_recv_ic_id, sizeof(mr->ic_info.rdma_recv_ic_id)); 1820 if (ret) 1821 return ret; 1822 1823 if (mr->ic_info.recv_ic_id_valid) 1824 ic_id_validity |= EFA_QUERY_MR_VALIDITY_RECV_IC_ID; 1825 if (mr->ic_info.rdma_read_ic_id_valid) 1826 ic_id_validity |= EFA_QUERY_MR_VALIDITY_RDMA_READ_IC_ID; 1827 if (mr->ic_info.rdma_recv_ic_id_valid) 1828 ic_id_validity |= EFA_QUERY_MR_VALIDITY_RDMA_RECV_IC_ID; 1829 1830 return uverbs_copy_to(attrs, EFA_IB_ATTR_QUERY_MR_RESP_IC_ID_VALIDITY, 1831 &ic_id_validity, sizeof(ic_id_validity)); 1832 } 1833 1834 int efa_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata) 1835 { 1836 struct efa_dev *dev = to_edev(ibmr->device); 1837 struct efa_com_dereg_mr_params params; 1838 struct efa_mr *mr = to_emr(ibmr); 1839 int err; 1840 1841 ibdev_dbg(&dev->ibdev, "Deregister mr[%d]\n", ibmr->lkey); 1842 1843 params.l_key = mr->ibmr.lkey; 1844 err = efa_com_dereg_mr(&dev->edev, ¶ms); 1845 if (err) 1846 return err; 1847 1848 ib_umem_release(mr->umem); 1849 kfree(mr); 1850 1851 return 0; 1852 } 1853 1854 int efa_get_port_immutable(struct ib_device *ibdev, u32 port_num, 1855 struct ib_port_immutable *immutable) 1856 { 1857 struct ib_port_attr attr; 1858 int err; 1859 1860 err = ib_query_port(ibdev, port_num, &attr); 1861 if (err) { 1862 ibdev_dbg(ibdev, "Couldn't query port err[%d]\n", err); 1863 return err; 1864 } 1865 1866 immutable->pkey_tbl_len = attr.pkey_tbl_len; 1867 immutable->gid_tbl_len = attr.gid_tbl_len; 1868 1869 return 0; 1870 } 1871 1872 static int efa_dealloc_uar(struct efa_dev *dev, u16 uarn) 1873 { 1874 struct efa_com_dealloc_uar_params params = { 1875 .uarn = uarn, 1876 }; 1877 1878 return efa_com_dealloc_uar(&dev->edev, ¶ms); 1879 } 1880 1881 #define EFA_CHECK_USER_SUPP(_dev, _supported_caps, _attr, _mask, _attr_str) \ 1882 (_attr_str = (!(_dev)->dev_attr._attr || ((_supported_caps) & (_mask))) ? \ 1883 NULL : #_attr) 1884 1885 static int efa_user_supp_handshake(const struct ib_ucontext *ibucontext, 1886 const struct efa_ibv_alloc_ucontext_cmd *cmd) 1887 { 1888 struct efa_dev *dev = to_edev(ibucontext->device); 1889 char *attr_str; 1890 1891 if (EFA_CHECK_USER_SUPP(dev, cmd->supported_caps, max_tx_batch, 1892 EFA_ALLOC_UCONTEXT_CMD_SUPP_CAPS_TX_BATCH, 1893 attr_str)) 1894 goto err; 1895 1896 if (EFA_CHECK_USER_SUPP(dev, cmd->supported_caps, min_sq_depth, 1897 EFA_ALLOC_UCONTEXT_CMD_SUPP_CAPS_MIN_SQ_WR, 1898 attr_str)) 1899 goto err; 1900 1901 return 0; 1902 1903 err: 1904 ibdev_dbg(&dev->ibdev, "Userspace handshake failed for %s attribute\n", 1905 attr_str); 1906 return -EOPNOTSUPP; 1907 } 1908 1909 int efa_alloc_ucontext(struct ib_ucontext *ibucontext, struct ib_udata *udata) 1910 { 1911 struct efa_ucontext *ucontext = to_eucontext(ibucontext); 1912 struct efa_dev *dev = to_edev(ibucontext->device); 1913 struct efa_ibv_alloc_ucontext_resp resp = {}; 1914 struct efa_ibv_alloc_ucontext_cmd cmd = {}; 1915 struct efa_com_alloc_uar_result result; 1916 int err; 1917 1918 /* 1919 * it's fine if the driver does not know all request fields, 1920 * we will ack input fields in our response. 1921 */ 1922 1923 err = ib_copy_from_udata(&cmd, udata, 1924 min(sizeof(cmd), udata->inlen)); 1925 if (err) { 1926 ibdev_dbg(&dev->ibdev, 1927 "Cannot copy udata for alloc_ucontext\n"); 1928 goto err_out; 1929 } 1930 1931 err = efa_user_supp_handshake(ibucontext, &cmd); 1932 if (err) 1933 goto err_out; 1934 1935 err = efa_com_alloc_uar(&dev->edev, &result); 1936 if (err) 1937 goto err_out; 1938 1939 ucontext->uarn = result.uarn; 1940 1941 resp.cmds_supp_udata_mask |= EFA_USER_CMDS_SUPP_UDATA_QUERY_DEVICE; 1942 resp.cmds_supp_udata_mask |= EFA_USER_CMDS_SUPP_UDATA_CREATE_AH; 1943 resp.sub_cqs_per_cq = dev->dev_attr.sub_cqs_per_cq; 1944 resp.inline_buf_size = dev->dev_attr.inline_buf_size; 1945 resp.inline_buf_size_ex = dev->dev_attr.inline_buf_size_ex; 1946 resp.max_llq_size = dev->dev_attr.max_llq_size; 1947 resp.max_tx_batch = dev->dev_attr.max_tx_batch; 1948 resp.min_sq_wr = dev->dev_attr.min_sq_depth; 1949 1950 err = ib_respond_udata(udata, resp); 1951 if (err) 1952 goto err_dealloc_uar; 1953 1954 return 0; 1955 1956 err_dealloc_uar: 1957 efa_dealloc_uar(dev, result.uarn); 1958 err_out: 1959 atomic64_inc(&dev->stats.alloc_ucontext_err); 1960 return err; 1961 } 1962 1963 void efa_dealloc_ucontext(struct ib_ucontext *ibucontext) 1964 { 1965 struct efa_ucontext *ucontext = to_eucontext(ibucontext); 1966 struct efa_dev *dev = to_edev(ibucontext->device); 1967 1968 efa_dealloc_uar(dev, ucontext->uarn); 1969 } 1970 1971 void efa_mmap_free(struct rdma_user_mmap_entry *rdma_entry) 1972 { 1973 struct efa_user_mmap_entry *entry = to_emmap(rdma_entry); 1974 1975 kfree(entry); 1976 } 1977 1978 static int __efa_mmap(struct efa_dev *dev, struct efa_ucontext *ucontext, 1979 struct vm_area_struct *vma) 1980 { 1981 struct rdma_user_mmap_entry *rdma_entry; 1982 struct efa_user_mmap_entry *entry; 1983 unsigned long va; 1984 int err = 0; 1985 u64 pfn; 1986 1987 rdma_entry = rdma_user_mmap_entry_get(&ucontext->ibucontext, vma); 1988 if (!rdma_entry) { 1989 ibdev_dbg(&dev->ibdev, 1990 "pgoff[%#lx] does not have valid entry\n", 1991 vma->vm_pgoff); 1992 atomic64_inc(&dev->stats.mmap_err); 1993 return -EINVAL; 1994 } 1995 entry = to_emmap(rdma_entry); 1996 1997 ibdev_dbg(&dev->ibdev, 1998 "Mapping address[%#llx], length[%#zx], mmap_flag[%d]\n", 1999 entry->address, rdma_entry->npages * PAGE_SIZE, 2000 entry->mmap_flag); 2001 2002 pfn = entry->address >> PAGE_SHIFT; 2003 switch (entry->mmap_flag) { 2004 case EFA_MMAP_IO_NC: 2005 err = rdma_user_mmap_io(&ucontext->ibucontext, vma, pfn, 2006 entry->rdma_entry.npages * PAGE_SIZE, 2007 pgprot_noncached(vma->vm_page_prot), 2008 rdma_entry); 2009 break; 2010 case EFA_MMAP_IO_WC: 2011 err = rdma_user_mmap_io(&ucontext->ibucontext, vma, pfn, 2012 entry->rdma_entry.npages * PAGE_SIZE, 2013 pgprot_writecombine(vma->vm_page_prot), 2014 rdma_entry); 2015 break; 2016 case EFA_MMAP_DMA_PAGE: 2017 for (va = vma->vm_start; va < vma->vm_end; 2018 va += PAGE_SIZE, pfn++) { 2019 err = vm_insert_page(vma, va, pfn_to_page(pfn)); 2020 if (err) 2021 break; 2022 } 2023 break; 2024 default: 2025 err = -EINVAL; 2026 } 2027 2028 if (err) { 2029 ibdev_dbg( 2030 &dev->ibdev, 2031 "Couldn't mmap address[%#llx] length[%#zx] mmap_flag[%d] err[%d]\n", 2032 entry->address, rdma_entry->npages * PAGE_SIZE, 2033 entry->mmap_flag, err); 2034 atomic64_inc(&dev->stats.mmap_err); 2035 } 2036 2037 rdma_user_mmap_entry_put(rdma_entry); 2038 return err; 2039 } 2040 2041 int efa_mmap(struct ib_ucontext *ibucontext, 2042 struct vm_area_struct *vma) 2043 { 2044 struct efa_ucontext *ucontext = to_eucontext(ibucontext); 2045 struct efa_dev *dev = to_edev(ibucontext->device); 2046 size_t length = vma->vm_end - vma->vm_start; 2047 2048 ibdev_dbg(&dev->ibdev, 2049 "start %#lx, end %#lx, length = %#zx, pgoff = %#lx\n", 2050 vma->vm_start, vma->vm_end, length, vma->vm_pgoff); 2051 2052 return __efa_mmap(dev, ucontext, vma); 2053 } 2054 2055 static int efa_ah_destroy(struct efa_dev *dev, struct efa_ah *ah) 2056 { 2057 struct efa_com_destroy_ah_params params = { 2058 .ah = ah->ah, 2059 .pdn = to_epd(ah->ibah.pd)->pdn, 2060 }; 2061 2062 return efa_com_destroy_ah(&dev->edev, ¶ms); 2063 } 2064 2065 int efa_create_ah(struct ib_ah *ibah, 2066 struct rdma_ah_init_attr *init_attr, 2067 struct ib_udata *udata) 2068 { 2069 struct rdma_ah_attr *ah_attr = init_attr->ah_attr; 2070 struct efa_dev *dev = to_edev(ibah->device); 2071 struct efa_com_create_ah_params params = {}; 2072 struct efa_ibv_create_ah_resp resp = {}; 2073 struct efa_com_create_ah_result result; 2074 struct efa_ah *ah = to_eah(ibah); 2075 int err; 2076 2077 if (!(init_attr->flags & RDMA_CREATE_AH_SLEEPABLE)) { 2078 ibdev_dbg(&dev->ibdev, 2079 "Create address handle is not supported in atomic context\n"); 2080 err = -EOPNOTSUPP; 2081 goto err_out; 2082 } 2083 2084 err = ib_is_udata_in_empty(udata); 2085 if (err) 2086 goto err_out; 2087 2088 memcpy(params.dest_addr, ah_attr->grh.dgid.raw, 2089 sizeof(params.dest_addr)); 2090 params.pdn = to_epd(ibah->pd)->pdn; 2091 err = efa_com_create_ah(&dev->edev, ¶ms, &result); 2092 if (err) 2093 goto err_out; 2094 2095 memcpy(ah->id, ah_attr->grh.dgid.raw, sizeof(ah->id)); 2096 ah->ah = result.ah; 2097 2098 resp.efa_address_handle = result.ah; 2099 2100 if (udata->outlen) { 2101 err = ib_respond_udata(udata, resp); 2102 if (err) 2103 goto err_destroy_ah; 2104 } 2105 ibdev_dbg(&dev->ibdev, "Created ah[%d]\n", ah->ah); 2106 2107 return 0; 2108 2109 err_destroy_ah: 2110 efa_ah_destroy(dev, ah); 2111 err_out: 2112 atomic64_inc(&dev->stats.create_ah_err); 2113 return err; 2114 } 2115 2116 int efa_destroy_ah(struct ib_ah *ibah, u32 flags) 2117 { 2118 struct efa_dev *dev = to_edev(ibah->pd->device); 2119 struct efa_ah *ah = to_eah(ibah); 2120 2121 ibdev_dbg(&dev->ibdev, "Destroy ah[%d]\n", ah->ah); 2122 2123 if (!(flags & RDMA_DESTROY_AH_SLEEPABLE)) { 2124 ibdev_dbg(&dev->ibdev, 2125 "Destroy address handle is not supported in atomic context\n"); 2126 return -EOPNOTSUPP; 2127 } 2128 2129 efa_ah_destroy(dev, ah); 2130 return 0; 2131 } 2132 2133 struct rdma_hw_stats *efa_alloc_hw_port_stats(struct ib_device *ibdev, 2134 u32 port_num) 2135 { 2136 return rdma_alloc_hw_stats_struct(efa_port_stats_descs, 2137 ARRAY_SIZE(efa_port_stats_descs), 2138 RDMA_HW_STATS_DEFAULT_LIFESPAN); 2139 } 2140 2141 struct rdma_hw_stats *efa_alloc_hw_device_stats(struct ib_device *ibdev) 2142 { 2143 return rdma_alloc_hw_stats_struct(efa_device_stats_descs, 2144 ARRAY_SIZE(efa_device_stats_descs), 2145 RDMA_HW_STATS_DEFAULT_LIFESPAN); 2146 } 2147 2148 static int efa_fill_device_stats(struct efa_dev *dev, 2149 struct rdma_hw_stats *stats) 2150 { 2151 struct efa_com_stats_admin *as = &dev->edev.aq.stats; 2152 struct efa_stats *s = &dev->stats; 2153 2154 stats->value[EFA_SUBMITTED_CMDS] = atomic64_read(&as->submitted_cmd); 2155 stats->value[EFA_COMPLETED_CMDS] = atomic64_read(&as->completed_cmd); 2156 stats->value[EFA_CMDS_ERR] = atomic64_read(&as->cmd_err); 2157 stats->value[EFA_NO_COMPLETION_CMDS] = atomic64_read(&as->no_completion); 2158 2159 stats->value[EFA_KEEP_ALIVE_RCVD] = atomic64_read(&s->keep_alive_rcvd); 2160 stats->value[EFA_ALLOC_PD_ERR] = atomic64_read(&s->alloc_pd_err); 2161 stats->value[EFA_CREATE_QP_ERR] = atomic64_read(&s->create_qp_err); 2162 stats->value[EFA_CREATE_CQ_ERR] = atomic64_read(&s->create_cq_err); 2163 stats->value[EFA_REG_MR_ERR] = atomic64_read(&s->reg_mr_err); 2164 stats->value[EFA_ALLOC_UCONTEXT_ERR] = 2165 atomic64_read(&s->alloc_ucontext_err); 2166 stats->value[EFA_CREATE_AH_ERR] = atomic64_read(&s->create_ah_err); 2167 stats->value[EFA_MMAP_ERR] = atomic64_read(&s->mmap_err); 2168 2169 return ARRAY_SIZE(efa_device_stats_descs); 2170 } 2171 2172 static int efa_fill_port_stats(struct efa_dev *dev, struct rdma_hw_stats *stats, 2173 u32 port_num) 2174 { 2175 struct efa_com_get_stats_params params = {}; 2176 union efa_com_get_stats_result result; 2177 struct efa_com_rdma_write_stats *rws; 2178 struct efa_com_rdma_read_stats *rrs; 2179 struct efa_com_messages_stats *ms; 2180 struct efa_com_network_stats *ns; 2181 struct efa_com_basic_stats *bs; 2182 int err; 2183 2184 params.scope = EFA_ADMIN_GET_STATS_SCOPE_ALL; 2185 params.type = EFA_ADMIN_GET_STATS_TYPE_BASIC; 2186 2187 err = efa_com_get_stats(&dev->edev, ¶ms, &result); 2188 if (err) 2189 return err; 2190 2191 bs = &result.basic_stats; 2192 stats->value[EFA_TX_BYTES] = bs->tx_bytes; 2193 stats->value[EFA_TX_PKTS] = bs->tx_pkts; 2194 stats->value[EFA_RX_BYTES] = bs->rx_bytes; 2195 stats->value[EFA_RX_PKTS] = bs->rx_pkts; 2196 stats->value[EFA_RX_DROPS] = bs->rx_drops; 2197 2198 params.type = EFA_ADMIN_GET_STATS_TYPE_MESSAGES; 2199 err = efa_com_get_stats(&dev->edev, ¶ms, &result); 2200 if (err) 2201 return err; 2202 2203 ms = &result.messages_stats; 2204 stats->value[EFA_SEND_BYTES] = ms->send_bytes; 2205 stats->value[EFA_SEND_WRS] = ms->send_wrs; 2206 stats->value[EFA_RECV_BYTES] = ms->recv_bytes; 2207 stats->value[EFA_RECV_WRS] = ms->recv_wrs; 2208 2209 params.type = EFA_ADMIN_GET_STATS_TYPE_RDMA_READ; 2210 err = efa_com_get_stats(&dev->edev, ¶ms, &result); 2211 if (err) 2212 return err; 2213 2214 rrs = &result.rdma_read_stats; 2215 stats->value[EFA_RDMA_READ_WRS] = rrs->read_wrs; 2216 stats->value[EFA_RDMA_READ_BYTES] = rrs->read_bytes; 2217 stats->value[EFA_RDMA_READ_WR_ERR] = rrs->read_wr_err; 2218 stats->value[EFA_RDMA_READ_RESP_BYTES] = rrs->read_resp_bytes; 2219 2220 if (EFA_DEV_CAP(dev, RDMA_WRITE)) { 2221 params.type = EFA_ADMIN_GET_STATS_TYPE_RDMA_WRITE; 2222 err = efa_com_get_stats(&dev->edev, ¶ms, &result); 2223 if (err) 2224 return err; 2225 2226 rws = &result.rdma_write_stats; 2227 stats->value[EFA_RDMA_WRITE_WRS] = rws->write_wrs; 2228 stats->value[EFA_RDMA_WRITE_BYTES] = rws->write_bytes; 2229 stats->value[EFA_RDMA_WRITE_WR_ERR] = rws->write_wr_err; 2230 stats->value[EFA_RDMA_WRITE_RECV_BYTES] = rws->write_recv_bytes; 2231 } 2232 2233 params.type = EFA_ADMIN_GET_STATS_TYPE_NETWORK; 2234 err = efa_com_get_stats(&dev->edev, ¶ms, &result); 2235 if (err) 2236 return err; 2237 2238 ns = &result.network_stats; 2239 stats->value[EFA_RETRANS_BYTES] = ns->retrans_bytes; 2240 stats->value[EFA_RETRANS_PKTS] = ns->retrans_pkts; 2241 stats->value[EFA_RETRANS_TIMEOUT_EVENS] = ns->retrans_timeout_events; 2242 stats->value[EFA_UNRESPONSIVE_REMOTE_EVENTS] = ns->unresponsive_remote_events; 2243 stats->value[EFA_IMPAIRED_REMOTE_CONN_EVENTS] = ns->impaired_remote_conn_events; 2244 2245 return ARRAY_SIZE(efa_port_stats_descs); 2246 } 2247 2248 int efa_get_hw_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats, 2249 u32 port_num, int index) 2250 { 2251 if (port_num) 2252 return efa_fill_port_stats(to_edev(ibdev), stats, port_num); 2253 else 2254 return efa_fill_device_stats(to_edev(ibdev), stats); 2255 } 2256 2257 enum rdma_link_layer efa_port_link_layer(struct ib_device *ibdev, 2258 u32 port_num) 2259 { 2260 return IB_LINK_LAYER_UNSPECIFIED; 2261 } 2262 2263 DECLARE_UVERBS_NAMED_METHOD(EFA_IB_METHOD_MR_QUERY, 2264 UVERBS_ATTR_IDR(EFA_IB_ATTR_QUERY_MR_HANDLE, 2265 UVERBS_OBJECT_MR, 2266 UVERBS_ACCESS_READ, 2267 UA_MANDATORY), 2268 UVERBS_ATTR_PTR_OUT(EFA_IB_ATTR_QUERY_MR_RESP_IC_ID_VALIDITY, 2269 UVERBS_ATTR_TYPE(u16), 2270 UA_MANDATORY), 2271 UVERBS_ATTR_PTR_OUT(EFA_IB_ATTR_QUERY_MR_RESP_RECV_IC_ID, 2272 UVERBS_ATTR_TYPE(u16), 2273 UA_MANDATORY), 2274 UVERBS_ATTR_PTR_OUT(EFA_IB_ATTR_QUERY_MR_RESP_RDMA_READ_IC_ID, 2275 UVERBS_ATTR_TYPE(u16), 2276 UA_MANDATORY), 2277 UVERBS_ATTR_PTR_OUT(EFA_IB_ATTR_QUERY_MR_RESP_RDMA_RECV_IC_ID, 2278 UVERBS_ATTR_TYPE(u16), 2279 UA_MANDATORY)); 2280 2281 ADD_UVERBS_METHODS(efa_mr, 2282 UVERBS_OBJECT_MR, 2283 &UVERBS_METHOD(EFA_IB_METHOD_MR_QUERY)); 2284 2285 const struct uapi_definition efa_uapi_defs[] = { 2286 UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_MR, 2287 &efa_mr), 2288 {}, 2289 }; 2290