1 // SPDX-License-Identifier: GPL-2.0
2
3 /* Authors: Cheng Xu <chengyou@linux.alibaba.com> */
4 /* Kai Shen <kaishen@linux.alibaba.com> */
5 /* Copyright (c) 2020-2022, Alibaba Group. */
6
7 /* Authors: Bernard Metzler <bmt@zurich.ibm.com> */
8 /* Copyright (c) 2008-2019, IBM Corporation */
9
10 /* Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved. */
11
12 #include <linux/vmalloc.h>
13 #include <net/addrconf.h>
14 #include <rdma/erdma-abi.h>
15 #include <rdma/ib_umem.h>
16 #include <rdma/uverbs_ioctl.h>
17
18 #include "erdma.h"
19 #include "erdma_cm.h"
20 #include "erdma_verbs.h"
21
assemble_qbuf_mtt_for_cmd(struct erdma_mem * mem,u32 * cfg,u64 * addr0,u64 * addr1)22 static void assemble_qbuf_mtt_for_cmd(struct erdma_mem *mem, u32 *cfg,
23 u64 *addr0, u64 *addr1)
24 {
25 struct erdma_mtt *mtt = mem->mtt;
26
27 if (mem->mtt_nents > ERDMA_MAX_INLINE_MTT_ENTRIES) {
28 *addr0 = mtt->buf_dma;
29 *cfg |= FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_LEVEL_MASK,
30 ERDMA_MR_MTT_1LEVEL);
31 } else {
32 *addr0 = mtt->buf[0];
33 memcpy(addr1, mtt->buf + 1, MTT_SIZE(mem->mtt_nents - 1));
34 *cfg |= FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_LEVEL_MASK,
35 ERDMA_MR_MTT_0LEVEL);
36 }
37 }
38
create_qp_cmd(struct erdma_ucontext * uctx,struct erdma_qp * qp)39 static int create_qp_cmd(struct erdma_ucontext *uctx, struct erdma_qp *qp)
40 {
41 struct erdma_dev *dev = to_edev(qp->ibqp.device);
42 struct erdma_pd *pd = to_epd(qp->ibqp.pd);
43 struct erdma_cmdq_create_qp_req req;
44 struct erdma_uqp *user_qp;
45 u64 resp0, resp1;
46 int err;
47
48 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
49 CMDQ_OPCODE_CREATE_QP);
50
51 req.cfg0 = FIELD_PREP(ERDMA_CMD_CREATE_QP_SQ_DEPTH_MASK,
52 ilog2(qp->attrs.sq_size)) |
53 FIELD_PREP(ERDMA_CMD_CREATE_QP_QPN_MASK, QP_ID(qp));
54 req.cfg1 = FIELD_PREP(ERDMA_CMD_CREATE_QP_RQ_DEPTH_MASK,
55 ilog2(qp->attrs.rq_size)) |
56 FIELD_PREP(ERDMA_CMD_CREATE_QP_PD_MASK, pd->pdn);
57
58 if (qp->ibqp.qp_type == IB_QPT_RC)
59 req.cfg2 = FIELD_PREP(ERDMA_CMD_CREATE_QP_TYPE_MASK,
60 ERDMA_QPT_RC);
61 else
62 req.cfg2 = FIELD_PREP(ERDMA_CMD_CREATE_QP_TYPE_MASK,
63 ERDMA_QPT_UD);
64
65 if (rdma_is_kernel_res(&qp->ibqp.res)) {
66 u32 pgsz_range = ilog2(SZ_1M) - ERDMA_HW_PAGE_SHIFT;
67
68 req.sq_cqn_mtt_cfg =
69 FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,
70 pgsz_range) |
71 FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->scq->cqn);
72 req.rq_cqn_mtt_cfg =
73 FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,
74 pgsz_range) |
75 FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->rcq->cqn);
76
77 req.sq_mtt_cfg =
78 FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_OFFSET_MASK, 0) |
79 FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK, 1) |
80 FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_LEVEL_MASK,
81 ERDMA_MR_MTT_0LEVEL);
82 req.rq_mtt_cfg = req.sq_mtt_cfg;
83
84 req.rq_buf_addr = qp->kern_qp.rq_buf_dma_addr;
85 req.sq_buf_addr = qp->kern_qp.sq_buf_dma_addr;
86 req.sq_dbrec_dma = qp->kern_qp.sq_dbrec_dma;
87 req.rq_dbrec_dma = qp->kern_qp.rq_dbrec_dma;
88 } else {
89 user_qp = &qp->user_qp;
90 req.sq_cqn_mtt_cfg = FIELD_PREP(
91 ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,
92 ilog2(user_qp->sq_mem.page_size) - ERDMA_HW_PAGE_SHIFT);
93 req.sq_cqn_mtt_cfg |=
94 FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->scq->cqn);
95
96 req.rq_cqn_mtt_cfg = FIELD_PREP(
97 ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,
98 ilog2(user_qp->rq_mem.page_size) - ERDMA_HW_PAGE_SHIFT);
99 req.rq_cqn_mtt_cfg |=
100 FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->rcq->cqn);
101
102 req.sq_mtt_cfg = user_qp->sq_mem.page_offset;
103 req.sq_mtt_cfg |= FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK,
104 user_qp->sq_mem.mtt_nents);
105
106 req.rq_mtt_cfg = user_qp->rq_mem.page_offset;
107 req.rq_mtt_cfg |= FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK,
108 user_qp->rq_mem.mtt_nents);
109
110 assemble_qbuf_mtt_for_cmd(&user_qp->sq_mem, &req.sq_mtt_cfg,
111 &req.sq_buf_addr, req.sq_mtt_entry);
112 assemble_qbuf_mtt_for_cmd(&user_qp->rq_mem, &req.rq_mtt_cfg,
113 &req.rq_buf_addr, req.rq_mtt_entry);
114
115 req.sq_dbrec_dma = user_qp->sq_dbrec_dma;
116 req.rq_dbrec_dma = user_qp->rq_dbrec_dma;
117
118 if (uctx->ext_db.enable) {
119 req.sq_cqn_mtt_cfg |=
120 FIELD_PREP(ERDMA_CMD_CREATE_QP_DB_CFG_MASK, 1);
121 req.db_cfg =
122 FIELD_PREP(ERDMA_CMD_CREATE_QP_SQDB_CFG_MASK,
123 uctx->ext_db.sdb_off) |
124 FIELD_PREP(ERDMA_CMD_CREATE_QP_RQDB_CFG_MASK,
125 uctx->ext_db.rdb_off);
126 }
127 }
128
129 err = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), &resp0, &resp1,
130 true);
131 if (!err && erdma_device_iwarp(dev))
132 qp->attrs.iwarp.cookie =
133 FIELD_GET(ERDMA_CMDQ_CREATE_QP_RESP_COOKIE_MASK, resp0);
134
135 return err;
136 }
137
regmr_cmd(struct erdma_dev * dev,struct erdma_mr * mr)138 static int regmr_cmd(struct erdma_dev *dev, struct erdma_mr *mr)
139 {
140 struct erdma_pd *pd = to_epd(mr->ibmr.pd);
141 u32 mtt_level = ERDMA_MR_MTT_0LEVEL;
142 struct erdma_cmdq_reg_mr_req req;
143
144 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA, CMDQ_OPCODE_REG_MR);
145
146 if (mr->type == ERDMA_MR_TYPE_FRMR ||
147 mr->mem.page_cnt > ERDMA_MAX_INLINE_MTT_ENTRIES) {
148 if (mr->mem.mtt->continuous) {
149 req.phy_addr[0] = mr->mem.mtt->buf_dma;
150 mtt_level = ERDMA_MR_MTT_1LEVEL;
151 } else {
152 req.phy_addr[0] = mr->mem.mtt->dma_addrs[0];
153 mtt_level = mr->mem.mtt->level;
154 }
155 } else if (mr->type != ERDMA_MR_TYPE_DMA) {
156 memcpy(req.phy_addr, mr->mem.mtt->buf,
157 MTT_SIZE(mr->mem.page_cnt));
158 }
159
160 req.cfg0 = FIELD_PREP(ERDMA_CMD_MR_VALID_MASK, mr->valid) |
161 FIELD_PREP(ERDMA_CMD_MR_KEY_MASK, mr->ibmr.lkey & 0xFF) |
162 FIELD_PREP(ERDMA_CMD_MR_MPT_IDX_MASK, mr->ibmr.lkey >> 8);
163 req.cfg1 = FIELD_PREP(ERDMA_CMD_REGMR_PD_MASK, pd->pdn) |
164 FIELD_PREP(ERDMA_CMD_REGMR_TYPE_MASK, mr->type) |
165 FIELD_PREP(ERDMA_CMD_REGMR_RIGHT_MASK, mr->access);
166 req.cfg2 = FIELD_PREP(ERDMA_CMD_REGMR_PAGESIZE_MASK,
167 ilog2(mr->mem.page_size)) |
168 FIELD_PREP(ERDMA_CMD_REGMR_MTT_LEVEL_MASK, mtt_level) |
169 FIELD_PREP(ERDMA_CMD_REGMR_MTT_CNT_MASK, mr->mem.page_cnt);
170
171 if (mr->type == ERDMA_MR_TYPE_DMA)
172 goto post_cmd;
173
174 if (mr->type == ERDMA_MR_TYPE_NORMAL) {
175 req.start_va = mr->mem.va;
176 req.size = mr->mem.len;
177 }
178
179 if (!mr->mem.mtt->continuous && mr->mem.mtt->level > 1) {
180 req.cfg0 |= FIELD_PREP(ERDMA_CMD_MR_VERSION_MASK, 1);
181 req.cfg2 |= FIELD_PREP(ERDMA_CMD_REGMR_MTT_PAGESIZE_MASK,
182 PAGE_SHIFT - ERDMA_HW_PAGE_SHIFT);
183 req.size_h = upper_32_bits(mr->mem.len);
184 req.mtt_cnt_h = mr->mem.page_cnt >> 20;
185 }
186
187 post_cmd:
188 return erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL,
189 true);
190 }
191
create_cq_cmd(struct erdma_ucontext * uctx,struct erdma_cq * cq)192 static int create_cq_cmd(struct erdma_ucontext *uctx, struct erdma_cq *cq)
193 {
194 struct erdma_dev *dev = to_edev(cq->ibcq.device);
195 struct erdma_cmdq_create_cq_req req;
196 struct erdma_mem *mem;
197 u32 page_size;
198
199 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
200 CMDQ_OPCODE_CREATE_CQ);
201
202 req.cfg0 = FIELD_PREP(ERDMA_CMD_CREATE_CQ_CQN_MASK, cq->cqn) |
203 FIELD_PREP(ERDMA_CMD_CREATE_CQ_DEPTH_MASK, ilog2(cq->depth));
204 req.cfg1 = FIELD_PREP(ERDMA_CMD_CREATE_CQ_EQN_MASK, cq->assoc_eqn);
205
206 if (rdma_is_kernel_res(&cq->ibcq.res)) {
207 page_size = SZ_32M;
208 req.cfg0 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_PAGESIZE_MASK,
209 ilog2(page_size) - ERDMA_HW_PAGE_SHIFT);
210 req.qbuf_addr_l = lower_32_bits(cq->kern_cq.qbuf_dma_addr);
211 req.qbuf_addr_h = upper_32_bits(cq->kern_cq.qbuf_dma_addr);
212
213 req.cfg1 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_CNT_MASK, 1) |
214 FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_LEVEL_MASK,
215 ERDMA_MR_MTT_0LEVEL);
216
217 req.first_page_offset = 0;
218 req.cq_dbrec_dma = cq->kern_cq.dbrec_dma;
219 } else {
220 mem = &cq->user_cq.qbuf_mem;
221 req.cfg0 |=
222 FIELD_PREP(ERDMA_CMD_CREATE_CQ_PAGESIZE_MASK,
223 ilog2(mem->page_size) - ERDMA_HW_PAGE_SHIFT);
224 if (mem->mtt_nents == 1) {
225 req.qbuf_addr_l = lower_32_bits(mem->mtt->buf[0]);
226 req.qbuf_addr_h = upper_32_bits(mem->mtt->buf[0]);
227 req.cfg1 |=
228 FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_LEVEL_MASK,
229 ERDMA_MR_MTT_0LEVEL);
230 } else {
231 req.qbuf_addr_l = lower_32_bits(mem->mtt->buf_dma);
232 req.qbuf_addr_h = upper_32_bits(mem->mtt->buf_dma);
233 req.cfg1 |=
234 FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_LEVEL_MASK,
235 ERDMA_MR_MTT_1LEVEL);
236 }
237 req.cfg1 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_CNT_MASK,
238 mem->mtt_nents);
239
240 req.first_page_offset = mem->page_offset;
241 req.cq_dbrec_dma = cq->user_cq.dbrec_dma;
242
243 if (uctx->ext_db.enable) {
244 req.cfg1 |= FIELD_PREP(
245 ERDMA_CMD_CREATE_CQ_MTT_DB_CFG_MASK, 1);
246 req.cfg2 = FIELD_PREP(ERDMA_CMD_CREATE_CQ_DB_CFG_MASK,
247 uctx->ext_db.cdb_off);
248 }
249 }
250
251 return erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL,
252 true);
253 }
254
erdma_alloc_idx(struct erdma_resource_cb * res_cb)255 static int erdma_alloc_idx(struct erdma_resource_cb *res_cb)
256 {
257 int idx;
258 unsigned long flags;
259
260 spin_lock_irqsave(&res_cb->lock, flags);
261 idx = find_next_zero_bit(res_cb->bitmap, res_cb->max_cap,
262 res_cb->next_alloc_idx);
263 if (idx == res_cb->max_cap) {
264 idx = find_first_zero_bit(res_cb->bitmap, res_cb->max_cap);
265 if (idx == res_cb->max_cap) {
266 res_cb->next_alloc_idx = 1;
267 spin_unlock_irqrestore(&res_cb->lock, flags);
268 return -ENOSPC;
269 }
270 }
271
272 set_bit(idx, res_cb->bitmap);
273 res_cb->next_alloc_idx = idx + 1;
274 spin_unlock_irqrestore(&res_cb->lock, flags);
275
276 return idx;
277 }
278
erdma_free_idx(struct erdma_resource_cb * res_cb,u32 idx)279 static inline void erdma_free_idx(struct erdma_resource_cb *res_cb, u32 idx)
280 {
281 unsigned long flags;
282 u32 used;
283
284 spin_lock_irqsave(&res_cb->lock, flags);
285 used = __test_and_clear_bit(idx, res_cb->bitmap);
286 spin_unlock_irqrestore(&res_cb->lock, flags);
287 WARN_ON(!used);
288 }
289
290 static struct rdma_user_mmap_entry *
erdma_user_mmap_entry_insert(struct erdma_ucontext * uctx,void * address,u32 size,u8 mmap_flag,u64 * mmap_offset)291 erdma_user_mmap_entry_insert(struct erdma_ucontext *uctx, void *address,
292 u32 size, u8 mmap_flag, u64 *mmap_offset)
293 {
294 struct erdma_user_mmap_entry *entry = kzalloc_obj(*entry);
295 int ret;
296
297 if (!entry)
298 return NULL;
299
300 entry->address = (u64)address;
301 entry->mmap_flag = mmap_flag;
302
303 size = PAGE_ALIGN(size);
304
305 ret = rdma_user_mmap_entry_insert(&uctx->ibucontext, &entry->rdma_entry,
306 size);
307 if (ret) {
308 kfree(entry);
309 return NULL;
310 }
311
312 *mmap_offset = rdma_user_mmap_get_offset(&entry->rdma_entry);
313
314 return &entry->rdma_entry;
315 }
316
erdma_query_device(struct ib_device * ibdev,struct ib_device_attr * attr,struct ib_udata * unused)317 int erdma_query_device(struct ib_device *ibdev, struct ib_device_attr *attr,
318 struct ib_udata *unused)
319 {
320 struct erdma_dev *dev = to_edev(ibdev);
321
322 memset(attr, 0, sizeof(*attr));
323
324 attr->max_mr_size = dev->attrs.max_mr_size;
325 attr->vendor_id = PCI_VENDOR_ID_ALIBABA;
326 attr->vendor_part_id = dev->pdev->device;
327 attr->hw_ver = dev->pdev->revision;
328 attr->max_qp = dev->attrs.max_qp - 1;
329 attr->max_qp_wr = min(dev->attrs.max_send_wr, dev->attrs.max_recv_wr);
330 attr->max_qp_rd_atom = dev->attrs.max_ord;
331 attr->max_qp_init_rd_atom = dev->attrs.max_ird;
332 attr->max_res_rd_atom = dev->attrs.max_qp * dev->attrs.max_ird;
333 attr->device_cap_flags = IB_DEVICE_MEM_MGT_EXTENSIONS;
334 attr->kernel_cap_flags = IBK_LOCAL_DMA_LKEY;
335 ibdev->local_dma_lkey = dev->attrs.local_dma_key;
336 attr->max_send_sge = dev->attrs.max_send_sge;
337 attr->max_recv_sge = dev->attrs.max_recv_sge;
338 attr->max_sge_rd = dev->attrs.max_sge_rd;
339 attr->max_cq = dev->attrs.max_cq - 1;
340 attr->max_cqe = dev->attrs.max_cqe;
341 attr->max_mr = dev->attrs.max_mr;
342 attr->max_pd = dev->attrs.max_pd;
343 attr->max_mw = dev->attrs.max_mw;
344 attr->max_fast_reg_page_list_len = ERDMA_MAX_FRMR_PA;
345 attr->page_size_cap = ERDMA_PAGE_SIZE_SUPPORT;
346
347 if (erdma_device_rocev2(dev)) {
348 attr->max_pkeys = ERDMA_MAX_PKEYS;
349 attr->max_ah = dev->attrs.max_ah;
350 }
351
352 if (dev->attrs.cap_flags & ERDMA_DEV_CAP_FLAGS_ATOMIC)
353 attr->atomic_cap = IB_ATOMIC_GLOB;
354
355 attr->fw_ver = dev->attrs.fw_version;
356
357 if (dev->netdev)
358 addrconf_addr_eui48((u8 *)&attr->sys_image_guid,
359 dev->netdev->dev_addr);
360
361 return 0;
362 }
363
erdma_query_gid(struct ib_device * ibdev,u32 port,int idx,union ib_gid * gid)364 int erdma_query_gid(struct ib_device *ibdev, u32 port, int idx,
365 union ib_gid *gid)
366 {
367 struct erdma_dev *dev = to_edev(ibdev);
368
369 memset(gid, 0, sizeof(*gid));
370 ether_addr_copy(gid->raw, dev->attrs.peer_addr);
371
372 return 0;
373 }
374
erdma_query_port(struct ib_device * ibdev,u32 port,struct ib_port_attr * attr)375 int erdma_query_port(struct ib_device *ibdev, u32 port,
376 struct ib_port_attr *attr)
377 {
378 struct erdma_dev *dev = to_edev(ibdev);
379 struct net_device *ndev = dev->netdev;
380
381 memset(attr, 0, sizeof(*attr));
382
383 if (erdma_device_iwarp(dev)) {
384 attr->gid_tbl_len = 1;
385 } else {
386 attr->gid_tbl_len = dev->attrs.max_gid;
387 attr->ip_gids = true;
388 attr->pkey_tbl_len = ERDMA_MAX_PKEYS;
389 }
390
391 attr->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_DEVICE_MGMT_SUP;
392 attr->max_msg_sz = -1;
393
394 if (!ndev)
395 goto out;
396
397 ib_get_eth_speed(ibdev, port, &attr->active_speed, &attr->active_width);
398 attr->max_mtu = ib_mtu_int_to_enum(ndev->mtu);
399 attr->active_mtu = ib_mtu_int_to_enum(ndev->mtu);
400 attr->state = ib_get_curr_port_state(ndev);
401
402 out:
403 if (attr->state == IB_PORT_ACTIVE)
404 attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP;
405 else
406 attr->phys_state = IB_PORT_PHYS_STATE_DISABLED;
407
408 return 0;
409 }
410
erdma_get_port_immutable(struct ib_device * ibdev,u32 port,struct ib_port_immutable * port_immutable)411 int erdma_get_port_immutable(struct ib_device *ibdev, u32 port,
412 struct ib_port_immutable *port_immutable)
413 {
414 struct erdma_dev *dev = to_edev(ibdev);
415
416 if (erdma_device_iwarp(dev)) {
417 port_immutable->core_cap_flags = RDMA_CORE_PORT_IWARP;
418 port_immutable->gid_tbl_len = 1;
419 } else {
420 port_immutable->core_cap_flags =
421 RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
422 port_immutable->max_mad_size = IB_MGMT_MAD_SIZE;
423 port_immutable->gid_tbl_len = dev->attrs.max_gid;
424 port_immutable->pkey_tbl_len = ERDMA_MAX_PKEYS;
425 }
426
427 return 0;
428 }
429
erdma_alloc_pd(struct ib_pd * ibpd,struct ib_udata * udata)430 int erdma_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
431 {
432 struct erdma_pd *pd = to_epd(ibpd);
433 struct erdma_dev *dev = to_edev(ibpd->device);
434 int pdn;
435
436 pdn = erdma_alloc_idx(&dev->res_cb[ERDMA_RES_TYPE_PD]);
437 if (pdn < 0)
438 return pdn;
439
440 pd->pdn = pdn;
441
442 return 0;
443 }
444
erdma_dealloc_pd(struct ib_pd * ibpd,struct ib_udata * udata)445 int erdma_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
446 {
447 struct erdma_pd *pd = to_epd(ibpd);
448 struct erdma_dev *dev = to_edev(ibpd->device);
449
450 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_PD], pd->pdn);
451
452 return 0;
453 }
454
erdma_flush_worker(struct work_struct * work)455 static void erdma_flush_worker(struct work_struct *work)
456 {
457 struct delayed_work *dwork = to_delayed_work(work);
458 struct erdma_qp *qp =
459 container_of(dwork, struct erdma_qp, reflush_dwork);
460 struct erdma_cmdq_reflush_req req;
461
462 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
463 CMDQ_OPCODE_REFLUSH);
464 req.qpn = QP_ID(qp);
465 req.sq_pi = qp->kern_qp.sq_pi;
466 req.rq_pi = qp->kern_qp.rq_pi;
467 erdma_post_cmd_wait(&qp->dev->cmdq, &req, sizeof(req), NULL, NULL,
468 true);
469 }
470
erdma_qp_validate_cap(struct erdma_dev * dev,struct ib_qp_init_attr * attrs)471 static int erdma_qp_validate_cap(struct erdma_dev *dev,
472 struct ib_qp_init_attr *attrs)
473 {
474 if ((attrs->cap.max_send_wr > dev->attrs.max_send_wr) ||
475 (attrs->cap.max_recv_wr > dev->attrs.max_recv_wr) ||
476 (attrs->cap.max_send_sge > dev->attrs.max_send_sge) ||
477 (attrs->cap.max_recv_sge > dev->attrs.max_recv_sge) ||
478 (attrs->cap.max_inline_data > ERDMA_MAX_INLINE) ||
479 !attrs->cap.max_send_wr || !attrs->cap.max_recv_wr) {
480 return -EINVAL;
481 }
482
483 return 0;
484 }
485
erdma_qp_validate_attr(struct erdma_dev * dev,struct ib_qp_init_attr * attrs)486 static int erdma_qp_validate_attr(struct erdma_dev *dev,
487 struct ib_qp_init_attr *attrs)
488 {
489 if (erdma_device_iwarp(dev) && attrs->qp_type != IB_QPT_RC)
490 return -EOPNOTSUPP;
491
492 if (erdma_device_rocev2(dev) && attrs->qp_type != IB_QPT_RC &&
493 attrs->qp_type != IB_QPT_UD && attrs->qp_type != IB_QPT_GSI)
494 return -EOPNOTSUPP;
495
496 if (attrs->srq)
497 return -EOPNOTSUPP;
498
499 if (!attrs->send_cq || !attrs->recv_cq)
500 return -EOPNOTSUPP;
501
502 return 0;
503 }
504
free_kernel_qp(struct erdma_qp * qp)505 static void free_kernel_qp(struct erdma_qp *qp)
506 {
507 struct erdma_dev *dev = qp->dev;
508
509 vfree(qp->kern_qp.swr_tbl);
510 vfree(qp->kern_qp.rwr_tbl);
511
512 if (qp->kern_qp.sq_buf)
513 dma_free_coherent(&dev->pdev->dev,
514 qp->attrs.sq_size << SQEBB_SHIFT,
515 qp->kern_qp.sq_buf,
516 qp->kern_qp.sq_buf_dma_addr);
517
518 if (qp->kern_qp.sq_dbrec)
519 dma_pool_free(dev->db_pool, qp->kern_qp.sq_dbrec,
520 qp->kern_qp.sq_dbrec_dma);
521
522 if (qp->kern_qp.rq_buf)
523 dma_free_coherent(&dev->pdev->dev,
524 qp->attrs.rq_size << RQE_SHIFT,
525 qp->kern_qp.rq_buf,
526 qp->kern_qp.rq_buf_dma_addr);
527
528 if (qp->kern_qp.rq_dbrec)
529 dma_pool_free(dev->db_pool, qp->kern_qp.rq_dbrec,
530 qp->kern_qp.rq_dbrec_dma);
531 }
532
init_kernel_qp(struct erdma_dev * dev,struct erdma_qp * qp,struct ib_qp_init_attr * attrs)533 static int init_kernel_qp(struct erdma_dev *dev, struct erdma_qp *qp,
534 struct ib_qp_init_attr *attrs)
535 {
536 struct erdma_kqp *kqp = &qp->kern_qp;
537 int size;
538
539 if (attrs->sq_sig_type == IB_SIGNAL_ALL_WR)
540 kqp->sig_all = 1;
541
542 kqp->sq_pi = 0;
543 kqp->sq_ci = 0;
544 kqp->rq_pi = 0;
545 kqp->rq_ci = 0;
546 kqp->hw_sq_db =
547 dev->func_bar + (ERDMA_SDB_SHARED_PAGE_INDEX << PAGE_SHIFT);
548 kqp->hw_rq_db = dev->func_bar + ERDMA_BAR_RQDB_SPACE_OFFSET;
549
550 kqp->swr_tbl = vmalloc_array(qp->attrs.sq_size, sizeof(u64));
551 kqp->rwr_tbl = vmalloc_array(qp->attrs.rq_size, sizeof(u64));
552 if (!kqp->swr_tbl || !kqp->rwr_tbl)
553 goto err_out;
554
555 size = qp->attrs.sq_size << SQEBB_SHIFT;
556 kqp->sq_buf = dma_alloc_coherent(&dev->pdev->dev, size,
557 &kqp->sq_buf_dma_addr, GFP_KERNEL);
558 if (!kqp->sq_buf)
559 goto err_out;
560
561 kqp->sq_dbrec =
562 dma_pool_zalloc(dev->db_pool, GFP_KERNEL, &kqp->sq_dbrec_dma);
563 if (!kqp->sq_dbrec)
564 goto err_out;
565
566 size = qp->attrs.rq_size << RQE_SHIFT;
567 kqp->rq_buf = dma_alloc_coherent(&dev->pdev->dev, size,
568 &kqp->rq_buf_dma_addr, GFP_KERNEL);
569 if (!kqp->rq_buf)
570 goto err_out;
571
572 kqp->rq_dbrec =
573 dma_pool_zalloc(dev->db_pool, GFP_KERNEL, &kqp->rq_dbrec_dma);
574 if (!kqp->rq_dbrec)
575 goto err_out;
576
577 return 0;
578
579 err_out:
580 free_kernel_qp(qp);
581 return -ENOMEM;
582 }
583
erdma_fill_bottom_mtt(struct erdma_dev * dev,struct erdma_mem * mem)584 static void erdma_fill_bottom_mtt(struct erdma_dev *dev, struct erdma_mem *mem)
585 {
586 struct erdma_mtt *mtt = mem->mtt;
587 struct ib_block_iter biter;
588 u32 idx = 0;
589
590 while (mtt->low_level)
591 mtt = mtt->low_level;
592
593 rdma_umem_for_each_dma_block(mem->umem, &biter, mem->page_size)
594 mtt->buf[idx++] = rdma_block_iter_dma_address(&biter);
595 }
596
erdma_create_cont_mtt(struct erdma_dev * dev,size_t size)597 static struct erdma_mtt *erdma_create_cont_mtt(struct erdma_dev *dev,
598 size_t size)
599 {
600 struct erdma_mtt *mtt;
601
602 mtt = kzalloc_obj(*mtt);
603 if (!mtt)
604 return ERR_PTR(-ENOMEM);
605
606 mtt->size = size;
607 mtt->buf = kzalloc(mtt->size, GFP_KERNEL);
608 if (!mtt->buf)
609 goto err_free_mtt;
610
611 mtt->continuous = true;
612 mtt->buf_dma = dma_map_single(&dev->pdev->dev, mtt->buf, mtt->size,
613 DMA_TO_DEVICE);
614 if (dma_mapping_error(&dev->pdev->dev, mtt->buf_dma))
615 goto err_free_mtt_buf;
616
617 return mtt;
618
619 err_free_mtt_buf:
620 kfree(mtt->buf);
621
622 err_free_mtt:
623 kfree(mtt);
624
625 return ERR_PTR(-ENOMEM);
626 }
627
erdma_unmap_page_list(struct erdma_dev * dev,dma_addr_t * pg_dma,u32 npages)628 static void erdma_unmap_page_list(struct erdma_dev *dev, dma_addr_t *pg_dma,
629 u32 npages)
630 {
631 u32 i;
632
633 for (i = 0; i < npages; i++)
634 dma_unmap_page(&dev->pdev->dev, pg_dma[i], PAGE_SIZE,
635 DMA_TO_DEVICE);
636 }
637
erdma_destroy_mtt_buf_dma_addrs(struct erdma_dev * dev,struct erdma_mtt * mtt)638 static void erdma_destroy_mtt_buf_dma_addrs(struct erdma_dev *dev,
639 struct erdma_mtt *mtt)
640 {
641 erdma_unmap_page_list(dev, mtt->dma_addrs, mtt->npages);
642 vfree(mtt->dma_addrs);
643 }
644
erdma_destroy_scatter_mtt(struct erdma_dev * dev,struct erdma_mtt * mtt)645 static void erdma_destroy_scatter_mtt(struct erdma_dev *dev,
646 struct erdma_mtt *mtt)
647 {
648 erdma_destroy_mtt_buf_dma_addrs(dev, mtt);
649 vfree(mtt->buf);
650 kfree(mtt);
651 }
652
erdma_init_middle_mtt(struct erdma_mtt * mtt,struct erdma_mtt * low_mtt)653 static void erdma_init_middle_mtt(struct erdma_mtt *mtt,
654 struct erdma_mtt *low_mtt)
655 {
656 dma_addr_t *pg_addr = mtt->buf;
657 u32 i;
658
659 for (i = 0; i < low_mtt->npages; i++)
660 pg_addr[i] = low_mtt->dma_addrs[i];
661 }
662
vmalloc_to_dma_addrs(struct erdma_dev * dev,dma_addr_t ** dma_addrs,void * buf,u64 len)663 static u32 vmalloc_to_dma_addrs(struct erdma_dev *dev, dma_addr_t **dma_addrs,
664 void *buf, u64 len)
665 {
666 dma_addr_t *pg_dma;
667 struct page *pg;
668 u32 npages, i;
669 void *addr;
670
671 npages = (PAGE_ALIGN((u64)buf + len) - PAGE_ALIGN_DOWN((u64)buf)) >>
672 PAGE_SHIFT;
673 pg_dma = vcalloc(npages, sizeof(*pg_dma));
674 if (!pg_dma)
675 return 0;
676
677 addr = buf;
678 for (i = 0; i < npages; i++) {
679 pg = vmalloc_to_page(addr);
680 if (!pg)
681 goto err;
682
683 pg_dma[i] = dma_map_page(&dev->pdev->dev, pg, 0, PAGE_SIZE,
684 DMA_TO_DEVICE);
685 if (dma_mapping_error(&dev->pdev->dev, pg_dma[i]))
686 goto err;
687
688 addr += PAGE_SIZE;
689 }
690
691 *dma_addrs = pg_dma;
692
693 return npages;
694 err:
695 erdma_unmap_page_list(dev, pg_dma, i);
696 vfree(pg_dma);
697
698 return 0;
699 }
700
erdma_create_mtt_buf_dma_addrs(struct erdma_dev * dev,struct erdma_mtt * mtt)701 static int erdma_create_mtt_buf_dma_addrs(struct erdma_dev *dev,
702 struct erdma_mtt *mtt)
703 {
704 dma_addr_t *addrs;
705 u32 npages;
706
707 /* Failed if buf is not page aligned */
708 if ((uintptr_t)mtt->buf & ~PAGE_MASK)
709 return -EINVAL;
710
711 npages = vmalloc_to_dma_addrs(dev, &addrs, mtt->buf, mtt->size);
712 if (!npages)
713 return -ENOMEM;
714
715 mtt->dma_addrs = addrs;
716 mtt->npages = npages;
717
718 return 0;
719 }
720
erdma_create_scatter_mtt(struct erdma_dev * dev,size_t size)721 static struct erdma_mtt *erdma_create_scatter_mtt(struct erdma_dev *dev,
722 size_t size)
723 {
724 struct erdma_mtt *mtt;
725 int ret = -ENOMEM;
726
727 mtt = kzalloc_obj(*mtt);
728 if (!mtt)
729 return ERR_PTR(-ENOMEM);
730
731 mtt->size = ALIGN(size, PAGE_SIZE);
732 mtt->buf = vzalloc(mtt->size);
733 mtt->continuous = false;
734 if (!mtt->buf)
735 goto err_free_mtt;
736
737 ret = erdma_create_mtt_buf_dma_addrs(dev, mtt);
738 if (ret)
739 goto err_free_mtt_buf;
740
741 ibdev_dbg(&dev->ibdev, "create scatter mtt, size:%lu, npages:%u\n",
742 mtt->size, mtt->npages);
743
744 return mtt;
745
746 err_free_mtt_buf:
747 vfree(mtt->buf);
748
749 err_free_mtt:
750 kfree(mtt);
751
752 return ERR_PTR(ret);
753 }
754
erdma_create_mtt(struct erdma_dev * dev,size_t size,bool force_continuous)755 static struct erdma_mtt *erdma_create_mtt(struct erdma_dev *dev, size_t size,
756 bool force_continuous)
757 {
758 struct erdma_mtt *mtt, *tmp_mtt;
759 int ret, level = 0;
760
761 ibdev_dbg(&dev->ibdev, "create_mtt, size:%lu, force cont:%d\n", size,
762 force_continuous);
763
764 if (!(dev->attrs.cap_flags & ERDMA_DEV_CAP_FLAGS_MTT_VA))
765 force_continuous = true;
766
767 if (force_continuous)
768 return erdma_create_cont_mtt(dev, size);
769
770 mtt = erdma_create_scatter_mtt(dev, size);
771 if (IS_ERR(mtt))
772 return mtt;
773 level = 1;
774
775 /* convergence the mtt table. */
776 while (mtt->npages != 1 && level <= 3) {
777 tmp_mtt = erdma_create_scatter_mtt(dev, MTT_SIZE(mtt->npages));
778 if (IS_ERR(tmp_mtt)) {
779 ret = PTR_ERR(tmp_mtt);
780 goto err_free_mtt;
781 }
782 erdma_init_middle_mtt(tmp_mtt, mtt);
783 tmp_mtt->low_level = mtt;
784 mtt = tmp_mtt;
785 level++;
786 }
787
788 if (level > 3) {
789 ret = -ENOMEM;
790 goto err_free_mtt;
791 }
792
793 mtt->level = level;
794 ibdev_dbg(&dev->ibdev, "top mtt: level:%d, dma_addr 0x%llx\n",
795 mtt->level, mtt->dma_addrs[0]);
796
797 return mtt;
798 err_free_mtt:
799 while (mtt) {
800 tmp_mtt = mtt->low_level;
801 erdma_destroy_scatter_mtt(dev, mtt);
802 mtt = tmp_mtt;
803 }
804
805 return ERR_PTR(ret);
806 }
807
erdma_destroy_mtt(struct erdma_dev * dev,struct erdma_mtt * mtt)808 static void erdma_destroy_mtt(struct erdma_dev *dev, struct erdma_mtt *mtt)
809 {
810 struct erdma_mtt *tmp_mtt;
811
812 if (mtt->continuous) {
813 dma_unmap_single(&dev->pdev->dev, mtt->buf_dma, mtt->size,
814 DMA_TO_DEVICE);
815 kfree(mtt->buf);
816 kfree(mtt);
817 } else {
818 while (mtt) {
819 tmp_mtt = mtt->low_level;
820 erdma_destroy_scatter_mtt(dev, mtt);
821 mtt = tmp_mtt;
822 }
823 }
824 }
825
get_mtt_entries(struct erdma_dev * dev,struct erdma_mem * mem,u64 start,u64 len,int access,u64 virt,unsigned long req_page_size,bool force_continuous)826 static int get_mtt_entries(struct erdma_dev *dev, struct erdma_mem *mem,
827 u64 start, u64 len, int access, u64 virt,
828 unsigned long req_page_size, bool force_continuous)
829 {
830 int ret = 0;
831
832 mem->umem = ib_umem_get(&dev->ibdev, start, len, access);
833 if (IS_ERR(mem->umem)) {
834 ret = PTR_ERR(mem->umem);
835 mem->umem = NULL;
836 return ret;
837 }
838
839 mem->va = virt;
840 mem->len = len;
841 mem->page_size = ib_umem_find_best_pgsz(mem->umem, req_page_size, virt);
842 mem->page_offset = start & (mem->page_size - 1);
843 mem->mtt_nents = ib_umem_num_dma_blocks(mem->umem, mem->page_size);
844 mem->page_cnt = mem->mtt_nents;
845 mem->mtt = erdma_create_mtt(dev, MTT_SIZE(mem->page_cnt),
846 force_continuous);
847 if (IS_ERR(mem->mtt)) {
848 ret = PTR_ERR(mem->mtt);
849 goto error_ret;
850 }
851
852 erdma_fill_bottom_mtt(dev, mem);
853
854 return 0;
855
856 error_ret:
857 if (mem->umem) {
858 ib_umem_release(mem->umem);
859 mem->umem = NULL;
860 }
861
862 return ret;
863 }
864
put_mtt_entries(struct erdma_dev * dev,struct erdma_mem * mem)865 static void put_mtt_entries(struct erdma_dev *dev, struct erdma_mem *mem)
866 {
867 if (mem->mtt)
868 erdma_destroy_mtt(dev, mem->mtt);
869
870 if (mem->umem) {
871 ib_umem_release(mem->umem);
872 mem->umem = NULL;
873 }
874 }
875
erdma_map_user_dbrecords(struct erdma_ucontext * ctx,u64 dbrecords_va,struct erdma_user_dbrecords_page ** dbr_page,dma_addr_t * dma_addr)876 static int erdma_map_user_dbrecords(struct erdma_ucontext *ctx,
877 u64 dbrecords_va,
878 struct erdma_user_dbrecords_page **dbr_page,
879 dma_addr_t *dma_addr)
880 {
881 struct erdma_user_dbrecords_page *page = NULL;
882 int rv = 0;
883
884 mutex_lock(&ctx->dbrecords_page_mutex);
885
886 list_for_each_entry(page, &ctx->dbrecords_page_list, list)
887 if (page->va == (dbrecords_va & PAGE_MASK))
888 goto found;
889
890 page = kmalloc_obj(*page);
891 if (!page) {
892 rv = -ENOMEM;
893 goto out;
894 }
895
896 page->va = (dbrecords_va & PAGE_MASK);
897 page->refcnt = 0;
898
899 page->umem = ib_umem_get(ctx->ibucontext.device,
900 dbrecords_va & PAGE_MASK, PAGE_SIZE, 0);
901 if (IS_ERR(page->umem)) {
902 rv = PTR_ERR(page->umem);
903 kfree(page);
904 goto out;
905 }
906
907 list_add(&page->list, &ctx->dbrecords_page_list);
908
909 found:
910 *dma_addr = sg_dma_address(page->umem->sgt_append.sgt.sgl) +
911 (dbrecords_va & ~PAGE_MASK);
912 *dbr_page = page;
913 page->refcnt++;
914
915 out:
916 mutex_unlock(&ctx->dbrecords_page_mutex);
917 return rv;
918 }
919
920 static void
erdma_unmap_user_dbrecords(struct erdma_ucontext * ctx,struct erdma_user_dbrecords_page ** dbr_page)921 erdma_unmap_user_dbrecords(struct erdma_ucontext *ctx,
922 struct erdma_user_dbrecords_page **dbr_page)
923 {
924 if (!ctx || !(*dbr_page))
925 return;
926
927 mutex_lock(&ctx->dbrecords_page_mutex);
928 if (--(*dbr_page)->refcnt == 0) {
929 list_del(&(*dbr_page)->list);
930 ib_umem_release((*dbr_page)->umem);
931 kfree(*dbr_page);
932 }
933
934 *dbr_page = NULL;
935 mutex_unlock(&ctx->dbrecords_page_mutex);
936 }
937
init_user_qp(struct erdma_qp * qp,struct erdma_ucontext * uctx,u64 va,u32 len,u64 dbrec_va)938 static int init_user_qp(struct erdma_qp *qp, struct erdma_ucontext *uctx,
939 u64 va, u32 len, u64 dbrec_va)
940 {
941 dma_addr_t dbrec_dma;
942 u32 rq_offset;
943 int ret;
944
945 if (len < (ALIGN(qp->attrs.sq_size * SQEBB_SIZE, ERDMA_HW_PAGE_SIZE) +
946 qp->attrs.rq_size * RQE_SIZE))
947 return -EINVAL;
948
949 ret = get_mtt_entries(qp->dev, &qp->user_qp.sq_mem, va,
950 qp->attrs.sq_size << SQEBB_SHIFT, 0, va,
951 (SZ_1M - SZ_4K), true);
952 if (ret)
953 return ret;
954
955 rq_offset = ALIGN(qp->attrs.sq_size << SQEBB_SHIFT, ERDMA_HW_PAGE_SIZE);
956 qp->user_qp.rq_offset = rq_offset;
957
958 ret = get_mtt_entries(qp->dev, &qp->user_qp.rq_mem, va + rq_offset,
959 qp->attrs.rq_size << RQE_SHIFT, 0, va + rq_offset,
960 (SZ_1M - SZ_4K), true);
961 if (ret)
962 goto put_sq_mtt;
963
964 ret = erdma_map_user_dbrecords(uctx, dbrec_va,
965 &qp->user_qp.user_dbr_page,
966 &dbrec_dma);
967 if (ret)
968 goto put_rq_mtt;
969
970 qp->user_qp.sq_dbrec_dma = dbrec_dma;
971 qp->user_qp.rq_dbrec_dma = dbrec_dma + ERDMA_DB_SIZE;
972
973 return 0;
974
975 put_rq_mtt:
976 put_mtt_entries(qp->dev, &qp->user_qp.rq_mem);
977
978 put_sq_mtt:
979 put_mtt_entries(qp->dev, &qp->user_qp.sq_mem);
980
981 return ret;
982 }
983
free_user_qp(struct erdma_qp * qp,struct erdma_ucontext * uctx)984 static void free_user_qp(struct erdma_qp *qp, struct erdma_ucontext *uctx)
985 {
986 put_mtt_entries(qp->dev, &qp->user_qp.sq_mem);
987 put_mtt_entries(qp->dev, &qp->user_qp.rq_mem);
988 erdma_unmap_user_dbrecords(uctx, &qp->user_qp.user_dbr_page);
989 }
990
erdma_create_qp(struct ib_qp * ibqp,struct ib_qp_init_attr * attrs,struct ib_udata * udata)991 int erdma_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs,
992 struct ib_udata *udata)
993 {
994 struct erdma_qp *qp = to_eqp(ibqp);
995 struct erdma_dev *dev = to_edev(ibqp->device);
996 struct erdma_ucontext *uctx = rdma_udata_to_drv_context(
997 udata, struct erdma_ucontext, ibucontext);
998 struct erdma_ureq_create_qp ureq;
999 struct erdma_uresp_create_qp uresp;
1000 void *old_entry;
1001 int ret = 0;
1002
1003 ret = erdma_qp_validate_cap(dev, attrs);
1004 if (ret)
1005 goto err_out;
1006
1007 ret = erdma_qp_validate_attr(dev, attrs);
1008 if (ret)
1009 goto err_out;
1010
1011 qp->scq = to_ecq(attrs->send_cq);
1012 qp->rcq = to_ecq(attrs->recv_cq);
1013 qp->dev = dev;
1014 qp->attrs.cc = dev->attrs.cc;
1015
1016 init_rwsem(&qp->state_lock);
1017 kref_init(&qp->ref);
1018 init_completion(&qp->safe_free);
1019
1020 if (qp->ibqp.qp_type == IB_QPT_GSI) {
1021 old_entry = xa_store(&dev->qp_xa, 1, qp, GFP_KERNEL);
1022 if (xa_is_err(old_entry))
1023 ret = xa_err(old_entry);
1024 else
1025 qp->ibqp.qp_num = 1;
1026 } else {
1027 ret = xa_alloc_cyclic(&dev->qp_xa, &qp->ibqp.qp_num, qp,
1028 XA_LIMIT(1, dev->attrs.max_qp - 1),
1029 &dev->next_alloc_qpn, GFP_KERNEL);
1030 }
1031
1032 if (ret < 0) {
1033 ret = -ENOMEM;
1034 goto err_out;
1035 }
1036
1037 qp->attrs.sq_size = roundup_pow_of_two(attrs->cap.max_send_wr *
1038 ERDMA_MAX_WQEBB_PER_SQE);
1039 qp->attrs.rq_size = roundup_pow_of_two(attrs->cap.max_recv_wr);
1040
1041 if (uctx) {
1042 ret = ib_copy_from_udata(&ureq, udata,
1043 min(sizeof(ureq), udata->inlen));
1044 if (ret)
1045 goto err_out_xa;
1046
1047 ret = init_user_qp(qp, uctx, ureq.qbuf_va, ureq.qbuf_len,
1048 ureq.db_record_va);
1049 if (ret)
1050 goto err_out_xa;
1051
1052 memset(&uresp, 0, sizeof(uresp));
1053
1054 uresp.num_sqe = qp->attrs.sq_size;
1055 uresp.num_rqe = qp->attrs.rq_size;
1056 uresp.qp_id = QP_ID(qp);
1057 uresp.rq_offset = qp->user_qp.rq_offset;
1058
1059 ret = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1060 if (ret)
1061 goto err_out_cmd;
1062 } else {
1063 ret = init_kernel_qp(dev, qp, attrs);
1064 if (ret)
1065 goto err_out_xa;
1066 }
1067
1068 qp->attrs.max_send_sge = attrs->cap.max_send_sge;
1069 qp->attrs.max_recv_sge = attrs->cap.max_recv_sge;
1070
1071 if (erdma_device_iwarp(qp->dev))
1072 qp->attrs.iwarp.state = ERDMA_QPS_IWARP_IDLE;
1073 else
1074 qp->attrs.rocev2.state = ERDMA_QPS_ROCEV2_RESET;
1075
1076 INIT_DELAYED_WORK(&qp->reflush_dwork, erdma_flush_worker);
1077
1078 ret = create_qp_cmd(uctx, qp);
1079 if (ret)
1080 goto err_out_cmd;
1081
1082 spin_lock_init(&qp->lock);
1083
1084 return 0;
1085
1086 err_out_cmd:
1087 if (uctx)
1088 free_user_qp(qp, uctx);
1089 else
1090 free_kernel_qp(qp);
1091 err_out_xa:
1092 xa_erase(&dev->qp_xa, QP_ID(qp));
1093 err_out:
1094 return ret;
1095 }
1096
erdma_create_stag(struct erdma_dev * dev,u32 * stag)1097 static int erdma_create_stag(struct erdma_dev *dev, u32 *stag)
1098 {
1099 int stag_idx;
1100
1101 stag_idx = erdma_alloc_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX]);
1102 if (stag_idx < 0)
1103 return stag_idx;
1104
1105 /* For now, we always let key field be zero. */
1106 *stag = (stag_idx << 8);
1107
1108 return 0;
1109 }
1110
erdma_get_dma_mr(struct ib_pd * ibpd,int acc)1111 struct ib_mr *erdma_get_dma_mr(struct ib_pd *ibpd, int acc)
1112 {
1113 struct erdma_dev *dev = to_edev(ibpd->device);
1114 struct erdma_mr *mr;
1115 u32 stag;
1116 int ret;
1117
1118 mr = kzalloc_obj(*mr);
1119 if (!mr)
1120 return ERR_PTR(-ENOMEM);
1121
1122 ret = erdma_create_stag(dev, &stag);
1123 if (ret)
1124 goto out_free;
1125
1126 mr->type = ERDMA_MR_TYPE_DMA;
1127
1128 mr->ibmr.lkey = stag;
1129 mr->ibmr.rkey = stag;
1130 mr->ibmr.pd = ibpd;
1131 mr->access = ERDMA_MR_ACC_LR | to_erdma_access_flags(acc);
1132 ret = regmr_cmd(dev, mr);
1133 if (ret)
1134 goto out_remove_stag;
1135
1136 return &mr->ibmr;
1137
1138 out_remove_stag:
1139 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX],
1140 mr->ibmr.lkey >> 8);
1141
1142 out_free:
1143 kfree(mr);
1144
1145 return ERR_PTR(ret);
1146 }
1147
erdma_ib_alloc_mr(struct ib_pd * ibpd,enum ib_mr_type mr_type,u32 max_num_sg)1148 struct ib_mr *erdma_ib_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
1149 u32 max_num_sg)
1150 {
1151 struct erdma_mr *mr;
1152 struct erdma_dev *dev = to_edev(ibpd->device);
1153 int ret;
1154 u32 stag;
1155
1156 if (mr_type != IB_MR_TYPE_MEM_REG)
1157 return ERR_PTR(-EOPNOTSUPP);
1158
1159 if (max_num_sg > ERDMA_MR_MAX_MTT_CNT)
1160 return ERR_PTR(-EINVAL);
1161
1162 mr = kzalloc_obj(*mr);
1163 if (!mr)
1164 return ERR_PTR(-ENOMEM);
1165
1166 ret = erdma_create_stag(dev, &stag);
1167 if (ret)
1168 goto out_free;
1169
1170 mr->type = ERDMA_MR_TYPE_FRMR;
1171
1172 mr->ibmr.lkey = stag;
1173 mr->ibmr.rkey = stag;
1174 mr->ibmr.pd = ibpd;
1175 /* update it in FRMR. */
1176 mr->access = ERDMA_MR_ACC_LR | ERDMA_MR_ACC_LW | ERDMA_MR_ACC_RR |
1177 ERDMA_MR_ACC_RW;
1178
1179 mr->mem.page_size = PAGE_SIZE; /* update it later. */
1180 mr->mem.page_cnt = max_num_sg;
1181 mr->mem.mtt = erdma_create_mtt(dev, MTT_SIZE(max_num_sg), true);
1182 if (IS_ERR(mr->mem.mtt)) {
1183 ret = PTR_ERR(mr->mem.mtt);
1184 goto out_remove_stag;
1185 }
1186
1187 ret = regmr_cmd(dev, mr);
1188 if (ret)
1189 goto out_destroy_mtt;
1190
1191 return &mr->ibmr;
1192
1193 out_destroy_mtt:
1194 erdma_destroy_mtt(dev, mr->mem.mtt);
1195
1196 out_remove_stag:
1197 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX],
1198 mr->ibmr.lkey >> 8);
1199
1200 out_free:
1201 kfree(mr);
1202
1203 return ERR_PTR(ret);
1204 }
1205
erdma_set_page(struct ib_mr * ibmr,u64 addr)1206 static int erdma_set_page(struct ib_mr *ibmr, u64 addr)
1207 {
1208 struct erdma_mr *mr = to_emr(ibmr);
1209
1210 if (mr->mem.mtt_nents >= mr->mem.page_cnt)
1211 return -1;
1212
1213 mr->mem.mtt->buf[mr->mem.mtt_nents] = addr;
1214 mr->mem.mtt_nents++;
1215
1216 return 0;
1217 }
1218
erdma_map_mr_sg(struct ib_mr * ibmr,struct scatterlist * sg,int sg_nents,unsigned int * sg_offset)1219 int erdma_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
1220 unsigned int *sg_offset)
1221 {
1222 struct erdma_mr *mr = to_emr(ibmr);
1223 int num;
1224
1225 mr->mem.mtt_nents = 0;
1226
1227 num = ib_sg_to_pages(&mr->ibmr, sg, sg_nents, sg_offset,
1228 erdma_set_page);
1229
1230 return num;
1231 }
1232
erdma_reg_user_mr(struct ib_pd * ibpd,u64 start,u64 len,u64 virt,int access,struct ib_dmah * dmah,struct ib_udata * udata)1233 struct ib_mr *erdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
1234 u64 virt, int access, struct ib_dmah *dmah,
1235 struct ib_udata *udata)
1236 {
1237 struct erdma_mr *mr = NULL;
1238 struct erdma_dev *dev = to_edev(ibpd->device);
1239 u32 stag;
1240 int ret;
1241
1242 if (dmah)
1243 return ERR_PTR(-EOPNOTSUPP);
1244
1245 if (!len || len > dev->attrs.max_mr_size)
1246 return ERR_PTR(-EINVAL);
1247
1248 mr = kzalloc_obj(*mr);
1249 if (!mr)
1250 return ERR_PTR(-ENOMEM);
1251
1252 ret = get_mtt_entries(dev, &mr->mem, start, len, access, virt,
1253 SZ_2G - SZ_4K, false);
1254 if (ret)
1255 goto err_out_free;
1256
1257 ret = erdma_create_stag(dev, &stag);
1258 if (ret)
1259 goto err_out_put_mtt;
1260
1261 mr->ibmr.lkey = mr->ibmr.rkey = stag;
1262 mr->ibmr.pd = ibpd;
1263 mr->mem.va = virt;
1264 mr->mem.len = len;
1265 mr->access = ERDMA_MR_ACC_LR | to_erdma_access_flags(access);
1266 mr->valid = 1;
1267 mr->type = ERDMA_MR_TYPE_NORMAL;
1268
1269 ret = regmr_cmd(dev, mr);
1270 if (ret)
1271 goto err_out_mr;
1272
1273 return &mr->ibmr;
1274
1275 err_out_mr:
1276 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX],
1277 mr->ibmr.lkey >> 8);
1278
1279 err_out_put_mtt:
1280 put_mtt_entries(dev, &mr->mem);
1281
1282 err_out_free:
1283 kfree(mr);
1284
1285 return ERR_PTR(ret);
1286 }
1287
erdma_dereg_mr(struct ib_mr * ibmr,struct ib_udata * udata)1288 int erdma_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)
1289 {
1290 struct erdma_mr *mr;
1291 struct erdma_dev *dev = to_edev(ibmr->device);
1292 struct erdma_cmdq_dereg_mr_req req;
1293 int ret;
1294
1295 mr = to_emr(ibmr);
1296
1297 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
1298 CMDQ_OPCODE_DEREG_MR);
1299
1300 req.cfg = FIELD_PREP(ERDMA_CMD_MR_MPT_IDX_MASK, ibmr->lkey >> 8) |
1301 FIELD_PREP(ERDMA_CMD_MR_KEY_MASK, ibmr->lkey & 0xFF);
1302
1303 ret = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL,
1304 true);
1305 if (ret)
1306 return ret;
1307
1308 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX], ibmr->lkey >> 8);
1309
1310 put_mtt_entries(dev, &mr->mem);
1311
1312 kfree(mr);
1313 return 0;
1314 }
1315
erdma_destroy_cq(struct ib_cq * ibcq,struct ib_udata * udata)1316 int erdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
1317 {
1318 struct erdma_cq *cq = to_ecq(ibcq);
1319 struct erdma_dev *dev = to_edev(ibcq->device);
1320 struct erdma_ucontext *ctx = rdma_udata_to_drv_context(
1321 udata, struct erdma_ucontext, ibucontext);
1322 int err;
1323 struct erdma_cmdq_destroy_cq_req req;
1324
1325 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
1326 CMDQ_OPCODE_DESTROY_CQ);
1327 req.cqn = cq->cqn;
1328
1329 err = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL,
1330 true);
1331 if (err)
1332 return err;
1333
1334 if (rdma_is_kernel_res(&cq->ibcq.res)) {
1335 dma_free_coherent(&dev->pdev->dev, cq->depth << CQE_SHIFT,
1336 cq->kern_cq.qbuf, cq->kern_cq.qbuf_dma_addr);
1337 dma_pool_free(dev->db_pool, cq->kern_cq.dbrec,
1338 cq->kern_cq.dbrec_dma);
1339 } else {
1340 erdma_unmap_user_dbrecords(ctx, &cq->user_cq.user_dbr_page);
1341 put_mtt_entries(dev, &cq->user_cq.qbuf_mem);
1342 }
1343
1344 xa_erase(&dev->cq_xa, cq->cqn);
1345
1346 return 0;
1347 }
1348
erdma_destroy_qp(struct ib_qp * ibqp,struct ib_udata * udata)1349 int erdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
1350 {
1351 struct erdma_qp *qp = to_eqp(ibqp);
1352 struct erdma_dev *dev = to_edev(ibqp->device);
1353 struct erdma_ucontext *ctx = rdma_udata_to_drv_context(
1354 udata, struct erdma_ucontext, ibucontext);
1355 struct erdma_cmdq_destroy_qp_req req;
1356 union erdma_mod_qp_params params;
1357 int err;
1358
1359 down_write(&qp->state_lock);
1360 if (erdma_device_iwarp(dev)) {
1361 params.iwarp.state = ERDMA_QPS_IWARP_ERROR;
1362 erdma_modify_qp_state_iwarp(qp, ¶ms.iwarp,
1363 ERDMA_QPA_IWARP_STATE);
1364 } else {
1365 params.rocev2.state = ERDMA_QPS_ROCEV2_ERROR;
1366 erdma_modify_qp_state_rocev2(qp, ¶ms.rocev2,
1367 ERDMA_QPA_ROCEV2_STATE);
1368 }
1369 up_write(&qp->state_lock);
1370
1371 cancel_delayed_work_sync(&qp->reflush_dwork);
1372
1373 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
1374 CMDQ_OPCODE_DESTROY_QP);
1375 req.qpn = QP_ID(qp);
1376
1377 err = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL,
1378 true);
1379 if (err)
1380 return err;
1381
1382 erdma_qp_put(qp);
1383 wait_for_completion(&qp->safe_free);
1384
1385 if (rdma_is_kernel_res(&qp->ibqp.res)) {
1386 free_kernel_qp(qp);
1387 } else {
1388 put_mtt_entries(dev, &qp->user_qp.sq_mem);
1389 put_mtt_entries(dev, &qp->user_qp.rq_mem);
1390 erdma_unmap_user_dbrecords(ctx, &qp->user_qp.user_dbr_page);
1391 }
1392
1393 if (qp->cep)
1394 erdma_cep_put(qp->cep);
1395 xa_erase(&dev->qp_xa, QP_ID(qp));
1396
1397 return 0;
1398 }
1399
erdma_qp_get_ref(struct ib_qp * ibqp)1400 void erdma_qp_get_ref(struct ib_qp *ibqp)
1401 {
1402 erdma_qp_get(to_eqp(ibqp));
1403 }
1404
erdma_qp_put_ref(struct ib_qp * ibqp)1405 void erdma_qp_put_ref(struct ib_qp *ibqp)
1406 {
1407 erdma_qp_put(to_eqp(ibqp));
1408 }
1409
erdma_mmap(struct ib_ucontext * ctx,struct vm_area_struct * vma)1410 int erdma_mmap(struct ib_ucontext *ctx, struct vm_area_struct *vma)
1411 {
1412 struct rdma_user_mmap_entry *rdma_entry;
1413 struct erdma_user_mmap_entry *entry;
1414 pgprot_t prot;
1415 int err;
1416
1417 rdma_entry = rdma_user_mmap_entry_get(ctx, vma);
1418 if (!rdma_entry)
1419 return -EINVAL;
1420
1421 entry = to_emmap(rdma_entry);
1422
1423 switch (entry->mmap_flag) {
1424 case ERDMA_MMAP_IO_NC:
1425 /* map doorbell. */
1426 prot = pgprot_device(vma->vm_page_prot);
1427 break;
1428 default:
1429 err = -EINVAL;
1430 goto put_entry;
1431 }
1432
1433 err = rdma_user_mmap_io(ctx, vma, PFN_DOWN(entry->address), PAGE_SIZE,
1434 prot, rdma_entry);
1435
1436 put_entry:
1437 rdma_user_mmap_entry_put(rdma_entry);
1438 return err;
1439 }
1440
erdma_mmap_free(struct rdma_user_mmap_entry * rdma_entry)1441 void erdma_mmap_free(struct rdma_user_mmap_entry *rdma_entry)
1442 {
1443 struct erdma_user_mmap_entry *entry = to_emmap(rdma_entry);
1444
1445 kfree(entry);
1446 }
1447
alloc_db_resources(struct erdma_dev * dev,struct erdma_ucontext * ctx,bool ext_db_en)1448 static int alloc_db_resources(struct erdma_dev *dev, struct erdma_ucontext *ctx,
1449 bool ext_db_en)
1450 {
1451 struct erdma_cmdq_ext_db_req req = {};
1452 u64 val0, val1;
1453 int ret;
1454
1455 /*
1456 * CAP_SYS_RAWIO is required if hardware does not support extend
1457 * doorbell mechanism.
1458 */
1459 if (!ext_db_en && !capable(CAP_SYS_RAWIO))
1460 return -EPERM;
1461
1462 if (!ext_db_en) {
1463 ctx->sdb = dev->func_bar_addr + ERDMA_BAR_SQDB_SPACE_OFFSET;
1464 ctx->rdb = dev->func_bar_addr + ERDMA_BAR_RQDB_SPACE_OFFSET;
1465 ctx->cdb = dev->func_bar_addr + ERDMA_BAR_CQDB_SPACE_OFFSET;
1466 return 0;
1467 }
1468
1469 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_COMMON,
1470 CMDQ_OPCODE_ALLOC_DB);
1471
1472 req.cfg = FIELD_PREP(ERDMA_CMD_EXT_DB_CQ_EN_MASK, 1) |
1473 FIELD_PREP(ERDMA_CMD_EXT_DB_RQ_EN_MASK, 1) |
1474 FIELD_PREP(ERDMA_CMD_EXT_DB_SQ_EN_MASK, 1);
1475
1476 ret = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), &val0, &val1,
1477 true);
1478 if (ret)
1479 return ret;
1480
1481 ctx->ext_db.enable = true;
1482 ctx->ext_db.sdb_off = ERDMA_GET(val0, ALLOC_DB_RESP_SDB);
1483 ctx->ext_db.rdb_off = ERDMA_GET(val0, ALLOC_DB_RESP_RDB);
1484 ctx->ext_db.cdb_off = ERDMA_GET(val0, ALLOC_DB_RESP_CDB);
1485
1486 ctx->sdb = dev->func_bar_addr + (ctx->ext_db.sdb_off << PAGE_SHIFT);
1487 ctx->cdb = dev->func_bar_addr + (ctx->ext_db.rdb_off << PAGE_SHIFT);
1488 ctx->rdb = dev->func_bar_addr + (ctx->ext_db.cdb_off << PAGE_SHIFT);
1489
1490 return 0;
1491 }
1492
free_db_resources(struct erdma_dev * dev,struct erdma_ucontext * ctx)1493 static void free_db_resources(struct erdma_dev *dev, struct erdma_ucontext *ctx)
1494 {
1495 struct erdma_cmdq_ext_db_req req = {};
1496 int ret;
1497
1498 if (!ctx->ext_db.enable)
1499 return;
1500
1501 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_COMMON,
1502 CMDQ_OPCODE_FREE_DB);
1503
1504 req.cfg = FIELD_PREP(ERDMA_CMD_EXT_DB_CQ_EN_MASK, 1) |
1505 FIELD_PREP(ERDMA_CMD_EXT_DB_RQ_EN_MASK, 1) |
1506 FIELD_PREP(ERDMA_CMD_EXT_DB_SQ_EN_MASK, 1);
1507
1508 req.sdb_off = ctx->ext_db.sdb_off;
1509 req.rdb_off = ctx->ext_db.rdb_off;
1510 req.cdb_off = ctx->ext_db.cdb_off;
1511
1512 ret = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL,
1513 true);
1514 if (ret)
1515 ibdev_err_ratelimited(&dev->ibdev,
1516 "free db resources failed %d", ret);
1517 }
1518
erdma_uctx_user_mmap_entries_remove(struct erdma_ucontext * uctx)1519 static void erdma_uctx_user_mmap_entries_remove(struct erdma_ucontext *uctx)
1520 {
1521 rdma_user_mmap_entry_remove(uctx->sq_db_mmap_entry);
1522 rdma_user_mmap_entry_remove(uctx->rq_db_mmap_entry);
1523 rdma_user_mmap_entry_remove(uctx->cq_db_mmap_entry);
1524 }
1525
erdma_alloc_ucontext(struct ib_ucontext * ibctx,struct ib_udata * udata)1526 int erdma_alloc_ucontext(struct ib_ucontext *ibctx, struct ib_udata *udata)
1527 {
1528 struct erdma_ucontext *ctx = to_ectx(ibctx);
1529 struct erdma_dev *dev = to_edev(ibctx->device);
1530 int ret;
1531 struct erdma_uresp_alloc_ctx uresp = {};
1532
1533 if (atomic_inc_return(&dev->num_ctx) > ERDMA_MAX_CONTEXT) {
1534 ret = -ENOMEM;
1535 goto err_out;
1536 }
1537
1538 if (udata->outlen < sizeof(uresp)) {
1539 ret = -EINVAL;
1540 goto err_out;
1541 }
1542
1543 INIT_LIST_HEAD(&ctx->dbrecords_page_list);
1544 mutex_init(&ctx->dbrecords_page_mutex);
1545
1546 ret = alloc_db_resources(dev, ctx,
1547 !!(dev->attrs.cap_flags &
1548 ERDMA_DEV_CAP_FLAGS_EXTEND_DB));
1549 if (ret)
1550 goto err_out;
1551
1552 ctx->sq_db_mmap_entry = erdma_user_mmap_entry_insert(
1553 ctx, (void *)ctx->sdb, PAGE_SIZE, ERDMA_MMAP_IO_NC, &uresp.sdb);
1554 if (!ctx->sq_db_mmap_entry) {
1555 ret = -ENOMEM;
1556 goto err_free_ext_db;
1557 }
1558
1559 ctx->rq_db_mmap_entry = erdma_user_mmap_entry_insert(
1560 ctx, (void *)ctx->rdb, PAGE_SIZE, ERDMA_MMAP_IO_NC, &uresp.rdb);
1561 if (!ctx->rq_db_mmap_entry) {
1562 ret = -EINVAL;
1563 goto err_put_mmap_entries;
1564 }
1565
1566 ctx->cq_db_mmap_entry = erdma_user_mmap_entry_insert(
1567 ctx, (void *)ctx->cdb, PAGE_SIZE, ERDMA_MMAP_IO_NC, &uresp.cdb);
1568 if (!ctx->cq_db_mmap_entry) {
1569 ret = -EINVAL;
1570 goto err_put_mmap_entries;
1571 }
1572
1573 uresp.dev_id = dev->pdev->device;
1574
1575 ret = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1576 if (ret)
1577 goto err_put_mmap_entries;
1578
1579 return 0;
1580
1581 err_put_mmap_entries:
1582 erdma_uctx_user_mmap_entries_remove(ctx);
1583
1584 err_free_ext_db:
1585 free_db_resources(dev, ctx);
1586
1587 err_out:
1588 atomic_dec(&dev->num_ctx);
1589 return ret;
1590 }
1591
erdma_dealloc_ucontext(struct ib_ucontext * ibctx)1592 void erdma_dealloc_ucontext(struct ib_ucontext *ibctx)
1593 {
1594 struct erdma_dev *dev = to_edev(ibctx->device);
1595 struct erdma_ucontext *ctx = to_ectx(ibctx);
1596
1597 erdma_uctx_user_mmap_entries_remove(ctx);
1598 free_db_resources(dev, ctx);
1599 atomic_dec(&dev->num_ctx);
1600 }
1601
erdma_attr_to_av(const struct rdma_ah_attr * ah_attr,struct erdma_av * av,u16 sport)1602 static void erdma_attr_to_av(const struct rdma_ah_attr *ah_attr,
1603 struct erdma_av *av, u16 sport)
1604 {
1605 const struct ib_global_route *grh = rdma_ah_read_grh(ah_attr);
1606
1607 av->port = rdma_ah_get_port_num(ah_attr);
1608 av->sgid_index = grh->sgid_index;
1609 av->hop_limit = grh->hop_limit;
1610 av->traffic_class = grh->traffic_class;
1611 av->sl = rdma_ah_get_sl(ah_attr);
1612
1613 av->flow_label = grh->flow_label;
1614 av->udp_sport = sport;
1615
1616 ether_addr_copy(av->dmac, ah_attr->roce.dmac);
1617 memcpy(av->dgid, grh->dgid.raw, ERDMA_ROCEV2_GID_SIZE);
1618
1619 if (ipv6_addr_v4mapped((struct in6_addr *)&grh->dgid))
1620 av->ntype = ERDMA_NETWORK_TYPE_IPV4;
1621 else
1622 av->ntype = ERDMA_NETWORK_TYPE_IPV6;
1623 }
1624
erdma_av_to_attr(struct erdma_av * av,struct rdma_ah_attr * ah_attr)1625 static void erdma_av_to_attr(struct erdma_av *av, struct rdma_ah_attr *ah_attr)
1626 {
1627 ah_attr->type = RDMA_AH_ATTR_TYPE_ROCE;
1628
1629 rdma_ah_set_sl(ah_attr, av->sl);
1630 rdma_ah_set_port_num(ah_attr, av->port);
1631 rdma_ah_set_ah_flags(ah_attr, IB_AH_GRH);
1632
1633 rdma_ah_set_grh(ah_attr, NULL, av->flow_label, av->sgid_index,
1634 av->hop_limit, av->traffic_class);
1635 rdma_ah_set_dgid_raw(ah_attr, av->dgid);
1636 }
1637
1638 static int ib_qps_to_erdma_qps[ERDMA_PROTO_COUNT][IB_QPS_ERR + 1] = {
1639 [ERDMA_PROTO_IWARP] = {
1640 [IB_QPS_RESET] = ERDMA_QPS_IWARP_IDLE,
1641 [IB_QPS_INIT] = ERDMA_QPS_IWARP_IDLE,
1642 [IB_QPS_RTR] = ERDMA_QPS_IWARP_RTR,
1643 [IB_QPS_RTS] = ERDMA_QPS_IWARP_RTS,
1644 [IB_QPS_SQD] = ERDMA_QPS_IWARP_CLOSING,
1645 [IB_QPS_SQE] = ERDMA_QPS_IWARP_TERMINATE,
1646 [IB_QPS_ERR] = ERDMA_QPS_IWARP_ERROR,
1647 },
1648 [ERDMA_PROTO_ROCEV2] = {
1649 [IB_QPS_RESET] = ERDMA_QPS_ROCEV2_RESET,
1650 [IB_QPS_INIT] = ERDMA_QPS_ROCEV2_INIT,
1651 [IB_QPS_RTR] = ERDMA_QPS_ROCEV2_RTR,
1652 [IB_QPS_RTS] = ERDMA_QPS_ROCEV2_RTS,
1653 [IB_QPS_SQD] = ERDMA_QPS_ROCEV2_SQD,
1654 [IB_QPS_SQE] = ERDMA_QPS_ROCEV2_SQE,
1655 [IB_QPS_ERR] = ERDMA_QPS_ROCEV2_ERROR,
1656 },
1657 };
1658
1659 static int erdma_qps_to_ib_qps[ERDMA_PROTO_COUNT][ERDMA_QPS_ROCEV2_COUNT] = {
1660 [ERDMA_PROTO_IWARP] = {
1661 [ERDMA_QPS_IWARP_IDLE] = IB_QPS_INIT,
1662 [ERDMA_QPS_IWARP_RTR] = IB_QPS_RTR,
1663 [ERDMA_QPS_IWARP_RTS] = IB_QPS_RTS,
1664 [ERDMA_QPS_IWARP_CLOSING] = IB_QPS_ERR,
1665 [ERDMA_QPS_IWARP_TERMINATE] = IB_QPS_ERR,
1666 [ERDMA_QPS_IWARP_ERROR] = IB_QPS_ERR,
1667 },
1668 [ERDMA_PROTO_ROCEV2] = {
1669 [ERDMA_QPS_ROCEV2_RESET] = IB_QPS_RESET,
1670 [ERDMA_QPS_ROCEV2_INIT] = IB_QPS_INIT,
1671 [ERDMA_QPS_ROCEV2_RTR] = IB_QPS_RTR,
1672 [ERDMA_QPS_ROCEV2_RTS] = IB_QPS_RTS,
1673 [ERDMA_QPS_ROCEV2_SQD] = IB_QPS_SQD,
1674 [ERDMA_QPS_ROCEV2_SQE] = IB_QPS_SQE,
1675 [ERDMA_QPS_ROCEV2_ERROR] = IB_QPS_ERR,
1676 },
1677 };
1678
ib_to_iwarp_qps(enum ib_qp_state state)1679 static inline enum erdma_qps_iwarp ib_to_iwarp_qps(enum ib_qp_state state)
1680 {
1681 return ib_qps_to_erdma_qps[ERDMA_PROTO_IWARP][state];
1682 }
1683
ib_to_rocev2_qps(enum ib_qp_state state)1684 static inline enum erdma_qps_rocev2 ib_to_rocev2_qps(enum ib_qp_state state)
1685 {
1686 return ib_qps_to_erdma_qps[ERDMA_PROTO_ROCEV2][state];
1687 }
1688
iwarp_to_ib_qps(enum erdma_qps_iwarp state)1689 static inline enum ib_qp_state iwarp_to_ib_qps(enum erdma_qps_iwarp state)
1690 {
1691 return erdma_qps_to_ib_qps[ERDMA_PROTO_IWARP][state];
1692 }
1693
rocev2_to_ib_qps(enum erdma_qps_rocev2 state)1694 static inline enum ib_qp_state rocev2_to_ib_qps(enum erdma_qps_rocev2 state)
1695 {
1696 return erdma_qps_to_ib_qps[ERDMA_PROTO_ROCEV2][state];
1697 }
1698
erdma_check_qp_attrs(struct erdma_qp * qp,struct ib_qp_attr * attr,int attr_mask)1699 static int erdma_check_qp_attrs(struct erdma_qp *qp, struct ib_qp_attr *attr,
1700 int attr_mask)
1701 {
1702 enum ib_qp_state cur_state, nxt_state;
1703 struct erdma_dev *dev = qp->dev;
1704 int ret = -EINVAL;
1705
1706 if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS) {
1707 ret = -EOPNOTSUPP;
1708 goto out;
1709 }
1710
1711 if ((attr_mask & IB_QP_PORT) &&
1712 !rdma_is_port_valid(&dev->ibdev, attr->port_num))
1713 goto out;
1714
1715 if (erdma_device_rocev2(dev)) {
1716 cur_state = (attr_mask & IB_QP_CUR_STATE) ?
1717 attr->cur_qp_state :
1718 rocev2_to_ib_qps(qp->attrs.rocev2.state);
1719
1720 nxt_state = (attr_mask & IB_QP_STATE) ? attr->qp_state :
1721 cur_state;
1722
1723 if (!ib_modify_qp_is_ok(cur_state, nxt_state, qp->ibqp.qp_type,
1724 attr_mask))
1725 goto out;
1726
1727 if ((attr_mask & IB_QP_AV) &&
1728 erdma_check_gid_attr(
1729 rdma_ah_read_grh(&attr->ah_attr)->sgid_attr))
1730 goto out;
1731
1732 if ((attr_mask & IB_QP_PKEY_INDEX) &&
1733 attr->pkey_index >= ERDMA_MAX_PKEYS)
1734 goto out;
1735 }
1736
1737 return 0;
1738
1739 out:
1740 return ret;
1741 }
1742
erdma_init_mod_qp_params_rocev2(struct erdma_qp * qp,struct erdma_mod_qp_params_rocev2 * params,int * erdma_attr_mask,struct ib_qp_attr * attr,int ib_attr_mask)1743 static void erdma_init_mod_qp_params_rocev2(
1744 struct erdma_qp *qp, struct erdma_mod_qp_params_rocev2 *params,
1745 int *erdma_attr_mask, struct ib_qp_attr *attr, int ib_attr_mask)
1746 {
1747 enum erdma_qpa_mask_rocev2 to_modify_attrs = 0;
1748 enum erdma_qps_rocev2 cur_state, nxt_state;
1749 u16 udp_sport;
1750
1751 if (ib_attr_mask & IB_QP_CUR_STATE)
1752 cur_state = ib_to_rocev2_qps(attr->cur_qp_state);
1753 else
1754 cur_state = qp->attrs.rocev2.state;
1755
1756 if (ib_attr_mask & IB_QP_STATE)
1757 nxt_state = ib_to_rocev2_qps(attr->qp_state);
1758 else
1759 nxt_state = cur_state;
1760
1761 to_modify_attrs |= ERDMA_QPA_ROCEV2_STATE;
1762 params->state = nxt_state;
1763
1764 if (ib_attr_mask & IB_QP_QKEY) {
1765 to_modify_attrs |= ERDMA_QPA_ROCEV2_QKEY;
1766 params->qkey = attr->qkey;
1767 }
1768
1769 if (ib_attr_mask & IB_QP_SQ_PSN) {
1770 to_modify_attrs |= ERDMA_QPA_ROCEV2_SQ_PSN;
1771 params->sq_psn = attr->sq_psn;
1772 }
1773
1774 if (ib_attr_mask & IB_QP_RQ_PSN) {
1775 to_modify_attrs |= ERDMA_QPA_ROCEV2_RQ_PSN;
1776 params->rq_psn = attr->rq_psn;
1777 }
1778
1779 if (ib_attr_mask & IB_QP_DEST_QPN) {
1780 to_modify_attrs |= ERDMA_QPA_ROCEV2_DST_QPN;
1781 params->dst_qpn = attr->dest_qp_num;
1782 }
1783
1784 if (ib_attr_mask & IB_QP_AV) {
1785 to_modify_attrs |= ERDMA_QPA_ROCEV2_AV;
1786 udp_sport = rdma_get_udp_sport(attr->ah_attr.grh.flow_label,
1787 QP_ID(qp), params->dst_qpn);
1788 erdma_attr_to_av(&attr->ah_attr, ¶ms->av, udp_sport);
1789 }
1790
1791 *erdma_attr_mask = to_modify_attrs;
1792 }
1793
erdma_modify_qp(struct ib_qp * ibqp,struct ib_qp_attr * attr,int attr_mask,struct ib_udata * udata)1794 int erdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask,
1795 struct ib_udata *udata)
1796 {
1797 struct erdma_qp *qp = to_eqp(ibqp);
1798 union erdma_mod_qp_params params;
1799 int ret = 0, erdma_attr_mask = 0;
1800
1801 down_write(&qp->state_lock);
1802
1803 ret = erdma_check_qp_attrs(qp, attr, attr_mask);
1804 if (ret)
1805 goto out;
1806
1807 if (erdma_device_iwarp(qp->dev)) {
1808 if (attr_mask & IB_QP_STATE) {
1809 erdma_attr_mask |= ERDMA_QPA_IWARP_STATE;
1810 params.iwarp.state = ib_to_iwarp_qps(attr->qp_state);
1811 }
1812
1813 ret = erdma_modify_qp_state_iwarp(qp, ¶ms.iwarp,
1814 erdma_attr_mask);
1815 } else {
1816 erdma_init_mod_qp_params_rocev2(
1817 qp, ¶ms.rocev2, &erdma_attr_mask, attr, attr_mask);
1818
1819 ret = erdma_modify_qp_state_rocev2(qp, ¶ms.rocev2,
1820 erdma_attr_mask);
1821 }
1822
1823 out:
1824 up_write(&qp->state_lock);
1825 return ret;
1826 }
1827
query_qp_state(struct erdma_qp * qp)1828 static enum ib_qp_state query_qp_state(struct erdma_qp *qp)
1829 {
1830 if (erdma_device_iwarp(qp->dev))
1831 return iwarp_to_ib_qps(qp->attrs.iwarp.state);
1832 else
1833 return rocev2_to_ib_qps(qp->attrs.rocev2.state);
1834 }
1835
erdma_query_qp(struct ib_qp * ibqp,struct ib_qp_attr * qp_attr,int qp_attr_mask,struct ib_qp_init_attr * qp_init_attr)1836 int erdma_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
1837 int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr)
1838 {
1839 struct erdma_cmdq_query_qp_req_rocev2 req;
1840 struct erdma_dev *dev;
1841 struct erdma_qp *qp;
1842 u64 resp0, resp1;
1843 int ret;
1844
1845 if (ibqp && qp_attr && qp_init_attr) {
1846 qp = to_eqp(ibqp);
1847 dev = to_edev(ibqp->device);
1848 } else {
1849 return -EINVAL;
1850 }
1851
1852 qp_attr->cap.max_inline_data = ERDMA_MAX_INLINE;
1853 qp_init_attr->cap.max_inline_data = ERDMA_MAX_INLINE;
1854
1855 qp_attr->cap.max_send_wr = qp->attrs.sq_size;
1856 qp_attr->cap.max_recv_wr = qp->attrs.rq_size;
1857 qp_attr->cap.max_send_sge = qp->attrs.max_send_sge;
1858 qp_attr->cap.max_recv_sge = qp->attrs.max_recv_sge;
1859
1860 qp_attr->path_mtu = ib_mtu_int_to_enum(dev->netdev->mtu);
1861 qp_attr->max_rd_atomic = qp->attrs.irq_size;
1862 qp_attr->max_dest_rd_atomic = qp->attrs.orq_size;
1863
1864 qp_attr->qp_access_flags = IB_ACCESS_LOCAL_WRITE |
1865 IB_ACCESS_REMOTE_WRITE |
1866 IB_ACCESS_REMOTE_READ;
1867
1868 qp_init_attr->cap = qp_attr->cap;
1869
1870 if (erdma_device_rocev2(dev)) {
1871 /* Query hardware to get some attributes */
1872 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
1873 CMDQ_OPCODE_QUERY_QP);
1874 req.qpn = QP_ID(qp);
1875
1876 ret = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), &resp0,
1877 &resp1, true);
1878 if (ret)
1879 return ret;
1880
1881 qp_attr->sq_psn =
1882 FIELD_GET(ERDMA_CMD_QUERY_QP_RESP_SQ_PSN_MASK, resp0);
1883 qp_attr->rq_psn =
1884 FIELD_GET(ERDMA_CMD_QUERY_QP_RESP_RQ_PSN_MASK, resp0);
1885 qp_attr->qp_state = rocev2_to_ib_qps(FIELD_GET(
1886 ERDMA_CMD_QUERY_QP_RESP_QP_STATE_MASK, resp0));
1887 qp_attr->cur_qp_state = qp_attr->qp_state;
1888 qp_attr->sq_draining = FIELD_GET(
1889 ERDMA_CMD_QUERY_QP_RESP_SQ_DRAINING_MASK, resp0);
1890
1891 qp_attr->pkey_index = 0;
1892 qp_attr->dest_qp_num = qp->attrs.rocev2.dst_qpn;
1893
1894 if (qp->ibqp.qp_type == IB_QPT_RC)
1895 erdma_av_to_attr(&qp->attrs.rocev2.av,
1896 &qp_attr->ah_attr);
1897 } else {
1898 qp_attr->qp_state = query_qp_state(qp);
1899 qp_attr->cur_qp_state = qp_attr->qp_state;
1900 }
1901
1902 return 0;
1903 }
1904
erdma_init_user_cq(struct erdma_ucontext * ctx,struct erdma_cq * cq,struct erdma_ureq_create_cq * ureq)1905 static int erdma_init_user_cq(struct erdma_ucontext *ctx, struct erdma_cq *cq,
1906 struct erdma_ureq_create_cq *ureq)
1907 {
1908 int ret;
1909 struct erdma_dev *dev = to_edev(cq->ibcq.device);
1910
1911 ret = get_mtt_entries(dev, &cq->user_cq.qbuf_mem, ureq->qbuf_va,
1912 ureq->qbuf_len, 0, ureq->qbuf_va, SZ_64M - SZ_4K,
1913 true);
1914 if (ret)
1915 return ret;
1916
1917 ret = erdma_map_user_dbrecords(ctx, ureq->db_record_va,
1918 &cq->user_cq.user_dbr_page,
1919 &cq->user_cq.dbrec_dma);
1920 if (ret)
1921 put_mtt_entries(dev, &cq->user_cq.qbuf_mem);
1922
1923 return ret;
1924 }
1925
erdma_init_kernel_cq(struct erdma_cq * cq)1926 static int erdma_init_kernel_cq(struct erdma_cq *cq)
1927 {
1928 struct erdma_dev *dev = to_edev(cq->ibcq.device);
1929
1930 cq->kern_cq.qbuf =
1931 dma_alloc_coherent(&dev->pdev->dev, cq->depth << CQE_SHIFT,
1932 &cq->kern_cq.qbuf_dma_addr, GFP_KERNEL);
1933 if (!cq->kern_cq.qbuf)
1934 return -ENOMEM;
1935
1936 cq->kern_cq.dbrec = dma_pool_zalloc(dev->db_pool, GFP_KERNEL,
1937 &cq->kern_cq.dbrec_dma);
1938 if (!cq->kern_cq.dbrec)
1939 goto err_out;
1940
1941 spin_lock_init(&cq->kern_cq.lock);
1942 /* use default cqdb addr */
1943 cq->kern_cq.db = dev->func_bar + ERDMA_BAR_CQDB_SPACE_OFFSET;
1944
1945 return 0;
1946
1947 err_out:
1948 dma_free_coherent(&dev->pdev->dev, cq->depth << CQE_SHIFT,
1949 cq->kern_cq.qbuf, cq->kern_cq.qbuf_dma_addr);
1950
1951 return -ENOMEM;
1952 }
1953
erdma_create_cq(struct ib_cq * ibcq,const struct ib_cq_init_attr * attr,struct uverbs_attr_bundle * attrs)1954 int erdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
1955 struct uverbs_attr_bundle *attrs)
1956 {
1957 struct ib_udata *udata = &attrs->driver_udata;
1958 struct erdma_cq *cq = to_ecq(ibcq);
1959 struct erdma_dev *dev = to_edev(ibcq->device);
1960 unsigned int depth = attr->cqe;
1961 int ret;
1962 struct erdma_ucontext *ctx = rdma_udata_to_drv_context(
1963 udata, struct erdma_ucontext, ibucontext);
1964
1965 if (depth > dev->attrs.max_cqe)
1966 return -EINVAL;
1967
1968 depth = roundup_pow_of_two(depth);
1969 cq->ibcq.cqe = depth;
1970 cq->depth = depth;
1971 cq->assoc_eqn = attr->comp_vector + 1;
1972
1973 ret = xa_alloc_cyclic(&dev->cq_xa, &cq->cqn, cq,
1974 XA_LIMIT(1, dev->attrs.max_cq - 1),
1975 &dev->next_alloc_cqn, GFP_KERNEL);
1976 if (ret < 0)
1977 return ret;
1978
1979 if (!rdma_is_kernel_res(&ibcq->res)) {
1980 struct erdma_ureq_create_cq ureq;
1981 struct erdma_uresp_create_cq uresp;
1982
1983 ret = ib_copy_from_udata(&ureq, udata,
1984 min(udata->inlen, sizeof(ureq)));
1985 if (ret)
1986 goto err_out_xa;
1987
1988 ret = erdma_init_user_cq(ctx, cq, &ureq);
1989 if (ret)
1990 goto err_out_xa;
1991
1992 uresp.cq_id = cq->cqn;
1993 uresp.num_cqe = depth;
1994
1995 ret = ib_copy_to_udata(udata, &uresp,
1996 min(sizeof(uresp), udata->outlen));
1997 if (ret)
1998 goto err_free_res;
1999 } else {
2000 ret = erdma_init_kernel_cq(cq);
2001 if (ret)
2002 goto err_out_xa;
2003 }
2004
2005 ret = create_cq_cmd(ctx, cq);
2006 if (ret)
2007 goto err_free_res;
2008
2009 return 0;
2010
2011 err_free_res:
2012 if (!rdma_is_kernel_res(&ibcq->res)) {
2013 erdma_unmap_user_dbrecords(ctx, &cq->user_cq.user_dbr_page);
2014 put_mtt_entries(dev, &cq->user_cq.qbuf_mem);
2015 } else {
2016 dma_free_coherent(&dev->pdev->dev, depth << CQE_SHIFT,
2017 cq->kern_cq.qbuf, cq->kern_cq.qbuf_dma_addr);
2018 dma_pool_free(dev->db_pool, cq->kern_cq.dbrec,
2019 cq->kern_cq.dbrec_dma);
2020 }
2021
2022 err_out_xa:
2023 xa_erase(&dev->cq_xa, cq->cqn);
2024
2025 return ret;
2026 }
2027
erdma_disassociate_ucontext(struct ib_ucontext * ibcontext)2028 void erdma_disassociate_ucontext(struct ib_ucontext *ibcontext)
2029 {
2030 }
2031
erdma_set_mtu(struct erdma_dev * dev,u32 mtu)2032 void erdma_set_mtu(struct erdma_dev *dev, u32 mtu)
2033 {
2034 struct erdma_cmdq_config_mtu_req req;
2035
2036 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_COMMON,
2037 CMDQ_OPCODE_CONF_MTU);
2038 req.mtu = mtu;
2039
2040 erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL, true);
2041 }
2042
erdma_port_event(struct erdma_dev * dev,enum ib_event_type reason)2043 void erdma_port_event(struct erdma_dev *dev, enum ib_event_type reason)
2044 {
2045 struct ib_event event;
2046
2047 event.device = &dev->ibdev;
2048 event.element.port_num = 1;
2049 event.event = reason;
2050
2051 ib_dispatch_event(&event);
2052 }
2053
2054 enum counters {
2055 ERDMA_STATS_TX_REQS_CNT,
2056 ERDMA_STATS_TX_PACKETS_CNT,
2057 ERDMA_STATS_TX_BYTES_CNT,
2058 ERDMA_STATS_TX_DISABLE_DROP_CNT,
2059 ERDMA_STATS_TX_BPS_METER_DROP_CNT,
2060 ERDMA_STATS_TX_PPS_METER_DROP_CNT,
2061
2062 ERDMA_STATS_RX_PACKETS_CNT,
2063 ERDMA_STATS_RX_BYTES_CNT,
2064 ERDMA_STATS_RX_DISABLE_DROP_CNT,
2065 ERDMA_STATS_RX_BPS_METER_DROP_CNT,
2066 ERDMA_STATS_RX_PPS_METER_DROP_CNT,
2067
2068 ERDMA_STATS_MAX
2069 };
2070
2071 static const struct rdma_stat_desc erdma_descs[] = {
2072 [ERDMA_STATS_TX_REQS_CNT].name = "tx_reqs_cnt",
2073 [ERDMA_STATS_TX_PACKETS_CNT].name = "tx_packets_cnt",
2074 [ERDMA_STATS_TX_BYTES_CNT].name = "tx_bytes_cnt",
2075 [ERDMA_STATS_TX_DISABLE_DROP_CNT].name = "tx_disable_drop_cnt",
2076 [ERDMA_STATS_TX_BPS_METER_DROP_CNT].name = "tx_bps_limit_drop_cnt",
2077 [ERDMA_STATS_TX_PPS_METER_DROP_CNT].name = "tx_pps_limit_drop_cnt",
2078 [ERDMA_STATS_RX_PACKETS_CNT].name = "rx_packets_cnt",
2079 [ERDMA_STATS_RX_BYTES_CNT].name = "rx_bytes_cnt",
2080 [ERDMA_STATS_RX_DISABLE_DROP_CNT].name = "rx_disable_drop_cnt",
2081 [ERDMA_STATS_RX_BPS_METER_DROP_CNT].name = "rx_bps_limit_drop_cnt",
2082 [ERDMA_STATS_RX_PPS_METER_DROP_CNT].name = "rx_pps_limit_drop_cnt",
2083 };
2084
erdma_alloc_hw_port_stats(struct ib_device * device,u32 port_num)2085 struct rdma_hw_stats *erdma_alloc_hw_port_stats(struct ib_device *device,
2086 u32 port_num)
2087 {
2088 return rdma_alloc_hw_stats_struct(erdma_descs, ERDMA_STATS_MAX,
2089 RDMA_HW_STATS_DEFAULT_LIFESPAN);
2090 }
2091
erdma_query_hw_stats(struct erdma_dev * dev,struct rdma_hw_stats * stats)2092 static int erdma_query_hw_stats(struct erdma_dev *dev,
2093 struct rdma_hw_stats *stats)
2094 {
2095 struct erdma_cmdq_query_stats_resp *resp;
2096 struct erdma_cmdq_query_req req;
2097 dma_addr_t dma_addr;
2098 int err;
2099
2100 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_COMMON,
2101 CMDQ_OPCODE_GET_STATS);
2102
2103 resp = dma_pool_zalloc(dev->resp_pool, GFP_KERNEL, &dma_addr);
2104 if (!resp)
2105 return -ENOMEM;
2106
2107 req.target_addr = dma_addr;
2108 req.target_length = ERDMA_HW_RESP_SIZE;
2109
2110 err = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL,
2111 true);
2112 if (err)
2113 goto out;
2114
2115 if (resp->hdr.magic != ERDMA_HW_RESP_MAGIC) {
2116 err = -EINVAL;
2117 goto out;
2118 }
2119
2120 memcpy(&stats->value[0], &resp->tx_req_cnt,
2121 sizeof(u64) * stats->num_counters);
2122
2123 out:
2124 dma_pool_free(dev->resp_pool, resp, dma_addr);
2125
2126 return err;
2127 }
2128
erdma_get_hw_stats(struct ib_device * ibdev,struct rdma_hw_stats * stats,u32 port,int index)2129 int erdma_get_hw_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats,
2130 u32 port, int index)
2131 {
2132 struct erdma_dev *dev = to_edev(ibdev);
2133 int ret;
2134
2135 if (port == 0)
2136 return 0;
2137
2138 ret = erdma_query_hw_stats(dev, stats);
2139 if (ret)
2140 return ret;
2141
2142 return stats->num_counters;
2143 }
2144
erdma_get_link_layer(struct ib_device * ibdev,u32 port_num)2145 enum rdma_link_layer erdma_get_link_layer(struct ib_device *ibdev, u32 port_num)
2146 {
2147 return IB_LINK_LAYER_ETHERNET;
2148 }
2149
erdma_set_gid(struct erdma_dev * dev,u8 op,u32 idx,const union ib_gid * gid)2150 static int erdma_set_gid(struct erdma_dev *dev, u8 op, u32 idx,
2151 const union ib_gid *gid)
2152 {
2153 struct erdma_cmdq_set_gid_req req;
2154 u8 ntype;
2155
2156 req.cfg = FIELD_PREP(ERDMA_CMD_SET_GID_SGID_IDX_MASK, idx) |
2157 FIELD_PREP(ERDMA_CMD_SET_GID_OP_MASK, op);
2158
2159 if (op == ERDMA_SET_GID_OP_ADD) {
2160 if (ipv6_addr_v4mapped((struct in6_addr *)gid))
2161 ntype = ERDMA_NETWORK_TYPE_IPV4;
2162 else
2163 ntype = ERDMA_NETWORK_TYPE_IPV6;
2164
2165 req.cfg |= FIELD_PREP(ERDMA_CMD_SET_GID_NTYPE_MASK, ntype);
2166
2167 memcpy(&req.gid, gid, ERDMA_ROCEV2_GID_SIZE);
2168 }
2169
2170 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
2171 CMDQ_OPCODE_SET_GID);
2172 return erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL,
2173 true);
2174 }
2175
erdma_add_gid(const struct ib_gid_attr * attr,void ** context)2176 int erdma_add_gid(const struct ib_gid_attr *attr, void **context)
2177 {
2178 struct erdma_dev *dev = to_edev(attr->device);
2179 int ret;
2180
2181 ret = erdma_check_gid_attr(attr);
2182 if (ret)
2183 return ret;
2184
2185 return erdma_set_gid(dev, ERDMA_SET_GID_OP_ADD, attr->index,
2186 &attr->gid);
2187 }
2188
erdma_del_gid(const struct ib_gid_attr * attr,void ** context)2189 int erdma_del_gid(const struct ib_gid_attr *attr, void **context)
2190 {
2191 return erdma_set_gid(to_edev(attr->device), ERDMA_SET_GID_OP_DEL,
2192 attr->index, NULL);
2193 }
2194
erdma_query_pkey(struct ib_device * ibdev,u32 port,u16 index,u16 * pkey)2195 int erdma_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey)
2196 {
2197 if (index >= ERDMA_MAX_PKEYS)
2198 return -EINVAL;
2199
2200 *pkey = ERDMA_DEFAULT_PKEY;
2201 return 0;
2202 }
2203
erdma_set_av_cfg(struct erdma_av_cfg * av_cfg,struct erdma_av * av)2204 void erdma_set_av_cfg(struct erdma_av_cfg *av_cfg, struct erdma_av *av)
2205 {
2206 av_cfg->cfg0 = FIELD_PREP(ERDMA_CMD_CREATE_AV_FL_MASK, av->flow_label) |
2207 FIELD_PREP(ERDMA_CMD_CREATE_AV_NTYPE_MASK, av->ntype);
2208
2209 av_cfg->traffic_class = av->traffic_class;
2210 av_cfg->hop_limit = av->hop_limit;
2211 av_cfg->sl = av->sl;
2212
2213 av_cfg->udp_sport = av->udp_sport;
2214 av_cfg->sgid_index = av->sgid_index;
2215
2216 ether_addr_copy(av_cfg->dmac, av->dmac);
2217 memcpy(av_cfg->dgid, av->dgid, ERDMA_ROCEV2_GID_SIZE);
2218 }
2219
erdma_create_ah(struct ib_ah * ibah,struct rdma_ah_init_attr * init_attr,struct ib_udata * udata)2220 int erdma_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr,
2221 struct ib_udata *udata)
2222 {
2223 const struct ib_global_route *grh =
2224 rdma_ah_read_grh(init_attr->ah_attr);
2225 struct erdma_dev *dev = to_edev(ibah->device);
2226 struct erdma_pd *pd = to_epd(ibah->pd);
2227 struct erdma_ah *ah = to_eah(ibah);
2228 struct erdma_cmdq_create_ah_req req;
2229 u32 udp_sport;
2230 int ret;
2231
2232 ret = erdma_check_gid_attr(grh->sgid_attr);
2233 if (ret)
2234 return ret;
2235
2236 ret = erdma_alloc_idx(&dev->res_cb[ERDMA_RES_TYPE_AH]);
2237 if (ret < 0)
2238 return ret;
2239
2240 ah->ahn = ret;
2241
2242 if (grh->flow_label)
2243 udp_sport = rdma_flow_label_to_udp_sport(grh->flow_label);
2244 else
2245 udp_sport =
2246 IB_ROCE_UDP_ENCAP_VALID_PORT_MIN + (ah->ahn & 0x3FFF);
2247
2248 erdma_attr_to_av(init_attr->ah_attr, &ah->av, udp_sport);
2249
2250 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
2251 CMDQ_OPCODE_CREATE_AH);
2252
2253 req.pdn = pd->pdn;
2254 req.ahn = ah->ahn;
2255 erdma_set_av_cfg(&req.av_cfg, &ah->av);
2256
2257 ret = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL,
2258 init_attr->flags & RDMA_CREATE_AH_SLEEPABLE);
2259 if (ret) {
2260 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_AH], ah->ahn);
2261 return ret;
2262 }
2263
2264 return 0;
2265 }
2266
erdma_destroy_ah(struct ib_ah * ibah,u32 flags)2267 int erdma_destroy_ah(struct ib_ah *ibah, u32 flags)
2268 {
2269 struct erdma_dev *dev = to_edev(ibah->device);
2270 struct erdma_pd *pd = to_epd(ibah->pd);
2271 struct erdma_ah *ah = to_eah(ibah);
2272 struct erdma_cmdq_destroy_ah_req req;
2273 int ret;
2274
2275 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
2276 CMDQ_OPCODE_DESTROY_AH);
2277
2278 req.pdn = pd->pdn;
2279 req.ahn = ah->ahn;
2280
2281 ret = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL,
2282 flags & RDMA_DESTROY_AH_SLEEPABLE);
2283 if (ret)
2284 return ret;
2285
2286 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_AH], ah->ahn);
2287
2288 return 0;
2289 }
2290
erdma_query_ah(struct ib_ah * ibah,struct rdma_ah_attr * ah_attr)2291 int erdma_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr)
2292 {
2293 struct erdma_ah *ah = to_eah(ibah);
2294
2295 memset(ah_attr, 0, sizeof(*ah_attr));
2296 erdma_av_to_attr(&ah->av, ah_attr);
2297
2298 return 0;
2299 }
2300