1 // SPDX-License-Identifier: GPL-2.0
2
3 /* Authors: Cheng Xu <chengyou@linux.alibaba.com> */
4 /* Kai Shen <kaishen@linux.alibaba.com> */
5 /* Copyright (c) 2020-2022, Alibaba Group. */
6
7 /* Authors: Bernard Metzler <bmt@zurich.ibm.com> */
8 /* Copyright (c) 2008-2019, IBM Corporation */
9
10 /* Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved. */
11
12 #include <linux/vmalloc.h>
13 #include <net/addrconf.h>
14 #include <rdma/erdma-abi.h>
15 #include <rdma/iter.h>
16 #include <rdma/uverbs_ioctl.h>
17
18 #include "erdma.h"
19 #include "erdma_cm.h"
20 #include "erdma_verbs.h"
21
assemble_qbuf_mtt_for_cmd(struct erdma_mem * mem,u32 * cfg,u64 * addr0,u64 * addr1)22 static void assemble_qbuf_mtt_for_cmd(struct erdma_mem *mem, u32 *cfg,
23 u64 *addr0, u64 *addr1)
24 {
25 struct erdma_mtt *mtt = mem->mtt;
26
27 if (mem->mtt_nents > ERDMA_MAX_INLINE_MTT_ENTRIES) {
28 *addr0 = mtt->buf_dma;
29 *cfg |= FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_LEVEL_MASK,
30 ERDMA_MR_MTT_1LEVEL);
31 } else {
32 *addr0 = mtt->buf[0];
33 memcpy(addr1, mtt->buf + 1, MTT_SIZE(mem->mtt_nents - 1));
34 *cfg |= FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_LEVEL_MASK,
35 ERDMA_MR_MTT_0LEVEL);
36 }
37 }
38
create_qp_cmd(struct erdma_ucontext * uctx,struct erdma_qp * qp)39 static int create_qp_cmd(struct erdma_ucontext *uctx, struct erdma_qp *qp)
40 {
41 struct erdma_dev *dev = to_edev(qp->ibqp.device);
42 struct erdma_pd *pd = to_epd(qp->ibqp.pd);
43 struct erdma_cmdq_create_qp_req req;
44 struct erdma_uqp *user_qp;
45 u64 resp0, resp1;
46 int err;
47
48 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
49 CMDQ_OPCODE_CREATE_QP);
50
51 req.cfg0 = FIELD_PREP(ERDMA_CMD_CREATE_QP_SQ_DEPTH_MASK,
52 ilog2(qp->attrs.sq_size)) |
53 FIELD_PREP(ERDMA_CMD_CREATE_QP_QPN_MASK, QP_ID(qp));
54 req.cfg1 = FIELD_PREP(ERDMA_CMD_CREATE_QP_RQ_DEPTH_MASK,
55 ilog2(qp->attrs.rq_size)) |
56 FIELD_PREP(ERDMA_CMD_CREATE_QP_PD_MASK, pd->pdn);
57
58 if (qp->ibqp.qp_type == IB_QPT_RC)
59 req.cfg2 = FIELD_PREP(ERDMA_CMD_CREATE_QP_TYPE_MASK,
60 ERDMA_QPT_RC);
61 else
62 req.cfg2 = FIELD_PREP(ERDMA_CMD_CREATE_QP_TYPE_MASK,
63 ERDMA_QPT_UD);
64
65 if (rdma_is_kernel_res(&qp->ibqp.res)) {
66 u32 pgsz_range = ilog2(SZ_1M) - ERDMA_HW_PAGE_SHIFT;
67
68 req.sq_cqn_mtt_cfg =
69 FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,
70 pgsz_range) |
71 FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->scq->cqn);
72 req.rq_cqn_mtt_cfg =
73 FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,
74 pgsz_range) |
75 FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->rcq->cqn);
76
77 req.sq_mtt_cfg =
78 FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_OFFSET_MASK, 0) |
79 FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK, 1) |
80 FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_LEVEL_MASK,
81 ERDMA_MR_MTT_0LEVEL);
82 req.rq_mtt_cfg = req.sq_mtt_cfg;
83
84 req.rq_buf_addr = qp->kern_qp.rq_buf_dma_addr;
85 req.sq_buf_addr = qp->kern_qp.sq_buf_dma_addr;
86 req.sq_dbrec_dma = qp->kern_qp.sq_dbrec_dma;
87 req.rq_dbrec_dma = qp->kern_qp.rq_dbrec_dma;
88 } else {
89 user_qp = &qp->user_qp;
90 req.sq_cqn_mtt_cfg = FIELD_PREP(
91 ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,
92 ilog2(user_qp->sq_mem.page_size) - ERDMA_HW_PAGE_SHIFT);
93 req.sq_cqn_mtt_cfg |=
94 FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->scq->cqn);
95
96 req.rq_cqn_mtt_cfg = FIELD_PREP(
97 ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,
98 ilog2(user_qp->rq_mem.page_size) - ERDMA_HW_PAGE_SHIFT);
99 req.rq_cqn_mtt_cfg |=
100 FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->rcq->cqn);
101
102 req.sq_mtt_cfg = user_qp->sq_mem.page_offset;
103 req.sq_mtt_cfg |= FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK,
104 user_qp->sq_mem.mtt_nents);
105
106 req.rq_mtt_cfg = user_qp->rq_mem.page_offset;
107 req.rq_mtt_cfg |= FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK,
108 user_qp->rq_mem.mtt_nents);
109
110 assemble_qbuf_mtt_for_cmd(&user_qp->sq_mem, &req.sq_mtt_cfg,
111 &req.sq_buf_addr, req.sq_mtt_entry);
112 assemble_qbuf_mtt_for_cmd(&user_qp->rq_mem, &req.rq_mtt_cfg,
113 &req.rq_buf_addr, req.rq_mtt_entry);
114
115 req.sq_dbrec_dma = user_qp->sq_dbrec_dma;
116 req.rq_dbrec_dma = user_qp->rq_dbrec_dma;
117
118 if (uctx->ext_db.enable) {
119 req.sq_cqn_mtt_cfg |=
120 FIELD_PREP(ERDMA_CMD_CREATE_QP_DB_CFG_MASK, 1);
121 req.db_cfg =
122 FIELD_PREP(ERDMA_CMD_CREATE_QP_SQDB_CFG_MASK,
123 uctx->ext_db.sdb_off) |
124 FIELD_PREP(ERDMA_CMD_CREATE_QP_RQDB_CFG_MASK,
125 uctx->ext_db.rdb_off);
126 }
127 }
128
129 err = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), &resp0, &resp1,
130 true);
131 if (!err && erdma_device_iwarp(dev))
132 qp->attrs.iwarp.cookie =
133 FIELD_GET(ERDMA_CMDQ_CREATE_QP_RESP_COOKIE_MASK, resp0);
134
135 return err;
136 }
137
regmr_cmd(struct erdma_dev * dev,struct erdma_mr * mr)138 static int regmr_cmd(struct erdma_dev *dev, struct erdma_mr *mr)
139 {
140 struct erdma_pd *pd = to_epd(mr->ibmr.pd);
141 u32 mtt_level = ERDMA_MR_MTT_0LEVEL;
142 struct erdma_cmdq_reg_mr_req req;
143
144 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA, CMDQ_OPCODE_REG_MR);
145
146 if (mr->type == ERDMA_MR_TYPE_FRMR ||
147 mr->mem.page_cnt > ERDMA_MAX_INLINE_MTT_ENTRIES) {
148 if (mr->mem.mtt->continuous) {
149 req.phy_addr[0] = mr->mem.mtt->buf_dma;
150 mtt_level = ERDMA_MR_MTT_1LEVEL;
151 } else {
152 req.phy_addr[0] = mr->mem.mtt->dma_addrs[0];
153 mtt_level = mr->mem.mtt->level;
154 }
155 } else if (mr->type != ERDMA_MR_TYPE_DMA) {
156 memcpy(req.phy_addr, mr->mem.mtt->buf,
157 MTT_SIZE(mr->mem.page_cnt));
158 }
159
160 req.cfg0 = FIELD_PREP(ERDMA_CMD_MR_VALID_MASK, mr->valid) |
161 FIELD_PREP(ERDMA_CMD_MR_KEY_MASK, mr->ibmr.lkey & 0xFF) |
162 FIELD_PREP(ERDMA_CMD_MR_MPT_IDX_MASK, mr->ibmr.lkey >> 8);
163 req.cfg1 = FIELD_PREP(ERDMA_CMD_REGMR_PD_MASK, pd->pdn) |
164 FIELD_PREP(ERDMA_CMD_REGMR_TYPE_MASK, mr->type) |
165 FIELD_PREP(ERDMA_CMD_REGMR_RIGHT_MASK, mr->access);
166 req.cfg2 = FIELD_PREP(ERDMA_CMD_REGMR_PAGESIZE_MASK,
167 ilog2(mr->mem.page_size)) |
168 FIELD_PREP(ERDMA_CMD_REGMR_MTT_LEVEL_MASK, mtt_level) |
169 FIELD_PREP(ERDMA_CMD_REGMR_MTT_CNT_MASK, mr->mem.page_cnt);
170
171 if (mr->type == ERDMA_MR_TYPE_DMA)
172 goto post_cmd;
173
174 if (mr->type == ERDMA_MR_TYPE_NORMAL) {
175 req.start_va = mr->mem.va;
176 req.size = mr->mem.len;
177 }
178
179 if (!mr->mem.mtt->continuous && mr->mem.mtt->level > 1) {
180 req.cfg0 |= FIELD_PREP(ERDMA_CMD_MR_VERSION_MASK, 1);
181 req.cfg2 |= FIELD_PREP(ERDMA_CMD_REGMR_MTT_PAGESIZE_MASK,
182 PAGE_SHIFT - ERDMA_HW_PAGE_SHIFT);
183 req.size_h = upper_32_bits(mr->mem.len);
184 req.mtt_cnt_h = mr->mem.page_cnt >> 20;
185 }
186
187 post_cmd:
188 return erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL,
189 true);
190 }
191
create_cq_cmd(struct erdma_ucontext * uctx,struct erdma_cq * cq)192 static int create_cq_cmd(struct erdma_ucontext *uctx, struct erdma_cq *cq)
193 {
194 struct erdma_dev *dev = to_edev(cq->ibcq.device);
195 struct erdma_cmdq_create_cq_req req;
196 struct erdma_mem *mem;
197 u32 page_size;
198
199 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
200 CMDQ_OPCODE_CREATE_CQ);
201
202 req.cfg0 = FIELD_PREP(ERDMA_CMD_CREATE_CQ_CQN_MASK, cq->cqn) |
203 FIELD_PREP(ERDMA_CMD_CREATE_CQ_DEPTH_MASK, ilog2(cq->depth));
204 req.cfg1 = FIELD_PREP(ERDMA_CMD_CREATE_CQ_EQN_MASK, cq->assoc_eqn);
205
206 if (rdma_is_kernel_res(&cq->ibcq.res)) {
207 page_size = SZ_32M;
208 req.cfg0 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_PAGESIZE_MASK,
209 ilog2(page_size) - ERDMA_HW_PAGE_SHIFT);
210 req.qbuf_addr_l = lower_32_bits(cq->kern_cq.qbuf_dma_addr);
211 req.qbuf_addr_h = upper_32_bits(cq->kern_cq.qbuf_dma_addr);
212
213 req.cfg1 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_CNT_MASK, 1) |
214 FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_LEVEL_MASK,
215 ERDMA_MR_MTT_0LEVEL);
216
217 req.first_page_offset = 0;
218 req.cq_dbrec_dma = cq->kern_cq.dbrec_dma;
219 } else {
220 mem = &cq->user_cq.qbuf_mem;
221 req.cfg0 |=
222 FIELD_PREP(ERDMA_CMD_CREATE_CQ_PAGESIZE_MASK,
223 ilog2(mem->page_size) - ERDMA_HW_PAGE_SHIFT);
224 if (mem->mtt_nents == 1) {
225 req.qbuf_addr_l = lower_32_bits(mem->mtt->buf[0]);
226 req.qbuf_addr_h = upper_32_bits(mem->mtt->buf[0]);
227 req.cfg1 |=
228 FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_LEVEL_MASK,
229 ERDMA_MR_MTT_0LEVEL);
230 } else {
231 req.qbuf_addr_l = lower_32_bits(mem->mtt->buf_dma);
232 req.qbuf_addr_h = upper_32_bits(mem->mtt->buf_dma);
233 req.cfg1 |=
234 FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_LEVEL_MASK,
235 ERDMA_MR_MTT_1LEVEL);
236 }
237 req.cfg1 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_CNT_MASK,
238 mem->mtt_nents);
239
240 req.first_page_offset = mem->page_offset;
241 req.cq_dbrec_dma = cq->user_cq.dbrec_dma;
242
243 if (uctx->ext_db.enable) {
244 req.cfg1 |= FIELD_PREP(
245 ERDMA_CMD_CREATE_CQ_MTT_DB_CFG_MASK, 1);
246 req.cfg2 = FIELD_PREP(ERDMA_CMD_CREATE_CQ_DB_CFG_MASK,
247 uctx->ext_db.cdb_off);
248 }
249 }
250
251 return erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL,
252 true);
253 }
254
erdma_alloc_idx(struct erdma_resource_cb * res_cb)255 static int erdma_alloc_idx(struct erdma_resource_cb *res_cb)
256 {
257 int idx;
258 unsigned long flags;
259
260 spin_lock_irqsave(&res_cb->lock, flags);
261 idx = find_next_zero_bit(res_cb->bitmap, res_cb->max_cap,
262 res_cb->next_alloc_idx);
263 if (idx == res_cb->max_cap) {
264 idx = find_first_zero_bit(res_cb->bitmap, res_cb->max_cap);
265 if (idx == res_cb->max_cap) {
266 res_cb->next_alloc_idx = 1;
267 spin_unlock_irqrestore(&res_cb->lock, flags);
268 return -ENOSPC;
269 }
270 }
271
272 set_bit(idx, res_cb->bitmap);
273 res_cb->next_alloc_idx = idx + 1;
274 spin_unlock_irqrestore(&res_cb->lock, flags);
275
276 return idx;
277 }
278
erdma_free_idx(struct erdma_resource_cb * res_cb,u32 idx)279 static inline void erdma_free_idx(struct erdma_resource_cb *res_cb, u32 idx)
280 {
281 unsigned long flags;
282 u32 used;
283
284 spin_lock_irqsave(&res_cb->lock, flags);
285 used = __test_and_clear_bit(idx, res_cb->bitmap);
286 spin_unlock_irqrestore(&res_cb->lock, flags);
287 WARN_ON(!used);
288 }
289
290 static struct rdma_user_mmap_entry *
erdma_user_mmap_entry_insert(struct erdma_ucontext * uctx,void * address,u32 size,u8 mmap_flag,u64 * mmap_offset)291 erdma_user_mmap_entry_insert(struct erdma_ucontext *uctx, void *address,
292 u32 size, u8 mmap_flag, u64 *mmap_offset)
293 {
294 struct erdma_user_mmap_entry *entry = kzalloc_obj(*entry);
295 int ret;
296
297 if (!entry)
298 return NULL;
299
300 entry->address = (u64)address;
301 entry->mmap_flag = mmap_flag;
302
303 size = PAGE_ALIGN(size);
304
305 ret = rdma_user_mmap_entry_insert(&uctx->ibucontext, &entry->rdma_entry,
306 size);
307 if (ret) {
308 kfree(entry);
309 return NULL;
310 }
311
312 *mmap_offset = rdma_user_mmap_get_offset(&entry->rdma_entry);
313
314 return &entry->rdma_entry;
315 }
316
erdma_query_device(struct ib_device * ibdev,struct ib_device_attr * attr,struct ib_udata * udata)317 int erdma_query_device(struct ib_device *ibdev, struct ib_device_attr *attr,
318 struct ib_udata *udata)
319 {
320 struct erdma_dev *dev = to_edev(ibdev);
321 int err;
322
323 err = ib_no_udata_io(udata);
324 if (err)
325 return err;
326
327 attr->max_mr_size = dev->attrs.max_mr_size;
328 attr->vendor_id = PCI_VENDOR_ID_ALIBABA;
329 attr->vendor_part_id = dev->pdev->device;
330 attr->hw_ver = dev->pdev->revision;
331 attr->max_qp = dev->attrs.max_qp - 1;
332 attr->max_qp_wr = min(dev->attrs.max_send_wr, dev->attrs.max_recv_wr);
333 attr->max_qp_rd_atom = dev->attrs.max_ord;
334 attr->max_qp_init_rd_atom = dev->attrs.max_ird;
335 attr->max_res_rd_atom = dev->attrs.max_qp * dev->attrs.max_ird;
336 attr->device_cap_flags = IB_DEVICE_MEM_MGT_EXTENSIONS;
337 attr->kernel_cap_flags = IBK_LOCAL_DMA_LKEY;
338 ibdev->local_dma_lkey = dev->attrs.local_dma_key;
339 attr->max_send_sge = dev->attrs.max_send_sge;
340 attr->max_recv_sge = dev->attrs.max_recv_sge;
341 attr->max_sge_rd = dev->attrs.max_sge_rd;
342 attr->max_cq = dev->attrs.max_cq - 1;
343 attr->max_cqe = dev->attrs.max_cqe;
344 attr->max_mr = dev->attrs.max_mr;
345 attr->max_pd = dev->attrs.max_pd;
346 attr->max_mw = dev->attrs.max_mw;
347 attr->max_fast_reg_page_list_len = ERDMA_MAX_FRMR_PA;
348 attr->page_size_cap = ERDMA_PAGE_SIZE_SUPPORT;
349
350 if (erdma_device_rocev2(dev)) {
351 attr->max_pkeys = ERDMA_MAX_PKEYS;
352 attr->max_ah = dev->attrs.max_ah;
353 }
354
355 if (dev->attrs.cap_flags & ERDMA_DEV_CAP_FLAGS_ATOMIC)
356 attr->atomic_cap = IB_ATOMIC_GLOB;
357
358 attr->fw_ver = dev->attrs.fw_version;
359
360 if (dev->netdev)
361 addrconf_addr_eui48((u8 *)&attr->sys_image_guid,
362 dev->netdev->dev_addr);
363
364 return 0;
365 }
366
erdma_query_gid(struct ib_device * ibdev,u32 port,int idx,union ib_gid * gid)367 int erdma_query_gid(struct ib_device *ibdev, u32 port, int idx,
368 union ib_gid *gid)
369 {
370 struct erdma_dev *dev = to_edev(ibdev);
371
372 memset(gid, 0, sizeof(*gid));
373 ether_addr_copy(gid->raw, dev->attrs.peer_addr);
374
375 return 0;
376 }
377
erdma_query_port(struct ib_device * ibdev,u32 port,struct ib_port_attr * attr)378 int erdma_query_port(struct ib_device *ibdev, u32 port,
379 struct ib_port_attr *attr)
380 {
381 struct erdma_dev *dev = to_edev(ibdev);
382 struct net_device *ndev = dev->netdev;
383
384 memset(attr, 0, sizeof(*attr));
385
386 if (erdma_device_iwarp(dev)) {
387 attr->gid_tbl_len = 1;
388 } else {
389 attr->gid_tbl_len = dev->attrs.max_gid;
390 attr->ip_gids = true;
391 attr->pkey_tbl_len = ERDMA_MAX_PKEYS;
392 }
393
394 attr->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_DEVICE_MGMT_SUP;
395 attr->max_msg_sz = -1;
396
397 if (!ndev)
398 goto out;
399
400 ib_get_eth_speed(ibdev, port, &attr->active_speed, &attr->active_width);
401 attr->max_mtu = ib_mtu_int_to_enum(ndev->mtu);
402 attr->active_mtu = ib_mtu_int_to_enum(ndev->mtu);
403 attr->state = ib_get_curr_port_state(ndev);
404
405 out:
406 if (attr->state == IB_PORT_ACTIVE)
407 attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP;
408 else
409 attr->phys_state = IB_PORT_PHYS_STATE_DISABLED;
410
411 return 0;
412 }
413
erdma_get_port_immutable(struct ib_device * ibdev,u32 port,struct ib_port_immutable * port_immutable)414 int erdma_get_port_immutable(struct ib_device *ibdev, u32 port,
415 struct ib_port_immutable *port_immutable)
416 {
417 struct erdma_dev *dev = to_edev(ibdev);
418
419 if (erdma_device_iwarp(dev)) {
420 port_immutable->core_cap_flags = RDMA_CORE_PORT_IWARP;
421 port_immutable->gid_tbl_len = 1;
422 } else {
423 port_immutable->core_cap_flags =
424 RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
425 port_immutable->max_mad_size = IB_MGMT_MAD_SIZE;
426 port_immutable->gid_tbl_len = dev->attrs.max_gid;
427 port_immutable->pkey_tbl_len = ERDMA_MAX_PKEYS;
428 }
429
430 return 0;
431 }
432
erdma_alloc_pd(struct ib_pd * ibpd,struct ib_udata * udata)433 int erdma_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
434 {
435 struct erdma_pd *pd = to_epd(ibpd);
436 struct erdma_dev *dev = to_edev(ibpd->device);
437 int pdn;
438
439 pdn = erdma_alloc_idx(&dev->res_cb[ERDMA_RES_TYPE_PD]);
440 if (pdn < 0)
441 return pdn;
442
443 pd->pdn = pdn;
444
445 return 0;
446 }
447
erdma_dealloc_pd(struct ib_pd * ibpd,struct ib_udata * udata)448 int erdma_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
449 {
450 struct erdma_pd *pd = to_epd(ibpd);
451 struct erdma_dev *dev = to_edev(ibpd->device);
452
453 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_PD], pd->pdn);
454
455 return 0;
456 }
457
erdma_flush_worker(struct work_struct * work)458 static void erdma_flush_worker(struct work_struct *work)
459 {
460 struct delayed_work *dwork = to_delayed_work(work);
461 struct erdma_qp *qp =
462 container_of(dwork, struct erdma_qp, reflush_dwork);
463 struct erdma_cmdq_reflush_req req;
464
465 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
466 CMDQ_OPCODE_REFLUSH);
467 req.qpn = QP_ID(qp);
468 req.sq_pi = qp->kern_qp.sq_pi;
469 req.rq_pi = qp->kern_qp.rq_pi;
470 erdma_post_cmd_wait(&qp->dev->cmdq, &req, sizeof(req), NULL, NULL,
471 true);
472 }
473
erdma_qp_validate_cap(struct erdma_dev * dev,struct ib_qp_init_attr * attrs)474 static int erdma_qp_validate_cap(struct erdma_dev *dev,
475 struct ib_qp_init_attr *attrs)
476 {
477 if ((attrs->cap.max_send_wr > dev->attrs.max_send_wr) ||
478 (attrs->cap.max_recv_wr > dev->attrs.max_recv_wr) ||
479 (attrs->cap.max_send_sge > dev->attrs.max_send_sge) ||
480 (attrs->cap.max_recv_sge > dev->attrs.max_recv_sge) ||
481 (attrs->cap.max_inline_data > ERDMA_MAX_INLINE) ||
482 !attrs->cap.max_send_wr || !attrs->cap.max_recv_wr) {
483 return -EINVAL;
484 }
485
486 return 0;
487 }
488
erdma_qp_validate_attr(struct erdma_dev * dev,struct ib_qp_init_attr * attrs)489 static int erdma_qp_validate_attr(struct erdma_dev *dev,
490 struct ib_qp_init_attr *attrs)
491 {
492 if (erdma_device_iwarp(dev) && attrs->qp_type != IB_QPT_RC)
493 return -EOPNOTSUPP;
494
495 if (erdma_device_rocev2(dev) && attrs->qp_type != IB_QPT_RC &&
496 attrs->qp_type != IB_QPT_UD && attrs->qp_type != IB_QPT_GSI)
497 return -EOPNOTSUPP;
498
499 if (attrs->srq)
500 return -EOPNOTSUPP;
501
502 if (!attrs->send_cq || !attrs->recv_cq)
503 return -EOPNOTSUPP;
504
505 return 0;
506 }
507
free_kernel_qp(struct erdma_qp * qp)508 static void free_kernel_qp(struct erdma_qp *qp)
509 {
510 struct erdma_dev *dev = qp->dev;
511
512 vfree(qp->kern_qp.swr_tbl);
513 vfree(qp->kern_qp.rwr_tbl);
514
515 if (qp->kern_qp.sq_buf)
516 dma_free_coherent(&dev->pdev->dev,
517 qp->attrs.sq_size << SQEBB_SHIFT,
518 qp->kern_qp.sq_buf,
519 qp->kern_qp.sq_buf_dma_addr);
520
521 if (qp->kern_qp.sq_dbrec)
522 dma_pool_free(dev->db_pool, qp->kern_qp.sq_dbrec,
523 qp->kern_qp.sq_dbrec_dma);
524
525 if (qp->kern_qp.rq_buf)
526 dma_free_coherent(&dev->pdev->dev,
527 qp->attrs.rq_size << RQE_SHIFT,
528 qp->kern_qp.rq_buf,
529 qp->kern_qp.rq_buf_dma_addr);
530
531 if (qp->kern_qp.rq_dbrec)
532 dma_pool_free(dev->db_pool, qp->kern_qp.rq_dbrec,
533 qp->kern_qp.rq_dbrec_dma);
534 }
535
init_kernel_qp(struct erdma_dev * dev,struct erdma_qp * qp,struct ib_qp_init_attr * attrs)536 static int init_kernel_qp(struct erdma_dev *dev, struct erdma_qp *qp,
537 struct ib_qp_init_attr *attrs)
538 {
539 struct erdma_kqp *kqp = &qp->kern_qp;
540 int size;
541
542 if (attrs->sq_sig_type == IB_SIGNAL_ALL_WR)
543 kqp->sig_all = 1;
544
545 kqp->sq_pi = 0;
546 kqp->sq_ci = 0;
547 kqp->rq_pi = 0;
548 kqp->rq_ci = 0;
549 kqp->hw_sq_db =
550 dev->func_bar + (ERDMA_SDB_SHARED_PAGE_INDEX << PAGE_SHIFT);
551 kqp->hw_rq_db = dev->func_bar + ERDMA_BAR_RQDB_SPACE_OFFSET;
552
553 kqp->swr_tbl = vmalloc_array(qp->attrs.sq_size, sizeof(u64));
554 kqp->rwr_tbl = vmalloc_array(qp->attrs.rq_size, sizeof(u64));
555 if (!kqp->swr_tbl || !kqp->rwr_tbl)
556 goto err_out;
557
558 size = qp->attrs.sq_size << SQEBB_SHIFT;
559 kqp->sq_buf = dma_alloc_coherent(&dev->pdev->dev, size,
560 &kqp->sq_buf_dma_addr, GFP_KERNEL);
561 if (!kqp->sq_buf)
562 goto err_out;
563
564 kqp->sq_dbrec =
565 dma_pool_zalloc(dev->db_pool, GFP_KERNEL, &kqp->sq_dbrec_dma);
566 if (!kqp->sq_dbrec)
567 goto err_out;
568
569 size = qp->attrs.rq_size << RQE_SHIFT;
570 kqp->rq_buf = dma_alloc_coherent(&dev->pdev->dev, size,
571 &kqp->rq_buf_dma_addr, GFP_KERNEL);
572 if (!kqp->rq_buf)
573 goto err_out;
574
575 kqp->rq_dbrec =
576 dma_pool_zalloc(dev->db_pool, GFP_KERNEL, &kqp->rq_dbrec_dma);
577 if (!kqp->rq_dbrec)
578 goto err_out;
579
580 return 0;
581
582 err_out:
583 free_kernel_qp(qp);
584 return -ENOMEM;
585 }
586
erdma_fill_bottom_mtt(struct erdma_dev * dev,struct erdma_mem * mem)587 static void erdma_fill_bottom_mtt(struct erdma_dev *dev, struct erdma_mem *mem)
588 {
589 struct erdma_mtt *mtt = mem->mtt;
590 struct ib_block_iter biter;
591 u32 idx = 0;
592
593 while (mtt->low_level)
594 mtt = mtt->low_level;
595
596 rdma_umem_for_each_dma_block(mem->umem, &biter, mem->page_size)
597 mtt->buf[idx++] = rdma_block_iter_dma_address(&biter);
598 }
599
erdma_create_cont_mtt(struct erdma_dev * dev,size_t size)600 static struct erdma_mtt *erdma_create_cont_mtt(struct erdma_dev *dev,
601 size_t size)
602 {
603 struct erdma_mtt *mtt;
604
605 mtt = kzalloc_obj(*mtt);
606 if (!mtt)
607 return ERR_PTR(-ENOMEM);
608
609 mtt->size = size;
610 mtt->buf = kzalloc(mtt->size, GFP_KERNEL);
611 if (!mtt->buf)
612 goto err_free_mtt;
613
614 mtt->continuous = true;
615 mtt->buf_dma = dma_map_single(&dev->pdev->dev, mtt->buf, mtt->size,
616 DMA_TO_DEVICE);
617 if (dma_mapping_error(&dev->pdev->dev, mtt->buf_dma))
618 goto err_free_mtt_buf;
619
620 return mtt;
621
622 err_free_mtt_buf:
623 kfree(mtt->buf);
624
625 err_free_mtt:
626 kfree(mtt);
627
628 return ERR_PTR(-ENOMEM);
629 }
630
erdma_unmap_page_list(struct erdma_dev * dev,dma_addr_t * pg_dma,u32 npages)631 static void erdma_unmap_page_list(struct erdma_dev *dev, dma_addr_t *pg_dma,
632 u32 npages)
633 {
634 u32 i;
635
636 for (i = 0; i < npages; i++)
637 dma_unmap_page(&dev->pdev->dev, pg_dma[i], PAGE_SIZE,
638 DMA_TO_DEVICE);
639 }
640
erdma_destroy_mtt_buf_dma_addrs(struct erdma_dev * dev,struct erdma_mtt * mtt)641 static void erdma_destroy_mtt_buf_dma_addrs(struct erdma_dev *dev,
642 struct erdma_mtt *mtt)
643 {
644 erdma_unmap_page_list(dev, mtt->dma_addrs, mtt->npages);
645 vfree(mtt->dma_addrs);
646 }
647
erdma_destroy_scatter_mtt(struct erdma_dev * dev,struct erdma_mtt * mtt)648 static void erdma_destroy_scatter_mtt(struct erdma_dev *dev,
649 struct erdma_mtt *mtt)
650 {
651 erdma_destroy_mtt_buf_dma_addrs(dev, mtt);
652 vfree(mtt->buf);
653 kfree(mtt);
654 }
655
erdma_init_middle_mtt(struct erdma_mtt * mtt,struct erdma_mtt * low_mtt)656 static void erdma_init_middle_mtt(struct erdma_mtt *mtt,
657 struct erdma_mtt *low_mtt)
658 {
659 dma_addr_t *pg_addr = mtt->buf;
660 u32 i;
661
662 for (i = 0; i < low_mtt->npages; i++)
663 pg_addr[i] = low_mtt->dma_addrs[i];
664 }
665
vmalloc_to_dma_addrs(struct erdma_dev * dev,dma_addr_t ** dma_addrs,void * buf,u64 len)666 static u32 vmalloc_to_dma_addrs(struct erdma_dev *dev, dma_addr_t **dma_addrs,
667 void *buf, u64 len)
668 {
669 dma_addr_t *pg_dma;
670 struct page *pg;
671 u32 npages, i;
672 void *addr;
673
674 npages = (PAGE_ALIGN((u64)buf + len) - PAGE_ALIGN_DOWN((u64)buf)) >>
675 PAGE_SHIFT;
676 pg_dma = vcalloc(npages, sizeof(*pg_dma));
677 if (!pg_dma)
678 return 0;
679
680 addr = buf;
681 for (i = 0; i < npages; i++) {
682 pg = vmalloc_to_page(addr);
683 if (!pg)
684 goto err;
685
686 pg_dma[i] = dma_map_page(&dev->pdev->dev, pg, 0, PAGE_SIZE,
687 DMA_TO_DEVICE);
688 if (dma_mapping_error(&dev->pdev->dev, pg_dma[i]))
689 goto err;
690
691 addr += PAGE_SIZE;
692 }
693
694 *dma_addrs = pg_dma;
695
696 return npages;
697 err:
698 erdma_unmap_page_list(dev, pg_dma, i);
699 vfree(pg_dma);
700
701 return 0;
702 }
703
erdma_create_mtt_buf_dma_addrs(struct erdma_dev * dev,struct erdma_mtt * mtt)704 static int erdma_create_mtt_buf_dma_addrs(struct erdma_dev *dev,
705 struct erdma_mtt *mtt)
706 {
707 dma_addr_t *addrs;
708 u32 npages;
709
710 /* Failed if buf is not page aligned */
711 if ((uintptr_t)mtt->buf & ~PAGE_MASK)
712 return -EINVAL;
713
714 npages = vmalloc_to_dma_addrs(dev, &addrs, mtt->buf, mtt->size);
715 if (!npages)
716 return -ENOMEM;
717
718 mtt->dma_addrs = addrs;
719 mtt->npages = npages;
720
721 return 0;
722 }
723
erdma_create_scatter_mtt(struct erdma_dev * dev,size_t size)724 static struct erdma_mtt *erdma_create_scatter_mtt(struct erdma_dev *dev,
725 size_t size)
726 {
727 struct erdma_mtt *mtt;
728 int ret = -ENOMEM;
729
730 mtt = kzalloc_obj(*mtt);
731 if (!mtt)
732 return ERR_PTR(-ENOMEM);
733
734 mtt->size = ALIGN(size, PAGE_SIZE);
735 mtt->buf = vzalloc(mtt->size);
736 mtt->continuous = false;
737 if (!mtt->buf)
738 goto err_free_mtt;
739
740 ret = erdma_create_mtt_buf_dma_addrs(dev, mtt);
741 if (ret)
742 goto err_free_mtt_buf;
743
744 ibdev_dbg(&dev->ibdev, "create scatter mtt, size:%lu, npages:%u\n",
745 mtt->size, mtt->npages);
746
747 return mtt;
748
749 err_free_mtt_buf:
750 vfree(mtt->buf);
751
752 err_free_mtt:
753 kfree(mtt);
754
755 return ERR_PTR(ret);
756 }
757
erdma_create_mtt(struct erdma_dev * dev,size_t size,bool force_continuous)758 static struct erdma_mtt *erdma_create_mtt(struct erdma_dev *dev, size_t size,
759 bool force_continuous)
760 {
761 struct erdma_mtt *mtt, *tmp_mtt;
762 int ret, level = 0;
763
764 ibdev_dbg(&dev->ibdev, "create_mtt, size:%lu, force cont:%d\n", size,
765 force_continuous);
766
767 if (!(dev->attrs.cap_flags & ERDMA_DEV_CAP_FLAGS_MTT_VA))
768 force_continuous = true;
769
770 if (force_continuous)
771 return erdma_create_cont_mtt(dev, size);
772
773 mtt = erdma_create_scatter_mtt(dev, size);
774 if (IS_ERR(mtt))
775 return mtt;
776 level = 1;
777
778 /* convergence the mtt table. */
779 while (mtt->npages != 1 && level <= 3) {
780 tmp_mtt = erdma_create_scatter_mtt(dev, MTT_SIZE(mtt->npages));
781 if (IS_ERR(tmp_mtt)) {
782 ret = PTR_ERR(tmp_mtt);
783 goto err_free_mtt;
784 }
785 erdma_init_middle_mtt(tmp_mtt, mtt);
786 tmp_mtt->low_level = mtt;
787 mtt = tmp_mtt;
788 level++;
789 }
790
791 if (level > 3) {
792 ret = -ENOMEM;
793 goto err_free_mtt;
794 }
795
796 mtt->level = level;
797 ibdev_dbg(&dev->ibdev, "top mtt: level:%d, dma_addr 0x%llx\n",
798 mtt->level, mtt->dma_addrs[0]);
799
800 return mtt;
801 err_free_mtt:
802 while (mtt) {
803 tmp_mtt = mtt->low_level;
804 erdma_destroy_scatter_mtt(dev, mtt);
805 mtt = tmp_mtt;
806 }
807
808 return ERR_PTR(ret);
809 }
810
erdma_destroy_mtt(struct erdma_dev * dev,struct erdma_mtt * mtt)811 static void erdma_destroy_mtt(struct erdma_dev *dev, struct erdma_mtt *mtt)
812 {
813 struct erdma_mtt *tmp_mtt;
814
815 if (mtt->continuous) {
816 dma_unmap_single(&dev->pdev->dev, mtt->buf_dma, mtt->size,
817 DMA_TO_DEVICE);
818 kfree(mtt->buf);
819 kfree(mtt);
820 } else {
821 while (mtt) {
822 tmp_mtt = mtt->low_level;
823 erdma_destroy_scatter_mtt(dev, mtt);
824 mtt = tmp_mtt;
825 }
826 }
827 }
828
get_mtt_entries(struct erdma_dev * dev,struct erdma_mem * mem,u64 start,u64 len,int access,u64 virt,unsigned long req_page_size,bool force_continuous)829 static int get_mtt_entries(struct erdma_dev *dev, struct erdma_mem *mem,
830 u64 start, u64 len, int access, u64 virt,
831 unsigned long req_page_size, bool force_continuous)
832 {
833 int ret = 0;
834
835 mem->umem = ib_umem_get_va(&dev->ibdev, start, len, access);
836 if (IS_ERR(mem->umem)) {
837 ret = PTR_ERR(mem->umem);
838 mem->umem = NULL;
839 return ret;
840 }
841
842 mem->va = virt;
843 mem->len = len;
844 mem->page_size = ib_umem_find_best_pgsz(mem->umem, req_page_size, virt);
845 mem->page_offset = start & (mem->page_size - 1);
846 mem->mtt_nents = ib_umem_num_dma_blocks(mem->umem, mem->page_size);
847 mem->page_cnt = mem->mtt_nents;
848 mem->mtt = erdma_create_mtt(dev, MTT_SIZE(mem->page_cnt),
849 force_continuous);
850 if (IS_ERR(mem->mtt)) {
851 ret = PTR_ERR(mem->mtt);
852 goto error_ret;
853 }
854
855 erdma_fill_bottom_mtt(dev, mem);
856
857 return 0;
858
859 error_ret:
860 if (mem->umem) {
861 ib_umem_release(mem->umem);
862 mem->umem = NULL;
863 }
864
865 return ret;
866 }
867
put_mtt_entries(struct erdma_dev * dev,struct erdma_mem * mem)868 static void put_mtt_entries(struct erdma_dev *dev, struct erdma_mem *mem)
869 {
870 if (mem->mtt)
871 erdma_destroy_mtt(dev, mem->mtt);
872
873 if (mem->umem) {
874 ib_umem_release(mem->umem);
875 mem->umem = NULL;
876 }
877 }
878
erdma_map_user_dbrecords(struct erdma_ucontext * ctx,u64 dbrecords_va,struct erdma_user_dbrecords_page ** dbr_page,dma_addr_t * dma_addr)879 static int erdma_map_user_dbrecords(struct erdma_ucontext *ctx,
880 u64 dbrecords_va,
881 struct erdma_user_dbrecords_page **dbr_page,
882 dma_addr_t *dma_addr)
883 {
884 struct erdma_user_dbrecords_page *page = NULL;
885 int rv = 0;
886
887 mutex_lock(&ctx->dbrecords_page_mutex);
888
889 list_for_each_entry(page, &ctx->dbrecords_page_list, list)
890 if (page->va == (dbrecords_va & PAGE_MASK))
891 goto found;
892
893 page = kmalloc_obj(*page);
894 if (!page) {
895 rv = -ENOMEM;
896 goto out;
897 }
898
899 page->va = (dbrecords_va & PAGE_MASK);
900 page->refcnt = 0;
901
902 page->umem = ib_umem_get_va(ctx->ibucontext.device,
903 dbrecords_va & PAGE_MASK, PAGE_SIZE, 0);
904 if (IS_ERR(page->umem)) {
905 rv = PTR_ERR(page->umem);
906 kfree(page);
907 goto out;
908 }
909
910 list_add(&page->list, &ctx->dbrecords_page_list);
911
912 found:
913 *dma_addr = sg_dma_address(page->umem->sgt_append.sgt.sgl) +
914 (dbrecords_va & ~PAGE_MASK);
915 *dbr_page = page;
916 page->refcnt++;
917
918 out:
919 mutex_unlock(&ctx->dbrecords_page_mutex);
920 return rv;
921 }
922
923 static void
erdma_unmap_user_dbrecords(struct erdma_ucontext * ctx,struct erdma_user_dbrecords_page ** dbr_page)924 erdma_unmap_user_dbrecords(struct erdma_ucontext *ctx,
925 struct erdma_user_dbrecords_page **dbr_page)
926 {
927 if (!ctx || !(*dbr_page))
928 return;
929
930 mutex_lock(&ctx->dbrecords_page_mutex);
931 if (--(*dbr_page)->refcnt == 0) {
932 list_del(&(*dbr_page)->list);
933 ib_umem_release((*dbr_page)->umem);
934 kfree(*dbr_page);
935 }
936
937 *dbr_page = NULL;
938 mutex_unlock(&ctx->dbrecords_page_mutex);
939 }
940
init_user_qp(struct erdma_qp * qp,struct erdma_ucontext * uctx,u64 va,u32 len,u64 dbrec_va)941 static int init_user_qp(struct erdma_qp *qp, struct erdma_ucontext *uctx,
942 u64 va, u32 len, u64 dbrec_va)
943 {
944 dma_addr_t dbrec_dma;
945 u32 rq_offset;
946 int ret;
947
948 if (len < (ALIGN(qp->attrs.sq_size * SQEBB_SIZE, ERDMA_HW_PAGE_SIZE) +
949 qp->attrs.rq_size * RQE_SIZE))
950 return -EINVAL;
951
952 ret = get_mtt_entries(qp->dev, &qp->user_qp.sq_mem, va,
953 qp->attrs.sq_size << SQEBB_SHIFT, 0, va,
954 (SZ_1M - SZ_4K), true);
955 if (ret)
956 return ret;
957
958 rq_offset = ALIGN(qp->attrs.sq_size << SQEBB_SHIFT, ERDMA_HW_PAGE_SIZE);
959 qp->user_qp.rq_offset = rq_offset;
960
961 ret = get_mtt_entries(qp->dev, &qp->user_qp.rq_mem, va + rq_offset,
962 qp->attrs.rq_size << RQE_SHIFT, 0, va + rq_offset,
963 (SZ_1M - SZ_4K), true);
964 if (ret)
965 goto put_sq_mtt;
966
967 ret = erdma_map_user_dbrecords(uctx, dbrec_va,
968 &qp->user_qp.user_dbr_page,
969 &dbrec_dma);
970 if (ret)
971 goto put_rq_mtt;
972
973 qp->user_qp.sq_dbrec_dma = dbrec_dma;
974 qp->user_qp.rq_dbrec_dma = dbrec_dma + ERDMA_DB_SIZE;
975
976 return 0;
977
978 put_rq_mtt:
979 put_mtt_entries(qp->dev, &qp->user_qp.rq_mem);
980
981 put_sq_mtt:
982 put_mtt_entries(qp->dev, &qp->user_qp.sq_mem);
983
984 return ret;
985 }
986
free_user_qp(struct erdma_qp * qp,struct erdma_ucontext * uctx)987 static void free_user_qp(struct erdma_qp *qp, struct erdma_ucontext *uctx)
988 {
989 put_mtt_entries(qp->dev, &qp->user_qp.sq_mem);
990 put_mtt_entries(qp->dev, &qp->user_qp.rq_mem);
991 erdma_unmap_user_dbrecords(uctx, &qp->user_qp.user_dbr_page);
992 }
993
erdma_create_qp(struct ib_qp * ibqp,struct ib_qp_init_attr * attrs,struct ib_udata * udata)994 int erdma_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs,
995 struct ib_udata *udata)
996 {
997 struct erdma_qp *qp = to_eqp(ibqp);
998 struct erdma_dev *dev = to_edev(ibqp->device);
999 struct erdma_ucontext *uctx = rdma_udata_to_drv_context(
1000 udata, struct erdma_ucontext, ibucontext);
1001 struct erdma_ureq_create_qp ureq;
1002 struct erdma_uresp_create_qp uresp = {};
1003 void *old_entry;
1004 int ret = 0;
1005
1006 ret = erdma_qp_validate_cap(dev, attrs);
1007 if (ret)
1008 goto err_out;
1009
1010 ret = erdma_qp_validate_attr(dev, attrs);
1011 if (ret)
1012 goto err_out;
1013
1014 qp->scq = to_ecq(attrs->send_cq);
1015 qp->rcq = to_ecq(attrs->recv_cq);
1016 qp->dev = dev;
1017 qp->attrs.cc = dev->attrs.cc;
1018
1019 init_rwsem(&qp->state_lock);
1020 kref_init(&qp->ref);
1021 init_completion(&qp->safe_free);
1022
1023 if (qp->ibqp.qp_type == IB_QPT_GSI) {
1024 old_entry = xa_store_irq(&dev->qp_xa, 1, qp, GFP_KERNEL);
1025 if (xa_is_err(old_entry))
1026 ret = xa_err(old_entry);
1027 else
1028 qp->ibqp.qp_num = 1;
1029 } else {
1030 ret = xa_alloc_cyclic_irq(&dev->qp_xa, &qp->ibqp.qp_num, qp,
1031 XA_LIMIT(1, dev->attrs.max_qp - 1),
1032 &dev->next_alloc_qpn, GFP_KERNEL);
1033 }
1034
1035 if (ret < 0) {
1036 ret = -ENOMEM;
1037 goto err_out;
1038 }
1039
1040 qp->attrs.sq_size = roundup_pow_of_two(attrs->cap.max_send_wr *
1041 ERDMA_MAX_WQEBB_PER_SQE);
1042 qp->attrs.rq_size = roundup_pow_of_two(attrs->cap.max_recv_wr);
1043
1044 if (uctx) {
1045 ret = ib_copy_validate_udata_in(udata, ureq, rsvd0);
1046 if (ret)
1047 goto err_out_xa;
1048
1049 ret = init_user_qp(qp, uctx, ureq.qbuf_va, ureq.qbuf_len,
1050 ureq.db_record_va);
1051 if (ret)
1052 goto err_out_xa;
1053
1054 uresp.num_sqe = qp->attrs.sq_size;
1055 uresp.num_rqe = qp->attrs.rq_size;
1056 uresp.qp_id = QP_ID(qp);
1057 uresp.rq_offset = qp->user_qp.rq_offset;
1058
1059 ret = ib_respond_udata(udata, uresp);
1060 if (ret)
1061 goto err_out_cmd;
1062 } else {
1063 ret = init_kernel_qp(dev, qp, attrs);
1064 if (ret)
1065 goto err_out_xa;
1066 }
1067
1068 qp->attrs.max_send_sge = attrs->cap.max_send_sge;
1069 qp->attrs.max_recv_sge = attrs->cap.max_recv_sge;
1070
1071 if (erdma_device_iwarp(qp->dev))
1072 qp->attrs.iwarp.state = ERDMA_QPS_IWARP_IDLE;
1073 else
1074 qp->attrs.rocev2.state = ERDMA_QPS_ROCEV2_RESET;
1075
1076 INIT_DELAYED_WORK(&qp->reflush_dwork, erdma_flush_worker);
1077
1078 ret = create_qp_cmd(uctx, qp);
1079 if (ret)
1080 goto err_out_cmd;
1081
1082 spin_lock_init(&qp->lock);
1083
1084 return 0;
1085
1086 err_out_cmd:
1087 if (uctx)
1088 free_user_qp(qp, uctx);
1089 else
1090 free_kernel_qp(qp);
1091 err_out_xa:
1092 xa_erase_irq(&dev->qp_xa, QP_ID(qp));
1093 err_out:
1094 return ret;
1095 }
1096
erdma_create_stag(struct erdma_dev * dev,u32 * stag)1097 static int erdma_create_stag(struct erdma_dev *dev, u32 *stag)
1098 {
1099 int stag_idx;
1100
1101 stag_idx = erdma_alloc_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX]);
1102 if (stag_idx < 0)
1103 return stag_idx;
1104
1105 /* For now, we always let key field be zero. */
1106 *stag = (stag_idx << 8);
1107
1108 return 0;
1109 }
1110
erdma_get_dma_mr(struct ib_pd * ibpd,int acc)1111 struct ib_mr *erdma_get_dma_mr(struct ib_pd *ibpd, int acc)
1112 {
1113 struct erdma_dev *dev = to_edev(ibpd->device);
1114 struct erdma_mr *mr;
1115 u32 stag;
1116 int ret;
1117
1118 mr = kzalloc_obj(*mr);
1119 if (!mr)
1120 return ERR_PTR(-ENOMEM);
1121
1122 ret = erdma_create_stag(dev, &stag);
1123 if (ret)
1124 goto out_free;
1125
1126 mr->type = ERDMA_MR_TYPE_DMA;
1127
1128 mr->ibmr.lkey = stag;
1129 mr->ibmr.rkey = stag;
1130 mr->ibmr.pd = ibpd;
1131 mr->access = ERDMA_MR_ACC_LR | to_erdma_access_flags(acc);
1132 ret = regmr_cmd(dev, mr);
1133 if (ret)
1134 goto out_remove_stag;
1135
1136 return &mr->ibmr;
1137
1138 out_remove_stag:
1139 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX],
1140 mr->ibmr.lkey >> 8);
1141
1142 out_free:
1143 kfree(mr);
1144
1145 return ERR_PTR(ret);
1146 }
1147
erdma_ib_alloc_mr(struct ib_pd * ibpd,enum ib_mr_type mr_type,u32 max_num_sg)1148 struct ib_mr *erdma_ib_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
1149 u32 max_num_sg)
1150 {
1151 struct erdma_mr *mr;
1152 struct erdma_dev *dev = to_edev(ibpd->device);
1153 int ret;
1154 u32 stag;
1155
1156 if (mr_type != IB_MR_TYPE_MEM_REG)
1157 return ERR_PTR(-EOPNOTSUPP);
1158
1159 if (max_num_sg > ERDMA_MR_MAX_MTT_CNT)
1160 return ERR_PTR(-EINVAL);
1161
1162 mr = kzalloc_obj(*mr);
1163 if (!mr)
1164 return ERR_PTR(-ENOMEM);
1165
1166 ret = erdma_create_stag(dev, &stag);
1167 if (ret)
1168 goto out_free;
1169
1170 mr->type = ERDMA_MR_TYPE_FRMR;
1171
1172 mr->ibmr.lkey = stag;
1173 mr->ibmr.rkey = stag;
1174 mr->ibmr.pd = ibpd;
1175 /* update it in FRMR. */
1176 mr->access = ERDMA_MR_ACC_LR | ERDMA_MR_ACC_LW | ERDMA_MR_ACC_RR |
1177 ERDMA_MR_ACC_RW;
1178
1179 mr->mem.page_size = PAGE_SIZE; /* update it later. */
1180 mr->mem.page_cnt = max_num_sg;
1181 mr->mem.mtt = erdma_create_mtt(dev, MTT_SIZE(max_num_sg), true);
1182 if (IS_ERR(mr->mem.mtt)) {
1183 ret = PTR_ERR(mr->mem.mtt);
1184 goto out_remove_stag;
1185 }
1186
1187 ret = regmr_cmd(dev, mr);
1188 if (ret)
1189 goto out_destroy_mtt;
1190
1191 return &mr->ibmr;
1192
1193 out_destroy_mtt:
1194 erdma_destroy_mtt(dev, mr->mem.mtt);
1195
1196 out_remove_stag:
1197 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX],
1198 mr->ibmr.lkey >> 8);
1199
1200 out_free:
1201 kfree(mr);
1202
1203 return ERR_PTR(ret);
1204 }
1205
erdma_set_page(struct ib_mr * ibmr,u64 addr)1206 static int erdma_set_page(struct ib_mr *ibmr, u64 addr)
1207 {
1208 struct erdma_mr *mr = to_emr(ibmr);
1209
1210 if (mr->mem.mtt_nents >= mr->mem.page_cnt)
1211 return -1;
1212
1213 mr->mem.mtt->buf[mr->mem.mtt_nents] = addr;
1214 mr->mem.mtt_nents++;
1215
1216 return 0;
1217 }
1218
erdma_map_mr_sg(struct ib_mr * ibmr,struct scatterlist * sg,int sg_nents,unsigned int * sg_offset)1219 int erdma_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
1220 unsigned int *sg_offset)
1221 {
1222 struct erdma_mr *mr = to_emr(ibmr);
1223 int num;
1224
1225 mr->mem.mtt_nents = 0;
1226
1227 num = ib_sg_to_pages(&mr->ibmr, sg, sg_nents, sg_offset,
1228 erdma_set_page);
1229
1230 return num;
1231 }
1232
erdma_reg_user_mr(struct ib_pd * ibpd,u64 start,u64 len,u64 virt,int access,struct ib_dmah * dmah,struct ib_udata * udata)1233 struct ib_mr *erdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
1234 u64 virt, int access, struct ib_dmah *dmah,
1235 struct ib_udata *udata)
1236 {
1237 struct erdma_mr *mr = NULL;
1238 struct erdma_dev *dev = to_edev(ibpd->device);
1239 u32 stag;
1240 int ret;
1241
1242 if (dmah)
1243 return ERR_PTR(-EOPNOTSUPP);
1244
1245 if (!len || len > dev->attrs.max_mr_size)
1246 return ERR_PTR(-EINVAL);
1247
1248 mr = kzalloc_obj(*mr);
1249 if (!mr)
1250 return ERR_PTR(-ENOMEM);
1251
1252 ret = get_mtt_entries(dev, &mr->mem, start, len, access, virt,
1253 SZ_2G - SZ_4K, false);
1254 if (ret)
1255 goto err_out_free;
1256
1257 ret = erdma_create_stag(dev, &stag);
1258 if (ret)
1259 goto err_out_put_mtt;
1260
1261 mr->ibmr.lkey = mr->ibmr.rkey = stag;
1262 mr->ibmr.pd = ibpd;
1263 mr->mem.va = virt;
1264 mr->mem.len = len;
1265 mr->access = ERDMA_MR_ACC_LR | to_erdma_access_flags(access);
1266 mr->valid = 1;
1267 mr->type = ERDMA_MR_TYPE_NORMAL;
1268
1269 ret = regmr_cmd(dev, mr);
1270 if (ret)
1271 goto err_out_mr;
1272
1273 return &mr->ibmr;
1274
1275 err_out_mr:
1276 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX],
1277 mr->ibmr.lkey >> 8);
1278
1279 err_out_put_mtt:
1280 put_mtt_entries(dev, &mr->mem);
1281
1282 err_out_free:
1283 kfree(mr);
1284
1285 return ERR_PTR(ret);
1286 }
1287
erdma_dereg_mr(struct ib_mr * ibmr,struct ib_udata * udata)1288 int erdma_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)
1289 {
1290 struct erdma_mr *mr;
1291 struct erdma_dev *dev = to_edev(ibmr->device);
1292 struct erdma_cmdq_dereg_mr_req req;
1293 int ret;
1294
1295 mr = to_emr(ibmr);
1296
1297 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
1298 CMDQ_OPCODE_DEREG_MR);
1299
1300 req.cfg = FIELD_PREP(ERDMA_CMD_MR_MPT_IDX_MASK, ibmr->lkey >> 8) |
1301 FIELD_PREP(ERDMA_CMD_MR_KEY_MASK, ibmr->lkey & 0xFF);
1302
1303 ret = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL,
1304 true);
1305 /*
1306 * A timeout disables the command queue, so retry cannot succeed. Treat
1307 * terminal command failures as diagnostic; propagating them can make
1308 * forced uverbs cleanup discard the last software resource pointers.
1309 */
1310 if (ret)
1311 ibdev_warn_ratelimited(&dev->ibdev,
1312 "failed to deregister MR 0x%x: %d\n",
1313 ibmr->lkey, ret);
1314
1315 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX], ibmr->lkey >> 8);
1316
1317 put_mtt_entries(dev, &mr->mem);
1318
1319 kfree(mr);
1320 return 0;
1321 }
1322
erdma_destroy_cq(struct ib_cq * ibcq,struct ib_udata * udata)1323 int erdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
1324 {
1325 struct erdma_cq *cq = to_ecq(ibcq);
1326 struct erdma_dev *dev = to_edev(ibcq->device);
1327 struct erdma_ucontext *ctx = rdma_udata_to_drv_context(
1328 udata, struct erdma_ucontext, ibucontext);
1329 unsigned long flags;
1330 int err;
1331 struct erdma_cmdq_destroy_cq_req req;
1332
1333 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
1334 CMDQ_OPCODE_DESTROY_CQ);
1335 req.cqn = cq->cqn;
1336
1337 err = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL,
1338 true);
1339 if (err)
1340 ibdev_warn_ratelimited(&dev->ibdev,
1341 "failed to destroy CQ %u: %d\n",
1342 cq->cqn, err);
1343
1344 xa_lock_irqsave(&dev->cq_xa, flags);
1345 __xa_erase(&dev->cq_xa, cq->cqn);
1346 xa_unlock_irqrestore(&dev->cq_xa, flags);
1347
1348 erdma_cq_put(cq);
1349 wait_for_completion(&cq->free);
1350
1351 if (rdma_is_kernel_res(&cq->ibcq.res)) {
1352 dma_free_coherent(&dev->pdev->dev, cq->depth << CQE_SHIFT,
1353 cq->kern_cq.qbuf, cq->kern_cq.qbuf_dma_addr);
1354 dma_pool_free(dev->db_pool, cq->kern_cq.dbrec,
1355 cq->kern_cq.dbrec_dma);
1356 } else {
1357 erdma_unmap_user_dbrecords(ctx, &cq->user_cq.user_dbr_page);
1358 put_mtt_entries(dev, &cq->user_cq.qbuf_mem);
1359 }
1360
1361 return 0;
1362 }
1363
erdma_destroy_qp(struct ib_qp * ibqp,struct ib_udata * udata)1364 int erdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
1365 {
1366 struct erdma_qp *qp = to_eqp(ibqp);
1367 struct erdma_dev *dev = to_edev(ibqp->device);
1368 struct erdma_ucontext *ctx = rdma_udata_to_drv_context(
1369 udata, struct erdma_ucontext, ibucontext);
1370 struct erdma_cmdq_destroy_qp_req req;
1371 union erdma_mod_qp_params params;
1372 unsigned long flags;
1373 int err;
1374
1375 down_write(&qp->state_lock);
1376 if (erdma_device_iwarp(dev)) {
1377 params.iwarp.state = ERDMA_QPS_IWARP_ERROR;
1378 erdma_modify_qp_state_iwarp(qp, ¶ms.iwarp,
1379 ERDMA_QPA_IWARP_STATE);
1380 } else {
1381 params.rocev2.state = ERDMA_QPS_ROCEV2_ERROR;
1382 erdma_modify_qp_state_rocev2(qp, ¶ms.rocev2,
1383 ERDMA_QPA_ROCEV2_STATE);
1384 }
1385 up_write(&qp->state_lock);
1386
1387 cancel_delayed_work_sync(&qp->reflush_dwork);
1388
1389 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
1390 CMDQ_OPCODE_DESTROY_QP);
1391 req.qpn = QP_ID(qp);
1392
1393 err = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL,
1394 true);
1395 if (err)
1396 ibdev_warn_ratelimited(&dev->ibdev,
1397 "failed to destroy QP %u: %d\n",
1398 QP_ID(qp), err);
1399
1400 xa_lock_irqsave(&dev->qp_xa, flags);
1401 __xa_erase(&dev->qp_xa, QP_ID(qp));
1402 xa_unlock_irqrestore(&dev->qp_xa, flags);
1403
1404 erdma_qp_put(qp);
1405 wait_for_completion(&qp->safe_free);
1406
1407 if (rdma_is_kernel_res(&qp->ibqp.res)) {
1408 free_kernel_qp(qp);
1409 } else {
1410 put_mtt_entries(dev, &qp->user_qp.sq_mem);
1411 put_mtt_entries(dev, &qp->user_qp.rq_mem);
1412 erdma_unmap_user_dbrecords(ctx, &qp->user_qp.user_dbr_page);
1413 }
1414
1415 if (qp->cep)
1416 erdma_cep_put(qp->cep);
1417
1418 return 0;
1419 }
1420
erdma_qp_get_ref(struct ib_qp * ibqp)1421 void erdma_qp_get_ref(struct ib_qp *ibqp)
1422 {
1423 erdma_qp_get(to_eqp(ibqp));
1424 }
1425
erdma_qp_put_ref(struct ib_qp * ibqp)1426 void erdma_qp_put_ref(struct ib_qp *ibqp)
1427 {
1428 erdma_qp_put(to_eqp(ibqp));
1429 }
1430
erdma_mmap(struct ib_ucontext * ctx,struct vm_area_struct * vma)1431 int erdma_mmap(struct ib_ucontext *ctx, struct vm_area_struct *vma)
1432 {
1433 struct rdma_user_mmap_entry *rdma_entry;
1434 struct erdma_user_mmap_entry *entry;
1435 pgprot_t prot;
1436 int err;
1437
1438 rdma_entry = rdma_user_mmap_entry_get(ctx, vma);
1439 if (!rdma_entry)
1440 return -EINVAL;
1441
1442 entry = to_emmap(rdma_entry);
1443
1444 switch (entry->mmap_flag) {
1445 case ERDMA_MMAP_IO_NC:
1446 /* map doorbell. */
1447 prot = pgprot_device(vma->vm_page_prot);
1448 break;
1449 default:
1450 err = -EINVAL;
1451 goto put_entry;
1452 }
1453
1454 err = rdma_user_mmap_io(ctx, vma, PFN_DOWN(entry->address), PAGE_SIZE,
1455 prot, rdma_entry);
1456
1457 put_entry:
1458 rdma_user_mmap_entry_put(rdma_entry);
1459 return err;
1460 }
1461
erdma_mmap_free(struct rdma_user_mmap_entry * rdma_entry)1462 void erdma_mmap_free(struct rdma_user_mmap_entry *rdma_entry)
1463 {
1464 struct erdma_user_mmap_entry *entry = to_emmap(rdma_entry);
1465
1466 kfree(entry);
1467 }
1468
alloc_db_resources(struct erdma_dev * dev,struct erdma_ucontext * ctx,bool ext_db_en)1469 static int alloc_db_resources(struct erdma_dev *dev, struct erdma_ucontext *ctx,
1470 bool ext_db_en)
1471 {
1472 struct erdma_cmdq_ext_db_req req = {};
1473 u64 val0, val1;
1474 int ret;
1475
1476 /*
1477 * CAP_SYS_RAWIO is required if hardware does not support extend
1478 * doorbell mechanism.
1479 */
1480 if (!ext_db_en && !capable(CAP_SYS_RAWIO))
1481 return -EPERM;
1482
1483 if (!ext_db_en) {
1484 ctx->sdb = dev->func_bar_addr + ERDMA_BAR_SQDB_SPACE_OFFSET;
1485 ctx->rdb = dev->func_bar_addr + ERDMA_BAR_RQDB_SPACE_OFFSET;
1486 ctx->cdb = dev->func_bar_addr + ERDMA_BAR_CQDB_SPACE_OFFSET;
1487 return 0;
1488 }
1489
1490 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_COMMON,
1491 CMDQ_OPCODE_ALLOC_DB);
1492
1493 req.cfg = FIELD_PREP(ERDMA_CMD_EXT_DB_CQ_EN_MASK, 1) |
1494 FIELD_PREP(ERDMA_CMD_EXT_DB_RQ_EN_MASK, 1) |
1495 FIELD_PREP(ERDMA_CMD_EXT_DB_SQ_EN_MASK, 1);
1496
1497 ret = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), &val0, &val1,
1498 true);
1499 if (ret)
1500 return ret;
1501
1502 ctx->ext_db.enable = true;
1503 ctx->ext_db.sdb_off = ERDMA_GET(val0, ALLOC_DB_RESP_SDB);
1504 ctx->ext_db.rdb_off = ERDMA_GET(val0, ALLOC_DB_RESP_RDB);
1505 ctx->ext_db.cdb_off = ERDMA_GET(val0, ALLOC_DB_RESP_CDB);
1506
1507 ctx->sdb = dev->func_bar_addr + (ctx->ext_db.sdb_off << PAGE_SHIFT);
1508 ctx->cdb = dev->func_bar_addr + (ctx->ext_db.rdb_off << PAGE_SHIFT);
1509 ctx->rdb = dev->func_bar_addr + (ctx->ext_db.cdb_off << PAGE_SHIFT);
1510
1511 return 0;
1512 }
1513
free_db_resources(struct erdma_dev * dev,struct erdma_ucontext * ctx)1514 static void free_db_resources(struct erdma_dev *dev, struct erdma_ucontext *ctx)
1515 {
1516 struct erdma_cmdq_ext_db_req req = {};
1517 int ret;
1518
1519 if (!ctx->ext_db.enable)
1520 return;
1521
1522 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_COMMON,
1523 CMDQ_OPCODE_FREE_DB);
1524
1525 req.cfg = FIELD_PREP(ERDMA_CMD_EXT_DB_CQ_EN_MASK, 1) |
1526 FIELD_PREP(ERDMA_CMD_EXT_DB_RQ_EN_MASK, 1) |
1527 FIELD_PREP(ERDMA_CMD_EXT_DB_SQ_EN_MASK, 1);
1528
1529 req.sdb_off = ctx->ext_db.sdb_off;
1530 req.rdb_off = ctx->ext_db.rdb_off;
1531 req.cdb_off = ctx->ext_db.cdb_off;
1532
1533 ret = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL,
1534 true);
1535 if (ret)
1536 ibdev_err_ratelimited(&dev->ibdev,
1537 "free db resources failed %d", ret);
1538 }
1539
erdma_uctx_user_mmap_entries_remove(struct erdma_ucontext * uctx)1540 static void erdma_uctx_user_mmap_entries_remove(struct erdma_ucontext *uctx)
1541 {
1542 rdma_user_mmap_entry_remove(uctx->sq_db_mmap_entry);
1543 rdma_user_mmap_entry_remove(uctx->rq_db_mmap_entry);
1544 rdma_user_mmap_entry_remove(uctx->cq_db_mmap_entry);
1545 }
1546
erdma_alloc_ucontext(struct ib_ucontext * ibctx,struct ib_udata * udata)1547 int erdma_alloc_ucontext(struct ib_ucontext *ibctx, struct ib_udata *udata)
1548 {
1549 struct erdma_ucontext *ctx = to_ectx(ibctx);
1550 struct erdma_dev *dev = to_edev(ibctx->device);
1551 int ret;
1552 struct erdma_uresp_alloc_ctx uresp = {};
1553
1554 if (atomic_inc_return(&dev->num_ctx) > ERDMA_MAX_CONTEXT) {
1555 ret = -ENOMEM;
1556 goto err_out;
1557 }
1558
1559 if (udata->outlen < sizeof(uresp)) {
1560 ret = -EINVAL;
1561 goto err_out;
1562 }
1563
1564 INIT_LIST_HEAD(&ctx->dbrecords_page_list);
1565 mutex_init(&ctx->dbrecords_page_mutex);
1566
1567 ret = alloc_db_resources(dev, ctx,
1568 !!(dev->attrs.cap_flags &
1569 ERDMA_DEV_CAP_FLAGS_EXTEND_DB));
1570 if (ret)
1571 goto err_out;
1572
1573 ctx->sq_db_mmap_entry = erdma_user_mmap_entry_insert(
1574 ctx, (void *)ctx->sdb, PAGE_SIZE, ERDMA_MMAP_IO_NC, &uresp.sdb);
1575 if (!ctx->sq_db_mmap_entry) {
1576 ret = -ENOMEM;
1577 goto err_free_ext_db;
1578 }
1579
1580 ctx->rq_db_mmap_entry = erdma_user_mmap_entry_insert(
1581 ctx, (void *)ctx->rdb, PAGE_SIZE, ERDMA_MMAP_IO_NC, &uresp.rdb);
1582 if (!ctx->rq_db_mmap_entry) {
1583 ret = -EINVAL;
1584 goto err_put_mmap_entries;
1585 }
1586
1587 ctx->cq_db_mmap_entry = erdma_user_mmap_entry_insert(
1588 ctx, (void *)ctx->cdb, PAGE_SIZE, ERDMA_MMAP_IO_NC, &uresp.cdb);
1589 if (!ctx->cq_db_mmap_entry) {
1590 ret = -EINVAL;
1591 goto err_put_mmap_entries;
1592 }
1593
1594 uresp.dev_id = dev->pdev->device;
1595
1596 ret = ib_respond_udata(udata, uresp);
1597 if (ret)
1598 goto err_put_mmap_entries;
1599
1600 return 0;
1601
1602 err_put_mmap_entries:
1603 erdma_uctx_user_mmap_entries_remove(ctx);
1604
1605 err_free_ext_db:
1606 free_db_resources(dev, ctx);
1607
1608 err_out:
1609 atomic_dec(&dev->num_ctx);
1610 return ret;
1611 }
1612
erdma_dealloc_ucontext(struct ib_ucontext * ibctx)1613 void erdma_dealloc_ucontext(struct ib_ucontext *ibctx)
1614 {
1615 struct erdma_dev *dev = to_edev(ibctx->device);
1616 struct erdma_ucontext *ctx = to_ectx(ibctx);
1617
1618 erdma_uctx_user_mmap_entries_remove(ctx);
1619 free_db_resources(dev, ctx);
1620 atomic_dec(&dev->num_ctx);
1621 }
1622
erdma_attr_to_av(const struct rdma_ah_attr * ah_attr,struct erdma_av * av,u16 sport)1623 static void erdma_attr_to_av(const struct rdma_ah_attr *ah_attr,
1624 struct erdma_av *av, u16 sport)
1625 {
1626 const struct ib_global_route *grh = rdma_ah_read_grh(ah_attr);
1627
1628 av->port = rdma_ah_get_port_num(ah_attr);
1629 av->sgid_index = grh->sgid_index;
1630 av->hop_limit = grh->hop_limit;
1631 av->traffic_class = grh->traffic_class;
1632 av->sl = rdma_ah_get_sl(ah_attr);
1633
1634 av->flow_label = grh->flow_label;
1635 av->udp_sport = sport;
1636
1637 ether_addr_copy(av->dmac, ah_attr->roce.dmac);
1638 memcpy(av->dgid, grh->dgid.raw, ERDMA_ROCEV2_GID_SIZE);
1639
1640 if (ipv6_addr_v4mapped((struct in6_addr *)&grh->dgid))
1641 av->ntype = ERDMA_NETWORK_TYPE_IPV4;
1642 else
1643 av->ntype = ERDMA_NETWORK_TYPE_IPV6;
1644 }
1645
erdma_av_to_attr(struct erdma_av * av,struct rdma_ah_attr * ah_attr)1646 static void erdma_av_to_attr(struct erdma_av *av, struct rdma_ah_attr *ah_attr)
1647 {
1648 ah_attr->type = RDMA_AH_ATTR_TYPE_ROCE;
1649
1650 rdma_ah_set_sl(ah_attr, av->sl);
1651 rdma_ah_set_port_num(ah_attr, av->port);
1652 rdma_ah_set_ah_flags(ah_attr, IB_AH_GRH);
1653
1654 rdma_ah_set_grh(ah_attr, NULL, av->flow_label, av->sgid_index,
1655 av->hop_limit, av->traffic_class);
1656 rdma_ah_set_dgid_raw(ah_attr, av->dgid);
1657 }
1658
1659 static int ib_qps_to_erdma_qps[ERDMA_PROTO_COUNT][IB_QPS_ERR + 1] = {
1660 [ERDMA_PROTO_IWARP] = {
1661 [IB_QPS_RESET] = ERDMA_QPS_IWARP_IDLE,
1662 [IB_QPS_INIT] = ERDMA_QPS_IWARP_IDLE,
1663 [IB_QPS_RTR] = ERDMA_QPS_IWARP_RTR,
1664 [IB_QPS_RTS] = ERDMA_QPS_IWARP_RTS,
1665 [IB_QPS_SQD] = ERDMA_QPS_IWARP_CLOSING,
1666 [IB_QPS_SQE] = ERDMA_QPS_IWARP_TERMINATE,
1667 [IB_QPS_ERR] = ERDMA_QPS_IWARP_ERROR,
1668 },
1669 [ERDMA_PROTO_ROCEV2] = {
1670 [IB_QPS_RESET] = ERDMA_QPS_ROCEV2_RESET,
1671 [IB_QPS_INIT] = ERDMA_QPS_ROCEV2_INIT,
1672 [IB_QPS_RTR] = ERDMA_QPS_ROCEV2_RTR,
1673 [IB_QPS_RTS] = ERDMA_QPS_ROCEV2_RTS,
1674 [IB_QPS_SQD] = ERDMA_QPS_ROCEV2_SQD,
1675 [IB_QPS_SQE] = ERDMA_QPS_ROCEV2_SQE,
1676 [IB_QPS_ERR] = ERDMA_QPS_ROCEV2_ERROR,
1677 },
1678 };
1679
1680 static int erdma_qps_to_ib_qps[ERDMA_PROTO_COUNT][ERDMA_QPS_ROCEV2_COUNT] = {
1681 [ERDMA_PROTO_IWARP] = {
1682 [ERDMA_QPS_IWARP_IDLE] = IB_QPS_INIT,
1683 [ERDMA_QPS_IWARP_RTR] = IB_QPS_RTR,
1684 [ERDMA_QPS_IWARP_RTS] = IB_QPS_RTS,
1685 [ERDMA_QPS_IWARP_CLOSING] = IB_QPS_ERR,
1686 [ERDMA_QPS_IWARP_TERMINATE] = IB_QPS_ERR,
1687 [ERDMA_QPS_IWARP_ERROR] = IB_QPS_ERR,
1688 },
1689 [ERDMA_PROTO_ROCEV2] = {
1690 [ERDMA_QPS_ROCEV2_RESET] = IB_QPS_RESET,
1691 [ERDMA_QPS_ROCEV2_INIT] = IB_QPS_INIT,
1692 [ERDMA_QPS_ROCEV2_RTR] = IB_QPS_RTR,
1693 [ERDMA_QPS_ROCEV2_RTS] = IB_QPS_RTS,
1694 [ERDMA_QPS_ROCEV2_SQD] = IB_QPS_SQD,
1695 [ERDMA_QPS_ROCEV2_SQE] = IB_QPS_SQE,
1696 [ERDMA_QPS_ROCEV2_ERROR] = IB_QPS_ERR,
1697 },
1698 };
1699
ib_to_iwarp_qps(enum ib_qp_state state)1700 static inline enum erdma_qps_iwarp ib_to_iwarp_qps(enum ib_qp_state state)
1701 {
1702 return ib_qps_to_erdma_qps[ERDMA_PROTO_IWARP][state];
1703 }
1704
ib_to_rocev2_qps(enum ib_qp_state state)1705 static inline enum erdma_qps_rocev2 ib_to_rocev2_qps(enum ib_qp_state state)
1706 {
1707 return ib_qps_to_erdma_qps[ERDMA_PROTO_ROCEV2][state];
1708 }
1709
iwarp_to_ib_qps(enum erdma_qps_iwarp state)1710 static inline enum ib_qp_state iwarp_to_ib_qps(enum erdma_qps_iwarp state)
1711 {
1712 return erdma_qps_to_ib_qps[ERDMA_PROTO_IWARP][state];
1713 }
1714
rocev2_to_ib_qps(enum erdma_qps_rocev2 state)1715 static inline enum ib_qp_state rocev2_to_ib_qps(enum erdma_qps_rocev2 state)
1716 {
1717 return erdma_qps_to_ib_qps[ERDMA_PROTO_ROCEV2][state];
1718 }
1719
erdma_check_qp_attrs(struct erdma_qp * qp,struct ib_qp_attr * attr,int attr_mask)1720 static int erdma_check_qp_attrs(struct erdma_qp *qp, struct ib_qp_attr *attr,
1721 int attr_mask)
1722 {
1723 enum ib_qp_state cur_state, nxt_state;
1724 struct erdma_dev *dev = qp->dev;
1725 int ret = -EINVAL;
1726
1727 if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS) {
1728 ret = -EOPNOTSUPP;
1729 goto out;
1730 }
1731
1732 if ((attr_mask & IB_QP_PORT) &&
1733 !rdma_is_port_valid(&dev->ibdev, attr->port_num))
1734 goto out;
1735
1736 if (erdma_device_rocev2(dev)) {
1737 cur_state = (attr_mask & IB_QP_CUR_STATE) ?
1738 attr->cur_qp_state :
1739 rocev2_to_ib_qps(qp->attrs.rocev2.state);
1740
1741 nxt_state = (attr_mask & IB_QP_STATE) ? attr->qp_state :
1742 cur_state;
1743
1744 if (!ib_modify_qp_is_ok(cur_state, nxt_state, qp->ibqp.qp_type,
1745 attr_mask))
1746 goto out;
1747
1748 if ((attr_mask & IB_QP_AV) &&
1749 erdma_check_gid_attr(
1750 rdma_ah_read_grh(&attr->ah_attr)->sgid_attr))
1751 goto out;
1752
1753 if ((attr_mask & IB_QP_PKEY_INDEX) &&
1754 attr->pkey_index >= ERDMA_MAX_PKEYS)
1755 goto out;
1756 }
1757
1758 return 0;
1759
1760 out:
1761 return ret;
1762 }
1763
erdma_init_mod_qp_params_rocev2(struct erdma_qp * qp,struct erdma_mod_qp_params_rocev2 * params,int * erdma_attr_mask,struct ib_qp_attr * attr,int ib_attr_mask)1764 static void erdma_init_mod_qp_params_rocev2(
1765 struct erdma_qp *qp, struct erdma_mod_qp_params_rocev2 *params,
1766 int *erdma_attr_mask, struct ib_qp_attr *attr, int ib_attr_mask)
1767 {
1768 enum erdma_qpa_mask_rocev2 to_modify_attrs = 0;
1769 enum erdma_qps_rocev2 cur_state, nxt_state;
1770 u16 udp_sport;
1771
1772 if (ib_attr_mask & IB_QP_CUR_STATE)
1773 cur_state = ib_to_rocev2_qps(attr->cur_qp_state);
1774 else
1775 cur_state = qp->attrs.rocev2.state;
1776
1777 if (ib_attr_mask & IB_QP_STATE)
1778 nxt_state = ib_to_rocev2_qps(attr->qp_state);
1779 else
1780 nxt_state = cur_state;
1781
1782 to_modify_attrs |= ERDMA_QPA_ROCEV2_STATE;
1783 params->state = nxt_state;
1784
1785 if (ib_attr_mask & IB_QP_QKEY) {
1786 to_modify_attrs |= ERDMA_QPA_ROCEV2_QKEY;
1787 params->qkey = attr->qkey;
1788 }
1789
1790 if (ib_attr_mask & IB_QP_SQ_PSN) {
1791 to_modify_attrs |= ERDMA_QPA_ROCEV2_SQ_PSN;
1792 params->sq_psn = attr->sq_psn;
1793 }
1794
1795 if (ib_attr_mask & IB_QP_RQ_PSN) {
1796 to_modify_attrs |= ERDMA_QPA_ROCEV2_RQ_PSN;
1797 params->rq_psn = attr->rq_psn;
1798 }
1799
1800 if (ib_attr_mask & IB_QP_DEST_QPN) {
1801 to_modify_attrs |= ERDMA_QPA_ROCEV2_DST_QPN;
1802 params->dst_qpn = attr->dest_qp_num;
1803 }
1804
1805 if (ib_attr_mask & IB_QP_AV) {
1806 to_modify_attrs |= ERDMA_QPA_ROCEV2_AV;
1807 udp_sport = rdma_get_udp_sport(attr->ah_attr.grh.flow_label,
1808 QP_ID(qp), params->dst_qpn);
1809 erdma_attr_to_av(&attr->ah_attr, ¶ms->av, udp_sport);
1810 }
1811
1812 *erdma_attr_mask = to_modify_attrs;
1813 }
1814
erdma_modify_qp(struct ib_qp * ibqp,struct ib_qp_attr * attr,int attr_mask,struct ib_udata * udata)1815 int erdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask,
1816 struct ib_udata *udata)
1817 {
1818 struct erdma_qp *qp = to_eqp(ibqp);
1819 union erdma_mod_qp_params params;
1820 int ret = 0, erdma_attr_mask = 0;
1821
1822 down_write(&qp->state_lock);
1823
1824 ret = erdma_check_qp_attrs(qp, attr, attr_mask);
1825 if (ret)
1826 goto out;
1827
1828 if (erdma_device_iwarp(qp->dev)) {
1829 if (attr_mask & IB_QP_STATE) {
1830 erdma_attr_mask |= ERDMA_QPA_IWARP_STATE;
1831 params.iwarp.state = ib_to_iwarp_qps(attr->qp_state);
1832 }
1833
1834 ret = erdma_modify_qp_state_iwarp(qp, ¶ms.iwarp,
1835 erdma_attr_mask);
1836 } else {
1837 erdma_init_mod_qp_params_rocev2(
1838 qp, ¶ms.rocev2, &erdma_attr_mask, attr, attr_mask);
1839
1840 ret = erdma_modify_qp_state_rocev2(qp, ¶ms.rocev2,
1841 erdma_attr_mask);
1842 }
1843
1844 out:
1845 up_write(&qp->state_lock);
1846 return ret;
1847 }
1848
query_qp_state(struct erdma_qp * qp)1849 static enum ib_qp_state query_qp_state(struct erdma_qp *qp)
1850 {
1851 if (erdma_device_iwarp(qp->dev))
1852 return iwarp_to_ib_qps(qp->attrs.iwarp.state);
1853 else
1854 return rocev2_to_ib_qps(qp->attrs.rocev2.state);
1855 }
1856
erdma_query_qp(struct ib_qp * ibqp,struct ib_qp_attr * qp_attr,int qp_attr_mask,struct ib_qp_init_attr * qp_init_attr)1857 int erdma_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
1858 int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr)
1859 {
1860 struct erdma_cmdq_query_qp_req_rocev2 req;
1861 struct erdma_dev *dev;
1862 struct erdma_qp *qp;
1863 u64 resp0, resp1;
1864 int ret;
1865
1866 if (ibqp && qp_attr && qp_init_attr) {
1867 qp = to_eqp(ibqp);
1868 dev = to_edev(ibqp->device);
1869 } else {
1870 return -EINVAL;
1871 }
1872
1873 qp_attr->cap.max_inline_data = ERDMA_MAX_INLINE;
1874 qp_init_attr->cap.max_inline_data = ERDMA_MAX_INLINE;
1875
1876 qp_attr->cap.max_send_wr = qp->attrs.sq_size;
1877 qp_attr->cap.max_recv_wr = qp->attrs.rq_size;
1878 qp_attr->cap.max_send_sge = qp->attrs.max_send_sge;
1879 qp_attr->cap.max_recv_sge = qp->attrs.max_recv_sge;
1880
1881 qp_attr->path_mtu = ib_mtu_int_to_enum(dev->netdev->mtu);
1882 qp_attr->max_rd_atomic = qp->attrs.irq_size;
1883 qp_attr->max_dest_rd_atomic = qp->attrs.orq_size;
1884
1885 qp_attr->qp_access_flags = IB_ACCESS_LOCAL_WRITE |
1886 IB_ACCESS_REMOTE_WRITE |
1887 IB_ACCESS_REMOTE_READ;
1888
1889 qp_init_attr->cap = qp_attr->cap;
1890
1891 if (erdma_device_rocev2(dev)) {
1892 /* Query hardware to get some attributes */
1893 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
1894 CMDQ_OPCODE_QUERY_QP);
1895 req.qpn = QP_ID(qp);
1896
1897 ret = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), &resp0,
1898 &resp1, true);
1899 if (ret)
1900 return ret;
1901
1902 qp_attr->sq_psn =
1903 FIELD_GET(ERDMA_CMD_QUERY_QP_RESP_SQ_PSN_MASK, resp0);
1904 qp_attr->rq_psn =
1905 FIELD_GET(ERDMA_CMD_QUERY_QP_RESP_RQ_PSN_MASK, resp0);
1906 qp_attr->qp_state = rocev2_to_ib_qps(FIELD_GET(
1907 ERDMA_CMD_QUERY_QP_RESP_QP_STATE_MASK, resp0));
1908 qp_attr->cur_qp_state = qp_attr->qp_state;
1909 qp_attr->sq_draining = FIELD_GET(
1910 ERDMA_CMD_QUERY_QP_RESP_SQ_DRAINING_MASK, resp0);
1911
1912 qp_attr->pkey_index = 0;
1913 qp_attr->dest_qp_num = qp->attrs.rocev2.dst_qpn;
1914
1915 if (qp->ibqp.qp_type == IB_QPT_RC)
1916 erdma_av_to_attr(&qp->attrs.rocev2.av,
1917 &qp_attr->ah_attr);
1918 } else {
1919 qp_attr->qp_state = query_qp_state(qp);
1920 qp_attr->cur_qp_state = qp_attr->qp_state;
1921 }
1922
1923 return 0;
1924 }
1925
erdma_init_user_cq(struct erdma_ucontext * ctx,struct erdma_cq * cq,struct erdma_ureq_create_cq * ureq)1926 static int erdma_init_user_cq(struct erdma_ucontext *ctx, struct erdma_cq *cq,
1927 struct erdma_ureq_create_cq *ureq)
1928 {
1929 int ret;
1930 struct erdma_dev *dev = to_edev(cq->ibcq.device);
1931
1932 ret = get_mtt_entries(dev, &cq->user_cq.qbuf_mem, ureq->qbuf_va,
1933 ureq->qbuf_len, 0, ureq->qbuf_va, SZ_64M - SZ_4K,
1934 true);
1935 if (ret)
1936 return ret;
1937
1938 ret = erdma_map_user_dbrecords(ctx, ureq->db_record_va,
1939 &cq->user_cq.user_dbr_page,
1940 &cq->user_cq.dbrec_dma);
1941 if (ret)
1942 put_mtt_entries(dev, &cq->user_cq.qbuf_mem);
1943
1944 return ret;
1945 }
1946
erdma_init_kernel_cq(struct erdma_cq * cq)1947 static int erdma_init_kernel_cq(struct erdma_cq *cq)
1948 {
1949 struct erdma_dev *dev = to_edev(cq->ibcq.device);
1950
1951 cq->kern_cq.qbuf =
1952 dma_alloc_coherent(&dev->pdev->dev, cq->depth << CQE_SHIFT,
1953 &cq->kern_cq.qbuf_dma_addr, GFP_KERNEL);
1954 if (!cq->kern_cq.qbuf)
1955 return -ENOMEM;
1956
1957 cq->kern_cq.dbrec = dma_pool_zalloc(dev->db_pool, GFP_KERNEL,
1958 &cq->kern_cq.dbrec_dma);
1959 if (!cq->kern_cq.dbrec)
1960 goto err_out;
1961
1962 spin_lock_init(&cq->kern_cq.lock);
1963 /* use default cqdb addr */
1964 cq->kern_cq.db = dev->func_bar + ERDMA_BAR_CQDB_SPACE_OFFSET;
1965
1966 return 0;
1967
1968 err_out:
1969 dma_free_coherent(&dev->pdev->dev, cq->depth << CQE_SHIFT,
1970 cq->kern_cq.qbuf, cq->kern_cq.qbuf_dma_addr);
1971
1972 return -ENOMEM;
1973 }
1974
erdma_create_cq(struct ib_cq * ibcq,const struct ib_cq_init_attr * attr,struct uverbs_attr_bundle * attrs)1975 int erdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
1976 struct uverbs_attr_bundle *attrs)
1977 {
1978 struct ib_udata *udata = &attrs->driver_udata;
1979 struct erdma_cq *cq = to_ecq(ibcq);
1980 struct erdma_dev *dev = to_edev(ibcq->device);
1981 unsigned int depth = attr->cqe;
1982 int ret;
1983 struct erdma_ucontext *ctx = rdma_udata_to_drv_context(
1984 udata, struct erdma_ucontext, ibucontext);
1985
1986 if (depth > dev->attrs.max_cqe)
1987 return -EINVAL;
1988
1989 depth = roundup_pow_of_two(depth);
1990 cq->ibcq.cqe = depth;
1991 cq->depth = depth;
1992 cq->assoc_eqn = attr->comp_vector + 1;
1993 refcount_set(&cq->refcount, 1);
1994 init_completion(&cq->free);
1995
1996 ret = xa_alloc_cyclic_irq(&dev->cq_xa, &cq->cqn, cq,
1997 XA_LIMIT(1, dev->attrs.max_cq - 1),
1998 &dev->next_alloc_cqn, GFP_KERNEL);
1999 if (ret < 0)
2000 return ret;
2001
2002 if (!rdma_is_kernel_res(&ibcq->res)) {
2003 struct erdma_ureq_create_cq ureq;
2004 struct erdma_uresp_create_cq uresp = {};
2005
2006 ret = ib_copy_validate_udata_in(udata, ureq, rsvd0);
2007 if (ret)
2008 goto err_out_xa;
2009
2010 ret = erdma_init_user_cq(ctx, cq, &ureq);
2011 if (ret)
2012 goto err_out_xa;
2013
2014 uresp.cq_id = cq->cqn;
2015 uresp.num_cqe = depth;
2016
2017 ret = ib_respond_udata(udata, uresp);
2018 if (ret)
2019 goto err_free_res;
2020 } else {
2021 ret = erdma_init_kernel_cq(cq);
2022 if (ret)
2023 goto err_out_xa;
2024 }
2025
2026 ret = create_cq_cmd(ctx, cq);
2027 if (ret)
2028 goto err_free_res;
2029
2030 return 0;
2031
2032 err_free_res:
2033 if (!rdma_is_kernel_res(&ibcq->res)) {
2034 erdma_unmap_user_dbrecords(ctx, &cq->user_cq.user_dbr_page);
2035 put_mtt_entries(dev, &cq->user_cq.qbuf_mem);
2036 } else {
2037 dma_free_coherent(&dev->pdev->dev, depth << CQE_SHIFT,
2038 cq->kern_cq.qbuf, cq->kern_cq.qbuf_dma_addr);
2039 dma_pool_free(dev->db_pool, cq->kern_cq.dbrec,
2040 cq->kern_cq.dbrec_dma);
2041 }
2042
2043 err_out_xa:
2044 xa_erase_irq(&dev->cq_xa, cq->cqn);
2045
2046 return ret;
2047 }
2048
erdma_disassociate_ucontext(struct ib_ucontext * ibcontext)2049 void erdma_disassociate_ucontext(struct ib_ucontext *ibcontext)
2050 {
2051 }
2052
erdma_set_mtu(struct erdma_dev * dev,u32 mtu)2053 void erdma_set_mtu(struct erdma_dev *dev, u32 mtu)
2054 {
2055 struct erdma_cmdq_config_mtu_req req;
2056
2057 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_COMMON,
2058 CMDQ_OPCODE_CONF_MTU);
2059 req.mtu = mtu;
2060
2061 erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL, true);
2062 }
2063
erdma_port_event(struct erdma_dev * dev,enum ib_event_type reason)2064 void erdma_port_event(struct erdma_dev *dev, enum ib_event_type reason)
2065 {
2066 struct ib_event event;
2067
2068 event.device = &dev->ibdev;
2069 event.element.port_num = 1;
2070 event.event = reason;
2071
2072 ib_dispatch_event(&event);
2073 }
2074
2075 enum counters {
2076 ERDMA_STATS_TX_REQS_CNT,
2077 ERDMA_STATS_TX_PACKETS_CNT,
2078 ERDMA_STATS_TX_BYTES_CNT,
2079 ERDMA_STATS_TX_DISABLE_DROP_CNT,
2080 ERDMA_STATS_TX_BPS_METER_DROP_CNT,
2081 ERDMA_STATS_TX_PPS_METER_DROP_CNT,
2082
2083 ERDMA_STATS_RX_PACKETS_CNT,
2084 ERDMA_STATS_RX_BYTES_CNT,
2085 ERDMA_STATS_RX_DISABLE_DROP_CNT,
2086 ERDMA_STATS_RX_BPS_METER_DROP_CNT,
2087 ERDMA_STATS_RX_PPS_METER_DROP_CNT,
2088
2089 ERDMA_STATS_MAX
2090 };
2091
2092 static const struct rdma_stat_desc erdma_descs[] = {
2093 [ERDMA_STATS_TX_REQS_CNT].name = "tx_reqs_cnt",
2094 [ERDMA_STATS_TX_PACKETS_CNT].name = "tx_packets_cnt",
2095 [ERDMA_STATS_TX_BYTES_CNT].name = "tx_bytes_cnt",
2096 [ERDMA_STATS_TX_DISABLE_DROP_CNT].name = "tx_disable_drop_cnt",
2097 [ERDMA_STATS_TX_BPS_METER_DROP_CNT].name = "tx_bps_limit_drop_cnt",
2098 [ERDMA_STATS_TX_PPS_METER_DROP_CNT].name = "tx_pps_limit_drop_cnt",
2099 [ERDMA_STATS_RX_PACKETS_CNT].name = "rx_packets_cnt",
2100 [ERDMA_STATS_RX_BYTES_CNT].name = "rx_bytes_cnt",
2101 [ERDMA_STATS_RX_DISABLE_DROP_CNT].name = "rx_disable_drop_cnt",
2102 [ERDMA_STATS_RX_BPS_METER_DROP_CNT].name = "rx_bps_limit_drop_cnt",
2103 [ERDMA_STATS_RX_PPS_METER_DROP_CNT].name = "rx_pps_limit_drop_cnt",
2104 };
2105
erdma_alloc_hw_port_stats(struct ib_device * device,u32 port_num)2106 struct rdma_hw_stats *erdma_alloc_hw_port_stats(struct ib_device *device,
2107 u32 port_num)
2108 {
2109 return rdma_alloc_hw_stats_struct(erdma_descs, ERDMA_STATS_MAX,
2110 RDMA_HW_STATS_DEFAULT_LIFESPAN);
2111 }
2112
erdma_query_hw_stats(struct erdma_dev * dev,struct rdma_hw_stats * stats)2113 static int erdma_query_hw_stats(struct erdma_dev *dev,
2114 struct rdma_hw_stats *stats)
2115 {
2116 struct erdma_cmdq_query_stats_resp *resp;
2117 struct erdma_cmdq_query_req req;
2118 dma_addr_t dma_addr;
2119 int err;
2120
2121 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_COMMON,
2122 CMDQ_OPCODE_GET_STATS);
2123
2124 resp = dma_pool_zalloc(dev->resp_pool, GFP_KERNEL, &dma_addr);
2125 if (!resp)
2126 return -ENOMEM;
2127
2128 req.target_addr = dma_addr;
2129 req.target_length = ERDMA_HW_RESP_SIZE;
2130
2131 err = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL,
2132 true);
2133 if (err)
2134 goto out;
2135
2136 if (resp->hdr.magic != ERDMA_HW_RESP_MAGIC) {
2137 err = -EINVAL;
2138 goto out;
2139 }
2140
2141 memcpy(&stats->value[0], &resp->tx_req_cnt,
2142 sizeof(u64) * stats->num_counters);
2143
2144 out:
2145 dma_pool_free(dev->resp_pool, resp, dma_addr);
2146
2147 return err;
2148 }
2149
erdma_get_hw_stats(struct ib_device * ibdev,struct rdma_hw_stats * stats,u32 port,int index)2150 int erdma_get_hw_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats,
2151 u32 port, int index)
2152 {
2153 struct erdma_dev *dev = to_edev(ibdev);
2154 int ret;
2155
2156 if (port == 0)
2157 return 0;
2158
2159 ret = erdma_query_hw_stats(dev, stats);
2160 if (ret)
2161 return ret;
2162
2163 return stats->num_counters;
2164 }
2165
erdma_get_link_layer(struct ib_device * ibdev,u32 port_num)2166 enum rdma_link_layer erdma_get_link_layer(struct ib_device *ibdev, u32 port_num)
2167 {
2168 return IB_LINK_LAYER_ETHERNET;
2169 }
2170
erdma_set_gid(struct erdma_dev * dev,u8 op,u32 idx,const union ib_gid * gid)2171 static int erdma_set_gid(struct erdma_dev *dev, u8 op, u32 idx,
2172 const union ib_gid *gid)
2173 {
2174 struct erdma_cmdq_set_gid_req req;
2175 u8 ntype;
2176
2177 req.cfg = FIELD_PREP(ERDMA_CMD_SET_GID_SGID_IDX_MASK, idx) |
2178 FIELD_PREP(ERDMA_CMD_SET_GID_OP_MASK, op);
2179
2180 if (op == ERDMA_SET_GID_OP_ADD) {
2181 if (ipv6_addr_v4mapped((struct in6_addr *)gid))
2182 ntype = ERDMA_NETWORK_TYPE_IPV4;
2183 else
2184 ntype = ERDMA_NETWORK_TYPE_IPV6;
2185
2186 req.cfg |= FIELD_PREP(ERDMA_CMD_SET_GID_NTYPE_MASK, ntype);
2187
2188 memcpy(&req.gid, gid, ERDMA_ROCEV2_GID_SIZE);
2189 }
2190
2191 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
2192 CMDQ_OPCODE_SET_GID);
2193 return erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL,
2194 true);
2195 }
2196
erdma_add_gid(const struct ib_gid_attr * attr,void ** context)2197 int erdma_add_gid(const struct ib_gid_attr *attr, void **context)
2198 {
2199 struct erdma_dev *dev = to_edev(attr->device);
2200 int ret;
2201
2202 ret = erdma_check_gid_attr(attr);
2203 if (ret)
2204 return ret;
2205
2206 return erdma_set_gid(dev, ERDMA_SET_GID_OP_ADD, attr->index,
2207 &attr->gid);
2208 }
2209
erdma_del_gid(const struct ib_gid_attr * attr,void ** context)2210 int erdma_del_gid(const struct ib_gid_attr *attr, void **context)
2211 {
2212 return erdma_set_gid(to_edev(attr->device), ERDMA_SET_GID_OP_DEL,
2213 attr->index, NULL);
2214 }
2215
erdma_query_pkey(struct ib_device * ibdev,u32 port,u16 index,u16 * pkey)2216 int erdma_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey)
2217 {
2218 if (index >= ERDMA_MAX_PKEYS)
2219 return -EINVAL;
2220
2221 *pkey = ERDMA_DEFAULT_PKEY;
2222 return 0;
2223 }
2224
erdma_set_av_cfg(struct erdma_av_cfg * av_cfg,struct erdma_av * av)2225 void erdma_set_av_cfg(struct erdma_av_cfg *av_cfg, struct erdma_av *av)
2226 {
2227 av_cfg->cfg0 = FIELD_PREP(ERDMA_CMD_CREATE_AV_FL_MASK, av->flow_label) |
2228 FIELD_PREP(ERDMA_CMD_CREATE_AV_NTYPE_MASK, av->ntype);
2229
2230 av_cfg->traffic_class = av->traffic_class;
2231 av_cfg->hop_limit = av->hop_limit;
2232 av_cfg->sl = av->sl;
2233
2234 av_cfg->udp_sport = av->udp_sport;
2235 av_cfg->sgid_index = av->sgid_index;
2236
2237 ether_addr_copy(av_cfg->dmac, av->dmac);
2238 memcpy(av_cfg->dgid, av->dgid, ERDMA_ROCEV2_GID_SIZE);
2239 }
2240
erdma_create_ah(struct ib_ah * ibah,struct rdma_ah_init_attr * init_attr,struct ib_udata * udata)2241 int erdma_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr,
2242 struct ib_udata *udata)
2243 {
2244 const struct ib_global_route *grh =
2245 rdma_ah_read_grh(init_attr->ah_attr);
2246 struct erdma_dev *dev = to_edev(ibah->device);
2247 struct erdma_pd *pd = to_epd(ibah->pd);
2248 struct erdma_ah *ah = to_eah(ibah);
2249 struct erdma_cmdq_create_ah_req req;
2250 u32 udp_sport;
2251 int ret;
2252
2253 ret = erdma_check_gid_attr(grh->sgid_attr);
2254 if (ret)
2255 return ret;
2256
2257 ret = erdma_alloc_idx(&dev->res_cb[ERDMA_RES_TYPE_AH]);
2258 if (ret < 0)
2259 return ret;
2260
2261 ah->ahn = ret;
2262
2263 if (grh->flow_label)
2264 udp_sport = rdma_flow_label_to_udp_sport(grh->flow_label);
2265 else
2266 udp_sport =
2267 IB_ROCE_UDP_ENCAP_VALID_PORT_MIN + (ah->ahn & 0x3FFF);
2268
2269 erdma_attr_to_av(init_attr->ah_attr, &ah->av, udp_sport);
2270
2271 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
2272 CMDQ_OPCODE_CREATE_AH);
2273
2274 req.pdn = pd->pdn;
2275 req.ahn = ah->ahn;
2276 erdma_set_av_cfg(&req.av_cfg, &ah->av);
2277
2278 ret = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL,
2279 init_attr->flags & RDMA_CREATE_AH_SLEEPABLE);
2280 if (ret) {
2281 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_AH], ah->ahn);
2282 return ret;
2283 }
2284
2285 return 0;
2286 }
2287
erdma_destroy_ah(struct ib_ah * ibah,u32 flags)2288 int erdma_destroy_ah(struct ib_ah *ibah, u32 flags)
2289 {
2290 struct erdma_dev *dev = to_edev(ibah->device);
2291 struct erdma_pd *pd = to_epd(ibah->pd);
2292 struct erdma_ah *ah = to_eah(ibah);
2293 struct erdma_cmdq_destroy_ah_req req;
2294 int ret;
2295
2296 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
2297 CMDQ_OPCODE_DESTROY_AH);
2298
2299 req.pdn = pd->pdn;
2300 req.ahn = ah->ahn;
2301
2302 ret = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL,
2303 flags & RDMA_DESTROY_AH_SLEEPABLE);
2304 if (ret)
2305 ibdev_warn_ratelimited(&dev->ibdev,
2306 "failed to destroy AH %u: %d\n",
2307 ah->ahn, ret);
2308
2309 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_AH], ah->ahn);
2310
2311 return 0;
2312 }
2313
erdma_query_ah(struct ib_ah * ibah,struct rdma_ah_attr * ah_attr)2314 int erdma_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr)
2315 {
2316 struct erdma_ah *ah = to_eah(ibah);
2317
2318 memset(ah_attr, 0, sizeof(*ah_attr));
2319 erdma_av_to_attr(&ah->av, ah_attr);
2320
2321 return 0;
2322 }
2323