1 // SPDX-License-Identifier: GPL-2.0
2
3 /* Authors: Cheng Xu <chengyou@linux.alibaba.com> */
4 /* Kai Shen <kaishen@linux.alibaba.com> */
5 /* Copyright (c) 2020-2022, Alibaba Group. */
6
7 /* Authors: Bernard Metzler <bmt@zurich.ibm.com> */
8 /* Copyright (c) 2008-2019, IBM Corporation */
9
10 /* Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved. */
11
12 #include <linux/vmalloc.h>
13 #include <net/addrconf.h>
14 #include <rdma/erdma-abi.h>
15 #include <rdma/ib_umem.h>
16 #include <rdma/uverbs_ioctl.h>
17
18 #include "erdma.h"
19 #include "erdma_cm.h"
20 #include "erdma_verbs.h"
21
assemble_qbuf_mtt_for_cmd(struct erdma_mem * mem,u32 * cfg,u64 * addr0,u64 * addr1)22 static void assemble_qbuf_mtt_for_cmd(struct erdma_mem *mem, u32 *cfg,
23 u64 *addr0, u64 *addr1)
24 {
25 struct erdma_mtt *mtt = mem->mtt;
26
27 if (mem->mtt_nents > ERDMA_MAX_INLINE_MTT_ENTRIES) {
28 *addr0 = mtt->buf_dma;
29 *cfg |= FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_LEVEL_MASK,
30 ERDMA_MR_MTT_1LEVEL);
31 } else {
32 *addr0 = mtt->buf[0];
33 memcpy(addr1, mtt->buf + 1, MTT_SIZE(mem->mtt_nents - 1));
34 *cfg |= FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_LEVEL_MASK,
35 ERDMA_MR_MTT_0LEVEL);
36 }
37 }
38
create_qp_cmd(struct erdma_ucontext * uctx,struct erdma_qp * qp)39 static int create_qp_cmd(struct erdma_ucontext *uctx, struct erdma_qp *qp)
40 {
41 struct erdma_dev *dev = to_edev(qp->ibqp.device);
42 struct erdma_pd *pd = to_epd(qp->ibqp.pd);
43 struct erdma_cmdq_create_qp_req req;
44 struct erdma_uqp *user_qp;
45 u64 resp0, resp1;
46 int err;
47
48 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
49 CMDQ_OPCODE_CREATE_QP);
50
51 req.cfg0 = FIELD_PREP(ERDMA_CMD_CREATE_QP_SQ_DEPTH_MASK,
52 ilog2(qp->attrs.sq_size)) |
53 FIELD_PREP(ERDMA_CMD_CREATE_QP_QPN_MASK, QP_ID(qp));
54 req.cfg1 = FIELD_PREP(ERDMA_CMD_CREATE_QP_RQ_DEPTH_MASK,
55 ilog2(qp->attrs.rq_size)) |
56 FIELD_PREP(ERDMA_CMD_CREATE_QP_PD_MASK, pd->pdn);
57
58 if (qp->ibqp.qp_type == IB_QPT_RC)
59 req.cfg2 = FIELD_PREP(ERDMA_CMD_CREATE_QP_TYPE_MASK,
60 ERDMA_QPT_RC);
61 else
62 req.cfg2 = FIELD_PREP(ERDMA_CMD_CREATE_QP_TYPE_MASK,
63 ERDMA_QPT_UD);
64
65 if (rdma_is_kernel_res(&qp->ibqp.res)) {
66 u32 pgsz_range = ilog2(SZ_1M) - ERDMA_HW_PAGE_SHIFT;
67
68 req.sq_cqn_mtt_cfg =
69 FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,
70 pgsz_range) |
71 FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->scq->cqn);
72 req.rq_cqn_mtt_cfg =
73 FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,
74 pgsz_range) |
75 FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->rcq->cqn);
76
77 req.sq_mtt_cfg =
78 FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_OFFSET_MASK, 0) |
79 FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK, 1) |
80 FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_LEVEL_MASK,
81 ERDMA_MR_MTT_0LEVEL);
82 req.rq_mtt_cfg = req.sq_mtt_cfg;
83
84 req.rq_buf_addr = qp->kern_qp.rq_buf_dma_addr;
85 req.sq_buf_addr = qp->kern_qp.sq_buf_dma_addr;
86 req.sq_dbrec_dma = qp->kern_qp.sq_dbrec_dma;
87 req.rq_dbrec_dma = qp->kern_qp.rq_dbrec_dma;
88 } else {
89 user_qp = &qp->user_qp;
90 req.sq_cqn_mtt_cfg = FIELD_PREP(
91 ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,
92 ilog2(user_qp->sq_mem.page_size) - ERDMA_HW_PAGE_SHIFT);
93 req.sq_cqn_mtt_cfg |=
94 FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->scq->cqn);
95
96 req.rq_cqn_mtt_cfg = FIELD_PREP(
97 ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,
98 ilog2(user_qp->rq_mem.page_size) - ERDMA_HW_PAGE_SHIFT);
99 req.rq_cqn_mtt_cfg |=
100 FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->rcq->cqn);
101
102 req.sq_mtt_cfg = user_qp->sq_mem.page_offset;
103 req.sq_mtt_cfg |= FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK,
104 user_qp->sq_mem.mtt_nents);
105
106 req.rq_mtt_cfg = user_qp->rq_mem.page_offset;
107 req.rq_mtt_cfg |= FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK,
108 user_qp->rq_mem.mtt_nents);
109
110 assemble_qbuf_mtt_for_cmd(&user_qp->sq_mem, &req.sq_mtt_cfg,
111 &req.sq_buf_addr, req.sq_mtt_entry);
112 assemble_qbuf_mtt_for_cmd(&user_qp->rq_mem, &req.rq_mtt_cfg,
113 &req.rq_buf_addr, req.rq_mtt_entry);
114
115 req.sq_dbrec_dma = user_qp->sq_dbrec_dma;
116 req.rq_dbrec_dma = user_qp->rq_dbrec_dma;
117
118 if (uctx->ext_db.enable) {
119 req.sq_cqn_mtt_cfg |=
120 FIELD_PREP(ERDMA_CMD_CREATE_QP_DB_CFG_MASK, 1);
121 req.db_cfg =
122 FIELD_PREP(ERDMA_CMD_CREATE_QP_SQDB_CFG_MASK,
123 uctx->ext_db.sdb_off) |
124 FIELD_PREP(ERDMA_CMD_CREATE_QP_RQDB_CFG_MASK,
125 uctx->ext_db.rdb_off);
126 }
127 }
128
129 err = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), &resp0, &resp1,
130 true);
131 if (!err && erdma_device_iwarp(dev))
132 qp->attrs.iwarp.cookie =
133 FIELD_GET(ERDMA_CMDQ_CREATE_QP_RESP_COOKIE_MASK, resp0);
134
135 return err;
136 }
137
regmr_cmd(struct erdma_dev * dev,struct erdma_mr * mr)138 static int regmr_cmd(struct erdma_dev *dev, struct erdma_mr *mr)
139 {
140 struct erdma_pd *pd = to_epd(mr->ibmr.pd);
141 u32 mtt_level = ERDMA_MR_MTT_0LEVEL;
142 struct erdma_cmdq_reg_mr_req req;
143
144 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA, CMDQ_OPCODE_REG_MR);
145
146 if (mr->type == ERDMA_MR_TYPE_FRMR ||
147 mr->mem.page_cnt > ERDMA_MAX_INLINE_MTT_ENTRIES) {
148 if (mr->mem.mtt->continuous) {
149 req.phy_addr[0] = mr->mem.mtt->buf_dma;
150 mtt_level = ERDMA_MR_MTT_1LEVEL;
151 } else {
152 req.phy_addr[0] = sg_dma_address(mr->mem.mtt->sglist);
153 mtt_level = mr->mem.mtt->level;
154 }
155 } else if (mr->type != ERDMA_MR_TYPE_DMA) {
156 memcpy(req.phy_addr, mr->mem.mtt->buf,
157 MTT_SIZE(mr->mem.page_cnt));
158 }
159
160 req.cfg0 = FIELD_PREP(ERDMA_CMD_MR_VALID_MASK, mr->valid) |
161 FIELD_PREP(ERDMA_CMD_MR_KEY_MASK, mr->ibmr.lkey & 0xFF) |
162 FIELD_PREP(ERDMA_CMD_MR_MPT_IDX_MASK, mr->ibmr.lkey >> 8);
163 req.cfg1 = FIELD_PREP(ERDMA_CMD_REGMR_PD_MASK, pd->pdn) |
164 FIELD_PREP(ERDMA_CMD_REGMR_TYPE_MASK, mr->type) |
165 FIELD_PREP(ERDMA_CMD_REGMR_RIGHT_MASK, mr->access);
166 req.cfg2 = FIELD_PREP(ERDMA_CMD_REGMR_PAGESIZE_MASK,
167 ilog2(mr->mem.page_size)) |
168 FIELD_PREP(ERDMA_CMD_REGMR_MTT_LEVEL_MASK, mtt_level) |
169 FIELD_PREP(ERDMA_CMD_REGMR_MTT_CNT_MASK, mr->mem.page_cnt);
170
171 if (mr->type == ERDMA_MR_TYPE_DMA)
172 goto post_cmd;
173
174 if (mr->type == ERDMA_MR_TYPE_NORMAL) {
175 req.start_va = mr->mem.va;
176 req.size = mr->mem.len;
177 }
178
179 if (!mr->mem.mtt->continuous && mr->mem.mtt->level > 1) {
180 req.cfg0 |= FIELD_PREP(ERDMA_CMD_MR_VERSION_MASK, 1);
181 req.cfg2 |= FIELD_PREP(ERDMA_CMD_REGMR_MTT_PAGESIZE_MASK,
182 PAGE_SHIFT - ERDMA_HW_PAGE_SHIFT);
183 req.size_h = upper_32_bits(mr->mem.len);
184 req.mtt_cnt_h = mr->mem.page_cnt >> 20;
185 }
186
187 post_cmd:
188 return erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL,
189 true);
190 }
191
create_cq_cmd(struct erdma_ucontext * uctx,struct erdma_cq * cq)192 static int create_cq_cmd(struct erdma_ucontext *uctx, struct erdma_cq *cq)
193 {
194 struct erdma_dev *dev = to_edev(cq->ibcq.device);
195 struct erdma_cmdq_create_cq_req req;
196 struct erdma_mem *mem;
197 u32 page_size;
198
199 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
200 CMDQ_OPCODE_CREATE_CQ);
201
202 req.cfg0 = FIELD_PREP(ERDMA_CMD_CREATE_CQ_CQN_MASK, cq->cqn) |
203 FIELD_PREP(ERDMA_CMD_CREATE_CQ_DEPTH_MASK, ilog2(cq->depth));
204 req.cfg1 = FIELD_PREP(ERDMA_CMD_CREATE_CQ_EQN_MASK, cq->assoc_eqn);
205
206 if (rdma_is_kernel_res(&cq->ibcq.res)) {
207 page_size = SZ_32M;
208 req.cfg0 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_PAGESIZE_MASK,
209 ilog2(page_size) - ERDMA_HW_PAGE_SHIFT);
210 req.qbuf_addr_l = lower_32_bits(cq->kern_cq.qbuf_dma_addr);
211 req.qbuf_addr_h = upper_32_bits(cq->kern_cq.qbuf_dma_addr);
212
213 req.cfg1 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_CNT_MASK, 1) |
214 FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_LEVEL_MASK,
215 ERDMA_MR_MTT_0LEVEL);
216
217 req.first_page_offset = 0;
218 req.cq_dbrec_dma = cq->kern_cq.dbrec_dma;
219 } else {
220 mem = &cq->user_cq.qbuf_mem;
221 req.cfg0 |=
222 FIELD_PREP(ERDMA_CMD_CREATE_CQ_PAGESIZE_MASK,
223 ilog2(mem->page_size) - ERDMA_HW_PAGE_SHIFT);
224 if (mem->mtt_nents == 1) {
225 req.qbuf_addr_l = lower_32_bits(mem->mtt->buf[0]);
226 req.qbuf_addr_h = upper_32_bits(mem->mtt->buf[0]);
227 req.cfg1 |=
228 FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_LEVEL_MASK,
229 ERDMA_MR_MTT_0LEVEL);
230 } else {
231 req.qbuf_addr_l = lower_32_bits(mem->mtt->buf_dma);
232 req.qbuf_addr_h = upper_32_bits(mem->mtt->buf_dma);
233 req.cfg1 |=
234 FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_LEVEL_MASK,
235 ERDMA_MR_MTT_1LEVEL);
236 }
237 req.cfg1 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_CNT_MASK,
238 mem->mtt_nents);
239
240 req.first_page_offset = mem->page_offset;
241 req.cq_dbrec_dma = cq->user_cq.dbrec_dma;
242
243 if (uctx->ext_db.enable) {
244 req.cfg1 |= FIELD_PREP(
245 ERDMA_CMD_CREATE_CQ_MTT_DB_CFG_MASK, 1);
246 req.cfg2 = FIELD_PREP(ERDMA_CMD_CREATE_CQ_DB_CFG_MASK,
247 uctx->ext_db.cdb_off);
248 }
249 }
250
251 return erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL,
252 true);
253 }
254
erdma_alloc_idx(struct erdma_resource_cb * res_cb)255 static int erdma_alloc_idx(struct erdma_resource_cb *res_cb)
256 {
257 int idx;
258 unsigned long flags;
259
260 spin_lock_irqsave(&res_cb->lock, flags);
261 idx = find_next_zero_bit(res_cb->bitmap, res_cb->max_cap,
262 res_cb->next_alloc_idx);
263 if (idx == res_cb->max_cap) {
264 idx = find_first_zero_bit(res_cb->bitmap, res_cb->max_cap);
265 if (idx == res_cb->max_cap) {
266 res_cb->next_alloc_idx = 1;
267 spin_unlock_irqrestore(&res_cb->lock, flags);
268 return -ENOSPC;
269 }
270 }
271
272 set_bit(idx, res_cb->bitmap);
273 res_cb->next_alloc_idx = idx + 1;
274 spin_unlock_irqrestore(&res_cb->lock, flags);
275
276 return idx;
277 }
278
erdma_free_idx(struct erdma_resource_cb * res_cb,u32 idx)279 static inline void erdma_free_idx(struct erdma_resource_cb *res_cb, u32 idx)
280 {
281 unsigned long flags;
282 u32 used;
283
284 spin_lock_irqsave(&res_cb->lock, flags);
285 used = __test_and_clear_bit(idx, res_cb->bitmap);
286 spin_unlock_irqrestore(&res_cb->lock, flags);
287 WARN_ON(!used);
288 }
289
290 static struct rdma_user_mmap_entry *
erdma_user_mmap_entry_insert(struct erdma_ucontext * uctx,void * address,u32 size,u8 mmap_flag,u64 * mmap_offset)291 erdma_user_mmap_entry_insert(struct erdma_ucontext *uctx, void *address,
292 u32 size, u8 mmap_flag, u64 *mmap_offset)
293 {
294 struct erdma_user_mmap_entry *entry =
295 kzalloc(sizeof(*entry), GFP_KERNEL);
296 int ret;
297
298 if (!entry)
299 return NULL;
300
301 entry->address = (u64)address;
302 entry->mmap_flag = mmap_flag;
303
304 size = PAGE_ALIGN(size);
305
306 ret = rdma_user_mmap_entry_insert(&uctx->ibucontext, &entry->rdma_entry,
307 size);
308 if (ret) {
309 kfree(entry);
310 return NULL;
311 }
312
313 *mmap_offset = rdma_user_mmap_get_offset(&entry->rdma_entry);
314
315 return &entry->rdma_entry;
316 }
317
erdma_query_device(struct ib_device * ibdev,struct ib_device_attr * attr,struct ib_udata * unused)318 int erdma_query_device(struct ib_device *ibdev, struct ib_device_attr *attr,
319 struct ib_udata *unused)
320 {
321 struct erdma_dev *dev = to_edev(ibdev);
322
323 memset(attr, 0, sizeof(*attr));
324
325 attr->max_mr_size = dev->attrs.max_mr_size;
326 attr->vendor_id = PCI_VENDOR_ID_ALIBABA;
327 attr->vendor_part_id = dev->pdev->device;
328 attr->hw_ver = dev->pdev->revision;
329 attr->max_qp = dev->attrs.max_qp - 1;
330 attr->max_qp_wr = min(dev->attrs.max_send_wr, dev->attrs.max_recv_wr);
331 attr->max_qp_rd_atom = dev->attrs.max_ord;
332 attr->max_qp_init_rd_atom = dev->attrs.max_ird;
333 attr->max_res_rd_atom = dev->attrs.max_qp * dev->attrs.max_ird;
334 attr->device_cap_flags = IB_DEVICE_MEM_MGT_EXTENSIONS;
335 attr->kernel_cap_flags = IBK_LOCAL_DMA_LKEY;
336 ibdev->local_dma_lkey = dev->attrs.local_dma_key;
337 attr->max_send_sge = dev->attrs.max_send_sge;
338 attr->max_recv_sge = dev->attrs.max_recv_sge;
339 attr->max_sge_rd = dev->attrs.max_sge_rd;
340 attr->max_cq = dev->attrs.max_cq - 1;
341 attr->max_cqe = dev->attrs.max_cqe;
342 attr->max_mr = dev->attrs.max_mr;
343 attr->max_pd = dev->attrs.max_pd;
344 attr->max_mw = dev->attrs.max_mw;
345 attr->max_fast_reg_page_list_len = ERDMA_MAX_FRMR_PA;
346 attr->page_size_cap = ERDMA_PAGE_SIZE_SUPPORT;
347
348 if (erdma_device_rocev2(dev)) {
349 attr->max_pkeys = ERDMA_MAX_PKEYS;
350 attr->max_ah = dev->attrs.max_ah;
351 }
352
353 if (dev->attrs.cap_flags & ERDMA_DEV_CAP_FLAGS_ATOMIC)
354 attr->atomic_cap = IB_ATOMIC_GLOB;
355
356 attr->fw_ver = dev->attrs.fw_version;
357
358 if (dev->netdev)
359 addrconf_addr_eui48((u8 *)&attr->sys_image_guid,
360 dev->netdev->dev_addr);
361
362 return 0;
363 }
364
erdma_query_gid(struct ib_device * ibdev,u32 port,int idx,union ib_gid * gid)365 int erdma_query_gid(struct ib_device *ibdev, u32 port, int idx,
366 union ib_gid *gid)
367 {
368 struct erdma_dev *dev = to_edev(ibdev);
369
370 memset(gid, 0, sizeof(*gid));
371 ether_addr_copy(gid->raw, dev->attrs.peer_addr);
372
373 return 0;
374 }
375
erdma_query_port(struct ib_device * ibdev,u32 port,struct ib_port_attr * attr)376 int erdma_query_port(struct ib_device *ibdev, u32 port,
377 struct ib_port_attr *attr)
378 {
379 struct erdma_dev *dev = to_edev(ibdev);
380 struct net_device *ndev = dev->netdev;
381
382 memset(attr, 0, sizeof(*attr));
383
384 if (erdma_device_iwarp(dev)) {
385 attr->gid_tbl_len = 1;
386 } else {
387 attr->gid_tbl_len = dev->attrs.max_gid;
388 attr->ip_gids = true;
389 attr->pkey_tbl_len = ERDMA_MAX_PKEYS;
390 }
391
392 attr->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_DEVICE_MGMT_SUP;
393 attr->max_msg_sz = -1;
394
395 if (!ndev)
396 goto out;
397
398 ib_get_eth_speed(ibdev, port, &attr->active_speed, &attr->active_width);
399 attr->max_mtu = ib_mtu_int_to_enum(ndev->mtu);
400 attr->active_mtu = ib_mtu_int_to_enum(ndev->mtu);
401 attr->state = ib_get_curr_port_state(ndev);
402
403 out:
404 if (attr->state == IB_PORT_ACTIVE)
405 attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP;
406 else
407 attr->phys_state = IB_PORT_PHYS_STATE_DISABLED;
408
409 return 0;
410 }
411
erdma_get_port_immutable(struct ib_device * ibdev,u32 port,struct ib_port_immutable * port_immutable)412 int erdma_get_port_immutable(struct ib_device *ibdev, u32 port,
413 struct ib_port_immutable *port_immutable)
414 {
415 struct erdma_dev *dev = to_edev(ibdev);
416
417 if (erdma_device_iwarp(dev)) {
418 port_immutable->core_cap_flags = RDMA_CORE_PORT_IWARP;
419 port_immutable->gid_tbl_len = 1;
420 } else {
421 port_immutable->core_cap_flags =
422 RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
423 port_immutable->max_mad_size = IB_MGMT_MAD_SIZE;
424 port_immutable->gid_tbl_len = dev->attrs.max_gid;
425 port_immutable->pkey_tbl_len = ERDMA_MAX_PKEYS;
426 }
427
428 return 0;
429 }
430
erdma_alloc_pd(struct ib_pd * ibpd,struct ib_udata * udata)431 int erdma_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
432 {
433 struct erdma_pd *pd = to_epd(ibpd);
434 struct erdma_dev *dev = to_edev(ibpd->device);
435 int pdn;
436
437 pdn = erdma_alloc_idx(&dev->res_cb[ERDMA_RES_TYPE_PD]);
438 if (pdn < 0)
439 return pdn;
440
441 pd->pdn = pdn;
442
443 return 0;
444 }
445
erdma_dealloc_pd(struct ib_pd * ibpd,struct ib_udata * udata)446 int erdma_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
447 {
448 struct erdma_pd *pd = to_epd(ibpd);
449 struct erdma_dev *dev = to_edev(ibpd->device);
450
451 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_PD], pd->pdn);
452
453 return 0;
454 }
455
erdma_flush_worker(struct work_struct * work)456 static void erdma_flush_worker(struct work_struct *work)
457 {
458 struct delayed_work *dwork = to_delayed_work(work);
459 struct erdma_qp *qp =
460 container_of(dwork, struct erdma_qp, reflush_dwork);
461 struct erdma_cmdq_reflush_req req;
462
463 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
464 CMDQ_OPCODE_REFLUSH);
465 req.qpn = QP_ID(qp);
466 req.sq_pi = qp->kern_qp.sq_pi;
467 req.rq_pi = qp->kern_qp.rq_pi;
468 erdma_post_cmd_wait(&qp->dev->cmdq, &req, sizeof(req), NULL, NULL,
469 true);
470 }
471
erdma_qp_validate_cap(struct erdma_dev * dev,struct ib_qp_init_attr * attrs)472 static int erdma_qp_validate_cap(struct erdma_dev *dev,
473 struct ib_qp_init_attr *attrs)
474 {
475 if ((attrs->cap.max_send_wr > dev->attrs.max_send_wr) ||
476 (attrs->cap.max_recv_wr > dev->attrs.max_recv_wr) ||
477 (attrs->cap.max_send_sge > dev->attrs.max_send_sge) ||
478 (attrs->cap.max_recv_sge > dev->attrs.max_recv_sge) ||
479 (attrs->cap.max_inline_data > ERDMA_MAX_INLINE) ||
480 !attrs->cap.max_send_wr || !attrs->cap.max_recv_wr) {
481 return -EINVAL;
482 }
483
484 return 0;
485 }
486
erdma_qp_validate_attr(struct erdma_dev * dev,struct ib_qp_init_attr * attrs)487 static int erdma_qp_validate_attr(struct erdma_dev *dev,
488 struct ib_qp_init_attr *attrs)
489 {
490 if (erdma_device_iwarp(dev) && attrs->qp_type != IB_QPT_RC)
491 return -EOPNOTSUPP;
492
493 if (erdma_device_rocev2(dev) && attrs->qp_type != IB_QPT_RC &&
494 attrs->qp_type != IB_QPT_UD && attrs->qp_type != IB_QPT_GSI)
495 return -EOPNOTSUPP;
496
497 if (attrs->srq)
498 return -EOPNOTSUPP;
499
500 if (!attrs->send_cq || !attrs->recv_cq)
501 return -EOPNOTSUPP;
502
503 return 0;
504 }
505
free_kernel_qp(struct erdma_qp * qp)506 static void free_kernel_qp(struct erdma_qp *qp)
507 {
508 struct erdma_dev *dev = qp->dev;
509
510 vfree(qp->kern_qp.swr_tbl);
511 vfree(qp->kern_qp.rwr_tbl);
512
513 if (qp->kern_qp.sq_buf)
514 dma_free_coherent(&dev->pdev->dev,
515 qp->attrs.sq_size << SQEBB_SHIFT,
516 qp->kern_qp.sq_buf,
517 qp->kern_qp.sq_buf_dma_addr);
518
519 if (qp->kern_qp.sq_dbrec)
520 dma_pool_free(dev->db_pool, qp->kern_qp.sq_dbrec,
521 qp->kern_qp.sq_dbrec_dma);
522
523 if (qp->kern_qp.rq_buf)
524 dma_free_coherent(&dev->pdev->dev,
525 qp->attrs.rq_size << RQE_SHIFT,
526 qp->kern_qp.rq_buf,
527 qp->kern_qp.rq_buf_dma_addr);
528
529 if (qp->kern_qp.rq_dbrec)
530 dma_pool_free(dev->db_pool, qp->kern_qp.rq_dbrec,
531 qp->kern_qp.rq_dbrec_dma);
532 }
533
init_kernel_qp(struct erdma_dev * dev,struct erdma_qp * qp,struct ib_qp_init_attr * attrs)534 static int init_kernel_qp(struct erdma_dev *dev, struct erdma_qp *qp,
535 struct ib_qp_init_attr *attrs)
536 {
537 struct erdma_kqp *kqp = &qp->kern_qp;
538 int size;
539
540 if (attrs->sq_sig_type == IB_SIGNAL_ALL_WR)
541 kqp->sig_all = 1;
542
543 kqp->sq_pi = 0;
544 kqp->sq_ci = 0;
545 kqp->rq_pi = 0;
546 kqp->rq_ci = 0;
547 kqp->hw_sq_db =
548 dev->func_bar + (ERDMA_SDB_SHARED_PAGE_INDEX << PAGE_SHIFT);
549 kqp->hw_rq_db = dev->func_bar + ERDMA_BAR_RQDB_SPACE_OFFSET;
550
551 kqp->swr_tbl = vmalloc_array(qp->attrs.sq_size, sizeof(u64));
552 kqp->rwr_tbl = vmalloc_array(qp->attrs.rq_size, sizeof(u64));
553 if (!kqp->swr_tbl || !kqp->rwr_tbl)
554 goto err_out;
555
556 size = qp->attrs.sq_size << SQEBB_SHIFT;
557 kqp->sq_buf = dma_alloc_coherent(&dev->pdev->dev, size,
558 &kqp->sq_buf_dma_addr, GFP_KERNEL);
559 if (!kqp->sq_buf)
560 goto err_out;
561
562 kqp->sq_dbrec =
563 dma_pool_zalloc(dev->db_pool, GFP_KERNEL, &kqp->sq_dbrec_dma);
564 if (!kqp->sq_dbrec)
565 goto err_out;
566
567 size = qp->attrs.rq_size << RQE_SHIFT;
568 kqp->rq_buf = dma_alloc_coherent(&dev->pdev->dev, size,
569 &kqp->rq_buf_dma_addr, GFP_KERNEL);
570 if (!kqp->rq_buf)
571 goto err_out;
572
573 kqp->rq_dbrec =
574 dma_pool_zalloc(dev->db_pool, GFP_KERNEL, &kqp->rq_dbrec_dma);
575 if (!kqp->rq_dbrec)
576 goto err_out;
577
578 return 0;
579
580 err_out:
581 free_kernel_qp(qp);
582 return -ENOMEM;
583 }
584
erdma_fill_bottom_mtt(struct erdma_dev * dev,struct erdma_mem * mem)585 static void erdma_fill_bottom_mtt(struct erdma_dev *dev, struct erdma_mem *mem)
586 {
587 struct erdma_mtt *mtt = mem->mtt;
588 struct ib_block_iter biter;
589 u32 idx = 0;
590
591 while (mtt->low_level)
592 mtt = mtt->low_level;
593
594 rdma_umem_for_each_dma_block(mem->umem, &biter, mem->page_size)
595 mtt->buf[idx++] = rdma_block_iter_dma_address(&biter);
596 }
597
erdma_create_cont_mtt(struct erdma_dev * dev,size_t size)598 static struct erdma_mtt *erdma_create_cont_mtt(struct erdma_dev *dev,
599 size_t size)
600 {
601 struct erdma_mtt *mtt;
602
603 mtt = kzalloc(sizeof(*mtt), GFP_KERNEL);
604 if (!mtt)
605 return ERR_PTR(-ENOMEM);
606
607 mtt->size = size;
608 mtt->buf = kzalloc(mtt->size, GFP_KERNEL);
609 if (!mtt->buf)
610 goto err_free_mtt;
611
612 mtt->continuous = true;
613 mtt->buf_dma = dma_map_single(&dev->pdev->dev, mtt->buf, mtt->size,
614 DMA_TO_DEVICE);
615 if (dma_mapping_error(&dev->pdev->dev, mtt->buf_dma))
616 goto err_free_mtt_buf;
617
618 return mtt;
619
620 err_free_mtt_buf:
621 kfree(mtt->buf);
622
623 err_free_mtt:
624 kfree(mtt);
625
626 return ERR_PTR(-ENOMEM);
627 }
628
erdma_destroy_mtt_buf_sg(struct erdma_dev * dev,struct erdma_mtt * mtt)629 static void erdma_destroy_mtt_buf_sg(struct erdma_dev *dev,
630 struct erdma_mtt *mtt)
631 {
632 dma_unmap_sg(&dev->pdev->dev, mtt->sglist,
633 DIV_ROUND_UP(mtt->size, PAGE_SIZE), DMA_TO_DEVICE);
634 vfree(mtt->sglist);
635 }
636
erdma_destroy_scatter_mtt(struct erdma_dev * dev,struct erdma_mtt * mtt)637 static void erdma_destroy_scatter_mtt(struct erdma_dev *dev,
638 struct erdma_mtt *mtt)
639 {
640 erdma_destroy_mtt_buf_sg(dev, mtt);
641 vfree(mtt->buf);
642 kfree(mtt);
643 }
644
erdma_init_middle_mtt(struct erdma_mtt * mtt,struct erdma_mtt * low_mtt)645 static void erdma_init_middle_mtt(struct erdma_mtt *mtt,
646 struct erdma_mtt *low_mtt)
647 {
648 struct scatterlist *sg;
649 u32 idx = 0, i;
650
651 for_each_sg(low_mtt->sglist, sg, low_mtt->nsg, i)
652 mtt->buf[idx++] = sg_dma_address(sg);
653 }
654
erdma_create_mtt_buf_sg(struct erdma_dev * dev,struct erdma_mtt * mtt)655 static int erdma_create_mtt_buf_sg(struct erdma_dev *dev, struct erdma_mtt *mtt)
656 {
657 struct scatterlist *sglist;
658 void *buf = mtt->buf;
659 u32 npages, i, nsg;
660 struct page *pg;
661
662 /* Failed if buf is not page aligned */
663 if ((uintptr_t)buf & ~PAGE_MASK)
664 return -EINVAL;
665
666 npages = DIV_ROUND_UP(mtt->size, PAGE_SIZE);
667 sglist = vzalloc(npages * sizeof(*sglist));
668 if (!sglist)
669 return -ENOMEM;
670
671 sg_init_table(sglist, npages);
672 for (i = 0; i < npages; i++) {
673 pg = vmalloc_to_page(buf);
674 if (!pg)
675 goto err;
676 sg_set_page(&sglist[i], pg, PAGE_SIZE, 0);
677 buf += PAGE_SIZE;
678 }
679
680 nsg = dma_map_sg(&dev->pdev->dev, sglist, npages, DMA_TO_DEVICE);
681 if (!nsg)
682 goto err;
683
684 mtt->sglist = sglist;
685 mtt->nsg = nsg;
686
687 return 0;
688 err:
689 vfree(sglist);
690
691 return -ENOMEM;
692 }
693
erdma_create_scatter_mtt(struct erdma_dev * dev,size_t size)694 static struct erdma_mtt *erdma_create_scatter_mtt(struct erdma_dev *dev,
695 size_t size)
696 {
697 struct erdma_mtt *mtt;
698 int ret = -ENOMEM;
699
700 mtt = kzalloc(sizeof(*mtt), GFP_KERNEL);
701 if (!mtt)
702 return ERR_PTR(-ENOMEM);
703
704 mtt->size = ALIGN(size, PAGE_SIZE);
705 mtt->buf = vzalloc(mtt->size);
706 mtt->continuous = false;
707 if (!mtt->buf)
708 goto err_free_mtt;
709
710 ret = erdma_create_mtt_buf_sg(dev, mtt);
711 if (ret)
712 goto err_free_mtt_buf;
713
714 ibdev_dbg(&dev->ibdev, "create scatter mtt, size:%lu, nsg:%u\n",
715 mtt->size, mtt->nsg);
716
717 return mtt;
718
719 err_free_mtt_buf:
720 vfree(mtt->buf);
721
722 err_free_mtt:
723 kfree(mtt);
724
725 return ERR_PTR(ret);
726 }
727
erdma_create_mtt(struct erdma_dev * dev,size_t size,bool force_continuous)728 static struct erdma_mtt *erdma_create_mtt(struct erdma_dev *dev, size_t size,
729 bool force_continuous)
730 {
731 struct erdma_mtt *mtt, *tmp_mtt;
732 int ret, level = 0;
733
734 ibdev_dbg(&dev->ibdev, "create_mtt, size:%lu, force cont:%d\n", size,
735 force_continuous);
736
737 if (!(dev->attrs.cap_flags & ERDMA_DEV_CAP_FLAGS_MTT_VA))
738 force_continuous = true;
739
740 if (force_continuous)
741 return erdma_create_cont_mtt(dev, size);
742
743 mtt = erdma_create_scatter_mtt(dev, size);
744 if (IS_ERR(mtt))
745 return mtt;
746 level = 1;
747
748 /* convergence the mtt table. */
749 while (mtt->nsg != 1 && level <= 3) {
750 tmp_mtt = erdma_create_scatter_mtt(dev, MTT_SIZE(mtt->nsg));
751 if (IS_ERR(tmp_mtt)) {
752 ret = PTR_ERR(tmp_mtt);
753 goto err_free_mtt;
754 }
755 erdma_init_middle_mtt(tmp_mtt, mtt);
756 tmp_mtt->low_level = mtt;
757 mtt = tmp_mtt;
758 level++;
759 }
760
761 if (level > 3) {
762 ret = -ENOMEM;
763 goto err_free_mtt;
764 }
765
766 mtt->level = level;
767 ibdev_dbg(&dev->ibdev, "top mtt: level:%d, dma_addr 0x%llx\n",
768 mtt->level, mtt->sglist[0].dma_address);
769
770 return mtt;
771 err_free_mtt:
772 while (mtt) {
773 tmp_mtt = mtt->low_level;
774 erdma_destroy_scatter_mtt(dev, mtt);
775 mtt = tmp_mtt;
776 }
777
778 return ERR_PTR(ret);
779 }
780
erdma_destroy_mtt(struct erdma_dev * dev,struct erdma_mtt * mtt)781 static void erdma_destroy_mtt(struct erdma_dev *dev, struct erdma_mtt *mtt)
782 {
783 struct erdma_mtt *tmp_mtt;
784
785 if (mtt->continuous) {
786 dma_unmap_single(&dev->pdev->dev, mtt->buf_dma, mtt->size,
787 DMA_TO_DEVICE);
788 kfree(mtt->buf);
789 kfree(mtt);
790 } else {
791 while (mtt) {
792 tmp_mtt = mtt->low_level;
793 erdma_destroy_scatter_mtt(dev, mtt);
794 mtt = tmp_mtt;
795 }
796 }
797 }
798
get_mtt_entries(struct erdma_dev * dev,struct erdma_mem * mem,u64 start,u64 len,int access,u64 virt,unsigned long req_page_size,bool force_continuous)799 static int get_mtt_entries(struct erdma_dev *dev, struct erdma_mem *mem,
800 u64 start, u64 len, int access, u64 virt,
801 unsigned long req_page_size, bool force_continuous)
802 {
803 int ret = 0;
804
805 mem->umem = ib_umem_get(&dev->ibdev, start, len, access);
806 if (IS_ERR(mem->umem)) {
807 ret = PTR_ERR(mem->umem);
808 mem->umem = NULL;
809 return ret;
810 }
811
812 mem->va = virt;
813 mem->len = len;
814 mem->page_size = ib_umem_find_best_pgsz(mem->umem, req_page_size, virt);
815 mem->page_offset = start & (mem->page_size - 1);
816 mem->mtt_nents = ib_umem_num_dma_blocks(mem->umem, mem->page_size);
817 mem->page_cnt = mem->mtt_nents;
818 mem->mtt = erdma_create_mtt(dev, MTT_SIZE(mem->page_cnt),
819 force_continuous);
820 if (IS_ERR(mem->mtt)) {
821 ret = PTR_ERR(mem->mtt);
822 goto error_ret;
823 }
824
825 erdma_fill_bottom_mtt(dev, mem);
826
827 return 0;
828
829 error_ret:
830 if (mem->umem) {
831 ib_umem_release(mem->umem);
832 mem->umem = NULL;
833 }
834
835 return ret;
836 }
837
put_mtt_entries(struct erdma_dev * dev,struct erdma_mem * mem)838 static void put_mtt_entries(struct erdma_dev *dev, struct erdma_mem *mem)
839 {
840 if (mem->mtt)
841 erdma_destroy_mtt(dev, mem->mtt);
842
843 if (mem->umem) {
844 ib_umem_release(mem->umem);
845 mem->umem = NULL;
846 }
847 }
848
erdma_map_user_dbrecords(struct erdma_ucontext * ctx,u64 dbrecords_va,struct erdma_user_dbrecords_page ** dbr_page,dma_addr_t * dma_addr)849 static int erdma_map_user_dbrecords(struct erdma_ucontext *ctx,
850 u64 dbrecords_va,
851 struct erdma_user_dbrecords_page **dbr_page,
852 dma_addr_t *dma_addr)
853 {
854 struct erdma_user_dbrecords_page *page = NULL;
855 int rv = 0;
856
857 mutex_lock(&ctx->dbrecords_page_mutex);
858
859 list_for_each_entry(page, &ctx->dbrecords_page_list, list)
860 if (page->va == (dbrecords_va & PAGE_MASK))
861 goto found;
862
863 page = kmalloc(sizeof(*page), GFP_KERNEL);
864 if (!page) {
865 rv = -ENOMEM;
866 goto out;
867 }
868
869 page->va = (dbrecords_va & PAGE_MASK);
870 page->refcnt = 0;
871
872 page->umem = ib_umem_get(ctx->ibucontext.device,
873 dbrecords_va & PAGE_MASK, PAGE_SIZE, 0);
874 if (IS_ERR(page->umem)) {
875 rv = PTR_ERR(page->umem);
876 kfree(page);
877 goto out;
878 }
879
880 list_add(&page->list, &ctx->dbrecords_page_list);
881
882 found:
883 *dma_addr = sg_dma_address(page->umem->sgt_append.sgt.sgl) +
884 (dbrecords_va & ~PAGE_MASK);
885 *dbr_page = page;
886 page->refcnt++;
887
888 out:
889 mutex_unlock(&ctx->dbrecords_page_mutex);
890 return rv;
891 }
892
893 static void
erdma_unmap_user_dbrecords(struct erdma_ucontext * ctx,struct erdma_user_dbrecords_page ** dbr_page)894 erdma_unmap_user_dbrecords(struct erdma_ucontext *ctx,
895 struct erdma_user_dbrecords_page **dbr_page)
896 {
897 if (!ctx || !(*dbr_page))
898 return;
899
900 mutex_lock(&ctx->dbrecords_page_mutex);
901 if (--(*dbr_page)->refcnt == 0) {
902 list_del(&(*dbr_page)->list);
903 ib_umem_release((*dbr_page)->umem);
904 kfree(*dbr_page);
905 }
906
907 *dbr_page = NULL;
908 mutex_unlock(&ctx->dbrecords_page_mutex);
909 }
910
init_user_qp(struct erdma_qp * qp,struct erdma_ucontext * uctx,u64 va,u32 len,u64 dbrec_va)911 static int init_user_qp(struct erdma_qp *qp, struct erdma_ucontext *uctx,
912 u64 va, u32 len, u64 dbrec_va)
913 {
914 dma_addr_t dbrec_dma;
915 u32 rq_offset;
916 int ret;
917
918 if (len < (ALIGN(qp->attrs.sq_size * SQEBB_SIZE, ERDMA_HW_PAGE_SIZE) +
919 qp->attrs.rq_size * RQE_SIZE))
920 return -EINVAL;
921
922 ret = get_mtt_entries(qp->dev, &qp->user_qp.sq_mem, va,
923 qp->attrs.sq_size << SQEBB_SHIFT, 0, va,
924 (SZ_1M - SZ_4K), true);
925 if (ret)
926 return ret;
927
928 rq_offset = ALIGN(qp->attrs.sq_size << SQEBB_SHIFT, ERDMA_HW_PAGE_SIZE);
929 qp->user_qp.rq_offset = rq_offset;
930
931 ret = get_mtt_entries(qp->dev, &qp->user_qp.rq_mem, va + rq_offset,
932 qp->attrs.rq_size << RQE_SHIFT, 0, va + rq_offset,
933 (SZ_1M - SZ_4K), true);
934 if (ret)
935 goto put_sq_mtt;
936
937 ret = erdma_map_user_dbrecords(uctx, dbrec_va,
938 &qp->user_qp.user_dbr_page,
939 &dbrec_dma);
940 if (ret)
941 goto put_rq_mtt;
942
943 qp->user_qp.sq_dbrec_dma = dbrec_dma;
944 qp->user_qp.rq_dbrec_dma = dbrec_dma + ERDMA_DB_SIZE;
945
946 return 0;
947
948 put_rq_mtt:
949 put_mtt_entries(qp->dev, &qp->user_qp.rq_mem);
950
951 put_sq_mtt:
952 put_mtt_entries(qp->dev, &qp->user_qp.sq_mem);
953
954 return ret;
955 }
956
free_user_qp(struct erdma_qp * qp,struct erdma_ucontext * uctx)957 static void free_user_qp(struct erdma_qp *qp, struct erdma_ucontext *uctx)
958 {
959 put_mtt_entries(qp->dev, &qp->user_qp.sq_mem);
960 put_mtt_entries(qp->dev, &qp->user_qp.rq_mem);
961 erdma_unmap_user_dbrecords(uctx, &qp->user_qp.user_dbr_page);
962 }
963
erdma_create_qp(struct ib_qp * ibqp,struct ib_qp_init_attr * attrs,struct ib_udata * udata)964 int erdma_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs,
965 struct ib_udata *udata)
966 {
967 struct erdma_qp *qp = to_eqp(ibqp);
968 struct erdma_dev *dev = to_edev(ibqp->device);
969 struct erdma_ucontext *uctx = rdma_udata_to_drv_context(
970 udata, struct erdma_ucontext, ibucontext);
971 struct erdma_ureq_create_qp ureq;
972 struct erdma_uresp_create_qp uresp;
973 void *old_entry;
974 int ret = 0;
975
976 ret = erdma_qp_validate_cap(dev, attrs);
977 if (ret)
978 goto err_out;
979
980 ret = erdma_qp_validate_attr(dev, attrs);
981 if (ret)
982 goto err_out;
983
984 qp->scq = to_ecq(attrs->send_cq);
985 qp->rcq = to_ecq(attrs->recv_cq);
986 qp->dev = dev;
987 qp->attrs.cc = dev->attrs.cc;
988
989 init_rwsem(&qp->state_lock);
990 kref_init(&qp->ref);
991 init_completion(&qp->safe_free);
992
993 if (qp->ibqp.qp_type == IB_QPT_GSI) {
994 old_entry = xa_store(&dev->qp_xa, 1, qp, GFP_KERNEL);
995 if (xa_is_err(old_entry))
996 ret = xa_err(old_entry);
997 else
998 qp->ibqp.qp_num = 1;
999 } else {
1000 ret = xa_alloc_cyclic(&dev->qp_xa, &qp->ibqp.qp_num, qp,
1001 XA_LIMIT(1, dev->attrs.max_qp - 1),
1002 &dev->next_alloc_qpn, GFP_KERNEL);
1003 }
1004
1005 if (ret < 0) {
1006 ret = -ENOMEM;
1007 goto err_out;
1008 }
1009
1010 qp->attrs.sq_size = roundup_pow_of_two(attrs->cap.max_send_wr *
1011 ERDMA_MAX_WQEBB_PER_SQE);
1012 qp->attrs.rq_size = roundup_pow_of_two(attrs->cap.max_recv_wr);
1013
1014 if (uctx) {
1015 ret = ib_copy_from_udata(&ureq, udata,
1016 min(sizeof(ureq), udata->inlen));
1017 if (ret)
1018 goto err_out_xa;
1019
1020 ret = init_user_qp(qp, uctx, ureq.qbuf_va, ureq.qbuf_len,
1021 ureq.db_record_va);
1022 if (ret)
1023 goto err_out_xa;
1024
1025 memset(&uresp, 0, sizeof(uresp));
1026
1027 uresp.num_sqe = qp->attrs.sq_size;
1028 uresp.num_rqe = qp->attrs.rq_size;
1029 uresp.qp_id = QP_ID(qp);
1030 uresp.rq_offset = qp->user_qp.rq_offset;
1031
1032 ret = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1033 if (ret)
1034 goto err_out_cmd;
1035 } else {
1036 ret = init_kernel_qp(dev, qp, attrs);
1037 if (ret)
1038 goto err_out_xa;
1039 }
1040
1041 qp->attrs.max_send_sge = attrs->cap.max_send_sge;
1042 qp->attrs.max_recv_sge = attrs->cap.max_recv_sge;
1043
1044 if (erdma_device_iwarp(qp->dev))
1045 qp->attrs.iwarp.state = ERDMA_QPS_IWARP_IDLE;
1046 else
1047 qp->attrs.rocev2.state = ERDMA_QPS_ROCEV2_RESET;
1048
1049 INIT_DELAYED_WORK(&qp->reflush_dwork, erdma_flush_worker);
1050
1051 ret = create_qp_cmd(uctx, qp);
1052 if (ret)
1053 goto err_out_cmd;
1054
1055 spin_lock_init(&qp->lock);
1056
1057 return 0;
1058
1059 err_out_cmd:
1060 if (uctx)
1061 free_user_qp(qp, uctx);
1062 else
1063 free_kernel_qp(qp);
1064 err_out_xa:
1065 xa_erase(&dev->qp_xa, QP_ID(qp));
1066 err_out:
1067 return ret;
1068 }
1069
erdma_create_stag(struct erdma_dev * dev,u32 * stag)1070 static int erdma_create_stag(struct erdma_dev *dev, u32 *stag)
1071 {
1072 int stag_idx;
1073
1074 stag_idx = erdma_alloc_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX]);
1075 if (stag_idx < 0)
1076 return stag_idx;
1077
1078 /* For now, we always let key field be zero. */
1079 *stag = (stag_idx << 8);
1080
1081 return 0;
1082 }
1083
erdma_get_dma_mr(struct ib_pd * ibpd,int acc)1084 struct ib_mr *erdma_get_dma_mr(struct ib_pd *ibpd, int acc)
1085 {
1086 struct erdma_dev *dev = to_edev(ibpd->device);
1087 struct erdma_mr *mr;
1088 u32 stag;
1089 int ret;
1090
1091 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1092 if (!mr)
1093 return ERR_PTR(-ENOMEM);
1094
1095 ret = erdma_create_stag(dev, &stag);
1096 if (ret)
1097 goto out_free;
1098
1099 mr->type = ERDMA_MR_TYPE_DMA;
1100
1101 mr->ibmr.lkey = stag;
1102 mr->ibmr.rkey = stag;
1103 mr->ibmr.pd = ibpd;
1104 mr->access = ERDMA_MR_ACC_LR | to_erdma_access_flags(acc);
1105 ret = regmr_cmd(dev, mr);
1106 if (ret)
1107 goto out_remove_stag;
1108
1109 return &mr->ibmr;
1110
1111 out_remove_stag:
1112 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX],
1113 mr->ibmr.lkey >> 8);
1114
1115 out_free:
1116 kfree(mr);
1117
1118 return ERR_PTR(ret);
1119 }
1120
erdma_ib_alloc_mr(struct ib_pd * ibpd,enum ib_mr_type mr_type,u32 max_num_sg)1121 struct ib_mr *erdma_ib_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
1122 u32 max_num_sg)
1123 {
1124 struct erdma_mr *mr;
1125 struct erdma_dev *dev = to_edev(ibpd->device);
1126 int ret;
1127 u32 stag;
1128
1129 if (mr_type != IB_MR_TYPE_MEM_REG)
1130 return ERR_PTR(-EOPNOTSUPP);
1131
1132 if (max_num_sg > ERDMA_MR_MAX_MTT_CNT)
1133 return ERR_PTR(-EINVAL);
1134
1135 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1136 if (!mr)
1137 return ERR_PTR(-ENOMEM);
1138
1139 ret = erdma_create_stag(dev, &stag);
1140 if (ret)
1141 goto out_free;
1142
1143 mr->type = ERDMA_MR_TYPE_FRMR;
1144
1145 mr->ibmr.lkey = stag;
1146 mr->ibmr.rkey = stag;
1147 mr->ibmr.pd = ibpd;
1148 /* update it in FRMR. */
1149 mr->access = ERDMA_MR_ACC_LR | ERDMA_MR_ACC_LW | ERDMA_MR_ACC_RR |
1150 ERDMA_MR_ACC_RW;
1151
1152 mr->mem.page_size = PAGE_SIZE; /* update it later. */
1153 mr->mem.page_cnt = max_num_sg;
1154 mr->mem.mtt = erdma_create_mtt(dev, MTT_SIZE(max_num_sg), true);
1155 if (IS_ERR(mr->mem.mtt)) {
1156 ret = PTR_ERR(mr->mem.mtt);
1157 goto out_remove_stag;
1158 }
1159
1160 ret = regmr_cmd(dev, mr);
1161 if (ret)
1162 goto out_destroy_mtt;
1163
1164 return &mr->ibmr;
1165
1166 out_destroy_mtt:
1167 erdma_destroy_mtt(dev, mr->mem.mtt);
1168
1169 out_remove_stag:
1170 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX],
1171 mr->ibmr.lkey >> 8);
1172
1173 out_free:
1174 kfree(mr);
1175
1176 return ERR_PTR(ret);
1177 }
1178
erdma_set_page(struct ib_mr * ibmr,u64 addr)1179 static int erdma_set_page(struct ib_mr *ibmr, u64 addr)
1180 {
1181 struct erdma_mr *mr = to_emr(ibmr);
1182
1183 if (mr->mem.mtt_nents >= mr->mem.page_cnt)
1184 return -1;
1185
1186 mr->mem.mtt->buf[mr->mem.mtt_nents] = addr;
1187 mr->mem.mtt_nents++;
1188
1189 return 0;
1190 }
1191
erdma_map_mr_sg(struct ib_mr * ibmr,struct scatterlist * sg,int sg_nents,unsigned int * sg_offset)1192 int erdma_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
1193 unsigned int *sg_offset)
1194 {
1195 struct erdma_mr *mr = to_emr(ibmr);
1196 int num;
1197
1198 mr->mem.mtt_nents = 0;
1199
1200 num = ib_sg_to_pages(&mr->ibmr, sg, sg_nents, sg_offset,
1201 erdma_set_page);
1202
1203 return num;
1204 }
1205
erdma_reg_user_mr(struct ib_pd * ibpd,u64 start,u64 len,u64 virt,int access,struct ib_dmah * dmah,struct ib_udata * udata)1206 struct ib_mr *erdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
1207 u64 virt, int access, struct ib_dmah *dmah,
1208 struct ib_udata *udata)
1209 {
1210 struct erdma_mr *mr = NULL;
1211 struct erdma_dev *dev = to_edev(ibpd->device);
1212 u32 stag;
1213 int ret;
1214
1215 if (dmah)
1216 return ERR_PTR(-EOPNOTSUPP);
1217
1218 if (!len || len > dev->attrs.max_mr_size)
1219 return ERR_PTR(-EINVAL);
1220
1221 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1222 if (!mr)
1223 return ERR_PTR(-ENOMEM);
1224
1225 ret = get_mtt_entries(dev, &mr->mem, start, len, access, virt,
1226 SZ_2G - SZ_4K, false);
1227 if (ret)
1228 goto err_out_free;
1229
1230 ret = erdma_create_stag(dev, &stag);
1231 if (ret)
1232 goto err_out_put_mtt;
1233
1234 mr->ibmr.lkey = mr->ibmr.rkey = stag;
1235 mr->ibmr.pd = ibpd;
1236 mr->mem.va = virt;
1237 mr->mem.len = len;
1238 mr->access = ERDMA_MR_ACC_LR | to_erdma_access_flags(access);
1239 mr->valid = 1;
1240 mr->type = ERDMA_MR_TYPE_NORMAL;
1241
1242 ret = regmr_cmd(dev, mr);
1243 if (ret)
1244 goto err_out_mr;
1245
1246 return &mr->ibmr;
1247
1248 err_out_mr:
1249 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX],
1250 mr->ibmr.lkey >> 8);
1251
1252 err_out_put_mtt:
1253 put_mtt_entries(dev, &mr->mem);
1254
1255 err_out_free:
1256 kfree(mr);
1257
1258 return ERR_PTR(ret);
1259 }
1260
erdma_dereg_mr(struct ib_mr * ibmr,struct ib_udata * udata)1261 int erdma_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)
1262 {
1263 struct erdma_mr *mr;
1264 struct erdma_dev *dev = to_edev(ibmr->device);
1265 struct erdma_cmdq_dereg_mr_req req;
1266 int ret;
1267
1268 mr = to_emr(ibmr);
1269
1270 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
1271 CMDQ_OPCODE_DEREG_MR);
1272
1273 req.cfg = FIELD_PREP(ERDMA_CMD_MR_MPT_IDX_MASK, ibmr->lkey >> 8) |
1274 FIELD_PREP(ERDMA_CMD_MR_KEY_MASK, ibmr->lkey & 0xFF);
1275
1276 ret = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL,
1277 true);
1278 if (ret)
1279 return ret;
1280
1281 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX], ibmr->lkey >> 8);
1282
1283 put_mtt_entries(dev, &mr->mem);
1284
1285 kfree(mr);
1286 return 0;
1287 }
1288
erdma_destroy_cq(struct ib_cq * ibcq,struct ib_udata * udata)1289 int erdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
1290 {
1291 struct erdma_cq *cq = to_ecq(ibcq);
1292 struct erdma_dev *dev = to_edev(ibcq->device);
1293 struct erdma_ucontext *ctx = rdma_udata_to_drv_context(
1294 udata, struct erdma_ucontext, ibucontext);
1295 int err;
1296 struct erdma_cmdq_destroy_cq_req req;
1297
1298 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
1299 CMDQ_OPCODE_DESTROY_CQ);
1300 req.cqn = cq->cqn;
1301
1302 err = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL,
1303 true);
1304 if (err)
1305 return err;
1306
1307 if (rdma_is_kernel_res(&cq->ibcq.res)) {
1308 dma_free_coherent(&dev->pdev->dev, cq->depth << CQE_SHIFT,
1309 cq->kern_cq.qbuf, cq->kern_cq.qbuf_dma_addr);
1310 dma_pool_free(dev->db_pool, cq->kern_cq.dbrec,
1311 cq->kern_cq.dbrec_dma);
1312 } else {
1313 erdma_unmap_user_dbrecords(ctx, &cq->user_cq.user_dbr_page);
1314 put_mtt_entries(dev, &cq->user_cq.qbuf_mem);
1315 }
1316
1317 xa_erase(&dev->cq_xa, cq->cqn);
1318
1319 return 0;
1320 }
1321
erdma_destroy_qp(struct ib_qp * ibqp,struct ib_udata * udata)1322 int erdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
1323 {
1324 struct erdma_qp *qp = to_eqp(ibqp);
1325 struct erdma_dev *dev = to_edev(ibqp->device);
1326 struct erdma_ucontext *ctx = rdma_udata_to_drv_context(
1327 udata, struct erdma_ucontext, ibucontext);
1328 struct erdma_cmdq_destroy_qp_req req;
1329 union erdma_mod_qp_params params;
1330 int err;
1331
1332 down_write(&qp->state_lock);
1333 if (erdma_device_iwarp(dev)) {
1334 params.iwarp.state = ERDMA_QPS_IWARP_ERROR;
1335 erdma_modify_qp_state_iwarp(qp, ¶ms.iwarp,
1336 ERDMA_QPA_IWARP_STATE);
1337 } else {
1338 params.rocev2.state = ERDMA_QPS_ROCEV2_ERROR;
1339 erdma_modify_qp_state_rocev2(qp, ¶ms.rocev2,
1340 ERDMA_QPA_ROCEV2_STATE);
1341 }
1342 up_write(&qp->state_lock);
1343
1344 cancel_delayed_work_sync(&qp->reflush_dwork);
1345
1346 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
1347 CMDQ_OPCODE_DESTROY_QP);
1348 req.qpn = QP_ID(qp);
1349
1350 err = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL,
1351 true);
1352 if (err)
1353 return err;
1354
1355 erdma_qp_put(qp);
1356 wait_for_completion(&qp->safe_free);
1357
1358 if (rdma_is_kernel_res(&qp->ibqp.res)) {
1359 free_kernel_qp(qp);
1360 } else {
1361 put_mtt_entries(dev, &qp->user_qp.sq_mem);
1362 put_mtt_entries(dev, &qp->user_qp.rq_mem);
1363 erdma_unmap_user_dbrecords(ctx, &qp->user_qp.user_dbr_page);
1364 }
1365
1366 if (qp->cep)
1367 erdma_cep_put(qp->cep);
1368 xa_erase(&dev->qp_xa, QP_ID(qp));
1369
1370 return 0;
1371 }
1372
erdma_qp_get_ref(struct ib_qp * ibqp)1373 void erdma_qp_get_ref(struct ib_qp *ibqp)
1374 {
1375 erdma_qp_get(to_eqp(ibqp));
1376 }
1377
erdma_qp_put_ref(struct ib_qp * ibqp)1378 void erdma_qp_put_ref(struct ib_qp *ibqp)
1379 {
1380 erdma_qp_put(to_eqp(ibqp));
1381 }
1382
erdma_mmap(struct ib_ucontext * ctx,struct vm_area_struct * vma)1383 int erdma_mmap(struct ib_ucontext *ctx, struct vm_area_struct *vma)
1384 {
1385 struct rdma_user_mmap_entry *rdma_entry;
1386 struct erdma_user_mmap_entry *entry;
1387 pgprot_t prot;
1388 int err;
1389
1390 rdma_entry = rdma_user_mmap_entry_get(ctx, vma);
1391 if (!rdma_entry)
1392 return -EINVAL;
1393
1394 entry = to_emmap(rdma_entry);
1395
1396 switch (entry->mmap_flag) {
1397 case ERDMA_MMAP_IO_NC:
1398 /* map doorbell. */
1399 prot = pgprot_device(vma->vm_page_prot);
1400 break;
1401 default:
1402 err = -EINVAL;
1403 goto put_entry;
1404 }
1405
1406 err = rdma_user_mmap_io(ctx, vma, PFN_DOWN(entry->address), PAGE_SIZE,
1407 prot, rdma_entry);
1408
1409 put_entry:
1410 rdma_user_mmap_entry_put(rdma_entry);
1411 return err;
1412 }
1413
erdma_mmap_free(struct rdma_user_mmap_entry * rdma_entry)1414 void erdma_mmap_free(struct rdma_user_mmap_entry *rdma_entry)
1415 {
1416 struct erdma_user_mmap_entry *entry = to_emmap(rdma_entry);
1417
1418 kfree(entry);
1419 }
1420
alloc_db_resources(struct erdma_dev * dev,struct erdma_ucontext * ctx,bool ext_db_en)1421 static int alloc_db_resources(struct erdma_dev *dev, struct erdma_ucontext *ctx,
1422 bool ext_db_en)
1423 {
1424 struct erdma_cmdq_ext_db_req req = {};
1425 u64 val0, val1;
1426 int ret;
1427
1428 /*
1429 * CAP_SYS_RAWIO is required if hardware does not support extend
1430 * doorbell mechanism.
1431 */
1432 if (!ext_db_en && !capable(CAP_SYS_RAWIO))
1433 return -EPERM;
1434
1435 if (!ext_db_en) {
1436 ctx->sdb = dev->func_bar_addr + ERDMA_BAR_SQDB_SPACE_OFFSET;
1437 ctx->rdb = dev->func_bar_addr + ERDMA_BAR_RQDB_SPACE_OFFSET;
1438 ctx->cdb = dev->func_bar_addr + ERDMA_BAR_CQDB_SPACE_OFFSET;
1439 return 0;
1440 }
1441
1442 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_COMMON,
1443 CMDQ_OPCODE_ALLOC_DB);
1444
1445 req.cfg = FIELD_PREP(ERDMA_CMD_EXT_DB_CQ_EN_MASK, 1) |
1446 FIELD_PREP(ERDMA_CMD_EXT_DB_RQ_EN_MASK, 1) |
1447 FIELD_PREP(ERDMA_CMD_EXT_DB_SQ_EN_MASK, 1);
1448
1449 ret = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), &val0, &val1,
1450 true);
1451 if (ret)
1452 return ret;
1453
1454 ctx->ext_db.enable = true;
1455 ctx->ext_db.sdb_off = ERDMA_GET(val0, ALLOC_DB_RESP_SDB);
1456 ctx->ext_db.rdb_off = ERDMA_GET(val0, ALLOC_DB_RESP_RDB);
1457 ctx->ext_db.cdb_off = ERDMA_GET(val0, ALLOC_DB_RESP_CDB);
1458
1459 ctx->sdb = dev->func_bar_addr + (ctx->ext_db.sdb_off << PAGE_SHIFT);
1460 ctx->cdb = dev->func_bar_addr + (ctx->ext_db.rdb_off << PAGE_SHIFT);
1461 ctx->rdb = dev->func_bar_addr + (ctx->ext_db.cdb_off << PAGE_SHIFT);
1462
1463 return 0;
1464 }
1465
free_db_resources(struct erdma_dev * dev,struct erdma_ucontext * ctx)1466 static void free_db_resources(struct erdma_dev *dev, struct erdma_ucontext *ctx)
1467 {
1468 struct erdma_cmdq_ext_db_req req = {};
1469 int ret;
1470
1471 if (!ctx->ext_db.enable)
1472 return;
1473
1474 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_COMMON,
1475 CMDQ_OPCODE_FREE_DB);
1476
1477 req.cfg = FIELD_PREP(ERDMA_CMD_EXT_DB_CQ_EN_MASK, 1) |
1478 FIELD_PREP(ERDMA_CMD_EXT_DB_RQ_EN_MASK, 1) |
1479 FIELD_PREP(ERDMA_CMD_EXT_DB_SQ_EN_MASK, 1);
1480
1481 req.sdb_off = ctx->ext_db.sdb_off;
1482 req.rdb_off = ctx->ext_db.rdb_off;
1483 req.cdb_off = ctx->ext_db.cdb_off;
1484
1485 ret = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL,
1486 true);
1487 if (ret)
1488 ibdev_err_ratelimited(&dev->ibdev,
1489 "free db resources failed %d", ret);
1490 }
1491
erdma_uctx_user_mmap_entries_remove(struct erdma_ucontext * uctx)1492 static void erdma_uctx_user_mmap_entries_remove(struct erdma_ucontext *uctx)
1493 {
1494 rdma_user_mmap_entry_remove(uctx->sq_db_mmap_entry);
1495 rdma_user_mmap_entry_remove(uctx->rq_db_mmap_entry);
1496 rdma_user_mmap_entry_remove(uctx->cq_db_mmap_entry);
1497 }
1498
erdma_alloc_ucontext(struct ib_ucontext * ibctx,struct ib_udata * udata)1499 int erdma_alloc_ucontext(struct ib_ucontext *ibctx, struct ib_udata *udata)
1500 {
1501 struct erdma_ucontext *ctx = to_ectx(ibctx);
1502 struct erdma_dev *dev = to_edev(ibctx->device);
1503 int ret;
1504 struct erdma_uresp_alloc_ctx uresp = {};
1505
1506 if (atomic_inc_return(&dev->num_ctx) > ERDMA_MAX_CONTEXT) {
1507 ret = -ENOMEM;
1508 goto err_out;
1509 }
1510
1511 if (udata->outlen < sizeof(uresp)) {
1512 ret = -EINVAL;
1513 goto err_out;
1514 }
1515
1516 INIT_LIST_HEAD(&ctx->dbrecords_page_list);
1517 mutex_init(&ctx->dbrecords_page_mutex);
1518
1519 ret = alloc_db_resources(dev, ctx,
1520 !!(dev->attrs.cap_flags &
1521 ERDMA_DEV_CAP_FLAGS_EXTEND_DB));
1522 if (ret)
1523 goto err_out;
1524
1525 ctx->sq_db_mmap_entry = erdma_user_mmap_entry_insert(
1526 ctx, (void *)ctx->sdb, PAGE_SIZE, ERDMA_MMAP_IO_NC, &uresp.sdb);
1527 if (!ctx->sq_db_mmap_entry) {
1528 ret = -ENOMEM;
1529 goto err_free_ext_db;
1530 }
1531
1532 ctx->rq_db_mmap_entry = erdma_user_mmap_entry_insert(
1533 ctx, (void *)ctx->rdb, PAGE_SIZE, ERDMA_MMAP_IO_NC, &uresp.rdb);
1534 if (!ctx->rq_db_mmap_entry) {
1535 ret = -EINVAL;
1536 goto err_put_mmap_entries;
1537 }
1538
1539 ctx->cq_db_mmap_entry = erdma_user_mmap_entry_insert(
1540 ctx, (void *)ctx->cdb, PAGE_SIZE, ERDMA_MMAP_IO_NC, &uresp.cdb);
1541 if (!ctx->cq_db_mmap_entry) {
1542 ret = -EINVAL;
1543 goto err_put_mmap_entries;
1544 }
1545
1546 uresp.dev_id = dev->pdev->device;
1547
1548 ret = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1549 if (ret)
1550 goto err_put_mmap_entries;
1551
1552 return 0;
1553
1554 err_put_mmap_entries:
1555 erdma_uctx_user_mmap_entries_remove(ctx);
1556
1557 err_free_ext_db:
1558 free_db_resources(dev, ctx);
1559
1560 err_out:
1561 atomic_dec(&dev->num_ctx);
1562 return ret;
1563 }
1564
erdma_dealloc_ucontext(struct ib_ucontext * ibctx)1565 void erdma_dealloc_ucontext(struct ib_ucontext *ibctx)
1566 {
1567 struct erdma_dev *dev = to_edev(ibctx->device);
1568 struct erdma_ucontext *ctx = to_ectx(ibctx);
1569
1570 erdma_uctx_user_mmap_entries_remove(ctx);
1571 free_db_resources(dev, ctx);
1572 atomic_dec(&dev->num_ctx);
1573 }
1574
erdma_attr_to_av(const struct rdma_ah_attr * ah_attr,struct erdma_av * av,u16 sport)1575 static void erdma_attr_to_av(const struct rdma_ah_attr *ah_attr,
1576 struct erdma_av *av, u16 sport)
1577 {
1578 const struct ib_global_route *grh = rdma_ah_read_grh(ah_attr);
1579
1580 av->port = rdma_ah_get_port_num(ah_attr);
1581 av->sgid_index = grh->sgid_index;
1582 av->hop_limit = grh->hop_limit;
1583 av->traffic_class = grh->traffic_class;
1584 av->sl = rdma_ah_get_sl(ah_attr);
1585
1586 av->flow_label = grh->flow_label;
1587 av->udp_sport = sport;
1588
1589 ether_addr_copy(av->dmac, ah_attr->roce.dmac);
1590 memcpy(av->dgid, grh->dgid.raw, ERDMA_ROCEV2_GID_SIZE);
1591
1592 if (ipv6_addr_v4mapped((struct in6_addr *)&grh->dgid))
1593 av->ntype = ERDMA_NETWORK_TYPE_IPV4;
1594 else
1595 av->ntype = ERDMA_NETWORK_TYPE_IPV6;
1596 }
1597
erdma_av_to_attr(struct erdma_av * av,struct rdma_ah_attr * ah_attr)1598 static void erdma_av_to_attr(struct erdma_av *av, struct rdma_ah_attr *ah_attr)
1599 {
1600 ah_attr->type = RDMA_AH_ATTR_TYPE_ROCE;
1601
1602 rdma_ah_set_sl(ah_attr, av->sl);
1603 rdma_ah_set_port_num(ah_attr, av->port);
1604 rdma_ah_set_ah_flags(ah_attr, IB_AH_GRH);
1605
1606 rdma_ah_set_grh(ah_attr, NULL, av->flow_label, av->sgid_index,
1607 av->hop_limit, av->traffic_class);
1608 rdma_ah_set_dgid_raw(ah_attr, av->dgid);
1609 }
1610
1611 static int ib_qps_to_erdma_qps[ERDMA_PROTO_COUNT][IB_QPS_ERR + 1] = {
1612 [ERDMA_PROTO_IWARP] = {
1613 [IB_QPS_RESET] = ERDMA_QPS_IWARP_IDLE,
1614 [IB_QPS_INIT] = ERDMA_QPS_IWARP_IDLE,
1615 [IB_QPS_RTR] = ERDMA_QPS_IWARP_RTR,
1616 [IB_QPS_RTS] = ERDMA_QPS_IWARP_RTS,
1617 [IB_QPS_SQD] = ERDMA_QPS_IWARP_CLOSING,
1618 [IB_QPS_SQE] = ERDMA_QPS_IWARP_TERMINATE,
1619 [IB_QPS_ERR] = ERDMA_QPS_IWARP_ERROR,
1620 },
1621 [ERDMA_PROTO_ROCEV2] = {
1622 [IB_QPS_RESET] = ERDMA_QPS_ROCEV2_RESET,
1623 [IB_QPS_INIT] = ERDMA_QPS_ROCEV2_INIT,
1624 [IB_QPS_RTR] = ERDMA_QPS_ROCEV2_RTR,
1625 [IB_QPS_RTS] = ERDMA_QPS_ROCEV2_RTS,
1626 [IB_QPS_SQD] = ERDMA_QPS_ROCEV2_SQD,
1627 [IB_QPS_SQE] = ERDMA_QPS_ROCEV2_SQE,
1628 [IB_QPS_ERR] = ERDMA_QPS_ROCEV2_ERROR,
1629 },
1630 };
1631
1632 static int erdma_qps_to_ib_qps[ERDMA_PROTO_COUNT][ERDMA_QPS_ROCEV2_COUNT] = {
1633 [ERDMA_PROTO_IWARP] = {
1634 [ERDMA_QPS_IWARP_IDLE] = IB_QPS_INIT,
1635 [ERDMA_QPS_IWARP_RTR] = IB_QPS_RTR,
1636 [ERDMA_QPS_IWARP_RTS] = IB_QPS_RTS,
1637 [ERDMA_QPS_IWARP_CLOSING] = IB_QPS_ERR,
1638 [ERDMA_QPS_IWARP_TERMINATE] = IB_QPS_ERR,
1639 [ERDMA_QPS_IWARP_ERROR] = IB_QPS_ERR,
1640 },
1641 [ERDMA_PROTO_ROCEV2] = {
1642 [ERDMA_QPS_ROCEV2_RESET] = IB_QPS_RESET,
1643 [ERDMA_QPS_ROCEV2_INIT] = IB_QPS_INIT,
1644 [ERDMA_QPS_ROCEV2_RTR] = IB_QPS_RTR,
1645 [ERDMA_QPS_ROCEV2_RTS] = IB_QPS_RTS,
1646 [ERDMA_QPS_ROCEV2_SQD] = IB_QPS_SQD,
1647 [ERDMA_QPS_ROCEV2_SQE] = IB_QPS_SQE,
1648 [ERDMA_QPS_ROCEV2_ERROR] = IB_QPS_ERR,
1649 },
1650 };
1651
ib_to_iwarp_qps(enum ib_qp_state state)1652 static inline enum erdma_qps_iwarp ib_to_iwarp_qps(enum ib_qp_state state)
1653 {
1654 return ib_qps_to_erdma_qps[ERDMA_PROTO_IWARP][state];
1655 }
1656
ib_to_rocev2_qps(enum ib_qp_state state)1657 static inline enum erdma_qps_rocev2 ib_to_rocev2_qps(enum ib_qp_state state)
1658 {
1659 return ib_qps_to_erdma_qps[ERDMA_PROTO_ROCEV2][state];
1660 }
1661
iwarp_to_ib_qps(enum erdma_qps_iwarp state)1662 static inline enum ib_qp_state iwarp_to_ib_qps(enum erdma_qps_iwarp state)
1663 {
1664 return erdma_qps_to_ib_qps[ERDMA_PROTO_IWARP][state];
1665 }
1666
rocev2_to_ib_qps(enum erdma_qps_rocev2 state)1667 static inline enum ib_qp_state rocev2_to_ib_qps(enum erdma_qps_rocev2 state)
1668 {
1669 return erdma_qps_to_ib_qps[ERDMA_PROTO_ROCEV2][state];
1670 }
1671
erdma_check_qp_attrs(struct erdma_qp * qp,struct ib_qp_attr * attr,int attr_mask)1672 static int erdma_check_qp_attrs(struct erdma_qp *qp, struct ib_qp_attr *attr,
1673 int attr_mask)
1674 {
1675 enum ib_qp_state cur_state, nxt_state;
1676 struct erdma_dev *dev = qp->dev;
1677 int ret = -EINVAL;
1678
1679 if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS) {
1680 ret = -EOPNOTSUPP;
1681 goto out;
1682 }
1683
1684 if ((attr_mask & IB_QP_PORT) &&
1685 !rdma_is_port_valid(&dev->ibdev, attr->port_num))
1686 goto out;
1687
1688 if (erdma_device_rocev2(dev)) {
1689 cur_state = (attr_mask & IB_QP_CUR_STATE) ?
1690 attr->cur_qp_state :
1691 rocev2_to_ib_qps(qp->attrs.rocev2.state);
1692
1693 nxt_state = (attr_mask & IB_QP_STATE) ? attr->qp_state :
1694 cur_state;
1695
1696 if (!ib_modify_qp_is_ok(cur_state, nxt_state, qp->ibqp.qp_type,
1697 attr_mask))
1698 goto out;
1699
1700 if ((attr_mask & IB_QP_AV) &&
1701 erdma_check_gid_attr(
1702 rdma_ah_read_grh(&attr->ah_attr)->sgid_attr))
1703 goto out;
1704
1705 if ((attr_mask & IB_QP_PKEY_INDEX) &&
1706 attr->pkey_index >= ERDMA_MAX_PKEYS)
1707 goto out;
1708 }
1709
1710 return 0;
1711
1712 out:
1713 return ret;
1714 }
1715
erdma_init_mod_qp_params_rocev2(struct erdma_qp * qp,struct erdma_mod_qp_params_rocev2 * params,int * erdma_attr_mask,struct ib_qp_attr * attr,int ib_attr_mask)1716 static void erdma_init_mod_qp_params_rocev2(
1717 struct erdma_qp *qp, struct erdma_mod_qp_params_rocev2 *params,
1718 int *erdma_attr_mask, struct ib_qp_attr *attr, int ib_attr_mask)
1719 {
1720 enum erdma_qpa_mask_rocev2 to_modify_attrs = 0;
1721 enum erdma_qps_rocev2 cur_state, nxt_state;
1722 u16 udp_sport;
1723
1724 if (ib_attr_mask & IB_QP_CUR_STATE)
1725 cur_state = ib_to_rocev2_qps(attr->cur_qp_state);
1726 else
1727 cur_state = qp->attrs.rocev2.state;
1728
1729 if (ib_attr_mask & IB_QP_STATE)
1730 nxt_state = ib_to_rocev2_qps(attr->qp_state);
1731 else
1732 nxt_state = cur_state;
1733
1734 to_modify_attrs |= ERDMA_QPA_ROCEV2_STATE;
1735 params->state = nxt_state;
1736
1737 if (ib_attr_mask & IB_QP_QKEY) {
1738 to_modify_attrs |= ERDMA_QPA_ROCEV2_QKEY;
1739 params->qkey = attr->qkey;
1740 }
1741
1742 if (ib_attr_mask & IB_QP_SQ_PSN) {
1743 to_modify_attrs |= ERDMA_QPA_ROCEV2_SQ_PSN;
1744 params->sq_psn = attr->sq_psn;
1745 }
1746
1747 if (ib_attr_mask & IB_QP_RQ_PSN) {
1748 to_modify_attrs |= ERDMA_QPA_ROCEV2_RQ_PSN;
1749 params->rq_psn = attr->rq_psn;
1750 }
1751
1752 if (ib_attr_mask & IB_QP_DEST_QPN) {
1753 to_modify_attrs |= ERDMA_QPA_ROCEV2_DST_QPN;
1754 params->dst_qpn = attr->dest_qp_num;
1755 }
1756
1757 if (ib_attr_mask & IB_QP_AV) {
1758 to_modify_attrs |= ERDMA_QPA_ROCEV2_AV;
1759 udp_sport = rdma_get_udp_sport(attr->ah_attr.grh.flow_label,
1760 QP_ID(qp), params->dst_qpn);
1761 erdma_attr_to_av(&attr->ah_attr, ¶ms->av, udp_sport);
1762 }
1763
1764 *erdma_attr_mask = to_modify_attrs;
1765 }
1766
erdma_modify_qp(struct ib_qp * ibqp,struct ib_qp_attr * attr,int attr_mask,struct ib_udata * udata)1767 int erdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask,
1768 struct ib_udata *udata)
1769 {
1770 struct erdma_qp *qp = to_eqp(ibqp);
1771 union erdma_mod_qp_params params;
1772 int ret = 0, erdma_attr_mask = 0;
1773
1774 down_write(&qp->state_lock);
1775
1776 ret = erdma_check_qp_attrs(qp, attr, attr_mask);
1777 if (ret)
1778 goto out;
1779
1780 if (erdma_device_iwarp(qp->dev)) {
1781 if (attr_mask & IB_QP_STATE) {
1782 erdma_attr_mask |= ERDMA_QPA_IWARP_STATE;
1783 params.iwarp.state = ib_to_iwarp_qps(attr->qp_state);
1784 }
1785
1786 ret = erdma_modify_qp_state_iwarp(qp, ¶ms.iwarp,
1787 erdma_attr_mask);
1788 } else {
1789 erdma_init_mod_qp_params_rocev2(
1790 qp, ¶ms.rocev2, &erdma_attr_mask, attr, attr_mask);
1791
1792 ret = erdma_modify_qp_state_rocev2(qp, ¶ms.rocev2,
1793 erdma_attr_mask);
1794 }
1795
1796 out:
1797 up_write(&qp->state_lock);
1798 return ret;
1799 }
1800
query_qp_state(struct erdma_qp * qp)1801 static enum ib_qp_state query_qp_state(struct erdma_qp *qp)
1802 {
1803 if (erdma_device_iwarp(qp->dev))
1804 return iwarp_to_ib_qps(qp->attrs.iwarp.state);
1805 else
1806 return rocev2_to_ib_qps(qp->attrs.rocev2.state);
1807 }
1808
erdma_query_qp(struct ib_qp * ibqp,struct ib_qp_attr * qp_attr,int qp_attr_mask,struct ib_qp_init_attr * qp_init_attr)1809 int erdma_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
1810 int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr)
1811 {
1812 struct erdma_cmdq_query_qp_req_rocev2 req;
1813 struct erdma_dev *dev;
1814 struct erdma_qp *qp;
1815 u64 resp0, resp1;
1816 int ret;
1817
1818 if (ibqp && qp_attr && qp_init_attr) {
1819 qp = to_eqp(ibqp);
1820 dev = to_edev(ibqp->device);
1821 } else {
1822 return -EINVAL;
1823 }
1824
1825 qp_attr->cap.max_inline_data = ERDMA_MAX_INLINE;
1826 qp_init_attr->cap.max_inline_data = ERDMA_MAX_INLINE;
1827
1828 qp_attr->cap.max_send_wr = qp->attrs.sq_size;
1829 qp_attr->cap.max_recv_wr = qp->attrs.rq_size;
1830 qp_attr->cap.max_send_sge = qp->attrs.max_send_sge;
1831 qp_attr->cap.max_recv_sge = qp->attrs.max_recv_sge;
1832
1833 qp_attr->path_mtu = ib_mtu_int_to_enum(dev->netdev->mtu);
1834 qp_attr->max_rd_atomic = qp->attrs.irq_size;
1835 qp_attr->max_dest_rd_atomic = qp->attrs.orq_size;
1836
1837 qp_attr->qp_access_flags = IB_ACCESS_LOCAL_WRITE |
1838 IB_ACCESS_REMOTE_WRITE |
1839 IB_ACCESS_REMOTE_READ;
1840
1841 qp_init_attr->cap = qp_attr->cap;
1842
1843 if (erdma_device_rocev2(dev)) {
1844 /* Query hardware to get some attributes */
1845 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
1846 CMDQ_OPCODE_QUERY_QP);
1847 req.qpn = QP_ID(qp);
1848
1849 ret = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), &resp0,
1850 &resp1, true);
1851 if (ret)
1852 return ret;
1853
1854 qp_attr->sq_psn =
1855 FIELD_GET(ERDMA_CMD_QUERY_QP_RESP_SQ_PSN_MASK, resp0);
1856 qp_attr->rq_psn =
1857 FIELD_GET(ERDMA_CMD_QUERY_QP_RESP_RQ_PSN_MASK, resp0);
1858 qp_attr->qp_state = rocev2_to_ib_qps(FIELD_GET(
1859 ERDMA_CMD_QUERY_QP_RESP_QP_STATE_MASK, resp0));
1860 qp_attr->cur_qp_state = qp_attr->qp_state;
1861 qp_attr->sq_draining = FIELD_GET(
1862 ERDMA_CMD_QUERY_QP_RESP_SQ_DRAINING_MASK, resp0);
1863
1864 qp_attr->pkey_index = 0;
1865 qp_attr->dest_qp_num = qp->attrs.rocev2.dst_qpn;
1866
1867 if (qp->ibqp.qp_type == IB_QPT_RC)
1868 erdma_av_to_attr(&qp->attrs.rocev2.av,
1869 &qp_attr->ah_attr);
1870 } else {
1871 qp_attr->qp_state = query_qp_state(qp);
1872 qp_attr->cur_qp_state = qp_attr->qp_state;
1873 }
1874
1875 return 0;
1876 }
1877
erdma_init_user_cq(struct erdma_ucontext * ctx,struct erdma_cq * cq,struct erdma_ureq_create_cq * ureq)1878 static int erdma_init_user_cq(struct erdma_ucontext *ctx, struct erdma_cq *cq,
1879 struct erdma_ureq_create_cq *ureq)
1880 {
1881 int ret;
1882 struct erdma_dev *dev = to_edev(cq->ibcq.device);
1883
1884 ret = get_mtt_entries(dev, &cq->user_cq.qbuf_mem, ureq->qbuf_va,
1885 ureq->qbuf_len, 0, ureq->qbuf_va, SZ_64M - SZ_4K,
1886 true);
1887 if (ret)
1888 return ret;
1889
1890 ret = erdma_map_user_dbrecords(ctx, ureq->db_record_va,
1891 &cq->user_cq.user_dbr_page,
1892 &cq->user_cq.dbrec_dma);
1893 if (ret)
1894 put_mtt_entries(dev, &cq->user_cq.qbuf_mem);
1895
1896 return ret;
1897 }
1898
erdma_init_kernel_cq(struct erdma_cq * cq)1899 static int erdma_init_kernel_cq(struct erdma_cq *cq)
1900 {
1901 struct erdma_dev *dev = to_edev(cq->ibcq.device);
1902
1903 cq->kern_cq.qbuf =
1904 dma_alloc_coherent(&dev->pdev->dev, cq->depth << CQE_SHIFT,
1905 &cq->kern_cq.qbuf_dma_addr, GFP_KERNEL);
1906 if (!cq->kern_cq.qbuf)
1907 return -ENOMEM;
1908
1909 cq->kern_cq.dbrec = dma_pool_zalloc(dev->db_pool, GFP_KERNEL,
1910 &cq->kern_cq.dbrec_dma);
1911 if (!cq->kern_cq.dbrec)
1912 goto err_out;
1913
1914 spin_lock_init(&cq->kern_cq.lock);
1915 /* use default cqdb addr */
1916 cq->kern_cq.db = dev->func_bar + ERDMA_BAR_CQDB_SPACE_OFFSET;
1917
1918 return 0;
1919
1920 err_out:
1921 dma_free_coherent(&dev->pdev->dev, cq->depth << CQE_SHIFT,
1922 cq->kern_cq.qbuf, cq->kern_cq.qbuf_dma_addr);
1923
1924 return -ENOMEM;
1925 }
1926
erdma_create_cq(struct ib_cq * ibcq,const struct ib_cq_init_attr * attr,struct uverbs_attr_bundle * attrs)1927 int erdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
1928 struct uverbs_attr_bundle *attrs)
1929 {
1930 struct ib_udata *udata = &attrs->driver_udata;
1931 struct erdma_cq *cq = to_ecq(ibcq);
1932 struct erdma_dev *dev = to_edev(ibcq->device);
1933 unsigned int depth = attr->cqe;
1934 int ret;
1935 struct erdma_ucontext *ctx = rdma_udata_to_drv_context(
1936 udata, struct erdma_ucontext, ibucontext);
1937
1938 if (depth > dev->attrs.max_cqe)
1939 return -EINVAL;
1940
1941 depth = roundup_pow_of_two(depth);
1942 cq->ibcq.cqe = depth;
1943 cq->depth = depth;
1944 cq->assoc_eqn = attr->comp_vector + 1;
1945
1946 ret = xa_alloc_cyclic(&dev->cq_xa, &cq->cqn, cq,
1947 XA_LIMIT(1, dev->attrs.max_cq - 1),
1948 &dev->next_alloc_cqn, GFP_KERNEL);
1949 if (ret < 0)
1950 return ret;
1951
1952 if (!rdma_is_kernel_res(&ibcq->res)) {
1953 struct erdma_ureq_create_cq ureq;
1954 struct erdma_uresp_create_cq uresp;
1955
1956 ret = ib_copy_from_udata(&ureq, udata,
1957 min(udata->inlen, sizeof(ureq)));
1958 if (ret)
1959 goto err_out_xa;
1960
1961 ret = erdma_init_user_cq(ctx, cq, &ureq);
1962 if (ret)
1963 goto err_out_xa;
1964
1965 uresp.cq_id = cq->cqn;
1966 uresp.num_cqe = depth;
1967
1968 ret = ib_copy_to_udata(udata, &uresp,
1969 min(sizeof(uresp), udata->outlen));
1970 if (ret)
1971 goto err_free_res;
1972 } else {
1973 ret = erdma_init_kernel_cq(cq);
1974 if (ret)
1975 goto err_out_xa;
1976 }
1977
1978 ret = create_cq_cmd(ctx, cq);
1979 if (ret)
1980 goto err_free_res;
1981
1982 return 0;
1983
1984 err_free_res:
1985 if (!rdma_is_kernel_res(&ibcq->res)) {
1986 erdma_unmap_user_dbrecords(ctx, &cq->user_cq.user_dbr_page);
1987 put_mtt_entries(dev, &cq->user_cq.qbuf_mem);
1988 } else {
1989 dma_free_coherent(&dev->pdev->dev, depth << CQE_SHIFT,
1990 cq->kern_cq.qbuf, cq->kern_cq.qbuf_dma_addr);
1991 dma_pool_free(dev->db_pool, cq->kern_cq.dbrec,
1992 cq->kern_cq.dbrec_dma);
1993 }
1994
1995 err_out_xa:
1996 xa_erase(&dev->cq_xa, cq->cqn);
1997
1998 return ret;
1999 }
2000
erdma_disassociate_ucontext(struct ib_ucontext * ibcontext)2001 void erdma_disassociate_ucontext(struct ib_ucontext *ibcontext)
2002 {
2003 }
2004
erdma_set_mtu(struct erdma_dev * dev,u32 mtu)2005 void erdma_set_mtu(struct erdma_dev *dev, u32 mtu)
2006 {
2007 struct erdma_cmdq_config_mtu_req req;
2008
2009 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_COMMON,
2010 CMDQ_OPCODE_CONF_MTU);
2011 req.mtu = mtu;
2012
2013 erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL, true);
2014 }
2015
erdma_port_event(struct erdma_dev * dev,enum ib_event_type reason)2016 void erdma_port_event(struct erdma_dev *dev, enum ib_event_type reason)
2017 {
2018 struct ib_event event;
2019
2020 event.device = &dev->ibdev;
2021 event.element.port_num = 1;
2022 event.event = reason;
2023
2024 ib_dispatch_event(&event);
2025 }
2026
2027 enum counters {
2028 ERDMA_STATS_TX_REQS_CNT,
2029 ERDMA_STATS_TX_PACKETS_CNT,
2030 ERDMA_STATS_TX_BYTES_CNT,
2031 ERDMA_STATS_TX_DISABLE_DROP_CNT,
2032 ERDMA_STATS_TX_BPS_METER_DROP_CNT,
2033 ERDMA_STATS_TX_PPS_METER_DROP_CNT,
2034
2035 ERDMA_STATS_RX_PACKETS_CNT,
2036 ERDMA_STATS_RX_BYTES_CNT,
2037 ERDMA_STATS_RX_DISABLE_DROP_CNT,
2038 ERDMA_STATS_RX_BPS_METER_DROP_CNT,
2039 ERDMA_STATS_RX_PPS_METER_DROP_CNT,
2040
2041 ERDMA_STATS_MAX
2042 };
2043
2044 static const struct rdma_stat_desc erdma_descs[] = {
2045 [ERDMA_STATS_TX_REQS_CNT].name = "tx_reqs_cnt",
2046 [ERDMA_STATS_TX_PACKETS_CNT].name = "tx_packets_cnt",
2047 [ERDMA_STATS_TX_BYTES_CNT].name = "tx_bytes_cnt",
2048 [ERDMA_STATS_TX_DISABLE_DROP_CNT].name = "tx_disable_drop_cnt",
2049 [ERDMA_STATS_TX_BPS_METER_DROP_CNT].name = "tx_bps_limit_drop_cnt",
2050 [ERDMA_STATS_TX_PPS_METER_DROP_CNT].name = "tx_pps_limit_drop_cnt",
2051 [ERDMA_STATS_RX_PACKETS_CNT].name = "rx_packets_cnt",
2052 [ERDMA_STATS_RX_BYTES_CNT].name = "rx_bytes_cnt",
2053 [ERDMA_STATS_RX_DISABLE_DROP_CNT].name = "rx_disable_drop_cnt",
2054 [ERDMA_STATS_RX_BPS_METER_DROP_CNT].name = "rx_bps_limit_drop_cnt",
2055 [ERDMA_STATS_RX_PPS_METER_DROP_CNT].name = "rx_pps_limit_drop_cnt",
2056 };
2057
erdma_alloc_hw_port_stats(struct ib_device * device,u32 port_num)2058 struct rdma_hw_stats *erdma_alloc_hw_port_stats(struct ib_device *device,
2059 u32 port_num)
2060 {
2061 return rdma_alloc_hw_stats_struct(erdma_descs, ERDMA_STATS_MAX,
2062 RDMA_HW_STATS_DEFAULT_LIFESPAN);
2063 }
2064
erdma_query_hw_stats(struct erdma_dev * dev,struct rdma_hw_stats * stats)2065 static int erdma_query_hw_stats(struct erdma_dev *dev,
2066 struct rdma_hw_stats *stats)
2067 {
2068 struct erdma_cmdq_query_stats_resp *resp;
2069 struct erdma_cmdq_query_req req;
2070 dma_addr_t dma_addr;
2071 int err;
2072
2073 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_COMMON,
2074 CMDQ_OPCODE_GET_STATS);
2075
2076 resp = dma_pool_zalloc(dev->resp_pool, GFP_KERNEL, &dma_addr);
2077 if (!resp)
2078 return -ENOMEM;
2079
2080 req.target_addr = dma_addr;
2081 req.target_length = ERDMA_HW_RESP_SIZE;
2082
2083 err = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL,
2084 true);
2085 if (err)
2086 goto out;
2087
2088 if (resp->hdr.magic != ERDMA_HW_RESP_MAGIC) {
2089 err = -EINVAL;
2090 goto out;
2091 }
2092
2093 memcpy(&stats->value[0], &resp->tx_req_cnt,
2094 sizeof(u64) * stats->num_counters);
2095
2096 out:
2097 dma_pool_free(dev->resp_pool, resp, dma_addr);
2098
2099 return err;
2100 }
2101
erdma_get_hw_stats(struct ib_device * ibdev,struct rdma_hw_stats * stats,u32 port,int index)2102 int erdma_get_hw_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats,
2103 u32 port, int index)
2104 {
2105 struct erdma_dev *dev = to_edev(ibdev);
2106 int ret;
2107
2108 if (port == 0)
2109 return 0;
2110
2111 ret = erdma_query_hw_stats(dev, stats);
2112 if (ret)
2113 return ret;
2114
2115 return stats->num_counters;
2116 }
2117
erdma_get_link_layer(struct ib_device * ibdev,u32 port_num)2118 enum rdma_link_layer erdma_get_link_layer(struct ib_device *ibdev, u32 port_num)
2119 {
2120 return IB_LINK_LAYER_ETHERNET;
2121 }
2122
erdma_set_gid(struct erdma_dev * dev,u8 op,u32 idx,const union ib_gid * gid)2123 static int erdma_set_gid(struct erdma_dev *dev, u8 op, u32 idx,
2124 const union ib_gid *gid)
2125 {
2126 struct erdma_cmdq_set_gid_req req;
2127 u8 ntype;
2128
2129 req.cfg = FIELD_PREP(ERDMA_CMD_SET_GID_SGID_IDX_MASK, idx) |
2130 FIELD_PREP(ERDMA_CMD_SET_GID_OP_MASK, op);
2131
2132 if (op == ERDMA_SET_GID_OP_ADD) {
2133 if (ipv6_addr_v4mapped((struct in6_addr *)gid))
2134 ntype = ERDMA_NETWORK_TYPE_IPV4;
2135 else
2136 ntype = ERDMA_NETWORK_TYPE_IPV6;
2137
2138 req.cfg |= FIELD_PREP(ERDMA_CMD_SET_GID_NTYPE_MASK, ntype);
2139
2140 memcpy(&req.gid, gid, ERDMA_ROCEV2_GID_SIZE);
2141 }
2142
2143 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
2144 CMDQ_OPCODE_SET_GID);
2145 return erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL,
2146 true);
2147 }
2148
erdma_add_gid(const struct ib_gid_attr * attr,void ** context)2149 int erdma_add_gid(const struct ib_gid_attr *attr, void **context)
2150 {
2151 struct erdma_dev *dev = to_edev(attr->device);
2152 int ret;
2153
2154 ret = erdma_check_gid_attr(attr);
2155 if (ret)
2156 return ret;
2157
2158 return erdma_set_gid(dev, ERDMA_SET_GID_OP_ADD, attr->index,
2159 &attr->gid);
2160 }
2161
erdma_del_gid(const struct ib_gid_attr * attr,void ** context)2162 int erdma_del_gid(const struct ib_gid_attr *attr, void **context)
2163 {
2164 return erdma_set_gid(to_edev(attr->device), ERDMA_SET_GID_OP_DEL,
2165 attr->index, NULL);
2166 }
2167
erdma_query_pkey(struct ib_device * ibdev,u32 port,u16 index,u16 * pkey)2168 int erdma_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey)
2169 {
2170 if (index >= ERDMA_MAX_PKEYS)
2171 return -EINVAL;
2172
2173 *pkey = ERDMA_DEFAULT_PKEY;
2174 return 0;
2175 }
2176
erdma_set_av_cfg(struct erdma_av_cfg * av_cfg,struct erdma_av * av)2177 void erdma_set_av_cfg(struct erdma_av_cfg *av_cfg, struct erdma_av *av)
2178 {
2179 av_cfg->cfg0 = FIELD_PREP(ERDMA_CMD_CREATE_AV_FL_MASK, av->flow_label) |
2180 FIELD_PREP(ERDMA_CMD_CREATE_AV_NTYPE_MASK, av->ntype);
2181
2182 av_cfg->traffic_class = av->traffic_class;
2183 av_cfg->hop_limit = av->hop_limit;
2184 av_cfg->sl = av->sl;
2185
2186 av_cfg->udp_sport = av->udp_sport;
2187 av_cfg->sgid_index = av->sgid_index;
2188
2189 ether_addr_copy(av_cfg->dmac, av->dmac);
2190 memcpy(av_cfg->dgid, av->dgid, ERDMA_ROCEV2_GID_SIZE);
2191 }
2192
erdma_create_ah(struct ib_ah * ibah,struct rdma_ah_init_attr * init_attr,struct ib_udata * udata)2193 int erdma_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr,
2194 struct ib_udata *udata)
2195 {
2196 const struct ib_global_route *grh =
2197 rdma_ah_read_grh(init_attr->ah_attr);
2198 struct erdma_dev *dev = to_edev(ibah->device);
2199 struct erdma_pd *pd = to_epd(ibah->pd);
2200 struct erdma_ah *ah = to_eah(ibah);
2201 struct erdma_cmdq_create_ah_req req;
2202 u32 udp_sport;
2203 int ret;
2204
2205 ret = erdma_check_gid_attr(grh->sgid_attr);
2206 if (ret)
2207 return ret;
2208
2209 ret = erdma_alloc_idx(&dev->res_cb[ERDMA_RES_TYPE_AH]);
2210 if (ret < 0)
2211 return ret;
2212
2213 ah->ahn = ret;
2214
2215 if (grh->flow_label)
2216 udp_sport = rdma_flow_label_to_udp_sport(grh->flow_label);
2217 else
2218 udp_sport =
2219 IB_ROCE_UDP_ENCAP_VALID_PORT_MIN + (ah->ahn & 0x3FFF);
2220
2221 erdma_attr_to_av(init_attr->ah_attr, &ah->av, udp_sport);
2222
2223 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
2224 CMDQ_OPCODE_CREATE_AH);
2225
2226 req.pdn = pd->pdn;
2227 req.ahn = ah->ahn;
2228 erdma_set_av_cfg(&req.av_cfg, &ah->av);
2229
2230 ret = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL,
2231 init_attr->flags & RDMA_CREATE_AH_SLEEPABLE);
2232 if (ret) {
2233 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_AH], ah->ahn);
2234 return ret;
2235 }
2236
2237 return 0;
2238 }
2239
erdma_destroy_ah(struct ib_ah * ibah,u32 flags)2240 int erdma_destroy_ah(struct ib_ah *ibah, u32 flags)
2241 {
2242 struct erdma_dev *dev = to_edev(ibah->device);
2243 struct erdma_pd *pd = to_epd(ibah->pd);
2244 struct erdma_ah *ah = to_eah(ibah);
2245 struct erdma_cmdq_destroy_ah_req req;
2246 int ret;
2247
2248 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
2249 CMDQ_OPCODE_DESTROY_AH);
2250
2251 req.pdn = pd->pdn;
2252 req.ahn = ah->ahn;
2253
2254 ret = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL,
2255 flags & RDMA_DESTROY_AH_SLEEPABLE);
2256 if (ret)
2257 return ret;
2258
2259 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_AH], ah->ahn);
2260
2261 return 0;
2262 }
2263
erdma_query_ah(struct ib_ah * ibah,struct rdma_ah_attr * ah_attr)2264 int erdma_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr)
2265 {
2266 struct erdma_ah *ah = to_eah(ibah);
2267
2268 memset(ah_attr, 0, sizeof(*ah_attr));
2269 erdma_av_to_attr(&ah->av, ah_attr);
2270
2271 return 0;
2272 }
2273