1 /*
2 * Broadcom NetXtreme-E RoCE driver.
3 *
4 * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term
5 * Broadcom refers to Broadcom Limited and/or its subsidiaries.
6 *
7 * This software is available to you under a choice of one of two
8 * licenses. You may choose to be licensed under the terms of the GNU
9 * General Public License (GPL) Version 2, available from the file
10 * COPYING in the main directory of this source tree, or the
11 * BSD license below:
12 *
13 * Redistribution and use in source and binary forms, with or without
14 * modification, are permitted provided that the following conditions
15 * are met:
16 *
17 * 1. Redistributions of source code must retain the above copyright
18 * notice, this list of conditions and the following disclaimer.
19 * 2. Redistributions in binary form must reproduce the above copyright
20 * notice, this list of conditions and the following disclaimer in
21 * the documentation and/or other materials provided with the
22 * distribution.
23 *
24 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
25 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
26 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
27 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
28 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
29 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
30 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
31 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
32 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
33 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
34 * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35 *
36 * Description: IB Verbs interpreter
37 */
38
39 #include <linux/interrupt.h>
40 #include <linux/types.h>
41 #include <linux/pci.h>
42 #include <linux/netdevice.h>
43 #include <linux/if_ether.h>
44 #include <net/addrconf.h>
45
46 #include <rdma/ib_verbs.h>
47 #include <rdma/ib_user_verbs.h>
48 #include <rdma/ib_umem.h>
49 #include <rdma/ib_addr.h>
50 #include <rdma/ib_mad.h>
51 #include <rdma/ib_cache.h>
52 #include <rdma/ib_pma.h>
53 #include <rdma/uverbs_ioctl.h>
54 #include <linux/hashtable.h>
55
56 #include "roce_hsi.h"
57 #include "qplib_res.h"
58 #include "qplib_sp.h"
59 #include "qplib_fp.h"
60 #include "qplib_rcfw.h"
61
62 #include "bnxt_re.h"
63 #include "ib_verbs.h"
64 #include "debugfs.h"
65
66 #include <rdma/uverbs_types.h>
67 #include <rdma/uverbs_std_types.h>
68
69 #include <rdma/ib_user_ioctl_cmds.h>
70
71 #define UVERBS_MODULE_NAME bnxt_re
72 #include <rdma/uverbs_named_ioctl.h>
73
74 #include <rdma/bnxt_re-abi.h>
75
__from_ib_access_flags(int iflags)76 static int __from_ib_access_flags(int iflags)
77 {
78 int qflags = 0;
79
80 if (iflags & IB_ACCESS_LOCAL_WRITE)
81 qflags |= BNXT_QPLIB_ACCESS_LOCAL_WRITE;
82 if (iflags & IB_ACCESS_REMOTE_READ)
83 qflags |= BNXT_QPLIB_ACCESS_REMOTE_READ;
84 if (iflags & IB_ACCESS_REMOTE_WRITE)
85 qflags |= BNXT_QPLIB_ACCESS_REMOTE_WRITE;
86 if (iflags & IB_ACCESS_REMOTE_ATOMIC)
87 qflags |= BNXT_QPLIB_ACCESS_REMOTE_ATOMIC;
88 if (iflags & IB_ACCESS_MW_BIND)
89 qflags |= BNXT_QPLIB_ACCESS_MW_BIND;
90 if (iflags & IB_ZERO_BASED)
91 qflags |= BNXT_QPLIB_ACCESS_ZERO_BASED;
92 if (iflags & IB_ACCESS_ON_DEMAND)
93 qflags |= BNXT_QPLIB_ACCESS_ON_DEMAND;
94 return qflags;
95 };
96
__to_ib_access_flags(int qflags)97 static int __to_ib_access_flags(int qflags)
98 {
99 int iflags = 0;
100
101 if (qflags & BNXT_QPLIB_ACCESS_LOCAL_WRITE)
102 iflags |= IB_ACCESS_LOCAL_WRITE;
103 if (qflags & BNXT_QPLIB_ACCESS_REMOTE_WRITE)
104 iflags |= IB_ACCESS_REMOTE_WRITE;
105 if (qflags & BNXT_QPLIB_ACCESS_REMOTE_READ)
106 iflags |= IB_ACCESS_REMOTE_READ;
107 if (qflags & BNXT_QPLIB_ACCESS_REMOTE_ATOMIC)
108 iflags |= IB_ACCESS_REMOTE_ATOMIC;
109 if (qflags & BNXT_QPLIB_ACCESS_MW_BIND)
110 iflags |= IB_ACCESS_MW_BIND;
111 if (qflags & BNXT_QPLIB_ACCESS_ZERO_BASED)
112 iflags |= IB_ZERO_BASED;
113 if (qflags & BNXT_QPLIB_ACCESS_ON_DEMAND)
114 iflags |= IB_ACCESS_ON_DEMAND;
115 return iflags;
116 }
117
__qp_access_flags_from_ib(struct bnxt_qplib_chip_ctx * cctx,int iflags)118 static u8 __qp_access_flags_from_ib(struct bnxt_qplib_chip_ctx *cctx, int iflags)
119 {
120 u8 qflags = 0;
121
122 if (!bnxt_qplib_is_chip_gen_p5_p7(cctx))
123 /* For Wh+ */
124 return (u8)__from_ib_access_flags(iflags);
125
126 /* For P5, P7 and later chips */
127 if (iflags & IB_ACCESS_LOCAL_WRITE)
128 qflags |= CMDQ_MODIFY_QP_ACCESS_LOCAL_WRITE;
129 if (iflags & IB_ACCESS_REMOTE_WRITE)
130 qflags |= CMDQ_MODIFY_QP_ACCESS_REMOTE_WRITE;
131 if (iflags & IB_ACCESS_REMOTE_READ)
132 qflags |= CMDQ_MODIFY_QP_ACCESS_REMOTE_READ;
133 if (iflags & IB_ACCESS_REMOTE_ATOMIC)
134 qflags |= CMDQ_MODIFY_QP_ACCESS_REMOTE_ATOMIC;
135
136 return qflags;
137 }
138
__qp_access_flags_to_ib(struct bnxt_qplib_chip_ctx * cctx,u8 qflags)139 static int __qp_access_flags_to_ib(struct bnxt_qplib_chip_ctx *cctx, u8 qflags)
140 {
141 int iflags = 0;
142
143 if (!bnxt_qplib_is_chip_gen_p5_p7(cctx))
144 /* For Wh+ */
145 return __to_ib_access_flags(qflags);
146
147 /* For P5, P7 and later chips */
148 if (qflags & CMDQ_MODIFY_QP_ACCESS_LOCAL_WRITE)
149 iflags |= IB_ACCESS_LOCAL_WRITE;
150 if (qflags & CMDQ_MODIFY_QP_ACCESS_REMOTE_WRITE)
151 iflags |= IB_ACCESS_REMOTE_WRITE;
152 if (qflags & CMDQ_MODIFY_QP_ACCESS_REMOTE_READ)
153 iflags |= IB_ACCESS_REMOTE_READ;
154 if (qflags & CMDQ_MODIFY_QP_ACCESS_REMOTE_ATOMIC)
155 iflags |= IB_ACCESS_REMOTE_ATOMIC;
156
157 return iflags;
158 }
159
bnxt_re_check_and_set_relaxed_ordering(struct bnxt_re_dev * rdev,struct bnxt_qplib_mrw * qplib_mr)160 static void bnxt_re_check_and_set_relaxed_ordering(struct bnxt_re_dev *rdev,
161 struct bnxt_qplib_mrw *qplib_mr)
162 {
163 if (_is_relaxed_ordering_supported(rdev->dev_attr->dev_cap_flags2) &&
164 pcie_relaxed_ordering_enabled(rdev->en_dev->pdev))
165 qplib_mr->flags |= CMDQ_REGISTER_MR_FLAGS_ENABLE_RO;
166 }
167
bnxt_re_build_sgl(struct ib_sge * ib_sg_list,struct bnxt_qplib_sge * sg_list,int num)168 static int bnxt_re_build_sgl(struct ib_sge *ib_sg_list,
169 struct bnxt_qplib_sge *sg_list, int num)
170 {
171 int i, total = 0;
172
173 for (i = 0; i < num; i++) {
174 sg_list[i].addr = ib_sg_list[i].addr;
175 sg_list[i].lkey = ib_sg_list[i].lkey;
176 sg_list[i].size = ib_sg_list[i].length;
177 total += sg_list[i].size;
178 }
179 return total;
180 }
181
182 /* Device */
bnxt_re_query_device(struct ib_device * ibdev,struct ib_device_attr * ib_attr,struct ib_udata * udata)183 int bnxt_re_query_device(struct ib_device *ibdev,
184 struct ib_device_attr *ib_attr,
185 struct ib_udata *udata)
186 {
187 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
188 struct bnxt_qplib_dev_attr *dev_attr = rdev->dev_attr;
189 struct bnxt_re_query_device_ex_resp resp = {};
190 int rc = 0;
191
192 rc = ib_is_udata_in_empty(udata);
193 if (rc)
194 return rc;
195
196 memset(ib_attr, 0, sizeof(*ib_attr));
197 memcpy(&ib_attr->fw_ver, dev_attr->fw_ver,
198 min(sizeof(dev_attr->fw_ver),
199 sizeof(ib_attr->fw_ver)));
200 addrconf_addr_eui48((u8 *)&ib_attr->sys_image_guid,
201 rdev->netdev->dev_addr);
202 ib_attr->max_mr_size = BNXT_RE_MAX_MR_SIZE;
203 ib_attr->page_size_cap = BNXT_RE_PAGE_SIZE_SUPPORTED;
204
205 ib_attr->vendor_id = rdev->en_dev->pdev->vendor;
206 ib_attr->vendor_part_id = rdev->en_dev->pdev->device;
207 ib_attr->hw_ver = rdev->en_dev->pdev->revision;
208 ib_attr->max_qp = dev_attr->max_qp;
209 ib_attr->max_qp_wr = dev_attr->max_qp_wqes;
210 ib_attr->device_cap_flags =
211 IB_DEVICE_CURR_QP_STATE_MOD
212 | IB_DEVICE_RC_RNR_NAK_GEN
213 | IB_DEVICE_SHUTDOWN_PORT
214 | IB_DEVICE_SYS_IMAGE_GUID
215 | IB_DEVICE_RESIZE_MAX_WR
216 | IB_DEVICE_PORT_ACTIVE_EVENT
217 | IB_DEVICE_N_NOTIFY_CQ
218 | IB_DEVICE_MEM_WINDOW
219 | IB_DEVICE_MEM_WINDOW_TYPE_2B
220 | IB_DEVICE_MEM_MGT_EXTENSIONS;
221 ib_attr->kernel_cap_flags = IBK_LOCAL_DMA_LKEY;
222 ib_attr->max_send_sge = dev_attr->max_qp_sges;
223 ib_attr->max_recv_sge = dev_attr->max_qp_sges;
224 ib_attr->max_sge_rd = dev_attr->max_qp_sges;
225 ib_attr->max_cq = dev_attr->max_cq;
226 ib_attr->max_cqe = dev_attr->max_cq_wqes;
227 ib_attr->max_mr = dev_attr->max_mr;
228 ib_attr->max_pd = dev_attr->max_pd;
229 ib_attr->max_qp_rd_atom = dev_attr->max_qp_rd_atom;
230 ib_attr->max_qp_init_rd_atom = dev_attr->max_qp_init_rd_atom;
231 ib_attr->atomic_cap = IB_ATOMIC_NONE;
232 ib_attr->masked_atomic_cap = IB_ATOMIC_NONE;
233 if (dev_attr->is_atomic) {
234 ib_attr->atomic_cap = IB_ATOMIC_GLOB;
235 ib_attr->masked_atomic_cap = IB_ATOMIC_GLOB;
236 }
237
238 ib_attr->max_ee_rd_atom = 0;
239 ib_attr->max_res_rd_atom = 0;
240 ib_attr->max_ee_init_rd_atom = 0;
241 ib_attr->max_ee = 0;
242 ib_attr->max_rdd = 0;
243 ib_attr->max_mw = dev_attr->max_mw;
244 ib_attr->max_raw_ipv6_qp = 0;
245 ib_attr->max_raw_ethy_qp = dev_attr->max_raw_ethy_qp;
246 ib_attr->max_mcast_grp = 0;
247 ib_attr->max_mcast_qp_attach = 0;
248 ib_attr->max_total_mcast_qp_attach = 0;
249 ib_attr->max_ah = dev_attr->max_ah;
250
251 ib_attr->max_srq = dev_attr->max_srq;
252 ib_attr->max_srq_wr = dev_attr->max_srq_wqes;
253 ib_attr->max_srq_sge = dev_attr->max_srq_sges;
254
255 ib_attr->max_fast_reg_page_list_len = MAX_PBL_LVL_1_PGS;
256
257 ib_attr->max_pkeys = 1;
258 ib_attr->local_ca_ack_delay = BNXT_RE_DEFAULT_ACK_DELAY;
259
260 if (_is_modify_qp_rate_limit_supported(dev_attr->dev_cap_flags2)) {
261 resp.packet_pacing_caps.qp_rate_limit_min =
262 dev_attr->rate_limit_min;
263 resp.packet_pacing_caps.qp_rate_limit_max =
264 dev_attr->rate_limit_max;
265 resp.packet_pacing_caps.supported_qpts =
266 1 << IB_QPT_RC;
267 }
268 return ib_respond_udata(udata, resp);
269 }
270
bnxt_re_modify_device(struct ib_device * ibdev,int device_modify_mask,struct ib_device_modify * device_modify)271 int bnxt_re_modify_device(struct ib_device *ibdev,
272 int device_modify_mask,
273 struct ib_device_modify *device_modify)
274 {
275 ibdev_dbg(ibdev, "Modify device with mask 0x%x", device_modify_mask);
276
277 if (device_modify_mask & ~IB_DEVICE_MODIFY_NODE_DESC)
278 return -EOPNOTSUPP;
279
280 if (!(device_modify_mask & IB_DEVICE_MODIFY_NODE_DESC))
281 return 0;
282
283 memcpy(ibdev->node_desc, device_modify->node_desc, IB_DEVICE_NODE_DESC_MAX);
284 return 0;
285 }
286
287 /* Port */
bnxt_re_query_port(struct ib_device * ibdev,u32 port_num,struct ib_port_attr * port_attr)288 int bnxt_re_query_port(struct ib_device *ibdev, u32 port_num,
289 struct ib_port_attr *port_attr)
290 {
291 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
292 struct bnxt_qplib_dev_attr *dev_attr = rdev->dev_attr;
293 int rc;
294
295 memset(port_attr, 0, sizeof(*port_attr));
296
297 if (netif_running(rdev->netdev) && netif_carrier_ok(rdev->netdev)) {
298 port_attr->state = IB_PORT_ACTIVE;
299 port_attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP;
300 } else {
301 port_attr->state = IB_PORT_DOWN;
302 port_attr->phys_state = IB_PORT_PHYS_STATE_DISABLED;
303 }
304 port_attr->max_mtu = IB_MTU_4096;
305 port_attr->active_mtu = iboe_get_mtu(rdev->netdev->mtu);
306 /* One GID is reserved for RawEth QP. Report one less */
307 port_attr->gid_tbl_len = (rdev->rcfw.roce_mirror ? (dev_attr->max_sgid - 1) :
308 dev_attr->max_sgid);
309 port_attr->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_REINIT_SUP |
310 IB_PORT_DEVICE_MGMT_SUP |
311 IB_PORT_VENDOR_CLASS_SUP;
312 port_attr->ip_gids = true;
313
314 port_attr->max_msg_sz = (u32)BNXT_RE_MAX_MR_SIZE_LOW;
315 port_attr->bad_pkey_cntr = 0;
316 port_attr->qkey_viol_cntr = 0;
317 port_attr->pkey_tbl_len = dev_attr->max_pkey;
318 port_attr->lid = 0;
319 port_attr->sm_lid = 0;
320 port_attr->lmc = 0;
321 port_attr->max_vl_num = 4;
322 port_attr->sm_sl = 0;
323 port_attr->subnet_timeout = 0;
324 port_attr->init_type_reply = 0;
325 rc = ib_get_eth_speed(&rdev->ibdev, port_num, &port_attr->active_speed,
326 &port_attr->active_width);
327
328 return rc;
329 }
330
bnxt_re_get_port_immutable(struct ib_device * ibdev,u32 port_num,struct ib_port_immutable * immutable)331 int bnxt_re_get_port_immutable(struct ib_device *ibdev, u32 port_num,
332 struct ib_port_immutable *immutable)
333 {
334 struct ib_port_attr port_attr;
335
336 if (bnxt_re_query_port(ibdev, port_num, &port_attr))
337 return -EINVAL;
338
339 immutable->pkey_tbl_len = port_attr.pkey_tbl_len;
340 immutable->gid_tbl_len = port_attr.gid_tbl_len;
341 immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE;
342 immutable->core_cap_flags |= RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP;
343 immutable->max_mad_size = IB_MGMT_MAD_SIZE;
344 return 0;
345 }
346
bnxt_re_query_fw_str(struct ib_device * ibdev,char * str)347 void bnxt_re_query_fw_str(struct ib_device *ibdev, char *str)
348 {
349 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
350
351 snprintf(str, IB_FW_VERSION_NAME_MAX, "%d.%d.%d.%d",
352 rdev->dev_attr->fw_ver[0], rdev->dev_attr->fw_ver[1],
353 rdev->dev_attr->fw_ver[2], rdev->dev_attr->fw_ver[3]);
354 }
355
bnxt_re_query_pkey(struct ib_device * ibdev,u32 port_num,u16 index,u16 * pkey)356 int bnxt_re_query_pkey(struct ib_device *ibdev, u32 port_num,
357 u16 index, u16 *pkey)
358 {
359 if (index > 0)
360 return -EINVAL;
361
362 *pkey = IB_DEFAULT_PKEY_FULL;
363
364 return 0;
365 }
366
bnxt_re_query_gid(struct ib_device * ibdev,u32 port_num,int index,union ib_gid * gid)367 int bnxt_re_query_gid(struct ib_device *ibdev, u32 port_num,
368 int index, union ib_gid *gid)
369 {
370 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
371 int rc;
372
373 /* Ignore port_num */
374 memset(gid, 0, sizeof(*gid));
375 rc = bnxt_qplib_get_sgid(&rdev->qplib_res,
376 &rdev->qplib_res.sgid_tbl, index,
377 (struct bnxt_qplib_gid *)gid);
378 return rc;
379 }
380
bnxt_re_del_gid(const struct ib_gid_attr * attr,void ** context)381 int bnxt_re_del_gid(const struct ib_gid_attr *attr, void **context)
382 {
383 int rc = 0;
384 struct bnxt_re_gid_ctx *ctx, **ctx_tbl;
385 struct bnxt_re_dev *rdev = to_bnxt_re_dev(attr->device, ibdev);
386 struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl;
387 struct bnxt_qplib_gid *gid_to_del;
388 u16 vlan_id = 0xFFFF;
389
390 /* Delete the entry from the hardware */
391 ctx = *context;
392 if (!ctx)
393 return -EINVAL;
394
395 if (sgid_tbl->active) {
396 if (ctx->idx >= sgid_tbl->max)
397 return -EINVAL;
398 gid_to_del = &sgid_tbl->tbl[ctx->idx].gid;
399 vlan_id = sgid_tbl->tbl[ctx->idx].vlan_id;
400 /* DEL_GID is called in WQ context(netdevice_event_work_handler)
401 * or via the ib_unregister_device path. In the former case QP1
402 * may not be destroyed yet, in which case just return as FW
403 * needs that entry to be present and will fail it's deletion.
404 * We could get invoked again after QP1 is destroyed OR get an
405 * ADD_GID call with a different GID value for the same index
406 * where we issue MODIFY_GID cmd to update the GID entry -- TBD
407 */
408 if (ctx->idx == 0 &&
409 rdma_link_local_addr((struct in6_addr *)gid_to_del) &&
410 ctx->refcnt == 1 && rdev->gsi_ctx.gsi_sqp) {
411 ibdev_dbg(&rdev->ibdev,
412 "Trying to delete GID0 while QP1 is alive\n");
413 return -EFAULT;
414 }
415 ctx->refcnt--;
416 if (!ctx->refcnt) {
417 rc = bnxt_qplib_del_sgid(sgid_tbl, gid_to_del,
418 vlan_id, true);
419 if (rc) {
420 ibdev_err(&rdev->ibdev,
421 "Failed to remove GID: %#x", rc);
422 } else {
423 ctx_tbl = sgid_tbl->ctx;
424 ctx_tbl[ctx->idx] = NULL;
425 kfree(ctx);
426 }
427 }
428 } else {
429 return -EINVAL;
430 }
431 return rc;
432 }
433
bnxt_re_add_gid(const struct ib_gid_attr * attr,void ** context)434 int bnxt_re_add_gid(const struct ib_gid_attr *attr, void **context)
435 {
436 int rc;
437 u32 tbl_idx = 0;
438 u16 vlan_id = 0xFFFF;
439 struct bnxt_re_gid_ctx *ctx, **ctx_tbl;
440 struct bnxt_re_dev *rdev = to_bnxt_re_dev(attr->device, ibdev);
441 struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl;
442
443 rc = rdma_read_gid_l2_fields(attr, &vlan_id, NULL);
444 if (rc)
445 return rc;
446
447 rc = bnxt_qplib_add_sgid(sgid_tbl, (struct bnxt_qplib_gid *)&attr->gid,
448 rdev->qplib_res.netdev->dev_addr,
449 vlan_id, true, &tbl_idx, false, 0);
450 if (rc == -EALREADY) {
451 ctx_tbl = sgid_tbl->ctx;
452 ctx_tbl[tbl_idx]->refcnt++;
453 *context = ctx_tbl[tbl_idx];
454 return 0;
455 }
456
457 if (rc < 0) {
458 ibdev_err(&rdev->ibdev, "Failed to add GID: %#x", rc);
459 return rc;
460 }
461
462 ctx = kmalloc_obj(*ctx);
463 if (!ctx)
464 return -ENOMEM;
465 ctx_tbl = sgid_tbl->ctx;
466 ctx->idx = tbl_idx;
467 ctx->refcnt = 1;
468 ctx_tbl[tbl_idx] = ctx;
469 *context = ctx;
470
471 return rc;
472 }
473
bnxt_re_get_link_layer(struct ib_device * ibdev,u32 port_num)474 enum rdma_link_layer bnxt_re_get_link_layer(struct ib_device *ibdev,
475 u32 port_num)
476 {
477 return IB_LINK_LAYER_ETHERNET;
478 }
479
480 #define BNXT_RE_FENCE_PBL_SIZE DIV_ROUND_UP(BNXT_RE_FENCE_BYTES, PAGE_SIZE)
481
bnxt_re_create_fence_wqe(struct bnxt_re_pd * pd)482 static void bnxt_re_create_fence_wqe(struct bnxt_re_pd *pd)
483 {
484 struct bnxt_re_fence_data *fence = &pd->fence;
485 struct ib_mr *ib_mr = &fence->mr->ib_mr;
486 struct bnxt_qplib_swqe *wqe = &fence->bind_wqe;
487 struct bnxt_re_dev *rdev = pd->rdev;
488
489 if (bnxt_qplib_is_chip_gen_p5_p7(rdev->chip_ctx))
490 return;
491
492 memset(wqe, 0, sizeof(*wqe));
493 wqe->type = BNXT_QPLIB_SWQE_TYPE_BIND_MW;
494 wqe->wr_id = BNXT_QPLIB_FENCE_WRID;
495 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
496 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
497 wqe->bind.zero_based = false;
498 wqe->bind.parent_l_key = ib_mr->lkey;
499 wqe->bind.va = (u64)(unsigned long)fence->va;
500 wqe->bind.length = fence->size;
501 wqe->bind.access_cntl = __from_ib_access_flags(IB_ACCESS_REMOTE_READ);
502 wqe->bind.mw_type = SQ_BIND_MW_TYPE_TYPE1;
503
504 /* Save the initial rkey in fence structure for now;
505 * wqe->bind.r_key will be set at (re)bind time.
506 */
507 fence->bind_rkey = ib_inc_rkey(fence->mw->rkey);
508 }
509
bnxt_re_bind_fence_mw(struct bnxt_qplib_qp * qplib_qp)510 static int bnxt_re_bind_fence_mw(struct bnxt_qplib_qp *qplib_qp)
511 {
512 struct bnxt_re_qp *qp = container_of(qplib_qp, struct bnxt_re_qp,
513 qplib_qp);
514 struct ib_pd *ib_pd = qp->ib_qp.pd;
515 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
516 struct bnxt_re_fence_data *fence = &pd->fence;
517 struct bnxt_qplib_swqe *fence_wqe = &fence->bind_wqe;
518 struct bnxt_qplib_swqe wqe;
519 int rc;
520
521 memcpy(&wqe, fence_wqe, sizeof(wqe));
522 wqe.bind.r_key = fence->bind_rkey;
523 fence->bind_rkey = ib_inc_rkey(fence->bind_rkey);
524
525 ibdev_dbg(&qp->rdev->ibdev,
526 "Posting bind fence-WQE: rkey: %#x QP: %d PD: %p\n",
527 wqe.bind.r_key, qp->qplib_qp.id, pd);
528 rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe);
529 if (rc) {
530 ibdev_err(&qp->rdev->ibdev, "Failed to bind fence-WQE\n");
531 return rc;
532 }
533 bnxt_qplib_post_send_db(&qp->qplib_qp);
534
535 return rc;
536 }
537
bnxt_re_destroy_fence_mr(struct bnxt_re_pd * pd)538 static void bnxt_re_destroy_fence_mr(struct bnxt_re_pd *pd)
539 {
540 struct bnxt_re_fence_data *fence = &pd->fence;
541 struct bnxt_re_dev *rdev = pd->rdev;
542 struct device *dev = &rdev->en_dev->pdev->dev;
543 struct bnxt_re_mr *mr = fence->mr;
544
545 if (bnxt_qplib_is_chip_gen_p5_p7(rdev->chip_ctx))
546 return;
547
548 if (fence->mw) {
549 bnxt_re_dealloc_mw(fence->mw);
550 fence->mw = NULL;
551 }
552 if (mr) {
553 if (mr->ib_mr.rkey)
554 bnxt_qplib_dereg_mrw(&rdev->qplib_res, &mr->qplib_mr,
555 true);
556 if (mr->ib_mr.lkey)
557 bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr);
558 kfree(mr);
559 fence->mr = NULL;
560 }
561 if (fence->dma_addr) {
562 dma_unmap_single(dev, fence->dma_addr, BNXT_RE_FENCE_BYTES,
563 DMA_BIDIRECTIONAL);
564 fence->dma_addr = 0;
565 }
566 }
567
bnxt_re_create_fence_mr(struct bnxt_re_pd * pd)568 static int bnxt_re_create_fence_mr(struct bnxt_re_pd *pd)
569 {
570 int mr_access_flags = IB_ACCESS_LOCAL_WRITE | IB_ACCESS_MW_BIND;
571 struct bnxt_re_fence_data *fence = &pd->fence;
572 struct bnxt_re_dev *rdev = pd->rdev;
573 struct device *dev = &rdev->en_dev->pdev->dev;
574 struct bnxt_re_mr *mr = NULL;
575 dma_addr_t dma_addr = 0;
576 struct ib_mw *mw;
577 int rc;
578
579 if (bnxt_qplib_is_chip_gen_p5_p7(rdev->chip_ctx))
580 return 0;
581
582 dma_addr = dma_map_single(dev, fence->va, BNXT_RE_FENCE_BYTES,
583 DMA_BIDIRECTIONAL);
584 rc = dma_mapping_error(dev, dma_addr);
585 if (rc) {
586 ibdev_err(&rdev->ibdev, "Failed to dma-map fence-MR-mem\n");
587 rc = -EIO;
588 fence->dma_addr = 0;
589 goto fail;
590 }
591 fence->dma_addr = dma_addr;
592
593 /* Allocate a MR */
594 mr = kzalloc_obj(*mr);
595 if (!mr) {
596 rc = -ENOMEM;
597 goto fail;
598 }
599 fence->mr = mr;
600 mr->rdev = rdev;
601 mr->qplib_mr.pd = &pd->qplib_pd;
602 mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR;
603 mr->qplib_mr.access_flags = __from_ib_access_flags(mr_access_flags);
604 if (!_is_alloc_mr_unified(rdev->dev_attr->dev_cap_flags)) {
605 rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr);
606 if (rc) {
607 ibdev_err(&rdev->ibdev, "Failed to alloc fence-HW-MR\n");
608 goto fail;
609 }
610
611 /* Register MR */
612 mr->ib_mr.lkey = mr->qplib_mr.lkey;
613 } else {
614 mr->qplib_mr.flags = CMDQ_REGISTER_MR_FLAGS_ALLOC_MR;
615 }
616 mr->qplib_mr.va = (u64)(unsigned long)fence->va;
617 mr->qplib_mr.total_size = BNXT_RE_FENCE_BYTES;
618 rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, NULL,
619 BNXT_RE_FENCE_PBL_SIZE, PAGE_SIZE,
620 _is_alloc_mr_unified(rdev->dev_attr->dev_cap_flags));
621 if (rc) {
622 ibdev_err(&rdev->ibdev, "Failed to register fence-MR\n");
623 goto fail;
624 }
625 mr->ib_mr.rkey = mr->qplib_mr.rkey;
626
627 /* Create a fence MW only for kernel consumers */
628 mw = bnxt_re_alloc_mw(&pd->ib_pd, IB_MW_TYPE_1, NULL);
629 if (IS_ERR(mw)) {
630 ibdev_err(&rdev->ibdev,
631 "Failed to create fence-MW for PD: %p\n", pd);
632 rc = PTR_ERR(mw);
633 goto fail;
634 }
635 fence->mw = mw;
636
637 bnxt_re_create_fence_wqe(pd);
638 return 0;
639
640 fail:
641 bnxt_re_destroy_fence_mr(pd);
642 return rc;
643 }
644
645 struct bnxt_re_user_mmap_entry*
bnxt_re_mmap_entry_insert(struct bnxt_re_ucontext * uctx,u64 mem_offset,enum bnxt_re_mmap_flag mmap_flag,u64 * offset)646 bnxt_re_mmap_entry_insert(struct bnxt_re_ucontext *uctx, u64 mem_offset,
647 enum bnxt_re_mmap_flag mmap_flag, u64 *offset)
648 {
649 struct bnxt_re_user_mmap_entry *entry;
650 int ret;
651
652 entry = kzalloc_obj(*entry);
653 if (!entry)
654 return NULL;
655
656 entry->mem_offset = mem_offset;
657 entry->mmap_flag = mmap_flag;
658 entry->uctx = uctx;
659
660 switch (mmap_flag) {
661 case BNXT_RE_MMAP_SH_PAGE:
662 ret = rdma_user_mmap_entry_insert_exact(&uctx->ib_uctx,
663 &entry->rdma_entry, PAGE_SIZE, 0);
664 break;
665 case BNXT_RE_MMAP_UC_DB:
666 case BNXT_RE_MMAP_WC_DB:
667 case BNXT_RE_MMAP_DBR_BAR:
668 case BNXT_RE_MMAP_DBR_PAGE:
669 case BNXT_RE_MMAP_TOGGLE_PAGE:
670 ret = rdma_user_mmap_entry_insert(&uctx->ib_uctx,
671 &entry->rdma_entry, PAGE_SIZE);
672 break;
673 default:
674 ret = -EINVAL;
675 break;
676 }
677
678 if (ret) {
679 kfree(entry);
680 return NULL;
681 }
682 if (offset)
683 *offset = rdma_user_mmap_get_offset(&entry->rdma_entry);
684
685 return entry;
686 }
687
688 /* Protection Domains */
bnxt_re_dealloc_pd(struct ib_pd * ib_pd,struct ib_udata * udata)689 int bnxt_re_dealloc_pd(struct ib_pd *ib_pd, struct ib_udata *udata)
690 {
691 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
692 struct bnxt_re_dev *rdev = pd->rdev;
693 int ret;
694
695 ret = ib_is_udata_in_empty(udata);
696 if (ret)
697 return ret;
698
699 if (udata) {
700 rdma_user_mmap_entry_remove(pd->pd_db_mmap);
701 pd->pd_db_mmap = NULL;
702 }
703
704 bnxt_re_destroy_fence_mr(pd);
705
706 if (pd->qplib_pd.id) {
707 if (!bnxt_qplib_dealloc_pd(&rdev->qplib_res,
708 &rdev->qplib_res.pd_tbl,
709 &pd->qplib_pd))
710 atomic_dec(&rdev->stats.res.pd_count);
711 }
712 return ib_respond_empty_udata(udata);
713 }
714
bnxt_re_alloc_pd(struct ib_pd * ibpd,struct ib_udata * udata)715 int bnxt_re_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
716 {
717 struct ib_device *ibdev = ibpd->device;
718 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
719 struct bnxt_re_ucontext *ucntx = rdma_udata_to_drv_context(
720 udata, struct bnxt_re_ucontext, ib_uctx);
721 struct bnxt_re_pd *pd = container_of(ibpd, struct bnxt_re_pd, ib_pd);
722 struct bnxt_re_user_mmap_entry *entry = NULL;
723 u32 active_pds;
724 int rc = 0;
725
726 rc = ib_is_udata_in_empty(udata);
727 if (rc)
728 return rc;
729
730 pd->rdev = rdev;
731 if (bnxt_qplib_alloc_pd(&rdev->qplib_res, &pd->qplib_pd)) {
732 ibdev_err(&rdev->ibdev, "Failed to allocate HW PD");
733 rc = -ENOMEM;
734 goto fail;
735 }
736
737 if (udata) {
738 struct bnxt_re_pd_resp resp = {};
739
740 if (!ucntx->dpi.dbr) {
741 /* Allocate DPI in alloc_pd to avoid failing of
742 * ibv_devinfo and family of application when DPIs
743 * are depleted.
744 */
745 if (bnxt_qplib_alloc_dpi(&rdev->qplib_res,
746 &ucntx->dpi, ucntx, BNXT_QPLIB_DPI_TYPE_UC)) {
747 rc = -ENOMEM;
748 goto dbfail;
749 }
750 }
751
752 resp.pdid = pd->qplib_pd.id;
753 /* Still allow mapping this DBR to the new user PD. */
754 resp.dpi = ucntx->dpi.dpi;
755
756 entry = bnxt_re_mmap_entry_insert(ucntx, (u64)ucntx->dpi.umdbr,
757 BNXT_RE_MMAP_UC_DB, &resp.dbr);
758
759 if (!entry) {
760 rc = -ENOMEM;
761 goto dbfail;
762 }
763
764 pd->pd_db_mmap = &entry->rdma_entry;
765
766 rc = ib_respond_udata(udata, resp);
767 if (rc) {
768 rdma_user_mmap_entry_remove(pd->pd_db_mmap);
769 rc = -EFAULT;
770 goto dbfail;
771 }
772 }
773
774 if (!udata)
775 if (bnxt_re_create_fence_mr(pd))
776 ibdev_warn(&rdev->ibdev,
777 "Failed to create Fence-MR\n");
778 active_pds = atomic_inc_return(&rdev->stats.res.pd_count);
779 if (active_pds > rdev->stats.res.pd_watermark)
780 rdev->stats.res.pd_watermark = active_pds;
781
782 return 0;
783 dbfail:
784 bnxt_qplib_dealloc_pd(&rdev->qplib_res, &rdev->qplib_res.pd_tbl,
785 &pd->qplib_pd);
786 fail:
787 return rc;
788 }
789
790 /* Address Handles */
bnxt_re_destroy_ah(struct ib_ah * ib_ah,u32 flags)791 int bnxt_re_destroy_ah(struct ib_ah *ib_ah, u32 flags)
792 {
793 struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ib_ah);
794 struct bnxt_re_dev *rdev = ah->rdev;
795 bool block = true;
796 int rc;
797
798 block = !(flags & RDMA_DESTROY_AH_SLEEPABLE);
799 rc = bnxt_qplib_destroy_ah(&rdev->qplib_res, &ah->qplib_ah, block);
800 if (BNXT_RE_CHECK_RC(rc)) {
801 if (rc == -ETIMEDOUT)
802 rc = 0;
803 else
804 goto fail;
805 }
806 atomic_dec(&rdev->stats.res.ah_count);
807 fail:
808 return rc;
809 }
810
bnxt_re_stack_to_dev_nw_type(enum rdma_network_type ntype)811 static u8 bnxt_re_stack_to_dev_nw_type(enum rdma_network_type ntype)
812 {
813 u8 nw_type;
814
815 switch (ntype) {
816 case RDMA_NETWORK_IPV4:
817 nw_type = CMDQ_CREATE_AH_TYPE_V2IPV4;
818 break;
819 case RDMA_NETWORK_IPV6:
820 nw_type = CMDQ_CREATE_AH_TYPE_V2IPV6;
821 break;
822 default:
823 nw_type = CMDQ_CREATE_AH_TYPE_V1;
824 break;
825 }
826 return nw_type;
827 }
828
bnxt_re_create_ah(struct ib_ah * ib_ah,struct rdma_ah_init_attr * init_attr,struct ib_udata * udata)829 int bnxt_re_create_ah(struct ib_ah *ib_ah, struct rdma_ah_init_attr *init_attr,
830 struct ib_udata *udata)
831 {
832 struct ib_pd *ib_pd = ib_ah->pd;
833 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
834 struct rdma_ah_attr *ah_attr = init_attr->ah_attr;
835 const struct ib_global_route *grh = rdma_ah_read_grh(ah_attr);
836 struct bnxt_re_dev *rdev = pd->rdev;
837 const struct ib_gid_attr *sgid_attr;
838 struct bnxt_re_gid_ctx *ctx;
839 struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ib_ah);
840 u32 active_ahs;
841 u8 nw_type;
842 int rc;
843
844 rc = ib_is_udata_in_empty(udata);
845 if (rc)
846 return rc;
847
848 if (!(rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH)) {
849 ibdev_err(&rdev->ibdev, "Failed to alloc AH: GRH not set");
850 return -EINVAL;
851 }
852
853 ah->rdev = rdev;
854 ah->qplib_ah.pd = &pd->qplib_pd;
855
856 /* Supply the configuration for the HW */
857 memcpy(ah->qplib_ah.dgid.data, grh->dgid.raw,
858 sizeof(union ib_gid));
859 sgid_attr = grh->sgid_attr;
860 /* Get the HW context of the GID. The reference
861 * of GID table entry is already taken by the caller.
862 */
863 ctx = rdma_read_gid_hw_context(sgid_attr);
864 ah->qplib_ah.sgid_index = ctx->idx;
865 ah->qplib_ah.host_sgid_index = grh->sgid_index;
866 ah->qplib_ah.traffic_class = grh->traffic_class;
867 ah->qplib_ah.flow_label = grh->flow_label;
868 ah->qplib_ah.hop_limit = grh->hop_limit;
869 ah->qplib_ah.sl = rdma_ah_get_sl(ah_attr);
870
871 /* Get network header type for this GID */
872 nw_type = rdma_gid_attr_network_type(sgid_attr);
873 ah->qplib_ah.nw_type = bnxt_re_stack_to_dev_nw_type(nw_type);
874
875 memcpy(ah->qplib_ah.dmac, ah_attr->roce.dmac, ETH_ALEN);
876 rc = bnxt_qplib_create_ah(&rdev->qplib_res, &ah->qplib_ah,
877 !(init_attr->flags &
878 RDMA_CREATE_AH_SLEEPABLE));
879 if (rc) {
880 ibdev_err(&rdev->ibdev, "Failed to allocate HW AH");
881 return rc;
882 }
883
884 /* Write AVID to shared page. */
885 if (udata) {
886 struct bnxt_re_ucontext *uctx = rdma_udata_to_drv_context(
887 udata, struct bnxt_re_ucontext, ib_uctx);
888 unsigned long flag;
889 u32 *wrptr;
890
891 spin_lock_irqsave(&uctx->sh_lock, flag);
892 wrptr = (u32 *)(uctx->shpg + BNXT_RE_AVID_OFFT);
893 *wrptr = ah->qplib_ah.id;
894 wmb(); /* make sure cache is updated. */
895 spin_unlock_irqrestore(&uctx->sh_lock, flag);
896 }
897 active_ahs = atomic_inc_return(&rdev->stats.res.ah_count);
898 if (active_ahs > rdev->stats.res.ah_watermark)
899 rdev->stats.res.ah_watermark = active_ahs;
900
901 return ib_respond_empty_udata(udata);
902 }
903
bnxt_re_query_ah(struct ib_ah * ib_ah,struct rdma_ah_attr * ah_attr)904 int bnxt_re_query_ah(struct ib_ah *ib_ah, struct rdma_ah_attr *ah_attr)
905 {
906 struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ib_ah);
907
908 ah_attr->type = ib_ah->type;
909 rdma_ah_set_sl(ah_attr, ah->qplib_ah.sl);
910 memcpy(ah_attr->roce.dmac, ah->qplib_ah.dmac, ETH_ALEN);
911 rdma_ah_set_grh(ah_attr, NULL, 0,
912 ah->qplib_ah.host_sgid_index,
913 0, ah->qplib_ah.traffic_class);
914 rdma_ah_set_dgid_raw(ah_attr, ah->qplib_ah.dgid.data);
915 rdma_ah_set_port_num(ah_attr, 1);
916 rdma_ah_set_static_rate(ah_attr, 0);
917 return 0;
918 }
919
bnxt_re_lock_cqs(struct bnxt_re_qp * qp)920 unsigned long bnxt_re_lock_cqs(struct bnxt_re_qp *qp)
921 __acquires(&qp->scq->cq_lock) __acquires(&qp->rcq->cq_lock)
922 {
923 unsigned long flags;
924
925 spin_lock_irqsave(&qp->scq->cq_lock, flags);
926 if (qp->rcq != qp->scq)
927 spin_lock(&qp->rcq->cq_lock);
928 else
929 __acquire(&qp->rcq->cq_lock);
930
931 return flags;
932 }
933
bnxt_re_unlock_cqs(struct bnxt_re_qp * qp,unsigned long flags)934 void bnxt_re_unlock_cqs(struct bnxt_re_qp *qp,
935 unsigned long flags)
936 __releases(&qp->scq->cq_lock) __releases(&qp->rcq->cq_lock)
937 {
938 if (qp->rcq != qp->scq)
939 spin_unlock(&qp->rcq->cq_lock);
940 else
941 __release(&qp->rcq->cq_lock);
942 spin_unlock_irqrestore(&qp->scq->cq_lock, flags);
943 }
944
bnxt_re_destroy_gsi_sqp(struct bnxt_re_qp * qp)945 static void bnxt_re_destroy_gsi_sqp(struct bnxt_re_qp *qp)
946 {
947 struct bnxt_re_qp *gsi_sqp;
948 struct bnxt_re_ah *gsi_sah;
949 struct bnxt_re_dev *rdev;
950 int rc;
951
952 rdev = qp->rdev;
953 gsi_sqp = rdev->gsi_ctx.gsi_sqp;
954 gsi_sah = rdev->gsi_ctx.gsi_sah;
955
956 ibdev_dbg(&rdev->ibdev, "Destroy the shadow AH\n");
957 bnxt_qplib_destroy_ah(&rdev->qplib_res,
958 &gsi_sah->qplib_ah,
959 true);
960 atomic_dec(&rdev->stats.res.ah_count);
961 bnxt_qplib_clean_qp(&qp->qplib_qp);
962
963 ibdev_dbg(&rdev->ibdev, "Destroy the shadow QP\n");
964 rc = bnxt_qplib_destroy_qp(&rdev->qplib_res, &gsi_sqp->qplib_qp);
965 if (rc)
966 ibdev_err(&rdev->ibdev, "Destroy Shadow QP failed");
967
968 bnxt_qplib_free_qp_res(&rdev->qplib_res, &gsi_sqp->qplib_qp);
969
970 /* remove from active qp list */
971 mutex_lock(&rdev->qp_lock);
972 list_del(&gsi_sqp->list);
973 mutex_unlock(&rdev->qp_lock);
974 atomic_dec(&rdev->stats.res.qp_count);
975
976 kfree(rdev->gsi_ctx.sqp_tbl);
977 kfree(gsi_sah);
978 kfree(gsi_sqp);
979 rdev->gsi_ctx.gsi_sqp = NULL;
980 rdev->gsi_ctx.gsi_sah = NULL;
981 rdev->gsi_ctx.sqp_tbl = NULL;
982 }
983
bnxt_re_del_unique_gid(struct bnxt_re_dev * rdev)984 static void bnxt_re_del_unique_gid(struct bnxt_re_dev *rdev)
985 {
986 int rc;
987
988 if (!rdev->rcfw.roce_mirror)
989 return;
990
991 rc = bnxt_qplib_del_sgid(&rdev->qplib_res.sgid_tbl,
992 (struct bnxt_qplib_gid *)&rdev->ugid,
993 0xFFFF, true);
994 if (rc)
995 dev_err(rdev_to_dev(rdev), "Failed to delete unique GID, rc: %d\n", rc);
996 }
997
bnxt_re_qp_free_umem(struct bnxt_re_qp * qp)998 static void bnxt_re_qp_free_umem(struct bnxt_re_qp *qp)
999 {
1000 ib_umem_release(qp->rumem);
1001 ib_umem_release(qp->sumem);
1002 }
1003
1004 /* Queue Pairs */
bnxt_re_destroy_qp(struct ib_qp * ib_qp,struct ib_udata * udata)1005 int bnxt_re_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata)
1006 {
1007 struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
1008 struct bnxt_qplib_qp *qplib_qp = &qp->qplib_qp;
1009 struct bnxt_re_dev *rdev = qp->rdev;
1010 struct bnxt_qplib_nq *scq_nq = NULL;
1011 struct bnxt_qplib_nq *rcq_nq = NULL;
1012 unsigned int flags;
1013 int rc;
1014
1015 rc = ib_is_udata_in_empty(udata);
1016 if (rc)
1017 return rc;
1018
1019 bnxt_re_debug_rem_qpinfo(rdev, qp);
1020
1021 bnxt_qplib_flush_cqn_wq(&qp->qplib_qp);
1022
1023 rc = bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp);
1024 if (rc)
1025 ibdev_err(&rdev->ibdev, "Failed to destroy HW QP");
1026
1027 if (rdma_is_kernel_res(&qp->ib_qp.res)) {
1028 flags = bnxt_re_lock_cqs(qp);
1029 bnxt_qplib_clean_qp(&qp->qplib_qp);
1030 bnxt_re_unlock_cqs(qp, flags);
1031 }
1032
1033 bnxt_qplib_free_qp_res(&rdev->qplib_res, &qp->qplib_qp);
1034
1035 if (ib_qp->qp_type == IB_QPT_GSI && rdev->gsi_ctx.gsi_sqp)
1036 bnxt_re_destroy_gsi_sqp(qp);
1037
1038 mutex_lock(&rdev->qp_lock);
1039 list_del(&qp->list);
1040 mutex_unlock(&rdev->qp_lock);
1041 atomic_dec(&rdev->stats.res.qp_count);
1042 if (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_RC)
1043 atomic_dec(&rdev->stats.res.rc_qp_count);
1044 else if (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_UD)
1045 atomic_dec(&rdev->stats.res.ud_qp_count);
1046
1047 if (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_RAW_ETHERTYPE)
1048 bnxt_re_del_unique_gid(rdev);
1049
1050 bnxt_re_qp_free_umem(qp);
1051
1052 /* Flush all the entries of notification queue associated with
1053 * given qp.
1054 */
1055 scq_nq = qplib_qp->scq->nq;
1056 rcq_nq = qplib_qp->rcq->nq;
1057 bnxt_re_synchronize_nq(scq_nq);
1058 if (scq_nq != rcq_nq)
1059 bnxt_re_synchronize_nq(rcq_nq);
1060
1061 return ib_respond_empty_udata(udata);
1062 }
1063
__from_ib_qp_type(enum ib_qp_type type)1064 static u8 __from_ib_qp_type(enum ib_qp_type type)
1065 {
1066 switch (type) {
1067 case IB_QPT_GSI:
1068 return CMDQ_CREATE_QP1_TYPE_GSI;
1069 case IB_QPT_RC:
1070 return CMDQ_CREATE_QP_TYPE_RC;
1071 case IB_QPT_UD:
1072 return CMDQ_CREATE_QP_TYPE_UD;
1073 case IB_QPT_RAW_PACKET:
1074 return CMDQ_CREATE_QP_TYPE_RAW_ETHERTYPE;
1075 default:
1076 return IB_QPT_MAX;
1077 }
1078 }
1079
bnxt_re_setup_rwqe_size(struct bnxt_qplib_qp * qplqp,int rsge,int max)1080 static u16 bnxt_re_setup_rwqe_size(struct bnxt_qplib_qp *qplqp,
1081 int rsge, int max)
1082 {
1083 if (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC)
1084 rsge = max;
1085 return bnxt_re_get_rwqe_size(rsge);
1086 }
1087
bnxt_re_get_wqe_size(int ilsize,int nsge)1088 static u16 bnxt_re_get_wqe_size(int ilsize, int nsge)
1089 {
1090 u16 wqe_size, calc_ils;
1091
1092 wqe_size = bnxt_re_get_swqe_size(nsge);
1093 if (ilsize) {
1094 calc_ils = sizeof(struct sq_send_hdr) + ilsize;
1095 wqe_size = max_t(u16, calc_ils, wqe_size);
1096 wqe_size = ALIGN(wqe_size, sizeof(struct sq_send_hdr));
1097 }
1098 return wqe_size;
1099 }
1100
bnxt_re_setup_swqe_size(struct bnxt_re_qp * qp,struct ib_qp_init_attr * init_attr)1101 static int bnxt_re_setup_swqe_size(struct bnxt_re_qp *qp,
1102 struct ib_qp_init_attr *init_attr)
1103 {
1104 struct bnxt_qplib_dev_attr *dev_attr;
1105 struct bnxt_qplib_qp *qplqp;
1106 struct bnxt_re_dev *rdev;
1107 struct bnxt_qplib_q *sq;
1108 int align, ilsize;
1109
1110 rdev = qp->rdev;
1111 qplqp = &qp->qplib_qp;
1112 sq = &qplqp->sq;
1113 dev_attr = rdev->dev_attr;
1114
1115 align = sizeof(struct sq_send_hdr);
1116 ilsize = ALIGN(init_attr->cap.max_inline_data, align);
1117
1118 /* For gen p4 and gen p5 fixed wqe compatibility mode
1119 * wqe size is fixed to 128 bytes - ie 6 SGEs
1120 */
1121 if (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) {
1122 sq->wqe_size = bnxt_re_get_swqe_size(BNXT_STATIC_MAX_SGE);
1123 sq->max_sge = BNXT_STATIC_MAX_SGE;
1124 } else {
1125 sq->wqe_size = bnxt_re_get_wqe_size(ilsize, sq->max_sge);
1126 if (sq->wqe_size > bnxt_re_get_swqe_size(dev_attr->max_qp_sges))
1127 return -EINVAL;
1128 }
1129
1130 if (init_attr->cap.max_inline_data) {
1131 qplqp->max_inline_data = sq->wqe_size -
1132 sizeof(struct sq_send_hdr);
1133 init_attr->cap.max_inline_data = qplqp->max_inline_data;
1134 }
1135
1136 return 0;
1137 }
1138
bnxt_re_init_user_qp(struct bnxt_re_dev * rdev,struct bnxt_re_pd * pd,struct bnxt_re_qp * qp,struct bnxt_re_ucontext * cntx,struct bnxt_re_qp_req * ureq)1139 static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev, struct bnxt_re_pd *pd,
1140 struct bnxt_re_qp *qp, struct bnxt_re_ucontext *cntx,
1141 struct bnxt_re_qp_req *ureq)
1142 {
1143 struct bnxt_qplib_qp *qplib_qp;
1144 int bytes = 0, psn_sz;
1145 struct ib_umem *umem;
1146 int psn_nume;
1147
1148 qplib_qp = &qp->qplib_qp;
1149
1150 bytes = (qplib_qp->sq.max_wqe * qplib_qp->sq.wqe_size);
1151 /* Consider mapping PSN search memory only for RC QPs. */
1152 if (qplib_qp->type == CMDQ_CREATE_QP_TYPE_RC) {
1153 psn_sz = bnxt_qplib_is_chip_gen_p5_p7(rdev->chip_ctx) ?
1154 sizeof(struct sq_psn_search_ext) :
1155 sizeof(struct sq_psn_search);
1156 if (cntx && bnxt_re_is_var_size_supported(rdev, cntx)) {
1157 psn_nume = ureq->sq_slots;
1158 } else {
1159 psn_nume = (qplib_qp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ?
1160 qplib_qp->sq.max_wqe : ((qplib_qp->sq.max_wqe * qplib_qp->sq.wqe_size) /
1161 sizeof(struct bnxt_qplib_sge));
1162 }
1163 if (_is_host_msn_table(rdev->qplib_res.dattr->dev_cap_flags2))
1164 psn_nume = roundup_pow_of_two(psn_nume);
1165 bytes += (psn_nume * psn_sz);
1166 }
1167
1168 bytes = PAGE_ALIGN(bytes);
1169 umem = ib_umem_get(&rdev->ibdev, ureq->qpsva, bytes,
1170 IB_ACCESS_LOCAL_WRITE);
1171 if (IS_ERR(umem))
1172 return PTR_ERR(umem);
1173
1174 qp->sumem = umem;
1175 qplib_qp->sq.sg_info.umem = umem;
1176 qplib_qp->sq.sg_info.pgsize = PAGE_SIZE;
1177 qplib_qp->sq.sg_info.pgshft = PAGE_SHIFT;
1178 qplib_qp->qp_handle = ureq->qp_handle;
1179
1180 if (!qp->qplib_qp.srq) {
1181 bytes = (qplib_qp->rq.max_wqe * qplib_qp->rq.wqe_size);
1182 bytes = PAGE_ALIGN(bytes);
1183 umem = ib_umem_get(&rdev->ibdev, ureq->qprva, bytes,
1184 IB_ACCESS_LOCAL_WRITE);
1185 if (IS_ERR(umem))
1186 goto rqfail;
1187 qp->rumem = umem;
1188 qplib_qp->rq.sg_info.umem = umem;
1189 qplib_qp->rq.sg_info.pgsize = PAGE_SIZE;
1190 qplib_qp->rq.sg_info.pgshft = PAGE_SHIFT;
1191 }
1192
1193 qplib_qp->dpi = &cntx->dpi;
1194 qplib_qp->is_user = true;
1195 return 0;
1196 rqfail:
1197 ib_umem_release(qp->sumem);
1198 qp->sumem = NULL;
1199 memset(&qplib_qp->sq.sg_info, 0, sizeof(qplib_qp->sq.sg_info));
1200
1201 return PTR_ERR(umem);
1202 }
1203
bnxt_re_create_shadow_qp_ah(struct bnxt_re_pd * pd,struct bnxt_qplib_res * qp1_res,struct bnxt_qplib_qp * qp1_qp)1204 static struct bnxt_re_ah *bnxt_re_create_shadow_qp_ah
1205 (struct bnxt_re_pd *pd,
1206 struct bnxt_qplib_res *qp1_res,
1207 struct bnxt_qplib_qp *qp1_qp)
1208 {
1209 struct bnxt_re_dev *rdev = pd->rdev;
1210 struct bnxt_re_ah *ah;
1211 union ib_gid sgid;
1212 int rc;
1213
1214 ah = kzalloc_obj(*ah);
1215 if (!ah)
1216 return NULL;
1217
1218 ah->rdev = rdev;
1219 ah->qplib_ah.pd = &pd->qplib_pd;
1220
1221 rc = bnxt_re_query_gid(&rdev->ibdev, 1, 0, &sgid);
1222 if (rc)
1223 goto fail;
1224
1225 /* supply the dgid data same as sgid */
1226 memcpy(ah->qplib_ah.dgid.data, &sgid.raw,
1227 sizeof(union ib_gid));
1228 ah->qplib_ah.sgid_index = 0;
1229
1230 ah->qplib_ah.traffic_class = 0;
1231 ah->qplib_ah.flow_label = 0;
1232 ah->qplib_ah.hop_limit = 1;
1233 ah->qplib_ah.sl = 0;
1234 /* Have DMAC same as SMAC */
1235 ether_addr_copy(ah->qplib_ah.dmac, rdev->netdev->dev_addr);
1236
1237 rc = bnxt_qplib_create_ah(&rdev->qplib_res, &ah->qplib_ah, false);
1238 if (rc) {
1239 ibdev_err(&rdev->ibdev,
1240 "Failed to allocate HW AH for Shadow QP");
1241 goto fail;
1242 }
1243 atomic_inc(&rdev->stats.res.ah_count);
1244
1245 return ah;
1246
1247 fail:
1248 kfree(ah);
1249 return NULL;
1250 }
1251
bnxt_re_qp_alloc_init_xrrq(struct bnxt_re_qp * qp)1252 static int bnxt_re_qp_alloc_init_xrrq(struct bnxt_re_qp *qp)
1253 {
1254 struct bnxt_qplib_res *res = &qp->rdev->qplib_res;
1255 struct bnxt_qplib_qp *qplib_qp = &qp->qplib_qp;
1256 struct bnxt_qplib_hwq_attr hwq_attr = {};
1257 struct bnxt_qplib_sg_info sginfo = {};
1258 struct bnxt_qplib_hwq *irrq, *orrq;
1259 int rc, req_size;
1260
1261 orrq = &qplib_qp->orrq;
1262 orrq->max_elements =
1263 ORD_LIMIT_TO_ORRQ_SLOTS(qplib_qp->max_rd_atomic);
1264 req_size = orrq->max_elements *
1265 BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE + PAGE_SIZE - 1;
1266 req_size &= ~(PAGE_SIZE - 1);
1267 sginfo.pgsize = req_size;
1268 sginfo.pgshft = PAGE_SHIFT;
1269
1270 hwq_attr.res = res;
1271 hwq_attr.sginfo = &sginfo;
1272 hwq_attr.depth = orrq->max_elements;
1273 hwq_attr.stride = BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE;
1274 hwq_attr.aux_stride = 0;
1275 hwq_attr.aux_depth = 0;
1276 hwq_attr.type = HWQ_TYPE_CTX;
1277 rc = bnxt_qplib_alloc_init_hwq(orrq, &hwq_attr);
1278 if (rc)
1279 return rc;
1280
1281 irrq = &qplib_qp->irrq;
1282 irrq->max_elements =
1283 IRD_LIMIT_TO_IRRQ_SLOTS(qplib_qp->max_dest_rd_atomic);
1284 req_size = irrq->max_elements *
1285 BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE + PAGE_SIZE - 1;
1286 req_size &= ~(PAGE_SIZE - 1);
1287 sginfo.pgsize = req_size;
1288 hwq_attr.sginfo = &sginfo;
1289 hwq_attr.depth = irrq->max_elements;
1290 hwq_attr.stride = BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE;
1291 rc = bnxt_qplib_alloc_init_hwq(irrq, &hwq_attr);
1292 if (rc)
1293 goto free_orrq_hwq;
1294 return 0;
1295 free_orrq_hwq:
1296 bnxt_qplib_free_hwq(res, orrq);
1297 return rc;
1298 }
1299
bnxt_re_setup_qp_hwqs(struct bnxt_re_qp * qp)1300 static int bnxt_re_setup_qp_hwqs(struct bnxt_re_qp *qp)
1301 {
1302 struct bnxt_qplib_res *res = &qp->rdev->qplib_res;
1303 struct bnxt_qplib_qp *qplib_qp = &qp->qplib_qp;
1304 struct bnxt_qplib_hwq_attr hwq_attr = {};
1305 struct bnxt_qplib_q *sq = &qplib_qp->sq;
1306 struct bnxt_qplib_q *rq = &qplib_qp->rq;
1307 u8 wqe_mode = qplib_qp->wqe_mode;
1308 u8 pg_sz_lvl;
1309 int rc;
1310
1311 hwq_attr.res = res;
1312 hwq_attr.sginfo = &sq->sg_info;
1313 hwq_attr.stride = bnxt_qplib_get_stride();
1314 hwq_attr.depth = bnxt_qplib_get_depth(sq, wqe_mode, true);
1315 hwq_attr.aux_stride = qplib_qp->psn_sz;
1316 hwq_attr.aux_depth = (qplib_qp->psn_sz) ?
1317 bnxt_qplib_set_sq_size(sq, wqe_mode) : 0;
1318 if (qplib_qp->is_host_msn_tbl && qplib_qp->psn_sz)
1319 hwq_attr.aux_depth = qplib_qp->msn_tbl_sz;
1320 hwq_attr.type = HWQ_TYPE_QUEUE;
1321 rc = bnxt_qplib_alloc_init_hwq(&sq->hwq, &hwq_attr);
1322 if (rc)
1323 return rc;
1324
1325 pg_sz_lvl = bnxt_qplib_base_pg_size(&sq->hwq) << CMDQ_CREATE_QP_SQ_PG_SIZE_SFT;
1326 pg_sz_lvl |= ((sq->hwq.level & CMDQ_CREATE_QP_SQ_LVL_MASK) <<
1327 CMDQ_CREATE_QP_SQ_LVL_SFT);
1328 sq->hwq.pg_sz_lvl = pg_sz_lvl;
1329
1330 hwq_attr.res = res;
1331 hwq_attr.sginfo = &rq->sg_info;
1332 hwq_attr.stride = bnxt_qplib_get_stride();
1333 hwq_attr.depth = bnxt_qplib_get_depth(rq, qplib_qp->wqe_mode, false);
1334 hwq_attr.aux_stride = 0;
1335 hwq_attr.aux_depth = 0;
1336 hwq_attr.type = HWQ_TYPE_QUEUE;
1337 rc = bnxt_qplib_alloc_init_hwq(&rq->hwq, &hwq_attr);
1338 if (rc)
1339 goto free_sq_hwq;
1340 pg_sz_lvl = bnxt_qplib_base_pg_size(&rq->hwq) <<
1341 CMDQ_CREATE_QP_RQ_PG_SIZE_SFT;
1342 pg_sz_lvl |= ((rq->hwq.level & CMDQ_CREATE_QP_RQ_LVL_MASK) <<
1343 CMDQ_CREATE_QP_RQ_LVL_SFT);
1344 rq->hwq.pg_sz_lvl = pg_sz_lvl;
1345
1346 if (qplib_qp->psn_sz) {
1347 rc = bnxt_re_qp_alloc_init_xrrq(qp);
1348 if (rc)
1349 goto free_rq_hwq;
1350 }
1351
1352 return 0;
1353 free_rq_hwq:
1354 bnxt_qplib_free_hwq(res, &rq->hwq);
1355 free_sq_hwq:
1356 bnxt_qplib_free_hwq(res, &sq->hwq);
1357 return rc;
1358 }
1359
bnxt_re_create_shadow_qp(struct bnxt_re_pd * pd,struct bnxt_qplib_res * qp1_res,struct bnxt_qplib_qp * qp1_qp)1360 static struct bnxt_re_qp *bnxt_re_create_shadow_qp
1361 (struct bnxt_re_pd *pd,
1362 struct bnxt_qplib_res *qp1_res,
1363 struct bnxt_qplib_qp *qp1_qp)
1364 {
1365 struct bnxt_re_dev *rdev = pd->rdev;
1366 struct bnxt_re_qp *qp;
1367 int rc;
1368
1369 qp = kzalloc_obj(*qp);
1370 if (!qp)
1371 return NULL;
1372
1373 qp->rdev = rdev;
1374
1375 /* Initialize the shadow QP structure from the QP1 values */
1376 ether_addr_copy(qp->qplib_qp.smac, rdev->netdev->dev_addr);
1377
1378 qp->qplib_qp.pd = &pd->qplib_pd;
1379 qp->qplib_qp.qp_handle = (u64)(unsigned long)(&qp->qplib_qp);
1380 qp->qplib_qp.type = IB_QPT_UD;
1381 qp->qplib_qp.cctx = rdev->chip_ctx;
1382
1383 qp->qplib_qp.max_inline_data = 0;
1384 qp->qplib_qp.sig_type = true;
1385
1386 /* Shadow QP SQ depth should be same as QP1 RQ depth */
1387 qp->qplib_qp.sq.wqe_size = bnxt_re_get_wqe_size(0, 6);
1388 qp->qplib_qp.sq.max_wqe = qp1_qp->rq.max_wqe;
1389 qp->qplib_qp.sq.max_sw_wqe = qp1_qp->rq.max_wqe;
1390 qp->qplib_qp.sq.max_sge = 2;
1391 /* Q full delta can be 1 since it is internal QP */
1392 qp->qplib_qp.sq.q_full_delta = 1;
1393 qp->qplib_qp.sq.sg_info.pgsize = PAGE_SIZE;
1394 qp->qplib_qp.sq.sg_info.pgshft = PAGE_SHIFT;
1395
1396 qp->qplib_qp.scq = qp1_qp->scq;
1397 qp->qplib_qp.rcq = qp1_qp->rcq;
1398
1399 qp->qplib_qp.rq.wqe_size = bnxt_re_get_rwqe_size(6);
1400 qp->qplib_qp.rq.max_wqe = qp1_qp->rq.max_wqe;
1401 qp->qplib_qp.rq.max_sw_wqe = qp1_qp->rq.max_wqe;
1402 qp->qplib_qp.rq.max_sge = qp1_qp->rq.max_sge;
1403 /* Q full delta can be 1 since it is internal QP */
1404 qp->qplib_qp.rq.q_full_delta = 1;
1405 qp->qplib_qp.rq.sg_info.pgsize = PAGE_SIZE;
1406 qp->qplib_qp.rq.sg_info.pgshft = PAGE_SHIFT;
1407
1408 qp->qplib_qp.mtu = qp1_qp->mtu;
1409
1410 qp->qplib_qp.sq_hdr_buf_size = 0;
1411 qp->qplib_qp.rq_hdr_buf_size = BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6;
1412 qp->qplib_qp.dpi = &rdev->dpi_privileged;
1413
1414 rc = bnxt_re_setup_qp_hwqs(qp);
1415 if (rc)
1416 goto fail;
1417
1418 rc = bnxt_qplib_create_qp(qp1_res, &qp->qplib_qp);
1419 if (rc)
1420 goto free_hwq;
1421
1422 spin_lock_init(&qp->sq_lock);
1423 INIT_LIST_HEAD(&qp->list);
1424 mutex_lock(&rdev->qp_lock);
1425 list_add_tail(&qp->list, &rdev->qp_list);
1426 atomic_inc(&rdev->stats.res.qp_count);
1427 mutex_unlock(&rdev->qp_lock);
1428 return qp;
1429
1430 free_hwq:
1431 bnxt_qplib_free_qp_res(&rdev->qplib_res, &qp->qplib_qp);
1432 fail:
1433 kfree(qp);
1434 return NULL;
1435 }
1436
bnxt_re_init_rq_attr(struct bnxt_re_qp * qp,struct ib_qp_init_attr * init_attr,struct bnxt_re_ucontext * uctx)1437 static int bnxt_re_init_rq_attr(struct bnxt_re_qp *qp,
1438 struct ib_qp_init_attr *init_attr,
1439 struct bnxt_re_ucontext *uctx)
1440 {
1441 struct bnxt_qplib_dev_attr *dev_attr;
1442 struct bnxt_qplib_qp *qplqp;
1443 struct bnxt_re_dev *rdev;
1444 struct bnxt_qplib_q *rq;
1445
1446 rdev = qp->rdev;
1447 qplqp = &qp->qplib_qp;
1448 rq = &qplqp->rq;
1449 dev_attr = rdev->dev_attr;
1450
1451 if (init_attr->srq) {
1452 struct bnxt_re_srq *srq;
1453
1454 srq = container_of(init_attr->srq, struct bnxt_re_srq, ib_srq);
1455 qplqp->srq = &srq->qplib_srq;
1456 rq->max_wqe = 0;
1457 } else {
1458 rq->max_sge = init_attr->cap.max_recv_sge;
1459 if (rq->max_sge > dev_attr->max_qp_sges)
1460 rq->max_sge = dev_attr->max_qp_sges;
1461 init_attr->cap.max_recv_sge = rq->max_sge;
1462 rq->wqe_size = bnxt_re_setup_rwqe_size(qplqp, rq->max_sge,
1463 dev_attr->max_qp_sges);
1464 /* Allocate 1 more than what's provided so posting max doesn't
1465 * mean empty.
1466 */
1467 rq->max_wqe = bnxt_re_init_depth(init_attr->cap.max_recv_wr + 1,
1468 dev_attr->max_qp_wqes + 1,
1469 uctx);
1470 rq->max_sw_wqe = rq->max_wqe;
1471 rq->q_full_delta = 0;
1472 rq->sg_info.pgsize = PAGE_SIZE;
1473 rq->sg_info.pgshft = PAGE_SHIFT;
1474 }
1475
1476 return 0;
1477 }
1478
bnxt_re_adjust_gsi_rq_attr(struct bnxt_re_qp * qp)1479 static void bnxt_re_adjust_gsi_rq_attr(struct bnxt_re_qp *qp)
1480 {
1481 struct bnxt_qplib_dev_attr *dev_attr;
1482 struct bnxt_qplib_qp *qplqp;
1483 struct bnxt_re_dev *rdev;
1484
1485 rdev = qp->rdev;
1486 qplqp = &qp->qplib_qp;
1487 dev_attr = rdev->dev_attr;
1488
1489 if (!bnxt_qplib_is_chip_gen_p5_p7(rdev->chip_ctx)) {
1490 qplqp->rq.max_sge = dev_attr->max_qp_sges;
1491 if (qplqp->rq.max_sge > dev_attr->max_qp_sges)
1492 qplqp->rq.max_sge = dev_attr->max_qp_sges;
1493 qplqp->rq.max_sge = 6;
1494 }
1495 }
1496
bnxt_re_init_sq_attr(struct bnxt_re_qp * qp,struct ib_qp_init_attr * init_attr,struct bnxt_re_ucontext * uctx,struct bnxt_re_qp_req * ureq)1497 static int bnxt_re_init_sq_attr(struct bnxt_re_qp *qp,
1498 struct ib_qp_init_attr *init_attr,
1499 struct bnxt_re_ucontext *uctx,
1500 struct bnxt_re_qp_req *ureq)
1501 {
1502 struct bnxt_qplib_dev_attr *dev_attr;
1503 struct bnxt_qplib_qp *qplqp;
1504 struct bnxt_re_dev *rdev;
1505 struct bnxt_qplib_q *sq;
1506 int diff = 0;
1507 int rc;
1508
1509 rdev = qp->rdev;
1510 qplqp = &qp->qplib_qp;
1511 sq = &qplqp->sq;
1512 dev_attr = rdev->dev_attr;
1513
1514 sq->max_sge = init_attr->cap.max_send_sge;
1515 if (uctx && qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE) {
1516 sq->max_wqe = ureq->sq_slots;
1517 sq->max_sw_wqe = ureq->sq_slots;
1518 sq->wqe_size = sizeof(struct sq_sge);
1519 } else {
1520 if (sq->max_sge > dev_attr->max_qp_sges) {
1521 sq->max_sge = dev_attr->max_qp_sges;
1522 init_attr->cap.max_send_sge = sq->max_sge;
1523 }
1524
1525 rc = bnxt_re_setup_swqe_size(qp, init_attr);
1526 if (rc)
1527 return rc;
1528
1529 /* Allocate 128 + 1 more than what's provided */
1530 if (qplqp->wqe_mode != BNXT_QPLIB_WQE_MODE_VARIABLE)
1531 diff = BNXT_QPLIB_RESERVED_QP_WRS;
1532 sq->max_wqe = bnxt_re_init_depth(
1533 init_attr->cap.max_send_wr + diff + 1,
1534 dev_attr->max_qp_wqes + diff + 1, uctx);
1535 if (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE)
1536 sq->max_sw_wqe = bnxt_qplib_get_depth(sq, qplqp->wqe_mode, true);
1537 else
1538 sq->max_sw_wqe = sq->max_wqe;
1539
1540 }
1541 sq->q_full_delta = diff + 1;
1542 /*
1543 * Reserving one slot for Phantom WQE. Application can
1544 * post one extra entry in this case. But allowing this to avoid
1545 * unexpected Queue full condition
1546 */
1547 qplqp->sq.q_full_delta -= 1;
1548 qplqp->sq.sg_info.pgsize = PAGE_SIZE;
1549 qplqp->sq.sg_info.pgshft = PAGE_SHIFT;
1550
1551 return 0;
1552 }
1553
bnxt_re_adjust_gsi_sq_attr(struct bnxt_re_qp * qp,struct ib_qp_init_attr * init_attr,struct bnxt_re_ucontext * uctx)1554 static void bnxt_re_adjust_gsi_sq_attr(struct bnxt_re_qp *qp,
1555 struct ib_qp_init_attr *init_attr,
1556 struct bnxt_re_ucontext *uctx)
1557 {
1558 struct bnxt_qplib_dev_attr *dev_attr;
1559 struct bnxt_qplib_qp *qplqp;
1560 struct bnxt_re_dev *rdev;
1561
1562 rdev = qp->rdev;
1563 qplqp = &qp->qplib_qp;
1564 dev_attr = rdev->dev_attr;
1565
1566 if (!bnxt_qplib_is_chip_gen_p5_p7(rdev->chip_ctx)) {
1567 qplqp->sq.max_wqe =
1568 bnxt_re_init_depth(init_attr->cap.max_send_wr + 1,
1569 dev_attr->max_qp_wqes + 1, uctx);
1570 qplqp->sq.q_full_delta = qplqp->sq.max_wqe -
1571 init_attr->cap.max_send_wr;
1572 qplqp->sq.max_sge++; /* Need one extra sge to put UD header */
1573 if (qplqp->sq.max_sge > dev_attr->max_qp_sges)
1574 qplqp->sq.max_sge = dev_attr->max_qp_sges;
1575 }
1576 }
1577
bnxt_re_init_qp_type(struct bnxt_re_dev * rdev,struct ib_qp_init_attr * init_attr)1578 static int bnxt_re_init_qp_type(struct bnxt_re_dev *rdev,
1579 struct ib_qp_init_attr *init_attr)
1580 {
1581 struct bnxt_qplib_chip_ctx *chip_ctx;
1582 int qptype;
1583
1584 chip_ctx = rdev->chip_ctx;
1585
1586 qptype = __from_ib_qp_type(init_attr->qp_type);
1587 if (qptype == IB_QPT_MAX) {
1588 ibdev_err(&rdev->ibdev, "QP type 0x%x not supported", qptype);
1589 qptype = -EOPNOTSUPP;
1590 goto out;
1591 }
1592
1593 if (bnxt_qplib_is_chip_gen_p5_p7(chip_ctx) &&
1594 init_attr->qp_type == IB_QPT_GSI)
1595 qptype = CMDQ_CREATE_QP_TYPE_GSI;
1596 out:
1597 return qptype;
1598 }
1599
bnxt_re_qp_calculate_msn_psn_size(struct bnxt_re_qp * qp)1600 static void bnxt_re_qp_calculate_msn_psn_size(struct bnxt_re_qp *qp)
1601 {
1602 struct bnxt_qplib_qp *qplib_qp = &qp->qplib_qp;
1603 struct bnxt_qplib_q *sq = &qplib_qp->sq;
1604 struct bnxt_re_dev *rdev = qp->rdev;
1605 u8 wqe_mode = qplib_qp->wqe_mode;
1606
1607 if (rdev->dev_attr)
1608 qplib_qp->is_host_msn_tbl =
1609 _is_host_msn_table(rdev->dev_attr->dev_cap_flags2);
1610
1611 if (qplib_qp->type == CMDQ_CREATE_QP_TYPE_RC) {
1612 qplib_qp->psn_sz = bnxt_qplib_is_chip_gen_p5_p7(rdev->chip_ctx) ?
1613 sizeof(struct sq_psn_search_ext) :
1614 sizeof(struct sq_psn_search);
1615 if (qplib_qp->is_host_msn_tbl) {
1616 qplib_qp->psn_sz = sizeof(struct sq_msn_search);
1617 qplib_qp->msn = 0;
1618 }
1619 }
1620
1621 /* Update msn tbl size */
1622 if (qplib_qp->is_host_msn_tbl && qplib_qp->psn_sz) {
1623 if (wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC)
1624 qplib_qp->msn_tbl_sz =
1625 roundup_pow_of_two(bnxt_qplib_set_sq_size(sq, wqe_mode));
1626 else
1627 qplib_qp->msn_tbl_sz =
1628 roundup_pow_of_two(bnxt_qplib_set_sq_size(sq, wqe_mode)) / 2;
1629 qplib_qp->msn = 0;
1630 }
1631 }
1632
bnxt_re_init_qp_attr(struct bnxt_re_qp * qp,struct bnxt_re_pd * pd,struct ib_qp_init_attr * init_attr,struct bnxt_re_ucontext * uctx,struct bnxt_re_qp_req * ureq)1633 static int bnxt_re_init_qp_attr(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd,
1634 struct ib_qp_init_attr *init_attr,
1635 struct bnxt_re_ucontext *uctx,
1636 struct bnxt_re_qp_req *ureq)
1637 {
1638 struct bnxt_qplib_dev_attr *dev_attr;
1639 struct bnxt_qplib_qp *qplqp;
1640 struct bnxt_re_dev *rdev;
1641 struct bnxt_re_cq *cq;
1642 int rc = 0, qptype;
1643
1644 rdev = qp->rdev;
1645 qplqp = &qp->qplib_qp;
1646 dev_attr = rdev->dev_attr;
1647
1648 /* Setup misc params */
1649 ether_addr_copy(qplqp->smac, rdev->netdev->dev_addr);
1650 qplqp->pd = &pd->qplib_pd;
1651 qplqp->qp_handle = (u64)qplqp;
1652 qplqp->max_inline_data = init_attr->cap.max_inline_data;
1653 qplqp->sig_type = init_attr->sq_sig_type == IB_SIGNAL_ALL_WR;
1654 qptype = bnxt_re_init_qp_type(rdev, init_attr);
1655 if (qptype < 0)
1656 return qptype;
1657 qplqp->type = (u8)qptype;
1658 qplqp->wqe_mode = bnxt_re_is_var_size_supported(rdev, uctx);
1659 qplqp->dev_cap_flags = dev_attr->dev_cap_flags;
1660 qplqp->cctx = rdev->chip_ctx;
1661 if (init_attr->qp_type == IB_QPT_RC) {
1662 qplqp->max_rd_atomic = dev_attr->max_qp_rd_atom;
1663 qplqp->max_dest_rd_atomic = dev_attr->max_qp_init_rd_atom;
1664 }
1665 qplqp->mtu = ib_mtu_enum_to_int(iboe_get_mtu(rdev->netdev->mtu));
1666 qplqp->dpi = &rdev->dpi_privileged; /* Doorbell page */
1667 if (init_attr->create_flags) {
1668 ibdev_dbg(&rdev->ibdev,
1669 "QP create flags 0x%x not supported",
1670 init_attr->create_flags);
1671 return -EOPNOTSUPP;
1672 }
1673
1674 /* Setup CQs */
1675 if (init_attr->send_cq) {
1676 cq = container_of(init_attr->send_cq, struct bnxt_re_cq, ib_cq);
1677 qplqp->scq = &cq->qplib_cq;
1678 qp->scq = cq;
1679 }
1680
1681 if (init_attr->recv_cq) {
1682 cq = container_of(init_attr->recv_cq, struct bnxt_re_cq, ib_cq);
1683 qplqp->rcq = &cq->qplib_cq;
1684 qp->rcq = cq;
1685 }
1686
1687 /* Setup RQ/SRQ */
1688 rc = bnxt_re_init_rq_attr(qp, init_attr, uctx);
1689 if (rc)
1690 return rc;
1691 if (init_attr->qp_type == IB_QPT_GSI)
1692 bnxt_re_adjust_gsi_rq_attr(qp);
1693
1694 /* Setup SQ */
1695 rc = bnxt_re_init_sq_attr(qp, init_attr, uctx, ureq);
1696 if (rc)
1697 return rc;
1698 if (init_attr->qp_type == IB_QPT_GSI)
1699 bnxt_re_adjust_gsi_sq_attr(qp, init_attr, uctx);
1700
1701 if (uctx) { /* This will update DPI and qp_handle */
1702 rc = bnxt_re_init_user_qp(rdev, pd, qp, uctx, ureq);
1703 if (rc)
1704 return rc;
1705 }
1706
1707 bnxt_re_qp_calculate_msn_psn_size(qp);
1708
1709 rc = bnxt_re_setup_qp_hwqs(qp);
1710 if (rc)
1711 goto free_umem;
1712
1713 return 0;
1714 free_umem:
1715 bnxt_re_qp_free_umem(qp);
1716 return rc;
1717 }
1718
bnxt_re_create_shadow_gsi(struct bnxt_re_qp * qp,struct bnxt_re_pd * pd)1719 static int bnxt_re_create_shadow_gsi(struct bnxt_re_qp *qp,
1720 struct bnxt_re_pd *pd)
1721 {
1722 struct bnxt_re_sqp_entries *sqp_tbl;
1723 struct bnxt_re_dev *rdev;
1724 struct bnxt_re_qp *sqp;
1725 struct bnxt_re_ah *sah;
1726 int rc = 0;
1727
1728 rdev = qp->rdev;
1729 /* Create a shadow QP to handle the QP1 traffic */
1730 sqp_tbl = kzalloc_objs(*sqp_tbl, BNXT_RE_MAX_GSI_SQP_ENTRIES);
1731 if (!sqp_tbl)
1732 return -ENOMEM;
1733 rdev->gsi_ctx.sqp_tbl = sqp_tbl;
1734
1735 sqp = bnxt_re_create_shadow_qp(pd, &rdev->qplib_res, &qp->qplib_qp);
1736 if (!sqp) {
1737 rc = -ENODEV;
1738 ibdev_err(&rdev->ibdev, "Failed to create Shadow QP for QP1");
1739 goto out;
1740 }
1741 rdev->gsi_ctx.gsi_sqp = sqp;
1742
1743 sqp->rcq = qp->rcq;
1744 sqp->scq = qp->scq;
1745 sah = bnxt_re_create_shadow_qp_ah(pd, &rdev->qplib_res,
1746 &qp->qplib_qp);
1747 if (!sah) {
1748 bnxt_qplib_destroy_qp(&rdev->qplib_res,
1749 &sqp->qplib_qp);
1750 rc = -ENODEV;
1751 ibdev_err(&rdev->ibdev,
1752 "Failed to create AH entry for ShadowQP");
1753 goto out;
1754 }
1755 rdev->gsi_ctx.gsi_sah = sah;
1756
1757 return 0;
1758 out:
1759 kfree(sqp_tbl);
1760 return rc;
1761 }
1762
bnxt_re_create_gsi_qp(struct bnxt_re_qp * qp,struct bnxt_re_pd * pd,struct ib_qp_init_attr * init_attr)1763 static int bnxt_re_create_gsi_qp(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd,
1764 struct ib_qp_init_attr *init_attr)
1765 {
1766 struct bnxt_re_dev *rdev;
1767 struct bnxt_qplib_qp *qplqp;
1768 int rc;
1769
1770 rdev = qp->rdev;
1771 qplqp = &qp->qplib_qp;
1772 qplqp->cctx = rdev->chip_ctx;
1773
1774 qplqp->rq_hdr_buf_size = BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2;
1775 qplqp->sq_hdr_buf_size = BNXT_QPLIB_MAX_QP1_SQ_HDR_SIZE_V2;
1776
1777 rc = bnxt_qplib_create_qp1(&rdev->qplib_res, qplqp);
1778 if (rc) {
1779 ibdev_err(&rdev->ibdev, "create HW QP1 failed!");
1780 goto out;
1781 }
1782
1783 rc = bnxt_re_create_shadow_gsi(qp, pd);
1784 out:
1785 return rc;
1786 }
1787
bnxt_re_test_qp_limits(struct bnxt_re_dev * rdev,struct ib_qp_init_attr * init_attr,struct bnxt_qplib_dev_attr * dev_attr)1788 static bool bnxt_re_test_qp_limits(struct bnxt_re_dev *rdev,
1789 struct ib_qp_init_attr *init_attr,
1790 struct bnxt_qplib_dev_attr *dev_attr)
1791 {
1792 bool rc = true;
1793
1794 if (init_attr->cap.max_send_wr > dev_attr->max_qp_wqes ||
1795 init_attr->cap.max_recv_wr > dev_attr->max_qp_wqes ||
1796 init_attr->cap.max_send_sge > dev_attr->max_qp_sges ||
1797 init_attr->cap.max_recv_sge > dev_attr->max_qp_sges ||
1798 init_attr->cap.max_inline_data > dev_attr->max_inline_data) {
1799 ibdev_err(&rdev->ibdev,
1800 "Create QP failed - max exceeded! 0x%x/0x%x 0x%x/0x%x 0x%x/0x%x 0x%x/0x%x 0x%x/0x%x",
1801 init_attr->cap.max_send_wr, dev_attr->max_qp_wqes,
1802 init_attr->cap.max_recv_wr, dev_attr->max_qp_wqes,
1803 init_attr->cap.max_send_sge, dev_attr->max_qp_sges,
1804 init_attr->cap.max_recv_sge, dev_attr->max_qp_sges,
1805 init_attr->cap.max_inline_data,
1806 dev_attr->max_inline_data);
1807 rc = false;
1808 }
1809 return rc;
1810 }
1811
bnxt_re_add_unique_gid(struct bnxt_re_dev * rdev)1812 static int bnxt_re_add_unique_gid(struct bnxt_re_dev *rdev)
1813 {
1814 struct bnxt_qplib_ctx *hctx = &rdev->qplib_ctx;
1815 struct bnxt_qplib_res *res = &rdev->qplib_res;
1816 int rc;
1817
1818 if (!rdev->rcfw.roce_mirror)
1819 return 0;
1820
1821 rdev->ugid.global.subnet_prefix = cpu_to_be64(0xfe8000000000abcdLL);
1822 addrconf_ifid_eui48(&rdev->ugid.raw[8], rdev->netdev);
1823
1824 rc = bnxt_qplib_add_sgid(&res->sgid_tbl,
1825 (struct bnxt_qplib_gid *)&rdev->ugid,
1826 rdev->qplib_res.netdev->dev_addr,
1827 0xFFFF, true, &rdev->ugid_index, true,
1828 hctx->stats3.fw_id);
1829 if (rc)
1830 dev_err(rdev_to_dev(rdev), "Failed to add unique GID. rc = %d\n", rc);
1831
1832 return rc;
1833 }
1834
bnxt_re_create_qp(struct ib_qp * ib_qp,struct ib_qp_init_attr * qp_init_attr,struct ib_udata * udata)1835 int bnxt_re_create_qp(struct ib_qp *ib_qp, struct ib_qp_init_attr *qp_init_attr,
1836 struct ib_udata *udata)
1837 {
1838 struct bnxt_qplib_dev_attr *dev_attr;
1839 struct bnxt_re_ucontext *uctx;
1840 struct bnxt_re_qp_req ureq;
1841 struct bnxt_re_dev *rdev;
1842 struct bnxt_re_pd *pd;
1843 struct bnxt_re_qp *qp;
1844 struct ib_pd *ib_pd;
1845 u32 active_qps;
1846 int rc;
1847
1848 ib_pd = ib_qp->pd;
1849 pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
1850 rdev = pd->rdev;
1851 dev_attr = rdev->dev_attr;
1852 qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
1853
1854 uctx = rdma_udata_to_drv_context(udata, struct bnxt_re_ucontext, ib_uctx);
1855 if (udata) {
1856 rc = ib_copy_validate_udata_in_cm(udata, ureq, qp_handle, 0);
1857 if (rc)
1858 return rc;
1859 }
1860
1861 rc = bnxt_re_test_qp_limits(rdev, qp_init_attr, dev_attr);
1862 if (!rc) {
1863 rc = -EINVAL;
1864 goto fail;
1865 }
1866
1867 qp->rdev = rdev;
1868 rc = bnxt_re_init_qp_attr(qp, pd, qp_init_attr, uctx, &ureq);
1869 if (rc)
1870 goto fail;
1871
1872 if (qp_init_attr->qp_type == IB_QPT_GSI &&
1873 !(bnxt_qplib_is_chip_gen_p5_p7(rdev->chip_ctx))) {
1874 rc = bnxt_re_create_gsi_qp(qp, pd, qp_init_attr);
1875 if (rc == -ENODEV)
1876 goto qp_destroy;
1877 if (rc)
1878 goto free_hwq;
1879 } else {
1880 rc = bnxt_qplib_create_qp(&rdev->qplib_res, &qp->qplib_qp);
1881 if (rc) {
1882 ibdev_err(&rdev->ibdev, "Failed to create HW QP");
1883 goto free_hwq;
1884 }
1885
1886 if (udata) {
1887 struct bnxt_re_qp_resp resp;
1888
1889 resp.qpid = qp->qplib_qp.id;
1890 resp.rsvd = 0;
1891 rc = ib_respond_udata(udata, resp);
1892 if (rc)
1893 goto qp_destroy;
1894 }
1895 }
1896
1897 /* Support for RawEth QP is added to capture TCP pkt dump.
1898 * So unique SGID is used to avoid incorrect statistics on per
1899 * function stats_ctx
1900 */
1901 if (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_RAW_ETHERTYPE) {
1902 rc = bnxt_re_add_unique_gid(rdev);
1903 if (rc)
1904 goto qp_destroy;
1905 qp->qplib_qp.ugid_index = rdev->ugid_index;
1906 }
1907
1908 qp->ib_qp.qp_num = qp->qplib_qp.id;
1909 if (qp_init_attr->qp_type == IB_QPT_GSI)
1910 rdev->gsi_ctx.gsi_qp = qp;
1911 spin_lock_init(&qp->sq_lock);
1912 spin_lock_init(&qp->rq_lock);
1913 INIT_LIST_HEAD(&qp->list);
1914 mutex_lock(&rdev->qp_lock);
1915 list_add_tail(&qp->list, &rdev->qp_list);
1916 mutex_unlock(&rdev->qp_lock);
1917 active_qps = atomic_inc_return(&rdev->stats.res.qp_count);
1918 if (active_qps > rdev->stats.res.qp_watermark)
1919 rdev->stats.res.qp_watermark = active_qps;
1920 if (qp_init_attr->qp_type == IB_QPT_RC) {
1921 active_qps = atomic_inc_return(&rdev->stats.res.rc_qp_count);
1922 if (active_qps > rdev->stats.res.rc_qp_watermark)
1923 rdev->stats.res.rc_qp_watermark = active_qps;
1924 } else if (qp_init_attr->qp_type == IB_QPT_UD) {
1925 active_qps = atomic_inc_return(&rdev->stats.res.ud_qp_count);
1926 if (active_qps > rdev->stats.res.ud_qp_watermark)
1927 rdev->stats.res.ud_qp_watermark = active_qps;
1928 }
1929 bnxt_re_debug_add_qpinfo(rdev, qp);
1930
1931 return 0;
1932 qp_destroy:
1933 bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp);
1934 free_hwq:
1935 bnxt_qplib_free_qp_res(&rdev->qplib_res, &qp->qplib_qp);
1936 bnxt_re_qp_free_umem(qp);
1937 fail:
1938 return rc;
1939 }
1940
__from_ib_qp_state(enum ib_qp_state state)1941 static u8 __from_ib_qp_state(enum ib_qp_state state)
1942 {
1943 switch (state) {
1944 case IB_QPS_RESET:
1945 return CMDQ_MODIFY_QP_NEW_STATE_RESET;
1946 case IB_QPS_INIT:
1947 return CMDQ_MODIFY_QP_NEW_STATE_INIT;
1948 case IB_QPS_RTR:
1949 return CMDQ_MODIFY_QP_NEW_STATE_RTR;
1950 case IB_QPS_RTS:
1951 return CMDQ_MODIFY_QP_NEW_STATE_RTS;
1952 case IB_QPS_SQD:
1953 return CMDQ_MODIFY_QP_NEW_STATE_SQD;
1954 case IB_QPS_SQE:
1955 return CMDQ_MODIFY_QP_NEW_STATE_SQE;
1956 case IB_QPS_ERR:
1957 default:
1958 return CMDQ_MODIFY_QP_NEW_STATE_ERR;
1959 }
1960 }
1961
__to_ib_qp_state(u8 state)1962 static enum ib_qp_state __to_ib_qp_state(u8 state)
1963 {
1964 switch (state) {
1965 case CMDQ_MODIFY_QP_NEW_STATE_RESET:
1966 return IB_QPS_RESET;
1967 case CMDQ_MODIFY_QP_NEW_STATE_INIT:
1968 return IB_QPS_INIT;
1969 case CMDQ_MODIFY_QP_NEW_STATE_RTR:
1970 return IB_QPS_RTR;
1971 case CMDQ_MODIFY_QP_NEW_STATE_RTS:
1972 return IB_QPS_RTS;
1973 case CMDQ_MODIFY_QP_NEW_STATE_SQD:
1974 return IB_QPS_SQD;
1975 case CMDQ_MODIFY_QP_NEW_STATE_SQE:
1976 return IB_QPS_SQE;
1977 case CMDQ_MODIFY_QP_NEW_STATE_ERR:
1978 default:
1979 return IB_QPS_ERR;
1980 }
1981 }
1982
__from_ib_mtu(enum ib_mtu mtu)1983 static u32 __from_ib_mtu(enum ib_mtu mtu)
1984 {
1985 switch (mtu) {
1986 case IB_MTU_256:
1987 return CMDQ_MODIFY_QP_PATH_MTU_MTU_256;
1988 case IB_MTU_512:
1989 return CMDQ_MODIFY_QP_PATH_MTU_MTU_512;
1990 case IB_MTU_1024:
1991 return CMDQ_MODIFY_QP_PATH_MTU_MTU_1024;
1992 case IB_MTU_2048:
1993 return CMDQ_MODIFY_QP_PATH_MTU_MTU_2048;
1994 case IB_MTU_4096:
1995 return CMDQ_MODIFY_QP_PATH_MTU_MTU_4096;
1996 default:
1997 return CMDQ_MODIFY_QP_PATH_MTU_MTU_2048;
1998 }
1999 }
2000
__to_ib_mtu(u32 mtu)2001 static enum ib_mtu __to_ib_mtu(u32 mtu)
2002 {
2003 switch (mtu & CREQ_QUERY_QP_RESP_SB_PATH_MTU_MASK) {
2004 case CMDQ_MODIFY_QP_PATH_MTU_MTU_256:
2005 return IB_MTU_256;
2006 case CMDQ_MODIFY_QP_PATH_MTU_MTU_512:
2007 return IB_MTU_512;
2008 case CMDQ_MODIFY_QP_PATH_MTU_MTU_1024:
2009 return IB_MTU_1024;
2010 case CMDQ_MODIFY_QP_PATH_MTU_MTU_2048:
2011 return IB_MTU_2048;
2012 case CMDQ_MODIFY_QP_PATH_MTU_MTU_4096:
2013 return IB_MTU_4096;
2014 default:
2015 return IB_MTU_2048;
2016 }
2017 }
2018
2019 /* Shared Receive Queues */
bnxt_re_destroy_srq(struct ib_srq * ib_srq,struct ib_udata * udata)2020 int bnxt_re_destroy_srq(struct ib_srq *ib_srq, struct ib_udata *udata)
2021 {
2022 struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq,
2023 ib_srq);
2024 struct bnxt_re_dev *rdev = srq->rdev;
2025 struct bnxt_qplib_srq *qplib_srq = &srq->qplib_srq;
2026 int ret;
2027
2028 ret = ib_is_udata_in_empty(udata);
2029 if (ret)
2030 return ret;
2031
2032 if (rdev->chip_ctx->modes.toggle_bits & BNXT_QPLIB_SRQ_TOGGLE_BIT) {
2033 free_page((unsigned long)srq->uctx_srq_page);
2034 hash_del(&srq->hash_entry);
2035 }
2036 bnxt_qplib_destroy_srq(&rdev->qplib_res, qplib_srq);
2037 ib_umem_release(srq->umem);
2038 atomic_dec(&rdev->stats.res.srq_count);
2039 return ib_respond_empty_udata(udata);
2040 }
2041
bnxt_re_init_user_srq(struct bnxt_re_dev * rdev,struct bnxt_re_pd * pd,struct bnxt_re_srq * srq,struct ib_udata * udata)2042 static int bnxt_re_init_user_srq(struct bnxt_re_dev *rdev,
2043 struct bnxt_re_pd *pd,
2044 struct bnxt_re_srq *srq,
2045 struct ib_udata *udata)
2046 {
2047 struct bnxt_re_srq_req ureq;
2048 struct bnxt_qplib_srq *qplib_srq = &srq->qplib_srq;
2049 struct ib_umem *umem;
2050 int bytes = 0;
2051 struct bnxt_re_ucontext *cntx = rdma_udata_to_drv_context(
2052 udata, struct bnxt_re_ucontext, ib_uctx);
2053 int rc;
2054
2055 rc = ib_copy_validate_udata_in(udata, ureq, srq_handle);
2056 if (rc)
2057 return rc;
2058
2059 bytes = (qplib_srq->max_wqe * qplib_srq->wqe_size);
2060 bytes = PAGE_ALIGN(bytes);
2061 umem = ib_umem_get(&rdev->ibdev, ureq.srqva, bytes,
2062 IB_ACCESS_LOCAL_WRITE);
2063 if (IS_ERR(umem))
2064 return PTR_ERR(umem);
2065
2066 srq->umem = umem;
2067 qplib_srq->sg_info.umem = umem;
2068 qplib_srq->sg_info.pgsize = PAGE_SIZE;
2069 qplib_srq->sg_info.pgshft = PAGE_SHIFT;
2070 qplib_srq->srq_handle = ureq.srq_handle;
2071 qplib_srq->dpi = &cntx->dpi;
2072
2073 return 0;
2074 }
2075
bnxt_re_create_srq(struct ib_srq * ib_srq,struct ib_srq_init_attr * srq_init_attr,struct ib_udata * udata)2076 int bnxt_re_create_srq(struct ib_srq *ib_srq,
2077 struct ib_srq_init_attr *srq_init_attr,
2078 struct ib_udata *udata)
2079 {
2080 struct bnxt_qplib_dev_attr *dev_attr;
2081 struct bnxt_re_ucontext *uctx;
2082 struct bnxt_re_dev *rdev;
2083 struct bnxt_re_srq *srq;
2084 struct bnxt_re_pd *pd;
2085 struct ib_pd *ib_pd;
2086 u32 active_srqs;
2087 int rc;
2088
2089 ib_pd = ib_srq->pd;
2090 pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
2091 rdev = pd->rdev;
2092 dev_attr = rdev->dev_attr;
2093 srq = container_of(ib_srq, struct bnxt_re_srq, ib_srq);
2094
2095 if (srq_init_attr->attr.max_wr >= dev_attr->max_srq_wqes) {
2096 ibdev_err(&rdev->ibdev, "Create CQ failed - max exceeded");
2097 rc = -EINVAL;
2098 goto exit;
2099 }
2100
2101 if (srq_init_attr->srq_type != IB_SRQT_BASIC) {
2102 rc = -EOPNOTSUPP;
2103 goto exit;
2104 }
2105
2106 uctx = rdma_udata_to_drv_context(udata, struct bnxt_re_ucontext, ib_uctx);
2107 srq->rdev = rdev;
2108 srq->qplib_srq.pd = &pd->qplib_pd;
2109 srq->qplib_srq.dpi = &rdev->dpi_privileged;
2110 /* Allocate 1 more than what's provided so posting max doesn't
2111 * mean empty
2112 */
2113 srq->qplib_srq.max_wqe =
2114 bnxt_re_init_depth(srq_init_attr->attr.max_wr + 1,
2115 dev_attr->max_srq_wqes + 1, uctx);
2116
2117 srq->qplib_srq.max_sge = srq_init_attr->attr.max_sge;
2118 /* 128 byte wqe size for SRQ . So use max sges */
2119 srq->qplib_srq.wqe_size = bnxt_re_get_rwqe_size(dev_attr->max_srq_sges);
2120 srq->qplib_srq.threshold = srq_init_attr->attr.srq_limit;
2121 srq->srq_limit = srq_init_attr->attr.srq_limit;
2122 srq->qplib_srq.eventq_hw_ring_id = rdev->nqr->nq[0].ring_id;
2123 srq->qplib_srq.sg_info.pgsize = PAGE_SIZE;
2124 srq->qplib_srq.sg_info.pgshft = PAGE_SHIFT;
2125
2126 if (udata) {
2127 rc = bnxt_re_init_user_srq(rdev, pd, srq, udata);
2128 if (rc)
2129 goto fail;
2130 }
2131
2132 rc = bnxt_qplib_create_srq(&rdev->qplib_res, &srq->qplib_srq);
2133 if (rc) {
2134 ibdev_err(&rdev->ibdev, "Create HW SRQ failed!");
2135 goto fail;
2136 }
2137
2138 if (udata) {
2139 struct bnxt_re_srq_resp resp = {};
2140
2141 resp.srqid = srq->qplib_srq.id;
2142 if (rdev->chip_ctx->modes.toggle_bits & BNXT_QPLIB_SRQ_TOGGLE_BIT) {
2143 hash_add(rdev->srq_hash, &srq->hash_entry, srq->qplib_srq.id);
2144 srq->uctx_srq_page = (void *)get_zeroed_page(GFP_KERNEL);
2145 if (!srq->uctx_srq_page) {
2146 rc = -ENOMEM;
2147 goto fail;
2148 }
2149 resp.comp_mask |= BNXT_RE_SRQ_TOGGLE_PAGE_SUPPORT;
2150 }
2151 rc = ib_respond_udata(udata, resp);
2152 if (rc) {
2153 bnxt_qplib_destroy_srq(&rdev->qplib_res,
2154 &srq->qplib_srq);
2155 goto fail;
2156 }
2157 }
2158 active_srqs = atomic_inc_return(&rdev->stats.res.srq_count);
2159 if (active_srqs > rdev->stats.res.srq_watermark)
2160 rdev->stats.res.srq_watermark = active_srqs;
2161 spin_lock_init(&srq->lock);
2162
2163 return 0;
2164
2165 fail:
2166 ib_umem_release(srq->umem);
2167 exit:
2168 return rc;
2169 }
2170
bnxt_re_modify_srq(struct ib_srq * ib_srq,struct ib_srq_attr * srq_attr,enum ib_srq_attr_mask srq_attr_mask,struct ib_udata * udata)2171 int bnxt_re_modify_srq(struct ib_srq *ib_srq, struct ib_srq_attr *srq_attr,
2172 enum ib_srq_attr_mask srq_attr_mask,
2173 struct ib_udata *udata)
2174 {
2175 struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq,
2176 ib_srq);
2177 struct bnxt_re_dev *rdev = srq->rdev;
2178 int ret;
2179
2180 ret = ib_is_udata_in_empty(udata);
2181 if (ret)
2182 return ret;
2183
2184 switch (srq_attr_mask) {
2185 case IB_SRQ_MAX_WR:
2186 /* SRQ resize is not supported */
2187 return -EINVAL;
2188 case IB_SRQ_LIMIT:
2189 /* Change the SRQ threshold */
2190 if (srq_attr->srq_limit > srq->qplib_srq.max_wqe)
2191 return -EINVAL;
2192
2193 srq->qplib_srq.threshold = srq_attr->srq_limit;
2194 bnxt_qplib_srq_arm_db(&srq->qplib_srq.dbinfo, srq->qplib_srq.threshold);
2195
2196 /* On success, update the shadow */
2197 srq->srq_limit = srq_attr->srq_limit;
2198 /* No need to Build and send response back to udata */
2199 return ib_respond_empty_udata(udata);
2200 default:
2201 ibdev_err(&rdev->ibdev,
2202 "Unsupported srq_attr_mask 0x%x", srq_attr_mask);
2203 return -EINVAL;
2204 }
2205 }
2206
bnxt_re_query_srq(struct ib_srq * ib_srq,struct ib_srq_attr * srq_attr)2207 int bnxt_re_query_srq(struct ib_srq *ib_srq, struct ib_srq_attr *srq_attr)
2208 {
2209 struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq,
2210 ib_srq);
2211 struct bnxt_re_srq tsrq;
2212 struct bnxt_re_dev *rdev = srq->rdev;
2213 int rc;
2214
2215 /* Get live SRQ attr */
2216 tsrq.qplib_srq.id = srq->qplib_srq.id;
2217 rc = bnxt_qplib_query_srq(&rdev->qplib_res, &tsrq.qplib_srq);
2218 if (rc) {
2219 ibdev_err(&rdev->ibdev, "Query HW SRQ failed!");
2220 return rc;
2221 }
2222 srq_attr->max_wr = srq->qplib_srq.max_wqe;
2223 srq_attr->max_sge = srq->qplib_srq.max_sge;
2224 srq_attr->srq_limit = tsrq.qplib_srq.threshold;
2225
2226 return 0;
2227 }
2228
bnxt_re_post_srq_recv(struct ib_srq * ib_srq,const struct ib_recv_wr * wr,const struct ib_recv_wr ** bad_wr)2229 int bnxt_re_post_srq_recv(struct ib_srq *ib_srq, const struct ib_recv_wr *wr,
2230 const struct ib_recv_wr **bad_wr)
2231 {
2232 struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq,
2233 ib_srq);
2234 struct bnxt_qplib_swqe wqe;
2235 unsigned long flags;
2236 int rc = 0;
2237
2238 spin_lock_irqsave(&srq->lock, flags);
2239 while (wr) {
2240 /* Transcribe each ib_recv_wr to qplib_swqe */
2241 wqe.num_sge = wr->num_sge;
2242 bnxt_re_build_sgl(wr->sg_list, wqe.sg_list, wr->num_sge);
2243 wqe.wr_id = wr->wr_id;
2244 wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV;
2245
2246 rc = bnxt_qplib_post_srq_recv(&srq->qplib_srq, &wqe);
2247 if (rc) {
2248 *bad_wr = wr;
2249 break;
2250 }
2251 wr = wr->next;
2252 }
2253 spin_unlock_irqrestore(&srq->lock, flags);
2254
2255 return rc;
2256 }
bnxt_re_modify_shadow_qp(struct bnxt_re_dev * rdev,struct bnxt_re_qp * qp1_qp,int qp_attr_mask)2257 static int bnxt_re_modify_shadow_qp(struct bnxt_re_dev *rdev,
2258 struct bnxt_re_qp *qp1_qp,
2259 int qp_attr_mask)
2260 {
2261 struct bnxt_re_qp *qp = rdev->gsi_ctx.gsi_sqp;
2262 int rc;
2263
2264 if (qp_attr_mask & IB_QP_STATE) {
2265 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_STATE;
2266 qp->qplib_qp.state = qp1_qp->qplib_qp.state;
2267 }
2268 if (qp_attr_mask & IB_QP_PKEY_INDEX) {
2269 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PKEY;
2270 qp->qplib_qp.pkey_index = qp1_qp->qplib_qp.pkey_index;
2271 }
2272
2273 if (qp_attr_mask & IB_QP_QKEY) {
2274 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_QKEY;
2275 /* Using a Random QKEY */
2276 qp->qplib_qp.qkey = 0x81818181;
2277 }
2278 if (qp_attr_mask & IB_QP_SQ_PSN) {
2279 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN;
2280 qp->qplib_qp.sq.psn = qp1_qp->qplib_qp.sq.psn;
2281 }
2282
2283 rc = bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp);
2284 if (rc)
2285 ibdev_err(&rdev->ibdev, "Failed to modify Shadow QP for QP1");
2286 return rc;
2287 }
2288
bnxt_re_modify_qp(struct ib_qp * ib_qp,struct ib_qp_attr * qp_attr,int qp_attr_mask,struct ib_udata * udata)2289 int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,
2290 int qp_attr_mask, struct ib_udata *udata)
2291 {
2292 struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
2293 struct bnxt_re_dev *rdev = qp->rdev;
2294 struct bnxt_qplib_dev_attr *dev_attr = rdev->dev_attr;
2295 enum ib_qp_state curr_qp_state, new_qp_state;
2296 int rc;
2297 unsigned int flags;
2298 u8 nw_type;
2299
2300 rc = ib_is_udata_in_empty(udata);
2301 if (rc)
2302 return rc;
2303
2304 if (qp_attr_mask & ~(IB_QP_ATTR_STANDARD_BITS | IB_QP_RATE_LIMIT))
2305 return -EOPNOTSUPP;
2306
2307 qp->qplib_qp.modify_flags = 0;
2308 qp->qplib_qp.ext_modify_flags = 0;
2309 if (qp_attr_mask & IB_QP_STATE) {
2310 curr_qp_state = __to_ib_qp_state(qp->qplib_qp.cur_qp_state);
2311 new_qp_state = qp_attr->qp_state;
2312 if (!ib_modify_qp_is_ok(curr_qp_state, new_qp_state,
2313 ib_qp->qp_type, qp_attr_mask)) {
2314 ibdev_err(&rdev->ibdev,
2315 "Invalid attribute mask: %#x specified ",
2316 qp_attr_mask);
2317 ibdev_err(&rdev->ibdev,
2318 "for qpn: %#x type: %#x",
2319 ib_qp->qp_num, ib_qp->qp_type);
2320 ibdev_err(&rdev->ibdev,
2321 "curr_qp_state=0x%x, new_qp_state=0x%x\n",
2322 curr_qp_state, new_qp_state);
2323 return -EINVAL;
2324 }
2325 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_STATE;
2326 qp->qplib_qp.state = __from_ib_qp_state(qp_attr->qp_state);
2327
2328 if (!qp->sumem &&
2329 qp->qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_ERR) {
2330 ibdev_dbg(&rdev->ibdev,
2331 "Move QP = %p to flush list\n", qp);
2332 flags = bnxt_re_lock_cqs(qp);
2333 bnxt_qplib_add_flush_qp(&qp->qplib_qp);
2334 bnxt_re_unlock_cqs(qp, flags);
2335 }
2336 if (!qp->sumem &&
2337 qp->qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_RESET) {
2338 ibdev_dbg(&rdev->ibdev,
2339 "Move QP = %p out of flush list\n", qp);
2340 flags = bnxt_re_lock_cqs(qp);
2341 bnxt_qplib_clean_qp(&qp->qplib_qp);
2342 bnxt_re_unlock_cqs(qp, flags);
2343 }
2344 }
2345
2346 if (qp_attr_mask & IB_QP_RATE_LIMIT) {
2347 if (qp->qplib_qp.type != IB_QPT_RC ||
2348 !_is_modify_qp_rate_limit_supported(dev_attr->dev_cap_flags2))
2349 return -EOPNOTSUPP;
2350 qp->qplib_qp.ext_modify_flags |=
2351 CMDQ_MODIFY_QP_EXT_MODIFY_MASK_RATE_LIMIT_VALID;
2352 qp->qplib_qp.rate_limit = qp_attr->rate_limit;
2353 }
2354 if (qp_attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY) {
2355 qp->qplib_qp.modify_flags |=
2356 CMDQ_MODIFY_QP_MODIFY_MASK_EN_SQD_ASYNC_NOTIFY;
2357 qp->qplib_qp.en_sqd_async_notify = true;
2358 }
2359 if (qp_attr_mask & IB_QP_ACCESS_FLAGS) {
2360 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS;
2361 qp->qplib_qp.access =
2362 __qp_access_flags_from_ib(qp->qplib_qp.cctx,
2363 qp_attr->qp_access_flags);
2364 /* LOCAL_WRITE access must be set to allow RC receive */
2365 qp->qplib_qp.access |= CMDQ_MODIFY_QP_ACCESS_LOCAL_WRITE;
2366 }
2367 if (qp_attr_mask & IB_QP_PKEY_INDEX) {
2368 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PKEY;
2369 qp->qplib_qp.pkey_index = qp_attr->pkey_index;
2370 }
2371 if (qp_attr_mask & IB_QP_QKEY) {
2372 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_QKEY;
2373 qp->qplib_qp.qkey = qp_attr->qkey;
2374 }
2375 if (qp_attr_mask & IB_QP_AV) {
2376 const struct ib_global_route *grh =
2377 rdma_ah_read_grh(&qp_attr->ah_attr);
2378 const struct ib_gid_attr *sgid_attr;
2379 struct bnxt_re_gid_ctx *ctx;
2380
2381 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_DGID |
2382 CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL |
2383 CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX |
2384 CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT |
2385 CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS |
2386 CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC |
2387 CMDQ_MODIFY_QP_MODIFY_MASK_VLAN_ID;
2388 memcpy(qp->qplib_qp.ah.dgid.data, grh->dgid.raw,
2389 sizeof(qp->qplib_qp.ah.dgid.data));
2390 qp->qplib_qp.ah.flow_label = grh->flow_label;
2391 sgid_attr = grh->sgid_attr;
2392 /* Get the HW context of the GID. The reference
2393 * of GID table entry is already taken by the caller.
2394 */
2395 ctx = rdma_read_gid_hw_context(sgid_attr);
2396 qp->qplib_qp.ah.sgid_index = ctx->idx;
2397 qp->qplib_qp.ah.host_sgid_index = grh->sgid_index;
2398 qp->qplib_qp.ah.hop_limit = grh->hop_limit;
2399 qp->qplib_qp.ah.traffic_class = grh->traffic_class >> 2;
2400 qp->qplib_qp.ah.sl = rdma_ah_get_sl(&qp_attr->ah_attr);
2401 ether_addr_copy(qp->qplib_qp.ah.dmac,
2402 qp_attr->ah_attr.roce.dmac);
2403
2404 rc = rdma_read_gid_l2_fields(sgid_attr, NULL,
2405 &qp->qplib_qp.smac[0]);
2406 if (rc)
2407 return rc;
2408
2409 nw_type = rdma_gid_attr_network_type(sgid_attr);
2410 switch (nw_type) {
2411 case RDMA_NETWORK_IPV4:
2412 qp->qplib_qp.nw_type =
2413 CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV4;
2414 break;
2415 case RDMA_NETWORK_IPV6:
2416 qp->qplib_qp.nw_type =
2417 CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV6;
2418 break;
2419 default:
2420 qp->qplib_qp.nw_type =
2421 CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV1;
2422 break;
2423 }
2424 }
2425
2426 if (qp_attr->qp_state == IB_QPS_RTR) {
2427 enum ib_mtu qpmtu;
2428
2429 qpmtu = iboe_get_mtu(rdev->netdev->mtu);
2430 if (qp_attr_mask & IB_QP_PATH_MTU) {
2431 if (ib_mtu_enum_to_int(qp_attr->path_mtu) >
2432 ib_mtu_enum_to_int(qpmtu))
2433 return -EINVAL;
2434 qpmtu = qp_attr->path_mtu;
2435 }
2436
2437 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU;
2438 qp->qplib_qp.path_mtu = __from_ib_mtu(qpmtu);
2439 qp->qplib_qp.mtu = ib_mtu_enum_to_int(qpmtu);
2440 }
2441
2442 if (qp_attr_mask & IB_QP_TIMEOUT) {
2443 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT;
2444 qp->qplib_qp.timeout = qp_attr->timeout;
2445 }
2446 if (qp_attr_mask & IB_QP_RETRY_CNT) {
2447 qp->qplib_qp.modify_flags |=
2448 CMDQ_MODIFY_QP_MODIFY_MASK_RETRY_CNT;
2449 qp->qplib_qp.retry_cnt = qp_attr->retry_cnt;
2450 }
2451 if (qp_attr_mask & IB_QP_RNR_RETRY) {
2452 qp->qplib_qp.modify_flags |=
2453 CMDQ_MODIFY_QP_MODIFY_MASK_RNR_RETRY;
2454 qp->qplib_qp.rnr_retry = qp_attr->rnr_retry;
2455 }
2456 if (qp_attr_mask & IB_QP_MIN_RNR_TIMER) {
2457 qp->qplib_qp.modify_flags |=
2458 CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER;
2459 qp->qplib_qp.min_rnr_timer = qp_attr->min_rnr_timer;
2460 }
2461 if (qp_attr_mask & IB_QP_RQ_PSN) {
2462 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN;
2463 qp->qplib_qp.rq.psn = qp_attr->rq_psn;
2464 }
2465 if (qp_attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
2466 qp->qplib_qp.modify_flags |=
2467 CMDQ_MODIFY_QP_MODIFY_MASK_MAX_RD_ATOMIC;
2468 /* Cap the max_rd_atomic to device max */
2469 qp->qplib_qp.max_rd_atomic = min_t(u32, qp_attr->max_rd_atomic,
2470 dev_attr->max_qp_rd_atom);
2471 }
2472 if (qp_attr_mask & IB_QP_SQ_PSN) {
2473 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN;
2474 qp->qplib_qp.sq.psn = qp_attr->sq_psn;
2475 }
2476 if (qp_attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
2477 if (qp_attr->max_dest_rd_atomic >
2478 dev_attr->max_qp_init_rd_atom) {
2479 ibdev_err(&rdev->ibdev,
2480 "max_dest_rd_atomic requested%d is > dev_max%d",
2481 qp_attr->max_dest_rd_atomic,
2482 dev_attr->max_qp_init_rd_atom);
2483 return -EINVAL;
2484 }
2485
2486 qp->qplib_qp.modify_flags |=
2487 CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC;
2488 qp->qplib_qp.max_dest_rd_atomic = qp_attr->max_dest_rd_atomic;
2489 }
2490 if (qp_attr_mask & IB_QP_CAP) {
2491 struct bnxt_re_ucontext *uctx =
2492 rdma_udata_to_drv_context(udata, struct bnxt_re_ucontext, ib_uctx);
2493
2494 qp->qplib_qp.modify_flags |=
2495 CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SIZE |
2496 CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SIZE |
2497 CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SGE |
2498 CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SGE |
2499 CMDQ_MODIFY_QP_MODIFY_MASK_MAX_INLINE_DATA;
2500 if ((qp_attr->cap.max_send_wr >= dev_attr->max_qp_wqes) ||
2501 (qp_attr->cap.max_recv_wr >= dev_attr->max_qp_wqes) ||
2502 (qp_attr->cap.max_send_sge >= dev_attr->max_qp_sges) ||
2503 (qp_attr->cap.max_recv_sge >= dev_attr->max_qp_sges) ||
2504 (qp_attr->cap.max_inline_data >=
2505 dev_attr->max_inline_data)) {
2506 ibdev_err(&rdev->ibdev,
2507 "Create QP failed - max exceeded");
2508 return -EINVAL;
2509 }
2510 qp->qplib_qp.sq.max_wqe =
2511 bnxt_re_init_depth(qp_attr->cap.max_send_wr,
2512 dev_attr->max_qp_wqes + 1, uctx);
2513 qp->qplib_qp.sq.q_full_delta = qp->qplib_qp.sq.max_wqe -
2514 qp_attr->cap.max_send_wr;
2515 /*
2516 * Reserving one slot for Phantom WQE. Some application can
2517 * post one extra entry in this case. Allowing this to avoid
2518 * unexpected Queue full condition
2519 */
2520 qp->qplib_qp.sq.q_full_delta -= 1;
2521 qp->qplib_qp.sq.max_sge = qp_attr->cap.max_send_sge;
2522 if (qp->qplib_qp.rq.max_wqe) {
2523 qp->qplib_qp.rq.max_wqe = bnxt_re_init_depth(
2524 qp_attr->cap.max_recv_wr,
2525 dev_attr->max_qp_wqes + 1, uctx);
2526 qp->qplib_qp.rq.max_sw_wqe = qp->qplib_qp.rq.max_wqe;
2527 qp->qplib_qp.rq.q_full_delta = qp->qplib_qp.rq.max_wqe -
2528 qp_attr->cap.max_recv_wr;
2529 qp->qplib_qp.rq.max_sge = qp_attr->cap.max_recv_sge;
2530 } else {
2531 /* SRQ was used prior, just ignore the RQ caps */
2532 }
2533 }
2534 if (qp_attr_mask & IB_QP_DEST_QPN) {
2535 qp->qplib_qp.modify_flags |=
2536 CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID;
2537 qp->qplib_qp.dest_qpn = qp_attr->dest_qp_num;
2538 }
2539 rc = bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp);
2540 if (rc) {
2541 ibdev_err(&rdev->ibdev, "Failed to modify HW QP");
2542 return rc;
2543 }
2544 if (ib_qp->qp_type == IB_QPT_GSI && rdev->gsi_ctx.gsi_sqp) {
2545 rc = bnxt_re_modify_shadow_qp(rdev, qp, qp_attr_mask);
2546 if (rc)
2547 return rc;
2548 }
2549 return ib_respond_empty_udata(udata);
2550 }
2551
bnxt_re_query_qp(struct ib_qp * ib_qp,struct ib_qp_attr * qp_attr,int qp_attr_mask,struct ib_qp_init_attr * qp_init_attr)2552 int bnxt_re_query_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,
2553 int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr)
2554 {
2555 struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
2556 struct bnxt_re_dev *rdev = qp->rdev;
2557 struct bnxt_qplib_qp *qplib_qp;
2558 int rc;
2559
2560 qplib_qp = kzalloc_obj(*qplib_qp);
2561 if (!qplib_qp)
2562 return -ENOMEM;
2563
2564 qplib_qp->id = qp->qplib_qp.id;
2565 qplib_qp->ah.host_sgid_index = qp->qplib_qp.ah.host_sgid_index;
2566
2567 rc = bnxt_qplib_query_qp(&rdev->qplib_res, qplib_qp);
2568 if (rc) {
2569 ibdev_err(&rdev->ibdev, "Failed to query HW QP");
2570 goto out;
2571 }
2572 qp_attr->qp_state = __to_ib_qp_state(qplib_qp->state);
2573 qp_attr->cur_qp_state = __to_ib_qp_state(qplib_qp->cur_qp_state);
2574 qp_attr->en_sqd_async_notify = qplib_qp->en_sqd_async_notify ? 1 : 0;
2575 qp_attr->qp_access_flags = __qp_access_flags_to_ib(qp->qplib_qp.cctx,
2576 qplib_qp->access);
2577 qp_attr->pkey_index = qplib_qp->pkey_index;
2578 qp_attr->qkey = qplib_qp->qkey;
2579 qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;
2580 rdma_ah_set_grh(&qp_attr->ah_attr, NULL, qplib_qp->udp_sport,
2581 qplib_qp->ah.host_sgid_index,
2582 qplib_qp->ah.hop_limit,
2583 qplib_qp->ah.traffic_class);
2584 rdma_ah_set_dgid_raw(&qp_attr->ah_attr, qplib_qp->ah.dgid.data);
2585 rdma_ah_set_sl(&qp_attr->ah_attr, qplib_qp->ah.sl);
2586 ether_addr_copy(qp_attr->ah_attr.roce.dmac, qplib_qp->ah.dmac);
2587 qp_attr->path_mtu = __to_ib_mtu(qplib_qp->path_mtu);
2588 qp_attr->timeout = qplib_qp->timeout;
2589 qp_attr->retry_cnt = qplib_qp->retry_cnt;
2590 qp_attr->rnr_retry = qplib_qp->rnr_retry;
2591 qp_attr->min_rnr_timer = qplib_qp->min_rnr_timer;
2592 qp_attr->port_num = __to_ib_port_num(qplib_qp->port_id);
2593 qp_attr->rq_psn = qplib_qp->rq.psn;
2594 qp_attr->max_rd_atomic = qplib_qp->max_rd_atomic;
2595 qp_attr->sq_psn = qplib_qp->sq.psn;
2596 qp_attr->max_dest_rd_atomic = qplib_qp->max_dest_rd_atomic;
2597 qp_init_attr->sq_sig_type = qplib_qp->sig_type ? IB_SIGNAL_ALL_WR :
2598 IB_SIGNAL_REQ_WR;
2599 qp_attr->dest_qp_num = qplib_qp->dest_qpn;
2600
2601 qp_attr->cap.max_send_wr = qp->qplib_qp.sq.max_wqe;
2602 qp_attr->cap.max_send_sge = qp->qplib_qp.sq.max_sge;
2603 qp_attr->cap.max_recv_wr = qp->qplib_qp.rq.max_wqe;
2604 qp_attr->cap.max_recv_sge = qp->qplib_qp.rq.max_sge;
2605 qp_attr->cap.max_inline_data = qp->qplib_qp.max_inline_data;
2606 qp_init_attr->cap = qp_attr->cap;
2607
2608 out:
2609 kfree(qplib_qp);
2610 return rc;
2611 }
2612
2613 /* Routine for sending QP1 packets for RoCE V1 an V2
2614 */
bnxt_re_build_qp1_send_v2(struct bnxt_re_qp * qp,const struct ib_send_wr * wr,struct bnxt_qplib_swqe * wqe,int payload_size)2615 static int bnxt_re_build_qp1_send_v2(struct bnxt_re_qp *qp,
2616 const struct ib_send_wr *wr,
2617 struct bnxt_qplib_swqe *wqe,
2618 int payload_size)
2619 {
2620 struct bnxt_re_ah *ah = container_of(ud_wr(wr)->ah, struct bnxt_re_ah,
2621 ib_ah);
2622 struct bnxt_qplib_ah *qplib_ah = &ah->qplib_ah;
2623 const struct ib_gid_attr *sgid_attr = ah->ib_ah.sgid_attr;
2624 struct bnxt_qplib_sge sge;
2625 u8 nw_type;
2626 u16 ether_type;
2627 union ib_gid dgid;
2628 bool is_eth = false;
2629 bool is_vlan = false;
2630 bool is_grh = false;
2631 bool is_udp = false;
2632 u8 ip_version = 0;
2633 u16 vlan_id = 0xFFFF;
2634 void *buf;
2635 int i, rc;
2636
2637 memset(&qp->qp1_hdr, 0, sizeof(qp->qp1_hdr));
2638
2639 rc = rdma_read_gid_l2_fields(sgid_attr, &vlan_id, NULL);
2640 if (rc)
2641 return rc;
2642
2643 /* Get network header type for this GID */
2644 nw_type = rdma_gid_attr_network_type(sgid_attr);
2645 switch (nw_type) {
2646 case RDMA_NETWORK_IPV4:
2647 nw_type = BNXT_RE_ROCEV2_IPV4_PACKET;
2648 break;
2649 case RDMA_NETWORK_IPV6:
2650 nw_type = BNXT_RE_ROCEV2_IPV6_PACKET;
2651 break;
2652 default:
2653 nw_type = BNXT_RE_ROCE_V1_PACKET;
2654 break;
2655 }
2656 memcpy(&dgid.raw, &qplib_ah->dgid, 16);
2657 is_udp = sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP;
2658 if (is_udp) {
2659 if (ipv6_addr_v4mapped((struct in6_addr *)&sgid_attr->gid)) {
2660 ip_version = 4;
2661 ether_type = ETH_P_IP;
2662 } else {
2663 ip_version = 6;
2664 ether_type = ETH_P_IPV6;
2665 }
2666 is_grh = false;
2667 } else {
2668 ether_type = ETH_P_IBOE;
2669 is_grh = true;
2670 }
2671
2672 is_eth = true;
2673 is_vlan = vlan_id && (vlan_id < 0x1000);
2674
2675 ib_ud_header_init(payload_size, !is_eth, is_eth, is_vlan, is_grh,
2676 ip_version, is_udp, 0, &qp->qp1_hdr);
2677
2678 /* ETH */
2679 ether_addr_copy(qp->qp1_hdr.eth.dmac_h, ah->qplib_ah.dmac);
2680 ether_addr_copy(qp->qp1_hdr.eth.smac_h, qp->qplib_qp.smac);
2681
2682 /* For vlan, check the sgid for vlan existence */
2683
2684 if (!is_vlan) {
2685 qp->qp1_hdr.eth.type = cpu_to_be16(ether_type);
2686 } else {
2687 qp->qp1_hdr.vlan.type = cpu_to_be16(ether_type);
2688 qp->qp1_hdr.vlan.tag = cpu_to_be16(vlan_id);
2689 }
2690
2691 if (is_grh || (ip_version == 6)) {
2692 memcpy(qp->qp1_hdr.grh.source_gid.raw, sgid_attr->gid.raw,
2693 sizeof(sgid_attr->gid));
2694 memcpy(qp->qp1_hdr.grh.destination_gid.raw, qplib_ah->dgid.data,
2695 sizeof(sgid_attr->gid));
2696 qp->qp1_hdr.grh.hop_limit = qplib_ah->hop_limit;
2697 }
2698
2699 if (ip_version == 4) {
2700 qp->qp1_hdr.ip4.tos = 0;
2701 qp->qp1_hdr.ip4.id = 0;
2702 qp->qp1_hdr.ip4.frag_off = htons(IP_DF);
2703 qp->qp1_hdr.ip4.ttl = qplib_ah->hop_limit;
2704
2705 memcpy(&qp->qp1_hdr.ip4.saddr, sgid_attr->gid.raw + 12, 4);
2706 memcpy(&qp->qp1_hdr.ip4.daddr, qplib_ah->dgid.data + 12, 4);
2707 qp->qp1_hdr.ip4.check = ib_ud_ip4_csum(&qp->qp1_hdr);
2708 }
2709
2710 if (is_udp) {
2711 qp->qp1_hdr.udp.dport = htons(ROCE_V2_UDP_DPORT);
2712 qp->qp1_hdr.udp.sport = htons(0x8CD1);
2713 qp->qp1_hdr.udp.csum = 0;
2714 }
2715
2716 /* BTH */
2717 if (wr->opcode == IB_WR_SEND_WITH_IMM) {
2718 qp->qp1_hdr.bth.opcode = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE;
2719 qp->qp1_hdr.immediate_present = 1;
2720 } else {
2721 qp->qp1_hdr.bth.opcode = IB_OPCODE_UD_SEND_ONLY;
2722 }
2723 if (wr->send_flags & IB_SEND_SOLICITED)
2724 qp->qp1_hdr.bth.solicited_event = 1;
2725 /* pad_count */
2726 qp->qp1_hdr.bth.pad_count = (4 - payload_size) & 3;
2727
2728 /* P_key for QP1 is for all members */
2729 qp->qp1_hdr.bth.pkey = cpu_to_be16(0xFFFF);
2730 qp->qp1_hdr.bth.destination_qpn = IB_QP1;
2731 qp->qp1_hdr.bth.ack_req = 0;
2732 qp->send_psn++;
2733 qp->send_psn &= BTH_PSN_MASK;
2734 qp->qp1_hdr.bth.psn = cpu_to_be32(qp->send_psn);
2735 /* DETH */
2736 /* Use the priviledged Q_Key for QP1 */
2737 qp->qp1_hdr.deth.qkey = cpu_to_be32(IB_QP1_QKEY);
2738 qp->qp1_hdr.deth.source_qpn = IB_QP1;
2739
2740 /* Pack the QP1 to the transmit buffer */
2741 buf = bnxt_qplib_get_qp1_sq_buf(&qp->qplib_qp, &sge);
2742 if (buf) {
2743 ib_ud_header_pack(&qp->qp1_hdr, buf);
2744 for (i = wqe->num_sge; i; i--) {
2745 wqe->sg_list[i].addr = wqe->sg_list[i - 1].addr;
2746 wqe->sg_list[i].lkey = wqe->sg_list[i - 1].lkey;
2747 wqe->sg_list[i].size = wqe->sg_list[i - 1].size;
2748 }
2749
2750 /*
2751 * Max Header buf size for IPV6 RoCE V2 is 86,
2752 * which is same as the QP1 SQ header buffer.
2753 * Header buf size for IPV4 RoCE V2 can be 66.
2754 * ETH(14) + VLAN(4)+ IP(20) + UDP (8) + BTH(20).
2755 * Subtract 20 bytes from QP1 SQ header buf size
2756 */
2757 if (is_udp && ip_version == 4)
2758 sge.size -= 20;
2759 /*
2760 * Max Header buf size for RoCE V1 is 78.
2761 * ETH(14) + VLAN(4) + GRH(40) + BTH(20).
2762 * Subtract 8 bytes from QP1 SQ header buf size
2763 */
2764 if (!is_udp)
2765 sge.size -= 8;
2766
2767 /* Subtract 4 bytes for non vlan packets */
2768 if (!is_vlan)
2769 sge.size -= 4;
2770
2771 wqe->sg_list[0].addr = sge.addr;
2772 wqe->sg_list[0].lkey = sge.lkey;
2773 wqe->sg_list[0].size = sge.size;
2774 wqe->num_sge++;
2775
2776 } else {
2777 ibdev_err(&qp->rdev->ibdev, "QP1 buffer is empty!");
2778 rc = -ENOMEM;
2779 }
2780 return rc;
2781 }
2782
2783 /* For the MAD layer, it only provides the recv SGE the size of
2784 * ib_grh + MAD datagram. No Ethernet headers, Ethertype, BTH, DETH,
2785 * nor RoCE iCRC. The Cu+ solution must provide buffer for the entire
2786 * receive packet (334 bytes) with no VLAN and then copy the GRH
2787 * and the MAD datagram out to the provided SGE.
2788 */
bnxt_re_build_qp1_shadow_qp_recv(struct bnxt_re_qp * qp,const struct ib_recv_wr * wr,struct bnxt_qplib_swqe * wqe,int payload_size)2789 static int bnxt_re_build_qp1_shadow_qp_recv(struct bnxt_re_qp *qp,
2790 const struct ib_recv_wr *wr,
2791 struct bnxt_qplib_swqe *wqe,
2792 int payload_size)
2793 {
2794 struct bnxt_re_sqp_entries *sqp_entry;
2795 struct bnxt_qplib_sge ref, sge;
2796 struct bnxt_re_dev *rdev;
2797 u32 rq_prod_index;
2798
2799 rdev = qp->rdev;
2800
2801 rq_prod_index = bnxt_qplib_get_rq_prod_index(&qp->qplib_qp);
2802
2803 if (!bnxt_qplib_get_qp1_rq_buf(&qp->qplib_qp, &sge))
2804 return -ENOMEM;
2805
2806 /* Create 1 SGE to receive the entire
2807 * ethernet packet
2808 */
2809 /* Save the reference from ULP */
2810 ref.addr = wqe->sg_list[0].addr;
2811 ref.lkey = wqe->sg_list[0].lkey;
2812 ref.size = wqe->sg_list[0].size;
2813
2814 sqp_entry = &rdev->gsi_ctx.sqp_tbl[rq_prod_index];
2815
2816 /* SGE 1 */
2817 wqe->sg_list[0].addr = sge.addr;
2818 wqe->sg_list[0].lkey = sge.lkey;
2819 wqe->sg_list[0].size = BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2;
2820 sge.size -= wqe->sg_list[0].size;
2821
2822 sqp_entry->sge.addr = ref.addr;
2823 sqp_entry->sge.lkey = ref.lkey;
2824 sqp_entry->sge.size = ref.size;
2825 /* Store the wrid for reporting completion */
2826 sqp_entry->wrid = wqe->wr_id;
2827 /* change the wqe->wrid to table index */
2828 wqe->wr_id = rq_prod_index;
2829 return 0;
2830 }
2831
is_ud_qp(struct bnxt_re_qp * qp)2832 static int is_ud_qp(struct bnxt_re_qp *qp)
2833 {
2834 return (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_UD ||
2835 qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_GSI);
2836 }
2837
bnxt_re_build_send_wqe(struct bnxt_re_qp * qp,const struct ib_send_wr * wr,struct bnxt_qplib_swqe * wqe)2838 static int bnxt_re_build_send_wqe(struct bnxt_re_qp *qp,
2839 const struct ib_send_wr *wr,
2840 struct bnxt_qplib_swqe *wqe)
2841 {
2842 struct bnxt_re_ah *ah = NULL;
2843
2844 if (is_ud_qp(qp)) {
2845 ah = container_of(ud_wr(wr)->ah, struct bnxt_re_ah, ib_ah);
2846 wqe->send.q_key = ud_wr(wr)->remote_qkey;
2847 wqe->send.dst_qp = ud_wr(wr)->remote_qpn;
2848 wqe->send.avid = ah->qplib_ah.id;
2849 }
2850 switch (wr->opcode) {
2851 case IB_WR_SEND:
2852 wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND;
2853 break;
2854 case IB_WR_SEND_WITH_IMM:
2855 wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM;
2856 wqe->send.imm_data = be32_to_cpu(wr->ex.imm_data);
2857 break;
2858 case IB_WR_SEND_WITH_INV:
2859 wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV;
2860 wqe->send.inv_key = wr->ex.invalidate_rkey;
2861 break;
2862 default:
2863 return -EINVAL;
2864 }
2865 if (wr->send_flags & IB_SEND_SIGNALED)
2866 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
2867 if (wr->send_flags & IB_SEND_FENCE)
2868 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
2869 if (wr->send_flags & IB_SEND_SOLICITED)
2870 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT;
2871 if (wr->send_flags & IB_SEND_INLINE)
2872 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_INLINE;
2873
2874 return 0;
2875 }
2876
bnxt_re_build_rdma_wqe(const struct ib_send_wr * wr,struct bnxt_qplib_swqe * wqe)2877 static int bnxt_re_build_rdma_wqe(const struct ib_send_wr *wr,
2878 struct bnxt_qplib_swqe *wqe)
2879 {
2880 switch (wr->opcode) {
2881 case IB_WR_RDMA_WRITE:
2882 wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE;
2883 break;
2884 case IB_WR_RDMA_WRITE_WITH_IMM:
2885 wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM;
2886 wqe->rdma.imm_data = be32_to_cpu(wr->ex.imm_data);
2887 break;
2888 case IB_WR_RDMA_READ:
2889 wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_READ;
2890 wqe->rdma.inv_key = wr->ex.invalidate_rkey;
2891 break;
2892 default:
2893 return -EINVAL;
2894 }
2895 wqe->rdma.remote_va = rdma_wr(wr)->remote_addr;
2896 wqe->rdma.r_key = rdma_wr(wr)->rkey;
2897 if (wr->send_flags & IB_SEND_SIGNALED)
2898 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
2899 if (wr->send_flags & IB_SEND_FENCE)
2900 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
2901 if (wr->send_flags & IB_SEND_SOLICITED)
2902 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT;
2903 if (wr->send_flags & IB_SEND_INLINE)
2904 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_INLINE;
2905
2906 return 0;
2907 }
2908
bnxt_re_build_atomic_wqe(const struct ib_send_wr * wr,struct bnxt_qplib_swqe * wqe)2909 static int bnxt_re_build_atomic_wqe(const struct ib_send_wr *wr,
2910 struct bnxt_qplib_swqe *wqe)
2911 {
2912 switch (wr->opcode) {
2913 case IB_WR_ATOMIC_CMP_AND_SWP:
2914 wqe->type = BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP;
2915 wqe->atomic.cmp_data = atomic_wr(wr)->compare_add;
2916 wqe->atomic.swap_data = atomic_wr(wr)->swap;
2917 break;
2918 case IB_WR_ATOMIC_FETCH_AND_ADD:
2919 wqe->type = BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD;
2920 wqe->atomic.cmp_data = atomic_wr(wr)->compare_add;
2921 break;
2922 default:
2923 return -EINVAL;
2924 }
2925 wqe->atomic.remote_va = atomic_wr(wr)->remote_addr;
2926 wqe->atomic.r_key = atomic_wr(wr)->rkey;
2927 if (wr->send_flags & IB_SEND_SIGNALED)
2928 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
2929 if (wr->send_flags & IB_SEND_FENCE)
2930 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
2931 if (wr->send_flags & IB_SEND_SOLICITED)
2932 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT;
2933 return 0;
2934 }
2935
bnxt_re_build_inv_wqe(const struct ib_send_wr * wr,struct bnxt_qplib_swqe * wqe)2936 static int bnxt_re_build_inv_wqe(const struct ib_send_wr *wr,
2937 struct bnxt_qplib_swqe *wqe)
2938 {
2939 wqe->type = BNXT_QPLIB_SWQE_TYPE_LOCAL_INV;
2940 wqe->local_inv.inv_l_key = wr->ex.invalidate_rkey;
2941
2942 if (wr->send_flags & IB_SEND_SIGNALED)
2943 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
2944 if (wr->send_flags & IB_SEND_SOLICITED)
2945 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT;
2946
2947 return 0;
2948 }
2949
bnxt_re_build_reg_wqe(const struct ib_reg_wr * wr,struct bnxt_qplib_swqe * wqe)2950 static int bnxt_re_build_reg_wqe(const struct ib_reg_wr *wr,
2951 struct bnxt_qplib_swqe *wqe)
2952 {
2953 struct bnxt_re_mr *mr = container_of(wr->mr, struct bnxt_re_mr, ib_mr);
2954 struct bnxt_qplib_frpl *qplib_frpl = &mr->qplib_frpl;
2955 int access = wr->access;
2956
2957 wqe->frmr.pbl_ptr = (__le64 *)qplib_frpl->hwq.pbl_ptr[0];
2958 wqe->frmr.pbl_dma_ptr = qplib_frpl->hwq.pbl_dma_ptr[0];
2959 wqe->frmr.page_list = mr->pages;
2960 wqe->frmr.page_list_len = mr->npages;
2961 wqe->frmr.levels = qplib_frpl->hwq.level;
2962 wqe->type = BNXT_QPLIB_SWQE_TYPE_REG_MR;
2963
2964 if (wr->wr.send_flags & IB_SEND_SIGNALED)
2965 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
2966
2967 if (access & IB_ACCESS_LOCAL_WRITE)
2968 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_LOCAL_WRITE;
2969 if (access & IB_ACCESS_REMOTE_READ)
2970 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_READ;
2971 if (access & IB_ACCESS_REMOTE_WRITE)
2972 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_WRITE;
2973 if (access & IB_ACCESS_REMOTE_ATOMIC)
2974 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_ATOMIC;
2975 if (access & IB_ACCESS_MW_BIND)
2976 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_WINDOW_BIND;
2977
2978 wqe->frmr.l_key = wr->key;
2979 wqe->frmr.length = wr->mr->length;
2980 wqe->frmr.pbl_pg_sz_log = ilog2(PAGE_SIZE >> PAGE_SHIFT_4K);
2981 wqe->frmr.pg_sz_log = ilog2(wr->mr->page_size >> PAGE_SHIFT_4K);
2982 wqe->frmr.va = wr->mr->iova;
2983 return 0;
2984 }
2985
bnxt_re_copy_inline_data(struct bnxt_re_dev * rdev,const struct ib_send_wr * wr,struct bnxt_qplib_swqe * wqe)2986 static int bnxt_re_copy_inline_data(struct bnxt_re_dev *rdev,
2987 const struct ib_send_wr *wr,
2988 struct bnxt_qplib_swqe *wqe)
2989 {
2990 /* Copy the inline data to the data field */
2991 u8 *in_data;
2992 u32 i, sge_len;
2993 void *sge_addr;
2994
2995 in_data = wqe->inline_data;
2996 for (i = 0; i < wr->num_sge; i++) {
2997 sge_addr = (void *)(unsigned long)
2998 wr->sg_list[i].addr;
2999 sge_len = wr->sg_list[i].length;
3000
3001 if ((sge_len + wqe->inline_len) >
3002 BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH) {
3003 ibdev_err(&rdev->ibdev,
3004 "Inline data size requested > supported value");
3005 return -EINVAL;
3006 }
3007 sge_len = wr->sg_list[i].length;
3008
3009 memcpy(in_data, sge_addr, sge_len);
3010 in_data += wr->sg_list[i].length;
3011 wqe->inline_len += wr->sg_list[i].length;
3012 }
3013 return wqe->inline_len;
3014 }
3015
bnxt_re_copy_wr_payload(struct bnxt_re_dev * rdev,const struct ib_send_wr * wr,struct bnxt_qplib_swqe * wqe)3016 static int bnxt_re_copy_wr_payload(struct bnxt_re_dev *rdev,
3017 const struct ib_send_wr *wr,
3018 struct bnxt_qplib_swqe *wqe)
3019 {
3020 int payload_sz = 0;
3021
3022 if (wr->send_flags & IB_SEND_INLINE)
3023 payload_sz = bnxt_re_copy_inline_data(rdev, wr, wqe);
3024 else
3025 payload_sz = bnxt_re_build_sgl(wr->sg_list, wqe->sg_list,
3026 wqe->num_sge);
3027
3028 return payload_sz;
3029 }
3030
bnxt_ud_qp_hw_stall_workaround(struct bnxt_re_qp * qp)3031 static void bnxt_ud_qp_hw_stall_workaround(struct bnxt_re_qp *qp)
3032 {
3033 if ((qp->ib_qp.qp_type == IB_QPT_UD ||
3034 qp->ib_qp.qp_type == IB_QPT_GSI ||
3035 qp->ib_qp.qp_type == IB_QPT_RAW_ETHERTYPE) &&
3036 qp->qplib_qp.wqe_cnt == BNXT_RE_UD_QP_HW_STALL) {
3037 int qp_attr_mask;
3038 struct ib_qp_attr qp_attr;
3039
3040 qp_attr_mask = IB_QP_STATE;
3041 qp_attr.qp_state = IB_QPS_RTS;
3042 bnxt_re_modify_qp(&qp->ib_qp, &qp_attr, qp_attr_mask, NULL);
3043 qp->qplib_qp.wqe_cnt = 0;
3044 }
3045 }
3046
bnxt_re_post_send_shadow_qp(struct bnxt_re_dev * rdev,struct bnxt_re_qp * qp,const struct ib_send_wr * wr)3047 static int bnxt_re_post_send_shadow_qp(struct bnxt_re_dev *rdev,
3048 struct bnxt_re_qp *qp,
3049 const struct ib_send_wr *wr)
3050 {
3051 int rc = 0, payload_sz = 0;
3052 unsigned long flags;
3053
3054 spin_lock_irqsave(&qp->sq_lock, flags);
3055 while (wr) {
3056 struct bnxt_qplib_swqe wqe = {};
3057
3058 /* Common */
3059 wqe.num_sge = wr->num_sge;
3060 if (wr->num_sge > qp->qplib_qp.sq.max_sge) {
3061 ibdev_err(&rdev->ibdev,
3062 "Limit exceeded for Send SGEs");
3063 rc = -EINVAL;
3064 goto bad;
3065 }
3066
3067 payload_sz = bnxt_re_copy_wr_payload(qp->rdev, wr, &wqe);
3068 if (payload_sz < 0) {
3069 rc = -EINVAL;
3070 goto bad;
3071 }
3072 wqe.wr_id = wr->wr_id;
3073
3074 wqe.type = BNXT_QPLIB_SWQE_TYPE_SEND;
3075
3076 rc = bnxt_re_build_send_wqe(qp, wr, &wqe);
3077 if (!rc)
3078 rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe);
3079 bad:
3080 if (rc) {
3081 ibdev_err(&rdev->ibdev,
3082 "Post send failed opcode = %#x rc = %d",
3083 wr->opcode, rc);
3084 break;
3085 }
3086 wr = wr->next;
3087 }
3088 bnxt_qplib_post_send_db(&qp->qplib_qp);
3089 if (!bnxt_qplib_is_chip_gen_p5_p7(qp->rdev->chip_ctx))
3090 bnxt_ud_qp_hw_stall_workaround(qp);
3091 spin_unlock_irqrestore(&qp->sq_lock, flags);
3092 return rc;
3093 }
3094
bnxt_re_legacy_set_uc_fence(struct bnxt_qplib_swqe * wqe)3095 static void bnxt_re_legacy_set_uc_fence(struct bnxt_qplib_swqe *wqe)
3096 {
3097 /* Need unconditional fence for non-wire memory opcode
3098 * to work as expected.
3099 */
3100 if (wqe->type == BNXT_QPLIB_SWQE_TYPE_LOCAL_INV ||
3101 wqe->type == BNXT_QPLIB_SWQE_TYPE_FAST_REG_MR ||
3102 wqe->type == BNXT_QPLIB_SWQE_TYPE_REG_MR ||
3103 wqe->type == BNXT_QPLIB_SWQE_TYPE_BIND_MW)
3104 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
3105 }
3106
bnxt_re_post_send(struct ib_qp * ib_qp,const struct ib_send_wr * wr,const struct ib_send_wr ** bad_wr)3107 int bnxt_re_post_send(struct ib_qp *ib_qp, const struct ib_send_wr *wr,
3108 const struct ib_send_wr **bad_wr)
3109 {
3110 struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
3111 struct bnxt_qplib_swqe wqe;
3112 int rc = 0, payload_sz = 0;
3113 unsigned long flags;
3114
3115 spin_lock_irqsave(&qp->sq_lock, flags);
3116 while (wr) {
3117 /* House keeping */
3118 memset(&wqe, 0, sizeof(wqe));
3119
3120 /* Common */
3121 wqe.num_sge = wr->num_sge;
3122 if (wr->num_sge > qp->qplib_qp.sq.max_sge) {
3123 ibdev_err(&qp->rdev->ibdev,
3124 "Limit exceeded for Send SGEs");
3125 rc = -EINVAL;
3126 goto bad;
3127 }
3128
3129 payload_sz = bnxt_re_copy_wr_payload(qp->rdev, wr, &wqe);
3130 if (payload_sz < 0) {
3131 rc = -EINVAL;
3132 goto bad;
3133 }
3134 wqe.wr_id = wr->wr_id;
3135
3136 switch (wr->opcode) {
3137 case IB_WR_SEND:
3138 case IB_WR_SEND_WITH_IMM:
3139 if (qp->qplib_qp.type == CMDQ_CREATE_QP1_TYPE_GSI) {
3140 rc = bnxt_re_build_qp1_send_v2(qp, wr, &wqe,
3141 payload_sz);
3142 if (rc)
3143 goto bad;
3144 wqe.rawqp1.lflags |=
3145 SQ_SEND_RAWETH_QP1_LFLAGS_ROCE_CRC;
3146 }
3147 if (wr->send_flags & IB_SEND_IP_CSUM)
3148 wqe.rawqp1.lflags |=
3149 SQ_SEND_RAWETH_QP1_LFLAGS_IP_CHKSUM;
3150 fallthrough;
3151 case IB_WR_SEND_WITH_INV:
3152 rc = bnxt_re_build_send_wqe(qp, wr, &wqe);
3153 break;
3154 case IB_WR_RDMA_WRITE:
3155 case IB_WR_RDMA_WRITE_WITH_IMM:
3156 case IB_WR_RDMA_READ:
3157 rc = bnxt_re_build_rdma_wqe(wr, &wqe);
3158 break;
3159 case IB_WR_ATOMIC_CMP_AND_SWP:
3160 case IB_WR_ATOMIC_FETCH_AND_ADD:
3161 rc = bnxt_re_build_atomic_wqe(wr, &wqe);
3162 break;
3163 case IB_WR_RDMA_READ_WITH_INV:
3164 ibdev_err(&qp->rdev->ibdev,
3165 "RDMA Read with Invalidate is not supported");
3166 rc = -EINVAL;
3167 goto bad;
3168 case IB_WR_LOCAL_INV:
3169 rc = bnxt_re_build_inv_wqe(wr, &wqe);
3170 break;
3171 case IB_WR_REG_MR:
3172 rc = bnxt_re_build_reg_wqe(reg_wr(wr), &wqe);
3173 break;
3174 default:
3175 /* Unsupported WRs */
3176 ibdev_err(&qp->rdev->ibdev,
3177 "WR (%#x) is not supported", wr->opcode);
3178 rc = -EINVAL;
3179 goto bad;
3180 }
3181 if (!rc) {
3182 if (!bnxt_qplib_is_chip_gen_p5_p7(qp->rdev->chip_ctx))
3183 bnxt_re_legacy_set_uc_fence(&wqe);
3184 rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe);
3185 }
3186 bad:
3187 if (rc) {
3188 ibdev_err(&qp->rdev->ibdev,
3189 "post_send failed op:%#x qps = %#x rc = %d\n",
3190 wr->opcode, qp->qplib_qp.state, rc);
3191 *bad_wr = wr;
3192 break;
3193 }
3194 wr = wr->next;
3195 }
3196 bnxt_qplib_post_send_db(&qp->qplib_qp);
3197 if (!bnxt_qplib_is_chip_gen_p5_p7(qp->rdev->chip_ctx))
3198 bnxt_ud_qp_hw_stall_workaround(qp);
3199 spin_unlock_irqrestore(&qp->sq_lock, flags);
3200
3201 return rc;
3202 }
3203
bnxt_re_post_recv_shadow_qp(struct bnxt_re_dev * rdev,struct bnxt_re_qp * qp,const struct ib_recv_wr * wr)3204 static int bnxt_re_post_recv_shadow_qp(struct bnxt_re_dev *rdev,
3205 struct bnxt_re_qp *qp,
3206 const struct ib_recv_wr *wr)
3207 {
3208 struct bnxt_qplib_swqe wqe;
3209 int rc = 0;
3210
3211 while (wr) {
3212 /* House keeping */
3213 memset(&wqe, 0, sizeof(wqe));
3214
3215 /* Common */
3216 wqe.num_sge = wr->num_sge;
3217 if (wr->num_sge > qp->qplib_qp.rq.max_sge) {
3218 ibdev_err(&rdev->ibdev,
3219 "Limit exceeded for Receive SGEs");
3220 rc = -EINVAL;
3221 break;
3222 }
3223 bnxt_re_build_sgl(wr->sg_list, wqe.sg_list, wr->num_sge);
3224 wqe.wr_id = wr->wr_id;
3225 wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV;
3226
3227 rc = bnxt_qplib_post_recv(&qp->qplib_qp, &wqe);
3228 if (rc)
3229 break;
3230
3231 wr = wr->next;
3232 }
3233 if (!rc)
3234 bnxt_qplib_post_recv_db(&qp->qplib_qp);
3235 return rc;
3236 }
3237
bnxt_re_post_recv(struct ib_qp * ib_qp,const struct ib_recv_wr * wr,const struct ib_recv_wr ** bad_wr)3238 int bnxt_re_post_recv(struct ib_qp *ib_qp, const struct ib_recv_wr *wr,
3239 const struct ib_recv_wr **bad_wr)
3240 {
3241 struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
3242 struct bnxt_qplib_swqe wqe;
3243 int rc = 0, payload_sz = 0;
3244 unsigned long flags;
3245 u32 count = 0;
3246
3247 spin_lock_irqsave(&qp->rq_lock, flags);
3248 while (wr) {
3249 /* House keeping */
3250 memset(&wqe, 0, sizeof(wqe));
3251
3252 /* Common */
3253 wqe.num_sge = wr->num_sge;
3254 if (wr->num_sge > qp->qplib_qp.rq.max_sge) {
3255 ibdev_err(&qp->rdev->ibdev,
3256 "Limit exceeded for Receive SGEs");
3257 rc = -EINVAL;
3258 *bad_wr = wr;
3259 break;
3260 }
3261
3262 payload_sz = bnxt_re_build_sgl(wr->sg_list, wqe.sg_list,
3263 wr->num_sge);
3264 wqe.wr_id = wr->wr_id;
3265 wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV;
3266
3267 if (ib_qp->qp_type == IB_QPT_GSI &&
3268 qp->qplib_qp.type != CMDQ_CREATE_QP_TYPE_GSI)
3269 rc = bnxt_re_build_qp1_shadow_qp_recv(qp, wr, &wqe,
3270 payload_sz);
3271 if (!rc)
3272 rc = bnxt_qplib_post_recv(&qp->qplib_qp, &wqe);
3273 if (rc) {
3274 *bad_wr = wr;
3275 break;
3276 }
3277
3278 /* Ring DB if the RQEs posted reaches a threshold value */
3279 if (++count >= BNXT_RE_RQ_WQE_THRESHOLD) {
3280 bnxt_qplib_post_recv_db(&qp->qplib_qp);
3281 count = 0;
3282 }
3283
3284 wr = wr->next;
3285 }
3286
3287 if (count)
3288 bnxt_qplib_post_recv_db(&qp->qplib_qp);
3289
3290 spin_unlock_irqrestore(&qp->rq_lock, flags);
3291
3292 return rc;
3293 }
3294
bnxt_re_get_nq(struct bnxt_re_dev * rdev)3295 static struct bnxt_qplib_nq *bnxt_re_get_nq(struct bnxt_re_dev *rdev)
3296 {
3297 int min, indx;
3298
3299 mutex_lock(&rdev->nqr->load_lock);
3300 for (indx = 0, min = 0; indx < (rdev->nqr->num_msix - 1); indx++) {
3301 if (rdev->nqr->nq[min].load > rdev->nqr->nq[indx].load)
3302 min = indx;
3303 }
3304 rdev->nqr->nq[min].load++;
3305 mutex_unlock(&rdev->nqr->load_lock);
3306
3307 return &rdev->nqr->nq[min];
3308 }
3309
bnxt_re_put_nq(struct bnxt_re_dev * rdev,struct bnxt_qplib_nq * nq)3310 static void bnxt_re_put_nq(struct bnxt_re_dev *rdev, struct bnxt_qplib_nq *nq)
3311 {
3312 mutex_lock(&rdev->nqr->load_lock);
3313 nq->load--;
3314 mutex_unlock(&rdev->nqr->load_lock);
3315 }
3316
3317 /* Completion Queues */
bnxt_re_destroy_cq(struct ib_cq * ib_cq,struct ib_udata * udata)3318 int bnxt_re_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata)
3319 {
3320 struct bnxt_qplib_chip_ctx *cctx;
3321 struct bnxt_qplib_nq *nq;
3322 struct bnxt_re_dev *rdev;
3323 struct bnxt_re_cq *cq;
3324 int ret;
3325
3326 cq = container_of(ib_cq, struct bnxt_re_cq, ib_cq);
3327 rdev = cq->rdev;
3328 nq = cq->qplib_cq.nq;
3329 cctx = rdev->chip_ctx;
3330
3331 ret = ib_is_udata_in_empty(udata);
3332 if (ret)
3333 return ret;
3334
3335 if (cctx->modes.toggle_bits & BNXT_QPLIB_CQ_TOGGLE_BIT) {
3336 free_page((unsigned long)cq->uctx_cq_page);
3337 hash_del(&cq->hash_entry);
3338 }
3339 bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq);
3340
3341 bnxt_re_put_nq(rdev, nq);
3342
3343 atomic_dec(&rdev->stats.res.cq_count);
3344 kfree(cq->cql);
3345 return ib_respond_empty_udata(udata);
3346 }
3347
bnxt_re_setup_sginfo(struct bnxt_re_dev * rdev,struct ib_umem * umem,struct bnxt_qplib_sg_info * sginfo)3348 static int bnxt_re_setup_sginfo(struct bnxt_re_dev *rdev,
3349 struct ib_umem *umem,
3350 struct bnxt_qplib_sg_info *sginfo)
3351 {
3352 unsigned long page_size;
3353
3354 if (!umem)
3355 return -EINVAL;
3356
3357 page_size = ib_umem_find_best_pgsz(umem, SZ_4K, 0);
3358 if (!page_size || page_size != SZ_4K)
3359 return -EINVAL;
3360
3361 sginfo->umem = umem;
3362 sginfo->npages = ib_umem_num_dma_blocks(umem, page_size);
3363 sginfo->pgsize = page_size;
3364 sginfo->pgshft = __builtin_ctz(page_size);
3365 return 0;
3366 }
3367
bnxt_re_create_user_cq(struct ib_cq * ibcq,const struct ib_cq_init_attr * attr,struct uverbs_attr_bundle * attrs)3368 int bnxt_re_create_user_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
3369 struct uverbs_attr_bundle *attrs)
3370 {
3371 struct bnxt_re_cq *cq = container_of(ibcq, struct bnxt_re_cq, ib_cq);
3372 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibcq->device, ibdev);
3373 struct ib_udata *udata = &attrs->driver_udata;
3374 struct bnxt_re_ucontext *uctx =
3375 rdma_udata_to_drv_context(udata, struct bnxt_re_ucontext, ib_uctx);
3376 struct bnxt_qplib_dev_attr *dev_attr = rdev->dev_attr;
3377 struct bnxt_qplib_chip_ctx *cctx;
3378 struct bnxt_re_cq_resp resp = {};
3379 struct bnxt_re_cq_req req;
3380 int rc;
3381 u32 active_cqs, entries;
3382
3383 if (attr->flags)
3384 return -EOPNOTSUPP;
3385
3386 /* Validate CQ fields */
3387 if (attr->cqe > dev_attr->max_cq_wqes)
3388 return -EINVAL;
3389
3390 cq->rdev = rdev;
3391 cctx = rdev->chip_ctx;
3392 cq->qplib_cq.cq_handle = (u64)(unsigned long)(&cq->qplib_cq);
3393
3394 rc = ib_copy_validate_udata_in_cm(udata, req, cq_handle,
3395 BNXT_RE_CQ_FIXED_NUM_CQE_ENABLE);
3396 if (rc)
3397 return rc;
3398
3399 if (req.comp_mask & BNXT_RE_CQ_FIXED_NUM_CQE_ENABLE)
3400 entries = attr->cqe;
3401 else
3402 entries = bnxt_re_init_depth(attr->cqe + 1,
3403 dev_attr->max_cq_wqes + 1, uctx);
3404
3405 if (!ibcq->umem) {
3406 ibcq->umem = ib_umem_get(&rdev->ibdev, req.cq_va,
3407 entries * sizeof(struct cq_base),
3408 IB_ACCESS_LOCAL_WRITE);
3409 if (IS_ERR(ibcq->umem))
3410 return PTR_ERR(ibcq->umem);
3411 }
3412
3413 rc = bnxt_re_setup_sginfo(rdev, ibcq->umem, &cq->qplib_cq.sg_info);
3414 if (rc)
3415 return rc;
3416
3417 cq->qplib_cq.dpi = &uctx->dpi;
3418 cq->qplib_cq.max_wqe = entries;
3419 cq->qplib_cq.coalescing = &rdev->cq_coalescing;
3420 cq->qplib_cq.nq = bnxt_re_get_nq(rdev);
3421 cq->qplib_cq.cnq_hw_ring_id = cq->qplib_cq.nq->ring_id;
3422
3423 rc = bnxt_qplib_create_cq(&rdev->qplib_res, &cq->qplib_cq);
3424 if (rc)
3425 return rc;
3426
3427 cq->ib_cq.cqe = entries;
3428 cq->cq_period = cq->qplib_cq.period;
3429 active_cqs = atomic_inc_return(&rdev->stats.res.cq_count);
3430 if (active_cqs > rdev->stats.res.cq_watermark)
3431 rdev->stats.res.cq_watermark = active_cqs;
3432 spin_lock_init(&cq->cq_lock);
3433
3434 if (cctx->modes.toggle_bits & BNXT_QPLIB_CQ_TOGGLE_BIT) {
3435 hash_add(rdev->cq_hash, &cq->hash_entry, cq->qplib_cq.id);
3436 /* Allocate a page */
3437 cq->uctx_cq_page = (void *)get_zeroed_page(GFP_KERNEL);
3438 if (!cq->uctx_cq_page)
3439 return -ENOMEM;
3440
3441 resp.comp_mask |= BNXT_RE_CQ_TOGGLE_PAGE_SUPPORT;
3442 }
3443 resp.cqid = cq->qplib_cq.id;
3444 resp.tail = cq->qplib_cq.hwq.cons;
3445 resp.phase = cq->qplib_cq.period;
3446 rc = ib_respond_udata(udata, resp);
3447 if (rc) {
3448 bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq);
3449 goto free_mem;
3450 }
3451
3452 return 0;
3453
3454 free_mem:
3455 free_page((unsigned long)cq->uctx_cq_page);
3456 return rc;
3457 }
3458
bnxt_re_create_cq(struct ib_cq * ibcq,const struct ib_cq_init_attr * attr,struct uverbs_attr_bundle * attrs)3459 int bnxt_re_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
3460 struct uverbs_attr_bundle *attrs)
3461 {
3462 struct bnxt_re_cq *cq = container_of(ibcq, struct bnxt_re_cq, ib_cq);
3463 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibcq->device, ibdev);
3464 struct bnxt_qplib_dev_attr *dev_attr = rdev->dev_attr;
3465 int rc;
3466 u32 active_cqs;
3467
3468 if (attr->flags)
3469 return -EOPNOTSUPP;
3470
3471 /* Validate CQ fields */
3472 if (attr->cqe > dev_attr->max_cq_wqes)
3473 return -EINVAL;
3474
3475 cq->rdev = rdev;
3476 cq->qplib_cq.cq_handle = (u64)(unsigned long)(&cq->qplib_cq);
3477
3478 cq->max_cql = attr->cqe + 1;
3479 cq->cql = kzalloc_objs(struct bnxt_qplib_cqe, cq->max_cql);
3480 if (!cq->cql)
3481 return -ENOMEM;
3482
3483 cq->qplib_cq.sg_info.pgsize = SZ_4K;
3484 cq->qplib_cq.sg_info.pgshft = __builtin_ctz(SZ_4K);
3485 cq->qplib_cq.dpi = &rdev->dpi_privileged;
3486 cq->qplib_cq.max_wqe = cq->max_cql;
3487 cq->qplib_cq.coalescing = &rdev->cq_coalescing;
3488 cq->qplib_cq.nq = bnxt_re_get_nq(rdev);
3489 cq->qplib_cq.cnq_hw_ring_id = cq->qplib_cq.nq->ring_id;
3490
3491 rc = bnxt_qplib_create_cq(&rdev->qplib_res, &cq->qplib_cq);
3492 if (rc) {
3493 ibdev_err(&rdev->ibdev, "Failed to create HW CQ");
3494 goto fail;
3495 }
3496
3497 cq->ib_cq.cqe = cq->max_cql;
3498 cq->cq_period = cq->qplib_cq.period;
3499 active_cqs = atomic_inc_return(&rdev->stats.res.cq_count);
3500 if (active_cqs > rdev->stats.res.cq_watermark)
3501 rdev->stats.res.cq_watermark = active_cqs;
3502 spin_lock_init(&cq->cq_lock);
3503
3504 return 0;
3505
3506 fail:
3507 kfree(cq->cql);
3508 return rc;
3509 }
3510
bnxt_re_resize_cq_complete(struct bnxt_re_cq * cq)3511 static void bnxt_re_resize_cq_complete(struct bnxt_re_cq *cq)
3512 {
3513 struct bnxt_re_dev *rdev = cq->rdev;
3514
3515 bnxt_qplib_resize_cq_complete(&rdev->qplib_res, &cq->qplib_cq);
3516
3517 cq->qplib_cq.max_wqe = cq->resize_cqe;
3518 if (cq->resize_umem) {
3519 ib_umem_release(cq->ib_cq.umem);
3520 cq->ib_cq.umem = cq->resize_umem;
3521 cq->resize_umem = NULL;
3522 cq->resize_cqe = 0;
3523 }
3524 }
3525
bnxt_re_resize_cq(struct ib_cq * ibcq,unsigned int cqe,struct ib_udata * udata)3526 int bnxt_re_resize_cq(struct ib_cq *ibcq, unsigned int cqe,
3527 struct ib_udata *udata)
3528 {
3529 struct bnxt_qplib_sg_info sg_info = {};
3530 struct bnxt_qplib_dpi *orig_dpi = NULL;
3531 struct bnxt_qplib_dev_attr *dev_attr;
3532 struct bnxt_re_ucontext *uctx = NULL;
3533 struct bnxt_re_resize_cq_req req;
3534 struct bnxt_re_dev *rdev;
3535 struct bnxt_re_cq *cq;
3536 int rc;
3537 u32 entries;
3538
3539 cq = container_of(ibcq, struct bnxt_re_cq, ib_cq);
3540 rdev = cq->rdev;
3541 dev_attr = rdev->dev_attr;
3542 if (!ibcq->uobject) {
3543 ibdev_err(&rdev->ibdev, "Kernel CQ Resize not supported");
3544 return -EOPNOTSUPP;
3545 }
3546
3547 if (cq->resize_umem) {
3548 ibdev_err(&rdev->ibdev, "Resize CQ %#x failed - Busy",
3549 cq->qplib_cq.id);
3550 return -EBUSY;
3551 }
3552
3553 /* Check the requested cq depth out of supported depth */
3554 if (cqe > dev_attr->max_cq_wqes)
3555 return -EINVAL;
3556
3557 uctx = rdma_udata_to_drv_context(udata, struct bnxt_re_ucontext, ib_uctx);
3558 entries = bnxt_re_init_depth(cqe + 1, dev_attr->max_cq_wqes + 1, uctx);
3559
3560 /* uverbs consumer */
3561 rc = ib_copy_validate_udata_in(udata, req, cq_va);
3562 if (rc)
3563 goto fail;
3564
3565 cq->resize_umem = ib_umem_get(&rdev->ibdev, req.cq_va,
3566 entries * sizeof(struct cq_base),
3567 IB_ACCESS_LOCAL_WRITE);
3568 if (IS_ERR(cq->resize_umem)) {
3569 rc = PTR_ERR(cq->resize_umem);
3570 ibdev_err(&rdev->ibdev, "%s: ib_umem_get failed! rc = %pe\n",
3571 __func__, cq->resize_umem);
3572 cq->resize_umem = NULL;
3573 goto fail;
3574 }
3575 cq->resize_cqe = entries;
3576 memcpy(&sg_info, &cq->qplib_cq.sg_info, sizeof(sg_info));
3577 orig_dpi = cq->qplib_cq.dpi;
3578
3579 cq->qplib_cq.sg_info.umem = cq->resize_umem;
3580 cq->qplib_cq.sg_info.pgsize = PAGE_SIZE;
3581 cq->qplib_cq.sg_info.pgshft = PAGE_SHIFT;
3582 cq->qplib_cq.dpi = &uctx->dpi;
3583
3584 rc = bnxt_qplib_resize_cq(&rdev->qplib_res, &cq->qplib_cq, entries);
3585 if (rc) {
3586 ibdev_err(&rdev->ibdev, "Resize HW CQ %#x failed!",
3587 cq->qplib_cq.id);
3588 goto fail;
3589 }
3590
3591 cq->ib_cq.cqe = cq->resize_cqe;
3592 atomic_inc(&rdev->stats.res.resize_count);
3593
3594 return ib_respond_empty_udata(udata);
3595
3596 fail:
3597 if (cq->resize_umem) {
3598 ib_umem_release(cq->resize_umem);
3599 cq->resize_umem = NULL;
3600 cq->resize_cqe = 0;
3601 memcpy(&cq->qplib_cq.sg_info, &sg_info, sizeof(sg_info));
3602 cq->qplib_cq.dpi = orig_dpi;
3603 }
3604 return rc;
3605 }
3606
__req_to_ib_wc_status(u8 qstatus)3607 static u8 __req_to_ib_wc_status(u8 qstatus)
3608 {
3609 switch (qstatus) {
3610 case CQ_REQ_STATUS_OK:
3611 return IB_WC_SUCCESS;
3612 case CQ_REQ_STATUS_BAD_RESPONSE_ERR:
3613 return IB_WC_BAD_RESP_ERR;
3614 case CQ_REQ_STATUS_LOCAL_LENGTH_ERR:
3615 return IB_WC_LOC_LEN_ERR;
3616 case CQ_REQ_STATUS_LOCAL_QP_OPERATION_ERR:
3617 return IB_WC_LOC_QP_OP_ERR;
3618 case CQ_REQ_STATUS_LOCAL_PROTECTION_ERR:
3619 return IB_WC_LOC_PROT_ERR;
3620 case CQ_REQ_STATUS_MEMORY_MGT_OPERATION_ERR:
3621 return IB_WC_GENERAL_ERR;
3622 case CQ_REQ_STATUS_REMOTE_INVALID_REQUEST_ERR:
3623 return IB_WC_REM_INV_REQ_ERR;
3624 case CQ_REQ_STATUS_REMOTE_ACCESS_ERR:
3625 return IB_WC_REM_ACCESS_ERR;
3626 case CQ_REQ_STATUS_REMOTE_OPERATION_ERR:
3627 return IB_WC_REM_OP_ERR;
3628 case CQ_REQ_STATUS_RNR_NAK_RETRY_CNT_ERR:
3629 return IB_WC_RNR_RETRY_EXC_ERR;
3630 case CQ_REQ_STATUS_TRANSPORT_RETRY_CNT_ERR:
3631 return IB_WC_RETRY_EXC_ERR;
3632 case CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR:
3633 return IB_WC_WR_FLUSH_ERR;
3634 default:
3635 return IB_WC_GENERAL_ERR;
3636 }
3637 return 0;
3638 }
3639
__rawqp1_to_ib_wc_status(u8 qstatus)3640 static u8 __rawqp1_to_ib_wc_status(u8 qstatus)
3641 {
3642 switch (qstatus) {
3643 case CQ_RES_RAWETH_QP1_STATUS_OK:
3644 return IB_WC_SUCCESS;
3645 case CQ_RES_RAWETH_QP1_STATUS_LOCAL_ACCESS_ERROR:
3646 return IB_WC_LOC_ACCESS_ERR;
3647 case CQ_RES_RAWETH_QP1_STATUS_HW_LOCAL_LENGTH_ERR:
3648 return IB_WC_LOC_LEN_ERR;
3649 case CQ_RES_RAWETH_QP1_STATUS_LOCAL_PROTECTION_ERR:
3650 return IB_WC_LOC_PROT_ERR;
3651 case CQ_RES_RAWETH_QP1_STATUS_LOCAL_QP_OPERATION_ERR:
3652 return IB_WC_LOC_QP_OP_ERR;
3653 case CQ_RES_RAWETH_QP1_STATUS_MEMORY_MGT_OPERATION_ERR:
3654 return IB_WC_GENERAL_ERR;
3655 case CQ_RES_RAWETH_QP1_STATUS_WORK_REQUEST_FLUSHED_ERR:
3656 return IB_WC_WR_FLUSH_ERR;
3657 case CQ_RES_RAWETH_QP1_STATUS_HW_FLUSH_ERR:
3658 return IB_WC_WR_FLUSH_ERR;
3659 default:
3660 return IB_WC_GENERAL_ERR;
3661 }
3662 }
3663
__rc_to_ib_wc_status(u8 qstatus)3664 static u8 __rc_to_ib_wc_status(u8 qstatus)
3665 {
3666 switch (qstatus) {
3667 case CQ_RES_RC_STATUS_OK:
3668 return IB_WC_SUCCESS;
3669 case CQ_RES_RC_STATUS_LOCAL_ACCESS_ERROR:
3670 return IB_WC_LOC_ACCESS_ERR;
3671 case CQ_RES_RC_STATUS_LOCAL_LENGTH_ERR:
3672 return IB_WC_LOC_LEN_ERR;
3673 case CQ_RES_RC_STATUS_LOCAL_PROTECTION_ERR:
3674 return IB_WC_LOC_PROT_ERR;
3675 case CQ_RES_RC_STATUS_LOCAL_QP_OPERATION_ERR:
3676 return IB_WC_LOC_QP_OP_ERR;
3677 case CQ_RES_RC_STATUS_MEMORY_MGT_OPERATION_ERR:
3678 return IB_WC_GENERAL_ERR;
3679 case CQ_RES_RC_STATUS_REMOTE_INVALID_REQUEST_ERR:
3680 return IB_WC_REM_INV_REQ_ERR;
3681 case CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR:
3682 return IB_WC_WR_FLUSH_ERR;
3683 case CQ_RES_RC_STATUS_HW_FLUSH_ERR:
3684 return IB_WC_WR_FLUSH_ERR;
3685 default:
3686 return IB_WC_GENERAL_ERR;
3687 }
3688 }
3689
bnxt_re_process_req_wc(struct ib_wc * wc,struct bnxt_qplib_cqe * cqe)3690 static void bnxt_re_process_req_wc(struct ib_wc *wc, struct bnxt_qplib_cqe *cqe)
3691 {
3692 switch (cqe->type) {
3693 case BNXT_QPLIB_SWQE_TYPE_SEND:
3694 wc->opcode = IB_WC_SEND;
3695 break;
3696 case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM:
3697 wc->opcode = IB_WC_SEND;
3698 wc->wc_flags |= IB_WC_WITH_IMM;
3699 break;
3700 case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV:
3701 wc->opcode = IB_WC_SEND;
3702 wc->wc_flags |= IB_WC_WITH_INVALIDATE;
3703 break;
3704 case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE:
3705 wc->opcode = IB_WC_RDMA_WRITE;
3706 break;
3707 case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM:
3708 wc->opcode = IB_WC_RDMA_WRITE;
3709 wc->wc_flags |= IB_WC_WITH_IMM;
3710 break;
3711 case BNXT_QPLIB_SWQE_TYPE_RDMA_READ:
3712 wc->opcode = IB_WC_RDMA_READ;
3713 break;
3714 case BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP:
3715 wc->opcode = IB_WC_COMP_SWAP;
3716 break;
3717 case BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD:
3718 wc->opcode = IB_WC_FETCH_ADD;
3719 break;
3720 case BNXT_QPLIB_SWQE_TYPE_LOCAL_INV:
3721 wc->opcode = IB_WC_LOCAL_INV;
3722 break;
3723 case BNXT_QPLIB_SWQE_TYPE_REG_MR:
3724 wc->opcode = IB_WC_REG_MR;
3725 break;
3726 default:
3727 wc->opcode = IB_WC_SEND;
3728 break;
3729 }
3730
3731 wc->status = __req_to_ib_wc_status(cqe->status);
3732 }
3733
bnxt_re_check_packet_type(u16 raweth_qp1_flags,u16 raweth_qp1_flags2)3734 static int bnxt_re_check_packet_type(u16 raweth_qp1_flags,
3735 u16 raweth_qp1_flags2)
3736 {
3737 bool is_ipv6 = false, is_ipv4 = false;
3738
3739 /* raweth_qp1_flags Bit 9-6 indicates itype */
3740 if ((raweth_qp1_flags & CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE)
3741 != CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE)
3742 return -1;
3743
3744 if (raweth_qp1_flags2 &
3745 CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_CS_CALC &&
3746 raweth_qp1_flags2 &
3747 CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_L4_CS_CALC) {
3748 /* raweth_qp1_flags2 Bit 8 indicates ip_type. 0-v4 1 - v6 */
3749 (raweth_qp1_flags2 &
3750 CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_TYPE) ?
3751 (is_ipv6 = true) : (is_ipv4 = true);
3752 return ((is_ipv6) ?
3753 BNXT_RE_ROCEV2_IPV6_PACKET :
3754 BNXT_RE_ROCEV2_IPV4_PACKET);
3755 } else {
3756 return BNXT_RE_ROCE_V1_PACKET;
3757 }
3758 }
3759
bnxt_re_to_ib_nw_type(int nw_type)3760 static int bnxt_re_to_ib_nw_type(int nw_type)
3761 {
3762 u8 nw_hdr_type = 0xFF;
3763
3764 switch (nw_type) {
3765 case BNXT_RE_ROCE_V1_PACKET:
3766 nw_hdr_type = RDMA_NETWORK_ROCE_V1;
3767 break;
3768 case BNXT_RE_ROCEV2_IPV4_PACKET:
3769 nw_hdr_type = RDMA_NETWORK_IPV4;
3770 break;
3771 case BNXT_RE_ROCEV2_IPV6_PACKET:
3772 nw_hdr_type = RDMA_NETWORK_IPV6;
3773 break;
3774 }
3775 return nw_hdr_type;
3776 }
3777
bnxt_re_is_loopback_packet(struct bnxt_re_dev * rdev,void * rq_hdr_buf)3778 static bool bnxt_re_is_loopback_packet(struct bnxt_re_dev *rdev,
3779 void *rq_hdr_buf)
3780 {
3781 u8 *tmp_buf = NULL;
3782 struct ethhdr *eth_hdr;
3783 u16 eth_type;
3784 bool rc = false;
3785
3786 tmp_buf = (u8 *)rq_hdr_buf;
3787 /*
3788 * If dest mac is not same as I/F mac, this could be a
3789 * loopback address or multicast address, check whether
3790 * it is a loopback packet
3791 */
3792 if (!ether_addr_equal(tmp_buf, rdev->netdev->dev_addr)) {
3793 tmp_buf += 4;
3794 /* Check the ether type */
3795 eth_hdr = (struct ethhdr *)tmp_buf;
3796 eth_type = ntohs(eth_hdr->h_proto);
3797 switch (eth_type) {
3798 case ETH_P_IBOE:
3799 rc = true;
3800 break;
3801 case ETH_P_IP:
3802 case ETH_P_IPV6: {
3803 u32 len;
3804 struct udphdr *udp_hdr;
3805
3806 len = (eth_type == ETH_P_IP ? sizeof(struct iphdr) :
3807 sizeof(struct ipv6hdr));
3808 tmp_buf += sizeof(struct ethhdr) + len;
3809 udp_hdr = (struct udphdr *)tmp_buf;
3810 if (ntohs(udp_hdr->dest) ==
3811 ROCE_V2_UDP_DPORT)
3812 rc = true;
3813 break;
3814 }
3815 default:
3816 break;
3817 }
3818 }
3819
3820 return rc;
3821 }
3822
bnxt_re_process_raw_qp_pkt_rx(struct bnxt_re_qp * gsi_qp,struct bnxt_qplib_cqe * cqe)3823 static int bnxt_re_process_raw_qp_pkt_rx(struct bnxt_re_qp *gsi_qp,
3824 struct bnxt_qplib_cqe *cqe)
3825 {
3826 struct bnxt_re_dev *rdev = gsi_qp->rdev;
3827 struct bnxt_re_sqp_entries *sqp_entry = NULL;
3828 struct bnxt_re_qp *gsi_sqp = rdev->gsi_ctx.gsi_sqp;
3829 dma_addr_t shrq_hdr_buf_map;
3830 struct ib_sge s_sge[2] = {};
3831 struct ib_sge r_sge[2] = {};
3832 struct bnxt_re_ah *gsi_sah;
3833 struct ib_recv_wr rwr = {};
3834 dma_addr_t rq_hdr_buf_map;
3835 struct ib_ud_wr udwr = {};
3836 struct ib_send_wr *swr;
3837 u32 skip_bytes = 0;
3838 int pkt_type = 0;
3839 void *rq_hdr_buf;
3840 u32 offset = 0;
3841 u32 tbl_idx;
3842 int rc;
3843
3844 swr = &udwr.wr;
3845 tbl_idx = cqe->wr_id;
3846
3847 rq_hdr_buf = gsi_qp->qplib_qp.rq_hdr_buf +
3848 (tbl_idx * gsi_qp->qplib_qp.rq_hdr_buf_size);
3849 rq_hdr_buf_map = bnxt_qplib_get_qp_buf_from_index(&gsi_qp->qplib_qp,
3850 tbl_idx);
3851
3852 /* Shadow QP header buffer */
3853 shrq_hdr_buf_map = bnxt_qplib_get_qp_buf_from_index(&gsi_qp->qplib_qp,
3854 tbl_idx);
3855 sqp_entry = &rdev->gsi_ctx.sqp_tbl[tbl_idx];
3856
3857 /* Store this cqe */
3858 memcpy(&sqp_entry->cqe, cqe, sizeof(struct bnxt_qplib_cqe));
3859 sqp_entry->qp1_qp = gsi_qp;
3860
3861 /* Find packet type from the cqe */
3862
3863 pkt_type = bnxt_re_check_packet_type(cqe->raweth_qp1_flags,
3864 cqe->raweth_qp1_flags2);
3865 if (pkt_type < 0) {
3866 ibdev_err(&rdev->ibdev, "Invalid packet\n");
3867 return -EINVAL;
3868 }
3869
3870 /* Adjust the offset for the user buffer and post in the rq */
3871
3872 if (pkt_type == BNXT_RE_ROCEV2_IPV4_PACKET)
3873 offset = 20;
3874
3875 /*
3876 * QP1 loopback packet has 4 bytes of internal header before
3877 * ether header. Skip these four bytes.
3878 */
3879 if (bnxt_re_is_loopback_packet(rdev, rq_hdr_buf))
3880 skip_bytes = 4;
3881
3882 /* First send SGE . Skip the ether header*/
3883 s_sge[0].addr = rq_hdr_buf_map + BNXT_QPLIB_MAX_QP1_RQ_ETH_HDR_SIZE
3884 + skip_bytes;
3885 s_sge[0].lkey = 0xFFFFFFFF;
3886 s_sge[0].length = offset ? BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV4 :
3887 BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6;
3888
3889 /* Second Send SGE */
3890 s_sge[1].addr = s_sge[0].addr + s_sge[0].length +
3891 BNXT_QPLIB_MAX_QP1_RQ_BDETH_HDR_SIZE;
3892 if (pkt_type != BNXT_RE_ROCE_V1_PACKET)
3893 s_sge[1].addr += 8;
3894 s_sge[1].lkey = 0xFFFFFFFF;
3895 s_sge[1].length = 256;
3896
3897 /* First recv SGE */
3898
3899 r_sge[0].addr = shrq_hdr_buf_map;
3900 r_sge[0].lkey = 0xFFFFFFFF;
3901 r_sge[0].length = 40;
3902
3903 r_sge[1].addr = sqp_entry->sge.addr + offset;
3904 r_sge[1].lkey = sqp_entry->sge.lkey;
3905 r_sge[1].length = BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6 + 256 - offset;
3906
3907 /* Create receive work request */
3908 rwr.num_sge = 2;
3909 rwr.sg_list = r_sge;
3910 rwr.wr_id = tbl_idx;
3911 rwr.next = NULL;
3912
3913 rc = bnxt_re_post_recv_shadow_qp(rdev, gsi_sqp, &rwr);
3914 if (rc) {
3915 ibdev_err(&rdev->ibdev,
3916 "Failed to post Rx buffers to shadow QP");
3917 return -ENOMEM;
3918 }
3919
3920 swr->num_sge = 2;
3921 swr->sg_list = s_sge;
3922 swr->wr_id = tbl_idx;
3923 swr->opcode = IB_WR_SEND;
3924 swr->next = NULL;
3925 gsi_sah = rdev->gsi_ctx.gsi_sah;
3926 udwr.ah = &gsi_sah->ib_ah;
3927 udwr.remote_qpn = gsi_sqp->qplib_qp.id;
3928 udwr.remote_qkey = gsi_sqp->qplib_qp.qkey;
3929
3930 /* post data received in the send queue */
3931 return bnxt_re_post_send_shadow_qp(rdev, gsi_sqp, swr);
3932 }
3933
bnxt_re_process_res_rawqp1_wc(struct ib_wc * wc,struct bnxt_qplib_cqe * cqe)3934 static void bnxt_re_process_res_rawqp1_wc(struct ib_wc *wc,
3935 struct bnxt_qplib_cqe *cqe)
3936 {
3937 wc->opcode = IB_WC_RECV;
3938 wc->status = __rawqp1_to_ib_wc_status(cqe->status);
3939 wc->wc_flags |= IB_WC_GRH;
3940 }
3941
bnxt_re_check_if_vlan_valid(struct bnxt_re_dev * rdev,u16 vlan_id)3942 static bool bnxt_re_check_if_vlan_valid(struct bnxt_re_dev *rdev,
3943 u16 vlan_id)
3944 {
3945 /*
3946 * Check if the vlan is configured in the host. If not configured, it
3947 * can be a transparent VLAN. So dont report the vlan id.
3948 */
3949 if (!__vlan_find_dev_deep_rcu(rdev->netdev,
3950 htons(ETH_P_8021Q), vlan_id))
3951 return false;
3952 return true;
3953 }
3954
bnxt_re_is_vlan_pkt(struct bnxt_qplib_cqe * orig_cqe,u16 * vid,u8 * sl)3955 static bool bnxt_re_is_vlan_pkt(struct bnxt_qplib_cqe *orig_cqe,
3956 u16 *vid, u8 *sl)
3957 {
3958 bool ret = false;
3959 u32 metadata;
3960 u16 tpid;
3961
3962 metadata = orig_cqe->raweth_qp1_metadata;
3963 if (orig_cqe->raweth_qp1_flags2 &
3964 CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_VLAN) {
3965 tpid = ((metadata &
3966 CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_MASK) >>
3967 CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_SFT);
3968 if (tpid == ETH_P_8021Q) {
3969 *vid = metadata &
3970 CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_VID_MASK;
3971 *sl = (metadata &
3972 CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_PRI_MASK) >>
3973 CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_PRI_SFT;
3974 ret = true;
3975 }
3976 }
3977
3978 return ret;
3979 }
3980
bnxt_re_process_res_rc_wc(struct ib_wc * wc,struct bnxt_qplib_cqe * cqe)3981 static void bnxt_re_process_res_rc_wc(struct ib_wc *wc,
3982 struct bnxt_qplib_cqe *cqe)
3983 {
3984 wc->opcode = IB_WC_RECV;
3985 wc->status = __rc_to_ib_wc_status(cqe->status);
3986
3987 if (cqe->flags & CQ_RES_RC_FLAGS_IMM)
3988 wc->wc_flags |= IB_WC_WITH_IMM;
3989 if (cqe->flags & CQ_RES_RC_FLAGS_INV)
3990 wc->wc_flags |= IB_WC_WITH_INVALIDATE;
3991 if ((cqe->flags & (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM)) ==
3992 (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM))
3993 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
3994 }
3995
bnxt_re_process_res_shadow_qp_wc(struct bnxt_re_qp * gsi_sqp,struct ib_wc * wc,struct bnxt_qplib_cqe * cqe)3996 static void bnxt_re_process_res_shadow_qp_wc(struct bnxt_re_qp *gsi_sqp,
3997 struct ib_wc *wc,
3998 struct bnxt_qplib_cqe *cqe)
3999 {
4000 struct bnxt_re_dev *rdev = gsi_sqp->rdev;
4001 struct bnxt_re_qp *gsi_qp = NULL;
4002 struct bnxt_qplib_cqe *orig_cqe = NULL;
4003 struct bnxt_re_sqp_entries *sqp_entry = NULL;
4004 int nw_type;
4005 u32 tbl_idx;
4006 u16 vlan_id;
4007 u8 sl;
4008
4009 tbl_idx = cqe->wr_id;
4010
4011 sqp_entry = &rdev->gsi_ctx.sqp_tbl[tbl_idx];
4012 gsi_qp = sqp_entry->qp1_qp;
4013 orig_cqe = &sqp_entry->cqe;
4014
4015 wc->wr_id = sqp_entry->wrid;
4016 wc->byte_len = orig_cqe->length;
4017 wc->qp = &gsi_qp->ib_qp;
4018
4019 wc->ex.imm_data = cpu_to_be32(orig_cqe->immdata);
4020 wc->src_qp = orig_cqe->src_qp;
4021 memcpy(wc->smac, orig_cqe->smac, ETH_ALEN);
4022 if (bnxt_re_is_vlan_pkt(orig_cqe, &vlan_id, &sl)) {
4023 if (bnxt_re_check_if_vlan_valid(rdev, vlan_id)) {
4024 wc->vlan_id = vlan_id;
4025 wc->sl = sl;
4026 wc->wc_flags |= IB_WC_WITH_VLAN;
4027 }
4028 }
4029 wc->port_num = 1;
4030 wc->vendor_err = orig_cqe->status;
4031
4032 wc->opcode = IB_WC_RECV;
4033 wc->status = __rawqp1_to_ib_wc_status(orig_cqe->status);
4034 wc->wc_flags |= IB_WC_GRH;
4035
4036 nw_type = bnxt_re_check_packet_type(orig_cqe->raweth_qp1_flags,
4037 orig_cqe->raweth_qp1_flags2);
4038 if (nw_type >= 0) {
4039 wc->network_hdr_type = bnxt_re_to_ib_nw_type(nw_type);
4040 wc->wc_flags |= IB_WC_WITH_NETWORK_HDR_TYPE;
4041 }
4042 }
4043
bnxt_re_process_res_ud_wc(struct bnxt_re_qp * qp,struct ib_wc * wc,struct bnxt_qplib_cqe * cqe)4044 static void bnxt_re_process_res_ud_wc(struct bnxt_re_qp *qp,
4045 struct ib_wc *wc,
4046 struct bnxt_qplib_cqe *cqe)
4047 {
4048 struct bnxt_re_dev *rdev;
4049 u16 vlan_id = 0;
4050 u8 nw_type;
4051
4052 rdev = qp->rdev;
4053 wc->opcode = IB_WC_RECV;
4054 wc->status = __rc_to_ib_wc_status(cqe->status);
4055
4056 if (cqe->flags & CQ_RES_UD_FLAGS_IMM)
4057 wc->wc_flags |= IB_WC_WITH_IMM;
4058 /* report only on GSI QP for Thor */
4059 if (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_GSI) {
4060 wc->wc_flags |= IB_WC_GRH;
4061 memcpy(wc->smac, cqe->smac, ETH_ALEN);
4062 wc->wc_flags |= IB_WC_WITH_SMAC;
4063 if (cqe->flags & CQ_RES_UD_FLAGS_META_FORMAT_VLAN) {
4064 vlan_id = (cqe->cfa_meta & 0xFFF);
4065 }
4066 /* Mark only if vlan_id is non zero */
4067 if (vlan_id && bnxt_re_check_if_vlan_valid(rdev, vlan_id)) {
4068 wc->vlan_id = vlan_id;
4069 wc->wc_flags |= IB_WC_WITH_VLAN;
4070 }
4071 nw_type = (cqe->flags & CQ_RES_UD_FLAGS_ROCE_IP_VER_MASK) >>
4072 CQ_RES_UD_FLAGS_ROCE_IP_VER_SFT;
4073 wc->network_hdr_type = bnxt_re_to_ib_nw_type(nw_type);
4074 wc->wc_flags |= IB_WC_WITH_NETWORK_HDR_TYPE;
4075 }
4076
4077 }
4078
send_phantom_wqe(struct bnxt_re_qp * qp)4079 static int send_phantom_wqe(struct bnxt_re_qp *qp)
4080 {
4081 struct bnxt_qplib_qp *lib_qp = &qp->qplib_qp;
4082 unsigned long flags;
4083 int rc;
4084
4085 spin_lock_irqsave(&qp->sq_lock, flags);
4086
4087 rc = bnxt_re_bind_fence_mw(lib_qp);
4088 if (!rc) {
4089 lib_qp->sq.phantom_wqe_cnt++;
4090 ibdev_dbg(&qp->rdev->ibdev,
4091 "qp %#x sq->prod %#x sw_prod %#x phantom_wqe_cnt %d\n",
4092 lib_qp->id, lib_qp->sq.hwq.prod,
4093 HWQ_CMP(lib_qp->sq.hwq.prod, &lib_qp->sq.hwq),
4094 lib_qp->sq.phantom_wqe_cnt);
4095 }
4096
4097 spin_unlock_irqrestore(&qp->sq_lock, flags);
4098 return rc;
4099 }
4100
bnxt_re_poll_cq(struct ib_cq * ib_cq,int num_entries,struct ib_wc * wc)4101 int bnxt_re_poll_cq(struct ib_cq *ib_cq, int num_entries, struct ib_wc *wc)
4102 {
4103 struct bnxt_re_cq *cq = container_of(ib_cq, struct bnxt_re_cq, ib_cq);
4104 struct bnxt_re_qp *qp, *sh_qp;
4105 struct bnxt_qplib_cqe *cqe;
4106 int i, ncqe, budget;
4107 struct bnxt_qplib_q *sq;
4108 struct bnxt_qplib_qp *lib_qp;
4109 u32 tbl_idx;
4110 struct bnxt_re_sqp_entries *sqp_entry = NULL;
4111 unsigned long flags;
4112
4113 /* User CQ; the only processing we do is to
4114 * complete any pending CQ resize operation.
4115 */
4116 if (cq->ib_cq.umem) {
4117 if (cq->resize_umem)
4118 bnxt_re_resize_cq_complete(cq);
4119 return 0;
4120 }
4121
4122 spin_lock_irqsave(&cq->cq_lock, flags);
4123 budget = min_t(u32, num_entries, cq->max_cql);
4124 num_entries = budget;
4125 if (!cq->cql) {
4126 ibdev_err(&cq->rdev->ibdev, "POLL CQ : no CQL to use");
4127 goto exit;
4128 }
4129 cqe = &cq->cql[0];
4130 while (budget) {
4131 lib_qp = NULL;
4132 ncqe = bnxt_qplib_poll_cq(&cq->qplib_cq, cqe, budget, &lib_qp);
4133 if (lib_qp) {
4134 sq = &lib_qp->sq;
4135 if (sq->send_phantom) {
4136 qp = container_of(lib_qp,
4137 struct bnxt_re_qp, qplib_qp);
4138 if (send_phantom_wqe(qp) == -ENOMEM)
4139 ibdev_err(&cq->rdev->ibdev,
4140 "Phantom failed! Scheduled to send again\n");
4141 else
4142 sq->send_phantom = false;
4143 }
4144 }
4145 if (ncqe < budget)
4146 ncqe += bnxt_qplib_process_flush_list(&cq->qplib_cq,
4147 cqe + ncqe,
4148 budget - ncqe);
4149
4150 if (!ncqe)
4151 break;
4152
4153 for (i = 0; i < ncqe; i++, cqe++) {
4154 /* Transcribe each qplib_wqe back to ib_wc */
4155 memset(wc, 0, sizeof(*wc));
4156
4157 wc->wr_id = cqe->wr_id;
4158 wc->byte_len = cqe->length;
4159 qp = container_of
4160 ((struct bnxt_qplib_qp *)
4161 (unsigned long)(cqe->qp_handle),
4162 struct bnxt_re_qp, qplib_qp);
4163 wc->qp = &qp->ib_qp;
4164 if (cqe->flags & CQ_RES_RC_FLAGS_IMM)
4165 wc->ex.imm_data = cpu_to_be32(cqe->immdata);
4166 else
4167 wc->ex.invalidate_rkey = cqe->invrkey;
4168 wc->src_qp = cqe->src_qp;
4169 memcpy(wc->smac, cqe->smac, ETH_ALEN);
4170 wc->port_num = 1;
4171 wc->vendor_err = cqe->status;
4172
4173 switch (cqe->opcode) {
4174 case CQ_BASE_CQE_TYPE_REQ:
4175 sh_qp = qp->rdev->gsi_ctx.gsi_sqp;
4176 if (sh_qp &&
4177 qp->qplib_qp.id == sh_qp->qplib_qp.id) {
4178 /* Handle this completion with
4179 * the stored completion
4180 */
4181 memset(wc, 0, sizeof(*wc));
4182 continue;
4183 }
4184 bnxt_re_process_req_wc(wc, cqe);
4185 break;
4186 case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1:
4187 if (!cqe->status) {
4188 int rc = 0;
4189
4190 rc = bnxt_re_process_raw_qp_pkt_rx
4191 (qp, cqe);
4192 if (!rc) {
4193 memset(wc, 0, sizeof(*wc));
4194 continue;
4195 }
4196 cqe->status = -1;
4197 }
4198 /* Errors need not be looped back.
4199 * But change the wr_id to the one
4200 * stored in the table
4201 */
4202 tbl_idx = cqe->wr_id;
4203 sqp_entry = &cq->rdev->gsi_ctx.sqp_tbl[tbl_idx];
4204 wc->wr_id = sqp_entry->wrid;
4205 bnxt_re_process_res_rawqp1_wc(wc, cqe);
4206 break;
4207 case CQ_BASE_CQE_TYPE_RES_RC:
4208 bnxt_re_process_res_rc_wc(wc, cqe);
4209 break;
4210 case CQ_BASE_CQE_TYPE_RES_UD:
4211 sh_qp = qp->rdev->gsi_ctx.gsi_sqp;
4212 if (sh_qp &&
4213 qp->qplib_qp.id == sh_qp->qplib_qp.id) {
4214 /* Handle this completion with
4215 * the stored completion
4216 */
4217 if (cqe->status) {
4218 continue;
4219 } else {
4220 bnxt_re_process_res_shadow_qp_wc
4221 (qp, wc, cqe);
4222 break;
4223 }
4224 }
4225 bnxt_re_process_res_ud_wc(qp, wc, cqe);
4226 break;
4227 default:
4228 ibdev_err(&cq->rdev->ibdev,
4229 "POLL CQ : type 0x%x not handled",
4230 cqe->opcode);
4231 continue;
4232 }
4233 wc++;
4234 budget--;
4235 }
4236 }
4237 exit:
4238 spin_unlock_irqrestore(&cq->cq_lock, flags);
4239 return num_entries - budget;
4240 }
4241
bnxt_re_req_notify_cq(struct ib_cq * ib_cq,enum ib_cq_notify_flags ib_cqn_flags)4242 int bnxt_re_req_notify_cq(struct ib_cq *ib_cq,
4243 enum ib_cq_notify_flags ib_cqn_flags)
4244 {
4245 struct bnxt_re_cq *cq = container_of(ib_cq, struct bnxt_re_cq, ib_cq);
4246 int type = 0, rc = 0;
4247 unsigned long flags;
4248
4249 spin_lock_irqsave(&cq->cq_lock, flags);
4250 /* Trigger on the very next completion */
4251 if (ib_cqn_flags & IB_CQ_NEXT_COMP)
4252 type = DBC_DBC_TYPE_CQ_ARMALL;
4253 /* Trigger on the next solicited completion */
4254 else if (ib_cqn_flags & IB_CQ_SOLICITED)
4255 type = DBC_DBC_TYPE_CQ_ARMSE;
4256
4257 /* Poll to see if there are missed events */
4258 if ((ib_cqn_flags & IB_CQ_REPORT_MISSED_EVENTS) &&
4259 !(bnxt_qplib_is_cq_empty(&cq->qplib_cq))) {
4260 rc = 1;
4261 goto exit;
4262 }
4263 bnxt_qplib_req_notify_cq(&cq->qplib_cq, type);
4264
4265 exit:
4266 spin_unlock_irqrestore(&cq->cq_lock, flags);
4267 return rc;
4268 }
4269
4270 /* Memory Regions */
bnxt_re_get_dma_mr(struct ib_pd * ib_pd,int mr_access_flags)4271 struct ib_mr *bnxt_re_get_dma_mr(struct ib_pd *ib_pd, int mr_access_flags)
4272 {
4273 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
4274 struct bnxt_re_dev *rdev = pd->rdev;
4275 struct bnxt_re_mr *mr;
4276 u32 active_mrs;
4277 int rc;
4278
4279 mr = kzalloc_obj(*mr);
4280 if (!mr)
4281 return ERR_PTR(-ENOMEM);
4282
4283 mr->rdev = rdev;
4284 mr->qplib_mr.pd = &pd->qplib_pd;
4285 mr->qplib_mr.access_flags = __from_ib_access_flags(mr_access_flags);
4286 mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR;
4287
4288 if (mr_access_flags & IB_ACCESS_RELAXED_ORDERING)
4289 bnxt_re_check_and_set_relaxed_ordering(rdev, &mr->qplib_mr);
4290
4291 /* Allocate and register 0 as the address */
4292 rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr);
4293 if (rc)
4294 goto fail;
4295
4296 mr->qplib_mr.hwq.level = PBL_LVL_MAX;
4297 mr->qplib_mr.total_size = -1; /* Infinte length */
4298 rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, NULL, 0,
4299 PAGE_SIZE, false);
4300 if (rc)
4301 goto fail_mr;
4302
4303 mr->ib_mr.lkey = mr->qplib_mr.lkey;
4304 if (mr_access_flags & (IB_ACCESS_REMOTE_WRITE | IB_ACCESS_REMOTE_READ |
4305 IB_ACCESS_REMOTE_ATOMIC))
4306 mr->ib_mr.rkey = mr->ib_mr.lkey;
4307 active_mrs = atomic_inc_return(&rdev->stats.res.mr_count);
4308 if (active_mrs > rdev->stats.res.mr_watermark)
4309 rdev->stats.res.mr_watermark = active_mrs;
4310
4311 return &mr->ib_mr;
4312
4313 fail_mr:
4314 bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr);
4315 fail:
4316 kfree(mr);
4317 return ERR_PTR(rc);
4318 }
4319
bnxt_re_dereg_mr(struct ib_mr * ib_mr,struct ib_udata * udata)4320 int bnxt_re_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
4321 {
4322 struct bnxt_re_mr *mr = container_of(ib_mr, struct bnxt_re_mr, ib_mr);
4323 struct bnxt_re_dev *rdev = mr->rdev;
4324 int rc;
4325
4326 rc = ib_is_udata_in_empty(udata);
4327 if (rc)
4328 return rc;
4329
4330 rc = bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr);
4331 if (rc) {
4332 ibdev_err(&rdev->ibdev, "Dereg MR failed: %#x\n", rc);
4333 return rc;
4334 }
4335
4336 if (mr->pages) {
4337 rc = bnxt_qplib_free_fast_reg_page_list(&rdev->qplib_res,
4338 &mr->qplib_frpl);
4339 kfree(mr->pages);
4340 mr->npages = 0;
4341 mr->pages = NULL;
4342 }
4343 ib_umem_release(mr->ib_umem);
4344
4345 kfree(mr);
4346 atomic_dec(&rdev->stats.res.mr_count);
4347 if (rc)
4348 return rc;
4349 return ib_respond_empty_udata(udata);
4350 }
4351
bnxt_re_set_page(struct ib_mr * ib_mr,u64 addr)4352 static int bnxt_re_set_page(struct ib_mr *ib_mr, u64 addr)
4353 {
4354 struct bnxt_re_mr *mr = container_of(ib_mr, struct bnxt_re_mr, ib_mr);
4355
4356 if (unlikely(mr->npages == mr->qplib_frpl.max_pg_ptrs))
4357 return -ENOMEM;
4358
4359 mr->pages[mr->npages++] = addr;
4360 return 0;
4361 }
4362
bnxt_re_map_mr_sg(struct ib_mr * ib_mr,struct scatterlist * sg,int sg_nents,unsigned int * sg_offset)4363 int bnxt_re_map_mr_sg(struct ib_mr *ib_mr, struct scatterlist *sg, int sg_nents,
4364 unsigned int *sg_offset)
4365 {
4366 struct bnxt_re_mr *mr = container_of(ib_mr, struct bnxt_re_mr, ib_mr);
4367
4368 mr->npages = 0;
4369 return ib_sg_to_pages(ib_mr, sg, sg_nents, sg_offset, bnxt_re_set_page);
4370 }
4371
bnxt_re_alloc_mr(struct ib_pd * ib_pd,enum ib_mr_type type,u32 max_num_sg)4372 struct ib_mr *bnxt_re_alloc_mr(struct ib_pd *ib_pd, enum ib_mr_type type,
4373 u32 max_num_sg)
4374 {
4375 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
4376 struct bnxt_re_dev *rdev = pd->rdev;
4377 struct bnxt_re_mr *mr = NULL;
4378 u32 active_mrs;
4379 int rc;
4380
4381 if (type != IB_MR_TYPE_MEM_REG) {
4382 ibdev_dbg(&rdev->ibdev, "MR type 0x%x not supported", type);
4383 return ERR_PTR(-EINVAL);
4384 }
4385 if (max_num_sg > MAX_PBL_LVL_1_PGS)
4386 return ERR_PTR(-EINVAL);
4387
4388 mr = kzalloc_obj(*mr);
4389 if (!mr)
4390 return ERR_PTR(-ENOMEM);
4391
4392 mr->rdev = rdev;
4393 mr->qplib_mr.pd = &pd->qplib_pd;
4394 mr->qplib_mr.access_flags = BNXT_QPLIB_FR_PMR;
4395 mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR;
4396
4397 rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr);
4398 if (rc)
4399 goto bail;
4400
4401 mr->ib_mr.lkey = mr->qplib_mr.lkey;
4402 mr->ib_mr.rkey = mr->ib_mr.lkey;
4403
4404 mr->pages = kzalloc_objs(u64, max_num_sg);
4405 if (!mr->pages) {
4406 rc = -ENOMEM;
4407 goto fail;
4408 }
4409 rc = bnxt_qplib_alloc_fast_reg_page_list(&rdev->qplib_res,
4410 &mr->qplib_frpl, max_num_sg);
4411 if (rc) {
4412 ibdev_err(&rdev->ibdev,
4413 "Failed to allocate HW FR page list");
4414 goto fail_mr;
4415 }
4416
4417 active_mrs = atomic_inc_return(&rdev->stats.res.mr_count);
4418 if (active_mrs > rdev->stats.res.mr_watermark)
4419 rdev->stats.res.mr_watermark = active_mrs;
4420 return &mr->ib_mr;
4421
4422 fail_mr:
4423 kfree(mr->pages);
4424 fail:
4425 bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr);
4426 bail:
4427 kfree(mr);
4428 return ERR_PTR(rc);
4429 }
4430
bnxt_re_alloc_mw(struct ib_pd * ib_pd,enum ib_mw_type type,struct ib_udata * udata)4431 struct ib_mw *bnxt_re_alloc_mw(struct ib_pd *ib_pd, enum ib_mw_type type,
4432 struct ib_udata *udata)
4433 {
4434 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
4435 struct bnxt_re_dev *rdev = pd->rdev;
4436 struct bnxt_re_mw *mw;
4437 u32 active_mws;
4438 int rc;
4439
4440 rc = ib_is_udata_in_empty(udata);
4441 if (rc)
4442 return ERR_PTR(rc);
4443
4444 mw = kzalloc_obj(*mw);
4445 if (!mw)
4446 return ERR_PTR(-ENOMEM);
4447 mw->rdev = rdev;
4448 mw->qplib_mw.pd = &pd->qplib_pd;
4449
4450 mw->qplib_mw.type = (type == IB_MW_TYPE_1 ?
4451 CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE1 :
4452 CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2B);
4453 rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mw->qplib_mw);
4454 if (rc) {
4455 ibdev_err(&rdev->ibdev, "Allocate MW failed!");
4456 goto fail;
4457 }
4458 mw->ib_mw.rkey = mw->qplib_mw.rkey;
4459
4460 active_mws = atomic_inc_return(&rdev->stats.res.mw_count);
4461 if (active_mws > rdev->stats.res.mw_watermark)
4462 rdev->stats.res.mw_watermark = active_mws;
4463 return &mw->ib_mw;
4464
4465 fail:
4466 kfree(mw);
4467 return ERR_PTR(rc);
4468 }
4469
bnxt_re_dealloc_mw(struct ib_mw * ib_mw)4470 int bnxt_re_dealloc_mw(struct ib_mw *ib_mw)
4471 {
4472 struct bnxt_re_mw *mw = container_of(ib_mw, struct bnxt_re_mw, ib_mw);
4473 struct bnxt_re_dev *rdev = mw->rdev;
4474 int rc;
4475
4476 rc = bnxt_qplib_free_mrw(&rdev->qplib_res, &mw->qplib_mw);
4477 if (rc) {
4478 ibdev_err(&rdev->ibdev, "Free MW failed: %#x\n", rc);
4479 return rc;
4480 }
4481
4482 kfree(mw);
4483 atomic_dec(&rdev->stats.res.mw_count);
4484 return rc;
4485 }
4486
__bnxt_re_user_reg_mr(struct ib_pd * ib_pd,u64 length,u64 virt_addr,int mr_access_flags,struct ib_umem * umem)4487 static struct ib_mr *__bnxt_re_user_reg_mr(struct ib_pd *ib_pd, u64 length, u64 virt_addr,
4488 int mr_access_flags, struct ib_umem *umem)
4489 {
4490 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
4491 struct bnxt_re_dev *rdev = pd->rdev;
4492 unsigned long page_size;
4493 struct bnxt_re_mr *mr;
4494 int umem_pgs, rc;
4495 u32 active_mrs;
4496
4497 if (length > BNXT_RE_MAX_MR_SIZE) {
4498 ibdev_err(&rdev->ibdev, "MR Size: %lld > Max supported:%lld\n",
4499 length, BNXT_RE_MAX_MR_SIZE);
4500 return ERR_PTR(-ENOMEM);
4501 }
4502
4503 page_size = ib_umem_find_best_pgsz(umem, BNXT_RE_PAGE_SIZE_SUPPORTED, virt_addr);
4504 if (!page_size) {
4505 ibdev_err(&rdev->ibdev, "umem page size unsupported!");
4506 return ERR_PTR(-EINVAL);
4507 }
4508
4509 mr = kzalloc_obj(*mr);
4510 if (!mr)
4511 return ERR_PTR(-ENOMEM);
4512
4513 mr->rdev = rdev;
4514 mr->qplib_mr.pd = &pd->qplib_pd;
4515 mr->qplib_mr.access_flags = __from_ib_access_flags(mr_access_flags);
4516 mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_MR;
4517
4518 if (!_is_alloc_mr_unified(rdev->dev_attr->dev_cap_flags)) {
4519 rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr);
4520 if (rc) {
4521 ibdev_err(&rdev->ibdev, "Failed to allocate MR rc = %d", rc);
4522 rc = -EIO;
4523 goto free_mr;
4524 }
4525 /* The fixed portion of the rkey is the same as the lkey */
4526 mr->ib_mr.rkey = mr->qplib_mr.rkey;
4527 } else {
4528 mr->qplib_mr.flags = CMDQ_REGISTER_MR_FLAGS_ALLOC_MR;
4529 }
4530 mr->ib_umem = umem;
4531 mr->qplib_mr.va = virt_addr;
4532 mr->qplib_mr.total_size = length;
4533
4534 if (mr_access_flags & IB_ACCESS_RELAXED_ORDERING)
4535 bnxt_re_check_and_set_relaxed_ordering(rdev, &mr->qplib_mr);
4536
4537 umem_pgs = ib_umem_num_dma_blocks(umem, page_size);
4538 rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, umem,
4539 umem_pgs, page_size,
4540 _is_alloc_mr_unified(rdev->dev_attr->dev_cap_flags));
4541 if (rc) {
4542 ibdev_err(&rdev->ibdev, "Failed to register user MR - rc = %d\n", rc);
4543 rc = -EIO;
4544 goto free_mrw;
4545 }
4546
4547 mr->ib_mr.lkey = mr->qplib_mr.lkey;
4548 mr->ib_mr.rkey = mr->qplib_mr.lkey;
4549 active_mrs = atomic_inc_return(&rdev->stats.res.mr_count);
4550 if (active_mrs > rdev->stats.res.mr_watermark)
4551 rdev->stats.res.mr_watermark = active_mrs;
4552
4553 return &mr->ib_mr;
4554
4555 free_mrw:
4556 bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr);
4557 free_mr:
4558 kfree(mr);
4559 return ERR_PTR(rc);
4560 }
4561
bnxt_re_reg_user_mr(struct ib_pd * ib_pd,u64 start,u64 length,u64 virt_addr,int mr_access_flags,struct ib_dmah * dmah,struct ib_udata * udata)4562 struct ib_mr *bnxt_re_reg_user_mr(struct ib_pd *ib_pd, u64 start, u64 length,
4563 u64 virt_addr, int mr_access_flags,
4564 struct ib_dmah *dmah,
4565 struct ib_udata *udata)
4566 {
4567 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
4568 struct bnxt_re_dev *rdev = pd->rdev;
4569 struct ib_umem *umem;
4570 struct ib_mr *ib_mr;
4571 int ret;
4572
4573 ret = ib_is_udata_in_empty(udata);
4574 if (ret)
4575 return ERR_PTR(ret);
4576
4577 if (dmah)
4578 return ERR_PTR(-EOPNOTSUPP);
4579
4580 umem = ib_umem_get(&rdev->ibdev, start, length, mr_access_flags);
4581 if (IS_ERR(umem))
4582 return ERR_CAST(umem);
4583
4584 ib_mr = __bnxt_re_user_reg_mr(ib_pd, length, virt_addr, mr_access_flags, umem);
4585 if (IS_ERR(ib_mr))
4586 ib_umem_release(umem);
4587 return ib_mr;
4588 }
4589
bnxt_re_reg_user_mr_dmabuf(struct ib_pd * ib_pd,u64 start,u64 length,u64 virt_addr,int fd,int mr_access_flags,struct ib_dmah * dmah,struct uverbs_attr_bundle * attrs)4590 struct ib_mr *bnxt_re_reg_user_mr_dmabuf(struct ib_pd *ib_pd, u64 start,
4591 u64 length, u64 virt_addr, int fd,
4592 int mr_access_flags,
4593 struct ib_dmah *dmah,
4594 struct uverbs_attr_bundle *attrs)
4595 {
4596 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
4597 struct bnxt_re_dev *rdev = pd->rdev;
4598 struct ib_umem_dmabuf *umem_dmabuf;
4599 struct ib_umem *umem;
4600 struct ib_mr *ib_mr;
4601
4602 if (dmah)
4603 return ERR_PTR(-EOPNOTSUPP);
4604
4605 umem_dmabuf = ib_umem_dmabuf_get_pinned(&rdev->ibdev, start, length,
4606 fd, mr_access_flags);
4607 if (IS_ERR(umem_dmabuf))
4608 return ERR_CAST(umem_dmabuf);
4609
4610 umem = &umem_dmabuf->umem;
4611
4612 ib_mr = __bnxt_re_user_reg_mr(ib_pd, length, virt_addr, mr_access_flags, umem);
4613 if (IS_ERR(ib_mr))
4614 ib_umem_release(umem);
4615 return ib_mr;
4616 }
4617
bnxt_re_alloc_ucontext(struct ib_ucontext * ctx,struct ib_udata * udata)4618 int bnxt_re_alloc_ucontext(struct ib_ucontext *ctx, struct ib_udata *udata)
4619 {
4620 struct ib_device *ibdev = ctx->device;
4621 struct bnxt_re_ucontext *uctx =
4622 container_of(ctx, struct bnxt_re_ucontext, ib_uctx);
4623 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
4624 struct bnxt_qplib_dev_attr *dev_attr = rdev->dev_attr;
4625 struct bnxt_re_user_mmap_entry *entry;
4626 struct bnxt_re_uctx_resp resp = {};
4627 struct bnxt_re_uctx_req ureq = {};
4628 u32 chip_met_rev_num = 0;
4629 int rc;
4630
4631 ibdev_dbg(ibdev, "ABI version requested %u", ibdev->ops.uverbs_abi_ver);
4632
4633 if (ibdev->ops.uverbs_abi_ver != BNXT_RE_ABI_VERSION) {
4634 ibdev_dbg(ibdev, " is different from the device %d ",
4635 BNXT_RE_ABI_VERSION);
4636 return -EPERM;
4637 }
4638
4639 uctx->rdev = rdev;
4640
4641 uctx->shpg = (void *)get_zeroed_page(GFP_KERNEL);
4642 if (!uctx->shpg) {
4643 rc = -ENOMEM;
4644 goto fail;
4645 }
4646 spin_lock_init(&uctx->sh_lock);
4647
4648 resp.comp_mask = BNXT_RE_UCNTX_CMASK_HAVE_CCTX;
4649 chip_met_rev_num = rdev->chip_ctx->chip_num;
4650 chip_met_rev_num |= ((u32)rdev->chip_ctx->chip_rev & 0xFF) <<
4651 BNXT_RE_CHIP_ID0_CHIP_REV_SFT;
4652 chip_met_rev_num |= ((u32)rdev->chip_ctx->chip_metal & 0xFF) <<
4653 BNXT_RE_CHIP_ID0_CHIP_MET_SFT;
4654 resp.chip_id0 = chip_met_rev_num;
4655 /*Temp, Use xa_alloc instead */
4656 resp.dev_id = rdev->en_dev->pdev->devfn;
4657 resp.max_qp = rdev->qplib_ctx.qpc_count;
4658 resp.pg_size = PAGE_SIZE;
4659 resp.cqe_sz = sizeof(struct cq_base);
4660 resp.max_cqd = dev_attr->max_cq_wqes;
4661
4662 if (rdev->chip_ctx->modes.db_push)
4663 resp.comp_mask |= BNXT_RE_UCNTX_CMASK_WC_DPI_ENABLED;
4664
4665 entry = bnxt_re_mmap_entry_insert(uctx, 0, BNXT_RE_MMAP_SH_PAGE, NULL);
4666 if (!entry) {
4667 rc = -ENOMEM;
4668 goto cfail;
4669 }
4670 uctx->shpage_mmap = &entry->rdma_entry;
4671 if (rdev->pacing.dbr_pacing)
4672 resp.comp_mask |= BNXT_RE_UCNTX_CMASK_DBR_PACING_ENABLED;
4673
4674 if (_is_host_msn_table(rdev->qplib_res.dattr->dev_cap_flags2))
4675 resp.comp_mask |= BNXT_RE_UCNTX_CMASK_MSN_TABLE_ENABLED;
4676
4677 if (_is_modify_qp_rate_limit_supported(dev_attr->dev_cap_flags2))
4678 resp.comp_mask |= BNXT_RE_UCNTX_CMASK_QP_RATE_LIMIT_ENABLED;
4679
4680 if (udata->inlen) {
4681 rc = ib_copy_validate_udata_in_cm(
4682 udata, ureq, comp_mask,
4683 BNXT_RE_COMP_MASK_REQ_UCNTX_POW2_SUPPORT |
4684 BNXT_RE_COMP_MASK_REQ_UCNTX_VAR_WQE_SUPPORT);
4685 if (rc)
4686 goto cfail;
4687 if (ureq.comp_mask & BNXT_RE_COMP_MASK_REQ_UCNTX_POW2_SUPPORT) {
4688 resp.comp_mask |= BNXT_RE_UCNTX_CMASK_POW2_DISABLED;
4689 uctx->cmask |= BNXT_RE_UCNTX_CAP_POW2_DISABLED;
4690 }
4691 if (ureq.comp_mask & BNXT_RE_COMP_MASK_REQ_UCNTX_VAR_WQE_SUPPORT) {
4692 resp.comp_mask |= BNXT_RE_UCNTX_CMASK_HAVE_MODE;
4693 resp.mode = rdev->chip_ctx->modes.wqe_mode;
4694 if (resp.mode == BNXT_QPLIB_WQE_MODE_VARIABLE)
4695 uctx->cmask |= BNXT_RE_UCNTX_CAP_VAR_WQE_ENABLED;
4696 }
4697 }
4698
4699 rc = ib_respond_udata(udata, resp);
4700 if (rc)
4701 goto cfail;
4702
4703 return 0;
4704 cfail:
4705 free_page((unsigned long)uctx->shpg);
4706 uctx->shpg = NULL;
4707 fail:
4708 return rc;
4709 }
4710
bnxt_re_dealloc_ucontext(struct ib_ucontext * ib_uctx)4711 void bnxt_re_dealloc_ucontext(struct ib_ucontext *ib_uctx)
4712 {
4713 struct bnxt_re_ucontext *uctx = container_of(ib_uctx,
4714 struct bnxt_re_ucontext,
4715 ib_uctx);
4716
4717 struct bnxt_re_dev *rdev = uctx->rdev;
4718
4719 rdma_user_mmap_entry_remove(uctx->shpage_mmap);
4720 uctx->shpage_mmap = NULL;
4721 if (uctx->shpg)
4722 free_page((unsigned long)uctx->shpg);
4723
4724 if (uctx->dpi.dbr) {
4725 /* Free DPI only if this is the first PD allocated by the
4726 * application and mark the context dpi as NULL
4727 */
4728 bnxt_qplib_dealloc_dpi(&rdev->qplib_res, &uctx->dpi);
4729 uctx->dpi.dbr = NULL;
4730 }
4731 }
4732
bnxt_re_setup_vnic(struct bnxt_re_dev * rdev,struct bnxt_re_qp * qp)4733 static int bnxt_re_setup_vnic(struct bnxt_re_dev *rdev, struct bnxt_re_qp *qp)
4734 {
4735 int rc;
4736
4737 rc = bnxt_re_hwrm_alloc_vnic(rdev);
4738 if (rc)
4739 return rc;
4740
4741 rc = bnxt_re_hwrm_cfg_vnic(rdev, qp->qplib_qp.id);
4742 if (rc)
4743 goto out_free_vnic;
4744
4745 return 0;
4746 out_free_vnic:
4747 bnxt_re_hwrm_free_vnic(rdev);
4748 return rc;
4749 }
4750
bnxt_re_create_flow(struct ib_qp * ib_qp,struct ib_flow_attr * attr,struct ib_udata * udata)4751 struct ib_flow *bnxt_re_create_flow(struct ib_qp *ib_qp,
4752 struct ib_flow_attr *attr,
4753 struct ib_udata *udata)
4754 {
4755 struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
4756 struct bnxt_re_dev *rdev = qp->rdev;
4757 struct bnxt_re_flow *flow;
4758 int rc;
4759
4760 rc = ib_is_udata_in_empty(udata);
4761 if (rc)
4762 return ERR_PTR(rc);
4763
4764 if (attr->type != IB_FLOW_ATTR_SNIFFER ||
4765 !rdev->rcfw.roce_mirror)
4766 return ERR_PTR(-EOPNOTSUPP);
4767
4768 mutex_lock(&rdev->qp_lock);
4769 if (rdev->sniffer_flow_created) {
4770 ibdev_err(&rdev->ibdev, "RoCE Mirroring is already Configured\n");
4771 mutex_unlock(&rdev->qp_lock);
4772 return ERR_PTR(-EBUSY);
4773 }
4774
4775 flow = kzalloc_obj(*flow);
4776 if (!flow) {
4777 mutex_unlock(&rdev->qp_lock);
4778 return ERR_PTR(-ENOMEM);
4779 }
4780
4781 flow->rdev = rdev;
4782
4783 rc = bnxt_re_setup_vnic(rdev, qp);
4784 if (rc)
4785 goto out_free_flow;
4786
4787 rc = bnxt_qplib_create_flow(&rdev->qplib_res);
4788 if (rc)
4789 goto out_free_vnic;
4790
4791 rdev->sniffer_flow_created = 1;
4792 mutex_unlock(&rdev->qp_lock);
4793
4794 return &flow->ib_flow;
4795
4796 out_free_vnic:
4797 bnxt_re_hwrm_free_vnic(rdev);
4798 out_free_flow:
4799 mutex_unlock(&rdev->qp_lock);
4800 kfree(flow);
4801 return ERR_PTR(rc);
4802 }
4803
bnxt_re_destroy_flow(struct ib_flow * flow_id)4804 int bnxt_re_destroy_flow(struct ib_flow *flow_id)
4805 {
4806 struct bnxt_re_flow *flow =
4807 container_of(flow_id, struct bnxt_re_flow, ib_flow);
4808 struct bnxt_re_dev *rdev = flow->rdev;
4809 int rc;
4810
4811 mutex_lock(&rdev->qp_lock);
4812 rc = bnxt_qplib_destroy_flow(&rdev->qplib_res);
4813 if (rc)
4814 ibdev_dbg(&rdev->ibdev, "failed to destroy_flow rc = %d\n", rc);
4815 rdev->sniffer_flow_created = 0;
4816
4817 bnxt_re_hwrm_free_vnic(rdev);
4818 mutex_unlock(&rdev->qp_lock);
4819 kfree(flow);
4820
4821 return rc;
4822 }
4823
4824 /* Helper function to mmap the virtual memory from user app */
bnxt_re_mmap(struct ib_ucontext * ib_uctx,struct vm_area_struct * vma)4825 int bnxt_re_mmap(struct ib_ucontext *ib_uctx, struct vm_area_struct *vma)
4826 {
4827 struct bnxt_re_ucontext *uctx = container_of(ib_uctx,
4828 struct bnxt_re_ucontext,
4829 ib_uctx);
4830 struct bnxt_re_user_mmap_entry *bnxt_entry;
4831 struct rdma_user_mmap_entry *rdma_entry;
4832 int ret = 0;
4833 u64 pfn;
4834
4835 rdma_entry = rdma_user_mmap_entry_get(&uctx->ib_uctx, vma);
4836 if (!rdma_entry)
4837 return -EINVAL;
4838
4839 bnxt_entry = container_of(rdma_entry, struct bnxt_re_user_mmap_entry,
4840 rdma_entry);
4841
4842 switch (bnxt_entry->mmap_flag) {
4843 case BNXT_RE_MMAP_WC_DB:
4844 pfn = bnxt_entry->mem_offset >> PAGE_SHIFT;
4845 ret = rdma_user_mmap_io(ib_uctx, vma, pfn, PAGE_SIZE,
4846 pgprot_writecombine(vma->vm_page_prot),
4847 rdma_entry);
4848 break;
4849 case BNXT_RE_MMAP_UC_DB:
4850 pfn = bnxt_entry->mem_offset >> PAGE_SHIFT;
4851 ret = rdma_user_mmap_io(ib_uctx, vma, pfn, PAGE_SIZE,
4852 pgprot_noncached(vma->vm_page_prot),
4853 rdma_entry);
4854 break;
4855 case BNXT_RE_MMAP_SH_PAGE:
4856 ret = vm_insert_page(vma, vma->vm_start, virt_to_page(uctx->shpg));
4857 break;
4858 case BNXT_RE_MMAP_DBR_BAR:
4859 pfn = bnxt_entry->mem_offset >> PAGE_SHIFT;
4860 ret = rdma_user_mmap_io(ib_uctx, vma, pfn, PAGE_SIZE,
4861 pgprot_noncached(vma->vm_page_prot),
4862 rdma_entry);
4863 break;
4864 case BNXT_RE_MMAP_DBR_PAGE:
4865 case BNXT_RE_MMAP_TOGGLE_PAGE:
4866 /* Driver doesn't expect write access for user space */
4867 if (vma->vm_flags & VM_WRITE)
4868 ret = -EFAULT;
4869 else
4870 ret = vm_insert_page(vma, vma->vm_start,
4871 virt_to_page((void *)bnxt_entry->mem_offset));
4872 break;
4873 default:
4874 ret = -EINVAL;
4875 break;
4876 }
4877
4878 rdma_user_mmap_entry_put(rdma_entry);
4879 return ret;
4880 }
4881
bnxt_re_mmap_free(struct rdma_user_mmap_entry * rdma_entry)4882 void bnxt_re_mmap_free(struct rdma_user_mmap_entry *rdma_entry)
4883 {
4884 struct bnxt_re_user_mmap_entry *bnxt_entry;
4885
4886 bnxt_entry = container_of(rdma_entry, struct bnxt_re_user_mmap_entry,
4887 rdma_entry);
4888
4889 kfree(bnxt_entry);
4890 }
4891
bnxt_re_process_mad(struct ib_device * ibdev,int mad_flags,u32 port_num,const struct ib_wc * in_wc,const struct ib_grh * in_grh,const struct ib_mad * in_mad,struct ib_mad * out_mad,size_t * out_mad_size,u16 * out_mad_pkey_index)4892 int bnxt_re_process_mad(struct ib_device *ibdev, int mad_flags,
4893 u32 port_num, const struct ib_wc *in_wc,
4894 const struct ib_grh *in_grh,
4895 const struct ib_mad *in_mad, struct ib_mad *out_mad,
4896 size_t *out_mad_size, u16 *out_mad_pkey_index)
4897 {
4898 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
4899 struct ib_class_port_info cpi = {};
4900 int ret = IB_MAD_RESULT_SUCCESS;
4901 int rc = 0;
4902
4903 if (in_mad->mad_hdr.mgmt_class != IB_MGMT_CLASS_PERF_MGMT)
4904 return ret;
4905
4906 switch (in_mad->mad_hdr.attr_id) {
4907 case IB_PMA_CLASS_PORT_INFO:
4908 cpi.capability_mask = IB_PMA_CLASS_CAP_EXT_WIDTH;
4909 memcpy((out_mad->data + 40), &cpi, sizeof(cpi));
4910 break;
4911 case IB_PMA_PORT_COUNTERS_EXT:
4912 rc = bnxt_re_assign_pma_port_ext_counters(rdev, out_mad);
4913 break;
4914 case IB_PMA_PORT_COUNTERS:
4915 rc = bnxt_re_assign_pma_port_counters(rdev, out_mad);
4916 break;
4917 default:
4918 rc = -EINVAL;
4919 break;
4920 }
4921 if (rc)
4922 return IB_MAD_RESULT_FAILURE;
4923 ret |= IB_MAD_RESULT_REPLY;
4924 return ret;
4925 }
4926