1 /*
2 * Copyright (c) 2015-2024, Broadcom. All rights reserved. The term
3 * Broadcom refers to Broadcom Limited and/or its subsidiaries.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 *
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in
13 * the documentation and/or other materials provided with the
14 * distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
17 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
18 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
20 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
23 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
24 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
25 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
26 * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 *
28 * Description: IB Verbs interpreter
29 */
30
31 #include <linux/if_ether.h>
32 #include <linux/etherdevice.h>
33 #include <rdma/uverbs_ioctl.h>
34
35 #include "bnxt_re.h"
36 #include "ib_verbs.h"
37
38 static inline
get_ib_umem_sgl(struct ib_umem * umem,u32 * nmap)39 struct scatterlist *get_ib_umem_sgl(struct ib_umem *umem, u32 *nmap)
40 {
41
42 *nmap = umem->nmap;
43 return umem->sg_head.sgl;
44 }
45
bnxt_re_peer_mem_release(struct ib_umem * umem)46 static inline void bnxt_re_peer_mem_release(struct ib_umem *umem)
47 {
48 dev_dbg(NULL, "ib_umem_release getting invoked \n");
49 ib_umem_release(umem);
50 }
51
bnxt_re_resolve_dmac_task(struct work_struct * work)52 void bnxt_re_resolve_dmac_task(struct work_struct *work)
53 {
54 int rc = -1;
55 struct bnxt_re_dev *rdev;
56 struct ib_ah_attr *ah_attr;
57 struct bnxt_re_resolve_dmac_work *dmac_work =
58 container_of(work, struct bnxt_re_resolve_dmac_work, work);
59
60 rdev = dmac_work->rdev;
61 ah_attr = dmac_work->ah_attr;
62 rc = ib_resolve_eth_dmac(&rdev->ibdev, ah_attr);
63 if (rc)
64 dev_err(rdev_to_dev(dmac_work->rdev),
65 "Failed to resolve dest mac rc = %d\n", rc);
66 atomic_set(&dmac_work->status_wait, rc << 8);
67 }
68
__from_ib_access_flags(int iflags)69 static int __from_ib_access_flags(int iflags)
70 {
71 int qflags = 0;
72
73 if (iflags & IB_ACCESS_LOCAL_WRITE)
74 qflags |= BNXT_QPLIB_ACCESS_LOCAL_WRITE;
75 if (iflags & IB_ACCESS_REMOTE_READ)
76 qflags |= BNXT_QPLIB_ACCESS_REMOTE_READ;
77 if (iflags & IB_ACCESS_REMOTE_WRITE)
78 qflags |= BNXT_QPLIB_ACCESS_REMOTE_WRITE;
79 if (iflags & IB_ACCESS_REMOTE_ATOMIC)
80 qflags |= BNXT_QPLIB_ACCESS_REMOTE_ATOMIC;
81 if (iflags & IB_ACCESS_MW_BIND)
82 qflags |= BNXT_QPLIB_ACCESS_MW_BIND;
83 if (iflags & IB_ZERO_BASED)
84 qflags |= BNXT_QPLIB_ACCESS_ZERO_BASED;
85 if (iflags & IB_ACCESS_ON_DEMAND)
86 qflags |= BNXT_QPLIB_ACCESS_ON_DEMAND;
87 return qflags;
88 };
89
__to_ib_access_flags(int qflags)90 static enum ib_access_flags __to_ib_access_flags(int qflags)
91 {
92 enum ib_access_flags iflags = 0;
93
94 if (qflags & BNXT_QPLIB_ACCESS_LOCAL_WRITE)
95 iflags |= IB_ACCESS_LOCAL_WRITE;
96 if (qflags & BNXT_QPLIB_ACCESS_REMOTE_WRITE)
97 iflags |= IB_ACCESS_REMOTE_WRITE;
98 if (qflags & BNXT_QPLIB_ACCESS_REMOTE_READ)
99 iflags |= IB_ACCESS_REMOTE_READ;
100 if (qflags & BNXT_QPLIB_ACCESS_REMOTE_ATOMIC)
101 iflags |= IB_ACCESS_REMOTE_ATOMIC;
102 if (qflags & BNXT_QPLIB_ACCESS_MW_BIND)
103 iflags |= IB_ACCESS_MW_BIND;
104 if (qflags & BNXT_QPLIB_ACCESS_ZERO_BASED)
105 iflags |= IB_ZERO_BASED;
106 if (qflags & BNXT_QPLIB_ACCESS_ON_DEMAND)
107 iflags |= IB_ACCESS_ON_DEMAND;
108 return iflags;
109 };
110
bnxt_re_copy_to_udata(struct bnxt_re_dev * rdev,void * data,int len,struct ib_udata * udata)111 static int bnxt_re_copy_to_udata(struct bnxt_re_dev *rdev, void *data,
112 int len, struct ib_udata *udata)
113 {
114 int rc;
115
116 rc = ib_copy_to_udata(udata, data, len);
117 if (rc)
118 dev_err(rdev_to_dev(rdev),
119 "ucontext copy failed from %ps rc %d\n",
120 __builtin_return_address(0), rc);
121
122 return rc;
123 }
124
bnxt_re_get_netdev(struct ib_device * ibdev,u8 port_num)125 struct ifnet *bnxt_re_get_netdev(struct ib_device *ibdev,
126 u8 port_num)
127 {
128 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
129 struct ifnet *netdev = NULL;
130
131 rcu_read_lock();
132
133 if (!rdev || !rdev->netdev)
134 goto end;
135
136 netdev = rdev->netdev;
137
138 /* In case of active-backup bond mode, return active slave */
139 if (netdev)
140 dev_hold(netdev);
141
142 end:
143 rcu_read_unlock();
144 return netdev;
145 }
146
bnxt_re_query_device(struct ib_device * ibdev,struct ib_device_attr * ib_attr,struct ib_udata * udata)147 int bnxt_re_query_device(struct ib_device *ibdev,
148 struct ib_device_attr *ib_attr,
149 struct ib_udata *udata)
150 {
151 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
152 struct bnxt_qplib_dev_attr *dev_attr = rdev->dev_attr;
153
154 memset(ib_attr, 0, sizeof(*ib_attr));
155
156 memcpy(&ib_attr->fw_ver, dev_attr->fw_ver, 4);
157 bnxt_qplib_get_guid(rdev->dev_addr, (u8 *)&ib_attr->sys_image_guid);
158 ib_attr->max_mr_size = BNXT_RE_MAX_MR_SIZE;
159 ib_attr->page_size_cap = dev_attr->page_size_cap;
160 ib_attr->vendor_id = rdev->en_dev->pdev->vendor;
161 ib_attr->vendor_part_id = rdev->en_dev->pdev->device;
162 ib_attr->hw_ver = rdev->en_dev->pdev->subsystem_device;
163 ib_attr->max_qp = dev_attr->max_qp;
164 ib_attr->max_qp_wr = dev_attr->max_qp_wqes;
165 /*
166 * Read and set from the module param 'min_tx_depth'
167 * only once after the driver load
168 */
169 if (rdev->min_tx_depth == 1 &&
170 min_tx_depth < dev_attr->max_qp_wqes)
171 rdev->min_tx_depth = min_tx_depth;
172 ib_attr->device_cap_flags =
173 IB_DEVICE_CURR_QP_STATE_MOD
174 | IB_DEVICE_RC_RNR_NAK_GEN
175 | IB_DEVICE_SHUTDOWN_PORT
176 | IB_DEVICE_SYS_IMAGE_GUID
177 | IB_DEVICE_LOCAL_DMA_LKEY
178 | IB_DEVICE_RESIZE_MAX_WR
179 | IB_DEVICE_PORT_ACTIVE_EVENT
180 | IB_DEVICE_N_NOTIFY_CQ
181 | IB_DEVICE_MEM_WINDOW
182 | IB_DEVICE_MEM_WINDOW_TYPE_2B
183 | IB_DEVICE_MEM_MGT_EXTENSIONS;
184 ib_attr->max_send_sge = dev_attr->max_qp_sges;
185 ib_attr->max_recv_sge = dev_attr->max_qp_sges;
186 ib_attr->max_sge_rd = dev_attr->max_qp_sges;
187 ib_attr->max_cq = dev_attr->max_cq;
188 ib_attr->max_cqe = dev_attr->max_cq_wqes;
189 ib_attr->max_mr = dev_attr->max_mr;
190 ib_attr->max_pd = dev_attr->max_pd;
191 ib_attr->max_qp_rd_atom = dev_attr->max_qp_rd_atom;
192 ib_attr->max_qp_init_rd_atom = dev_attr->max_qp_init_rd_atom;
193 if (dev_attr->is_atomic) {
194 ib_attr->atomic_cap = IB_ATOMIC_GLOB;
195 ib_attr->masked_atomic_cap = IB_ATOMIC_GLOB;
196 }
197 ib_attr->max_ee_rd_atom = 0;
198 ib_attr->max_res_rd_atom = 0;
199 ib_attr->max_ee_init_rd_atom = 0;
200 ib_attr->max_ee = 0;
201 ib_attr->max_rdd = 0;
202 ib_attr->max_mw = dev_attr->max_mw;
203 ib_attr->max_raw_ipv6_qp = 0;
204 ib_attr->max_raw_ethy_qp = dev_attr->max_raw_ethy_qp;
205 ib_attr->max_mcast_grp = 0;
206 ib_attr->max_mcast_qp_attach = 0;
207 ib_attr->max_total_mcast_qp_attach = 0;
208 ib_attr->max_ah = dev_attr->max_ah;
209 ib_attr->max_srq = dev_attr->max_srq;
210 ib_attr->max_srq_wr = dev_attr->max_srq_wqes;
211 ib_attr->max_srq_sge = dev_attr->max_srq_sges;
212
213 ib_attr->max_fast_reg_page_list_len = MAX_PBL_LVL_1_PGS;
214 ib_attr->max_pkeys = 1;
215 ib_attr->local_ca_ack_delay = BNXT_RE_DEFAULT_ACK_DELAY;
216 ib_attr->sig_prot_cap = 0;
217 ib_attr->sig_guard_cap = 0;
218 ib_attr->odp_caps.general_caps = 0;
219
220 return 0;
221 }
222
bnxt_re_modify_device(struct ib_device * ibdev,int device_modify_mask,struct ib_device_modify * device_modify)223 int bnxt_re_modify_device(struct ib_device *ibdev,
224 int device_modify_mask,
225 struct ib_device_modify *device_modify)
226 {
227 dev_dbg(rdev_to_dev(rdev), "Modify device with mask 0x%x\n",
228 device_modify_mask);
229
230 switch (device_modify_mask) {
231 case IB_DEVICE_MODIFY_SYS_IMAGE_GUID:
232 /* Modify the GUID requires the modification of the GID table */
233 /* GUID should be made as READ-ONLY */
234 break;
235 case IB_DEVICE_MODIFY_NODE_DESC:
236 /* Node Desc should be made as READ-ONLY */
237 break;
238 default:
239 break;
240 }
241 return 0;
242 }
243
__to_ib_speed_width(u32 espeed,u8 lanes,u8 * speed,u8 * width)244 static void __to_ib_speed_width(u32 espeed, u8 lanes, u8 *speed, u8 *width)
245 {
246 if (!lanes) {
247 switch (espeed) {
248 case SPEED_1000:
249 *speed = IB_SPEED_SDR;
250 *width = IB_WIDTH_1X;
251 break;
252 case SPEED_10000:
253 *speed = IB_SPEED_QDR;
254 *width = IB_WIDTH_1X;
255 break;
256 case SPEED_20000:
257 *speed = IB_SPEED_DDR;
258 *width = IB_WIDTH_4X;
259 break;
260 case SPEED_25000:
261 *speed = IB_SPEED_EDR;
262 *width = IB_WIDTH_1X;
263 break;
264 case SPEED_40000:
265 *speed = IB_SPEED_QDR;
266 *width = IB_WIDTH_4X;
267 break;
268 case SPEED_50000:
269 *speed = IB_SPEED_EDR;
270 *width = IB_WIDTH_2X;
271 break;
272 case SPEED_100000:
273 *speed = IB_SPEED_EDR;
274 *width = IB_WIDTH_4X;
275 break;
276 case SPEED_200000:
277 *speed = IB_SPEED_HDR;
278 *width = IB_WIDTH_4X;
279 break;
280 case SPEED_400000:
281 *speed = IB_SPEED_NDR;
282 *width = IB_WIDTH_4X;
283 break;
284 default:
285 *speed = IB_SPEED_SDR;
286 *width = IB_WIDTH_1X;
287 break;
288 }
289 return;
290 }
291
292 switch (lanes) {
293 case 1:
294 *width = IB_WIDTH_1X;
295 break;
296 case 2:
297 *width = IB_WIDTH_2X;
298 break;
299 case 4:
300 *width = IB_WIDTH_4X;
301 break;
302 case 8:
303 *width = IB_WIDTH_8X;
304 break;
305 case 12:
306 *width = IB_WIDTH_12X;
307 break;
308 default:
309 *width = IB_WIDTH_1X;
310 }
311
312 switch (espeed / lanes) {
313 case SPEED_2500:
314 *speed = IB_SPEED_SDR;
315 break;
316 case SPEED_5000:
317 *speed = IB_SPEED_DDR;
318 break;
319 case SPEED_10000:
320 *speed = IB_SPEED_FDR10;
321 break;
322 case SPEED_14000:
323 *speed = IB_SPEED_FDR;
324 break;
325 case SPEED_25000:
326 *speed = IB_SPEED_EDR;
327 break;
328 case SPEED_50000:
329 *speed = IB_SPEED_HDR;
330 break;
331 case SPEED_100000:
332 *speed = IB_SPEED_NDR;
333 break;
334 default:
335 *speed = IB_SPEED_SDR;
336 }
337 }
338
339 /* Port */
bnxt_re_query_port(struct ib_device * ibdev,u8 port_num,struct ib_port_attr * port_attr)340 int bnxt_re_query_port(struct ib_device *ibdev, u8 port_num,
341 struct ib_port_attr *port_attr)
342 {
343 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
344 struct bnxt_qplib_dev_attr *dev_attr = rdev->dev_attr;
345 u8 active_speed = 0, active_width = 0;
346
347 dev_dbg(rdev_to_dev(rdev), "QUERY PORT with port_num 0x%x\n", port_num);
348 memset(port_attr, 0, sizeof(*port_attr));
349
350 port_attr->phys_state = IB_PORT_PHYS_STATE_DISABLED;
351 port_attr->state = bnxt_re_get_link_state(rdev);
352 if (port_attr->state == IB_PORT_ACTIVE)
353 port_attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP;
354 port_attr->max_mtu = IB_MTU_4096;
355 port_attr->active_mtu = iboe_get_mtu(if_getmtu(rdev->netdev));
356 port_attr->gid_tbl_len = dev_attr->max_sgid;
357 port_attr->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_REINIT_SUP |
358 IB_PORT_DEVICE_MGMT_SUP |
359 IB_PORT_VENDOR_CLASS_SUP |
360 IB_PORT_IP_BASED_GIDS;
361
362 port_attr->max_msg_sz = (u32)BNXT_RE_MAX_MR_SIZE_LOW;
363 port_attr->bad_pkey_cntr = 0;
364 port_attr->qkey_viol_cntr = 0;
365 port_attr->pkey_tbl_len = dev_attr->max_pkey;
366 port_attr->lid = 0;
367 port_attr->sm_lid = 0;
368 port_attr->lmc = 0;
369 port_attr->max_vl_num = 4;
370 port_attr->sm_sl = 0;
371 port_attr->subnet_timeout = 0;
372 port_attr->init_type_reply = 0;
373 rdev->espeed = rdev->en_dev->espeed;
374 rdev->lanes = rdev->en_dev->lanes;
375
376 if (test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags))
377 __to_ib_speed_width(rdev->espeed, rdev->lanes, &active_speed,
378 &active_width);
379
380 port_attr->active_speed = active_speed;
381 port_attr->active_width = active_width;
382
383 return 0;
384 }
385
bnxt_re_modify_port(struct ib_device * ibdev,u8 port_num,int port_modify_mask,struct ib_port_modify * port_modify)386 int bnxt_re_modify_port(struct ib_device *ibdev, u8 port_num,
387 int port_modify_mask,
388 struct ib_port_modify *port_modify)
389 {
390 dev_dbg(rdev_to_dev(rdev), "Modify port with mask 0x%x\n",
391 port_modify_mask);
392
393 switch (port_modify_mask) {
394 case IB_PORT_SHUTDOWN:
395 break;
396 case IB_PORT_INIT_TYPE:
397 break;
398 case IB_PORT_RESET_QKEY_CNTR:
399 break;
400 default:
401 break;
402 }
403 return 0;
404 }
405
bnxt_re_get_port_immutable(struct ib_device * ibdev,u8 port_num,struct ib_port_immutable * immutable)406 int bnxt_re_get_port_immutable(struct ib_device *ibdev, u8 port_num,
407 struct ib_port_immutable *immutable)
408 {
409 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
410 struct ib_port_attr port_attr;
411
412 if (bnxt_re_query_port(ibdev, port_num, &port_attr))
413 return -EINVAL;
414
415 immutable->pkey_tbl_len = port_attr.pkey_tbl_len;
416 immutable->gid_tbl_len = port_attr.gid_tbl_len;
417 if (rdev->roce_mode == BNXT_RE_FLAG_ROCEV1_CAP)
418 immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE;
419 else if (rdev->roce_mode == BNXT_RE_FLAG_ROCEV2_CAP)
420 immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
421 else
422 immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE |
423 RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
424 immutable->max_mad_size = IB_MGMT_MAD_SIZE;
425 return 0;
426 }
427
bnxt_re_compat_qfwstr(void)428 void bnxt_re_compat_qfwstr(void)
429 {
430 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
431
432 sprintf(str, "%d.%d.%d.%d", rdev->dev_attr->fw_ver[0],
433 rdev->dev_attr->fw_ver[1], rdev->dev_attr->fw_ver[2],
434 rdev->dev_attr->fw_ver[3]);
435 }
436
bnxt_re_query_pkey(struct ib_device * ibdev,u8 port_num,u16 index,u16 * pkey)437 int bnxt_re_query_pkey(struct ib_device *ibdev, u8 port_num,
438 u16 index, u16 *pkey)
439 {
440 if (index > 0)
441 return -EINVAL;
442
443 *pkey = IB_DEFAULT_PKEY_FULL;
444
445 return 0;
446 }
447
bnxt_re_query_gid(struct ib_device * ibdev,u8 port_num,int index,union ib_gid * gid)448 int bnxt_re_query_gid(struct ib_device *ibdev, u8 port_num,
449 int index, union ib_gid *gid)
450 {
451 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
452 int rc = 0;
453
454 /* Ignore port_num */
455 memset(gid, 0, sizeof(*gid));
456 rc = bnxt_qplib_get_sgid(&rdev->qplib_res,
457 &rdev->qplib_res.sgid_tbl, index,
458 (struct bnxt_qplib_gid *)gid);
459 return rc;
460 }
461
bnxt_re_del_gid(struct ib_device * ibdev,u8 port_num,unsigned int index,void ** context)462 int bnxt_re_del_gid(struct ib_device *ibdev, u8 port_num,
463 unsigned int index, void **context)
464 {
465 int rc = 0;
466 struct bnxt_re_gid_ctx *ctx, **ctx_tbl;
467 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
468 struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl;
469 struct bnxt_qplib_gid *gid_to_del;
470 u16 vlan_id = 0xFFFF;
471
472 /* Delete the entry from the hardware */
473 ctx = *context;
474 if (!ctx) {
475 dev_err(rdev_to_dev(rdev), "GID entry has no ctx?!\n");
476 return -EINVAL;
477 }
478 if (sgid_tbl && sgid_tbl->active) {
479 if (ctx->idx >= sgid_tbl->max) {
480 dev_dbg(rdev_to_dev(rdev), "GID index out of range?!\n");
481 return -EINVAL;
482 }
483 gid_to_del = &sgid_tbl->tbl[ctx->idx].gid;
484 vlan_id = sgid_tbl->tbl[ctx->idx].vlan_id;
485 ctx->refcnt--;
486 /* DEL_GID is called via WQ context(netdevice_event_work_handler)
487 * or via the ib_unregister_device path. In the former case QP1
488 * may not be destroyed yet, in which case just return as FW
489 * needs that entry to be present and will fail it's deletion.
490 * We could get invoked again after QP1 is destroyed OR get an
491 * ADD_GID call with a different GID value for the same index
492 * where we issue MODIFY_GID cmd to update the GID entry -- TBD
493 */
494 if (ctx->idx == 0 &&
495 rdma_link_local_addr((struct in6_addr *)gid_to_del) &&
496 (rdev->gsi_ctx.gsi_sqp ||
497 rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_UD)) {
498 dev_dbg(rdev_to_dev(rdev),
499 "Trying to delete GID0 while QP1 is alive\n");
500 if (!ctx->refcnt) {
501 rdev->gid_map[index] = -1;
502 ctx_tbl = sgid_tbl->ctx;
503 ctx_tbl[ctx->idx] = NULL;
504 kfree(ctx);
505 }
506 return 0;
507 }
508 rdev->gid_map[index] = -1;
509 if (!ctx->refcnt) {
510 rc = bnxt_qplib_del_sgid(sgid_tbl, gid_to_del,
511 vlan_id, true);
512 if (!rc) {
513 dev_dbg(rdev_to_dev(rdev), "GID remove success\n");
514 ctx_tbl = sgid_tbl->ctx;
515 ctx_tbl[ctx->idx] = NULL;
516 kfree(ctx);
517 } else {
518 dev_err(rdev_to_dev(rdev),
519 "Remove GID failed rc = 0x%x\n", rc);
520 }
521 }
522 } else {
523 dev_dbg(rdev_to_dev(rdev), "GID sgid_tbl does not exist!\n");
524 return -EINVAL;
525 }
526 return rc;
527 }
528
bnxt_re_add_gid(struct ib_device * ibdev,u8 port_num,unsigned int index,const union ib_gid * gid,const struct ib_gid_attr * attr,void ** context)529 int bnxt_re_add_gid(struct ib_device *ibdev, u8 port_num,
530 unsigned int index, const union ib_gid *gid,
531 const struct ib_gid_attr *attr, void **context)
532 {
533 int rc;
534 u32 tbl_idx = 0;
535 u16 vlan_id = 0xFFFF;
536 struct bnxt_re_gid_ctx *ctx, **ctx_tbl;
537 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
538 struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl;
539 if ((attr->ndev) && is_vlan_dev(attr->ndev))
540 vlan_id = vlan_dev_vlan_id(attr->ndev);
541
542 rc = bnxt_qplib_add_sgid(sgid_tbl, gid,
543 rdev->dev_addr,
544 vlan_id, true, &tbl_idx);
545 if (rc == -EALREADY) {
546 dev_dbg(rdev_to_dev(rdev), "GID %pI6 is already present\n", gid);
547 ctx_tbl = sgid_tbl->ctx;
548 if (!ctx_tbl[tbl_idx]) {
549 ctx = kmalloc(sizeof(*ctx), GFP_KERNEL);
550 if (!ctx)
551 return -ENOMEM;
552 ctx->idx = tbl_idx;
553 ctx->refcnt = 1;
554 ctx_tbl[tbl_idx] = ctx;
555 } else {
556 ctx_tbl[tbl_idx]->refcnt++;
557 }
558 *context = ctx_tbl[tbl_idx];
559 /* tbl_idx is the HW table index and index is the stack index */
560 rdev->gid_map[index] = tbl_idx;
561 return 0;
562 } else if (rc < 0) {
563 dev_err(rdev_to_dev(rdev), "Add GID failed rc = 0x%x\n", rc);
564 return rc;
565 } else {
566 ctx = kmalloc(sizeof(*ctx), GFP_KERNEL);
567 if (!ctx) {
568 dev_err(rdev_to_dev(rdev), "Add GID ctx failed\n");
569 return -ENOMEM;
570 }
571 ctx_tbl = sgid_tbl->ctx;
572 ctx->idx = tbl_idx;
573 ctx->refcnt = 1;
574 ctx_tbl[tbl_idx] = ctx;
575 /* tbl_idx is the HW table index and index is the stack index */
576 rdev->gid_map[index] = tbl_idx;
577 *context = ctx;
578 }
579 return rc;
580 }
581
bnxt_re_get_link_layer(struct ib_device * ibdev,u8 port_num)582 enum rdma_link_layer bnxt_re_get_link_layer(struct ib_device *ibdev,
583 u8 port_num)
584 {
585 return IB_LINK_LAYER_ETHERNET;
586 }
587
bnxt_re_legacy_create_fence_wqe(struct bnxt_re_pd * pd)588 static void bnxt_re_legacy_create_fence_wqe(struct bnxt_re_pd *pd)
589 {
590 struct bnxt_re_legacy_fence_data *fence = &pd->fence;
591 struct ib_mr *ib_mr = &fence->mr->ib_mr;
592 struct bnxt_qplib_swqe *wqe = &fence->bind_wqe;
593 struct bnxt_re_dev *rdev = pd->rdev;
594
595 if (!_is_chip_gen_p5_p7(rdev->chip_ctx))
596 return;
597
598 memset(wqe, 0, sizeof(*wqe));
599 wqe->type = BNXT_QPLIB_SWQE_TYPE_BIND_MW;
600 wqe->wr_id = BNXT_QPLIB_FENCE_WRID;
601 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
602 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
603 wqe->bind.zero_based = false;
604 wqe->bind.parent_l_key = ib_mr->lkey;
605 wqe->bind.va = (u64)fence->va;
606 wqe->bind.length = fence->size;
607 wqe->bind.access_cntl = __from_ib_access_flags(IB_ACCESS_REMOTE_READ);
608 wqe->bind.mw_type = SQ_BIND_MW_TYPE_TYPE1;
609
610 /* Save the initial rkey in fence structure for now;
611 * wqe->bind.r_key will be set at (re)bind time.
612 */
613 fence->bind_rkey = ib_inc_rkey(fence->mw->rkey);
614 }
615
bnxt_re_legacy_bind_fence_mw(struct bnxt_qplib_qp * qplib_qp)616 static int bnxt_re_legacy_bind_fence_mw(struct bnxt_qplib_qp *qplib_qp)
617 {
618 struct bnxt_re_qp *qp = container_of(qplib_qp, struct bnxt_re_qp,
619 qplib_qp);
620 struct ib_pd *ib_pd = qp->ib_qp.pd;
621 struct bnxt_re_pd *pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd);
622 struct bnxt_re_legacy_fence_data *fence = &pd->fence;
623 struct bnxt_qplib_swqe *fence_wqe = &fence->bind_wqe;
624 struct bnxt_qplib_swqe wqe;
625 int rc;
626
627 /* TODO: Need SQ locking here when Fence WQE
628 * posting moves up into bnxt_re from bnxt_qplib.
629 */
630 memcpy(&wqe, fence_wqe, sizeof(wqe));
631 wqe.bind.r_key = fence->bind_rkey;
632 fence->bind_rkey = ib_inc_rkey(fence->bind_rkey);
633
634 dev_dbg(rdev_to_dev(qp->rdev),
635 "Posting bind fence-WQE: rkey: %#x QP: %d PD: %p\n",
636 wqe.bind.r_key, qp->qplib_qp.id, pd);
637 rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe);
638 if (rc) {
639 dev_err(rdev_to_dev(qp->rdev), "Failed to bind fence-WQE\n");
640 return rc;
641 }
642 bnxt_qplib_post_send_db(&qp->qplib_qp);
643
644 return rc;
645 }
646
bnxt_re_legacy_create_fence_mr(struct bnxt_re_pd * pd)647 static int bnxt_re_legacy_create_fence_mr(struct bnxt_re_pd *pd)
648 {
649 int mr_access_flags = IB_ACCESS_LOCAL_WRITE | IB_ACCESS_MW_BIND;
650 struct bnxt_re_legacy_fence_data *fence = &pd->fence;
651 struct bnxt_re_dev *rdev = pd->rdev;
652 struct bnxt_qplib_mrinfo mrinfo;
653 struct bnxt_re_mr *mr = NULL;
654 struct ib_mw *ib_mw = NULL;
655 dma_addr_t dma_addr = 0;
656 u32 max_mr_count;
657 u64 pbl_tbl;
658 int rc;
659
660 if (!_is_chip_gen_p5_p7(rdev->chip_ctx))
661 return 0;
662
663 memset(&mrinfo, 0, sizeof(mrinfo));
664 /* Allocate a small chunk of memory and dma-map it */
665 fence->va = kzalloc(BNXT_RE_LEGACY_FENCE_BYTES, GFP_KERNEL);
666 if (!fence->va)
667 return -ENOMEM;
668 dma_addr = ib_dma_map_single(&rdev->ibdev, fence->va,
669 BNXT_RE_LEGACY_FENCE_BYTES,
670 DMA_BIDIRECTIONAL);
671 rc = ib_dma_mapping_error(&rdev->ibdev, dma_addr);
672 if (rc) {
673 dev_err(rdev_to_dev(rdev), "Failed to dma-map fence-MR-mem\n");
674 rc = -EIO;
675 fence->dma_addr = 0;
676 goto free_va;
677 }
678 fence->dma_addr = dma_addr;
679
680 /* Allocate a MR */
681 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
682 if (!mr)
683 goto free_dma_addr;
684 fence->mr = mr;
685 mr->rdev = rdev;
686 mr->qplib_mr.pd = &pd->qplib_pd;
687 mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR;
688 mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags);
689 if (!_is_alloc_mr_unified(rdev->qplib_res.dattr)) {
690 rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr);
691 if (rc) {
692 dev_err(rdev_to_dev(rdev), "Failed to alloc fence-HW-MR\n");
693 goto free_mr;
694 }
695 /* Register MR */
696 mr->ib_mr.lkey = mr->qplib_mr.lkey;
697 }
698 mr->qplib_mr.va = (u64)fence->va;
699 mr->qplib_mr.total_size = BNXT_RE_LEGACY_FENCE_BYTES;
700 pbl_tbl = dma_addr;
701
702 mrinfo.mrw = &mr->qplib_mr;
703 mrinfo.ptes = &pbl_tbl;
704 mrinfo.sg.npages = BNXT_RE_LEGACY_FENCE_PBL_SIZE;
705
706 mrinfo.sg.nmap = 0;
707 mrinfo.sg.sghead = 0;
708 mrinfo.sg.pgshft = PAGE_SHIFT;
709 mrinfo.sg.pgsize = PAGE_SIZE;
710 rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mrinfo, false);
711 if (rc) {
712 dev_err(rdev_to_dev(rdev), "Failed to register fence-MR\n");
713 goto free_mr;
714 }
715 mr->ib_mr.lkey = mr->qplib_mr.lkey;
716 mr->ib_mr.rkey = mr->qplib_mr.rkey;
717 atomic_inc(&rdev->stats.rsors.mr_count);
718 max_mr_count = atomic_read(&rdev->stats.rsors.mr_count);
719 if (max_mr_count > (atomic_read(&rdev->stats.rsors.max_mr_count)))
720 atomic_set(&rdev->stats.rsors.max_mr_count, max_mr_count);
721
722 ib_mw = bnxt_re_alloc_mw(&pd->ibpd, IB_MW_TYPE_1, NULL);
723 /* Create a fence MW only for kernel consumers */
724 if (!ib_mw) {
725 dev_err(rdev_to_dev(rdev),
726 "Failed to create fence-MW for PD: %p\n", pd);
727 rc = -EINVAL;
728 goto free_mr;
729 }
730 fence->mw = ib_mw;
731
732 bnxt_re_legacy_create_fence_wqe(pd);
733 return 0;
734
735 free_mr:
736 if (mr->ib_mr.lkey) {
737 bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr);
738 atomic_dec(&rdev->stats.rsors.mr_count);
739 }
740 kfree(mr);
741 fence->mr = NULL;
742
743 free_dma_addr:
744 ib_dma_unmap_single(&rdev->ibdev, fence->dma_addr,
745 BNXT_RE_LEGACY_FENCE_BYTES, DMA_BIDIRECTIONAL);
746 fence->dma_addr = 0;
747
748 free_va:
749 kfree(fence->va);
750 fence->va = NULL;
751 return rc;
752 }
753
bnxt_re_legacy_destroy_fence_mr(struct bnxt_re_pd * pd)754 static void bnxt_re_legacy_destroy_fence_mr(struct bnxt_re_pd *pd)
755 {
756 struct bnxt_re_legacy_fence_data *fence = &pd->fence;
757 struct bnxt_re_dev *rdev = pd->rdev;
758 struct bnxt_re_mr *mr = fence->mr;
759
760 if (!_is_chip_gen_p5_p7(rdev->chip_ctx))
761 return;
762
763 if (fence->mw) {
764 bnxt_re_dealloc_mw(fence->mw);
765 fence->mw = NULL;
766 }
767 if (mr) {
768 if (mr->ib_mr.rkey)
769 bnxt_qplib_dereg_mrw(&rdev->qplib_res, &mr->qplib_mr,
770 false);
771 if (mr->ib_mr.lkey)
772 bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr);
773 kfree(mr);
774 fence->mr = NULL;
775 atomic_dec(&rdev->stats.rsors.mr_count);
776 }
777 if (fence->dma_addr) {
778 ib_dma_unmap_single(&rdev->ibdev, fence->dma_addr,
779 BNXT_RE_LEGACY_FENCE_BYTES,
780 DMA_BIDIRECTIONAL);
781 fence->dma_addr = 0;
782 }
783 kfree(fence->va);
784 fence->va = NULL;
785 }
786
787
bnxt_re_get_user_dpi(struct bnxt_re_dev * rdev,struct bnxt_re_ucontext * cntx)788 static int bnxt_re_get_user_dpi(struct bnxt_re_dev *rdev,
789 struct bnxt_re_ucontext *cntx)
790 {
791 struct bnxt_qplib_chip_ctx *cctx = rdev->chip_ctx;
792 int ret = 0;
793 u8 type;
794 /* Allocate DPI in alloc_pd or in create_cq to avoid failing of
795 * ibv_devinfo and family of application when DPIs are depleted.
796 */
797 type = BNXT_QPLIB_DPI_TYPE_UC;
798 ret = bnxt_qplib_alloc_dpi(&rdev->qplib_res, &cntx->dpi, cntx, type);
799 if (ret) {
800 dev_err(rdev_to_dev(rdev), "Alloc doorbell page failed!\n");
801 goto out;
802 }
803
804 if (cctx->modes.db_push) {
805 type = BNXT_QPLIB_DPI_TYPE_WC;
806 ret = bnxt_qplib_alloc_dpi(&rdev->qplib_res, &cntx->wcdpi,
807 cntx, type);
808 if (ret)
809 dev_err(rdev_to_dev(rdev), "push dp alloc failed\n");
810 }
811 out:
812 return ret;
813 }
814
815 /* Protection Domains */
bnxt_re_dealloc_pd(struct ib_pd * ib_pd,struct ib_udata * udata)816 void bnxt_re_dealloc_pd(struct ib_pd *ib_pd, struct ib_udata *udata)
817 {
818 struct bnxt_re_pd *pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd);
819 struct bnxt_re_dev *rdev = pd->rdev;
820 int rc;
821
822 bnxt_re_legacy_destroy_fence_mr(pd);
823
824 rc = bnxt_qplib_dealloc_pd(&rdev->qplib_res,
825 &rdev->qplib_res.pd_tbl,
826 &pd->qplib_pd);
827 if (rc)
828 dev_err_ratelimited(rdev_to_dev(rdev),
829 "%s failed rc = %d\n", __func__, rc);
830 atomic_dec(&rdev->stats.rsors.pd_count);
831
832 return;
833 }
834
bnxt_re_alloc_pd(struct ib_pd * pd_in,struct ib_udata * udata)835 int bnxt_re_alloc_pd(struct ib_pd *pd_in,
836 struct ib_udata *udata)
837 {
838 struct ib_pd *ibpd = pd_in;
839 struct ib_device *ibdev = ibpd->device;
840 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
841 struct bnxt_re_ucontext *ucntx =
842 rdma_udata_to_drv_context(udata, struct bnxt_re_ucontext,
843 ibucontext);
844 u32 max_pd_count;
845 int rc;
846 struct bnxt_re_pd *pd = container_of(ibpd, struct bnxt_re_pd, ibpd);
847
848 pd->rdev = rdev;
849 if (bnxt_qplib_alloc_pd(&rdev->qplib_res, &pd->qplib_pd)) {
850 dev_err(rdev_to_dev(rdev),
851 "Allocate HW Protection Domain failed!\n");
852 rc = -ENOMEM;
853 goto fail;
854 }
855
856 if (udata) {
857 struct bnxt_re_pd_resp resp = {};
858
859 if (!ucntx->dpi.dbr) {
860 rc = bnxt_re_get_user_dpi(rdev, ucntx);
861 if (rc)
862 goto dbfail;
863 }
864
865 resp.pdid = pd->qplib_pd.id;
866 /* Still allow mapping this DBR to the new user PD. */
867 resp.dpi = ucntx->dpi.dpi;
868 resp.dbr = (u64)ucntx->dpi.umdbr;
869 /* Copy only on a valid wcpdi */
870 if (ucntx->wcdpi.dpi) {
871 resp.wcdpi = ucntx->wcdpi.dpi;
872 resp.comp_mask = BNXT_RE_COMP_MASK_PD_HAS_WC_DPI;
873 }
874 if (rdev->dbr_pacing) {
875 WARN_ON(!rdev->dbr_bar_addr);
876 resp.dbr_bar_addr = (u64)rdev->dbr_bar_addr;
877 resp.comp_mask |= BNXT_RE_COMP_MASK_PD_HAS_DBR_BAR_ADDR;
878 }
879
880 rc = bnxt_re_copy_to_udata(rdev, &resp,
881 min(udata->outlen, sizeof(resp)),
882 udata);
883 if (rc)
884 goto dbfail;
885 }
886
887 if (!udata)
888 if (bnxt_re_legacy_create_fence_mr(pd))
889 dev_warn(rdev_to_dev(rdev),
890 "Failed to create Fence-MR\n");
891
892 atomic_inc(&rdev->stats.rsors.pd_count);
893 max_pd_count = atomic_read(&rdev->stats.rsors.pd_count);
894 if (max_pd_count > atomic_read(&rdev->stats.rsors.max_pd_count))
895 atomic_set(&rdev->stats.rsors.max_pd_count, max_pd_count);
896
897 return 0;
898 dbfail:
899 (void)bnxt_qplib_dealloc_pd(&rdev->qplib_res, &rdev->qplib_res.pd_tbl,
900 &pd->qplib_pd);
901 fail:
902 return rc;
903 }
904
905 /* Address Handles */
bnxt_re_destroy_ah(struct ib_ah * ib_ah,u32 flags)906 void bnxt_re_destroy_ah(struct ib_ah *ib_ah, u32 flags)
907 {
908 struct bnxt_re_ah *ah = to_bnxt_re(ib_ah, struct bnxt_re_ah, ibah);
909 struct bnxt_re_dev *rdev = ah->rdev;
910 int rc = 0;
911 bool block = true;
912
913 block = !(flags & RDMA_DESTROY_AH_SLEEPABLE);
914
915 rc = bnxt_qplib_destroy_ah(&rdev->qplib_res, &ah->qplib_ah, block);
916 if (rc)
917 dev_err_ratelimited(rdev_to_dev(rdev),
918 "%s id = %d blocking %d failed rc = %d\n",
919 __func__, ah->qplib_ah.id, block, rc);
920 atomic_dec(&rdev->stats.rsors.ah_count);
921
922 return;
923 }
924
_to_bnxt_re_nw_type(enum rdma_network_type ntype)925 static u8 _to_bnxt_re_nw_type(enum rdma_network_type ntype)
926 {
927 u8 nw_type;
928 switch (ntype) {
929 case RDMA_NETWORK_IPV4:
930 nw_type = CMDQ_CREATE_AH_TYPE_V2IPV4;
931 break;
932 case RDMA_NETWORK_IPV6:
933 nw_type = CMDQ_CREATE_AH_TYPE_V2IPV6;
934 break;
935 default:
936 nw_type = CMDQ_CREATE_AH_TYPE_V1;
937 break;
938 }
939 return nw_type;
940 }
941
942 static inline int
bnxt_re_get_cached_gid(struct ib_device * dev,u8 port_num,int index,union ib_gid * sgid,struct ib_gid_attr ** sgid_attr,struct ib_global_route * grh,struct ib_ah * ah)943 bnxt_re_get_cached_gid(struct ib_device *dev, u8 port_num, int index,
944 union ib_gid *sgid, struct ib_gid_attr **sgid_attr,
945 struct ib_global_route *grh, struct ib_ah *ah)
946 {
947 int ret = 0;
948
949 ret = ib_get_cached_gid(dev, port_num, index, sgid, *sgid_attr);
950 return ret;
951 }
952
953 static inline enum rdma_network_type
bnxt_re_gid_to_network_type(struct ib_gid_attr * sgid_attr,union ib_gid * sgid)954 bnxt_re_gid_to_network_type(struct ib_gid_attr *sgid_attr,
955 union ib_gid *sgid)
956 {
957 return ib_gid_to_network_type(sgid_attr->gid_type, sgid);
958 }
959
bnxt_re_get_ah_info(struct bnxt_re_dev * rdev,struct ib_ah_attr * ah_attr,struct bnxt_re_ah_info * ah_info)960 static int bnxt_re_get_ah_info(struct bnxt_re_dev *rdev,
961 struct ib_ah_attr *ah_attr,
962 struct bnxt_re_ah_info *ah_info)
963 {
964 struct ib_gid_attr *gattr;
965 enum rdma_network_type ib_ntype;
966 u8 ntype;
967 union ib_gid *gid;
968 int rc = 0;
969
970 gid = &ah_info->sgid;
971 gattr = &ah_info->sgid_attr;
972
973 rc = bnxt_re_get_cached_gid(&rdev->ibdev, 1, ah_attr->grh.sgid_index,
974 gid, &gattr, &ah_attr->grh, NULL);
975 if (rc)
976 return rc;
977
978 /* Get vlan tag */
979 if (gattr->ndev) {
980 if (is_vlan_dev(gattr->ndev))
981 ah_info->vlan_tag = vlan_dev_vlan_id(gattr->ndev);
982 if_rele(gattr->ndev);
983 }
984
985 /* Get network header type for this GID */
986
987 ib_ntype = bnxt_re_gid_to_network_type(gattr, gid);
988 ntype = _to_bnxt_re_nw_type(ib_ntype);
989 ah_info->nw_type = ntype;
990
991 return rc;
992 }
993
_get_sgid_index(struct bnxt_re_dev * rdev,u8 gindx)994 static u8 _get_sgid_index(struct bnxt_re_dev *rdev, u8 gindx)
995 {
996 gindx = rdev->gid_map[gindx];
997 return gindx;
998 }
999
bnxt_re_init_dmac(struct bnxt_re_dev * rdev,struct ib_ah_attr * ah_attr,struct bnxt_re_ah_info * ah_info,bool is_user,struct bnxt_re_ah * ah)1000 static int bnxt_re_init_dmac(struct bnxt_re_dev *rdev, struct ib_ah_attr *ah_attr,
1001 struct bnxt_re_ah_info *ah_info, bool is_user,
1002 struct bnxt_re_ah *ah)
1003 {
1004 int rc = 0;
1005 u8 *dmac;
1006
1007 if (is_user && !rdma_is_multicast_addr((struct in6_addr *)
1008 ah_attr->grh.dgid.raw) &&
1009 !rdma_link_local_addr((struct in6_addr *)ah_attr->grh.dgid.raw)) {
1010
1011 u32 retry_count = BNXT_RE_RESOLVE_RETRY_COUNT_US;
1012 struct bnxt_re_resolve_dmac_work *resolve_dmac_work;
1013
1014
1015 resolve_dmac_work = kzalloc(sizeof(*resolve_dmac_work), GFP_ATOMIC);
1016
1017 resolve_dmac_work->rdev = rdev;
1018 resolve_dmac_work->ah_attr = ah_attr;
1019 resolve_dmac_work->ah_info = ah_info;
1020
1021 atomic_set(&resolve_dmac_work->status_wait, 1);
1022 INIT_WORK(&resolve_dmac_work->work, bnxt_re_resolve_dmac_task);
1023 queue_work(rdev->resolve_wq, &resolve_dmac_work->work);
1024
1025 do {
1026 rc = atomic_read(&resolve_dmac_work->status_wait) & 0xFF;
1027 if (!rc)
1028 break;
1029 udelay(1);
1030 } while (--retry_count);
1031 if (atomic_read(&resolve_dmac_work->status_wait)) {
1032 INIT_LIST_HEAD(&resolve_dmac_work->list);
1033 list_add_tail(&resolve_dmac_work->list,
1034 &rdev->mac_wq_list);
1035 return -EFAULT;
1036 }
1037 kfree(resolve_dmac_work);
1038 }
1039 dmac = ROCE_DMAC(ah_attr);
1040 if (dmac)
1041 memcpy(ah->qplib_ah.dmac, dmac, ETH_ALEN);
1042 return rc;
1043 }
1044
bnxt_re_create_ah(struct ib_ah * ah_in,struct ib_ah_attr * attr,u32 flags,struct ib_udata * udata)1045 int bnxt_re_create_ah(struct ib_ah *ah_in, struct ib_ah_attr *attr,
1046 u32 flags, struct ib_udata *udata)
1047 {
1048
1049 struct ib_ah *ib_ah = ah_in;
1050 struct ib_pd *ib_pd = ib_ah->pd;
1051 struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ibah);
1052 struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ibpd);
1053 struct bnxt_re_dev *rdev = pd->rdev;
1054 struct bnxt_re_ah_info ah_info;
1055 u32 max_ah_count;
1056 bool is_user;
1057 int rc;
1058 bool block = true;
1059 struct ib_ah_attr *ah_attr = attr;
1060 block = !(flags & RDMA_CREATE_AH_SLEEPABLE);
1061
1062 if (!(ah_attr->ah_flags & IB_AH_GRH))
1063 dev_err(rdev_to_dev(rdev), "ah_attr->ah_flags GRH is not set\n");
1064
1065 ah->rdev = rdev;
1066 ah->qplib_ah.pd = &pd->qplib_pd;
1067 is_user = ib_pd->uobject ? true : false;
1068
1069 /* Supply the configuration for the HW */
1070 memcpy(ah->qplib_ah.dgid.data, ah_attr->grh.dgid.raw,
1071 sizeof(union ib_gid));
1072 ah->qplib_ah.sgid_index = _get_sgid_index(rdev, ah_attr->grh.sgid_index);
1073 if (ah->qplib_ah.sgid_index == 0xFF) {
1074 dev_err(rdev_to_dev(rdev), "invalid sgid_index!\n");
1075 rc = -EINVAL;
1076 goto fail;
1077 }
1078 ah->qplib_ah.host_sgid_index = ah_attr->grh.sgid_index;
1079 ah->qplib_ah.traffic_class = ah_attr->grh.traffic_class;
1080 ah->qplib_ah.flow_label = ah_attr->grh.flow_label;
1081 ah->qplib_ah.hop_limit = ah_attr->grh.hop_limit;
1082 ah->qplib_ah.sl = ah_attr->sl;
1083 rc = bnxt_re_get_ah_info(rdev, ah_attr, &ah_info);
1084 if (rc)
1085 goto fail;
1086 ah->qplib_ah.nw_type = ah_info.nw_type;
1087
1088 rc = bnxt_re_init_dmac(rdev, ah_attr, &ah_info, is_user, ah);
1089 if (rc)
1090 goto fail;
1091
1092 rc = bnxt_qplib_create_ah(&rdev->qplib_res, &ah->qplib_ah, block);
1093 if (rc) {
1094 dev_err(rdev_to_dev(rdev),
1095 "Allocate HW Address Handle failed!\n");
1096 goto fail;
1097 }
1098
1099 /* Write AVID to shared page. */
1100 if (ib_pd->uobject) {
1101 struct ib_ucontext *ib_uctx = ib_pd->uobject->context;
1102 struct bnxt_re_ucontext *uctx;
1103 unsigned long flag;
1104 u32 *wrptr;
1105
1106 uctx = to_bnxt_re(ib_uctx, struct bnxt_re_ucontext, ibucontext);
1107 spin_lock_irqsave(&uctx->sh_lock, flag);
1108 wrptr = (u32 *)((u8 *)uctx->shpg + BNXT_RE_AVID_OFFT);
1109 *wrptr = ah->qplib_ah.id;
1110 wmb(); /* make sure cache is updated. */
1111 spin_unlock_irqrestore(&uctx->sh_lock, flag);
1112 }
1113 atomic_inc(&rdev->stats.rsors.ah_count);
1114 max_ah_count = atomic_read(&rdev->stats.rsors.ah_count);
1115 if (max_ah_count > atomic_read(&rdev->stats.rsors.max_ah_count))
1116 atomic_set(&rdev->stats.rsors.max_ah_count, max_ah_count);
1117
1118 return 0;
1119 fail:
1120 return rc;
1121 }
1122
bnxt_re_modify_ah(struct ib_ah * ib_ah,struct ib_ah_attr * ah_attr)1123 int bnxt_re_modify_ah(struct ib_ah *ib_ah, struct ib_ah_attr *ah_attr)
1124 {
1125 return 0;
1126 }
1127
bnxt_re_query_ah(struct ib_ah * ib_ah,struct ib_ah_attr * ah_attr)1128 int bnxt_re_query_ah(struct ib_ah *ib_ah, struct ib_ah_attr *ah_attr)
1129 {
1130 struct bnxt_re_ah *ah = to_bnxt_re(ib_ah, struct bnxt_re_ah, ibah);
1131
1132 memcpy(ah_attr->grh.dgid.raw, ah->qplib_ah.dgid.data,
1133 sizeof(union ib_gid));
1134 ah_attr->grh.sgid_index = ah->qplib_ah.host_sgid_index;
1135 ah_attr->grh.traffic_class = ah->qplib_ah.traffic_class;
1136 ah_attr->sl = ah->qplib_ah.sl;
1137 memcpy(ROCE_DMAC(ah_attr), ah->qplib_ah.dmac, ETH_ALEN);
1138 ah_attr->ah_flags = IB_AH_GRH;
1139 ah_attr->port_num = 1;
1140 ah_attr->static_rate = 0;
1141
1142 return 0;
1143 }
1144
1145 /* Shared Receive Queues */
bnxt_re_destroy_srq(struct ib_srq * ib_srq,struct ib_udata * udata)1146 void bnxt_re_destroy_srq(struct ib_srq *ib_srq,
1147 struct ib_udata *udata)
1148 {
1149 struct bnxt_re_srq *srq = to_bnxt_re(ib_srq, struct bnxt_re_srq, ibsrq);
1150 struct bnxt_re_dev *rdev = srq->rdev;
1151 struct bnxt_qplib_srq *qplib_srq = &srq->qplib_srq;
1152 int rc = 0;
1153
1154
1155 rc = bnxt_qplib_destroy_srq(&rdev->qplib_res, qplib_srq);
1156 if (rc)
1157 dev_err_ratelimited(rdev_to_dev(rdev),
1158 "%s id = %d failed rc = %d\n",
1159 __func__, qplib_srq->id, rc);
1160
1161 if (srq->umem && !IS_ERR(srq->umem))
1162 ib_umem_release(srq->umem);
1163
1164 atomic_dec(&rdev->stats.rsors.srq_count);
1165
1166 return;
1167 }
1168
_max_rwqe_sz(int nsge)1169 static u16 _max_rwqe_sz(int nsge)
1170 {
1171 return sizeof(struct rq_wqe_hdr) + (nsge * sizeof(struct sq_sge));
1172 }
1173
bnxt_re_get_rwqe_size(struct bnxt_qplib_qp * qplqp,int rsge,int max)1174 static u16 bnxt_re_get_rwqe_size(struct bnxt_qplib_qp *qplqp,
1175 int rsge, int max)
1176 {
1177 if (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC)
1178 rsge = max;
1179
1180 return _max_rwqe_sz(rsge);
1181 }
1182
1183 static inline
ib_umem_get_compat(struct bnxt_re_dev * rdev,struct ib_ucontext * ucontext,struct ib_udata * udata,unsigned long addr,size_t size,int access,int dmasync)1184 struct ib_umem *ib_umem_get_compat(struct bnxt_re_dev *rdev,
1185 struct ib_ucontext *ucontext,
1186 struct ib_udata *udata,
1187 unsigned long addr,
1188 size_t size, int access, int dmasync)
1189 {
1190 return ib_umem_get(ucontext, addr, size, access, dmasync);
1191 }
1192
1193 static inline
ib_umem_get_flags_compat(struct bnxt_re_dev * rdev,struct ib_ucontext * ucontext,struct ib_udata * udata,unsigned long addr,size_t size,int access,int dmasync)1194 struct ib_umem *ib_umem_get_flags_compat(struct bnxt_re_dev *rdev,
1195 struct ib_ucontext *ucontext,
1196 struct ib_udata *udata,
1197 unsigned long addr,
1198 size_t size, int access, int dmasync)
1199 {
1200 return ib_umem_get_compat(rdev, ucontext, udata, addr, size,
1201 access, 0);
1202 }
1203
ib_umem_num_pages_compat(struct ib_umem * umem)1204 static inline size_t ib_umem_num_pages_compat(struct ib_umem *umem)
1205 {
1206 return ib_umem_num_pages(umem);
1207 }
1208
bnxt_re_init_user_srq(struct bnxt_re_dev * rdev,struct bnxt_re_pd * pd,struct bnxt_re_srq * srq,struct ib_udata * udata)1209 static int bnxt_re_init_user_srq(struct bnxt_re_dev *rdev,
1210 struct bnxt_re_pd *pd,
1211 struct bnxt_re_srq *srq,
1212 struct ib_udata *udata)
1213 {
1214 struct bnxt_qplib_sg_info *sginfo;
1215 struct bnxt_qplib_srq *qplib_srq;
1216 struct bnxt_re_ucontext *cntx;
1217 struct ib_ucontext *context;
1218 struct bnxt_re_srq_req ureq;
1219 struct ib_umem *umem;
1220 int rc, bytes = 0;
1221
1222 context = pd->ibpd.uobject->context;
1223 cntx = to_bnxt_re(context, struct bnxt_re_ucontext, ibucontext);
1224 qplib_srq = &srq->qplib_srq;
1225 sginfo = &qplib_srq->sginfo;
1226
1227 if (udata->inlen < sizeof(ureq))
1228 dev_warn(rdev_to_dev(rdev),
1229 "Update the library ulen %d klen %d\n",
1230 (unsigned int)udata->inlen,
1231 (unsigned int)sizeof(ureq));
1232
1233 rc = ib_copy_from_udata(&ureq, udata,
1234 min(udata->inlen, sizeof(ureq)));
1235 if (rc)
1236 return rc;
1237
1238 bytes = (qplib_srq->max_wqe * qplib_srq->wqe_size);
1239 bytes = PAGE_ALIGN(bytes);
1240 umem = ib_umem_get_compat(rdev, context, udata, ureq.srqva, bytes,
1241 IB_ACCESS_LOCAL_WRITE, 1);
1242 if (IS_ERR(umem)) {
1243 dev_err(rdev_to_dev(rdev), "%s: ib_umem_get failed with %ld\n",
1244 __func__, PTR_ERR(umem));
1245 return PTR_ERR(umem);
1246 }
1247
1248 srq->umem = umem;
1249 sginfo->sghead = get_ib_umem_sgl(umem, &sginfo->nmap);
1250 sginfo->npages = ib_umem_num_pages_compat(umem);
1251 qplib_srq->srq_handle = ureq.srq_handle;
1252 qplib_srq->dpi = &cntx->dpi;
1253 qplib_srq->is_user = true;
1254
1255 return 0;
1256 }
1257
bnxt_re_create_srq(struct ib_srq * srq_in,struct ib_srq_init_attr * srq_init_attr,struct ib_udata * udata)1258 int bnxt_re_create_srq(struct ib_srq *srq_in, struct ib_srq_init_attr *srq_init_attr,
1259 struct ib_udata *udata)
1260 {
1261 struct bnxt_qplib_dev_attr *dev_attr;
1262 struct bnxt_re_ucontext *cntx = NULL;
1263 struct ib_ucontext *context;
1264 struct bnxt_re_dev *rdev;
1265 struct bnxt_re_pd *pd;
1266 int rc, entries;
1267 struct ib_srq *ib_srq = srq_in;
1268 struct ib_pd *ib_pd = ib_srq->pd;
1269 struct bnxt_re_srq *srq =
1270 container_of(ib_srq, struct bnxt_re_srq, ibsrq);
1271 u32 max_srq_count;
1272
1273 pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd);
1274 rdev = pd->rdev;
1275 dev_attr = rdev->dev_attr;
1276
1277 if (rdev->mod_exit) {
1278 dev_dbg(rdev_to_dev(rdev), "%s(): in mod_exit, just return!\n", __func__);
1279 rc = -EIO;
1280 goto exit;
1281 }
1282
1283 if (srq_init_attr->srq_type != IB_SRQT_BASIC) {
1284 dev_err(rdev_to_dev(rdev), "SRQ type not supported\n");
1285 rc = -ENOTSUPP;
1286 goto exit;
1287 }
1288
1289 if (udata) {
1290 context = pd->ibpd.uobject->context;
1291 cntx = to_bnxt_re(context, struct bnxt_re_ucontext, ibucontext);
1292 }
1293
1294 if (atomic_read(&rdev->stats.rsors.srq_count) >= dev_attr->max_srq) {
1295 dev_err(rdev_to_dev(rdev), "Create SRQ failed - max exceeded(SRQs)\n");
1296 rc = -EINVAL;
1297 goto exit;
1298 }
1299
1300 if (srq_init_attr->attr.max_wr >= dev_attr->max_srq_wqes) {
1301 dev_err(rdev_to_dev(rdev), "Create SRQ failed - max exceeded(SRQ_WQs)\n");
1302 rc = -EINVAL;
1303 goto exit;
1304 }
1305
1306 srq->rdev = rdev;
1307 srq->qplib_srq.pd = &pd->qplib_pd;
1308 srq->qplib_srq.dpi = &rdev->dpi_privileged;
1309
1310 /* Allocate 1 more than what's provided so posting max doesn't
1311 mean empty */
1312 entries = srq_init_attr->attr.max_wr + 1;
1313 entries = bnxt_re_init_depth(entries, cntx);
1314 if (entries > dev_attr->max_srq_wqes + 1)
1315 entries = dev_attr->max_srq_wqes + 1;
1316
1317 srq->qplib_srq.wqe_size = _max_rwqe_sz(6); /* 128 byte wqe size */
1318 srq->qplib_srq.max_wqe = entries;
1319 srq->qplib_srq.max_sge = srq_init_attr->attr.max_sge;
1320 srq->qplib_srq.threshold = srq_init_attr->attr.srq_limit;
1321 srq->srq_limit = srq_init_attr->attr.srq_limit;
1322 srq->qplib_srq.eventq_hw_ring_id = rdev->nqr.nq[0].ring_id;
1323 srq->qplib_srq.sginfo.pgsize = PAGE_SIZE;
1324 srq->qplib_srq.sginfo.pgshft = PAGE_SHIFT;
1325
1326 if (udata) {
1327 rc = bnxt_re_init_user_srq(rdev, pd, srq, udata);
1328 if (rc)
1329 goto fail;
1330 }
1331
1332 rc = bnxt_qplib_create_srq(&rdev->qplib_res, &srq->qplib_srq);
1333 if (rc) {
1334 dev_err(rdev_to_dev(rdev), "Create HW SRQ failed!\n");
1335 goto fail;
1336 }
1337
1338 if (udata) {
1339 struct bnxt_re_srq_resp resp;
1340
1341 resp.srqid = srq->qplib_srq.id;
1342 rc = bnxt_re_copy_to_udata(rdev, &resp,
1343 min(udata->outlen, sizeof(resp)),
1344 udata);
1345 if (rc) {
1346 bnxt_qplib_destroy_srq(&rdev->qplib_res, &srq->qplib_srq);
1347 goto fail;
1348 }
1349 }
1350 atomic_inc(&rdev->stats.rsors.srq_count);
1351 max_srq_count = atomic_read(&rdev->stats.rsors.srq_count);
1352 if (max_srq_count > atomic_read(&rdev->stats.rsors.max_srq_count))
1353 atomic_set(&rdev->stats.rsors.max_srq_count, max_srq_count);
1354 spin_lock_init(&srq->lock);
1355
1356 return 0;
1357 fail:
1358 if (udata && srq->umem && !IS_ERR(srq->umem)) {
1359 ib_umem_release(srq->umem);
1360 srq->umem = NULL;
1361 }
1362 exit:
1363 return rc;
1364 }
1365
bnxt_re_modify_srq(struct ib_srq * ib_srq,struct ib_srq_attr * srq_attr,enum ib_srq_attr_mask srq_attr_mask,struct ib_udata * udata)1366 int bnxt_re_modify_srq(struct ib_srq *ib_srq, struct ib_srq_attr *srq_attr,
1367 enum ib_srq_attr_mask srq_attr_mask,
1368 struct ib_udata *udata)
1369 {
1370 struct bnxt_re_srq *srq = to_bnxt_re(ib_srq, struct bnxt_re_srq,
1371 ibsrq);
1372 struct bnxt_re_dev *rdev = srq->rdev;
1373 int rc;
1374
1375 switch (srq_attr_mask) {
1376 case IB_SRQ_MAX_WR:
1377 /* SRQ resize is not supported */
1378 break;
1379 case IB_SRQ_LIMIT:
1380 /* Change the SRQ threshold */
1381 if (srq_attr->srq_limit > srq->qplib_srq.max_wqe)
1382 return -EINVAL;
1383
1384 srq->qplib_srq.threshold = srq_attr->srq_limit;
1385 rc = bnxt_qplib_modify_srq(&rdev->qplib_res, &srq->qplib_srq);
1386 if (rc) {
1387 dev_err(rdev_to_dev(rdev), "Modify HW SRQ failed!\n");
1388 return rc;
1389 }
1390 /* On success, update the shadow */
1391 srq->srq_limit = srq_attr->srq_limit;
1392
1393 if (udata) {
1394 /* Build and send response back to udata */
1395 rc = bnxt_re_copy_to_udata(rdev, srq, 0, udata);
1396 if (rc)
1397 return rc;
1398 }
1399 break;
1400 default:
1401 dev_err(rdev_to_dev(rdev),
1402 "Unsupported srq_attr_mask 0x%x\n", srq_attr_mask);
1403 return -EINVAL;
1404 }
1405 return 0;
1406 }
1407
bnxt_re_query_srq(struct ib_srq * ib_srq,struct ib_srq_attr * srq_attr)1408 int bnxt_re_query_srq(struct ib_srq *ib_srq, struct ib_srq_attr *srq_attr)
1409 {
1410 struct bnxt_re_srq *srq = to_bnxt_re(ib_srq, struct bnxt_re_srq,
1411 ibsrq);
1412 struct bnxt_re_dev *rdev = srq->rdev;
1413 int rc;
1414
1415 rc = bnxt_qplib_query_srq(&rdev->qplib_res, &srq->qplib_srq);
1416 if (rc) {
1417 dev_err(rdev_to_dev(rdev), "Query HW SRQ (0x%x) failed! rc = %d\n",
1418 srq->qplib_srq.id, rc);
1419 return rc;
1420 }
1421 srq_attr->max_wr = srq->qplib_srq.max_wqe;
1422 srq_attr->max_sge = srq->qplib_srq.max_sge;
1423 srq_attr->srq_limit = srq->qplib_srq.threshold;
1424
1425 return 0;
1426 }
1427
bnxt_re_post_srq_recv(struct ib_srq * ib_srq,const struct ib_recv_wr * wr,const struct ib_recv_wr ** bad_wr)1428 int bnxt_re_post_srq_recv(struct ib_srq *ib_srq, const struct ib_recv_wr *wr,
1429 const struct ib_recv_wr **bad_wr)
1430 {
1431 struct bnxt_re_srq *srq = to_bnxt_re(ib_srq, struct bnxt_re_srq,
1432 ibsrq);
1433 struct bnxt_qplib_swqe wqe = {};
1434 unsigned long flags;
1435 int rc = 0;
1436
1437 spin_lock_irqsave(&srq->lock, flags);
1438 while (wr) {
1439 /* Transcribe each ib_recv_wr to qplib_swqe */
1440 wqe.num_sge = wr->num_sge;
1441 wqe.sg_list = (struct bnxt_qplib_sge *)wr->sg_list;
1442 wqe.wr_id = wr->wr_id;
1443 wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV;
1444 rc = bnxt_qplib_post_srq_recv(&srq->qplib_srq, &wqe);
1445 if (rc) {
1446 *bad_wr = wr;
1447 break;
1448 }
1449 wr = wr->next;
1450 }
1451 spin_unlock_irqrestore(&srq->lock, flags);
1452
1453 return rc;
1454 }
1455
bnxt_re_lock_cqs(struct bnxt_re_qp * qp)1456 unsigned long bnxt_re_lock_cqs(struct bnxt_re_qp *qp)
1457 {
1458 unsigned long flags;
1459
1460 spin_lock_irqsave(&qp->scq->cq_lock, flags);
1461 if (qp->rcq && qp->rcq != qp->scq)
1462 spin_lock(&qp->rcq->cq_lock);
1463
1464 return flags;
1465 }
1466
bnxt_re_unlock_cqs(struct bnxt_re_qp * qp,unsigned long flags)1467 void bnxt_re_unlock_cqs(struct bnxt_re_qp *qp,
1468 unsigned long flags)
1469 {
1470 if (qp->rcq && qp->rcq != qp->scq)
1471 spin_unlock(&qp->rcq->cq_lock);
1472 spin_unlock_irqrestore(&qp->scq->cq_lock, flags);
1473 }
1474
1475 /* Queue Pairs */
bnxt_re_destroy_gsi_sqp(struct bnxt_re_qp * qp)1476 static int bnxt_re_destroy_gsi_sqp(struct bnxt_re_qp *qp)
1477 {
1478 struct bnxt_re_qp *gsi_sqp;
1479 struct bnxt_re_ah *gsi_sah;
1480 struct bnxt_re_dev *rdev;
1481 unsigned long flags;
1482 int rc = 0;
1483
1484 rdev = qp->rdev;
1485 gsi_sqp = rdev->gsi_ctx.gsi_sqp;
1486 gsi_sah = rdev->gsi_ctx.gsi_sah;
1487
1488 /* remove from active qp list */
1489 mutex_lock(&rdev->qp_lock);
1490 list_del(&gsi_sqp->list);
1491 mutex_unlock(&rdev->qp_lock);
1492
1493 if (gsi_sah) {
1494 dev_dbg(rdev_to_dev(rdev), "Destroy the shadow AH\n");
1495 rc = bnxt_qplib_destroy_ah(&rdev->qplib_res, &gsi_sah->qplib_ah,
1496 true);
1497 if (rc)
1498 dev_err(rdev_to_dev(rdev),
1499 "Destroy HW AH for shadow QP failed!\n");
1500 atomic_dec(&rdev->stats.rsors.ah_count);
1501 }
1502
1503 dev_dbg(rdev_to_dev(rdev), "Destroy the shadow QP\n");
1504 rc = bnxt_qplib_destroy_qp(&rdev->qplib_res, &gsi_sqp->qplib_qp);
1505 if (rc)
1506 dev_err(rdev_to_dev(rdev), "Destroy Shadow QP failed\n");
1507
1508 /* Clean the CQ for shadow QP completions */
1509 flags = bnxt_re_lock_cqs(gsi_sqp);
1510 bnxt_qplib_clean_qp(&gsi_sqp->qplib_qp);
1511 bnxt_re_unlock_cqs(gsi_sqp, flags);
1512
1513 bnxt_qplib_free_qp_res(&rdev->qplib_res, &gsi_sqp->qplib_qp);
1514 bnxt_qplib_free_hdr_buf(&rdev->qplib_res, &gsi_sqp->qplib_qp);
1515 kfree(rdev->gsi_ctx.sqp_tbl);
1516 kfree(gsi_sah);
1517 kfree(gsi_sqp);
1518 rdev->gsi_ctx.gsi_sqp = NULL;
1519 rdev->gsi_ctx.gsi_sah = NULL;
1520 rdev->gsi_ctx.sqp_tbl = NULL;
1521 atomic_dec(&rdev->stats.rsors.qp_count);
1522
1523 return 0;
1524 }
1525
bnxt_re_dump_debug_stats(struct bnxt_re_dev * rdev,u32 active_qps)1526 static void bnxt_re_dump_debug_stats(struct bnxt_re_dev *rdev, u32 active_qps)
1527 {
1528 u32 total_qp = 0;
1529 u64 avg_time = 0;
1530 int i;
1531
1532 if (!rdev->rcfw.sp_perf_stats_enabled)
1533 return;
1534
1535 switch (active_qps) {
1536 case 1:
1537 /* Potential hint for Test Stop */
1538 for (i = 0; i < RCFW_MAX_STAT_INDEX; i++) {
1539 if (rdev->rcfw.qp_destroy_stats[i]) {
1540 total_qp++;
1541 avg_time += rdev->rcfw.qp_destroy_stats[i];
1542 }
1543 }
1544 if (total_qp >= 0 || avg_time >= 0)
1545 dev_dbg(rdev_to_dev(rdev),
1546 "Perf Debug: %ps Total (%d) QP destroyed in (%ld) msec\n",
1547 __builtin_return_address(0), total_qp,
1548 (long)jiffies_to_msecs(avg_time));
1549 break;
1550 case 2:
1551 /* Potential hint for Test Start */
1552 dev_dbg(rdev_to_dev(rdev),
1553 "Perf Debug: %ps active_qps = %d\n",
1554 __builtin_return_address(0), active_qps);
1555 break;
1556 default:
1557 /* Potential hint to know latency of QP destroy.
1558 * Average time taken for 1K QP Destroy.
1559 */
1560 if (active_qps > 1024 && !(active_qps % 1024))
1561 dev_dbg(rdev_to_dev(rdev),
1562 "Perf Debug: %ps Active QP (%d) Watermark (%d)\n",
1563 __builtin_return_address(0), active_qps,
1564 atomic_read(&rdev->stats.rsors.max_qp_count));
1565 break;
1566 }
1567 }
1568
bnxt_re_destroy_qp(struct ib_qp * ib_qp,struct ib_udata * udata)1569 int bnxt_re_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata)
1570 {
1571 struct bnxt_re_qp *qp = to_bnxt_re(ib_qp, struct bnxt_re_qp, ib_qp);
1572 struct bnxt_re_dev *rdev = qp->rdev;
1573 unsigned long flags;
1574 u32 active_qps;
1575 int rc;
1576
1577 mutex_lock(&rdev->qp_lock);
1578 list_del(&qp->list);
1579 active_qps = atomic_dec_return(&rdev->stats.rsors.qp_count);
1580 if (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_RC)
1581 atomic_dec(&rdev->stats.rsors.rc_qp_count);
1582 else if (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_UD)
1583 atomic_dec(&rdev->stats.rsors.ud_qp_count);
1584 mutex_unlock(&rdev->qp_lock);
1585
1586 rc = bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp);
1587 if (rc)
1588 dev_err_ratelimited(rdev_to_dev(rdev),
1589 "%s id = %d failed rc = %d\n",
1590 __func__, qp->qplib_qp.id, rc);
1591
1592 if (!ib_qp->uobject) {
1593 flags = bnxt_re_lock_cqs(qp);
1594 bnxt_qplib_clean_qp(&qp->qplib_qp);
1595 bnxt_re_unlock_cqs(qp, flags);
1596 }
1597
1598 bnxt_qplib_free_qp_res(&rdev->qplib_res, &qp->qplib_qp);
1599 if (ib_qp->qp_type == IB_QPT_GSI &&
1600 rdev->gsi_ctx.gsi_qp_mode != BNXT_RE_GSI_MODE_UD) {
1601 if (rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_ALL &&
1602 rdev->gsi_ctx.gsi_sqp) {
1603 bnxt_re_destroy_gsi_sqp(qp);
1604 }
1605 bnxt_qplib_free_hdr_buf(&rdev->qplib_res, &qp->qplib_qp);
1606 }
1607
1608 if (qp->rumem && !IS_ERR(qp->rumem))
1609 ib_umem_release(qp->rumem);
1610 if (qp->sumem && !IS_ERR(qp->sumem))
1611 ib_umem_release(qp->sumem);
1612 kfree(qp);
1613
1614 bnxt_re_dump_debug_stats(rdev, active_qps);
1615
1616 return 0;
1617 }
1618
__from_ib_qp_type(enum ib_qp_type type)1619 static u8 __from_ib_qp_type(enum ib_qp_type type)
1620 {
1621 switch (type) {
1622 case IB_QPT_GSI:
1623 return CMDQ_CREATE_QP1_TYPE_GSI;
1624 case IB_QPT_RC:
1625 return CMDQ_CREATE_QP_TYPE_RC;
1626 case IB_QPT_UD:
1627 return CMDQ_CREATE_QP_TYPE_UD;
1628 case IB_QPT_RAW_ETHERTYPE:
1629 return CMDQ_CREATE_QP_TYPE_RAW_ETHERTYPE;
1630 default:
1631 return IB_QPT_MAX;
1632 }
1633 }
1634
_get_swqe_sz(int nsge)1635 static u16 _get_swqe_sz(int nsge)
1636 {
1637 return sizeof(struct sq_send_hdr) + nsge * sizeof(struct sq_sge);
1638 }
1639
bnxt_re_get_swqe_size(int ilsize,int nsge)1640 static int bnxt_re_get_swqe_size(int ilsize, int nsge)
1641 {
1642 u16 wqe_size, calc_ils;
1643
1644 wqe_size = _get_swqe_sz(nsge);
1645 if (ilsize) {
1646 calc_ils = (sizeof(struct sq_send_hdr) + ilsize);
1647 wqe_size = max_t(int, calc_ils, wqe_size);
1648 wqe_size = ALIGN(wqe_size, 32);
1649 }
1650 return wqe_size;
1651 }
1652
bnxt_re_setup_swqe_size(struct bnxt_re_qp * qp,struct ib_qp_init_attr * init_attr)1653 static int bnxt_re_setup_swqe_size(struct bnxt_re_qp *qp,
1654 struct ib_qp_init_attr *init_attr)
1655 {
1656 struct bnxt_qplib_dev_attr *dev_attr;
1657 struct bnxt_qplib_qp *qplqp;
1658 struct bnxt_re_dev *rdev;
1659 struct bnxt_qplib_q *sq;
1660 int align, ilsize;
1661
1662 rdev = qp->rdev;
1663 qplqp = &qp->qplib_qp;
1664 sq = &qplqp->sq;
1665 dev_attr = rdev->dev_attr;
1666
1667 align = sizeof(struct sq_send_hdr);
1668 ilsize = ALIGN(init_attr->cap.max_inline_data, align);
1669
1670 /* For gen p4 and gen p5 fixed wqe compatibility mode
1671 * wqe size is fixed to 128 bytes - ie 6 SGEs
1672 */
1673 if (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) {
1674 sq->wqe_size = _get_swqe_sz(BNXT_STATIC_MAX_SGE);
1675 sq->max_sge = BNXT_STATIC_MAX_SGE;
1676 } else {
1677 sq->wqe_size = bnxt_re_get_swqe_size(ilsize, sq->max_sge);
1678 if (sq->wqe_size > _get_swqe_sz(dev_attr->max_qp_sges))
1679 return -EINVAL;
1680 }
1681
1682
1683 if (init_attr->cap.max_inline_data) {
1684 qplqp->max_inline_data = sq->wqe_size -
1685 sizeof(struct sq_send_hdr);
1686 init_attr->cap.max_inline_data = qplqp->max_inline_data;
1687 if (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC)
1688 sq->max_sge = qplqp->max_inline_data /
1689 sizeof(struct sq_sge);
1690 }
1691
1692 return 0;
1693 }
1694
bnxt_re_init_user_qp(struct bnxt_re_dev * rdev,struct bnxt_re_pd * pd,struct bnxt_re_qp * qp,struct ib_udata * udata)1695 static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev,
1696 struct bnxt_re_pd *pd, struct bnxt_re_qp *qp,
1697 struct ib_udata *udata)
1698 {
1699 struct bnxt_qplib_sg_info *sginfo;
1700 struct bnxt_qplib_qp *qplib_qp;
1701 struct bnxt_re_ucontext *cntx;
1702 struct ib_ucontext *context;
1703 struct bnxt_re_qp_req ureq;
1704 struct ib_umem *umem;
1705 int rc, bytes = 0;
1706 int psn_nume;
1707 int psn_sz;
1708
1709 qplib_qp = &qp->qplib_qp;
1710 context = pd->ibpd.uobject->context;
1711 cntx = to_bnxt_re(context, struct bnxt_re_ucontext, ibucontext);
1712 sginfo = &qplib_qp->sq.sginfo;
1713
1714 if (udata->inlen < sizeof(ureq))
1715 dev_warn(rdev_to_dev(rdev),
1716 "Update the library ulen %d klen %d\n",
1717 (unsigned int)udata->inlen,
1718 (unsigned int)sizeof(ureq));
1719
1720 rc = ib_copy_from_udata(&ureq, udata,
1721 min(udata->inlen, sizeof(ureq)));
1722 if (rc)
1723 return rc;
1724
1725 bytes = (qplib_qp->sq.max_wqe * qplib_qp->sq.wqe_size);
1726 bytes = PAGE_ALIGN(bytes);
1727 /* Consider mapping PSN search memory only for RC QPs. */
1728 if (qplib_qp->type == CMDQ_CREATE_QP_TYPE_RC) {
1729 psn_sz = _is_chip_gen_p5_p7(rdev->chip_ctx) ?
1730 sizeof(struct sq_psn_search_ext) :
1731 sizeof(struct sq_psn_search);
1732 if (rdev->dev_attr && _is_host_msn_table(rdev->dev_attr->dev_cap_ext_flags2))
1733 psn_sz = sizeof(struct sq_msn_search);
1734 if (cntx && bnxt_re_is_var_size_supported(rdev, cntx)) {
1735 psn_nume = ureq.sq_slots;
1736 } else {
1737 psn_nume = (qplib_qp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ?
1738 qplib_qp->sq.max_wqe :
1739 ((qplib_qp->sq.max_wqe * qplib_qp->sq.wqe_size) /
1740 sizeof(struct bnxt_qplib_sge));
1741 }
1742 if (rdev->dev_attr && _is_host_msn_table(rdev->dev_attr->dev_cap_ext_flags2))
1743 psn_nume = roundup_pow_of_two(psn_nume);
1744
1745 bytes += (psn_nume * psn_sz);
1746 bytes = PAGE_ALIGN(bytes);
1747 }
1748 umem = ib_umem_get_compat(rdev, context, udata, ureq.qpsva, bytes,
1749 IB_ACCESS_LOCAL_WRITE, 1);
1750 if (IS_ERR(umem)) {
1751 dev_err(rdev_to_dev(rdev), "%s: ib_umem_get failed with %ld\n",
1752 __func__, PTR_ERR(umem));
1753 return PTR_ERR(umem);
1754 }
1755
1756 qp->sumem = umem;
1757 /* pgsize and pgshft were initialize already. */
1758 sginfo->sghead = get_ib_umem_sgl(umem, &sginfo->nmap);
1759 sginfo->npages = ib_umem_num_pages_compat(umem);
1760 qplib_qp->qp_handle = ureq.qp_handle;
1761
1762 if (!qp->qplib_qp.srq) {
1763 sginfo = &qplib_qp->rq.sginfo;
1764 bytes = (qplib_qp->rq.max_wqe * qplib_qp->rq.wqe_size);
1765 bytes = PAGE_ALIGN(bytes);
1766 umem = ib_umem_get_compat(rdev,
1767 context, udata, ureq.qprva, bytes,
1768 IB_ACCESS_LOCAL_WRITE, 1);
1769 if (IS_ERR(umem)) {
1770 dev_err(rdev_to_dev(rdev),
1771 "%s: ib_umem_get failed ret =%ld\n",
1772 __func__, PTR_ERR(umem));
1773 goto rqfail;
1774 }
1775 qp->rumem = umem;
1776 /* pgsize and pgshft were initialize already. */
1777 sginfo->sghead = get_ib_umem_sgl(umem, &sginfo->nmap);
1778 sginfo->npages = ib_umem_num_pages_compat(umem);
1779 }
1780
1781 qplib_qp->dpi = &cntx->dpi;
1782 qplib_qp->is_user = true;
1783
1784 return 0;
1785 rqfail:
1786 ib_umem_release(qp->sumem);
1787 qp->sumem = NULL;
1788 qplib_qp->sq.sginfo.sghead = NULL;
1789 qplib_qp->sq.sginfo.nmap = 0;
1790
1791 return PTR_ERR(umem);
1792 }
1793
bnxt_re_create_shadow_qp_ah(struct bnxt_re_pd * pd,struct bnxt_qplib_res * qp1_res,struct bnxt_qplib_qp * qp1_qp)1794 static struct bnxt_re_ah *bnxt_re_create_shadow_qp_ah(struct bnxt_re_pd *pd,
1795 struct bnxt_qplib_res *qp1_res,
1796 struct bnxt_qplib_qp *qp1_qp)
1797 {
1798 struct bnxt_re_dev *rdev = pd->rdev;
1799 struct bnxt_re_ah *ah;
1800 union ib_gid sgid;
1801 int rc;
1802
1803 ah = kzalloc(sizeof(*ah), GFP_KERNEL);
1804 if (!ah) {
1805 dev_err(rdev_to_dev(rdev), "Allocate Address Handle failed!\n");
1806 return NULL;
1807 }
1808 memset(ah, 0, sizeof(*ah));
1809 ah->rdev = rdev;
1810 ah->qplib_ah.pd = &pd->qplib_pd;
1811
1812 rc = bnxt_re_query_gid(&rdev->ibdev, 1, 0, &sgid);
1813 if (rc)
1814 goto fail;
1815
1816 /* supply the dgid data same as sgid */
1817 memcpy(ah->qplib_ah.dgid.data, &sgid.raw,
1818 sizeof(union ib_gid));
1819 ah->qplib_ah.sgid_index = 0;
1820
1821 ah->qplib_ah.traffic_class = 0;
1822 ah->qplib_ah.flow_label = 0;
1823 ah->qplib_ah.hop_limit = 1;
1824 ah->qplib_ah.sl = 0;
1825 /* Have DMAC same as SMAC */
1826 ether_addr_copy(ah->qplib_ah.dmac, rdev->dev_addr);
1827 dev_dbg(rdev_to_dev(rdev), "ah->qplib_ah.dmac = %x:%x:%x:%x:%x:%x\n",
1828 ah->qplib_ah.dmac[0], ah->qplib_ah.dmac[1], ah->qplib_ah.dmac[2],
1829 ah->qplib_ah.dmac[3], ah->qplib_ah.dmac[4], ah->qplib_ah.dmac[5]);
1830
1831 rc = bnxt_qplib_create_ah(&rdev->qplib_res, &ah->qplib_ah, true);
1832 if (rc) {
1833 dev_err(rdev_to_dev(rdev),
1834 "Allocate HW AH for Shadow QP failed!\n");
1835 goto fail;
1836 }
1837 dev_dbg(rdev_to_dev(rdev), "AH ID = %d\n", ah->qplib_ah.id);
1838 atomic_inc(&rdev->stats.rsors.ah_count);
1839
1840 return ah;
1841 fail:
1842 kfree(ah);
1843 return NULL;
1844 }
1845
bnxt_re_update_shadow_ah(struct bnxt_re_dev * rdev)1846 void bnxt_re_update_shadow_ah(struct bnxt_re_dev *rdev)
1847 {
1848 struct bnxt_re_qp *gsi_qp;
1849 struct bnxt_re_ah *sah;
1850 struct bnxt_re_pd *pd;
1851 struct ib_pd *ib_pd;
1852 int rc;
1853
1854 if (!rdev)
1855 return;
1856
1857 sah = rdev->gsi_ctx.gsi_sah;
1858
1859 dev_dbg(rdev_to_dev(rdev), "Updating the AH\n");
1860 if (sah) {
1861 /* Check if the AH created with current mac address */
1862 if (!compare_ether_header(sah->qplib_ah.dmac, rdev->dev_addr)) {
1863 dev_dbg(rdev_to_dev(rdev),
1864 "Not modifying shadow AH during AH update\n");
1865 return;
1866 }
1867
1868 gsi_qp = rdev->gsi_ctx.gsi_qp;
1869 ib_pd = gsi_qp->ib_qp.pd;
1870 pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd);
1871 rc = bnxt_qplib_destroy_ah(&rdev->qplib_res,
1872 &sah->qplib_ah, false);
1873 if (rc) {
1874 dev_err(rdev_to_dev(rdev),
1875 "Failed to destroy shadow AH during AH update\n");
1876 return;
1877 }
1878 atomic_dec(&rdev->stats.rsors.ah_count);
1879 kfree(sah);
1880 rdev->gsi_ctx.gsi_sah = NULL;
1881
1882 sah = bnxt_re_create_shadow_qp_ah(pd, &rdev->qplib_res,
1883 &gsi_qp->qplib_qp);
1884 if (!sah) {
1885 dev_err(rdev_to_dev(rdev),
1886 "Failed to update AH for ShadowQP\n");
1887 return;
1888 }
1889 rdev->gsi_ctx.gsi_sah = sah;
1890 atomic_inc(&rdev->stats.rsors.ah_count);
1891 }
1892 }
1893
bnxt_re_create_shadow_qp(struct bnxt_re_pd * pd,struct bnxt_qplib_res * qp1_res,struct bnxt_qplib_qp * qp1_qp)1894 static struct bnxt_re_qp *bnxt_re_create_shadow_qp(struct bnxt_re_pd *pd,
1895 struct bnxt_qplib_res *qp1_res,
1896 struct bnxt_qplib_qp *qp1_qp)
1897 {
1898 struct bnxt_re_dev *rdev = pd->rdev;
1899 struct bnxt_re_qp *qp;
1900 int rc;
1901
1902 qp = kzalloc(sizeof(*qp), GFP_KERNEL);
1903 if (!qp) {
1904 dev_err(rdev_to_dev(rdev), "Allocate internal UD QP failed!\n");
1905 return NULL;
1906 }
1907 memset(qp, 0, sizeof(*qp));
1908 qp->rdev = rdev;
1909
1910 /* Initialize the shadow QP structure from the QP1 values */
1911 ether_addr_copy(qp->qplib_qp.smac, rdev->dev_addr);
1912 qp->qplib_qp.pd = &pd->qplib_pd;
1913 qp->qplib_qp.qp_handle = (u64)&qp->qplib_qp;
1914 qp->qplib_qp.type = IB_QPT_UD;
1915
1916 qp->qplib_qp.max_inline_data = 0;
1917 qp->qplib_qp.sig_type = true;
1918
1919 /* Shadow QP SQ depth should be same as QP1 RQ depth */
1920 qp->qplib_qp.sq.wqe_size = bnxt_re_get_swqe_size(0, 6);
1921 qp->qplib_qp.sq.max_wqe = qp1_qp->rq.max_wqe;
1922 qp->qplib_qp.sq.max_sw_wqe = qp1_qp->rq.max_wqe;
1923 qp->qplib_qp.sq.max_sge = 2;
1924 /* Q full delta can be 1 since it is internal QP */
1925 qp->qplib_qp.sq.q_full_delta = 1;
1926 qp->qplib_qp.sq.sginfo.pgsize = PAGE_SIZE;
1927 qp->qplib_qp.sq.sginfo.pgshft = PAGE_SHIFT;
1928
1929 qp->qplib_qp.scq = qp1_qp->scq;
1930 qp->qplib_qp.rcq = qp1_qp->rcq;
1931
1932 qp->qplib_qp.rq.wqe_size = _max_rwqe_sz(6); /* 128 Byte wqe size */
1933 qp->qplib_qp.rq.max_wqe = qp1_qp->rq.max_wqe;
1934 qp->qplib_qp.rq.max_sw_wqe = qp1_qp->rq.max_wqe;
1935 qp->qplib_qp.rq.max_sge = qp1_qp->rq.max_sge;
1936 qp->qplib_qp.rq.sginfo.pgsize = PAGE_SIZE;
1937 qp->qplib_qp.rq.sginfo.pgshft = PAGE_SHIFT;
1938 /* Q full delta can be 1 since it is internal QP */
1939 qp->qplib_qp.rq.q_full_delta = 1;
1940 qp->qplib_qp.mtu = qp1_qp->mtu;
1941 qp->qplib_qp.dpi = &rdev->dpi_privileged;
1942
1943 rc = bnxt_qplib_alloc_hdr_buf(qp1_res, &qp->qplib_qp, 0,
1944 BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6);
1945 if (rc)
1946 goto fail;
1947
1948 rc = bnxt_qplib_create_qp(qp1_res, &qp->qplib_qp);
1949 if (rc) {
1950 dev_err(rdev_to_dev(rdev), "create HW QP failed!\n");
1951 goto qp_fail;
1952 }
1953
1954 dev_dbg(rdev_to_dev(rdev), "Created shadow QP with ID = %d\n",
1955 qp->qplib_qp.id);
1956 spin_lock_init(&qp->sq_lock);
1957 INIT_LIST_HEAD(&qp->list);
1958 mutex_lock(&rdev->qp_lock);
1959 list_add_tail(&qp->list, &rdev->qp_list);
1960 atomic_inc(&rdev->stats.rsors.qp_count);
1961 mutex_unlock(&rdev->qp_lock);
1962 return qp;
1963 qp_fail:
1964 bnxt_qplib_free_hdr_buf(qp1_res, &qp->qplib_qp);
1965 fail:
1966 kfree(qp);
1967 return NULL;
1968 }
1969
bnxt_re_init_rq_attr(struct bnxt_re_qp * qp,struct ib_qp_init_attr * init_attr,void * cntx)1970 static int bnxt_re_init_rq_attr(struct bnxt_re_qp *qp,
1971 struct ib_qp_init_attr *init_attr, void *cntx)
1972 {
1973 struct bnxt_qplib_dev_attr *dev_attr;
1974 struct bnxt_qplib_qp *qplqp;
1975 struct bnxt_re_dev *rdev;
1976 struct bnxt_qplib_q *rq;
1977 int entries;
1978
1979 rdev = qp->rdev;
1980 qplqp = &qp->qplib_qp;
1981 rq = &qplqp->rq;
1982 dev_attr = rdev->dev_attr;
1983
1984 if (init_attr->srq) {
1985 struct bnxt_re_srq *srq;
1986
1987 srq = to_bnxt_re(init_attr->srq, struct bnxt_re_srq, ibsrq);
1988 if (!srq) {
1989 dev_err(rdev_to_dev(rdev), "SRQ not found\n");
1990 return -EINVAL;
1991 }
1992 qplqp->srq = &srq->qplib_srq;
1993 rq->max_wqe = 0;
1994 } else {
1995 rq->max_sge = init_attr->cap.max_recv_sge;
1996 if (rq->max_sge > dev_attr->max_qp_sges)
1997 rq->max_sge = dev_attr->max_qp_sges;
1998 init_attr->cap.max_recv_sge = rq->max_sge;
1999 rq->wqe_size = bnxt_re_get_rwqe_size(qplqp, rq->max_sge,
2000 dev_attr->max_qp_sges);
2001
2002 /* Allocate 1 more than what's provided so posting max doesn't
2003 mean empty */
2004 entries = init_attr->cap.max_recv_wr + 1;
2005 entries = bnxt_re_init_depth(entries, cntx);
2006 rq->max_wqe = min_t(u32, entries, dev_attr->max_qp_wqes + 1);
2007 rq->max_sw_wqe = rq->max_wqe;
2008 rq->q_full_delta = 0;
2009 rq->sginfo.pgsize = PAGE_SIZE;
2010 rq->sginfo.pgshft = PAGE_SHIFT;
2011 }
2012
2013 return 0;
2014 }
2015
bnxt_re_adjust_gsi_rq_attr(struct bnxt_re_qp * qp)2016 static void bnxt_re_adjust_gsi_rq_attr(struct bnxt_re_qp *qp)
2017 {
2018 struct bnxt_qplib_dev_attr *dev_attr;
2019 struct bnxt_qplib_qp *qplqp;
2020 struct bnxt_re_dev *rdev;
2021
2022 rdev = qp->rdev;
2023 qplqp = &qp->qplib_qp;
2024 dev_attr = rdev->dev_attr;
2025
2026 if (rdev->gsi_ctx.gsi_qp_mode != BNXT_RE_GSI_MODE_UD)
2027 qplqp->rq.max_sge = dev_attr->max_qp_sges;
2028 }
2029
bnxt_re_init_sq_attr(struct bnxt_re_qp * qp,struct ib_qp_init_attr * init_attr,void * cntx,struct ib_udata * udata)2030 static int bnxt_re_init_sq_attr(struct bnxt_re_qp *qp,
2031 struct ib_qp_init_attr *init_attr,
2032 void *cntx, struct ib_udata *udata)
2033 {
2034 struct bnxt_qplib_dev_attr *dev_attr;
2035 struct bnxt_qplib_qp *qplqp;
2036 struct bnxt_re_qp_req ureq;
2037 struct bnxt_re_dev *rdev;
2038 struct bnxt_qplib_q *sq;
2039 int diff = 0;
2040 int entries;
2041 int rc;
2042
2043 rdev = qp->rdev;
2044 qplqp = &qp->qplib_qp;
2045 sq = &qplqp->sq;
2046 dev_attr = rdev->dev_attr;
2047
2048 if (udata) {
2049 rc = ib_copy_from_udata(&ureq, udata,
2050 min(udata->inlen, sizeof(ureq)));
2051 if (rc)
2052 return -EINVAL;
2053 }
2054
2055 sq->max_sge = init_attr->cap.max_send_sge;
2056 entries = init_attr->cap.max_send_wr;
2057 if (cntx && udata && qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE) {
2058 sq->max_wqe = ureq.sq_slots;
2059 sq->max_sw_wqe = ureq.sq_slots;
2060 sq->wqe_size = sizeof(struct sq_sge);
2061 } else {
2062 if (sq->max_sge > dev_attr->max_qp_sges) {
2063 sq->max_sge = dev_attr->max_qp_sges;
2064 init_attr->cap.max_send_sge = sq->max_sge;
2065 }
2066 rc = bnxt_re_setup_swqe_size(qp, init_attr);
2067 if (rc)
2068 return rc;
2069 /*
2070 * Change the SQ depth if user has requested minimum using
2071 * configfs. Only supported for kernel consumers. Setting
2072 * min_tx_depth to 4096 to handle iser SQ full condition
2073 * in most of the newer OS distros
2074 */
2075
2076 if (!cntx && rdev->min_tx_depth && init_attr->qp_type != IB_QPT_GSI) {
2077 /*
2078 * If users specify any value greater than 1 use min_tx_depth
2079 * provided by user for comparison. Else, compare it with the
2080 * BNXT_RE_MIN_KERNEL_QP_TX_DEPTH and adjust it accordingly.
2081 */
2082 if (rdev->min_tx_depth > 1 && entries < rdev->min_tx_depth)
2083 entries = rdev->min_tx_depth;
2084 else if (entries < BNXT_RE_MIN_KERNEL_QP_TX_DEPTH)
2085 entries = BNXT_RE_MIN_KERNEL_QP_TX_DEPTH;
2086 }
2087 diff = bnxt_re_get_diff(cntx, rdev->chip_ctx);
2088 entries = bnxt_re_init_depth(entries + diff + 1, cntx);
2089 sq->max_wqe = min_t(u32, entries, dev_attr->max_qp_wqes + diff + 1);
2090 if (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE)
2091 sq->max_sw_wqe = bnxt_qplib_get_depth(sq, qplqp->wqe_mode, true);
2092 else
2093 sq->max_sw_wqe = sq->max_wqe;
2094 }
2095 sq->q_full_delta = diff + 1;
2096 /*
2097 * Reserving one slot for Phantom WQE. Application can
2098 * post one extra entry in this case. But allowing this to avoid
2099 * unexpected Queue full condition
2100 */
2101 sq->q_full_delta -= 1; /* becomes 0 for gen-p5 */
2102 sq->sginfo.pgsize = PAGE_SIZE;
2103 sq->sginfo.pgshft = PAGE_SHIFT;
2104 return 0;
2105 }
2106
bnxt_re_adjust_gsi_sq_attr(struct bnxt_re_qp * qp,struct ib_qp_init_attr * init_attr,void * cntx)2107 static void bnxt_re_adjust_gsi_sq_attr(struct bnxt_re_qp *qp,
2108 struct ib_qp_init_attr *init_attr,
2109 void *cntx)
2110 {
2111 struct bnxt_qplib_dev_attr *dev_attr;
2112 struct bnxt_qplib_qp *qplqp;
2113 struct bnxt_re_dev *rdev;
2114 int entries;
2115
2116 rdev = qp->rdev;
2117 qplqp = &qp->qplib_qp;
2118 dev_attr = rdev->dev_attr;
2119
2120 if (rdev->gsi_ctx.gsi_qp_mode != BNXT_RE_GSI_MODE_UD) {
2121 entries = init_attr->cap.max_send_wr + 1;
2122 entries = bnxt_re_init_depth(entries, cntx);
2123 qplqp->sq.max_wqe = min_t(u32, entries,
2124 dev_attr->max_qp_wqes + 1);
2125 qplqp->sq.q_full_delta = qplqp->sq.max_wqe -
2126 init_attr->cap.max_send_wr;
2127 qplqp->sq.max_sge++; /* Need one extra sge to put UD header */
2128 if (qplqp->sq.max_sge > dev_attr->max_qp_sges)
2129 qplqp->sq.max_sge = dev_attr->max_qp_sges;
2130 }
2131 }
2132
bnxt_re_init_qp_type(struct bnxt_re_dev * rdev,struct ib_qp_init_attr * init_attr)2133 static int bnxt_re_init_qp_type(struct bnxt_re_dev *rdev,
2134 struct ib_qp_init_attr *init_attr)
2135 {
2136 struct bnxt_qplib_chip_ctx *chip_ctx;
2137 struct bnxt_re_gsi_context *gsi_ctx;
2138 int qptype;
2139
2140 chip_ctx = rdev->chip_ctx;
2141 gsi_ctx = &rdev->gsi_ctx;
2142
2143 qptype = __from_ib_qp_type(init_attr->qp_type);
2144 if (qptype == IB_QPT_MAX) {
2145 dev_err(rdev_to_dev(rdev), "QP type 0x%x not supported\n",
2146 qptype);
2147 qptype = -EINVAL;
2148 goto out;
2149 }
2150
2151 if (_is_chip_gen_p5_p7(chip_ctx) && init_attr->qp_type == IB_QPT_GSI) {
2152 /* For Thor always force UD mode. */
2153 qptype = CMDQ_CREATE_QP_TYPE_GSI;
2154 gsi_ctx->gsi_qp_mode = BNXT_RE_GSI_MODE_UD;
2155 }
2156 out:
2157 return qptype;
2158 }
2159
bnxt_re_init_qp_attr(struct bnxt_re_qp * qp,struct bnxt_re_pd * pd,struct ib_qp_init_attr * init_attr,struct ib_udata * udata)2160 static int bnxt_re_init_qp_attr(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd,
2161 struct ib_qp_init_attr *init_attr,
2162 struct ib_udata *udata)
2163 {
2164 struct bnxt_qplib_dev_attr *dev_attr;
2165 struct bnxt_re_ucontext *cntx = NULL;
2166 struct ib_ucontext *context;
2167 struct bnxt_qplib_qp *qplqp;
2168 struct bnxt_re_dev *rdev;
2169 struct bnxt_re_cq *cq;
2170 int rc = 0, qptype;
2171
2172 rdev = qp->rdev;
2173 qplqp = &qp->qplib_qp;
2174 dev_attr = rdev->dev_attr;
2175
2176 if (udata) {
2177 context = pd->ibpd.uobject->context;
2178 cntx = to_bnxt_re(context, struct bnxt_re_ucontext, ibucontext);
2179 }
2180
2181 /* Setup misc params */
2182 qplqp->is_user = false;
2183 qplqp->pd = &pd->qplib_pd;
2184 qplqp->qp_handle = (u64)qplqp;
2185 qplqp->sig_type = ((init_attr->sq_sig_type == IB_SIGNAL_ALL_WR) ?
2186 true : false);
2187 qptype = bnxt_re_init_qp_type(rdev, init_attr);
2188 if (qptype < 0) {
2189 rc = qptype;
2190 goto out;
2191 }
2192 qplqp->type = (u8)qptype;
2193 qplqp->wqe_mode = bnxt_re_is_var_size_supported(rdev, cntx);
2194 ether_addr_copy(qplqp->smac, rdev->dev_addr);
2195
2196 if (init_attr->qp_type == IB_QPT_RC) {
2197 qplqp->max_rd_atomic = dev_attr->max_qp_rd_atom;
2198 qplqp->max_dest_rd_atomic = dev_attr->max_qp_init_rd_atom;
2199 }
2200 qplqp->mtu = ib_mtu_enum_to_int(iboe_get_mtu(if_getmtu(rdev->netdev)));
2201 qplqp->dpi = &rdev->dpi_privileged; /* Doorbell page */
2202 if (init_attr->create_flags) {
2203 dev_dbg(rdev_to_dev(rdev),
2204 "QP create flags 0x%x not supported\n",
2205 init_attr->create_flags);
2206 return -EOPNOTSUPP;
2207 }
2208
2209 /* Setup CQs */
2210 if (init_attr->send_cq) {
2211 cq = to_bnxt_re(init_attr->send_cq, struct bnxt_re_cq, ibcq);
2212 if (!cq) {
2213 dev_err(rdev_to_dev(rdev), "Send CQ not found\n");
2214 rc = -EINVAL;
2215 goto out;
2216 }
2217 qplqp->scq = &cq->qplib_cq;
2218 qp->scq = cq;
2219 }
2220
2221 if (init_attr->recv_cq) {
2222 cq = to_bnxt_re(init_attr->recv_cq, struct bnxt_re_cq, ibcq);
2223 if (!cq) {
2224 dev_err(rdev_to_dev(rdev), "Receive CQ not found\n");
2225 rc = -EINVAL;
2226 goto out;
2227 }
2228 qplqp->rcq = &cq->qplib_cq;
2229 qp->rcq = cq;
2230 }
2231
2232 /* Setup RQ/SRQ */
2233 rc = bnxt_re_init_rq_attr(qp, init_attr, cntx);
2234 if (rc)
2235 goto out;
2236 if (init_attr->qp_type == IB_QPT_GSI)
2237 bnxt_re_adjust_gsi_rq_attr(qp);
2238
2239 /* Setup SQ */
2240 rc = bnxt_re_init_sq_attr(qp, init_attr, cntx, udata);
2241 if (rc)
2242 goto out;
2243 if (init_attr->qp_type == IB_QPT_GSI)
2244 bnxt_re_adjust_gsi_sq_attr(qp, init_attr, cntx);
2245
2246 if (udata) /* This will update DPI and qp_handle */
2247 rc = bnxt_re_init_user_qp(rdev, pd, qp, udata);
2248 out:
2249 return rc;
2250 }
2251
bnxt_re_create_shadow_gsi(struct bnxt_re_qp * qp,struct bnxt_re_pd * pd)2252 static int bnxt_re_create_shadow_gsi(struct bnxt_re_qp *qp,
2253 struct bnxt_re_pd *pd)
2254 {
2255 struct bnxt_re_sqp_entries *sqp_tbl = NULL;
2256 struct bnxt_re_dev *rdev;
2257 struct bnxt_re_qp *sqp;
2258 struct bnxt_re_ah *sah;
2259 int rc = 0;
2260
2261 rdev = qp->rdev;
2262 /* Create a shadow QP to handle the QP1 traffic */
2263 sqp_tbl = kzalloc(sizeof(*sqp_tbl) * BNXT_RE_MAX_GSI_SQP_ENTRIES,
2264 GFP_KERNEL);
2265 if (!sqp_tbl)
2266 return -ENOMEM;
2267 rdev->gsi_ctx.sqp_tbl = sqp_tbl;
2268
2269 sqp = bnxt_re_create_shadow_qp(pd, &rdev->qplib_res, &qp->qplib_qp);
2270 if (!sqp) {
2271 rc = -ENODEV;
2272 dev_err(rdev_to_dev(rdev),
2273 "Failed to create Shadow QP for QP1\n");
2274 goto out;
2275 }
2276 rdev->gsi_ctx.gsi_sqp = sqp;
2277
2278 sqp->rcq = qp->rcq;
2279 sqp->scq = qp->scq;
2280 sah = bnxt_re_create_shadow_qp_ah(pd, &rdev->qplib_res,
2281 &qp->qplib_qp);
2282 if (!sah) {
2283 bnxt_qplib_destroy_qp(&rdev->qplib_res,
2284 &sqp->qplib_qp);
2285 rc = -ENODEV;
2286 dev_err(rdev_to_dev(rdev),
2287 "Failed to create AH entry for ShadowQP\n");
2288 goto out;
2289 }
2290 rdev->gsi_ctx.gsi_sah = sah;
2291
2292 return 0;
2293 out:
2294 kfree(sqp_tbl);
2295 return rc;
2296 }
2297
__get_rq_hdr_buf_size(u8 gsi_mode)2298 static int __get_rq_hdr_buf_size(u8 gsi_mode)
2299 {
2300 return (gsi_mode == BNXT_RE_GSI_MODE_ALL) ?
2301 BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2 :
2302 BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE;
2303 }
2304
__get_sq_hdr_buf_size(u8 gsi_mode)2305 static int __get_sq_hdr_buf_size(u8 gsi_mode)
2306 {
2307 return (gsi_mode != BNXT_RE_GSI_MODE_ROCE_V1) ?
2308 BNXT_QPLIB_MAX_QP1_SQ_HDR_SIZE_V2 :
2309 BNXT_QPLIB_MAX_QP1_SQ_HDR_SIZE;
2310 }
2311
bnxt_re_create_gsi_qp(struct bnxt_re_qp * qp,struct bnxt_re_pd * pd)2312 static int bnxt_re_create_gsi_qp(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd)
2313 {
2314 struct bnxt_qplib_qp *qplqp;
2315 struct bnxt_qplib_res *res;
2316 struct bnxt_re_dev *rdev;
2317 u32 sstep, rstep;
2318 u8 gsi_mode;
2319 int rc = 0;
2320
2321 rdev = qp->rdev;
2322 qplqp = &qp->qplib_qp;
2323 res = &rdev->qplib_res;
2324 gsi_mode = rdev->gsi_ctx.gsi_qp_mode;
2325
2326 rstep = __get_rq_hdr_buf_size(gsi_mode);
2327 sstep = __get_sq_hdr_buf_size(gsi_mode);
2328 rc = bnxt_qplib_alloc_hdr_buf(res, qplqp, sstep, rstep);
2329 if (rc)
2330 goto out;
2331
2332 rc = bnxt_qplib_create_qp1(res, qplqp);
2333 if (rc) {
2334 dev_err(rdev_to_dev(rdev), "create HW QP1 failed!\n");
2335 goto out;
2336 }
2337
2338 if (gsi_mode == BNXT_RE_GSI_MODE_ALL)
2339 rc = bnxt_re_create_shadow_gsi(qp, pd);
2340 out:
2341 return rc;
2342 }
2343
bnxt_re_test_qp_limits(struct bnxt_re_dev * rdev,struct ib_qp_init_attr * init_attr,struct bnxt_qplib_dev_attr * dev_attr)2344 static bool bnxt_re_test_qp_limits(struct bnxt_re_dev *rdev,
2345 struct ib_qp_init_attr *init_attr,
2346 struct bnxt_qplib_dev_attr *dev_attr)
2347 {
2348 bool rc = true;
2349 int ilsize;
2350
2351 ilsize = ALIGN(init_attr->cap.max_inline_data, sizeof(struct sq_sge));
2352 if ((init_attr->cap.max_send_wr > dev_attr->max_qp_wqes) ||
2353 (init_attr->cap.max_recv_wr > dev_attr->max_qp_wqes) ||
2354 (init_attr->cap.max_send_sge > dev_attr->max_qp_sges) ||
2355 (init_attr->cap.max_recv_sge > dev_attr->max_qp_sges) ||
2356 (ilsize > dev_attr->max_inline_data)) {
2357 dev_err(rdev_to_dev(rdev), "Create QP failed - max exceeded! "
2358 "0x%x/0x%x 0x%x/0x%x 0x%x/0x%x "
2359 "0x%x/0x%x 0x%x/0x%x\n",
2360 init_attr->cap.max_send_wr, dev_attr->max_qp_wqes,
2361 init_attr->cap.max_recv_wr, dev_attr->max_qp_wqes,
2362 init_attr->cap.max_send_sge, dev_attr->max_qp_sges,
2363 init_attr->cap.max_recv_sge, dev_attr->max_qp_sges,
2364 init_attr->cap.max_inline_data,
2365 dev_attr->max_inline_data);
2366 rc = false;
2367 }
2368 return rc;
2369 }
2370
2371 static inline struct
__get_qp_from_qp_in(struct ib_pd * qp_in,struct bnxt_re_dev * rdev)2372 bnxt_re_qp *__get_qp_from_qp_in(struct ib_pd *qp_in,
2373 struct bnxt_re_dev *rdev)
2374 {
2375 struct bnxt_re_qp *qp;
2376
2377 qp = kzalloc(sizeof(*qp), GFP_KERNEL);
2378 if (!qp)
2379 dev_err(rdev_to_dev(rdev), "Allocate QP failed!\n");
2380 return qp;
2381 }
2382
bnxt_re_create_qp(struct ib_pd * qp_in,struct ib_qp_init_attr * qp_init_attr,struct ib_udata * udata)2383 struct ib_qp *bnxt_re_create_qp(struct ib_pd *qp_in,
2384 struct ib_qp_init_attr *qp_init_attr,
2385 struct ib_udata *udata)
2386 {
2387 struct bnxt_re_pd *pd;
2388 struct ib_pd *ib_pd = qp_in;
2389 struct bnxt_qplib_dev_attr *dev_attr;
2390 struct bnxt_re_dev *rdev;
2391 u32 active_qps, tmp_qps;
2392 struct bnxt_re_qp *qp;
2393 int rc;
2394
2395 pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd);
2396 rdev = pd->rdev;
2397 dev_attr = rdev->dev_attr;
2398 if (rdev->mod_exit) {
2399 rc = -EIO;
2400 dev_dbg(rdev_to_dev(rdev), "%s(): in mod_exit, just return!\n", __func__);
2401 goto exit;
2402 }
2403
2404 if (atomic_read(&rdev->stats.rsors.qp_count) >= dev_attr->max_qp) {
2405 dev_err(rdev_to_dev(rdev), "Create QP failed - max exceeded(QPs Alloc'd %u of max %u)\n",
2406 atomic_read(&rdev->stats.rsors.qp_count), dev_attr->max_qp);
2407 rc = -EINVAL;
2408 goto exit;
2409 }
2410
2411 rc = bnxt_re_test_qp_limits(rdev, qp_init_attr, dev_attr);
2412 if (!rc) {
2413 rc = -EINVAL;
2414 goto exit;
2415 }
2416 qp = __get_qp_from_qp_in(qp_in, rdev);
2417 if (!qp) {
2418 rc = -ENOMEM;
2419 goto exit;
2420 }
2421 qp->rdev = rdev;
2422
2423 rc = bnxt_re_init_qp_attr(qp, pd, qp_init_attr, udata);
2424 if (rc)
2425 goto fail;
2426
2427 if (qp_init_attr->qp_type == IB_QPT_GSI &&
2428 !_is_chip_gen_p5_p7(rdev->chip_ctx)) {
2429 rc = bnxt_re_create_gsi_qp(qp, pd);
2430 if (rc == -ENODEV)
2431 goto qp_destroy;
2432 if (rc)
2433 goto fail;
2434 } else {
2435 rc = bnxt_qplib_create_qp(&rdev->qplib_res, &qp->qplib_qp);
2436 if (rc) {
2437 dev_err(rdev_to_dev(rdev), "create HW QP failed!\n");
2438 goto free_umem;
2439 }
2440
2441 if (udata) {
2442 struct bnxt_re_qp_resp resp;
2443
2444 resp.qpid = qp->qplib_qp.id;
2445 rc = bnxt_re_copy_to_udata(rdev, &resp,
2446 min(udata->outlen, sizeof(resp)),
2447 udata);
2448 if (rc)
2449 goto qp_destroy;
2450 }
2451 }
2452
2453 qp->ib_qp.qp_num = qp->qplib_qp.id;
2454 if (qp_init_attr->qp_type == IB_QPT_GSI)
2455 rdev->gsi_ctx.gsi_qp = qp;
2456 spin_lock_init(&qp->sq_lock);
2457 spin_lock_init(&qp->rq_lock);
2458 INIT_LIST_HEAD(&qp->list);
2459 mutex_lock(&rdev->qp_lock);
2460 list_add_tail(&qp->list, &rdev->qp_list);
2461 mutex_unlock(&rdev->qp_lock);
2462 atomic_inc(&rdev->stats.rsors.qp_count);
2463 active_qps = atomic_read(&rdev->stats.rsors.qp_count);
2464 if (active_qps > atomic_read(&rdev->stats.rsors.max_qp_count))
2465 atomic_set(&rdev->stats.rsors.max_qp_count, active_qps);
2466
2467 bnxt_re_dump_debug_stats(rdev, active_qps);
2468
2469 /* Get the counters for RC QPs and UD QPs */
2470 if (qp_init_attr->qp_type == IB_QPT_RC) {
2471 tmp_qps = atomic_inc_return(&rdev->stats.rsors.rc_qp_count);
2472 if (tmp_qps > atomic_read(&rdev->stats.rsors.max_rc_qp_count))
2473 atomic_set(&rdev->stats.rsors.max_rc_qp_count, tmp_qps);
2474 } else if (qp_init_attr->qp_type == IB_QPT_UD) {
2475 tmp_qps = atomic_inc_return(&rdev->stats.rsors.ud_qp_count);
2476 if (tmp_qps > atomic_read(&rdev->stats.rsors.max_ud_qp_count))
2477 atomic_set(&rdev->stats.rsors.max_ud_qp_count, tmp_qps);
2478 }
2479
2480 return &qp->ib_qp;
2481
2482 qp_destroy:
2483 bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp);
2484 free_umem:
2485 if (udata) {
2486 if (qp->rumem && !IS_ERR(qp->rumem))
2487 ib_umem_release(qp->rumem);
2488 if (qp->sumem && !IS_ERR(qp->sumem))
2489 ib_umem_release(qp->sumem);
2490 }
2491 fail:
2492 kfree(qp);
2493 exit:
2494 return ERR_PTR(rc);
2495 }
2496
bnxt_re_modify_shadow_qp(struct bnxt_re_dev * rdev,struct bnxt_re_qp * qp1_qp,int qp_attr_mask)2497 static int bnxt_re_modify_shadow_qp(struct bnxt_re_dev *rdev,
2498 struct bnxt_re_qp *qp1_qp,
2499 int qp_attr_mask)
2500 {
2501 struct bnxt_re_qp *qp = rdev->gsi_ctx.gsi_sqp;
2502 int rc = 0;
2503
2504 if (qp_attr_mask & IB_QP_STATE) {
2505 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_STATE;
2506 qp->qplib_qp.state = qp1_qp->qplib_qp.state;
2507 }
2508 if (qp_attr_mask & IB_QP_PKEY_INDEX) {
2509 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PKEY;
2510 qp->qplib_qp.pkey_index = qp1_qp->qplib_qp.pkey_index;
2511 }
2512
2513 if (qp_attr_mask & IB_QP_QKEY) {
2514 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_QKEY;
2515 /* Using a Random QKEY */
2516 qp->qplib_qp.qkey = BNXT_RE_QP_RANDOM_QKEY;
2517 }
2518 if (qp_attr_mask & IB_QP_SQ_PSN) {
2519 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN;
2520 qp->qplib_qp.sq.psn = qp1_qp->qplib_qp.sq.psn;
2521 }
2522
2523 rc = bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp);
2524 if (rc)
2525 dev_err(rdev_to_dev(rdev), "Modify Shadow QP for QP1 failed\n");
2526 return rc;
2527 }
2528
ipv4_from_gid(u8 * gid)2529 static u32 ipv4_from_gid(u8 *gid)
2530 {
2531 return (gid[15] << 24 | gid[14] << 16 | gid[13] << 8 | gid[12]);
2532 }
2533
get_source_port(struct bnxt_re_dev * rdev,struct bnxt_re_qp * qp)2534 static u16 get_source_port(struct bnxt_re_dev *rdev,
2535 struct bnxt_re_qp *qp)
2536 {
2537 u8 ip_off, data[48], smac[ETH_ALEN];
2538 u16 crc = 0, buf_len = 0, i;
2539 u8 addr_len;
2540 u32 qpn;
2541
2542 if (qp->qplib_qp.nw_type == CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV6) {
2543 addr_len = 6;
2544 ip_off = 10;
2545 } else {
2546 addr_len = 4;
2547 ip_off = 12;
2548 }
2549
2550 memcpy(smac, qp->qplib_qp.smac, ETH_ALEN);
2551
2552 memset(data, 0, 48);
2553 memcpy(data, qp->qplib_qp.ah.dmac, ETH_ALEN);
2554 buf_len += ETH_ALEN;
2555
2556 memcpy(data + buf_len, smac, ETH_ALEN);
2557 buf_len += ETH_ALEN;
2558
2559 memcpy(data + buf_len, qp->qplib_qp.ah.dgid.data + ip_off, addr_len);
2560 buf_len += addr_len;
2561
2562 memcpy(data + buf_len, qp->qp_info_entry.sgid.raw + ip_off, addr_len);
2563 buf_len += addr_len;
2564
2565 qpn = htonl(qp->qplib_qp.dest_qpn);
2566 memcpy(data + buf_len, (u8 *)&qpn + 1, 3);
2567 buf_len += 3;
2568
2569 for (i = 0; i < buf_len; i++)
2570 crc = crc16(crc, (data + i), 1);
2571
2572 return crc;
2573 }
2574
bnxt_re_update_qp_info(struct bnxt_re_dev * rdev,struct bnxt_re_qp * qp)2575 static void bnxt_re_update_qp_info(struct bnxt_re_dev *rdev, struct bnxt_re_qp *qp)
2576 {
2577 u16 type;
2578
2579 type = __from_hw_to_ib_qp_type(qp->qplib_qp.type);
2580
2581 /* User-space can extract ip address with sgid_index. */
2582 if (ipv6_addr_v4mapped((struct in6_addr *)&qp->qplib_qp.ah.dgid)) {
2583 qp->qp_info_entry.s_ip.ipv4_addr = ipv4_from_gid(qp->qp_info_entry.sgid.raw);
2584 qp->qp_info_entry.d_ip.ipv4_addr = ipv4_from_gid(qp->qplib_qp.ah.dgid.data);
2585 } else {
2586 memcpy(&qp->qp_info_entry.s_ip.ipv6_addr, qp->qp_info_entry.sgid.raw,
2587 sizeof(qp->qp_info_entry.s_ip.ipv6_addr));
2588 memcpy(&qp->qp_info_entry.d_ip.ipv6_addr, qp->qplib_qp.ah.dgid.data,
2589 sizeof(qp->qp_info_entry.d_ip.ipv6_addr));
2590 }
2591
2592 if (type == IB_QPT_RC &&
2593 (qp->qplib_qp.nw_type == CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV4 ||
2594 qp->qplib_qp.nw_type == CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV6)) {
2595 qp->qp_info_entry.s_port = get_source_port(rdev, qp);
2596 }
2597 qp->qp_info_entry.d_port = BNXT_RE_QP_DEST_PORT;
2598 }
2599
bnxt_qplib_manage_flush_qp(struct bnxt_re_qp * qp)2600 static void bnxt_qplib_manage_flush_qp(struct bnxt_re_qp *qp)
2601 {
2602 struct bnxt_qplib_q *rq, *sq;
2603 unsigned long flags;
2604
2605 if (qp->sumem)
2606 return;
2607
2608 if (qp->qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_ERR) {
2609 rq = &qp->qplib_qp.rq;
2610 sq = &qp->qplib_qp.sq;
2611
2612 dev_dbg(rdev_to_dev(qp->rdev),
2613 "Move QP = %p to flush list\n", qp);
2614 flags = bnxt_re_lock_cqs(qp);
2615 bnxt_qplib_add_flush_qp(&qp->qplib_qp);
2616 bnxt_re_unlock_cqs(qp, flags);
2617
2618 if (sq->hwq.prod != sq->hwq.cons)
2619 bnxt_re_handle_cqn(&qp->scq->qplib_cq);
2620
2621 if (qp->rcq && (qp->rcq != qp->scq) &&
2622 (rq->hwq.prod != rq->hwq.cons))
2623 bnxt_re_handle_cqn(&qp->rcq->qplib_cq);
2624 }
2625
2626 if (qp->qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_RESET) {
2627 dev_dbg(rdev_to_dev(qp->rdev),
2628 "Move QP = %p out of flush list\n", qp);
2629 flags = bnxt_re_lock_cqs(qp);
2630 bnxt_qplib_clean_qp(&qp->qplib_qp);
2631 bnxt_re_unlock_cqs(qp, flags);
2632 }
2633 }
2634
ib_modify_qp_is_ok_compat(enum ib_qp_state cur_state,enum ib_qp_state next_state,enum ib_qp_type type,enum ib_qp_attr_mask mask)2635 bool ib_modify_qp_is_ok_compat(enum ib_qp_state cur_state,
2636 enum ib_qp_state next_state,
2637 enum ib_qp_type type,
2638 enum ib_qp_attr_mask mask)
2639 {
2640 return (ib_modify_qp_is_ok(cur_state, next_state,
2641 type, mask));
2642 }
2643
bnxt_re_modify_qp(struct ib_qp * ib_qp,struct ib_qp_attr * qp_attr,int qp_attr_mask,struct ib_udata * udata)2644 int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,
2645 int qp_attr_mask, struct ib_udata *udata)
2646 {
2647 enum ib_qp_state curr_qp_state, new_qp_state;
2648 struct bnxt_re_modify_qp_ex_resp resp = {};
2649 struct bnxt_re_modify_qp_ex_req ureq = {};
2650 struct bnxt_qplib_dev_attr *dev_attr;
2651 struct bnxt_qplib_ppp *ppp = NULL;
2652 struct bnxt_re_dev *rdev;
2653 struct bnxt_re_qp *qp;
2654 struct ib_gid_attr *sgid_attr;
2655 struct ib_gid_attr gid_attr;
2656 union ib_gid sgid, *gid_ptr = NULL;
2657 u8 nw_type;
2658 int rc, entries, status;
2659 bool is_copy_to_udata = false;
2660 bool is_qpmtu_high = false;
2661
2662 qp = to_bnxt_re(ib_qp, struct bnxt_re_qp, ib_qp);
2663 rdev = qp->rdev;
2664 dev_attr = rdev->dev_attr;
2665
2666 qp->qplib_qp.modify_flags = 0;
2667 ppp = &qp->qplib_qp.ppp;
2668 if (qp_attr_mask & IB_QP_STATE) {
2669 curr_qp_state = __to_ib_qp_state(qp->qplib_qp.cur_qp_state);
2670 new_qp_state = qp_attr->qp_state;
2671 if (!ib_modify_qp_is_ok_compat(curr_qp_state, new_qp_state,
2672 ib_qp->qp_type, qp_attr_mask)) {
2673 dev_err(rdev_to_dev(rdev),"invalid attribute mask=0x%x"
2674 " specified for qpn=0x%x of type=0x%x"
2675 " current_qp_state=0x%x, new_qp_state=0x%x\n",
2676 qp_attr_mask, ib_qp->qp_num, ib_qp->qp_type,
2677 curr_qp_state, new_qp_state);
2678 return -EINVAL;
2679 }
2680 dev_dbg(rdev_to_dev(rdev), "%s:%d INFO attribute mask=0x%x qpn=0x%x "
2681 "of type=0x%x current_qp_state=0x%x, new_qp_state=0x%x\n",
2682 __func__, __LINE__, qp_attr_mask, ib_qp->qp_num,
2683 ib_qp->qp_type, curr_qp_state, new_qp_state);
2684 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_STATE;
2685 qp->qplib_qp.state = __from_ib_qp_state(qp_attr->qp_state);
2686
2687 if (udata && curr_qp_state == IB_QPS_RESET &&
2688 new_qp_state == IB_QPS_INIT) {
2689 if (!ib_copy_from_udata(&ureq, udata, sizeof(ureq))) {
2690 if (ureq.comp_mask &
2691 BNXT_RE_COMP_MASK_MQP_EX_PPP_REQ_EN_MASK) {
2692 ppp->req = BNXT_QPLIB_PPP_REQ;
2693 ppp->dpi = ureq.dpi;
2694 }
2695 }
2696 }
2697 }
2698 if (qp_attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY) {
2699 qp->qplib_qp.modify_flags |=
2700 CMDQ_MODIFY_QP_MODIFY_MASK_EN_SQD_ASYNC_NOTIFY;
2701 qp->qplib_qp.en_sqd_async_notify = true;
2702 }
2703 if (qp_attr_mask & IB_QP_ACCESS_FLAGS) {
2704 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS;
2705 qp->qplib_qp.access =
2706 __from_ib_access_flags(qp_attr->qp_access_flags);
2707 /* LOCAL_WRITE access must be set to allow RC receive */
2708 qp->qplib_qp.access |= BNXT_QPLIB_ACCESS_LOCAL_WRITE;
2709 qp->qplib_qp.access |= CMDQ_MODIFY_QP_ACCESS_REMOTE_WRITE;
2710 qp->qplib_qp.access |= CMDQ_MODIFY_QP_ACCESS_REMOTE_READ;
2711 }
2712 if (qp_attr_mask & IB_QP_PKEY_INDEX) {
2713 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PKEY;
2714 qp->qplib_qp.pkey_index = qp_attr->pkey_index;
2715 }
2716 if (qp_attr_mask & IB_QP_QKEY) {
2717 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_QKEY;
2718 qp->qplib_qp.qkey = qp_attr->qkey;
2719 }
2720 if (qp_attr_mask & IB_QP_AV) {
2721 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_DGID |
2722 CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL |
2723 CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX |
2724 CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT |
2725 CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS |
2726 CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC |
2727 CMDQ_MODIFY_QP_MODIFY_MASK_VLAN_ID;
2728 memcpy(qp->qplib_qp.ah.dgid.data, qp_attr->ah_attr.grh.dgid.raw,
2729 sizeof(qp->qplib_qp.ah.dgid.data));
2730 qp->qplib_qp.ah.flow_label = qp_attr->ah_attr.grh.flow_label;
2731 qp->qplib_qp.ah.sgid_index = _get_sgid_index(rdev,
2732 qp_attr->ah_attr.grh.sgid_index);
2733 qp->qplib_qp.ah.host_sgid_index = qp_attr->ah_attr.grh.sgid_index;
2734 qp->qplib_qp.ah.hop_limit = qp_attr->ah_attr.grh.hop_limit;
2735 qp->qplib_qp.ah.traffic_class =
2736 qp_attr->ah_attr.grh.traffic_class;
2737 qp->qplib_qp.ah.sl = qp_attr->ah_attr.sl;
2738 ether_addr_copy(qp->qplib_qp.ah.dmac, ROCE_DMAC(&qp_attr->ah_attr));
2739 sgid_attr = &gid_attr;
2740 status = bnxt_re_get_cached_gid(&rdev->ibdev, 1,
2741 qp_attr->ah_attr.grh.sgid_index,
2742 &sgid, &sgid_attr,
2743 &qp_attr->ah_attr.grh, NULL);
2744 if (!status)
2745 if_rele(sgid_attr->ndev);
2746 gid_ptr = &sgid;
2747 if (sgid_attr->ndev) {
2748 memcpy(qp->qplib_qp.smac, rdev->dev_addr,
2749 ETH_ALEN);
2750 nw_type = bnxt_re_gid_to_network_type(sgid_attr, &sgid);
2751 dev_dbg(rdev_to_dev(rdev),
2752 "Connection using the nw_type %d\n", nw_type);
2753 switch (nw_type) {
2754 case RDMA_NETWORK_IPV4:
2755 qp->qplib_qp.nw_type =
2756 CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV4;
2757 break;
2758 case RDMA_NETWORK_IPV6:
2759 qp->qplib_qp.nw_type =
2760 CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV6;
2761 break;
2762 default:
2763 qp->qplib_qp.nw_type =
2764 CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV1;
2765 break;
2766 }
2767 }
2768 memcpy(&qp->qp_info_entry.sgid, gid_ptr, sizeof(qp->qp_info_entry.sgid));
2769 }
2770
2771 /* MTU settings allowed only during INIT -> RTR */
2772 if (qp_attr->qp_state == IB_QPS_RTR) {
2773 bnxt_re_init_qpmtu(qp, if_getmtu(rdev->netdev), qp_attr_mask, qp_attr,
2774 &is_qpmtu_high);
2775 if (udata && !ib_copy_from_udata(&ureq, udata, sizeof(ureq))) {
2776 if (ureq.comp_mask & BNXT_RE_COMP_MASK_MQP_EX_PATH_MTU_MASK) {
2777 resp.comp_mask |= BNXT_RE_COMP_MASK_MQP_EX_PATH_MTU_MASK;
2778 resp.path_mtu = qp->qplib_qp.mtu;
2779 is_copy_to_udata = true;
2780 } else if (is_qpmtu_high) {
2781 dev_err(rdev_to_dev(rdev), "qp %#x invalid mtu\n",
2782 qp->qplib_qp.id);
2783 return -EINVAL;
2784 }
2785 }
2786 }
2787
2788 if (qp_attr_mask & IB_QP_TIMEOUT) {
2789 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT;
2790 qp->qplib_qp.timeout = qp_attr->timeout;
2791 }
2792 if (qp_attr_mask & IB_QP_RETRY_CNT) {
2793 qp->qplib_qp.modify_flags |=
2794 CMDQ_MODIFY_QP_MODIFY_MASK_RETRY_CNT;
2795 qp->qplib_qp.retry_cnt = qp_attr->retry_cnt;
2796 }
2797 if (qp_attr_mask & IB_QP_RNR_RETRY) {
2798 qp->qplib_qp.modify_flags |=
2799 CMDQ_MODIFY_QP_MODIFY_MASK_RNR_RETRY;
2800 qp->qplib_qp.rnr_retry = qp_attr->rnr_retry;
2801 }
2802 if (qp_attr_mask & IB_QP_MIN_RNR_TIMER) {
2803 qp->qplib_qp.modify_flags |=
2804 CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER;
2805 qp->qplib_qp.min_rnr_timer = qp_attr->min_rnr_timer;
2806 }
2807 if (qp_attr_mask & IB_QP_RQ_PSN) {
2808 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN;
2809 qp->qplib_qp.rq.psn = qp_attr->rq_psn;
2810 }
2811 if (qp_attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
2812 qp->qplib_qp.modify_flags |=
2813 CMDQ_MODIFY_QP_MODIFY_MASK_MAX_RD_ATOMIC;
2814 /* Cap the max_rd_atomic to device max */
2815 if (qp_attr->max_rd_atomic > dev_attr->max_qp_rd_atom)
2816 dev_dbg(rdev_to_dev(rdev),
2817 "max_rd_atomic requested %d is > device max %d\n",
2818 qp_attr->max_rd_atomic,
2819 dev_attr->max_qp_rd_atom);
2820 qp->qplib_qp.max_rd_atomic = min_t(u32, qp_attr->max_rd_atomic,
2821 dev_attr->max_qp_rd_atom);
2822 }
2823 if (qp_attr_mask & IB_QP_SQ_PSN) {
2824 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN;
2825 qp->qplib_qp.sq.psn = qp_attr->sq_psn;
2826 }
2827 if (qp_attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
2828 if (qp_attr->max_dest_rd_atomic >
2829 dev_attr->max_qp_init_rd_atom) {
2830 dev_err(rdev_to_dev(rdev),
2831 "max_dest_rd_atomic requested %d is > device max %d\n",
2832 qp_attr->max_dest_rd_atomic,
2833 dev_attr->max_qp_init_rd_atom);
2834 return -EINVAL;
2835 }
2836 qp->qplib_qp.modify_flags |=
2837 CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC;
2838 qp->qplib_qp.max_dest_rd_atomic = qp_attr->max_dest_rd_atomic;
2839 }
2840 if (qp_attr_mask & IB_QP_CAP) {
2841 qp->qplib_qp.modify_flags |=
2842 CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SIZE |
2843 CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SIZE |
2844 CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SGE |
2845 CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SGE |
2846 CMDQ_MODIFY_QP_MODIFY_MASK_MAX_INLINE_DATA;
2847 if ((qp_attr->cap.max_send_wr >= dev_attr->max_qp_wqes) ||
2848 (qp_attr->cap.max_recv_wr >= dev_attr->max_qp_wqes) ||
2849 (qp_attr->cap.max_send_sge >= dev_attr->max_qp_sges) ||
2850 (qp_attr->cap.max_recv_sge >= dev_attr->max_qp_sges) ||
2851 (qp_attr->cap.max_inline_data >=
2852 dev_attr->max_inline_data)) {
2853 dev_err(rdev_to_dev(rdev),
2854 "Create QP failed - max exceeded\n");
2855 return -EINVAL;
2856 }
2857 entries = roundup_pow_of_two(qp_attr->cap.max_send_wr);
2858 if (entries > dev_attr->max_qp_wqes)
2859 entries = dev_attr->max_qp_wqes;
2860 entries = min_t(u32, entries, dev_attr->max_qp_wqes);
2861 qp->qplib_qp.sq.max_wqe = entries;
2862 qp->qplib_qp.sq.q_full_delta = qp->qplib_qp.sq.max_wqe -
2863 qp_attr->cap.max_send_wr;
2864 /*
2865 * Reserving one slot for Phantom WQE. Some application can
2866 * post one extra entry in this case. Allowing this to avoid
2867 * unexpected Queue full condition
2868 */
2869 qp->qplib_qp.sq.q_full_delta -= 1;
2870 qp->qplib_qp.sq.max_sge = qp_attr->cap.max_send_sge;
2871 if (qp->qplib_qp.rq.max_wqe) {
2872 entries = roundup_pow_of_two(qp_attr->cap.max_recv_wr);
2873 if (entries > dev_attr->max_qp_wqes)
2874 entries = dev_attr->max_qp_wqes;
2875 qp->qplib_qp.rq.max_wqe = entries;
2876 qp->qplib_qp.rq.max_sw_wqe = qp->qplib_qp.rq.max_wqe;
2877 qp->qplib_qp.rq.q_full_delta = qp->qplib_qp.rq.max_wqe -
2878 qp_attr->cap.max_recv_wr;
2879 qp->qplib_qp.rq.max_sge = qp_attr->cap.max_recv_sge;
2880 } else {
2881 /* SRQ was used prior, just ignore the RQ caps */
2882 }
2883 }
2884 if (qp_attr_mask & IB_QP_DEST_QPN) {
2885 qp->qplib_qp.modify_flags |=
2886 CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID;
2887 qp->qplib_qp.dest_qpn = qp_attr->dest_qp_num;
2888 }
2889
2890 rc = bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp);
2891 if (rc) {
2892 dev_err(rdev_to_dev(rdev), "Modify HW QP failed!\n");
2893 return rc;
2894 }
2895 if (qp_attr_mask & IB_QP_STATE)
2896 bnxt_qplib_manage_flush_qp(qp);
2897 if (ureq.comp_mask & BNXT_RE_COMP_MASK_MQP_EX_PPP_REQ_EN_MASK &&
2898 ppp->st_idx_en & CREQ_MODIFY_QP_RESP_PINGPONG_PUSH_ENABLED) {
2899 resp.comp_mask |= BNXT_RE_COMP_MASK_MQP_EX_PPP_REQ_EN;
2900 resp.ppp_st_idx = ppp->st_idx_en >>
2901 BNXT_QPLIB_PPP_ST_IDX_SHIFT;
2902 is_copy_to_udata = true;
2903 }
2904
2905 if (is_copy_to_udata) {
2906 rc = bnxt_re_copy_to_udata(rdev, &resp,
2907 min(udata->outlen, sizeof(resp)),
2908 udata);
2909 if (rc)
2910 return rc;
2911 }
2912
2913 if (ib_qp->qp_type == IB_QPT_GSI &&
2914 rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_ALL &&
2915 rdev->gsi_ctx.gsi_sqp)
2916 rc = bnxt_re_modify_shadow_qp(rdev, qp, qp_attr_mask);
2917 /*
2918 * Update info when qp_info_info
2919 */
2920 bnxt_re_update_qp_info(rdev, qp);
2921 return rc;
2922 }
2923
bnxt_re_query_qp(struct ib_qp * ib_qp,struct ib_qp_attr * qp_attr,int qp_attr_mask,struct ib_qp_init_attr * qp_init_attr)2924 int bnxt_re_query_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,
2925 int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr)
2926 {
2927 struct bnxt_re_qp *qp = to_bnxt_re(ib_qp, struct bnxt_re_qp, ib_qp);
2928 struct bnxt_re_dev *rdev = qp->rdev;
2929 struct bnxt_qplib_qp *qplib_qp;
2930 int rc;
2931
2932 qplib_qp = kcalloc(1, sizeof(*qplib_qp), GFP_KERNEL);
2933 if (!qplib_qp)
2934 return -ENOMEM;
2935
2936 qplib_qp->id = qp->qplib_qp.id;
2937 qplib_qp->ah.host_sgid_index = qp->qplib_qp.ah.host_sgid_index;
2938
2939 rc = bnxt_qplib_query_qp(&rdev->qplib_res, qplib_qp);
2940 if (rc) {
2941 dev_err(rdev_to_dev(rdev), "Query HW QP (0x%x) failed! rc = %d\n",
2942 qplib_qp->id, rc);
2943 goto free_mem;
2944 }
2945 qp_attr->qp_state = __to_ib_qp_state(qplib_qp->state);
2946 qp_attr->cur_qp_state = __to_ib_qp_state(qplib_qp->cur_qp_state);
2947 qp_attr->en_sqd_async_notify = qplib_qp->en_sqd_async_notify ? 1 : 0;
2948 qp_attr->qp_access_flags = __to_ib_access_flags(qplib_qp->access);
2949 qp_attr->pkey_index = qplib_qp->pkey_index;
2950 qp_attr->qkey = qplib_qp->qkey;
2951 memcpy(qp_attr->ah_attr.grh.dgid.raw, qplib_qp->ah.dgid.data,
2952 sizeof(qplib_qp->ah.dgid.data));
2953 qp_attr->ah_attr.grh.flow_label = qplib_qp->ah.flow_label;
2954 qp_attr->ah_attr.grh.sgid_index = qplib_qp->ah.host_sgid_index;
2955 qp_attr->ah_attr.grh.hop_limit = qplib_qp->ah.hop_limit;
2956 qp_attr->ah_attr.grh.traffic_class = qplib_qp->ah.traffic_class;
2957 qp_attr->ah_attr.sl = qplib_qp->ah.sl;
2958 ether_addr_copy(ROCE_DMAC(&qp_attr->ah_attr), qplib_qp->ah.dmac);
2959 qp_attr->path_mtu = __to_ib_mtu(qplib_qp->path_mtu);
2960 qp_attr->timeout = qplib_qp->timeout;
2961 qp_attr->retry_cnt = qplib_qp->retry_cnt;
2962 qp_attr->rnr_retry = qplib_qp->rnr_retry;
2963 qp_attr->min_rnr_timer = qplib_qp->min_rnr_timer;
2964 qp_attr->rq_psn = qplib_qp->rq.psn;
2965 qp_attr->max_rd_atomic = qplib_qp->max_rd_atomic;
2966 qp_attr->sq_psn = qplib_qp->sq.psn;
2967 qp_attr->max_dest_rd_atomic = qplib_qp->max_dest_rd_atomic;
2968 qp_init_attr->sq_sig_type = qplib_qp->sig_type ? IB_SIGNAL_ALL_WR :
2969 IB_SIGNAL_REQ_WR;
2970 qp_attr->dest_qp_num = qplib_qp->dest_qpn;
2971
2972 qp_attr->cap.max_send_wr = qp->qplib_qp.sq.max_wqe;
2973 qp_attr->cap.max_send_sge = qp->qplib_qp.sq.max_sge;
2974 qp_attr->cap.max_recv_wr = qp->qplib_qp.rq.max_wqe;
2975 qp_attr->cap.max_recv_sge = qp->qplib_qp.rq.max_sge;
2976 qp_attr->cap.max_inline_data = qp->qplib_qp.max_inline_data;
2977 qp_init_attr->cap = qp_attr->cap;
2978
2979 free_mem:
2980 kfree(qplib_qp);
2981 return rc;
2982 }
2983
2984 /* Builders */
2985
2986 /* For Raw, the application is responsible to build the entire packet */
bnxt_re_build_raw_send(const struct ib_send_wr * wr,struct bnxt_qplib_swqe * wqe)2987 static void bnxt_re_build_raw_send(const struct ib_send_wr *wr,
2988 struct bnxt_qplib_swqe *wqe)
2989 {
2990 switch (wr->send_flags) {
2991 case IB_SEND_IP_CSUM:
2992 wqe->rawqp1.lflags |= SQ_SEND_RAWETH_QP1_LFLAGS_IP_CHKSUM;
2993 break;
2994 default:
2995 /* Pad HW RoCE iCRC */
2996 wqe->rawqp1.lflags |= SQ_SEND_RAWETH_QP1_LFLAGS_ROCE_CRC;
2997 break;
2998 }
2999 }
3000
3001 /* For QP1, the driver must build the entire RoCE (v1/v2) packet hdr
3002 * as according to the sgid and AV
3003 */
bnxt_re_build_qp1_send(struct bnxt_re_qp * qp,const struct ib_send_wr * wr,struct bnxt_qplib_swqe * wqe,int payload_size)3004 static int bnxt_re_build_qp1_send(struct bnxt_re_qp *qp, const struct ib_send_wr *wr,
3005 struct bnxt_qplib_swqe *wqe, int payload_size)
3006 {
3007 struct bnxt_re_ah *ah = to_bnxt_re(ud_wr(wr)->ah, struct bnxt_re_ah,
3008 ibah);
3009 struct bnxt_qplib_ah *qplib_ah = &ah->qplib_ah;
3010 struct bnxt_qplib_sge sge;
3011 int i, rc = 0;
3012 union ib_gid sgid;
3013 u16 vlan_id;
3014 u8 *ptmac;
3015 void *buf;
3016
3017 memset(&qp->qp1_hdr, 0, sizeof(qp->qp1_hdr));
3018
3019 /* Get sgid */
3020 rc = bnxt_re_query_gid(&qp->rdev->ibdev, 1, qplib_ah->sgid_index, &sgid);
3021 if (rc)
3022 return rc;
3023
3024 /* ETH */
3025 qp->qp1_hdr.eth_present = 1;
3026 ptmac = ah->qplib_ah.dmac;
3027 memcpy(qp->qp1_hdr.eth.dmac_h, ptmac, 4);
3028 ptmac += 4;
3029 memcpy(qp->qp1_hdr.eth.dmac_l, ptmac, 2);
3030
3031 ptmac = qp->qplib_qp.smac;
3032 memcpy(qp->qp1_hdr.eth.smac_h, ptmac, 2);
3033 ptmac += 2;
3034 memcpy(qp->qp1_hdr.eth.smac_l, ptmac, 4);
3035
3036 qp->qp1_hdr.eth.type = cpu_to_be16(BNXT_QPLIB_ETHTYPE_ROCEV1);
3037
3038 /* For vlan, check the sgid for vlan existence */
3039 vlan_id = rdma_get_vlan_id(&sgid);
3040 if (vlan_id && vlan_id < 0x1000) {
3041 qp->qp1_hdr.vlan_present = 1;
3042 qp->qp1_hdr.eth.type = cpu_to_be16(ETH_P_8021Q);
3043 }
3044 /* GRH */
3045 qp->qp1_hdr.grh_present = 1;
3046 qp->qp1_hdr.grh.ip_version = 6;
3047 qp->qp1_hdr.grh.payload_length =
3048 cpu_to_be16((IB_BTH_BYTES + IB_DETH_BYTES + payload_size + 7)
3049 & ~3);
3050 qp->qp1_hdr.grh.next_header = 0x1b;
3051 memcpy(qp->qp1_hdr.grh.source_gid.raw, sgid.raw, sizeof(sgid));
3052 memcpy(qp->qp1_hdr.grh.destination_gid.raw, qplib_ah->dgid.data,
3053 sizeof(sgid));
3054
3055 /* BTH */
3056 if (wr->opcode == IB_WR_SEND_WITH_IMM) {
3057 qp->qp1_hdr.bth.opcode = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE;
3058 qp->qp1_hdr.immediate_present = 1;
3059 } else {
3060 qp->qp1_hdr.bth.opcode = IB_OPCODE_UD_SEND_ONLY;
3061 }
3062 if (wr->send_flags & IB_SEND_SOLICITED)
3063 qp->qp1_hdr.bth.solicited_event = 1;
3064 qp->qp1_hdr.bth.pad_count = (4 - payload_size) & 3;
3065 /* P_key for QP1 is for all members */
3066 qp->qp1_hdr.bth.pkey = cpu_to_be16(0xFFFF);
3067 qp->qp1_hdr.bth.destination_qpn = IB_QP1;
3068 qp->qp1_hdr.bth.ack_req = 0;
3069 qp->send_psn++;
3070 qp->send_psn &= BTH_PSN_MASK;
3071 qp->qp1_hdr.bth.psn = cpu_to_be32(qp->send_psn);
3072 /* DETH */
3073 /* Use the priviledged Q_Key for QP1 */
3074 qp->qp1_hdr.deth.qkey = cpu_to_be32(IB_QP1_QKEY);
3075 qp->qp1_hdr.deth.source_qpn = IB_QP1;
3076
3077 /* Pack the QP1 to the transmit buffer */
3078 buf = bnxt_qplib_get_qp1_sq_buf(&qp->qplib_qp, &sge);
3079 if (!buf) {
3080 dev_err(rdev_to_dev(qp->rdev), "QP1 buffer is empty!\n");
3081 rc = -ENOMEM;
3082 }
3083 for (i = wqe->num_sge; i; i--) {
3084 wqe->sg_list[i].addr = wqe->sg_list[i - 1].addr;
3085 wqe->sg_list[i].lkey = wqe->sg_list[i - 1].lkey;
3086 wqe->sg_list[i].size = wqe->sg_list[i - 1].size;
3087 }
3088 wqe->sg_list[0].addr = sge.addr;
3089 wqe->sg_list[0].lkey = sge.lkey;
3090 wqe->sg_list[0].size = sge.size;
3091 wqe->num_sge++;
3092
3093 return rc;
3094 }
3095
bnxt_re_build_gsi_send(struct bnxt_re_qp * qp,const struct ib_send_wr * wr,struct bnxt_qplib_swqe * wqe)3096 static int bnxt_re_build_gsi_send(struct bnxt_re_qp *qp,
3097 const struct ib_send_wr *wr,
3098 struct bnxt_qplib_swqe *wqe)
3099 {
3100 struct bnxt_re_dev *rdev;
3101 int rc, indx, len = 0;
3102
3103 rdev = qp->rdev;
3104
3105 /* Mode UD is applicable to Gen P5 only */
3106 if (rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_UD)
3107 return 0;
3108
3109 for (indx = 0; indx < wr->num_sge; indx++) {
3110 wqe->sg_list[indx].addr = wr->sg_list[indx].addr;
3111 wqe->sg_list[indx].lkey = wr->sg_list[indx].lkey;
3112 wqe->sg_list[indx].size = wr->sg_list[indx].length;
3113 len += wr->sg_list[indx].length;
3114 }
3115 rc = bnxt_re_build_qp1_send(qp, wr, wqe, len);
3116 wqe->rawqp1.lflags |= SQ_SEND_RAWETH_QP1_LFLAGS_ROCE_CRC;
3117
3118 return rc;
3119 }
3120
3121 /* For the MAD layer, it only provides the recv SGE the size of
3122 ib_grh + MAD datagram. No Ethernet headers, Ethertype, BTH, DETH,
3123 nor RoCE iCRC. The Cu+ solution must provide buffer for the entire
3124 receive packet (334 bytes) with no VLAN and then copy the GRH
3125 and the MAD datagram out to the provided SGE.
3126 */
3127
bnxt_re_build_qp1_recv(struct bnxt_re_qp * qp,const struct ib_recv_wr * wr,struct bnxt_qplib_swqe * wqe)3128 static int bnxt_re_build_qp1_recv(struct bnxt_re_qp *qp,
3129 const struct ib_recv_wr *wr,
3130 struct bnxt_qplib_swqe *wqe)
3131 {
3132 struct bnxt_re_dev *rdev = qp->rdev;
3133 struct bnxt_qplib_sge ref, sge;
3134 u8 udp_hdr_size = 0;
3135 u8 ip_hdr_size = 0;
3136 int rc = 0;
3137 int size;
3138
3139 if (bnxt_qplib_get_qp1_rq_buf(&qp->qplib_qp, &sge)) {
3140 /* Create 5 SGEs as according to the following:
3141 * Ethernet header (14)
3142 * ib_grh (40) - as provided from the wr
3143 * ib_bth + ib_deth + UDP(RoCE v2 only) (28)
3144 * MAD (256) - as provided from the wr
3145 * iCRC (4)
3146 */
3147
3148 /* Set RoCE v2 header size and offsets */
3149 if (rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_ROCE_V2_IPV4)
3150 ip_hdr_size = 20;
3151 if (rdev->gsi_ctx.gsi_qp_mode != BNXT_RE_GSI_MODE_ROCE_V1)
3152 udp_hdr_size = 8;
3153
3154 /* Save the reference from ULP */
3155 ref.addr = wr->sg_list[0].addr;
3156 ref.lkey = wr->sg_list[0].lkey;
3157 ref.size = wr->sg_list[0].length;
3158
3159 /* SGE 1 */
3160 size = sge.size;
3161 wqe->sg_list[0].addr = sge.addr;
3162 wqe->sg_list[0].lkey = sge.lkey;
3163 wqe->sg_list[0].size = BNXT_QPLIB_MAX_QP1_RQ_ETH_HDR_SIZE;
3164 size -= wqe->sg_list[0].size;
3165 if (size <= 0) {
3166 dev_err(rdev_to_dev(qp->rdev),"QP1 rq buffer is empty!\n");
3167 rc = -ENOMEM;
3168 goto done;
3169 }
3170 sge.size = (u32)size;
3171 sge.addr += wqe->sg_list[0].size;
3172
3173 /* SGE 2 */
3174 /* In case of RoCE v2 ipv4 lower 20 bytes should have IP hdr */
3175 wqe->sg_list[1].addr = ref.addr + ip_hdr_size;
3176 wqe->sg_list[1].lkey = ref.lkey;
3177 wqe->sg_list[1].size = sizeof(struct ib_grh) - ip_hdr_size;
3178 ref.size -= wqe->sg_list[1].size;
3179 if (ref.size <= 0) {
3180 dev_err(rdev_to_dev(qp->rdev),
3181 "QP1 ref buffer is empty!\n");
3182 rc = -ENOMEM;
3183 goto done;
3184 }
3185 ref.addr += wqe->sg_list[1].size + ip_hdr_size;
3186
3187 /* SGE 3 */
3188 wqe->sg_list[2].addr = sge.addr;
3189 wqe->sg_list[2].lkey = sge.lkey;
3190 wqe->sg_list[2].size = BNXT_QPLIB_MAX_QP1_RQ_BDETH_HDR_SIZE +
3191 udp_hdr_size;
3192 size -= wqe->sg_list[2].size;
3193 if (size <= 0) {
3194 dev_err(rdev_to_dev(qp->rdev),
3195 "QP1 rq buffer is empty!\n");
3196 rc = -ENOMEM;
3197 goto done;
3198 }
3199 sge.size = (u32)size;
3200 sge.addr += wqe->sg_list[2].size;
3201
3202 /* SGE 4 */
3203 wqe->sg_list[3].addr = ref.addr;
3204 wqe->sg_list[3].lkey = ref.lkey;
3205 wqe->sg_list[3].size = ref.size;
3206 ref.size -= wqe->sg_list[3].size;
3207 if (ref.size) {
3208 dev_err(rdev_to_dev(qp->rdev),
3209 "QP1 ref buffer is incorrect!\n");
3210 rc = -ENOMEM;
3211 goto done;
3212 }
3213 /* SGE 5 */
3214 wqe->sg_list[4].addr = sge.addr;
3215 wqe->sg_list[4].lkey = sge.lkey;
3216 wqe->sg_list[4].size = sge.size;
3217 size -= wqe->sg_list[4].size;
3218 if (size) {
3219 dev_err(rdev_to_dev(qp->rdev),
3220 "QP1 rq buffer is incorrect!\n");
3221 rc = -ENOMEM;
3222 goto done;
3223 }
3224 sge.size = (u32)size;
3225 wqe->num_sge = 5;
3226 } else {
3227 dev_err(rdev_to_dev(qp->rdev), "QP1 buffer is empty!\n");
3228 rc = -ENOMEM;
3229 }
3230 done:
3231 return rc;
3232 }
3233
bnxt_re_build_qp1_shadow_qp_recv(struct bnxt_re_qp * qp,const struct ib_recv_wr * wr,struct bnxt_qplib_swqe * wqe)3234 static int bnxt_re_build_qp1_shadow_qp_recv(struct bnxt_re_qp *qp,
3235 const struct ib_recv_wr *wr,
3236 struct bnxt_qplib_swqe *wqe)
3237 {
3238 struct bnxt_re_sqp_entries *sqp_entry;
3239 struct bnxt_qplib_sge sge;
3240 struct bnxt_re_dev *rdev;
3241 u32 rq_prod_index;
3242 int rc = 0;
3243
3244 rdev = qp->rdev;
3245
3246 rq_prod_index = bnxt_qplib_get_rq_prod_index(&qp->qplib_qp);
3247
3248 if (bnxt_qplib_get_qp1_rq_buf(&qp->qplib_qp, &sge)) {
3249 /* Create 1 SGE to receive the entire
3250 * ethernet packet
3251 */
3252 /* SGE 1 */
3253 wqe->sg_list[0].addr = sge.addr;
3254 /* TODO check the lkey to be used */
3255 wqe->sg_list[0].lkey = sge.lkey;
3256 wqe->sg_list[0].size = BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2;
3257 if (sge.size < wqe->sg_list[0].size) {
3258 dev_err(rdev_to_dev(qp->rdev),
3259 "QP1 rq buffer is empty!\n");
3260 rc = -ENOMEM;
3261 goto done;
3262 }
3263
3264 sqp_entry = &rdev->gsi_ctx.sqp_tbl[rq_prod_index];
3265 sqp_entry->sge.addr = wr->sg_list[0].addr;
3266 sqp_entry->sge.lkey = wr->sg_list[0].lkey;
3267 sqp_entry->sge.size = wr->sg_list[0].length;
3268 /* Store the wrid for reporting completion */
3269 sqp_entry->wrid = wqe->wr_id;
3270 /* change the wqe->wrid to table index */
3271 wqe->wr_id = rq_prod_index;
3272 }
3273 done:
3274 return rc;
3275 }
3276
is_ud_qp(struct bnxt_re_qp * qp)3277 static bool is_ud_qp(struct bnxt_re_qp *qp)
3278 {
3279 return (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_UD ||
3280 qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_GSI);
3281 }
3282
bnxt_re_build_send_wqe(struct bnxt_re_qp * qp,const struct ib_send_wr * wr,struct bnxt_qplib_swqe * wqe)3283 static int bnxt_re_build_send_wqe(struct bnxt_re_qp *qp,
3284 const struct ib_send_wr *wr,
3285 struct bnxt_qplib_swqe *wqe)
3286 {
3287 struct bnxt_re_ah *ah = NULL;
3288
3289 if(is_ud_qp(qp)) {
3290 ah = to_bnxt_re(ud_wr(wr)->ah, struct bnxt_re_ah, ibah);
3291 wqe->send.q_key = ud_wr(wr)->remote_qkey;
3292 wqe->send.dst_qp = ud_wr(wr)->remote_qpn;
3293 wqe->send.avid = ah->qplib_ah.id;
3294 }
3295 switch (wr->opcode) {
3296 case IB_WR_SEND:
3297 wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND;
3298 break;
3299 case IB_WR_SEND_WITH_IMM:
3300 wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM;
3301 wqe->send.imm_data = wr->ex.imm_data;
3302 break;
3303 case IB_WR_SEND_WITH_INV:
3304 wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV;
3305 wqe->send.inv_key = wr->ex.invalidate_rkey;
3306 break;
3307 default:
3308 dev_err(rdev_to_dev(qp->rdev), "%s Invalid opcode %d!\n",
3309 __func__, wr->opcode);
3310 return -EINVAL;
3311 }
3312 if (wr->send_flags & IB_SEND_SIGNALED)
3313 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
3314 if (wr->send_flags & IB_SEND_FENCE)
3315 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
3316 if (wr->send_flags & IB_SEND_SOLICITED)
3317 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT;
3318 if (wr->send_flags & IB_SEND_INLINE)
3319 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_INLINE;
3320
3321 return 0;
3322 }
3323
bnxt_re_build_rdma_wqe(const struct ib_send_wr * wr,struct bnxt_qplib_swqe * wqe)3324 static int bnxt_re_build_rdma_wqe(const struct ib_send_wr *wr,
3325 struct bnxt_qplib_swqe *wqe)
3326 {
3327 switch (wr->opcode) {
3328 case IB_WR_RDMA_WRITE:
3329 wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE;
3330 break;
3331 case IB_WR_RDMA_WRITE_WITH_IMM:
3332 wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM;
3333 wqe->rdma.imm_data = wr->ex.imm_data;
3334 break;
3335 case IB_WR_RDMA_READ:
3336 wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_READ;
3337 wqe->rdma.inv_key = wr->ex.invalidate_rkey;
3338 break;
3339 default:
3340 return -EINVAL;
3341 }
3342 wqe->rdma.remote_va = rdma_wr(wr)->remote_addr;
3343 wqe->rdma.r_key = rdma_wr(wr)->rkey;
3344 if (wr->send_flags & IB_SEND_SIGNALED)
3345 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
3346 if (wr->send_flags & IB_SEND_FENCE)
3347 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
3348 if (wr->send_flags & IB_SEND_SOLICITED)
3349 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT;
3350 if (wr->send_flags & IB_SEND_INLINE)
3351 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_INLINE;
3352
3353 return 0;
3354 }
3355
bnxt_re_build_atomic_wqe(const struct ib_send_wr * wr,struct bnxt_qplib_swqe * wqe)3356 static int bnxt_re_build_atomic_wqe(const struct ib_send_wr *wr,
3357 struct bnxt_qplib_swqe *wqe)
3358 {
3359 switch (wr->opcode) {
3360 case IB_WR_ATOMIC_CMP_AND_SWP:
3361 wqe->type = BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP;
3362 wqe->atomic.cmp_data = atomic_wr(wr)->compare_add;
3363 wqe->atomic.swap_data = atomic_wr(wr)->swap;
3364 break;
3365 case IB_WR_ATOMIC_FETCH_AND_ADD:
3366 wqe->type = BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD;
3367 wqe->atomic.cmp_data = atomic_wr(wr)->compare_add;
3368 break;
3369 default:
3370 return -EINVAL;
3371 }
3372 wqe->atomic.remote_va = atomic_wr(wr)->remote_addr;
3373 wqe->atomic.r_key = atomic_wr(wr)->rkey;
3374 if (wr->send_flags & IB_SEND_SIGNALED)
3375 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
3376 if (wr->send_flags & IB_SEND_FENCE)
3377 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
3378 if (wr->send_flags & IB_SEND_SOLICITED)
3379 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT;
3380 return 0;
3381 }
3382
bnxt_re_build_inv_wqe(const struct ib_send_wr * wr,struct bnxt_qplib_swqe * wqe)3383 static int bnxt_re_build_inv_wqe(const struct ib_send_wr *wr,
3384 struct bnxt_qplib_swqe *wqe)
3385 {
3386 wqe->type = BNXT_QPLIB_SWQE_TYPE_LOCAL_INV;
3387 wqe->local_inv.inv_l_key = wr->ex.invalidate_rkey;
3388 if (wr->send_flags & IB_SEND_SIGNALED)
3389 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
3390 if (wr->send_flags & IB_SEND_FENCE)
3391 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
3392 if (wr->send_flags & IB_SEND_SOLICITED)
3393 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT;
3394
3395 return 0;
3396 }
3397
bnxt_re_build_reg_wqe(const struct ib_reg_wr * wr,struct bnxt_qplib_swqe * wqe)3398 static int bnxt_re_build_reg_wqe(const struct ib_reg_wr *wr,
3399 struct bnxt_qplib_swqe *wqe)
3400 {
3401 struct bnxt_re_mr *mr = to_bnxt_re(wr->mr, struct bnxt_re_mr, ib_mr);
3402 struct bnxt_qplib_frpl *qplib_frpl = &mr->qplib_frpl;
3403 int reg_len, i, access = wr->access;
3404
3405 if (mr->npages > qplib_frpl->max_pg_ptrs) {
3406 dev_err_ratelimited(rdev_to_dev(mr->rdev),
3407 " %s: failed npages %d > %d\n", __func__,
3408 mr->npages, qplib_frpl->max_pg_ptrs);
3409 return -EINVAL;
3410 }
3411
3412 wqe->frmr.pbl_ptr = (__le64 *)qplib_frpl->hwq.pbl_ptr[0];
3413 wqe->frmr.pbl_dma_ptr = qplib_frpl->hwq.pbl_dma_ptr[0];
3414 wqe->frmr.levels = qplib_frpl->hwq.level;
3415 wqe->frmr.page_list = mr->pages;
3416 wqe->frmr.page_list_len = mr->npages;
3417 wqe->type = BNXT_QPLIB_SWQE_TYPE_REG_MR;
3418
3419 if (wr->wr.send_flags & IB_SEND_SIGNALED)
3420 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
3421 if (access & IB_ACCESS_LOCAL_WRITE)
3422 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_LOCAL_WRITE;
3423 if (access & IB_ACCESS_REMOTE_READ)
3424 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_READ;
3425 if (access & IB_ACCESS_REMOTE_WRITE)
3426 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_WRITE;
3427 if (access & IB_ACCESS_REMOTE_ATOMIC)
3428 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_ATOMIC;
3429 if (access & IB_ACCESS_MW_BIND)
3430 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_WINDOW_BIND;
3431
3432 /* TODO: OFED provides the rkey of the MR instead of the lkey */
3433 wqe->frmr.l_key = wr->key;
3434 wqe->frmr.length = wr->mr->length;
3435 wqe->frmr.pbl_pg_sz_log = ilog2(PAGE_SIZE >> PAGE_SHIFT_4K);
3436 wqe->frmr.pg_sz_log = ilog2(wr->mr->page_size >> PAGE_SHIFT_4K);
3437 wqe->frmr.va = wr->mr->iova;
3438 reg_len = wqe->frmr.page_list_len * wr->mr->page_size;
3439
3440 if (wqe->frmr.length > reg_len) {
3441 dev_err_ratelimited(rdev_to_dev(mr->rdev),
3442 "%s: bnxt_re_mr 0x%px len (%d > %d)\n",
3443 __func__, (void *)mr, wqe->frmr.length,
3444 reg_len);
3445
3446 for (i = 0; i < mr->npages; i++)
3447 dev_dbg(rdev_to_dev(mr->rdev),
3448 "%s: build_reg_wqe page[%d] = 0x%llx\n",
3449 __func__, i, mr->pages[i]);
3450
3451 return -EINVAL;
3452 }
3453
3454 return 0;
3455 }
3456
bnxt_re_set_sg_list(const struct ib_send_wr * wr,struct bnxt_qplib_swqe * wqe)3457 static void bnxt_re_set_sg_list(const struct ib_send_wr *wr,
3458 struct bnxt_qplib_swqe *wqe)
3459 {
3460 wqe->sg_list = (struct bnxt_qplib_sge *)wr->sg_list;
3461 wqe->num_sge = wr->num_sge;
3462 }
3463
bnxt_ud_qp_hw_stall_workaround(struct bnxt_re_qp * qp)3464 static void bnxt_ud_qp_hw_stall_workaround(struct bnxt_re_qp *qp)
3465 {
3466 if ((qp->ib_qp.qp_type == IB_QPT_UD || qp->ib_qp.qp_type == IB_QPT_GSI ||
3467 qp->ib_qp.qp_type == IB_QPT_RAW_ETHERTYPE) &&
3468 qp->qplib_qp.wqe_cnt == BNXT_RE_UD_QP_HW_STALL) {
3469 int qp_attr_mask;
3470 struct ib_qp_attr qp_attr;
3471
3472 qp_attr_mask = IB_QP_STATE;
3473 qp_attr.qp_state = IB_QPS_RTS;
3474 bnxt_re_modify_qp(&qp->ib_qp, &qp_attr, qp_attr_mask, NULL);
3475 qp->qplib_qp.wqe_cnt = 0;
3476 }
3477 }
3478
bnxt_re_post_send_shadow_qp(struct bnxt_re_dev * rdev,struct bnxt_re_qp * qp,const struct ib_send_wr * wr)3479 static int bnxt_re_post_send_shadow_qp(struct bnxt_re_dev *rdev,
3480 struct bnxt_re_qp *qp,
3481 const struct ib_send_wr *wr)
3482 {
3483 struct bnxt_qplib_swqe wqe;
3484 unsigned long flags;
3485 int rc = 0;
3486
3487 spin_lock_irqsave(&qp->sq_lock, flags);
3488 while (wr) {
3489 /* House keeping */
3490 memset(&wqe, 0, sizeof(wqe));
3491 /* Common */
3492 if (wr->num_sge > qp->qplib_qp.sq.max_sge) {
3493 dev_err(rdev_to_dev(rdev),
3494 "Limit exceeded for Send SGEs\n");
3495 rc = -EINVAL;
3496 break;
3497 }
3498
3499 bnxt_re_set_sg_list(wr, &wqe);
3500 wqe.wr_id = wr->wr_id;
3501 wqe.type = BNXT_QPLIB_SWQE_TYPE_SEND;
3502 rc = bnxt_re_build_send_wqe(qp, wr, &wqe);
3503 if (rc)
3504 break;
3505
3506 rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe);
3507 if (rc) {
3508 dev_err(rdev_to_dev(rdev),
3509 "bad_wr seen with opcode = 0x%x rc = %d\n",
3510 wr->opcode, rc);
3511 break;
3512 }
3513 wr = wr->next;
3514 }
3515 bnxt_qplib_post_send_db(&qp->qplib_qp);
3516 bnxt_ud_qp_hw_stall_workaround(qp);
3517 spin_unlock_irqrestore(&qp->sq_lock, flags);
3518 return rc;
3519 }
3520
bnxt_re_legacy_set_uc_fence(struct bnxt_qplib_swqe * wqe)3521 static void bnxt_re_legacy_set_uc_fence(struct bnxt_qplib_swqe *wqe)
3522 {
3523 /* Need unconditional fence for non-wire memory opcode
3524 * to work as expected.
3525 */
3526 if (wqe->type == BNXT_QPLIB_SWQE_TYPE_LOCAL_INV ||
3527 wqe->type == BNXT_QPLIB_SWQE_TYPE_FAST_REG_MR ||
3528 wqe->type == BNXT_QPLIB_SWQE_TYPE_REG_MR ||
3529 wqe->type == BNXT_QPLIB_SWQE_TYPE_BIND_MW)
3530 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
3531 }
3532
bnxt_re_post_send(struct ib_qp * ib_qp,const struct ib_send_wr * wr,const struct ib_send_wr ** bad_wr)3533 int bnxt_re_post_send(struct ib_qp *ib_qp, const struct ib_send_wr *wr,
3534 const struct ib_send_wr **bad_wr)
3535 {
3536 struct bnxt_re_qp *qp = to_bnxt_re(ib_qp, struct bnxt_re_qp, ib_qp);
3537 struct bnxt_qplib_sge sge[6];
3538 struct bnxt_qplib_swqe wqe;
3539 struct bnxt_re_dev *rdev;
3540 unsigned long flags;
3541 int rc = 0;
3542
3543 rdev = qp->rdev;
3544 spin_lock_irqsave(&qp->sq_lock, flags);
3545 while (wr) {
3546 /* House keeping */
3547 memset(&wqe, 0, sizeof(wqe));
3548 /* Common */
3549 if (wr->num_sge > qp->qplib_qp.sq.max_sge) {
3550 dev_err(rdev_to_dev(rdev),
3551 "Limit exceeded for Send SGEs\n");
3552 rc = -EINVAL;
3553 goto bad;
3554 }
3555
3556 bnxt_re_set_sg_list(wr, &wqe);
3557 wqe.wr_id = wr->wr_id;
3558
3559 switch (wr->opcode) {
3560 case IB_WR_SEND:
3561 case IB_WR_SEND_WITH_IMM:
3562 if (ib_qp->qp_type == IB_QPT_GSI &&
3563 rdev->gsi_ctx.gsi_qp_mode != BNXT_RE_GSI_MODE_UD) {
3564 memset(sge, 0, sizeof(sge));
3565 wqe.sg_list = sge;
3566 rc = bnxt_re_build_gsi_send(qp, wr, &wqe);
3567 if (rc)
3568 goto bad;
3569 } else if (ib_qp->qp_type == IB_QPT_RAW_ETHERTYPE) {
3570 bnxt_re_build_raw_send(wr, &wqe);
3571 }
3572 switch (wr->send_flags) {
3573 case IB_SEND_IP_CSUM:
3574 wqe.rawqp1.lflags |=
3575 SQ_SEND_RAWETH_QP1_LFLAGS_IP_CHKSUM;
3576 break;
3577 default:
3578 break;
3579 }
3580 fallthrough;
3581 case IB_WR_SEND_WITH_INV:
3582 rc = bnxt_re_build_send_wqe(qp, wr, &wqe);
3583 break;
3584 case IB_WR_RDMA_WRITE:
3585 case IB_WR_RDMA_WRITE_WITH_IMM:
3586 case IB_WR_RDMA_READ:
3587 rc = bnxt_re_build_rdma_wqe(wr, &wqe);
3588 break;
3589 case IB_WR_ATOMIC_CMP_AND_SWP:
3590 case IB_WR_ATOMIC_FETCH_AND_ADD:
3591 rc = bnxt_re_build_atomic_wqe(wr, &wqe);
3592 break;
3593 case IB_WR_RDMA_READ_WITH_INV:
3594 dev_err(rdev_to_dev(rdev),
3595 "RDMA Read with Invalidate is not supported\n");
3596 rc = -EINVAL;
3597 goto bad;
3598 case IB_WR_LOCAL_INV:
3599 rc = bnxt_re_build_inv_wqe(wr, &wqe);
3600 break;
3601 case IB_WR_REG_MR:
3602 rc = bnxt_re_build_reg_wqe(reg_wr(wr), &wqe);
3603 break;
3604 default:
3605 /* Unsupported WRs */
3606 dev_err(rdev_to_dev(rdev),
3607 "WR (0x%x) is not supported\n", wr->opcode);
3608 rc = -EINVAL;
3609 goto bad;
3610 }
3611
3612 if (likely(!rc)) {
3613 if (!_is_chip_gen_p5_p7(rdev->chip_ctx))
3614 bnxt_re_legacy_set_uc_fence(&wqe);
3615 rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe);
3616 }
3617 bad:
3618 if (unlikely(rc)) {
3619 dev_err(rdev_to_dev(rdev),
3620 "bad_wr seen with opcode = 0x%x\n", wr->opcode);
3621 *bad_wr = wr;
3622 break;
3623 }
3624 wr = wr->next;
3625 }
3626 bnxt_qplib_post_send_db(&qp->qplib_qp);
3627 if (!_is_chip_gen_p5_p7(rdev->chip_ctx))
3628 bnxt_ud_qp_hw_stall_workaround(qp);
3629 spin_unlock_irqrestore(&qp->sq_lock, flags);
3630
3631 return rc;
3632 }
3633
bnxt_re_post_recv_shadow_qp(struct bnxt_re_dev * rdev,struct bnxt_re_qp * qp,struct ib_recv_wr * wr)3634 static int bnxt_re_post_recv_shadow_qp(struct bnxt_re_dev *rdev,
3635 struct bnxt_re_qp *qp,
3636 struct ib_recv_wr *wr)
3637 {
3638 struct bnxt_qplib_swqe wqe;
3639 int rc = 0;
3640
3641 /* rq lock can be pardoned here. */
3642 while (wr) {
3643 /* House keeping */
3644 memset(&wqe, 0, sizeof(wqe));
3645 /* Common */
3646 if (wr->num_sge > qp->qplib_qp.rq.max_sge) {
3647 dev_err(rdev_to_dev(rdev),
3648 "Limit exceeded for Receive SGEs\n");
3649 rc = -EINVAL;
3650 goto bad;
3651 }
3652
3653 wqe.sg_list = (struct bnxt_qplib_sge *)wr->sg_list;
3654 wqe.num_sge = wr->num_sge;
3655 wqe.wr_id = wr->wr_id;
3656 wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV;
3657 rc = bnxt_qplib_post_recv(&qp->qplib_qp, &wqe);
3658 bad:
3659 if (rc) {
3660 dev_err(rdev_to_dev(rdev),
3661 "bad_wr seen with RQ post\n");
3662 break;
3663 }
3664 wr = wr->next;
3665 }
3666 bnxt_qplib_post_recv_db(&qp->qplib_qp);
3667 return rc;
3668 }
3669
bnxt_re_build_gsi_recv(struct bnxt_re_qp * qp,const struct ib_recv_wr * wr,struct bnxt_qplib_swqe * wqe)3670 static int bnxt_re_build_gsi_recv(struct bnxt_re_qp *qp,
3671 const struct ib_recv_wr *wr,
3672 struct bnxt_qplib_swqe *wqe)
3673 {
3674 struct bnxt_re_dev *rdev = qp->rdev;
3675 int rc = 0;
3676
3677 if (rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_ALL)
3678 rc = bnxt_re_build_qp1_shadow_qp_recv(qp, wr, wqe);
3679 else
3680 rc = bnxt_re_build_qp1_recv(qp, wr, wqe);
3681
3682 return rc;
3683 }
3684
bnxt_re_post_recv(struct ib_qp * ib_qp,const struct ib_recv_wr * wr,const struct ib_recv_wr ** bad_wr)3685 int bnxt_re_post_recv(struct ib_qp *ib_qp, const struct ib_recv_wr *wr,
3686 const struct ib_recv_wr **bad_wr)
3687 {
3688 struct bnxt_re_qp *qp = to_bnxt_re(ib_qp, struct bnxt_re_qp, ib_qp);
3689 struct bnxt_qplib_sge sge[6];
3690 struct bnxt_qplib_swqe wqe;
3691 unsigned long flags;
3692 u32 count = 0;
3693 int rc = 0;
3694
3695 spin_lock_irqsave(&qp->rq_lock, flags);
3696 while (wr) {
3697 memset(&wqe, 0, sizeof(wqe));
3698 if (wr->num_sge > qp->qplib_qp.rq.max_sge) {
3699 dev_err(rdev_to_dev(qp->rdev),
3700 "Limit exceeded for Receive SGEs\n");
3701 rc = -EINVAL;
3702 goto bad;
3703 }
3704 wqe.num_sge = wr->num_sge;
3705 wqe.sg_list = (struct bnxt_qplib_sge *)wr->sg_list;
3706 wqe.wr_id = wr->wr_id;
3707 wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV;
3708
3709 if (ib_qp->qp_type == IB_QPT_GSI &&
3710 qp->rdev->gsi_ctx.gsi_qp_mode != BNXT_RE_GSI_MODE_UD) {
3711 memset(sge, 0, sizeof(sge));
3712 wqe.sg_list = sge;
3713 rc = bnxt_re_build_gsi_recv(qp, wr, &wqe);
3714 if (rc)
3715 goto bad;
3716 }
3717 rc = bnxt_qplib_post_recv(&qp->qplib_qp, &wqe);
3718 bad:
3719 if (rc) {
3720 dev_err(rdev_to_dev(qp->rdev),
3721 "bad_wr seen with RQ post\n");
3722 *bad_wr = wr;
3723 break;
3724 }
3725 /* Ring DB if the RQEs posted reaches a threshold value */
3726 if (++count >= BNXT_RE_RQ_WQE_THRESHOLD) {
3727 bnxt_qplib_post_recv_db(&qp->qplib_qp);
3728 count = 0;
3729 }
3730 wr = wr->next;
3731 }
3732
3733 if (count)
3734 bnxt_qplib_post_recv_db(&qp->qplib_qp);
3735 spin_unlock_irqrestore(&qp->rq_lock, flags);
3736
3737 return rc;
3738 }
3739
3740 /* Completion Queues */
bnxt_re_destroy_cq(struct ib_cq * ib_cq,struct ib_udata * udata)3741 void bnxt_re_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata)
3742 {
3743 struct bnxt_re_cq *cq = to_bnxt_re(ib_cq, struct bnxt_re_cq, ibcq);
3744 struct bnxt_re_dev *rdev = cq->rdev;
3745 int rc = 0;
3746
3747 if (cq->uctx_cq_page) {
3748 BNXT_RE_CQ_PAGE_LIST_DEL(cq->uctx, cq);
3749 free_page((u64)cq->uctx_cq_page);
3750 cq->uctx_cq_page = NULL;
3751 }
3752 if (cq->is_dbr_soft_cq && cq->uctx) {
3753 void *dbr_page;
3754
3755 if (cq->uctx->dbr_recov_cq) {
3756 dbr_page = cq->uctx->dbr_recov_cq_page;
3757 cq->uctx->dbr_recov_cq_page = NULL;
3758 cq->uctx->dbr_recov_cq = NULL;
3759 free_page((unsigned long)dbr_page);
3760 }
3761 goto end;
3762 }
3763 /* CQ getting destroyed. Set this state for cqn handler */
3764 spin_lock_bh(&cq->qplib_cq.compl_lock);
3765 cq->qplib_cq.destroyed = true;
3766 spin_unlock_bh(&cq->qplib_cq.compl_lock);
3767 if (ib_cq->poll_ctx == IB_POLL_WORKQUEUE ||
3768 ib_cq->poll_ctx == IB_POLL_UNBOUND_WORKQUEUE)
3769 cancel_work_sync(&ib_cq->work);
3770
3771 rc = bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq);
3772 if (rc)
3773 dev_err_ratelimited(rdev_to_dev(rdev),
3774 "%s id = %d failed rc = %d\n",
3775 __func__, cq->qplib_cq.id, rc);
3776
3777 bnxt_re_put_nq(rdev, cq->qplib_cq.nq);
3778 if (cq->umem && !IS_ERR(cq->umem))
3779 ib_umem_release(cq->umem);
3780
3781 kfree(cq->cql);
3782 atomic_dec(&rdev->stats.rsors.cq_count);
3783 end:
3784 return;
3785 }
3786
3787 static inline struct
__get_cq_from_cq_in(struct ib_cq * cq_in,struct bnxt_re_dev * rdev)3788 bnxt_re_cq *__get_cq_from_cq_in(struct ib_cq *cq_in,
3789 struct bnxt_re_dev *rdev)
3790 {
3791 struct bnxt_re_cq *cq;
3792 cq = container_of(cq_in, struct bnxt_re_cq, ibcq);
3793 return cq;
3794 }
3795
bnxt_re_create_cq(struct ib_cq * cq_in,const struct ib_cq_init_attr * attr,struct ib_udata * udata)3796 int bnxt_re_create_cq(struct ib_cq *cq_in,
3797 const struct ib_cq_init_attr *attr,
3798 struct ib_udata *udata)
3799 {
3800 struct bnxt_qplib_dev_attr *dev_attr;
3801 struct bnxt_re_ucontext *uctx = NULL;
3802 struct ib_ucontext *context = NULL;
3803 struct bnxt_qplib_cq *qplcq;
3804 struct bnxt_re_cq_req ureq;
3805 struct bnxt_re_dev *rdev;
3806 int rc, entries;
3807 struct bnxt_re_cq *cq;
3808 u32 max_active_cqs;
3809 int cqe = attr->cqe;
3810
3811 if (attr->flags)
3812 return -EOPNOTSUPP;
3813
3814 rdev = rdev_from_cq_in(cq_in);
3815 if (rdev->mod_exit) {
3816 rc = -EIO;
3817 dev_dbg(rdev_to_dev(rdev), "%s(): in mod_exit, just return!\n", __func__);
3818 goto exit;
3819 }
3820 if (udata) {
3821 uctx = rdma_udata_to_drv_context(udata,
3822 struct bnxt_re_ucontext,
3823 ibucontext);
3824 context = &uctx->ibucontext;
3825 }
3826 dev_attr = rdev->dev_attr;
3827
3828 if (atomic_read(&rdev->stats.rsors.cq_count) >= dev_attr->max_cq) {
3829 dev_err(rdev_to_dev(rdev), "Create CQ failed - max exceeded(CQs)\n");
3830 rc = -EINVAL;
3831 goto exit;
3832 }
3833 /* Validate CQ fields */
3834 if (cqe < 1 || cqe > dev_attr->max_cq_wqes) {
3835 dev_err(rdev_to_dev(rdev), "Create CQ failed - max exceeded(CQ_WQs)\n");
3836 rc = -EINVAL;
3837 goto exit;
3838 }
3839
3840 cq = __get_cq_from_cq_in(cq_in, rdev);
3841 if (!cq) {
3842 rc = -ENOMEM;
3843 goto exit;
3844 }
3845 cq->rdev = rdev;
3846 cq->uctx = uctx;
3847 qplcq = &cq->qplib_cq;
3848 qplcq->cq_handle = (u64)qplcq;
3849 /*
3850 * Since CQ is for QP1 is shared with Shadow CQ, the size
3851 * should be double the size. There is no way to identify
3852 * whether this CQ is for GSI QP. So assuming that the first
3853 * CQ created is for QP1
3854 */
3855 if (!udata && !rdev->gsi_ctx.first_cq_created &&
3856 rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_ALL) {
3857 rdev->gsi_ctx.first_cq_created = true;
3858 /*
3859 * Total CQE required for the CQ = CQE for QP1 RQ +
3860 * CQE for Shadow QP SQEs + CQE for Shadow QP RQEs.
3861 * Max entries of shadow QP SQ and RQ = QP1 RQEs = cqe
3862 */
3863 cqe *= 3;
3864 }
3865
3866 entries = bnxt_re_init_depth(cqe + 1, uctx);
3867 if (entries > dev_attr->max_cq_wqes + 1)
3868 entries = dev_attr->max_cq_wqes + 1;
3869
3870 qplcq->sginfo.pgshft = PAGE_SHIFT;
3871 qplcq->sginfo.pgsize = PAGE_SIZE;
3872 if (udata) {
3873 if (udata->inlen < sizeof(ureq))
3874 dev_warn(rdev_to_dev(rdev),
3875 "Update the library ulen %d klen %d\n",
3876 (unsigned int)udata->inlen,
3877 (unsigned int)sizeof(ureq));
3878
3879 rc = ib_copy_from_udata(&ureq, udata,
3880 min(udata->inlen, sizeof(ureq)));
3881 if (rc)
3882 goto fail;
3883
3884 if (BNXT_RE_IS_DBR_PACING_NOTIFY_CQ(ureq)) {
3885 cq->is_dbr_soft_cq = true;
3886 goto success;
3887 }
3888
3889 if (BNXT_RE_IS_DBR_RECOV_CQ(ureq)) {
3890 void *dbr_page;
3891 u32 *epoch;
3892
3893 dbr_page = (void *)__get_free_page(GFP_KERNEL);
3894 if (!dbr_page) {
3895 dev_err(rdev_to_dev(rdev),
3896 "DBR recov CQ page allocation failed!");
3897 rc = -ENOMEM;
3898 goto fail;
3899 }
3900
3901 /* memset the epoch and epoch_ack to 0 */
3902 epoch = dbr_page;
3903 epoch[0] = 0x0;
3904 epoch[1] = 0x0;
3905
3906 uctx->dbr_recov_cq = cq;
3907 uctx->dbr_recov_cq_page = dbr_page;
3908
3909 cq->is_dbr_soft_cq = true;
3910 goto success;
3911 }
3912
3913 cq->umem = ib_umem_get_compat
3914 (rdev, context, udata, ureq.cq_va,
3915 entries * sizeof(struct cq_base),
3916 IB_ACCESS_LOCAL_WRITE, 1);
3917 if (IS_ERR(cq->umem)) {
3918 rc = PTR_ERR(cq->umem);
3919 dev_err(rdev_to_dev(rdev),
3920 "%s: ib_umem_get failed! rc = %d\n",
3921 __func__, rc);
3922 goto fail;
3923 }
3924 qplcq->sginfo.sghead = get_ib_umem_sgl(cq->umem,
3925 &qplcq->sginfo.nmap);
3926 qplcq->sginfo.npages = ib_umem_num_pages_compat(cq->umem);
3927 if (!uctx->dpi.dbr) {
3928 rc = bnxt_re_get_user_dpi(rdev, uctx);
3929 if (rc)
3930 goto c2fail;
3931 }
3932 qplcq->dpi = &uctx->dpi;
3933 } else {
3934 cq->max_cql = entries > MAX_CQL_PER_POLL ? MAX_CQL_PER_POLL : entries;
3935 cq->cql = kcalloc(cq->max_cql, sizeof(struct bnxt_qplib_cqe),
3936 GFP_KERNEL);
3937 if (!cq->cql) {
3938 dev_err(rdev_to_dev(rdev),
3939 "Allocate CQL for %d failed!\n", cq->max_cql);
3940 rc = -ENOMEM;
3941 goto fail;
3942 }
3943 qplcq->dpi = &rdev->dpi_privileged;
3944 }
3945 /*
3946 * Allocating the NQ in a round robin fashion. nq_alloc_cnt is a
3947 * used for getting the NQ index.
3948 */
3949 qplcq->max_wqe = entries;
3950 qplcq->nq = bnxt_re_get_nq(rdev);
3951 qplcq->cnq_hw_ring_id = qplcq->nq->ring_id;
3952
3953 rc = bnxt_qplib_create_cq(&rdev->qplib_res, qplcq);
3954 if (rc) {
3955 dev_err(rdev_to_dev(rdev), "Create HW CQ failed!\n");
3956 goto fail;
3957 }
3958
3959 INIT_LIST_HEAD(&cq->cq_list);
3960 cq->ibcq.cqe = entries;
3961 cq->cq_period = qplcq->period;
3962
3963 atomic_inc(&rdev->stats.rsors.cq_count);
3964 max_active_cqs = atomic_read(&rdev->stats.rsors.cq_count);
3965 if (max_active_cqs > atomic_read(&rdev->stats.rsors.max_cq_count))
3966 atomic_set(&rdev->stats.rsors.max_cq_count, max_active_cqs);
3967 spin_lock_init(&cq->cq_lock);
3968
3969 if (udata) {
3970 struct bnxt_re_cq_resp resp;
3971
3972 resp.cqid = qplcq->id;
3973 resp.tail = qplcq->hwq.cons;
3974 resp.phase = qplcq->period;
3975 resp.comp_mask = 0;
3976 resp.dbr = (u64)uctx->dpi.umdbr;
3977 resp.dpi = uctx->dpi.dpi;
3978 resp.comp_mask |= BNXT_RE_COMP_MASK_CQ_HAS_DB_INFO;
3979 /* Copy only on a valid wcpdi */
3980 if (uctx->wcdpi.dpi) {
3981 resp.wcdpi = uctx->wcdpi.dpi;
3982 resp.comp_mask |= BNXT_RE_COMP_MASK_CQ_HAS_WC_DPI;
3983 }
3984
3985 if (_is_chip_p7(rdev->chip_ctx)) {
3986 cq->uctx_cq_page = (void *)__get_free_page(GFP_KERNEL);
3987
3988 if (!cq->uctx_cq_page) {
3989 dev_err(rdev_to_dev(rdev),
3990 "CQ page allocation failed!\n");
3991 bnxt_qplib_destroy_cq(&rdev->qplib_res, qplcq);
3992 rc = -ENOMEM;
3993 goto c2fail;
3994 }
3995
3996 resp.uctx_cq_page = (u64)cq->uctx_cq_page;
3997 resp.comp_mask |= BNXT_RE_COMP_MASK_CQ_HAS_CQ_PAGE;
3998 }
3999
4000 rc = bnxt_re_copy_to_udata(rdev, &resp,
4001 min(udata->outlen, sizeof(resp)),
4002 udata);
4003 if (rc) {
4004 free_page((u64)cq->uctx_cq_page);
4005 cq->uctx_cq_page = NULL;
4006 bnxt_qplib_destroy_cq(&rdev->qplib_res, qplcq);
4007 goto c2fail;
4008 }
4009
4010 if (cq->uctx_cq_page)
4011 BNXT_RE_CQ_PAGE_LIST_ADD(uctx, cq);
4012 }
4013
4014 success:
4015 return 0;
4016 c2fail:
4017 if (udata && cq->umem && !IS_ERR(cq->umem))
4018 ib_umem_release(cq->umem);
4019 fail:
4020 if (cq) {
4021 if (cq->cql)
4022 kfree(cq->cql);
4023 }
4024 exit:
4025 return rc;
4026 }
4027
bnxt_re_modify_cq(struct ib_cq * ib_cq,u16 cq_count,u16 cq_period)4028 int bnxt_re_modify_cq(struct ib_cq *ib_cq, u16 cq_count, u16 cq_period)
4029 {
4030 struct bnxt_re_cq *cq = to_bnxt_re(ib_cq, struct bnxt_re_cq, ibcq);
4031 struct bnxt_re_dev *rdev = cq->rdev;
4032 int rc;
4033
4034 if ((cq->cq_count != cq_count) || (cq->cq_period != cq_period)) {
4035 cq->qplib_cq.count = cq_count;
4036 cq->qplib_cq.period = cq_period;
4037 rc = bnxt_qplib_modify_cq(&rdev->qplib_res, &cq->qplib_cq);
4038 if (rc) {
4039 dev_err(rdev_to_dev(rdev), "Modify HW CQ %#x failed!\n",
4040 cq->qplib_cq.id);
4041 return rc;
4042 }
4043 /* On success, update the shadow */
4044 cq->cq_count = cq_count;
4045 cq->cq_period = cq_period;
4046 }
4047 return 0;
4048 }
4049
bnxt_re_resize_cq_complete(struct bnxt_re_cq * cq)4050 static void bnxt_re_resize_cq_complete(struct bnxt_re_cq *cq)
4051 {
4052 struct bnxt_re_dev *rdev = cq->rdev;
4053
4054 bnxt_qplib_resize_cq_complete(&rdev->qplib_res, &cq->qplib_cq);
4055
4056 cq->qplib_cq.max_wqe = cq->resize_cqe;
4057 if (cq->resize_umem) {
4058 ib_umem_release(cq->umem);
4059 cq->umem = cq->resize_umem;
4060 cq->resize_umem = NULL;
4061 cq->resize_cqe = 0;
4062 }
4063 }
4064
bnxt_re_resize_cq(struct ib_cq * ib_cq,int cqe,struct ib_udata * udata)4065 int bnxt_re_resize_cq(struct ib_cq *ib_cq, int cqe, struct ib_udata *udata)
4066 {
4067 struct bnxt_qplib_sg_info sginfo = {};
4068 struct bnxt_qplib_dpi *orig_dpi = NULL;
4069 struct bnxt_qplib_dev_attr *dev_attr;
4070 struct bnxt_re_ucontext *uctx = NULL;
4071 struct bnxt_re_resize_cq_req ureq;
4072 struct ib_ucontext *context = NULL;
4073 struct bnxt_re_dev *rdev;
4074 struct bnxt_re_cq *cq;
4075 int rc, entries;
4076
4077 /* Don't allow more than one resize request at the same time.
4078 * TODO: need a mutex here when we support kernel consumers of resize.
4079 */
4080 cq = to_bnxt_re(ib_cq, struct bnxt_re_cq, ibcq);
4081 rdev = cq->rdev;
4082 dev_attr = rdev->dev_attr;
4083 if (ib_cq->uobject) {
4084 uctx = rdma_udata_to_drv_context(udata,
4085 struct bnxt_re_ucontext,
4086 ibucontext);
4087 context = &uctx->ibucontext;
4088 }
4089
4090 if (cq->resize_umem) {
4091 dev_err(rdev_to_dev(rdev), "Resize CQ %#x failed - Busy\n",
4092 cq->qplib_cq.id);
4093 return -EBUSY;
4094 }
4095
4096 /* Check the requested cq depth out of supported depth */
4097 if (cqe < 1 || cqe > dev_attr->max_cq_wqes) {
4098 dev_err(rdev_to_dev(rdev), "Resize CQ %#x failed - max exceeded\n",
4099 cq->qplib_cq.id);
4100 return -EINVAL;
4101 }
4102
4103 entries = bnxt_re_init_depth(cqe + 1, uctx);
4104 entries = min_t(u32, (u32)entries, dev_attr->max_cq_wqes + 1);
4105
4106 /* Check to see if the new requested size can be handled by already
4107 * existing CQ
4108 */
4109 if (entries == cq->ibcq.cqe) {
4110 dev_info(rdev_to_dev(rdev), "CQ is already at size %d\n", cqe);
4111 return 0;
4112 }
4113
4114 if (ib_cq->uobject && udata) {
4115 if (udata->inlen < sizeof(ureq))
4116 dev_warn(rdev_to_dev(rdev),
4117 "Update the library ulen %d klen %d\n",
4118 (unsigned int)udata->inlen,
4119 (unsigned int)sizeof(ureq));
4120
4121 rc = ib_copy_from_udata(&ureq, udata,
4122 min(udata->inlen, sizeof(ureq)));
4123 if (rc)
4124 goto fail;
4125
4126 dev_dbg(rdev_to_dev(rdev), "%s: va %p\n", __func__,
4127 (void *)ureq.cq_va);
4128 cq->resize_umem = ib_umem_get_compat
4129 (rdev,
4130 context, udata, ureq.cq_va,
4131 entries * sizeof(struct cq_base),
4132 IB_ACCESS_LOCAL_WRITE, 1);
4133 if (IS_ERR(cq->resize_umem)) {
4134 rc = PTR_ERR(cq->resize_umem);
4135 cq->resize_umem = NULL;
4136 dev_err(rdev_to_dev(rdev), "%s: ib_umem_get failed! rc = %d\n",
4137 __func__, rc);
4138 goto fail;
4139 }
4140 cq->resize_cqe = entries;
4141 dev_dbg(rdev_to_dev(rdev), "%s: ib_umem_get() success\n",
4142 __func__);
4143 memcpy(&sginfo, &cq->qplib_cq.sginfo, sizeof(sginfo));
4144 orig_dpi = cq->qplib_cq.dpi;
4145
4146 cq->qplib_cq.sginfo.sghead = get_ib_umem_sgl(cq->resize_umem,
4147 &cq->qplib_cq.sginfo.nmap);
4148 cq->qplib_cq.sginfo.npages =
4149 ib_umem_num_pages_compat(cq->resize_umem);
4150 cq->qplib_cq.sginfo.pgsize = PAGE_SIZE;
4151 cq->qplib_cq.sginfo.pgshft = PAGE_SHIFT;
4152 cq->qplib_cq.dpi = &uctx->dpi;
4153 } else {
4154 /* TODO: kernel consumer */
4155 }
4156
4157 rc = bnxt_qplib_resize_cq(&rdev->qplib_res, &cq->qplib_cq, entries);
4158 if (rc) {
4159 dev_err(rdev_to_dev(rdev), "Resize HW CQ %#x failed!\n",
4160 cq->qplib_cq.id);
4161 goto fail;
4162 }
4163
4164 cq->ibcq.cqe = cq->resize_cqe;
4165 /* For kernel consumers complete resize here. For uverbs consumers,
4166 * we complete it in the context of ibv_poll_cq().
4167 */
4168 if (!cq->resize_umem)
4169 bnxt_qplib_resize_cq_complete(&rdev->qplib_res, &cq->qplib_cq);
4170
4171 atomic_inc(&rdev->stats.rsors.resize_count);
4172 return 0;
4173
4174 fail:
4175 if (cq->resize_umem) {
4176 ib_umem_release(cq->resize_umem);
4177 cq->resize_umem = NULL;
4178 cq->resize_cqe = 0;
4179 memcpy(&cq->qplib_cq.sginfo, &sginfo, sizeof(sginfo));
4180 cq->qplib_cq.dpi = orig_dpi;
4181 }
4182 return rc;
4183 }
4184
__req_to_ib_wc_status(u8 qstatus)4185 static enum ib_wc_status __req_to_ib_wc_status(u8 qstatus)
4186 {
4187 switch(qstatus) {
4188 case CQ_REQ_STATUS_OK:
4189 return IB_WC_SUCCESS;
4190 case CQ_REQ_STATUS_BAD_RESPONSE_ERR:
4191 return IB_WC_BAD_RESP_ERR;
4192 case CQ_REQ_STATUS_LOCAL_LENGTH_ERR:
4193 return IB_WC_LOC_LEN_ERR;
4194 case CQ_REQ_STATUS_LOCAL_QP_OPERATION_ERR:
4195 return IB_WC_LOC_QP_OP_ERR;
4196 case CQ_REQ_STATUS_LOCAL_PROTECTION_ERR:
4197 return IB_WC_LOC_PROT_ERR;
4198 case CQ_REQ_STATUS_MEMORY_MGT_OPERATION_ERR:
4199 return IB_WC_GENERAL_ERR;
4200 case CQ_REQ_STATUS_REMOTE_INVALID_REQUEST_ERR:
4201 return IB_WC_REM_INV_REQ_ERR;
4202 case CQ_REQ_STATUS_REMOTE_ACCESS_ERR:
4203 return IB_WC_REM_ACCESS_ERR;
4204 case CQ_REQ_STATUS_REMOTE_OPERATION_ERR:
4205 return IB_WC_REM_OP_ERR;
4206 case CQ_REQ_STATUS_RNR_NAK_RETRY_CNT_ERR:
4207 return IB_WC_RNR_RETRY_EXC_ERR;
4208 case CQ_REQ_STATUS_TRANSPORT_RETRY_CNT_ERR:
4209 return IB_WC_RETRY_EXC_ERR;
4210 case CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR:
4211 return IB_WC_WR_FLUSH_ERR;
4212 default:
4213 return IB_WC_GENERAL_ERR;
4214 }
4215 return 0;
4216 }
4217
__rawqp1_to_ib_wc_status(u8 qstatus)4218 static enum ib_wc_status __rawqp1_to_ib_wc_status(u8 qstatus)
4219 {
4220 switch(qstatus) {
4221 case CQ_RES_RAWETH_QP1_STATUS_OK:
4222 return IB_WC_SUCCESS;
4223 case CQ_RES_RAWETH_QP1_STATUS_LOCAL_ACCESS_ERROR:
4224 return IB_WC_LOC_ACCESS_ERR;
4225 case CQ_RES_RAWETH_QP1_STATUS_HW_LOCAL_LENGTH_ERR:
4226 return IB_WC_LOC_LEN_ERR;
4227 case CQ_RES_RAWETH_QP1_STATUS_LOCAL_PROTECTION_ERR:
4228 return IB_WC_LOC_PROT_ERR;
4229 case CQ_RES_RAWETH_QP1_STATUS_LOCAL_QP_OPERATION_ERR:
4230 return IB_WC_LOC_QP_OP_ERR;
4231 case CQ_RES_RAWETH_QP1_STATUS_MEMORY_MGT_OPERATION_ERR:
4232 return IB_WC_GENERAL_ERR;
4233 case CQ_RES_RAWETH_QP1_STATUS_WORK_REQUEST_FLUSHED_ERR:
4234 return IB_WC_WR_FLUSH_ERR;
4235 case CQ_RES_RAWETH_QP1_STATUS_HW_FLUSH_ERR:
4236 return IB_WC_WR_FLUSH_ERR;
4237 default:
4238 return IB_WC_GENERAL_ERR;
4239 }
4240 }
4241
__rc_to_ib_wc_status(u8 qstatus)4242 static enum ib_wc_status __rc_to_ib_wc_status(u8 qstatus)
4243 {
4244 switch(qstatus) {
4245 case CQ_RES_RC_STATUS_OK:
4246 return IB_WC_SUCCESS;
4247 case CQ_RES_RC_STATUS_LOCAL_ACCESS_ERROR:
4248 return IB_WC_LOC_ACCESS_ERR;
4249 case CQ_RES_RC_STATUS_LOCAL_LENGTH_ERR:
4250 return IB_WC_LOC_LEN_ERR;
4251 case CQ_RES_RC_STATUS_LOCAL_PROTECTION_ERR:
4252 return IB_WC_LOC_PROT_ERR;
4253 case CQ_RES_RC_STATUS_LOCAL_QP_OPERATION_ERR:
4254 return IB_WC_LOC_QP_OP_ERR;
4255 case CQ_RES_RC_STATUS_MEMORY_MGT_OPERATION_ERR:
4256 return IB_WC_GENERAL_ERR;
4257 case CQ_RES_RC_STATUS_REMOTE_INVALID_REQUEST_ERR:
4258 return IB_WC_REM_INV_REQ_ERR;
4259 case CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR:
4260 return IB_WC_WR_FLUSH_ERR;
4261 case CQ_RES_RC_STATUS_HW_FLUSH_ERR:
4262 return IB_WC_WR_FLUSH_ERR;
4263 default:
4264 return IB_WC_GENERAL_ERR;
4265 }
4266 }
4267
bnxt_re_process_req_wc(struct ib_wc * wc,struct bnxt_qplib_cqe * cqe)4268 static void bnxt_re_process_req_wc(struct ib_wc *wc, struct bnxt_qplib_cqe *cqe)
4269 {
4270 switch (cqe->type) {
4271 case BNXT_QPLIB_SWQE_TYPE_SEND:
4272 wc->opcode = IB_WC_SEND;
4273 break;
4274 case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM:
4275 wc->opcode = IB_WC_SEND;
4276 wc->wc_flags |= IB_WC_WITH_IMM;
4277 break;
4278 case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV:
4279 wc->opcode = IB_WC_SEND;
4280 wc->wc_flags |= IB_WC_WITH_INVALIDATE;
4281 break;
4282 case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE:
4283 wc->opcode = IB_WC_RDMA_WRITE;
4284 break;
4285 case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM:
4286 wc->opcode = IB_WC_RDMA_WRITE;
4287 wc->wc_flags |= IB_WC_WITH_IMM;
4288 break;
4289 case BNXT_QPLIB_SWQE_TYPE_RDMA_READ:
4290 wc->opcode = IB_WC_RDMA_READ;
4291 break;
4292 case BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP:
4293 wc->opcode = IB_WC_COMP_SWAP;
4294 break;
4295 case BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD:
4296 wc->opcode = IB_WC_FETCH_ADD;
4297 break;
4298 case BNXT_QPLIB_SWQE_TYPE_LOCAL_INV:
4299 wc->opcode = IB_WC_LOCAL_INV;
4300 break;
4301 case BNXT_QPLIB_SWQE_TYPE_REG_MR:
4302 wc->opcode = IB_WC_REG_MR;
4303 break;
4304 default:
4305 wc->opcode = IB_WC_SEND;
4306 break;
4307 }
4308
4309 wc->status = __req_to_ib_wc_status(cqe->status);
4310 }
4311
bnxt_re_check_packet_type(u16 raweth_qp1_flags,u16 raweth_qp1_flags2)4312 static int bnxt_re_check_packet_type(u16 raweth_qp1_flags, u16 raweth_qp1_flags2)
4313 {
4314 bool is_ipv6 = false, is_ipv4 = false;
4315
4316 /* raweth_qp1_flags Bit 9-6 indicates itype */
4317
4318 if ((raweth_qp1_flags & CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE)
4319 != CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE)
4320 return -1;
4321
4322 if (raweth_qp1_flags2 &
4323 CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_CS_CALC &&
4324 raweth_qp1_flags2 &
4325 CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_L4_CS_CALC) {
4326 /* raweth_qp1_flags2 Bit 8 indicates ip_type. 0-v4 1 - v6 */
4327 (raweth_qp1_flags2 &
4328 CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_TYPE) ?
4329 (is_ipv6 = true) : (is_ipv4 = true);
4330 return ((is_ipv6) ?
4331 BNXT_RE_ROCEV2_IPV6_PACKET :
4332 BNXT_RE_ROCEV2_IPV4_PACKET);
4333 } else {
4334 return BNXT_RE_ROCE_V1_PACKET;
4335 }
4336 }
4337
bnxt_re_is_loopback_packet(struct bnxt_re_dev * rdev,void * rq_hdr_buf)4338 static bool bnxt_re_is_loopback_packet(struct bnxt_re_dev *rdev,
4339 void *rq_hdr_buf)
4340 {
4341 u8 *tmp_buf = NULL;
4342 struct ethhdr *eth_hdr;
4343 u16 eth_type;
4344 bool rc = false;
4345
4346 tmp_buf = (u8 *)rq_hdr_buf;
4347 /*
4348 * If dest mac is not same as I/F mac, this could be a
4349 * loopback address or multicast address, check whether
4350 * it is a loopback packet
4351 */
4352 if (!ether_addr_equal(tmp_buf, rdev->dev_addr)) {
4353 tmp_buf += 4;
4354 /* Check the ether type */
4355 eth_hdr = (struct ethhdr *)tmp_buf;
4356 eth_type = ntohs(eth_hdr->h_proto);
4357 switch (eth_type) {
4358 case BNXT_QPLIB_ETHTYPE_ROCEV1:
4359 rc = true;
4360 break;
4361 default:
4362 break;
4363 }
4364 }
4365
4366 return rc;
4367 }
4368
bnxt_re_is_vlan_in_packet(struct bnxt_re_dev * rdev,void * rq_hdr_buf,struct bnxt_qplib_cqe * cqe)4369 static bool bnxt_re_is_vlan_in_packet(struct bnxt_re_dev *rdev,
4370 void *rq_hdr_buf,
4371 struct bnxt_qplib_cqe *cqe)
4372 {
4373 struct vlan_hdr *vlan_hdr;
4374 struct ethhdr *eth_hdr;
4375 u8 *tmp_buf = NULL;
4376 u16 eth_type;
4377
4378 tmp_buf = (u8 *)rq_hdr_buf;
4379 /* Check the ether type */
4380 eth_hdr = (struct ethhdr *)tmp_buf;
4381 eth_type = ntohs(eth_hdr->h_proto);
4382 if (eth_type == ETH_P_8021Q) {
4383 tmp_buf += sizeof(struct ethhdr);
4384 vlan_hdr = (struct vlan_hdr *)tmp_buf;
4385 cqe->raweth_qp1_metadata =
4386 ntohs(vlan_hdr->h_vlan_TCI) |
4387 (eth_type <<
4388 CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_SFT);
4389 cqe->raweth_qp1_flags2 |=
4390 CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_VLAN;
4391 return true;
4392 }
4393
4394 return false;
4395 }
4396
bnxt_re_process_raw_qp_packet_receive(struct bnxt_re_qp * gsi_qp,struct bnxt_qplib_cqe * cqe)4397 static int bnxt_re_process_raw_qp_packet_receive(struct bnxt_re_qp *gsi_qp,
4398 struct bnxt_qplib_cqe *cqe)
4399 {
4400 struct bnxt_re_sqp_entries *sqp_entry = NULL;
4401 struct bnxt_qplib_hdrbuf *hdr_buf;
4402 dma_addr_t shrq_hdr_buf_map;
4403 struct ib_sge s_sge[2] = {};
4404 struct ib_sge r_sge[2] = {};
4405 struct ib_recv_wr rwr = {};
4406 struct bnxt_re_ah *gsi_sah;
4407 struct bnxt_re_qp *gsi_sqp;
4408 dma_addr_t rq_hdr_buf_map;
4409 struct bnxt_re_dev *rdev;
4410 struct ib_send_wr *swr;
4411 u32 skip_bytes = 0;
4412 void *rq_hdr_buf;
4413 int pkt_type = 0;
4414 u32 offset = 0;
4415 u32 tbl_idx;
4416 int rc;
4417 struct ib_ud_wr udwr = {};
4418
4419 swr = &udwr.wr;
4420 rdev = gsi_qp->rdev;
4421 gsi_sqp = rdev->gsi_ctx.gsi_sqp;
4422 tbl_idx = cqe->wr_id;
4423
4424 hdr_buf = gsi_qp->qplib_qp.rq_hdr_buf;
4425 rq_hdr_buf = (u8 *) hdr_buf->va + tbl_idx * hdr_buf->step;
4426 rq_hdr_buf_map = bnxt_qplib_get_qp_buf_from_index(&gsi_qp->qplib_qp,
4427 tbl_idx);
4428 /* Shadow QP header buffer */
4429 shrq_hdr_buf_map = bnxt_qplib_get_qp_buf_from_index(&gsi_sqp->qplib_qp,
4430 tbl_idx);
4431 sqp_entry = &rdev->gsi_ctx.sqp_tbl[tbl_idx];
4432
4433 /* Find packet type from the cqe */
4434 pkt_type = bnxt_re_check_packet_type(cqe->raweth_qp1_flags,
4435 cqe->raweth_qp1_flags2);
4436 if (pkt_type < 0) {
4437 dev_err(rdev_to_dev(rdev), "Not handling this packet\n");
4438 return -EINVAL;
4439 }
4440
4441 /* Adjust the offset for the user buffer and post in the rq */
4442
4443 if (pkt_type == BNXT_RE_ROCEV2_IPV4_PACKET)
4444 offset = 20;
4445
4446 /*
4447 * QP1 loopback packet has 4 bytes of internal header before
4448 * ether header. Skip these four bytes.
4449 */
4450 if (bnxt_re_is_loopback_packet(rdev, rq_hdr_buf))
4451 skip_bytes = 4;
4452
4453 if (bnxt_re_is_vlan_in_packet(rdev, rq_hdr_buf, cqe))
4454 skip_bytes += VLAN_HLEN;
4455
4456 /* Store this cqe */
4457 memcpy(&sqp_entry->cqe, cqe, sizeof(struct bnxt_qplib_cqe));
4458 sqp_entry->qp1_qp = gsi_qp;
4459
4460 /* First send SGE . Skip the ether header*/
4461 s_sge[0].addr = rq_hdr_buf_map + BNXT_QPLIB_MAX_QP1_RQ_ETH_HDR_SIZE
4462 + skip_bytes;
4463 s_sge[0].lkey = 0xFFFFFFFF;
4464 s_sge[0].length = offset ? BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV4 :
4465 BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6;
4466
4467 /* Second Send SGE */
4468 s_sge[1].addr = s_sge[0].addr + s_sge[0].length +
4469 BNXT_QPLIB_MAX_QP1_RQ_BDETH_HDR_SIZE;
4470 if (pkt_type != BNXT_RE_ROCE_V1_PACKET)
4471 s_sge[1].addr += 8;
4472 s_sge[1].lkey = 0xFFFFFFFF;
4473 s_sge[1].length = 256;
4474
4475 /* First recv SGE */
4476 r_sge[0].addr = shrq_hdr_buf_map;
4477 r_sge[0].lkey = 0xFFFFFFFF;
4478 r_sge[0].length = 40;
4479
4480 r_sge[1].addr = sqp_entry->sge.addr + offset;
4481 r_sge[1].lkey = sqp_entry->sge.lkey;
4482 r_sge[1].length = BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6 + 256 - offset;
4483
4484 /* Create receive work request */
4485 rwr.num_sge = 2;
4486 rwr.sg_list = r_sge;
4487 rwr.wr_id = tbl_idx;
4488 rwr.next = NULL;
4489
4490 rc = bnxt_re_post_recv_shadow_qp(rdev, gsi_sqp, &rwr);
4491 if (rc) {
4492 dev_err(rdev_to_dev(rdev),
4493 "Failed to post Rx buffers to shadow QP\n");
4494 return -ENOMEM;
4495 }
4496
4497 swr->num_sge = 2;
4498 swr->sg_list = s_sge;
4499 swr->wr_id = tbl_idx;
4500 swr->opcode = IB_WR_SEND;
4501 swr->next = NULL;
4502
4503 gsi_sah = rdev->gsi_ctx.gsi_sah;
4504 udwr.ah = &gsi_sah->ibah;
4505 udwr.remote_qpn = gsi_sqp->qplib_qp.id;
4506 udwr.remote_qkey = gsi_sqp->qplib_qp.qkey;
4507 /* post data received in the send queue */
4508 rc = bnxt_re_post_send_shadow_qp(rdev, gsi_sqp, swr);
4509
4510 return rc;
4511 }
4512
bnxt_re_process_res_rawqp1_wc(struct ib_wc * wc,struct bnxt_qplib_cqe * cqe)4513 static void bnxt_re_process_res_rawqp1_wc(struct ib_wc *wc,
4514 struct bnxt_qplib_cqe *cqe)
4515 {
4516 wc->opcode = IB_WC_RECV;
4517 wc->status = __rawqp1_to_ib_wc_status(cqe->status);
4518 wc->wc_flags |= IB_WC_GRH;
4519 }
4520
bnxt_re_process_res_rc_wc(struct ib_wc * wc,struct bnxt_qplib_cqe * cqe)4521 static void bnxt_re_process_res_rc_wc(struct ib_wc *wc,
4522 struct bnxt_qplib_cqe *cqe)
4523 {
4524 wc->opcode = IB_WC_RECV;
4525 wc->status = __rc_to_ib_wc_status(cqe->status);
4526
4527 if (cqe->flags & CQ_RES_RC_FLAGS_IMM)
4528 wc->wc_flags |= IB_WC_WITH_IMM;
4529 if (cqe->flags & CQ_RES_RC_FLAGS_INV)
4530 wc->wc_flags |= IB_WC_WITH_INVALIDATE;
4531 if ((cqe->flags & (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM)) ==
4532 (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM))
4533 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
4534 }
4535
4536 /* Returns TRUE if pkt has valid VLAN and if VLAN id is non-zero */
bnxt_re_is_nonzero_vlanid_pkt(struct bnxt_qplib_cqe * orig_cqe,u16 * vid,u8 * sl)4537 static bool bnxt_re_is_nonzero_vlanid_pkt(struct bnxt_qplib_cqe *orig_cqe,
4538 u16 *vid, u8 *sl)
4539 {
4540 u32 metadata;
4541 u16 tpid;
4542 bool ret = false;
4543 metadata = orig_cqe->raweth_qp1_metadata;
4544 if (orig_cqe->raweth_qp1_flags2 &
4545 CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_VLAN) {
4546 tpid = ((metadata &
4547 CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_MASK) >>
4548 CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_SFT);
4549 if (tpid == ETH_P_8021Q) {
4550 *vid = metadata &
4551 CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_VID_MASK;
4552 *sl = (metadata &
4553 CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_PRI_MASK) >>
4554 CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_PRI_SFT;
4555 ret = !!(*vid);
4556 }
4557 }
4558
4559 return ret;
4560 }
4561
bnxt_re_process_res_shadow_qp_wc(struct bnxt_re_qp * gsi_sqp,struct ib_wc * wc,struct bnxt_qplib_cqe * cqe)4562 static void bnxt_re_process_res_shadow_qp_wc(struct bnxt_re_qp *gsi_sqp,
4563 struct ib_wc *wc,
4564 struct bnxt_qplib_cqe *cqe)
4565 {
4566 u32 tbl_idx;
4567 struct bnxt_re_dev *rdev = gsi_sqp->rdev;
4568 struct bnxt_re_qp *gsi_qp = NULL;
4569 struct bnxt_qplib_cqe *orig_cqe = NULL;
4570 struct bnxt_re_sqp_entries *sqp_entry = NULL;
4571 int nw_type;
4572 u16 vlan_id;
4573 u8 sl;
4574
4575 tbl_idx = cqe->wr_id;
4576
4577 sqp_entry = &rdev->gsi_ctx.sqp_tbl[tbl_idx];
4578 gsi_qp = sqp_entry->qp1_qp;
4579 orig_cqe = &sqp_entry->cqe;
4580
4581 wc->wr_id = sqp_entry->wrid;
4582 wc->byte_len = orig_cqe->length;
4583 wc->qp = &gsi_qp->ib_qp;
4584
4585 wc->ex.imm_data = orig_cqe->immdata;
4586 wc->src_qp = orig_cqe->src_qp;
4587 memcpy(wc->smac, orig_cqe->smac, ETH_ALEN);
4588 if (bnxt_re_is_nonzero_vlanid_pkt(orig_cqe, &vlan_id, &sl)) {
4589 if (bnxt_re_check_if_vlan_valid(rdev, vlan_id)) {
4590 wc->sl = sl;
4591 wc->vlan_id = vlan_id;
4592 wc->wc_flags |= IB_WC_WITH_VLAN;
4593 }
4594 }
4595 wc->port_num = 1;
4596 wc->vendor_err = orig_cqe->status;
4597
4598 wc->opcode = IB_WC_RECV;
4599 wc->status = __rawqp1_to_ib_wc_status(orig_cqe->status);
4600 wc->wc_flags |= IB_WC_GRH;
4601
4602 nw_type = bnxt_re_check_packet_type(orig_cqe->raweth_qp1_flags,
4603 orig_cqe->raweth_qp1_flags2);
4604 if(nw_type >= 0)
4605 dev_dbg(rdev_to_dev(rdev), "%s nw_type = %d\n", __func__, nw_type);
4606 }
4607
bnxt_re_process_res_ud_wc(struct bnxt_re_dev * rdev,struct bnxt_re_qp * qp,struct ib_wc * wc,struct bnxt_qplib_cqe * cqe)4608 static void bnxt_re_process_res_ud_wc(struct bnxt_re_dev *rdev,
4609 struct bnxt_re_qp *qp, struct ib_wc *wc,
4610 struct bnxt_qplib_cqe *cqe)
4611 {
4612 u16 vlan_id = 0;
4613
4614 wc->opcode = IB_WC_RECV;
4615 wc->status = __rc_to_ib_wc_status(cqe->status);
4616 if (cqe->flags & CQ_RES_UD_FLAGS_IMM)
4617 wc->wc_flags |= IB_WC_WITH_IMM;
4618 if (cqe->flags & CQ_RES_RC_FLAGS_INV)
4619 wc->wc_flags |= IB_WC_WITH_INVALIDATE;
4620 /* report only on GSI QP for Thor */
4621 if (rdev->gsi_ctx.gsi_qp->qplib_qp.id == qp->qplib_qp.id &&
4622 rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_UD) {
4623 wc->wc_flags |= IB_WC_GRH;
4624 memcpy(wc->smac, cqe->smac, ETH_ALEN);
4625 wc->wc_flags |= IB_WC_WITH_SMAC;
4626 if (_is_cqe_v2_supported(rdev->dev_attr->dev_cap_flags)) {
4627 if (cqe->flags & CQ_RES_UD_V2_FLAGS_META_FORMAT_MASK) {
4628 if (cqe->cfa_meta &
4629 BNXT_QPLIB_CQE_CFA_META1_VALID)
4630 vlan_id = (cqe->cfa_meta & 0xFFF);
4631 }
4632 } else if (cqe->flags & CQ_RES_UD_FLAGS_META_FORMAT_VLAN) {
4633 vlan_id = (cqe->cfa_meta & 0xFFF);
4634 }
4635 /* Mark only if vlan_id is non zero */
4636 if (vlan_id && bnxt_re_check_if_vlan_valid(rdev, vlan_id)) {
4637 wc->vlan_id = vlan_id;
4638 wc->wc_flags |= IB_WC_WITH_VLAN;
4639 }
4640 }
4641 }
4642
bnxt_re_legacy_send_phantom_wqe(struct bnxt_re_qp * qp)4643 static int bnxt_re_legacy_send_phantom_wqe(struct bnxt_re_qp *qp)
4644 {
4645 struct bnxt_qplib_qp *lib_qp = &qp->qplib_qp;
4646 unsigned long flags;
4647 int rc = 0;
4648
4649 spin_lock_irqsave(&qp->sq_lock, flags);
4650
4651 rc = bnxt_re_legacy_bind_fence_mw(lib_qp);
4652 if (!rc) {
4653 lib_qp->sq.phantom_wqe_cnt++;
4654 dev_dbg(&lib_qp->sq.hwq.pdev->dev,
4655 "qp %#x sq->prod %#x sw_prod %#x phantom_wqe_cnt %d\n",
4656 lib_qp->id, lib_qp->sq.hwq.prod,
4657 HWQ_CMP(lib_qp->sq.hwq.prod, &lib_qp->sq.hwq),
4658 lib_qp->sq.phantom_wqe_cnt);
4659 }
4660
4661 spin_unlock_irqrestore(&qp->sq_lock, flags);
4662 return rc;
4663 }
4664
bnxt_re_poll_cq(struct ib_cq * ib_cq,int num_entries,struct ib_wc * wc)4665 int bnxt_re_poll_cq(struct ib_cq *ib_cq, int num_entries, struct ib_wc *wc)
4666 {
4667 struct bnxt_re_cq *cq = to_bnxt_re(ib_cq, struct bnxt_re_cq, ibcq);
4668 struct bnxt_re_dev *rdev = cq->rdev;
4669 struct bnxt_re_qp *qp;
4670 struct bnxt_qplib_cqe *cqe;
4671 int i, ncqe, budget, init_budget;
4672 struct bnxt_qplib_q *sq;
4673 struct bnxt_qplib_qp *lib_qp;
4674 u32 tbl_idx;
4675 struct bnxt_re_sqp_entries *sqp_entry = NULL;
4676 unsigned long flags;
4677 u8 gsi_mode;
4678
4679 /*
4680 * DB recovery CQ; only process the door bell pacing alert from
4681 * the user lib
4682 */
4683 if (cq->is_dbr_soft_cq) {
4684 bnxt_re_pacing_alert(rdev);
4685 return 0;
4686 }
4687
4688 /* User CQ; the only processing we do is to
4689 * complete any pending CQ resize operation.
4690 */
4691 if (cq->umem) {
4692 if (cq->resize_umem)
4693 bnxt_re_resize_cq_complete(cq);
4694 return 0;
4695 }
4696
4697 spin_lock_irqsave(&cq->cq_lock, flags);
4698
4699 budget = min_t(u32, num_entries, cq->max_cql);
4700 init_budget = budget;
4701 if (!cq->cql) {
4702 dev_err(rdev_to_dev(rdev), "POLL CQ no CQL to use\n");
4703 goto exit;
4704 }
4705 cqe = &cq->cql[0];
4706 gsi_mode = rdev->gsi_ctx.gsi_qp_mode;
4707 while (budget) {
4708 lib_qp = NULL;
4709 ncqe = bnxt_qplib_poll_cq(&cq->qplib_cq, cqe, budget, &lib_qp);
4710 if (lib_qp) {
4711 sq = &lib_qp->sq;
4712 if (sq->legacy_send_phantom == true) {
4713 qp = container_of(lib_qp, struct bnxt_re_qp, qplib_qp);
4714 if (bnxt_re_legacy_send_phantom_wqe(qp) == -ENOMEM)
4715 dev_err(rdev_to_dev(rdev),
4716 "Phantom failed! Scheduled to send again\n");
4717 else
4718 sq->legacy_send_phantom = false;
4719 }
4720 }
4721 if (ncqe < budget)
4722 ncqe += bnxt_qplib_process_flush_list(&cq->qplib_cq,
4723 cqe + ncqe,
4724 budget - ncqe);
4725
4726 if (!ncqe)
4727 break;
4728
4729 for (i = 0; i < ncqe; i++, cqe++) {
4730 /* Transcribe each qplib_wqe back to ib_wc */
4731 memset(wc, 0, sizeof(*wc));
4732
4733 wc->wr_id = cqe->wr_id;
4734 wc->byte_len = cqe->length;
4735 qp = to_bnxt_re((struct bnxt_qplib_qp *)cqe->qp_handle,
4736 struct bnxt_re_qp, qplib_qp);
4737 if (!qp) {
4738 dev_err(rdev_to_dev(rdev),
4739 "POLL CQ bad QP handle\n");
4740 continue;
4741 }
4742 wc->qp = &qp->ib_qp;
4743 wc->ex.imm_data = cqe->immdata;
4744 wc->src_qp = cqe->src_qp;
4745 memcpy(wc->smac, cqe->smac, ETH_ALEN);
4746 wc->port_num = 1;
4747 wc->vendor_err = cqe->status;
4748
4749 switch(cqe->opcode) {
4750 case CQ_BASE_CQE_TYPE_REQ:
4751 if (gsi_mode == BNXT_RE_GSI_MODE_ALL &&
4752 qp->qplib_qp.id ==
4753 rdev->gsi_ctx.gsi_sqp->qplib_qp.id) {
4754 /* Handle this completion with
4755 * the stored completion */
4756 dev_dbg(rdev_to_dev(rdev),
4757 "Skipping this UD Send CQ\n");
4758 memset(wc, 0, sizeof(*wc));
4759 continue;
4760 }
4761 bnxt_re_process_req_wc(wc, cqe);
4762 break;
4763 case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1:
4764 if (gsi_mode == BNXT_RE_GSI_MODE_ALL) {
4765 if (!cqe->status) {
4766 int rc = 0;
4767 rc = bnxt_re_process_raw_qp_packet_receive(qp, cqe);
4768 if (!rc) {
4769 memset(wc, 0,
4770 sizeof(*wc));
4771 continue;
4772 }
4773 cqe->status = -1;
4774 }
4775 /* Errors need not be looped back.
4776 * But change the wr_id to the one
4777 * stored in the table
4778 */
4779 tbl_idx = cqe->wr_id;
4780 sqp_entry = &rdev->gsi_ctx.sqp_tbl[tbl_idx];
4781 wc->wr_id = sqp_entry->wrid;
4782 }
4783
4784 bnxt_re_process_res_rawqp1_wc(wc, cqe);
4785 break;
4786 case CQ_BASE_CQE_TYPE_RES_RC:
4787 bnxt_re_process_res_rc_wc(wc, cqe);
4788 break;
4789 case CQ_BASE_CQE_TYPE_RES_UD:
4790 if (gsi_mode == BNXT_RE_GSI_MODE_ALL &&
4791 qp->qplib_qp.id ==
4792 rdev->gsi_ctx.gsi_sqp->qplib_qp.id) {
4793 /* Handle this completion with
4794 * the stored completion
4795 */
4796 dev_dbg(rdev_to_dev(rdev),
4797 "Handling the UD receive CQ\n");
4798 if (cqe->status) {
4799 /* TODO handle this completion as a failure in
4800 * loopback porocedure
4801 */
4802 continue;
4803 } else {
4804 bnxt_re_process_res_shadow_qp_wc(qp, wc, cqe);
4805 break;
4806 }
4807 }
4808 bnxt_re_process_res_ud_wc(rdev, qp, wc, cqe);
4809 break;
4810 default:
4811 dev_err(rdev_to_dev(cq->rdev),
4812 "POLL CQ type 0x%x not handled, skip!\n",
4813 cqe->opcode);
4814 continue;
4815 }
4816 wc++;
4817 budget--;
4818 }
4819 }
4820 exit:
4821 spin_unlock_irqrestore(&cq->cq_lock, flags);
4822 return init_budget - budget;
4823 }
4824
bnxt_re_req_notify_cq(struct ib_cq * ib_cq,enum ib_cq_notify_flags ib_cqn_flags)4825 int bnxt_re_req_notify_cq(struct ib_cq *ib_cq,
4826 enum ib_cq_notify_flags ib_cqn_flags)
4827 {
4828 struct bnxt_re_cq *cq = to_bnxt_re(ib_cq, struct bnxt_re_cq, ibcq);
4829 int type = 0, rc = 0;
4830 unsigned long flags;
4831
4832 spin_lock_irqsave(&cq->cq_lock, flags);
4833 /* Trigger on the very next completion */
4834 if (ib_cqn_flags & IB_CQ_NEXT_COMP)
4835 type = DBC_DBC_TYPE_CQ_ARMALL;
4836 /* Trigger on the next solicited completion */
4837 else if (ib_cqn_flags & IB_CQ_SOLICITED)
4838 type = DBC_DBC_TYPE_CQ_ARMSE;
4839
4840 bnxt_qplib_req_notify_cq(&cq->qplib_cq, type);
4841
4842 /* Poll to see if there are missed events */
4843 if ((ib_cqn_flags & IB_CQ_REPORT_MISSED_EVENTS) &&
4844 !(bnxt_qplib_is_cq_empty(&cq->qplib_cq)))
4845 rc = 1;
4846
4847 spin_unlock_irqrestore(&cq->cq_lock, flags);
4848
4849 return rc;
4850 }
4851
4852 /* Memory Regions */
bnxt_re_get_dma_mr(struct ib_pd * ib_pd,int mr_access_flags)4853 struct ib_mr *bnxt_re_get_dma_mr(struct ib_pd *ib_pd, int mr_access_flags)
4854 {
4855 struct bnxt_qplib_mrinfo mrinfo;
4856 struct bnxt_re_dev *rdev;
4857 struct bnxt_re_mr *mr;
4858 struct bnxt_re_pd *pd;
4859 u32 max_mr_count;
4860 u64 pbl = 0;
4861 int rc;
4862
4863 memset(&mrinfo, 0, sizeof(mrinfo));
4864 pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd);
4865 rdev = pd->rdev;
4866
4867 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
4868 if (!mr) {
4869 dev_err(rdev_to_dev(rdev),
4870 "Allocate memory for DMA MR failed!\n");
4871 return ERR_PTR(-ENOMEM);
4872 }
4873 mr->rdev = rdev;
4874 mr->qplib_mr.pd = &pd->qplib_pd;
4875 mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags);
4876 mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR;
4877
4878 /* Allocate and register 0 as the address */
4879 rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr);
4880 if (rc) {
4881 dev_err(rdev_to_dev(rdev), "Allocate DMA MR failed!\n");
4882 goto fail;
4883 }
4884 mr->qplib_mr.total_size = -1; /* Infinite length */
4885 mrinfo.ptes = &pbl;
4886 mrinfo.sg.npages = 0;
4887 mrinfo.sg.pgsize = PAGE_SIZE;
4888 mrinfo.sg.pgshft = PAGE_SHIFT;
4889 mrinfo.sg.pgsize = PAGE_SIZE;
4890 mrinfo.mrw = &mr->qplib_mr;
4891 mrinfo.is_dma = true;
4892 rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mrinfo, false);
4893 if (rc) {
4894 dev_err(rdev_to_dev(rdev), "Register DMA MR failed!\n");
4895 goto fail_mr;
4896 }
4897 mr->ib_mr.lkey = mr->qplib_mr.lkey;
4898 if (mr_access_flags & (IB_ACCESS_REMOTE_WRITE | IB_ACCESS_REMOTE_READ |
4899 IB_ACCESS_REMOTE_ATOMIC))
4900 mr->ib_mr.rkey = mr->ib_mr.lkey;
4901 atomic_inc(&rdev->stats.rsors.mr_count);
4902 max_mr_count = atomic_read(&rdev->stats.rsors.mr_count);
4903 if (max_mr_count > atomic_read(&rdev->stats.rsors.max_mr_count))
4904 atomic_set(&rdev->stats.rsors.max_mr_count, max_mr_count);
4905
4906 return &mr->ib_mr;
4907
4908 fail_mr:
4909 bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr);
4910 fail:
4911 kfree(mr);
4912 return ERR_PTR(rc);
4913 }
4914
bnxt_re_dereg_mr(struct ib_mr * ib_mr,struct ib_udata * udata)4915 int bnxt_re_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
4916 {
4917 struct bnxt_re_mr *mr = to_bnxt_re(ib_mr, struct bnxt_re_mr, ib_mr);
4918 struct bnxt_re_dev *rdev = mr->rdev;
4919 int rc = 0;
4920
4921 rc = bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr);
4922 if (rc)
4923 dev_err(rdev_to_dev(rdev), "Dereg MR failed (%d): rc - %#x\n",
4924 mr->qplib_mr.lkey, rc);
4925
4926 if (mr->pages) {
4927 bnxt_qplib_free_fast_reg_page_list(&rdev->qplib_res,
4928 &mr->qplib_frpl);
4929 kfree(mr->pages);
4930 mr->npages = 0;
4931 mr->pages = NULL;
4932 }
4933 if (!IS_ERR(mr->ib_umem) && mr->ib_umem) {
4934 mr->is_invalcb_active = false;
4935 bnxt_re_peer_mem_release(mr->ib_umem);
4936 }
4937 kfree(mr);
4938 atomic_dec(&rdev->stats.rsors.mr_count);
4939 return 0;
4940 }
4941
bnxt_re_set_page(struct ib_mr * ib_mr,u64 addr)4942 static int bnxt_re_set_page(struct ib_mr *ib_mr, u64 addr)
4943 {
4944 struct bnxt_re_mr *mr = to_bnxt_re(ib_mr, struct bnxt_re_mr, ib_mr);
4945
4946 if (unlikely(mr->npages == mr->qplib_frpl.max_pg_ptrs))
4947 return -ENOMEM;
4948
4949 mr->pages[mr->npages++] = addr;
4950 dev_dbg(NULL, "%s: ibdev %p Set MR pages[%d] = 0x%lx\n",
4951 ROCE_DRV_MODULE_NAME, ib_mr->device, mr->npages - 1,
4952 mr->pages[mr->npages - 1]);
4953 return 0;
4954 }
4955
bnxt_re_map_mr_sg(struct ib_mr * ib_mr,struct scatterlist * sg,int sg_nents,unsigned int * sg_offset)4956 int bnxt_re_map_mr_sg(struct ib_mr *ib_mr, struct scatterlist *sg,
4957 int sg_nents, unsigned int *sg_offset)
4958 {
4959 struct bnxt_re_mr *mr = to_bnxt_re(ib_mr, struct bnxt_re_mr, ib_mr);
4960
4961 mr->npages = 0;
4962 return ib_sg_to_pages(ib_mr, sg, sg_nents,
4963 sg_offset, bnxt_re_set_page);
4964 }
4965
bnxt_re_alloc_mr(struct ib_pd * ib_pd,enum ib_mr_type type,u32 max_num_sg,struct ib_udata * udata)4966 struct ib_mr *bnxt_re_alloc_mr(struct ib_pd *ib_pd, enum ib_mr_type type,
4967 u32 max_num_sg, struct ib_udata *udata)
4968 {
4969 struct bnxt_re_pd *pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd);
4970 struct bnxt_re_dev *rdev = pd->rdev;
4971 struct bnxt_re_mr *mr;
4972 u32 max_mr_count;
4973 int rc;
4974
4975 dev_dbg(rdev_to_dev(rdev), "Alloc MR\n");
4976 if (type != IB_MR_TYPE_MEM_REG) {
4977 dev_dbg(rdev_to_dev(rdev), "MR type 0x%x not supported\n", type);
4978 return ERR_PTR(-EINVAL);
4979 }
4980 if (max_num_sg > MAX_PBL_LVL_1_PGS) {
4981 dev_dbg(rdev_to_dev(rdev), "Max SG exceeded\n");
4982 return ERR_PTR(-EINVAL);
4983 }
4984 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
4985 if (!mr) {
4986 dev_err(rdev_to_dev(rdev), "Allocate MR mem failed!\n");
4987 return ERR_PTR(-ENOMEM);
4988 }
4989 mr->rdev = rdev;
4990 mr->qplib_mr.pd = &pd->qplib_pd;
4991 mr->qplib_mr.flags = BNXT_QPLIB_FR_PMR;
4992 mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR;
4993
4994 rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr);
4995 if (rc) {
4996 dev_err(rdev_to_dev(rdev), "Allocate MR failed!\n");
4997 goto fail;
4998 }
4999 mr->ib_mr.lkey = mr->qplib_mr.lkey;
5000 mr->ib_mr.rkey = mr->ib_mr.lkey;
5001 mr->pages = kzalloc(sizeof(u64) * max_num_sg, GFP_KERNEL);
5002 if (!mr->pages) {
5003 dev_err(rdev_to_dev(rdev),
5004 "Allocate MR page list mem failed!\n");
5005 rc = -ENOMEM;
5006 goto fail_mr;
5007 }
5008 rc = bnxt_qplib_alloc_fast_reg_page_list(&rdev->qplib_res,
5009 &mr->qplib_frpl, max_num_sg);
5010 if (rc) {
5011 dev_err(rdev_to_dev(rdev),
5012 "Allocate HW Fast reg page list failed!\n");
5013 goto free_page;
5014 }
5015 dev_dbg(rdev_to_dev(rdev), "Alloc MR pages = 0x%p\n", mr->pages);
5016
5017 atomic_inc(&rdev->stats.rsors.mr_count);
5018 max_mr_count = atomic_read(&rdev->stats.rsors.mr_count);
5019 if (max_mr_count > atomic_read(&rdev->stats.rsors.max_mr_count))
5020 atomic_set(&rdev->stats.rsors.max_mr_count, max_mr_count);
5021 return &mr->ib_mr;
5022
5023 free_page:
5024 kfree(mr->pages);
5025 fail_mr:
5026 bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr);
5027 fail:
5028 kfree(mr);
5029 return ERR_PTR(rc);
5030 }
5031
5032 /* Memory Windows */
bnxt_re_alloc_mw(struct ib_pd * ib_pd,enum ib_mw_type type,struct ib_udata * udata)5033 struct ib_mw *bnxt_re_alloc_mw(struct ib_pd *ib_pd, enum ib_mw_type type,
5034 struct ib_udata *udata)
5035 {
5036 struct bnxt_re_pd *pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd);
5037 struct bnxt_re_dev *rdev = pd->rdev;
5038 struct bnxt_re_mw *mw;
5039 u32 max_mw_count;
5040 int rc;
5041
5042 mw = kzalloc(sizeof(*mw), GFP_KERNEL);
5043 if (!mw) {
5044 dev_err(rdev_to_dev(rdev), "Allocate MW failed!\n");
5045 rc = -ENOMEM;
5046 goto exit;
5047 }
5048 mw->rdev = rdev;
5049 mw->qplib_mw.pd = &pd->qplib_pd;
5050
5051 mw->qplib_mw.type = (type == IB_MW_TYPE_1 ?
5052 CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE1 :
5053 CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2B);
5054 rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mw->qplib_mw);
5055 if (rc) {
5056 dev_err(rdev_to_dev(rdev), "Allocate MW failed!\n");
5057 goto fail;
5058 }
5059 mw->ib_mw.rkey = mw->qplib_mw.rkey;
5060 atomic_inc(&rdev->stats.rsors.mw_count);
5061 max_mw_count = atomic_read(&rdev->stats.rsors.mw_count);
5062 if (max_mw_count > atomic_read(&rdev->stats.rsors.max_mw_count))
5063 atomic_set(&rdev->stats.rsors.max_mw_count, max_mw_count);
5064
5065 return &mw->ib_mw;
5066 fail:
5067 kfree(mw);
5068 exit:
5069 return ERR_PTR(rc);
5070 }
5071
bnxt_re_dealloc_mw(struct ib_mw * ib_mw)5072 int bnxt_re_dealloc_mw(struct ib_mw *ib_mw)
5073 {
5074 struct bnxt_re_mw *mw = to_bnxt_re(ib_mw, struct bnxt_re_mw, ib_mw);
5075 struct bnxt_re_dev *rdev = mw->rdev;
5076 int rc;
5077
5078 rc = bnxt_qplib_free_mrw(&rdev->qplib_res, &mw->qplib_mw);
5079 if (rc) {
5080 dev_err(rdev_to_dev(rdev), "Free MW failed: %#x\n", rc);
5081 return rc;
5082 }
5083
5084 kfree(mw);
5085 atomic_dec(&rdev->stats.rsors.mw_count);
5086 return rc;
5087 }
5088
bnxt_re_page_size_ok(int page_shift)5089 static int bnxt_re_page_size_ok(int page_shift)
5090 {
5091 switch (page_shift) {
5092 case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_4K:
5093 case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_8K:
5094 case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_64K:
5095 case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_2M:
5096 case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_256K:
5097 case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_1M:
5098 case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_4M:
5099 case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_256MB:
5100 case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_1G:
5101 return 1;
5102 default:
5103 return 0;
5104 }
5105 }
5106
bnxt_re_get_page_shift(struct ib_umem * umem,u64 va,u64 st,u64 cmask)5107 static int bnxt_re_get_page_shift(struct ib_umem *umem,
5108 u64 va, u64 st, u64 cmask)
5109 {
5110 int pgshft;
5111
5112 pgshft = ilog2(umem->page_size);
5113
5114 return pgshft;
5115 }
5116
bnxt_re_get_num_pages(struct ib_umem * umem,u64 start,u64 length,int page_shift)5117 static int bnxt_re_get_num_pages(struct ib_umem *umem, u64 start, u64 length, int page_shift)
5118 {
5119 int npages = 0;
5120
5121 if (page_shift == PAGE_SHIFT) {
5122 npages = ib_umem_num_pages_compat(umem);
5123 } else {
5124 npages = ALIGN(length, BIT(page_shift)) / BIT(page_shift);
5125 if (start % BIT(page_shift))
5126 npages++;
5127 }
5128 return npages;
5129 }
5130
5131 /* uverbs */
bnxt_re_reg_user_mr(struct ib_pd * ib_pd,u64 start,u64 length,u64 virt_addr,int mr_access_flags,struct ib_udata * udata)5132 struct ib_mr *bnxt_re_reg_user_mr(struct ib_pd *ib_pd, u64 start, u64 length,
5133 u64 virt_addr, int mr_access_flags,
5134 struct ib_udata *udata)
5135 {
5136 struct bnxt_re_pd *pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd);
5137 struct bnxt_re_dev *rdev = pd->rdev;
5138 struct bnxt_qplib_mrinfo mrinfo;
5139 int umem_pgs, page_shift, rc;
5140 struct bnxt_re_mr *mr;
5141 struct ib_umem *umem;
5142 u32 max_mr_count;
5143 int npages;
5144
5145 dev_dbg(rdev_to_dev(rdev), "Reg user MR\n");
5146
5147 if (bnxt_re_get_total_mr_mw_count(rdev) >= rdev->dev_attr->max_mr)
5148 return ERR_PTR(-ENOMEM);
5149
5150 if (rdev->mod_exit) {
5151 dev_dbg(rdev_to_dev(rdev), "%s(): in mod_exit, just return!\n", __func__);
5152 return ERR_PTR(-EIO);
5153 }
5154 memset(&mrinfo, 0, sizeof(mrinfo));
5155 if (length > BNXT_RE_MAX_MR_SIZE) {
5156 dev_err(rdev_to_dev(rdev), "Requested MR Size: %lu "
5157 "> Max supported: %ld\n", length, BNXT_RE_MAX_MR_SIZE);
5158 return ERR_PTR(-ENOMEM);
5159 }
5160 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
5161 if (!mr) {
5162 dev_err(rdev_to_dev(rdev), "Allocate MR failed!\n");
5163 return ERR_PTR (-ENOMEM);
5164 }
5165 mr->rdev = rdev;
5166 mr->qplib_mr.pd = &pd->qplib_pd;
5167 mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags);
5168 mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_MR;
5169
5170 if (!_is_alloc_mr_unified(rdev->qplib_res.dattr)) {
5171 rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr);
5172 if (rc) {
5173 dev_err(rdev_to_dev(rdev), "Alloc MR failed!\n");
5174 goto fail;
5175 }
5176 /* The fixed portion of the rkey is the same as the lkey */
5177 mr->ib_mr.rkey = mr->qplib_mr.rkey;
5178 }
5179
5180 umem = ib_umem_get_flags_compat(rdev, ib_pd->uobject->context,
5181 udata, start, length,
5182 mr_access_flags, 0);
5183 if (IS_ERR(umem)) {
5184 rc = PTR_ERR(umem);
5185 dev_err(rdev_to_dev(rdev), "%s: ib_umem_get failed! rc = %d\n",
5186 __func__, rc);
5187 goto free_mr;
5188 }
5189 mr->ib_umem = umem;
5190
5191 mr->qplib_mr.va = virt_addr;
5192 umem_pgs = ib_umem_num_pages_compat(umem);
5193 if (!umem_pgs) {
5194 dev_err(rdev_to_dev(rdev), "umem is invalid!\n");
5195 rc = -EINVAL;
5196 goto free_umem;
5197 }
5198 mr->qplib_mr.total_size = length;
5199 page_shift = bnxt_re_get_page_shift(umem, virt_addr, start,
5200 rdev->dev_attr->page_size_cap);
5201 if (!bnxt_re_page_size_ok(page_shift)) {
5202 dev_err(rdev_to_dev(rdev), "umem page size unsupported!\n");
5203 rc = -EFAULT;
5204 goto free_umem;
5205 }
5206 npages = bnxt_re_get_num_pages(umem, start, length, page_shift);
5207
5208 /* Map umem buf ptrs to the PBL */
5209 mrinfo.sg.npages = npages;
5210 mrinfo.sg.sghead = get_ib_umem_sgl(umem, &mrinfo.sg.nmap);
5211 mrinfo.sg.pgshft = page_shift;
5212 mrinfo.sg.pgsize = BIT(page_shift);
5213
5214 mrinfo.mrw = &mr->qplib_mr;
5215
5216 rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mrinfo, false);
5217 if (rc) {
5218 dev_err(rdev_to_dev(rdev), "Reg user MR failed!\n");
5219 goto free_umem;
5220 }
5221
5222 mr->ib_mr.lkey = mr->ib_mr.rkey = mr->qplib_mr.lkey;
5223 atomic_inc(&rdev->stats.rsors.mr_count);
5224 max_mr_count = atomic_read(&rdev->stats.rsors.mr_count);
5225 if (max_mr_count > atomic_read(&rdev->stats.rsors.max_mr_count))
5226 atomic_set(&rdev->stats.rsors.max_mr_count, max_mr_count);
5227
5228 return &mr->ib_mr;
5229
5230 free_umem:
5231 bnxt_re_peer_mem_release(mr->ib_umem);
5232 free_mr:
5233 if (!_is_alloc_mr_unified(rdev->qplib_res.dattr))
5234 bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr);
5235 fail:
5236 kfree(mr);
5237 return ERR_PTR(rc);
5238 }
5239
5240 int
bnxt_re_rereg_user_mr(struct ib_mr * ib_mr,int flags,u64 start,u64 length,u64 virt_addr,int mr_access_flags,struct ib_pd * ib_pd,struct ib_udata * udata)5241 bnxt_re_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start, u64 length,
5242 u64 virt_addr, int mr_access_flags,
5243 struct ib_pd *ib_pd, struct ib_udata *udata)
5244 {
5245 struct bnxt_re_mr *mr = to_bnxt_re(ib_mr, struct bnxt_re_mr, ib_mr);
5246 struct bnxt_re_pd *pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd);
5247 int umem_pgs = 0, page_shift = PAGE_SHIFT, rc;
5248 struct bnxt_re_dev *rdev = mr->rdev;
5249 struct bnxt_qplib_mrinfo mrinfo;
5250 struct ib_umem *umem;
5251 u32 npages;
5252
5253 /* TODO: Must decipher what to modify based on the flags */
5254 memset(&mrinfo, 0, sizeof(mrinfo));
5255 if (flags & IB_MR_REREG_TRANS) {
5256 umem = ib_umem_get_flags_compat(rdev, ib_pd->uobject->context,
5257 udata, start, length,
5258 mr_access_flags, 0);
5259 if (IS_ERR(umem)) {
5260 rc = PTR_ERR(umem);
5261 dev_err(rdev_to_dev(rdev),
5262 "%s: ib_umem_get failed! ret = %d\n",
5263 __func__, rc);
5264 goto fail;
5265 }
5266 mr->ib_umem = umem;
5267
5268 mr->qplib_mr.va = virt_addr;
5269 umem_pgs = ib_umem_num_pages_compat(umem);
5270 if (!umem_pgs) {
5271 dev_err(rdev_to_dev(rdev), "umem is invalid!\n");
5272 rc = -EINVAL;
5273 goto fail_free_umem;
5274 }
5275 mr->qplib_mr.total_size = length;
5276 page_shift = bnxt_re_get_page_shift(umem, virt_addr, start,
5277 rdev->dev_attr->page_size_cap);
5278 if (!bnxt_re_page_size_ok(page_shift)) {
5279 dev_err(rdev_to_dev(rdev),
5280 "umem page size unsupported!\n");
5281 rc = -EFAULT;
5282 goto fail_free_umem;
5283 }
5284 npages = bnxt_re_get_num_pages(umem, start, length, page_shift);
5285 /* Map umem buf ptrs to the PBL */
5286 mrinfo.sg.npages = npages;
5287 mrinfo.sg.sghead = get_ib_umem_sgl(umem, &mrinfo.sg.nmap);
5288 mrinfo.sg.pgshft = page_shift;
5289 mrinfo.sg.pgsize = BIT(page_shift);
5290 }
5291
5292 mrinfo.mrw = &mr->qplib_mr;
5293 if (flags & IB_MR_REREG_PD)
5294 mr->qplib_mr.pd = &pd->qplib_pd;
5295
5296 if (flags & IB_MR_REREG_ACCESS)
5297 mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags);
5298
5299 rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mrinfo, false);
5300 if (rc) {
5301 dev_err(rdev_to_dev(rdev), "Rereg user MR failed!\n");
5302 goto fail_free_umem;
5303 }
5304 mr->ib_mr.rkey = mr->qplib_mr.rkey;
5305
5306 return 0;
5307
5308 fail_free_umem:
5309 bnxt_re_peer_mem_release(mr->ib_umem);
5310 fail:
5311 return rc;
5312 }
5313
bnxt_re_check_abi_version(struct bnxt_re_dev * rdev)5314 static int bnxt_re_check_abi_version(struct bnxt_re_dev *rdev)
5315 {
5316 struct ib_device *ibdev = &rdev->ibdev;
5317 u32 uverbs_abi_ver;
5318
5319 uverbs_abi_ver = GET_UVERBS_ABI_VERSION(ibdev);
5320 dev_dbg(rdev_to_dev(rdev), "ABI version requested %d\n",
5321 uverbs_abi_ver);
5322 if (uverbs_abi_ver != BNXT_RE_ABI_VERSION) {
5323 dev_dbg(rdev_to_dev(rdev), " is different from the device %d \n",
5324 BNXT_RE_ABI_VERSION);
5325 return -EPERM;
5326 }
5327 return 0;
5328 }
5329
bnxt_re_alloc_ucontext(struct ib_ucontext * uctx_in,struct ib_udata * udata)5330 int bnxt_re_alloc_ucontext(struct ib_ucontext *uctx_in,
5331 struct ib_udata *udata)
5332 {
5333 struct ib_ucontext *ctx = uctx_in;
5334 struct ib_device *ibdev = ctx->device;
5335 struct bnxt_re_ucontext *uctx =
5336 container_of(ctx, struct bnxt_re_ucontext, ibucontext);
5337
5338 struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
5339 struct bnxt_qplib_dev_attr *dev_attr = rdev->dev_attr;
5340 struct bnxt_re_uctx_resp resp = {};
5341 struct bnxt_re_uctx_req ureq = {};
5342 struct bnxt_qplib_chip_ctx *cctx;
5343 u32 chip_met_rev_num;
5344 bool genp5 = false;
5345 int rc;
5346
5347 cctx = rdev->chip_ctx;
5348 rc = bnxt_re_check_abi_version(rdev);
5349 if (rc)
5350 goto fail;
5351
5352 uctx->rdev = rdev;
5353 uctx->shpg = (void *)__get_free_page(GFP_KERNEL);
5354 if (!uctx->shpg) {
5355 dev_err(rdev_to_dev(rdev), "shared memory allocation failed!\n");
5356 rc = -ENOMEM;
5357 goto fail;
5358 }
5359 spin_lock_init(&uctx->sh_lock);
5360 if (BNXT_RE_ABI_VERSION >= 4) {
5361 chip_met_rev_num = cctx->chip_num;
5362 chip_met_rev_num |= ((u32)cctx->chip_rev & 0xFF) <<
5363 BNXT_RE_CHIP_ID0_CHIP_REV_SFT;
5364 chip_met_rev_num |= ((u32)cctx->chip_metal & 0xFF) <<
5365 BNXT_RE_CHIP_ID0_CHIP_MET_SFT;
5366 resp.chip_id0 = chip_met_rev_num;
5367 resp.chip_id1 = 0; /* future extension of chip info */
5368 }
5369
5370 if (BNXT_RE_ABI_VERSION != 4) {
5371 /*Temp, Use idr_alloc instead*/
5372 resp.dev_id = rdev->en_dev->pdev->devfn;
5373 resp.max_qp = rdev->qplib_res.hctx->qp_ctx.max;
5374 }
5375
5376 genp5 = _is_chip_gen_p5_p7(cctx);
5377 resp.mode = genp5 ? cctx->modes.wqe_mode : 0;
5378 if (rdev->dev_attr && _is_host_msn_table(rdev->dev_attr->dev_cap_ext_flags2))
5379 resp.comp_mask = BNXT_RE_COMP_MASK_UCNTX_HW_RETX_ENABLED;
5380
5381 resp.pg_size = PAGE_SIZE;
5382 resp.cqe_sz = sizeof(struct cq_base);
5383 resp.max_cqd = dev_attr->max_cq_wqes;
5384 if (genp5 && cctx->modes.db_push) {
5385 resp.comp_mask |= BNXT_RE_COMP_MASK_UCNTX_WC_DPI_ENABLED;
5386 if (_is_chip_p7(cctx) &&
5387 !(dev_attr->dev_cap_flags &
5388 CREQ_QUERY_FUNC_RESP_SB_PINGPONG_PUSH_MODE))
5389 resp.comp_mask &=
5390 ~BNXT_RE_COMP_MASK_UCNTX_WC_DPI_ENABLED;
5391 }
5392
5393 resp.comp_mask |= BNXT_RE_COMP_MASK_UCNTX_MQP_EX_SUPPORTED;
5394
5395 if (rdev->dbr_pacing)
5396 resp.comp_mask |= BNXT_RE_COMP_MASK_UCNTX_DBR_PACING_ENABLED;
5397
5398 if (rdev->dbr_drop_recov && rdev->user_dbr_drop_recov)
5399 resp.comp_mask |= BNXT_RE_COMP_MASK_UCNTX_DBR_RECOVERY_ENABLED;
5400
5401 if (udata->inlen >= sizeof(ureq)) {
5402 rc = ib_copy_from_udata(&ureq, udata,
5403 min(udata->inlen, sizeof(ureq)));
5404 if (rc)
5405 goto cfail;
5406 if (bnxt_re_init_pow2_flag(&ureq, &resp))
5407 dev_warn(rdev_to_dev(rdev),
5408 "Enabled roundup logic. Library bug?\n");
5409 if (bnxt_re_init_rsvd_wqe_flag(&ureq, &resp, genp5))
5410 dev_warn(rdev_to_dev(rdev),
5411 "Rsvd wqe in use! Try the updated library.\n");
5412 if (ureq.comp_mask & BNXT_RE_COMP_MASK_REQ_UCNTX_VAR_WQE_SUPPORT) {
5413 resp.comp_mask |= BNXT_RE_COMP_MASK_UCNTX_CMASK_HAVE_MODE;
5414 resp.mode = rdev->chip_ctx->modes.wqe_mode;
5415 if (resp.mode == BNXT_QPLIB_WQE_MODE_VARIABLE)
5416 resp.comp_mask |= BNXT_RE_UCNTX_CAP_VAR_WQE_ENABLED;
5417 }
5418 } else {
5419 dev_warn(rdev_to_dev(rdev),
5420 "Enabled roundup logic. Update the library!\n");
5421 resp.comp_mask &= ~BNXT_RE_COMP_MASK_UCNTX_POW2_DISABLED;
5422
5423 dev_warn(rdev_to_dev(rdev),
5424 "Rsvd wqe in use. Update the library!\n");
5425 resp.comp_mask &= ~BNXT_RE_COMP_MASK_UCNTX_RSVD_WQE_DISABLED;
5426 }
5427
5428 uctx->cmask = (uint64_t)resp.comp_mask;
5429 rc = bnxt_re_copy_to_udata(rdev, &resp,
5430 min(udata->outlen, sizeof(resp)),
5431 udata);
5432 if (rc)
5433 goto cfail;
5434
5435 INIT_LIST_HEAD(&uctx->cq_list);
5436 mutex_init(&uctx->cq_lock);
5437
5438 return 0;
5439 cfail:
5440 free_page((u64)uctx->shpg);
5441 uctx->shpg = NULL;
5442 fail:
5443 return rc;
5444 }
5445
bnxt_re_dealloc_ucontext(struct ib_ucontext * ib_uctx)5446 void bnxt_re_dealloc_ucontext(struct ib_ucontext *ib_uctx)
5447 {
5448 struct bnxt_re_ucontext *uctx = to_bnxt_re(ib_uctx,
5449 struct bnxt_re_ucontext,
5450 ibucontext);
5451 struct bnxt_re_dev *rdev = uctx->rdev;
5452 int rc = 0;
5453
5454 if (uctx->shpg)
5455 free_page((u64)uctx->shpg);
5456
5457 if (uctx->dpi.dbr) {
5458 /* Free DPI only if this is the first PD allocated by the
5459 * application and mark the context dpi as NULL
5460 */
5461 if (_is_chip_gen_p5_p7(rdev->chip_ctx) && uctx->wcdpi.dbr) {
5462 rc = bnxt_qplib_dealloc_dpi(&rdev->qplib_res,
5463 &uctx->wcdpi);
5464 if (rc)
5465 dev_err(rdev_to_dev(rdev),
5466 "dealloc push dp failed\n");
5467 uctx->wcdpi.dbr = NULL;
5468 }
5469
5470 rc = bnxt_qplib_dealloc_dpi(&rdev->qplib_res,
5471 &uctx->dpi);
5472 if (rc)
5473 dev_err(rdev_to_dev(rdev), "Deallocte HW DPI failed!\n");
5474 /* Don't fail, continue*/
5475 uctx->dpi.dbr = NULL;
5476 }
5477 return;
5478 }
5479
is_bnxt_re_cq_page(struct bnxt_re_ucontext * uctx,u64 pg_off)5480 static struct bnxt_re_cq *is_bnxt_re_cq_page(struct bnxt_re_ucontext *uctx,
5481 u64 pg_off)
5482 {
5483 struct bnxt_re_cq *cq = NULL, *tmp_cq;
5484
5485 if (!_is_chip_p7(uctx->rdev->chip_ctx))
5486 return NULL;
5487
5488 mutex_lock(&uctx->cq_lock);
5489 list_for_each_entry(tmp_cq, &uctx->cq_list, cq_list) {
5490 if (((u64)tmp_cq->uctx_cq_page >> PAGE_SHIFT) == pg_off) {
5491 cq = tmp_cq;
5492 break;
5493 }
5494 }
5495 mutex_unlock(&uctx->cq_lock);
5496 return cq;
5497 }
5498
5499 /* Helper function to mmap the virtual memory from user app */
bnxt_re_mmap(struct ib_ucontext * ib_uctx,struct vm_area_struct * vma)5500 int bnxt_re_mmap(struct ib_ucontext *ib_uctx, struct vm_area_struct *vma)
5501 {
5502 struct bnxt_re_ucontext *uctx = to_bnxt_re(ib_uctx,
5503 struct bnxt_re_ucontext,
5504 ibucontext);
5505 struct bnxt_re_dev *rdev = uctx->rdev;
5506 struct bnxt_re_cq *cq = NULL;
5507 int rc = 0;
5508 u64 pfn;
5509
5510 switch (vma->vm_pgoff) {
5511 case BNXT_RE_MAP_SH_PAGE:
5512 pfn = vtophys(uctx->shpg) >> PAGE_SHIFT;
5513 return rdma_user_mmap_io(&uctx->ibucontext, vma, pfn, PAGE_SIZE, vma->vm_page_prot, NULL);
5514 dev_dbg(rdev_to_dev(rdev), "%s:%d uctx->shpg 0x%lx, vtophys(uctx->shpg) 0x%lx, pfn = 0x%lx \n",
5515 __func__, __LINE__, (u64) uctx->shpg, vtophys(uctx->shpg), pfn);
5516 if (rc) {
5517 dev_err(rdev_to_dev(rdev), "Shared page mapping failed!\n");
5518 rc = -EAGAIN;
5519 }
5520 return rc;
5521 case BNXT_RE_MAP_WC:
5522 vma->vm_page_prot =
5523 pgprot_writecombine(vma->vm_page_prot);
5524 pfn = (uctx->wcdpi.umdbr >> PAGE_SHIFT);
5525 if (!pfn)
5526 return -EFAULT;
5527 break;
5528 case BNXT_RE_DBR_PAGE:
5529 /* Driver doesn't expect write access request */
5530 if (vma->vm_flags & VM_WRITE)
5531 return -EFAULT;
5532
5533 pfn = vtophys(rdev->dbr_page) >> PAGE_SHIFT;
5534 if (!pfn)
5535 return -EFAULT;
5536 break;
5537 case BNXT_RE_MAP_DB_RECOVERY_PAGE:
5538 pfn = vtophys(uctx->dbr_recov_cq_page) >> PAGE_SHIFT;
5539 if (!pfn)
5540 return -EFAULT;
5541 break;
5542 default:
5543 cq = is_bnxt_re_cq_page(uctx, vma->vm_pgoff);
5544 if (cq) {
5545 pfn = vtophys((void *)cq->uctx_cq_page) >> PAGE_SHIFT;
5546 rc = rdma_user_mmap_io(&uctx->ibucontext, vma, pfn, PAGE_SIZE, vma->vm_page_prot, NULL);
5547 if (rc) {
5548 dev_err(rdev_to_dev(rdev),
5549 "CQ page mapping failed!\n");
5550 rc = -EAGAIN;
5551 }
5552 goto out;
5553 } else {
5554 vma->vm_page_prot =
5555 pgprot_noncached(vma->vm_page_prot);
5556 pfn = vma->vm_pgoff;
5557 }
5558 break;
5559 }
5560
5561 rc = rdma_user_mmap_io(&uctx->ibucontext, vma, pfn, PAGE_SIZE, vma->vm_page_prot, NULL);
5562 if (rc) {
5563 dev_err(rdev_to_dev(rdev), "DPI mapping failed!\n");
5564 return -EAGAIN;
5565 }
5566 rc = __bnxt_re_set_vma_data(uctx, vma);
5567 out:
5568 return rc;
5569 }
5570
bnxt_re_process_mad(struct ib_device * ibdev,int mad_flags,u8 port_num,const struct ib_wc * wc,const struct ib_grh * grh,const struct ib_mad_hdr * in_mad,size_t in_mad_size,struct ib_mad_hdr * out_mad,size_t * out_mad_size,u16 * out_mad_pkey_index)5571 int bnxt_re_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num,
5572 const struct ib_wc *wc, const struct ib_grh *grh,
5573 const struct ib_mad_hdr *in_mad, size_t in_mad_size,
5574 struct ib_mad_hdr *out_mad, size_t *out_mad_size,
5575 u16 *out_mad_pkey_index)
5576 {
5577 return IB_MAD_RESULT_SUCCESS;
5578 }
5579
bnxt_re_disassociate_ucntx(struct ib_ucontext * ib_uctx)5580 void bnxt_re_disassociate_ucntx(struct ib_ucontext *ib_uctx)
5581 {
5582 }
5583