1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2009-2013 Chelsio, Inc. All rights reserved.
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 */
34 #include <sys/cdefs.h>
35 #include "opt_inet.h"
36
37 #ifdef TCP_OFFLOAD
38 #include <sys/types.h>
39 #include <sys/malloc.h>
40 #include <sys/socket.h>
41 #include <sys/socketvar.h>
42 #include <sys/sockio.h>
43 #include <sys/taskqueue.h>
44 #include <netinet/in.h>
45 #include <net/route.h>
46
47 #include <netinet/in_systm.h>
48 #include <netinet/in_pcb.h>
49 #include <netinet/ip.h>
50 #include <netinet/ip_var.h>
51 #include <netinet/tcp_var.h>
52 #include <netinet/tcp.h>
53 #include <netinet/tcpip.h>
54
55 #include <netinet/toecore.h>
56
57 struct sge_iq;
58 struct rss_header;
59 struct cpl_set_tcb_rpl;
60 #include <linux/types.h>
61 #include "offload.h"
62 #include "tom/t4_tom.h"
63
64 #include "iw_cxgbe.h"
65 #include "user.h"
66
67 static int creds(struct toepcb *toep, struct inpcb *inp, size_t wrsize);
68 static int max_fr_immd = T4_MAX_FR_IMMD;//SYSCTL parameter later...
69
alloc_ird(struct c4iw_dev * dev,u32 ird)70 static int alloc_ird(struct c4iw_dev *dev, u32 ird)
71 {
72 int ret = 0;
73
74 spin_lock_irq(&dev->lock);
75 if (ird <= dev->avail_ird)
76 dev->avail_ird -= ird;
77 else
78 ret = -ENOMEM;
79 spin_unlock_irq(&dev->lock);
80
81 if (ret)
82 log(LOG_WARNING, "%s: device IRD resources exhausted\n",
83 device_get_nameunit(dev->rdev.adap->dev));
84
85 return ret;
86 }
87
free_ird(struct c4iw_dev * dev,int ird)88 static void free_ird(struct c4iw_dev *dev, int ird)
89 {
90 spin_lock_irq(&dev->lock);
91 dev->avail_ird += ird;
92 spin_unlock_irq(&dev->lock);
93 }
94
set_state(struct c4iw_qp * qhp,enum c4iw_qp_state state)95 static void set_state(struct c4iw_qp *qhp, enum c4iw_qp_state state)
96 {
97 unsigned long flag;
98 spin_lock_irqsave(&qhp->lock, flag);
99 qhp->attr.state = state;
100 spin_unlock_irqrestore(&qhp->lock, flag);
101 }
102
destroy_qp(struct c4iw_rdev * rdev,struct t4_wq * wq,struct c4iw_dev_ucontext * uctx)103 static int destroy_qp(struct c4iw_rdev *rdev, struct t4_wq *wq,
104 struct c4iw_dev_ucontext *uctx)
105 {
106 struct c4iw_dev *rhp = rdev_to_c4iw_dev(rdev);
107 /*
108 * uP clears EQ contexts when the connection exits rdma mode,
109 * so no need to post a RESET WR for these EQs.
110 */
111 dma_free_coherent(rhp->ibdev.dma_device,
112 wq->rq.memsize, wq->rq.queue,
113 dma_unmap_addr(&wq->rq, mapping));
114 dma_free_coherent(rhp->ibdev.dma_device,
115 wq->sq.memsize, wq->sq.queue,
116 dma_unmap_addr(&wq->sq, mapping));
117 c4iw_rqtpool_free(rdev, wq->rq.rqt_hwaddr, wq->rq.rqt_size);
118 kfree(wq->rq.sw_rq);
119 kfree(wq->sq.sw_sq);
120 c4iw_put_qpid(rdev, wq->rq.qid, uctx);
121 c4iw_put_qpid(rdev, wq->sq.qid, uctx);
122 return 0;
123 }
124
create_qp(struct c4iw_rdev * rdev,struct t4_wq * wq,struct t4_cq * rcq,struct t4_cq * scq,struct c4iw_dev_ucontext * uctx)125 static int create_qp(struct c4iw_rdev *rdev, struct t4_wq *wq,
126 struct t4_cq *rcq, struct t4_cq *scq,
127 struct c4iw_dev_ucontext *uctx)
128 {
129 struct adapter *sc = rdev->adap;
130 struct c4iw_dev *rhp = rdev_to_c4iw_dev(rdev);
131 int user = (uctx != &rdev->uctx);
132 struct fw_ri_res_wr *res_wr;
133 struct fw_ri_res *res;
134 int wr_len;
135 struct c4iw_wr_wait wr_wait;
136 int ret = 0;
137 int eqsize;
138 struct wrqe *wr;
139 u64 sq_bar2_qoffset = 0, rq_bar2_qoffset = 0;
140
141 if (__predict_false(c4iw_stopped(rdev)))
142 return -EIO;
143
144 wq->sq.qid = c4iw_get_qpid(rdev, uctx);
145 if (!wq->sq.qid)
146 return -ENOMEM;
147
148 wq->rq.qid = c4iw_get_qpid(rdev, uctx);
149 if (!wq->rq.qid) {
150 ret = -ENOMEM;
151 goto free_sq_qid;
152 }
153
154 if (!user) {
155 wq->sq.sw_sq = kzalloc(wq->sq.size * sizeof *wq->sq.sw_sq,
156 GFP_KERNEL);
157 if (!wq->sq.sw_sq) {
158 ret = -ENOMEM;
159 goto free_rq_qid;
160 }
161
162 wq->rq.sw_rq = kzalloc(wq->rq.size * sizeof *wq->rq.sw_rq,
163 GFP_KERNEL);
164 if (!wq->rq.sw_rq) {
165 ret = -ENOMEM;
166 goto free_sw_sq;
167 }
168 }
169
170 /*
171 * RQT must be a power of 2 and at least 16 deep.
172 */
173 wq->rq.rqt_size = roundup_pow_of_two(max_t(u16, wq->rq.size, 16));
174 wq->rq.rqt_hwaddr = c4iw_rqtpool_alloc(rdev, wq->rq.rqt_size);
175 if (!wq->rq.rqt_hwaddr) {
176 ret = -ENOMEM;
177 goto free_sw_rq;
178 }
179
180 /*QP memory, allocate DMAable memory for Send & Receive Queues */
181 wq->sq.queue = dma_alloc_coherent(rhp->ibdev.dma_device, wq->sq.memsize,
182 &(wq->sq.dma_addr), GFP_KERNEL);
183 if (!wq->sq.queue) {
184 ret = -ENOMEM;
185 goto free_hwaddr;
186 }
187 wq->sq.phys_addr = vtophys(wq->sq.queue);
188 dma_unmap_addr_set(&wq->sq, mapping, wq->sq.dma_addr);
189 memset(wq->sq.queue, 0, wq->sq.memsize);
190
191 wq->rq.queue = dma_alloc_coherent(rhp->ibdev.dma_device,
192 wq->rq.memsize, &(wq->rq.dma_addr), GFP_KERNEL);
193 if (!wq->rq.queue) {
194 ret = -ENOMEM;
195 goto free_sq_dma;
196 }
197 wq->rq.phys_addr = vtophys(wq->rq.queue);
198 dma_unmap_addr_set(&wq->rq, mapping, wq->rq.dma_addr);
199 memset(wq->rq.queue, 0, wq->rq.memsize);
200
201 CTR5(KTR_IW_CXGBE,
202 "%s QP sq base va 0x%p pa 0x%llx rq base va 0x%p pa 0x%llx",
203 __func__,
204 wq->sq.queue, (unsigned long long)wq->sq.phys_addr,
205 wq->rq.queue, (unsigned long long)wq->rq.phys_addr);
206
207 /* Doorbell/WC regions, determine the BAR2 queue offset and qid. */
208 t4_bar2_sge_qregs(rdev->adap, wq->sq.qid, T4_BAR2_QTYPE_EGRESS, user,
209 &sq_bar2_qoffset, &wq->sq.bar2_qid);
210 t4_bar2_sge_qregs(rdev->adap, wq->rq.qid, T4_BAR2_QTYPE_EGRESS, user,
211 &rq_bar2_qoffset, &wq->rq.bar2_qid);
212
213 if (user) {
214 /* Compute BAR2 DB/WC physical address(page-aligned) for
215 * Userspace mapping.
216 */
217 wq->sq.bar2_pa = (rdev->bar2_pa + sq_bar2_qoffset) & PAGE_MASK;
218 wq->rq.bar2_pa = (rdev->bar2_pa + rq_bar2_qoffset) & PAGE_MASK;
219 CTR3(KTR_IW_CXGBE,
220 "%s BAR2 DB/WC sq base pa 0x%llx rq base pa 0x%llx",
221 __func__, (unsigned long long)wq->sq.bar2_pa,
222 (unsigned long long)wq->rq.bar2_pa);
223 } else {
224 /* Compute BAR2 DB/WC virtual address to access in kernel. */
225 wq->sq.bar2_va = (void __iomem *)((u64)rdev->bar2_kva +
226 sq_bar2_qoffset);
227 wq->rq.bar2_va = (void __iomem *)((u64)rdev->bar2_kva +
228 rq_bar2_qoffset);
229 CTR3(KTR_IW_CXGBE, "%s BAR2 DB/WC sq base va %p rq base va %p",
230 __func__, (unsigned long long)wq->sq.bar2_va,
231 (unsigned long long)wq->rq.bar2_va);
232 }
233
234 wq->rdev = rdev;
235 wq->rq.msn = 1;
236
237 /* build fw_ri_res_wr */
238 wr_len = sizeof *res_wr + 2 * sizeof *res;
239
240 wr = alloc_wrqe(wr_len, &sc->sge.ctrlq[0]);
241 if (wr == NULL) {
242 ret = -ENOMEM;
243 goto free_rq_dma;
244 }
245 res_wr = wrtod(wr);
246
247 memset(res_wr, 0, wr_len);
248 res_wr->op_nres = cpu_to_be32(
249 V_FW_WR_OP(FW_RI_RES_WR) |
250 V_FW_RI_RES_WR_NRES(2) |
251 F_FW_WR_COMPL);
252 res_wr->len16_pkd = cpu_to_be32(DIV_ROUND_UP(wr_len, 16));
253 res_wr->cookie = (unsigned long) &wr_wait;
254 res = res_wr->res;
255 res->u.sqrq.restype = FW_RI_RES_TYPE_SQ;
256 res->u.sqrq.op = FW_RI_RES_OP_WRITE;
257
258 /* eqsize is the number of 64B entries plus the status page size. */
259 eqsize = wq->sq.size * T4_SQ_NUM_SLOTS +
260 rdev->hw_queue.t4_eq_status_entries;
261
262 res->u.sqrq.fetchszm_to_iqid = cpu_to_be32(
263 V_FW_RI_RES_WR_HOSTFCMODE(0) | /* no host cidx updates */
264 V_FW_RI_RES_WR_CPRIO(0) | /* don't keep in chip cache */
265 V_FW_RI_RES_WR_PCIECHN(0) | /* set by uP at ri_init time */
266 V_FW_RI_RES_WR_IQID(scq->cqid));
267 res->u.sqrq.dcaen_to_eqsize = cpu_to_be32(
268 V_FW_RI_RES_WR_DCAEN(0) |
269 V_FW_RI_RES_WR_DCACPU(0) |
270 V_FW_RI_RES_WR_FBMIN(chip_id(sc) <= CHELSIO_T5 ?
271 X_FETCHBURSTMIN_64B : X_FETCHBURSTMIN_64B_T6) |
272 V_FW_RI_RES_WR_FBMAX(3) |
273 V_FW_RI_RES_WR_CIDXFTHRESHO(0) |
274 V_FW_RI_RES_WR_CIDXFTHRESH(0) |
275 V_FW_RI_RES_WR_EQSIZE(eqsize));
276 res->u.sqrq.eqid = cpu_to_be32(wq->sq.qid);
277 res->u.sqrq.eqaddr = cpu_to_be64(wq->sq.dma_addr);
278 res++;
279 res->u.sqrq.restype = FW_RI_RES_TYPE_RQ;
280 res->u.sqrq.op = FW_RI_RES_OP_WRITE;
281
282 /* eqsize is the number of 64B entries plus the status page size. */
283 eqsize = wq->rq.size * T4_RQ_NUM_SLOTS +
284 rdev->hw_queue.t4_eq_status_entries;
285 res->u.sqrq.fetchszm_to_iqid = cpu_to_be32(
286 V_FW_RI_RES_WR_HOSTFCMODE(0) | /* no host cidx updates */
287 V_FW_RI_RES_WR_CPRIO(0) | /* don't keep in chip cache */
288 V_FW_RI_RES_WR_PCIECHN(0) | /* set by uP at ri_init time */
289 V_FW_RI_RES_WR_IQID(rcq->cqid));
290 res->u.sqrq.dcaen_to_eqsize = cpu_to_be32(
291 V_FW_RI_RES_WR_DCAEN(0) |
292 V_FW_RI_RES_WR_DCACPU(0) |
293 V_FW_RI_RES_WR_FBMIN(chip_id(sc) <= CHELSIO_T5 ?
294 X_FETCHBURSTMIN_64B : X_FETCHBURSTMIN_64B_T6) |
295 V_FW_RI_RES_WR_FBMAX(3) |
296 V_FW_RI_RES_WR_CIDXFTHRESHO(0) |
297 V_FW_RI_RES_WR_CIDXFTHRESH(0) |
298 V_FW_RI_RES_WR_EQSIZE(eqsize));
299 res->u.sqrq.eqid = cpu_to_be32(wq->rq.qid);
300 res->u.sqrq.eqaddr = cpu_to_be64(wq->rq.dma_addr);
301
302 c4iw_init_wr_wait(&wr_wait);
303
304 t4_wrq_tx(sc, wr);
305 ret = c4iw_wait_for_reply(rdev, &wr_wait, 0, wq->sq.qid,
306 NULL, __func__);
307 if (ret)
308 goto free_rq_dma;
309
310 CTR5(KTR_IW_CXGBE,
311 "%s sqid 0x%x rqid 0x%x kdb 0x%p squdb 0x%llx rqudb 0x%llx",
312 __func__, wq->sq.qid, wq->rq.qid,
313 (unsigned long long)wq->sq.bar2_va,
314 (unsigned long long)wq->rq.bar2_va);
315
316 return 0;
317 free_rq_dma:
318 dma_free_coherent(rhp->ibdev.dma_device,
319 wq->rq.memsize, wq->rq.queue,
320 dma_unmap_addr(&wq->rq, mapping));
321 free_sq_dma:
322 dma_free_coherent(rhp->ibdev.dma_device,
323 wq->sq.memsize, wq->sq.queue,
324 dma_unmap_addr(&wq->sq, mapping));
325 free_hwaddr:
326 c4iw_rqtpool_free(rdev, wq->rq.rqt_hwaddr, wq->rq.rqt_size);
327 free_sw_rq:
328 kfree(wq->rq.sw_rq);
329 free_sw_sq:
330 kfree(wq->sq.sw_sq);
331 free_rq_qid:
332 c4iw_put_qpid(rdev, wq->rq.qid, uctx);
333 free_sq_qid:
334 c4iw_put_qpid(rdev, wq->sq.qid, uctx);
335 return ret;
336 }
337
build_immd(struct t4_sq * sq,struct fw_ri_immd * immdp,const struct ib_send_wr * wr,int max,u32 * plenp)338 static int build_immd(struct t4_sq *sq, struct fw_ri_immd *immdp,
339 const struct ib_send_wr *wr, int max, u32 *plenp)
340 {
341 u8 *dstp, *srcp;
342 u32 plen = 0;
343 int i;
344 int rem, len;
345
346 dstp = (u8 *)immdp->data;
347 for (i = 0; i < wr->num_sge; i++) {
348 if ((plen + wr->sg_list[i].length) > max)
349 return -EMSGSIZE;
350 srcp = (u8 *)(unsigned long)wr->sg_list[i].addr;
351 plen += wr->sg_list[i].length;
352 rem = wr->sg_list[i].length;
353 while (rem) {
354 if (dstp == (u8 *)&sq->queue[sq->size])
355 dstp = (u8 *)sq->queue;
356 if (rem <= (u8 *)&sq->queue[sq->size] - dstp)
357 len = rem;
358 else
359 len = (u8 *)&sq->queue[sq->size] - dstp;
360 memcpy(dstp, srcp, len);
361 dstp += len;
362 srcp += len;
363 rem -= len;
364 }
365 }
366 len = roundup(plen + sizeof *immdp, 16) - (plen + sizeof *immdp);
367 if (len)
368 memset(dstp, 0, len);
369 immdp->op = FW_RI_DATA_IMMD;
370 immdp->r1 = 0;
371 immdp->r2 = 0;
372 immdp->immdlen = cpu_to_be32(plen);
373 *plenp = plen;
374 return 0;
375 }
376
build_isgl(__be64 * queue_start,__be64 * queue_end,struct fw_ri_isgl * isglp,struct ib_sge * sg_list,int num_sge,u32 * plenp)377 static int build_isgl(__be64 *queue_start, __be64 *queue_end,
378 struct fw_ri_isgl *isglp, struct ib_sge *sg_list,
379 int num_sge, u32 *plenp)
380
381 {
382 int i;
383 u32 plen = 0;
384 __be64 *flitp = (__be64 *)isglp->sge;
385
386 for (i = 0; i < num_sge; i++) {
387 if ((plen + sg_list[i].length) < plen)
388 return -EMSGSIZE;
389 plen += sg_list[i].length;
390 *flitp = cpu_to_be64(((u64)sg_list[i].lkey << 32) |
391 sg_list[i].length);
392 if (++flitp == queue_end)
393 flitp = queue_start;
394 *flitp = cpu_to_be64(sg_list[i].addr);
395 if (++flitp == queue_end)
396 flitp = queue_start;
397 }
398 *flitp = (__force __be64)0;
399 isglp->op = FW_RI_DATA_ISGL;
400 isglp->r1 = 0;
401 isglp->nsge = cpu_to_be16(num_sge);
402 isglp->r2 = 0;
403 if (plenp)
404 *plenp = plen;
405 return 0;
406 }
407
build_rdma_send(struct t4_sq * sq,union t4_wr * wqe,const struct ib_send_wr * wr,u8 * len16)408 static int build_rdma_send(struct t4_sq *sq, union t4_wr *wqe,
409 const struct ib_send_wr *wr, u8 *len16)
410 {
411 u32 plen;
412 int size;
413 int ret;
414
415 if (wr->num_sge > T4_MAX_SEND_SGE)
416 return -EINVAL;
417 switch (wr->opcode) {
418 case IB_WR_SEND:
419 if (wr->send_flags & IB_SEND_SOLICITED)
420 wqe->send.sendop_pkd = cpu_to_be32(
421 V_FW_RI_SEND_WR_SENDOP(FW_RI_SEND_WITH_SE));
422 else
423 wqe->send.sendop_pkd = cpu_to_be32(
424 V_FW_RI_SEND_WR_SENDOP(FW_RI_SEND));
425 wqe->send.stag_inv = 0;
426 break;
427 case IB_WR_SEND_WITH_INV:
428 if (wr->send_flags & IB_SEND_SOLICITED)
429 wqe->send.sendop_pkd = cpu_to_be32(
430 V_FW_RI_SEND_WR_SENDOP(FW_RI_SEND_WITH_SE_INV));
431 else
432 wqe->send.sendop_pkd = cpu_to_be32(
433 V_FW_RI_SEND_WR_SENDOP(FW_RI_SEND_WITH_INV));
434 wqe->send.stag_inv = cpu_to_be32(wr->ex.invalidate_rkey);
435 break;
436
437 default:
438 return -EINVAL;
439 }
440 wqe->send.r3 = 0;
441 wqe->send.r4 = 0;
442
443 plen = 0;
444 if (wr->num_sge) {
445 if (wr->send_flags & IB_SEND_INLINE) {
446 ret = build_immd(sq, wqe->send.u.immd_src, wr,
447 T4_MAX_SEND_INLINE, &plen);
448 if (ret)
449 return ret;
450 size = sizeof wqe->send + sizeof(struct fw_ri_immd) +
451 plen;
452 } else {
453 ret = build_isgl((__be64 *)sq->queue,
454 (__be64 *)&sq->queue[sq->size],
455 wqe->send.u.isgl_src,
456 wr->sg_list, wr->num_sge, &plen);
457 if (ret)
458 return ret;
459 size = sizeof wqe->send + sizeof(struct fw_ri_isgl) +
460 wr->num_sge * sizeof(struct fw_ri_sge);
461 }
462 } else {
463 wqe->send.u.immd_src[0].op = FW_RI_DATA_IMMD;
464 wqe->send.u.immd_src[0].r1 = 0;
465 wqe->send.u.immd_src[0].r2 = 0;
466 wqe->send.u.immd_src[0].immdlen = 0;
467 size = sizeof wqe->send + sizeof(struct fw_ri_immd);
468 plen = 0;
469 }
470 *len16 = DIV_ROUND_UP(size, 16);
471 wqe->send.plen = cpu_to_be32(plen);
472 return 0;
473 }
474
build_rdma_write(struct t4_sq * sq,union t4_wr * wqe,const struct ib_send_wr * wr,u8 * len16)475 static int build_rdma_write(struct t4_sq *sq, union t4_wr *wqe,
476 const struct ib_send_wr *wr, u8 *len16)
477 {
478 u32 plen;
479 int size;
480 int ret;
481
482 if (wr->num_sge > T4_MAX_SEND_SGE)
483 return -EINVAL;
484 wqe->write.immd_data = 0;
485 wqe->write.stag_sink = cpu_to_be32(rdma_wr(wr)->rkey);
486 wqe->write.to_sink = cpu_to_be64(rdma_wr(wr)->remote_addr);
487 if (wr->num_sge) {
488 if (wr->send_flags & IB_SEND_INLINE) {
489 ret = build_immd(sq, wqe->write.u.immd_src, wr,
490 T4_MAX_WRITE_INLINE, &plen);
491 if (ret)
492 return ret;
493 size = sizeof wqe->write + sizeof(struct fw_ri_immd) +
494 plen;
495 } else {
496 ret = build_isgl((__be64 *)sq->queue,
497 (__be64 *)&sq->queue[sq->size],
498 wqe->write.u.isgl_src,
499 wr->sg_list, wr->num_sge, &plen);
500 if (ret)
501 return ret;
502 size = sizeof wqe->write + sizeof(struct fw_ri_isgl) +
503 wr->num_sge * sizeof(struct fw_ri_sge);
504 }
505 } else {
506 wqe->write.u.immd_src[0].op = FW_RI_DATA_IMMD;
507 wqe->write.u.immd_src[0].r1 = 0;
508 wqe->write.u.immd_src[0].r2 = 0;
509 wqe->write.u.immd_src[0].immdlen = 0;
510 size = sizeof wqe->write + sizeof(struct fw_ri_immd);
511 plen = 0;
512 }
513 *len16 = DIV_ROUND_UP(size, 16);
514 wqe->write.plen = cpu_to_be32(plen);
515 return 0;
516 }
517
build_rdma_read(union t4_wr * wqe,const struct ib_send_wr * wr,u8 * len16)518 static int build_rdma_read(union t4_wr *wqe, const struct ib_send_wr *wr, u8 *len16)
519 {
520 if (wr->num_sge > 1)
521 return -EINVAL;
522 if (wr->num_sge && wr->sg_list[0].length) {
523 wqe->read.stag_src = cpu_to_be32(rdma_wr(wr)->rkey);
524 wqe->read.to_src_hi = cpu_to_be32((u32)(rdma_wr(wr)->remote_addr
525 >> 32));
526 wqe->read.to_src_lo =
527 cpu_to_be32((u32)rdma_wr(wr)->remote_addr);
528 wqe->read.stag_sink = cpu_to_be32(wr->sg_list[0].lkey);
529 wqe->read.plen = cpu_to_be32(wr->sg_list[0].length);
530 wqe->read.to_sink_hi = cpu_to_be32((u32)(wr->sg_list[0].addr
531 >> 32));
532 wqe->read.to_sink_lo = cpu_to_be32((u32)(wr->sg_list[0].addr));
533 } else {
534 wqe->read.stag_src = cpu_to_be32(2);
535 wqe->read.to_src_hi = 0;
536 wqe->read.to_src_lo = 0;
537 wqe->read.stag_sink = cpu_to_be32(2);
538 wqe->read.plen = 0;
539 wqe->read.to_sink_hi = 0;
540 wqe->read.to_sink_lo = 0;
541 }
542 wqe->read.r2 = 0;
543 wqe->read.r5 = 0;
544 *len16 = DIV_ROUND_UP(sizeof wqe->read, 16);
545 return 0;
546 }
547
build_rdma_recv(struct c4iw_qp * qhp,union t4_recv_wr * wqe,const struct ib_recv_wr * wr,u8 * len16)548 static int build_rdma_recv(struct c4iw_qp *qhp, union t4_recv_wr *wqe,
549 const struct ib_recv_wr *wr, u8 *len16)
550 {
551 int ret;
552
553 ret = build_isgl((__be64 *)qhp->wq.rq.queue,
554 (__be64 *)&qhp->wq.rq.queue[qhp->wq.rq.size],
555 &wqe->recv.isgl, wr->sg_list, wr->num_sge, NULL);
556 if (ret)
557 return ret;
558 *len16 = DIV_ROUND_UP(sizeof wqe->recv +
559 wr->num_sge * sizeof(struct fw_ri_sge), 16);
560 return 0;
561 }
562
build_inv_stag(union t4_wr * wqe,const struct ib_send_wr * wr,u8 * len16)563 static int build_inv_stag(union t4_wr *wqe, const struct ib_send_wr *wr,
564 u8 *len16)
565 {
566 wqe->inv.stag_inv = cpu_to_be32(wr->ex.invalidate_rkey);
567 wqe->inv.r2 = 0;
568 *len16 = DIV_ROUND_UP(sizeof wqe->inv, 16);
569 return 0;
570 }
571
free_qp_work(struct work_struct * work)572 static void free_qp_work(struct work_struct *work)
573 {
574 struct c4iw_ucontext *ucontext;
575 struct c4iw_qp *qhp;
576 struct c4iw_dev *rhp;
577
578 qhp = container_of(work, struct c4iw_qp, free_work);
579 ucontext = qhp->ucontext;
580 rhp = qhp->rhp;
581
582 CTR3(KTR_IW_CXGBE, "%s qhp %p ucontext %p", __func__,
583 qhp, ucontext);
584 destroy_qp(&rhp->rdev, &qhp->wq,
585 ucontext ? &ucontext->uctx : &rhp->rdev.uctx);
586
587 kfree(qhp);
588 }
589
queue_qp_free(struct kref * kref)590 static void queue_qp_free(struct kref *kref)
591 {
592 struct c4iw_qp *qhp;
593
594 qhp = container_of(kref, struct c4iw_qp, kref);
595 CTR2(KTR_IW_CXGBE, "%s qhp %p", __func__, qhp);
596 queue_work(qhp->rhp->rdev.free_workq, &qhp->free_work);
597 }
598
c4iw_qp_add_ref(struct ib_qp * qp)599 void c4iw_qp_add_ref(struct ib_qp *qp)
600 {
601 CTR2(KTR_IW_CXGBE, "%s ib_qp %p", __func__, qp);
602 kref_get(&to_c4iw_qp(qp)->kref);
603 }
604
c4iw_qp_rem_ref(struct ib_qp * qp)605 void c4iw_qp_rem_ref(struct ib_qp *qp)
606 {
607 CTR2(KTR_IW_CXGBE, "%s ib_qp %p", __func__, qp);
608 kref_put(&to_c4iw_qp(qp)->kref, queue_qp_free);
609 }
610
complete_sq_drain_wr(struct c4iw_qp * qhp,const struct ib_send_wr * wr)611 static void complete_sq_drain_wr(struct c4iw_qp *qhp, const struct ib_send_wr *wr)
612 {
613 struct t4_cqe cqe = {};
614 struct c4iw_cq *schp;
615 unsigned long flag;
616 struct t4_cq *cq;
617
618 schp = to_c4iw_cq(qhp->ibqp.send_cq);
619 cq = &schp->cq;
620
621 PDBG("%s drain sq id %u\n", __func__, qhp->wq.sq.qid);
622 cqe.u.drain_cookie = wr->wr_id;
623 cqe.header = cpu_to_be32(V_CQE_STATUS(T4_ERR_SWFLUSH) |
624 V_CQE_OPCODE(C4IW_DRAIN_OPCODE) |
625 V_CQE_TYPE(1) |
626 V_CQE_SWCQE(1) |
627 V_CQE_QPID(qhp->wq.sq.qid));
628
629 spin_lock_irqsave(&schp->lock, flag);
630 cqe.bits_type_ts = cpu_to_be64(V_CQE_GENBIT((u64)cq->gen));
631 cq->sw_queue[cq->sw_pidx] = cqe;
632 t4_swcq_produce(cq);
633 spin_unlock_irqrestore(&schp->lock, flag);
634
635 spin_lock_irqsave(&schp->comp_handler_lock, flag);
636 (*schp->ibcq.comp_handler)(&schp->ibcq,
637 schp->ibcq.cq_context);
638 spin_unlock_irqrestore(&schp->comp_handler_lock, flag);
639 }
640
complete_rq_drain_wr(struct c4iw_qp * qhp,const struct ib_recv_wr * wr)641 static void complete_rq_drain_wr(struct c4iw_qp *qhp, const struct ib_recv_wr *wr)
642 {
643 struct t4_cqe cqe = {};
644 struct c4iw_cq *rchp;
645 unsigned long flag;
646 struct t4_cq *cq;
647
648 rchp = to_c4iw_cq(qhp->ibqp.recv_cq);
649 cq = &rchp->cq;
650
651 PDBG("%s drain rq id %u\n", __func__, qhp->wq.sq.qid);
652 cqe.u.drain_cookie = wr->wr_id;
653 cqe.header = cpu_to_be32(V_CQE_STATUS(T4_ERR_SWFLUSH) |
654 V_CQE_OPCODE(C4IW_DRAIN_OPCODE) |
655 V_CQE_TYPE(0) |
656 V_CQE_SWCQE(1) |
657 V_CQE_QPID(qhp->wq.sq.qid));
658
659 spin_lock_irqsave(&rchp->lock, flag);
660 cqe.bits_type_ts = cpu_to_be64(V_CQE_GENBIT((u64)cq->gen));
661 cq->sw_queue[cq->sw_pidx] = cqe;
662 t4_swcq_produce(cq);
663 spin_unlock_irqrestore(&rchp->lock, flag);
664
665 spin_lock_irqsave(&rchp->comp_handler_lock, flag);
666 (*rchp->ibcq.comp_handler)(&rchp->ibcq,
667 rchp->ibcq.cq_context);
668 spin_unlock_irqrestore(&rchp->comp_handler_lock, flag);
669 }
670
build_tpte_memreg(struct fw_ri_fr_nsmr_tpte_wr * fr,const struct ib_reg_wr * wr,struct c4iw_mr * mhp,u8 * len16)671 static int build_tpte_memreg(struct fw_ri_fr_nsmr_tpte_wr *fr,
672 const struct ib_reg_wr *wr, struct c4iw_mr *mhp, u8 *len16)
673 {
674 __be64 *p = (__be64 *)fr->pbl;
675
676 if (wr->mr->page_size > C4IW_MAX_PAGE_SIZE)
677 return -EINVAL;
678
679 fr->r2 = cpu_to_be32(0);
680 fr->stag = cpu_to_be32(mhp->ibmr.rkey);
681
682 fr->tpte.valid_to_pdid = cpu_to_be32(F_FW_RI_TPTE_VALID |
683 V_FW_RI_TPTE_STAGKEY((mhp->ibmr.rkey & M_FW_RI_TPTE_STAGKEY)) |
684 V_FW_RI_TPTE_STAGSTATE(1) |
685 V_FW_RI_TPTE_STAGTYPE(FW_RI_STAG_NSMR) |
686 V_FW_RI_TPTE_PDID(mhp->attr.pdid));
687 fr->tpte.locread_to_qpid = cpu_to_be32(
688 V_FW_RI_TPTE_PERM(c4iw_ib_to_tpt_access(wr->access)) |
689 V_FW_RI_TPTE_ADDRTYPE(FW_RI_VA_BASED_TO) |
690 V_FW_RI_TPTE_PS(ilog2(wr->mr->page_size) - 12));
691 fr->tpte.nosnoop_pbladdr = cpu_to_be32(V_FW_RI_TPTE_PBLADDR(
692 PBL_OFF(&mhp->rhp->rdev, mhp->attr.pbl_addr)>>3));
693 fr->tpte.dca_mwbcnt_pstag = cpu_to_be32(0);
694 fr->tpte.len_hi = cpu_to_be32(mhp->ibmr.length >> 32);
695 fr->tpte.len_lo = cpu_to_be32(mhp->ibmr.length & 0xffffffff);
696 fr->tpte.va_hi = cpu_to_be32(mhp->ibmr.iova >> 32);
697 fr->tpte.va_lo_fbo = cpu_to_be32(mhp->ibmr.iova & 0xffffffff);
698
699 p[0] = cpu_to_be64((u64)mhp->mpl[0]);
700 p[1] = cpu_to_be64((u64)mhp->mpl[1]);
701
702 *len16 = DIV_ROUND_UP(sizeof(*fr), 16);
703 return 0;
704 }
705
build_memreg(struct t4_sq * sq,union t4_wr * wqe,const struct ib_reg_wr * wr,struct c4iw_mr * mhp,u8 * len16,bool dsgl_supported)706 static int build_memreg(struct t4_sq *sq, union t4_wr *wqe,
707 const struct ib_reg_wr *wr, struct c4iw_mr *mhp, u8 *len16,
708 bool dsgl_supported)
709 {
710 struct fw_ri_immd *imdp;
711 __be64 *p;
712 int i;
713 int pbllen = roundup(mhp->mpl_len * sizeof(u64), 32);
714 int rem;
715
716 if (mhp->mpl_len > t4_max_fr_depth(&mhp->rhp->rdev, use_dsgl))
717 return -EINVAL;
718 if (wr->mr->page_size > C4IW_MAX_PAGE_SIZE)
719 return -EINVAL;
720
721 wqe->fr.qpbinde_to_dcacpu = 0;
722 wqe->fr.pgsz_shift = ilog2(wr->mr->page_size) - 12;
723 wqe->fr.addr_type = FW_RI_VA_BASED_TO;
724 wqe->fr.mem_perms = c4iw_ib_to_tpt_access(wr->access);
725 wqe->fr.len_hi = cpu_to_be32(mhp->ibmr.length >> 32);
726 wqe->fr.len_lo = cpu_to_be32(mhp->ibmr.length & 0xffffffff);
727 wqe->fr.stag = cpu_to_be32(wr->key);
728 wqe->fr.va_hi = cpu_to_be32(mhp->ibmr.iova >> 32);
729 wqe->fr.va_lo_fbo = cpu_to_be32(mhp->ibmr.iova & 0xffffffff);
730
731 if (dsgl_supported && use_dsgl && (pbllen > max_fr_immd)) {
732 struct fw_ri_dsgl *sglp;
733
734 for (i = 0; i < mhp->mpl_len; i++)
735 mhp->mpl[i] =
736 (__force u64)cpu_to_be64((u64)mhp->mpl[i]);
737
738 sglp = (struct fw_ri_dsgl *)(&wqe->fr + 1);
739 sglp->op = FW_RI_DATA_DSGL;
740 sglp->r1 = 0;
741 sglp->nsge = cpu_to_be16(1);
742 sglp->addr0 = cpu_to_be64(mhp->mpl_addr);
743 sglp->len0 = cpu_to_be32(pbllen);
744
745 *len16 = DIV_ROUND_UP(sizeof(wqe->fr) + sizeof(*sglp), 16);
746 } else {
747 imdp = (struct fw_ri_immd *)(&wqe->fr + 1);
748 imdp->op = FW_RI_DATA_IMMD;
749 imdp->r1 = 0;
750 imdp->r2 = 0;
751 imdp->immdlen = cpu_to_be32(pbllen);
752 p = (__be64 *)(imdp + 1);
753 rem = pbllen;
754 for (i = 0; i < mhp->mpl_len; i++) {
755 *p = cpu_to_be64((u64)mhp->mpl[i]);
756 rem -= sizeof(*p);
757 if (++p == (__be64 *)&sq->queue[sq->size])
758 p = (__be64 *)sq->queue;
759 }
760 BUG_ON(rem < 0);
761 while (rem) {
762 *p = 0;
763 rem -= sizeof(*p);
764 if (++p == (__be64 *)&sq->queue[sq->size])
765 p = (__be64 *)sq->queue;
766 }
767 *len16 = DIV_ROUND_UP(sizeof(wqe->fr) + sizeof(*imdp)
768 + pbllen, 16);
769 }
770
771 return 0;
772 }
773
c4iw_post_send(struct ib_qp * ibqp,const struct ib_send_wr * wr,const struct ib_send_wr ** bad_wr)774 int c4iw_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
775 const struct ib_send_wr **bad_wr)
776 {
777 int err = 0;
778 u8 len16 = 0;
779 enum fw_wr_opcodes fw_opcode = 0;
780 enum fw_ri_wr_flags fw_flags;
781 struct c4iw_qp *qhp;
782 union t4_wr *wqe = NULL;
783 u32 num_wrs;
784 struct t4_swsqe *swsqe;
785 unsigned long flag;
786 u16 idx = 0;
787 struct c4iw_rdev *rdev;
788
789 qhp = to_c4iw_qp(ibqp);
790 rdev = &qhp->rhp->rdev;
791 if (__predict_false(c4iw_stopped(rdev)))
792 return -EIO;
793 spin_lock_irqsave(&qhp->lock, flag);
794 if (t4_wq_in_error(&qhp->wq)) {
795 spin_unlock_irqrestore(&qhp->lock, flag);
796 complete_sq_drain_wr(qhp, wr);
797 return err;
798 }
799 num_wrs = t4_sq_avail(&qhp->wq);
800 if (num_wrs == 0) {
801 spin_unlock_irqrestore(&qhp->lock, flag);
802 *bad_wr = wr;
803 return -ENOMEM;
804 }
805 while (wr) {
806 if (num_wrs == 0) {
807 err = -ENOMEM;
808 *bad_wr = wr;
809 break;
810 }
811 wqe = (union t4_wr *)((u8 *)qhp->wq.sq.queue +
812 qhp->wq.sq.wq_pidx * T4_EQ_ENTRY_SIZE);
813
814 fw_flags = 0;
815 if (wr->send_flags & IB_SEND_SOLICITED)
816 fw_flags |= FW_RI_SOLICITED_EVENT_FLAG;
817 if (wr->send_flags & IB_SEND_SIGNALED || qhp->sq_sig_all)
818 fw_flags |= FW_RI_COMPLETION_FLAG;
819 swsqe = &qhp->wq.sq.sw_sq[qhp->wq.sq.pidx];
820 switch (wr->opcode) {
821 case IB_WR_SEND_WITH_INV:
822 case IB_WR_SEND:
823 if (wr->send_flags & IB_SEND_FENCE)
824 fw_flags |= FW_RI_READ_FENCE_FLAG;
825 fw_opcode = FW_RI_SEND_WR;
826 if (wr->opcode == IB_WR_SEND)
827 swsqe->opcode = FW_RI_SEND;
828 else
829 swsqe->opcode = FW_RI_SEND_WITH_INV;
830 err = build_rdma_send(&qhp->wq.sq, wqe, wr, &len16);
831 break;
832 case IB_WR_RDMA_WRITE:
833 fw_opcode = FW_RI_RDMA_WRITE_WR;
834 swsqe->opcode = FW_RI_RDMA_WRITE;
835 err = build_rdma_write(&qhp->wq.sq, wqe, wr, &len16);
836 break;
837 case IB_WR_RDMA_READ:
838 case IB_WR_RDMA_READ_WITH_INV:
839 fw_opcode = FW_RI_RDMA_READ_WR;
840 swsqe->opcode = FW_RI_READ_REQ;
841 if (wr->opcode == IB_WR_RDMA_READ_WITH_INV) {
842 c4iw_invalidate_mr(qhp->rhp,
843 wr->sg_list[0].lkey);
844 fw_flags = FW_RI_RDMA_READ_INVALIDATE;
845 } else {
846 fw_flags = 0;
847 }
848 err = build_rdma_read(wqe, wr, &len16);
849 if (err)
850 break;
851 swsqe->read_len = wr->sg_list[0].length;
852 if (!qhp->wq.sq.oldest_read)
853 qhp->wq.sq.oldest_read = swsqe;
854 break;
855 case IB_WR_REG_MR: {
856 struct c4iw_mr *mhp = to_c4iw_mr(reg_wr(wr)->mr);
857
858 swsqe->opcode = FW_RI_FAST_REGISTER;
859 if (rdev->adap->params.fr_nsmr_tpte_wr_support &&
860 !mhp->attr.state && mhp->mpl_len <= 2) {
861 fw_opcode = FW_RI_FR_NSMR_TPTE_WR;
862 err = build_tpte_memreg(&wqe->fr_tpte, reg_wr(wr),
863 mhp, &len16);
864 } else {
865 fw_opcode = FW_RI_FR_NSMR_WR;
866 err = build_memreg(&qhp->wq.sq, wqe, reg_wr(wr),
867 mhp, &len16,
868 rdev->adap->params.ulptx_memwrite_dsgl);
869 }
870 if (err)
871 break;
872 mhp->attr.state = 1;
873 break;
874 }
875 case IB_WR_LOCAL_INV:
876 if (wr->send_flags & IB_SEND_FENCE)
877 fw_flags |= FW_RI_LOCAL_FENCE_FLAG;
878 fw_opcode = FW_RI_INV_LSTAG_WR;
879 swsqe->opcode = FW_RI_LOCAL_INV;
880 err = build_inv_stag(wqe, wr, &len16);
881 c4iw_invalidate_mr(qhp->rhp, wr->ex.invalidate_rkey);
882 break;
883 default:
884 CTR2(KTR_IW_CXGBE, "%s post of type =%d TBD!", __func__,
885 wr->opcode);
886 err = -EINVAL;
887 }
888 if (err) {
889 *bad_wr = wr;
890 break;
891 }
892 swsqe->idx = qhp->wq.sq.pidx;
893 swsqe->complete = 0;
894 swsqe->signaled = (wr->send_flags & IB_SEND_SIGNALED) ||
895 qhp->sq_sig_all;
896 swsqe->flushed = 0;
897 swsqe->wr_id = wr->wr_id;
898
899 init_wr_hdr(wqe, qhp->wq.sq.pidx, fw_opcode, fw_flags, len16);
900
901 CTR5(KTR_IW_CXGBE,
902 "%s cookie 0x%llx pidx 0x%x opcode 0x%x read_len %u",
903 __func__, (unsigned long long)wr->wr_id, qhp->wq.sq.pidx,
904 swsqe->opcode, swsqe->read_len);
905 wr = wr->next;
906 num_wrs--;
907 t4_sq_produce(&qhp->wq, len16);
908 idx += DIV_ROUND_UP(len16*16, T4_EQ_ENTRY_SIZE);
909 }
910
911 t4_ring_sq_db(&qhp->wq, idx, wqe, rdev->adap->iwt.wc_en);
912 spin_unlock_irqrestore(&qhp->lock, flag);
913 return err;
914 }
915
c4iw_post_receive(struct ib_qp * ibqp,const struct ib_recv_wr * wr,const struct ib_recv_wr ** bad_wr)916 int c4iw_post_receive(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
917 const struct ib_recv_wr **bad_wr)
918 {
919 int err = 0;
920 struct c4iw_qp *qhp;
921 union t4_recv_wr *wqe = NULL;
922 u32 num_wrs;
923 u8 len16 = 0;
924 unsigned long flag;
925 u16 idx = 0;
926
927 qhp = to_c4iw_qp(ibqp);
928 if (__predict_false(c4iw_stopped(&qhp->rhp->rdev)))
929 return -EIO;
930 spin_lock_irqsave(&qhp->lock, flag);
931 if (t4_wq_in_error(&qhp->wq)) {
932 spin_unlock_irqrestore(&qhp->lock, flag);
933 complete_rq_drain_wr(qhp, wr);
934 return err;
935 }
936 num_wrs = t4_rq_avail(&qhp->wq);
937 if (num_wrs == 0) {
938 spin_unlock_irqrestore(&qhp->lock, flag);
939 *bad_wr = wr;
940 return -ENOMEM;
941 }
942 while (wr) {
943 if (wr->num_sge > T4_MAX_RECV_SGE) {
944 err = -EINVAL;
945 *bad_wr = wr;
946 break;
947 }
948 wqe = (union t4_recv_wr *)((u8 *)qhp->wq.rq.queue +
949 qhp->wq.rq.wq_pidx *
950 T4_EQ_ENTRY_SIZE);
951 if (num_wrs)
952 err = build_rdma_recv(qhp, wqe, wr, &len16);
953 else
954 err = -ENOMEM;
955 if (err) {
956 *bad_wr = wr;
957 break;
958 }
959
960 qhp->wq.rq.sw_rq[qhp->wq.rq.pidx].wr_id = wr->wr_id;
961
962 wqe->recv.opcode = FW_RI_RECV_WR;
963 wqe->recv.r1 = 0;
964 wqe->recv.wrid = qhp->wq.rq.pidx;
965 wqe->recv.r2[0] = 0;
966 wqe->recv.r2[1] = 0;
967 wqe->recv.r2[2] = 0;
968 wqe->recv.len16 = len16;
969 CTR3(KTR_IW_CXGBE, "%s cookie 0x%llx pidx %u", __func__,
970 (unsigned long long) wr->wr_id, qhp->wq.rq.pidx);
971 t4_rq_produce(&qhp->wq, len16);
972 idx += DIV_ROUND_UP(len16*16, T4_EQ_ENTRY_SIZE);
973 wr = wr->next;
974 num_wrs--;
975 }
976
977 t4_ring_rq_db(&qhp->wq, idx, wqe, qhp->rhp->rdev.adap->iwt.wc_en);
978 spin_unlock_irqrestore(&qhp->lock, flag);
979 return err;
980 }
981
build_term_codes(struct t4_cqe * err_cqe,u8 * layer_type,u8 * ecode)982 static inline void build_term_codes(struct t4_cqe *err_cqe, u8 *layer_type,
983 u8 *ecode)
984 {
985 int status;
986 int tagged;
987 int opcode;
988 int rqtype;
989 int send_inv;
990
991 if (!err_cqe) {
992 *layer_type = LAYER_RDMAP|DDP_LOCAL_CATA;
993 *ecode = 0;
994 return;
995 }
996
997 status = CQE_STATUS(err_cqe);
998 opcode = CQE_OPCODE(err_cqe);
999 rqtype = RQ_TYPE(err_cqe);
1000 send_inv = (opcode == FW_RI_SEND_WITH_INV) ||
1001 (opcode == FW_RI_SEND_WITH_SE_INV);
1002 tagged = (opcode == FW_RI_RDMA_WRITE) ||
1003 (rqtype && (opcode == FW_RI_READ_RESP));
1004
1005 switch (status) {
1006 case T4_ERR_STAG:
1007 if (send_inv) {
1008 *layer_type = LAYER_RDMAP|RDMAP_REMOTE_OP;
1009 *ecode = RDMAP_CANT_INV_STAG;
1010 } else {
1011 *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT;
1012 *ecode = RDMAP_INV_STAG;
1013 }
1014 break;
1015 case T4_ERR_PDID:
1016 *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT;
1017 if ((opcode == FW_RI_SEND_WITH_INV) ||
1018 (opcode == FW_RI_SEND_WITH_SE_INV))
1019 *ecode = RDMAP_CANT_INV_STAG;
1020 else
1021 *ecode = RDMAP_STAG_NOT_ASSOC;
1022 break;
1023 case T4_ERR_QPID:
1024 *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT;
1025 *ecode = RDMAP_STAG_NOT_ASSOC;
1026 break;
1027 case T4_ERR_ACCESS:
1028 *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT;
1029 *ecode = RDMAP_ACC_VIOL;
1030 break;
1031 case T4_ERR_WRAP:
1032 *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT;
1033 *ecode = RDMAP_TO_WRAP;
1034 break;
1035 case T4_ERR_BOUND:
1036 if (tagged) {
1037 *layer_type = LAYER_DDP|DDP_TAGGED_ERR;
1038 *ecode = DDPT_BASE_BOUNDS;
1039 } else {
1040 *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT;
1041 *ecode = RDMAP_BASE_BOUNDS;
1042 }
1043 break;
1044 case T4_ERR_INVALIDATE_SHARED_MR:
1045 case T4_ERR_INVALIDATE_MR_WITH_MW_BOUND:
1046 *layer_type = LAYER_RDMAP|RDMAP_REMOTE_OP;
1047 *ecode = RDMAP_CANT_INV_STAG;
1048 break;
1049 case T4_ERR_ECC:
1050 case T4_ERR_ECC_PSTAG:
1051 case T4_ERR_INTERNAL_ERR:
1052 *layer_type = LAYER_RDMAP|RDMAP_LOCAL_CATA;
1053 *ecode = 0;
1054 break;
1055 case T4_ERR_OUT_OF_RQE:
1056 *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR;
1057 *ecode = DDPU_INV_MSN_NOBUF;
1058 break;
1059 case T4_ERR_PBL_ADDR_BOUND:
1060 *layer_type = LAYER_DDP|DDP_TAGGED_ERR;
1061 *ecode = DDPT_BASE_BOUNDS;
1062 break;
1063 case T4_ERR_CRC:
1064 *layer_type = LAYER_MPA|DDP_LLP;
1065 *ecode = MPA_CRC_ERR;
1066 break;
1067 case T4_ERR_MARKER:
1068 *layer_type = LAYER_MPA|DDP_LLP;
1069 *ecode = MPA_MARKER_ERR;
1070 break;
1071 case T4_ERR_PDU_LEN_ERR:
1072 *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR;
1073 *ecode = DDPU_MSG_TOOBIG;
1074 break;
1075 case T4_ERR_DDP_VERSION:
1076 if (tagged) {
1077 *layer_type = LAYER_DDP|DDP_TAGGED_ERR;
1078 *ecode = DDPT_INV_VERS;
1079 } else {
1080 *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR;
1081 *ecode = DDPU_INV_VERS;
1082 }
1083 break;
1084 case T4_ERR_RDMA_VERSION:
1085 *layer_type = LAYER_RDMAP|RDMAP_REMOTE_OP;
1086 *ecode = RDMAP_INV_VERS;
1087 break;
1088 case T4_ERR_OPCODE:
1089 *layer_type = LAYER_RDMAP|RDMAP_REMOTE_OP;
1090 *ecode = RDMAP_INV_OPCODE;
1091 break;
1092 case T4_ERR_DDP_QUEUE_NUM:
1093 *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR;
1094 *ecode = DDPU_INV_QN;
1095 break;
1096 case T4_ERR_MSN:
1097 case T4_ERR_MSN_GAP:
1098 case T4_ERR_MSN_RANGE:
1099 case T4_ERR_IRD_OVERFLOW:
1100 *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR;
1101 *ecode = DDPU_INV_MSN_RANGE;
1102 break;
1103 case T4_ERR_TBIT:
1104 *layer_type = LAYER_DDP|DDP_LOCAL_CATA;
1105 *ecode = 0;
1106 break;
1107 case T4_ERR_MO:
1108 *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR;
1109 *ecode = DDPU_INV_MO;
1110 break;
1111 default:
1112 *layer_type = LAYER_RDMAP|DDP_LOCAL_CATA;
1113 *ecode = 0;
1114 break;
1115 }
1116 }
1117
post_terminate(struct c4iw_qp * qhp,struct t4_cqe * err_cqe,gfp_t gfp)1118 static void post_terminate(struct c4iw_qp *qhp, struct t4_cqe *err_cqe,
1119 gfp_t gfp)
1120 {
1121 int ret;
1122 struct fw_ri_wr *wqe;
1123 struct terminate_message *term;
1124 struct wrqe *wr;
1125 struct socket *so = qhp->ep->com.so;
1126 struct inpcb *inp = sotoinpcb(so);
1127 struct tcpcb *tp = intotcpcb(inp);
1128 struct toepcb *toep = tp->t_toe;
1129
1130 CTR4(KTR_IW_CXGBE, "%s qhp %p qid 0x%x tid %u", __func__, qhp,
1131 qhp->wq.sq.qid, qhp->ep->hwtid);
1132
1133 wr = alloc_wrqe(sizeof(*wqe), &toep->ofld_txq->wrq);
1134 if (wr == NULL)
1135 return;
1136 wqe = wrtod(wr);
1137
1138 memset(wqe, 0, sizeof *wqe);
1139 wqe->op_compl = cpu_to_be32(V_FW_WR_OP(FW_RI_WR));
1140 wqe->flowid_len16 = cpu_to_be32(
1141 V_FW_WR_FLOWID(qhp->ep->hwtid) |
1142 V_FW_WR_LEN16(DIV_ROUND_UP(sizeof *wqe, 16)));
1143
1144 wqe->u.terminate.type = FW_RI_TYPE_TERMINATE;
1145 wqe->u.terminate.immdlen = cpu_to_be32(sizeof *term);
1146 term = (struct terminate_message *)wqe->u.terminate.termmsg;
1147 if (qhp->attr.layer_etype == (LAYER_MPA|DDP_LLP)) {
1148 term->layer_etype = qhp->attr.layer_etype;
1149 term->ecode = qhp->attr.ecode;
1150 } else
1151 build_term_codes(err_cqe, &term->layer_etype, &term->ecode);
1152 ret = creds(toep, inp, sizeof(*wqe));
1153 if (ret) {
1154 free_wrqe(wr);
1155 return;
1156 }
1157 t4_wrq_tx(qhp->rhp->rdev.adap, wr);
1158 }
1159
1160 /* Assumes qhp lock is held. */
__flush_qp(struct c4iw_qp * qhp,struct c4iw_cq * rchp,struct c4iw_cq * schp)1161 static void __flush_qp(struct c4iw_qp *qhp, struct c4iw_cq *rchp,
1162 struct c4iw_cq *schp)
1163 {
1164 int count;
1165 int rq_flushed, sq_flushed;
1166 unsigned long flag;
1167
1168 CTR4(KTR_IW_CXGBE, "%s qhp %p rchp %p schp %p", __func__, qhp, rchp,
1169 schp);
1170
1171 /* locking hierarchy: cq lock first, then qp lock. */
1172 spin_lock_irqsave(&rchp->lock, flag);
1173 spin_lock(&qhp->lock);
1174
1175 if (qhp->wq.flushed) {
1176 spin_unlock(&qhp->lock);
1177 spin_unlock_irqrestore(&rchp->lock, flag);
1178 return;
1179 }
1180 qhp->wq.flushed = 1;
1181
1182 c4iw_flush_hw_cq(rchp);
1183 c4iw_count_rcqes(&rchp->cq, &qhp->wq, &count);
1184 rq_flushed = c4iw_flush_rq(&qhp->wq, &rchp->cq, count);
1185 spin_unlock(&qhp->lock);
1186 spin_unlock_irqrestore(&rchp->lock, flag);
1187
1188 /* locking hierarchy: cq lock first, then qp lock. */
1189 spin_lock_irqsave(&schp->lock, flag);
1190 spin_lock(&qhp->lock);
1191 if (schp != rchp)
1192 c4iw_flush_hw_cq(schp);
1193 sq_flushed = c4iw_flush_sq(qhp);
1194 spin_unlock(&qhp->lock);
1195 spin_unlock_irqrestore(&schp->lock, flag);
1196
1197 if (schp == rchp) {
1198 if (t4_clear_cq_armed(&rchp->cq) &&
1199 (rq_flushed || sq_flushed)) {
1200 spin_lock_irqsave(&rchp->comp_handler_lock, flag);
1201 (*rchp->ibcq.comp_handler)(&rchp->ibcq,
1202 rchp->ibcq.cq_context);
1203 spin_unlock_irqrestore(&rchp->comp_handler_lock, flag);
1204 }
1205 } else {
1206 if (t4_clear_cq_armed(&rchp->cq) && rq_flushed) {
1207 spin_lock_irqsave(&rchp->comp_handler_lock, flag);
1208 (*rchp->ibcq.comp_handler)(&rchp->ibcq,
1209 rchp->ibcq.cq_context);
1210 spin_unlock_irqrestore(&rchp->comp_handler_lock, flag);
1211 }
1212 if (t4_clear_cq_armed(&schp->cq) && sq_flushed) {
1213 spin_lock_irqsave(&schp->comp_handler_lock, flag);
1214 (*schp->ibcq.comp_handler)(&schp->ibcq,
1215 schp->ibcq.cq_context);
1216 spin_unlock_irqrestore(&schp->comp_handler_lock, flag);
1217 }
1218 }
1219 }
1220
flush_qp(struct c4iw_qp * qhp)1221 static void flush_qp(struct c4iw_qp *qhp)
1222 {
1223 struct c4iw_cq *rchp, *schp;
1224 unsigned long flag;
1225
1226 rchp = to_c4iw_cq(qhp->ibqp.recv_cq);
1227 schp = to_c4iw_cq(qhp->ibqp.send_cq);
1228
1229 t4_set_wq_in_error(&qhp->wq);
1230 if (qhp->ibqp.uobject) {
1231 t4_set_cq_in_error(&rchp->cq);
1232 spin_lock_irqsave(&rchp->comp_handler_lock, flag);
1233 (*rchp->ibcq.comp_handler)(&rchp->ibcq, rchp->ibcq.cq_context);
1234 spin_unlock_irqrestore(&rchp->comp_handler_lock, flag);
1235 if (schp != rchp) {
1236 t4_set_cq_in_error(&schp->cq);
1237 spin_lock_irqsave(&schp->comp_handler_lock, flag);
1238 (*schp->ibcq.comp_handler)(&schp->ibcq,
1239 schp->ibcq.cq_context);
1240 spin_unlock_irqrestore(&schp->comp_handler_lock, flag);
1241 }
1242 return;
1243 }
1244 __flush_qp(qhp, rchp, schp);
1245 }
1246
1247 static int
rdma_fini(struct c4iw_dev * rhp,struct c4iw_qp * qhp,struct c4iw_ep * ep)1248 rdma_fini(struct c4iw_dev *rhp, struct c4iw_qp *qhp, struct c4iw_ep *ep)
1249 {
1250 struct c4iw_rdev *rdev = &rhp->rdev;
1251 struct adapter *sc = rdev->adap;
1252 struct fw_ri_wr *wqe;
1253 int ret;
1254 struct wrqe *wr;
1255 struct socket *so = ep->com.so;
1256 struct inpcb *inp = sotoinpcb(so);
1257 struct tcpcb *tp = intotcpcb(inp);
1258 struct toepcb *toep = tp->t_toe;
1259
1260 KASSERT(rhp == qhp->rhp && ep == qhp->ep, ("%s: EDOOFUS", __func__));
1261
1262 CTR5(KTR_IW_CXGBE, "%s qhp %p qid 0x%x ep %p tid %u", __func__, qhp,
1263 qhp->wq.sq.qid, ep, ep->hwtid);
1264
1265 wr = alloc_wrqe(sizeof(*wqe), &toep->ofld_txq->wrq);
1266 if (wr == NULL)
1267 return (0);
1268 wqe = wrtod(wr);
1269
1270 memset(wqe, 0, sizeof *wqe);
1271
1272 wqe->op_compl = cpu_to_be32(V_FW_WR_OP(FW_RI_WR) | F_FW_WR_COMPL);
1273 wqe->flowid_len16 = cpu_to_be32(V_FW_WR_FLOWID(ep->hwtid) |
1274 V_FW_WR_LEN16(DIV_ROUND_UP(sizeof *wqe, 16)));
1275 wqe->cookie = (unsigned long) &ep->com.wr_wait;
1276 wqe->u.fini.type = FW_RI_TYPE_FINI;
1277
1278 c4iw_init_wr_wait(&ep->com.wr_wait);
1279
1280 ret = creds(toep, inp, sizeof(*wqe));
1281 if (ret) {
1282 free_wrqe(wr);
1283 return ret;
1284 }
1285 t4_wrq_tx(sc, wr);
1286
1287 ret = c4iw_wait_for_reply(rdev, &ep->com.wr_wait, ep->hwtid,
1288 qhp->wq.sq.qid, ep->com.so, __func__);
1289 return ret;
1290 }
1291
build_rtr_msg(u8 p2p_type,struct fw_ri_init * init)1292 static void build_rtr_msg(u8 p2p_type, struct fw_ri_init *init)
1293 {
1294 CTR2(KTR_IW_CXGBE, "%s p2p_type = %d", __func__, p2p_type);
1295 memset(&init->u, 0, sizeof init->u);
1296 switch (p2p_type) {
1297 case FW_RI_INIT_P2PTYPE_RDMA_WRITE:
1298 init->u.write.opcode = FW_RI_RDMA_WRITE_WR;
1299 init->u.write.stag_sink = cpu_to_be32(1);
1300 init->u.write.to_sink = cpu_to_be64(1);
1301 init->u.write.u.immd_src[0].op = FW_RI_DATA_IMMD;
1302 init->u.write.len16 = DIV_ROUND_UP(sizeof init->u.write +
1303 sizeof(struct fw_ri_immd),
1304 16);
1305 break;
1306 case FW_RI_INIT_P2PTYPE_READ_REQ:
1307 init->u.write.opcode = FW_RI_RDMA_READ_WR;
1308 init->u.read.stag_src = cpu_to_be32(1);
1309 init->u.read.to_src_lo = cpu_to_be32(1);
1310 init->u.read.stag_sink = cpu_to_be32(1);
1311 init->u.read.to_sink_lo = cpu_to_be32(1);
1312 init->u.read.len16 = DIV_ROUND_UP(sizeof init->u.read, 16);
1313 break;
1314 }
1315 }
1316
1317 static int
creds(struct toepcb * toep,struct inpcb * inp,size_t wrsize)1318 creds(struct toepcb *toep, struct inpcb *inp, size_t wrsize)
1319 {
1320 struct ofld_tx_sdesc *txsd;
1321
1322 CTR3(KTR_IW_CXGBE, "%s:creB %p %u", __func__, toep , wrsize);
1323 INP_WLOCK(inp);
1324 if ((inp->inp_flags & INP_DROPPED) != 0) {
1325 INP_WUNLOCK(inp);
1326 return (EINVAL);
1327 }
1328 txsd = &toep->txsd[toep->txsd_pidx];
1329 txsd->tx_credits = howmany(wrsize, 16);
1330 txsd->plen = 0;
1331 KASSERT(toep->tx_credits >= txsd->tx_credits && toep->txsd_avail > 0,
1332 ("%s: not enough credits (%d)", __func__, toep->tx_credits));
1333 toep->tx_credits -= txsd->tx_credits;
1334 if (__predict_false(++toep->txsd_pidx == toep->txsd_total))
1335 toep->txsd_pidx = 0;
1336 toep->txsd_avail--;
1337 INP_WUNLOCK(inp);
1338 CTR5(KTR_IW_CXGBE, "%s:creE %p %u %u %u", __func__, toep ,
1339 txsd->tx_credits, toep->tx_credits, toep->txsd_pidx);
1340 return (0);
1341 }
1342
rdma_init(struct c4iw_dev * rhp,struct c4iw_qp * qhp)1343 static int rdma_init(struct c4iw_dev *rhp, struct c4iw_qp *qhp)
1344 {
1345 struct fw_ri_wr *wqe;
1346 int ret;
1347 struct wrqe *wr;
1348 struct c4iw_ep *ep = qhp->ep;
1349 struct c4iw_rdev *rdev = &qhp->rhp->rdev;
1350 struct adapter *sc = rdev->adap;
1351 struct socket *so = ep->com.so;
1352 struct inpcb *inp = sotoinpcb(so);
1353 struct tcpcb *tp = intotcpcb(inp);
1354 struct toepcb *toep = tp->t_toe;
1355
1356 CTR5(KTR_IW_CXGBE, "%s qhp %p qid 0x%x ep %p tid %u", __func__, qhp,
1357 qhp->wq.sq.qid, ep, ep->hwtid);
1358
1359 wr = alloc_wrqe(sizeof(*wqe), &toep->ofld_txq->wrq);
1360 if (wr == NULL)
1361 return (0);
1362 wqe = wrtod(wr);
1363 ret = alloc_ird(rhp, qhp->attr.max_ird);
1364 if (ret) {
1365 qhp->attr.max_ird = 0;
1366 free_wrqe(wr);
1367 return ret;
1368 }
1369
1370 memset(wqe, 0, sizeof *wqe);
1371
1372 wqe->op_compl = cpu_to_be32(
1373 V_FW_WR_OP(FW_RI_WR) |
1374 F_FW_WR_COMPL);
1375 wqe->flowid_len16 = cpu_to_be32(V_FW_WR_FLOWID(ep->hwtid) |
1376 V_FW_WR_LEN16(DIV_ROUND_UP(sizeof *wqe, 16)));
1377
1378 wqe->cookie = (unsigned long) &ep->com.wr_wait;
1379
1380 wqe->u.init.type = FW_RI_TYPE_INIT;
1381 wqe->u.init.mpareqbit_p2ptype =
1382 V_FW_RI_WR_MPAREQBIT(qhp->attr.mpa_attr.initiator) |
1383 V_FW_RI_WR_P2PTYPE(qhp->attr.mpa_attr.p2p_type);
1384 wqe->u.init.mpa_attrs = FW_RI_MPA_IETF_ENABLE;
1385 if (qhp->attr.mpa_attr.recv_marker_enabled)
1386 wqe->u.init.mpa_attrs |= FW_RI_MPA_RX_MARKER_ENABLE;
1387 if (qhp->attr.mpa_attr.xmit_marker_enabled)
1388 wqe->u.init.mpa_attrs |= FW_RI_MPA_TX_MARKER_ENABLE;
1389 if (qhp->attr.mpa_attr.crc_enabled)
1390 wqe->u.init.mpa_attrs |= FW_RI_MPA_CRC_ENABLE;
1391
1392 wqe->u.init.qp_caps = FW_RI_QP_RDMA_READ_ENABLE |
1393 FW_RI_QP_RDMA_WRITE_ENABLE |
1394 FW_RI_QP_BIND_ENABLE;
1395 if (!qhp->ibqp.uobject)
1396 wqe->u.init.qp_caps |= FW_RI_QP_FAST_REGISTER_ENABLE |
1397 FW_RI_QP_STAG0_ENABLE;
1398 wqe->u.init.nrqe = cpu_to_be16(t4_rqes_posted(&qhp->wq));
1399 wqe->u.init.pdid = cpu_to_be32(qhp->attr.pd);
1400 wqe->u.init.qpid = cpu_to_be32(qhp->wq.sq.qid);
1401 wqe->u.init.sq_eqid = cpu_to_be32(qhp->wq.sq.qid);
1402 wqe->u.init.rq_eqid = cpu_to_be32(qhp->wq.rq.qid);
1403 wqe->u.init.scqid = cpu_to_be32(qhp->attr.scq);
1404 wqe->u.init.rcqid = cpu_to_be32(qhp->attr.rcq);
1405 wqe->u.init.ord_max = cpu_to_be32(qhp->attr.max_ord);
1406 wqe->u.init.ird_max = cpu_to_be32(qhp->attr.max_ird);
1407 wqe->u.init.iss = cpu_to_be32(ep->snd_seq);
1408 wqe->u.init.irs = cpu_to_be32(ep->rcv_seq);
1409 wqe->u.init.hwrqsize = cpu_to_be32(qhp->wq.rq.rqt_size);
1410 wqe->u.init.hwrqaddr = cpu_to_be32(qhp->wq.rq.rqt_hwaddr -
1411 sc->vres.rq.start);
1412 if (qhp->attr.mpa_attr.initiator)
1413 build_rtr_msg(qhp->attr.mpa_attr.p2p_type, &wqe->u.init);
1414
1415 c4iw_init_wr_wait(&ep->com.wr_wait);
1416
1417 ret = creds(toep, inp, sizeof(*wqe));
1418 if (ret) {
1419 free_wrqe(wr);
1420 free_ird(rhp, qhp->attr.max_ird);
1421 return ret;
1422 }
1423 t4_wrq_tx(sc, wr);
1424
1425 ret = c4iw_wait_for_reply(rdev, &ep->com.wr_wait, ep->hwtid,
1426 qhp->wq.sq.qid, ep->com.so, __func__);
1427
1428 toep->params.ulp_mode = ULP_MODE_RDMA;
1429 free_ird(rhp, qhp->attr.max_ird);
1430
1431 return ret;
1432 }
1433
c4iw_modify_qp(struct c4iw_dev * rhp,struct c4iw_qp * qhp,enum c4iw_qp_attr_mask mask,struct c4iw_qp_attributes * attrs,int internal)1434 int c4iw_modify_qp(struct c4iw_dev *rhp, struct c4iw_qp *qhp,
1435 enum c4iw_qp_attr_mask mask,
1436 struct c4iw_qp_attributes *attrs,
1437 int internal)
1438 {
1439 int ret = 0;
1440 struct c4iw_qp_attributes newattr = qhp->attr;
1441 int disconnect = 0;
1442 int terminate = 0;
1443 int abort = 0;
1444 int free = 0;
1445 struct c4iw_ep *ep = NULL;
1446
1447 CTR5(KTR_IW_CXGBE, "%s qhp %p sqid 0x%x rqid 0x%x ep %p", __func__, qhp,
1448 qhp->wq.sq.qid, qhp->wq.rq.qid, qhp->ep);
1449 CTR3(KTR_IW_CXGBE, "%s state %d -> %d", __func__, qhp->attr.state,
1450 (mask & C4IW_QP_ATTR_NEXT_STATE) ? attrs->next_state : -1);
1451
1452 mutex_lock(&qhp->mutex);
1453
1454 /* Process attr changes if in IDLE */
1455 if (mask & C4IW_QP_ATTR_VALID_MODIFY) {
1456 if (qhp->attr.state != C4IW_QP_STATE_IDLE) {
1457 ret = -EIO;
1458 goto out;
1459 }
1460 if (mask & C4IW_QP_ATTR_ENABLE_RDMA_READ)
1461 newattr.enable_rdma_read = attrs->enable_rdma_read;
1462 if (mask & C4IW_QP_ATTR_ENABLE_RDMA_WRITE)
1463 newattr.enable_rdma_write = attrs->enable_rdma_write;
1464 if (mask & C4IW_QP_ATTR_ENABLE_RDMA_BIND)
1465 newattr.enable_bind = attrs->enable_bind;
1466 if (mask & C4IW_QP_ATTR_MAX_ORD) {
1467 if (attrs->max_ord > c4iw_max_read_depth) {
1468 ret = -EINVAL;
1469 goto out;
1470 }
1471 newattr.max_ord = attrs->max_ord;
1472 }
1473 if (mask & C4IW_QP_ATTR_MAX_IRD) {
1474 if (attrs->max_ird > cur_max_read_depth(rhp)) {
1475 ret = -EINVAL;
1476 goto out;
1477 }
1478 newattr.max_ird = attrs->max_ird;
1479 }
1480 qhp->attr = newattr;
1481 }
1482
1483 if (!(mask & C4IW_QP_ATTR_NEXT_STATE))
1484 goto out;
1485 if (qhp->attr.state == attrs->next_state)
1486 goto out;
1487
1488 /* Return EINPROGRESS if QP is already in transition state.
1489 * Eg: CLOSING->IDLE transition or *->ERROR transition.
1490 * This can happen while connection is switching(due to rdma_fini)
1491 * from iWARP/RDDP to TOE mode and any inflight RDMA RX data will
1492 * reach TOE driver -> TCP stack -> iWARP driver. In this way
1493 * iWARP driver keep receiving inflight RDMA RX data until socket
1494 * is closed or aborted. And if iWARP CM is in FPDU sate, then
1495 * it tries to put QP in TERM state and disconnects endpoint.
1496 * But as QP is already in transition state, this event is ignored.
1497 */
1498 if ((qhp->attr.state >= C4IW_QP_STATE_ERROR) &&
1499 (attrs->next_state == C4IW_QP_STATE_TERMINATE)) {
1500 ret = -EINPROGRESS;
1501 goto out;
1502 }
1503
1504 switch (qhp->attr.state) {
1505 case C4IW_QP_STATE_IDLE:
1506 switch (attrs->next_state) {
1507 case C4IW_QP_STATE_RTS:
1508 if (!(mask & C4IW_QP_ATTR_LLP_STREAM_HANDLE)) {
1509 ret = -EINVAL;
1510 goto out;
1511 }
1512 if (!(mask & C4IW_QP_ATTR_MPA_ATTR)) {
1513 ret = -EINVAL;
1514 goto out;
1515 }
1516 qhp->attr.mpa_attr = attrs->mpa_attr;
1517 qhp->attr.llp_stream_handle = attrs->llp_stream_handle;
1518 qhp->ep = qhp->attr.llp_stream_handle;
1519 set_state(qhp, C4IW_QP_STATE_RTS);
1520
1521 /*
1522 * Ref the endpoint here and deref when we
1523 * disassociate the endpoint from the QP. This
1524 * happens in CLOSING->IDLE transition or *->ERROR
1525 * transition.
1526 */
1527 c4iw_get_ep(&qhp->ep->com);
1528 ret = rdma_init(rhp, qhp);
1529 if (ret)
1530 goto err;
1531 break;
1532 case C4IW_QP_STATE_ERROR:
1533 set_state(qhp, C4IW_QP_STATE_ERROR);
1534 flush_qp(qhp);
1535 break;
1536 default:
1537 ret = -EINVAL;
1538 goto out;
1539 }
1540 break;
1541 case C4IW_QP_STATE_RTS:
1542 switch (attrs->next_state) {
1543 case C4IW_QP_STATE_CLOSING:
1544 BUG_ON(kref_read(&qhp->ep->com.kref) < 2);
1545 t4_set_wq_in_error(&qhp->wq);
1546 set_state(qhp, C4IW_QP_STATE_CLOSING);
1547 ep = qhp->ep;
1548 if (!internal) {
1549 abort = 0;
1550 disconnect = 1;
1551 c4iw_get_ep(&qhp->ep->com);
1552 }
1553 ret = rdma_fini(rhp, qhp, ep);
1554 if (ret)
1555 goto err;
1556 break;
1557 case C4IW_QP_STATE_TERMINATE:
1558 t4_set_wq_in_error(&qhp->wq);
1559 set_state(qhp, C4IW_QP_STATE_TERMINATE);
1560 qhp->attr.layer_etype = attrs->layer_etype;
1561 qhp->attr.ecode = attrs->ecode;
1562 ep = qhp->ep;
1563 if (!internal) {
1564 c4iw_get_ep(&qhp->ep->com);
1565 terminate = 1;
1566 disconnect = 1;
1567 } else {
1568 terminate = qhp->attr.send_term;
1569 ret = rdma_fini(rhp, qhp, ep);
1570 if (ret)
1571 goto err;
1572 }
1573 break;
1574 case C4IW_QP_STATE_ERROR:
1575 t4_set_wq_in_error(&qhp->wq);
1576 set_state(qhp, C4IW_QP_STATE_ERROR);
1577 if (!internal) {
1578 abort = 1;
1579 disconnect = 1;
1580 ep = qhp->ep;
1581 c4iw_get_ep(&qhp->ep->com);
1582 }
1583 goto err;
1584 break;
1585 default:
1586 ret = -EINVAL;
1587 goto out;
1588 }
1589 break;
1590 case C4IW_QP_STATE_CLOSING:
1591
1592 /*
1593 * Allow kernel users to move to ERROR for qp draining.
1594 */
1595 if (!internal && (qhp->ibqp.uobject || attrs->next_state !=
1596 C4IW_QP_STATE_ERROR)) {
1597 ret = -EINVAL;
1598 goto out;
1599 }
1600 switch (attrs->next_state) {
1601 case C4IW_QP_STATE_IDLE:
1602 flush_qp(qhp);
1603 set_state(qhp, C4IW_QP_STATE_IDLE);
1604 qhp->attr.llp_stream_handle = NULL;
1605 c4iw_put_ep(&qhp->ep->com);
1606 qhp->ep = NULL;
1607 wake_up(&qhp->wait);
1608 break;
1609 case C4IW_QP_STATE_ERROR:
1610 goto err;
1611 default:
1612 ret = -EINVAL;
1613 goto err;
1614 }
1615 break;
1616 case C4IW_QP_STATE_ERROR:
1617 if (attrs->next_state != C4IW_QP_STATE_IDLE) {
1618 ret = -EINVAL;
1619 goto out;
1620 }
1621 if (!t4_sq_empty(&qhp->wq) || !t4_rq_empty(&qhp->wq)) {
1622 ret = -EINVAL;
1623 goto out;
1624 }
1625 set_state(qhp, C4IW_QP_STATE_IDLE);
1626 break;
1627 case C4IW_QP_STATE_TERMINATE:
1628 if (!internal) {
1629 ret = -EINVAL;
1630 goto out;
1631 }
1632 goto err;
1633 break;
1634 default:
1635 printf("%s in a bad state %d\n",
1636 __func__, qhp->attr.state);
1637 ret = -EINVAL;
1638 goto err;
1639 break;
1640 }
1641 goto out;
1642 err:
1643 CTR3(KTR_IW_CXGBE, "%s disassociating ep %p qpid 0x%x", __func__,
1644 qhp->ep, qhp->wq.sq.qid);
1645
1646 /* disassociate the LLP connection */
1647 qhp->attr.llp_stream_handle = NULL;
1648 if (!ep)
1649 ep = qhp->ep;
1650 qhp->ep = NULL;
1651 set_state(qhp, C4IW_QP_STATE_ERROR);
1652 free = 1;
1653 abort = 1;
1654 BUG_ON(!ep);
1655 flush_qp(qhp);
1656 wake_up(&qhp->wait);
1657 out:
1658 mutex_unlock(&qhp->mutex);
1659
1660 if (terminate)
1661 post_terminate(qhp, NULL, internal ? GFP_ATOMIC : GFP_KERNEL);
1662
1663 /*
1664 * If disconnect is 1, then we need to initiate a disconnect
1665 * on the EP. This can be a normal close (RTS->CLOSING) or
1666 * an abnormal close (RTS/CLOSING->ERROR).
1667 */
1668 if (disconnect) {
1669 __c4iw_ep_disconnect(ep, abort, internal ? GFP_ATOMIC :
1670 GFP_KERNEL);
1671 c4iw_put_ep(&ep->com);
1672 }
1673
1674 /*
1675 * If free is 1, then we've disassociated the EP from the QP
1676 * and we need to dereference the EP.
1677 */
1678 if (free)
1679 c4iw_put_ep(&ep->com);
1680 CTR2(KTR_IW_CXGBE, "%s exit state %d", __func__, qhp->attr.state);
1681 return ret;
1682 }
1683
c4iw_destroy_qp(struct ib_qp * ib_qp,struct ib_udata * udata)1684 int c4iw_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata)
1685 {
1686 struct c4iw_dev *rhp;
1687 struct c4iw_qp *qhp;
1688 struct c4iw_qp_attributes attrs;
1689
1690 CTR2(KTR_IW_CXGBE, "%s ib_qp %p", __func__, ib_qp);
1691 qhp = to_c4iw_qp(ib_qp);
1692 rhp = qhp->rhp;
1693
1694 attrs.next_state = C4IW_QP_STATE_ERROR;
1695 if (qhp->attr.state == C4IW_QP_STATE_TERMINATE)
1696 c4iw_modify_qp(rhp, qhp, C4IW_QP_ATTR_NEXT_STATE, &attrs, 1);
1697 else
1698 c4iw_modify_qp(rhp, qhp, C4IW_QP_ATTR_NEXT_STATE, &attrs, 0);
1699 wait_event(qhp->wait, !qhp->ep);
1700
1701 remove_handle(rhp, &rhp->qpidr, qhp->wq.sq.qid);
1702
1703 free_ird(rhp, qhp->attr.max_ird);
1704 c4iw_qp_rem_ref(ib_qp);
1705
1706 CTR3(KTR_IW_CXGBE, "%s ib_qp %p qpid 0x%0x", __func__, ib_qp,
1707 qhp->wq.sq.qid);
1708 return 0;
1709 }
1710
1711 struct ib_qp *
c4iw_create_qp(struct ib_pd * pd,struct ib_qp_init_attr * attrs,struct ib_udata * udata)1712 c4iw_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *attrs,
1713 struct ib_udata *udata)
1714 {
1715 struct c4iw_dev *rhp;
1716 struct c4iw_qp *qhp;
1717 struct c4iw_pd *php;
1718 struct c4iw_cq *schp;
1719 struct c4iw_cq *rchp;
1720 struct c4iw_create_qp_resp uresp;
1721 unsigned int sqsize, rqsize;
1722 struct c4iw_ucontext *ucontext;
1723 int ret;
1724 struct c4iw_mm_entry *sq_key_mm = NULL, *rq_key_mm = NULL;
1725 struct c4iw_mm_entry *sq_db_key_mm = NULL, *rq_db_key_mm = NULL;
1726
1727 CTR2(KTR_IW_CXGBE, "%s ib_pd %p", __func__, pd);
1728
1729 if (attrs->qp_type != IB_QPT_RC)
1730 return ERR_PTR(-EINVAL);
1731
1732 php = to_c4iw_pd(pd);
1733 rhp = php->rhp;
1734 schp = get_chp(rhp, ((struct c4iw_cq *)attrs->send_cq)->cq.cqid);
1735 rchp = get_chp(rhp, ((struct c4iw_cq *)attrs->recv_cq)->cq.cqid);
1736 if (!schp || !rchp)
1737 return ERR_PTR(-EINVAL);
1738
1739 if (attrs->cap.max_inline_data > T4_MAX_SEND_INLINE)
1740 return ERR_PTR(-EINVAL);
1741
1742 if (attrs->cap.max_recv_wr > rhp->rdev.hw_queue.t4_max_rq_size)
1743 return ERR_PTR(-E2BIG);
1744 rqsize = attrs->cap.max_recv_wr + 1;
1745 if (rqsize < 8)
1746 rqsize = 8;
1747
1748 if (attrs->cap.max_send_wr > rhp->rdev.hw_queue.t4_max_sq_size)
1749 return ERR_PTR(-E2BIG);
1750 sqsize = attrs->cap.max_send_wr + 1;
1751 if (sqsize < 8)
1752 sqsize = 8;
1753
1754 ucontext = pd->uobject ? to_c4iw_ucontext(pd->uobject->context) : NULL;
1755
1756 qhp = kzalloc(sizeof(*qhp), GFP_KERNEL);
1757 if (!qhp)
1758 return ERR_PTR(-ENOMEM);
1759 qhp->wq.sq.size = sqsize;
1760 qhp->wq.sq.memsize =
1761 (sqsize + rhp->rdev.hw_queue.t4_eq_status_entries) *
1762 sizeof(*qhp->wq.sq.queue) + 16 * sizeof(__be64);
1763 qhp->wq.sq.flush_cidx = -1;
1764 qhp->wq.rq.size = rqsize;
1765 qhp->wq.rq.memsize =
1766 (rqsize + rhp->rdev.hw_queue.t4_eq_status_entries) *
1767 sizeof(*qhp->wq.rq.queue);
1768
1769 if (ucontext) {
1770 qhp->wq.sq.memsize = roundup(qhp->wq.sq.memsize, PAGE_SIZE);
1771 qhp->wq.rq.memsize = roundup(qhp->wq.rq.memsize, PAGE_SIZE);
1772 }
1773
1774 CTR5(KTR_IW_CXGBE, "%s sqsize %u sqmemsize %zu rqsize %u rqmemsize %zu",
1775 __func__, sqsize, qhp->wq.sq.memsize, rqsize, qhp->wq.rq.memsize);
1776
1777 ret = create_qp(&rhp->rdev, &qhp->wq, &schp->cq, &rchp->cq,
1778 ucontext ? &ucontext->uctx : &rhp->rdev.uctx);
1779 if (ret)
1780 goto err1;
1781
1782 attrs->cap.max_recv_wr = rqsize - 1;
1783 attrs->cap.max_send_wr = sqsize - 1;
1784 attrs->cap.max_inline_data = T4_MAX_SEND_INLINE;
1785
1786 qhp->rhp = rhp;
1787 qhp->attr.pd = php->pdid;
1788 qhp->attr.scq = ((struct c4iw_cq *) attrs->send_cq)->cq.cqid;
1789 qhp->attr.rcq = ((struct c4iw_cq *) attrs->recv_cq)->cq.cqid;
1790 qhp->attr.sq_num_entries = attrs->cap.max_send_wr;
1791 qhp->attr.rq_num_entries = attrs->cap.max_recv_wr;
1792 qhp->attr.sq_max_sges = attrs->cap.max_send_sge;
1793 qhp->attr.sq_max_sges_rdma_write = attrs->cap.max_send_sge;
1794 qhp->attr.rq_max_sges = attrs->cap.max_recv_sge;
1795 qhp->attr.state = C4IW_QP_STATE_IDLE;
1796 qhp->attr.next_state = C4IW_QP_STATE_IDLE;
1797 qhp->attr.enable_rdma_read = 1;
1798 qhp->attr.enable_rdma_write = 1;
1799 qhp->attr.enable_bind = 1;
1800 qhp->attr.max_ord = 0;
1801 qhp->attr.max_ird = 0;
1802 qhp->sq_sig_all = attrs->sq_sig_type == IB_SIGNAL_ALL_WR;
1803 spin_lock_init(&qhp->lock);
1804 mutex_init(&qhp->mutex);
1805 init_waitqueue_head(&qhp->wait);
1806 kref_init(&qhp->kref);
1807 INIT_WORK(&qhp->free_work, free_qp_work);
1808
1809 ret = insert_handle(rhp, &rhp->qpidr, qhp, qhp->wq.sq.qid);
1810 if (ret)
1811 goto err2;
1812
1813 if (udata) {
1814 sq_key_mm = kmalloc(sizeof(*sq_key_mm), GFP_KERNEL);
1815 if (!sq_key_mm) {
1816 ret = -ENOMEM;
1817 goto err3;
1818 }
1819 rq_key_mm = kmalloc(sizeof(*rq_key_mm), GFP_KERNEL);
1820 if (!rq_key_mm) {
1821 ret = -ENOMEM;
1822 goto err4;
1823 }
1824 sq_db_key_mm = kmalloc(sizeof(*sq_db_key_mm), GFP_KERNEL);
1825 if (!sq_db_key_mm) {
1826 ret = -ENOMEM;
1827 goto err5;
1828 }
1829 rq_db_key_mm = kmalloc(sizeof(*rq_db_key_mm), GFP_KERNEL);
1830 if (!rq_db_key_mm) {
1831 ret = -ENOMEM;
1832 goto err6;
1833 }
1834 uresp.flags = 0;
1835 uresp.qid_mask = rhp->rdev.qpmask;
1836 uresp.sqid = qhp->wq.sq.qid;
1837 uresp.sq_size = qhp->wq.sq.size;
1838 uresp.sq_memsize = qhp->wq.sq.memsize;
1839 uresp.rqid = qhp->wq.rq.qid;
1840 uresp.rq_size = qhp->wq.rq.size;
1841 uresp.rq_memsize = qhp->wq.rq.memsize;
1842 spin_lock(&ucontext->mmap_lock);
1843 uresp.ma_sync_key = 0;
1844 uresp.sq_key = ucontext->key;
1845 ucontext->key += PAGE_SIZE;
1846 uresp.rq_key = ucontext->key;
1847 ucontext->key += PAGE_SIZE;
1848 uresp.sq_db_gts_key = ucontext->key;
1849 ucontext->key += PAGE_SIZE;
1850 uresp.rq_db_gts_key = ucontext->key;
1851 ucontext->key += PAGE_SIZE;
1852 spin_unlock(&ucontext->mmap_lock);
1853 ret = ib_copy_to_udata(udata, &uresp, sizeof uresp);
1854 if (ret)
1855 goto err7;
1856 sq_key_mm->key = uresp.sq_key;
1857 sq_key_mm->addr = qhp->wq.sq.phys_addr;
1858 sq_key_mm->len = PAGE_ALIGN(qhp->wq.sq.memsize);
1859 CTR4(KTR_IW_CXGBE, "%s sq_key_mm %x, %x, %d", __func__,
1860 sq_key_mm->key, sq_key_mm->addr,
1861 sq_key_mm->len);
1862 insert_mmap(ucontext, sq_key_mm);
1863 rq_key_mm->key = uresp.rq_key;
1864 rq_key_mm->addr = qhp->wq.rq.phys_addr;
1865 rq_key_mm->len = PAGE_ALIGN(qhp->wq.rq.memsize);
1866 CTR4(KTR_IW_CXGBE, "%s rq_key_mm %x, %x, %d", __func__,
1867 rq_key_mm->key, rq_key_mm->addr,
1868 rq_key_mm->len);
1869 insert_mmap(ucontext, rq_key_mm);
1870 sq_db_key_mm->key = uresp.sq_db_gts_key;
1871 sq_db_key_mm->addr = (u64)qhp->wq.sq.bar2_pa;
1872 sq_db_key_mm->len = PAGE_SIZE;
1873 CTR4(KTR_IW_CXGBE, "%s sq_db_key_mm %x, %x, %d", __func__,
1874 sq_db_key_mm->key, sq_db_key_mm->addr,
1875 sq_db_key_mm->len);
1876 insert_mmap(ucontext, sq_db_key_mm);
1877 rq_db_key_mm->key = uresp.rq_db_gts_key;
1878 rq_db_key_mm->addr = (u64)qhp->wq.rq.bar2_pa;
1879 rq_db_key_mm->len = PAGE_SIZE;
1880 CTR4(KTR_IW_CXGBE, "%s rq_db_key_mm %x, %x, %d", __func__,
1881 rq_db_key_mm->key, rq_db_key_mm->addr,
1882 rq_db_key_mm->len);
1883 insert_mmap(ucontext, rq_db_key_mm);
1884
1885 qhp->ucontext = ucontext;
1886 }
1887 qhp->ibqp.qp_num = qhp->wq.sq.qid;
1888 init_timer(&(qhp->timer));
1889
1890 CTR5(KTR_IW_CXGBE, "%s sq id %u size %u memsize %zu num_entries %u",
1891 __func__, qhp->wq.sq.qid,
1892 qhp->wq.sq.size, qhp->wq.sq.memsize, attrs->cap.max_send_wr);
1893 CTR5(KTR_IW_CXGBE, "%s rq id %u size %u memsize %zu num_entries %u",
1894 __func__, qhp->wq.rq.qid,
1895 qhp->wq.rq.size, qhp->wq.rq.memsize, attrs->cap.max_recv_wr);
1896 return &qhp->ibqp;
1897 err7:
1898 kfree(rq_db_key_mm);
1899 err6:
1900 kfree(sq_db_key_mm);
1901 err5:
1902 kfree(rq_key_mm);
1903 err4:
1904 kfree(sq_key_mm);
1905 err3:
1906 remove_handle(rhp, &rhp->qpidr, qhp->wq.sq.qid);
1907 err2:
1908 destroy_qp(&rhp->rdev, &qhp->wq,
1909 ucontext ? &ucontext->uctx : &rhp->rdev.uctx);
1910 err1:
1911 kfree(qhp);
1912 return ERR_PTR(ret);
1913 }
1914
c4iw_ib_modify_qp(struct ib_qp * ibqp,struct ib_qp_attr * attr,int attr_mask,struct ib_udata * udata)1915 int c4iw_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
1916 int attr_mask, struct ib_udata *udata)
1917 {
1918 struct c4iw_dev *rhp;
1919 struct c4iw_qp *qhp;
1920 enum c4iw_qp_attr_mask mask = 0;
1921 struct c4iw_qp_attributes attrs;
1922
1923 CTR2(KTR_IW_CXGBE, "%s ib_qp %p", __func__, ibqp);
1924
1925 /* iwarp does not support the RTR state */
1926 if ((attr_mask & IB_QP_STATE) && (attr->qp_state == IB_QPS_RTR))
1927 attr_mask &= ~IB_QP_STATE;
1928
1929 /* Make sure we still have something left to do */
1930 if (!attr_mask)
1931 return 0;
1932
1933 memset(&attrs, 0, sizeof attrs);
1934 qhp = to_c4iw_qp(ibqp);
1935 rhp = qhp->rhp;
1936
1937 attrs.next_state = c4iw_convert_state(attr->qp_state);
1938 attrs.enable_rdma_read = (attr->qp_access_flags &
1939 IB_ACCESS_REMOTE_READ) ? 1 : 0;
1940 attrs.enable_rdma_write = (attr->qp_access_flags &
1941 IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
1942 attrs.enable_bind = (attr->qp_access_flags & IB_ACCESS_MW_BIND) ? 1 : 0;
1943
1944
1945 mask |= (attr_mask & IB_QP_STATE) ? C4IW_QP_ATTR_NEXT_STATE : 0;
1946 mask |= (attr_mask & IB_QP_ACCESS_FLAGS) ?
1947 (C4IW_QP_ATTR_ENABLE_RDMA_READ |
1948 C4IW_QP_ATTR_ENABLE_RDMA_WRITE |
1949 C4IW_QP_ATTR_ENABLE_RDMA_BIND) : 0;
1950
1951 return c4iw_modify_qp(rhp, qhp, mask, &attrs, 0);
1952 }
1953
c4iw_get_qp(struct ib_device * dev,int qpn)1954 struct ib_qp *c4iw_get_qp(struct ib_device *dev, int qpn)
1955 {
1956 CTR3(KTR_IW_CXGBE, "%s ib_dev %p qpn 0x%x", __func__, dev, qpn);
1957 return (struct ib_qp *)get_qhp(to_c4iw_dev(dev), qpn);
1958 }
1959
c4iw_ib_query_qp(struct ib_qp * ibqp,struct ib_qp_attr * attr,int attr_mask,struct ib_qp_init_attr * init_attr)1960 int c4iw_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
1961 int attr_mask, struct ib_qp_init_attr *init_attr)
1962 {
1963 struct c4iw_qp *qhp = to_c4iw_qp(ibqp);
1964
1965 memset(attr, 0, sizeof *attr);
1966 memset(init_attr, 0, sizeof *init_attr);
1967 attr->qp_state = to_ib_qp_state(qhp->attr.state);
1968 init_attr->cap.max_send_wr = qhp->attr.sq_num_entries;
1969 init_attr->cap.max_recv_wr = qhp->attr.rq_num_entries;
1970 init_attr->cap.max_send_sge = qhp->attr.sq_max_sges;
1971 init_attr->cap.max_recv_sge = qhp->attr.sq_max_sges;
1972 init_attr->cap.max_inline_data = T4_MAX_SEND_INLINE;
1973 init_attr->sq_sig_type = qhp->sq_sig_all ? IB_SIGNAL_ALL_WR : 0;
1974 return 0;
1975 }
1976 #endif
1977