1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2009-2013 Chelsio, Inc. All rights reserved.
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 */
34 #include <sys/cdefs.h>
35 #include "opt_inet.h"
36
37 #ifdef TCP_OFFLOAD
38 #include <linux/slab.h>
39
40 #include "iw_cxgbe.h"
41
print_tpte(struct adapter * sc,const u32 stag,const struct fw_ri_tpte * tpte)42 static void print_tpte(struct adapter *sc, const u32 stag,
43 const struct fw_ri_tpte *tpte)
44 {
45 const __be64 *p = (const void *)tpte;
46
47 CH_ERR(sc, "stag idx 0x%x valid %d key 0x%x state %d pdid %d "
48 "perm 0x%x ps %d len 0x%016llx va 0x%016llx\n",
49 stag & 0xffffff00,
50 G_FW_RI_TPTE_VALID(ntohl(tpte->valid_to_pdid)),
51 G_FW_RI_TPTE_STAGKEY(ntohl(tpte->valid_to_pdid)),
52 G_FW_RI_TPTE_STAGSTATE(ntohl(tpte->valid_to_pdid)),
53 G_FW_RI_TPTE_PDID(ntohl(tpte->valid_to_pdid)),
54 G_FW_RI_TPTE_PERM(ntohl(tpte->locread_to_qpid)),
55 G_FW_RI_TPTE_PS(ntohl(tpte->locread_to_qpid)),
56 (long long)(((u64)ntohl(tpte->len_hi) << 32) | ntohl(tpte->len_lo)),
57 (long long)(((u64)ntohl(tpte->va_hi) << 32) | ntohl(tpte->va_lo_fbo)));
58 CH_ERR(sc, "stag idx 0x%x: %016llx %016llx %016llx %016llx\n",
59 stag & 0xffffff00,
60 (long long)be64_to_cpu(p[0]), (long long)be64_to_cpu(p[1]),
61 (long long)be64_to_cpu(p[2]), (long long)be64_to_cpu(p[3]));
62 }
63
t4_dump_stag(struct adapter * sc,const u32 stag)64 void t4_dump_stag(struct adapter *sc, const u32 stag)
65 {
66 struct fw_ri_tpte tpte __aligned(sizeof(__be64)) = {0};
67 const u32 offset = sc->vres.stag.start + ((stag >> 8) * 32);
68
69 if (offset > sc->vres.stag.start + sc->vres.stag.size - 32) {
70 CH_ERR(sc, "stag 0x%x is invalid for current configuration.\n",
71 stag);
72 return;
73 }
74 read_via_memwin(sc, 0, offset, (u32 *)&tpte, 32);
75 print_tpte(sc, stag, &tpte);
76 }
77
t4_dump_all_stag(struct adapter * sc)78 void t4_dump_all_stag(struct adapter *sc)
79 {
80 struct fw_ri_tpte tpte __aligned(sizeof(__be64)) = {0};
81 const u32 first = sc->vres.stag.start;
82 const u32 last = first + sc->vres.stag.size - 32;
83 u32 offset, i;
84
85 for (i = 0, offset = first; offset <= last; i++, offset += 32) {
86 tpte.valid_to_pdid = 0;
87 read_via_memwin(sc, 0, offset, (u32 *)&tpte, 4);
88 if (tpte.valid_to_pdid != 0) {
89 read_via_memwin(sc, 0, offset, (u32 *)&tpte, 32);
90 print_tpte(sc, i << 8, &tpte);
91 }
92 }
93 }
94
dump_err_cqe(struct c4iw_dev * dev,struct t4_cqe * err_cqe)95 static void dump_err_cqe(struct c4iw_dev *dev, struct t4_cqe *err_cqe)
96 {
97 struct adapter *sc = dev->rdev.adap;
98 __be64 *p = (void *)err_cqe;
99
100 CH_ERR(sc, "AE qpid 0x%x opcode %d status 0x%x "
101 "type %d wrid.hi 0x%x wrid.lo 0x%x\n",
102 CQE_QPID(err_cqe), CQE_OPCODE(err_cqe),
103 CQE_STATUS(err_cqe), CQE_TYPE(err_cqe),
104 CQE_WRID_HI(err_cqe), CQE_WRID_LOW(err_cqe));
105 CH_ERR(sc, "%016llx %016llx %016llx %016llx\n",
106 (long long)be64_to_cpu(p[0]), (long long)be64_to_cpu(p[1]),
107 (long long)be64_to_cpu(p[2]), (long long)be64_to_cpu(p[3]));
108
109 /*
110 * Ingress WRITE and READ_RESP errors provide
111 * the offending stag, so parse and log it.
112 */
113 if (RQ_TYPE(err_cqe) && (CQE_OPCODE(err_cqe) == FW_RI_RDMA_WRITE ||
114 CQE_OPCODE(err_cqe) == FW_RI_READ_RESP))
115 t4_dump_stag(sc, CQE_WRID_STAG(err_cqe));
116 }
117
post_qp_event(struct c4iw_dev * dev,struct c4iw_cq * chp,struct c4iw_qp * qhp,struct t4_cqe * err_cqe,enum ib_event_type ib_event)118 static void post_qp_event(struct c4iw_dev *dev, struct c4iw_cq *chp,
119 struct c4iw_qp *qhp,
120 struct t4_cqe *err_cqe,
121 enum ib_event_type ib_event)
122 {
123 struct ib_event event;
124 struct c4iw_qp_attributes attrs;
125 unsigned long flag;
126
127 if ((qhp->attr.state == C4IW_QP_STATE_ERROR) ||
128 (qhp->attr.state == C4IW_QP_STATE_TERMINATE)) {
129 CTR4(KTR_IW_CXGBE, "%s AE received after RTS - "
130 "qp state %d qpid 0x%x status 0x%x", __func__,
131 qhp->attr.state, qhp->wq.sq.qid, CQE_STATUS(err_cqe));
132 return;
133 }
134
135 dump_err_cqe(dev, err_cqe);
136
137 if (qhp->attr.state == C4IW_QP_STATE_RTS) {
138 attrs.next_state = C4IW_QP_STATE_TERMINATE;
139 c4iw_modify_qp(qhp->rhp, qhp, C4IW_QP_ATTR_NEXT_STATE,
140 &attrs, 0);
141 }
142
143 event.event = ib_event;
144 event.device = chp->ibcq.device;
145 if (ib_event == IB_EVENT_CQ_ERR)
146 event.element.cq = &chp->ibcq;
147 else
148 event.element.qp = &qhp->ibqp;
149 if (qhp->ibqp.event_handler)
150 (*qhp->ibqp.event_handler)(&event, qhp->ibqp.qp_context);
151
152 spin_lock_irqsave(&chp->comp_handler_lock, flag);
153 (*chp->ibcq.comp_handler)(&chp->ibcq, chp->ibcq.cq_context);
154 spin_unlock_irqrestore(&chp->comp_handler_lock, flag);
155 }
156
c4iw_ev_dispatch(struct c4iw_dev * dev,struct t4_cqe * err_cqe)157 void c4iw_ev_dispatch(struct c4iw_dev *dev, struct t4_cqe *err_cqe)
158 {
159 struct c4iw_cq *chp;
160 struct c4iw_qp *qhp;
161 u32 cqid;
162
163 spin_lock_irq(&dev->lock);
164 qhp = get_qhp(dev, CQE_QPID(err_cqe));
165 if (!qhp) {
166 printf("BAD AE qpid 0x%x opcode %d "
167 "status 0x%x type %d wrid.hi 0x%x wrid.lo 0x%x\n",
168 CQE_QPID(err_cqe),
169 CQE_OPCODE(err_cqe), CQE_STATUS(err_cqe),
170 CQE_TYPE(err_cqe), CQE_WRID_HI(err_cqe),
171 CQE_WRID_LOW(err_cqe));
172 spin_unlock_irq(&dev->lock);
173 goto out;
174 }
175
176 if (SQ_TYPE(err_cqe))
177 cqid = qhp->attr.scq;
178 else
179 cqid = qhp->attr.rcq;
180 chp = get_chp(dev, cqid);
181 if (!chp) {
182 printf("BAD AE cqid 0x%x qpid 0x%x opcode %d "
183 "status 0x%x type %d wrid.hi 0x%x wrid.lo 0x%x\n",
184 cqid, CQE_QPID(err_cqe),
185 CQE_OPCODE(err_cqe), CQE_STATUS(err_cqe),
186 CQE_TYPE(err_cqe), CQE_WRID_HI(err_cqe),
187 CQE_WRID_LOW(err_cqe));
188 spin_unlock_irq(&dev->lock);
189 goto out;
190 }
191
192 c4iw_qp_add_ref(&qhp->ibqp);
193 atomic_inc(&chp->refcnt);
194 spin_unlock_irq(&dev->lock);
195
196 /* Bad incoming write */
197 if (RQ_TYPE(err_cqe) &&
198 (CQE_OPCODE(err_cqe) == FW_RI_RDMA_WRITE)) {
199 post_qp_event(dev, chp, qhp, err_cqe, IB_EVENT_QP_REQ_ERR);
200 goto done;
201 }
202
203 switch (CQE_STATUS(err_cqe)) {
204
205 /* Completion Events */
206 case T4_ERR_SUCCESS:
207 printf(KERN_ERR MOD "AE with status 0!\n");
208 break;
209
210 case T4_ERR_STAG:
211 case T4_ERR_PDID:
212 case T4_ERR_QPID:
213 case T4_ERR_ACCESS:
214 case T4_ERR_WRAP:
215 case T4_ERR_BOUND:
216 case T4_ERR_INVALIDATE_SHARED_MR:
217 case T4_ERR_INVALIDATE_MR_WITH_MW_BOUND:
218 post_qp_event(dev, chp, qhp, err_cqe, IB_EVENT_QP_ACCESS_ERR);
219 break;
220
221 /* Device Fatal Errors */
222 case T4_ERR_ECC:
223 case T4_ERR_ECC_PSTAG:
224 case T4_ERR_INTERNAL_ERR:
225 post_qp_event(dev, chp, qhp, err_cqe, IB_EVENT_DEVICE_FATAL);
226 break;
227
228 /* QP Fatal Errors */
229 case T4_ERR_OUT_OF_RQE:
230 case T4_ERR_PBL_ADDR_BOUND:
231 case T4_ERR_CRC:
232 case T4_ERR_MARKER:
233 case T4_ERR_PDU_LEN_ERR:
234 case T4_ERR_DDP_VERSION:
235 case T4_ERR_RDMA_VERSION:
236 case T4_ERR_OPCODE:
237 case T4_ERR_DDP_QUEUE_NUM:
238 case T4_ERR_MSN:
239 case T4_ERR_TBIT:
240 case T4_ERR_MO:
241 case T4_ERR_MSN_GAP:
242 case T4_ERR_MSN_RANGE:
243 case T4_ERR_RQE_ADDR_BOUND:
244 case T4_ERR_IRD_OVERFLOW:
245 post_qp_event(dev, chp, qhp, err_cqe, IB_EVENT_QP_FATAL);
246 break;
247
248 default:
249 printf("Unknown T4 status 0x%x QPID 0x%x\n",
250 CQE_STATUS(err_cqe), qhp->wq.sq.qid);
251 post_qp_event(dev, chp, qhp, err_cqe, IB_EVENT_QP_FATAL);
252 break;
253 }
254 done:
255 if (atomic_dec_and_test(&chp->refcnt))
256 wake_up(&chp->wait);
257 c4iw_qp_rem_ref(&qhp->ibqp);
258 out:
259 return;
260 }
261
c4iw_ev_handler(struct sge_iq * iq,const struct rsp_ctrl * rc)262 int c4iw_ev_handler(struct sge_iq *iq, const struct rsp_ctrl *rc)
263 {
264 struct c4iw_dev *dev = iq->adapter->iwarp_softc;
265 u32 qid = be32_to_cpu(rc->pldbuflen_qid);
266 struct c4iw_cq *chp;
267 unsigned long flag;
268
269 spin_lock_irqsave(&dev->lock, flag);
270 chp = get_chp(dev, qid);
271 if (chp) {
272 atomic_inc(&chp->refcnt);
273 spin_unlock_irqrestore(&dev->lock, flag);
274
275 spin_lock_irqsave(&chp->comp_handler_lock, flag);
276 (*chp->ibcq.comp_handler)(&chp->ibcq, chp->ibcq.cq_context);
277 spin_unlock_irqrestore(&chp->comp_handler_lock, flag);
278 if (atomic_dec_and_test(&chp->refcnt))
279 wake_up(&chp->wait);
280 } else {
281 CTR2(KTR_IW_CXGBE, "%s unknown cqid 0x%x", __func__, qid);
282 spin_unlock_irqrestore(&dev->lock, flag);
283 }
284
285 return 0;
286 }
287 #endif
288