1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause OR GPL-2.0
3 *
4 * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved.
5 * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
6 * Copyright (c) 2005 Mellanox Technologies. All rights reserved.
7 * Copyright (c) 2004, 2005 Voltaire, Inc. All rights reserved.
8 *
9 * This software is available to you under a choice of one of two
10 * licenses. You may choose to be licensed under the terms of the GNU
11 * General Public License (GPL) Version 2, available from the file
12 * COPYING in the main directory of this source tree, or the
13 * OpenIB.org BSD license below:
14 *
15 * Redistribution and use in source and binary forms, with or
16 * without modification, are permitted provided that the following
17 * conditions are met:
18 *
19 * - Redistributions of source code must retain the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer.
22 *
23 * - Redistributions in binary form must reproduce the above
24 * copyright notice, this list of conditions and the following
25 * disclaimer in the documentation and/or other materials
26 * provided with the distribution.
27 *
28 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
29 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
30 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
31 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
32 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
33 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
34 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
35 * SOFTWARE.
36 */
37
38 #include <sys/cdefs.h>
39 #include "ipoib.h"
40
41 #include <rdma/ib_cache.h>
42
43 #include <security/mac/mac_framework.h>
44
45 #include <linux/delay.h>
46 #include <linux/dma-mapping.h>
47
48 #ifdef CONFIG_INFINIBAND_IPOIB_DEBUG_DATA
49 static int data_debug_level;
50
51 module_param(data_debug_level, int, 0644);
52 MODULE_PARM_DESC(data_debug_level,
53 "Enable data path debug tracing if > 0");
54 #endif
55
56 static DEFINE_MUTEX(pkey_mutex);
57
ipoib_create_ah(struct ipoib_dev_priv * priv,struct ib_pd * pd,struct rdma_ah_attr * attr)58 struct ipoib_ah *ipoib_create_ah(struct ipoib_dev_priv *priv,
59 struct ib_pd *pd, struct rdma_ah_attr *attr)
60 {
61 struct ipoib_ah *ah;
62 struct ib_ah *vah;
63
64 ah = kmalloc(sizeof *ah, GFP_KERNEL);
65 if (!ah)
66 return ERR_PTR(-ENOMEM);
67
68 ah->priv = priv;
69 ah->last_send = 0;
70 kref_init(&ah->ref);
71
72 vah = rdma_create_ah(pd, attr, RDMA_CREATE_AH_SLEEPABLE);
73 if (IS_ERR(vah)) {
74 kfree(ah);
75 ah = (struct ipoib_ah *)vah;
76 } else {
77 ah->ah = vah;
78 ipoib_dbg(priv, "Created ah %p\n", ah->ah);
79 }
80
81 return ah;
82 }
83
ipoib_free_ah(struct kref * kref)84 void ipoib_free_ah(struct kref *kref)
85 {
86 struct ipoib_ah *ah = container_of(kref, struct ipoib_ah, ref);
87 struct ipoib_dev_priv *priv = ah->priv;
88
89 unsigned long flags;
90
91 spin_lock_irqsave(&priv->lock, flags);
92 list_add_tail(&ah->list, &priv->dead_ahs);
93 spin_unlock_irqrestore(&priv->lock, flags);
94 }
95
96 void
ipoib_dma_unmap_rx(struct ipoib_dev_priv * priv,struct ipoib_rx_buf * rx_req)97 ipoib_dma_unmap_rx(struct ipoib_dev_priv *priv, struct ipoib_rx_buf *rx_req)
98 {
99 struct mbuf *m;
100 int i;
101
102 for (i = 0, m = rx_req->mb; m != NULL; m = m->m_next, i++)
103 ib_dma_unmap_single(priv->ca, rx_req->mapping[i], m->m_len,
104 DMA_FROM_DEVICE);
105 }
106
107 void
ipoib_dma_mb(struct ipoib_dev_priv * priv,struct mbuf * mb,unsigned int length)108 ipoib_dma_mb(struct ipoib_dev_priv *priv, struct mbuf *mb, unsigned int length)
109 {
110
111 m_adj(mb, -(mb->m_pkthdr.len - length));
112 }
113
114 struct mbuf *
ipoib_alloc_map_mb(struct ipoib_dev_priv * priv,struct ipoib_rx_buf * rx_req,int align,int size,int max_frags)115 ipoib_alloc_map_mb(struct ipoib_dev_priv *priv, struct ipoib_rx_buf *rx_req,
116 int align, int size, int max_frags)
117 {
118 struct mbuf *mb, *m;
119 int i, j;
120
121 rx_req->mb = NULL;
122 mb = m_getm2(NULL, align + size, M_NOWAIT, MT_DATA, M_PKTHDR);
123 if (mb == NULL)
124 return (NULL);
125 for (i = 0, m = mb; m != NULL; m = m->m_next, i++) {
126 MPASS(i < max_frags);
127
128 m->m_len = M_SIZE(m) - align;
129 m->m_data += align;
130 align = 0;
131 mb->m_pkthdr.len += m->m_len;
132 rx_req->mapping[i] = ib_dma_map_single(priv->ca,
133 mtod(m, void *), m->m_len, DMA_FROM_DEVICE);
134 if (unlikely(ib_dma_mapping_error(priv->ca,
135 rx_req->mapping[i])))
136 goto error;
137
138 }
139 rx_req->mb = mb;
140 return (mb);
141 error:
142 for (j = 0, m = mb; j < i; m = m->m_next, j++)
143 ib_dma_unmap_single(priv->ca, rx_req->mapping[j], m->m_len,
144 DMA_FROM_DEVICE);
145 m_freem(mb);
146 return (NULL);
147
148 }
149
ipoib_ib_post_receive(struct ipoib_dev_priv * priv,int id)150 static int ipoib_ib_post_receive(struct ipoib_dev_priv *priv, int id)
151 {
152 struct ipoib_rx_buf *rx_req;
153 struct mbuf *m;
154 int ret;
155 int i;
156
157 rx_req = &priv->rx_ring[id];
158 for (m = rx_req->mb, i = 0; m != NULL; m = m->m_next, i++) {
159 priv->rx_sge[i].addr = rx_req->mapping[i];
160 priv->rx_sge[i].length = m->m_len;
161 }
162 priv->rx_wr.num_sge = i;
163 priv->rx_wr.wr_id = id | IPOIB_OP_RECV;
164
165 ret = ib_post_recv(priv->qp, &priv->rx_wr, NULL);
166 if (unlikely(ret)) {
167 ipoib_warn(priv, "receive failed for buf %d (%d)\n", id, ret);
168 ipoib_dma_unmap_rx(priv, &priv->rx_ring[id]);
169 m_freem(priv->rx_ring[id].mb);
170 priv->rx_ring[id].mb = NULL;
171 }
172
173 return ret;
174 }
175
176 static struct mbuf *
ipoib_alloc_rx_mb(struct ipoib_dev_priv * priv,int id)177 ipoib_alloc_rx_mb(struct ipoib_dev_priv *priv, int id)
178 {
179 return ipoib_alloc_map_mb(priv, &priv->rx_ring[id],
180 0, priv->max_ib_mtu + IB_GRH_BYTES, IPOIB_UD_RX_SG);
181 }
182
ipoib_ib_post_receives(struct ipoib_dev_priv * priv)183 static int ipoib_ib_post_receives(struct ipoib_dev_priv *priv)
184 {
185 int i;
186
187 for (i = 0; i < ipoib_recvq_size; ++i) {
188 if (!ipoib_alloc_rx_mb(priv, i)) {
189 ipoib_warn(priv, "failed to allocate receive buffer %d\n", i);
190 return -ENOMEM;
191 }
192 if (ipoib_ib_post_receive(priv, i)) {
193 ipoib_warn(priv, "ipoib_ib_post_receive failed for buf %d\n", i);
194 return -EIO;
195 }
196 }
197
198 return 0;
199 }
200
201 static void
ipoib_ib_handle_rx_wc(struct ipoib_dev_priv * priv,struct ib_wc * wc)202 ipoib_ib_handle_rx_wc(struct ipoib_dev_priv *priv, struct ib_wc *wc)
203 {
204 struct ipoib_rx_buf saverx;
205 unsigned int wr_id = wc->wr_id & ~IPOIB_OP_RECV;
206 if_t dev = priv->dev;
207 struct ipoib_header *eh;
208 struct mbuf *mb;
209
210 ipoib_dbg_data(priv, "recv completion: id %d, status: %d\n",
211 wr_id, wc->status);
212
213 if (unlikely(wr_id >= ipoib_recvq_size)) {
214 ipoib_warn(priv, "recv completion event with wrid %d (> %d)\n",
215 wr_id, ipoib_recvq_size);
216 return;
217 }
218
219 mb = priv->rx_ring[wr_id].mb;
220
221 if (unlikely(wc->status != IB_WC_SUCCESS)) {
222 if (wc->status != IB_WC_WR_FLUSH_ERR) {
223 ipoib_warn(priv, "failed recv event "
224 "(status=%d, wrid=%d vend_err %x)\n",
225 wc->status, wr_id, wc->vendor_err);
226 goto repost;
227 }
228 if (mb) {
229 ipoib_dma_unmap_rx(priv, &priv->rx_ring[wr_id]);
230 m_freem(mb);
231 priv->rx_ring[wr_id].mb = NULL;
232 }
233 return;
234 }
235
236 /*
237 * Drop packets that this interface sent, ie multicast packets
238 * that the HCA has replicated.
239 */
240 if (wc->slid == priv->local_lid && wc->src_qp == priv->qp->qp_num)
241 goto repost;
242
243 memcpy(&saverx, &priv->rx_ring[wr_id], sizeof(saverx));
244 /*
245 * If we can't allocate a new RX buffer, dump
246 * this packet and reuse the old buffer.
247 */
248 if (unlikely(!ipoib_alloc_rx_mb(priv, wr_id))) {
249 memcpy(&priv->rx_ring[wr_id], &saverx, sizeof(saverx));
250 if_inc_counter(dev, IFCOUNTER_IQDROPS, 1);
251 goto repost;
252 }
253
254 ipoib_dbg_data(priv, "received %d bytes, SLID 0x%04x\n",
255 wc->byte_len, wc->slid);
256
257 ipoib_dma_unmap_rx(priv, &saverx);
258 ipoib_dma_mb(priv, mb, wc->byte_len);
259
260 if_inc_counter(dev, IFCOUNTER_IPACKETS, 1);
261 if_inc_counter(dev, IFCOUNTER_IBYTES, mb->m_pkthdr.len);
262 mb->m_pkthdr.rcvif = dev;
263 m_adj(mb, sizeof(struct ib_grh) - INFINIBAND_ALEN);
264 eh = mtod(mb, struct ipoib_header *);
265 bzero(eh->hwaddr, 4); /* Zero the queue pair, only dgid is in grh */
266
267 if (test_bit(IPOIB_FLAG_CSUM, &priv->flags) && likely(wc->wc_flags & IB_WC_IP_CSUM_OK))
268 mb->m_pkthdr.csum_flags = CSUM_IP_CHECKED | CSUM_IP_VALID;
269
270 if_input(dev, mb);
271
272 repost:
273 if (unlikely(ipoib_ib_post_receive(priv, wr_id)))
274 ipoib_warn(priv, "ipoib_ib_post_receive failed "
275 "for buf %d\n", wr_id);
276 }
277
ipoib_dma_map_tx(struct ib_device * ca,struct ipoib_tx_buf * tx_req,int max)278 int ipoib_dma_map_tx(struct ib_device *ca, struct ipoib_tx_buf *tx_req, int max)
279 {
280 struct mbuf *mb = tx_req->mb;
281 u64 *mapping = tx_req->mapping;
282 struct mbuf *m, *p;
283 int error;
284 int i;
285
286 for (m = mb, p = NULL, i = 0; m != NULL; p = m, m = m->m_next, i++) {
287 if (m->m_len != 0)
288 continue;
289 if (p == NULL)
290 panic("ipoib_dma_map_tx: First mbuf empty\n");
291 p->m_next = m_free(m);
292 m = p;
293 i--;
294 }
295 i--;
296 if (i >= max) {
297 tx_req->mb = mb = m_defrag(mb, M_NOWAIT);
298 if (mb == NULL)
299 return -EIO;
300 for (m = mb, i = 0; m != NULL; m = m->m_next, i++);
301 if (i >= max)
302 return -EIO;
303 }
304 error = 0;
305 for (m = mb, i = 0; m != NULL; m = m->m_next, i++) {
306 mapping[i] = ib_dma_map_single(ca, mtod(m, void *),
307 m->m_len, DMA_TO_DEVICE);
308 if (unlikely(ib_dma_mapping_error(ca, mapping[i]))) {
309 error = -EIO;
310 break;
311 }
312 }
313 if (error) {
314 int end;
315
316 end = i;
317 for (m = mb, i = 0; i < end; m = m->m_next, i++)
318 ib_dma_unmap_single(ca, mapping[i], m->m_len,
319 DMA_TO_DEVICE);
320 }
321 return error;
322 }
323
ipoib_dma_unmap_tx(struct ib_device * ca,struct ipoib_tx_buf * tx_req)324 void ipoib_dma_unmap_tx(struct ib_device *ca, struct ipoib_tx_buf *tx_req)
325 {
326 struct mbuf *mb = tx_req->mb;
327 u64 *mapping = tx_req->mapping;
328 struct mbuf *m;
329 int i;
330
331 for (m = mb, i = 0; m != NULL; m = m->m_next, i++)
332 ib_dma_unmap_single(ca, mapping[i], m->m_len, DMA_TO_DEVICE);
333 }
334
ipoib_ib_handle_tx_wc(struct ipoib_dev_priv * priv,struct ib_wc * wc)335 static void ipoib_ib_handle_tx_wc(struct ipoib_dev_priv *priv, struct ib_wc *wc)
336 {
337 if_t dev = priv->dev;
338 unsigned int wr_id = wc->wr_id;
339 struct ipoib_tx_buf *tx_req;
340
341 ipoib_dbg_data(priv, "send completion: id %d, status: %d\n",
342 wr_id, wc->status);
343
344 if (unlikely(wr_id >= ipoib_sendq_size)) {
345 ipoib_warn(priv, "send completion event with wrid %d (> %d)\n",
346 wr_id, ipoib_sendq_size);
347 return;
348 }
349
350 tx_req = &priv->tx_ring[wr_id];
351
352 ipoib_dma_unmap_tx(priv->ca, tx_req);
353
354 if_inc_counter(dev, IFCOUNTER_OPACKETS, 1);
355
356 m_freem(tx_req->mb);
357
358 ++priv->tx_tail;
359 if (unlikely(--priv->tx_outstanding == ipoib_sendq_size >> 1) &&
360 (if_getdrvflags(dev) & IFF_DRV_OACTIVE) &&
361 test_bit(IPOIB_FLAG_ADMIN_UP, &priv->flags))
362 if_setdrvflagbits(dev, 0, IFF_DRV_OACTIVE);
363
364 if (wc->status != IB_WC_SUCCESS &&
365 wc->status != IB_WC_WR_FLUSH_ERR)
366 ipoib_warn(priv, "failed send event "
367 "(status=%d, wrid=%d vend_err %x)\n",
368 wc->status, wr_id, wc->vendor_err);
369 }
370
371 int
ipoib_poll_tx(struct ipoib_dev_priv * priv,bool do_start)372 ipoib_poll_tx(struct ipoib_dev_priv *priv, bool do_start)
373 {
374 int n, i;
375
376 n = ib_poll_cq(priv->send_cq, MAX_SEND_CQE, priv->send_wc);
377 for (i = 0; i < n; ++i) {
378 struct ib_wc *wc = priv->send_wc + i;
379 if (wc->wr_id & IPOIB_OP_CM)
380 ipoib_cm_handle_tx_wc(priv, wc);
381 else
382 ipoib_ib_handle_tx_wc(priv, wc);
383 }
384
385 if (do_start && n != 0)
386 ipoib_start_locked(priv->dev, priv);
387
388 return n == MAX_SEND_CQE;
389 }
390
391 static void
ipoib_poll(struct ipoib_dev_priv * priv)392 ipoib_poll(struct ipoib_dev_priv *priv)
393 {
394 int n, i;
395
396 poll_more:
397 spin_lock(&priv->drain_lock);
398 for (;;) {
399 n = ib_poll_cq(priv->recv_cq, IPOIB_NUM_WC, priv->ibwc);
400 for (i = 0; i < n; i++) {
401 struct ib_wc *wc = priv->ibwc + i;
402
403 if ((wc->wr_id & IPOIB_OP_RECV) == 0)
404 panic("ipoib_poll: Bad wr_id 0x%jX\n",
405 (intmax_t)wc->wr_id);
406 if (wc->wr_id & IPOIB_OP_CM)
407 ipoib_cm_handle_rx_wc(priv, wc);
408 else
409 ipoib_ib_handle_rx_wc(priv, wc);
410 }
411
412 if (n != IPOIB_NUM_WC)
413 break;
414 }
415 spin_unlock(&priv->drain_lock);
416
417 if (ib_req_notify_cq(priv->recv_cq,
418 IB_CQ_NEXT_COMP | IB_CQ_REPORT_MISSED_EVENTS) > 0)
419 goto poll_more;
420 }
421
ipoib_ib_completion(struct ib_cq * cq,void * dev_ptr)422 void ipoib_ib_completion(struct ib_cq *cq, void *dev_ptr)
423 {
424 struct ipoib_dev_priv *priv = dev_ptr;
425
426 ipoib_poll(priv);
427 }
428
drain_tx_cq(struct ipoib_dev_priv * priv)429 static void drain_tx_cq(struct ipoib_dev_priv *priv)
430 {
431 if_t dev = priv->dev;
432
433 spin_lock(&priv->lock);
434 while (ipoib_poll_tx(priv, true))
435 ; /* nothing */
436
437 if (if_getdrvflags(dev) & IFF_DRV_OACTIVE)
438 mod_timer(&priv->poll_timer, jiffies + 1);
439
440 spin_unlock(&priv->lock);
441 }
442
ipoib_send_comp_handler(struct ib_cq * cq,void * dev_ptr)443 void ipoib_send_comp_handler(struct ib_cq *cq, void *dev_ptr)
444 {
445 struct ipoib_dev_priv *priv = dev_ptr;
446
447 mod_timer(&priv->poll_timer, jiffies);
448 }
449
450 static inline int
post_send(struct ipoib_dev_priv * priv,unsigned int wr_id,struct ib_ah * address,u32 qpn,struct ipoib_tx_buf * tx_req,void * head,int hlen)451 post_send(struct ipoib_dev_priv *priv, unsigned int wr_id,
452 struct ib_ah *address, u32 qpn, struct ipoib_tx_buf *tx_req, void *head,
453 int hlen)
454 {
455 struct mbuf *mb = tx_req->mb;
456 u64 *mapping = tx_req->mapping;
457 struct mbuf *m;
458 int i;
459
460 for (m = mb, i = 0; m != NULL; m = m->m_next, i++) {
461 priv->tx_sge[i].addr = mapping[i];
462 priv->tx_sge[i].length = m->m_len;
463 }
464 priv->tx_wr.wr.num_sge = i;
465 priv->tx_wr.wr.wr_id = wr_id;
466 priv->tx_wr.remote_qpn = qpn;
467 priv->tx_wr.ah = address;
468
469 if (head) {
470 priv->tx_wr.mss = 0; /* XXX mb_shinfo(mb)->gso_size; */
471 priv->tx_wr.header = head;
472 priv->tx_wr.hlen = hlen;
473 priv->tx_wr.wr.opcode = IB_WR_LSO;
474 } else
475 priv->tx_wr.wr.opcode = IB_WR_SEND;
476
477 return ib_post_send(priv->qp, &priv->tx_wr.wr, NULL);
478 }
479
480 void
ipoib_send(struct ipoib_dev_priv * priv,struct mbuf * mb,struct ipoib_ah * address,u32 qpn)481 ipoib_send(struct ipoib_dev_priv *priv, struct mbuf *mb,
482 struct ipoib_ah *address, u32 qpn)
483 {
484 if_t dev = priv->dev;
485 struct ipoib_tx_buf *tx_req;
486 int hlen;
487 void *phead;
488
489 if (unlikely(priv->tx_outstanding > MAX_SEND_CQE))
490 while (ipoib_poll_tx(priv, false))
491 ; /* nothing */
492
493 m_adj(mb, sizeof (struct ipoib_pseudoheader));
494 if (0 /* XXX segment offload mb_is_gso(mb) */) {
495 /* XXX hlen = mb_transport_offset(mb) + tcp_hdrlen(mb); */
496 phead = mtod(mb, void *);
497 if (mb->m_len < hlen) {
498 ipoib_warn(priv, "linear data too small\n");
499 if_inc_counter(dev, IFCOUNTER_OERRORS, 1);
500 m_freem(mb);
501 return;
502 }
503 m_adj(mb, hlen);
504 } else {
505 if (unlikely(mb->m_pkthdr.len - IPOIB_ENCAP_LEN > priv->mcast_mtu)) {
506 ipoib_warn(priv, "packet len %d (> %d) too long to send, dropping\n",
507 mb->m_pkthdr.len, priv->mcast_mtu);
508 if_inc_counter(dev, IFCOUNTER_OERRORS, 1);
509 ipoib_cm_mb_too_long(priv, mb, priv->mcast_mtu);
510 return;
511 }
512 phead = NULL;
513 hlen = 0;
514 }
515
516 ipoib_dbg_data(priv, "sending packet, length=%d address=%p qpn=0x%06x\n",
517 mb->m_pkthdr.len, address, qpn);
518
519 /*
520 * We put the mb into the tx_ring _before_ we call post_send()
521 * because it's entirely possible that the completion handler will
522 * run before we execute anything after the post_send(). That
523 * means we have to make sure everything is properly recorded and
524 * our state is consistent before we call post_send().
525 */
526 tx_req = &priv->tx_ring[priv->tx_head & (ipoib_sendq_size - 1)];
527 tx_req->mb = mb;
528 if (unlikely(ipoib_dma_map_tx(priv->ca, tx_req, IPOIB_UD_TX_SG))) {
529 if_inc_counter(dev, IFCOUNTER_OERRORS, 1);
530 if (tx_req->mb)
531 m_freem(tx_req->mb);
532 return;
533 }
534
535 if (mb->m_pkthdr.csum_flags & (CSUM_IP|CSUM_TCP|CSUM_UDP))
536 priv->tx_wr.wr.send_flags |= IB_SEND_IP_CSUM;
537 else
538 priv->tx_wr.wr.send_flags &= ~IB_SEND_IP_CSUM;
539
540 if (++priv->tx_outstanding == ipoib_sendq_size) {
541 ipoib_dbg(priv, "TX ring full, stopping kernel net queue\n");
542 if (ib_req_notify_cq(priv->send_cq, IB_CQ_NEXT_COMP))
543 ipoib_warn(priv, "request notify on send CQ failed\n");
544 if_setdrvflagbits(dev, IFF_DRV_OACTIVE, 0);
545 }
546
547 if (unlikely(post_send(priv,
548 priv->tx_head & (ipoib_sendq_size - 1), address->ah, qpn,
549 tx_req, phead, hlen))) {
550 ipoib_warn(priv, "post_send failed\n");
551 if_inc_counter(dev, IFCOUNTER_OERRORS, 1);
552 --priv->tx_outstanding;
553 ipoib_dma_unmap_tx(priv->ca, tx_req);
554 m_freem(mb);
555 if (if_getdrvflags(dev) & IFF_DRV_OACTIVE)
556 if_setdrvflagbits(dev, 0, IFF_DRV_OACTIVE);
557 } else {
558 address->last_send = priv->tx_head;
559 ++priv->tx_head;
560 }
561 }
562
__ipoib_reap_ah(struct ipoib_dev_priv * priv)563 static void __ipoib_reap_ah(struct ipoib_dev_priv *priv)
564 {
565 struct ipoib_ah *ah, *tah;
566 LIST_HEAD(remove_list);
567 unsigned long flags;
568
569 spin_lock_irqsave(&priv->lock, flags);
570
571 list_for_each_entry_safe(ah, tah, &priv->dead_ahs, list)
572 if ((int) priv->tx_tail - (int) ah->last_send >= 0) {
573 list_del(&ah->list);
574 rdma_destroy_ah(ah->ah, 0);
575 kfree(ah);
576 }
577
578 spin_unlock_irqrestore(&priv->lock, flags);
579 }
580
ipoib_reap_ah(struct work_struct * work)581 void ipoib_reap_ah(struct work_struct *work)
582 {
583 struct ipoib_dev_priv *priv =
584 container_of(work, struct ipoib_dev_priv, ah_reap_task.work);
585
586 __ipoib_reap_ah(priv);
587
588 if (!test_bit(IPOIB_STOP_REAPER, &priv->flags))
589 queue_delayed_work(ipoib_workqueue, &priv->ah_reap_task,
590 HZ);
591 }
592
ipoib_ah_dev_cleanup(struct ipoib_dev_priv * priv)593 static void ipoib_ah_dev_cleanup(struct ipoib_dev_priv *priv)
594 {
595 unsigned long begin;
596
597 begin = jiffies;
598
599 while (!list_empty(&priv->dead_ahs)) {
600 __ipoib_reap_ah(priv);
601
602 if (time_after(jiffies, begin + HZ)) {
603 ipoib_warn(priv, "timing out; will leak address handles\n");
604 break;
605 }
606
607 msleep(1);
608 }
609 }
610
ipoib_ib_tx_timer_func(struct timer_list * timer)611 static void ipoib_ib_tx_timer_func(struct timer_list *timer)
612 {
613 struct ipoib_dev_priv *priv = timer_container_of(priv, timer,
614 poll_timer);
615
616 drain_tx_cq(priv);
617 }
618
ipoib_ib_dev_open(struct ipoib_dev_priv * priv)619 int ipoib_ib_dev_open(struct ipoib_dev_priv *priv)
620 {
621 int ret;
622
623 if (ib_find_pkey(priv->ca, priv->port, priv->pkey, &priv->pkey_index)) {
624 ipoib_warn(priv, "P_Key 0x%04x not found\n", priv->pkey);
625 clear_bit(IPOIB_PKEY_ASSIGNED, &priv->flags);
626 return -1;
627 }
628 set_bit(IPOIB_PKEY_ASSIGNED, &priv->flags);
629
630 ret = ipoib_init_qp(priv);
631 if (ret) {
632 ipoib_warn(priv, "ipoib_init_qp returned %d\n", ret);
633 return -1;
634 }
635
636 ret = ipoib_ib_post_receives(priv);
637 if (ret) {
638 ipoib_warn(priv, "ipoib_ib_post_receives returned %d\n", ret);
639 ipoib_ib_dev_stop(priv, 1);
640 return -1;
641 }
642
643 ret = ipoib_cm_dev_open(priv);
644 if (ret) {
645 ipoib_warn(priv, "ipoib_cm_dev_open returned %d\n", ret);
646 ipoib_ib_dev_stop(priv, 1);
647 return -1;
648 }
649
650 clear_bit(IPOIB_STOP_REAPER, &priv->flags);
651 queue_delayed_work(ipoib_workqueue, &priv->ah_reap_task, HZ);
652
653 set_bit(IPOIB_FLAG_INITIALIZED, &priv->flags);
654
655 return 0;
656 }
657
ipoib_pkey_dev_check_presence(struct ipoib_dev_priv * priv)658 static void ipoib_pkey_dev_check_presence(struct ipoib_dev_priv *priv)
659 {
660 u16 pkey_index = 0;
661
662 if (ib_find_pkey(priv->ca, priv->port, priv->pkey, &pkey_index))
663 clear_bit(IPOIB_PKEY_ASSIGNED, &priv->flags);
664 else
665 set_bit(IPOIB_PKEY_ASSIGNED, &priv->flags);
666 }
667
ipoib_ib_dev_up(struct ipoib_dev_priv * priv)668 int ipoib_ib_dev_up(struct ipoib_dev_priv *priv)
669 {
670
671 ipoib_pkey_dev_check_presence(priv);
672
673 if (!test_bit(IPOIB_PKEY_ASSIGNED, &priv->flags)) {
674 ipoib_dbg(priv, "PKEY is not assigned.\n");
675 return 0;
676 }
677
678 set_bit(IPOIB_FLAG_OPER_UP, &priv->flags);
679
680 return ipoib_mcast_start_thread(priv);
681 }
682
ipoib_ib_dev_down(struct ipoib_dev_priv * priv,int flush)683 int ipoib_ib_dev_down(struct ipoib_dev_priv *priv, int flush)
684 {
685
686 ipoib_dbg(priv, "downing ib_dev\n");
687
688 clear_bit(IPOIB_FLAG_OPER_UP, &priv->flags);
689 if_link_state_change(priv->dev, LINK_STATE_DOWN);
690
691 /* Shutdown the P_Key thread if still active */
692 if (!test_bit(IPOIB_PKEY_ASSIGNED, &priv->flags)) {
693 mutex_lock(&pkey_mutex);
694 set_bit(IPOIB_PKEY_STOP, &priv->flags);
695 cancel_delayed_work(&priv->pkey_poll_task);
696 mutex_unlock(&pkey_mutex);
697 if (flush)
698 flush_workqueue(ipoib_workqueue);
699 }
700
701 ipoib_mcast_stop_thread(priv, flush);
702 ipoib_mcast_dev_flush(priv);
703
704 ipoib_flush_paths(priv);
705
706 return 0;
707 }
708
recvs_pending(struct ipoib_dev_priv * priv)709 static int recvs_pending(struct ipoib_dev_priv *priv)
710 {
711 int pending = 0;
712 int i;
713
714 for (i = 0; i < ipoib_recvq_size; ++i)
715 if (priv->rx_ring[i].mb)
716 ++pending;
717
718 return pending;
719 }
720
check_qp_movement_and_print(struct ipoib_dev_priv * priv,struct ib_qp * qp,enum ib_qp_state new_state)721 static void check_qp_movement_and_print(struct ipoib_dev_priv *priv,
722 struct ib_qp *qp,
723 enum ib_qp_state new_state)
724 {
725 struct ib_qp_attr qp_attr;
726 struct ib_qp_init_attr query_init_attr;
727 int ret;
728
729 ret = ib_query_qp(qp, &qp_attr, IB_QP_STATE, &query_init_attr);
730 if (ret) {
731 ipoib_warn(priv, "%s: Failed to query QP (%d)\n", __func__, ret);
732 return;
733 }
734
735 /* print according to the new-state and the previous state */
736 if (new_state == IB_QPS_ERR && qp_attr.qp_state == IB_QPS_RESET) {
737 ipoib_dbg(priv, "Failed to modify QP %d->%d, acceptable\n",
738 qp_attr.qp_state, new_state);
739 } else {
740 ipoib_warn(priv, "Failed to modify QP %d->%d\n",
741 qp_attr.qp_state, new_state);
742 }
743 }
744
ipoib_drain_cq(struct ipoib_dev_priv * priv)745 void ipoib_drain_cq(struct ipoib_dev_priv *priv)
746 {
747 int i, n;
748
749 spin_lock(&priv->drain_lock);
750 do {
751 n = ib_poll_cq(priv->recv_cq, IPOIB_NUM_WC, priv->ibwc);
752 for (i = 0; i < n; ++i) {
753 /*
754 * Convert any successful completions to flush
755 * errors to avoid passing packets up the
756 * stack after bringing the device down.
757 */
758 if (priv->ibwc[i].status == IB_WC_SUCCESS)
759 priv->ibwc[i].status = IB_WC_WR_FLUSH_ERR;
760
761 if ((priv->ibwc[i].wr_id & IPOIB_OP_RECV) == 0)
762 panic("ipoib_drain_cq: Bad wrid 0x%jX\n",
763 (intmax_t)priv->ibwc[i].wr_id);
764 if (priv->ibwc[i].wr_id & IPOIB_OP_CM)
765 ipoib_cm_handle_rx_wc(priv, priv->ibwc + i);
766 else
767 ipoib_ib_handle_rx_wc(priv, priv->ibwc + i);
768 }
769 } while (n == IPOIB_NUM_WC);
770 spin_unlock(&priv->drain_lock);
771
772 spin_lock(&priv->lock);
773 while (ipoib_poll_tx(priv, true))
774 ; /* nothing */
775
776 spin_unlock(&priv->lock);
777 }
778
ipoib_ib_dev_stop(struct ipoib_dev_priv * priv,int flush)779 int ipoib_ib_dev_stop(struct ipoib_dev_priv *priv, int flush)
780 {
781 struct ib_qp_attr qp_attr;
782 unsigned long begin;
783 struct ipoib_tx_buf *tx_req;
784 int i;
785
786 clear_bit(IPOIB_FLAG_INITIALIZED, &priv->flags);
787
788 ipoib_cm_dev_stop(priv);
789
790 /*
791 * Move our QP to the error state and then reinitialize in
792 * when all work requests have completed or have been flushed.
793 */
794 qp_attr.qp_state = IB_QPS_ERR;
795 if (ib_modify_qp(priv->qp, &qp_attr, IB_QP_STATE))
796 check_qp_movement_and_print(priv, priv->qp, IB_QPS_ERR);
797
798 /* Wait for all sends and receives to complete */
799 begin = jiffies;
800
801 while (priv->tx_head != priv->tx_tail || recvs_pending(priv)) {
802 if (time_after(jiffies, begin + 5 * HZ)) {
803 ipoib_warn(priv, "timing out; %d sends %d receives not completed\n",
804 priv->tx_head - priv->tx_tail, recvs_pending(priv));
805
806 /*
807 * assume the HW is wedged and just free up
808 * all our pending work requests.
809 */
810 while ((int) priv->tx_tail - (int) priv->tx_head < 0) {
811 tx_req = &priv->tx_ring[priv->tx_tail &
812 (ipoib_sendq_size - 1)];
813 ipoib_dma_unmap_tx(priv->ca, tx_req);
814 m_freem(tx_req->mb);
815 ++priv->tx_tail;
816 --priv->tx_outstanding;
817 }
818
819 for (i = 0; i < ipoib_recvq_size; ++i) {
820 struct ipoib_rx_buf *rx_req;
821
822 rx_req = &priv->rx_ring[i];
823 if (!rx_req->mb)
824 continue;
825 ipoib_dma_unmap_rx(priv, &priv->rx_ring[i]);
826 m_freem(rx_req->mb);
827 rx_req->mb = NULL;
828 }
829
830 goto timeout;
831 }
832
833 ipoib_drain_cq(priv);
834
835 msleep(1);
836 }
837
838 ipoib_dbg(priv, "All sends and receives done.\n");
839
840 timeout:
841 del_timer_sync(&priv->poll_timer);
842 qp_attr.qp_state = IB_QPS_RESET;
843 if (ib_modify_qp(priv->qp, &qp_attr, IB_QP_STATE))
844 ipoib_warn(priv, "Failed to modify QP to RESET state\n");
845
846 /* Wait for all AHs to be reaped */
847 set_bit(IPOIB_STOP_REAPER, &priv->flags);
848 cancel_delayed_work(&priv->ah_reap_task);
849 if (flush)
850 flush_workqueue(ipoib_workqueue);
851
852 ipoib_ah_dev_cleanup(priv);
853
854 ib_req_notify_cq(priv->recv_cq, IB_CQ_NEXT_COMP);
855
856 return 0;
857 }
858
ipoib_ib_dev_init(struct ipoib_dev_priv * priv,struct ib_device * ca,int port)859 int ipoib_ib_dev_init(struct ipoib_dev_priv *priv, struct ib_device *ca, int port)
860 {
861 if_t dev = priv->dev;
862
863 priv->ca = ca;
864 priv->port = port;
865 priv->qp = NULL;
866
867 if (ipoib_transport_dev_init(priv, ca)) {
868 printk(KERN_WARNING "%s: ipoib_transport_dev_init failed\n", ca->name);
869 return -ENODEV;
870 }
871
872 timer_setup(&priv->poll_timer, ipoib_ib_tx_timer_func, 0);
873
874 if (if_getflags(dev) & IFF_UP) {
875 if (ipoib_ib_dev_open(priv)) {
876 ipoib_transport_dev_cleanup(priv);
877 return -ENODEV;
878 }
879 }
880
881 return 0;
882 }
883
__ipoib_ib_dev_flush(struct ipoib_dev_priv * priv,enum ipoib_flush_level level)884 static void __ipoib_ib_dev_flush(struct ipoib_dev_priv *priv,
885 enum ipoib_flush_level level)
886 {
887 struct ipoib_dev_priv *cpriv;
888 u16 new_index;
889
890 mutex_lock(&priv->vlan_mutex);
891
892 /*
893 * Flush any child interfaces too -- they might be up even if
894 * the parent is down.
895 */
896 list_for_each_entry(cpriv, &priv->child_intfs, list)
897 __ipoib_ib_dev_flush(cpriv, level);
898
899 mutex_unlock(&priv->vlan_mutex);
900
901 if (!test_bit(IPOIB_FLAG_INITIALIZED, &priv->flags)) {
902 ipoib_dbg(priv, "Not flushing - IPOIB_FLAG_INITIALIZED not set.\n");
903 return;
904 }
905
906 if (!test_bit(IPOIB_FLAG_ADMIN_UP, &priv->flags)) {
907 ipoib_dbg(priv, "Not flushing - IPOIB_FLAG_ADMIN_UP not set.\n");
908 return;
909 }
910
911 if (level == IPOIB_FLUSH_HEAVY) {
912 if (ib_find_pkey(priv->ca, priv->port, priv->pkey, &new_index)) {
913 clear_bit(IPOIB_PKEY_ASSIGNED, &priv->flags);
914 ipoib_ib_dev_down(priv, 0);
915 ipoib_ib_dev_stop(priv, 0);
916 if (ipoib_pkey_dev_delay_open(priv))
917 return;
918 }
919
920 /* restart QP only if P_Key index is changed */
921 if (test_and_set_bit(IPOIB_PKEY_ASSIGNED, &priv->flags) &&
922 new_index == priv->pkey_index) {
923 ipoib_dbg(priv, "Not flushing - P_Key index not changed.\n");
924 return;
925 }
926 priv->pkey_index = new_index;
927 }
928
929 if (level == IPOIB_FLUSH_LIGHT) {
930 ipoib_mark_paths_invalid(priv);
931 ipoib_mcast_dev_flush(priv);
932 }
933
934 if (level >= IPOIB_FLUSH_NORMAL)
935 ipoib_ib_dev_down(priv, 0);
936
937 if (level == IPOIB_FLUSH_HEAVY) {
938 ipoib_ib_dev_stop(priv, 0);
939 ipoib_ib_dev_open(priv);
940 }
941
942 /*
943 * The device could have been brought down between the start and when
944 * we get here, don't bring it back up if it's not configured up
945 */
946 if (test_bit(IPOIB_FLAG_ADMIN_UP, &priv->flags)) {
947 if (level >= IPOIB_FLUSH_NORMAL)
948 ipoib_ib_dev_up(priv);
949 ipoib_mcast_restart_task(&priv->restart_task);
950 }
951 }
952
ipoib_ib_dev_flush_light(struct work_struct * work)953 void ipoib_ib_dev_flush_light(struct work_struct *work)
954 {
955 struct ipoib_dev_priv *priv =
956 container_of(work, struct ipoib_dev_priv, flush_light);
957
958 __ipoib_ib_dev_flush(priv, IPOIB_FLUSH_LIGHT);
959 }
960
ipoib_ib_dev_flush_normal(struct work_struct * work)961 void ipoib_ib_dev_flush_normal(struct work_struct *work)
962 {
963 struct ipoib_dev_priv *priv =
964 container_of(work, struct ipoib_dev_priv, flush_normal);
965
966 __ipoib_ib_dev_flush(priv, IPOIB_FLUSH_NORMAL);
967 }
968
ipoib_ib_dev_flush_heavy(struct work_struct * work)969 void ipoib_ib_dev_flush_heavy(struct work_struct *work)
970 {
971 struct ipoib_dev_priv *priv =
972 container_of(work, struct ipoib_dev_priv, flush_heavy);
973
974 __ipoib_ib_dev_flush(priv, IPOIB_FLUSH_HEAVY);
975 }
976
ipoib_ib_dev_cleanup(struct ipoib_dev_priv * priv)977 void ipoib_ib_dev_cleanup(struct ipoib_dev_priv *priv)
978 {
979
980 ipoib_dbg(priv, "cleaning up ib_dev\n");
981
982 ipoib_mcast_stop_thread(priv, 1);
983 ipoib_mcast_dev_flush(priv);
984
985 ipoib_ah_dev_cleanup(priv);
986 ipoib_transport_dev_cleanup(priv);
987 }
988
989 /*
990 * Delayed P_Key Assigment Interim Support
991 *
992 * The following is initial implementation of delayed P_Key assigment
993 * mechanism. It is using the same approach implemented for the multicast
994 * group join. The single goal of this implementation is to quickly address
995 * Bug #2507. This implementation will probably be removed when the P_Key
996 * change async notification is available.
997 */
998
ipoib_pkey_poll(struct work_struct * work)999 void ipoib_pkey_poll(struct work_struct *work)
1000 {
1001 struct ipoib_dev_priv *priv =
1002 container_of(work, struct ipoib_dev_priv, pkey_poll_task.work);
1003
1004 ipoib_pkey_dev_check_presence(priv);
1005
1006 if (test_bit(IPOIB_PKEY_ASSIGNED, &priv->flags))
1007 ipoib_open(priv);
1008 else {
1009 mutex_lock(&pkey_mutex);
1010 if (!test_bit(IPOIB_PKEY_STOP, &priv->flags))
1011 queue_delayed_work(ipoib_workqueue,
1012 &priv->pkey_poll_task,
1013 HZ);
1014 mutex_unlock(&pkey_mutex);
1015 }
1016 }
1017
ipoib_pkey_dev_delay_open(struct ipoib_dev_priv * priv)1018 int ipoib_pkey_dev_delay_open(struct ipoib_dev_priv *priv)
1019 {
1020
1021 /* Look for the interface pkey value in the IB Port P_Key table and */
1022 /* set the interface pkey assigment flag */
1023 ipoib_pkey_dev_check_presence(priv);
1024
1025 /* P_Key value not assigned yet - start polling */
1026 if (!test_bit(IPOIB_PKEY_ASSIGNED, &priv->flags)) {
1027 mutex_lock(&pkey_mutex);
1028 clear_bit(IPOIB_PKEY_STOP, &priv->flags);
1029 queue_delayed_work(ipoib_workqueue,
1030 &priv->pkey_poll_task,
1031 HZ);
1032 mutex_unlock(&pkey_mutex);
1033 return 1;
1034 }
1035
1036 return 0;
1037 }
1038