xref: /linux/drivers/infiniband/hw/qib/qib_sdma.c (revision 4ab5a5d2a4a2289c2af07accbec7170ca5671f41)
1 /*
2  * Copyright (c) 2012 Intel Corporation. All rights reserved.
3  * Copyright (c) 2007 - 2012 QLogic Corporation. All rights reserved.
4  *
5  * This software is available to you under a choice of one of two
6  * licenses.  You may choose to be licensed under the terms of the GNU
7  * General Public License (GPL) Version 2, available from the file
8  * COPYING in the main directory of this source tree, or the
9  * OpenIB.org BSD license below:
10  *
11  *     Redistribution and use in source and binary forms, with or
12  *     without modification, are permitted provided that the following
13  *     conditions are met:
14  *
15  *      - Redistributions of source code must retain the above
16  *        copyright notice, this list of conditions and the following
17  *        disclaimer.
18  *
19  *      - Redistributions in binary form must reproduce the above
20  *        copyright notice, this list of conditions and the following
21  *        disclaimer in the documentation and/or other materials
22  *        provided with the distribution.
23  *
24  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31  * SOFTWARE.
32  */
33 
34 #include <linux/spinlock.h>
35 #include <linux/netdevice.h>
36 #include <linux/moduleparam.h>
37 
38 #include "qib.h"
39 #include "qib_common.h"
40 
41 /* default pio off, sdma on */
42 static ushort sdma_descq_cnt = 256;
43 module_param_named(sdma_descq_cnt, sdma_descq_cnt, ushort, S_IRUGO);
44 MODULE_PARM_DESC(sdma_descq_cnt, "Number of SDMA descq entries");
45 
46 /*
47  * Bits defined in the send DMA descriptor.
48  */
49 #define SDMA_DESC_LAST          (1ULL << 11)
50 #define SDMA_DESC_FIRST         (1ULL << 12)
51 #define SDMA_DESC_DMA_HEAD      (1ULL << 13)
52 #define SDMA_DESC_USE_LARGE_BUF (1ULL << 14)
53 #define SDMA_DESC_INTR          (1ULL << 15)
54 #define SDMA_DESC_COUNT_LSB     16
55 #define SDMA_DESC_GEN_LSB       30
56 
57 /* declare all statics here rather than keep sorting */
58 static int alloc_sdma(struct qib_pportdata *);
59 static void sdma_complete(struct kref *);
60 static void sdma_finalput(struct qib_sdma_state *);
61 static void sdma_get(struct qib_sdma_state *);
62 static void sdma_put(struct qib_sdma_state *);
63 static void sdma_set_state(struct qib_pportdata *, enum qib_sdma_states);
64 static void sdma_start_sw_clean_up(struct qib_pportdata *);
65 static void sdma_sw_clean_up_task(unsigned long);
66 static void unmap_desc(struct qib_pportdata *, unsigned);
67 
68 static void sdma_get(struct qib_sdma_state *ss)
69 {
70 	kref_get(&ss->kref);
71 }
72 
73 static void sdma_complete(struct kref *kref)
74 {
75 	struct qib_sdma_state *ss =
76 		container_of(kref, struct qib_sdma_state, kref);
77 
78 	complete(&ss->comp);
79 }
80 
81 static void sdma_put(struct qib_sdma_state *ss)
82 {
83 	kref_put(&ss->kref, sdma_complete);
84 }
85 
86 static void sdma_finalput(struct qib_sdma_state *ss)
87 {
88 	sdma_put(ss);
89 	wait_for_completion(&ss->comp);
90 }
91 
92 /*
93  * Complete all the sdma requests on the active list, in the correct
94  * order, and with appropriate processing.   Called when cleaning up
95  * after sdma shutdown, and when new sdma requests are submitted for
96  * a link that is down.   This matches what is done for requests
97  * that complete normally, it's just the full list.
98  *
99  * Must be called with sdma_lock held
100  */
101 static void clear_sdma_activelist(struct qib_pportdata *ppd)
102 {
103 	struct qib_sdma_txreq *txp, *txp_next;
104 
105 	list_for_each_entry_safe(txp, txp_next, &ppd->sdma_activelist, list) {
106 		list_del_init(&txp->list);
107 		if (txp->flags & QIB_SDMA_TXREQ_F_FREEDESC) {
108 			unsigned idx;
109 
110 			idx = txp->start_idx;
111 			while (idx != txp->next_descq_idx) {
112 				unmap_desc(ppd, idx);
113 				if (++idx == ppd->sdma_descq_cnt)
114 					idx = 0;
115 			}
116 		}
117 		if (txp->callback)
118 			(*txp->callback)(txp, QIB_SDMA_TXREQ_S_ABORTED);
119 	}
120 }
121 
122 static void sdma_sw_clean_up_task(unsigned long opaque)
123 {
124 	struct qib_pportdata *ppd = (struct qib_pportdata *) opaque;
125 	unsigned long flags;
126 
127 	spin_lock_irqsave(&ppd->sdma_lock, flags);
128 
129 	/*
130 	 * At this point, the following should always be true:
131 	 * - We are halted, so no more descriptors are getting retired.
132 	 * - We are not running, so no one is submitting new work.
133 	 * - Only we can send the e40_sw_cleaned, so we can't start
134 	 *   running again until we say so.  So, the active list and
135 	 *   descq are ours to play with.
136 	 */
137 
138 	/* Process all retired requests. */
139 	qib_sdma_make_progress(ppd);
140 
141 	clear_sdma_activelist(ppd);
142 
143 	/*
144 	 * Resync count of added and removed.  It is VERY important that
145 	 * sdma_descq_removed NEVER decrement - user_sdma depends on it.
146 	 */
147 	ppd->sdma_descq_removed = ppd->sdma_descq_added;
148 
149 	/*
150 	 * Reset our notion of head and tail.
151 	 * Note that the HW registers will be reset when switching states
152 	 * due to calling __qib_sdma_process_event() below.
153 	 */
154 	ppd->sdma_descq_tail = 0;
155 	ppd->sdma_descq_head = 0;
156 	ppd->sdma_head_dma[0] = 0;
157 	ppd->sdma_generation = 0;
158 
159 	__qib_sdma_process_event(ppd, qib_sdma_event_e40_sw_cleaned);
160 
161 	spin_unlock_irqrestore(&ppd->sdma_lock, flags);
162 }
163 
164 /*
165  * This is called when changing to state qib_sdma_state_s10_hw_start_up_wait
166  * as a result of send buffer errors or send DMA descriptor errors.
167  * We want to disarm the buffers in these cases.
168  */
169 static void sdma_hw_start_up(struct qib_pportdata *ppd)
170 {
171 	struct qib_sdma_state *ss = &ppd->sdma_state;
172 	unsigned bufno;
173 
174 	for (bufno = ss->first_sendbuf; bufno < ss->last_sendbuf; ++bufno)
175 		ppd->dd->f_sendctrl(ppd, QIB_SENDCTRL_DISARM_BUF(bufno));
176 
177 	ppd->dd->f_sdma_hw_start_up(ppd);
178 }
179 
180 static void sdma_sw_tear_down(struct qib_pportdata *ppd)
181 {
182 	struct qib_sdma_state *ss = &ppd->sdma_state;
183 
184 	/* Releasing this reference means the state machine has stopped. */
185 	sdma_put(ss);
186 }
187 
188 static void sdma_start_sw_clean_up(struct qib_pportdata *ppd)
189 {
190 	tasklet_hi_schedule(&ppd->sdma_sw_clean_up_task);
191 }
192 
193 static void sdma_set_state(struct qib_pportdata *ppd,
194 	enum qib_sdma_states next_state)
195 {
196 	struct qib_sdma_state *ss = &ppd->sdma_state;
197 	struct sdma_set_state_action *action = ss->set_state_action;
198 	unsigned op = 0;
199 
200 	/* debugging bookkeeping */
201 	ss->previous_state = ss->current_state;
202 	ss->previous_op = ss->current_op;
203 
204 	ss->current_state = next_state;
205 
206 	if (action[next_state].op_enable)
207 		op |= QIB_SDMA_SENDCTRL_OP_ENABLE;
208 
209 	if (action[next_state].op_intenable)
210 		op |= QIB_SDMA_SENDCTRL_OP_INTENABLE;
211 
212 	if (action[next_state].op_halt)
213 		op |= QIB_SDMA_SENDCTRL_OP_HALT;
214 
215 	if (action[next_state].op_drain)
216 		op |= QIB_SDMA_SENDCTRL_OP_DRAIN;
217 
218 	if (action[next_state].go_s99_running_tofalse)
219 		ss->go_s99_running = 0;
220 
221 	if (action[next_state].go_s99_running_totrue)
222 		ss->go_s99_running = 1;
223 
224 	ss->current_op = op;
225 
226 	ppd->dd->f_sdma_sendctrl(ppd, ss->current_op);
227 }
228 
229 static void unmap_desc(struct qib_pportdata *ppd, unsigned head)
230 {
231 	__le64 *descqp = &ppd->sdma_descq[head].qw[0];
232 	u64 desc[2];
233 	dma_addr_t addr;
234 	size_t len;
235 
236 	desc[0] = le64_to_cpu(descqp[0]);
237 	desc[1] = le64_to_cpu(descqp[1]);
238 
239 	addr = (desc[1] << 32) | (desc[0] >> 32);
240 	len = (desc[0] >> 14) & (0x7ffULL << 2);
241 	dma_unmap_single(&ppd->dd->pcidev->dev, addr, len, DMA_TO_DEVICE);
242 }
243 
244 static int alloc_sdma(struct qib_pportdata *ppd)
245 {
246 	ppd->sdma_descq_cnt = sdma_descq_cnt;
247 	if (!ppd->sdma_descq_cnt)
248 		ppd->sdma_descq_cnt = 256;
249 
250 	/* Allocate memory for SendDMA descriptor FIFO */
251 	ppd->sdma_descq = dma_alloc_coherent(&ppd->dd->pcidev->dev,
252 		ppd->sdma_descq_cnt * sizeof(u64[2]), &ppd->sdma_descq_phys,
253 		GFP_KERNEL);
254 
255 	if (!ppd->sdma_descq) {
256 		qib_dev_err(ppd->dd,
257 			"failed to allocate SendDMA descriptor FIFO memory\n");
258 		goto bail;
259 	}
260 
261 	/* Allocate memory for DMA of head register to memory */
262 	ppd->sdma_head_dma = dma_alloc_coherent(&ppd->dd->pcidev->dev,
263 		PAGE_SIZE, &ppd->sdma_head_phys, GFP_KERNEL);
264 	if (!ppd->sdma_head_dma) {
265 		qib_dev_err(ppd->dd,
266 			"failed to allocate SendDMA head memory\n");
267 		goto cleanup_descq;
268 	}
269 	ppd->sdma_head_dma[0] = 0;
270 	return 0;
271 
272 cleanup_descq:
273 	dma_free_coherent(&ppd->dd->pcidev->dev,
274 		ppd->sdma_descq_cnt * sizeof(u64[2]), (void *)ppd->sdma_descq,
275 		ppd->sdma_descq_phys);
276 	ppd->sdma_descq = NULL;
277 	ppd->sdma_descq_phys = 0;
278 bail:
279 	ppd->sdma_descq_cnt = 0;
280 	return -ENOMEM;
281 }
282 
283 static void free_sdma(struct qib_pportdata *ppd)
284 {
285 	struct qib_devdata *dd = ppd->dd;
286 
287 	if (ppd->sdma_head_dma) {
288 		dma_free_coherent(&dd->pcidev->dev, PAGE_SIZE,
289 				  (void *)ppd->sdma_head_dma,
290 				  ppd->sdma_head_phys);
291 		ppd->sdma_head_dma = NULL;
292 		ppd->sdma_head_phys = 0;
293 	}
294 
295 	if (ppd->sdma_descq) {
296 		dma_free_coherent(&dd->pcidev->dev,
297 				  ppd->sdma_descq_cnt * sizeof(u64[2]),
298 				  ppd->sdma_descq, ppd->sdma_descq_phys);
299 		ppd->sdma_descq = NULL;
300 		ppd->sdma_descq_phys = 0;
301 	}
302 }
303 
304 static inline void make_sdma_desc(struct qib_pportdata *ppd,
305 				  u64 *sdmadesc, u64 addr, u64 dwlen,
306 				  u64 dwoffset)
307 {
308 
309 	WARN_ON(addr & 3);
310 	/* SDmaPhyAddr[47:32] */
311 	sdmadesc[1] = addr >> 32;
312 	/* SDmaPhyAddr[31:0] */
313 	sdmadesc[0] = (addr & 0xfffffffcULL) << 32;
314 	/* SDmaGeneration[1:0] */
315 	sdmadesc[0] |= (ppd->sdma_generation & 3ULL) <<
316 		SDMA_DESC_GEN_LSB;
317 	/* SDmaDwordCount[10:0] */
318 	sdmadesc[0] |= (dwlen & 0x7ffULL) << SDMA_DESC_COUNT_LSB;
319 	/* SDmaBufOffset[12:2] */
320 	sdmadesc[0] |= dwoffset & 0x7ffULL;
321 }
322 
323 /* sdma_lock must be held */
324 int qib_sdma_make_progress(struct qib_pportdata *ppd)
325 {
326 	struct list_head *lp = NULL;
327 	struct qib_sdma_txreq *txp = NULL;
328 	struct qib_devdata *dd = ppd->dd;
329 	int progress = 0;
330 	u16 hwhead;
331 	u16 idx = 0;
332 
333 	hwhead = dd->f_sdma_gethead(ppd);
334 
335 	/* The reason for some of the complexity of this code is that
336 	 * not all descriptors have corresponding txps.  So, we have to
337 	 * be able to skip over descs until we wander into the range of
338 	 * the next txp on the list.
339 	 */
340 
341 	if (!list_empty(&ppd->sdma_activelist)) {
342 		lp = ppd->sdma_activelist.next;
343 		txp = list_entry(lp, struct qib_sdma_txreq, list);
344 		idx = txp->start_idx;
345 	}
346 
347 	while (ppd->sdma_descq_head != hwhead) {
348 		/* if desc is part of this txp, unmap if needed */
349 		if (txp && (txp->flags & QIB_SDMA_TXREQ_F_FREEDESC) &&
350 		    (idx == ppd->sdma_descq_head)) {
351 			unmap_desc(ppd, ppd->sdma_descq_head);
352 			if (++idx == ppd->sdma_descq_cnt)
353 				idx = 0;
354 		}
355 
356 		/* increment dequed desc count */
357 		ppd->sdma_descq_removed++;
358 
359 		/* advance head, wrap if needed */
360 		if (++ppd->sdma_descq_head == ppd->sdma_descq_cnt)
361 			ppd->sdma_descq_head = 0;
362 
363 		/* if now past this txp's descs, do the callback */
364 		if (txp && txp->next_descq_idx == ppd->sdma_descq_head) {
365 			/* remove from active list */
366 			list_del_init(&txp->list);
367 			if (txp->callback)
368 				(*txp->callback)(txp, QIB_SDMA_TXREQ_S_OK);
369 			/* see if there is another txp */
370 			if (list_empty(&ppd->sdma_activelist))
371 				txp = NULL;
372 			else {
373 				lp = ppd->sdma_activelist.next;
374 				txp = list_entry(lp, struct qib_sdma_txreq,
375 					list);
376 				idx = txp->start_idx;
377 			}
378 		}
379 		progress = 1;
380 	}
381 	if (progress)
382 		qib_verbs_sdma_desc_avail(ppd, qib_sdma_descq_freecnt(ppd));
383 	return progress;
384 }
385 
386 /*
387  * This is called from interrupt context.
388  */
389 void qib_sdma_intr(struct qib_pportdata *ppd)
390 {
391 	unsigned long flags;
392 
393 	spin_lock_irqsave(&ppd->sdma_lock, flags);
394 
395 	__qib_sdma_intr(ppd);
396 
397 	spin_unlock_irqrestore(&ppd->sdma_lock, flags);
398 }
399 
400 void __qib_sdma_intr(struct qib_pportdata *ppd)
401 {
402 	if (__qib_sdma_running(ppd)) {
403 		qib_sdma_make_progress(ppd);
404 		if (!list_empty(&ppd->sdma_userpending))
405 			qib_user_sdma_send_desc(ppd, &ppd->sdma_userpending);
406 	}
407 }
408 
409 int qib_setup_sdma(struct qib_pportdata *ppd)
410 {
411 	struct qib_devdata *dd = ppd->dd;
412 	unsigned long flags;
413 	int ret = 0;
414 
415 	ret = alloc_sdma(ppd);
416 	if (ret)
417 		goto bail;
418 
419 	/* set consistent sdma state */
420 	ppd->dd->f_sdma_init_early(ppd);
421 	spin_lock_irqsave(&ppd->sdma_lock, flags);
422 	sdma_set_state(ppd, qib_sdma_state_s00_hw_down);
423 	spin_unlock_irqrestore(&ppd->sdma_lock, flags);
424 
425 	/* set up reference counting */
426 	kref_init(&ppd->sdma_state.kref);
427 	init_completion(&ppd->sdma_state.comp);
428 
429 	ppd->sdma_generation = 0;
430 	ppd->sdma_descq_head = 0;
431 	ppd->sdma_descq_removed = 0;
432 	ppd->sdma_descq_added = 0;
433 
434 	ppd->sdma_intrequest = 0;
435 	INIT_LIST_HEAD(&ppd->sdma_userpending);
436 
437 	INIT_LIST_HEAD(&ppd->sdma_activelist);
438 
439 	tasklet_init(&ppd->sdma_sw_clean_up_task, sdma_sw_clean_up_task,
440 		(unsigned long)ppd);
441 
442 	ret = dd->f_init_sdma_regs(ppd);
443 	if (ret)
444 		goto bail_alloc;
445 
446 	qib_sdma_process_event(ppd, qib_sdma_event_e10_go_hw_start);
447 
448 	return 0;
449 
450 bail_alloc:
451 	qib_teardown_sdma(ppd);
452 bail:
453 	return ret;
454 }
455 
456 void qib_teardown_sdma(struct qib_pportdata *ppd)
457 {
458 	qib_sdma_process_event(ppd, qib_sdma_event_e00_go_hw_down);
459 
460 	/*
461 	 * This waits for the state machine to exit so it is not
462 	 * necessary to kill the sdma_sw_clean_up_task to make sure
463 	 * it is not running.
464 	 */
465 	sdma_finalput(&ppd->sdma_state);
466 
467 	free_sdma(ppd);
468 }
469 
470 int qib_sdma_running(struct qib_pportdata *ppd)
471 {
472 	unsigned long flags;
473 	int ret;
474 
475 	spin_lock_irqsave(&ppd->sdma_lock, flags);
476 	ret = __qib_sdma_running(ppd);
477 	spin_unlock_irqrestore(&ppd->sdma_lock, flags);
478 
479 	return ret;
480 }
481 
482 /*
483  * Complete a request when sdma not running; likely only request
484  * but to simplify the code, always queue it, then process the full
485  * activelist.  We process the entire list to ensure that this particular
486  * request does get it's callback, but in the correct order.
487  * Must be called with sdma_lock held
488  */
489 static void complete_sdma_err_req(struct qib_pportdata *ppd,
490 				  struct qib_verbs_txreq *tx)
491 {
492 	struct qib_qp_priv *priv = tx->qp->priv;
493 
494 	atomic_inc(&priv->s_dma_busy);
495 	/* no sdma descriptors, so no unmap_desc */
496 	tx->txreq.start_idx = 0;
497 	tx->txreq.next_descq_idx = 0;
498 	list_add_tail(&tx->txreq.list, &ppd->sdma_activelist);
499 	clear_sdma_activelist(ppd);
500 }
501 
502 /*
503  * This function queues one IB packet onto the send DMA queue per call.
504  * The caller is responsible for checking:
505  * 1) The number of send DMA descriptor entries is less than the size of
506  *    the descriptor queue.
507  * 2) The IB SGE addresses and lengths are 32-bit aligned
508  *    (except possibly the last SGE's length)
509  * 3) The SGE addresses are suitable for passing to dma_map_single().
510  */
511 int qib_sdma_verbs_send(struct qib_pportdata *ppd,
512 			struct rvt_sge_state *ss, u32 dwords,
513 			struct qib_verbs_txreq *tx)
514 {
515 	unsigned long flags;
516 	struct rvt_sge *sge;
517 	struct rvt_qp *qp;
518 	int ret = 0;
519 	u16 tail;
520 	__le64 *descqp;
521 	u64 sdmadesc[2];
522 	u32 dwoffset;
523 	dma_addr_t addr;
524 	struct qib_qp_priv *priv;
525 
526 	spin_lock_irqsave(&ppd->sdma_lock, flags);
527 
528 retry:
529 	if (unlikely(!__qib_sdma_running(ppd))) {
530 		complete_sdma_err_req(ppd, tx);
531 		goto unlock;
532 	}
533 
534 	if (tx->txreq.sg_count > qib_sdma_descq_freecnt(ppd)) {
535 		if (qib_sdma_make_progress(ppd))
536 			goto retry;
537 		if (ppd->dd->flags & QIB_HAS_SDMA_TIMEOUT)
538 			ppd->dd->f_sdma_set_desc_cnt(ppd,
539 					ppd->sdma_descq_cnt / 2);
540 		goto busy;
541 	}
542 
543 	dwoffset = tx->hdr_dwords;
544 	make_sdma_desc(ppd, sdmadesc, (u64) tx->txreq.addr, dwoffset, 0);
545 
546 	sdmadesc[0] |= SDMA_DESC_FIRST;
547 	if (tx->txreq.flags & QIB_SDMA_TXREQ_F_USELARGEBUF)
548 		sdmadesc[0] |= SDMA_DESC_USE_LARGE_BUF;
549 
550 	/* write to the descq */
551 	tail = ppd->sdma_descq_tail;
552 	descqp = &ppd->sdma_descq[tail].qw[0];
553 	*descqp++ = cpu_to_le64(sdmadesc[0]);
554 	*descqp++ = cpu_to_le64(sdmadesc[1]);
555 
556 	/* increment the tail */
557 	if (++tail == ppd->sdma_descq_cnt) {
558 		tail = 0;
559 		descqp = &ppd->sdma_descq[0].qw[0];
560 		++ppd->sdma_generation;
561 	}
562 
563 	tx->txreq.start_idx = tail;
564 
565 	sge = &ss->sge;
566 	while (dwords) {
567 		u32 dw;
568 		u32 len;
569 
570 		len = dwords << 2;
571 		if (len > sge->length)
572 			len = sge->length;
573 		if (len > sge->sge_length)
574 			len = sge->sge_length;
575 		BUG_ON(len == 0);
576 		dw = (len + 3) >> 2;
577 		addr = dma_map_single(&ppd->dd->pcidev->dev, sge->vaddr,
578 				      dw << 2, DMA_TO_DEVICE);
579 		if (dma_mapping_error(&ppd->dd->pcidev->dev, addr))
580 			goto unmap;
581 		sdmadesc[0] = 0;
582 		make_sdma_desc(ppd, sdmadesc, (u64) addr, dw, dwoffset);
583 		/* SDmaUseLargeBuf has to be set in every descriptor */
584 		if (tx->txreq.flags & QIB_SDMA_TXREQ_F_USELARGEBUF)
585 			sdmadesc[0] |= SDMA_DESC_USE_LARGE_BUF;
586 		/* write to the descq */
587 		*descqp++ = cpu_to_le64(sdmadesc[0]);
588 		*descqp++ = cpu_to_le64(sdmadesc[1]);
589 
590 		/* increment the tail */
591 		if (++tail == ppd->sdma_descq_cnt) {
592 			tail = 0;
593 			descqp = &ppd->sdma_descq[0].qw[0];
594 			++ppd->sdma_generation;
595 		}
596 		sge->vaddr += len;
597 		sge->length -= len;
598 		sge->sge_length -= len;
599 		if (sge->sge_length == 0) {
600 			if (--ss->num_sge)
601 				*sge = *ss->sg_list++;
602 		} else if (sge->length == 0 && sge->mr->lkey) {
603 			if (++sge->n >= RVT_SEGSZ) {
604 				if (++sge->m >= sge->mr->mapsz)
605 					break;
606 				sge->n = 0;
607 			}
608 			sge->vaddr =
609 				sge->mr->map[sge->m]->segs[sge->n].vaddr;
610 			sge->length =
611 				sge->mr->map[sge->m]->segs[sge->n].length;
612 		}
613 
614 		dwoffset += dw;
615 		dwords -= dw;
616 	}
617 
618 	if (!tail)
619 		descqp = &ppd->sdma_descq[ppd->sdma_descq_cnt].qw[0];
620 	descqp -= 2;
621 	descqp[0] |= cpu_to_le64(SDMA_DESC_LAST);
622 	if (tx->txreq.flags & QIB_SDMA_TXREQ_F_HEADTOHOST)
623 		descqp[0] |= cpu_to_le64(SDMA_DESC_DMA_HEAD);
624 	if (tx->txreq.flags & QIB_SDMA_TXREQ_F_INTREQ)
625 		descqp[0] |= cpu_to_le64(SDMA_DESC_INTR);
626 	priv = tx->qp->priv;
627 	atomic_inc(&priv->s_dma_busy);
628 	tx->txreq.next_descq_idx = tail;
629 	ppd->dd->f_sdma_update_tail(ppd, tail);
630 	ppd->sdma_descq_added += tx->txreq.sg_count;
631 	list_add_tail(&tx->txreq.list, &ppd->sdma_activelist);
632 	goto unlock;
633 
634 unmap:
635 	for (;;) {
636 		if (!tail)
637 			tail = ppd->sdma_descq_cnt - 1;
638 		else
639 			tail--;
640 		if (tail == ppd->sdma_descq_tail)
641 			break;
642 		unmap_desc(ppd, tail);
643 	}
644 	qp = tx->qp;
645 	priv = qp->priv;
646 	qib_put_txreq(tx);
647 	spin_lock(&qp->r_lock);
648 	spin_lock(&qp->s_lock);
649 	if (qp->ibqp.qp_type == IB_QPT_RC) {
650 		/* XXX what about error sending RDMA read responses? */
651 		if (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK)
652 			rvt_error_qp(qp, IB_WC_GENERAL_ERR);
653 	} else if (qp->s_wqe)
654 		rvt_send_complete(qp, qp->s_wqe, IB_WC_GENERAL_ERR);
655 	spin_unlock(&qp->s_lock);
656 	spin_unlock(&qp->r_lock);
657 	/* return zero to process the next send work request */
658 	goto unlock;
659 
660 busy:
661 	qp = tx->qp;
662 	priv = qp->priv;
663 	spin_lock(&qp->s_lock);
664 	if (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK) {
665 		struct qib_ibdev *dev;
666 
667 		/*
668 		 * If we couldn't queue the DMA request, save the info
669 		 * and try again later rather than destroying the
670 		 * buffer and undoing the side effects of the copy.
671 		 */
672 		tx->ss = ss;
673 		tx->dwords = dwords;
674 		priv->s_tx = tx;
675 		dev = &ppd->dd->verbs_dev;
676 		spin_lock(&dev->rdi.pending_lock);
677 		if (list_empty(&priv->iowait)) {
678 			struct qib_ibport *ibp;
679 
680 			ibp = &ppd->ibport_data;
681 			ibp->rvp.n_dmawait++;
682 			qp->s_flags |= RVT_S_WAIT_DMA_DESC;
683 			list_add_tail(&priv->iowait, &dev->dmawait);
684 		}
685 		spin_unlock(&dev->rdi.pending_lock);
686 		qp->s_flags &= ~RVT_S_BUSY;
687 		spin_unlock(&qp->s_lock);
688 		ret = -EBUSY;
689 	} else {
690 		spin_unlock(&qp->s_lock);
691 		qib_put_txreq(tx);
692 	}
693 unlock:
694 	spin_unlock_irqrestore(&ppd->sdma_lock, flags);
695 	return ret;
696 }
697 
698 /*
699  * sdma_lock should be acquired before calling this routine
700  */
701 void dump_sdma_state(struct qib_pportdata *ppd)
702 {
703 	struct qib_sdma_desc *descq;
704 	struct qib_sdma_txreq *txp, *txpnext;
705 	__le64 *descqp;
706 	u64 desc[2];
707 	u64 addr;
708 	u16 gen, dwlen, dwoffset;
709 	u16 head, tail, cnt;
710 
711 	head = ppd->sdma_descq_head;
712 	tail = ppd->sdma_descq_tail;
713 	cnt = qib_sdma_descq_freecnt(ppd);
714 	descq = ppd->sdma_descq;
715 
716 	qib_dev_porterr(ppd->dd, ppd->port,
717 		"SDMA ppd->sdma_descq_head: %u\n", head);
718 	qib_dev_porterr(ppd->dd, ppd->port,
719 		"SDMA ppd->sdma_descq_tail: %u\n", tail);
720 	qib_dev_porterr(ppd->dd, ppd->port,
721 		"SDMA sdma_descq_freecnt: %u\n", cnt);
722 
723 	/* print info for each entry in the descriptor queue */
724 	while (head != tail) {
725 		char flags[6] = { 'x', 'x', 'x', 'x', 'x', 0 };
726 
727 		descqp = &descq[head].qw[0];
728 		desc[0] = le64_to_cpu(descqp[0]);
729 		desc[1] = le64_to_cpu(descqp[1]);
730 		flags[0] = (desc[0] & 1<<15) ? 'I' : '-';
731 		flags[1] = (desc[0] & 1<<14) ? 'L' : 'S';
732 		flags[2] = (desc[0] & 1<<13) ? 'H' : '-';
733 		flags[3] = (desc[0] & 1<<12) ? 'F' : '-';
734 		flags[4] = (desc[0] & 1<<11) ? 'L' : '-';
735 		addr = (desc[1] << 32) | ((desc[0] >> 32) & 0xfffffffcULL);
736 		gen = (desc[0] >> 30) & 3ULL;
737 		dwlen = (desc[0] >> 14) & (0x7ffULL << 2);
738 		dwoffset = (desc[0] & 0x7ffULL) << 2;
739 		qib_dev_porterr(ppd->dd, ppd->port,
740 			"SDMA sdmadesc[%u]: flags:%s addr:0x%016llx gen:%u len:%u bytes offset:%u bytes\n",
741 			 head, flags, addr, gen, dwlen, dwoffset);
742 		if (++head == ppd->sdma_descq_cnt)
743 			head = 0;
744 	}
745 
746 	/* print dma descriptor indices from the TX requests */
747 	list_for_each_entry_safe(txp, txpnext, &ppd->sdma_activelist,
748 				 list)
749 		qib_dev_porterr(ppd->dd, ppd->port,
750 			"SDMA txp->start_idx: %u txp->next_descq_idx: %u\n",
751 			txp->start_idx, txp->next_descq_idx);
752 }
753 
754 void qib_sdma_process_event(struct qib_pportdata *ppd,
755 	enum qib_sdma_events event)
756 {
757 	unsigned long flags;
758 
759 	spin_lock_irqsave(&ppd->sdma_lock, flags);
760 
761 	__qib_sdma_process_event(ppd, event);
762 
763 	if (ppd->sdma_state.current_state == qib_sdma_state_s99_running)
764 		qib_verbs_sdma_desc_avail(ppd, qib_sdma_descq_freecnt(ppd));
765 
766 	spin_unlock_irqrestore(&ppd->sdma_lock, flags);
767 }
768 
769 void __qib_sdma_process_event(struct qib_pportdata *ppd,
770 	enum qib_sdma_events event)
771 {
772 	struct qib_sdma_state *ss = &ppd->sdma_state;
773 
774 	switch (ss->current_state) {
775 	case qib_sdma_state_s00_hw_down:
776 		switch (event) {
777 		case qib_sdma_event_e00_go_hw_down:
778 			break;
779 		case qib_sdma_event_e30_go_running:
780 			/*
781 			 * If down, but running requested (usually result
782 			 * of link up, then we need to start up.
783 			 * This can happen when hw down is requested while
784 			 * bringing the link up with traffic active on
785 			 * 7220, e.g. */
786 			ss->go_s99_running = 1;
787 			/* fall through -- and start dma engine */
788 		case qib_sdma_event_e10_go_hw_start:
789 			/* This reference means the state machine is started */
790 			sdma_get(&ppd->sdma_state);
791 			sdma_set_state(ppd,
792 				       qib_sdma_state_s10_hw_start_up_wait);
793 			break;
794 		case qib_sdma_event_e20_hw_started:
795 			break;
796 		case qib_sdma_event_e40_sw_cleaned:
797 			sdma_sw_tear_down(ppd);
798 			break;
799 		case qib_sdma_event_e50_hw_cleaned:
800 			break;
801 		case qib_sdma_event_e60_hw_halted:
802 			break;
803 		case qib_sdma_event_e70_go_idle:
804 			break;
805 		case qib_sdma_event_e7220_err_halted:
806 			break;
807 		case qib_sdma_event_e7322_err_halted:
808 			break;
809 		case qib_sdma_event_e90_timer_tick:
810 			break;
811 		}
812 		break;
813 
814 	case qib_sdma_state_s10_hw_start_up_wait:
815 		switch (event) {
816 		case qib_sdma_event_e00_go_hw_down:
817 			sdma_set_state(ppd, qib_sdma_state_s00_hw_down);
818 			sdma_sw_tear_down(ppd);
819 			break;
820 		case qib_sdma_event_e10_go_hw_start:
821 			break;
822 		case qib_sdma_event_e20_hw_started:
823 			sdma_set_state(ppd, ss->go_s99_running ?
824 				       qib_sdma_state_s99_running :
825 				       qib_sdma_state_s20_idle);
826 			break;
827 		case qib_sdma_event_e30_go_running:
828 			ss->go_s99_running = 1;
829 			break;
830 		case qib_sdma_event_e40_sw_cleaned:
831 			break;
832 		case qib_sdma_event_e50_hw_cleaned:
833 			break;
834 		case qib_sdma_event_e60_hw_halted:
835 			break;
836 		case qib_sdma_event_e70_go_idle:
837 			ss->go_s99_running = 0;
838 			break;
839 		case qib_sdma_event_e7220_err_halted:
840 			break;
841 		case qib_sdma_event_e7322_err_halted:
842 			break;
843 		case qib_sdma_event_e90_timer_tick:
844 			break;
845 		}
846 		break;
847 
848 	case qib_sdma_state_s20_idle:
849 		switch (event) {
850 		case qib_sdma_event_e00_go_hw_down:
851 			sdma_set_state(ppd, qib_sdma_state_s00_hw_down);
852 			sdma_sw_tear_down(ppd);
853 			break;
854 		case qib_sdma_event_e10_go_hw_start:
855 			break;
856 		case qib_sdma_event_e20_hw_started:
857 			break;
858 		case qib_sdma_event_e30_go_running:
859 			sdma_set_state(ppd, qib_sdma_state_s99_running);
860 			ss->go_s99_running = 1;
861 			break;
862 		case qib_sdma_event_e40_sw_cleaned:
863 			break;
864 		case qib_sdma_event_e50_hw_cleaned:
865 			break;
866 		case qib_sdma_event_e60_hw_halted:
867 			break;
868 		case qib_sdma_event_e70_go_idle:
869 			break;
870 		case qib_sdma_event_e7220_err_halted:
871 			break;
872 		case qib_sdma_event_e7322_err_halted:
873 			break;
874 		case qib_sdma_event_e90_timer_tick:
875 			break;
876 		}
877 		break;
878 
879 	case qib_sdma_state_s30_sw_clean_up_wait:
880 		switch (event) {
881 		case qib_sdma_event_e00_go_hw_down:
882 			sdma_set_state(ppd, qib_sdma_state_s00_hw_down);
883 			break;
884 		case qib_sdma_event_e10_go_hw_start:
885 			break;
886 		case qib_sdma_event_e20_hw_started:
887 			break;
888 		case qib_sdma_event_e30_go_running:
889 			ss->go_s99_running = 1;
890 			break;
891 		case qib_sdma_event_e40_sw_cleaned:
892 			sdma_set_state(ppd,
893 				       qib_sdma_state_s10_hw_start_up_wait);
894 			sdma_hw_start_up(ppd);
895 			break;
896 		case qib_sdma_event_e50_hw_cleaned:
897 			break;
898 		case qib_sdma_event_e60_hw_halted:
899 			break;
900 		case qib_sdma_event_e70_go_idle:
901 			ss->go_s99_running = 0;
902 			break;
903 		case qib_sdma_event_e7220_err_halted:
904 			break;
905 		case qib_sdma_event_e7322_err_halted:
906 			break;
907 		case qib_sdma_event_e90_timer_tick:
908 			break;
909 		}
910 		break;
911 
912 	case qib_sdma_state_s40_hw_clean_up_wait:
913 		switch (event) {
914 		case qib_sdma_event_e00_go_hw_down:
915 			sdma_set_state(ppd, qib_sdma_state_s00_hw_down);
916 			sdma_start_sw_clean_up(ppd);
917 			break;
918 		case qib_sdma_event_e10_go_hw_start:
919 			break;
920 		case qib_sdma_event_e20_hw_started:
921 			break;
922 		case qib_sdma_event_e30_go_running:
923 			ss->go_s99_running = 1;
924 			break;
925 		case qib_sdma_event_e40_sw_cleaned:
926 			break;
927 		case qib_sdma_event_e50_hw_cleaned:
928 			sdma_set_state(ppd,
929 				       qib_sdma_state_s30_sw_clean_up_wait);
930 			sdma_start_sw_clean_up(ppd);
931 			break;
932 		case qib_sdma_event_e60_hw_halted:
933 			break;
934 		case qib_sdma_event_e70_go_idle:
935 			ss->go_s99_running = 0;
936 			break;
937 		case qib_sdma_event_e7220_err_halted:
938 			break;
939 		case qib_sdma_event_e7322_err_halted:
940 			break;
941 		case qib_sdma_event_e90_timer_tick:
942 			break;
943 		}
944 		break;
945 
946 	case qib_sdma_state_s50_hw_halt_wait:
947 		switch (event) {
948 		case qib_sdma_event_e00_go_hw_down:
949 			sdma_set_state(ppd, qib_sdma_state_s00_hw_down);
950 			sdma_start_sw_clean_up(ppd);
951 			break;
952 		case qib_sdma_event_e10_go_hw_start:
953 			break;
954 		case qib_sdma_event_e20_hw_started:
955 			break;
956 		case qib_sdma_event_e30_go_running:
957 			ss->go_s99_running = 1;
958 			break;
959 		case qib_sdma_event_e40_sw_cleaned:
960 			break;
961 		case qib_sdma_event_e50_hw_cleaned:
962 			break;
963 		case qib_sdma_event_e60_hw_halted:
964 			sdma_set_state(ppd,
965 				       qib_sdma_state_s40_hw_clean_up_wait);
966 			ppd->dd->f_sdma_hw_clean_up(ppd);
967 			break;
968 		case qib_sdma_event_e70_go_idle:
969 			ss->go_s99_running = 0;
970 			break;
971 		case qib_sdma_event_e7220_err_halted:
972 			break;
973 		case qib_sdma_event_e7322_err_halted:
974 			break;
975 		case qib_sdma_event_e90_timer_tick:
976 			break;
977 		}
978 		break;
979 
980 	case qib_sdma_state_s99_running:
981 		switch (event) {
982 		case qib_sdma_event_e00_go_hw_down:
983 			sdma_set_state(ppd, qib_sdma_state_s00_hw_down);
984 			sdma_start_sw_clean_up(ppd);
985 			break;
986 		case qib_sdma_event_e10_go_hw_start:
987 			break;
988 		case qib_sdma_event_e20_hw_started:
989 			break;
990 		case qib_sdma_event_e30_go_running:
991 			break;
992 		case qib_sdma_event_e40_sw_cleaned:
993 			break;
994 		case qib_sdma_event_e50_hw_cleaned:
995 			break;
996 		case qib_sdma_event_e60_hw_halted:
997 			sdma_set_state(ppd,
998 				       qib_sdma_state_s30_sw_clean_up_wait);
999 			sdma_start_sw_clean_up(ppd);
1000 			break;
1001 		case qib_sdma_event_e70_go_idle:
1002 			sdma_set_state(ppd, qib_sdma_state_s50_hw_halt_wait);
1003 			ss->go_s99_running = 0;
1004 			break;
1005 		case qib_sdma_event_e7220_err_halted:
1006 			sdma_set_state(ppd,
1007 				       qib_sdma_state_s30_sw_clean_up_wait);
1008 			sdma_start_sw_clean_up(ppd);
1009 			break;
1010 		case qib_sdma_event_e7322_err_halted:
1011 			sdma_set_state(ppd, qib_sdma_state_s50_hw_halt_wait);
1012 			break;
1013 		case qib_sdma_event_e90_timer_tick:
1014 			break;
1015 		}
1016 		break;
1017 	}
1018 
1019 	ss->last_event = event;
1020 }
1021