1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
2 /*
3 * Copyright(c) 2015 - 2018 Intel Corporation.
4 */
5
6 #include <linux/spinlock.h>
7 #include <linux/seqlock.h>
8 #include <linux/netdevice.h>
9 #include <linux/moduleparam.h>
10 #include <linux/bitops.h>
11 #include <linux/timer.h>
12 #include <linux/vmalloc.h>
13 #include <linux/highmem.h>
14 #include <linux/sysfs.h>
15
16 #include "hfi.h"
17 #include "common.h"
18 #include "qp.h"
19 #include "sdma.h"
20 #include "iowait.h"
21 #include "trace.h"
22
23 /* must be a power of 2 >= 64 <= 32768 */
24 #define SDMA_DESCQ_CNT 2048
25 #define SDMA_DESC_INTR 64
26 #define INVALID_TAIL 0xffff
27 #define SDMA_PAD max_t(size_t, MAX_16B_PADDING, sizeof(u32))
28
29 static uint sdma_descq_cnt = SDMA_DESCQ_CNT;
30 module_param(sdma_descq_cnt, uint, S_IRUGO);
31 MODULE_PARM_DESC(sdma_descq_cnt, "Number of SDMA descq entries");
32
33 static uint sdma_idle_cnt = 250;
34 module_param(sdma_idle_cnt, uint, S_IRUGO);
35 MODULE_PARM_DESC(sdma_idle_cnt, "sdma interrupt idle delay (ns,default 250)");
36
37 uint mod_num_sdma;
38 module_param_named(num_sdma, mod_num_sdma, uint, S_IRUGO);
39 MODULE_PARM_DESC(num_sdma, "Set max number SDMA engines to use");
40
41 static uint sdma_desct_intr = SDMA_DESC_INTR;
42 module_param_named(desct_intr, sdma_desct_intr, uint, S_IRUGO | S_IWUSR);
43 MODULE_PARM_DESC(desct_intr, "Number of SDMA descriptor before interrupt");
44
45 #define SDMA_WAIT_BATCH_SIZE 20
46 /* max wait time for a SDMA engine to indicate it has halted */
47 #define SDMA_ERR_HALT_TIMEOUT 10 /* ms */
48 /* all SDMA engine errors that cause a halt */
49
50 #define SD(name) SEND_DMA_##name
51 #define ALL_SDMA_ENG_HALT_ERRS \
52 (SD(ENG_ERR_STATUS_SDMA_WRONG_DW_ERR_SMASK) \
53 | SD(ENG_ERR_STATUS_SDMA_GEN_MISMATCH_ERR_SMASK) \
54 | SD(ENG_ERR_STATUS_SDMA_TOO_LONG_ERR_SMASK) \
55 | SD(ENG_ERR_STATUS_SDMA_TAIL_OUT_OF_BOUNDS_ERR_SMASK) \
56 | SD(ENG_ERR_STATUS_SDMA_FIRST_DESC_ERR_SMASK) \
57 | SD(ENG_ERR_STATUS_SDMA_MEM_READ_ERR_SMASK) \
58 | SD(ENG_ERR_STATUS_SDMA_HALT_ERR_SMASK) \
59 | SD(ENG_ERR_STATUS_SDMA_LENGTH_MISMATCH_ERR_SMASK) \
60 | SD(ENG_ERR_STATUS_SDMA_PACKET_DESC_OVERFLOW_ERR_SMASK) \
61 | SD(ENG_ERR_STATUS_SDMA_HEADER_SELECT_ERR_SMASK) \
62 | SD(ENG_ERR_STATUS_SDMA_HEADER_ADDRESS_ERR_SMASK) \
63 | SD(ENG_ERR_STATUS_SDMA_HEADER_LENGTH_ERR_SMASK) \
64 | SD(ENG_ERR_STATUS_SDMA_TIMEOUT_ERR_SMASK) \
65 | SD(ENG_ERR_STATUS_SDMA_DESC_TABLE_UNC_ERR_SMASK) \
66 | SD(ENG_ERR_STATUS_SDMA_ASSEMBLY_UNC_ERR_SMASK) \
67 | SD(ENG_ERR_STATUS_SDMA_PACKET_TRACKING_UNC_ERR_SMASK) \
68 | SD(ENG_ERR_STATUS_SDMA_HEADER_STORAGE_UNC_ERR_SMASK) \
69 | SD(ENG_ERR_STATUS_SDMA_HEADER_REQUEST_FIFO_UNC_ERR_SMASK))
70
71 /* sdma_sendctrl operations */
72 #define SDMA_SENDCTRL_OP_ENABLE BIT(0)
73 #define SDMA_SENDCTRL_OP_INTENABLE BIT(1)
74 #define SDMA_SENDCTRL_OP_HALT BIT(2)
75 #define SDMA_SENDCTRL_OP_CLEANUP BIT(3)
76
77 /* handle long defines */
78 #define SDMA_EGRESS_PACKET_OCCUPANCY_SMASK \
79 SEND_EGRESS_SEND_DMA_STATUS_SDMA_EGRESS_PACKET_OCCUPANCY_SMASK
80 #define SDMA_EGRESS_PACKET_OCCUPANCY_SHIFT \
81 SEND_EGRESS_SEND_DMA_STATUS_SDMA_EGRESS_PACKET_OCCUPANCY_SHIFT
82
83 static const char * const sdma_state_names[] = {
84 [sdma_state_s00_hw_down] = "s00_HwDown",
85 [sdma_state_s10_hw_start_up_halt_wait] = "s10_HwStartUpHaltWait",
86 [sdma_state_s15_hw_start_up_clean_wait] = "s15_HwStartUpCleanWait",
87 [sdma_state_s20_idle] = "s20_Idle",
88 [sdma_state_s30_sw_clean_up_wait] = "s30_SwCleanUpWait",
89 [sdma_state_s40_hw_clean_up_wait] = "s40_HwCleanUpWait",
90 [sdma_state_s50_hw_halt_wait] = "s50_HwHaltWait",
91 [sdma_state_s60_idle_halt_wait] = "s60_IdleHaltWait",
92 [sdma_state_s80_hw_freeze] = "s80_HwFreeze",
93 [sdma_state_s82_freeze_sw_clean] = "s82_FreezeSwClean",
94 [sdma_state_s99_running] = "s99_Running",
95 };
96
97 #ifdef CONFIG_SDMA_VERBOSITY
98 static const char * const sdma_event_names[] = {
99 [sdma_event_e00_go_hw_down] = "e00_GoHwDown",
100 [sdma_event_e10_go_hw_start] = "e10_GoHwStart",
101 [sdma_event_e15_hw_halt_done] = "e15_HwHaltDone",
102 [sdma_event_e25_hw_clean_up_done] = "e25_HwCleanUpDone",
103 [sdma_event_e30_go_running] = "e30_GoRunning",
104 [sdma_event_e40_sw_cleaned] = "e40_SwCleaned",
105 [sdma_event_e50_hw_cleaned] = "e50_HwCleaned",
106 [sdma_event_e60_hw_halted] = "e60_HwHalted",
107 [sdma_event_e70_go_idle] = "e70_GoIdle",
108 [sdma_event_e80_hw_freeze] = "e80_HwFreeze",
109 [sdma_event_e81_hw_frozen] = "e81_HwFrozen",
110 [sdma_event_e82_hw_unfreeze] = "e82_HwUnfreeze",
111 [sdma_event_e85_link_down] = "e85_LinkDown",
112 [sdma_event_e90_sw_halted] = "e90_SwHalted",
113 };
114 #endif
115
116 static const struct sdma_set_state_action sdma_action_table[] = {
117 [sdma_state_s00_hw_down] = {
118 .go_s99_running_tofalse = 1,
119 .op_enable = 0,
120 .op_intenable = 0,
121 .op_halt = 0,
122 .op_cleanup = 0,
123 },
124 [sdma_state_s10_hw_start_up_halt_wait] = {
125 .op_enable = 0,
126 .op_intenable = 0,
127 .op_halt = 1,
128 .op_cleanup = 0,
129 },
130 [sdma_state_s15_hw_start_up_clean_wait] = {
131 .op_enable = 0,
132 .op_intenable = 1,
133 .op_halt = 0,
134 .op_cleanup = 1,
135 },
136 [sdma_state_s20_idle] = {
137 .op_enable = 0,
138 .op_intenable = 1,
139 .op_halt = 0,
140 .op_cleanup = 0,
141 },
142 [sdma_state_s30_sw_clean_up_wait] = {
143 .op_enable = 0,
144 .op_intenable = 0,
145 .op_halt = 0,
146 .op_cleanup = 0,
147 },
148 [sdma_state_s40_hw_clean_up_wait] = {
149 .op_enable = 0,
150 .op_intenable = 0,
151 .op_halt = 0,
152 .op_cleanup = 1,
153 },
154 [sdma_state_s50_hw_halt_wait] = {
155 .op_enable = 0,
156 .op_intenable = 0,
157 .op_halt = 0,
158 .op_cleanup = 0,
159 },
160 [sdma_state_s60_idle_halt_wait] = {
161 .go_s99_running_tofalse = 1,
162 .op_enable = 0,
163 .op_intenable = 0,
164 .op_halt = 1,
165 .op_cleanup = 0,
166 },
167 [sdma_state_s80_hw_freeze] = {
168 .op_enable = 0,
169 .op_intenable = 0,
170 .op_halt = 0,
171 .op_cleanup = 0,
172 },
173 [sdma_state_s82_freeze_sw_clean] = {
174 .op_enable = 0,
175 .op_intenable = 0,
176 .op_halt = 0,
177 .op_cleanup = 0,
178 },
179 [sdma_state_s99_running] = {
180 .op_enable = 1,
181 .op_intenable = 1,
182 .op_halt = 0,
183 .op_cleanup = 0,
184 .go_s99_running_totrue = 1,
185 },
186 };
187
188 #define SDMA_TAIL_UPDATE_THRESH 0x1F
189
190 /* declare all statics here rather than keep sorting */
191 static void sdma_complete(struct kref *);
192 static void sdma_finalput(struct sdma_state *);
193 static void sdma_get(struct sdma_state *);
194 static void sdma_hw_clean_up_task(struct tasklet_struct *);
195 static void sdma_put(struct sdma_state *);
196 static void sdma_set_state(struct sdma_engine *, enum sdma_states);
197 static void sdma_start_hw_clean_up(struct sdma_engine *);
198 static void sdma_sw_clean_up_task(struct tasklet_struct *);
199 static void sdma_sendctrl(struct sdma_engine *, unsigned);
200 static void init_sdma_regs(struct sdma_engine *, u32, uint);
201 static void sdma_process_event(
202 struct sdma_engine *sde,
203 enum sdma_events event);
204 static void __sdma_process_event(
205 struct sdma_engine *sde,
206 enum sdma_events event);
207 static void dump_sdma_state(struct sdma_engine *sde);
208 static void sdma_make_progress(struct sdma_engine *sde, u64 status);
209 static void sdma_desc_avail(struct sdma_engine *sde, uint avail);
210 static void sdma_flush_descq(struct sdma_engine *sde);
211
212 /**
213 * sdma_state_name() - return state string from enum
214 * @state: state
215 */
sdma_state_name(enum sdma_states state)216 static const char *sdma_state_name(enum sdma_states state)
217 {
218 return sdma_state_names[state];
219 }
220
sdma_get(struct sdma_state * ss)221 static void sdma_get(struct sdma_state *ss)
222 {
223 kref_get(&ss->kref);
224 }
225
sdma_complete(struct kref * kref)226 static void sdma_complete(struct kref *kref)
227 {
228 struct sdma_state *ss =
229 container_of(kref, struct sdma_state, kref);
230
231 complete(&ss->comp);
232 }
233
sdma_put(struct sdma_state * ss)234 static void sdma_put(struct sdma_state *ss)
235 {
236 kref_put(&ss->kref, sdma_complete);
237 }
238
sdma_finalput(struct sdma_state * ss)239 static void sdma_finalput(struct sdma_state *ss)
240 {
241 sdma_put(ss);
242 wait_for_completion(&ss->comp);
243 }
244
write_sde_csr(struct sdma_engine * sde,u32 offset0,u64 value)245 static inline void write_sde_csr(
246 struct sdma_engine *sde,
247 u32 offset0,
248 u64 value)
249 {
250 write_kctxt_csr(sde->dd, sde->this_idx, offset0, value);
251 }
252
read_sde_csr(struct sdma_engine * sde,u32 offset0)253 static inline u64 read_sde_csr(
254 struct sdma_engine *sde,
255 u32 offset0)
256 {
257 return read_kctxt_csr(sde->dd, sde->this_idx, offset0);
258 }
259
260 /*
261 * sdma_wait_for_packet_egress() - wait for the VL FIFO occupancy for
262 * sdma engine 'sde' to drop to 0.
263 */
sdma_wait_for_packet_egress(struct sdma_engine * sde,int pause)264 static void sdma_wait_for_packet_egress(struct sdma_engine *sde,
265 int pause)
266 {
267 u64 off = 8 * sde->this_idx;
268 struct hfi1_devdata *dd = sde->dd;
269 int lcnt = 0;
270 u64 reg_prev;
271 u64 reg = 0;
272
273 while (1) {
274 reg_prev = reg;
275 reg = read_csr(dd, off + SEND_EGRESS_SEND_DMA_STATUS);
276
277 reg &= SDMA_EGRESS_PACKET_OCCUPANCY_SMASK;
278 reg >>= SDMA_EGRESS_PACKET_OCCUPANCY_SHIFT;
279 if (reg == 0)
280 break;
281 /* counter is reest if accupancy count changes */
282 if (reg != reg_prev)
283 lcnt = 0;
284 if (lcnt++ > 500) {
285 /* timed out - bounce the link */
286 dd_dev_err(dd, "%s: engine %u timeout waiting for packets to egress, remaining count %u, bouncing link\n",
287 __func__, sde->this_idx, (u32)reg);
288 queue_work(dd->pport->link_wq,
289 &dd->pport->link_bounce_work);
290 break;
291 }
292 udelay(1);
293 }
294 }
295
296 /*
297 * sdma_wait() - wait for packet egress to complete for all SDMA engines,
298 * and pause for credit return.
299 */
sdma_wait(struct hfi1_devdata * dd)300 void sdma_wait(struct hfi1_devdata *dd)
301 {
302 int i;
303
304 for (i = 0; i < dd->num_sdma; i++) {
305 struct sdma_engine *sde = &dd->per_sdma[i];
306
307 sdma_wait_for_packet_egress(sde, 0);
308 }
309 }
310
sdma_set_desc_cnt(struct sdma_engine * sde,unsigned cnt)311 static inline void sdma_set_desc_cnt(struct sdma_engine *sde, unsigned cnt)
312 {
313 u64 reg;
314
315 if (!(sde->dd->flags & HFI1_HAS_SDMA_TIMEOUT))
316 return;
317 reg = cnt;
318 reg &= SD(DESC_CNT_CNT_MASK);
319 reg <<= SD(DESC_CNT_CNT_SHIFT);
320 write_sde_csr(sde, SD(DESC_CNT), reg);
321 }
322
complete_tx(struct sdma_engine * sde,struct sdma_txreq * tx,int res)323 static inline void complete_tx(struct sdma_engine *sde,
324 struct sdma_txreq *tx,
325 int res)
326 {
327 /* protect against complete modifying */
328 struct iowait *wait = tx->wait;
329 callback_t complete = tx->complete;
330
331 #ifdef CONFIG_HFI1_DEBUG_SDMA_ORDER
332 trace_hfi1_sdma_out_sn(sde, tx->sn);
333 if (WARN_ON_ONCE(sde->head_sn != tx->sn))
334 dd_dev_err(sde->dd, "expected %llu got %llu\n",
335 sde->head_sn, tx->sn);
336 sde->head_sn++;
337 #endif
338 __sdma_txclean(sde->dd, tx);
339 if (complete)
340 (*complete)(tx, res);
341 if (iowait_sdma_dec(wait))
342 iowait_drain_wakeup(wait);
343 }
344
345 /*
346 * Complete all the sdma requests with a SDMA_TXREQ_S_ABORTED status
347 *
348 * Depending on timing there can be txreqs in two places:
349 * - in the descq ring
350 * - in the flush list
351 *
352 * To avoid ordering issues the descq ring needs to be flushed
353 * first followed by the flush list.
354 *
355 * This routine is called from two places
356 * - From a work queue item
357 * - Directly from the state machine just before setting the
358 * state to running
359 *
360 * Must be called with head_lock held
361 *
362 */
sdma_flush(struct sdma_engine * sde)363 static void sdma_flush(struct sdma_engine *sde)
364 {
365 struct sdma_txreq *txp, *txp_next;
366 LIST_HEAD(flushlist);
367 unsigned long flags;
368 uint seq;
369
370 /* flush from head to tail */
371 sdma_flush_descq(sde);
372 spin_lock_irqsave(&sde->flushlist_lock, flags);
373 /* copy flush list */
374 list_splice_init(&sde->flushlist, &flushlist);
375 spin_unlock_irqrestore(&sde->flushlist_lock, flags);
376 /* flush from flush list */
377 list_for_each_entry_safe(txp, txp_next, &flushlist, list)
378 complete_tx(sde, txp, SDMA_TXREQ_S_ABORTED);
379 /* wakeup QPs orphaned on the dmawait list */
380 do {
381 struct iowait *w, *nw;
382
383 seq = read_seqbegin(&sde->waitlock);
384 if (!list_empty(&sde->dmawait)) {
385 write_seqlock(&sde->waitlock);
386 list_for_each_entry_safe(w, nw, &sde->dmawait, list) {
387 if (w->wakeup) {
388 w->wakeup(w, SDMA_AVAIL_REASON);
389 list_del_init(&w->list);
390 }
391 }
392 write_sequnlock(&sde->waitlock);
393 }
394 } while (read_seqretry(&sde->waitlock, seq));
395 }
396
397 /*
398 * Fields a work request for flushing the descq ring
399 * and the flush list
400 *
401 * If the engine has been brought to running during
402 * the scheduling delay, the flush is ignored, assuming
403 * that the process of bringing the engine to running
404 * would have done this flush prior to going to running.
405 *
406 */
sdma_field_flush(struct work_struct * work)407 static void sdma_field_flush(struct work_struct *work)
408 {
409 unsigned long flags;
410 struct sdma_engine *sde =
411 container_of(work, struct sdma_engine, flush_worker);
412
413 write_seqlock_irqsave(&sde->head_lock, flags);
414 if (!__sdma_running(sde))
415 sdma_flush(sde);
416 write_sequnlock_irqrestore(&sde->head_lock, flags);
417 }
418
sdma_err_halt_wait(struct work_struct * work)419 static void sdma_err_halt_wait(struct work_struct *work)
420 {
421 struct sdma_engine *sde = container_of(work, struct sdma_engine,
422 err_halt_worker);
423 u64 statuscsr;
424 unsigned long timeout;
425
426 timeout = jiffies + msecs_to_jiffies(SDMA_ERR_HALT_TIMEOUT);
427 while (1) {
428 statuscsr = read_sde_csr(sde, SD(STATUS));
429 statuscsr &= SD(STATUS_ENG_HALTED_SMASK);
430 if (statuscsr)
431 break;
432 if (time_after(jiffies, timeout)) {
433 dd_dev_err(sde->dd,
434 "SDMA engine %d - timeout waiting for engine to halt\n",
435 sde->this_idx);
436 /*
437 * Continue anyway. This could happen if there was
438 * an uncorrectable error in the wrong spot.
439 */
440 break;
441 }
442 usleep_range(80, 120);
443 }
444
445 sdma_process_event(sde, sdma_event_e15_hw_halt_done);
446 }
447
sdma_err_progress_check_schedule(struct sdma_engine * sde)448 static void sdma_err_progress_check_schedule(struct sdma_engine *sde)
449 {
450 if (!is_bx(sde->dd) && HFI1_CAP_IS_KSET(SDMA_AHG)) {
451 unsigned index;
452 struct hfi1_devdata *dd = sde->dd;
453
454 for (index = 0; index < dd->num_sdma; index++) {
455 struct sdma_engine *curr_sdma = &dd->per_sdma[index];
456
457 if (curr_sdma != sde)
458 curr_sdma->progress_check_head =
459 curr_sdma->descq_head;
460 }
461 dd_dev_err(sde->dd,
462 "SDMA engine %d - check scheduled\n",
463 sde->this_idx);
464 mod_timer(&sde->err_progress_check_timer, jiffies + 10);
465 }
466 }
467
sdma_err_progress_check(struct timer_list * t)468 static void sdma_err_progress_check(struct timer_list *t)
469 {
470 unsigned index;
471 struct sdma_engine *sde = timer_container_of(sde, t,
472 err_progress_check_timer);
473
474 dd_dev_err(sde->dd, "SDE progress check event\n");
475 for (index = 0; index < sde->dd->num_sdma; index++) {
476 struct sdma_engine *curr_sde = &sde->dd->per_sdma[index];
477 unsigned long flags;
478
479 /* check progress on each engine except the current one */
480 if (curr_sde == sde)
481 continue;
482 /*
483 * We must lock interrupts when acquiring sde->lock,
484 * to avoid a deadlock if interrupt triggers and spins on
485 * the same lock on same CPU
486 */
487 spin_lock_irqsave(&curr_sde->tail_lock, flags);
488 write_seqlock(&curr_sde->head_lock);
489
490 /* skip non-running queues */
491 if (curr_sde->state.current_state != sdma_state_s99_running) {
492 write_sequnlock(&curr_sde->head_lock);
493 spin_unlock_irqrestore(&curr_sde->tail_lock, flags);
494 continue;
495 }
496
497 if ((curr_sde->descq_head != curr_sde->descq_tail) &&
498 (curr_sde->descq_head ==
499 curr_sde->progress_check_head))
500 __sdma_process_event(curr_sde,
501 sdma_event_e90_sw_halted);
502 write_sequnlock(&curr_sde->head_lock);
503 spin_unlock_irqrestore(&curr_sde->tail_lock, flags);
504 }
505 schedule_work(&sde->err_halt_worker);
506 }
507
sdma_hw_clean_up_task(struct tasklet_struct * t)508 static void sdma_hw_clean_up_task(struct tasklet_struct *t)
509 {
510 struct sdma_engine *sde = from_tasklet(sde, t,
511 sdma_hw_clean_up_task);
512 u64 statuscsr;
513
514 while (1) {
515 #ifdef CONFIG_SDMA_VERBOSITY
516 dd_dev_err(sde->dd, "CONFIG SDMA(%u) %s:%d %s()\n",
517 sde->this_idx, slashstrip(__FILE__), __LINE__,
518 __func__);
519 #endif
520 statuscsr = read_sde_csr(sde, SD(STATUS));
521 statuscsr &= SD(STATUS_ENG_CLEANED_UP_SMASK);
522 if (statuscsr)
523 break;
524 udelay(10);
525 }
526
527 sdma_process_event(sde, sdma_event_e25_hw_clean_up_done);
528 }
529
get_txhead(struct sdma_engine * sde)530 static inline struct sdma_txreq *get_txhead(struct sdma_engine *sde)
531 {
532 return sde->tx_ring[sde->tx_head & sde->sdma_mask];
533 }
534
535 /*
536 * flush ring for recovery
537 */
sdma_flush_descq(struct sdma_engine * sde)538 static void sdma_flush_descq(struct sdma_engine *sde)
539 {
540 u16 head, tail;
541 int progress = 0;
542 struct sdma_txreq *txp = get_txhead(sde);
543
544 /* The reason for some of the complexity of this code is that
545 * not all descriptors have corresponding txps. So, we have to
546 * be able to skip over descs until we wander into the range of
547 * the next txp on the list.
548 */
549 head = sde->descq_head & sde->sdma_mask;
550 tail = sde->descq_tail & sde->sdma_mask;
551 while (head != tail) {
552 /* advance head, wrap if needed */
553 head = ++sde->descq_head & sde->sdma_mask;
554 /* if now past this txp's descs, do the callback */
555 if (txp && txp->next_descq_idx == head) {
556 /* remove from list */
557 sde->tx_ring[sde->tx_head++ & sde->sdma_mask] = NULL;
558 complete_tx(sde, txp, SDMA_TXREQ_S_ABORTED);
559 trace_hfi1_sdma_progress(sde, head, tail, txp);
560 txp = get_txhead(sde);
561 }
562 progress++;
563 }
564 if (progress)
565 sdma_desc_avail(sde, sdma_descq_freecnt(sde));
566 }
567
sdma_sw_clean_up_task(struct tasklet_struct * t)568 static void sdma_sw_clean_up_task(struct tasklet_struct *t)
569 {
570 struct sdma_engine *sde = from_tasklet(sde, t, sdma_sw_clean_up_task);
571 unsigned long flags;
572
573 spin_lock_irqsave(&sde->tail_lock, flags);
574 write_seqlock(&sde->head_lock);
575
576 /*
577 * At this point, the following should always be true:
578 * - We are halted, so no more descriptors are getting retired.
579 * - We are not running, so no one is submitting new work.
580 * - Only we can send the e40_sw_cleaned, so we can't start
581 * running again until we say so. So, the active list and
582 * descq are ours to play with.
583 */
584
585 /*
586 * In the error clean up sequence, software clean must be called
587 * before the hardware clean so we can use the hardware head in
588 * the progress routine. A hardware clean or SPC unfreeze will
589 * reset the hardware head.
590 *
591 * Process all retired requests. The progress routine will use the
592 * latest physical hardware head - we are not running so speed does
593 * not matter.
594 */
595 sdma_make_progress(sde, 0);
596
597 sdma_flush(sde);
598
599 /*
600 * Reset our notion of head and tail.
601 * Note that the HW registers have been reset via an earlier
602 * clean up.
603 */
604 sde->descq_tail = 0;
605 sde->descq_head = 0;
606 sde->desc_avail = sdma_descq_freecnt(sde);
607 *sde->head_dma = 0;
608
609 __sdma_process_event(sde, sdma_event_e40_sw_cleaned);
610
611 write_sequnlock(&sde->head_lock);
612 spin_unlock_irqrestore(&sde->tail_lock, flags);
613 }
614
sdma_sw_tear_down(struct sdma_engine * sde)615 static void sdma_sw_tear_down(struct sdma_engine *sde)
616 {
617 struct sdma_state *ss = &sde->state;
618
619 /* Releasing this reference means the state machine has stopped. */
620 sdma_put(ss);
621
622 /* stop waiting for all unfreeze events to complete */
623 atomic_set(&sde->dd->sdma_unfreeze_count, -1);
624 wake_up_interruptible(&sde->dd->sdma_unfreeze_wq);
625 }
626
sdma_start_hw_clean_up(struct sdma_engine * sde)627 static void sdma_start_hw_clean_up(struct sdma_engine *sde)
628 {
629 tasklet_hi_schedule(&sde->sdma_hw_clean_up_task);
630 }
631
sdma_set_state(struct sdma_engine * sde,enum sdma_states next_state)632 static void sdma_set_state(struct sdma_engine *sde,
633 enum sdma_states next_state)
634 {
635 struct sdma_state *ss = &sde->state;
636 const struct sdma_set_state_action *action = sdma_action_table;
637 unsigned op = 0;
638
639 trace_hfi1_sdma_state(
640 sde,
641 sdma_state_names[ss->current_state],
642 sdma_state_names[next_state]);
643
644 /* debugging bookkeeping */
645 ss->previous_state = ss->current_state;
646 ss->previous_op = ss->current_op;
647 ss->current_state = next_state;
648
649 if (ss->previous_state != sdma_state_s99_running &&
650 next_state == sdma_state_s99_running)
651 sdma_flush(sde);
652
653 if (action[next_state].op_enable)
654 op |= SDMA_SENDCTRL_OP_ENABLE;
655
656 if (action[next_state].op_intenable)
657 op |= SDMA_SENDCTRL_OP_INTENABLE;
658
659 if (action[next_state].op_halt)
660 op |= SDMA_SENDCTRL_OP_HALT;
661
662 if (action[next_state].op_cleanup)
663 op |= SDMA_SENDCTRL_OP_CLEANUP;
664
665 if (action[next_state].go_s99_running_tofalse)
666 ss->go_s99_running = 0;
667
668 if (action[next_state].go_s99_running_totrue)
669 ss->go_s99_running = 1;
670
671 ss->current_op = op;
672 sdma_sendctrl(sde, ss->current_op);
673 }
674
675 /**
676 * sdma_get_descq_cnt() - called when device probed
677 *
678 * Return a validated descq count.
679 *
680 * This is currently only used in the verbs initialization to build the tx
681 * list.
682 *
683 * This will probably be deleted in favor of a more scalable approach to
684 * alloc tx's.
685 *
686 */
sdma_get_descq_cnt(void)687 u16 sdma_get_descq_cnt(void)
688 {
689 u16 count = sdma_descq_cnt;
690
691 if (!count)
692 return SDMA_DESCQ_CNT;
693 /* count must be a power of 2 greater than 64 and less than
694 * 32768. Otherwise return default.
695 */
696 if (!is_power_of_2(count))
697 return SDMA_DESCQ_CNT;
698 if (count < 64 || count > 32768)
699 return SDMA_DESCQ_CNT;
700 return count;
701 }
702
703 /**
704 * sdma_engine_get_vl() - return vl for a given sdma engine
705 * @sde: sdma engine
706 *
707 * This function returns the vl mapped to a given engine, or an error if
708 * the mapping can't be found. The mapping fields are protected by RCU.
709 */
sdma_engine_get_vl(struct sdma_engine * sde)710 int sdma_engine_get_vl(struct sdma_engine *sde)
711 {
712 struct hfi1_devdata *dd = sde->dd;
713 struct sdma_vl_map *m;
714 u8 vl;
715
716 if (sde->this_idx >= TXE_NUM_SDMA_ENGINES)
717 return -EINVAL;
718
719 rcu_read_lock();
720 m = rcu_dereference(dd->sdma_map);
721 if (unlikely(!m)) {
722 rcu_read_unlock();
723 return -EINVAL;
724 }
725 vl = m->engine_to_vl[sde->this_idx];
726 rcu_read_unlock();
727
728 return vl;
729 }
730
731 /**
732 * sdma_select_engine_vl() - select sdma engine
733 * @dd: devdata
734 * @selector: a spreading factor
735 * @vl: this vl
736 *
737 *
738 * This function returns an engine based on the selector and a vl. The
739 * mapping fields are protected by RCU.
740 */
sdma_select_engine_vl(struct hfi1_devdata * dd,u32 selector,u8 vl)741 struct sdma_engine *sdma_select_engine_vl(
742 struct hfi1_devdata *dd,
743 u32 selector,
744 u8 vl)
745 {
746 struct sdma_vl_map *m;
747 struct sdma_map_elem *e;
748 struct sdma_engine *rval;
749
750 /* NOTE This should only happen if SC->VL changed after the initial
751 * checks on the QP/AH
752 * Default will return engine 0 below
753 */
754 if (vl >= num_vls) {
755 rval = NULL;
756 goto done;
757 }
758
759 rcu_read_lock();
760 m = rcu_dereference(dd->sdma_map);
761 if (unlikely(!m)) {
762 rcu_read_unlock();
763 return &dd->per_sdma[0];
764 }
765 e = m->map[vl & m->mask];
766 rval = e->sde[selector & e->mask];
767 rcu_read_unlock();
768
769 done:
770 rval = !rval ? &dd->per_sdma[0] : rval;
771 trace_hfi1_sdma_engine_select(dd, selector, vl, rval->this_idx);
772 return rval;
773 }
774
775 /**
776 * sdma_select_engine_sc() - select sdma engine
777 * @dd: devdata
778 * @selector: a spreading factor
779 * @sc5: the 5 bit sc
780 *
781 *
782 * This function returns an engine based on the selector and an sc.
783 */
sdma_select_engine_sc(struct hfi1_devdata * dd,u32 selector,u8 sc5)784 struct sdma_engine *sdma_select_engine_sc(
785 struct hfi1_devdata *dd,
786 u32 selector,
787 u8 sc5)
788 {
789 u8 vl = sc_to_vlt(dd, sc5);
790
791 return sdma_select_engine_vl(dd, selector, vl);
792 }
793
794 struct sdma_rht_map_elem {
795 u32 mask;
796 u8 ctr;
797 struct sdma_engine *sde[];
798 };
799
800 struct sdma_rht_node {
801 unsigned long cpu_id;
802 struct sdma_rht_map_elem *map[HFI1_MAX_VLS_SUPPORTED];
803 struct rhash_head node;
804 };
805
806 #define NR_CPUS_HINT 192
807
808 static const struct rhashtable_params sdma_rht_params = {
809 .nelem_hint = NR_CPUS_HINT,
810 .head_offset = offsetof(struct sdma_rht_node, node),
811 .key_offset = offsetof(struct sdma_rht_node, cpu_id),
812 .key_len = sizeof_field(struct sdma_rht_node, cpu_id),
813 .max_size = NR_CPUS,
814 .min_size = 8,
815 .automatic_shrinking = true,
816 };
817
818 /*
819 * sdma_select_user_engine() - select sdma engine based on user setup
820 * @dd: devdata
821 * @selector: a spreading factor
822 * @vl: this vl
823 *
824 * This function returns an sdma engine for a user sdma request.
825 * User defined sdma engine affinity setting is honored when applicable,
826 * otherwise system default sdma engine mapping is used. To ensure correct
827 * ordering, the mapping from <selector, vl> to sde must remain unchanged.
828 */
sdma_select_user_engine(struct hfi1_devdata * dd,u32 selector,u8 vl)829 struct sdma_engine *sdma_select_user_engine(struct hfi1_devdata *dd,
830 u32 selector, u8 vl)
831 {
832 struct sdma_rht_node *rht_node;
833 struct sdma_engine *sde = NULL;
834 unsigned long cpu_id;
835
836 /*
837 * To ensure that always the same sdma engine(s) will be
838 * selected make sure the process is pinned to this CPU only.
839 */
840 if (current->nr_cpus_allowed != 1)
841 goto out;
842
843 rcu_read_lock();
844 cpu_id = smp_processor_id();
845 rht_node = rhashtable_lookup(dd->sdma_rht, &cpu_id,
846 sdma_rht_params);
847
848 if (rht_node && rht_node->map[vl]) {
849 struct sdma_rht_map_elem *map = rht_node->map[vl];
850
851 sde = map->sde[selector & map->mask];
852 }
853 rcu_read_unlock();
854
855 if (sde)
856 return sde;
857
858 out:
859 return sdma_select_engine_vl(dd, selector, vl);
860 }
861
sdma_populate_sde_map(struct sdma_rht_map_elem * map)862 static void sdma_populate_sde_map(struct sdma_rht_map_elem *map)
863 {
864 int i;
865
866 for (i = 0; i < roundup_pow_of_two(map->ctr ? : 1) - map->ctr; i++)
867 map->sde[map->ctr + i] = map->sde[i];
868 }
869
sdma_cleanup_sde_map(struct sdma_rht_map_elem * map,struct sdma_engine * sde)870 static void sdma_cleanup_sde_map(struct sdma_rht_map_elem *map,
871 struct sdma_engine *sde)
872 {
873 unsigned int i, pow;
874
875 /* only need to check the first ctr entries for a match */
876 for (i = 0; i < map->ctr; i++) {
877 if (map->sde[i] == sde) {
878 memmove(&map->sde[i], &map->sde[i + 1],
879 (map->ctr - i - 1) * sizeof(map->sde[0]));
880 map->ctr--;
881 pow = roundup_pow_of_two(map->ctr ? : 1);
882 map->mask = pow - 1;
883 sdma_populate_sde_map(map);
884 break;
885 }
886 }
887 }
888
889 /*
890 * Prevents concurrent reads and writes of the sdma engine cpu_mask
891 */
892 static DEFINE_MUTEX(process_to_sde_mutex);
893
sdma_set_cpu_to_sde_map(struct sdma_engine * sde,const char * buf,size_t count)894 ssize_t sdma_set_cpu_to_sde_map(struct sdma_engine *sde, const char *buf,
895 size_t count)
896 {
897 struct hfi1_devdata *dd = sde->dd;
898 cpumask_var_t mask, new_mask;
899 unsigned long cpu;
900 int ret, vl, sz;
901 struct sdma_rht_node *rht_node;
902
903 vl = sdma_engine_get_vl(sde);
904 if (unlikely(vl < 0 || vl >= ARRAY_SIZE(rht_node->map)))
905 return -EINVAL;
906
907 ret = zalloc_cpumask_var(&mask, GFP_KERNEL);
908 if (!ret)
909 return -ENOMEM;
910
911 ret = zalloc_cpumask_var(&new_mask, GFP_KERNEL);
912 if (!ret) {
913 free_cpumask_var(mask);
914 return -ENOMEM;
915 }
916 ret = cpulist_parse(buf, mask);
917 if (ret)
918 goto out_free;
919
920 if (!cpumask_subset(mask, cpu_online_mask)) {
921 dd_dev_warn(sde->dd, "Invalid CPU mask\n");
922 ret = -EINVAL;
923 goto out_free;
924 }
925
926 sz = sizeof(struct sdma_rht_map_elem) +
927 (TXE_NUM_SDMA_ENGINES * sizeof(struct sdma_engine *));
928
929 mutex_lock(&process_to_sde_mutex);
930
931 for_each_cpu(cpu, mask) {
932 /* Check if we have this already mapped */
933 if (cpumask_test_cpu(cpu, &sde->cpu_mask)) {
934 cpumask_set_cpu(cpu, new_mask);
935 continue;
936 }
937
938 rht_node = rhashtable_lookup_fast(dd->sdma_rht, &cpu,
939 sdma_rht_params);
940 if (!rht_node) {
941 rht_node = kzalloc_obj(*rht_node);
942 if (!rht_node) {
943 ret = -ENOMEM;
944 goto out;
945 }
946
947 rht_node->map[vl] = kzalloc(sz, GFP_KERNEL);
948 if (!rht_node->map[vl]) {
949 kfree(rht_node);
950 ret = -ENOMEM;
951 goto out;
952 }
953 rht_node->cpu_id = cpu;
954 rht_node->map[vl]->mask = 0;
955 rht_node->map[vl]->ctr = 1;
956 rht_node->map[vl]->sde[0] = sde;
957
958 ret = rhashtable_insert_fast(dd->sdma_rht,
959 &rht_node->node,
960 sdma_rht_params);
961 if (ret) {
962 kfree(rht_node->map[vl]);
963 kfree(rht_node);
964 dd_dev_err(sde->dd, "Failed to set process to sde affinity for cpu %lu\n",
965 cpu);
966 goto out;
967 }
968
969 } else {
970 int ctr, pow;
971
972 /* Add new user mappings */
973 if (!rht_node->map[vl])
974 rht_node->map[vl] = kzalloc(sz, GFP_KERNEL);
975
976 if (!rht_node->map[vl]) {
977 ret = -ENOMEM;
978 goto out;
979 }
980
981 rht_node->map[vl]->ctr++;
982 ctr = rht_node->map[vl]->ctr;
983 rht_node->map[vl]->sde[ctr - 1] = sde;
984 pow = roundup_pow_of_two(ctr);
985 rht_node->map[vl]->mask = pow - 1;
986
987 /* Populate the sde map table */
988 sdma_populate_sde_map(rht_node->map[vl]);
989 }
990 cpumask_set_cpu(cpu, new_mask);
991 }
992
993 /* Clean up old mappings */
994 for_each_online_cpu(cpu) {
995 struct sdma_rht_node *rht_node;
996
997 /* Don't cleanup sdes that are set in the new mask */
998 if (cpumask_test_cpu(cpu, mask))
999 continue;
1000
1001 rht_node = rhashtable_lookup_fast(dd->sdma_rht, &cpu,
1002 sdma_rht_params);
1003 if (rht_node) {
1004 bool empty = true;
1005 int i;
1006
1007 /* Remove mappings for old sde */
1008 for (i = 0; i < HFI1_MAX_VLS_SUPPORTED; i++)
1009 if (rht_node->map[i])
1010 sdma_cleanup_sde_map(rht_node->map[i],
1011 sde);
1012
1013 /* Free empty hash table entries */
1014 for (i = 0; i < HFI1_MAX_VLS_SUPPORTED; i++) {
1015 if (!rht_node->map[i])
1016 continue;
1017
1018 if (rht_node->map[i]->ctr) {
1019 empty = false;
1020 break;
1021 }
1022 }
1023
1024 if (empty) {
1025 ret = rhashtable_remove_fast(dd->sdma_rht,
1026 &rht_node->node,
1027 sdma_rht_params);
1028 WARN_ON(ret);
1029
1030 for (i = 0; i < HFI1_MAX_VLS_SUPPORTED; i++)
1031 kfree(rht_node->map[i]);
1032
1033 kfree(rht_node);
1034 }
1035 }
1036 }
1037
1038 cpumask_copy(&sde->cpu_mask, new_mask);
1039 out:
1040 mutex_unlock(&process_to_sde_mutex);
1041 out_free:
1042 free_cpumask_var(mask);
1043 free_cpumask_var(new_mask);
1044 return ret ? : strnlen(buf, PAGE_SIZE);
1045 }
1046
sdma_get_cpu_to_sde_map(struct sdma_engine * sde,char * buf)1047 ssize_t sdma_get_cpu_to_sde_map(struct sdma_engine *sde, char *buf)
1048 {
1049 mutex_lock(&process_to_sde_mutex);
1050 if (cpumask_empty(&sde->cpu_mask))
1051 snprintf(buf, PAGE_SIZE, "%s\n", "empty");
1052 else
1053 sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(&sde->cpu_mask));
1054 mutex_unlock(&process_to_sde_mutex);
1055 return strnlen(buf, PAGE_SIZE);
1056 }
1057
sdma_rht_free(void * ptr,void * arg)1058 static void sdma_rht_free(void *ptr, void *arg)
1059 {
1060 struct sdma_rht_node *rht_node = ptr;
1061 int i;
1062
1063 for (i = 0; i < HFI1_MAX_VLS_SUPPORTED; i++)
1064 kfree(rht_node->map[i]);
1065
1066 kfree(rht_node);
1067 }
1068
1069 /**
1070 * sdma_seqfile_dump_cpu_list() - debugfs dump the cpu to sdma mappings
1071 * @s: seq file
1072 * @dd: hfi1_devdata
1073 * @cpuid: cpu id
1074 *
1075 * This routine dumps the process to sde mappings per cpu
1076 */
sdma_seqfile_dump_cpu_list(struct seq_file * s,struct hfi1_devdata * dd,unsigned long cpuid)1077 void sdma_seqfile_dump_cpu_list(struct seq_file *s,
1078 struct hfi1_devdata *dd,
1079 unsigned long cpuid)
1080 {
1081 struct sdma_rht_node *rht_node;
1082 int i, j;
1083
1084 rht_node = rhashtable_lookup_fast(dd->sdma_rht, &cpuid,
1085 sdma_rht_params);
1086 if (!rht_node)
1087 return;
1088
1089 seq_printf(s, "cpu%3lu: ", cpuid);
1090 for (i = 0; i < HFI1_MAX_VLS_SUPPORTED; i++) {
1091 if (!rht_node->map[i] || !rht_node->map[i]->ctr)
1092 continue;
1093
1094 seq_printf(s, " vl%d: [", i);
1095
1096 for (j = 0; j < rht_node->map[i]->ctr; j++) {
1097 if (!rht_node->map[i]->sde[j])
1098 continue;
1099
1100 if (j > 0)
1101 seq_puts(s, ",");
1102
1103 seq_printf(s, " sdma%2d",
1104 rht_node->map[i]->sde[j]->this_idx);
1105 }
1106 seq_puts(s, " ]");
1107 }
1108
1109 seq_puts(s, "\n");
1110 }
1111
1112 /*
1113 * Free the indicated map struct
1114 */
sdma_map_free(struct sdma_vl_map * m)1115 static void sdma_map_free(struct sdma_vl_map *m)
1116 {
1117 int i;
1118
1119 for (i = 0; m && i < m->actual_vls; i++)
1120 kfree(m->map[i]);
1121 kfree(m);
1122 }
1123
1124 /*
1125 * Handle RCU callback
1126 */
sdma_map_rcu_callback(struct rcu_head * list)1127 static void sdma_map_rcu_callback(struct rcu_head *list)
1128 {
1129 struct sdma_vl_map *m = container_of(list, struct sdma_vl_map, list);
1130
1131 sdma_map_free(m);
1132 }
1133
1134 /**
1135 * sdma_map_init - called when # vls change
1136 * @dd: hfi1_devdata
1137 * @port: port number
1138 * @num_vls: number of vls
1139 * @vl_engines: per vl engine mapping (optional)
1140 *
1141 * This routine changes the mapping based on the number of vls.
1142 *
1143 * vl_engines is used to specify a non-uniform vl/engine loading. NULL
1144 * implies auto computing the loading and giving each VLs a uniform
1145 * distribution of engines per VL.
1146 *
1147 * The auto algorithm computes the sde_per_vl and the number of extra
1148 * engines. Any extra engines are added from the last VL on down.
1149 *
1150 * rcu locking is used here to control access to the mapping fields.
1151 *
1152 * If either the num_vls or num_sdma are non-power of 2, the array sizes
1153 * in the struct sdma_vl_map and the struct sdma_map_elem are rounded
1154 * up to the next highest power of 2 and the first entry is reused
1155 * in a round robin fashion.
1156 *
1157 * If an error occurs the map change is not done and the mapping is
1158 * not changed.
1159 *
1160 */
sdma_map_init(struct hfi1_devdata * dd,u8 port,u8 num_vls,u8 * vl_engines)1161 int sdma_map_init(struct hfi1_devdata *dd, u8 port, u8 num_vls, u8 *vl_engines)
1162 {
1163 int i, j;
1164 int extra, sde_per_vl;
1165 int engine = 0;
1166 u8 lvl_engines[OPA_MAX_VLS];
1167 struct sdma_vl_map *oldmap, *newmap;
1168
1169 if (!(dd->flags & HFI1_HAS_SEND_DMA))
1170 return 0;
1171
1172 if (!vl_engines) {
1173 /* truncate divide */
1174 sde_per_vl = dd->num_sdma / num_vls;
1175 /* extras */
1176 extra = dd->num_sdma % num_vls;
1177 vl_engines = lvl_engines;
1178 /* add extras from last vl down */
1179 for (i = num_vls - 1; i >= 0; i--, extra--)
1180 vl_engines[i] = sde_per_vl + (extra > 0 ? 1 : 0);
1181 }
1182 /* build new map */
1183 newmap = kzalloc(
1184 sizeof(struct sdma_vl_map) +
1185 roundup_pow_of_two(num_vls) *
1186 sizeof(struct sdma_map_elem *),
1187 GFP_KERNEL);
1188 if (!newmap)
1189 goto bail;
1190 newmap->actual_vls = num_vls;
1191 newmap->vls = roundup_pow_of_two(num_vls);
1192 newmap->mask = (1 << ilog2(newmap->vls)) - 1;
1193 /* initialize back-map */
1194 for (i = 0; i < TXE_NUM_SDMA_ENGINES; i++)
1195 newmap->engine_to_vl[i] = -1;
1196 for (i = 0; i < newmap->vls; i++) {
1197 /* save for wrap around */
1198 int first_engine = engine;
1199
1200 if (i < newmap->actual_vls) {
1201 int sz = roundup_pow_of_two(vl_engines[i]);
1202
1203 /* only allocate once */
1204 newmap->map[i] = kzalloc(
1205 sizeof(struct sdma_map_elem) +
1206 sz * sizeof(struct sdma_engine *),
1207 GFP_KERNEL);
1208 if (!newmap->map[i])
1209 goto bail;
1210 newmap->map[i]->mask = (1 << ilog2(sz)) - 1;
1211 /* assign engines */
1212 for (j = 0; j < sz; j++) {
1213 newmap->map[i]->sde[j] =
1214 &dd->per_sdma[engine];
1215 if (++engine >= first_engine + vl_engines[i])
1216 /* wrap back to first engine */
1217 engine = first_engine;
1218 }
1219 /* assign back-map */
1220 for (j = 0; j < vl_engines[i]; j++)
1221 newmap->engine_to_vl[first_engine + j] = i;
1222 } else {
1223 /* just re-use entry without allocating */
1224 newmap->map[i] = newmap->map[i % num_vls];
1225 }
1226 engine = first_engine + vl_engines[i];
1227 }
1228 /* newmap in hand, save old map */
1229 spin_lock_irq(&dd->sde_map_lock);
1230 oldmap = rcu_dereference_protected(dd->sdma_map,
1231 lockdep_is_held(&dd->sde_map_lock));
1232
1233 /* publish newmap */
1234 rcu_assign_pointer(dd->sdma_map, newmap);
1235
1236 spin_unlock_irq(&dd->sde_map_lock);
1237 /* success, free any old map after grace period */
1238 if (oldmap)
1239 call_rcu(&oldmap->list, sdma_map_rcu_callback);
1240 return 0;
1241 bail:
1242 /* free any partial allocation */
1243 sdma_map_free(newmap);
1244 return -ENOMEM;
1245 }
1246
1247 /**
1248 * sdma_clean - Clean up allocated memory
1249 * @dd: struct hfi1_devdata
1250 * @num_engines: num sdma engines
1251 *
1252 * This routine can be called regardless of the success of
1253 * sdma_init()
1254 */
sdma_clean(struct hfi1_devdata * dd,size_t num_engines)1255 void sdma_clean(struct hfi1_devdata *dd, size_t num_engines)
1256 {
1257 size_t i;
1258 struct sdma_engine *sde;
1259 struct sdma_vl_map *map;
1260
1261 if (dd->sdma_pad_dma) {
1262 dma_free_coherent(&dd->pcidev->dev, SDMA_PAD,
1263 (void *)dd->sdma_pad_dma,
1264 dd->sdma_pad_phys);
1265 dd->sdma_pad_dma = NULL;
1266 dd->sdma_pad_phys = 0;
1267 }
1268 if (dd->sdma_heads_dma) {
1269 dma_free_coherent(&dd->pcidev->dev, dd->sdma_heads_size,
1270 (void *)dd->sdma_heads_dma,
1271 dd->sdma_heads_phys);
1272 dd->sdma_heads_dma = NULL;
1273 dd->sdma_heads_phys = 0;
1274 }
1275 for (i = 0; dd->per_sdma && i < num_engines; ++i) {
1276 sde = &dd->per_sdma[i];
1277
1278 sde->head_dma = NULL;
1279 sde->head_phys = 0;
1280
1281 if (sde->descq) {
1282 dma_free_coherent(
1283 &dd->pcidev->dev,
1284 sde->descq_cnt * sizeof(u64[2]),
1285 sde->descq,
1286 sde->descq_phys
1287 );
1288 sde->descq = NULL;
1289 sde->descq_phys = 0;
1290 }
1291 kvfree(sde->tx_ring);
1292 sde->tx_ring = NULL;
1293 }
1294 if (rcu_access_pointer(dd->sdma_map)) {
1295 spin_lock_irq(&dd->sde_map_lock);
1296 map = rcu_access_pointer(dd->sdma_map);
1297 RCU_INIT_POINTER(dd->sdma_map, NULL);
1298 spin_unlock_irq(&dd->sde_map_lock);
1299 synchronize_rcu();
1300 sdma_map_free(map);
1301 }
1302 kfree(dd->per_sdma);
1303 dd->per_sdma = NULL;
1304
1305 if (dd->sdma_rht) {
1306 rhashtable_free_and_destroy(dd->sdma_rht, sdma_rht_free, NULL);
1307 kfree(dd->sdma_rht);
1308 dd->sdma_rht = NULL;
1309 }
1310 }
1311
1312 /**
1313 * sdma_init() - called when device probed
1314 * @dd: hfi1_devdata
1315 * @port: port number (currently only zero)
1316 *
1317 * Initializes each sde and its csrs.
1318 * Interrupts are not required to be enabled.
1319 *
1320 * Returns:
1321 * 0 - success, -errno on failure
1322 */
sdma_init(struct hfi1_devdata * dd,u8 port)1323 int sdma_init(struct hfi1_devdata *dd, u8 port)
1324 {
1325 unsigned this_idx;
1326 struct sdma_engine *sde;
1327 struct rhashtable *tmp_sdma_rht;
1328 u16 descq_cnt;
1329 void *curr_head;
1330 struct hfi1_pportdata *ppd = dd->pport + port;
1331 u32 per_sdma_credits;
1332 uint idle_cnt = sdma_idle_cnt;
1333 size_t num_engines = chip_sdma_engines(dd);
1334 int ret = -ENOMEM;
1335
1336 if (!HFI1_CAP_IS_KSET(SDMA)) {
1337 HFI1_CAP_CLEAR(SDMA_AHG);
1338 return 0;
1339 }
1340 if (mod_num_sdma &&
1341 /* can't exceed chip support */
1342 mod_num_sdma <= chip_sdma_engines(dd) &&
1343 /* count must be >= vls */
1344 mod_num_sdma >= num_vls)
1345 num_engines = mod_num_sdma;
1346
1347 dd_dev_info(dd, "SDMA mod_num_sdma: %u\n", mod_num_sdma);
1348 dd_dev_info(dd, "SDMA chip_sdma_engines: %u\n", chip_sdma_engines(dd));
1349 dd_dev_info(dd, "SDMA chip_sdma_mem_size: %u\n",
1350 chip_sdma_mem_size(dd));
1351
1352 per_sdma_credits =
1353 chip_sdma_mem_size(dd) / (num_engines * SDMA_BLOCK_SIZE);
1354
1355 /* set up freeze waitqueue */
1356 init_waitqueue_head(&dd->sdma_unfreeze_wq);
1357 atomic_set(&dd->sdma_unfreeze_count, 0);
1358
1359 descq_cnt = sdma_get_descq_cnt();
1360 dd_dev_info(dd, "SDMA engines %zu descq_cnt %u\n",
1361 num_engines, descq_cnt);
1362
1363 /* alloc memory for array of send engines */
1364 dd->per_sdma = kcalloc_node(num_engines, sizeof(*dd->per_sdma),
1365 GFP_KERNEL, dd->node);
1366 if (!dd->per_sdma)
1367 return ret;
1368
1369 idle_cnt = ns_to_cclock(dd, idle_cnt);
1370 if (idle_cnt)
1371 dd->default_desc1 =
1372 SDMA_DESC1_HEAD_TO_HOST_FLAG;
1373 else
1374 dd->default_desc1 =
1375 SDMA_DESC1_INT_REQ_FLAG;
1376
1377 if (!sdma_desct_intr)
1378 sdma_desct_intr = SDMA_DESC_INTR;
1379
1380 /* Allocate memory for SendDMA descriptor FIFOs */
1381 for (this_idx = 0; this_idx < num_engines; ++this_idx) {
1382 sde = &dd->per_sdma[this_idx];
1383 sde->dd = dd;
1384 sde->ppd = ppd;
1385 sde->this_idx = this_idx;
1386 sde->descq_cnt = descq_cnt;
1387 sde->desc_avail = sdma_descq_freecnt(sde);
1388 sde->sdma_shift = ilog2(descq_cnt);
1389 sde->sdma_mask = (1 << sde->sdma_shift) - 1;
1390
1391 /* Create a mask specifically for each interrupt source */
1392 sde->int_mask = (u64)1 << (0 * TXE_NUM_SDMA_ENGINES +
1393 this_idx);
1394 sde->progress_mask = (u64)1 << (1 * TXE_NUM_SDMA_ENGINES +
1395 this_idx);
1396 sde->idle_mask = (u64)1 << (2 * TXE_NUM_SDMA_ENGINES +
1397 this_idx);
1398 /* Create a combined mask to cover all 3 interrupt sources */
1399 sde->imask = sde->int_mask | sde->progress_mask |
1400 sde->idle_mask;
1401
1402 spin_lock_init(&sde->tail_lock);
1403 seqlock_init(&sde->head_lock);
1404 spin_lock_init(&sde->senddmactrl_lock);
1405 spin_lock_init(&sde->flushlist_lock);
1406 seqlock_init(&sde->waitlock);
1407 /* insure there is always a zero bit */
1408 sde->ahg_bits = 0xfffffffe00000000ULL;
1409
1410 sdma_set_state(sde, sdma_state_s00_hw_down);
1411
1412 /* set up reference counting */
1413 kref_init(&sde->state.kref);
1414 init_completion(&sde->state.comp);
1415
1416 INIT_LIST_HEAD(&sde->flushlist);
1417 INIT_LIST_HEAD(&sde->dmawait);
1418
1419 sde->tail_csr =
1420 get_kctxt_csr_addr(dd, this_idx, SD(TAIL));
1421
1422 tasklet_setup(&sde->sdma_hw_clean_up_task,
1423 sdma_hw_clean_up_task);
1424 tasklet_setup(&sde->sdma_sw_clean_up_task,
1425 sdma_sw_clean_up_task);
1426 INIT_WORK(&sde->err_halt_worker, sdma_err_halt_wait);
1427 INIT_WORK(&sde->flush_worker, sdma_field_flush);
1428
1429 sde->progress_check_head = 0;
1430
1431 timer_setup(&sde->err_progress_check_timer,
1432 sdma_err_progress_check, 0);
1433
1434 sde->descq = dma_alloc_coherent(&dd->pcidev->dev,
1435 descq_cnt * sizeof(u64[2]),
1436 &sde->descq_phys, GFP_KERNEL);
1437 if (!sde->descq)
1438 goto bail;
1439 sde->tx_ring =
1440 kvzalloc_node(array_size(descq_cnt,
1441 sizeof(struct sdma_txreq *)),
1442 GFP_KERNEL, dd->node);
1443 if (!sde->tx_ring)
1444 goto bail;
1445 }
1446
1447 dd->sdma_heads_size = L1_CACHE_BYTES * num_engines;
1448 /* Allocate memory for DMA of head registers to memory */
1449 dd->sdma_heads_dma = dma_alloc_coherent(&dd->pcidev->dev,
1450 dd->sdma_heads_size,
1451 &dd->sdma_heads_phys,
1452 GFP_KERNEL);
1453 if (!dd->sdma_heads_dma) {
1454 dd_dev_err(dd, "failed to allocate SendDMA head memory\n");
1455 goto bail;
1456 }
1457
1458 /* Allocate memory for pad */
1459 dd->sdma_pad_dma = dma_alloc_coherent(&dd->pcidev->dev, SDMA_PAD,
1460 &dd->sdma_pad_phys, GFP_KERNEL);
1461 if (!dd->sdma_pad_dma) {
1462 dd_dev_err(dd, "failed to allocate SendDMA pad memory\n");
1463 goto bail;
1464 }
1465
1466 /* assign each engine to different cacheline and init registers */
1467 curr_head = (void *)dd->sdma_heads_dma;
1468 for (this_idx = 0; this_idx < num_engines; ++this_idx) {
1469 unsigned long phys_offset;
1470
1471 sde = &dd->per_sdma[this_idx];
1472
1473 sde->head_dma = curr_head;
1474 curr_head += L1_CACHE_BYTES;
1475 phys_offset = (unsigned long)sde->head_dma -
1476 (unsigned long)dd->sdma_heads_dma;
1477 sde->head_phys = dd->sdma_heads_phys + phys_offset;
1478 init_sdma_regs(sde, per_sdma_credits, idle_cnt);
1479 }
1480 dd->flags |= HFI1_HAS_SEND_DMA;
1481 dd->flags |= idle_cnt ? HFI1_HAS_SDMA_TIMEOUT : 0;
1482 dd->num_sdma = num_engines;
1483 ret = sdma_map_init(dd, port, ppd->vls_operational, NULL);
1484 if (ret < 0)
1485 goto bail;
1486
1487 tmp_sdma_rht = kzalloc_obj(*tmp_sdma_rht);
1488 if (!tmp_sdma_rht) {
1489 ret = -ENOMEM;
1490 goto bail;
1491 }
1492
1493 ret = rhashtable_init(tmp_sdma_rht, &sdma_rht_params);
1494 if (ret < 0) {
1495 kfree(tmp_sdma_rht);
1496 goto bail;
1497 }
1498
1499 dd->sdma_rht = tmp_sdma_rht;
1500
1501 dd_dev_info(dd, "SDMA num_sdma: %u\n", dd->num_sdma);
1502 return 0;
1503
1504 bail:
1505 sdma_clean(dd, num_engines);
1506 return ret;
1507 }
1508
1509 /**
1510 * sdma_all_running() - called when the link goes up
1511 * @dd: hfi1_devdata
1512 *
1513 * This routine moves all engines to the running state.
1514 */
sdma_all_running(struct hfi1_devdata * dd)1515 void sdma_all_running(struct hfi1_devdata *dd)
1516 {
1517 struct sdma_engine *sde;
1518 unsigned int i;
1519
1520 /* move all engines to running */
1521 for (i = 0; i < dd->num_sdma; ++i) {
1522 sde = &dd->per_sdma[i];
1523 sdma_process_event(sde, sdma_event_e30_go_running);
1524 }
1525 }
1526
1527 /**
1528 * sdma_start() - called to kick off state processing for all engines
1529 * @dd: hfi1_devdata
1530 *
1531 * This routine is for kicking off the state processing for all required
1532 * sdma engines. Interrupts need to be working at this point.
1533 *
1534 */
sdma_start(struct hfi1_devdata * dd)1535 void sdma_start(struct hfi1_devdata *dd)
1536 {
1537 unsigned i;
1538 struct sdma_engine *sde;
1539
1540 /* kick off the engines state processing */
1541 for (i = 0; i < dd->num_sdma; ++i) {
1542 sde = &dd->per_sdma[i];
1543 sdma_process_event(sde, sdma_event_e10_go_hw_start);
1544 }
1545 }
1546
1547 /**
1548 * sdma_exit() - used when module is removed
1549 * @dd: hfi1_devdata
1550 */
sdma_exit(struct hfi1_devdata * dd)1551 void sdma_exit(struct hfi1_devdata *dd)
1552 {
1553 unsigned this_idx;
1554 struct sdma_engine *sde;
1555
1556 for (this_idx = 0; dd->per_sdma && this_idx < dd->num_sdma;
1557 ++this_idx) {
1558 sde = &dd->per_sdma[this_idx];
1559 if (!list_empty(&sde->dmawait))
1560 dd_dev_err(dd, "sde %u: dmawait list not empty!\n",
1561 sde->this_idx);
1562 sdma_process_event(sde, sdma_event_e00_go_hw_down);
1563
1564 timer_delete_sync(&sde->err_progress_check_timer);
1565
1566 /*
1567 * This waits for the state machine to exit so it is not
1568 * necessary to kill the sdma_sw_clean_up_task to make sure
1569 * it is not running.
1570 */
1571 sdma_finalput(&sde->state);
1572 }
1573 }
1574
1575 /*
1576 * unmap the indicated descriptor
1577 */
sdma_unmap_desc(struct hfi1_devdata * dd,struct sdma_desc * descp)1578 static inline void sdma_unmap_desc(
1579 struct hfi1_devdata *dd,
1580 struct sdma_desc *descp)
1581 {
1582 switch (sdma_mapping_type(descp)) {
1583 case SDMA_MAP_SINGLE:
1584 dma_unmap_single(&dd->pcidev->dev, sdma_mapping_addr(descp),
1585 sdma_mapping_len(descp), DMA_TO_DEVICE);
1586 break;
1587 case SDMA_MAP_PAGE:
1588 dma_unmap_page(&dd->pcidev->dev, sdma_mapping_addr(descp),
1589 sdma_mapping_len(descp), DMA_TO_DEVICE);
1590 break;
1591 }
1592
1593 if (descp->pinning_ctx && descp->ctx_put)
1594 descp->ctx_put(descp->pinning_ctx);
1595 descp->pinning_ctx = NULL;
1596 }
1597
1598 /*
1599 * return the mode as indicated by the first
1600 * descriptor in the tx.
1601 */
ahg_mode(struct sdma_txreq * tx)1602 static inline u8 ahg_mode(struct sdma_txreq *tx)
1603 {
1604 return (tx->descp[0].qw[1] & SDMA_DESC1_HEADER_MODE_SMASK)
1605 >> SDMA_DESC1_HEADER_MODE_SHIFT;
1606 }
1607
1608 /**
1609 * __sdma_txclean() - clean tx of mappings, descp *kmalloc's
1610 * @dd: hfi1_devdata for unmapping
1611 * @tx: tx request to clean
1612 *
1613 * This is used in the progress routine to clean the tx or
1614 * by the ULP to toss an in-process tx build.
1615 *
1616 * The code can be called multiple times without issue.
1617 *
1618 */
__sdma_txclean(struct hfi1_devdata * dd,struct sdma_txreq * tx)1619 void __sdma_txclean(
1620 struct hfi1_devdata *dd,
1621 struct sdma_txreq *tx)
1622 {
1623 u16 i;
1624
1625 if (tx->num_desc) {
1626 u8 skip = 0, mode = ahg_mode(tx);
1627
1628 /* unmap first */
1629 sdma_unmap_desc(dd, &tx->descp[0]);
1630 /* determine number of AHG descriptors to skip */
1631 if (mode > SDMA_AHG_APPLY_UPDATE1)
1632 skip = mode >> 1;
1633 for (i = 1 + skip; i < tx->num_desc; i++)
1634 sdma_unmap_desc(dd, &tx->descp[i]);
1635 tx->num_desc = 0;
1636 }
1637 kfree(tx->coalesce_buf);
1638 tx->coalesce_buf = NULL;
1639 /* kmalloc'ed descp */
1640 if (unlikely(tx->desc_limit > ARRAY_SIZE(tx->descs))) {
1641 tx->desc_limit = ARRAY_SIZE(tx->descs);
1642 kfree(tx->descp);
1643 }
1644 }
1645
sdma_gethead(struct sdma_engine * sde)1646 static inline u16 sdma_gethead(struct sdma_engine *sde)
1647 {
1648 struct hfi1_devdata *dd = sde->dd;
1649 int use_dmahead;
1650 u16 hwhead;
1651
1652 #ifdef CONFIG_SDMA_VERBOSITY
1653 dd_dev_err(sde->dd, "CONFIG SDMA(%u) %s:%d %s()\n",
1654 sde->this_idx, slashstrip(__FILE__), __LINE__, __func__);
1655 #endif
1656
1657 retry:
1658 use_dmahead = HFI1_CAP_IS_KSET(USE_SDMA_HEAD) && __sdma_running(sde) &&
1659 (dd->flags & HFI1_HAS_SDMA_TIMEOUT);
1660 hwhead = use_dmahead ?
1661 (u16)le64_to_cpu(*sde->head_dma) :
1662 (u16)read_sde_csr(sde, SD(HEAD));
1663
1664 if (unlikely(HFI1_CAP_IS_KSET(SDMA_HEAD_CHECK))) {
1665 u16 cnt;
1666 u16 swtail;
1667 u16 swhead;
1668 int sane;
1669
1670 swhead = sde->descq_head & sde->sdma_mask;
1671 /* this code is really bad for cache line trading */
1672 swtail = READ_ONCE(sde->descq_tail) & sde->sdma_mask;
1673 cnt = sde->descq_cnt;
1674
1675 if (swhead < swtail)
1676 /* not wrapped */
1677 sane = (hwhead >= swhead) & (hwhead <= swtail);
1678 else if (swhead > swtail)
1679 /* wrapped around */
1680 sane = ((hwhead >= swhead) && (hwhead < cnt)) ||
1681 (hwhead <= swtail);
1682 else
1683 /* empty */
1684 sane = (hwhead == swhead);
1685
1686 if (unlikely(!sane)) {
1687 dd_dev_err(dd, "SDMA(%u) bad head (%s) hwhd=%u swhd=%u swtl=%u cnt=%u\n",
1688 sde->this_idx,
1689 use_dmahead ? "dma" : "kreg",
1690 hwhead, swhead, swtail, cnt);
1691 if (use_dmahead) {
1692 /* try one more time, using csr */
1693 use_dmahead = 0;
1694 goto retry;
1695 }
1696 /* proceed as if no progress */
1697 hwhead = swhead;
1698 }
1699 }
1700 return hwhead;
1701 }
1702
1703 /*
1704 * This is called when there are send DMA descriptors that might be
1705 * available.
1706 *
1707 * This is called with head_lock held.
1708 */
sdma_desc_avail(struct sdma_engine * sde,uint avail)1709 static void sdma_desc_avail(struct sdma_engine *sde, uint avail)
1710 {
1711 struct iowait *wait, *nw, *twait;
1712 struct iowait *waits[SDMA_WAIT_BATCH_SIZE];
1713 uint i, n = 0, seq, tidx = 0;
1714
1715 #ifdef CONFIG_SDMA_VERBOSITY
1716 dd_dev_err(sde->dd, "CONFIG SDMA(%u) %s:%d %s()\n", sde->this_idx,
1717 slashstrip(__FILE__), __LINE__, __func__);
1718 dd_dev_err(sde->dd, "avail: %u\n", avail);
1719 #endif
1720
1721 do {
1722 seq = read_seqbegin(&sde->waitlock);
1723 if (!list_empty(&sde->dmawait)) {
1724 /* at least one item */
1725 write_seqlock(&sde->waitlock);
1726 /* Harvest waiters wanting DMA descriptors */
1727 list_for_each_entry_safe(
1728 wait,
1729 nw,
1730 &sde->dmawait,
1731 list) {
1732 u32 num_desc;
1733
1734 if (!wait->wakeup)
1735 continue;
1736 if (n == ARRAY_SIZE(waits))
1737 break;
1738 iowait_init_priority(wait);
1739 num_desc = iowait_get_all_desc(wait);
1740 if (num_desc > avail)
1741 break;
1742 avail -= num_desc;
1743 /* Find the top-priority wait memeber */
1744 if (n) {
1745 twait = waits[tidx];
1746 tidx =
1747 iowait_priority_update_top(wait,
1748 twait,
1749 n,
1750 tidx);
1751 }
1752 list_del_init(&wait->list);
1753 waits[n++] = wait;
1754 }
1755 write_sequnlock(&sde->waitlock);
1756 break;
1757 }
1758 } while (read_seqretry(&sde->waitlock, seq));
1759
1760 /* Schedule the top-priority entry first */
1761 if (n)
1762 waits[tidx]->wakeup(waits[tidx], SDMA_AVAIL_REASON);
1763
1764 for (i = 0; i < n; i++)
1765 if (i != tidx)
1766 waits[i]->wakeup(waits[i], SDMA_AVAIL_REASON);
1767 }
1768
1769 /* head_lock must be held */
sdma_make_progress(struct sdma_engine * sde,u64 status)1770 static void sdma_make_progress(struct sdma_engine *sde, u64 status)
1771 {
1772 struct sdma_txreq *txp = NULL;
1773 int progress = 0;
1774 u16 hwhead, swhead;
1775 int idle_check_done = 0;
1776
1777 hwhead = sdma_gethead(sde);
1778
1779 /* The reason for some of the complexity of this code is that
1780 * not all descriptors have corresponding txps. So, we have to
1781 * be able to skip over descs until we wander into the range of
1782 * the next txp on the list.
1783 */
1784
1785 retry:
1786 txp = get_txhead(sde);
1787 swhead = sde->descq_head & sde->sdma_mask;
1788 trace_hfi1_sdma_progress(sde, hwhead, swhead, txp);
1789 while (swhead != hwhead) {
1790 /* advance head, wrap if needed */
1791 swhead = ++sde->descq_head & sde->sdma_mask;
1792
1793 /* if now past this txp's descs, do the callback */
1794 if (txp && txp->next_descq_idx == swhead) {
1795 /* remove from list */
1796 sde->tx_ring[sde->tx_head++ & sde->sdma_mask] = NULL;
1797 complete_tx(sde, txp, SDMA_TXREQ_S_OK);
1798 /* see if there is another txp */
1799 txp = get_txhead(sde);
1800 }
1801 trace_hfi1_sdma_progress(sde, hwhead, swhead, txp);
1802 progress++;
1803 }
1804
1805 /*
1806 * The SDMA idle interrupt is not guaranteed to be ordered with respect
1807 * to updates to the dma_head location in host memory. The head
1808 * value read might not be fully up to date. If there are pending
1809 * descriptors and the SDMA idle interrupt fired then read from the
1810 * CSR SDMA head instead to get the latest value from the hardware.
1811 * The hardware SDMA head should be read at most once in this invocation
1812 * of sdma_make_progress(..) which is ensured by idle_check_done flag
1813 */
1814 if ((status & sde->idle_mask) && !idle_check_done) {
1815 u16 swtail;
1816
1817 swtail = READ_ONCE(sde->descq_tail) & sde->sdma_mask;
1818 if (swtail != hwhead) {
1819 hwhead = (u16)read_sde_csr(sde, SD(HEAD));
1820 idle_check_done = 1;
1821 goto retry;
1822 }
1823 }
1824
1825 sde->last_status = status;
1826 if (progress)
1827 sdma_desc_avail(sde, sdma_descq_freecnt(sde));
1828 }
1829
1830 /*
1831 * sdma_engine_interrupt() - interrupt handler for engine
1832 * @sde: sdma engine
1833 * @status: sdma interrupt reason
1834 *
1835 * Status is a mask of the 3 possible interrupts for this engine. It will
1836 * contain bits _only_ for this SDMA engine. It will contain at least one
1837 * bit, it may contain more.
1838 */
sdma_engine_interrupt(struct sdma_engine * sde,u64 status)1839 void sdma_engine_interrupt(struct sdma_engine *sde, u64 status)
1840 {
1841 trace_hfi1_sdma_engine_interrupt(sde, status);
1842 write_seqlock(&sde->head_lock);
1843 sdma_set_desc_cnt(sde, sdma_desct_intr);
1844 if (status & sde->idle_mask)
1845 sde->idle_int_cnt++;
1846 else if (status & sde->progress_mask)
1847 sde->progress_int_cnt++;
1848 else if (status & sde->int_mask)
1849 sde->sdma_int_cnt++;
1850 sdma_make_progress(sde, status);
1851 write_sequnlock(&sde->head_lock);
1852 }
1853
1854 /**
1855 * sdma_engine_error() - error handler for engine
1856 * @sde: sdma engine
1857 * @status: sdma interrupt reason
1858 */
sdma_engine_error(struct sdma_engine * sde,u64 status)1859 void sdma_engine_error(struct sdma_engine *sde, u64 status)
1860 {
1861 unsigned long flags;
1862
1863 #ifdef CONFIG_SDMA_VERBOSITY
1864 dd_dev_err(sde->dd, "CONFIG SDMA(%u) error status 0x%llx state %s\n",
1865 sde->this_idx,
1866 (unsigned long long)status,
1867 sdma_state_names[sde->state.current_state]);
1868 #endif
1869 spin_lock_irqsave(&sde->tail_lock, flags);
1870 write_seqlock(&sde->head_lock);
1871 if (status & ALL_SDMA_ENG_HALT_ERRS)
1872 __sdma_process_event(sde, sdma_event_e60_hw_halted);
1873 if (status & ~SD(ENG_ERR_STATUS_SDMA_HALT_ERR_SMASK)) {
1874 dd_dev_err(sde->dd,
1875 "SDMA (%u) engine error: 0x%llx state %s\n",
1876 sde->this_idx,
1877 (unsigned long long)status,
1878 sdma_state_names[sde->state.current_state]);
1879 dump_sdma_state(sde);
1880 }
1881 write_sequnlock(&sde->head_lock);
1882 spin_unlock_irqrestore(&sde->tail_lock, flags);
1883 }
1884
sdma_sendctrl(struct sdma_engine * sde,unsigned op)1885 static void sdma_sendctrl(struct sdma_engine *sde, unsigned op)
1886 {
1887 u64 set_senddmactrl = 0;
1888 u64 clr_senddmactrl = 0;
1889 unsigned long flags;
1890
1891 #ifdef CONFIG_SDMA_VERBOSITY
1892 dd_dev_err(sde->dd, "CONFIG SDMA(%u) senddmactrl E=%d I=%d H=%d C=%d\n",
1893 sde->this_idx,
1894 (op & SDMA_SENDCTRL_OP_ENABLE) ? 1 : 0,
1895 (op & SDMA_SENDCTRL_OP_INTENABLE) ? 1 : 0,
1896 (op & SDMA_SENDCTRL_OP_HALT) ? 1 : 0,
1897 (op & SDMA_SENDCTRL_OP_CLEANUP) ? 1 : 0);
1898 #endif
1899
1900 if (op & SDMA_SENDCTRL_OP_ENABLE)
1901 set_senddmactrl |= SD(CTRL_SDMA_ENABLE_SMASK);
1902 else
1903 clr_senddmactrl |= SD(CTRL_SDMA_ENABLE_SMASK);
1904
1905 if (op & SDMA_SENDCTRL_OP_INTENABLE)
1906 set_senddmactrl |= SD(CTRL_SDMA_INT_ENABLE_SMASK);
1907 else
1908 clr_senddmactrl |= SD(CTRL_SDMA_INT_ENABLE_SMASK);
1909
1910 if (op & SDMA_SENDCTRL_OP_HALT)
1911 set_senddmactrl |= SD(CTRL_SDMA_HALT_SMASK);
1912 else
1913 clr_senddmactrl |= SD(CTRL_SDMA_HALT_SMASK);
1914
1915 spin_lock_irqsave(&sde->senddmactrl_lock, flags);
1916
1917 sde->p_senddmactrl |= set_senddmactrl;
1918 sde->p_senddmactrl &= ~clr_senddmactrl;
1919
1920 if (op & SDMA_SENDCTRL_OP_CLEANUP)
1921 write_sde_csr(sde, SD(CTRL),
1922 sde->p_senddmactrl |
1923 SD(CTRL_SDMA_CLEANUP_SMASK));
1924 else
1925 write_sde_csr(sde, SD(CTRL), sde->p_senddmactrl);
1926
1927 spin_unlock_irqrestore(&sde->senddmactrl_lock, flags);
1928
1929 #ifdef CONFIG_SDMA_VERBOSITY
1930 sdma_dumpstate(sde);
1931 #endif
1932 }
1933
sdma_setlengen(struct sdma_engine * sde)1934 static void sdma_setlengen(struct sdma_engine *sde)
1935 {
1936 #ifdef CONFIG_SDMA_VERBOSITY
1937 dd_dev_err(sde->dd, "CONFIG SDMA(%u) %s:%d %s()\n",
1938 sde->this_idx, slashstrip(__FILE__), __LINE__, __func__);
1939 #endif
1940
1941 /*
1942 * Set SendDmaLenGen and clear-then-set the MSB of the generation
1943 * count to enable generation checking and load the internal
1944 * generation counter.
1945 */
1946 write_sde_csr(sde, SD(LEN_GEN),
1947 (sde->descq_cnt / 64) << SD(LEN_GEN_LENGTH_SHIFT));
1948 write_sde_csr(sde, SD(LEN_GEN),
1949 ((sde->descq_cnt / 64) << SD(LEN_GEN_LENGTH_SHIFT)) |
1950 (4ULL << SD(LEN_GEN_GENERATION_SHIFT)));
1951 }
1952
sdma_update_tail(struct sdma_engine * sde,u16 tail)1953 static inline void sdma_update_tail(struct sdma_engine *sde, u16 tail)
1954 {
1955 /* Commit writes to memory and advance the tail on the chip */
1956 smp_wmb(); /* see get_txhead() */
1957 writeq(tail, sde->tail_csr);
1958 }
1959
1960 /*
1961 * This is called when changing to state s10_hw_start_up_halt_wait as
1962 * a result of send buffer errors or send DMA descriptor errors.
1963 */
sdma_hw_start_up(struct sdma_engine * sde)1964 static void sdma_hw_start_up(struct sdma_engine *sde)
1965 {
1966 u64 reg;
1967
1968 #ifdef CONFIG_SDMA_VERBOSITY
1969 dd_dev_err(sde->dd, "CONFIG SDMA(%u) %s:%d %s()\n",
1970 sde->this_idx, slashstrip(__FILE__), __LINE__, __func__);
1971 #endif
1972
1973 sdma_setlengen(sde);
1974 sdma_update_tail(sde, 0); /* Set SendDmaTail */
1975 *sde->head_dma = 0;
1976
1977 reg = SD(ENG_ERR_CLEAR_SDMA_HEADER_REQUEST_FIFO_UNC_ERR_MASK) <<
1978 SD(ENG_ERR_CLEAR_SDMA_HEADER_REQUEST_FIFO_UNC_ERR_SHIFT);
1979 write_sde_csr(sde, SD(ENG_ERR_CLEAR), reg);
1980 }
1981
1982 /*
1983 * set_sdma_integrity
1984 *
1985 * Set the SEND_DMA_CHECK_ENABLE register for send DMA engine 'sde'.
1986 */
set_sdma_integrity(struct sdma_engine * sde)1987 static void set_sdma_integrity(struct sdma_engine *sde)
1988 {
1989 struct hfi1_devdata *dd = sde->dd;
1990
1991 write_sde_csr(sde, SD(CHECK_ENABLE),
1992 hfi1_pkt_base_sdma_integrity(dd));
1993 }
1994
init_sdma_regs(struct sdma_engine * sde,u32 credits,uint idle_cnt)1995 static void init_sdma_regs(
1996 struct sdma_engine *sde,
1997 u32 credits,
1998 uint idle_cnt)
1999 {
2000 u8 opval, opmask;
2001 #ifdef CONFIG_SDMA_VERBOSITY
2002 struct hfi1_devdata *dd = sde->dd;
2003
2004 dd_dev_err(dd, "CONFIG SDMA(%u) %s:%d %s()\n",
2005 sde->this_idx, slashstrip(__FILE__), __LINE__, __func__);
2006 #endif
2007
2008 write_sde_csr(sde, SD(BASE_ADDR), sde->descq_phys);
2009 sdma_setlengen(sde);
2010 sdma_update_tail(sde, 0); /* Set SendDmaTail */
2011 write_sde_csr(sde, SD(RELOAD_CNT), idle_cnt);
2012 write_sde_csr(sde, SD(DESC_CNT), 0);
2013 write_sde_csr(sde, SD(HEAD_ADDR), sde->head_phys);
2014 write_sde_csr(sde, SD(MEMORY),
2015 ((u64)credits << SD(MEMORY_SDMA_MEMORY_CNT_SHIFT)) |
2016 ((u64)(credits * sde->this_idx) <<
2017 SD(MEMORY_SDMA_MEMORY_INDEX_SHIFT)));
2018 write_sde_csr(sde, SD(ENG_ERR_MASK), ~0ull);
2019 set_sdma_integrity(sde);
2020 opmask = OPCODE_CHECK_MASK_DISABLED;
2021 opval = OPCODE_CHECK_VAL_DISABLED;
2022 write_sde_csr(sde, SD(CHECK_OPCODE),
2023 (opmask << SEND_CTXT_CHECK_OPCODE_MASK_SHIFT) |
2024 (opval << SEND_CTXT_CHECK_OPCODE_VALUE_SHIFT));
2025 }
2026
2027 #ifdef CONFIG_SDMA_VERBOSITY
2028
2029 #define sdma_dumpstate_helper0(reg) do { \
2030 csr = read_csr(sde->dd, reg); \
2031 dd_dev_err(sde->dd, "%36s 0x%016llx\n", #reg, csr); \
2032 } while (0)
2033
2034 #define sdma_dumpstate_helper(reg) do { \
2035 csr = read_sde_csr(sde, reg); \
2036 dd_dev_err(sde->dd, "%36s[%02u] 0x%016llx\n", \
2037 #reg, sde->this_idx, csr); \
2038 } while (0)
2039
2040 #define sdma_dumpstate_helper2(reg) do { \
2041 csr = read_csr(sde->dd, reg + (8 * i)); \
2042 dd_dev_err(sde->dd, "%33s_%02u 0x%016llx\n", \
2043 #reg, i, csr); \
2044 } while (0)
2045
sdma_dumpstate(struct sdma_engine * sde)2046 void sdma_dumpstate(struct sdma_engine *sde)
2047 {
2048 u64 csr;
2049 unsigned i;
2050
2051 sdma_dumpstate_helper(SD(CTRL));
2052 sdma_dumpstate_helper(SD(STATUS));
2053 sdma_dumpstate_helper0(SD(ERR_STATUS));
2054 sdma_dumpstate_helper0(SD(ERR_MASK));
2055 sdma_dumpstate_helper(SD(ENG_ERR_STATUS));
2056 sdma_dumpstate_helper(SD(ENG_ERR_MASK));
2057
2058 for (i = 0; i < CCE_NUM_INT_CSRS; ++i) {
2059 sdma_dumpstate_helper2(CCE_INT_STATUS);
2060 sdma_dumpstate_helper2(CCE_INT_MASK);
2061 sdma_dumpstate_helper2(CCE_INT_BLOCKED);
2062 }
2063
2064 sdma_dumpstate_helper(SD(TAIL));
2065 sdma_dumpstate_helper(SD(HEAD));
2066 sdma_dumpstate_helper(SD(PRIORITY_THLD));
2067 sdma_dumpstate_helper(SD(IDLE_CNT));
2068 sdma_dumpstate_helper(SD(RELOAD_CNT));
2069 sdma_dumpstate_helper(SD(DESC_CNT));
2070 sdma_dumpstate_helper(SD(DESC_FETCHED_CNT));
2071 sdma_dumpstate_helper(SD(MEMORY));
2072 sdma_dumpstate_helper0(SD(ENGINES));
2073 sdma_dumpstate_helper0(SD(MEM_SIZE));
2074 /* sdma_dumpstate_helper(SEND_EGRESS_SEND_DMA_STATUS); */
2075 sdma_dumpstate_helper(SD(BASE_ADDR));
2076 sdma_dumpstate_helper(SD(LEN_GEN));
2077 sdma_dumpstate_helper(SD(HEAD_ADDR));
2078 sdma_dumpstate_helper(SD(CHECK_ENABLE));
2079 sdma_dumpstate_helper(SD(CHECK_VL));
2080 sdma_dumpstate_helper(SD(CHECK_JOB_KEY));
2081 sdma_dumpstate_helper(SD(CHECK_PARTITION_KEY));
2082 sdma_dumpstate_helper(SD(CHECK_SLID));
2083 sdma_dumpstate_helper(SD(CHECK_OPCODE));
2084 }
2085 #endif
2086
dump_sdma_state(struct sdma_engine * sde)2087 static void dump_sdma_state(struct sdma_engine *sde)
2088 {
2089 struct hw_sdma_desc *descqp;
2090 u64 desc[2];
2091 u64 addr;
2092 u8 gen;
2093 u16 len;
2094 u16 head, tail, cnt;
2095
2096 head = sde->descq_head & sde->sdma_mask;
2097 tail = sde->descq_tail & sde->sdma_mask;
2098 cnt = sdma_descq_freecnt(sde);
2099
2100 dd_dev_err(sde->dd,
2101 "SDMA (%u) descq_head: %u descq_tail: %u freecnt: %u FLE %d\n",
2102 sde->this_idx, head, tail, cnt,
2103 !list_empty(&sde->flushlist));
2104
2105 /* print info for each entry in the descriptor queue */
2106 while (head != tail) {
2107 char flags[6] = { 'x', 'x', 'x', 'x', 0 };
2108
2109 descqp = &sde->descq[head];
2110 desc[0] = le64_to_cpu(descqp->qw[0]);
2111 desc[1] = le64_to_cpu(descqp->qw[1]);
2112 flags[0] = (desc[1] & SDMA_DESC1_INT_REQ_FLAG) ? 'I' : '-';
2113 flags[1] = (desc[1] & SDMA_DESC1_HEAD_TO_HOST_FLAG) ?
2114 'H' : '-';
2115 flags[2] = (desc[0] & SDMA_DESC0_FIRST_DESC_FLAG) ? 'F' : '-';
2116 flags[3] = (desc[0] & SDMA_DESC0_LAST_DESC_FLAG) ? 'L' : '-';
2117 addr = (desc[0] >> SDMA_DESC0_PHY_ADDR_SHIFT)
2118 & SDMA_DESC0_PHY_ADDR_MASK;
2119 gen = (desc[1] >> SDMA_DESC1_GENERATION_SHIFT)
2120 & SDMA_DESC1_GENERATION_MASK;
2121 len = (desc[0] >> SDMA_DESC0_BYTE_COUNT_SHIFT)
2122 & SDMA_DESC0_BYTE_COUNT_MASK;
2123 dd_dev_err(sde->dd,
2124 "SDMA sdmadesc[%u]: flags:%s addr:0x%016llx gen:%u len:%u bytes\n",
2125 head, flags, addr, gen, len);
2126 dd_dev_err(sde->dd,
2127 "\tdesc0:0x%016llx desc1 0x%016llx\n",
2128 desc[0], desc[1]);
2129 if (desc[0] & SDMA_DESC0_FIRST_DESC_FLAG)
2130 dd_dev_err(sde->dd,
2131 "\taidx: %u amode: %u alen: %u\n",
2132 (u8)((desc[1] &
2133 SDMA_DESC1_HEADER_INDEX_SMASK) >>
2134 SDMA_DESC1_HEADER_INDEX_SHIFT),
2135 (u8)((desc[1] &
2136 SDMA_DESC1_HEADER_MODE_SMASK) >>
2137 SDMA_DESC1_HEADER_MODE_SHIFT),
2138 (u8)((desc[1] &
2139 SDMA_DESC1_HEADER_DWS_SMASK) >>
2140 SDMA_DESC1_HEADER_DWS_SHIFT));
2141 head++;
2142 head &= sde->sdma_mask;
2143 }
2144 }
2145
2146 #define SDE_FMT \
2147 "SDE %u CPU %d STE %s C 0x%llx S 0x%016llx E 0x%llx T(HW) 0x%llx T(SW) 0x%x H(HW) 0x%llx H(SW) 0x%x H(D) 0x%llx DM 0x%llx GL 0x%llx R 0x%llx LIS 0x%llx AHGI 0x%llx TXT %u TXH %u DT %u DH %u FLNE %d DQF %u SLC 0x%llx\n"
2148 /**
2149 * sdma_seqfile_dump_sde() - debugfs dump of sde
2150 * @s: seq file
2151 * @sde: send dma engine to dump
2152 *
2153 * This routine dumps the sde to the indicated seq file.
2154 */
sdma_seqfile_dump_sde(struct seq_file * s,struct sdma_engine * sde)2155 void sdma_seqfile_dump_sde(struct seq_file *s, struct sdma_engine *sde)
2156 {
2157 u16 head, tail;
2158 struct hw_sdma_desc *descqp;
2159 u64 desc[2];
2160 u64 addr;
2161 u8 gen;
2162 u16 len;
2163
2164 head = sde->descq_head & sde->sdma_mask;
2165 tail = READ_ONCE(sde->descq_tail) & sde->sdma_mask;
2166 seq_printf(s, SDE_FMT, sde->this_idx,
2167 sde->cpu,
2168 sdma_state_name(sde->state.current_state),
2169 (unsigned long long)read_sde_csr(sde, SD(CTRL)),
2170 (unsigned long long)read_sde_csr(sde, SD(STATUS)),
2171 (unsigned long long)read_sde_csr(sde, SD(ENG_ERR_STATUS)),
2172 (unsigned long long)read_sde_csr(sde, SD(TAIL)), tail,
2173 (unsigned long long)read_sde_csr(sde, SD(HEAD)), head,
2174 (unsigned long long)le64_to_cpu(*sde->head_dma),
2175 (unsigned long long)read_sde_csr(sde, SD(MEMORY)),
2176 (unsigned long long)read_sde_csr(sde, SD(LEN_GEN)),
2177 (unsigned long long)read_sde_csr(sde, SD(RELOAD_CNT)),
2178 (unsigned long long)sde->last_status,
2179 (unsigned long long)sde->ahg_bits,
2180 sde->tx_tail,
2181 sde->tx_head,
2182 sde->descq_tail,
2183 sde->descq_head,
2184 !list_empty(&sde->flushlist),
2185 sde->descq_full_count,
2186 (unsigned long long)read_sde_csr(sde, SEND_DMA_CHECK_SLID));
2187
2188 /* print info for each entry in the descriptor queue */
2189 while (head != tail) {
2190 char flags[6] = { 'x', 'x', 'x', 'x', 0 };
2191
2192 descqp = &sde->descq[head];
2193 desc[0] = le64_to_cpu(descqp->qw[0]);
2194 desc[1] = le64_to_cpu(descqp->qw[1]);
2195 flags[0] = (desc[1] & SDMA_DESC1_INT_REQ_FLAG) ? 'I' : '-';
2196 flags[1] = (desc[1] & SDMA_DESC1_HEAD_TO_HOST_FLAG) ?
2197 'H' : '-';
2198 flags[2] = (desc[0] & SDMA_DESC0_FIRST_DESC_FLAG) ? 'F' : '-';
2199 flags[3] = (desc[0] & SDMA_DESC0_LAST_DESC_FLAG) ? 'L' : '-';
2200 addr = (desc[0] >> SDMA_DESC0_PHY_ADDR_SHIFT)
2201 & SDMA_DESC0_PHY_ADDR_MASK;
2202 gen = (desc[1] >> SDMA_DESC1_GENERATION_SHIFT)
2203 & SDMA_DESC1_GENERATION_MASK;
2204 len = (desc[0] >> SDMA_DESC0_BYTE_COUNT_SHIFT)
2205 & SDMA_DESC0_BYTE_COUNT_MASK;
2206 seq_printf(s,
2207 "\tdesc[%u]: flags:%s addr:0x%016llx gen:%u len:%u bytes\n",
2208 head, flags, addr, gen, len);
2209 if (desc[0] & SDMA_DESC0_FIRST_DESC_FLAG)
2210 seq_printf(s, "\t\tahgidx: %u ahgmode: %u\n",
2211 (u8)((desc[1] &
2212 SDMA_DESC1_HEADER_INDEX_SMASK) >>
2213 SDMA_DESC1_HEADER_INDEX_SHIFT),
2214 (u8)((desc[1] &
2215 SDMA_DESC1_HEADER_MODE_SMASK) >>
2216 SDMA_DESC1_HEADER_MODE_SHIFT));
2217 head = (head + 1) & sde->sdma_mask;
2218 }
2219 }
2220
2221 /*
2222 * add the generation number into
2223 * the qw1 and return
2224 */
add_gen(struct sdma_engine * sde,u64 qw1)2225 static inline u64 add_gen(struct sdma_engine *sde, u64 qw1)
2226 {
2227 u8 generation = (sde->descq_tail >> sde->sdma_shift) & 3;
2228
2229 qw1 &= ~SDMA_DESC1_GENERATION_SMASK;
2230 qw1 |= ((u64)generation & SDMA_DESC1_GENERATION_MASK)
2231 << SDMA_DESC1_GENERATION_SHIFT;
2232 return qw1;
2233 }
2234
2235 /*
2236 * This routine submits the indicated tx
2237 *
2238 * Space has already been guaranteed and
2239 * tail side of ring is locked.
2240 *
2241 * The hardware tail update is done
2242 * in the caller and that is facilitated
2243 * by returning the new tail.
2244 *
2245 * There is special case logic for ahg
2246 * to not add the generation number for
2247 * up to 2 descriptors that follow the
2248 * first descriptor.
2249 *
2250 */
submit_tx(struct sdma_engine * sde,struct sdma_txreq * tx)2251 static inline u16 submit_tx(struct sdma_engine *sde, struct sdma_txreq *tx)
2252 {
2253 int i;
2254 u16 tail;
2255 struct sdma_desc *descp = tx->descp;
2256 u8 skip = 0, mode = ahg_mode(tx);
2257
2258 tail = sde->descq_tail & sde->sdma_mask;
2259 sde->descq[tail].qw[0] = cpu_to_le64(descp->qw[0]);
2260 sde->descq[tail].qw[1] = cpu_to_le64(add_gen(sde, descp->qw[1]));
2261 trace_hfi1_sdma_descriptor(sde, descp->qw[0], descp->qw[1],
2262 tail, &sde->descq[tail]);
2263 tail = ++sde->descq_tail & sde->sdma_mask;
2264 descp++;
2265 if (mode > SDMA_AHG_APPLY_UPDATE1)
2266 skip = mode >> 1;
2267 for (i = 1; i < tx->num_desc; i++, descp++) {
2268 u64 qw1;
2269
2270 sde->descq[tail].qw[0] = cpu_to_le64(descp->qw[0]);
2271 if (skip) {
2272 /* edits don't have generation */
2273 qw1 = descp->qw[1];
2274 skip--;
2275 } else {
2276 /* replace generation with real one for non-edits */
2277 qw1 = add_gen(sde, descp->qw[1]);
2278 }
2279 sde->descq[tail].qw[1] = cpu_to_le64(qw1);
2280 trace_hfi1_sdma_descriptor(sde, descp->qw[0], qw1,
2281 tail, &sde->descq[tail]);
2282 tail = ++sde->descq_tail & sde->sdma_mask;
2283 }
2284 tx->next_descq_idx = tail;
2285 #ifdef CONFIG_HFI1_DEBUG_SDMA_ORDER
2286 tx->sn = sde->tail_sn++;
2287 trace_hfi1_sdma_in_sn(sde, tx->sn);
2288 WARN_ON_ONCE(sde->tx_ring[sde->tx_tail & sde->sdma_mask]);
2289 #endif
2290 sde->tx_ring[sde->tx_tail++ & sde->sdma_mask] = tx;
2291 sde->desc_avail -= tx->num_desc;
2292 return tail;
2293 }
2294
2295 /*
2296 * Check for progress
2297 */
sdma_check_progress(struct sdma_engine * sde,struct iowait_work * wait,struct sdma_txreq * tx,bool pkts_sent)2298 static int sdma_check_progress(
2299 struct sdma_engine *sde,
2300 struct iowait_work *wait,
2301 struct sdma_txreq *tx,
2302 bool pkts_sent)
2303 {
2304 int ret;
2305
2306 sde->desc_avail = sdma_descq_freecnt(sde);
2307 if (tx->num_desc <= sde->desc_avail)
2308 return -EAGAIN;
2309 /* pulse the head_lock */
2310 if (wait && iowait_ioww_to_iow(wait)->sleep) {
2311 unsigned seq;
2312
2313 seq = raw_seqcount_begin(
2314 (const seqcount_t *)&sde->head_lock.seqcount);
2315 ret = wait->iow->sleep(sde, wait, tx, seq, pkts_sent);
2316 if (ret == -EAGAIN)
2317 sde->desc_avail = sdma_descq_freecnt(sde);
2318 } else {
2319 ret = -EBUSY;
2320 }
2321 return ret;
2322 }
2323
2324 /**
2325 * sdma_send_txreq() - submit a tx req to ring
2326 * @sde: sdma engine to use
2327 * @wait: SE wait structure to use when full (may be NULL)
2328 * @tx: sdma_txreq to submit
2329 * @pkts_sent: has any packet been sent yet?
2330 *
2331 * The call submits the tx into the ring. If a iowait structure is non-NULL
2332 * the packet will be queued to the list in wait.
2333 *
2334 * Return:
2335 * 0 - Success, -EINVAL - sdma_txreq incomplete, -EBUSY - no space in
2336 * ring (wait == NULL)
2337 * -EIOCBQUEUED - tx queued to iowait, -ECOMM bad sdma state
2338 */
sdma_send_txreq(struct sdma_engine * sde,struct iowait_work * wait,struct sdma_txreq * tx,bool pkts_sent)2339 int sdma_send_txreq(struct sdma_engine *sde,
2340 struct iowait_work *wait,
2341 struct sdma_txreq *tx,
2342 bool pkts_sent)
2343 {
2344 int ret = 0;
2345 u16 tail;
2346 unsigned long flags;
2347
2348 /* user should have supplied entire packet */
2349 if (unlikely(tx->tlen))
2350 return -EINVAL;
2351 tx->wait = iowait_ioww_to_iow(wait);
2352 spin_lock_irqsave(&sde->tail_lock, flags);
2353 retry:
2354 if (unlikely(!__sdma_running(sde)))
2355 goto unlock_noconn;
2356 if (unlikely(tx->num_desc > sde->desc_avail))
2357 goto nodesc;
2358 tail = submit_tx(sde, tx);
2359 if (wait)
2360 iowait_sdma_inc(iowait_ioww_to_iow(wait));
2361 sdma_update_tail(sde, tail);
2362 unlock:
2363 spin_unlock_irqrestore(&sde->tail_lock, flags);
2364 return ret;
2365 unlock_noconn:
2366 if (wait)
2367 iowait_sdma_inc(iowait_ioww_to_iow(wait));
2368 tx->next_descq_idx = 0;
2369 #ifdef CONFIG_HFI1_DEBUG_SDMA_ORDER
2370 tx->sn = sde->tail_sn++;
2371 trace_hfi1_sdma_in_sn(sde, tx->sn);
2372 #endif
2373 spin_lock(&sde->flushlist_lock);
2374 list_add_tail(&tx->list, &sde->flushlist);
2375 spin_unlock(&sde->flushlist_lock);
2376 iowait_inc_wait_count(wait, tx->num_desc);
2377 queue_work_on(sde->cpu, system_highpri_wq, &sde->flush_worker);
2378 ret = -ECOMM;
2379 goto unlock;
2380 nodesc:
2381 ret = sdma_check_progress(sde, wait, tx, pkts_sent);
2382 if (ret == -EAGAIN) {
2383 ret = 0;
2384 goto retry;
2385 }
2386 sde->descq_full_count++;
2387 goto unlock;
2388 }
2389
2390 /**
2391 * sdma_send_txlist() - submit a list of tx req to ring
2392 * @sde: sdma engine to use
2393 * @wait: SE wait structure to use when full (may be NULL)
2394 * @tx_list: list of sdma_txreqs to submit
2395 * @count_out: pointer to a u16 which, after return will contain the total number of
2396 * sdma_txreqs removed from the tx_list. This will include sdma_txreqs
2397 * whose SDMA descriptors are submitted to the ring and the sdma_txreqs
2398 * which are added to SDMA engine flush list if the SDMA engine state is
2399 * not running.
2400 *
2401 * The call submits the list into the ring.
2402 *
2403 * If the iowait structure is non-NULL and not equal to the iowait list
2404 * the unprocessed part of the list will be appended to the list in wait.
2405 *
2406 * In all cases, the tx_list will be updated so the head of the tx_list is
2407 * the list of descriptors that have yet to be transmitted.
2408 *
2409 * The intent of this call is to provide a more efficient
2410 * way of submitting multiple packets to SDMA while holding the tail
2411 * side locking.
2412 *
2413 * Return:
2414 * 0 - Success,
2415 * -EINVAL - sdma_txreq incomplete, -EBUSY - no space in ring (wait == NULL)
2416 * -EIOCBQUEUED - tx queued to iowait, -ECOMM bad sdma state
2417 */
sdma_send_txlist(struct sdma_engine * sde,struct iowait_work * wait,struct list_head * tx_list,u16 * count_out)2418 int sdma_send_txlist(struct sdma_engine *sde, struct iowait_work *wait,
2419 struct list_head *tx_list, u16 *count_out)
2420 {
2421 struct sdma_txreq *tx, *tx_next;
2422 int ret = 0;
2423 unsigned long flags;
2424 u16 tail = INVALID_TAIL;
2425 u32 submit_count = 0, flush_count = 0, total_count;
2426
2427 spin_lock_irqsave(&sde->tail_lock, flags);
2428 retry:
2429 list_for_each_entry_safe(tx, tx_next, tx_list, list) {
2430 tx->wait = iowait_ioww_to_iow(wait);
2431 if (unlikely(!__sdma_running(sde)))
2432 goto unlock_noconn;
2433 if (unlikely(tx->num_desc > sde->desc_avail))
2434 goto nodesc;
2435 if (unlikely(tx->tlen)) {
2436 ret = -EINVAL;
2437 goto update_tail;
2438 }
2439 list_del_init(&tx->list);
2440 tail = submit_tx(sde, tx);
2441 submit_count++;
2442 if (tail != INVALID_TAIL &&
2443 (submit_count & SDMA_TAIL_UPDATE_THRESH) == 0) {
2444 sdma_update_tail(sde, tail);
2445 tail = INVALID_TAIL;
2446 }
2447 }
2448 update_tail:
2449 total_count = submit_count + flush_count;
2450 if (wait) {
2451 iowait_sdma_add(iowait_ioww_to_iow(wait), total_count);
2452 iowait_starve_clear(submit_count > 0,
2453 iowait_ioww_to_iow(wait));
2454 }
2455 if (tail != INVALID_TAIL)
2456 sdma_update_tail(sde, tail);
2457 spin_unlock_irqrestore(&sde->tail_lock, flags);
2458 *count_out = total_count;
2459 return ret;
2460 unlock_noconn:
2461 spin_lock(&sde->flushlist_lock);
2462 list_for_each_entry_safe(tx, tx_next, tx_list, list) {
2463 tx->wait = iowait_ioww_to_iow(wait);
2464 list_del_init(&tx->list);
2465 tx->next_descq_idx = 0;
2466 #ifdef CONFIG_HFI1_DEBUG_SDMA_ORDER
2467 tx->sn = sde->tail_sn++;
2468 trace_hfi1_sdma_in_sn(sde, tx->sn);
2469 #endif
2470 list_add_tail(&tx->list, &sde->flushlist);
2471 flush_count++;
2472 iowait_inc_wait_count(wait, tx->num_desc);
2473 }
2474 spin_unlock(&sde->flushlist_lock);
2475 queue_work_on(sde->cpu, system_highpri_wq, &sde->flush_worker);
2476 ret = -ECOMM;
2477 goto update_tail;
2478 nodesc:
2479 ret = sdma_check_progress(sde, wait, tx, submit_count > 0);
2480 if (ret == -EAGAIN) {
2481 ret = 0;
2482 goto retry;
2483 }
2484 sde->descq_full_count++;
2485 goto update_tail;
2486 }
2487
sdma_process_event(struct sdma_engine * sde,enum sdma_events event)2488 static void sdma_process_event(struct sdma_engine *sde, enum sdma_events event)
2489 {
2490 unsigned long flags;
2491
2492 spin_lock_irqsave(&sde->tail_lock, flags);
2493 write_seqlock(&sde->head_lock);
2494
2495 __sdma_process_event(sde, event);
2496
2497 if (sde->state.current_state == sdma_state_s99_running)
2498 sdma_desc_avail(sde, sdma_descq_freecnt(sde));
2499
2500 write_sequnlock(&sde->head_lock);
2501 spin_unlock_irqrestore(&sde->tail_lock, flags);
2502 }
2503
__sdma_process_event(struct sdma_engine * sde,enum sdma_events event)2504 static void __sdma_process_event(struct sdma_engine *sde,
2505 enum sdma_events event)
2506 {
2507 struct sdma_state *ss = &sde->state;
2508 int need_progress = 0;
2509
2510 /* CONFIG SDMA temporary */
2511 #ifdef CONFIG_SDMA_VERBOSITY
2512 dd_dev_err(sde->dd, "CONFIG SDMA(%u) [%s] %s\n", sde->this_idx,
2513 sdma_state_names[ss->current_state],
2514 sdma_event_names[event]);
2515 #endif
2516
2517 switch (ss->current_state) {
2518 case sdma_state_s00_hw_down:
2519 switch (event) {
2520 case sdma_event_e00_go_hw_down:
2521 break;
2522 case sdma_event_e30_go_running:
2523 /*
2524 * If down, but running requested (usually result
2525 * of link up, then we need to start up.
2526 * This can happen when hw down is requested while
2527 * bringing the link up with traffic active on
2528 * 7220, e.g.
2529 */
2530 ss->go_s99_running = 1;
2531 fallthrough; /* and start dma engine */
2532 case sdma_event_e10_go_hw_start:
2533 /* This reference means the state machine is started */
2534 sdma_get(&sde->state);
2535 sdma_set_state(sde,
2536 sdma_state_s10_hw_start_up_halt_wait);
2537 break;
2538 case sdma_event_e15_hw_halt_done:
2539 break;
2540 case sdma_event_e25_hw_clean_up_done:
2541 break;
2542 case sdma_event_e40_sw_cleaned:
2543 sdma_sw_tear_down(sde);
2544 break;
2545 case sdma_event_e50_hw_cleaned:
2546 break;
2547 case sdma_event_e60_hw_halted:
2548 break;
2549 case sdma_event_e70_go_idle:
2550 break;
2551 case sdma_event_e80_hw_freeze:
2552 break;
2553 case sdma_event_e81_hw_frozen:
2554 break;
2555 case sdma_event_e82_hw_unfreeze:
2556 break;
2557 case sdma_event_e85_link_down:
2558 break;
2559 case sdma_event_e90_sw_halted:
2560 break;
2561 }
2562 break;
2563
2564 case sdma_state_s10_hw_start_up_halt_wait:
2565 switch (event) {
2566 case sdma_event_e00_go_hw_down:
2567 sdma_set_state(sde, sdma_state_s00_hw_down);
2568 sdma_sw_tear_down(sde);
2569 break;
2570 case sdma_event_e10_go_hw_start:
2571 break;
2572 case sdma_event_e15_hw_halt_done:
2573 sdma_set_state(sde,
2574 sdma_state_s15_hw_start_up_clean_wait);
2575 sdma_start_hw_clean_up(sde);
2576 break;
2577 case sdma_event_e25_hw_clean_up_done:
2578 break;
2579 case sdma_event_e30_go_running:
2580 ss->go_s99_running = 1;
2581 break;
2582 case sdma_event_e40_sw_cleaned:
2583 break;
2584 case sdma_event_e50_hw_cleaned:
2585 break;
2586 case sdma_event_e60_hw_halted:
2587 schedule_work(&sde->err_halt_worker);
2588 break;
2589 case sdma_event_e70_go_idle:
2590 ss->go_s99_running = 0;
2591 break;
2592 case sdma_event_e80_hw_freeze:
2593 break;
2594 case sdma_event_e81_hw_frozen:
2595 break;
2596 case sdma_event_e82_hw_unfreeze:
2597 break;
2598 case sdma_event_e85_link_down:
2599 break;
2600 case sdma_event_e90_sw_halted:
2601 break;
2602 }
2603 break;
2604
2605 case sdma_state_s15_hw_start_up_clean_wait:
2606 switch (event) {
2607 case sdma_event_e00_go_hw_down:
2608 sdma_set_state(sde, sdma_state_s00_hw_down);
2609 sdma_sw_tear_down(sde);
2610 break;
2611 case sdma_event_e10_go_hw_start:
2612 break;
2613 case sdma_event_e15_hw_halt_done:
2614 break;
2615 case sdma_event_e25_hw_clean_up_done:
2616 sdma_hw_start_up(sde);
2617 sdma_set_state(sde, ss->go_s99_running ?
2618 sdma_state_s99_running :
2619 sdma_state_s20_idle);
2620 break;
2621 case sdma_event_e30_go_running:
2622 ss->go_s99_running = 1;
2623 break;
2624 case sdma_event_e40_sw_cleaned:
2625 break;
2626 case sdma_event_e50_hw_cleaned:
2627 break;
2628 case sdma_event_e60_hw_halted:
2629 break;
2630 case sdma_event_e70_go_idle:
2631 ss->go_s99_running = 0;
2632 break;
2633 case sdma_event_e80_hw_freeze:
2634 break;
2635 case sdma_event_e81_hw_frozen:
2636 break;
2637 case sdma_event_e82_hw_unfreeze:
2638 break;
2639 case sdma_event_e85_link_down:
2640 break;
2641 case sdma_event_e90_sw_halted:
2642 break;
2643 }
2644 break;
2645
2646 case sdma_state_s20_idle:
2647 switch (event) {
2648 case sdma_event_e00_go_hw_down:
2649 sdma_set_state(sde, sdma_state_s00_hw_down);
2650 sdma_sw_tear_down(sde);
2651 break;
2652 case sdma_event_e10_go_hw_start:
2653 break;
2654 case sdma_event_e15_hw_halt_done:
2655 break;
2656 case sdma_event_e25_hw_clean_up_done:
2657 break;
2658 case sdma_event_e30_go_running:
2659 sdma_set_state(sde, sdma_state_s99_running);
2660 ss->go_s99_running = 1;
2661 break;
2662 case sdma_event_e40_sw_cleaned:
2663 break;
2664 case sdma_event_e50_hw_cleaned:
2665 break;
2666 case sdma_event_e60_hw_halted:
2667 sdma_set_state(sde, sdma_state_s50_hw_halt_wait);
2668 schedule_work(&sde->err_halt_worker);
2669 break;
2670 case sdma_event_e70_go_idle:
2671 break;
2672 case sdma_event_e85_link_down:
2673 case sdma_event_e80_hw_freeze:
2674 sdma_set_state(sde, sdma_state_s80_hw_freeze);
2675 atomic_dec(&sde->dd->sdma_unfreeze_count);
2676 wake_up_interruptible(&sde->dd->sdma_unfreeze_wq);
2677 break;
2678 case sdma_event_e81_hw_frozen:
2679 break;
2680 case sdma_event_e82_hw_unfreeze:
2681 break;
2682 case sdma_event_e90_sw_halted:
2683 break;
2684 }
2685 break;
2686
2687 case sdma_state_s30_sw_clean_up_wait:
2688 switch (event) {
2689 case sdma_event_e00_go_hw_down:
2690 sdma_set_state(sde, sdma_state_s00_hw_down);
2691 break;
2692 case sdma_event_e10_go_hw_start:
2693 break;
2694 case sdma_event_e15_hw_halt_done:
2695 break;
2696 case sdma_event_e25_hw_clean_up_done:
2697 break;
2698 case sdma_event_e30_go_running:
2699 ss->go_s99_running = 1;
2700 break;
2701 case sdma_event_e40_sw_cleaned:
2702 sdma_set_state(sde, sdma_state_s40_hw_clean_up_wait);
2703 sdma_start_hw_clean_up(sde);
2704 break;
2705 case sdma_event_e50_hw_cleaned:
2706 break;
2707 case sdma_event_e60_hw_halted:
2708 break;
2709 case sdma_event_e70_go_idle:
2710 ss->go_s99_running = 0;
2711 break;
2712 case sdma_event_e80_hw_freeze:
2713 break;
2714 case sdma_event_e81_hw_frozen:
2715 break;
2716 case sdma_event_e82_hw_unfreeze:
2717 break;
2718 case sdma_event_e85_link_down:
2719 ss->go_s99_running = 0;
2720 break;
2721 case sdma_event_e90_sw_halted:
2722 break;
2723 }
2724 break;
2725
2726 case sdma_state_s40_hw_clean_up_wait:
2727 switch (event) {
2728 case sdma_event_e00_go_hw_down:
2729 sdma_set_state(sde, sdma_state_s00_hw_down);
2730 tasklet_hi_schedule(&sde->sdma_sw_clean_up_task);
2731 break;
2732 case sdma_event_e10_go_hw_start:
2733 break;
2734 case sdma_event_e15_hw_halt_done:
2735 break;
2736 case sdma_event_e25_hw_clean_up_done:
2737 sdma_hw_start_up(sde);
2738 sdma_set_state(sde, ss->go_s99_running ?
2739 sdma_state_s99_running :
2740 sdma_state_s20_idle);
2741 break;
2742 case sdma_event_e30_go_running:
2743 ss->go_s99_running = 1;
2744 break;
2745 case sdma_event_e40_sw_cleaned:
2746 break;
2747 case sdma_event_e50_hw_cleaned:
2748 break;
2749 case sdma_event_e60_hw_halted:
2750 break;
2751 case sdma_event_e70_go_idle:
2752 ss->go_s99_running = 0;
2753 break;
2754 case sdma_event_e80_hw_freeze:
2755 break;
2756 case sdma_event_e81_hw_frozen:
2757 break;
2758 case sdma_event_e82_hw_unfreeze:
2759 break;
2760 case sdma_event_e85_link_down:
2761 ss->go_s99_running = 0;
2762 break;
2763 case sdma_event_e90_sw_halted:
2764 break;
2765 }
2766 break;
2767
2768 case sdma_state_s50_hw_halt_wait:
2769 switch (event) {
2770 case sdma_event_e00_go_hw_down:
2771 sdma_set_state(sde, sdma_state_s00_hw_down);
2772 tasklet_hi_schedule(&sde->sdma_sw_clean_up_task);
2773 break;
2774 case sdma_event_e10_go_hw_start:
2775 break;
2776 case sdma_event_e15_hw_halt_done:
2777 sdma_set_state(sde, sdma_state_s30_sw_clean_up_wait);
2778 tasklet_hi_schedule(&sde->sdma_sw_clean_up_task);
2779 break;
2780 case sdma_event_e25_hw_clean_up_done:
2781 break;
2782 case sdma_event_e30_go_running:
2783 ss->go_s99_running = 1;
2784 break;
2785 case sdma_event_e40_sw_cleaned:
2786 break;
2787 case sdma_event_e50_hw_cleaned:
2788 break;
2789 case sdma_event_e60_hw_halted:
2790 schedule_work(&sde->err_halt_worker);
2791 break;
2792 case sdma_event_e70_go_idle:
2793 ss->go_s99_running = 0;
2794 break;
2795 case sdma_event_e80_hw_freeze:
2796 break;
2797 case sdma_event_e81_hw_frozen:
2798 break;
2799 case sdma_event_e82_hw_unfreeze:
2800 break;
2801 case sdma_event_e85_link_down:
2802 ss->go_s99_running = 0;
2803 break;
2804 case sdma_event_e90_sw_halted:
2805 break;
2806 }
2807 break;
2808
2809 case sdma_state_s60_idle_halt_wait:
2810 switch (event) {
2811 case sdma_event_e00_go_hw_down:
2812 sdma_set_state(sde, sdma_state_s00_hw_down);
2813 tasklet_hi_schedule(&sde->sdma_sw_clean_up_task);
2814 break;
2815 case sdma_event_e10_go_hw_start:
2816 break;
2817 case sdma_event_e15_hw_halt_done:
2818 sdma_set_state(sde, sdma_state_s30_sw_clean_up_wait);
2819 tasklet_hi_schedule(&sde->sdma_sw_clean_up_task);
2820 break;
2821 case sdma_event_e25_hw_clean_up_done:
2822 break;
2823 case sdma_event_e30_go_running:
2824 ss->go_s99_running = 1;
2825 break;
2826 case sdma_event_e40_sw_cleaned:
2827 break;
2828 case sdma_event_e50_hw_cleaned:
2829 break;
2830 case sdma_event_e60_hw_halted:
2831 schedule_work(&sde->err_halt_worker);
2832 break;
2833 case sdma_event_e70_go_idle:
2834 ss->go_s99_running = 0;
2835 break;
2836 case sdma_event_e80_hw_freeze:
2837 break;
2838 case sdma_event_e81_hw_frozen:
2839 break;
2840 case sdma_event_e82_hw_unfreeze:
2841 break;
2842 case sdma_event_e85_link_down:
2843 break;
2844 case sdma_event_e90_sw_halted:
2845 break;
2846 }
2847 break;
2848
2849 case sdma_state_s80_hw_freeze:
2850 switch (event) {
2851 case sdma_event_e00_go_hw_down:
2852 sdma_set_state(sde, sdma_state_s00_hw_down);
2853 tasklet_hi_schedule(&sde->sdma_sw_clean_up_task);
2854 break;
2855 case sdma_event_e10_go_hw_start:
2856 break;
2857 case sdma_event_e15_hw_halt_done:
2858 break;
2859 case sdma_event_e25_hw_clean_up_done:
2860 break;
2861 case sdma_event_e30_go_running:
2862 ss->go_s99_running = 1;
2863 break;
2864 case sdma_event_e40_sw_cleaned:
2865 break;
2866 case sdma_event_e50_hw_cleaned:
2867 break;
2868 case sdma_event_e60_hw_halted:
2869 break;
2870 case sdma_event_e70_go_idle:
2871 ss->go_s99_running = 0;
2872 break;
2873 case sdma_event_e80_hw_freeze:
2874 break;
2875 case sdma_event_e81_hw_frozen:
2876 sdma_set_state(sde, sdma_state_s82_freeze_sw_clean);
2877 tasklet_hi_schedule(&sde->sdma_sw_clean_up_task);
2878 break;
2879 case sdma_event_e82_hw_unfreeze:
2880 break;
2881 case sdma_event_e85_link_down:
2882 break;
2883 case sdma_event_e90_sw_halted:
2884 break;
2885 }
2886 break;
2887
2888 case sdma_state_s82_freeze_sw_clean:
2889 switch (event) {
2890 case sdma_event_e00_go_hw_down:
2891 sdma_set_state(sde, sdma_state_s00_hw_down);
2892 tasklet_hi_schedule(&sde->sdma_sw_clean_up_task);
2893 break;
2894 case sdma_event_e10_go_hw_start:
2895 break;
2896 case sdma_event_e15_hw_halt_done:
2897 break;
2898 case sdma_event_e25_hw_clean_up_done:
2899 break;
2900 case sdma_event_e30_go_running:
2901 ss->go_s99_running = 1;
2902 break;
2903 case sdma_event_e40_sw_cleaned:
2904 /* notify caller this engine is done cleaning */
2905 atomic_dec(&sde->dd->sdma_unfreeze_count);
2906 wake_up_interruptible(&sde->dd->sdma_unfreeze_wq);
2907 break;
2908 case sdma_event_e50_hw_cleaned:
2909 break;
2910 case sdma_event_e60_hw_halted:
2911 break;
2912 case sdma_event_e70_go_idle:
2913 ss->go_s99_running = 0;
2914 break;
2915 case sdma_event_e80_hw_freeze:
2916 break;
2917 case sdma_event_e81_hw_frozen:
2918 break;
2919 case sdma_event_e82_hw_unfreeze:
2920 sdma_hw_start_up(sde);
2921 sdma_set_state(sde, ss->go_s99_running ?
2922 sdma_state_s99_running :
2923 sdma_state_s20_idle);
2924 break;
2925 case sdma_event_e85_link_down:
2926 break;
2927 case sdma_event_e90_sw_halted:
2928 break;
2929 }
2930 break;
2931
2932 case sdma_state_s99_running:
2933 switch (event) {
2934 case sdma_event_e00_go_hw_down:
2935 sdma_set_state(sde, sdma_state_s00_hw_down);
2936 tasklet_hi_schedule(&sde->sdma_sw_clean_up_task);
2937 break;
2938 case sdma_event_e10_go_hw_start:
2939 break;
2940 case sdma_event_e15_hw_halt_done:
2941 break;
2942 case sdma_event_e25_hw_clean_up_done:
2943 break;
2944 case sdma_event_e30_go_running:
2945 break;
2946 case sdma_event_e40_sw_cleaned:
2947 break;
2948 case sdma_event_e50_hw_cleaned:
2949 break;
2950 case sdma_event_e60_hw_halted:
2951 need_progress = 1;
2952 sdma_err_progress_check_schedule(sde);
2953 fallthrough;
2954 case sdma_event_e90_sw_halted:
2955 /*
2956 * SW initiated halt does not perform engines
2957 * progress check
2958 */
2959 sdma_set_state(sde, sdma_state_s50_hw_halt_wait);
2960 schedule_work(&sde->err_halt_worker);
2961 break;
2962 case sdma_event_e70_go_idle:
2963 sdma_set_state(sde, sdma_state_s60_idle_halt_wait);
2964 break;
2965 case sdma_event_e85_link_down:
2966 ss->go_s99_running = 0;
2967 fallthrough;
2968 case sdma_event_e80_hw_freeze:
2969 sdma_set_state(sde, sdma_state_s80_hw_freeze);
2970 atomic_dec(&sde->dd->sdma_unfreeze_count);
2971 wake_up_interruptible(&sde->dd->sdma_unfreeze_wq);
2972 break;
2973 case sdma_event_e81_hw_frozen:
2974 break;
2975 case sdma_event_e82_hw_unfreeze:
2976 break;
2977 }
2978 break;
2979 }
2980
2981 ss->last_event = event;
2982 if (need_progress)
2983 sdma_make_progress(sde, 0);
2984 }
2985
2986 /*
2987 * _extend_sdma_tx_descs() - helper to extend txreq
2988 *
2989 * This is called once the initial nominal allocation
2990 * of descriptors in the sdma_txreq is exhausted.
2991 *
2992 * The code will bump the allocation up to the max
2993 * of MAX_DESC (64) descriptors. There doesn't seem
2994 * much point in an interim step. The last descriptor
2995 * is reserved for coalesce buffer in order to support
2996 * cases where input packet has >MAX_DESC iovecs.
2997 *
2998 */
_extend_sdma_tx_descs(struct hfi1_devdata * dd,struct sdma_txreq * tx)2999 static int _extend_sdma_tx_descs(struct hfi1_devdata *dd, struct sdma_txreq *tx)
3000 {
3001 int i;
3002 struct sdma_desc *descp;
3003
3004 /* Handle last descriptor */
3005 if (unlikely((tx->num_desc == (MAX_DESC - 1)))) {
3006 /* if tlen is 0, it is for padding, release last descriptor */
3007 if (!tx->tlen) {
3008 tx->desc_limit = MAX_DESC;
3009 } else if (!tx->coalesce_buf) {
3010 /* allocate coalesce buffer with space for padding */
3011 tx->coalesce_buf = kmalloc(tx->tlen + sizeof(u32),
3012 GFP_ATOMIC);
3013 if (!tx->coalesce_buf)
3014 goto enomem;
3015 tx->coalesce_idx = 0;
3016 }
3017 return 0;
3018 }
3019
3020 if (unlikely(tx->num_desc == MAX_DESC))
3021 goto enomem;
3022
3023 descp = kmalloc_objs(struct sdma_desc, MAX_DESC, GFP_ATOMIC);
3024 if (!descp)
3025 goto enomem;
3026 tx->descp = descp;
3027
3028 /* reserve last descriptor for coalescing */
3029 tx->desc_limit = MAX_DESC - 1;
3030 /* copy ones already built */
3031 for (i = 0; i < tx->num_desc; i++)
3032 tx->descp[i] = tx->descs[i];
3033 return 0;
3034 enomem:
3035 __sdma_txclean(dd, tx);
3036 return -ENOMEM;
3037 }
3038
3039 /*
3040 * ext_coal_sdma_tx_descs() - extend or coalesce sdma tx descriptors
3041 *
3042 * This is called once the initial nominal allocation of descriptors
3043 * in the sdma_txreq is exhausted.
3044 *
3045 * This function calls _extend_sdma_tx_descs to extend or allocate
3046 * coalesce buffer. If there is a allocated coalesce buffer, it will
3047 * copy the input packet data into the coalesce buffer. It also adds
3048 * coalesce buffer descriptor once when whole packet is received.
3049 *
3050 * Return:
3051 * <0 - error
3052 * 0 - coalescing, don't populate descriptor
3053 * 1 - continue with populating descriptor
3054 */
ext_coal_sdma_tx_descs(struct hfi1_devdata * dd,struct sdma_txreq * tx,int type,void * kvaddr,struct page * page,unsigned long offset,u16 len)3055 int ext_coal_sdma_tx_descs(struct hfi1_devdata *dd, struct sdma_txreq *tx,
3056 int type, void *kvaddr, struct page *page,
3057 unsigned long offset, u16 len)
3058 {
3059 int pad_len, rval;
3060 dma_addr_t addr;
3061
3062 rval = _extend_sdma_tx_descs(dd, tx);
3063 if (rval) {
3064 __sdma_txclean(dd, tx);
3065 return rval;
3066 }
3067
3068 /* If coalesce buffer is allocated, copy data into it */
3069 if (tx->coalesce_buf) {
3070 if (type == SDMA_MAP_NONE) {
3071 __sdma_txclean(dd, tx);
3072 return -EINVAL;
3073 }
3074
3075 if (type == SDMA_MAP_PAGE) {
3076 kvaddr = kmap_local_page(page);
3077 kvaddr += offset;
3078 } else if (WARN_ON(!kvaddr)) {
3079 __sdma_txclean(dd, tx);
3080 return -EINVAL;
3081 }
3082
3083 memcpy(tx->coalesce_buf + tx->coalesce_idx, kvaddr, len);
3084 tx->coalesce_idx += len;
3085 if (type == SDMA_MAP_PAGE)
3086 kunmap_local(kvaddr);
3087
3088 /* If there is more data, return */
3089 if (tx->tlen - tx->coalesce_idx)
3090 return 0;
3091
3092 /* Whole packet is received; add any padding */
3093 pad_len = tx->packet_len & (sizeof(u32) - 1);
3094 if (pad_len) {
3095 pad_len = sizeof(u32) - pad_len;
3096 memset(tx->coalesce_buf + tx->coalesce_idx, 0, pad_len);
3097 /* padding is taken care of for coalescing case */
3098 tx->packet_len += pad_len;
3099 tx->tlen += pad_len;
3100 }
3101
3102 /* dma map the coalesce buffer */
3103 addr = dma_map_single(&dd->pcidev->dev,
3104 tx->coalesce_buf,
3105 tx->tlen,
3106 DMA_TO_DEVICE);
3107
3108 if (unlikely(dma_mapping_error(&dd->pcidev->dev, addr))) {
3109 __sdma_txclean(dd, tx);
3110 return -ENOSPC;
3111 }
3112
3113 /* Add descriptor for coalesce buffer */
3114 tx->desc_limit = MAX_DESC;
3115 return _sdma_txadd_daddr(dd, SDMA_MAP_SINGLE, tx,
3116 addr, tx->tlen, NULL, NULL, NULL);
3117 }
3118
3119 return 1;
3120 }
3121
3122 /* Update sdes when the lmc changes */
sdma_update_lmc(struct hfi1_devdata * dd,u64 mask,u32 lid)3123 void sdma_update_lmc(struct hfi1_devdata *dd, u64 mask, u32 lid)
3124 {
3125 struct sdma_engine *sde;
3126 int i;
3127 u64 sreg;
3128
3129 sreg = ((mask & SD(CHECK_SLID_MASK_MASK)) <<
3130 SD(CHECK_SLID_MASK_SHIFT)) |
3131 (((lid & mask) & SD(CHECK_SLID_VALUE_MASK)) <<
3132 SD(CHECK_SLID_VALUE_SHIFT));
3133
3134 for (i = 0; i < dd->num_sdma; i++) {
3135 hfi1_cdbg(LINKVERB, "SendDmaEngine[%d].SLID_CHECK = 0x%x",
3136 i, (u32)sreg);
3137 sde = &dd->per_sdma[i];
3138 write_sde_csr(sde, SD(CHECK_SLID), sreg);
3139 }
3140 }
3141
3142 /* tx not dword sized - pad */
_pad_sdma_tx_descs(struct hfi1_devdata * dd,struct sdma_txreq * tx)3143 int _pad_sdma_tx_descs(struct hfi1_devdata *dd, struct sdma_txreq *tx)
3144 {
3145 int rval = 0;
3146
3147 if ((unlikely(tx->num_desc == tx->desc_limit))) {
3148 rval = _extend_sdma_tx_descs(dd, tx);
3149 if (rval) {
3150 __sdma_txclean(dd, tx);
3151 return rval;
3152 }
3153 }
3154
3155 /* finish the one just added */
3156 make_tx_sdma_desc(
3157 tx,
3158 SDMA_MAP_NONE,
3159 dd->sdma_pad_phys,
3160 sizeof(u32) - (tx->packet_len & (sizeof(u32) - 1)),
3161 NULL, NULL, NULL);
3162 tx->num_desc++;
3163 _sdma_close_tx(dd, tx);
3164 return rval;
3165 }
3166
3167 /*
3168 * Add ahg to the sdma_txreq
3169 *
3170 * The logic will consume up to 3
3171 * descriptors at the beginning of
3172 * sdma_txreq.
3173 */
_sdma_txreq_ahgadd(struct sdma_txreq * tx,u8 num_ahg,u8 ahg_entry,u32 * ahg,u8 ahg_hlen)3174 void _sdma_txreq_ahgadd(
3175 struct sdma_txreq *tx,
3176 u8 num_ahg,
3177 u8 ahg_entry,
3178 u32 *ahg,
3179 u8 ahg_hlen)
3180 {
3181 u32 i, shift = 0, desc = 0;
3182 u8 mode;
3183
3184 WARN_ON_ONCE(num_ahg > 9 || (ahg_hlen & 3) || ahg_hlen == 4);
3185 /* compute mode */
3186 if (num_ahg == 1)
3187 mode = SDMA_AHG_APPLY_UPDATE1;
3188 else if (num_ahg <= 5)
3189 mode = SDMA_AHG_APPLY_UPDATE2;
3190 else
3191 mode = SDMA_AHG_APPLY_UPDATE3;
3192 tx->num_desc++;
3193 /* initialize to consumed descriptors to zero */
3194 switch (mode) {
3195 case SDMA_AHG_APPLY_UPDATE3:
3196 tx->num_desc++;
3197 tx->descs[2].qw[0] = 0;
3198 tx->descs[2].qw[1] = 0;
3199 fallthrough;
3200 case SDMA_AHG_APPLY_UPDATE2:
3201 tx->num_desc++;
3202 tx->descs[1].qw[0] = 0;
3203 tx->descs[1].qw[1] = 0;
3204 break;
3205 }
3206 ahg_hlen >>= 2;
3207 tx->descs[0].qw[1] |=
3208 (((u64)ahg_entry & SDMA_DESC1_HEADER_INDEX_MASK)
3209 << SDMA_DESC1_HEADER_INDEX_SHIFT) |
3210 (((u64)ahg_hlen & SDMA_DESC1_HEADER_DWS_MASK)
3211 << SDMA_DESC1_HEADER_DWS_SHIFT) |
3212 (((u64)mode & SDMA_DESC1_HEADER_MODE_MASK)
3213 << SDMA_DESC1_HEADER_MODE_SHIFT) |
3214 (((u64)ahg[0] & SDMA_DESC1_HEADER_UPDATE1_MASK)
3215 << SDMA_DESC1_HEADER_UPDATE1_SHIFT);
3216 for (i = 0; i < (num_ahg - 1); i++) {
3217 if (!shift && !(i & 2))
3218 desc++;
3219 tx->descs[desc].qw[!!(i & 2)] |=
3220 (((u64)ahg[i + 1])
3221 << shift);
3222 shift = (shift + 32) & 63;
3223 }
3224 }
3225
3226 /**
3227 * sdma_ahg_alloc - allocate an AHG entry
3228 * @sde: engine to allocate from
3229 *
3230 * Return:
3231 * 0-31 when successful, -EOPNOTSUPP if AHG is not enabled,
3232 * -ENOSPC if an entry is not available
3233 */
sdma_ahg_alloc(struct sdma_engine * sde)3234 int sdma_ahg_alloc(struct sdma_engine *sde)
3235 {
3236 int nr;
3237 int oldbit;
3238
3239 if (!sde) {
3240 trace_hfi1_ahg_allocate(sde, -EINVAL);
3241 return -EINVAL;
3242 }
3243 while (1) {
3244 nr = ffz(READ_ONCE(sde->ahg_bits));
3245 if (nr > 31) {
3246 trace_hfi1_ahg_allocate(sde, -ENOSPC);
3247 return -ENOSPC;
3248 }
3249 oldbit = test_and_set_bit(nr, &sde->ahg_bits);
3250 if (!oldbit)
3251 break;
3252 cpu_relax();
3253 }
3254 trace_hfi1_ahg_allocate(sde, nr);
3255 return nr;
3256 }
3257
3258 /**
3259 * sdma_ahg_free - free an AHG entry
3260 * @sde: engine to return AHG entry
3261 * @ahg_index: index to free
3262 *
3263 * This routine frees the indicate AHG entry.
3264 */
sdma_ahg_free(struct sdma_engine * sde,int ahg_index)3265 void sdma_ahg_free(struct sdma_engine *sde, int ahg_index)
3266 {
3267 if (!sde)
3268 return;
3269 trace_hfi1_ahg_deallocate(sde, ahg_index);
3270 if (ahg_index < 0 || ahg_index > 31)
3271 return;
3272 clear_bit(ahg_index, &sde->ahg_bits);
3273 }
3274
3275 /*
3276 * SPC freeze handling for SDMA engines. Called when the driver knows
3277 * the SPC is going into a freeze but before the freeze is fully
3278 * settled. Generally an error interrupt.
3279 *
3280 * This event will pull the engine out of running so no more entries can be
3281 * added to the engine's queue.
3282 */
sdma_freeze_notify(struct hfi1_devdata * dd,int link_down)3283 void sdma_freeze_notify(struct hfi1_devdata *dd, int link_down)
3284 {
3285 int i;
3286 enum sdma_events event = link_down ? sdma_event_e85_link_down :
3287 sdma_event_e80_hw_freeze;
3288
3289 /* set up the wait but do not wait here */
3290 atomic_set(&dd->sdma_unfreeze_count, dd->num_sdma);
3291
3292 /* tell all engines to stop running and wait */
3293 for (i = 0; i < dd->num_sdma; i++)
3294 sdma_process_event(&dd->per_sdma[i], event);
3295
3296 /* sdma_freeze() will wait for all engines to have stopped */
3297 }
3298
3299 /*
3300 * SPC freeze handling for SDMA engines. Called when the driver knows
3301 * the SPC is fully frozen.
3302 */
sdma_freeze(struct hfi1_devdata * dd)3303 void sdma_freeze(struct hfi1_devdata *dd)
3304 {
3305 int i;
3306 int ret;
3307
3308 /*
3309 * Make sure all engines have moved out of the running state before
3310 * continuing.
3311 */
3312 ret = wait_event_interruptible(dd->sdma_unfreeze_wq,
3313 atomic_read(&dd->sdma_unfreeze_count) <=
3314 0);
3315 /* interrupted or count is negative, then unloading - just exit */
3316 if (ret || atomic_read(&dd->sdma_unfreeze_count) < 0)
3317 return;
3318
3319 /* set up the count for the next wait */
3320 atomic_set(&dd->sdma_unfreeze_count, dd->num_sdma);
3321
3322 /* tell all engines that the SPC is frozen, they can start cleaning */
3323 for (i = 0; i < dd->num_sdma; i++)
3324 sdma_process_event(&dd->per_sdma[i], sdma_event_e81_hw_frozen);
3325
3326 /*
3327 * Wait for everyone to finish software clean before exiting. The
3328 * software clean will read engine CSRs, so must be completed before
3329 * the next step, which will clear the engine CSRs.
3330 */
3331 (void)wait_event_interruptible(dd->sdma_unfreeze_wq,
3332 atomic_read(&dd->sdma_unfreeze_count) <= 0);
3333 /* no need to check results - done no matter what */
3334 }
3335
3336 /*
3337 * SPC freeze handling for the SDMA engines. Called after the SPC is unfrozen.
3338 *
3339 * The SPC freeze acts like a SDMA halt and a hardware clean combined. All
3340 * that is left is a software clean. We could do it after the SPC is fully
3341 * frozen, but then we'd have to add another state to wait for the unfreeze.
3342 * Instead, just defer the software clean until the unfreeze step.
3343 */
sdma_unfreeze(struct hfi1_devdata * dd)3344 void sdma_unfreeze(struct hfi1_devdata *dd)
3345 {
3346 int i;
3347
3348 /* tell all engines start freeze clean up */
3349 for (i = 0; i < dd->num_sdma; i++)
3350 sdma_process_event(&dd->per_sdma[i],
3351 sdma_event_e82_hw_unfreeze);
3352 }
3353
3354 /**
3355 * _sdma_engine_progress_schedule() - schedule progress on engine
3356 * @sde: sdma_engine to schedule progress
3357 *
3358 */
_sdma_engine_progress_schedule(struct sdma_engine * sde)3359 void _sdma_engine_progress_schedule(
3360 struct sdma_engine *sde)
3361 {
3362 trace_hfi1_sdma_engine_progress(sde, sde->progress_mask);
3363 /* assume we have selected a good cpu */
3364 write_csr(sde->dd,
3365 CCE_INT_FORCE + (8 * (IS_SDMA_START / 64)),
3366 sde->progress_mask);
3367 }
3368