1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause 2 /* 3 * Copyright(c) 2015 - 2018 Intel Corporation. 4 */ 5 6 #include <linux/spinlock.h> 7 #include <linux/seqlock.h> 8 #include <linux/netdevice.h> 9 #include <linux/moduleparam.h> 10 #include <linux/bitops.h> 11 #include <linux/timer.h> 12 #include <linux/vmalloc.h> 13 #include <linux/highmem.h> 14 15 #include "hfi.h" 16 #include "common.h" 17 #include "qp.h" 18 #include "sdma.h" 19 #include "iowait.h" 20 #include "trace.h" 21 22 /* must be a power of 2 >= 64 <= 32768 */ 23 #define SDMA_DESCQ_CNT 2048 24 #define SDMA_DESC_INTR 64 25 #define INVALID_TAIL 0xffff 26 #define SDMA_PAD max_t(size_t, MAX_16B_PADDING, sizeof(u32)) 27 28 static uint sdma_descq_cnt = SDMA_DESCQ_CNT; 29 module_param(sdma_descq_cnt, uint, S_IRUGO); 30 MODULE_PARM_DESC(sdma_descq_cnt, "Number of SDMA descq entries"); 31 32 static uint sdma_idle_cnt = 250; 33 module_param(sdma_idle_cnt, uint, S_IRUGO); 34 MODULE_PARM_DESC(sdma_idle_cnt, "sdma interrupt idle delay (ns,default 250)"); 35 36 uint mod_num_sdma; 37 module_param_named(num_sdma, mod_num_sdma, uint, S_IRUGO); 38 MODULE_PARM_DESC(num_sdma, "Set max number SDMA engines to use"); 39 40 static uint sdma_desct_intr = SDMA_DESC_INTR; 41 module_param_named(desct_intr, sdma_desct_intr, uint, S_IRUGO | S_IWUSR); 42 MODULE_PARM_DESC(desct_intr, "Number of SDMA descriptor before interrupt"); 43 44 #define SDMA_WAIT_BATCH_SIZE 20 45 /* max wait time for a SDMA engine to indicate it has halted */ 46 #define SDMA_ERR_HALT_TIMEOUT 10 /* ms */ 47 /* all SDMA engine errors that cause a halt */ 48 49 #define SD(name) SEND_DMA_##name 50 #define ALL_SDMA_ENG_HALT_ERRS \ 51 (SD(ENG_ERR_STATUS_SDMA_WRONG_DW_ERR_SMASK) \ 52 | SD(ENG_ERR_STATUS_SDMA_GEN_MISMATCH_ERR_SMASK) \ 53 | SD(ENG_ERR_STATUS_SDMA_TOO_LONG_ERR_SMASK) \ 54 | SD(ENG_ERR_STATUS_SDMA_TAIL_OUT_OF_BOUNDS_ERR_SMASK) \ 55 | SD(ENG_ERR_STATUS_SDMA_FIRST_DESC_ERR_SMASK) \ 56 | SD(ENG_ERR_STATUS_SDMA_MEM_READ_ERR_SMASK) \ 57 | SD(ENG_ERR_STATUS_SDMA_HALT_ERR_SMASK) \ 58 | SD(ENG_ERR_STATUS_SDMA_LENGTH_MISMATCH_ERR_SMASK) \ 59 | SD(ENG_ERR_STATUS_SDMA_PACKET_DESC_OVERFLOW_ERR_SMASK) \ 60 | SD(ENG_ERR_STATUS_SDMA_HEADER_SELECT_ERR_SMASK) \ 61 | SD(ENG_ERR_STATUS_SDMA_HEADER_ADDRESS_ERR_SMASK) \ 62 | SD(ENG_ERR_STATUS_SDMA_HEADER_LENGTH_ERR_SMASK) \ 63 | SD(ENG_ERR_STATUS_SDMA_TIMEOUT_ERR_SMASK) \ 64 | SD(ENG_ERR_STATUS_SDMA_DESC_TABLE_UNC_ERR_SMASK) \ 65 | SD(ENG_ERR_STATUS_SDMA_ASSEMBLY_UNC_ERR_SMASK) \ 66 | SD(ENG_ERR_STATUS_SDMA_PACKET_TRACKING_UNC_ERR_SMASK) \ 67 | SD(ENG_ERR_STATUS_SDMA_HEADER_STORAGE_UNC_ERR_SMASK) \ 68 | SD(ENG_ERR_STATUS_SDMA_HEADER_REQUEST_FIFO_UNC_ERR_SMASK)) 69 70 /* sdma_sendctrl operations */ 71 #define SDMA_SENDCTRL_OP_ENABLE BIT(0) 72 #define SDMA_SENDCTRL_OP_INTENABLE BIT(1) 73 #define SDMA_SENDCTRL_OP_HALT BIT(2) 74 #define SDMA_SENDCTRL_OP_CLEANUP BIT(3) 75 76 /* handle long defines */ 77 #define SDMA_EGRESS_PACKET_OCCUPANCY_SMASK \ 78 SEND_EGRESS_SEND_DMA_STATUS_SDMA_EGRESS_PACKET_OCCUPANCY_SMASK 79 #define SDMA_EGRESS_PACKET_OCCUPANCY_SHIFT \ 80 SEND_EGRESS_SEND_DMA_STATUS_SDMA_EGRESS_PACKET_OCCUPANCY_SHIFT 81 82 static const char * const sdma_state_names[] = { 83 [sdma_state_s00_hw_down] = "s00_HwDown", 84 [sdma_state_s10_hw_start_up_halt_wait] = "s10_HwStartUpHaltWait", 85 [sdma_state_s15_hw_start_up_clean_wait] = "s15_HwStartUpCleanWait", 86 [sdma_state_s20_idle] = "s20_Idle", 87 [sdma_state_s30_sw_clean_up_wait] = "s30_SwCleanUpWait", 88 [sdma_state_s40_hw_clean_up_wait] = "s40_HwCleanUpWait", 89 [sdma_state_s50_hw_halt_wait] = "s50_HwHaltWait", 90 [sdma_state_s60_idle_halt_wait] = "s60_IdleHaltWait", 91 [sdma_state_s80_hw_freeze] = "s80_HwFreeze", 92 [sdma_state_s82_freeze_sw_clean] = "s82_FreezeSwClean", 93 [sdma_state_s99_running] = "s99_Running", 94 }; 95 96 #ifdef CONFIG_SDMA_VERBOSITY 97 static const char * const sdma_event_names[] = { 98 [sdma_event_e00_go_hw_down] = "e00_GoHwDown", 99 [sdma_event_e10_go_hw_start] = "e10_GoHwStart", 100 [sdma_event_e15_hw_halt_done] = "e15_HwHaltDone", 101 [sdma_event_e25_hw_clean_up_done] = "e25_HwCleanUpDone", 102 [sdma_event_e30_go_running] = "e30_GoRunning", 103 [sdma_event_e40_sw_cleaned] = "e40_SwCleaned", 104 [sdma_event_e50_hw_cleaned] = "e50_HwCleaned", 105 [sdma_event_e60_hw_halted] = "e60_HwHalted", 106 [sdma_event_e70_go_idle] = "e70_GoIdle", 107 [sdma_event_e80_hw_freeze] = "e80_HwFreeze", 108 [sdma_event_e81_hw_frozen] = "e81_HwFrozen", 109 [sdma_event_e82_hw_unfreeze] = "e82_HwUnfreeze", 110 [sdma_event_e85_link_down] = "e85_LinkDown", 111 [sdma_event_e90_sw_halted] = "e90_SwHalted", 112 }; 113 #endif 114 115 static const struct sdma_set_state_action sdma_action_table[] = { 116 [sdma_state_s00_hw_down] = { 117 .go_s99_running_tofalse = 1, 118 .op_enable = 0, 119 .op_intenable = 0, 120 .op_halt = 0, 121 .op_cleanup = 0, 122 }, 123 [sdma_state_s10_hw_start_up_halt_wait] = { 124 .op_enable = 0, 125 .op_intenable = 0, 126 .op_halt = 1, 127 .op_cleanup = 0, 128 }, 129 [sdma_state_s15_hw_start_up_clean_wait] = { 130 .op_enable = 0, 131 .op_intenable = 1, 132 .op_halt = 0, 133 .op_cleanup = 1, 134 }, 135 [sdma_state_s20_idle] = { 136 .op_enable = 0, 137 .op_intenable = 1, 138 .op_halt = 0, 139 .op_cleanup = 0, 140 }, 141 [sdma_state_s30_sw_clean_up_wait] = { 142 .op_enable = 0, 143 .op_intenable = 0, 144 .op_halt = 0, 145 .op_cleanup = 0, 146 }, 147 [sdma_state_s40_hw_clean_up_wait] = { 148 .op_enable = 0, 149 .op_intenable = 0, 150 .op_halt = 0, 151 .op_cleanup = 1, 152 }, 153 [sdma_state_s50_hw_halt_wait] = { 154 .op_enable = 0, 155 .op_intenable = 0, 156 .op_halt = 0, 157 .op_cleanup = 0, 158 }, 159 [sdma_state_s60_idle_halt_wait] = { 160 .go_s99_running_tofalse = 1, 161 .op_enable = 0, 162 .op_intenable = 0, 163 .op_halt = 1, 164 .op_cleanup = 0, 165 }, 166 [sdma_state_s80_hw_freeze] = { 167 .op_enable = 0, 168 .op_intenable = 0, 169 .op_halt = 0, 170 .op_cleanup = 0, 171 }, 172 [sdma_state_s82_freeze_sw_clean] = { 173 .op_enable = 0, 174 .op_intenable = 0, 175 .op_halt = 0, 176 .op_cleanup = 0, 177 }, 178 [sdma_state_s99_running] = { 179 .op_enable = 1, 180 .op_intenable = 1, 181 .op_halt = 0, 182 .op_cleanup = 0, 183 .go_s99_running_totrue = 1, 184 }, 185 }; 186 187 #define SDMA_TAIL_UPDATE_THRESH 0x1F 188 189 /* declare all statics here rather than keep sorting */ 190 static void sdma_complete(struct kref *); 191 static void sdma_finalput(struct sdma_state *); 192 static void sdma_get(struct sdma_state *); 193 static void sdma_hw_clean_up_task(struct tasklet_struct *); 194 static void sdma_put(struct sdma_state *); 195 static void sdma_set_state(struct sdma_engine *, enum sdma_states); 196 static void sdma_start_hw_clean_up(struct sdma_engine *); 197 static void sdma_sw_clean_up_task(struct tasklet_struct *); 198 static void sdma_sendctrl(struct sdma_engine *, unsigned); 199 static void init_sdma_regs(struct sdma_engine *, u32, uint); 200 static void sdma_process_event( 201 struct sdma_engine *sde, 202 enum sdma_events event); 203 static void __sdma_process_event( 204 struct sdma_engine *sde, 205 enum sdma_events event); 206 static void dump_sdma_state(struct sdma_engine *sde); 207 static void sdma_make_progress(struct sdma_engine *sde, u64 status); 208 static void sdma_desc_avail(struct sdma_engine *sde, uint avail); 209 static void sdma_flush_descq(struct sdma_engine *sde); 210 211 /** 212 * sdma_state_name() - return state string from enum 213 * @state: state 214 */ 215 static const char *sdma_state_name(enum sdma_states state) 216 { 217 return sdma_state_names[state]; 218 } 219 220 static void sdma_get(struct sdma_state *ss) 221 { 222 kref_get(&ss->kref); 223 } 224 225 static void sdma_complete(struct kref *kref) 226 { 227 struct sdma_state *ss = 228 container_of(kref, struct sdma_state, kref); 229 230 complete(&ss->comp); 231 } 232 233 static void sdma_put(struct sdma_state *ss) 234 { 235 kref_put(&ss->kref, sdma_complete); 236 } 237 238 static void sdma_finalput(struct sdma_state *ss) 239 { 240 sdma_put(ss); 241 wait_for_completion(&ss->comp); 242 } 243 244 static inline void write_sde_csr( 245 struct sdma_engine *sde, 246 u32 offset0, 247 u64 value) 248 { 249 write_kctxt_csr(sde->dd, sde->this_idx, offset0, value); 250 } 251 252 static inline u64 read_sde_csr( 253 struct sdma_engine *sde, 254 u32 offset0) 255 { 256 return read_kctxt_csr(sde->dd, sde->this_idx, offset0); 257 } 258 259 /* 260 * sdma_wait_for_packet_egress() - wait for the VL FIFO occupancy for 261 * sdma engine 'sde' to drop to 0. 262 */ 263 static void sdma_wait_for_packet_egress(struct sdma_engine *sde, 264 int pause) 265 { 266 u64 off = 8 * sde->this_idx; 267 struct hfi1_devdata *dd = sde->dd; 268 int lcnt = 0; 269 u64 reg_prev; 270 u64 reg = 0; 271 272 while (1) { 273 reg_prev = reg; 274 reg = read_csr(dd, off + SEND_EGRESS_SEND_DMA_STATUS); 275 276 reg &= SDMA_EGRESS_PACKET_OCCUPANCY_SMASK; 277 reg >>= SDMA_EGRESS_PACKET_OCCUPANCY_SHIFT; 278 if (reg == 0) 279 break; 280 /* counter is reest if accupancy count changes */ 281 if (reg != reg_prev) 282 lcnt = 0; 283 if (lcnt++ > 500) { 284 /* timed out - bounce the link */ 285 dd_dev_err(dd, "%s: engine %u timeout waiting for packets to egress, remaining count %u, bouncing link\n", 286 __func__, sde->this_idx, (u32)reg); 287 queue_work(dd->pport->link_wq, 288 &dd->pport->link_bounce_work); 289 break; 290 } 291 udelay(1); 292 } 293 } 294 295 /* 296 * sdma_wait() - wait for packet egress to complete for all SDMA engines, 297 * and pause for credit return. 298 */ 299 void sdma_wait(struct hfi1_devdata *dd) 300 { 301 int i; 302 303 for (i = 0; i < dd->num_sdma; i++) { 304 struct sdma_engine *sde = &dd->per_sdma[i]; 305 306 sdma_wait_for_packet_egress(sde, 0); 307 } 308 } 309 310 static inline void sdma_set_desc_cnt(struct sdma_engine *sde, unsigned cnt) 311 { 312 u64 reg; 313 314 if (!(sde->dd->flags & HFI1_HAS_SDMA_TIMEOUT)) 315 return; 316 reg = cnt; 317 reg &= SD(DESC_CNT_CNT_MASK); 318 reg <<= SD(DESC_CNT_CNT_SHIFT); 319 write_sde_csr(sde, SD(DESC_CNT), reg); 320 } 321 322 static inline void complete_tx(struct sdma_engine *sde, 323 struct sdma_txreq *tx, 324 int res) 325 { 326 /* protect against complete modifying */ 327 struct iowait *wait = tx->wait; 328 callback_t complete = tx->complete; 329 330 #ifdef CONFIG_HFI1_DEBUG_SDMA_ORDER 331 trace_hfi1_sdma_out_sn(sde, tx->sn); 332 if (WARN_ON_ONCE(sde->head_sn != tx->sn)) 333 dd_dev_err(sde->dd, "expected %llu got %llu\n", 334 sde->head_sn, tx->sn); 335 sde->head_sn++; 336 #endif 337 __sdma_txclean(sde->dd, tx); 338 if (complete) 339 (*complete)(tx, res); 340 if (iowait_sdma_dec(wait)) 341 iowait_drain_wakeup(wait); 342 } 343 344 /* 345 * Complete all the sdma requests with a SDMA_TXREQ_S_ABORTED status 346 * 347 * Depending on timing there can be txreqs in two places: 348 * - in the descq ring 349 * - in the flush list 350 * 351 * To avoid ordering issues the descq ring needs to be flushed 352 * first followed by the flush list. 353 * 354 * This routine is called from two places 355 * - From a work queue item 356 * - Directly from the state machine just before setting the 357 * state to running 358 * 359 * Must be called with head_lock held 360 * 361 */ 362 static void sdma_flush(struct sdma_engine *sde) 363 { 364 struct sdma_txreq *txp, *txp_next; 365 LIST_HEAD(flushlist); 366 unsigned long flags; 367 uint seq; 368 369 /* flush from head to tail */ 370 sdma_flush_descq(sde); 371 spin_lock_irqsave(&sde->flushlist_lock, flags); 372 /* copy flush list */ 373 list_splice_init(&sde->flushlist, &flushlist); 374 spin_unlock_irqrestore(&sde->flushlist_lock, flags); 375 /* flush from flush list */ 376 list_for_each_entry_safe(txp, txp_next, &flushlist, list) 377 complete_tx(sde, txp, SDMA_TXREQ_S_ABORTED); 378 /* wakeup QPs orphaned on the dmawait list */ 379 do { 380 struct iowait *w, *nw; 381 382 seq = read_seqbegin(&sde->waitlock); 383 if (!list_empty(&sde->dmawait)) { 384 write_seqlock(&sde->waitlock); 385 list_for_each_entry_safe(w, nw, &sde->dmawait, list) { 386 if (w->wakeup) { 387 w->wakeup(w, SDMA_AVAIL_REASON); 388 list_del_init(&w->list); 389 } 390 } 391 write_sequnlock(&sde->waitlock); 392 } 393 } while (read_seqretry(&sde->waitlock, seq)); 394 } 395 396 /* 397 * Fields a work request for flushing the descq ring 398 * and the flush list 399 * 400 * If the engine has been brought to running during 401 * the scheduling delay, the flush is ignored, assuming 402 * that the process of bringing the engine to running 403 * would have done this flush prior to going to running. 404 * 405 */ 406 static void sdma_field_flush(struct work_struct *work) 407 { 408 unsigned long flags; 409 struct sdma_engine *sde = 410 container_of(work, struct sdma_engine, flush_worker); 411 412 write_seqlock_irqsave(&sde->head_lock, flags); 413 if (!__sdma_running(sde)) 414 sdma_flush(sde); 415 write_sequnlock_irqrestore(&sde->head_lock, flags); 416 } 417 418 static void sdma_err_halt_wait(struct work_struct *work) 419 { 420 struct sdma_engine *sde = container_of(work, struct sdma_engine, 421 err_halt_worker); 422 u64 statuscsr; 423 unsigned long timeout; 424 425 timeout = jiffies + msecs_to_jiffies(SDMA_ERR_HALT_TIMEOUT); 426 while (1) { 427 statuscsr = read_sde_csr(sde, SD(STATUS)); 428 statuscsr &= SD(STATUS_ENG_HALTED_SMASK); 429 if (statuscsr) 430 break; 431 if (time_after(jiffies, timeout)) { 432 dd_dev_err(sde->dd, 433 "SDMA engine %d - timeout waiting for engine to halt\n", 434 sde->this_idx); 435 /* 436 * Continue anyway. This could happen if there was 437 * an uncorrectable error in the wrong spot. 438 */ 439 break; 440 } 441 usleep_range(80, 120); 442 } 443 444 sdma_process_event(sde, sdma_event_e15_hw_halt_done); 445 } 446 447 static void sdma_err_progress_check_schedule(struct sdma_engine *sde) 448 { 449 if (!is_bx(sde->dd) && HFI1_CAP_IS_KSET(SDMA_AHG)) { 450 unsigned index; 451 struct hfi1_devdata *dd = sde->dd; 452 453 for (index = 0; index < dd->num_sdma; index++) { 454 struct sdma_engine *curr_sdma = &dd->per_sdma[index]; 455 456 if (curr_sdma != sde) 457 curr_sdma->progress_check_head = 458 curr_sdma->descq_head; 459 } 460 dd_dev_err(sde->dd, 461 "SDMA engine %d - check scheduled\n", 462 sde->this_idx); 463 mod_timer(&sde->err_progress_check_timer, jiffies + 10); 464 } 465 } 466 467 static void sdma_err_progress_check(struct timer_list *t) 468 { 469 unsigned index; 470 struct sdma_engine *sde = from_timer(sde, t, err_progress_check_timer); 471 472 dd_dev_err(sde->dd, "SDE progress check event\n"); 473 for (index = 0; index < sde->dd->num_sdma; index++) { 474 struct sdma_engine *curr_sde = &sde->dd->per_sdma[index]; 475 unsigned long flags; 476 477 /* check progress on each engine except the current one */ 478 if (curr_sde == sde) 479 continue; 480 /* 481 * We must lock interrupts when acquiring sde->lock, 482 * to avoid a deadlock if interrupt triggers and spins on 483 * the same lock on same CPU 484 */ 485 spin_lock_irqsave(&curr_sde->tail_lock, flags); 486 write_seqlock(&curr_sde->head_lock); 487 488 /* skip non-running queues */ 489 if (curr_sde->state.current_state != sdma_state_s99_running) { 490 write_sequnlock(&curr_sde->head_lock); 491 spin_unlock_irqrestore(&curr_sde->tail_lock, flags); 492 continue; 493 } 494 495 if ((curr_sde->descq_head != curr_sde->descq_tail) && 496 (curr_sde->descq_head == 497 curr_sde->progress_check_head)) 498 __sdma_process_event(curr_sde, 499 sdma_event_e90_sw_halted); 500 write_sequnlock(&curr_sde->head_lock); 501 spin_unlock_irqrestore(&curr_sde->tail_lock, flags); 502 } 503 schedule_work(&sde->err_halt_worker); 504 } 505 506 static void sdma_hw_clean_up_task(struct tasklet_struct *t) 507 { 508 struct sdma_engine *sde = from_tasklet(sde, t, 509 sdma_hw_clean_up_task); 510 u64 statuscsr; 511 512 while (1) { 513 #ifdef CONFIG_SDMA_VERBOSITY 514 dd_dev_err(sde->dd, "CONFIG SDMA(%u) %s:%d %s()\n", 515 sde->this_idx, slashstrip(__FILE__), __LINE__, 516 __func__); 517 #endif 518 statuscsr = read_sde_csr(sde, SD(STATUS)); 519 statuscsr &= SD(STATUS_ENG_CLEANED_UP_SMASK); 520 if (statuscsr) 521 break; 522 udelay(10); 523 } 524 525 sdma_process_event(sde, sdma_event_e25_hw_clean_up_done); 526 } 527 528 static inline struct sdma_txreq *get_txhead(struct sdma_engine *sde) 529 { 530 return sde->tx_ring[sde->tx_head & sde->sdma_mask]; 531 } 532 533 /* 534 * flush ring for recovery 535 */ 536 static void sdma_flush_descq(struct sdma_engine *sde) 537 { 538 u16 head, tail; 539 int progress = 0; 540 struct sdma_txreq *txp = get_txhead(sde); 541 542 /* The reason for some of the complexity of this code is that 543 * not all descriptors have corresponding txps. So, we have to 544 * be able to skip over descs until we wander into the range of 545 * the next txp on the list. 546 */ 547 head = sde->descq_head & sde->sdma_mask; 548 tail = sde->descq_tail & sde->sdma_mask; 549 while (head != tail) { 550 /* advance head, wrap if needed */ 551 head = ++sde->descq_head & sde->sdma_mask; 552 /* if now past this txp's descs, do the callback */ 553 if (txp && txp->next_descq_idx == head) { 554 /* remove from list */ 555 sde->tx_ring[sde->tx_head++ & sde->sdma_mask] = NULL; 556 complete_tx(sde, txp, SDMA_TXREQ_S_ABORTED); 557 trace_hfi1_sdma_progress(sde, head, tail, txp); 558 txp = get_txhead(sde); 559 } 560 progress++; 561 } 562 if (progress) 563 sdma_desc_avail(sde, sdma_descq_freecnt(sde)); 564 } 565 566 static void sdma_sw_clean_up_task(struct tasklet_struct *t) 567 { 568 struct sdma_engine *sde = from_tasklet(sde, t, sdma_sw_clean_up_task); 569 unsigned long flags; 570 571 spin_lock_irqsave(&sde->tail_lock, flags); 572 write_seqlock(&sde->head_lock); 573 574 /* 575 * At this point, the following should always be true: 576 * - We are halted, so no more descriptors are getting retired. 577 * - We are not running, so no one is submitting new work. 578 * - Only we can send the e40_sw_cleaned, so we can't start 579 * running again until we say so. So, the active list and 580 * descq are ours to play with. 581 */ 582 583 /* 584 * In the error clean up sequence, software clean must be called 585 * before the hardware clean so we can use the hardware head in 586 * the progress routine. A hardware clean or SPC unfreeze will 587 * reset the hardware head. 588 * 589 * Process all retired requests. The progress routine will use the 590 * latest physical hardware head - we are not running so speed does 591 * not matter. 592 */ 593 sdma_make_progress(sde, 0); 594 595 sdma_flush(sde); 596 597 /* 598 * Reset our notion of head and tail. 599 * Note that the HW registers have been reset via an earlier 600 * clean up. 601 */ 602 sde->descq_tail = 0; 603 sde->descq_head = 0; 604 sde->desc_avail = sdma_descq_freecnt(sde); 605 *sde->head_dma = 0; 606 607 __sdma_process_event(sde, sdma_event_e40_sw_cleaned); 608 609 write_sequnlock(&sde->head_lock); 610 spin_unlock_irqrestore(&sde->tail_lock, flags); 611 } 612 613 static void sdma_sw_tear_down(struct sdma_engine *sde) 614 { 615 struct sdma_state *ss = &sde->state; 616 617 /* Releasing this reference means the state machine has stopped. */ 618 sdma_put(ss); 619 620 /* stop waiting for all unfreeze events to complete */ 621 atomic_set(&sde->dd->sdma_unfreeze_count, -1); 622 wake_up_interruptible(&sde->dd->sdma_unfreeze_wq); 623 } 624 625 static void sdma_start_hw_clean_up(struct sdma_engine *sde) 626 { 627 tasklet_hi_schedule(&sde->sdma_hw_clean_up_task); 628 } 629 630 static void sdma_set_state(struct sdma_engine *sde, 631 enum sdma_states next_state) 632 { 633 struct sdma_state *ss = &sde->state; 634 const struct sdma_set_state_action *action = sdma_action_table; 635 unsigned op = 0; 636 637 trace_hfi1_sdma_state( 638 sde, 639 sdma_state_names[ss->current_state], 640 sdma_state_names[next_state]); 641 642 /* debugging bookkeeping */ 643 ss->previous_state = ss->current_state; 644 ss->previous_op = ss->current_op; 645 ss->current_state = next_state; 646 647 if (ss->previous_state != sdma_state_s99_running && 648 next_state == sdma_state_s99_running) 649 sdma_flush(sde); 650 651 if (action[next_state].op_enable) 652 op |= SDMA_SENDCTRL_OP_ENABLE; 653 654 if (action[next_state].op_intenable) 655 op |= SDMA_SENDCTRL_OP_INTENABLE; 656 657 if (action[next_state].op_halt) 658 op |= SDMA_SENDCTRL_OP_HALT; 659 660 if (action[next_state].op_cleanup) 661 op |= SDMA_SENDCTRL_OP_CLEANUP; 662 663 if (action[next_state].go_s99_running_tofalse) 664 ss->go_s99_running = 0; 665 666 if (action[next_state].go_s99_running_totrue) 667 ss->go_s99_running = 1; 668 669 ss->current_op = op; 670 sdma_sendctrl(sde, ss->current_op); 671 } 672 673 /** 674 * sdma_get_descq_cnt() - called when device probed 675 * 676 * Return a validated descq count. 677 * 678 * This is currently only used in the verbs initialization to build the tx 679 * list. 680 * 681 * This will probably be deleted in favor of a more scalable approach to 682 * alloc tx's. 683 * 684 */ 685 u16 sdma_get_descq_cnt(void) 686 { 687 u16 count = sdma_descq_cnt; 688 689 if (!count) 690 return SDMA_DESCQ_CNT; 691 /* count must be a power of 2 greater than 64 and less than 692 * 32768. Otherwise return default. 693 */ 694 if (!is_power_of_2(count)) 695 return SDMA_DESCQ_CNT; 696 if (count < 64 || count > 32768) 697 return SDMA_DESCQ_CNT; 698 return count; 699 } 700 701 /** 702 * sdma_engine_get_vl() - return vl for a given sdma engine 703 * @sde: sdma engine 704 * 705 * This function returns the vl mapped to a given engine, or an error if 706 * the mapping can't be found. The mapping fields are protected by RCU. 707 */ 708 int sdma_engine_get_vl(struct sdma_engine *sde) 709 { 710 struct hfi1_devdata *dd = sde->dd; 711 struct sdma_vl_map *m; 712 u8 vl; 713 714 if (sde->this_idx >= TXE_NUM_SDMA_ENGINES) 715 return -EINVAL; 716 717 rcu_read_lock(); 718 m = rcu_dereference(dd->sdma_map); 719 if (unlikely(!m)) { 720 rcu_read_unlock(); 721 return -EINVAL; 722 } 723 vl = m->engine_to_vl[sde->this_idx]; 724 rcu_read_unlock(); 725 726 return vl; 727 } 728 729 /** 730 * sdma_select_engine_vl() - select sdma engine 731 * @dd: devdata 732 * @selector: a spreading factor 733 * @vl: this vl 734 * 735 * 736 * This function returns an engine based on the selector and a vl. The 737 * mapping fields are protected by RCU. 738 */ 739 struct sdma_engine *sdma_select_engine_vl( 740 struct hfi1_devdata *dd, 741 u32 selector, 742 u8 vl) 743 { 744 struct sdma_vl_map *m; 745 struct sdma_map_elem *e; 746 struct sdma_engine *rval; 747 748 /* NOTE This should only happen if SC->VL changed after the initial 749 * checks on the QP/AH 750 * Default will return engine 0 below 751 */ 752 if (vl >= num_vls) { 753 rval = NULL; 754 goto done; 755 } 756 757 rcu_read_lock(); 758 m = rcu_dereference(dd->sdma_map); 759 if (unlikely(!m)) { 760 rcu_read_unlock(); 761 return &dd->per_sdma[0]; 762 } 763 e = m->map[vl & m->mask]; 764 rval = e->sde[selector & e->mask]; 765 rcu_read_unlock(); 766 767 done: 768 rval = !rval ? &dd->per_sdma[0] : rval; 769 trace_hfi1_sdma_engine_select(dd, selector, vl, rval->this_idx); 770 return rval; 771 } 772 773 /** 774 * sdma_select_engine_sc() - select sdma engine 775 * @dd: devdata 776 * @selector: a spreading factor 777 * @sc5: the 5 bit sc 778 * 779 * 780 * This function returns an engine based on the selector and an sc. 781 */ 782 struct sdma_engine *sdma_select_engine_sc( 783 struct hfi1_devdata *dd, 784 u32 selector, 785 u8 sc5) 786 { 787 u8 vl = sc_to_vlt(dd, sc5); 788 789 return sdma_select_engine_vl(dd, selector, vl); 790 } 791 792 struct sdma_rht_map_elem { 793 u32 mask; 794 u8 ctr; 795 struct sdma_engine *sde[]; 796 }; 797 798 struct sdma_rht_node { 799 unsigned long cpu_id; 800 struct sdma_rht_map_elem *map[HFI1_MAX_VLS_SUPPORTED]; 801 struct rhash_head node; 802 }; 803 804 #define NR_CPUS_HINT 192 805 806 static const struct rhashtable_params sdma_rht_params = { 807 .nelem_hint = NR_CPUS_HINT, 808 .head_offset = offsetof(struct sdma_rht_node, node), 809 .key_offset = offsetof(struct sdma_rht_node, cpu_id), 810 .key_len = sizeof_field(struct sdma_rht_node, cpu_id), 811 .max_size = NR_CPUS, 812 .min_size = 8, 813 .automatic_shrinking = true, 814 }; 815 816 /* 817 * sdma_select_user_engine() - select sdma engine based on user setup 818 * @dd: devdata 819 * @selector: a spreading factor 820 * @vl: this vl 821 * 822 * This function returns an sdma engine for a user sdma request. 823 * User defined sdma engine affinity setting is honored when applicable, 824 * otherwise system default sdma engine mapping is used. To ensure correct 825 * ordering, the mapping from <selector, vl> to sde must remain unchanged. 826 */ 827 struct sdma_engine *sdma_select_user_engine(struct hfi1_devdata *dd, 828 u32 selector, u8 vl) 829 { 830 struct sdma_rht_node *rht_node; 831 struct sdma_engine *sde = NULL; 832 unsigned long cpu_id; 833 834 /* 835 * To ensure that always the same sdma engine(s) will be 836 * selected make sure the process is pinned to this CPU only. 837 */ 838 if (current->nr_cpus_allowed != 1) 839 goto out; 840 841 rcu_read_lock(); 842 cpu_id = smp_processor_id(); 843 rht_node = rhashtable_lookup(dd->sdma_rht, &cpu_id, 844 sdma_rht_params); 845 846 if (rht_node && rht_node->map[vl]) { 847 struct sdma_rht_map_elem *map = rht_node->map[vl]; 848 849 sde = map->sde[selector & map->mask]; 850 } 851 rcu_read_unlock(); 852 853 if (sde) 854 return sde; 855 856 out: 857 return sdma_select_engine_vl(dd, selector, vl); 858 } 859 860 static void sdma_populate_sde_map(struct sdma_rht_map_elem *map) 861 { 862 int i; 863 864 for (i = 0; i < roundup_pow_of_two(map->ctr ? : 1) - map->ctr; i++) 865 map->sde[map->ctr + i] = map->sde[i]; 866 } 867 868 static void sdma_cleanup_sde_map(struct sdma_rht_map_elem *map, 869 struct sdma_engine *sde) 870 { 871 unsigned int i, pow; 872 873 /* only need to check the first ctr entries for a match */ 874 for (i = 0; i < map->ctr; i++) { 875 if (map->sde[i] == sde) { 876 memmove(&map->sde[i], &map->sde[i + 1], 877 (map->ctr - i - 1) * sizeof(map->sde[0])); 878 map->ctr--; 879 pow = roundup_pow_of_two(map->ctr ? : 1); 880 map->mask = pow - 1; 881 sdma_populate_sde_map(map); 882 break; 883 } 884 } 885 } 886 887 /* 888 * Prevents concurrent reads and writes of the sdma engine cpu_mask 889 */ 890 static DEFINE_MUTEX(process_to_sde_mutex); 891 892 ssize_t sdma_set_cpu_to_sde_map(struct sdma_engine *sde, const char *buf, 893 size_t count) 894 { 895 struct hfi1_devdata *dd = sde->dd; 896 cpumask_var_t mask, new_mask; 897 unsigned long cpu; 898 int ret, vl, sz; 899 struct sdma_rht_node *rht_node; 900 901 vl = sdma_engine_get_vl(sde); 902 if (unlikely(vl < 0 || vl >= ARRAY_SIZE(rht_node->map))) 903 return -EINVAL; 904 905 ret = zalloc_cpumask_var(&mask, GFP_KERNEL); 906 if (!ret) 907 return -ENOMEM; 908 909 ret = zalloc_cpumask_var(&new_mask, GFP_KERNEL); 910 if (!ret) { 911 free_cpumask_var(mask); 912 return -ENOMEM; 913 } 914 ret = cpulist_parse(buf, mask); 915 if (ret) 916 goto out_free; 917 918 if (!cpumask_subset(mask, cpu_online_mask)) { 919 dd_dev_warn(sde->dd, "Invalid CPU mask\n"); 920 ret = -EINVAL; 921 goto out_free; 922 } 923 924 sz = sizeof(struct sdma_rht_map_elem) + 925 (TXE_NUM_SDMA_ENGINES * sizeof(struct sdma_engine *)); 926 927 mutex_lock(&process_to_sde_mutex); 928 929 for_each_cpu(cpu, mask) { 930 /* Check if we have this already mapped */ 931 if (cpumask_test_cpu(cpu, &sde->cpu_mask)) { 932 cpumask_set_cpu(cpu, new_mask); 933 continue; 934 } 935 936 rht_node = rhashtable_lookup_fast(dd->sdma_rht, &cpu, 937 sdma_rht_params); 938 if (!rht_node) { 939 rht_node = kzalloc(sizeof(*rht_node), GFP_KERNEL); 940 if (!rht_node) { 941 ret = -ENOMEM; 942 goto out; 943 } 944 945 rht_node->map[vl] = kzalloc(sz, GFP_KERNEL); 946 if (!rht_node->map[vl]) { 947 kfree(rht_node); 948 ret = -ENOMEM; 949 goto out; 950 } 951 rht_node->cpu_id = cpu; 952 rht_node->map[vl]->mask = 0; 953 rht_node->map[vl]->ctr = 1; 954 rht_node->map[vl]->sde[0] = sde; 955 956 ret = rhashtable_insert_fast(dd->sdma_rht, 957 &rht_node->node, 958 sdma_rht_params); 959 if (ret) { 960 kfree(rht_node->map[vl]); 961 kfree(rht_node); 962 dd_dev_err(sde->dd, "Failed to set process to sde affinity for cpu %lu\n", 963 cpu); 964 goto out; 965 } 966 967 } else { 968 int ctr, pow; 969 970 /* Add new user mappings */ 971 if (!rht_node->map[vl]) 972 rht_node->map[vl] = kzalloc(sz, GFP_KERNEL); 973 974 if (!rht_node->map[vl]) { 975 ret = -ENOMEM; 976 goto out; 977 } 978 979 rht_node->map[vl]->ctr++; 980 ctr = rht_node->map[vl]->ctr; 981 rht_node->map[vl]->sde[ctr - 1] = sde; 982 pow = roundup_pow_of_two(ctr); 983 rht_node->map[vl]->mask = pow - 1; 984 985 /* Populate the sde map table */ 986 sdma_populate_sde_map(rht_node->map[vl]); 987 } 988 cpumask_set_cpu(cpu, new_mask); 989 } 990 991 /* Clean up old mappings */ 992 for_each_cpu(cpu, cpu_online_mask) { 993 struct sdma_rht_node *rht_node; 994 995 /* Don't cleanup sdes that are set in the new mask */ 996 if (cpumask_test_cpu(cpu, mask)) 997 continue; 998 999 rht_node = rhashtable_lookup_fast(dd->sdma_rht, &cpu, 1000 sdma_rht_params); 1001 if (rht_node) { 1002 bool empty = true; 1003 int i; 1004 1005 /* Remove mappings for old sde */ 1006 for (i = 0; i < HFI1_MAX_VLS_SUPPORTED; i++) 1007 if (rht_node->map[i]) 1008 sdma_cleanup_sde_map(rht_node->map[i], 1009 sde); 1010 1011 /* Free empty hash table entries */ 1012 for (i = 0; i < HFI1_MAX_VLS_SUPPORTED; i++) { 1013 if (!rht_node->map[i]) 1014 continue; 1015 1016 if (rht_node->map[i]->ctr) { 1017 empty = false; 1018 break; 1019 } 1020 } 1021 1022 if (empty) { 1023 ret = rhashtable_remove_fast(dd->sdma_rht, 1024 &rht_node->node, 1025 sdma_rht_params); 1026 WARN_ON(ret); 1027 1028 for (i = 0; i < HFI1_MAX_VLS_SUPPORTED; i++) 1029 kfree(rht_node->map[i]); 1030 1031 kfree(rht_node); 1032 } 1033 } 1034 } 1035 1036 cpumask_copy(&sde->cpu_mask, new_mask); 1037 out: 1038 mutex_unlock(&process_to_sde_mutex); 1039 out_free: 1040 free_cpumask_var(mask); 1041 free_cpumask_var(new_mask); 1042 return ret ? : strnlen(buf, PAGE_SIZE); 1043 } 1044 1045 ssize_t sdma_get_cpu_to_sde_map(struct sdma_engine *sde, char *buf) 1046 { 1047 mutex_lock(&process_to_sde_mutex); 1048 if (cpumask_empty(&sde->cpu_mask)) 1049 snprintf(buf, PAGE_SIZE, "%s\n", "empty"); 1050 else 1051 cpumap_print_to_pagebuf(true, buf, &sde->cpu_mask); 1052 mutex_unlock(&process_to_sde_mutex); 1053 return strnlen(buf, PAGE_SIZE); 1054 } 1055 1056 static void sdma_rht_free(void *ptr, void *arg) 1057 { 1058 struct sdma_rht_node *rht_node = ptr; 1059 int i; 1060 1061 for (i = 0; i < HFI1_MAX_VLS_SUPPORTED; i++) 1062 kfree(rht_node->map[i]); 1063 1064 kfree(rht_node); 1065 } 1066 1067 /** 1068 * sdma_seqfile_dump_cpu_list() - debugfs dump the cpu to sdma mappings 1069 * @s: seq file 1070 * @dd: hfi1_devdata 1071 * @cpuid: cpu id 1072 * 1073 * This routine dumps the process to sde mappings per cpu 1074 */ 1075 void sdma_seqfile_dump_cpu_list(struct seq_file *s, 1076 struct hfi1_devdata *dd, 1077 unsigned long cpuid) 1078 { 1079 struct sdma_rht_node *rht_node; 1080 int i, j; 1081 1082 rht_node = rhashtable_lookup_fast(dd->sdma_rht, &cpuid, 1083 sdma_rht_params); 1084 if (!rht_node) 1085 return; 1086 1087 seq_printf(s, "cpu%3lu: ", cpuid); 1088 for (i = 0; i < HFI1_MAX_VLS_SUPPORTED; i++) { 1089 if (!rht_node->map[i] || !rht_node->map[i]->ctr) 1090 continue; 1091 1092 seq_printf(s, " vl%d: [", i); 1093 1094 for (j = 0; j < rht_node->map[i]->ctr; j++) { 1095 if (!rht_node->map[i]->sde[j]) 1096 continue; 1097 1098 if (j > 0) 1099 seq_puts(s, ","); 1100 1101 seq_printf(s, " sdma%2d", 1102 rht_node->map[i]->sde[j]->this_idx); 1103 } 1104 seq_puts(s, " ]"); 1105 } 1106 1107 seq_puts(s, "\n"); 1108 } 1109 1110 /* 1111 * Free the indicated map struct 1112 */ 1113 static void sdma_map_free(struct sdma_vl_map *m) 1114 { 1115 int i; 1116 1117 for (i = 0; m && i < m->actual_vls; i++) 1118 kfree(m->map[i]); 1119 kfree(m); 1120 } 1121 1122 /* 1123 * Handle RCU callback 1124 */ 1125 static void sdma_map_rcu_callback(struct rcu_head *list) 1126 { 1127 struct sdma_vl_map *m = container_of(list, struct sdma_vl_map, list); 1128 1129 sdma_map_free(m); 1130 } 1131 1132 /** 1133 * sdma_map_init - called when # vls change 1134 * @dd: hfi1_devdata 1135 * @port: port number 1136 * @num_vls: number of vls 1137 * @vl_engines: per vl engine mapping (optional) 1138 * 1139 * This routine changes the mapping based on the number of vls. 1140 * 1141 * vl_engines is used to specify a non-uniform vl/engine loading. NULL 1142 * implies auto computing the loading and giving each VLs a uniform 1143 * distribution of engines per VL. 1144 * 1145 * The auto algorithm computes the sde_per_vl and the number of extra 1146 * engines. Any extra engines are added from the last VL on down. 1147 * 1148 * rcu locking is used here to control access to the mapping fields. 1149 * 1150 * If either the num_vls or num_sdma are non-power of 2, the array sizes 1151 * in the struct sdma_vl_map and the struct sdma_map_elem are rounded 1152 * up to the next highest power of 2 and the first entry is reused 1153 * in a round robin fashion. 1154 * 1155 * If an error occurs the map change is not done and the mapping is 1156 * not changed. 1157 * 1158 */ 1159 int sdma_map_init(struct hfi1_devdata *dd, u8 port, u8 num_vls, u8 *vl_engines) 1160 { 1161 int i, j; 1162 int extra, sde_per_vl; 1163 int engine = 0; 1164 u8 lvl_engines[OPA_MAX_VLS]; 1165 struct sdma_vl_map *oldmap, *newmap; 1166 1167 if (!(dd->flags & HFI1_HAS_SEND_DMA)) 1168 return 0; 1169 1170 if (!vl_engines) { 1171 /* truncate divide */ 1172 sde_per_vl = dd->num_sdma / num_vls; 1173 /* extras */ 1174 extra = dd->num_sdma % num_vls; 1175 vl_engines = lvl_engines; 1176 /* add extras from last vl down */ 1177 for (i = num_vls - 1; i >= 0; i--, extra--) 1178 vl_engines[i] = sde_per_vl + (extra > 0 ? 1 : 0); 1179 } 1180 /* build new map */ 1181 newmap = kzalloc( 1182 sizeof(struct sdma_vl_map) + 1183 roundup_pow_of_two(num_vls) * 1184 sizeof(struct sdma_map_elem *), 1185 GFP_KERNEL); 1186 if (!newmap) 1187 goto bail; 1188 newmap->actual_vls = num_vls; 1189 newmap->vls = roundup_pow_of_two(num_vls); 1190 newmap->mask = (1 << ilog2(newmap->vls)) - 1; 1191 /* initialize back-map */ 1192 for (i = 0; i < TXE_NUM_SDMA_ENGINES; i++) 1193 newmap->engine_to_vl[i] = -1; 1194 for (i = 0; i < newmap->vls; i++) { 1195 /* save for wrap around */ 1196 int first_engine = engine; 1197 1198 if (i < newmap->actual_vls) { 1199 int sz = roundup_pow_of_two(vl_engines[i]); 1200 1201 /* only allocate once */ 1202 newmap->map[i] = kzalloc( 1203 sizeof(struct sdma_map_elem) + 1204 sz * sizeof(struct sdma_engine *), 1205 GFP_KERNEL); 1206 if (!newmap->map[i]) 1207 goto bail; 1208 newmap->map[i]->mask = (1 << ilog2(sz)) - 1; 1209 /* assign engines */ 1210 for (j = 0; j < sz; j++) { 1211 newmap->map[i]->sde[j] = 1212 &dd->per_sdma[engine]; 1213 if (++engine >= first_engine + vl_engines[i]) 1214 /* wrap back to first engine */ 1215 engine = first_engine; 1216 } 1217 /* assign back-map */ 1218 for (j = 0; j < vl_engines[i]; j++) 1219 newmap->engine_to_vl[first_engine + j] = i; 1220 } else { 1221 /* just re-use entry without allocating */ 1222 newmap->map[i] = newmap->map[i % num_vls]; 1223 } 1224 engine = first_engine + vl_engines[i]; 1225 } 1226 /* newmap in hand, save old map */ 1227 spin_lock_irq(&dd->sde_map_lock); 1228 oldmap = rcu_dereference_protected(dd->sdma_map, 1229 lockdep_is_held(&dd->sde_map_lock)); 1230 1231 /* publish newmap */ 1232 rcu_assign_pointer(dd->sdma_map, newmap); 1233 1234 spin_unlock_irq(&dd->sde_map_lock); 1235 /* success, free any old map after grace period */ 1236 if (oldmap) 1237 call_rcu(&oldmap->list, sdma_map_rcu_callback); 1238 return 0; 1239 bail: 1240 /* free any partial allocation */ 1241 sdma_map_free(newmap); 1242 return -ENOMEM; 1243 } 1244 1245 /** 1246 * sdma_clean - Clean up allocated memory 1247 * @dd: struct hfi1_devdata 1248 * @num_engines: num sdma engines 1249 * 1250 * This routine can be called regardless of the success of 1251 * sdma_init() 1252 */ 1253 void sdma_clean(struct hfi1_devdata *dd, size_t num_engines) 1254 { 1255 size_t i; 1256 struct sdma_engine *sde; 1257 1258 if (dd->sdma_pad_dma) { 1259 dma_free_coherent(&dd->pcidev->dev, SDMA_PAD, 1260 (void *)dd->sdma_pad_dma, 1261 dd->sdma_pad_phys); 1262 dd->sdma_pad_dma = NULL; 1263 dd->sdma_pad_phys = 0; 1264 } 1265 if (dd->sdma_heads_dma) { 1266 dma_free_coherent(&dd->pcidev->dev, dd->sdma_heads_size, 1267 (void *)dd->sdma_heads_dma, 1268 dd->sdma_heads_phys); 1269 dd->sdma_heads_dma = NULL; 1270 dd->sdma_heads_phys = 0; 1271 } 1272 for (i = 0; dd->per_sdma && i < num_engines; ++i) { 1273 sde = &dd->per_sdma[i]; 1274 1275 sde->head_dma = NULL; 1276 sde->head_phys = 0; 1277 1278 if (sde->descq) { 1279 dma_free_coherent( 1280 &dd->pcidev->dev, 1281 sde->descq_cnt * sizeof(u64[2]), 1282 sde->descq, 1283 sde->descq_phys 1284 ); 1285 sde->descq = NULL; 1286 sde->descq_phys = 0; 1287 } 1288 kvfree(sde->tx_ring); 1289 sde->tx_ring = NULL; 1290 } 1291 if (rcu_access_pointer(dd->sdma_map)) { 1292 spin_lock_irq(&dd->sde_map_lock); 1293 sdma_map_free(rcu_access_pointer(dd->sdma_map)); 1294 RCU_INIT_POINTER(dd->sdma_map, NULL); 1295 spin_unlock_irq(&dd->sde_map_lock); 1296 synchronize_rcu(); 1297 } 1298 kfree(dd->per_sdma); 1299 dd->per_sdma = NULL; 1300 1301 if (dd->sdma_rht) { 1302 rhashtable_free_and_destroy(dd->sdma_rht, sdma_rht_free, NULL); 1303 kfree(dd->sdma_rht); 1304 dd->sdma_rht = NULL; 1305 } 1306 } 1307 1308 /** 1309 * sdma_init() - called when device probed 1310 * @dd: hfi1_devdata 1311 * @port: port number (currently only zero) 1312 * 1313 * Initializes each sde and its csrs. 1314 * Interrupts are not required to be enabled. 1315 * 1316 * Returns: 1317 * 0 - success, -errno on failure 1318 */ 1319 int sdma_init(struct hfi1_devdata *dd, u8 port) 1320 { 1321 unsigned this_idx; 1322 struct sdma_engine *sde; 1323 struct rhashtable *tmp_sdma_rht; 1324 u16 descq_cnt; 1325 void *curr_head; 1326 struct hfi1_pportdata *ppd = dd->pport + port; 1327 u32 per_sdma_credits; 1328 uint idle_cnt = sdma_idle_cnt; 1329 size_t num_engines = chip_sdma_engines(dd); 1330 int ret = -ENOMEM; 1331 1332 if (!HFI1_CAP_IS_KSET(SDMA)) { 1333 HFI1_CAP_CLEAR(SDMA_AHG); 1334 return 0; 1335 } 1336 if (mod_num_sdma && 1337 /* can't exceed chip support */ 1338 mod_num_sdma <= chip_sdma_engines(dd) && 1339 /* count must be >= vls */ 1340 mod_num_sdma >= num_vls) 1341 num_engines = mod_num_sdma; 1342 1343 dd_dev_info(dd, "SDMA mod_num_sdma: %u\n", mod_num_sdma); 1344 dd_dev_info(dd, "SDMA chip_sdma_engines: %u\n", chip_sdma_engines(dd)); 1345 dd_dev_info(dd, "SDMA chip_sdma_mem_size: %u\n", 1346 chip_sdma_mem_size(dd)); 1347 1348 per_sdma_credits = 1349 chip_sdma_mem_size(dd) / (num_engines * SDMA_BLOCK_SIZE); 1350 1351 /* set up freeze waitqueue */ 1352 init_waitqueue_head(&dd->sdma_unfreeze_wq); 1353 atomic_set(&dd->sdma_unfreeze_count, 0); 1354 1355 descq_cnt = sdma_get_descq_cnt(); 1356 dd_dev_info(dd, "SDMA engines %zu descq_cnt %u\n", 1357 num_engines, descq_cnt); 1358 1359 /* alloc memory for array of send engines */ 1360 dd->per_sdma = kcalloc_node(num_engines, sizeof(*dd->per_sdma), 1361 GFP_KERNEL, dd->node); 1362 if (!dd->per_sdma) 1363 return ret; 1364 1365 idle_cnt = ns_to_cclock(dd, idle_cnt); 1366 if (idle_cnt) 1367 dd->default_desc1 = 1368 SDMA_DESC1_HEAD_TO_HOST_FLAG; 1369 else 1370 dd->default_desc1 = 1371 SDMA_DESC1_INT_REQ_FLAG; 1372 1373 if (!sdma_desct_intr) 1374 sdma_desct_intr = SDMA_DESC_INTR; 1375 1376 /* Allocate memory for SendDMA descriptor FIFOs */ 1377 for (this_idx = 0; this_idx < num_engines; ++this_idx) { 1378 sde = &dd->per_sdma[this_idx]; 1379 sde->dd = dd; 1380 sde->ppd = ppd; 1381 sde->this_idx = this_idx; 1382 sde->descq_cnt = descq_cnt; 1383 sde->desc_avail = sdma_descq_freecnt(sde); 1384 sde->sdma_shift = ilog2(descq_cnt); 1385 sde->sdma_mask = (1 << sde->sdma_shift) - 1; 1386 1387 /* Create a mask specifically for each interrupt source */ 1388 sde->int_mask = (u64)1 << (0 * TXE_NUM_SDMA_ENGINES + 1389 this_idx); 1390 sde->progress_mask = (u64)1 << (1 * TXE_NUM_SDMA_ENGINES + 1391 this_idx); 1392 sde->idle_mask = (u64)1 << (2 * TXE_NUM_SDMA_ENGINES + 1393 this_idx); 1394 /* Create a combined mask to cover all 3 interrupt sources */ 1395 sde->imask = sde->int_mask | sde->progress_mask | 1396 sde->idle_mask; 1397 1398 spin_lock_init(&sde->tail_lock); 1399 seqlock_init(&sde->head_lock); 1400 spin_lock_init(&sde->senddmactrl_lock); 1401 spin_lock_init(&sde->flushlist_lock); 1402 seqlock_init(&sde->waitlock); 1403 /* insure there is always a zero bit */ 1404 sde->ahg_bits = 0xfffffffe00000000ULL; 1405 1406 sdma_set_state(sde, sdma_state_s00_hw_down); 1407 1408 /* set up reference counting */ 1409 kref_init(&sde->state.kref); 1410 init_completion(&sde->state.comp); 1411 1412 INIT_LIST_HEAD(&sde->flushlist); 1413 INIT_LIST_HEAD(&sde->dmawait); 1414 1415 sde->tail_csr = 1416 get_kctxt_csr_addr(dd, this_idx, SD(TAIL)); 1417 1418 tasklet_setup(&sde->sdma_hw_clean_up_task, 1419 sdma_hw_clean_up_task); 1420 tasklet_setup(&sde->sdma_sw_clean_up_task, 1421 sdma_sw_clean_up_task); 1422 INIT_WORK(&sde->err_halt_worker, sdma_err_halt_wait); 1423 INIT_WORK(&sde->flush_worker, sdma_field_flush); 1424 1425 sde->progress_check_head = 0; 1426 1427 timer_setup(&sde->err_progress_check_timer, 1428 sdma_err_progress_check, 0); 1429 1430 sde->descq = dma_alloc_coherent(&dd->pcidev->dev, 1431 descq_cnt * sizeof(u64[2]), 1432 &sde->descq_phys, GFP_KERNEL); 1433 if (!sde->descq) 1434 goto bail; 1435 sde->tx_ring = 1436 kvzalloc_node(array_size(descq_cnt, 1437 sizeof(struct sdma_txreq *)), 1438 GFP_KERNEL, dd->node); 1439 if (!sde->tx_ring) 1440 goto bail; 1441 } 1442 1443 dd->sdma_heads_size = L1_CACHE_BYTES * num_engines; 1444 /* Allocate memory for DMA of head registers to memory */ 1445 dd->sdma_heads_dma = dma_alloc_coherent(&dd->pcidev->dev, 1446 dd->sdma_heads_size, 1447 &dd->sdma_heads_phys, 1448 GFP_KERNEL); 1449 if (!dd->sdma_heads_dma) { 1450 dd_dev_err(dd, "failed to allocate SendDMA head memory\n"); 1451 goto bail; 1452 } 1453 1454 /* Allocate memory for pad */ 1455 dd->sdma_pad_dma = dma_alloc_coherent(&dd->pcidev->dev, SDMA_PAD, 1456 &dd->sdma_pad_phys, GFP_KERNEL); 1457 if (!dd->sdma_pad_dma) { 1458 dd_dev_err(dd, "failed to allocate SendDMA pad memory\n"); 1459 goto bail; 1460 } 1461 1462 /* assign each engine to different cacheline and init registers */ 1463 curr_head = (void *)dd->sdma_heads_dma; 1464 for (this_idx = 0; this_idx < num_engines; ++this_idx) { 1465 unsigned long phys_offset; 1466 1467 sde = &dd->per_sdma[this_idx]; 1468 1469 sde->head_dma = curr_head; 1470 curr_head += L1_CACHE_BYTES; 1471 phys_offset = (unsigned long)sde->head_dma - 1472 (unsigned long)dd->sdma_heads_dma; 1473 sde->head_phys = dd->sdma_heads_phys + phys_offset; 1474 init_sdma_regs(sde, per_sdma_credits, idle_cnt); 1475 } 1476 dd->flags |= HFI1_HAS_SEND_DMA; 1477 dd->flags |= idle_cnt ? HFI1_HAS_SDMA_TIMEOUT : 0; 1478 dd->num_sdma = num_engines; 1479 ret = sdma_map_init(dd, port, ppd->vls_operational, NULL); 1480 if (ret < 0) 1481 goto bail; 1482 1483 tmp_sdma_rht = kzalloc(sizeof(*tmp_sdma_rht), GFP_KERNEL); 1484 if (!tmp_sdma_rht) { 1485 ret = -ENOMEM; 1486 goto bail; 1487 } 1488 1489 ret = rhashtable_init(tmp_sdma_rht, &sdma_rht_params); 1490 if (ret < 0) { 1491 kfree(tmp_sdma_rht); 1492 goto bail; 1493 } 1494 1495 dd->sdma_rht = tmp_sdma_rht; 1496 1497 dd_dev_info(dd, "SDMA num_sdma: %u\n", dd->num_sdma); 1498 return 0; 1499 1500 bail: 1501 sdma_clean(dd, num_engines); 1502 return ret; 1503 } 1504 1505 /** 1506 * sdma_all_running() - called when the link goes up 1507 * @dd: hfi1_devdata 1508 * 1509 * This routine moves all engines to the running state. 1510 */ 1511 void sdma_all_running(struct hfi1_devdata *dd) 1512 { 1513 struct sdma_engine *sde; 1514 unsigned int i; 1515 1516 /* move all engines to running */ 1517 for (i = 0; i < dd->num_sdma; ++i) { 1518 sde = &dd->per_sdma[i]; 1519 sdma_process_event(sde, sdma_event_e30_go_running); 1520 } 1521 } 1522 1523 /** 1524 * sdma_start() - called to kick off state processing for all engines 1525 * @dd: hfi1_devdata 1526 * 1527 * This routine is for kicking off the state processing for all required 1528 * sdma engines. Interrupts need to be working at this point. 1529 * 1530 */ 1531 void sdma_start(struct hfi1_devdata *dd) 1532 { 1533 unsigned i; 1534 struct sdma_engine *sde; 1535 1536 /* kick off the engines state processing */ 1537 for (i = 0; i < dd->num_sdma; ++i) { 1538 sde = &dd->per_sdma[i]; 1539 sdma_process_event(sde, sdma_event_e10_go_hw_start); 1540 } 1541 } 1542 1543 /** 1544 * sdma_exit() - used when module is removed 1545 * @dd: hfi1_devdata 1546 */ 1547 void sdma_exit(struct hfi1_devdata *dd) 1548 { 1549 unsigned this_idx; 1550 struct sdma_engine *sde; 1551 1552 for (this_idx = 0; dd->per_sdma && this_idx < dd->num_sdma; 1553 ++this_idx) { 1554 sde = &dd->per_sdma[this_idx]; 1555 if (!list_empty(&sde->dmawait)) 1556 dd_dev_err(dd, "sde %u: dmawait list not empty!\n", 1557 sde->this_idx); 1558 sdma_process_event(sde, sdma_event_e00_go_hw_down); 1559 1560 timer_delete_sync(&sde->err_progress_check_timer); 1561 1562 /* 1563 * This waits for the state machine to exit so it is not 1564 * necessary to kill the sdma_sw_clean_up_task to make sure 1565 * it is not running. 1566 */ 1567 sdma_finalput(&sde->state); 1568 } 1569 } 1570 1571 /* 1572 * unmap the indicated descriptor 1573 */ 1574 static inline void sdma_unmap_desc( 1575 struct hfi1_devdata *dd, 1576 struct sdma_desc *descp) 1577 { 1578 switch (sdma_mapping_type(descp)) { 1579 case SDMA_MAP_SINGLE: 1580 dma_unmap_single(&dd->pcidev->dev, sdma_mapping_addr(descp), 1581 sdma_mapping_len(descp), DMA_TO_DEVICE); 1582 break; 1583 case SDMA_MAP_PAGE: 1584 dma_unmap_page(&dd->pcidev->dev, sdma_mapping_addr(descp), 1585 sdma_mapping_len(descp), DMA_TO_DEVICE); 1586 break; 1587 } 1588 1589 if (descp->pinning_ctx && descp->ctx_put) 1590 descp->ctx_put(descp->pinning_ctx); 1591 descp->pinning_ctx = NULL; 1592 } 1593 1594 /* 1595 * return the mode as indicated by the first 1596 * descriptor in the tx. 1597 */ 1598 static inline u8 ahg_mode(struct sdma_txreq *tx) 1599 { 1600 return (tx->descp[0].qw[1] & SDMA_DESC1_HEADER_MODE_SMASK) 1601 >> SDMA_DESC1_HEADER_MODE_SHIFT; 1602 } 1603 1604 /** 1605 * __sdma_txclean() - clean tx of mappings, descp *kmalloc's 1606 * @dd: hfi1_devdata for unmapping 1607 * @tx: tx request to clean 1608 * 1609 * This is used in the progress routine to clean the tx or 1610 * by the ULP to toss an in-process tx build. 1611 * 1612 * The code can be called multiple times without issue. 1613 * 1614 */ 1615 void __sdma_txclean( 1616 struct hfi1_devdata *dd, 1617 struct sdma_txreq *tx) 1618 { 1619 u16 i; 1620 1621 if (tx->num_desc) { 1622 u8 skip = 0, mode = ahg_mode(tx); 1623 1624 /* unmap first */ 1625 sdma_unmap_desc(dd, &tx->descp[0]); 1626 /* determine number of AHG descriptors to skip */ 1627 if (mode > SDMA_AHG_APPLY_UPDATE1) 1628 skip = mode >> 1; 1629 for (i = 1 + skip; i < tx->num_desc; i++) 1630 sdma_unmap_desc(dd, &tx->descp[i]); 1631 tx->num_desc = 0; 1632 } 1633 kfree(tx->coalesce_buf); 1634 tx->coalesce_buf = NULL; 1635 /* kmalloc'ed descp */ 1636 if (unlikely(tx->desc_limit > ARRAY_SIZE(tx->descs))) { 1637 tx->desc_limit = ARRAY_SIZE(tx->descs); 1638 kfree(tx->descp); 1639 } 1640 } 1641 1642 static inline u16 sdma_gethead(struct sdma_engine *sde) 1643 { 1644 struct hfi1_devdata *dd = sde->dd; 1645 int use_dmahead; 1646 u16 hwhead; 1647 1648 #ifdef CONFIG_SDMA_VERBOSITY 1649 dd_dev_err(sde->dd, "CONFIG SDMA(%u) %s:%d %s()\n", 1650 sde->this_idx, slashstrip(__FILE__), __LINE__, __func__); 1651 #endif 1652 1653 retry: 1654 use_dmahead = HFI1_CAP_IS_KSET(USE_SDMA_HEAD) && __sdma_running(sde) && 1655 (dd->flags & HFI1_HAS_SDMA_TIMEOUT); 1656 hwhead = use_dmahead ? 1657 (u16)le64_to_cpu(*sde->head_dma) : 1658 (u16)read_sde_csr(sde, SD(HEAD)); 1659 1660 if (unlikely(HFI1_CAP_IS_KSET(SDMA_HEAD_CHECK))) { 1661 u16 cnt; 1662 u16 swtail; 1663 u16 swhead; 1664 int sane; 1665 1666 swhead = sde->descq_head & sde->sdma_mask; 1667 /* this code is really bad for cache line trading */ 1668 swtail = READ_ONCE(sde->descq_tail) & sde->sdma_mask; 1669 cnt = sde->descq_cnt; 1670 1671 if (swhead < swtail) 1672 /* not wrapped */ 1673 sane = (hwhead >= swhead) & (hwhead <= swtail); 1674 else if (swhead > swtail) 1675 /* wrapped around */ 1676 sane = ((hwhead >= swhead) && (hwhead < cnt)) || 1677 (hwhead <= swtail); 1678 else 1679 /* empty */ 1680 sane = (hwhead == swhead); 1681 1682 if (unlikely(!sane)) { 1683 dd_dev_err(dd, "SDMA(%u) bad head (%s) hwhd=%u swhd=%u swtl=%u cnt=%u\n", 1684 sde->this_idx, 1685 use_dmahead ? "dma" : "kreg", 1686 hwhead, swhead, swtail, cnt); 1687 if (use_dmahead) { 1688 /* try one more time, using csr */ 1689 use_dmahead = 0; 1690 goto retry; 1691 } 1692 /* proceed as if no progress */ 1693 hwhead = swhead; 1694 } 1695 } 1696 return hwhead; 1697 } 1698 1699 /* 1700 * This is called when there are send DMA descriptors that might be 1701 * available. 1702 * 1703 * This is called with head_lock held. 1704 */ 1705 static void sdma_desc_avail(struct sdma_engine *sde, uint avail) 1706 { 1707 struct iowait *wait, *nw, *twait; 1708 struct iowait *waits[SDMA_WAIT_BATCH_SIZE]; 1709 uint i, n = 0, seq, tidx = 0; 1710 1711 #ifdef CONFIG_SDMA_VERBOSITY 1712 dd_dev_err(sde->dd, "CONFIG SDMA(%u) %s:%d %s()\n", sde->this_idx, 1713 slashstrip(__FILE__), __LINE__, __func__); 1714 dd_dev_err(sde->dd, "avail: %u\n", avail); 1715 #endif 1716 1717 do { 1718 seq = read_seqbegin(&sde->waitlock); 1719 if (!list_empty(&sde->dmawait)) { 1720 /* at least one item */ 1721 write_seqlock(&sde->waitlock); 1722 /* Harvest waiters wanting DMA descriptors */ 1723 list_for_each_entry_safe( 1724 wait, 1725 nw, 1726 &sde->dmawait, 1727 list) { 1728 u32 num_desc; 1729 1730 if (!wait->wakeup) 1731 continue; 1732 if (n == ARRAY_SIZE(waits)) 1733 break; 1734 iowait_init_priority(wait); 1735 num_desc = iowait_get_all_desc(wait); 1736 if (num_desc > avail) 1737 break; 1738 avail -= num_desc; 1739 /* Find the top-priority wait memeber */ 1740 if (n) { 1741 twait = waits[tidx]; 1742 tidx = 1743 iowait_priority_update_top(wait, 1744 twait, 1745 n, 1746 tidx); 1747 } 1748 list_del_init(&wait->list); 1749 waits[n++] = wait; 1750 } 1751 write_sequnlock(&sde->waitlock); 1752 break; 1753 } 1754 } while (read_seqretry(&sde->waitlock, seq)); 1755 1756 /* Schedule the top-priority entry first */ 1757 if (n) 1758 waits[tidx]->wakeup(waits[tidx], SDMA_AVAIL_REASON); 1759 1760 for (i = 0; i < n; i++) 1761 if (i != tidx) 1762 waits[i]->wakeup(waits[i], SDMA_AVAIL_REASON); 1763 } 1764 1765 /* head_lock must be held */ 1766 static void sdma_make_progress(struct sdma_engine *sde, u64 status) 1767 { 1768 struct sdma_txreq *txp = NULL; 1769 int progress = 0; 1770 u16 hwhead, swhead; 1771 int idle_check_done = 0; 1772 1773 hwhead = sdma_gethead(sde); 1774 1775 /* The reason for some of the complexity of this code is that 1776 * not all descriptors have corresponding txps. So, we have to 1777 * be able to skip over descs until we wander into the range of 1778 * the next txp on the list. 1779 */ 1780 1781 retry: 1782 txp = get_txhead(sde); 1783 swhead = sde->descq_head & sde->sdma_mask; 1784 trace_hfi1_sdma_progress(sde, hwhead, swhead, txp); 1785 while (swhead != hwhead) { 1786 /* advance head, wrap if needed */ 1787 swhead = ++sde->descq_head & sde->sdma_mask; 1788 1789 /* if now past this txp's descs, do the callback */ 1790 if (txp && txp->next_descq_idx == swhead) { 1791 /* remove from list */ 1792 sde->tx_ring[sde->tx_head++ & sde->sdma_mask] = NULL; 1793 complete_tx(sde, txp, SDMA_TXREQ_S_OK); 1794 /* see if there is another txp */ 1795 txp = get_txhead(sde); 1796 } 1797 trace_hfi1_sdma_progress(sde, hwhead, swhead, txp); 1798 progress++; 1799 } 1800 1801 /* 1802 * The SDMA idle interrupt is not guaranteed to be ordered with respect 1803 * to updates to the dma_head location in host memory. The head 1804 * value read might not be fully up to date. If there are pending 1805 * descriptors and the SDMA idle interrupt fired then read from the 1806 * CSR SDMA head instead to get the latest value from the hardware. 1807 * The hardware SDMA head should be read at most once in this invocation 1808 * of sdma_make_progress(..) which is ensured by idle_check_done flag 1809 */ 1810 if ((status & sde->idle_mask) && !idle_check_done) { 1811 u16 swtail; 1812 1813 swtail = READ_ONCE(sde->descq_tail) & sde->sdma_mask; 1814 if (swtail != hwhead) { 1815 hwhead = (u16)read_sde_csr(sde, SD(HEAD)); 1816 idle_check_done = 1; 1817 goto retry; 1818 } 1819 } 1820 1821 sde->last_status = status; 1822 if (progress) 1823 sdma_desc_avail(sde, sdma_descq_freecnt(sde)); 1824 } 1825 1826 /* 1827 * sdma_engine_interrupt() - interrupt handler for engine 1828 * @sde: sdma engine 1829 * @status: sdma interrupt reason 1830 * 1831 * Status is a mask of the 3 possible interrupts for this engine. It will 1832 * contain bits _only_ for this SDMA engine. It will contain at least one 1833 * bit, it may contain more. 1834 */ 1835 void sdma_engine_interrupt(struct sdma_engine *sde, u64 status) 1836 { 1837 trace_hfi1_sdma_engine_interrupt(sde, status); 1838 write_seqlock(&sde->head_lock); 1839 sdma_set_desc_cnt(sde, sdma_desct_intr); 1840 if (status & sde->idle_mask) 1841 sde->idle_int_cnt++; 1842 else if (status & sde->progress_mask) 1843 sde->progress_int_cnt++; 1844 else if (status & sde->int_mask) 1845 sde->sdma_int_cnt++; 1846 sdma_make_progress(sde, status); 1847 write_sequnlock(&sde->head_lock); 1848 } 1849 1850 /** 1851 * sdma_engine_error() - error handler for engine 1852 * @sde: sdma engine 1853 * @status: sdma interrupt reason 1854 */ 1855 void sdma_engine_error(struct sdma_engine *sde, u64 status) 1856 { 1857 unsigned long flags; 1858 1859 #ifdef CONFIG_SDMA_VERBOSITY 1860 dd_dev_err(sde->dd, "CONFIG SDMA(%u) error status 0x%llx state %s\n", 1861 sde->this_idx, 1862 (unsigned long long)status, 1863 sdma_state_names[sde->state.current_state]); 1864 #endif 1865 spin_lock_irqsave(&sde->tail_lock, flags); 1866 write_seqlock(&sde->head_lock); 1867 if (status & ALL_SDMA_ENG_HALT_ERRS) 1868 __sdma_process_event(sde, sdma_event_e60_hw_halted); 1869 if (status & ~SD(ENG_ERR_STATUS_SDMA_HALT_ERR_SMASK)) { 1870 dd_dev_err(sde->dd, 1871 "SDMA (%u) engine error: 0x%llx state %s\n", 1872 sde->this_idx, 1873 (unsigned long long)status, 1874 sdma_state_names[sde->state.current_state]); 1875 dump_sdma_state(sde); 1876 } 1877 write_sequnlock(&sde->head_lock); 1878 spin_unlock_irqrestore(&sde->tail_lock, flags); 1879 } 1880 1881 static void sdma_sendctrl(struct sdma_engine *sde, unsigned op) 1882 { 1883 u64 set_senddmactrl = 0; 1884 u64 clr_senddmactrl = 0; 1885 unsigned long flags; 1886 1887 #ifdef CONFIG_SDMA_VERBOSITY 1888 dd_dev_err(sde->dd, "CONFIG SDMA(%u) senddmactrl E=%d I=%d H=%d C=%d\n", 1889 sde->this_idx, 1890 (op & SDMA_SENDCTRL_OP_ENABLE) ? 1 : 0, 1891 (op & SDMA_SENDCTRL_OP_INTENABLE) ? 1 : 0, 1892 (op & SDMA_SENDCTRL_OP_HALT) ? 1 : 0, 1893 (op & SDMA_SENDCTRL_OP_CLEANUP) ? 1 : 0); 1894 #endif 1895 1896 if (op & SDMA_SENDCTRL_OP_ENABLE) 1897 set_senddmactrl |= SD(CTRL_SDMA_ENABLE_SMASK); 1898 else 1899 clr_senddmactrl |= SD(CTRL_SDMA_ENABLE_SMASK); 1900 1901 if (op & SDMA_SENDCTRL_OP_INTENABLE) 1902 set_senddmactrl |= SD(CTRL_SDMA_INT_ENABLE_SMASK); 1903 else 1904 clr_senddmactrl |= SD(CTRL_SDMA_INT_ENABLE_SMASK); 1905 1906 if (op & SDMA_SENDCTRL_OP_HALT) 1907 set_senddmactrl |= SD(CTRL_SDMA_HALT_SMASK); 1908 else 1909 clr_senddmactrl |= SD(CTRL_SDMA_HALT_SMASK); 1910 1911 spin_lock_irqsave(&sde->senddmactrl_lock, flags); 1912 1913 sde->p_senddmactrl |= set_senddmactrl; 1914 sde->p_senddmactrl &= ~clr_senddmactrl; 1915 1916 if (op & SDMA_SENDCTRL_OP_CLEANUP) 1917 write_sde_csr(sde, SD(CTRL), 1918 sde->p_senddmactrl | 1919 SD(CTRL_SDMA_CLEANUP_SMASK)); 1920 else 1921 write_sde_csr(sde, SD(CTRL), sde->p_senddmactrl); 1922 1923 spin_unlock_irqrestore(&sde->senddmactrl_lock, flags); 1924 1925 #ifdef CONFIG_SDMA_VERBOSITY 1926 sdma_dumpstate(sde); 1927 #endif 1928 } 1929 1930 static void sdma_setlengen(struct sdma_engine *sde) 1931 { 1932 #ifdef CONFIG_SDMA_VERBOSITY 1933 dd_dev_err(sde->dd, "CONFIG SDMA(%u) %s:%d %s()\n", 1934 sde->this_idx, slashstrip(__FILE__), __LINE__, __func__); 1935 #endif 1936 1937 /* 1938 * Set SendDmaLenGen and clear-then-set the MSB of the generation 1939 * count to enable generation checking and load the internal 1940 * generation counter. 1941 */ 1942 write_sde_csr(sde, SD(LEN_GEN), 1943 (sde->descq_cnt / 64) << SD(LEN_GEN_LENGTH_SHIFT)); 1944 write_sde_csr(sde, SD(LEN_GEN), 1945 ((sde->descq_cnt / 64) << SD(LEN_GEN_LENGTH_SHIFT)) | 1946 (4ULL << SD(LEN_GEN_GENERATION_SHIFT))); 1947 } 1948 1949 static inline void sdma_update_tail(struct sdma_engine *sde, u16 tail) 1950 { 1951 /* Commit writes to memory and advance the tail on the chip */ 1952 smp_wmb(); /* see get_txhead() */ 1953 writeq(tail, sde->tail_csr); 1954 } 1955 1956 /* 1957 * This is called when changing to state s10_hw_start_up_halt_wait as 1958 * a result of send buffer errors or send DMA descriptor errors. 1959 */ 1960 static void sdma_hw_start_up(struct sdma_engine *sde) 1961 { 1962 u64 reg; 1963 1964 #ifdef CONFIG_SDMA_VERBOSITY 1965 dd_dev_err(sde->dd, "CONFIG SDMA(%u) %s:%d %s()\n", 1966 sde->this_idx, slashstrip(__FILE__), __LINE__, __func__); 1967 #endif 1968 1969 sdma_setlengen(sde); 1970 sdma_update_tail(sde, 0); /* Set SendDmaTail */ 1971 *sde->head_dma = 0; 1972 1973 reg = SD(ENG_ERR_CLEAR_SDMA_HEADER_REQUEST_FIFO_UNC_ERR_MASK) << 1974 SD(ENG_ERR_CLEAR_SDMA_HEADER_REQUEST_FIFO_UNC_ERR_SHIFT); 1975 write_sde_csr(sde, SD(ENG_ERR_CLEAR), reg); 1976 } 1977 1978 /* 1979 * set_sdma_integrity 1980 * 1981 * Set the SEND_DMA_CHECK_ENABLE register for send DMA engine 'sde'. 1982 */ 1983 static void set_sdma_integrity(struct sdma_engine *sde) 1984 { 1985 struct hfi1_devdata *dd = sde->dd; 1986 1987 write_sde_csr(sde, SD(CHECK_ENABLE), 1988 hfi1_pkt_base_sdma_integrity(dd)); 1989 } 1990 1991 static void init_sdma_regs( 1992 struct sdma_engine *sde, 1993 u32 credits, 1994 uint idle_cnt) 1995 { 1996 u8 opval, opmask; 1997 #ifdef CONFIG_SDMA_VERBOSITY 1998 struct hfi1_devdata *dd = sde->dd; 1999 2000 dd_dev_err(dd, "CONFIG SDMA(%u) %s:%d %s()\n", 2001 sde->this_idx, slashstrip(__FILE__), __LINE__, __func__); 2002 #endif 2003 2004 write_sde_csr(sde, SD(BASE_ADDR), sde->descq_phys); 2005 sdma_setlengen(sde); 2006 sdma_update_tail(sde, 0); /* Set SendDmaTail */ 2007 write_sde_csr(sde, SD(RELOAD_CNT), idle_cnt); 2008 write_sde_csr(sde, SD(DESC_CNT), 0); 2009 write_sde_csr(sde, SD(HEAD_ADDR), sde->head_phys); 2010 write_sde_csr(sde, SD(MEMORY), 2011 ((u64)credits << SD(MEMORY_SDMA_MEMORY_CNT_SHIFT)) | 2012 ((u64)(credits * sde->this_idx) << 2013 SD(MEMORY_SDMA_MEMORY_INDEX_SHIFT))); 2014 write_sde_csr(sde, SD(ENG_ERR_MASK), ~0ull); 2015 set_sdma_integrity(sde); 2016 opmask = OPCODE_CHECK_MASK_DISABLED; 2017 opval = OPCODE_CHECK_VAL_DISABLED; 2018 write_sde_csr(sde, SD(CHECK_OPCODE), 2019 (opmask << SEND_CTXT_CHECK_OPCODE_MASK_SHIFT) | 2020 (opval << SEND_CTXT_CHECK_OPCODE_VALUE_SHIFT)); 2021 } 2022 2023 #ifdef CONFIG_SDMA_VERBOSITY 2024 2025 #define sdma_dumpstate_helper0(reg) do { \ 2026 csr = read_csr(sde->dd, reg); \ 2027 dd_dev_err(sde->dd, "%36s 0x%016llx\n", #reg, csr); \ 2028 } while (0) 2029 2030 #define sdma_dumpstate_helper(reg) do { \ 2031 csr = read_sde_csr(sde, reg); \ 2032 dd_dev_err(sde->dd, "%36s[%02u] 0x%016llx\n", \ 2033 #reg, sde->this_idx, csr); \ 2034 } while (0) 2035 2036 #define sdma_dumpstate_helper2(reg) do { \ 2037 csr = read_csr(sde->dd, reg + (8 * i)); \ 2038 dd_dev_err(sde->dd, "%33s_%02u 0x%016llx\n", \ 2039 #reg, i, csr); \ 2040 } while (0) 2041 2042 void sdma_dumpstate(struct sdma_engine *sde) 2043 { 2044 u64 csr; 2045 unsigned i; 2046 2047 sdma_dumpstate_helper(SD(CTRL)); 2048 sdma_dumpstate_helper(SD(STATUS)); 2049 sdma_dumpstate_helper0(SD(ERR_STATUS)); 2050 sdma_dumpstate_helper0(SD(ERR_MASK)); 2051 sdma_dumpstate_helper(SD(ENG_ERR_STATUS)); 2052 sdma_dumpstate_helper(SD(ENG_ERR_MASK)); 2053 2054 for (i = 0; i < CCE_NUM_INT_CSRS; ++i) { 2055 sdma_dumpstate_helper2(CCE_INT_STATUS); 2056 sdma_dumpstate_helper2(CCE_INT_MASK); 2057 sdma_dumpstate_helper2(CCE_INT_BLOCKED); 2058 } 2059 2060 sdma_dumpstate_helper(SD(TAIL)); 2061 sdma_dumpstate_helper(SD(HEAD)); 2062 sdma_dumpstate_helper(SD(PRIORITY_THLD)); 2063 sdma_dumpstate_helper(SD(IDLE_CNT)); 2064 sdma_dumpstate_helper(SD(RELOAD_CNT)); 2065 sdma_dumpstate_helper(SD(DESC_CNT)); 2066 sdma_dumpstate_helper(SD(DESC_FETCHED_CNT)); 2067 sdma_dumpstate_helper(SD(MEMORY)); 2068 sdma_dumpstate_helper0(SD(ENGINES)); 2069 sdma_dumpstate_helper0(SD(MEM_SIZE)); 2070 /* sdma_dumpstate_helper(SEND_EGRESS_SEND_DMA_STATUS); */ 2071 sdma_dumpstate_helper(SD(BASE_ADDR)); 2072 sdma_dumpstate_helper(SD(LEN_GEN)); 2073 sdma_dumpstate_helper(SD(HEAD_ADDR)); 2074 sdma_dumpstate_helper(SD(CHECK_ENABLE)); 2075 sdma_dumpstate_helper(SD(CHECK_VL)); 2076 sdma_dumpstate_helper(SD(CHECK_JOB_KEY)); 2077 sdma_dumpstate_helper(SD(CHECK_PARTITION_KEY)); 2078 sdma_dumpstate_helper(SD(CHECK_SLID)); 2079 sdma_dumpstate_helper(SD(CHECK_OPCODE)); 2080 } 2081 #endif 2082 2083 static void dump_sdma_state(struct sdma_engine *sde) 2084 { 2085 struct hw_sdma_desc *descqp; 2086 u64 desc[2]; 2087 u64 addr; 2088 u8 gen; 2089 u16 len; 2090 u16 head, tail, cnt; 2091 2092 head = sde->descq_head & sde->sdma_mask; 2093 tail = sde->descq_tail & sde->sdma_mask; 2094 cnt = sdma_descq_freecnt(sde); 2095 2096 dd_dev_err(sde->dd, 2097 "SDMA (%u) descq_head: %u descq_tail: %u freecnt: %u FLE %d\n", 2098 sde->this_idx, head, tail, cnt, 2099 !list_empty(&sde->flushlist)); 2100 2101 /* print info for each entry in the descriptor queue */ 2102 while (head != tail) { 2103 char flags[6] = { 'x', 'x', 'x', 'x', 0 }; 2104 2105 descqp = &sde->descq[head]; 2106 desc[0] = le64_to_cpu(descqp->qw[0]); 2107 desc[1] = le64_to_cpu(descqp->qw[1]); 2108 flags[0] = (desc[1] & SDMA_DESC1_INT_REQ_FLAG) ? 'I' : '-'; 2109 flags[1] = (desc[1] & SDMA_DESC1_HEAD_TO_HOST_FLAG) ? 2110 'H' : '-'; 2111 flags[2] = (desc[0] & SDMA_DESC0_FIRST_DESC_FLAG) ? 'F' : '-'; 2112 flags[3] = (desc[0] & SDMA_DESC0_LAST_DESC_FLAG) ? 'L' : '-'; 2113 addr = (desc[0] >> SDMA_DESC0_PHY_ADDR_SHIFT) 2114 & SDMA_DESC0_PHY_ADDR_MASK; 2115 gen = (desc[1] >> SDMA_DESC1_GENERATION_SHIFT) 2116 & SDMA_DESC1_GENERATION_MASK; 2117 len = (desc[0] >> SDMA_DESC0_BYTE_COUNT_SHIFT) 2118 & SDMA_DESC0_BYTE_COUNT_MASK; 2119 dd_dev_err(sde->dd, 2120 "SDMA sdmadesc[%u]: flags:%s addr:0x%016llx gen:%u len:%u bytes\n", 2121 head, flags, addr, gen, len); 2122 dd_dev_err(sde->dd, 2123 "\tdesc0:0x%016llx desc1 0x%016llx\n", 2124 desc[0], desc[1]); 2125 if (desc[0] & SDMA_DESC0_FIRST_DESC_FLAG) 2126 dd_dev_err(sde->dd, 2127 "\taidx: %u amode: %u alen: %u\n", 2128 (u8)((desc[1] & 2129 SDMA_DESC1_HEADER_INDEX_SMASK) >> 2130 SDMA_DESC1_HEADER_INDEX_SHIFT), 2131 (u8)((desc[1] & 2132 SDMA_DESC1_HEADER_MODE_SMASK) >> 2133 SDMA_DESC1_HEADER_MODE_SHIFT), 2134 (u8)((desc[1] & 2135 SDMA_DESC1_HEADER_DWS_SMASK) >> 2136 SDMA_DESC1_HEADER_DWS_SHIFT)); 2137 head++; 2138 head &= sde->sdma_mask; 2139 } 2140 } 2141 2142 #define SDE_FMT \ 2143 "SDE %u CPU %d STE %s C 0x%llx S 0x%016llx E 0x%llx T(HW) 0x%llx T(SW) 0x%x H(HW) 0x%llx H(SW) 0x%x H(D) 0x%llx DM 0x%llx GL 0x%llx R 0x%llx LIS 0x%llx AHGI 0x%llx TXT %u TXH %u DT %u DH %u FLNE %d DQF %u SLC 0x%llx\n" 2144 /** 2145 * sdma_seqfile_dump_sde() - debugfs dump of sde 2146 * @s: seq file 2147 * @sde: send dma engine to dump 2148 * 2149 * This routine dumps the sde to the indicated seq file. 2150 */ 2151 void sdma_seqfile_dump_sde(struct seq_file *s, struct sdma_engine *sde) 2152 { 2153 u16 head, tail; 2154 struct hw_sdma_desc *descqp; 2155 u64 desc[2]; 2156 u64 addr; 2157 u8 gen; 2158 u16 len; 2159 2160 head = sde->descq_head & sde->sdma_mask; 2161 tail = READ_ONCE(sde->descq_tail) & sde->sdma_mask; 2162 seq_printf(s, SDE_FMT, sde->this_idx, 2163 sde->cpu, 2164 sdma_state_name(sde->state.current_state), 2165 (unsigned long long)read_sde_csr(sde, SD(CTRL)), 2166 (unsigned long long)read_sde_csr(sde, SD(STATUS)), 2167 (unsigned long long)read_sde_csr(sde, SD(ENG_ERR_STATUS)), 2168 (unsigned long long)read_sde_csr(sde, SD(TAIL)), tail, 2169 (unsigned long long)read_sde_csr(sde, SD(HEAD)), head, 2170 (unsigned long long)le64_to_cpu(*sde->head_dma), 2171 (unsigned long long)read_sde_csr(sde, SD(MEMORY)), 2172 (unsigned long long)read_sde_csr(sde, SD(LEN_GEN)), 2173 (unsigned long long)read_sde_csr(sde, SD(RELOAD_CNT)), 2174 (unsigned long long)sde->last_status, 2175 (unsigned long long)sde->ahg_bits, 2176 sde->tx_tail, 2177 sde->tx_head, 2178 sde->descq_tail, 2179 sde->descq_head, 2180 !list_empty(&sde->flushlist), 2181 sde->descq_full_count, 2182 (unsigned long long)read_sde_csr(sde, SEND_DMA_CHECK_SLID)); 2183 2184 /* print info for each entry in the descriptor queue */ 2185 while (head != tail) { 2186 char flags[6] = { 'x', 'x', 'x', 'x', 0 }; 2187 2188 descqp = &sde->descq[head]; 2189 desc[0] = le64_to_cpu(descqp->qw[0]); 2190 desc[1] = le64_to_cpu(descqp->qw[1]); 2191 flags[0] = (desc[1] & SDMA_DESC1_INT_REQ_FLAG) ? 'I' : '-'; 2192 flags[1] = (desc[1] & SDMA_DESC1_HEAD_TO_HOST_FLAG) ? 2193 'H' : '-'; 2194 flags[2] = (desc[0] & SDMA_DESC0_FIRST_DESC_FLAG) ? 'F' : '-'; 2195 flags[3] = (desc[0] & SDMA_DESC0_LAST_DESC_FLAG) ? 'L' : '-'; 2196 addr = (desc[0] >> SDMA_DESC0_PHY_ADDR_SHIFT) 2197 & SDMA_DESC0_PHY_ADDR_MASK; 2198 gen = (desc[1] >> SDMA_DESC1_GENERATION_SHIFT) 2199 & SDMA_DESC1_GENERATION_MASK; 2200 len = (desc[0] >> SDMA_DESC0_BYTE_COUNT_SHIFT) 2201 & SDMA_DESC0_BYTE_COUNT_MASK; 2202 seq_printf(s, 2203 "\tdesc[%u]: flags:%s addr:0x%016llx gen:%u len:%u bytes\n", 2204 head, flags, addr, gen, len); 2205 if (desc[0] & SDMA_DESC0_FIRST_DESC_FLAG) 2206 seq_printf(s, "\t\tahgidx: %u ahgmode: %u\n", 2207 (u8)((desc[1] & 2208 SDMA_DESC1_HEADER_INDEX_SMASK) >> 2209 SDMA_DESC1_HEADER_INDEX_SHIFT), 2210 (u8)((desc[1] & 2211 SDMA_DESC1_HEADER_MODE_SMASK) >> 2212 SDMA_DESC1_HEADER_MODE_SHIFT)); 2213 head = (head + 1) & sde->sdma_mask; 2214 } 2215 } 2216 2217 /* 2218 * add the generation number into 2219 * the qw1 and return 2220 */ 2221 static inline u64 add_gen(struct sdma_engine *sde, u64 qw1) 2222 { 2223 u8 generation = (sde->descq_tail >> sde->sdma_shift) & 3; 2224 2225 qw1 &= ~SDMA_DESC1_GENERATION_SMASK; 2226 qw1 |= ((u64)generation & SDMA_DESC1_GENERATION_MASK) 2227 << SDMA_DESC1_GENERATION_SHIFT; 2228 return qw1; 2229 } 2230 2231 /* 2232 * This routine submits the indicated tx 2233 * 2234 * Space has already been guaranteed and 2235 * tail side of ring is locked. 2236 * 2237 * The hardware tail update is done 2238 * in the caller and that is facilitated 2239 * by returning the new tail. 2240 * 2241 * There is special case logic for ahg 2242 * to not add the generation number for 2243 * up to 2 descriptors that follow the 2244 * first descriptor. 2245 * 2246 */ 2247 static inline u16 submit_tx(struct sdma_engine *sde, struct sdma_txreq *tx) 2248 { 2249 int i; 2250 u16 tail; 2251 struct sdma_desc *descp = tx->descp; 2252 u8 skip = 0, mode = ahg_mode(tx); 2253 2254 tail = sde->descq_tail & sde->sdma_mask; 2255 sde->descq[tail].qw[0] = cpu_to_le64(descp->qw[0]); 2256 sde->descq[tail].qw[1] = cpu_to_le64(add_gen(sde, descp->qw[1])); 2257 trace_hfi1_sdma_descriptor(sde, descp->qw[0], descp->qw[1], 2258 tail, &sde->descq[tail]); 2259 tail = ++sde->descq_tail & sde->sdma_mask; 2260 descp++; 2261 if (mode > SDMA_AHG_APPLY_UPDATE1) 2262 skip = mode >> 1; 2263 for (i = 1; i < tx->num_desc; i++, descp++) { 2264 u64 qw1; 2265 2266 sde->descq[tail].qw[0] = cpu_to_le64(descp->qw[0]); 2267 if (skip) { 2268 /* edits don't have generation */ 2269 qw1 = descp->qw[1]; 2270 skip--; 2271 } else { 2272 /* replace generation with real one for non-edits */ 2273 qw1 = add_gen(sde, descp->qw[1]); 2274 } 2275 sde->descq[tail].qw[1] = cpu_to_le64(qw1); 2276 trace_hfi1_sdma_descriptor(sde, descp->qw[0], qw1, 2277 tail, &sde->descq[tail]); 2278 tail = ++sde->descq_tail & sde->sdma_mask; 2279 } 2280 tx->next_descq_idx = tail; 2281 #ifdef CONFIG_HFI1_DEBUG_SDMA_ORDER 2282 tx->sn = sde->tail_sn++; 2283 trace_hfi1_sdma_in_sn(sde, tx->sn); 2284 WARN_ON_ONCE(sde->tx_ring[sde->tx_tail & sde->sdma_mask]); 2285 #endif 2286 sde->tx_ring[sde->tx_tail++ & sde->sdma_mask] = tx; 2287 sde->desc_avail -= tx->num_desc; 2288 return tail; 2289 } 2290 2291 /* 2292 * Check for progress 2293 */ 2294 static int sdma_check_progress( 2295 struct sdma_engine *sde, 2296 struct iowait_work *wait, 2297 struct sdma_txreq *tx, 2298 bool pkts_sent) 2299 { 2300 int ret; 2301 2302 sde->desc_avail = sdma_descq_freecnt(sde); 2303 if (tx->num_desc <= sde->desc_avail) 2304 return -EAGAIN; 2305 /* pulse the head_lock */ 2306 if (wait && iowait_ioww_to_iow(wait)->sleep) { 2307 unsigned seq; 2308 2309 seq = raw_seqcount_begin( 2310 (const seqcount_t *)&sde->head_lock.seqcount); 2311 ret = wait->iow->sleep(sde, wait, tx, seq, pkts_sent); 2312 if (ret == -EAGAIN) 2313 sde->desc_avail = sdma_descq_freecnt(sde); 2314 } else { 2315 ret = -EBUSY; 2316 } 2317 return ret; 2318 } 2319 2320 /** 2321 * sdma_send_txreq() - submit a tx req to ring 2322 * @sde: sdma engine to use 2323 * @wait: SE wait structure to use when full (may be NULL) 2324 * @tx: sdma_txreq to submit 2325 * @pkts_sent: has any packet been sent yet? 2326 * 2327 * The call submits the tx into the ring. If a iowait structure is non-NULL 2328 * the packet will be queued to the list in wait. 2329 * 2330 * Return: 2331 * 0 - Success, -EINVAL - sdma_txreq incomplete, -EBUSY - no space in 2332 * ring (wait == NULL) 2333 * -EIOCBQUEUED - tx queued to iowait, -ECOMM bad sdma state 2334 */ 2335 int sdma_send_txreq(struct sdma_engine *sde, 2336 struct iowait_work *wait, 2337 struct sdma_txreq *tx, 2338 bool pkts_sent) 2339 { 2340 int ret = 0; 2341 u16 tail; 2342 unsigned long flags; 2343 2344 /* user should have supplied entire packet */ 2345 if (unlikely(tx->tlen)) 2346 return -EINVAL; 2347 tx->wait = iowait_ioww_to_iow(wait); 2348 spin_lock_irqsave(&sde->tail_lock, flags); 2349 retry: 2350 if (unlikely(!__sdma_running(sde))) 2351 goto unlock_noconn; 2352 if (unlikely(tx->num_desc > sde->desc_avail)) 2353 goto nodesc; 2354 tail = submit_tx(sde, tx); 2355 if (wait) 2356 iowait_sdma_inc(iowait_ioww_to_iow(wait)); 2357 sdma_update_tail(sde, tail); 2358 unlock: 2359 spin_unlock_irqrestore(&sde->tail_lock, flags); 2360 return ret; 2361 unlock_noconn: 2362 if (wait) 2363 iowait_sdma_inc(iowait_ioww_to_iow(wait)); 2364 tx->next_descq_idx = 0; 2365 #ifdef CONFIG_HFI1_DEBUG_SDMA_ORDER 2366 tx->sn = sde->tail_sn++; 2367 trace_hfi1_sdma_in_sn(sde, tx->sn); 2368 #endif 2369 spin_lock(&sde->flushlist_lock); 2370 list_add_tail(&tx->list, &sde->flushlist); 2371 spin_unlock(&sde->flushlist_lock); 2372 iowait_inc_wait_count(wait, tx->num_desc); 2373 queue_work_on(sde->cpu, system_highpri_wq, &sde->flush_worker); 2374 ret = -ECOMM; 2375 goto unlock; 2376 nodesc: 2377 ret = sdma_check_progress(sde, wait, tx, pkts_sent); 2378 if (ret == -EAGAIN) { 2379 ret = 0; 2380 goto retry; 2381 } 2382 sde->descq_full_count++; 2383 goto unlock; 2384 } 2385 2386 /** 2387 * sdma_send_txlist() - submit a list of tx req to ring 2388 * @sde: sdma engine to use 2389 * @wait: SE wait structure to use when full (may be NULL) 2390 * @tx_list: list of sdma_txreqs to submit 2391 * @count_out: pointer to a u16 which, after return will contain the total number of 2392 * sdma_txreqs removed from the tx_list. This will include sdma_txreqs 2393 * whose SDMA descriptors are submitted to the ring and the sdma_txreqs 2394 * which are added to SDMA engine flush list if the SDMA engine state is 2395 * not running. 2396 * 2397 * The call submits the list into the ring. 2398 * 2399 * If the iowait structure is non-NULL and not equal to the iowait list 2400 * the unprocessed part of the list will be appended to the list in wait. 2401 * 2402 * In all cases, the tx_list will be updated so the head of the tx_list is 2403 * the list of descriptors that have yet to be transmitted. 2404 * 2405 * The intent of this call is to provide a more efficient 2406 * way of submitting multiple packets to SDMA while holding the tail 2407 * side locking. 2408 * 2409 * Return: 2410 * 0 - Success, 2411 * -EINVAL - sdma_txreq incomplete, -EBUSY - no space in ring (wait == NULL) 2412 * -EIOCBQUEUED - tx queued to iowait, -ECOMM bad sdma state 2413 */ 2414 int sdma_send_txlist(struct sdma_engine *sde, struct iowait_work *wait, 2415 struct list_head *tx_list, u16 *count_out) 2416 { 2417 struct sdma_txreq *tx, *tx_next; 2418 int ret = 0; 2419 unsigned long flags; 2420 u16 tail = INVALID_TAIL; 2421 u32 submit_count = 0, flush_count = 0, total_count; 2422 2423 spin_lock_irqsave(&sde->tail_lock, flags); 2424 retry: 2425 list_for_each_entry_safe(tx, tx_next, tx_list, list) { 2426 tx->wait = iowait_ioww_to_iow(wait); 2427 if (unlikely(!__sdma_running(sde))) 2428 goto unlock_noconn; 2429 if (unlikely(tx->num_desc > sde->desc_avail)) 2430 goto nodesc; 2431 if (unlikely(tx->tlen)) { 2432 ret = -EINVAL; 2433 goto update_tail; 2434 } 2435 list_del_init(&tx->list); 2436 tail = submit_tx(sde, tx); 2437 submit_count++; 2438 if (tail != INVALID_TAIL && 2439 (submit_count & SDMA_TAIL_UPDATE_THRESH) == 0) { 2440 sdma_update_tail(sde, tail); 2441 tail = INVALID_TAIL; 2442 } 2443 } 2444 update_tail: 2445 total_count = submit_count + flush_count; 2446 if (wait) { 2447 iowait_sdma_add(iowait_ioww_to_iow(wait), total_count); 2448 iowait_starve_clear(submit_count > 0, 2449 iowait_ioww_to_iow(wait)); 2450 } 2451 if (tail != INVALID_TAIL) 2452 sdma_update_tail(sde, tail); 2453 spin_unlock_irqrestore(&sde->tail_lock, flags); 2454 *count_out = total_count; 2455 return ret; 2456 unlock_noconn: 2457 spin_lock(&sde->flushlist_lock); 2458 list_for_each_entry_safe(tx, tx_next, tx_list, list) { 2459 tx->wait = iowait_ioww_to_iow(wait); 2460 list_del_init(&tx->list); 2461 tx->next_descq_idx = 0; 2462 #ifdef CONFIG_HFI1_DEBUG_SDMA_ORDER 2463 tx->sn = sde->tail_sn++; 2464 trace_hfi1_sdma_in_sn(sde, tx->sn); 2465 #endif 2466 list_add_tail(&tx->list, &sde->flushlist); 2467 flush_count++; 2468 iowait_inc_wait_count(wait, tx->num_desc); 2469 } 2470 spin_unlock(&sde->flushlist_lock); 2471 queue_work_on(sde->cpu, system_highpri_wq, &sde->flush_worker); 2472 ret = -ECOMM; 2473 goto update_tail; 2474 nodesc: 2475 ret = sdma_check_progress(sde, wait, tx, submit_count > 0); 2476 if (ret == -EAGAIN) { 2477 ret = 0; 2478 goto retry; 2479 } 2480 sde->descq_full_count++; 2481 goto update_tail; 2482 } 2483 2484 static void sdma_process_event(struct sdma_engine *sde, enum sdma_events event) 2485 { 2486 unsigned long flags; 2487 2488 spin_lock_irqsave(&sde->tail_lock, flags); 2489 write_seqlock(&sde->head_lock); 2490 2491 __sdma_process_event(sde, event); 2492 2493 if (sde->state.current_state == sdma_state_s99_running) 2494 sdma_desc_avail(sde, sdma_descq_freecnt(sde)); 2495 2496 write_sequnlock(&sde->head_lock); 2497 spin_unlock_irqrestore(&sde->tail_lock, flags); 2498 } 2499 2500 static void __sdma_process_event(struct sdma_engine *sde, 2501 enum sdma_events event) 2502 { 2503 struct sdma_state *ss = &sde->state; 2504 int need_progress = 0; 2505 2506 /* CONFIG SDMA temporary */ 2507 #ifdef CONFIG_SDMA_VERBOSITY 2508 dd_dev_err(sde->dd, "CONFIG SDMA(%u) [%s] %s\n", sde->this_idx, 2509 sdma_state_names[ss->current_state], 2510 sdma_event_names[event]); 2511 #endif 2512 2513 switch (ss->current_state) { 2514 case sdma_state_s00_hw_down: 2515 switch (event) { 2516 case sdma_event_e00_go_hw_down: 2517 break; 2518 case sdma_event_e30_go_running: 2519 /* 2520 * If down, but running requested (usually result 2521 * of link up, then we need to start up. 2522 * This can happen when hw down is requested while 2523 * bringing the link up with traffic active on 2524 * 7220, e.g. 2525 */ 2526 ss->go_s99_running = 1; 2527 fallthrough; /* and start dma engine */ 2528 case sdma_event_e10_go_hw_start: 2529 /* This reference means the state machine is started */ 2530 sdma_get(&sde->state); 2531 sdma_set_state(sde, 2532 sdma_state_s10_hw_start_up_halt_wait); 2533 break; 2534 case sdma_event_e15_hw_halt_done: 2535 break; 2536 case sdma_event_e25_hw_clean_up_done: 2537 break; 2538 case sdma_event_e40_sw_cleaned: 2539 sdma_sw_tear_down(sde); 2540 break; 2541 case sdma_event_e50_hw_cleaned: 2542 break; 2543 case sdma_event_e60_hw_halted: 2544 break; 2545 case sdma_event_e70_go_idle: 2546 break; 2547 case sdma_event_e80_hw_freeze: 2548 break; 2549 case sdma_event_e81_hw_frozen: 2550 break; 2551 case sdma_event_e82_hw_unfreeze: 2552 break; 2553 case sdma_event_e85_link_down: 2554 break; 2555 case sdma_event_e90_sw_halted: 2556 break; 2557 } 2558 break; 2559 2560 case sdma_state_s10_hw_start_up_halt_wait: 2561 switch (event) { 2562 case sdma_event_e00_go_hw_down: 2563 sdma_set_state(sde, sdma_state_s00_hw_down); 2564 sdma_sw_tear_down(sde); 2565 break; 2566 case sdma_event_e10_go_hw_start: 2567 break; 2568 case sdma_event_e15_hw_halt_done: 2569 sdma_set_state(sde, 2570 sdma_state_s15_hw_start_up_clean_wait); 2571 sdma_start_hw_clean_up(sde); 2572 break; 2573 case sdma_event_e25_hw_clean_up_done: 2574 break; 2575 case sdma_event_e30_go_running: 2576 ss->go_s99_running = 1; 2577 break; 2578 case sdma_event_e40_sw_cleaned: 2579 break; 2580 case sdma_event_e50_hw_cleaned: 2581 break; 2582 case sdma_event_e60_hw_halted: 2583 schedule_work(&sde->err_halt_worker); 2584 break; 2585 case sdma_event_e70_go_idle: 2586 ss->go_s99_running = 0; 2587 break; 2588 case sdma_event_e80_hw_freeze: 2589 break; 2590 case sdma_event_e81_hw_frozen: 2591 break; 2592 case sdma_event_e82_hw_unfreeze: 2593 break; 2594 case sdma_event_e85_link_down: 2595 break; 2596 case sdma_event_e90_sw_halted: 2597 break; 2598 } 2599 break; 2600 2601 case sdma_state_s15_hw_start_up_clean_wait: 2602 switch (event) { 2603 case sdma_event_e00_go_hw_down: 2604 sdma_set_state(sde, sdma_state_s00_hw_down); 2605 sdma_sw_tear_down(sde); 2606 break; 2607 case sdma_event_e10_go_hw_start: 2608 break; 2609 case sdma_event_e15_hw_halt_done: 2610 break; 2611 case sdma_event_e25_hw_clean_up_done: 2612 sdma_hw_start_up(sde); 2613 sdma_set_state(sde, ss->go_s99_running ? 2614 sdma_state_s99_running : 2615 sdma_state_s20_idle); 2616 break; 2617 case sdma_event_e30_go_running: 2618 ss->go_s99_running = 1; 2619 break; 2620 case sdma_event_e40_sw_cleaned: 2621 break; 2622 case sdma_event_e50_hw_cleaned: 2623 break; 2624 case sdma_event_e60_hw_halted: 2625 break; 2626 case sdma_event_e70_go_idle: 2627 ss->go_s99_running = 0; 2628 break; 2629 case sdma_event_e80_hw_freeze: 2630 break; 2631 case sdma_event_e81_hw_frozen: 2632 break; 2633 case sdma_event_e82_hw_unfreeze: 2634 break; 2635 case sdma_event_e85_link_down: 2636 break; 2637 case sdma_event_e90_sw_halted: 2638 break; 2639 } 2640 break; 2641 2642 case sdma_state_s20_idle: 2643 switch (event) { 2644 case sdma_event_e00_go_hw_down: 2645 sdma_set_state(sde, sdma_state_s00_hw_down); 2646 sdma_sw_tear_down(sde); 2647 break; 2648 case sdma_event_e10_go_hw_start: 2649 break; 2650 case sdma_event_e15_hw_halt_done: 2651 break; 2652 case sdma_event_e25_hw_clean_up_done: 2653 break; 2654 case sdma_event_e30_go_running: 2655 sdma_set_state(sde, sdma_state_s99_running); 2656 ss->go_s99_running = 1; 2657 break; 2658 case sdma_event_e40_sw_cleaned: 2659 break; 2660 case sdma_event_e50_hw_cleaned: 2661 break; 2662 case sdma_event_e60_hw_halted: 2663 sdma_set_state(sde, sdma_state_s50_hw_halt_wait); 2664 schedule_work(&sde->err_halt_worker); 2665 break; 2666 case sdma_event_e70_go_idle: 2667 break; 2668 case sdma_event_e85_link_down: 2669 case sdma_event_e80_hw_freeze: 2670 sdma_set_state(sde, sdma_state_s80_hw_freeze); 2671 atomic_dec(&sde->dd->sdma_unfreeze_count); 2672 wake_up_interruptible(&sde->dd->sdma_unfreeze_wq); 2673 break; 2674 case sdma_event_e81_hw_frozen: 2675 break; 2676 case sdma_event_e82_hw_unfreeze: 2677 break; 2678 case sdma_event_e90_sw_halted: 2679 break; 2680 } 2681 break; 2682 2683 case sdma_state_s30_sw_clean_up_wait: 2684 switch (event) { 2685 case sdma_event_e00_go_hw_down: 2686 sdma_set_state(sde, sdma_state_s00_hw_down); 2687 break; 2688 case sdma_event_e10_go_hw_start: 2689 break; 2690 case sdma_event_e15_hw_halt_done: 2691 break; 2692 case sdma_event_e25_hw_clean_up_done: 2693 break; 2694 case sdma_event_e30_go_running: 2695 ss->go_s99_running = 1; 2696 break; 2697 case sdma_event_e40_sw_cleaned: 2698 sdma_set_state(sde, sdma_state_s40_hw_clean_up_wait); 2699 sdma_start_hw_clean_up(sde); 2700 break; 2701 case sdma_event_e50_hw_cleaned: 2702 break; 2703 case sdma_event_e60_hw_halted: 2704 break; 2705 case sdma_event_e70_go_idle: 2706 ss->go_s99_running = 0; 2707 break; 2708 case sdma_event_e80_hw_freeze: 2709 break; 2710 case sdma_event_e81_hw_frozen: 2711 break; 2712 case sdma_event_e82_hw_unfreeze: 2713 break; 2714 case sdma_event_e85_link_down: 2715 ss->go_s99_running = 0; 2716 break; 2717 case sdma_event_e90_sw_halted: 2718 break; 2719 } 2720 break; 2721 2722 case sdma_state_s40_hw_clean_up_wait: 2723 switch (event) { 2724 case sdma_event_e00_go_hw_down: 2725 sdma_set_state(sde, sdma_state_s00_hw_down); 2726 tasklet_hi_schedule(&sde->sdma_sw_clean_up_task); 2727 break; 2728 case sdma_event_e10_go_hw_start: 2729 break; 2730 case sdma_event_e15_hw_halt_done: 2731 break; 2732 case sdma_event_e25_hw_clean_up_done: 2733 sdma_hw_start_up(sde); 2734 sdma_set_state(sde, ss->go_s99_running ? 2735 sdma_state_s99_running : 2736 sdma_state_s20_idle); 2737 break; 2738 case sdma_event_e30_go_running: 2739 ss->go_s99_running = 1; 2740 break; 2741 case sdma_event_e40_sw_cleaned: 2742 break; 2743 case sdma_event_e50_hw_cleaned: 2744 break; 2745 case sdma_event_e60_hw_halted: 2746 break; 2747 case sdma_event_e70_go_idle: 2748 ss->go_s99_running = 0; 2749 break; 2750 case sdma_event_e80_hw_freeze: 2751 break; 2752 case sdma_event_e81_hw_frozen: 2753 break; 2754 case sdma_event_e82_hw_unfreeze: 2755 break; 2756 case sdma_event_e85_link_down: 2757 ss->go_s99_running = 0; 2758 break; 2759 case sdma_event_e90_sw_halted: 2760 break; 2761 } 2762 break; 2763 2764 case sdma_state_s50_hw_halt_wait: 2765 switch (event) { 2766 case sdma_event_e00_go_hw_down: 2767 sdma_set_state(sde, sdma_state_s00_hw_down); 2768 tasklet_hi_schedule(&sde->sdma_sw_clean_up_task); 2769 break; 2770 case sdma_event_e10_go_hw_start: 2771 break; 2772 case sdma_event_e15_hw_halt_done: 2773 sdma_set_state(sde, sdma_state_s30_sw_clean_up_wait); 2774 tasklet_hi_schedule(&sde->sdma_sw_clean_up_task); 2775 break; 2776 case sdma_event_e25_hw_clean_up_done: 2777 break; 2778 case sdma_event_e30_go_running: 2779 ss->go_s99_running = 1; 2780 break; 2781 case sdma_event_e40_sw_cleaned: 2782 break; 2783 case sdma_event_e50_hw_cleaned: 2784 break; 2785 case sdma_event_e60_hw_halted: 2786 schedule_work(&sde->err_halt_worker); 2787 break; 2788 case sdma_event_e70_go_idle: 2789 ss->go_s99_running = 0; 2790 break; 2791 case sdma_event_e80_hw_freeze: 2792 break; 2793 case sdma_event_e81_hw_frozen: 2794 break; 2795 case sdma_event_e82_hw_unfreeze: 2796 break; 2797 case sdma_event_e85_link_down: 2798 ss->go_s99_running = 0; 2799 break; 2800 case sdma_event_e90_sw_halted: 2801 break; 2802 } 2803 break; 2804 2805 case sdma_state_s60_idle_halt_wait: 2806 switch (event) { 2807 case sdma_event_e00_go_hw_down: 2808 sdma_set_state(sde, sdma_state_s00_hw_down); 2809 tasklet_hi_schedule(&sde->sdma_sw_clean_up_task); 2810 break; 2811 case sdma_event_e10_go_hw_start: 2812 break; 2813 case sdma_event_e15_hw_halt_done: 2814 sdma_set_state(sde, sdma_state_s30_sw_clean_up_wait); 2815 tasklet_hi_schedule(&sde->sdma_sw_clean_up_task); 2816 break; 2817 case sdma_event_e25_hw_clean_up_done: 2818 break; 2819 case sdma_event_e30_go_running: 2820 ss->go_s99_running = 1; 2821 break; 2822 case sdma_event_e40_sw_cleaned: 2823 break; 2824 case sdma_event_e50_hw_cleaned: 2825 break; 2826 case sdma_event_e60_hw_halted: 2827 schedule_work(&sde->err_halt_worker); 2828 break; 2829 case sdma_event_e70_go_idle: 2830 ss->go_s99_running = 0; 2831 break; 2832 case sdma_event_e80_hw_freeze: 2833 break; 2834 case sdma_event_e81_hw_frozen: 2835 break; 2836 case sdma_event_e82_hw_unfreeze: 2837 break; 2838 case sdma_event_e85_link_down: 2839 break; 2840 case sdma_event_e90_sw_halted: 2841 break; 2842 } 2843 break; 2844 2845 case sdma_state_s80_hw_freeze: 2846 switch (event) { 2847 case sdma_event_e00_go_hw_down: 2848 sdma_set_state(sde, sdma_state_s00_hw_down); 2849 tasklet_hi_schedule(&sde->sdma_sw_clean_up_task); 2850 break; 2851 case sdma_event_e10_go_hw_start: 2852 break; 2853 case sdma_event_e15_hw_halt_done: 2854 break; 2855 case sdma_event_e25_hw_clean_up_done: 2856 break; 2857 case sdma_event_e30_go_running: 2858 ss->go_s99_running = 1; 2859 break; 2860 case sdma_event_e40_sw_cleaned: 2861 break; 2862 case sdma_event_e50_hw_cleaned: 2863 break; 2864 case sdma_event_e60_hw_halted: 2865 break; 2866 case sdma_event_e70_go_idle: 2867 ss->go_s99_running = 0; 2868 break; 2869 case sdma_event_e80_hw_freeze: 2870 break; 2871 case sdma_event_e81_hw_frozen: 2872 sdma_set_state(sde, sdma_state_s82_freeze_sw_clean); 2873 tasklet_hi_schedule(&sde->sdma_sw_clean_up_task); 2874 break; 2875 case sdma_event_e82_hw_unfreeze: 2876 break; 2877 case sdma_event_e85_link_down: 2878 break; 2879 case sdma_event_e90_sw_halted: 2880 break; 2881 } 2882 break; 2883 2884 case sdma_state_s82_freeze_sw_clean: 2885 switch (event) { 2886 case sdma_event_e00_go_hw_down: 2887 sdma_set_state(sde, sdma_state_s00_hw_down); 2888 tasklet_hi_schedule(&sde->sdma_sw_clean_up_task); 2889 break; 2890 case sdma_event_e10_go_hw_start: 2891 break; 2892 case sdma_event_e15_hw_halt_done: 2893 break; 2894 case sdma_event_e25_hw_clean_up_done: 2895 break; 2896 case sdma_event_e30_go_running: 2897 ss->go_s99_running = 1; 2898 break; 2899 case sdma_event_e40_sw_cleaned: 2900 /* notify caller this engine is done cleaning */ 2901 atomic_dec(&sde->dd->sdma_unfreeze_count); 2902 wake_up_interruptible(&sde->dd->sdma_unfreeze_wq); 2903 break; 2904 case sdma_event_e50_hw_cleaned: 2905 break; 2906 case sdma_event_e60_hw_halted: 2907 break; 2908 case sdma_event_e70_go_idle: 2909 ss->go_s99_running = 0; 2910 break; 2911 case sdma_event_e80_hw_freeze: 2912 break; 2913 case sdma_event_e81_hw_frozen: 2914 break; 2915 case sdma_event_e82_hw_unfreeze: 2916 sdma_hw_start_up(sde); 2917 sdma_set_state(sde, ss->go_s99_running ? 2918 sdma_state_s99_running : 2919 sdma_state_s20_idle); 2920 break; 2921 case sdma_event_e85_link_down: 2922 break; 2923 case sdma_event_e90_sw_halted: 2924 break; 2925 } 2926 break; 2927 2928 case sdma_state_s99_running: 2929 switch (event) { 2930 case sdma_event_e00_go_hw_down: 2931 sdma_set_state(sde, sdma_state_s00_hw_down); 2932 tasklet_hi_schedule(&sde->sdma_sw_clean_up_task); 2933 break; 2934 case sdma_event_e10_go_hw_start: 2935 break; 2936 case sdma_event_e15_hw_halt_done: 2937 break; 2938 case sdma_event_e25_hw_clean_up_done: 2939 break; 2940 case sdma_event_e30_go_running: 2941 break; 2942 case sdma_event_e40_sw_cleaned: 2943 break; 2944 case sdma_event_e50_hw_cleaned: 2945 break; 2946 case sdma_event_e60_hw_halted: 2947 need_progress = 1; 2948 sdma_err_progress_check_schedule(sde); 2949 fallthrough; 2950 case sdma_event_e90_sw_halted: 2951 /* 2952 * SW initiated halt does not perform engines 2953 * progress check 2954 */ 2955 sdma_set_state(sde, sdma_state_s50_hw_halt_wait); 2956 schedule_work(&sde->err_halt_worker); 2957 break; 2958 case sdma_event_e70_go_idle: 2959 sdma_set_state(sde, sdma_state_s60_idle_halt_wait); 2960 break; 2961 case sdma_event_e85_link_down: 2962 ss->go_s99_running = 0; 2963 fallthrough; 2964 case sdma_event_e80_hw_freeze: 2965 sdma_set_state(sde, sdma_state_s80_hw_freeze); 2966 atomic_dec(&sde->dd->sdma_unfreeze_count); 2967 wake_up_interruptible(&sde->dd->sdma_unfreeze_wq); 2968 break; 2969 case sdma_event_e81_hw_frozen: 2970 break; 2971 case sdma_event_e82_hw_unfreeze: 2972 break; 2973 } 2974 break; 2975 } 2976 2977 ss->last_event = event; 2978 if (need_progress) 2979 sdma_make_progress(sde, 0); 2980 } 2981 2982 /* 2983 * _extend_sdma_tx_descs() - helper to extend txreq 2984 * 2985 * This is called once the initial nominal allocation 2986 * of descriptors in the sdma_txreq is exhausted. 2987 * 2988 * The code will bump the allocation up to the max 2989 * of MAX_DESC (64) descriptors. There doesn't seem 2990 * much point in an interim step. The last descriptor 2991 * is reserved for coalesce buffer in order to support 2992 * cases where input packet has >MAX_DESC iovecs. 2993 * 2994 */ 2995 static int _extend_sdma_tx_descs(struct hfi1_devdata *dd, struct sdma_txreq *tx) 2996 { 2997 int i; 2998 struct sdma_desc *descp; 2999 3000 /* Handle last descriptor */ 3001 if (unlikely((tx->num_desc == (MAX_DESC - 1)))) { 3002 /* if tlen is 0, it is for padding, release last descriptor */ 3003 if (!tx->tlen) { 3004 tx->desc_limit = MAX_DESC; 3005 } else if (!tx->coalesce_buf) { 3006 /* allocate coalesce buffer with space for padding */ 3007 tx->coalesce_buf = kmalloc(tx->tlen + sizeof(u32), 3008 GFP_ATOMIC); 3009 if (!tx->coalesce_buf) 3010 goto enomem; 3011 tx->coalesce_idx = 0; 3012 } 3013 return 0; 3014 } 3015 3016 if (unlikely(tx->num_desc == MAX_DESC)) 3017 goto enomem; 3018 3019 descp = kmalloc_array(MAX_DESC, sizeof(struct sdma_desc), GFP_ATOMIC); 3020 if (!descp) 3021 goto enomem; 3022 tx->descp = descp; 3023 3024 /* reserve last descriptor for coalescing */ 3025 tx->desc_limit = MAX_DESC - 1; 3026 /* copy ones already built */ 3027 for (i = 0; i < tx->num_desc; i++) 3028 tx->descp[i] = tx->descs[i]; 3029 return 0; 3030 enomem: 3031 __sdma_txclean(dd, tx); 3032 return -ENOMEM; 3033 } 3034 3035 /* 3036 * ext_coal_sdma_tx_descs() - extend or coalesce sdma tx descriptors 3037 * 3038 * This is called once the initial nominal allocation of descriptors 3039 * in the sdma_txreq is exhausted. 3040 * 3041 * This function calls _extend_sdma_tx_descs to extend or allocate 3042 * coalesce buffer. If there is a allocated coalesce buffer, it will 3043 * copy the input packet data into the coalesce buffer. It also adds 3044 * coalesce buffer descriptor once when whole packet is received. 3045 * 3046 * Return: 3047 * <0 - error 3048 * 0 - coalescing, don't populate descriptor 3049 * 1 - continue with populating descriptor 3050 */ 3051 int ext_coal_sdma_tx_descs(struct hfi1_devdata *dd, struct sdma_txreq *tx, 3052 int type, void *kvaddr, struct page *page, 3053 unsigned long offset, u16 len) 3054 { 3055 int pad_len, rval; 3056 dma_addr_t addr; 3057 3058 rval = _extend_sdma_tx_descs(dd, tx); 3059 if (rval) { 3060 __sdma_txclean(dd, tx); 3061 return rval; 3062 } 3063 3064 /* If coalesce buffer is allocated, copy data into it */ 3065 if (tx->coalesce_buf) { 3066 if (type == SDMA_MAP_NONE) { 3067 __sdma_txclean(dd, tx); 3068 return -EINVAL; 3069 } 3070 3071 if (type == SDMA_MAP_PAGE) { 3072 kvaddr = kmap_local_page(page); 3073 kvaddr += offset; 3074 } else if (WARN_ON(!kvaddr)) { 3075 __sdma_txclean(dd, tx); 3076 return -EINVAL; 3077 } 3078 3079 memcpy(tx->coalesce_buf + tx->coalesce_idx, kvaddr, len); 3080 tx->coalesce_idx += len; 3081 if (type == SDMA_MAP_PAGE) 3082 kunmap_local(kvaddr); 3083 3084 /* If there is more data, return */ 3085 if (tx->tlen - tx->coalesce_idx) 3086 return 0; 3087 3088 /* Whole packet is received; add any padding */ 3089 pad_len = tx->packet_len & (sizeof(u32) - 1); 3090 if (pad_len) { 3091 pad_len = sizeof(u32) - pad_len; 3092 memset(tx->coalesce_buf + tx->coalesce_idx, 0, pad_len); 3093 /* padding is taken care of for coalescing case */ 3094 tx->packet_len += pad_len; 3095 tx->tlen += pad_len; 3096 } 3097 3098 /* dma map the coalesce buffer */ 3099 addr = dma_map_single(&dd->pcidev->dev, 3100 tx->coalesce_buf, 3101 tx->tlen, 3102 DMA_TO_DEVICE); 3103 3104 if (unlikely(dma_mapping_error(&dd->pcidev->dev, addr))) { 3105 __sdma_txclean(dd, tx); 3106 return -ENOSPC; 3107 } 3108 3109 /* Add descriptor for coalesce buffer */ 3110 tx->desc_limit = MAX_DESC; 3111 return _sdma_txadd_daddr(dd, SDMA_MAP_SINGLE, tx, 3112 addr, tx->tlen, NULL, NULL, NULL); 3113 } 3114 3115 return 1; 3116 } 3117 3118 /* Update sdes when the lmc changes */ 3119 void sdma_update_lmc(struct hfi1_devdata *dd, u64 mask, u32 lid) 3120 { 3121 struct sdma_engine *sde; 3122 int i; 3123 u64 sreg; 3124 3125 sreg = ((mask & SD(CHECK_SLID_MASK_MASK)) << 3126 SD(CHECK_SLID_MASK_SHIFT)) | 3127 (((lid & mask) & SD(CHECK_SLID_VALUE_MASK)) << 3128 SD(CHECK_SLID_VALUE_SHIFT)); 3129 3130 for (i = 0; i < dd->num_sdma; i++) { 3131 hfi1_cdbg(LINKVERB, "SendDmaEngine[%d].SLID_CHECK = 0x%x", 3132 i, (u32)sreg); 3133 sde = &dd->per_sdma[i]; 3134 write_sde_csr(sde, SD(CHECK_SLID), sreg); 3135 } 3136 } 3137 3138 /* tx not dword sized - pad */ 3139 int _pad_sdma_tx_descs(struct hfi1_devdata *dd, struct sdma_txreq *tx) 3140 { 3141 int rval = 0; 3142 3143 if ((unlikely(tx->num_desc == tx->desc_limit))) { 3144 rval = _extend_sdma_tx_descs(dd, tx); 3145 if (rval) { 3146 __sdma_txclean(dd, tx); 3147 return rval; 3148 } 3149 } 3150 3151 /* finish the one just added */ 3152 make_tx_sdma_desc( 3153 tx, 3154 SDMA_MAP_NONE, 3155 dd->sdma_pad_phys, 3156 sizeof(u32) - (tx->packet_len & (sizeof(u32) - 1)), 3157 NULL, NULL, NULL); 3158 tx->num_desc++; 3159 _sdma_close_tx(dd, tx); 3160 return rval; 3161 } 3162 3163 /* 3164 * Add ahg to the sdma_txreq 3165 * 3166 * The logic will consume up to 3 3167 * descriptors at the beginning of 3168 * sdma_txreq. 3169 */ 3170 void _sdma_txreq_ahgadd( 3171 struct sdma_txreq *tx, 3172 u8 num_ahg, 3173 u8 ahg_entry, 3174 u32 *ahg, 3175 u8 ahg_hlen) 3176 { 3177 u32 i, shift = 0, desc = 0; 3178 u8 mode; 3179 3180 WARN_ON_ONCE(num_ahg > 9 || (ahg_hlen & 3) || ahg_hlen == 4); 3181 /* compute mode */ 3182 if (num_ahg == 1) 3183 mode = SDMA_AHG_APPLY_UPDATE1; 3184 else if (num_ahg <= 5) 3185 mode = SDMA_AHG_APPLY_UPDATE2; 3186 else 3187 mode = SDMA_AHG_APPLY_UPDATE3; 3188 tx->num_desc++; 3189 /* initialize to consumed descriptors to zero */ 3190 switch (mode) { 3191 case SDMA_AHG_APPLY_UPDATE3: 3192 tx->num_desc++; 3193 tx->descs[2].qw[0] = 0; 3194 tx->descs[2].qw[1] = 0; 3195 fallthrough; 3196 case SDMA_AHG_APPLY_UPDATE2: 3197 tx->num_desc++; 3198 tx->descs[1].qw[0] = 0; 3199 tx->descs[1].qw[1] = 0; 3200 break; 3201 } 3202 ahg_hlen >>= 2; 3203 tx->descs[0].qw[1] |= 3204 (((u64)ahg_entry & SDMA_DESC1_HEADER_INDEX_MASK) 3205 << SDMA_DESC1_HEADER_INDEX_SHIFT) | 3206 (((u64)ahg_hlen & SDMA_DESC1_HEADER_DWS_MASK) 3207 << SDMA_DESC1_HEADER_DWS_SHIFT) | 3208 (((u64)mode & SDMA_DESC1_HEADER_MODE_MASK) 3209 << SDMA_DESC1_HEADER_MODE_SHIFT) | 3210 (((u64)ahg[0] & SDMA_DESC1_HEADER_UPDATE1_MASK) 3211 << SDMA_DESC1_HEADER_UPDATE1_SHIFT); 3212 for (i = 0; i < (num_ahg - 1); i++) { 3213 if (!shift && !(i & 2)) 3214 desc++; 3215 tx->descs[desc].qw[!!(i & 2)] |= 3216 (((u64)ahg[i + 1]) 3217 << shift); 3218 shift = (shift + 32) & 63; 3219 } 3220 } 3221 3222 /** 3223 * sdma_ahg_alloc - allocate an AHG entry 3224 * @sde: engine to allocate from 3225 * 3226 * Return: 3227 * 0-31 when successful, -EOPNOTSUPP if AHG is not enabled, 3228 * -ENOSPC if an entry is not available 3229 */ 3230 int sdma_ahg_alloc(struct sdma_engine *sde) 3231 { 3232 int nr; 3233 int oldbit; 3234 3235 if (!sde) { 3236 trace_hfi1_ahg_allocate(sde, -EINVAL); 3237 return -EINVAL; 3238 } 3239 while (1) { 3240 nr = ffz(READ_ONCE(sde->ahg_bits)); 3241 if (nr > 31) { 3242 trace_hfi1_ahg_allocate(sde, -ENOSPC); 3243 return -ENOSPC; 3244 } 3245 oldbit = test_and_set_bit(nr, &sde->ahg_bits); 3246 if (!oldbit) 3247 break; 3248 cpu_relax(); 3249 } 3250 trace_hfi1_ahg_allocate(sde, nr); 3251 return nr; 3252 } 3253 3254 /** 3255 * sdma_ahg_free - free an AHG entry 3256 * @sde: engine to return AHG entry 3257 * @ahg_index: index to free 3258 * 3259 * This routine frees the indicate AHG entry. 3260 */ 3261 void sdma_ahg_free(struct sdma_engine *sde, int ahg_index) 3262 { 3263 if (!sde) 3264 return; 3265 trace_hfi1_ahg_deallocate(sde, ahg_index); 3266 if (ahg_index < 0 || ahg_index > 31) 3267 return; 3268 clear_bit(ahg_index, &sde->ahg_bits); 3269 } 3270 3271 /* 3272 * SPC freeze handling for SDMA engines. Called when the driver knows 3273 * the SPC is going into a freeze but before the freeze is fully 3274 * settled. Generally an error interrupt. 3275 * 3276 * This event will pull the engine out of running so no more entries can be 3277 * added to the engine's queue. 3278 */ 3279 void sdma_freeze_notify(struct hfi1_devdata *dd, int link_down) 3280 { 3281 int i; 3282 enum sdma_events event = link_down ? sdma_event_e85_link_down : 3283 sdma_event_e80_hw_freeze; 3284 3285 /* set up the wait but do not wait here */ 3286 atomic_set(&dd->sdma_unfreeze_count, dd->num_sdma); 3287 3288 /* tell all engines to stop running and wait */ 3289 for (i = 0; i < dd->num_sdma; i++) 3290 sdma_process_event(&dd->per_sdma[i], event); 3291 3292 /* sdma_freeze() will wait for all engines to have stopped */ 3293 } 3294 3295 /* 3296 * SPC freeze handling for SDMA engines. Called when the driver knows 3297 * the SPC is fully frozen. 3298 */ 3299 void sdma_freeze(struct hfi1_devdata *dd) 3300 { 3301 int i; 3302 int ret; 3303 3304 /* 3305 * Make sure all engines have moved out of the running state before 3306 * continuing. 3307 */ 3308 ret = wait_event_interruptible(dd->sdma_unfreeze_wq, 3309 atomic_read(&dd->sdma_unfreeze_count) <= 3310 0); 3311 /* interrupted or count is negative, then unloading - just exit */ 3312 if (ret || atomic_read(&dd->sdma_unfreeze_count) < 0) 3313 return; 3314 3315 /* set up the count for the next wait */ 3316 atomic_set(&dd->sdma_unfreeze_count, dd->num_sdma); 3317 3318 /* tell all engines that the SPC is frozen, they can start cleaning */ 3319 for (i = 0; i < dd->num_sdma; i++) 3320 sdma_process_event(&dd->per_sdma[i], sdma_event_e81_hw_frozen); 3321 3322 /* 3323 * Wait for everyone to finish software clean before exiting. The 3324 * software clean will read engine CSRs, so must be completed before 3325 * the next step, which will clear the engine CSRs. 3326 */ 3327 (void)wait_event_interruptible(dd->sdma_unfreeze_wq, 3328 atomic_read(&dd->sdma_unfreeze_count) <= 0); 3329 /* no need to check results - done no matter what */ 3330 } 3331 3332 /* 3333 * SPC freeze handling for the SDMA engines. Called after the SPC is unfrozen. 3334 * 3335 * The SPC freeze acts like a SDMA halt and a hardware clean combined. All 3336 * that is left is a software clean. We could do it after the SPC is fully 3337 * frozen, but then we'd have to add another state to wait for the unfreeze. 3338 * Instead, just defer the software clean until the unfreeze step. 3339 */ 3340 void sdma_unfreeze(struct hfi1_devdata *dd) 3341 { 3342 int i; 3343 3344 /* tell all engines start freeze clean up */ 3345 for (i = 0; i < dd->num_sdma; i++) 3346 sdma_process_event(&dd->per_sdma[i], 3347 sdma_event_e82_hw_unfreeze); 3348 } 3349 3350 /** 3351 * _sdma_engine_progress_schedule() - schedule progress on engine 3352 * @sde: sdma_engine to schedule progress 3353 * 3354 */ 3355 void _sdma_engine_progress_schedule( 3356 struct sdma_engine *sde) 3357 { 3358 trace_hfi1_sdma_engine_progress(sde, sde->progress_mask); 3359 /* assume we have selected a good cpu */ 3360 write_csr(sde->dd, 3361 CCE_INT_FORCE + (8 * (IS_SDMA_START / 64)), 3362 sde->progress_mask); 3363 } 3364